Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion pyiceberg/catalog/dynamodb.py
Original file line number Diff line number Diff line change
Expand Up @@ -226,7 +226,10 @@ def create_table(
try:
self._put_dynamo_item(
item=_get_create_table_item(
database_name=database_name, table_name=table_name, properties=properties, metadata_location=metadata_location
database_name=database_name,
table_name=table_name,
properties=metadata.properties,
metadata_location=metadata_location,
),
condition_expression=f"attribute_not_exists({DYNAMODB_COL_IDENTIFIER})",
)
Expand Down
8 changes: 6 additions & 2 deletions pyiceberg/catalog/glue.py
Original file line number Diff line number Diff line change
Expand Up @@ -546,7 +546,9 @@ def _create_table_s3tables(
# Write metadata and update the Glue table with the metadata pointer.
# Skip the exist check before writing; S3 Tables doesn't support ListObjectsV2.
self._write_metadata_no_exist_check(staged_table.metadata, staged_table.io, staged_table.metadata_location)
table_input = _construct_table_input(table_name, staged_table.metadata_location, properties, staged_table.metadata)
table_input = _construct_table_input(
table_name, staged_table.metadata_location, staged_table.metadata.properties, staged_table.metadata
)
version_id = glue_table.get("VersionId")
if not version_id:
raise CommitFailedException(
Expand Down Expand Up @@ -630,7 +632,9 @@ def create_table(
)

self._write_metadata(staged_table.metadata, staged_table.io, staged_table.metadata_location)
table_input = _construct_table_input(table_name, staged_table.metadata_location, properties, staged_table.metadata)
table_input = _construct_table_input(
table_name, staged_table.metadata_location, staged_table.metadata.properties, staged_table.metadata
)
self._create_glue_table(database_name=database_name, table_name=table_name, table_input=table_input)

return Table(
Expand Down
3 changes: 2 additions & 1 deletion pyiceberg/table/metadata.py
Original file line number Diff line number Diff line change
Expand Up @@ -668,7 +668,8 @@ def new_table_metadata(
) -> TableMetadata:
from pyiceberg.table import TableProperties

# Remove format-version so it does not get persisted
# Remove format-version so it does not get persisted, without changing the caller's properties
properties = dict(properties)
format_version = int(properties.pop(TableProperties.FORMAT_VERSION, TableProperties.DEFAULT_FORMAT_VERSION))

schema.check_format_version_compatibility(format_version)
Expand Down
14 changes: 14 additions & 0 deletions tests/catalog/test_catalog_behaviors.py
Original file line number Diff line number Diff line change
Expand Up @@ -199,6 +199,20 @@ def test_create_table_default_sort_order(
catalog.drop_table(test_table_identifier)


def test_create_v1_tables_with_shared_properties(catalog: Catalog, table_schema_nested: Schema) -> None:
catalog.create_namespace("default")
properties = {"format-version": "1", "owner": "etl"}
catalog.create_table("default.first", table_schema_nested, properties=properties)
catalog.create_table_if_not_exists("default.first", table_schema_nested, properties=properties)
catalog.create_table("default.second", table_schema_nested, properties=properties)
catalog.create_table_transaction("default.third", table_schema_nested, properties=properties).commit_transaction()

tables = [catalog.load_table(f"default.{name}") for name in ("first", "second", "third")]
assert [table.format_version for table in tables] == [1, 1, 1]
assert [table.properties for table in tables] == [{"owner": "etl"}] * 3
assert properties == {"format-version": "1", "owner": "etl"}


def test_create_v1_table(catalog: Catalog, table_schema_nested: Schema, test_table_identifier: Identifier) -> None:
namespace = Catalog.namespace_from(test_table_identifier)
catalog.create_namespace(namespace)
Expand Down
20 changes: 20 additions & 0 deletions tests/catalog/test_dynamodb.py
Original file line number Diff line number Diff line change
Expand Up @@ -215,6 +215,26 @@ def test_create_table_if_not_exists_duplicated_table(
assert table1.name() == table2.name()


@mock_aws
def test_create_v1_table(
_bucket_initialize: None, moto_endpoint_url: str, table_schema_nested: Schema, database_name: str, table_name: str
) -> None:
test_catalog = DynamoDbCatalog(
"test_ddb_catalog", client=None, **{"warehouse": f"s3://{BUCKET_NAME}", "s3.endpoint": moto_endpoint_url}
)
test_catalog.create_namespace(namespace=database_name)
properties = {"format-version": "1", "owner": "etl", "enabled": True}
table = test_catalog.create_table((database_name, table_name), table_schema_nested, properties=properties)
assert table.format_version == 1
assert properties == {"format-version": "1", "owner": "etl", "enabled": True}

# pylint: disable=W0212
item = test_catalog._get_iceberg_table_item(database_name=database_name, table_name=table_name)
assert item[_add_property_prefix("owner")] == {"S": "etl"}
assert item[_add_property_prefix("enabled")] == {"S": "true"}
assert _add_property_prefix("format-version") not in item


@mock_aws
def test_load_table(
_bucket_initialize: None, moto_endpoint_url: str, table_schema_nested: Schema, database_name: str, table_name: str
Expand Down
13 changes: 11 additions & 2 deletions tests/catalog/test_glue.py
Original file line number Diff line number Diff line change
Expand Up @@ -145,13 +145,20 @@ def test_create_v1_table(
catalog_name = "glue"
test_catalog = GlueCatalog(catalog_name, client=None, **{"s3.endpoint": moto_endpoint_url})
test_catalog.create_namespace(namespace=database_name, properties={"location": f"s3://{BUCKET_NAME}/{database_name}.db"})
table = test_catalog.create_table((database_name, table_name), table_schema_nested, properties={"format-version": "1"})
table = test_catalog.create_table(
(database_name, table_name), table_schema_nested, properties={"format-version": "1", "enabled": True}
)
assert table.format_version == 1

table_info = _glue.get_table(
DatabaseName=database_name,
Name=table_name,
)
assert table_info["Table"]["Parameters"] == {
"table_type": "ICEBERG",
"metadata_location": table.metadata_location,
"enabled": "true",
}

storage_descriptor = table_info["Table"]["StorageDescriptor"]
columns = storage_descriptor["Columns"]
Expand Down Expand Up @@ -1254,12 +1261,14 @@ def test_create_table_s3tables(
test_catalog = GlueCatalog("s3tables", client=None, **{"s3.endpoint": moto_endpoint_url})
_create_s3tables_database(test_catalog, database_name)

table = test_catalog.create_table(identifier, table_schema_nested)
table = test_catalog.create_table(identifier, table_schema_nested, properties={"format-version": "2"})
assert table.name() == identifier
assert table.location() == f"s3://{S3TABLES_WAREHOUSE_LOCATION}/{database_name}/{table_name}"
assert table.metadata_location.startswith(f"s3://{S3TABLES_WAREHOUSE_LOCATION}/{database_name}/{table_name}/metadata/00000-")
assert table.metadata_location.endswith(".metadata.json")
assert test_catalog._parse_metadata_version(table.metadata_location) == 0
parameters = test_catalog.glue.get_table(DatabaseName=database_name, Name=table_name)["Table"]["Parameters"]
assert parameters == {"table_type": "ICEBERG", "metadata_location": table.metadata_location}


@mock_aws
Expand Down
9 changes: 9 additions & 0 deletions tests/table/test_metadata.py
Original file line number Diff line number Diff line change
Expand Up @@ -283,6 +283,15 @@ def test_new_table_metadata_with_explicit_v1_format() -> None:
assert actual.sort_orders == [expected_sort_order]


def test_new_table_metadata_does_not_change_properties() -> None:
properties = {"format-version": "1", "owner": "etl"}
metadata = new_table_metadata(
schema=Schema(), partition_spec=PartitionSpec(), sort_order=SortOrder(), location="s3://bucket", properties=properties
)
assert (metadata.format_version, metadata.properties) == (1, {"owner": "etl"})
assert properties == {"format-version": "1", "owner": "etl"}


def test_invalid_format_version(example_table_metadata_v1: dict[str, Any]) -> None:
"""Test the exception when trying to load an unknown version"""

Expand Down
Loading