diff --git a/pyiceberg/catalog/dynamodb.py b/pyiceberg/catalog/dynamodb.py index 62a4da5108..2157c76e39 100644 --- a/pyiceberg/catalog/dynamodb.py +++ b/pyiceberg/catalog/dynamodb.py @@ -226,7 +226,10 @@ def create_table( try: self._put_dynamo_item( item=_get_create_table_item( - database_name=database_name, table_name=table_name, properties=properties, metadata_location=metadata_location + database_name=database_name, + table_name=table_name, + properties=metadata.properties, + metadata_location=metadata_location, ), condition_expression=f"attribute_not_exists({DYNAMODB_COL_IDENTIFIER})", ) diff --git a/pyiceberg/catalog/glue.py b/pyiceberg/catalog/glue.py index bbb954fcda..9ba8775747 100644 --- a/pyiceberg/catalog/glue.py +++ b/pyiceberg/catalog/glue.py @@ -546,7 +546,9 @@ def _create_table_s3tables( # Write metadata and update the Glue table with the metadata pointer. # Skip the exist check before writing; S3 Tables doesn't support ListObjectsV2. self._write_metadata_no_exist_check(staged_table.metadata, staged_table.io, staged_table.metadata_location) - table_input = _construct_table_input(table_name, staged_table.metadata_location, properties, staged_table.metadata) + table_input = _construct_table_input( + table_name, staged_table.metadata_location, staged_table.metadata.properties, staged_table.metadata + ) version_id = glue_table.get("VersionId") if not version_id: raise CommitFailedException( @@ -630,7 +632,9 @@ def create_table( ) self._write_metadata(staged_table.metadata, staged_table.io, staged_table.metadata_location) - table_input = _construct_table_input(table_name, staged_table.metadata_location, properties, staged_table.metadata) + table_input = _construct_table_input( + table_name, staged_table.metadata_location, staged_table.metadata.properties, staged_table.metadata + ) self._create_glue_table(database_name=database_name, table_name=table_name, table_input=table_input) return Table( diff --git a/pyiceberg/table/metadata.py b/pyiceberg/table/metadata.py index c71338510f..f915ed929e 100644 --- a/pyiceberg/table/metadata.py +++ b/pyiceberg/table/metadata.py @@ -668,7 +668,8 @@ def new_table_metadata( ) -> TableMetadata: from pyiceberg.table import TableProperties - # Remove format-version so it does not get persisted + # Remove format-version so it does not get persisted, without changing the caller's properties + properties = dict(properties) format_version = int(properties.pop(TableProperties.FORMAT_VERSION, TableProperties.DEFAULT_FORMAT_VERSION)) schema.check_format_version_compatibility(format_version) diff --git a/tests/catalog/test_catalog_behaviors.py b/tests/catalog/test_catalog_behaviors.py index a5099e4e6b..69d817e95a 100644 --- a/tests/catalog/test_catalog_behaviors.py +++ b/tests/catalog/test_catalog_behaviors.py @@ -199,6 +199,20 @@ def test_create_table_default_sort_order( catalog.drop_table(test_table_identifier) +def test_create_v1_tables_with_shared_properties(catalog: Catalog, table_schema_nested: Schema) -> None: + catalog.create_namespace("default") + properties = {"format-version": "1", "owner": "etl"} + catalog.create_table("default.first", table_schema_nested, properties=properties) + catalog.create_table_if_not_exists("default.first", table_schema_nested, properties=properties) + catalog.create_table("default.second", table_schema_nested, properties=properties) + catalog.create_table_transaction("default.third", table_schema_nested, properties=properties).commit_transaction() + + tables = [catalog.load_table(f"default.{name}") for name in ("first", "second", "third")] + assert [table.format_version for table in tables] == [1, 1, 1] + assert [table.properties for table in tables] == [{"owner": "etl"}] * 3 + assert properties == {"format-version": "1", "owner": "etl"} + + def test_create_v1_table(catalog: Catalog, table_schema_nested: Schema, test_table_identifier: Identifier) -> None: namespace = Catalog.namespace_from(test_table_identifier) catalog.create_namespace(namespace) diff --git a/tests/catalog/test_dynamodb.py b/tests/catalog/test_dynamodb.py index 803cb2cd82..113fd5c1af 100644 --- a/tests/catalog/test_dynamodb.py +++ b/tests/catalog/test_dynamodb.py @@ -215,6 +215,26 @@ def test_create_table_if_not_exists_duplicated_table( assert table1.name() == table2.name() +@mock_aws +def test_create_v1_table( + _bucket_initialize: None, moto_endpoint_url: str, table_schema_nested: Schema, database_name: str, table_name: str +) -> None: + test_catalog = DynamoDbCatalog( + "test_ddb_catalog", client=None, **{"warehouse": f"s3://{BUCKET_NAME}", "s3.endpoint": moto_endpoint_url} + ) + test_catalog.create_namespace(namespace=database_name) + properties = {"format-version": "1", "owner": "etl", "enabled": True} + table = test_catalog.create_table((database_name, table_name), table_schema_nested, properties=properties) + assert table.format_version == 1 + assert properties == {"format-version": "1", "owner": "etl", "enabled": True} + + # pylint: disable=W0212 + item = test_catalog._get_iceberg_table_item(database_name=database_name, table_name=table_name) + assert item[_add_property_prefix("owner")] == {"S": "etl"} + assert item[_add_property_prefix("enabled")] == {"S": "true"} + assert _add_property_prefix("format-version") not in item + + @mock_aws def test_load_table( _bucket_initialize: None, moto_endpoint_url: str, table_schema_nested: Schema, database_name: str, table_name: str diff --git a/tests/catalog/test_glue.py b/tests/catalog/test_glue.py index f3949a9a80..5f1ecab2c7 100644 --- a/tests/catalog/test_glue.py +++ b/tests/catalog/test_glue.py @@ -145,13 +145,20 @@ def test_create_v1_table( catalog_name = "glue" test_catalog = GlueCatalog(catalog_name, client=None, **{"s3.endpoint": moto_endpoint_url}) test_catalog.create_namespace(namespace=database_name, properties={"location": f"s3://{BUCKET_NAME}/{database_name}.db"}) - table = test_catalog.create_table((database_name, table_name), table_schema_nested, properties={"format-version": "1"}) + table = test_catalog.create_table( + (database_name, table_name), table_schema_nested, properties={"format-version": "1", "enabled": True} + ) assert table.format_version == 1 table_info = _glue.get_table( DatabaseName=database_name, Name=table_name, ) + assert table_info["Table"]["Parameters"] == { + "table_type": "ICEBERG", + "metadata_location": table.metadata_location, + "enabled": "true", + } storage_descriptor = table_info["Table"]["StorageDescriptor"] columns = storage_descriptor["Columns"] @@ -1254,12 +1261,14 @@ def test_create_table_s3tables( test_catalog = GlueCatalog("s3tables", client=None, **{"s3.endpoint": moto_endpoint_url}) _create_s3tables_database(test_catalog, database_name) - table = test_catalog.create_table(identifier, table_schema_nested) + table = test_catalog.create_table(identifier, table_schema_nested, properties={"format-version": "2"}) assert table.name() == identifier assert table.location() == f"s3://{S3TABLES_WAREHOUSE_LOCATION}/{database_name}/{table_name}" assert table.metadata_location.startswith(f"s3://{S3TABLES_WAREHOUSE_LOCATION}/{database_name}/{table_name}/metadata/00000-") assert table.metadata_location.endswith(".metadata.json") assert test_catalog._parse_metadata_version(table.metadata_location) == 0 + parameters = test_catalog.glue.get_table(DatabaseName=database_name, Name=table_name)["Table"]["Parameters"] + assert parameters == {"table_type": "ICEBERG", "metadata_location": table.metadata_location} @mock_aws diff --git a/tests/table/test_metadata.py b/tests/table/test_metadata.py index d2f02319f7..c4460ef768 100644 --- a/tests/table/test_metadata.py +++ b/tests/table/test_metadata.py @@ -283,6 +283,15 @@ def test_new_table_metadata_with_explicit_v1_format() -> None: assert actual.sort_orders == [expected_sort_order] +def test_new_table_metadata_does_not_change_properties() -> None: + properties = {"format-version": "1", "owner": "etl"} + metadata = new_table_metadata( + schema=Schema(), partition_spec=PartitionSpec(), sort_order=SortOrder(), location="s3://bucket", properties=properties + ) + assert (metadata.format_version, metadata.properties) == (1, {"owner": "etl"}) + assert properties == {"format-version": "1", "owner": "etl"} + + def test_invalid_format_version(example_table_metadata_v1: dict[str, Any]) -> None: """Test the exception when trying to load an unknown version"""