diff --git a/pyiceberg/io/pyarrow.py b/pyiceberg/io/pyarrow.py index 20333882b5..1eae25fff9 100644 --- a/pyiceberg/io/pyarrow.py +++ b/pyiceberg/io/pyarrow.py @@ -3324,7 +3324,9 @@ def _dataframe_to_data_files( ), ) else: - partitions = _determine_partitions(spec=table_metadata.spec(), schema=table_metadata.schema(), arrow_table=df) + partitions = _determine_partitions( + spec=table_metadata.spec(), schema=table_metadata.schema(), arrow_table=df, input_schema=task_schema + ) yield from write_file( io=io, table_metadata=table_metadata, @@ -3348,9 +3350,14 @@ class _TablePartition: arrow_table_partition: pa.Table -def _determine_partitions(spec: PartitionSpec, schema: Schema, arrow_table: pa.Table) -> Iterable[_TablePartition]: +def _determine_partitions( + spec: PartitionSpec, schema: Schema, arrow_table: pa.Table, input_schema: Schema | None = None +) -> Iterable[_TablePartition]: """Based on the iceberg table partition spec, filter the arrow table into partitions with their keys. + The source columns are found by field ID in input_schema, the Iceberg schema of the arrow table (schema if not + given), whose names can differ from schema, e.g. when the arrow table has field IDs from before a rename. + Example: Input: An arrow table with partition key of ['n_legs', 'year'] and with data of @@ -3378,10 +3385,7 @@ def _determine_partitions(spec: PartitionSpec, schema: Schema, arrow_table: pa.T for partition, name in zip(spec.fields, partition_fields, strict=True): source_field = schema.find_field(partition.source_id) - full_field_name = schema.find_column_name(partition.source_id) - if full_field_name is None: - raise ValueError(f"Could not find column name for field ID: {partition.source_id}") - field_array = _get_field_from_arrow_table(arrow_table, full_field_name) + field_array = _get_field_from_arrow_table(arrow_table, schema, partition.source_id, input_schema) partition_array = partition.transform.pyarrow_transform(source_field.field_type)(field_array) # PyArrow has no group by or equality kernels for extension types such as arrow.uuid, so use their storage type if isinstance(partition_array.type, pa.BaseExtensionType): @@ -3427,30 +3431,36 @@ def _partition_value_filter(partition_field_name: str, value: Any) -> pc.Express return pc.field(partition_field_name) == value -def _get_field_from_arrow_table(arrow_table: pa.Table, field_path: str) -> pa.Array: - """Get a field from an Arrow table, supporting both literal field names and nested field paths. +def _get_field_from_arrow_table( + arrow_table: pa.Table, schema: Schema, field_id: int, input_schema: Schema | None = None +) -> pa.Array: + """Get a field from an Arrow table by its field ID, where the field can be nested in structs. - This function handles two cases: - 1. Literal field names that may contain dots (e.g., "some.id") - 2. Nested field paths using dot notation (e.g., "bar.baz" for nested access) + Follow the target schema's parent IDs and resolve each name in input_schema, matching the writer. + A name can contain dots (e.g., "some.id"), so a nested field's full name cannot be split into these names. Args: arrow_table: The Arrow table containing the field - field_path: Field name or dot-separated path + schema: The target Iceberg table schema + field_id: The ID of the field + input_schema: The Iceberg schema of the Arrow table, if different from schema Returns: The field as a PyArrow Array Raises: + ValueError: If a field ID on the path is missing from either schema KeyError: If the field path cannot be resolved """ - # Try exact column name match (handles field names containing literal dots) - if field_path in arrow_table.column_names: - return arrow_table[field_path] - - # If not found as exact name, treat as nested field path - path_parts = field_path.split(".") - # Get the struct column from the table (e.g., "bar" from "bar.baz") - field_array = arrow_table[path_parts[0]] - # Navigate into the struct using the remaining path parts - return pc.struct_field(field_array, path_parts[1:]) + source_schema = input_schema if input_schema is not None else schema + # Match ArrowAccessor: follow the target's parent IDs using each field's input name. + names = [source_schema.find_field(field_id).name] + parent_id = schema._lazy_id_to_parent.get(field_id) + while parent_id is not None: + names.append(source_schema.find_field(parent_id).name) + parent_id = schema._lazy_id_to_parent.get(parent_id) + + # Get the column from the table (e.g., "bar" for "bar.baz") and navigate into its structs + column_name, *nested_names = reversed(names) + field_array = arrow_table[column_name] + return pc.struct_field(field_array, nested_names) if nested_names else field_array diff --git a/tests/catalog/test_catalog_behaviors.py b/tests/catalog/test_catalog_behaviors.py index a5099e4e6b..6eaf7f1bfb 100644 --- a/tests/catalog/test_catalog_behaviors.py +++ b/tests/catalog/test_catalog_behaviors.py @@ -744,6 +744,72 @@ def test_table_properties_raise_for_none_value( # Append table +def test_append_to_identity_partitioned_table_with_dotted_names(catalog: Catalog) -> None: + catalog.create_namespace("default") + # The names from the top-level column down to the partition source, where a name can contain dots + paths = [("a.b",), ("bar", "baz"), ("foo", "bar", "baz"), ("a.b", "c"), ("a", "b.c"), ("root", "a.b", "c")] + for i, path in enumerate(paths): + column_type: pa.DataType = pa.string() + values: list[Any] = ["x", "y", None] + for name in reversed(path[1:]): + column_type = pa.struct([(name, column_type)]) + values = [{name: value} for value in values] + arrow_schema = pa.schema([("id", pa.int64()), (path[0], column_type)]) + df = pa.Table.from_pydict({"id": [1, 2, 3], path[0]: values}, schema=arrow_schema) + tbl = catalog.create_table(f"default.dotted_partition_source_{i}", schema=df.schema) + with tbl.update_spec() as update: + update.add_field(".".join(path), IdentityTransform(), "p") + + tbl.append(df) + + assert sorted(tbl.scan().to_arrow().to_pylist(), key=lambda row: row["id"]) == df.to_pylist(), path + assert {task.file.partition[0] for task in tbl.scan().plan_files()} == {"x", "y", None}, path + + +def test_append_with_field_ids_after_swapping_column_names(catalog: Catalog) -> None: + catalog.create_namespace("default") + struct_type = pa.struct([("c", pa.string())]) + # Each case swaps the names of the two columns after the Arrow table took its field IDs from the table schema + cases: list[tuple[pa.Schema, dict[str, Any], str, str]] = [ + (pa.schema([("a", pa.string()), ("b", pa.string())]), {"a": "x", "b": "y"}, "a", "b"), + (pa.schema([("left", struct_type), ("a.b", struct_type)]), {"left": {"c": "x"}, "a.b": {"c": "y"}}, "left.c", "a.b.c"), + ] + for i, (arrow_schema, row, source_name, renamed_source_name) in enumerate(cases): + tbl = catalog.create_table(f"default.swapped_names_{i}", schema=arrow_schema) + with tbl.update_spec() as spec_update: + spec_update.add_field(source_name, IdentityTransform(), "p") + df = pa.Table.from_pylist([row], schema=tbl.schema().as_arrow()) + first, second = arrow_schema.names + for old_name, new_name in [(first, "tmp"), (second, first), ("tmp", second)]: + with tbl.update_schema() as schema_update: + schema_update.rename_column(old_name, new_name) + + tbl.append(df) + + # The partition value comes from the column that the source field is written from + assert [task.file.partition[0] for task in tbl.scan().plan_files()] == ["x"], source_name + swapped_row = {first: row[second], second: row[first]} + assert tbl.scan(row_filter=f"{renamed_source_name} = 'x'").to_arrow().to_pylist() == [swapped_row], source_name + + +def test_partition_values_follow_writer_struct_path(catalog: Catalog) -> None: + catalog.create_namespace("default") + struct_type = pa.struct([("x", pa.string())]) + tbl = catalog.create_table("default.partition_input_parents", schema=pa.schema([("s", struct_type), ("t", struct_type)])) + with tbl.update_spec() as spec_update: + spec_update.add_field("s.x", IdentityTransform(), "p") + # Swap the struct types, so that each leaf field ID of the input is under the other parent + s_field, t_field = tbl.schema().as_arrow() + input_schema = pa.schema([s_field.with_type(t_field.type), t_field.with_type(s_field.type)]) + tbl.append(pa.Table.from_pylist([{"s": {"x": "X"}, "t": {"x": "Y"}}], schema=input_schema)) + + # The partition value and the filtered scan agree with the row that was written + written_rows = tbl.scan().to_arrow().to_pylist() + value = written_rows[0]["s"]["x"] + assert [task.file.partition[0] for task in tbl.scan().plan_files()] == [value] + assert tbl.scan(row_filter=f"s.x = '{value}'").to_arrow().to_pylist() == written_rows + + def test_append_table(catalog: Catalog, table_schema_simple: Schema, test_table_identifier: Identifier) -> None: namespace = Catalog.namespace_from(test_table_identifier) catalog.create_namespace(namespace) diff --git a/tests/io/test_pyarrow.py b/tests/io/test_pyarrow.py index 439405d121..18dad07f17 100644 --- a/tests/io/test_pyarrow.py +++ b/tests/io/test_pyarrow.py @@ -3436,6 +3436,32 @@ def test_partition_for_nested_field() -> None: assert partition_values == {486729, 486730} +def test_partition_for_nested_field_with_dotted_names() -> None: + # The full names a.b.c, d.e.f and g.h.i.j cannot be split on dots into the names of the nested fields + schema = Schema( + NestedField(1, "a.b", StructType(NestedField(2, "c", StringType()))), + NestedField(3, "d", StructType(NestedField(4, "e.f", StringType()))), + NestedField(5, "g", StructType(NestedField(6, "h.i", StructType(NestedField(7, "j", StringType()))))), + ) + spec = PartitionSpec( + PartitionField(source_id=2, field_id=1000, transform=IdentityTransform(), name="c"), + PartitionField(source_id=4, field_id=1001, transform=IdentityTransform(), name="e_f"), + PartitionField(source_id=7, field_id=1002, transform=IdentityTransform(), name="j"), + ) + rows = [ + {"a.b": {"c": "x"}, "d": {"e.f": "y"}, "g": {"h.i": {"j": "z"}}}, + {"a.b": {"c": "x"}, "d": {"e.f": None}, "g": None}, + ] + arrow_table = pa.Table.from_pylist(rows, schema=schema.as_arrow()) + + partitions = list(_determine_partitions(spec, schema, arrow_table)) + + assert {p.partition_key.partition: p.arrow_table_partition.to_pylist() for p in partitions} == { + Record("x", "y", "z"): [rows[0]], + Record("x", None, None): [rows[1]], + } + + def test_partition_for_deep_nested_field() -> None: schema = Schema( NestedField(