Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
54 changes: 32 additions & 22 deletions pyiceberg/io/pyarrow.py
Original file line number Diff line number Diff line change
Expand Up @@ -3324,7 +3324,9 @@ def _dataframe_to_data_files(
),
)
else:
partitions = _determine_partitions(spec=table_metadata.spec(), schema=table_metadata.schema(), arrow_table=df)
partitions = _determine_partitions(
spec=table_metadata.spec(), schema=table_metadata.schema(), arrow_table=df, input_schema=task_schema
)
yield from write_file(
io=io,
table_metadata=table_metadata,
Expand All @@ -3348,9 +3350,14 @@ class _TablePartition:
arrow_table_partition: pa.Table


def _determine_partitions(spec: PartitionSpec, schema: Schema, arrow_table: pa.Table) -> Iterable[_TablePartition]:
def _determine_partitions(
spec: PartitionSpec, schema: Schema, arrow_table: pa.Table, input_schema: Schema | None = None
) -> Iterable[_TablePartition]:
"""Based on the iceberg table partition spec, filter the arrow table into partitions with their keys.

The source columns are found by field ID in input_schema, the Iceberg schema of the arrow table (schema if not
given), whose names can differ from schema, e.g. when the arrow table has field IDs from before a rename.

Example:
Input:
An arrow table with partition key of ['n_legs', 'year'] and with data of
Expand Down Expand Up @@ -3378,10 +3385,7 @@ def _determine_partitions(spec: PartitionSpec, schema: Schema, arrow_table: pa.T

for partition, name in zip(spec.fields, partition_fields, strict=True):
source_field = schema.find_field(partition.source_id)
full_field_name = schema.find_column_name(partition.source_id)
if full_field_name is None:
raise ValueError(f"Could not find column name for field ID: {partition.source_id}")
field_array = _get_field_from_arrow_table(arrow_table, full_field_name)
field_array = _get_field_from_arrow_table(arrow_table, schema, partition.source_id, input_schema)
partition_array = partition.transform.pyarrow_transform(source_field.field_type)(field_array)
# PyArrow has no group by or equality kernels for extension types such as arrow.uuid, so use their storage type
if isinstance(partition_array.type, pa.BaseExtensionType):
Expand Down Expand Up @@ -3427,30 +3431,36 @@ def _partition_value_filter(partition_field_name: str, value: Any) -> pc.Express
return pc.field(partition_field_name) == value


def _get_field_from_arrow_table(arrow_table: pa.Table, field_path: str) -> pa.Array:
"""Get a field from an Arrow table, supporting both literal field names and nested field paths.
def _get_field_from_arrow_table(
arrow_table: pa.Table, schema: Schema, field_id: int, input_schema: Schema | None = None
) -> pa.Array:
"""Get a field from an Arrow table by its field ID, where the field can be nested in structs.

This function handles two cases:
1. Literal field names that may contain dots (e.g., "some.id")
2. Nested field paths using dot notation (e.g., "bar.baz" for nested access)
Follow the target schema's parent IDs and resolve each name in input_schema, matching the writer.
A name can contain dots (e.g., "some.id"), so a nested field's full name cannot be split into these names.

Args:
arrow_table: The Arrow table containing the field
field_path: Field name or dot-separated path
schema: The target Iceberg table schema
field_id: The ID of the field
input_schema: The Iceberg schema of the Arrow table, if different from schema

Returns:
The field as a PyArrow Array

Raises:
ValueError: If a field ID on the path is missing from either schema
KeyError: If the field path cannot be resolved
"""
# Try exact column name match (handles field names containing literal dots)
if field_path in arrow_table.column_names:
return arrow_table[field_path]

# If not found as exact name, treat as nested field path
path_parts = field_path.split(".")
# Get the struct column from the table (e.g., "bar" from "bar.baz")
field_array = arrow_table[path_parts[0]]
# Navigate into the struct using the remaining path parts
return pc.struct_field(field_array, path_parts[1:])
source_schema = input_schema if input_schema is not None else schema
# Match ArrowAccessor: follow the target's parent IDs using each field's input name.
names = [source_schema.find_field(field_id).name]
parent_id = schema._lazy_id_to_parent.get(field_id)
while parent_id is not None:
names.append(source_schema.find_field(parent_id).name)
parent_id = schema._lazy_id_to_parent.get(parent_id)

# Get the column from the table (e.g., "bar" for "bar.baz") and navigate into its structs
column_name, *nested_names = reversed(names)
field_array = arrow_table[column_name]
return pc.struct_field(field_array, nested_names) if nested_names else field_array
66 changes: 66 additions & 0 deletions tests/catalog/test_catalog_behaviors.py
Original file line number Diff line number Diff line change
Expand Up @@ -744,6 +744,72 @@ def test_table_properties_raise_for_none_value(
# Append table


def test_append_to_identity_partitioned_table_with_dotted_names(catalog: Catalog) -> None:
catalog.create_namespace("default")
# The names from the top-level column down to the partition source, where a name can contain dots
paths = [("a.b",), ("bar", "baz"), ("foo", "bar", "baz"), ("a.b", "c"), ("a", "b.c"), ("root", "a.b", "c")]
for i, path in enumerate(paths):
column_type: pa.DataType = pa.string()
values: list[Any] = ["x", "y", None]
for name in reversed(path[1:]):
column_type = pa.struct([(name, column_type)])
values = [{name: value} for value in values]
arrow_schema = pa.schema([("id", pa.int64()), (path[0], column_type)])
df = pa.Table.from_pydict({"id": [1, 2, 3], path[0]: values}, schema=arrow_schema)
tbl = catalog.create_table(f"default.dotted_partition_source_{i}", schema=df.schema)
with tbl.update_spec() as update:
update.add_field(".".join(path), IdentityTransform(), "p")

tbl.append(df)

assert sorted(tbl.scan().to_arrow().to_pylist(), key=lambda row: row["id"]) == df.to_pylist(), path
assert {task.file.partition[0] for task in tbl.scan().plan_files()} == {"x", "y", None}, path


def test_append_with_field_ids_after_swapping_column_names(catalog: Catalog) -> None:
catalog.create_namespace("default")
struct_type = pa.struct([("c", pa.string())])
# Each case swaps the names of the two columns after the Arrow table took its field IDs from the table schema
cases: list[tuple[pa.Schema, dict[str, Any], str, str]] = [
(pa.schema([("a", pa.string()), ("b", pa.string())]), {"a": "x", "b": "y"}, "a", "b"),
(pa.schema([("left", struct_type), ("a.b", struct_type)]), {"left": {"c": "x"}, "a.b": {"c": "y"}}, "left.c", "a.b.c"),
]
for i, (arrow_schema, row, source_name, renamed_source_name) in enumerate(cases):
tbl = catalog.create_table(f"default.swapped_names_{i}", schema=arrow_schema)
with tbl.update_spec() as spec_update:
spec_update.add_field(source_name, IdentityTransform(), "p")
df = pa.Table.from_pylist([row], schema=tbl.schema().as_arrow())
first, second = arrow_schema.names
for old_name, new_name in [(first, "tmp"), (second, first), ("tmp", second)]:
with tbl.update_schema() as schema_update:
schema_update.rename_column(old_name, new_name)

tbl.append(df)

# The partition value comes from the column that the source field is written from
assert [task.file.partition[0] for task in tbl.scan().plan_files()] == ["x"], source_name
swapped_row = {first: row[second], second: row[first]}
assert tbl.scan(row_filter=f"{renamed_source_name} = 'x'").to_arrow().to_pylist() == [swapped_row], source_name


def test_partition_values_follow_writer_struct_path(catalog: Catalog) -> None:
catalog.create_namespace("default")
struct_type = pa.struct([("x", pa.string())])
tbl = catalog.create_table("default.partition_input_parents", schema=pa.schema([("s", struct_type), ("t", struct_type)]))
with tbl.update_spec() as spec_update:
spec_update.add_field("s.x", IdentityTransform(), "p")
# Swap the struct types, so that each leaf field ID of the input is under the other parent
s_field, t_field = tbl.schema().as_arrow()
input_schema = pa.schema([s_field.with_type(t_field.type), t_field.with_type(s_field.type)])
tbl.append(pa.Table.from_pylist([{"s": {"x": "X"}, "t": {"x": "Y"}}], schema=input_schema))

# The partition value and the filtered scan agree with the row that was written
written_rows = tbl.scan().to_arrow().to_pylist()
value = written_rows[0]["s"]["x"]
assert [task.file.partition[0] for task in tbl.scan().plan_files()] == [value]
assert tbl.scan(row_filter=f"s.x = '{value}'").to_arrow().to_pylist() == written_rows


def test_append_table(catalog: Catalog, table_schema_simple: Schema, test_table_identifier: Identifier) -> None:
namespace = Catalog.namespace_from(test_table_identifier)
catalog.create_namespace(namespace)
Expand Down
26 changes: 26 additions & 0 deletions tests/io/test_pyarrow.py
Original file line number Diff line number Diff line change
Expand Up @@ -3436,6 +3436,32 @@ def test_partition_for_nested_field() -> None:
assert partition_values == {486729, 486730}


def test_partition_for_nested_field_with_dotted_names() -> None:
# The full names a.b.c, d.e.f and g.h.i.j cannot be split on dots into the names of the nested fields
schema = Schema(
NestedField(1, "a.b", StructType(NestedField(2, "c", StringType()))),
NestedField(3, "d", StructType(NestedField(4, "e.f", StringType()))),
NestedField(5, "g", StructType(NestedField(6, "h.i", StructType(NestedField(7, "j", StringType()))))),
)
spec = PartitionSpec(
PartitionField(source_id=2, field_id=1000, transform=IdentityTransform(), name="c"),
PartitionField(source_id=4, field_id=1001, transform=IdentityTransform(), name="e_f"),
PartitionField(source_id=7, field_id=1002, transform=IdentityTransform(), name="j"),
)
rows = [
{"a.b": {"c": "x"}, "d": {"e.f": "y"}, "g": {"h.i": {"j": "z"}}},
{"a.b": {"c": "x"}, "d": {"e.f": None}, "g": None},
]
arrow_table = pa.Table.from_pylist(rows, schema=schema.as_arrow())

partitions = list(_determine_partitions(spec, schema, arrow_table))

assert {p.partition_key.partition: p.arrow_table_partition.to_pylist() for p in partitions} == {
Record("x", "y", "z"): [rows[0]],
Record("x", None, None): [rows[1]],
}


def test_partition_for_deep_nested_field() -> None:
schema = Schema(
NestedField(
Expand Down
Loading