diff --git a/mkdocs/docs/api.md b/mkdocs/docs/api.md index aa7c111329..a659e51eb5 100644 --- a/mkdocs/docs/api.md +++ b/mkdocs/docs/api.md @@ -324,6 +324,8 @@ tbl.append(df) tbl.overwrite(df) ``` +`append` and `overwrite` also accept any object that implements the Arrow PyCapsule stream interface, such as a Polars DataFrame or an arro3 table, and import it as a PyArrow table. + Now, the data is written to the table, and the table can be read using `tbl.scan().to_arrow()`: ```python diff --git a/pyiceberg/table/__init__.py b/pyiceberg/table/__init__.py index 3ea41e3f22..d231aa5963 100644 --- a/pyiceberg/table/__init__.py +++ b/pyiceberg/table/__init__.py @@ -106,6 +106,7 @@ from pyiceberg.transforms import IdentityTransform from pyiceberg.typedef import ( EMPTY_DICT, + ArrowStreamExportable, IcebergBaseModel, IcebergRootModel, Identifier, @@ -522,15 +523,16 @@ def update_partition_statistics(self) -> UpdatePartitionStatistics: def append( self, - df: pa.Table | pa.RecordBatchReader, + df: pa.Table | pa.RecordBatchReader | ArrowStreamExportable, snapshot_properties: dict[str, str] = EMPTY_DICT, branch: str | None = MAIN_BRANCH, ) -> None: """ Shorthand API for appending PyArrow data to a table transaction. - Accepts either a fully materialised ``pa.Table`` or a streaming - ``pa.RecordBatchReader``. Streaming is microbatched by + Accepts either a fully materialised ``pa.Table``, a streaming + ``pa.RecordBatchReader``, or an object implementing the Arrow PyCapsule + stream interface, which is imported as a ``pa.Table``. Streaming is microbatched by ``write.target-file-size-bytes`` so memory stays bounded; the reader is consumed once and cannot be reused. @@ -577,8 +579,11 @@ def append( from pyiceberg.io.pyarrow import _check_pyarrow_schema_compatible, _dataframe_to_data_files + if isinstance(df, ArrowStreamExportable) and not isinstance(df, (pa.Table, pa.RecordBatchReader)): + df = pa.table(df) + if not isinstance(df, (pa.Table, pa.RecordBatchReader)): - raise ValueError(f"Expected pa.Table or pa.RecordBatchReader, got: {df}") + raise ValueError(f"Expected pa.Table, pa.RecordBatchReader, or an Arrow PyCapsule exporter, got: {df}") downcast_ns_timestamp_to_us = Config().get_bool(DOWNCAST_NS_TIMESTAMP_TO_US_ON_WRITE) or False _check_pyarrow_schema_compatible( @@ -676,7 +681,7 @@ def dynamic_partition_overwrite( def overwrite( self, - df: pa.Table | pa.RecordBatchReader, + df: pa.Table | pa.RecordBatchReader | ArrowStreamExportable, overwrite_filter: BooleanExpression | str = ALWAYS_TRUE, snapshot_properties: dict[str, str] = EMPTY_DICT, case_sensitive: bool = True, @@ -685,8 +690,9 @@ def overwrite( """ Shorthand for adding a table overwrite with a PyArrow table or RecordBatchReader to the transaction. - Accepts either a fully materialised ``pa.Table`` or a streaming - ``pa.RecordBatchReader``. Streaming is microbatched by + Accepts either a fully materialised ``pa.Table``, a streaming + ``pa.RecordBatchReader``, or an object implementing the Arrow PyCapsule + stream interface, which is imported as a ``pa.Table``. Streaming is microbatched by ``write.target-file-size-bytes`` so memory stays bounded; the reader is consumed once and cannot be reused. @@ -742,8 +748,11 @@ def overwrite( from pyiceberg.io.pyarrow import _check_pyarrow_schema_compatible, _dataframe_to_data_files + if isinstance(df, ArrowStreamExportable) and not isinstance(df, (pa.Table, pa.RecordBatchReader)): + df = pa.table(df) + if not isinstance(df, (pa.Table, pa.RecordBatchReader)): - raise ValueError(f"Expected pa.Table or pa.RecordBatchReader, got: {df}") + raise ValueError(f"Expected pa.Table, pa.RecordBatchReader, or an Arrow PyCapsule exporter, got: {df}") downcast_ns_timestamp_to_us = Config().get_bool(DOWNCAST_NS_TIMESTAMP_TO_US_ON_WRITE) or False _check_pyarrow_schema_compatible( @@ -1819,14 +1828,15 @@ def upsert( def append( self, - df: pa.Table | pa.RecordBatchReader, + df: pa.Table | pa.RecordBatchReader | ArrowStreamExportable, snapshot_properties: dict[str, str] = EMPTY_DICT, branch: str | None = MAIN_BRANCH, ) -> None: """ Shorthand API for appending PyArrow data to the table. - Accepts either a ``pa.Table`` or a streaming ``pa.RecordBatchReader``. + Accepts a ``pa.Table``, a streaming ``pa.RecordBatchReader``, or any object + implementing the Arrow PyCapsule stream interface. See :meth:`Transaction.append` for streaming semantics and partition limitations. @@ -1854,7 +1864,7 @@ def dynamic_partition_overwrite( def overwrite( self, - df: pa.Table | pa.RecordBatchReader, + df: pa.Table | pa.RecordBatchReader | ArrowStreamExportable, overwrite_filter: BooleanExpression | str = ALWAYS_TRUE, snapshot_properties: dict[str, str] = EMPTY_DICT, case_sensitive: bool = True, @@ -1863,7 +1873,8 @@ def overwrite( """ Shorthand for overwriting the table with a PyArrow Table or RecordBatchReader. - Accepts either a ``pa.Table`` or a streaming ``pa.RecordBatchReader``. + Accepts a ``pa.Table``, a streaming ``pa.RecordBatchReader``, or any object + implementing the Arrow PyCapsule stream interface. See :meth:`Transaction.overwrite` for streaming semantics and partition limitations. diff --git a/pyiceberg/typedef.py b/pyiceberg/typedef.py index de4d036fe7..3acd14ecdf 100644 --- a/pyiceberg/typedef.py +++ b/pyiceberg/typedef.py @@ -100,6 +100,14 @@ def __missing__(self, key: K) -> V: L = TypeVar("L", str, bool, int, float, bytes, UUID, Decimal, datetime, date, time, covariant=True) +@runtime_checkable +class ArrowStreamExportable(Protocol): # pragma: no cover + """An object that exports an Arrow stream through the Arrow PyCapsule interface.""" + + def __arrow_c_stream__(self, requested_schema: object | None = None) -> object: + """Export this object as an Arrow C stream capsule.""" + + @runtime_checkable class StructProtocol(Protocol): # pragma: no cover """A generic protocol used by accessors to get and set at positions of an object.""" diff --git a/tests/catalog/test_catalog_behaviors.py b/tests/catalog/test_catalog_behaviors.py index a5099e4e6b..25a26bf230 100644 --- a/tests/catalog/test_catalog_behaviors.py +++ b/tests/catalog/test_catalog_behaviors.py @@ -1475,7 +1475,7 @@ def test_append_invalid_input_type_raises(catalog: Catalog) -> None: identifier = f"default.append_invalid_input_{catalog.name}" pa_table = _simple_arrow_table() tbl = catalog.create_table(identifier=identifier, schema=pa_table.schema) - with pytest.raises(ValueError, match="Expected pa.Table or pa.RecordBatchReader"): + with pytest.raises(ValueError, match="Expected pa.Table, pa.RecordBatchReader, or an Arrow PyCapsule exporter"): tbl.append("not an arrow object") diff --git a/tests/integration/test_writes/test_partitioned_writes.py b/tests/integration/test_writes/test_partitioned_writes.py index 1d1488255f..798f61fbe4 100644 --- a/tests/integration/test_writes/test_partitioned_writes.py +++ b/tests/integration/test_writes/test_partitioned_writes.py @@ -768,7 +768,7 @@ def test_invalid_arguments(spark: SparkSession, session_catalog: Catalog) -> Non properties={"format-version": "1"}, ) - with pytest.raises(ValueError, match="Expected pa.Table or pa.RecordBatchReader, got: not a df"): + with pytest.raises(ValueError, match="Expected pa.Table, pa.RecordBatchReader, or an Arrow PyCapsule exporter"): tbl.append("not a df") diff --git a/tests/integration/test_writes/test_writes.py b/tests/integration/test_writes/test_writes.py index e0d647c2e9..824a8b985d 100644 --- a/tests/integration/test_writes/test_writes.py +++ b/tests/integration/test_writes/test_writes.py @@ -792,10 +792,10 @@ def test_invalid_arguments(spark: SparkSession, session_catalog: Catalog, arrow_ identifier = "default.arrow_data_files" tbl = _create_table(session_catalog, identifier, {"format-version": "1"}, []) - with pytest.raises(ValueError, match="Expected pa.Table or pa.RecordBatchReader, got: not a df"): + with pytest.raises(ValueError, match="Expected pa.Table, pa.RecordBatchReader, or an Arrow PyCapsule exporter"): tbl.overwrite("not a df") - with pytest.raises(ValueError, match="Expected pa.Table or pa.RecordBatchReader, got: not a df"): + with pytest.raises(ValueError, match="Expected pa.Table, pa.RecordBatchReader, or an Arrow PyCapsule exporter"): tbl.append("not a df") diff --git a/tests/table/test_init.py b/tests/table/test_init.py index 547ca94fc9..9d7c3b96b8 100644 --- a/tests/table/test_init.py +++ b/tests/table/test_init.py @@ -2595,6 +2595,58 @@ def test_overwrite_delete_data_file_on_bucket_partition(catalog: Catalog) -> Non assert table.scan().to_arrow().num_rows == 0 +def test_append_accepts_arrow_capsule_exporter(catalog: Catalog) -> None: + import pyarrow as pa + + from pyiceberg.io.pyarrow import schema_to_pyarrow + + class ArrowCapsuleTable: + def __init__(self, table: pa.Table) -> None: + self._table = table + + def __arrow_c_stream__(self, requested_schema: Any = None) -> Any: + return self._table.__arrow_c_stream__(requested_schema) + + catalog.create_namespace("default") + schema = Schema(NestedField(1, "x", IntegerType(), required=False)) + table = catalog.create_table("default.arrow_capsule_append", schema=schema) + + table.append(ArrowCapsuleTable(pa.table({"x": [1, 2]}, schema=schema_to_pyarrow(schema)))) + + assert table.scan().to_arrow().to_pylist() == [{"x": 1}, {"x": 2}] + + +def test_overwrite_accepts_arrow_capsule_exporter(catalog: Catalog) -> None: + import pyarrow as pa + + from pyiceberg.io.pyarrow import schema_to_pyarrow + + class ArrowCapsuleTable: + def __init__(self, table: pa.Table) -> None: + self._table = table + + def __arrow_c_stream__(self, requested_schema: Any = None) -> Any: + return self._table.__arrow_c_stream__(requested_schema) + + catalog.create_namespace("default") + schema = Schema(NestedField(1, "x", IntegerType(), required=False)) + table = catalog.create_table("default.arrow_capsule_overwrite", schema=schema) + table.append(pa.table({"x": [1, 2]}, schema=schema_to_pyarrow(schema))) + + table.overwrite(ArrowCapsuleTable(pa.table({"x": [3]}, schema=schema_to_pyarrow(schema)))) + + assert table.scan().to_arrow().to_pylist() == [{"x": 3}] + + +def test_append_rejects_object_without_arrow_capsule(catalog: Catalog) -> None: + catalog.create_namespace("default") + schema = Schema(NestedField(1, "x", IntegerType(), required=False)) + table = catalog.create_table("default.arrow_capsule_rejected", schema=schema) + + with pytest.raises(ValueError, match="Arrow PyCapsule"): + table.append(object()) # type: ignore[arg-type] + + def test_partial_delete_keeps_earlier_deletes_outside_the_filter(catalog: Catalog) -> None: """Rewriting a file applies all of its delete files, not only those whose bounds can match the delete filter.""" import pyarrow as pa