Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions mkdocs/docs/api.md
Original file line number Diff line number Diff line change
Expand Up @@ -324,6 +324,8 @@ tbl.append(df)
tbl.overwrite(df)
```

`append` and `overwrite` also accept any object that implements the Arrow PyCapsule stream interface, such as a Polars DataFrame or an arro3 table, and import it as a PyArrow table.

Now, the data is written to the table, and the table can be read using `tbl.scan().to_arrow()`:

```python
Expand Down
35 changes: 23 additions & 12 deletions pyiceberg/table/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -106,6 +106,7 @@
from pyiceberg.transforms import IdentityTransform
from pyiceberg.typedef import (
EMPTY_DICT,
ArrowStreamExportable,
IcebergBaseModel,
IcebergRootModel,
Identifier,
Expand Down Expand Up @@ -522,15 +523,16 @@ def update_partition_statistics(self) -> UpdatePartitionStatistics:

def append(
self,
df: pa.Table | pa.RecordBatchReader,
df: pa.Table | pa.RecordBatchReader | ArrowStreamExportable,
snapshot_properties: dict[str, str] = EMPTY_DICT,
branch: str | None = MAIN_BRANCH,
) -> None:
"""
Shorthand API for appending PyArrow data to a table transaction.

Accepts either a fully materialised ``pa.Table`` or a streaming
``pa.RecordBatchReader``. Streaming is microbatched by
Accepts either a fully materialised ``pa.Table``, a streaming
``pa.RecordBatchReader``, or an object implementing the Arrow PyCapsule
stream interface, which is imported as a ``pa.Table``. Streaming is microbatched by
``write.target-file-size-bytes`` so memory stays bounded; the reader is
consumed once and cannot be reused.

Expand Down Expand Up @@ -577,8 +579,11 @@ def append(

from pyiceberg.io.pyarrow import _check_pyarrow_schema_compatible, _dataframe_to_data_files

if isinstance(df, ArrowStreamExportable) and not isinstance(df, (pa.Table, pa.RecordBatchReader)):
df = pa.table(df)

if not isinstance(df, (pa.Table, pa.RecordBatchReader)):
raise ValueError(f"Expected pa.Table or pa.RecordBatchReader, got: {df}")
raise ValueError(f"Expected pa.Table, pa.RecordBatchReader, or an Arrow PyCapsule exporter, got: {df}")

downcast_ns_timestamp_to_us = Config().get_bool(DOWNCAST_NS_TIMESTAMP_TO_US_ON_WRITE) or False
_check_pyarrow_schema_compatible(
Expand Down Expand Up @@ -676,7 +681,7 @@ def dynamic_partition_overwrite(

def overwrite(
self,
df: pa.Table | pa.RecordBatchReader,
df: pa.Table | pa.RecordBatchReader | ArrowStreamExportable,
overwrite_filter: BooleanExpression | str = ALWAYS_TRUE,
snapshot_properties: dict[str, str] = EMPTY_DICT,
case_sensitive: bool = True,
Expand All @@ -685,8 +690,9 @@ def overwrite(
"""
Shorthand for adding a table overwrite with a PyArrow table or RecordBatchReader to the transaction.

Accepts either a fully materialised ``pa.Table`` or a streaming
``pa.RecordBatchReader``. Streaming is microbatched by
Accepts either a fully materialised ``pa.Table``, a streaming
``pa.RecordBatchReader``, or an object implementing the Arrow PyCapsule
stream interface, which is imported as a ``pa.Table``. Streaming is microbatched by
``write.target-file-size-bytes`` so memory stays bounded; the reader is
consumed once and cannot be reused.

Expand Down Expand Up @@ -742,8 +748,11 @@ def overwrite(

from pyiceberg.io.pyarrow import _check_pyarrow_schema_compatible, _dataframe_to_data_files

if isinstance(df, ArrowStreamExportable) and not isinstance(df, (pa.Table, pa.RecordBatchReader)):
df = pa.table(df)

if not isinstance(df, (pa.Table, pa.RecordBatchReader)):
raise ValueError(f"Expected pa.Table or pa.RecordBatchReader, got: {df}")
raise ValueError(f"Expected pa.Table, pa.RecordBatchReader, or an Arrow PyCapsule exporter, got: {df}")

downcast_ns_timestamp_to_us = Config().get_bool(DOWNCAST_NS_TIMESTAMP_TO_US_ON_WRITE) or False
_check_pyarrow_schema_compatible(
Expand Down Expand Up @@ -1819,14 +1828,15 @@ def upsert(

def append(
self,
df: pa.Table | pa.RecordBatchReader,
df: pa.Table | pa.RecordBatchReader | ArrowStreamExportable,
snapshot_properties: dict[str, str] = EMPTY_DICT,
branch: str | None = MAIN_BRANCH,
) -> None:
"""
Shorthand API for appending PyArrow data to the table.

Accepts either a ``pa.Table`` or a streaming ``pa.RecordBatchReader``.
Accepts a ``pa.Table``, a streaming ``pa.RecordBatchReader``, or any object
implementing the Arrow PyCapsule stream interface.
See :meth:`Transaction.append` for streaming semantics and partition
limitations.

Expand Down Expand Up @@ -1854,7 +1864,7 @@ def dynamic_partition_overwrite(

def overwrite(
self,
df: pa.Table | pa.RecordBatchReader,
df: pa.Table | pa.RecordBatchReader | ArrowStreamExportable,
overwrite_filter: BooleanExpression | str = ALWAYS_TRUE,
snapshot_properties: dict[str, str] = EMPTY_DICT,
case_sensitive: bool = True,
Expand All @@ -1863,7 +1873,8 @@ def overwrite(
"""
Shorthand for overwriting the table with a PyArrow Table or RecordBatchReader.

Accepts either a ``pa.Table`` or a streaming ``pa.RecordBatchReader``.
Accepts a ``pa.Table``, a streaming ``pa.RecordBatchReader``, or any object
implementing the Arrow PyCapsule stream interface.
See :meth:`Transaction.overwrite` for streaming semantics and partition
limitations.

Expand Down
8 changes: 8 additions & 0 deletions pyiceberg/typedef.py
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,14 @@ def __missing__(self, key: K) -> V:
L = TypeVar("L", str, bool, int, float, bytes, UUID, Decimal, datetime, date, time, covariant=True)


@runtime_checkable
class ArrowStreamExportable(Protocol): # pragma: no cover
"""An object that exports an Arrow stream through the Arrow PyCapsule interface."""

def __arrow_c_stream__(self, requested_schema: object | None = None) -> object:
"""Export this object as an Arrow C stream capsule."""


@runtime_checkable
class StructProtocol(Protocol): # pragma: no cover
"""A generic protocol used by accessors to get and set at positions of an object."""
Expand Down
2 changes: 1 addition & 1 deletion tests/catalog/test_catalog_behaviors.py
Original file line number Diff line number Diff line change
Expand Up @@ -1475,7 +1475,7 @@ def test_append_invalid_input_type_raises(catalog: Catalog) -> None:
identifier = f"default.append_invalid_input_{catalog.name}"
pa_table = _simple_arrow_table()
tbl = catalog.create_table(identifier=identifier, schema=pa_table.schema)
with pytest.raises(ValueError, match="Expected pa.Table or pa.RecordBatchReader"):
with pytest.raises(ValueError, match="Expected pa.Table, pa.RecordBatchReader, or an Arrow PyCapsule exporter"):
tbl.append("not an arrow object")


Expand Down
2 changes: 1 addition & 1 deletion tests/integration/test_writes/test_partitioned_writes.py
Original file line number Diff line number Diff line change
Expand Up @@ -768,7 +768,7 @@ def test_invalid_arguments(spark: SparkSession, session_catalog: Catalog) -> Non
properties={"format-version": "1"},
)

with pytest.raises(ValueError, match="Expected pa.Table or pa.RecordBatchReader, got: not a df"):
with pytest.raises(ValueError, match="Expected pa.Table, pa.RecordBatchReader, or an Arrow PyCapsule exporter"):
tbl.append("not a df")


Expand Down
4 changes: 2 additions & 2 deletions tests/integration/test_writes/test_writes.py
Original file line number Diff line number Diff line change
Expand Up @@ -792,10 +792,10 @@ def test_invalid_arguments(spark: SparkSession, session_catalog: Catalog, arrow_
identifier = "default.arrow_data_files"
tbl = _create_table(session_catalog, identifier, {"format-version": "1"}, [])

with pytest.raises(ValueError, match="Expected pa.Table or pa.RecordBatchReader, got: not a df"):
with pytest.raises(ValueError, match="Expected pa.Table, pa.RecordBatchReader, or an Arrow PyCapsule exporter"):
tbl.overwrite("not a df")

with pytest.raises(ValueError, match="Expected pa.Table or pa.RecordBatchReader, got: not a df"):
with pytest.raises(ValueError, match="Expected pa.Table, pa.RecordBatchReader, or an Arrow PyCapsule exporter"):
tbl.append("not a df")


Expand Down
52 changes: 52 additions & 0 deletions tests/table/test_init.py
Original file line number Diff line number Diff line change
Expand Up @@ -2595,6 +2595,58 @@ def test_overwrite_delete_data_file_on_bucket_partition(catalog: Catalog) -> Non
assert table.scan().to_arrow().num_rows == 0


def test_append_accepts_arrow_capsule_exporter(catalog: Catalog) -> None:
import pyarrow as pa

from pyiceberg.io.pyarrow import schema_to_pyarrow

class ArrowCapsuleTable:
def __init__(self, table: pa.Table) -> None:
self._table = table

def __arrow_c_stream__(self, requested_schema: Any = None) -> Any:
return self._table.__arrow_c_stream__(requested_schema)

catalog.create_namespace("default")
schema = Schema(NestedField(1, "x", IntegerType(), required=False))
table = catalog.create_table("default.arrow_capsule_append", schema=schema)

table.append(ArrowCapsuleTable(pa.table({"x": [1, 2]}, schema=schema_to_pyarrow(schema))))

assert table.scan().to_arrow().to_pylist() == [{"x": 1}, {"x": 2}]


def test_overwrite_accepts_arrow_capsule_exporter(catalog: Catalog) -> None:
import pyarrow as pa

from pyiceberg.io.pyarrow import schema_to_pyarrow

class ArrowCapsuleTable:
def __init__(self, table: pa.Table) -> None:
self._table = table

def __arrow_c_stream__(self, requested_schema: Any = None) -> Any:
return self._table.__arrow_c_stream__(requested_schema)

catalog.create_namespace("default")
schema = Schema(NestedField(1, "x", IntegerType(), required=False))
table = catalog.create_table("default.arrow_capsule_overwrite", schema=schema)
table.append(pa.table({"x": [1, 2]}, schema=schema_to_pyarrow(schema)))

table.overwrite(ArrowCapsuleTable(pa.table({"x": [3]}, schema=schema_to_pyarrow(schema))))

assert table.scan().to_arrow().to_pylist() == [{"x": 3}]


def test_append_rejects_object_without_arrow_capsule(catalog: Catalog) -> None:
catalog.create_namespace("default")
schema = Schema(NestedField(1, "x", IntegerType(), required=False))
table = catalog.create_table("default.arrow_capsule_rejected", schema=schema)

with pytest.raises(ValueError, match="Arrow PyCapsule"):
table.append(object()) # type: ignore[arg-type]


def test_partial_delete_keeps_earlier_deletes_outside_the_filter(catalog: Catalog) -> None:
"""Rewriting a file applies all of its delete files, not only those whose bounds can match the delete filter."""
import pyarrow as pa
Expand Down
Loading