Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions mkdocs/docs/api.md
Original file line number Diff line number Diff line change
Expand Up @@ -311,8 +311,8 @@ Next, write the data to the table. Both `append` and `overwrite` produce the sam

<!-- prettier-ignore-start -->

!!! note inline end "Fast append"
PyIceberg defaults to the [fast append](https://iceberg.apache.org/spec/#snapshots) to minimize the amount of data written. This enables fast commit operations, reducing the possibility of conflicts. The downside of the fast append is that it creates more metadata than a merge commit. [Compaction is planned](https://github.com/apache/iceberg-python/issues/270) and will automatically rewrite all the metadata when a threshold is hit, to maintain performant reads.
!!! note inline end "Merge and fast append"
PyIceberg defaults to [merge append](https://iceberg.apache.org/spec/#snapshots), which automatically merges manifests when the configured threshold is reached to keep metadata compact and reads performant. Fast append remains available by setting the table property `commit.manifest-merge.enabled` to `False`. Fast append minimizes the metadata written during a commit, enabling faster commits and reducing the possibility of conflicts, but accumulates more manifest metadata over time.

<!-- prettier-ignore-end -->

Expand Down
6 changes: 3 additions & 3 deletions mkdocs/docs/configuration.md
Original file line number Diff line number Diff line change
Expand Up @@ -106,12 +106,12 @@ Iceberg tables support table properties to configure table behavior.
| ------------------------------------ | ------------------- | ------------- | ----------------------------------------------------------- |
| `commit.manifest.target-size-bytes` | Size in bytes | 8388608 (8MB) | Target size when merging manifest files |
| `commit.manifest.min-count-to-merge` | Number of manifests | 100 | Minimum number of manifests to accumulate before merging |
| `commit.manifest-merge.enabled` | Boolean | False | Controls whether to automatically merge manifests on writes |
| `commit.manifest-merge.enabled` | Boolean | True | Controls whether to automatically merge manifests on writes |

<!-- prettier-ignore-start -->

!!! note "Fast append"
Unlike Java implementation, PyIceberg default to the [fast append](api.md#write-to-a-table) and thus `commit.manifest-merge.enabled` is set to `False` by default.
!!! note "Append modes"
PyIceberg defaults to [merge append](api.md#write-to-a-table) to keep manifest metadata compact. Set `commit.manifest-merge.enabled` to `False` on a table to use fast append, which writes less metadata during each commit but accumulates more manifests over time.

<!-- prettier-ignore-end -->

Expand Down
2 changes: 1 addition & 1 deletion pyiceberg/table/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -226,7 +226,7 @@ class TableProperties:
MANIFEST_MIN_MERGE_COUNT_DEFAULT = 100

MANIFEST_MERGE_ENABLED = "commit.manifest-merge.enabled"
MANIFEST_MERGE_ENABLED_DEFAULT = False
MANIFEST_MERGE_ENABLED_DEFAULT = True

METADATA_COMPRESSION = "write.metadata.compression-codec"
METADATA_COMPRESSION_DEFAULT = "none"
Expand Down
7 changes: 5 additions & 2 deletions tests/integration/test_writes/test_partitioned_writes.py
Original file line number Diff line number Diff line change
Expand Up @@ -748,8 +748,11 @@ def test_dynamic_partition_overwrite_evolve_partition(spark: SparkSession, sessi
tbl.dynamic_partition_overwrite(arrow_table)
result = tbl.scan().to_arrow()

assert result["place"].to_pylist() == ["Groningen", "Amsterdam", "Drachten"]
assert result["inhabitants"].to_pylist() == [238147, 921402, 44940]
assert sorted(zip(result["place"].to_pylist(), result["inhabitants"].to_pylist(), strict=True)) == [
("Amsterdam", 921402),
("Drachten", 44940),
("Groningen", 238147),
]


@pytest.mark.integration
Expand Down
8 changes: 6 additions & 2 deletions tests/integration/test_writes/test_writes.py
Original file line number Diff line number Diff line change
Expand Up @@ -1572,7 +1572,7 @@ def test_merge_manifests(session_catalog: Catalog, arrow_table_with_null: pa.Tab
tbl_a = _create_table(
session_catalog,
"default.merge_manifest_a",
{"commit.manifest-merge.enabled": "true", "commit.manifest.min-count-to-merge": "1", "format-version": format_version},
{"commit.manifest.min-count-to-merge": "1", "format-version": format_version},
[],
)
tbl_b = _create_table(
Expand All @@ -1589,7 +1589,11 @@ def test_merge_manifests(session_catalog: Catalog, arrow_table_with_null: pa.Tab
tbl_c = _create_table(
session_catalog,
"default.merge_manifest_c",
{"commit.manifest.min-count-to-merge": "1", "format-version": format_version},
{
"commit.manifest-merge.enabled": "false",
"commit.manifest.min-count-to-merge": "1",
"format-version": format_version,
},
[],
)

Expand Down
19 changes: 19 additions & 0 deletions tests/table/test_snapshots.py
Original file line number Diff line number Diff line change
Expand Up @@ -814,6 +814,25 @@ def test_overwrite_rejects_explicit_delete_missing_from_base_snapshot(catalog: C
assert _total_data_file_count(committed) == 0


def test_append_producer_defaults_to_merge_append(table_v2: Table) -> None:
from pyiceberg.table.update.snapshot import _MergeAppendFiles

append = table_v2.transaction()._append_snapshot_producer({})

assert isinstance(append, _MergeAppendFiles)


def test_append_producer_uses_fast_append_when_merge_disabled(table_v2: Table) -> None:
from pyiceberg.table.update.snapshot import _FastAppendFiles

transaction = table_v2.transaction()
transaction.set_properties({"commit.manifest-merge.enabled": "false"})

append = transaction._append_snapshot_producer({})

assert isinstance(append, _FastAppendFiles)


def test_overwrite_rejects_explicit_delete_without_parent_snapshot(
catalog: Catalog, overwrite_table: Table, arrow_table_simple: pa.Table
) -> None:
Expand Down
Loading