From e6c8619d9e66d28fc13784544c7edfd88ce833f6 Mon Sep 17 00:00:00 2001 From: Gayathri Srividya Rajavarapu Date: Fri, 20 Nov 2026 21:30:08 +0530 Subject: [PATCH 1/2] Fix duplicate file checks after commit retry Revalidate add_files duplicate paths against the refreshed branch snapshot before rebuilding a retried snapshot. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- pyiceberg/table/__init__.py | 19 +++++++++++++++---- pyiceberg/table/update/snapshot.py | 9 +++++++++ tests/table/test_commit_retry.py | 24 ++++++++++++++++++++++++ 3 files changed, 48 insertions(+), 4 deletions(-) diff --git a/pyiceberg/table/__init__.py b/pyiceberg/table/__init__.py index 3ea41e3f22..f011316a09 100644 --- a/pyiceberg/table/__init__.py +++ b/pyiceberg/table/__init__.py @@ -1031,9 +1031,9 @@ def upsert( return UpsertResult(rows_updated=update_row_cnt, rows_inserted=insert_row_cnt) - def _find_referenced_data_files(self, file_paths: list[str]) -> list[str]: + def _find_referenced_data_files(self, file_paths: list[str], branch: str | None = MAIN_BRANCH) -> list[str]: """Return file_paths already referenced by data files in the current snapshot.""" - snapshot = self.table_metadata.current_snapshot() + snapshot = self.table_metadata.snapshot_by_name(branch) if snapshot is None: return [] @@ -1075,7 +1075,7 @@ def add_files( raise ValueError("File paths must be unique") if check_duplicate_files: - referenced_files = self._find_referenced_data_files(file_paths) + referenced_files = self._find_referenced_data_files(file_paths, branch) if referenced_files: raise ValueError(f"Cannot add files that are already referenced by table, files: {', '.join(referenced_files)}") @@ -1084,6 +1084,16 @@ def add_files( **{TableProperties.DEFAULT_NAME_MAPPING: self.table_metadata.schema().name_mapping.model_dump_json()} ) with self._append_snapshot_producer(snapshot_properties, branch=branch) as append_files: + if check_duplicate_files: + + def validate_duplicate_files() -> None: + referenced_files = self._find_referenced_data_files(file_paths, branch) + if referenced_files: + raise ValueError( + f"Cannot add files that are already referenced by table, files: {', '.join(referenced_files)}" + ) + + append_files._set_retry_validator(validate_duplicate_files) data_files = _parquet_files_to_data_files( table_metadata=self.table_metadata, file_paths=file_paths, io=self._table.io ) @@ -1200,7 +1210,7 @@ def commit_transaction(self) -> Table: self._cleanup_uncommitted_manifests() break self._rebuild_snapshot_updates() - except (CommitFailedException, ValidationException): + except (CommitFailedException, ValidationException, ValueError): # These exceptions guarantee the commit did not land, so it is safe to delete the # files written for it. Any other exception (unknown outcome, or a commit that already # succeeded) is re-raised without deleting, since those files may be referenced by the @@ -1285,6 +1295,7 @@ def _rebuild_snapshot_updates(self) -> None: self._resolve_commit_windows() for producer in self._snapshot_producers: producer._refresh_for_retry() + producer._validate_retry() producer._validate_concurrency() updates, requirements = producer._commit() self._stage(updates, requirements) diff --git a/pyiceberg/table/update/snapshot.py b/pyiceberg/table/update/snapshot.py index 0c9831450e..51c0eef0b8 100644 --- a/pyiceberg/table/update/snapshot.py +++ b/pyiceberg/table/update/snapshot.py @@ -152,6 +152,7 @@ class _SnapshotProducer(UpdateTableMetadata[U], Generic[U]): _uncommitted_manifests: list[str] _written_manifest_lists: list[str] _isolation_operation: Operation + _retry_validator: Callable[[], None] | None def __init__( self, @@ -186,6 +187,14 @@ def __init__( self._case_sensitive = True self._commit_window = None self._isolation_operation: Operation = Operation.DELETE + self._retry_validator = None + + def _set_retry_validator(self, validator: Callable[[], None]) -> None: + self._retry_validator = validator + + def _validate_retry(self) -> None: + if self._retry_validator is not None: + self._retry_validator() def _validate_target_branch(self, branch: str | None) -> str | None: # if branch is none, write will be written into a staging snapshot diff --git a/tests/table/test_commit_retry.py b/tests/table/test_commit_retry.py index 67653be957..c26dbd8e94 100644 --- a/tests/table/test_commit_retry.py +++ b/tests/table/test_commit_retry.py @@ -15,9 +15,11 @@ # specific language governing permissions and limitations # under the License. import uuid +from pathlib import Path from typing import Any from unittest.mock import patch +import pyarrow.parquet as pq import pytest from pyiceberg.catalog import Catalog @@ -97,6 +99,28 @@ def counting_rebuild(self_tx: Any) -> None: assert len(result) == 6 +def test_concurrent_add_files_retry_rechecks_duplicate_paths(catalog: Catalog, tmp_path: Path) -> None: + """A retried add_files call rejects a file committed concurrently.""" + catalog.create_namespace("default") + catalog.create_table("default.add_files_retry_test", schema=_test_schema()) + + import pyarrow as pa + + data_file = tmp_path / "data.parquet" + pq.write_table(pa.table({"x": [1, 2]}), data_file) + + tbl1 = catalog.load_table("default.add_files_retry_test") + tbl2 = catalog.load_table("default.add_files_retry_test") + + tbl1.add_files([str(data_file)]) + + with pytest.raises(ValueError, match="Cannot add files that are already referenced by table"): + tbl2.add_files([str(data_file)]) + + refreshed = catalog.load_table("default.add_files_retry_test") + assert refreshed.scan().to_arrow()["x"].to_pylist() == [1, 2] + + def test_no_retry_without_snapshot_producers(catalog: Catalog) -> None: """Verify that a transaction with no snapshot producers has an empty producer list.""" catalog.create_namespace("default") From 1fb97d5013f0d99cf3f8545094737b8ecc292137 Mon Sep 17 00:00:00 2001 From: Gayathri Srividya Rajavarapu Date: Sat, 21 Nov 2026 13:53:35 +0530 Subject: [PATCH 2/2] Merge retry validation into concurrency checks --- pyiceberg/table/__init__.py | 1 - pyiceberg/table/update/snapshot.py | 8 +++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/pyiceberg/table/__init__.py b/pyiceberg/table/__init__.py index f011316a09..2c9417b1ff 100644 --- a/pyiceberg/table/__init__.py +++ b/pyiceberg/table/__init__.py @@ -1295,7 +1295,6 @@ def _rebuild_snapshot_updates(self) -> None: self._resolve_commit_windows() for producer in self._snapshot_producers: producer._refresh_for_retry() - producer._validate_retry() producer._validate_concurrency() updates, requirements = producer._commit() self._stage(updates, requirements) diff --git a/pyiceberg/table/update/snapshot.py b/pyiceberg/table/update/snapshot.py index 51c0eef0b8..7a37b76287 100644 --- a/pyiceberg/table/update/snapshot.py +++ b/pyiceberg/table/update/snapshot.py @@ -192,9 +192,11 @@ def __init__( def _set_retry_validator(self, validator: Callable[[], None]) -> None: self._retry_validator = validator - def _validate_retry(self) -> None: + def _validate_concurrency(self) -> None: + """Validate retry-specific invariants and conflicts with concurrent changes.""" if self._retry_validator is not None: self._retry_validator() + self._validate_concurrent_changes() def _validate_target_branch(self, branch: str | None) -> str | None: # if branch is none, write will be written into a staging snapshot @@ -477,7 +479,7 @@ def _refresh_for_retry(self) -> None: self._manifest_num_counter = itertools.count(0) self.commit_uuid = uuid.uuid4() - def _validate_concurrency(self) -> None: + def _validate_concurrent_changes(self) -> None: """Validate that concurrent changes do not conflict with this operation. Uses the CommitWindow to determine which catalog commits to validate against. @@ -699,7 +701,7 @@ def _refresh_for_retry(self) -> None: class _FastAppendFiles(_SnapshotProducer["_FastAppendFiles"]): - def _validate_concurrency(self) -> None: + def _validate_concurrent_changes(self) -> None: """Skip validation; appends do not conflict with other operations.""" def _existing_manifests(self) -> list[ManifestFile]: