Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 14 additions & 4 deletions pyiceberg/table/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -1031,9 +1031,9 @@ def upsert(

return UpsertResult(rows_updated=update_row_cnt, rows_inserted=insert_row_cnt)

def _find_referenced_data_files(self, file_paths: list[str]) -> list[str]:
def _find_referenced_data_files(self, file_paths: list[str], branch: str | None = MAIN_BRANCH) -> list[str]:
"""Return file_paths already referenced by data files in the current snapshot."""
snapshot = self.table_metadata.current_snapshot()
snapshot = self.table_metadata.snapshot_by_name(branch)
if snapshot is None:
return []

Expand Down Expand Up @@ -1075,7 +1075,7 @@ def add_files(
raise ValueError("File paths must be unique")

if check_duplicate_files:
referenced_files = self._find_referenced_data_files(file_paths)
referenced_files = self._find_referenced_data_files(file_paths, branch)
if referenced_files:
raise ValueError(f"Cannot add files that are already referenced by table, files: {', '.join(referenced_files)}")

Expand All @@ -1084,6 +1084,16 @@ def add_files(
**{TableProperties.DEFAULT_NAME_MAPPING: self.table_metadata.schema().name_mapping.model_dump_json()}
)
with self._append_snapshot_producer(snapshot_properties, branch=branch) as append_files:
if check_duplicate_files:

def validate_duplicate_files() -> None:
referenced_files = self._find_referenced_data_files(file_paths, branch)
if referenced_files:
raise ValueError(
f"Cannot add files that are already referenced by table, files: {', '.join(referenced_files)}"
)

append_files._set_retry_validator(validate_duplicate_files)
data_files = _parquet_files_to_data_files(
table_metadata=self.table_metadata, file_paths=file_paths, io=self._table.io
)
Expand Down Expand Up @@ -1200,7 +1210,7 @@ def commit_transaction(self) -> Table:
self._cleanup_uncommitted_manifests()
break
self._rebuild_snapshot_updates()
except (CommitFailedException, ValidationException):
except (CommitFailedException, ValidationException, ValueError):
# These exceptions guarantee the commit did not land, so it is safe to delete the
# files written for it. Any other exception (unknown outcome, or a commit that already
# succeeded) is re-raised without deleting, since those files may be referenced by the
Expand Down
15 changes: 13 additions & 2 deletions pyiceberg/table/update/snapshot.py
Original file line number Diff line number Diff line change
Expand Up @@ -152,6 +152,7 @@ class _SnapshotProducer(UpdateTableMetadata[U], Generic[U]):
_uncommitted_manifests: list[str]
_written_manifest_lists: list[str]
_isolation_operation: Operation
_retry_validator: Callable[[], None] | None

def __init__(
self,
Expand Down Expand Up @@ -186,6 +187,16 @@ def __init__(
self._case_sensitive = True
self._commit_window = None
self._isolation_operation: Operation = Operation.DELETE
self._retry_validator = None

def _set_retry_validator(self, validator: Callable[[], None]) -> None:
self._retry_validator = validator

def _validate_concurrency(self) -> None:
"""Validate retry-specific invariants and conflicts with concurrent changes."""
if self._retry_validator is not None:
self._retry_validator()
self._validate_concurrent_changes()

def _validate_target_branch(self, branch: str | None) -> str | None:
# if branch is none, write will be written into a staging snapshot
Expand Down Expand Up @@ -468,7 +479,7 @@ def _refresh_for_retry(self) -> None:
self._manifest_num_counter = itertools.count(0)
self.commit_uuid = uuid.uuid4()

def _validate_concurrency(self) -> None:
def _validate_concurrent_changes(self) -> None:
"""Validate that concurrent changes do not conflict with this operation.

Uses the CommitWindow to determine which catalog commits to validate against.
Expand Down Expand Up @@ -690,7 +701,7 @@ def _refresh_for_retry(self) -> None:


class _FastAppendFiles(_SnapshotProducer["_FastAppendFiles"]):
def _validate_concurrency(self) -> None:
def _validate_concurrent_changes(self) -> None:
"""Skip validation; appends do not conflict with other operations."""

def _existing_manifests(self) -> list[ManifestFile]:
Expand Down
24 changes: 24 additions & 0 deletions tests/table/test_commit_retry.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,9 +15,11 @@
# specific language governing permissions and limitations
# under the License.
import uuid
from pathlib import Path
from typing import Any
from unittest.mock import patch

import pyarrow.parquet as pq
import pytest

from pyiceberg.catalog import Catalog
Expand Down Expand Up @@ -97,6 +99,28 @@ def counting_rebuild(self_tx: Any) -> None:
assert len(result) == 6


def test_concurrent_add_files_retry_rechecks_duplicate_paths(catalog: Catalog, tmp_path: Path) -> None:
"""A retried add_files call rejects a file committed concurrently."""
catalog.create_namespace("default")
catalog.create_table("default.add_files_retry_test", schema=_test_schema())

import pyarrow as pa

data_file = tmp_path / "data.parquet"
pq.write_table(pa.table({"x": [1, 2]}), data_file)

tbl1 = catalog.load_table("default.add_files_retry_test")
tbl2 = catalog.load_table("default.add_files_retry_test")

tbl1.add_files([str(data_file)])

with pytest.raises(ValueError, match="Cannot add files that are already referenced by table"):
tbl2.add_files([str(data_file)])

refreshed = catalog.load_table("default.add_files_retry_test")
assert refreshed.scan().to_arrow()["x"].to_pylist() == [1, 2]


def test_no_retry_without_snapshot_producers(catalog: Catalog) -> None:
"""Verify that a transaction with no snapshot producers has an empty producer list."""
catalog.create_namespace("default")
Expand Down
Loading