From 515fab1e8129c7697f8c9d571529600d8cb7e7c4 Mon Sep 17 00:00:00 2001 From: Anurag Singh Date: Sat, 21 Nov 2026 02:13:31 +0530 Subject: [PATCH 1/2] Core: Add delete_files for removing data files by path Tables can add existing data files but had no way to remove them, which is the remaining gap with Java DeleteFiles. Add Table.delete_files and Transaction.delete_files taking full file paths, validating that each path is unique and still referenced by the target branch, and committing a copy-on-write delete snapshot with optional properties. --- mkdocs/docs/api.md | 27 +++++++++++++++ pyiceberg/table/__init__.py | 69 +++++++++++++++++++++++++++++++++++++ tests/table/test_init.py | 41 ++++++++++++++++++++++ 3 files changed, 137 insertions(+) diff --git a/mkdocs/docs/api.md b/mkdocs/docs/api.md index aa7c111329..6283b92bea 100644 --- a/mkdocs/docs/api.md +++ b/mkdocs/docs/api.md @@ -1156,6 +1156,33 @@ assert tbl.name_mapping() is not None assert tbl.metadata.snapshots[-1].summary["abc"] == "def" ``` +## Delete Files + +Data files can be removed from the current snapshot by path without rewriting the remaining files. Every path must reference a live data file in the target branch. + +| Parameter | Required? | Type | Description | +| ----------------------- | --------- | -------------- | ----------------------------------------------------------------------- | +| `file_paths` | ✔️ | List[str] | The full paths of the data files to remove | +| `snapshot_properties` | | Dict[str, str] | Properties to set for the new snapshot. Defaults to an empty dictionary | +| `branch` | | str | Branch to remove the files from. Defaults to `main` | + +### Example + +```python +# Assume an existing Iceberg table object `tbl` + +file_paths = [ + "s3a://warehouse/default/existing-1.parquet", +] + +tbl.delete_files(file_paths=file_paths) + +# The files are no longer part of the current snapshot and their rows are not scanned +``` + +!!! note "Concurrent changes" + If a file is no longer referenced when the delete commits, the commit fails. Reload the table and retry. + ## Schema evolution PyIceberg supports full schema evolution through the Python API. It takes care of setting the field-IDs and makes sure that only non-breaking changes are done (can be overridden). diff --git a/pyiceberg/table/__init__.py b/pyiceberg/table/__init__.py index 3ea41e3f22..6b73fb56d9 100644 --- a/pyiceberg/table/__init__.py +++ b/pyiceberg/table/__init__.py @@ -1090,6 +1090,56 @@ def add_files( for data_file in data_files: append_files.append_data_file(data_file) + def delete_files( + self, + file_paths: list[str], + snapshot_properties: dict[str, str] = EMPTY_DICT, + branch: str | None = MAIN_BRANCH, + ) -> None: + """Remove data files from the table transaction. + + Args: + file_paths: The full paths of the data files to remove. + snapshot_properties: Custom properties to be added to the snapshot summary. + branch: Branch Reference to run the delete operation. + + Raises: + ValueError: If a path is duplicated, or is not referenced by the table. + """ + if not file_paths: + return + + if len(file_paths) != len(set(file_paths)): + raise ValueError("File paths must be unique") + + snapshot = self.table_metadata.snapshot_by_name(branch) if branch is not None else self.table_metadata.current_snapshot() + if snapshot is None: + raise ValueError("Cannot delete files from a table without a snapshot") + + candidates = set(file_paths) + io = self._table.io + data_manifests = [manifest for manifest in snapshot.manifests(io) if manifest.content == ManifestContent.DATA] + + def path_filter(data_file: DataFile) -> bool: + return data_file.file_path in candidates + + executor = ExecutorFactory.get_or_create() + entries = chain.from_iterable( + executor.map( + lambda args: _open_manifest(*args), + [(io, manifest, path_filter, lambda _: True) for manifest in data_manifests], + ) + ) + data_files = {entry.data_file.file_path: entry.data_file for entry in entries} + + missing = candidates - data_files.keys() + if missing: + raise ValueError(f"Cannot delete files that are not referenced by table, files: {', '.join(sorted(missing))}") + + with self.update_snapshot(snapshot_properties=snapshot_properties, branch=branch).overwrite() as overwrite: + for data_file in data_files.values(): + overwrite.delete_data_file(data_file) + def update_spec(self) -> UpdateSpec: """Create a new UpdateSpec to update the partitioning of the table. @@ -1935,6 +1985,25 @@ def add_files( branch=branch, ) + def delete_files( + self, + file_paths: list[str], + snapshot_properties: dict[str, str] = EMPTY_DICT, + branch: str | None = MAIN_BRANCH, + ) -> None: + """Shorthand API for removing data files from the table. + + Args: + file_paths: The full paths of the data files to remove. + snapshot_properties: Custom properties to be added to the snapshot summary. + branch: Branch Reference to run the delete operation. + + Raises: + ValueError: If a path is duplicated, or is not referenced by the table. + """ + with self.transaction() as tx: + tx.delete_files(file_paths=file_paths, snapshot_properties=snapshot_properties, branch=branch) + def update_spec(self, case_sensitive: bool = True) -> UpdateSpec: return UpdateSpec(Transaction(self, autocommit=True), case_sensitive=case_sensitive) diff --git a/tests/table/test_init.py b/tests/table/test_init.py index 547ca94fc9..7e3d71476d 100644 --- a/tests/table/test_init.py +++ b/tests/table/test_init.py @@ -2595,6 +2595,47 @@ def test_overwrite_delete_data_file_on_bucket_partition(catalog: Catalog) -> Non assert table.scan().to_arrow().num_rows == 0 +def test_delete_files_removes_data_files(catalog: Catalog) -> None: + import pyarrow as pa + + from pyiceberg.io.pyarrow import schema_to_pyarrow + + catalog.create_namespace("default") + schema = Schema(NestedField(1, "x", IntegerType(), required=False)) + table = catalog.create_table("default.delete_files", schema=schema) + table.append(pa.table({"x": [1, 2]}, schema=schema_to_pyarrow(schema))) + table.append(pa.table({"x": [3]}, schema=schema_to_pyarrow(schema))) + table = catalog.load_table("default.delete_files") + + files = table.inspect.files().to_pylist() + assert len(files) == 2 + file_to_delete = next(data_file["file_path"] for data_file in files if data_file["record_count"] == 1) + + table.delete_files([file_to_delete], snapshot_properties={"source": "test"}) + + table = catalog.load_table("default.delete_files") + assert table.scan().to_arrow().to_pylist() == [{"x": 1}, {"x": 2}] + assert table.current_snapshot().summary.additional_properties["source"] == "test" + + +def test_delete_files_rejects_missing_and_duplicate_paths(catalog: Catalog) -> None: + import pyarrow as pa + + from pyiceberg.io.pyarrow import schema_to_pyarrow + + catalog.create_namespace("default") + schema = Schema(NestedField(1, "x", IntegerType(), required=False)) + table = catalog.create_table("default.delete_files_errors", schema=schema) + table.append(pa.table({"x": [1]}, schema=schema_to_pyarrow(schema))) + file_path = next(data_file["file_path"] for data_file in table.inspect.files().to_pylist()) + + with pytest.raises(ValueError, match="File paths must be unique"): + table.delete_files([file_path, file_path]) + + with pytest.raises(ValueError, match="not referenced by table"): + table.delete_files(["file:///missing.parquet"]) + + def test_partial_delete_keeps_earlier_deletes_outside_the_filter(catalog: Catalog) -> None: """Rewriting a file applies all of its delete files, not only those whose bounds can match the delete filter.""" import pyarrow as pa From 0d956cfdd04f5abedf1ef12fd79f2cd6e99180c6 Mon Sep 17 00:00:00 2001 From: Anurag Singh Date: Sat, 21 Nov 2026 03:06:23 +0530 Subject: [PATCH 2/2] Assert the delete snapshot exists before reading its properties --- tests/table/test_init.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/table/test_init.py b/tests/table/test_init.py index 7e3d71476d..33d173e36b 100644 --- a/tests/table/test_init.py +++ b/tests/table/test_init.py @@ -2615,7 +2615,10 @@ def test_delete_files_removes_data_files(catalog: Catalog) -> None: table = catalog.load_table("default.delete_files") assert table.scan().to_arrow().to_pylist() == [{"x": 1}, {"x": 2}] - assert table.current_snapshot().summary.additional_properties["source"] == "test" + snapshot = table.current_snapshot() + assert snapshot is not None + assert snapshot.summary is not None + assert snapshot.summary.additional_properties["source"] == "test" def test_delete_files_rejects_missing_and_duplicate_paths(catalog: Catalog) -> None: