From fe57f68599180ea49ec7c4f6ef84f1db9fc195f3 Mon Sep 17 00:00:00 2001 From: Daniel Weindl Date: Tue, 17 Nov 2026 23:24:48 +0100 Subject: [PATCH 1/2] v2: check for missing required columns when loading tables Creating v2 tables from DataFrames or TSV files now raises `MissingColumnsError` if required columns are missing, naming the table, the missing columns, and the file. Previously, this resulted in unspecific pydantic errors or `KeyError`s, or was silently accepted (e.g., a missing `estimate` column defaulted to `True`). `petablint` reports this error without a traceback. `BaseTable.from_df` now handles `None`/empty input, moves named index levels to columns, and checks required columns before delegating to the new abstract `_from_df`. The sciml `HybridizationTable` gets the same check. Relates to #369. Co-Authored-By: Claude Opus 5.5 --- changelog.d/+v2-missing-columns.breaking.md | 7 ++ petab/petablint.py | 4 + petab/v2/core.py | 103 ++++++++++++++------ petab/v2/extensions/sciml.py | 5 + tests/v2/test_core.py | 81 +++++++++++++++ tests/v2/test_sciml.py | 16 +++ 6 files changed, 186 insertions(+), 30 deletions(-) create mode 100644 changelog.d/+v2-missing-columns.breaking.md diff --git a/changelog.d/+v2-missing-columns.breaking.md b/changelog.d/+v2-missing-columns.breaking.md new file mode 100644 index 00000000..f0827e63 --- /dev/null +++ b/changelog.d/+v2-missing-columns.breaking.md @@ -0,0 +1,7 @@ +v2: Creating a table via `from_df` or `from_tsv` (including the +`Problem.*_df` setters) now raises `petab.v2.MissingColumnsError` if required +columns are missing, naming the table, the missing columns, and the file. +Previously, this led to unspecific errors or, e.g., for missing `estimate` or +`experimentId` columns, was silently accepted. `petablint` reports this error +without a traceback +([#XXX](https://github.com/PEtab-dev/libpetab-python/pull/XXX)) diff --git a/petab/petablint.py b/petab/petablint.py index 4c03b172..78846547 100755 --- a/petab/petablint.py +++ b/petab/petablint.py @@ -202,6 +202,7 @@ def main(): ret = petab.lint.lint_problem(problem) sys.exit(ret) case 2: + from petab.v2.core import MissingColumnsError from petab.v2.lint import ( ValidationIssueSeverity, lint_problem, @@ -232,6 +233,9 @@ def main(): # https://github.com/pydantic/pydantic/issues/7224 logger.error(f"Error in field(s) `{loc}`: {msg}") sys.exit(1) + except MissingColumnsError as e: + logger.error(str(e)) + sys.exit(1) case _: logger.error( diff --git a/petab/v2/core.py b/petab/v2/core.py index c719811a..f7f1c231 100644 --- a/petab/v2/core.py +++ b/petab/v2/core.py @@ -73,6 +73,7 @@ "MappingTable", "Measurement", "MeasurementTable", + "MissingColumnsError", "NoiseDistribution", "Observable", "ObservableTable", @@ -87,6 +88,10 @@ logger = logging.getLogger(__name__) +class MissingColumnsError(ValueError): + """A PEtab table lacks required columns.""" + + def _field_to_column(info: ValidationInfo) -> str: """The PEtab column name for the validated field.""" return {"lb": C.LOWER_BOUND, "ub": C.UPPER_BOUND}.get( @@ -138,6 +143,43 @@ def _reset_index(df: pd.DataFrame) -> pd.DataFrame: return df.reset_index(drop=all(name is None for name in df.index.names)) +def _prepare_table_df( + df: pd.DataFrame | None, + table_name: str, + required_columns: list[str], + rel_path: str | Path | AnyUrl | None = None, + base_path: str | Path | AnyUrl | None = None, + **_, +) -> pd.DataFrame | None: + """Prepare a DataFrame for creating a PEtab table. + + Moves named index levels to columns and checks for required columns. + + :param df: The table. + :param table_name: The table name for the error message. + :param required_columns: The required columns. + :param rel_path: The table file, for the error message. + :param base_path: The base path of ``rel_path``. + :return: The prepared DataFrame, or ``None`` if ``df`` is ``None`` or + has no columns, i.e., the table is empty. + :raises MissingColumnsError: If any required column is missing. + """ + if df is None: + return None + df = _reset_index(df) + if df.columns.empty: + return None + if missing := [c for c in required_columns if c not in df.columns]: + msg = ( + f"{table_name} is missing required column(s): " + f"{', '.join(missing)}." + ) + if rel_path is not None: + msg += f" File: {_generate_path(rel_path, base_path)}" + raise MissingColumnsError(msg) + return df + + def _column_name(name: str, field: FieldInfo) -> str: """Get the table column name for a model field. @@ -301,10 +343,30 @@ def __getitem__(self, id_: str) -> T: raise KeyError(f"{T.__name__} ID {id_} not found") + @classmethod + def from_df(cls, df: pd.DataFrame | None, **kwargs) -> BaseTable[T]: + """Create a table from a DataFrame. + + :param df: The table. Named index levels are treated as columns. + ``None`` or a DataFrame without any columns gives an empty table. + :param kwargs: Passed to the constructor + (e.g., ``rel_path``, ``base_path``). + :raises MissingColumnsError: If any required column is missing. + """ + df = _prepare_table_df( + df, cls.__name__, cls._required_columns, **kwargs + ) + if df is None: + return cls(**kwargs) + return cls._from_df(df, **kwargs) + @classmethod @abstractmethod - def from_df(cls, df: pd.DataFrame, **kwargs) -> BaseTable[T]: - """Create a table from a DataFrame.""" + def _from_df(cls, df: pd.DataFrame, **kwargs) -> BaseTable[T]: + """Create a table from a DataFrame with all required columns. + + ``df`` has no named index levels. + """ def to_df( self, *, columns: TableColumns | str | None = None @@ -523,12 +585,10 @@ def free_symbols(self) -> set[sp.Symbol]: ) @classmethod - def from_df(cls, df: pd.DataFrame, **kwargs) -> ObservableTable: + def _from_df(cls, df: pd.DataFrame, **kwargs) -> ObservableTable: """Create an ObservableTable from a DataFrame.""" - if df is None: - return cls(**kwargs) - df = get_observable_df(df.copy()) + df = get_observable_df(df) observables = [ Observable(**row.to_dict()) for _, row in _reset_index(df).iterrows() @@ -654,11 +714,8 @@ def conditions(self) -> list[Condition]: return self.elements @classmethod - def from_df(cls, df: pd.DataFrame, **kwargs) -> ConditionTable: + def _from_df(cls, df: pd.DataFrame, **kwargs) -> ConditionTable: """Create a ConditionTable from a DataFrame.""" - if df is None or df.empty: - return cls(**kwargs) - conditions = [] for condition_id, sub_df in df.groupby(C.CONDITION_ID): changes = [ @@ -817,10 +874,8 @@ def experiments(self) -> list[Experiment]: return self.elements @classmethod - def from_df(cls, df: pd.DataFrame, **kwargs) -> ExperimentTable: + def _from_df(cls, df: pd.DataFrame, **kwargs) -> ExperimentTable: """Create an ExperimentTable from a DataFrame.""" - if df is None: - return cls(**kwargs) experiments = [] for experiment_id, cur_exp_df in df.groupby(C.EXPERIMENT_ID): @@ -952,12 +1007,9 @@ def measurements(self) -> list[Measurement]: return self.elements @classmethod - def from_df(cls, df: pd.DataFrame, **kwargs) -> MeasurementTable: + def _from_df(cls, df: pd.DataFrame, **kwargs) -> MeasurementTable: """Create a MeasurementTable from a DataFrame.""" - if df is None: - return cls(**kwargs) - df = _reset_index(df) if C.MODEL_ID in df.columns: df[C.MODEL_ID] = df[C.MODEL_ID].apply(_convert_nan_to_none) @@ -1036,14 +1088,10 @@ def mappings(self) -> list[Mapping]: return self.elements @classmethod - def from_df(cls, df: pd.DataFrame, **kwargs) -> MappingTable: + def _from_df(cls, df: pd.DataFrame, **kwargs) -> MappingTable: """Create a MappingTable from a DataFrame.""" - if df is None: - return cls(**kwargs) - mappings = [ - Mapping(**row.to_dict()) for _, row in _reset_index(df).iterrows() - ] + mappings = [Mapping(**row.to_dict()) for _, row in df.iterrows()] return cls(mappings, **kwargs) def _to_df(self) -> pd.DataFrame: @@ -1261,15 +1309,10 @@ def parameters(self) -> list[Parameter]: return self.elements @classmethod - def from_df(cls, df: pd.DataFrame, **kwargs) -> ParameterTable: + def _from_df(cls, df: pd.DataFrame, **kwargs) -> ParameterTable: """Create a ParameterTable from a DataFrame.""" - if df is None: - return cls(**kwargs) - parameters = [ - Parameter(**row.to_dict()) - for _, row in _reset_index(df).iterrows() - ] + parameters = [Parameter(**row.to_dict()) for _, row in df.iterrows()] return cls(parameters, **kwargs) diff --git a/petab/v2/extensions/sciml.py b/petab/v2/extensions/sciml.py index c8bd54b9..e0ada67f 100644 --- a/petab/v2/extensions/sciml.py +++ b/petab/v2/extensions/sciml.py @@ -79,6 +79,11 @@ def elements(self) -> list[Hybridization]: @classmethod def from_df(cls, df: pd.DataFrame, **kwargs) -> HybridizationTable: """Create a HybridizationTable from a DataFrame.""" + from ..core import _prepare_table_df + + df = _prepare_table_df( + df, cls.__name__, [C.TARGET_ID, C.TARGET_VALUE], **kwargs + ) if df is None: return cls(**kwargs) diff --git a/tests/v2/test_core.py b/tests/v2/test_core.py index f715edc1..ffb50cd1 100644 --- a/tests/v2/test_core.py +++ b/tests/v2/test_core.py @@ -95,6 +95,7 @@ def test_condition_table_round_trip(): pd.DataFrame( { C.OBSERVABLE_ID: ["obs1", "obs2"], + C.EXPERIMENT_ID: ["exp1", "exp1"], C.TIME: [0.0, 1.0], C.MEASUREMENT: [1.0, 2.0], } @@ -129,6 +130,72 @@ def test_from_df_ignores_unnamed_index(table_cls, df): assert table2 == table +_VALID_TABLE_ROWS = { + ObservableTable: { + C.OBSERVABLE_ID: "obs1", + C.OBSERVABLE_FORMULA: "x", + C.NOISE_FORMULA: "1", + }, + ConditionTable: { + C.CONDITION_ID: "c1", + C.TARGET_ID: "x", + C.TARGET_VALUE: "1", + }, + ExperimentTable: { + C.EXPERIMENT_ID: "e1", + C.TIME: 0.0, + C.CONDITION_ID: "c1", + }, + MeasurementTable: { + C.OBSERVABLE_ID: "obs1", + C.EXPERIMENT_ID: "e1", + C.TIME: 1.0, + C.MEASUREMENT: 2.0, + }, + ParameterTable: { + C.PARAMETER_ID: "p1", + C.LOWER_BOUND: 0.0, + C.UPPER_BOUND: 1.0, + C.NOMINAL_VALUE: 0.5, + C.ESTIMATE: "true", + }, + MappingTable: { + PETAB_ENTITY_ID: "a", + MODEL_ENTITY_ID: "1a", + }, +} + + +@pytest.mark.parametrize( + "table_cls", _VALID_TABLE_ROWS, ids=lambda cls: cls.__name__ +) +def test_from_df_missing_required_columns(table_cls): + """Missing required columns are reported.""" + row = _VALID_TABLE_ROWS[table_cls] + assert set(row) == set(table_cls._required_columns) + df = pd.DataFrame([row]) + table = table_cls.from_df(df) + assert len(table.elements) == 1 + # required columns may be in the index + assert table_cls.from_df(df.set_index(df.columns[0])) == table + assert table_cls.from_df(df.set_index(list(df.columns))) == table + # a DataFrame without any columns is an empty table + assert table_cls.from_df(pd.DataFrame()).elements == [] + + for column in row: + with pytest.raises( + MissingColumnsError, match=f"column.*: {column}\\.$" + ): + table_cls.from_df(df.drop(columns=column)) + + with tempfile.TemporaryDirectory() as tmp_dir: + df.drop(columns=df.columns[-1]).to_csv( + Path(tmp_dir, "table.tsv"), sep="\t", index=False + ) + with pytest.raises(MissingColumnsError, match=r"File: .*table\.tsv"): + table_cls.from_tsv("table.tsv", base_path=tmp_dir) + + @pytest.mark.filterwarnings( "ignore:.*Parameter scales are not supported in PEtab v2.*:UserWarning" ) @@ -1242,6 +1309,20 @@ def test_petablint_v2(tmpdir): ) assert result.returncode == 1 + # missing required columns are reported without a traceback + parameter_file = Path(tmpdir, problem.parameter_tables[0].rel_path) + df = pd.read_csv(parameter_file, sep="\t") + df.drop(columns=C.ESTIMATE).to_csv(parameter_file, sep="\t", index=False) + result = subprocess.run( # noqa S607 + ["petablint", str(Path(tmpdir, "problem.yaml"))], # noqa S607 + capture_output=True, + text=True, + ) + output = result.stdout + result.stderr + assert result.returncode == 1 + assert "ParameterTable is missing required column(s): estimate" in output + assert "Traceback" not in output + def test_problem_id(tmpdir): """Test that the problem ID works as expected.""" diff --git a/tests/v2/test_sciml.py b/tests/v2/test_sciml.py index c8d7e23f..9752a7fd 100644 --- a/tests/v2/test_sciml.py +++ b/tests/v2/test_sciml.py @@ -1,4 +1,5 @@ import numpy as np +import pandas as pd import pytest from petab_sciml import Input, Node from pydantic import ConfigDict @@ -327,6 +328,21 @@ def test_hybridization_table_to_df_petab_math(): assert table.to_df()[C.TARGET_VALUE].tolist() == ["x ^ 2", "abs(y)"] +def test_hybridization_table_missing_required_columns(): + """Missing required columns are reported.""" + df = pd.DataFrame({C.TARGET_ID: ["a"], C.TARGET_VALUE: ["x"]}) + assert len(HybridizationTable.from_df(df).elements) == 1 + assert ( + len(HybridizationTable.from_df(df.set_index(C.TARGET_ID)).elements) + == 1 + ) + for column in df.columns: + with pytest.raises( + MissingColumnsError, match=f"column.*: {column}\\.$" + ): + HybridizationTable.from_df(df.drop(columns=column)) + + def test_hybridization_pre_initialization_not_evaluable(): problem = _get_test_problem() # net1_input1 is assigned the species `A`, which is not available before From b188bb38d763d3f05e5b8b8123faf8a48a9d5e67 Mon Sep 17 00:00:00 2001 From: Daniel Weindl Date: Tue, 17 Nov 2026 23:37:31 +0100 Subject: [PATCH 2/2] changelog: add PR number Co-Authored-By: Claude Opus 5.5 --- changelog.d/+v2-missing-columns.breaking.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog.d/+v2-missing-columns.breaking.md b/changelog.d/+v2-missing-columns.breaking.md index f0827e63..69e29f67 100644 --- a/changelog.d/+v2-missing-columns.breaking.md +++ b/changelog.d/+v2-missing-columns.breaking.md @@ -4,4 +4,4 @@ columns are missing, naming the table, the missing columns, and the file. Previously, this led to unspecific errors or, e.g., for missing `estimate` or `experimentId` columns, was silently accepted. `petablint` reports this error without a traceback -([#XXX](https://github.com/PEtab-dev/libpetab-python/pull/XXX)) +([#610](https://github.com/PEtab-dev/libpetab-python/pull/610))