Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions changelog.d/+v2-missing-columns.breaking.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
v2: Creating a table via `from_df` or `from_tsv` (including the
`Problem.*_df` setters) now raises `petab.v2.MissingColumnsError` if required
columns are missing, naming the table, the missing columns, and the file.
Previously, this led to unspecific errors or, e.g., for missing `estimate` or
`experimentId` columns, was silently accepted. `petablint` reports this error
without a traceback
([#610](https://github.com/PEtab-dev/libpetab-python/pull/610))
4 changes: 4 additions & 0 deletions petab/petablint.py
Original file line number Diff line number Diff line change
Expand Up @@ -202,6 +202,7 @@ def main():
ret = petab.lint.lint_problem(problem)
sys.exit(ret)
case 2:
from petab.v2.core import MissingColumnsError
from petab.v2.lint import (
ValidationIssueSeverity,
lint_problem,
Expand Down Expand Up @@ -232,6 +233,9 @@ def main():
# https://github.com/pydantic/pydantic/issues/7224
logger.error(f"Error in field(s) `{loc}`: {msg}")
sys.exit(1)
except MissingColumnsError as e:
logger.error(str(e))
sys.exit(1)

case _:
logger.error(
Expand Down
103 changes: 73 additions & 30 deletions petab/v2/core.py
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,7 @@
"MappingTable",
"Measurement",
"MeasurementTable",
"MissingColumnsError",
"NoiseDistribution",
"Observable",
"ObservableTable",
Expand All @@ -87,6 +88,10 @@
logger = logging.getLogger(__name__)


class MissingColumnsError(ValueError):
"""A PEtab table lacks required columns."""


def _field_to_column(info: ValidationInfo) -> str:
"""The PEtab column name for the validated field."""
return {"lb": C.LOWER_BOUND, "ub": C.UPPER_BOUND}.get(
Expand Down Expand Up @@ -138,6 +143,43 @@ def _reset_index(df: pd.DataFrame) -> pd.DataFrame:
return df.reset_index(drop=all(name is None for name in df.index.names))


def _prepare_table_df(
df: pd.DataFrame | None,
table_name: str,
required_columns: list[str],
rel_path: str | Path | AnyUrl | None = None,
base_path: str | Path | AnyUrl | None = None,
**_,
) -> pd.DataFrame | None:
"""Prepare a DataFrame for creating a PEtab table.

Moves named index levels to columns and checks for required columns.

:param df: The table.
:param table_name: The table name for the error message.
:param required_columns: The required columns.
:param rel_path: The table file, for the error message.
:param base_path: The base path of ``rel_path``.
:return: The prepared DataFrame, or ``None`` if ``df`` is ``None`` or
has no columns, i.e., the table is empty.
:raises MissingColumnsError: If any required column is missing.
"""
if df is None:
return None
df = _reset_index(df)
if df.columns.empty:
return None
if missing := [c for c in required_columns if c not in df.columns]:
msg = (
f"{table_name} is missing required column(s): "
f"{', '.join(missing)}."
)
if rel_path is not None:
msg += f" File: {_generate_path(rel_path, base_path)}"
raise MissingColumnsError(msg)
return df


def _column_name(name: str, field: FieldInfo) -> str:
"""Get the table column name for a model field.

Expand Down Expand Up @@ -301,10 +343,30 @@ def __getitem__(self, id_: str) -> T:

raise KeyError(f"{T.__name__} ID {id_} not found")

@classmethod
def from_df(cls, df: pd.DataFrame | None, **kwargs) -> BaseTable[T]:
"""Create a table from a DataFrame.

:param df: The table. Named index levels are treated as columns.
``None`` or a DataFrame without any columns gives an empty table.
:param kwargs: Passed to the constructor
(e.g., ``rel_path``, ``base_path``).
:raises MissingColumnsError: If any required column is missing.
"""
df = _prepare_table_df(
df, cls.__name__, cls._required_columns, **kwargs
)
if df is None:
return cls(**kwargs)
return cls._from_df(df, **kwargs)

@classmethod
@abstractmethod
def from_df(cls, df: pd.DataFrame, **kwargs) -> BaseTable[T]:
"""Create a table from a DataFrame."""
def _from_df(cls, df: pd.DataFrame, **kwargs) -> BaseTable[T]:
"""Create a table from a DataFrame with all required columns.

``df`` has no named index levels.
"""

def to_df(
self, *, columns: TableColumns | str | None = None
Expand Down Expand Up @@ -523,12 +585,10 @@ def free_symbols(self) -> set[sp.Symbol]:
)

@classmethod
def from_df(cls, df: pd.DataFrame, **kwargs) -> ObservableTable:
def _from_df(cls, df: pd.DataFrame, **kwargs) -> ObservableTable:
"""Create an ObservableTable from a DataFrame."""
if df is None:
return cls(**kwargs)

df = get_observable_df(df.copy())
df = get_observable_df(df)
observables = [
Observable(**row.to_dict())
for _, row in _reset_index(df).iterrows()
Expand Down Expand Up @@ -654,11 +714,8 @@ def conditions(self) -> list[Condition]:
return self.elements

@classmethod
def from_df(cls, df: pd.DataFrame, **kwargs) -> ConditionTable:
def _from_df(cls, df: pd.DataFrame, **kwargs) -> ConditionTable:
"""Create a ConditionTable from a DataFrame."""
if df is None or df.empty:
return cls(**kwargs)

conditions = []
for condition_id, sub_df in df.groupby(C.CONDITION_ID):
changes = [
Expand Down Expand Up @@ -817,10 +874,8 @@ def experiments(self) -> list[Experiment]:
return self.elements

@classmethod
def from_df(cls, df: pd.DataFrame, **kwargs) -> ExperimentTable:
def _from_df(cls, df: pd.DataFrame, **kwargs) -> ExperimentTable:
"""Create an ExperimentTable from a DataFrame."""
if df is None:
return cls(**kwargs)

experiments = []
for experiment_id, cur_exp_df in df.groupby(C.EXPERIMENT_ID):
Expand Down Expand Up @@ -952,12 +1007,9 @@ def measurements(self) -> list[Measurement]:
return self.elements

@classmethod
def from_df(cls, df: pd.DataFrame, **kwargs) -> MeasurementTable:
def _from_df(cls, df: pd.DataFrame, **kwargs) -> MeasurementTable:
"""Create a MeasurementTable from a DataFrame."""
if df is None:
return cls(**kwargs)

df = _reset_index(df)
if C.MODEL_ID in df.columns:
df[C.MODEL_ID] = df[C.MODEL_ID].apply(_convert_nan_to_none)

Expand Down Expand Up @@ -1036,14 +1088,10 @@ def mappings(self) -> list[Mapping]:
return self.elements

@classmethod
def from_df(cls, df: pd.DataFrame, **kwargs) -> MappingTable:
def _from_df(cls, df: pd.DataFrame, **kwargs) -> MappingTable:
"""Create a MappingTable from a DataFrame."""
if df is None:
return cls(**kwargs)

mappings = [
Mapping(**row.to_dict()) for _, row in _reset_index(df).iterrows()
]
mappings = [Mapping(**row.to_dict()) for _, row in df.iterrows()]
return cls(mappings, **kwargs)

def _to_df(self) -> pd.DataFrame:
Expand Down Expand Up @@ -1261,15 +1309,10 @@ def parameters(self) -> list[Parameter]:
return self.elements

@classmethod
def from_df(cls, df: pd.DataFrame, **kwargs) -> ParameterTable:
def _from_df(cls, df: pd.DataFrame, **kwargs) -> ParameterTable:
"""Create a ParameterTable from a DataFrame."""
if df is None:
return cls(**kwargs)

parameters = [
Parameter(**row.to_dict())
for _, row in _reset_index(df).iterrows()
]
parameters = [Parameter(**row.to_dict()) for _, row in df.iterrows()]

return cls(parameters, **kwargs)

Expand Down
5 changes: 5 additions & 0 deletions petab/v2/extensions/sciml.py
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,11 @@ def elements(self) -> list[Hybridization]:
@classmethod
def from_df(cls, df: pd.DataFrame, **kwargs) -> HybridizationTable:
"""Create a HybridizationTable from a DataFrame."""
from ..core import _prepare_table_df

df = _prepare_table_df(
df, cls.__name__, [C.TARGET_ID, C.TARGET_VALUE], **kwargs
)
if df is None:
return cls(**kwargs)

Expand Down
81 changes: 81 additions & 0 deletions tests/v2/test_core.py
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,7 @@ def test_condition_table_round_trip():
pd.DataFrame(
{
C.OBSERVABLE_ID: ["obs1", "obs2"],
C.EXPERIMENT_ID: ["exp1", "exp1"],
C.TIME: [0.0, 1.0],
C.MEASUREMENT: [1.0, 2.0],
}
Expand Down Expand Up @@ -129,6 +130,72 @@ def test_from_df_ignores_unnamed_index(table_cls, df):
assert table2 == table


_VALID_TABLE_ROWS = {
ObservableTable: {
C.OBSERVABLE_ID: "obs1",
C.OBSERVABLE_FORMULA: "x",
C.NOISE_FORMULA: "1",
},
ConditionTable: {
C.CONDITION_ID: "c1",
C.TARGET_ID: "x",
C.TARGET_VALUE: "1",
},
ExperimentTable: {
C.EXPERIMENT_ID: "e1",
C.TIME: 0.0,
C.CONDITION_ID: "c1",
},
MeasurementTable: {
C.OBSERVABLE_ID: "obs1",
C.EXPERIMENT_ID: "e1",
C.TIME: 1.0,
C.MEASUREMENT: 2.0,
},
ParameterTable: {
C.PARAMETER_ID: "p1",
C.LOWER_BOUND: 0.0,
C.UPPER_BOUND: 1.0,
C.NOMINAL_VALUE: 0.5,
C.ESTIMATE: "true",
},
MappingTable: {
PETAB_ENTITY_ID: "a",
MODEL_ENTITY_ID: "1a",
},
}


@pytest.mark.parametrize(
"table_cls", _VALID_TABLE_ROWS, ids=lambda cls: cls.__name__
)
def test_from_df_missing_required_columns(table_cls):
"""Missing required columns are reported."""
row = _VALID_TABLE_ROWS[table_cls]
assert set(row) == set(table_cls._required_columns)
df = pd.DataFrame([row])
table = table_cls.from_df(df)
assert len(table.elements) == 1
# required columns may be in the index
assert table_cls.from_df(df.set_index(df.columns[0])) == table
assert table_cls.from_df(df.set_index(list(df.columns))) == table
# a DataFrame without any columns is an empty table
assert table_cls.from_df(pd.DataFrame()).elements == []

for column in row:
with pytest.raises(
MissingColumnsError, match=f"column.*: {column}\\.$"
):
table_cls.from_df(df.drop(columns=column))

with tempfile.TemporaryDirectory() as tmp_dir:
df.drop(columns=df.columns[-1]).to_csv(
Path(tmp_dir, "table.tsv"), sep="\t", index=False
)
with pytest.raises(MissingColumnsError, match=r"File: .*table\.tsv"):
table_cls.from_tsv("table.tsv", base_path=tmp_dir)


@pytest.mark.filterwarnings(
"ignore:.*Parameter scales are not supported in PEtab v2.*:UserWarning"
)
Expand Down Expand Up @@ -1242,6 +1309,20 @@ def test_petablint_v2(tmpdir):
)
assert result.returncode == 1

# missing required columns are reported without a traceback
parameter_file = Path(tmpdir, problem.parameter_tables[0].rel_path)
df = pd.read_csv(parameter_file, sep="\t")
df.drop(columns=C.ESTIMATE).to_csv(parameter_file, sep="\t", index=False)
result = subprocess.run( # noqa S607
["petablint", str(Path(tmpdir, "problem.yaml"))], # noqa S607
capture_output=True,
text=True,
)
output = result.stdout + result.stderr
assert result.returncode == 1
assert "ParameterTable is missing required column(s): estimate" in output
assert "Traceback" not in output


def test_problem_id(tmpdir):
"""Test that the problem ID works as expected."""
Expand Down
16 changes: 16 additions & 0 deletions tests/v2/test_sciml.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import numpy as np
import pandas as pd
import pytest
from petab_sciml import Input, Node
from pydantic import ConfigDict
Expand Down Expand Up @@ -327,6 +328,21 @@ def test_hybridization_table_to_df_petab_math():
assert table.to_df()[C.TARGET_VALUE].tolist() == ["x ^ 2", "abs(y)"]


def test_hybridization_table_missing_required_columns():
"""Missing required columns are reported."""
df = pd.DataFrame({C.TARGET_ID: ["a"], C.TARGET_VALUE: ["x"]})
assert len(HybridizationTable.from_df(df).elements) == 1
assert (
len(HybridizationTable.from_df(df.set_index(C.TARGET_ID)).elements)
== 1
)
for column in df.columns:
with pytest.raises(
MissingColumnsError, match=f"column.*: {column}\\.$"
):
HybridizationTable.from_df(df.drop(columns=column))


def test_hybridization_pre_initialization_not_evaluable():
problem = _get_test_problem()
# net1_input1 is assigned the species `A`, which is not available before
Expand Down
Loading