diff --git a/changelog.d/+skip-unnecessary-observable-flattening.fix.md b/changelog.d/+skip-unnecessary-observable-flattening.fix.md new file mode 100644 index 00000000..d4db79ec --- /dev/null +++ b/changelog.d/+skip-unnecessary-observable-flattening.fix.md @@ -0,0 +1,3 @@ +Preserve the PEtab problem tables when measurement overrides do not vary by +timepoint or replicate within a simulation condition +([#617](https://github.com/PEtab-dev/libpetab-python/pull/617)). diff --git a/petab/v1/core.py b/petab/v1/core.py index 5262f6e1..c9620239 100644 --- a/petab/v1/core.py +++ b/petab/v1/core.py @@ -251,21 +251,33 @@ def flatten_timepoint_specific_output_overrides( This is a helper function for some tools which may not support such timepoint-specific mappings. The observable table and measurement table - are modified in place. + are modified in place only if timepoint-specific overrides exist. Arguments: petab_problem: PEtab problem to work on. Modified in place. """ + from .lint import measurement_table_has_timepoint_specific_mappings + + measurement_df = petab_problem.measurement_df + if ( + measurement_df is None + or measurement_df.empty + or not measurement_table_has_timepoint_specific_mappings( + measurement_df + ) + ): + return + new_measurement_dfs = [] new_observable_dfs = [] groupvars = get_notnull_columns( - petab_problem.measurement_df, POSSIBLE_GROUPVARS_FLATTENED_PROBLEM + measurement_df, POSSIBLE_GROUPVARS_FLATTENED_PROBLEM ) mappings = get_flattened_id_mappings(petab_problem) - for groupvar, measurements in petab_problem.measurement_df.groupby( + for groupvar, measurements in measurement_df.groupby( groupvars, dropna=False ): obs_id = groupvar[groupvars.index(OBSERVABLE_ID)] @@ -341,6 +353,13 @@ def unflatten_simulation_df( Returns: The simulation dataframe for the unflattened PEtab problem. """ + if ( + simulation_df[OBSERVABLE_ID] + .isin(petab_problem.observable_df.index) + .all() + ): + return simulation_df.copy() + mappings = get_flattened_id_mappings(petab_problem) original_observable_ids = simulation_df[OBSERVABLE_ID].replace( mappings[OBSERVABLE_ID] diff --git a/petab/v1/lint.py b/petab/v1/lint.py index 9d142af8..257f9c12 100644 --- a/petab/v1/lint.py +++ b/petab/v1/lint.py @@ -871,7 +871,7 @@ def measurement_table_has_timepoint_specific_mappings( PREEQUILIBRATION_CONDITION_ID, ], ) - grouped_df2 = measurement_df.groupby(grouping_cols) + grouped_df2 = measurement_df.groupby(grouping_cols, dropna=False) # data frame has timepoint specific overrides if grouping by noise # parameters and observable parameters in addition to observable, # condition and preeq id yields more groups diff --git a/tests/v1/test_petab.py b/tests/v1/test_petab.py index a352fb58..b6c11c1d 100644 --- a/tests/v1/test_petab.py +++ b/tests/v1/test_petab.py @@ -359,6 +359,60 @@ def test_create_parameter_df( assert parameter_df.loc["p0", NOMINAL_VALUE] == 3.0 +@pytest.mark.parametrize("overrides", [False, True]) +def test_flatten_without_timepoint_specific_overrides(overrides): + """Preserve the input tables when overrides do not vary within a group.""" + measurement_df = pd.DataFrame( + { + OBSERVABLE_ID: ["obs"] * 3, + SIMULATION_CONDITION_ID: ["c1", "c1", "c2"], + PREEQUILIBRATION_CONDITION_ID: [None, None, "pre"], + TIME: [0, 1, 0], + MEASUREMENT: [1.0, 2.0, 3.0], + } + ) + if overrides: + measurement_df[NOISE_PARAMETERS] = ["noise1", "noise1", "noise2"] + observable_df = pd.DataFrame( + {OBSERVABLE_FORMULA: ["x", "y"], NOISE_FORMULA: ["1", "1"]}, + index=pd.Index(["obs", "unused"], name=OBSERVABLE_ID), + ) + visualization_df = pd.DataFrame({Y_VALUES: ["obs"]}) + problem = petab.Problem( + measurement_df=measurement_df, + observable_df=observable_df, + visualization_df=visualization_df, + ) + simulation_df = measurement_df.rename(columns={MEASUREMENT: SIMULATION}) + petab.flatten_timepoint_specific_output_overrides(problem) + + pd.testing.assert_frame_equal(problem.measurement_df, measurement_df) + pd.testing.assert_frame_equal(problem.observable_df, observable_df) + pd.testing.assert_frame_equal(problem.visualization_df, visualization_df) + pd.testing.assert_frame_equal( + petab.core.unflatten_simulation_df(simulation_df, problem), + simulation_df, + ) + + +def test_unflatten_observable_ids_from_older_conversion(): + """Previously generated synthetic IDs still map back to the inputs.""" + measurement_df = pd.DataFrame( + {OBSERVABLE_ID: ["obs"], SIMULATION_CONDITION_ID: ["c1"]} + ) + problem = petab.Problem( + measurement_df=measurement_df, + observable_df=pd.DataFrame( + index=pd.Index(["obs"], name=OBSERVABLE_ID) + ), + ) + simulation_df = measurement_df.assign(observableId="obs__c1", simulation=1) + pd.testing.assert_frame_equal( + petab.core.unflatten_simulation_df(simulation_df, problem), + measurement_df.assign(simulation=1), + ) + + def test_flatten_timepoint_specific_output_overrides(): """Test flatten_timepoint_specific_output_overrides""" observable_df = pd.DataFrame(