From 8f16bfe5d2bb0dccaddd7a3521dbf0eba19383be Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 18 Nov 2026 23:00:41 +0000 Subject: [PATCH 01/15] perf(events): wait for work instead of polling while event exporting is idle The dispatch and notification loops woke every 10 ms even with nothing queued, taking the GIL about a hundred times a second each. In a long-lived process, such as a location's runtime, that slowed every other thread: a plan preview spent about a quarter of its time waiting to get the GIL back. Idle loops now wait for the notification that queued work, shutdown, failures and summaries already send, with a one-second safety timeout; busy rounds still poll. --- .../src/event_exporting/constants.rs | 3 ++ .../src/event_exporting/models.rs | 23 +++++++- .../event_exporting/classes/_test_types.py | 8 +++ .../classes/test_dispatcher.py | 52 +++++++++++++++++++ 4 files changed, 84 insertions(+), 2 deletions(-) diff --git a/crates/sqlbuild-python-base/src/event_exporting/constants.rs b/crates/sqlbuild-python-base/src/event_exporting/constants.rs index 9a195b54d6..07aaf03de8 100644 --- a/crates/sqlbuild-python-base/src/event_exporting/constants.rs +++ b/crates/sqlbuild-python-base/src/event_exporting/constants.rs @@ -19,6 +19,9 @@ pub const WORKER_ATTRIBUTE: &str = "_sqlbuild_event_exporter_worker"; /// The longest single wait of a polling delivery loop, as the Python dispatcher polled. pub const POLL_SECONDS: f64 = 0.01; +/// The longest an idle dispatcher loop waits for the notification that brings it work. +pub const IDLE_POLL_SECONDS: f64 = 1.0; + /// The most one dispatch round charges a sink call, so the dispatcher's GIL waits are not counted. pub const CHARGE_LIMIT_SECONDS: f64 = 2.0 * POLL_SECONDS; diff --git a/crates/sqlbuild-python-output/src/event_exporting/models.rs b/crates/sqlbuild-python-output/src/event_exporting/models.rs index e8b23c67bd..a7de6b396a 100644 --- a/crates/sqlbuild-python-output/src/event_exporting/models.rs +++ b/crates/sqlbuild-python-output/src/event_exporting/models.rs @@ -47,6 +47,7 @@ use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::event_exporting::constants::DISPATCHER_THREAD_NAME; use sqlbuild_python_base::event_exporting::constants::EXIT_JOIN_SECONDS; use sqlbuild_python_base::event_exporting::constants::HEALTH_THREAD_NAME; +use sqlbuild_python_base::event_exporting::constants::IDLE_POLL_SECONDS; use sqlbuild_python_base::event_exporting::constants::JOIN_GRACE_SECONDS; use sqlbuild_python_base::event_exporting::constants::NO_SECONDS; use sqlbuild_python_base::event_exporting::constants::NOTIFIER_THREAD_NAME; @@ -543,6 +544,7 @@ impl DispatcherWorker { notify |= state.feed(py, round)?; state.stopping && !state.busy() && state.queue.items.is_empty() && state.held.is_empty() }; + let idle: bool = !stopped && Self::idle_round(&state); let blocked: Vec = std::mem::take(&mut state.newly_blocked); drop(state); for name in blocked { @@ -552,11 +554,28 @@ impl DispatcherWorker { shared.notify_all(py)?; } if !stopped { - shared.wait(py, POLL_SECONDS)?; + shared.wait( + py, + if idle { + IDLE_POLL_SECONDS + } else { + POLL_SECONDS + }, + )?; } Ok(!stopped) } + /// Whether a round has nothing to watch: no queued, held or pending work, and no shutdown. + fn idle_round(state: &State) -> bool { + !state.busy() + && state.queue.items.is_empty() + && state.held.is_empty() + && !state.stopping + && !state.force_stop + && state.deadline.is_none() + } + /// Every `(token, sanitized error type)` delivery threads reported since the last round. pub(crate) fn reported(&self, py: Python<'_>) -> PyResult)>> { let results: &Bound<'_, PyAny> = self.shared.results.bind(py); @@ -593,7 +612,7 @@ impl DispatcherWorker { return Ok(Notification::Stop); } drop(state); - shared.wait(py, POLL_SECONDS)?; + shared.wait(py, IDLE_POLL_SECONDS)?; } } diff --git a/tests/unit/src/sqlbuild/runtime/event_exporting/classes/_test_types.py b/tests/unit/src/sqlbuild/runtime/event_exporting/classes/_test_types.py index 5b534e4ed2..70c0e22891 100644 --- a/tests/unit/src/sqlbuild/runtime/event_exporting/classes/_test_types.py +++ b/tests/unit/src/sqlbuild/runtime/event_exporting/classes/_test_types.py @@ -1,6 +1,14 @@ from dataclasses import dataclass +@dataclass(frozen=True) +class IdleDispatcherTestCase: + description: str + idle_seconds: float + expected_max_waits: int + expected_max_delivery_seconds: float + + @dataclass(frozen=True) class EventExporterDispatcherTestCase: description: str diff --git a/tests/unit/src/sqlbuild/runtime/event_exporting/classes/test_dispatcher.py b/tests/unit/src/sqlbuild/runtime/event_exporting/classes/test_dispatcher.py index a751b26fbf..fb20a5471a 100644 --- a/tests/unit/src/sqlbuild/runtime/event_exporting/classes/test_dispatcher.py +++ b/tests/unit/src/sqlbuild/runtime/event_exporting/classes/test_dispatcher.py @@ -29,6 +29,7 @@ from tests.unit.src.sqlbuild.runtime.event_exporting.classes._test_types import ( EventExporterDispatcherTestCase, HealthIntervalTestCase, + IdleDispatcherTestCase, OutcomeDeliveryTestCase, ) from tests.unit.src.sqlbuild.runtime.event_exporting.classes.helpers import ( @@ -123,6 +124,57 @@ def publish(*, event: LifecycleEvent) -> None: assert dispatcher.shutdown().delivered == test_case.expected_delivered +@pytest.mark.parametrize( + "test_case", + ( + IdleDispatcherTestCase( + description="an idle dispatcher waits for work instead of polling", + idle_seconds=0.5, + expected_max_waits=10, + expected_max_delivery_seconds=0.5, + ), + ), + ids=lambda case: case.description, +) +def test_given_idle_dispatcher_when_waiting_then_it_does_not_poll_and_still_delivers_promptly( + test_case: IdleDispatcherTestCase, + monkeypatch: pytest.MonkeyPatch, +) -> None: + delivered: list[threading.Event] = [threading.Event(), threading.Event()] + published: list[int] = [] + caller_thread: int = threading.get_ident() + waits: list[tuple[int, float | None]] = [] + condition_wait: Callable[[threading.Condition, float | None], bool] = threading.Condition.wait + + def counted_wait(self: threading.Condition, timeout: float | None = None) -> bool: + waits.append((threading.get_ident(), timeout)) + return condition_wait(self, timeout) + + monkeypatch.setattr(threading.Condition, "wait", counted_wait) + + def publish(*, event: LifecycleEvent) -> None: + del event + delivered[len(published)].set() + published.append(1) + + dispatcher: EventExporterDispatcher = EventExporterDispatcher( + exporters=(BoundEventExporter("publish", publish, {}),), + ) + dispatcher.enqueue(lifecycle_event()) + assert delivered[0].wait(timeout=2.0) + time.sleep(0.05) + waits.clear() + time.sleep(test_case.idle_seconds) + idle_waits: int = sum(thread != caller_thread for thread, _ in waits) + started: float = time.monotonic() + dispatcher.enqueue(lifecycle_event()) + + assert delivered[1].wait(timeout=2.0) + assert time.monotonic() - started < test_case.expected_max_delivery_seconds + assert idle_waits <= test_case.expected_max_waits + assert dispatcher.shutdown().delivered == 2 + + @pytest.mark.parametrize( "test_case", (EventExporterDispatcherTestCase("failure isolation", 1),), From 748649e4a8f9e4a6abb397e096283204e54f0601 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 18 Nov 2026 23:00:43 +0000 Subject: [PATCH 02/15] perf(plan): build migration relations natively Migration planning named each compiled relation location through a Python helper call per model. It now builds the same three-field MigrationRelation record natively. --- .../src/entries/_helpers/executor.rs | 5 ----- crates/sqlbuild-host/src/entries/models.rs | 1 - .../_helpers/column/column_entries.rs | 9 ++++----- .../_helpers/discovery/discovery_events.rs | 8 +++----- .../_helpers/model/model_entries.rs | 5 ++--- .../_helpers/old_name/old_name_objects.rs | 5 ++--- .../_helpers/planning/location_relation.rs | 19 +++++++++++++++++++ .../_helpers/planning/mod.rs | 1 + src/sqlbuild/_native_host.py | 4 ---- src/sqlbuild/_native_host.pyi | 3 --- 10 files changed, 31 insertions(+), 29 deletions(-) create mode 100644 crates/sqlbuild-python-planner/src/planner_migrations/_helpers/planning/location_relation.rs diff --git a/crates/sqlbuild-host/src/entries/_helpers/executor.rs b/crates/sqlbuild-host/src/entries/_helpers/executor.rs index 17302822a0..2041eb81f9 100644 --- a/crates/sqlbuild-host/src/entries/_helpers/executor.rs +++ b/crates/sqlbuild-host/src/entries/_helpers/executor.rs @@ -309,11 +309,6 @@ pub(crate) const ENTRIES: &[HostEntry] = &[ kind: HostKind::Class, edge: HostEdge::Executor, }, - HostEntry { - name: "migration_relation_for_location", - kind: HostKind::Function, - edge: HostEdge::Executor, - }, HostEntry { name: "is_compatibility_view", kind: HostKind::Function, diff --git a/crates/sqlbuild-host/src/entries/models.rs b/crates/sqlbuild-host/src/entries/models.rs index 4c39f96c72..0c11914fac 100644 --- a/crates/sqlbuild-host/src/entries/models.rs +++ b/crates/sqlbuild-host/src/entries/models.rs @@ -268,7 +268,6 @@ pub enum ExecutorHost { LoadCallbacks, LoadExecutionResult, LoadRuntimeParams, - MigrationRelationForLocation, IsCompatibilityView, AuditExecutionError, WarehouseConnectionError, diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_entries.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_entries.rs index 436d400ded..7164bdb915 100644 --- a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_entries.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_entries.rs @@ -1,8 +1,8 @@ //! Materialize column decisions from native proofs and live evidence. +use crate::planner_migrations::_helpers::planning::location_relation::location_relation; use crate::planner_phases::main::inputs::runtime_reads::runtime_reads; use crate::planner_warehouse::models::{AdapterRead, Read}; -use sqlbuild_host::entries::models::ExecutorHost; use crate::planner_migrations::_helpers::column::column_candidates::declared_names; use crate::planner_migrations::_helpers::column::column_objects::lower; @@ -19,7 +19,6 @@ use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyListMethods, PyTuple}; use sqlbuild_analysis::planner_migrations::main::column_decision::column_decision; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::keywords::keywords; -use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::construct::enum_from_value::enum_from_value; use sqlbuild_python_base::records::main::construct::enum_member::enum_member; use sqlbuild_python_base::records::main::fields::field::field as native_field; @@ -74,11 +73,11 @@ pub(crate) fn decide<'py>( outputs.push(name.to_lowercase()); } } - let relation = host_entry(py, ExecutorHost::MigrationRelationForLocation)?.call1(( - sqlbuild_python_project::compiled_views::main::compiled_models::destination::destination( + let relation = location_relation( + &sqlbuild_python_project::compiled_views::main::compiled_models::destination::destination( &candidate.model, )?, - ))?; + )?; let mut entries: Vec> = Vec::new(); for (origin_name, destination, discovery) in pairs { let origin_key = lower(&origin_name)?; diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/discovery/discovery_events.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/discovery/discovery_events.rs index ddfd3b7cfa..01acf3c4ce 100644 --- a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/discovery/discovery_events.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/discovery/discovery_events.rs @@ -1,13 +1,12 @@ //! Preserve event and declared-origin precedence at the inspection boundary. -use sqlbuild_host::entries::models::ExecutorHost; +use crate::planner_migrations::_helpers::planning::location_relation::location_relation; use crate::planner_migrations::_helpers::column::column_objects::config; use crate::planner_migrations::models::ModelMigrationDeclaration; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods, PyTuple}; use sqlbuild_python_base::boundary::main::call_with::call_with; -use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_front::state_rows::main::migrations::newest_move::newest_move; use sqlbuild_python_project::compiled_views::main::compiled_models::name_object::name_object; @@ -26,13 +25,12 @@ pub(crate) fn newest<'py>( events: &Bound<'py, PyAny>, location: &Bound<'py, PyAny>, ) -> PyResult> { - let relation = host_entry(location.py(), ExecutorHost::MigrationRelationForLocation)? - .call1((location,))?; + let relation = location_relation(location)?; newest_move(events, &relation) } pub(crate) fn relation<'py>(location: &Bound<'py, PyAny>) -> PyResult> { - host_entry(location.py(), ExecutorHost::MigrationRelationForLocation)?.call1((location,)) + location_relation(location) } pub(crate) fn declared<'py>( diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/model_entries.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/model_entries.rs index 66f9710514..d85bf58fcd 100644 --- a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/model_entries.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/model_entries.rs @@ -1,6 +1,6 @@ //! Assemble model migration decisions from inspected warehouse facts. -use sqlbuild_host::entries::models::ExecutorHost; +use crate::planner_migrations::_helpers::planning::location_relation::location_relation; use crate::planner_migrations::_helpers::column::column_objects::lower; use crate::planner_migrations::_helpers::column::column_objects::model; @@ -16,7 +16,6 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::PyBool; use sqlbuild_analysis::planner_migrations::main::model_decision::model_decision; use sqlbuild_python_base::boundary::main::call_with::call_with; -use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::construct::enum_from_value::enum_from_value; use sqlbuild_python_base::records::main::construct::enum_member::enum_member; use sqlbuild_python_base::records::main::construct::error_class::error_class; @@ -27,7 +26,7 @@ use sqlbuild_python_front::state_rows::main::migrations::newest_move::newest_mov use sqlbuild_python_project::compiled_views::main::compiled_models::name_object::name_object; pub(crate) fn relation<'py>(location: &Bound<'py, PyAny>) -> PyResult> { - host_entry(location.py(), ExecutorHost::MigrationRelationForLocation)?.call1((location,)) + location_relation(location) } fn key(location: &Bound<'_, PyAny>) -> PyResult<(String, String)> { diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_name_objects.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_name_objects.rs index cb3a98cf8c..61fe861929 100644 --- a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_name_objects.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_name_objects.rs @@ -1,14 +1,13 @@ //! Old-name planner object conversions and existing native helper calls. +use crate::planner_migrations::_helpers::planning::location_relation::location_relation; use crate::planner_phases::main::inputs::runtime_dialect::runtime_dialect; -use sqlbuild_host::entries::models::ExecutorHost; use sqlbuild_python_base::native_host::types::Syntax; use crate::planner_migrations::_helpers::column::column_objects::config; use crate::planner_migrations::_helpers::column::column_objects::model; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use sqlbuild_python_base::boundary::main::call_with::call_with; -use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; @@ -17,7 +16,7 @@ use sqlbuild_python_front::state_rows::main::migrations::old_name_retention::old use sqlbuild_python_project::compiled_views::main::compiled_models::destination::destination; pub(crate) fn relation<'py>(location: &Bound<'py, PyAny>) -> PyResult> { - host_entry(location.py(), ExecutorHost::MigrationRelationForLocation)?.call1((location,)) + location_relation(location) } pub(crate) fn location<'py>( diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/planning/location_relation.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/planning/location_relation.rs new file mode 100644 index 0000000000..ebb0eedec5 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/planning/location_relation.rs @@ -0,0 +1,19 @@ +//! The migration relation that names one compiled relation location, built natively. + +use pyo3::prelude::{Bound, PyAny, PyResult}; +use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::types::RecordId; + +/// The `MigrationRelation` that names `location`, as `migration_relation_for_location` builds it. +pub(crate) fn location_relation<'py>(location: &Bound<'py, PyAny>) -> PyResult> { + new_record_with( + location.py(), + RecordId::MigrationRelation, + &[ + ("database", native_field(location, "database")?), + ("schema", native_field(location, "schema")?), + ("name", native_field(location, "name")?), + ], + ) +} diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/planning/mod.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/planning/mod.rs index e8f2dc0ad7..fe92772606 100644 --- a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/planning/mod.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/planning/mod.rs @@ -1,4 +1,5 @@ pub(crate) mod compatibility; +pub(crate) mod location_relation; pub(crate) mod matching; pub(crate) mod planning_boundary; pub(crate) mod planning_execution; diff --git a/src/sqlbuild/_native_host.py b/src/sqlbuild/_native_host.py index f215c77928..6b2841f41e 100644 --- a/src/sqlbuild/_native_host.py +++ b/src/sqlbuild/_native_host.py @@ -424,10 +424,6 @@ "LoadCallbacks": ("sqlbuild.executor.load.models", "LoadCallbacks"), "LoadExecutionResult": ("sqlbuild.executor.load.models", "LoadExecutionResult"), "LoadRuntimeParams": ("sqlbuild.executor.load.models", "LoadRuntimeParams"), - "migration_relation_for_location": ( - "sqlbuild.executor.migrations._helpers.relation_for_location", - "migration_relation_for_location", - ), "is_compatibility_view": ( "sqlbuild.executor.old_name_views._helpers.janitor", "is_compatibility_view", diff --git a/src/sqlbuild/_native_host.pyi b/src/sqlbuild/_native_host.pyi index 39099b17ac..6e75602349 100644 --- a/src/sqlbuild/_native_host.pyi +++ b/src/sqlbuild/_native_host.pyi @@ -288,9 +288,6 @@ from sqlbuild.executor.load.main.run import run_load_pipeline as run_load_pipeli from sqlbuild.executor.load.models import LoadCallbacks as LoadCallbacks from sqlbuild.executor.load.models import LoadExecutionResult as LoadExecutionResult from sqlbuild.executor.load.models import LoadRuntimeParams as LoadRuntimeParams -from sqlbuild.executor.migrations._helpers.relation_for_location import ( - migration_relation_for_location as migration_relation_for_location, -) from sqlbuild.executor.old_name_views._helpers.janitor import ( is_compatibility_view as is_compatibility_view, ) From 26ec1c60c381c2e481b5735f1fb2c4e893871398 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 18 Nov 2026 23:00:44 +0000 Subject: [PATCH 03/15] perf(plan): index relations when planning old-name views Old-name view planning compared each moved relation with every model's destination and every recorded view, and each model with every recorded rename, through Python-level matches calls. It now indexes them once by lowered schema and name and keeps the same matching rules: the same lowered schema and name, the same database unless either is unknown, and the first match in order. --- .../_helpers/old_name/identity_index.rs | 54 +++++++++++ .../_helpers/old_name/mod.rs | 1 + .../_helpers/old_name/old_name_entries.rs | 95 ++++++++++++++----- .../_helpers/old_name/old_names.rs | 52 ---------- .../src/planner_migrations/models.rs | 8 ++ 5 files changed, 132 insertions(+), 78 deletions(-) create mode 100644 crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/identity_index.rs diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/identity_index.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/identity_index.rs new file mode 100644 index 0000000000..500104c890 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/identity_index.rs @@ -0,0 +1,54 @@ +//! Migration relations indexed by schema and name, matched as `MigrationRelation.matches` matches. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use std::collections::HashMap; + +/// A relation's `identity`: `(database.lower() or None, (schema or "").lower(), name.lower())`. +type Identity = (Option, String, String); + +/// Each indexed relation's lowered database and position, in input order. +type Candidates = Vec<(Option, usize)>; + +/// The positions of indexed relations by lowered `(schema, name)`, each list in input order. +#[derive(Debug)] +pub(crate) struct IdentityIndex { + by_name: HashMap<(String, String), Candidates>, +} + +fn identity(relation: &Bound<'_, PyAny>) -> PyResult { + relation.getattr("identity")?.extract() +} + +impl IdentityIndex { + /// Index `relations` by position. + pub(crate) fn of<'py>(relations: &[Bound<'py, PyAny>]) -> PyResult { + let mut by_name: HashMap<(String, String), Candidates> = HashMap::new(); + for (position, relation) in relations.iter().enumerate() { + let (database, schema, name) = identity(relation)?; + by_name + .entry((schema, name)) + .or_default() + .push((database, position)); + } + Ok(Self { by_name }) + } + + /// Positions of the indexed relations `relation.matches(...)` accepts, in input order. + pub(crate) fn matching(&self, relation: &Bound<'_, PyAny>) -> PyResult> { + let (database, schema, name) = identity(relation)?; + let mut positions: Vec = Vec::new(); + let Some(candidates) = self.by_name.get(&(schema, name)) else { + return Ok(positions); + }; + for (candidate, position) in candidates { + let same_database: bool = match (candidate, &database) { + (Some(left), Some(right)) => left == right, + _ => true, + }; + if same_database { + positions.push(*position); + } + } + Ok(positions) + } +} diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/mod.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/mod.rs index 87e886af43..7dc59c904b 100644 --- a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/mod.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/mod.rs @@ -1,3 +1,4 @@ +pub(crate) mod identity_index; pub(crate) mod old_name_entries; pub(crate) mod old_name_objects; pub(crate) mod old_name_planning; diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_name_entries.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_name_entries.rs index 693488e50e..fd444e50f0 100644 --- a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_name_entries.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_name_entries.rs @@ -2,13 +2,13 @@ use crate::planner_migrations::_helpers::column::column_objects::model; use crate::planner_migrations::_helpers::column::column_objects::text; +use crate::planner_migrations::_helpers::old_name::identity_index::IdentityIndex; use crate::planner_migrations::_helpers::old_name::old_name_objects::expires; use crate::planner_migrations::_helpers::old_name::old_name_objects::location; use crate::planner_migrations::_helpers::old_name::old_name_objects::relation; use crate::planner_migrations::_helpers::old_name::old_name_objects::retention; use crate::planner_migrations::_helpers::old_name::old_name_objects::stores_history; use crate::planner_migrations::_helpers::old_name::old_names::declared_column_aliases; -use crate::planner_migrations::_helpers::old_name::old_names::old_name_view_skip_reason; use crate::planner_migrations::constants::PENDING_ARCHIVE; use crate::planner_migrations::constants::PENDING_VIEW; use crate::planner_migrations::models::OldNamePlanner; @@ -16,9 +16,9 @@ use crate::planner_phases::main::inputs::runtime_dialect::runtime_dialect; use crate::planner_phases::main::inputs::runtime_field::runtime_field; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyBool, PyList, PyTuple}; +use sqlbuild_analysis::planner_migrations::main::old_name_skip_reason::old_name_skip_reason; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; -use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::native_host::types::Syntax; use sqlbuild_python_base::records::main::construct::enum_member::enum_member; use sqlbuild_python_base::records::main::construct::new_record::new_record; @@ -28,6 +28,7 @@ use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_project::compiled_views::main::compiled_models::destination::destination; use sqlbuild_python_project::compiled_views::main::compiled_models::name_object::name_object; use sqlbuild_python_project::planner_phases::main::object::object; +use std::cell::OnceCell; impl<'py> OldNamePlanner<'py> { pub(super) fn new( @@ -52,6 +53,51 @@ impl<'py> OldNamePlanner<'py> { retention: native_field(&config, "old_name_views")?, now: runtime_field(&runtime, |runtime| &runtime.invocation_time)?, runtime, + model_index: OnceCell::new(), + view_index: OnceCell::new(), + history_index: OnceCell::new(), + }) + } + + /// `cell`'s index, building it from `relations` the first time. + fn index<'a>( + cell: &'a OnceCell, + relations: impl FnOnce() -> PyResult>>, + ) -> PyResult<&'a IdentityIndex> { + if let Some(index) = cell.get() { + return Ok(index); + } + let built: IdentityIndex = IdentityIndex::of(&relations()?)?; + Ok(cell.get_or_init(|| built)) + } + + /// Model positions by destination relation. + fn model_index(&self) -> PyResult<&IdentityIndex> { + Self::index(&self.model_index, || { + self.models + .iter() + .map(|item| relation(&destination(item)?)) + .collect() + }) + } + + /// Recorded view positions by old relation. + fn view_index(&self) -> PyResult<&IdentityIndex> { + Self::index(&self.view_index, || { + self.views + .try_iter()? + .map(|view| relation(&native_field(&view?, "old")?)) + .collect() + }) + } + + /// History positions by their move's destination relation. + fn history_index(&self) -> PyResult<&IdentityIndex> { + Self::index(&self.history_index, || { + self.histories + .iter() + .map(|history| native_field(&native_field(history, "move")?, "destination")) + .collect() }) } @@ -61,14 +107,9 @@ impl<'py> OldNamePlanner<'py> { origin: Option<&Bound<'py, PyAny>>, ) -> PyResult>> { let mut newest: Option<(Bound<'py, PyTuple>, Bound<'py, PyAny>)> = None; - for history in &self.histories { + for position in self.history_index()?.matching(destination)? { + let history = &self.histories[position]; let movement = native_field(history, "move")?; - if !native_field(&movement, "destination")? - .call_method1("matches", (destination,))? - .is_truthy()? - { - continue; - } if let Some(origin) = origin && !native_field(&movement, "origin")? .call_method1("matches", (origin,))? @@ -145,30 +186,32 @@ impl<'py> OldNamePlanner<'py> { } } + /// Why no old-name view is kept for `origin`: a model reuses its name or a view already serves it. fn skip( &self, origin: &Bound<'py, PyAny>, tracked: Bound<'py, PyAny>, retention: &Bound<'py, PyAny>, ) -> PyResult> { + let py = self.runtime.py(); + if retention.is_none() { + return object(py, old_name_skip_reason(false, None, false, None)); + } + let old = relation(origin)?; + let reuser: Option = match self.model_index()?.matching(&old)?.first() { + Some(position) => Some(name_object(&self.models[*position])?.extract()?), + None => None, + }; + let compatibility_view: bool = + reuser.is_none() && !self.view_index()?.matching(&old)?.is_empty(); object( - self.runtime.py(), - old_name_view_skip_reason( - self.runtime.py(), - Some(keywords( - self.runtime.py(), - &[ - ("origin", origin.clone()), - ("origin_tracked", tracked), - ("retention", retention.clone()), - ( - "models", - PyTuple::new(self.runtime.py(), &self.models)?.into_any(), - ), - ("views", self.views.clone()), - ], - )?), - )?, + py, + old_name_skip_reason( + true, + reuser.as_deref(), + compatibility_view, + tracked.extract()?, + ), ) } diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_names.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_names.rs index b870bbb145..ae9bb518cf 100644 --- a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_names.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_names.rs @@ -8,7 +8,6 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PySet, PyTuple}; use sqlbuild_analysis::planner_migrations::main::{ declared_column_aliases as aliases, old_name_claim_message as claim, old_name_schemas, - old_name_skip_reason, }; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::native_host::main::host_entry::host_entry; @@ -132,54 +131,3 @@ pub(crate) fn old_name_claim_errors<'py>( } PyTuple::new(py, errors) } - -pub(crate) fn old_name_view_skip_reason<'py>( - py: Python<'py>, - options: Option>, -) -> PyResult> { - let arguments = KeywordArguments::read( - py, - "old_name_view_skip_reason", - options, - ( - &["origin", "origin_tracked", "retention", "models", "views"], - &[], - ), - )?; - let retained = !arguments.value("retention")?.is_none(); - if !retained { - return Ok(old_name_skip_reason::old_name_skip_reason( - false, None, false, None, - )); - } - let old = relation(&arguments.value("origin")?)?; - let mut reuser: Option = None; - for model in arguments.value("models")?.try_iter()? { - let model = model?; - if relation(&sqlbuild_python_project::compiled_views::main::compiled_models::destination::destination(&model)?)? - .call_method1("matches", (&old,))? - .is_truthy()? - { - reuser = Some(name_object(&model)?.extract()?); - break; - } - } - let mut compatibility_view = false; - if reuser.is_none() { - for view in arguments.value("views")?.try_iter()? { - if relation(&native_field(&view?, "old")?)? - .call_method1("matches", (&old,))? - .is_truthy()? - { - compatibility_view = true; - break; - } - } - } - Ok(old_name_skip_reason::old_name_skip_reason( - true, - reuser.as_deref(), - compatibility_view, - arguments.value("origin_tracked")?.extract()?, - )) -} diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/models.rs b/crates/sqlbuild-python-planner/src/planner_migrations/models.rs index 9dbc21f7eb..1d16a3d952 100644 --- a/crates/sqlbuild-python-planner/src/planner_migrations/models.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/models.rs @@ -7,6 +7,7 @@ use crate::planner_migrations::_helpers::discovery::state_inspection::pending; use crate::planner_migrations::_helpers::discovery::state_inspection::relation_key; use crate::planner_migrations::_helpers::discovery::state_inspection::schema_or_empty; use crate::planner_migrations::_helpers::discovery::state_inspection::tuple_of; +use crate::planner_migrations::_helpers::old_name::identity_index::IdentityIndex; use crate::planner_warehouse::main::inspection::columns_for::columns_for; use crate::planner_warehouse::main::inspection::list_relations::list_relations; use crate::planner_warehouse::models::FingerprintRead; @@ -41,6 +42,7 @@ use sqlbuild_python_front::state_rows::main::migrations::layout::layout; use sqlbuild_python_front::state_rows::main::migrations::old_name_views::old_name_views; use sqlbuild_state::events::main::newest_fingerprint::newest_fingerprint; use sqlbuild_state::events::models::{EventRelation, FingerprintKey}; +use std::cell::OnceCell; use std::collections::HashMap; use std::collections::HashSet; @@ -53,6 +55,12 @@ pub(crate) struct OldNamePlanner<'py> { pub columns: Bound<'py, PyAny>, pub retention: Bound<'py, PyAny>, pub now: Bound<'py, PyAny>, + /// `models` by destination relation, built on first use. + pub model_index: OnceCell, + /// `views` by old relation, built on first use. + pub view_index: OnceCell, + /// `histories` by their move's destination, built on first use. + pub history_index: OnceCell, } #[derive(Debug)] From 3421255e3687406fdb445266c9940370aec63901 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 18 Nov 2026 23:00:47 +0000 Subject: [PATCH 04/15] perf(plan): walk each audit target's lineage once Whether an attached audit reads outside its target's lineage walked the target's whole upstream graph once per audit, and many audits attach to the same model. Selected audits are now grouped by target, each target's lineage walked once and dropped before the next. --- .../_helpers/audit_entries.rs | 16 +++++-- .../src/planner_strategy/_helpers/audits.rs | 46 +++++++++++++++++-- .../build_selected_audit_entries.rs | 7 ++- 3 files changed, 61 insertions(+), 8 deletions(-) diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/audit_entries.rs b/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/audit_entries.rs index 0e50277e5f..9793caa19d 100644 --- a/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/audit_entries.rs +++ b/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/audit_entries.rs @@ -114,13 +114,17 @@ pub(crate) fn plan_audit_of<'py>( })), )?, }, + None, ) } /// One audit's plan entry, rendered against the planning relations. +/// +/// `outside_lineage` is whether the audit reads outside its target's lineage, when already known. pub(crate) fn plan_audit<'py>( audit: &Bound<'py, PyAny>, request: &AuditPlanning<'py>, + outside_lineage: Option, ) -> PyResult> { let py = audit.py(); let audit = audit.clone(); @@ -215,9 +219,15 @@ pub(crate) fn plan_audit<'py>( ("attached_target_name", target), ( "reads_outside_target_lineage", - PyBool::new(py, outside_target_lineage(&audit, lineage.graph())?) - .to_owned() - .into_any(), + PyBool::new( + py, + match outside_lineage { + Some(outside) => outside, + None => outside_target_lineage(&audit, lineage.graph())?, + }, + ) + .to_owned() + .into_any(), ), ] { values.set_item(field, value)?; diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/audits.rs b/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/audits.rs index d393d68ca8..0dbc27999b 100644 --- a/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/audits.rs +++ b/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/audits.rs @@ -4,6 +4,7 @@ use crate::planner_strategy::main::values::input_error::input_error; use crate::planner_strategy::main::values::text::text; use std::cell::RefCell; use std::collections::HashMap; +use std::collections::HashSet; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::PyTuple; @@ -149,16 +150,55 @@ pub(crate) fn outside_target_lineage( audit: &Bound<'_, PyAny>, graph: &ProjectGraph, ) -> PyResult { + let Some(target) = lineage_target(audit)? else { + return Ok(false); + }; + let lineage = reachable_graph_keys(&[target], graph, EdgeDirection::Upstream); + outside_lineage(audit, &lineage) +} + +/// `outside_target_lineage` of each audit, walking each distinct target's lineage once. +/// +/// Many audits attach to one model; walking its upstream lineage per audit dominated planning. +/// One target's lineage is held at a time. +pub(crate) fn outside_target_lineages( + audits: &[Bound<'_, PyAny>], + graph: &ProjectGraph, +) -> PyResult> { + let mut outside: Vec = vec![false; audits.len()]; + let mut by_target: HashMap<(String, String), Vec> = HashMap::new(); + for (position, audit) in audits.iter().enumerate() { + if let Some(target) = lineage_target(audit)? { + by_target.entry(target).or_default().push(position); + } + } + for (target, positions) in by_target { + let lineage = reachable_graph_keys(&[target], graph, EdgeDirection::Upstream); + for position in positions { + outside[position] = outside_lineage(&audits[position], &lineage)?; + } + } + Ok(outside) +} + +/// The key an attached audit's target lineage starts from, or `None` for an unattached audit. +fn lineage_target(audit: &Bound<'_, PyAny>) -> PyResult> { let py: Python<'_> = audit.py(); let kind: Bound<'_, PyAny> = attached_target_kind(audit)?; let name: Bound<'_, PyAny> = attached_target_name(audit)?; if kind.is_none() || name.is_none() { - return Ok(false); + return Ok(None); } let kind: String = text(&imported(py, COMPILE_TYPES_MODULE, "AttachedAuditTargetKind")?.call1((kind,))?)?; - let target = (kind, text(&name)?); - let lineage = reachable_graph_keys(&[target], graph, EdgeDirection::Upstream); + Ok(Some((kind, text(&name)?))) +} + +/// Whether the audit reads a dependency outside `lineage`. +fn outside_lineage( + audit: &Bound<'_, PyAny>, + lineage: &HashSet<(String, String)>, +) -> PyResult { for dependency in scope_deps(audit)?.try_iter()? { if !lineage.contains(&native_key(&dependency?)?) { return Ok(true); diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/main/audit_entries/build_selected_audit_entries.rs b/crates/sqlbuild-python-planner/src/planner_strategy/main/audit_entries/build_selected_audit_entries.rs index 69c28222c5..9013084966 100644 --- a/crates/sqlbuild-python-planner/src/planner_strategy/main/audit_entries/build_selected_audit_entries.rs +++ b/crates/sqlbuild-python-planner/src/planner_strategy/main/audit_entries/build_selected_audit_entries.rs @@ -1,6 +1,7 @@ //! Assemble selected audits in compiled-project order without querying lineage for an empty selection. use crate::planner_strategy::_helpers::audit_entries::plan_audit; +use crate::planner_strategy::_helpers::audits::outside_target_lineages; use crate::planner_strategy::main::audit_entries::audit_is_selected::audit_is_selected; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyList, PyListMethods}; @@ -39,8 +40,10 @@ pub fn build_selected_audit_entries<'py>( model_materializations: request.get("model_materializations")?, lineage: native_field(&project, "lineage_graph")?.extract()?, }; - for audit in selected { - entries.append(plan_audit(&audit, &planning)?)?; + let outside: Vec = + outside_target_lineages(&selected, planning.lineage.bind(py).borrow().graph())?; + for (audit, outside) in selected.iter().zip(outside) { + entries.append(plan_audit(audit, &planning, Some(outside))?)?; } Ok(entries) } From 6ec712200896fd3a3648ca81dcf77874e64c159b Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 18 Nov 2026 23:00:48 +0000 Subject: [PATCH 05/15] perf(plan): read remembered query fingerprints without releasing the GIL Change detection released and reacquired the GIL for every model's query fingerprint, even when the fingerprint was already remembered. A remembered fingerprint is now read directly, keyed so a lookup borrows the SQL text, and a recorded baseline that cannot differ skips the release too. --- .../src/planner_identity/_helpers/hashing.rs | 59 ++++++++++++++----- 1 file changed, 43 insertions(+), 16 deletions(-) diff --git a/crates/sqlbuild-python-project/src/planner_identity/_helpers/hashing.rs b/crates/sqlbuild-python-project/src/planner_identity/_helpers/hashing.rs index 88460a1e35..2eb6a8a87b 100644 --- a/crates/sqlbuild-python-project/src/planner_identity/_helpers/hashing.rs +++ b/crates/sqlbuild-python-project/src/planner_identity/_helpers/hashing.rs @@ -23,8 +23,24 @@ pub(crate) static QUERY_FINGERPRINTS: Mutex> = Mutex:: /// Distinct SQL texts the memo keeps before starting over; a 10K-model project holds about 10K. const QUERY_FINGERPRINTS_LIMIT: usize = 100_000; -/// Fingerprint results by `(dialect, SQL text)`. -pub(crate) type QueryFingerprints = HashMap<(Option, String), Result>; +/// Fingerprint results by dialect, then by SQL text, so a lookup borrows the SQL text. +pub(crate) type QueryFingerprints = Vec<(Option, HashMap>)>; + +/// The remembered fingerprint of `sql` under `dialect`, if any. +fn remembered( + memo: &Mutex>, + sql: &str, + dialect: Option<&str>, +) -> Option> { + memo.lock() + .unwrap_or_else(PoisonError::into_inner) + .as_ref()? + .iter() + .find(|(known, _)| known.as_deref() == dialect)? + .1 + .get(sql) + .cloned() +} /// `compiler_query_fingerprint`, remembered by dialect and SQL text. pub(crate) fn query_fingerprint( @@ -32,22 +48,27 @@ pub(crate) fn query_fingerprint( sql: &str, dialect: Option<&str>, ) -> Result { - let key: (Option, String) = (dialect.map(str::to_owned), sql.to_owned()); - if let Some(known) = memo - .lock() - .unwrap_or_else(PoisonError::into_inner) - .as_ref() - .and_then(|memo| memo.get(&key)) - { - return known.clone(); + if let Some(known) = remembered(memo, sql, dialect) { + return known; } let fingerprint: Result = compiler_query_fingerprint(sql, dialect); let mut memo = memo.lock().unwrap_or_else(PoisonError::into_inner); - let entries: &mut QueryFingerprints = memo.get_or_insert_with(HashMap::new); - if entries.len() > QUERY_FINGERPRINTS_LIMIT { - entries.clear(); + let dialects: &mut QueryFingerprints = memo.get_or_insert_with(Vec::new); + if dialects.iter().map(|(_, texts)| texts.len()).sum::() > QUERY_FINGERPRINTS_LIMIT { + dialects.clear(); + } + match dialects + .iter_mut() + .find(|(known, _)| known.as_deref() == dialect) + { + Some((_, texts)) => { + texts.insert(sql.to_owned(), fingerprint.clone()); + } + None => dialects.push(( + dialect.map(str::to_owned), + HashMap::from([(sql.to_owned(), fingerprint.clone())]), + )), } - entries.insert(key, fingerprint.clone()); fingerprint } @@ -57,8 +78,11 @@ pub(crate) fn model_query_hash( model_name: &str, (query_sql, dialect): (&str, Option<&str>), ) -> PyResult { - py.compiler_detach(|| query_fingerprint(&QUERY_FINGERPRINTS, query_sql, dialect)) - .map_err(|error| fingerprint_error(py, &format!("Model '{model_name}' SQL"), error)) + match remembered(&QUERY_FINGERPRINTS, query_sql, dialect) { + Some(known) => known, + None => py.compiler_detach(|| query_fingerprint(&QUERY_FINGERPRINTS, query_sql, dialect)), + } + .map_err(|error| fingerprint_error(py, &format!("Model '{model_name}' SQL"), error)) } /// `current`, or the fingerprint recorded with a `Fingerprint` row whose definition fingerprints to `current`. @@ -73,6 +97,9 @@ pub(crate) fn recorded_baseline( } let definition: String = native_field(recorded, "definition")?.extract()?; let recorded_hash: String = native_field(recorded, "definition_hash")?.extract()?; + if recorded_hash == current || recorded_hash.is_empty() || definition.is_empty() { + return Ok(current); + } py.compiler_detach(|| { Ok(baseline_fingerprint( current, From f6a5b86ecdcd73e0a554f1f1ce76cbd8a346aefe Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 19 Nov 2026 00:45:13 +0000 Subject: [PATCH 06/15] perf(plan): skip the lineage walk for audits that read only their target or its parents An attached audit's target lineage always holds the target and its direct parents, so an audit that reads nothing else needs no walk. Only the rest walk their target's lineage, once per target. --- .../_helpers/audit_entries.rs | 21 ++++++++-- .../src/planner_strategy/_helpers/audits.rs | 38 ++++++++++++++----- .../build_selected_audit_entries.rs | 9 ++--- 3 files changed, 48 insertions(+), 20 deletions(-) diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/audit_entries.rs b/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/audit_entries.rs index 9793caa19d..d7296f9142 100644 --- a/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/audit_entries.rs +++ b/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/audit_entries.rs @@ -5,7 +5,7 @@ use sqlbuild_analysis::graph::models::GraphIndexes; use sqlbuild_python_base::native_host::models::Dialect; use crate::planner_strategy::_helpers::audits::{ - attachment, outside_target_lineage, resolve_effective_run_scope, + attachment, outside_target_lineage, outside_target_lineages, resolve_effective_run_scope, }; use pyo3::IntoPyObject; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult}; @@ -118,9 +118,22 @@ pub(crate) fn plan_audit_of<'py>( ) } -/// One audit's plan entry, rendered against the planning relations. -/// -/// `outside_lineage` is whether the audit reads outside its target's lineage, when already known. +/// Plan entries of `audits`, walking each target's lineage at most once. +pub(crate) fn plan_audits<'py>( + audits: &[Bound<'py, PyAny>], + planning: &AuditPlanning<'py>, +) -> PyResult>> { + let py = planning.model_locations.py(); + let outside: Vec = + outside_target_lineages(audits, planning.lineage.bind(py).borrow().graph())?; + let mut entries: Vec> = Vec::new(); + for (audit, outside) in audits.iter().zip(outside) { + entries.push(plan_audit(audit, planning, Some(outside))?); + } + Ok(entries) +} + +/// One audit's plan entry, rendered against the planning relations, with `outside_lineage` if known. pub(crate) fn plan_audit<'py>( audit: &Bound<'py, PyAny>, request: &AuditPlanning<'py>, diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/audits.rs b/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/audits.rs index 0dbc27999b..8e01a0215e 100644 --- a/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/audits.rs +++ b/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/audits.rs @@ -10,6 +10,7 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::PyTuple; use pyo3::{IntoPyObject, pyfunction}; +use sqlbuild_analysis::graph::main::edge_keys::edge_keys; use sqlbuild_analysis::graph::main::graph_from_indexes::graph_from_indexes; use sqlbuild_analysis::graph::main::reachable_graph_keys::reachable_graph_keys; use sqlbuild_analysis::graph::models::{EdgeDirection, GraphIndexes, ProjectGraph}; @@ -157,25 +158,42 @@ pub(crate) fn outside_target_lineage( outside_lineage(audit, &lineage) } -/// `outside_target_lineage` of each audit, walking each distinct target's lineage once. -/// -/// Many audits attach to one model; walking its upstream lineage per audit dominated planning. -/// One target's lineage is held at a time. +/// An audit's position among the selected audits and the keys of what it reads. +type AuditDependencies = (usize, Vec<(String, String)>); + +/// `outside_target_lineage` of each audit, walking a target's lineage once and only when needed. pub(crate) fn outside_target_lineages( audits: &[Bound<'_, PyAny>], graph: &ProjectGraph, ) -> PyResult> { let mut outside: Vec = vec![false; audits.len()]; - let mut by_target: HashMap<(String, String), Vec> = HashMap::new(); + let mut by_target: HashMap<(String, String), Vec> = HashMap::new(); for (position, audit) in audits.iter().enumerate() { - if let Some(target) = lineage_target(audit)? { - by_target.entry(target).or_default().push(position); + let Some(target) = lineage_target(audit)? else { + continue; + }; + let dependencies: Vec<(String, String)> = scope_deps(audit)? + .try_iter()? + .map(|dependency| native_key(&dependency?)) + .collect::>()?; + let parents = edge_keys(graph, &target, EdgeDirection::Upstream); + if dependencies + .iter() + .all(|dependency| *dependency == target || parents.contains(dependency)) + { + continue; } + by_target + .entry(target) + .or_default() + .push((position, dependencies)); } - for (target, positions) in by_target { + for (target, walked) in by_target { let lineage = reachable_graph_keys(&[target], graph, EdgeDirection::Upstream); - for position in positions { - outside[position] = outside_lineage(&audits[position], &lineage)?; + for (position, dependencies) in walked { + outside[position] = dependencies + .iter() + .any(|dependency| !lineage.contains(dependency)); } } Ok(outside) diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/main/audit_entries/build_selected_audit_entries.rs b/crates/sqlbuild-python-planner/src/planner_strategy/main/audit_entries/build_selected_audit_entries.rs index 9013084966..06a2ffbafe 100644 --- a/crates/sqlbuild-python-planner/src/planner_strategy/main/audit_entries/build_selected_audit_entries.rs +++ b/crates/sqlbuild-python-planner/src/planner_strategy/main/audit_entries/build_selected_audit_entries.rs @@ -1,7 +1,6 @@ //! Assemble selected audits in compiled-project order without querying lineage for an empty selection. -use crate::planner_strategy::_helpers::audit_entries::plan_audit; -use crate::planner_strategy::_helpers::audits::outside_target_lineages; +use crate::planner_strategy::_helpers::audit_entries::plan_audits; use crate::planner_strategy::main::audit_entries::audit_is_selected::audit_is_selected; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyList, PyListMethods}; @@ -40,10 +39,8 @@ pub fn build_selected_audit_entries<'py>( model_materializations: request.get("model_materializations")?, lineage: native_field(&project, "lineage_graph")?.extract()?, }; - let outside: Vec = - outside_target_lineages(&selected, planning.lineage.bind(py).borrow().graph())?; - for (audit, outside) in selected.iter().zip(outside) { - entries.append(plan_audit(audit, &planning, Some(outside))?)?; + for entry in plan_audits(&selected, &planning)? { + entries.append(entry)?; } Ok(entries) } From 145363cc952b0f81093f1cf45c1be5adb3388ec6 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 19 Nov 2026 00:45:15 +0000 Subject: [PATCH 07/15] feat(location)!: return the kept plan for an identical preview request A location keeps its last 16 finished plans for the published generation and synced copy it planned from, in memory, and answers an identical request (the same plan arguments, selection and consumption) with the kept plan instead of planning again. The key holds everything the plan reads: the frozen settings become the plan arguments, and named jobs' settings and per-node warehouses come from the published project (the server freezes job selectors to node keys and refuses --no-job-settings). A newer generation or synced copy drops every kept plan. The reply's timings say it was reused and when that plan was made. Plan again in the web UI sends fresh: true, which plans anew. The 10K preview guard plans its warm preview fresh, so it still measures planning, and checks that the same request again returns the kept plan. BREAKING CHANGE: plan requests (POST .../targets/{target}/plans) and plan location work now require a fresh field. Clients must send "fresh": false or true, and servers and locations must be upgraded together; a location from an earlier release rejects plan work from this server. --- clients/typescript/src/schema.d.ts | 4 + .../native_commands/_helpers/location/mod.rs | 1 + .../_helpers/location/plan_reuse.rs | 85 +++++++++ .../src/native_commands/constants.rs | 3 + .../main/_location_changed_build.rs | 2 +- .../native_commands/main/_location_plan.rs | 27 ++- .../native_commands/main/_location_work.rs | 12 +- .../src/native_commands/models.rs | 30 ++++ .../src/native_commands/tests/helpers.rs | 46 ++++- .../tests/test_location_plan.rs | 166 +++++++++++++++++- .../src/native_commands/tests/test_types.rs | 28 +++ .../src/platform/models.rs | 4 + .../src/runs/_helpers/requests/plan.rs | 1 + .../tests/api/plans/helpers.rs | 1 + .../tests/api/test_person_plans.rs | 29 ++- .../tests/api/test_types.rs | 2 + .../sqlbuild/references/docs/cli/server.md | 10 +- .../cli/commands/main/plan/helpers.py | 13 +- .../test_duckshop_10k_preview_performance.py | 12 +- .../publication/_test_types.py | 2 + .../publication/catalog/helpers.py | 14 +- .../compiled_project/publication/helpers.py | 14 +- .../publication/test_plan_preview.py | 11 +- ui/src/lib/api/client.ts | 7 +- ui/src/lib/api/http-audit.test.ts | 2 +- ui/src/lib/api/http-client.test.ts | 28 +++ ui/src/lib/api/http-client.ts | 4 +- ui/src/lib/api/mock/mock-client.ts | 2 +- ui/src/routes/plan/+page.svelte | 7 +- website/src/content/docs/docs/cli/server.mdx | 10 +- 30 files changed, 532 insertions(+), 45 deletions(-) create mode 100644 crates/sqlbuild-cli/src/native_commands/_helpers/location/plan_reuse.rs diff --git a/clients/typescript/src/schema.d.ts b/clients/typescript/src/schema.d.ts index 63288c6dbc..ae7d871adf 100644 --- a/clients/typescript/src/schema.d.ts +++ b/clients/typescript/src/schema.d.ts @@ -2311,6 +2311,8 @@ export interface components { kind: "evaluate_sensor"; } | { command: string[]; + /** @description Plan again even when the location kept a plan for the same request. */ + fresh: boolean; /** @enum {string} */ kind: "plan"; selection: string[]; @@ -2812,6 +2814,8 @@ export interface components { /** @description A person's build command to plan on the location; the server freezes it like an ad-hoc run. */ PlanRequest: { command: string[]; + /** @description Plan again instead of returning the plan the location kept for the same request. */ + fresh: boolean; /** Format: uuid */ request_id: string; }; diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/location/mod.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/location/mod.rs index dc88ad9111..ad8a09edea 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/location/mod.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/location/mod.rs @@ -3,6 +3,7 @@ pub(crate) mod execution; pub(crate) mod held_copy; pub(crate) mod plan_document; pub(crate) mod plan_request; +pub(crate) mod plan_reuse; pub(crate) mod sync_outcomes; pub(crate) mod sync_report; pub(crate) mod work; diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/location/plan_reuse.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/location/plan_reuse.rs new file mode 100644 index 0000000000..d5b3ad26b1 --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/location/plan_reuse.rs @@ -0,0 +1,85 @@ +//! The location's last finished plans, in memory, returned again for an identical request. + +use std::collections::VecDeque; +use std::sync::{Mutex, PoisonError}; + +use chrono::{DateTime, SecondsFormat, Utc}; +use serde_json::{Value, json}; + +use crate::argv::models::Stream; +use crate::native_commands::_helpers::command::terminal::write_output; +use crate::native_commands::constants::REUSED_PLANS; +use crate::native_commands::models::{PlanKey, PlanReuse, ReusedPlan}; +use crate::native_commands::types::LocationPlan; + +/// The location's kept plans; one location serves one target. +pub(crate) static PLAN_REUSE: Mutex> = Mutex::new(None); + +/// The kept plan for `key` under this generation and copy, marked as reused. +pub(crate) fn reused_plan( + reuse: &Mutex>, + (generation, version): (i64, &str), + key: &PlanKey, +) -> Option { + let kept: ReusedPlan = reuse + .lock() + .unwrap_or_else(PoisonError::into_inner) + .as_ref() + .filter(|reuse| reuse.generation == generation && reuse.version == version)? + .plans + .iter() + .find(|plan| plan.key == *key)? + .clone(); + let mut document: Value = match serde_json::from_str(&kept.document) { + Ok(document) => document, + Err(error) => { + drop(write_output(&[( + Stream::Stderr, + &format!("A kept plan could not be read, so the location plans again: {error}\n"), + )])); + return None; + } + }; + let planned_from: Value = document["timings"]["planned_from"].clone(); + document["timings"] = json!({ + "planned_from": planned_from, + "reused_from": kept.planned_at, + "total_seconds": 0.0, + "steps": [{"name": format!("Reused plan from {}", kept.planned_at), "seconds": 0.0}], + }); + Some((document, kept.diffs)) +} + +/// Keep a finished plan for `key`, unless it failed; a newer generation or copy replaces all. +pub(crate) fn remember_plan( + reuse: &Mutex>, + (generation, version): (i64, &str), + key: PlanKey, + ((document, diffs), planned_at): (&LocationPlan, DateTime), +) { + if !document["error"].is_null() { + return; + } + let Ok(text) = serde_json::to_string(document) else { + return; + }; + let mut reuse = reuse.lock().unwrap_or_else(PoisonError::into_inner); + let current: &mut PlanReuse = match reuse.as_mut() { + Some(current) if current.generation == generation && current.version == version => current, + _ => reuse.insert(PlanReuse { + generation, + version: version.to_owned(), + plans: VecDeque::new(), + }), + }; + current.plans.retain(|plan| plan.key != key); + current.plans.push_back(ReusedPlan { + key, + document: text, + diffs: diffs.clone(), + planned_at: planned_at.to_rfc3339_opts(SecondsFormat::Secs, true), + }); + while current.plans.len() > REUSED_PLANS { + current.plans.pop_front(); + } +} diff --git a/crates/sqlbuild-cli/src/native_commands/constants.rs b/crates/sqlbuild-cli/src/native_commands/constants.rs index caf2ead03a..696430830a 100644 --- a/crates/sqlbuild-cli/src/native_commands/constants.rs +++ b/crates/sqlbuild-cli/src/native_commands/constants.rs @@ -218,6 +218,9 @@ pub(crate) const CHANGED_BUILD_COMMAND: [&str; 2] = ["build", "--changed"]; /// How long an idle location waits between checks for named work; after work it checks at once. pub(crate) const LOCATION_WORK_IDLE_POLL: std::time::Duration = std::time::Duration::from_secs(2); +/// Finished plans a location keeps for identical requests; a newer generation or synced copy drops them. +pub(crate) const REUSED_PLANS: usize = 16; + /// What a location reports when `sqb plan` produced no plan. pub(crate) const PLAN_NOT_PRODUCED: &str = "sqb plan did not produce a plan; see the location's log"; diff --git a/crates/sqlbuild-cli/src/native_commands/main/_location_changed_build.rs b/crates/sqlbuild-cli/src/native_commands/main/_location_changed_build.rs index 02ccfba216..389d3053af 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/_location_changed_build.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/_location_changed_build.rs @@ -46,7 +46,7 @@ pub(super) async fn report_changed_build( report("Planning what changed from the synced warehouse copy...\n"); let Some((document, _)) = plan_location( (options, runtime), - (&BTreeSet::new(), &settings), + (&BTreeSet::new(), &settings, false), (client, session_id, generation), stopped, ) diff --git a/crates/sqlbuild-cli/src/native_commands/main/_location_plan.rs b/crates/sqlbuild-cli/src/native_commands/main/_location_plan.rs index 0ba6251feb..2412f702a6 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/_location_plan.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/_location_plan.rs @@ -3,6 +3,7 @@ use std::collections::{BTreeMap, BTreeSet}; use std::future::Future; +use chrono::Utc; use serde_json::{Value, json}; use sqlbuild_orchestration::platform::models::LocationWorkClient; use sqlbuild_orchestration::store::errors::ControlError; @@ -15,13 +16,16 @@ use crate::native_commands::_helpers::location::held_copy::{HELD_COPY, held_copy use crate::native_commands::_helpers::location::plan_request::{ plan_arguments, run_plan, selected_names, }; -use crate::native_commands::models::{HeldCopy, LocationRuntimeRequest, LocationServe}; +use crate::native_commands::_helpers::location::plan_reuse::{ + PLAN_REUSE, remember_plan, reused_plan, +}; +use crate::native_commands::models::{HeldCopy, LocationRuntimeRequest, LocationServe, PlanKey}; use crate::native_commands::types::LocationPlan; /// The plan document with the diffs it left out, an `{"error": ...}` report, or `None` when stopped first. pub(super) async fn plan_location( (options, runtime): (&LocationServe, &mpsc::Sender), - (selection, settings): (&BTreeSet, &BTreeMap), + (selection, settings, fresh): (&BTreeSet, &BTreeMap, bool), (client, session_id, generation): (&LocationWorkClient, Uuid, i64), stopped: impl Future>, ) -> Result, ControlError> { @@ -54,6 +58,14 @@ pub(super) async fn plan_location( Some(held) if copy.unchanged => held, _ => hold_copy(&HELD_COPY, options, generation, (copy, read_consumption))?, }; + let key: PlanKey = PlanKey { + argv: argv.clone(), + select: select.clone(), + consumption: planned.consumption, + }; + if !fresh && let Some(reused) = reused_plan(&PLAN_REUSE, (generation, &planned.version), &key) { + return Ok(Some(reused)); + } let request = json!({"plan": { "argv": argv, "select": select, @@ -61,5 +73,14 @@ pub(super) async fn plan_location( "version": planned.version, "consumption": planned.consumption, }}); - run_plan(runtime, request, stopped).await + let plan: Option = run_plan(runtime, request, stopped).await?; + if let Some(plan) = &plan { + remember_plan( + &PLAN_REUSE, + (generation, &planned.version), + key, + (plan, Utc::now()), + ); + } + Ok(plan) } diff --git a/crates/sqlbuild-cli/src/native_commands/main/_location_work.rs b/crates/sqlbuild-cli/src/native_commands/main/_location_work.rs index c90cf0c516..7dd5e6c509 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/_location_work.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/_location_work.rs @@ -185,9 +185,15 @@ async fn perform_work( LocationWork::Plan { selection, settings, + fresh, .. - } => plan_location((options, runtime), (&selection, &settings), copy, expires) - .await? - .map(|(document, diffs)| (LocationWorkOutcome::Plan { document, diffs }, Vec::new())), + } => plan_location( + (options, runtime), + (&selection, &settings, fresh), + copy, + expires, + ) + .await? + .map(|(document, diffs)| (LocationWorkOutcome::Plan { document, diffs }, Vec::new())), }) } diff --git a/crates/sqlbuild-cli/src/native_commands/models.rs b/crates/sqlbuild-cli/src/native_commands/models.rs index b7ef802894..45469b43b3 100644 --- a/crates/sqlbuild-cli/src/native_commands/models.rs +++ b/crates/sqlbuild-cli/src/native_commands/models.rs @@ -565,6 +565,36 @@ pub(crate) enum LostStdout { Raised(String), } +/// What makes two plan requests identical once the generation and synced copy are the same. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct PlanKey { + /// The `sqb plan` arguments the location built from the request's settings. + pub(crate) argv: Vec, + /// The node names the server froze from the request's selection. + pub(crate) select: Vec, + /// Whether the plan read the copy's consumption. + pub(crate) consumption: bool, +} + +/// One finished plan, kept as its JSON text with the diffs that travel beside it. +#[derive(Clone, Debug)] +pub(crate) struct ReusedPlan { + pub(crate) key: PlanKey, + pub(crate) document: String, + pub(crate) diffs: + std::collections::BTreeMap, + /// When the plan finished, as RFC 3339 text. + pub(crate) planned_at: String, +} + +/// The finished plans of one published generation and synced copy, oldest first. +#[derive(Clone, Debug)] +pub(crate) struct PlanReuse { + pub(crate) generation: i64, + pub(crate) version: String, + pub(crate) plans: std::collections::VecDeque, +} + /// The copy file the location last read, the mirror version it holds and whether it has consumption. #[derive(Clone, Debug)] pub(crate) struct HeldCopy { diff --git a/crates/sqlbuild-cli/src/native_commands/tests/helpers.rs b/crates/sqlbuild-cli/src/native_commands/tests/helpers.rs index ec83c94d0a..866679ba7a 100644 --- a/crates/sqlbuild-cli/src/native_commands/tests/helpers.rs +++ b/crates/sqlbuild-cli/src/native_commands/tests/helpers.rs @@ -1,4 +1,8 @@ use std::path::Path; +use std::sync::Mutex; + +use chrono::Utc; +use serde_json::json; use sqlbuild_orchestration::warehouse_metadata::models::{MetadataBatch, WarehouseStateLocation}; use uuid::Uuid; @@ -10,7 +14,11 @@ use crate::native_commands::_helpers::command::paths::location_watch; use crate::native_commands::_helpers::location::execution::{ acknowledge_metadata_page, settle_publication, }; -use crate::native_commands::models::{LocationMetadataPage, LocationMetadataSync, LocationServe}; +use crate::native_commands::_helpers::location::plan_reuse::remember_plan; +use crate::native_commands::models::{ + LocationMetadataPage, LocationMetadataSync, LocationServe, PlanKey, PlanReuse, +}; +use crate::native_commands::types::LocationPlan; use sqlbuild_orchestration::platform::main::work::connect_location_work::connect_location_work; use sqlbuild_orchestration::platform::models::{ ConnectionCheckTrigger, LocationWorkClient, LocationWorkPoll, @@ -132,3 +140,39 @@ pub(super) fn published_location( state.idle = false; ((options, script), (client, poll), state) } + +/// The plan key of a `--changed` plan of `select`. +pub(super) fn reuse_key(select: &[&str]) -> PlanKey { + PlanKey { + argv: vec!["plan".into(), "--json".into(), "--changed".into()], + select: select.iter().map(|name| (*name).to_owned()).collect(), + consumption: true, + } +} + +/// A finished plan document, or a failed plan's error report. +pub(super) fn finished_plan(failed: bool) -> LocationPlan { + let documents = [ + json!({ + "models": [{"name": "orders"}], + "timings": {"planned_from": "synced_state", "total_seconds": 4.2, + "steps": [{"name": "Detecting changes", "seconds": 4.2}]}, + }), + json!({"error": {"code": "plan_failed", "message": "no plan"}}), + ]; + (documents[usize::from(failed)].clone(), Default::default()) +} + +/// `count` finished plans of `model_` under generation 1 and `copy-1`, oldest first. +pub(super) fn kept_plans(count: usize) -> Mutex> { + let reuse: Mutex> = Mutex::new(None); + for index in 0..count { + remember_plan( + &reuse, + (1, "copy-1"), + reuse_key(&[format!("model_{index}").as_str()]), + (&finished_plan(false), Utc::now()), + ); + } + reuse +} diff --git a/crates/sqlbuild-cli/src/native_commands/tests/test_location_plan.rs b/crates/sqlbuild-cli/src/native_commands/tests/test_location_plan.rs index 7059286714..06f440066b 100644 --- a/crates/sqlbuild-cli/src/native_commands/tests/test_location_plan.rs +++ b/crates/sqlbuild-cli/src/native_commands/tests/test_location_plan.rs @@ -1,12 +1,24 @@ use serde_json::{Value, json}; use sqlbuild_orchestration::platform::constants::PLAN_DOCUMENT_BYTES; +use std::sync::Mutex; + +use chrono::{TimeZone, Utc}; + use crate::native_commands::_helpers::location::plan_document::plan_document; +use crate::native_commands::_helpers::location::plan_request::plan_arguments; +use crate::native_commands::_helpers::location::plan_reuse::{remember_plan, reused_plan}; use crate::native_commands::_helpers::location::work::next_work_check; use crate::native_commands::constants::LOCATION_WORK_IDLE_POLL; +use crate::native_commands::constants::REUSED_PLANS; +use crate::native_commands::models::LocationServe; use crate::native_commands::models::LocationWorkTurn; -use crate::native_commands::tests::helpers::plan_output; -use crate::native_commands::tests::test_types::{PlanDocumentTestCase, WorkCheckTestCase}; +use crate::native_commands::models::PlanReuse; +use crate::native_commands::tests::helpers::{finished_plan, kept_plans, plan_output, reuse_key}; +use crate::native_commands::tests::test_types::{ + PlanArgumentsTestCase, PlanDocumentTestCase, PlanReuseBoundTestCase, PlanReuseTestCase, + WorkCheckTestCase, +}; #[test] fn given_plan_output_when_reading_it_on_a_location_then_diffs_always_travel_beside_the_document() { @@ -103,3 +115,153 @@ fn given_a_turn_of_location_work_when_scheduling_the_next_check_then_only_an_idl ); } } + +#[test] +fn given_a_kept_plan_when_the_same_request_comes_again_then_only_the_same_generation_copy_and_request_reuse_it() + { + let reused_timings = json!({"planned_from": "synced_state", "reused_from": "2026-01-02T03:04:05Z", + "total_seconds": 0.0, + "steps": [{"name": "Reused plan from 2026-01-02T03:04:05Z", "seconds": 0.0}]}); + let test_cases = [ + PlanReuseTestCase { + description: "the same generation, copy and request reuse the plan and say when it was made", + kept: (3, "copy-1", false), + asked: (3, "copy-1", &["orders"]), + expected_timings: Some(reused_timings), + }, + PlanReuseTestCase { + description: "a newer synced copy plans again", + kept: (3, "copy-1", false), + asked: (3, "copy-2", &["orders"]), + expected_timings: None, + }, + PlanReuseTestCase { + description: "a newer generation plans again", + kept: (3, "copy-1", false), + asked: (4, "copy-1", &["orders"]), + expected_timings: None, + }, + PlanReuseTestCase { + description: "another selection plans again", + kept: (3, "copy-1", false), + asked: (3, "copy-1", &["customers"]), + expected_timings: None, + }, + PlanReuseTestCase { + description: "a failed plan is not kept", + kept: (3, "copy-1", true), + asked: (3, "copy-1", &["orders"]), + expected_timings: None, + }, + ]; + let planned_at = Utc + .with_ymd_and_hms(2026, 1, 2, 3, 4, 5) + .single() + .expect("time"); + for test_case in test_cases { + let reuse: Mutex> = Mutex::new(None); + let (generation, version, failed) = test_case.kept; + remember_plan( + &reuse, + (generation, version), + reuse_key(&["orders"]), + (&finished_plan(failed), planned_at), + ); + let (generation, version, select) = test_case.asked; + let reused = reused_plan(&reuse, (generation, version), &reuse_key(select)); + assert_eq!( + reused.map(|(document, _)| document["timings"].clone()), + test_case.expected_timings, + "{description}", + description = test_case.description + ); + } +} + +#[test] +fn given_more_finished_plans_than_kept_when_planning_then_the_oldest_is_dropped_and_a_newer_copy_drops_all() + { + let test_cases = [ + PlanReuseBoundTestCase { + description: "the oldest of one plan too many is dropped", + newer_copy: false, + asked: ("model_0", "copy-1"), + expected_reused: false, + }, + PlanReuseBoundTestCase { + description: "the next oldest is kept", + newer_copy: false, + asked: ("model_1", "copy-1"), + expected_reused: true, + }, + PlanReuseBoundTestCase { + description: "a newer synced copy drops every kept plan", + newer_copy: true, + asked: ("model_1", "copy-2"), + expected_reused: false, + }, + PlanReuseBoundTestCase { + description: "a plan of the newer copy is kept", + newer_copy: true, + asked: ("model_new", "copy-2"), + expected_reused: true, + }, + ]; + for test_case in test_cases { + let reuse: Mutex> = kept_plans(REUSED_PLANS + 1); + let newer: &[(&str, &str)] = + [&[][..], &[("model_new", "copy-2")][..]][usize::from(test_case.newer_copy)]; + for &(name, version) in newer { + remember_plan( + &reuse, + (1, version), + reuse_key(&[name]), + (&finished_plan(false), Utc::now()), + ); + } + let (name, version) = test_case.asked; + assert_eq!( + reused_plan(&reuse, (1, version), &reuse_key(&[name])).is_some(), + test_case.expected_reused, + "{description}", + description = test_case.description + ); + } +} + +#[test] +fn given_frozen_plan_settings_when_building_the_plan_arguments_then_run_request_settings_never_vary_a_kept_plan() + { + let test_cases = [ + PlanArgumentsTestCase { + description: "build settings become the plan arguments the kept plan is keyed on", + settings: json!({"changed": true}), + expected_refused: false, + }, + PlanArgumentsTestCase { + description: "--no-job-settings belongs to the run request and never reaches a plan", + settings: json!({"changed": true, "job_settings": false}), + expected_refused: true, + }, + ]; + let options = LocationServe { + project_dir: "/project".into(), + server_url: "http://127.0.0.1:1".into(), + location_id: uuid::Uuid::nil(), + target: "development".into(), + image_id: None, + metadata_reconcile_seconds: 3600, + safety_sync_seconds: 0, + dev_events: false, + }; + for test_case in test_cases { + let settings = serde_json::from_value(test_case.settings).expect("settings"); + let argv: Result, String> = plan_arguments(&options, &settings); + assert_eq!( + argv.is_err(), + test_case.expected_refused, + "{description}: {argv:?}", + description = test_case.description + ); + } +} diff --git a/crates/sqlbuild-cli/src/native_commands/tests/test_types.rs b/crates/sqlbuild-cli/src/native_commands/tests/test_types.rs index 2bf84f524f..590deafa4c 100644 --- a/crates/sqlbuild-cli/src/native_commands/tests/test_types.rs +++ b/crates/sqlbuild-cli/src/native_commands/tests/test_types.rs @@ -90,6 +90,34 @@ pub(super) struct InitFailureTestCase { pub(super) expected_exit_code: i32, } +/// The frozen settings of one plan request. +pub(super) struct PlanArgumentsTestCase { + pub(super) description: &'static str, + pub(super) settings: serde_json::Value, + /// Whether the location refuses the settings instead of planning, and keeping, them. + pub(super) expected_refused: bool, +} + +/// One kept plan, then one request for a plan. +pub(super) struct PlanReuseTestCase { + pub(super) description: &'static str, + /// The kept plan's generation, copy version and whether it failed. + pub(super) kept: (i64, &'static str, bool), + /// The generation, copy version and selected names of the request. + pub(super) asked: (i64, &'static str, &'static [&'static str]), + /// The reused plan's timings, or `None` when the location plans again. + pub(super) expected_timings: Option, +} + +/// One more finished plan than a location keeps, maybe a plan of a newer copy, then one request. +pub(super) struct PlanReuseBoundTestCase { + pub(super) description: &'static str, + pub(super) newer_copy: bool, + /// The selected name and copy version of the request. + pub(super) asked: (&'static str, &'static str), + pub(super) expected_reused: bool, +} + /// One `sqb plan --json` output, whether it exited as a plan, and the first model the server receives. pub(super) struct PlanDocumentTestCase { pub(super) description: &'static str, diff --git a/crates/sqlbuild-orchestration/src/platform/models.rs b/crates/sqlbuild-orchestration/src/platform/models.rs index 4bee656326..b79c73a059 100644 --- a/crates/sqlbuild-orchestration/src/platform/models.rs +++ b/crates/sqlbuild-orchestration/src/platform/models.rs @@ -214,6 +214,8 @@ pub enum LocationWork { selection: std::collections::BTreeSet, #[schema(value_type = Object)] settings: std::collections::BTreeMap, + /// Plan again even when the location kept a plan for the same request. + fresh: bool, }, } @@ -233,6 +235,8 @@ pub struct LocationWorkRequest { pub struct PlanRequest { pub request_id: Uuid, pub command: Vec, + /// Plan again instead of returning the plan the location kept for the same request. + pub fresh: bool, } /// A person asks a location for one named read-only operation, such as a connection test. diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/plan.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/plan.rs index cb0ca4ef81..1b5dbdbd69 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/plan.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/plan.rs @@ -121,6 +121,7 @@ pub(in crate::runs) async fn request_plan_in( .into_iter() .filter(|(name, _)| name != PLAN_IGNORED_SETTING) .collect(), + fresh: request.fresh, }, }; let transaction = enqueue_location_work( diff --git a/crates/sqlbuild-orchestration/tests/api/plans/helpers.rs b/crates/sqlbuild-orchestration/tests/api/plans/helpers.rs index c693b359d4..266d72c524 100644 --- a/crates/sqlbuild-orchestration/tests/api/plans/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/api/plans/helpers.rs @@ -23,6 +23,7 @@ pub(crate) fn plan_request(tokens: &[&str]) -> PlanRequest { PlanRequest { request_id: Uuid::now_v7(), command: tokens.iter().map(|token| (*token).to_owned()).collect(), + fresh: false, } } diff --git a/crates/sqlbuild-orchestration/tests/api/test_person_plans.rs b/crates/sqlbuild-orchestration/tests/api/test_person_plans.rs index 0226a28950..6bdf1bd25c 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_person_plans.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_person_plans.rs @@ -38,12 +38,14 @@ fn given_person_plans_when_requested_claimed_and_read_then_they_stay_location_wo PlanTestCase { description: "SQLite plans", backend: Backend::Sqlite, + fresh: true, expected_selection: &["customers", "orders"], expected_warning: "connect failed: db_password=[REDACTED]", }, PlanTestCase { description: "PostgreSQL plans", backend: Backend::Postgres, + fresh: false, expected_selection: &["customers", "orders"], expected_warning: "connect failed: db_password=[REDACTED]", }, @@ -100,14 +102,17 @@ fn given_person_plans_when_requested_claimed_and_read_then_they_stay_location_wo ) .await; let outsider = person(store, owner, ("orders-outsider", location), &[]).await; - let first = plan_request(&[ - "build", - "customers", - "orders", - "--full-refresh", - "--warehouse", - "WH_PLAN", - ]); + let first = PlanRequest { + fresh: test_case.fresh, + ..plan_request(&[ + "build", + "customers", + "orders", + "--full-refresh", + "--warehouse", + "WH_PLAN", + ]) + }; let silent = ask(store, (&context, viewer), location, &first).await; assert!( @@ -225,6 +230,7 @@ fn given_person_plans_when_requested_claimed_and_read_then_they_stay_location_wo command: vec!["build".into(), "orders".into()], selection: ["orders".to_owned()].into(), settings: Default::default(), + fresh: false, }, }; assert!( @@ -278,6 +284,7 @@ fn given_person_plans_when_requested_claimed_and_read_then_they_stay_location_wo let LocationWork::Plan { selection, settings, + fresh, .. } = &claimed.request.work else { @@ -286,6 +293,12 @@ fn given_person_plans_when_requested_claimed_and_read_then_they_stay_location_wo description = test_case.description ) }; + assert_eq!( + *fresh, + test_case.fresh, + "{description}: the location sees whether to plan again", + description = test_case.description + ); assert_eq!( selection.iter().map(String::as_str).collect::>(), test_case.expected_selection, diff --git a/crates/sqlbuild-orchestration/tests/api/test_types.rs b/crates/sqlbuild-orchestration/tests/api/test_types.rs index 54c8c21af7..45b0fd01cb 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_types.rs @@ -319,6 +319,8 @@ pub(crate) struct OlderNodeFinished { pub(crate) struct PlanTestCase { pub(crate) description: &'static str, pub(crate) backend: Backend, + /// Whether the person asks to plan again rather than reuse a kept plan. + pub(crate) fresh: bool, pub(crate) expected_selection: &'static [&'static str], pub(crate) expected_warning: &'static str, } diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md index 7e900811e0..d527a8c1e2 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md @@ -892,11 +892,17 @@ These stay unknown until it does: - the columns of an expression source whose SQL changed since it was last described, which the plan takes as declared. -Over the API, queue a plan, then read it until `result_json` is set: +A location keeps its last 16 plans for the current publication and synced copy, in memory, and +answers an identical request with the plan it kept instead of planning again. A newer publication +or synced copy drops them all. A kept plan's `timings` say so in `reused_from`, the time it was +made. The page's **Plan again** asks for a fresh plan. + +Over the API, queue a plan, then read it until `result_json` is set. `fresh` is required: `true` +plans again even when the location kept a plan for the same command. ```json POST /api/control/locations/{location_id}/targets/{target}/plans -{ "request_id": "0191f0aa-0000-7000-8000-0000000000a1", "command": ["build", "--select", "+orders"] } +{ "request_id": "0191f0aa-0000-7000-8000-0000000000a1", "command": ["build", "--select", "+orders"], "fresh": false } GET /api/control/locations/{location_id}/targets/{target}/plans/{request_id} ``` diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/plan/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/plan/helpers.py index 1fd65c5fa5..b203e0b3da 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/plan/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/plan/helpers.py @@ -477,11 +477,18 @@ def page_of(kind: str, records: list[dict[str, Any]]) -> dict[str, Any]: def timed_plan( - served: ServedLocation, selection: list[str], settings: dict[str, Any] + served: ServedLocation, + selection: list[str], + settings: dict[str, Any], + *, + fresh: bool = False, ) -> tuple[float, dict[str, Any]]: - """A person's plan from the moment the server queues it until the location stored it.""" + """A person's plan from the moment the server queues it until the location stored it. + + `fresh` is Plan again, so the location plans anew instead of returning a plan it kept. + """ started: float = time.monotonic() - request_id: str = request_plan(served.control, selection, settings) + request_id: str = request_plan(served.control, selection, settings, fresh=fresh) deadline: float = started + 600.0 result: str | None = None while result is None: diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/plan/test_duckshop_10k_preview_performance.py b/tests/e2e/src/sqlbuild/cli/commands/main/plan/test_duckshop_10k_preview_performance.py index 2c621ff0a7..d204e94573 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/plan/test_duckshop_10k_preview_performance.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/plan/test_duckshop_10k_preview_performance.py @@ -108,8 +108,9 @@ def test_given_a_synced_10k_model_warehouse_when_previewing_a_changed_build_then mirror_synced_state(served, batches) selection: list[str] = published_keys(served.control) cold, _ = timed_plan(served, selection, {"changed": True}) - warm, document = timed_plan(served, selection, {"changed": True}) - _ = wait_for_log_count(served, PREVIEW_DONE, 2, 60.0) + warm, document = timed_plan(served, selection, {"changed": True}, fresh=True) + reused_seconds, reused = timed_plan(served, selection, {"changed": True}) + _ = wait_for_log_count(served, PREVIEW_DONE, 3, 60.0) admin: dict[str, Any] = provision_person( served.control.parents[1], "create-user", "plan-admin" ) @@ -139,7 +140,8 @@ def test_given_a_synced_10k_model_warehouse_when_previewing_a_changed_build_then print( f"duckshop 10k preview ({test_case.description}): cold={cold:.2f}s warm={warm:.2f}s " f"publish={served.published_seconds:.2f}s warm_to_publish={warm_to_publish:.2f} " - f"check_median={check_seconds * 1000:.0f}ms runs={len(reasons)} unbuilt={unbuilt} " + f"reused={reused_seconds:.2f}s check_median={check_seconds * 1000:.0f}ms " + f"runs={len(reasons)} unbuilt={unbuilt} " f"definition_changes={definition_changes} models={len(document['models'])} " f"live={live.seconds:.2f}s live_to_publish={live.seconds / served.published_seconds:.2f} " f"live_entries={live.model_entries} rebinding={live.source_rebinding_analyses} " @@ -161,6 +163,10 @@ def test_given_a_synced_10k_model_warehouse_when_previewing_a_changed_build_then assert warm < test_case.expected_max_warm_seconds, ( "a warm preview of 10K models must stay under ten seconds" ) + assert reused["timings"]["reused_from"] and reused["models"] == document["models"], ( + "the same request again returns the plan the location kept" + ) + assert reused_seconds < warm, "a kept plan comes back without planning again" assert warm_to_publish < test_case.expected_max_warm_to_publish, ( "planning grew faster than compiling: look for a lookup over every model or relation" ) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py index 57a4eabdeb..fe3b1a3fbd 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py @@ -532,9 +532,11 @@ class PlanPreviewTestCase: class HeldPlanTestCase: description: str settings: tuple[dict[str, object], ...] + fresh: frozenset[int] expected_runs: tuple[tuple[str, ...], ...] expected_compiles: int expected_held_plans: int + expected_reused: tuple[bool, ...] @dataclass(frozen=True) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/catalog/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/catalog/helpers.py index 0aa5817bea..96ea4148a0 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/catalog/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/catalog/helpers.py @@ -90,9 +90,16 @@ def wait_for_statistics( def request_plan( - control: Path, selection: list[str], settings: dict[str, object] | None = None + control: Path, + selection: list[str], + settings: dict[str, object] | None = None, + *, + fresh: bool = False, ) -> str: - """Queue the plan work a person's Plan creates, frozen to exact node keys and settings.""" + """Queue the plan work a person's Plan creates, frozen to exact node keys and settings. + + `fresh` is Plan again: the location plans anew rather than return a plan it kept. + """ request_id: str = str(uuid4()) now: datetime = datetime.now(UTC) with closing(sqlite3.connect(control)) as database: @@ -101,7 +108,7 @@ def request_plan( "request_json,requested_by,requested_at,expires_at) SELECT ?,c.location_id,c.target," "c.snapshot_id,s.generation,json_object('request_id',?,'location_id',c.location_id," "'target',c.target,'reason','Plan','work',json_object('kind','plan','command'," - "json('[\"build\"]'),'selection',json(?),'settings',json(?)))," + "json('[\"build\"]'),'selection',json(?),'settings',json(?),'fresh',json(?)))," "'test-admin',?,? FROM sqb_current_snapshots c JOIN sqb_snapshots s " "ON s.snapshot_id=c.snapshot_id", ( @@ -109,6 +116,7 @@ def request_plan( request_id, json.dumps(selection), json.dumps(settings or {}), + json.dumps(fresh), now.strftime("%Y-%m-%dT%H:%M:%S.%fZ"), (now + timedelta(minutes=5)).strftime("%Y-%m-%dT%H:%M:%S.%fZ"), ), diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py index fdc1552912..2b646a73be 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py @@ -1886,9 +1886,15 @@ def changed_build_and_plan_from_copy( def plans_from_copy( - tmp_path: Path, project: Path, requests: list[tuple[list[str], dict[str, object]]] + tmp_path: Path, + project: Path, + requests: list[tuple[list[str], dict[str, object]]], + fresh: frozenset[int] = frozenset(), ) -> list[dict[str, Any]]: - """Serve `project` until its first sync completes, then plan each request from the copy.""" + """Serve `project` until its first sync completes, then plan each request from the copy. + + The requests at the `fresh` positions are Plan again, which plans anew instead of reusing. + """ control: Path = tmp_path / "control/.sqlbuild/control.sqlite3" log: Path = tmp_path / "location.log" started: datetime = datetime.now(UTC) @@ -1922,8 +1928,8 @@ def plans_from_copy( log, after=started, ) - for selection, settings in requests: - request_id: str = request_plan(control, selection, settings) + for position, (selection, settings) in enumerate(requests): + request_id: str = request_plan(control, selection, settings, fresh=position in fresh) result: dict[str, Any] = wait_for_location_work(api, request_id, location, log) outcome: dict[str, Any] = result["result_json"]["outcome"] assert outcome["kind"] == "plan", outcome diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_plan_preview.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_plan_preview.py index 793533dc8a..f1fd970c08 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_plan_preview.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_plan_preview.py @@ -455,16 +455,20 @@ def changed_build() -> dict[str, list[str]]: HeldPlanTestCase( description=( "only the plan that sets variables compiles the project again; the other two " - "and the changed-build count plan from the published project" + "and the changed-build count plan from the published project, and the same " + "request again returns the kept plan unless it asks to plan again" ), settings=( {"changed": True}, {"changed": True, "vars": {"batch": "2"}}, {"changed": True}, + {"changed": True}, ), - expected_runs=((), ("model:order_count",), ()), + fresh=frozenset({2}), + expected_runs=((), ("model:order_count",), (), ()), expected_compiles=1, expected_held_plans=3, + expected_reused=(False, False, False, True), ), ], ids=lambda case: case.description, @@ -486,10 +490,11 @@ def test_given_plan_requests_when_the_location_plans_them_then_only_variables_co (["model:order_count"], settings) for settings in test_case.settings ] - plans: list[dict[str, Any]] = plans_from_copy(tmp_path, project, requests) + plans: list[dict[str, Any]] = plans_from_copy(tmp_path, project, requests, test_case.fresh) log: str = (tmp_path / "location.log").read_text() assert tuple(map(changed_runs, plans)) == test_case.expected_runs, log + assert tuple("reused_from" in plan["timings"] for plan in plans) == test_case.expected_reused assert log.count("Compiling project...") == test_case.expected_compiles, log assert log.count(_HELD) == test_case.expected_held_plans, log assert log.count(_SYNCED_READING) == test_case.expected_held_plans + 1, log diff --git a/ui/src/lib/api/client.ts b/ui/src/lib/api/client.ts index cbd2bf022b..832972e4f8 100644 --- a/ui/src/lib/api/client.ts +++ b/ui/src/lib/api/client.ts @@ -149,8 +149,11 @@ export interface ApiClient extends PlatformApi { /** The server's pre-launch check: resolution, write set and authorization. */ checkAdhocRun(scope: Scope, request: AdhocRunRequest): Promise; launchAdhocRun(scope: Scope, request: AdhocRunRequest): Promise; - /** Asks the location to run `sqb plan` on a build command against the warehouse, and waits for it. */ - planBuild(scope: Scope, command: string[]): Promise; + /** + * Asks the location to run `sqb plan` on a build command against the warehouse, and waits for it. + * The location may answer with the plan it kept for the same command; `fresh` makes it plan again. + */ + planBuild(scope: Scope, command: string[], options?: { fresh?: boolean }): Promise; /** One page of a large section of the plan `requestId` names, which its summary only counts. */ /** One page of a plan section; `name` finds rows whose name contains it, ignoring case. */ planRows(scope: Scope, requestId: string, section: PlanPageSection, offset: number, limit?: number, name?: string): Promise; diff --git a/ui/src/lib/api/http-audit.test.ts b/ui/src/lib/api/http-audit.test.ts index 9c2b905039..7a507556d8 100644 --- a/ui/src/lib/api/http-audit.test.ts +++ b/ui/src/lib/api/http-audit.test.ts @@ -60,7 +60,7 @@ describe('toAuditEntry', () => { const allowed = (action: string, after_json: Record) => toAuditEntry({ ...base, actor: 'principal:p-1', actor_name: 'ana', action, granting_role: 'admin', scope_json: { location_id: 'loc-1', target: 'prod' }, after_json }).summary; const work = (work: Record) => allowed('location.work_requested', { request_id: 'q-1', reason: 'plan', work }); - expect(work({ kind: 'plan', command: ['build', '--select', 'orders'], selection: ['model:orders'], settings: {} })).toBe('Plan sqb build --select orders'); + expect(work({ kind: 'plan', command: ['build', '--select', 'orders'], selection: ['model:orders'], settings: {}, fresh: false })).toBe('Plan sqb build --select orders'); expect(work({ kind: 'sync_warehouse' })).toBe('Sync the warehouse copy'); expect(allowed('error.read', { run_id: 'd36ac8b2-314c-46aa', found: true })).toBe('Run d36ac8b2'); expect(allowed('output.read', { run_id: 'd36ac8b2-314c-46aa', found: false })).toBe('Run d36ac8b2, not found'); diff --git a/ui/src/lib/api/http-client.test.ts b/ui/src/lib/api/http-client.test.ts index c7ed8084dd..68970a6f9d 100644 --- a/ui/src/lib/api/http-client.test.ts +++ b/ui/src/lib/api/http-client.test.ts @@ -952,3 +952,31 @@ describe('listWarehouseConnections', () => { expect(health.filter((p) => p.includes('/development/'))).toHaveLength(2); }); }); + +describe('planning a build', () => { + function server() { + const bodies: Record[] = []; + vi.stubGlobal('document', { cookie: '' }); + vi.stubGlobal( + 'fetch', + vi.fn(async (request: Request) => { + if (request.method === 'POST') { + bodies.push(await request.json()); + return Response.json({ applied: true }); + } + return Response.json({ expires_at: '2999-01-01T00:00:00Z', outcome: { kind: 'plan', document: { error: { code: 'plan_failed', message: 'stop here' } } } }); + }) + ); + return { bodies, client: createHttpClient({ baseUrl: 'http://ui.test', onUnauthorized: () => {} }) }; + } + + it('asks the location to plan again only when the person asks for a fresh plan', async () => { + const { bodies, client } = server(); + for (const options of [undefined, { fresh: true }]) { + const planned = client.planBuild(ORDERS, ['build', '--changed'], options); + await vi.advanceTimersByTimeAsync(1000); + expect((await planned).error).toBe('plan_failed: stop here'); + } + expect(bodies.map((body) => body.fresh)).toEqual([false, true]); + }); +}); diff --git a/ui/src/lib/api/http-client.ts b/ui/src/lib/api/http-client.ts index 63b8b3ba3c..251502a20c 100644 --- a/ui/src/lib/api/http-client.ts +++ b/ui/src/lib/api/http-client.ts @@ -599,12 +599,12 @@ export function createHttpClient({ baseUrl = '', onUnauthorized, fetch: answer, * Queues the plan, then waits for the location's summary: changed models in full and the large sections counted. * The first page of each large section loads with it; a plan the location could not make is a result, not a throw. */ - async planBuild(scope: Scope, command: string[]): Promise { + async planBuild(scope: Scope, command: string[], options: { fresh?: boolean } = {}): Promise { const { locationId, target } = parseLocationKey(scope.locationId); const path = { location_id: locationId, target }; const requestId = newRequestId(); const asked = await http.POST('/api/control/locations/{location_id}/targets/{target}/plans', { - body: { request_id: requestId, command }, + body: { request_id: requestId, command, fresh: options.fresh ?? false }, params: { path, header: { 'x-sqb-csrf': csrfToken(document.cookie) } } }); if (asked.response.status === 401) onUnauthorized(); diff --git a/ui/src/lib/api/mock/mock-client.ts b/ui/src/lib/api/mock/mock-client.ts index 0b1881a4ec..4a82d99801 100644 --- a/ui/src/lib/api/mock/mock-client.ts +++ b/ui/src/lib/api/mock/mock-client.ts @@ -371,7 +371,7 @@ export class MockApiClient implements ApiClient { return this.#operate.launchAdhocRun(scope, request); } - async planBuild(scope: Scope, command: string[]): Promise { + async planBuild(scope: Scope, command: string[], _options: { fresh?: boolean } = {}): Promise { this.#location(scope); // `--changed` starts from the nodes whose own definition changed, as the catalog marks them. const changed = command.some((t) => t === '--changed' || t.startsWith('--changed=')) diff --git a/ui/src/routes/plan/+page.svelte b/ui/src/routes/plan/+page.svelte index 93ca16f293..58d48d9636 100644 --- a/ui/src/routes/plan/+page.svelte +++ b/ui/src/routes/plan/+page.svelte @@ -133,6 +133,8 @@ let plannedKey = $state(''); let planning = $state(false); let planError = $state<{ key: string; message: string } | null>(null); + /** Set by "Plan again": the next plan skips the plan the location kept for the same command. */ + let planFresh = false; const wantsPlan = $derived(fresh && check?.verb === 'build' && !check.error); $effect(() => { const key = requestKey; @@ -141,7 +143,9 @@ const timer = setTimeout(async () => { planning = true; try { - const result = await app.client.planBuild(scope, request.command); + const replanned = planFresh; + planFresh = false; + const result = await app.client.planBuild(scope, request.command, { fresh: replanned }); if (result.plan) { plan = result.plan; planRequestId = result.requestId ?? null; @@ -179,6 +183,7 @@ return app.client.planDiff(scope, requestId, model); } function replan() { + planFresh = true; plannedKey = ''; planError = null; } diff --git a/website/src/content/docs/docs/cli/server.mdx b/website/src/content/docs/docs/cli/server.mdx index 2cc6e0df5c..1be6f9b93f 100644 --- a/website/src/content/docs/docs/cli/server.mdx +++ b/website/src/content/docs/docs/cli/server.mdx @@ -871,11 +871,17 @@ These stay unknown until it does: - the columns of an expression source whose SQL changed since it was last described, which the plan takes as declared. -Over the API, queue a plan, then read it until `result_json` is set: +A location keeps its last 16 plans for the current publication and synced copy, in memory, and +answers an identical request with the plan it kept instead of planning again. A newer publication +or synced copy drops them all. A kept plan's `timings` say so in `reused_from`, the time it was +made. The page's **Plan again** asks for a fresh plan. + +Over the API, queue a plan, then read it until `result_json` is set. `fresh` is required: `true` +plans again even when the location kept a plan for the same command. ```json POST /api/control/locations/{location_id}/targets/{target}/plans -{ "request_id": "0191f0aa-0000-7000-8000-0000000000a1", "command": ["build", "--select", "+orders"] } +{ "request_id": "0191f0aa-0000-7000-8000-0000000000a1", "command": ["build", "--select", "+orders"], "fresh": false } GET /api/control/locations/{location_id}/targets/{target}/plans/{request_id} ``` From 2f1a733b83d03cb32e0478f20f56065407c13b47 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 19 Nov 2026 00:45:26 +0000 Subject: [PATCH 08/15] perf(location): plan a held project's SQL tests once per generation Planning SQL tests reads only the compiled project and the dialect, yet each preview planned them again. A location's previews now plan every test once for the published generation, on the first preview, keep the results on the held project and take each preview's selection from them. Live plans, and a held project where planning every test fails, plan their selection as before. --- .../_helpers/plans/plan_output.rs | 104 +++++++++++++++--- .../_helpers/plans/test_planning.rs | 31 +++++- .../main/plans/build_execution_plan.rs | 13 +-- .../src/planner_warehouse/models.rs | 4 + .../src/planner_warehouse/types.rs | 5 + .../src/compiled_project/models.rs | 9 ++ 6 files changed, 139 insertions(+), 27 deletions(-) diff --git a/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/plan_output.rs b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/plan_output.rs index fb7e92c43d..112b761f16 100644 --- a/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/plan_output.rs +++ b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/plan_output.rs @@ -8,8 +8,10 @@ use crate::planner_entries::models::PlanOutputInputs; use crate::planner_resolution::main::references::resolve_function_sql::resolve_function_sql; use crate::planner_strategy::main::audit_entries::build_selected_audit_entries::build_selected_audit_entries; use crate::planner_strategy::main::model_entries::extract_seed_columns::extract_seed_columns; +use pyo3::prelude::Py; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::pyfunction; +use pyo3::types::PyTupleMethods; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyList, PyTuple}; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::boundary::models::KeywordArguments; @@ -19,7 +21,9 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_project::compiled_project::models::NativeCompiledProject; use sqlbuild_python_project::compiled_views::main::project::sql_tests::sql_tests; +use sqlbuild_python_project::compiled_views::models::NativeProjectView; use sqlbuild_python_project::planner_phases::main::object::object; use sqlbuild_python_project::planner_scope::main::build_source_load_entries::build_source_load_entries; use sqlbuild_python_project::planner_scope::main::upstream_loader_dependency_names::upstream_loader_dependency_names; @@ -38,6 +42,7 @@ use sqlbuild_python_project::resource_views::main::seed_reads::schema_entry::sch use sqlbuild_python_project::resource_views::main::seed_reads::seed_file::seed_file; use sqlbuild_python_project::sql_test_glue::main::assemble_sql_test_plans::assemble_sql_test_plans; use sqlbuild_python_project::sql_test_glue::models::SqlTestAssembly; +use std::sync::{Arc, OnceLock}; const LOADER_CONSUMER: &str = "loader"; const HOOK_CONSUMER: &str = "hook"; @@ -506,29 +511,48 @@ pub(crate) fn selected_test_entries<'py>( } selected.push(test); } - let mut entries: Vec> = Vec::new(); - let mut warnings: Vec> = Vec::new(); if selected.is_empty() { - return Ok(SelectedTests { entries, warnings }); + return Ok(SelectedTests { + entries: Vec::new(), + warnings: Vec::new(), + }); } + let results = test_plan_results((project, dialect), &selected)?; + selected_from_results(py, results.try_iter()?.collect::>>()?) +} + +/// One plan result per test in `tests`, in order, with fixture diagnostics in place of failed plans. +fn test_plan_results<'py>( + (project, dialect): (&Bound<'py, PyAny>, &Bound<'py, Dialect>), + tests: &[Bound<'py, PyAny>], +) -> PyResult> { + let py = project.py(); let analysis = native_field(&native_field(project, "settings")?, "sql_analysis")?; let fixture_context = if analysis.is_truthy()? { Bound::new(py, planning_context(project, None)?)?.into_any() } else { none(py) }; - let results = assemble_sql_test_plans( - &selected, + assemble_sql_test_plans( + tests, (project, dialect), &SqlTestAssembly { sql_analysis_enabled: analysis, validate_fixtures: true, fixture_planning_context: fixture_context, }, - )?; + ) +} + +/// The entries and warnings of planned results, failing on invalid fixtures. +fn selected_from_results<'py>( + py: Python<'py>, + results: Vec>, +) -> PyResult> { + let mut entries: Vec> = Vec::new(); + let mut warnings: Vec> = Vec::new(); let mut diagnostics: Vec = Vec::new(); - for result in results.try_iter()? { - let result = result?; + for result in results { let entry = native_field(&result, "entry")?; if entry.is_none() { for diagnostic in native_field(&result, "fixture_diagnostics")?.try_iter()? { @@ -552,15 +576,63 @@ pub(crate) fn selected_test_entries<'py>( Ok(SelectedTests { entries, warnings }) } -/// SQL tests selected by name or overlapping one selection, planned into a reusable result. +/// The selection's entries from every test a held project planned once, or `None` to plan it. +fn held_selected_test_entries<'py>( + (project, dialect): (&Bound<'py, PyAny>, &Bound<'py, Dialect>), + selected_keys: &Bound<'py, PyAny>, + selection: &Bound<'py, PyAny>, +) -> PyResult>> { + let py = project.py(); + let held: Py = match project.cast::() { + Ok(view) => view.get().native_project(py), + Err(_) => match project.cast::() { + Ok(held) => held.clone().unbind(), + Err(_) => return Ok(None), + }, + }; + let tests: Vec> = sql_tests(project)?.try_iter()?.collect::>()?; + let results: Arc>>> = + held.bind(py).borrow().held_sql_test_results(); + if results.get().is_none() { + let planned: Option> = match test_plan_results((project, dialect), &tests) { + Ok(planned) => Some(planned.unbind()), + Err(_) => None, + }; + drop(results.set(planned)); + } + let Some(Some(planned)) = results.get() else { + return Ok(None); + }; + let planned: &Bound<'py, PyTuple> = planned.bind(py); + if planned.len() != tests.len() { + return Ok(None); + } + let mut selected: Vec> = Vec::new(); + for (position, test) in tests.iter().enumerate() { + if test_selected(test, selected_keys, selection)? { + selected.push(planned.get_item(position)?); + } + } + selected_from_results(py, selected).map(Some) +} + +/// SQL tests selected by name or overlapping one selection; `held` reads a held project's plans. fn planned_sql_tests<'py>( project: &Bound<'py, PyAny>, dialect: &Bound<'py, Dialect>, - selected_keys: &Bound<'py, PyAny>, - selection: &Bound<'py, PyAny>, + (selected_keys, selection): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + held: bool, ) -> PyResult> { let py = project.py(); - let selected = selected_test_entries((project, dialect), selected_keys, selection, &none(py))?; + let reused: Option> = if held { + held_selected_test_entries((project, dialect), selected_keys, selection)? + } else { + None + }; + let selected = match reused { + Some(selected) => selected, + None => selected_test_entries((project, dialect), selected_keys, selection, &none(py))?, + }; native( py, RecordId::PlannedSqlTests, @@ -660,7 +732,7 @@ pub(crate) fn plan_output_fields<'py>( || native_field(&planned, "selected_keys")?.ne(&selected_keys)? || native_field(&planned, "sql_test_selection")?.ne(&selection)? { - planned = planned_sql_tests(&project, &dialect, &selected_keys, &selection)?; + planned = planned_sql_tests(&project, &dialect, (&selected_keys, &selection), false)?; } let test_entries = native_field(&planned, "entries")?; let mut test_keys: Vec> = Vec::new(); @@ -763,10 +835,10 @@ pub(crate) fn plan_output_fields<'py>( pub(crate) fn plan_selected_sql_tests<'py>( project: &Bound<'py, PyAny>, dialect: &Bound<'py, Dialect>, - selected_keys: &Bound<'py, PyAny>, - sql_test_selection: &Bound<'py, PyAny>, + (selected_keys, sql_test_selection): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + held: bool, ) -> PyResult> { - planned_sql_tests(project, dialect, selected_keys, sql_test_selection) + planned_sql_tests(project, dialect, (selected_keys, sql_test_selection), held) } /// Selected SQL test entries and warnings, optionally narrowed to one parameterized case. diff --git a/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/test_planning.rs b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/test_planning.rs index bbaab1bd4e..65798ff518 100644 --- a/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/test_planning.rs +++ b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/test_planning.rs @@ -13,10 +13,12 @@ use pyo3::types::PyDict; use pyo3::{pyclass, pyfunction, pymethods}; use crate::planner_entries::_helpers::plans::plan_output::plan_selected_sql_tests; +use crate::planner_warehouse::models::PlannerReads; use sqlbuild_python_base::boundary::main::compiler_guard::compiler_guard; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::native_threads::main::signal_interpreter_exit::signal_interpreter_exit; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; +use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; const TEST_PLANNING_THREAD: &str = "sqlbuild-test-planning"; @@ -155,6 +157,8 @@ pub(crate) struct PlanningRequest { pub(crate) dialect: Py, pub(crate) selected_keys: Py, pub(crate) sql_test_selection: Py, + /// Plan from the tests a location's held project planned once for its generation. + pub(crate) held: bool, } /// Plan the request's selected SQL tests, reporting a panic as a native compiler error. @@ -163,8 +167,11 @@ fn plan(py: Python<'_>, request: &PlanningRequest) -> Planned { plan_selected_sql_tests( request.project.bind(py), request.dialect.bind(py), - request.selected_keys.bind(py), - request.sql_test_selection.bind(py), + ( + request.selected_keys.bind(py), + request.sql_test_selection.bind(py), + ), + request.held, ) .map(Bound::unbind) }) @@ -211,6 +218,25 @@ pub(crate) fn start_planning( Py::new(py, SqlTestPlanning::of(Some(receiver), None)) } +/// Start planning the scope's SQL tests; a location's previews read its held project's plans. +pub(crate) fn start_scope_planning<'py>( + (project, dialect): (&Bound<'py, PyAny>, Bound<'py, Dialect>), + (scope, reads): (&Bound<'py, PyAny>, &Bound<'py, PlannerReads>), + enabled: bool, +) -> PyResult> { + start_planning( + project.py(), + PlanningRequest { + project: project.clone().unbind(), + dialect: dialect.unbind(), + selected_keys: native_field(scope, "selected_keys")?.unbind(), + sql_test_selection: native_field(scope, "sql_test_selection")?.unbind(), + held: reads.get().reads().preview(), + }, + enabled, + ) +} + /// Join SQL test planning threads at exit: finalization ending one inside native planning aborts. #[pyfunction] pub(crate) fn join_sql_test_planning_threads(py: Python<'_>) -> PyResult<()> { @@ -249,6 +275,7 @@ pub(crate) fn start_sql_test_planning<'py>( dialect: Dialect::passed(args.value("dialect")?)?.unbind(), selected_keys: args.value("selected_keys")?.unbind(), sql_test_selection: args.value("sql_test_selection")?.unbind(), + held: false, }, args.flag("enabled")?, ) diff --git a/crates/sqlbuild-python-planner/src/planner_entries/main/plans/build_execution_plan.rs b/crates/sqlbuild-python-planner/src/planner_entries/main/plans/build_execution_plan.rs index ff8ea412c4..9cf617e05b 100644 --- a/crates/sqlbuild-python-planner/src/planner_entries/main/plans/build_execution_plan.rs +++ b/crates/sqlbuild-python-planner/src/planner_entries/main/plans/build_execution_plan.rs @@ -2,7 +2,7 @@ use crate::planner_entries::_helpers::plans::execution_plan::planned_output; use crate::planner_entries::_helpers::plans::execution_plan::scopes_and_cursor_overrides; -use crate::planner_entries::_helpers::plans::test_planning::{PlanningRequest, start_planning}; +use crate::planner_entries::_helpers::plans::test_planning::start_scope_planning; use crate::planner_phases::main::inputs::class_field::class_field; use crate::planner_phases::models::PlannerRuntime; use crate::planner_warehouse::models::PlannerReads; @@ -63,14 +63,9 @@ pub fn build_execution_plan<'py>( .into_any(); let (scopes, overrides) = scopes_and_cursor_overrides(&args)?; let inspection = class_field(&scopes, |s: &PlannerScopeResolution| &s.inspection_scope)?; - let test_planning = start_planning( - py, - PlanningRequest { - project: project.clone().unbind(), - dialect: dialect.unbind(), - selected_keys: native_field(&inspection, "selected_keys")?.unbind(), - sql_test_selection: native_field(&inspection, "sql_test_selection")?.unbind(), - }, + let test_planning = start_scope_planning( + (&project, dialect), + (&inspection, &reads), native_field(&args.value("policies")?, "plan_sql_tests")?.is_truthy()?, )?; let planned = reads.get().reads().within_inspection(py, &mut || { diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/models.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/models.rs index bf6b4d06d9..50c131f53c 100644 --- a/crates/sqlbuild-python-planner/src/planner_warehouse/models.rs +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/models.rs @@ -568,6 +568,10 @@ impl SyncedReads { } impl WarehouseReads for SyncedReads { + fn preview(&self) -> bool { + true + } + fn relations<'py>( &self, py: Python<'py>, diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/types.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/types.rs index cbf790687b..6e8a17dfde 100644 --- a/crates/sqlbuild-python-planner/src/planner_warehouse/types.rs +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/types.rs @@ -32,6 +32,11 @@ pub trait CursorProgress<'py> { /// Everything a plan reads from the warehouse, live or from a synced state that may answer unknown. pub trait WarehouseReads: Send + Sync + std::fmt::Debug { + /// Whether these reads answer a location's preview from its synced state. + fn preview(&self) -> bool { + false + } + /// The listed relations of `database` narrowed to `schemas` and `names`, as `RelationInfo`s. fn relations<'py>( &self, diff --git a/crates/sqlbuild-python-project/src/compiled_project/models.rs b/crates/sqlbuild-python-project/src/compiled_project/models.rs index 73eb20c543..4d18108958 100644 --- a/crates/sqlbuild-python-project/src/compiled_project/models.rs +++ b/crates/sqlbuild-python-project/src/compiled_project/models.rs @@ -58,6 +58,7 @@ use pyo3::pymethods; use pyo3::types::PyDict; use pyo3::types::PyDictMethods; use pyo3::types::PyString; +use pyo3::types::PyTuple; use sqlbuild_analysis::compiled_project::main::project_models::project_models; use sqlbuild_analysis::compiled_project::main::record_model_analysis::record_model_analysis; use sqlbuild_analysis::compiled_project::main::record_skipped_analysis::record_skipped_analysis; @@ -75,6 +76,7 @@ use sqlbuild_python_base::cache::models::SqlTestScanStore; use sqlbuild_python_front::render::models::ConstantRendering; use std::collections::HashMap; use std::sync::Mutex; +use std::sync::{Arc, OnceLock}; /// A function's paths, stem, discovered file, header values, body and SQL file contents. #[derive(Debug)] @@ -579,9 +581,16 @@ pub struct NativeCompiledProject { pub(crate) inferred_column_projections: Mutex>>, /// The model loop and its constant rendering, which attach the project's other resources. pub(crate) attachment: Option<(Py, Py)>, + /// Every SQL test's plan result for a location's previews, set once; `None` when planning failed. + pub(crate) held_sql_test_results: Arc>>>, } impl NativeCompiledProject { + /// Every SQL test's plan result for a location's previews, planned once per published project. + pub fn held_sql_test_results(&self) -> Arc>>> { + Arc::clone(&self.held_sql_test_results) + } + /// The facts recorded so far, for native consumers such as the rules request. pub fn facts(&self) -> &CompiledProjectFacts { &self.project.facts From f3e918cc6c7fc18c79fd61176282a27664d39c9d Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 19 Nov 2026 01:11:53 +0000 Subject: [PATCH 09/15] chore(fensu): set the Python boundary budgets for faster plan previews Lower the crossing budgets of the migration planners that now call into Python less. Budget the old-name index's one identity read, and the held project's planned SQL tests, one more Python-typed field. --- fensu.toml | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/fensu.toml b/fensu.toml index 73dcbfa978..45cd1f8b9e 100644 --- a/fensu.toml +++ b/fensu.toml @@ -2569,27 +2569,28 @@ pending_budgets = [ "crates/sqlbuild-python-planner/src/planner_identity/main/hook_version_hashes.rs=1", "crates/sqlbuild-python-planner/src/planner_identity/main/merge_recomputed_model_changes.rs=1", "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_candidates.rs=3", - "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_entries.rs=3", + "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_entries.rs=2", "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_objects.rs=1", "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_overlay.rs=8", "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_planning.rs=1", "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_warnings.rs=1", "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/columns.rs=1", - "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/discovery/discovery_events.rs=6", + "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/discovery/discovery_events.rs=4", "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/discovery/discovery_models.rs=1", "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/discovery/discovery_requests.rs=2", "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/discovery/model_discovery.rs=1", "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/discovery/state_inspection.rs=2", - "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/model_entries.rs=9", + "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/model_entries.rs=8", "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/model_handover.rs=5", "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/model_overlay.rs=4", "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/model_planning.rs=1", "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/model_warnings.rs=2", - "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_name_entries.rs=9", - "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_name_objects.rs=4", + "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/identity_index.rs=1", + "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_name_entries.rs=8", + "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_name_objects.rs=3", "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_name_planning.rs=2", "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_name_references.rs=7", - "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_names.rs=7", + "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_names.rs=5", "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/planning/planning_execution.rs=2", "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/planning/planning_origin.rs=6", "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/projections/fingerprint.rs=1", @@ -3017,7 +3018,7 @@ pending_budgets = [ "crates/sqlbuild-python-planner/src/planner_warehouse/models.rs=8", "crates/sqlbuild-python-project/src/analysis/models.rs=9", "crates/sqlbuild-python-project/src/analysis_session/models.rs=1", - "crates/sqlbuild-python-project/src/compiled_project/models.rs=36", + "crates/sqlbuild-python-project/src/compiled_project/models.rs=37", "crates/sqlbuild-python-project/src/compiled_views/_helpers/compiled_models/compiled_model_view.rs=3", "crates/sqlbuild-python-project/src/compiled_views/_helpers/model_inputs/inputs_view.rs=24", "crates/sqlbuild-python-project/src/compiled_views/_helpers/project/project_view.rs=26", From 9f98cbd1ae02931cacbe3615a6d65fa0d67d43ba Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 19 Nov 2026 19:39:54 +0000 Subject: [PATCH 10/15] fix(plan): show a reused plan's age and refuse plan timings of another shape The server's plan timings rejected the reused_from field a location adds to a kept plan, and the summary read then dropped the timings as being of an unknown shape, so the plan page never said the plan was reused. Plan timings now carry reused_from, and the page labels a kept plan by its age instead of a planning time. Timings that do not match the shape fail the read instead of vanishing. A plan request ID retried with a different fresh is refused like any other change. --- clients/typescript/src/schema.d.ts | 5 + .../person_reads/read_person_plan_summary.rs | 11 +-- .../src/platform/models.rs | 5 +- .../src/runs/_helpers/requests/plan.rs | 3 +- .../tests/api/test_person_plans.rs | 94 ++++++++++++++++++- .../tests/api/test_types.rs | 9 ++ ui/src/lib/api/http-client.test.ts | 20 ++++ ui/src/lib/api/types/operate.ts | 2 + ui/src/lib/run/plan-timings.svelte | 8 +- ui/src/lib/run/plan-timings.test.ts | 15 ++- ui/src/lib/run/plan-timings.ts | 13 ++- ui/src/routes/plan/+page.svelte | 2 +- 12 files changed, 164 insertions(+), 23 deletions(-) diff --git a/clients/typescript/src/schema.d.ts b/clients/typescript/src/schema.d.ts index ae7d871adf..238a3410c8 100644 --- a/clients/typescript/src/schema.d.ts +++ b/clients/typescript/src/schema.d.ts @@ -2850,6 +2850,11 @@ export interface components { PlanTimings: { /** @description `synced_state` for a preview over the synced warehouse state; `live` when it read the warehouse. */ planned_from: string; + /** + * Format: date-time + * @description When the plan the location kept, and answered with again, was made; `None` for a new plan. + */ + reused_from?: string | null; steps: components["schemas"]["PlanTimingStep"][]; /** Format: double */ total_seconds: number; diff --git a/crates/sqlbuild-orchestration/src/platform/main/person_reads/read_person_plan_summary.rs b/crates/sqlbuild-orchestration/src/platform/main/person_reads/read_person_plan_summary.rs index ca6da1caab..805fb8c7d2 100644 --- a/crates/sqlbuild-orchestration/src/platform/main/person_reads/read_person_plan_summary.rs +++ b/crates/sqlbuild-orchestration/src/platform/main/person_reads/read_person_plan_summary.rs @@ -1,7 +1,6 @@ //! A plan request's state with its summary, which the plan page polls instead of the whole document. use serde_json::Value; -use serde_json::error::Category; use sqlbuild_core::plan_display::main::plan_summary::plan_summary; use uuid::Uuid; @@ -43,14 +42,12 @@ pub async fn read_person_plan_summary( })) } -/// The document's planning timings; absent from older locations, left out if of an unknown shape. +/// The document's planning timings; a failed plan's report has none. fn plan_timings(document: &Value) -> Result, ControlError> { let Some(timings) = document.get(PLAN_TIMINGS_KEY) else { return Ok(None); }; - match serde_json::from_value(timings.clone()) { - Ok(timings) => Ok(Some(timings)), - Err(error) if error.classify() == Category::Data => Ok(None), - Err(error) => Err(ControlError::Invalid(format!("plan timings: {error}"))), - } + serde_json::from_value(timings.clone()) + .map(Some) + .map_err(|error| ControlError::Invalid(format!("plan timings: {error}"))) } diff --git a/crates/sqlbuild-orchestration/src/platform/models.rs b/crates/sqlbuild-orchestration/src/platform/models.rs index b79c73a059..68afeb0e15 100644 --- a/crates/sqlbuild-orchestration/src/platform/models.rs +++ b/crates/sqlbuild-orchestration/src/platform/models.rs @@ -746,7 +746,7 @@ pub struct PlanSummaryRead { pub expires_at: DateTime, /// `None` until the location answers; a plan's document is its summary. pub outcome: Option, - /// How long the location took to plan, step by step; `None` for plans recorded without timings. + /// How long the location took to plan, step by step; `None` until it planned, or when planning failed. #[serde(default, skip_serializing_if = "Option::is_none")] pub timings: Option, } @@ -759,6 +759,9 @@ pub struct PlanTimings { pub steps: Vec, /// `synced_state` for a preview over the synced warehouse state; `live` when it read the warehouse. pub planned_from: String, + /// When the plan the location kept, and answered with again, was made; `None` for a new plan. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub reused_from: Option>, } /// One planning step and its wall time. diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/plan.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/plan.rs index 1b5dbdbd69..bafcaff7f1 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/plan.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/plan.rs @@ -37,7 +37,8 @@ pub(in crate::runs) async fn request_plan_in( && previous.record.request_json.location_id == location_id && previous.record.request_json.target == target && matches!(&previous.record.request_json.work, - LocationWork::Plan { command, .. } if *command == request.command); + LocationWork::Plan { command, fresh, .. } + if *command == request.command && *fresh == request.fresh); if !same { return Err(ControlError::Invalid( "location work ID already has different content or actor".into(), diff --git a/crates/sqlbuild-orchestration/tests/api/test_person_plans.rs b/crates/sqlbuild-orchestration/tests/api/test_person_plans.rs index 6bdf1bd25c..d539033fb4 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_person_plans.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_person_plans.rs @@ -7,7 +7,7 @@ use sqlbuild_core::plan_display::main::detach_model_diffs::detach_model_diffs; use uuid::Uuid; use crate::api::tests::plans_helpers::{ask, person, plan_request, register_instance, run_rows}; -use crate::api::tests::test_types::{PlanDiffTestCase, PlanTestCase}; +use crate::api::tests::test_types::{PlanDiffTestCase, PlanTestCase, PlanTimingsReadTestCase}; use crate::api::tests::work_helpers::execute; use crate::authorization::main::people::bootstrap_user::bootstrap_user; use crate::credentials::models::{AuditContext, BuiltinRole}; @@ -39,6 +39,7 @@ fn given_person_plans_when_requested_claimed_and_read_then_they_stay_location_wo description: "SQLite plans", backend: Backend::Sqlite, fresh: true, + reused_from: None, expected_selection: &["customers", "orders"], expected_warning: "connect failed: db_password=[REDACTED]", }, @@ -46,6 +47,7 @@ fn given_person_plans_when_requested_claimed_and_read_then_they_stay_location_wo description: "PostgreSQL plans", backend: Backend::Postgres, fresh: false, + reused_from: Some("2026-01-02T03:04:05Z"), expected_selection: &["customers", "orders"], expected_warning: "connect failed: db_password=[REDACTED]", }, @@ -184,6 +186,17 @@ fn given_person_plans_when_requested_claimed_and_read_then_they_stay_location_wo "{description}: a request ID cannot carry another command", description = test_case.description ); + let refreshed = PlanRequest { + fresh: !first.fresh, + ..first.clone() + }; + assert!( + ask(store, (&context, viewer), location, &refreshed) + .await + .is_err(), + "{description}: a request ID cannot change whether it plans again", + description = test_case.description + ); assert!( ask(store, (&context, developer), location, &first) .await @@ -356,7 +369,8 @@ fn given_person_plans_when_requested_claimed_and_read_then_they_stay_location_wo "config_change": "full_refresh true → false"}], "warnings": [{"severity": "warning", "message": "connect failed: db_password=orders-pass"}], "python_nodes": [{"name": "orders_export", "identity_diff": {"source_diff": ["-a", "+b"]}}], - "timings": {"total_seconds": 1.5, "steps": [{"name": "Reading warehouse state", "seconds": 1.0}, {"name": "Other", "seconds": 0.5}], "planned_from": "synced_state"}, + "timings": {"total_seconds": 1.5, "steps": [{"name": "Reading warehouse state", "seconds": 1.0}, {"name": "Other", "seconds": 0.5}], "planned_from": "synced_state", + "reused_from": test_case.reused_from}, }); assert!( complete_location_work( @@ -432,8 +446,11 @@ fn given_person_plans_when_requested_claimed_and_read_then_they_stay_location_wo }, ], planned_from: "synced_state".to_owned(), + reused_from: test_case + .reused_from + .map(|at| at.parse().expect("reused_from")), }), - "{description}: typed plan timings", + "{description}: typed plan timings, with when a reused plan was made", description = test_case.description ); let Some(LocationWorkOutcome::Plan { @@ -717,3 +734,74 @@ fn given_a_plan_whose_document_left_diffs_out_when_read_then_pages_count_them_an }); } } + +#[test] +fn given_a_plan_with_timings_when_its_summary_is_read_then_timings_of_another_shape_are_an_error() { + let test_cases = [ + PlanTimingsReadTestCase { + description: "a reused plan's timings say when it was made", + timings: json!({"total_seconds": 1.5, "steps": [], "planned_from": "synced_state", + "reused_from": "2026-01-02T03:04:05Z"}), + expected_read: true, + }, + PlanTimingsReadTestCase { + description: "timings of another shape fail the read rather than vanish", + timings: json!({"total_seconds": 1.5, "steps": [], "planned_from": "synced_state", + "reused": true}), + expected_read: false, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(Backend::Sqlite).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let context = AuditContext { + actor: "control-store-owner".into(), + granting_role: "control-store-owner".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + let admin = bootstrap_user(store, &context, "orders-admin") + .await + .expect("bootstrap"); + let location = create_location(store, "orders", "test-owner") + .await + .expect("location"); + publish_snapshot(store, &snapshot(location)) + .await + .expect("publish"); + let poll = register_instance(store, location).await; + let request = plan_request(&["build", "orders"]); + ask(store, (&context, admin), location, &request) + .await + .expect("plan"); + let claimed = claim_location_work(store, location, &poll) + .await + .expect("claim") + .expect("plan work"); + let result = LocationWorkResult { + claim_id: claimed.claim_id, + session_id: poll.session_id, + outcome: LocationWorkOutcome::Plan { + document: json!({"models": [], "timings": test_case.timings}), + diffs: BTreeMap::new(), + }, + }; + complete_location_work(store, (location, request.request_id), &result) + .await + .expect("complete"); + let plan = (location, "development", request.request_id); + assert_eq!( + read_person_plan_summary(store, (&context, admin), plan) + .await + .is_ok(), + test_case.expected_read, + "{}", + test_case.description + ); + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/api/test_types.rs b/crates/sqlbuild-orchestration/tests/api/test_types.rs index 45b0fd01cb..34412fed36 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_types.rs @@ -321,10 +321,19 @@ pub(crate) struct PlanTestCase { pub(crate) backend: Backend, /// Whether the person asks to plan again rather than reuse a kept plan. pub(crate) fresh: bool, + /// When the plan the location answered with was made, for a reused plan. + pub(crate) reused_from: Option<&'static str>, pub(crate) expected_selection: &'static [&'static str], pub(crate) expected_warning: &'static str, } +pub(crate) struct PlanTimingsReadTestCase { + pub(crate) description: &'static str, + pub(crate) timings: serde_json::Value, + /// Whether the plan's summary reads. + pub(crate) expected_read: bool, +} + pub(crate) struct UpcomingTestCase { pub(crate) description: &'static str, pub(crate) backend: Backend, diff --git a/ui/src/lib/api/http-client.test.ts b/ui/src/lib/api/http-client.test.ts index 68970a6f9d..b29ed14774 100644 --- a/ui/src/lib/api/http-client.test.ts +++ b/ui/src/lib/api/http-client.test.ts @@ -1,5 +1,6 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { HttpChangeFeed, createHttpClient, feedAllowed } from './http-client'; +import { timingTitle } from '$lib/run/plan-timings'; import type { ChangedSignal, ConnectionState } from './signals'; /** A stand-in `EventSource` the test opens, fails and feeds by hand. */ @@ -979,4 +980,23 @@ describe('planning a build', () => { } expect(bodies.map((body) => body.fresh)).toEqual([false, true]); }); + + it("carries a kept plan's age from the server's summary to the plan it returns", async () => { + const timings = { total_seconds: 4.2, steps: [{ name: 'Detecting changes', seconds: 4.2 }], planned_from: 'synced_state', reused_from: '2026-01-02T03:04:05Z' }; + vi.stubGlobal('document', { cookie: '' }); + vi.stubGlobal( + 'fetch', + vi.fn(async (request: Request) => + request.method === 'POST' + ? Response.json({ applied: true }) + : Response.json({ expires_at: '2999-01-01T00:00:00Z', outcome: { kind: 'plan', document: { selected_count: 1, models: [] }, diffs: {} }, timings }) + ) + ); + const client = createHttpClient({ baseUrl: 'http://ui.test', onUnauthorized: () => {} }); + const planned = client.planBuild(ORDERS, ['build', '--changed']); + await vi.advanceTimersByTimeAsync(1000); + const { plan } = await planned; + expect(plan?.timings).toEqual(timings); + expect(timingTitle(plan!.timings!, Date.parse('2026-01-02T03:10:05Z'))).toBe('Reused plan from 6m ago'); + }); }); diff --git a/ui/src/lib/api/types/operate.ts b/ui/src/lib/api/types/operate.ts index 4d3dc595c0..3257a30382 100644 --- a/ui/src/lib/api/types/operate.ts +++ b/ui/src/lib/api/types/operate.ts @@ -780,6 +780,8 @@ export type PlanTimings = { steps: { name: string; seconds: number }[]; /** `synced_state` for a location preview over the synced warehouse state; `live` when it read the warehouse. */ planned_from: string; + /** When the plan the location kept, and answered with again, was made; absent for a new plan. */ + reused_from?: string | null; }; /** The server's synced copy of warehouse metadata a location's plan preview planned from. */ diff --git a/ui/src/lib/run/plan-timings.svelte b/ui/src/lib/run/plan-timings.svelte index 96a9784494..065ef90b10 100644 --- a/ui/src/lib/run/plan-timings.svelte +++ b/ui/src/lib/run/plan-timings.svelte @@ -5,15 +5,15 @@ import { cn } from '$lib/utils'; import TimerIcon from '@lucide/svelte/icons/timer'; import type { PlanDocument, PlanTimings } from '$lib/api'; - import { seconds, timingFootnote, timingRows } from './plan-timings'; + import { seconds, timingFootnote, timingRows, timingTitle } from './plan-timings'; - let { plan, timings }: { plan: PlanDocument; timings: PlanTimings } = $props(); + let { plan, timings, now }: { plan: PlanDocument; timings: PlanTimings; now: number } = $props(); const rows = $derived(timingRows(timings)); - Planned in {seconds(timings.total_seconds)} s + {timingTitle(timings, now)}
@@ -32,7 +32,7 @@ Total{seconds(timings.total_seconds)} -

{timingFootnote(plan, timings)}

+

{timingFootnote(plan, timings, now)}

diff --git a/ui/src/lib/run/plan-timings.test.ts b/ui/src/lib/run/plan-timings.test.ts index d7152cd026..208eb6042f 100644 --- a/ui/src/lib/run/plan-timings.test.ts +++ b/ui/src/lib/run/plan-timings.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from 'vitest'; import type { PlanDocument, PlanTimings } from '$lib/api'; -import { timingFootnote, timingRows } from './plan-timings'; +import { timingFootnote, timingRows, timingTitle } from './plan-timings'; const timings: PlanTimings = { total_seconds: 10, @@ -25,7 +25,16 @@ describe('plan timings', () => { it('says where the plan ran and how many nodes it planned', () => { const plan = { selected_count: 2418 } as PlanDocument; - expect(timingFootnote(plan, timings)).toBe('On the location, from the synced warehouse state · 2,418 nodes'); - expect(timingFootnote(plan, { ...timings, planned_from: 'live' })).toBe('On the location, live from the warehouse · 2,418 nodes'); + const now = Date.parse('2026-01-02T03:10:05Z'); + expect(timingFootnote(plan, timings, now)).toBe('On the location, from the synced warehouse state · 2,418 nodes'); + expect(timingFootnote(plan, { ...timings, planned_from: 'live' }, now)).toBe('On the location, live from the warehouse · 2,418 nodes'); + }); + + it("labels a kept plan by its age instead of a planning time", () => { + const plan = { selected_count: 2418 } as PlanDocument; + const now = Date.parse('2026-01-02T03:10:05Z'); + const reused = { ...timings, reused_from: '2026-01-02T03:04:05Z' }; + expect([timingTitle(timings, now), timingTitle(reused, now)]).toEqual(['Planned in 10.0 s', 'Reused plan from 6m ago']); + expect(timingFootnote(plan, reused, now)).toBe("On the location, from the synced warehouse state · 2,418 nodes · the location's plan from 6m ago; Plan again for a new one"); }); }); diff --git a/ui/src/lib/run/plan-timings.ts b/ui/src/lib/run/plan-timings.ts index 4215f659b1..8e953847d0 100644 --- a/ui/src/lib/run/plan-timings.ts +++ b/ui/src/lib/run/plan-timings.ts @@ -2,6 +2,7 @@ import type { PlanDocument, PlanTimings } from '$lib/api'; import { plural } from '$lib/format/plural'; +import { formatAgo } from '$lib/format/time'; export type TimingRow = { name: string; seconds: string; share: number; percent: string }; @@ -19,8 +20,14 @@ export function timingRows(timings: PlanTimings): TimingRow[] { }); } -/** Where the plan ran and from what, with the nodes it planned. */ -export function timingFootnote(plan: PlanDocument, timings: PlanTimings): string { +/** The popover's label: how long planning took, or how old the kept plan the location answered with is. */ +export function timingTitle(timings: PlanTimings, now: number): string { + return timings.reused_from ? `Reused plan from ${formatAgo(timings.reused_from, now)}` : `Planned in ${seconds(timings.total_seconds)} s`; +} + +/** Where the plan ran and from what, with the nodes it planned and, for a kept plan, when it was made. */ +export function timingFootnote(plan: PlanDocument, timings: PlanTimings, now: number): string { const from = timings.planned_from === 'live' ? 'On the location, live from the warehouse' : 'On the location, from the synced warehouse state'; - return `${from} · ${plural(plan.selected_count, 'node')}`; + const reused = timings.reused_from ? ` · the location's plan from ${formatAgo(timings.reused_from, now)}; Plan again for a new one` : ''; + return `${from} · ${plural(plan.selected_count, 'node')}${reused}`; } diff --git a/ui/src/routes/plan/+page.svelte b/ui/src/routes/plan/+page.svelte index 58d48d9636..1ed41efbed 100644 --- a/ui/src/routes/plan/+page.svelte +++ b/ui/src/routes/plan/+page.svelte @@ -460,7 +460,7 @@ (view = v)} /> {#if shown.verb === 'build' && plan} - {#if plan.timings}{/if} + {#if plan.timings}{/if} {/if} From f55ab843ee933ada1f3d6a3ffafc9b677293b677 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 19 Nov 2026 19:39:55 +0000 Subject: [PATCH 11/15] fix(location): reuse a kept plan for ten minutes, within a byte budget, once its checks pass A plan's cursor windows and expiry follow the time it was made, so a kept plan answers identical requests for 10 minutes, then the location plans again. Kept plans are held behind Arc within 256 MiB as well as 16 plans, oldest out first, and read outside the lock. A plan is kept only after the checks that the project files did not change while it planned, for both requested plans and the post-sync plan of what changed. --- .../_helpers/location/plan_reuse.rs | 127 ++++++++++++------ .../src/native_commands/constants.rs | 6 + .../main/_location_changed_build.rs | 4 +- .../native_commands/main/_location_plan.rs | 54 +++++--- .../native_commands/main/_location_work.rs | 29 +++- .../src/native_commands/models.rs | 27 ++-- .../src/native_commands/tests/helpers.rs | 40 ++++-- .../tests/test_location_plan.rs | 82 ++++++++--- .../src/native_commands/tests/test_types.rs | 6 +- 9 files changed, 265 insertions(+), 110 deletions(-) diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/location/plan_reuse.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/location/plan_reuse.rs index d5b3ad26b1..50fe2ce4e7 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/location/plan_reuse.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/location/plan_reuse.rs @@ -1,37 +1,50 @@ //! The location's last finished plans, in memory, returned again for an identical request. -use std::collections::VecDeque; -use std::sync::{Mutex, PoisonError}; +use std::collections::{BTreeMap, VecDeque}; +use std::sync::{Arc, Mutex, PoisonError}; use chrono::{DateTime, SecondsFormat, Utc}; use serde_json::{Value, json}; +use sqlbuild_orchestration::platform::models::PlanModelDiff; use crate::argv::models::Stream; use crate::native_commands::_helpers::command::terminal::write_output; -use crate::native_commands::constants::REUSED_PLANS; -use crate::native_commands::models::{PlanKey, PlanReuse, ReusedPlan}; +use crate::native_commands::constants::{REUSED_PLAN_AGE, REUSED_PLAN_BYTES, REUSED_PLANS}; +use crate::native_commands::models::{PlanKey, PlanReuse, PlanToKeep, ReusedPlan}; use crate::native_commands::types::LocationPlan; /// The location's kept plans; one location serves one target. pub(crate) static PLAN_REUSE: Mutex> = Mutex::new(None); -/// The kept plan for `key` under this generation and copy, marked as reused. +/// The kept plan for `key` under this generation and copy, if younger than the reuse age at `now`. pub(crate) fn reused_plan( reuse: &Mutex>, (generation, version): (i64, &str), key: &PlanKey, + now: DateTime, ) -> Option { - let kept: ReusedPlan = reuse - .lock() - .unwrap_or_else(PoisonError::into_inner) - .as_ref() - .filter(|reuse| reuse.generation == generation && reuse.version == version)? - .plans - .iter() - .find(|plan| plan.key == *key)? - .clone(); - let mut document: Value = match serde_json::from_str(&kept.document) { - Ok(document) => document, + let kept: Arc = { + let mut reuse = reuse.lock().unwrap_or_else(PoisonError::into_inner); + let current: &mut PlanReuse = reuse + .as_mut() + .filter(|reuse| reuse.generation == generation && reuse.version == version)?; + while current + .plans + .front() + .is_some_and(|plan| now - plan.planned_at >= REUSED_PLAN_AGE) + { + if let Some(expired) = current.plans.pop_front() { + current.bytes -= plan_bytes(&expired); + } + } + Arc::clone(current.plans.iter().find(|plan| plan.key == *key)?) + }; + let read = serde_json::from_str::(&kept.document).and_then(|document| { + serde_json::from_str::>(&kept.diffs) + .map(|diffs| (document, diffs)) + }); + let (mut document, diffs) = match read { + Ok(read) => read, Err(error) => { drop(write_output(&[( Stream::Stderr, @@ -40,46 +53,76 @@ pub(crate) fn reused_plan( return None; } }; - let planned_from: Value = document["timings"]["planned_from"].clone(); - document["timings"] = json!({ - "planned_from": planned_from, - "reused_from": kept.planned_at, - "total_seconds": 0.0, - "steps": [{"name": format!("Reused plan from {}", kept.planned_at), "seconds": 0.0}], - }); - Some((document, kept.diffs)) + document["timings"]["reused_from"] = + json!(kept.planned_at.to_rfc3339_opts(SecondsFormat::Secs, true)); + Some((document, diffs)) } -/// Keep a finished plan for `key`, unless it failed; a newer generation or copy replaces all. +/// Keep a checked, successful plan within `(plans, bytes)`, oldest out first; a newer generation or copy replaces all. pub(crate) fn remember_plan( - reuse: &Mutex>, - (generation, version): (i64, &str), - key: PlanKey, - ((document, diffs), planned_at): (&LocationPlan, DateTime), + (reuse, (most_plans, most_bytes)): (&Mutex>, (usize, usize)), + keep: PlanToKeep, + (document, diffs): (&Value, &BTreeMap), ) { if !document["error"].is_null() { return; } - let Ok(text) = serde_json::to_string(document) else { + let (Ok(text), Ok(diff_text)) = ( + serde_json::to_string(document), + serde_json::to_string(diffs), + ) else { return; }; + let kept = Arc::new(ReusedPlan { + key: keep.key, + document: text, + diffs: diff_text, + planned_at: keep.planned_at, + }); + if plan_bytes(&kept) > most_bytes { + return; + } let mut reuse = reuse.lock().unwrap_or_else(PoisonError::into_inner); let current: &mut PlanReuse = match reuse.as_mut() { - Some(current) if current.generation == generation && current.version == version => current, + Some(current) + if current.generation == keep.generation && current.version == keep.version => + { + current + } _ => reuse.insert(PlanReuse { - generation, - version: version.to_owned(), + generation: keep.generation, + version: keep.version, plans: VecDeque::new(), + bytes: 0, }), }; - current.plans.retain(|plan| plan.key != key); - current.plans.push_back(ReusedPlan { - key, - document: text, - diffs: diffs.clone(), - planned_at: planned_at.to_rfc3339_opts(SecondsFormat::Secs, true), - }); - while current.plans.len() > REUSED_PLANS { - current.plans.pop_front(); + let replaced: usize = current + .plans + .iter() + .filter(|plan| plan.key == kept.key) + .map(|plan| plan_bytes(plan)) + .sum(); + current.plans.retain(|plan| plan.key != kept.key); + current.bytes = current.bytes - replaced + plan_bytes(&kept); + current.plans.push_back(kept); + while current.plans.len() > most_plans || current.bytes > most_bytes { + if let Some(dropped) = current.plans.pop_front() { + current.bytes -= plan_bytes(&dropped); + } } } + +/// Keep a plan for identical requests once the checks after planning passed. +pub(crate) fn keep_plan( + keep: Option, + plan: (&Value, &BTreeMap), +) { + if let Some(keep) = keep { + remember_plan((&PLAN_REUSE, (REUSED_PLANS, REUSED_PLAN_BYTES)), keep, plan); + } +} + +/// The bytes a kept plan's text holds. +fn plan_bytes(plan: &ReusedPlan) -> usize { + plan.document.len() + plan.diffs.len() +} diff --git a/crates/sqlbuild-cli/src/native_commands/constants.rs b/crates/sqlbuild-cli/src/native_commands/constants.rs index 696430830a..8f89b1fe35 100644 --- a/crates/sqlbuild-cli/src/native_commands/constants.rs +++ b/crates/sqlbuild-cli/src/native_commands/constants.rs @@ -221,6 +221,12 @@ pub(crate) const LOCATION_WORK_IDLE_POLL: std::time::Duration = std::time::Durat /// Finished plans a location keeps for identical requests; a newer generation or synced copy drops them. pub(crate) const REUSED_PLANS: usize = 16; +/// The most bytes of plan text a location keeps for reuse; the oldest plans go first. +pub(crate) const REUSED_PLAN_BYTES: usize = 256 * 1024 * 1024; + +/// How long a kept plan may answer a request: its windows and expiry follow the time it planned at. +pub(crate) const REUSED_PLAN_AGE: chrono::TimeDelta = chrono::TimeDelta::minutes(10); + /// What a location reports when `sqb plan` produced no plan. pub(crate) const PLAN_NOT_PRODUCED: &str = "sqb plan did not produce a plan; see the location's log"; diff --git a/crates/sqlbuild-cli/src/native_commands/main/_location_changed_build.rs b/crates/sqlbuild-cli/src/native_commands/main/_location_changed_build.rs index 389d3053af..2901c1bbae 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/_location_changed_build.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/_location_changed_build.rs @@ -16,6 +16,7 @@ use crate::native_commands::_helpers::command::terminal::write_output; use crate::native_commands::_helpers::location::changed_build::{ changed_build_nodes, changed_build_settings, }; +use crate::native_commands::_helpers::location::plan_reuse::keep_plan; use crate::native_commands::main::_location_plan::plan_location; use crate::native_commands::models::{LocationRuntimeRequest, LocationServe, LocationWatch}; @@ -44,7 +45,7 @@ pub(super) async fn report_changed_build( } }; report("Planning what changed from the synced warehouse copy...\n"); - let Some((document, _)) = plan_location( + let Some(((document, diffs), keep)) = plan_location( (options, runtime), (&BTreeSet::new(), &settings, false), (client, session_id, generation), @@ -58,6 +59,7 @@ pub(super) async fn report_changed_build( report("Project files changed while planning what changed; not reported.\n"); return Ok(Some(false)); } + keep_plan(keep, (&document, &diffs)); if let Some(reason) = document["error"]["message"].as_str() { report(&format!("Could not plan what changed: {reason}\n")); let request = ChangedBuildReport { diff --git a/crates/sqlbuild-cli/src/native_commands/main/_location_plan.rs b/crates/sqlbuild-cli/src/native_commands/main/_location_plan.rs index 2412f702a6..4443cf2113 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/_location_plan.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/_location_plan.rs @@ -3,7 +3,7 @@ use std::collections::{BTreeMap, BTreeSet}; use std::future::Future; -use chrono::Utc; +use chrono::{DateTime, Utc}; use serde_json::{Value, json}; use sqlbuild_orchestration::platform::models::LocationWorkClient; use sqlbuild_orchestration::store::errors::ControlError; @@ -16,24 +16,24 @@ use crate::native_commands::_helpers::location::held_copy::{HELD_COPY, held_copy use crate::native_commands::_helpers::location::plan_request::{ plan_arguments, run_plan, selected_names, }; -use crate::native_commands::_helpers::location::plan_reuse::{ - PLAN_REUSE, remember_plan, reused_plan, +use crate::native_commands::_helpers::location::plan_reuse::{PLAN_REUSE, reused_plan}; +use crate::native_commands::models::{ + HeldCopy, LocationRuntimeRequest, LocationServe, PlanKey, PlanToKeep, }; -use crate::native_commands::models::{HeldCopy, LocationRuntimeRequest, LocationServe, PlanKey}; use crate::native_commands::types::LocationPlan; -/// The plan document with the diffs it left out, an `{"error": ...}` report, or `None` when stopped first. +/// The plan with what to keep once its checks pass, an `{"error": ...}` report, or `None` when stopped first. pub(super) async fn plan_location( (options, runtime): (&LocationServe, &mpsc::Sender), (selection, settings, fresh): (&BTreeSet, &BTreeMap, bool), (client, session_id, generation): (&LocationWorkClient, Uuid, i64), stopped: impl Future>, -) -> Result, ControlError> { +) -> Result)>, ControlError> { let (argv, select) = match plan_arguments(options, settings) .and_then(|argv| Ok((argv, selected_names(selection)?))) { Ok(planned) => planned, - Err(message) => return Ok(Some(without_diffs(plan_failure(&message)))), + Err(message) => return Ok(Some((without_diffs(plan_failure(&message)), None))), }; let consumption: bool = settings .get("changed") @@ -49,9 +49,12 @@ pub(super) async fn plan_location( let copy = match client.preview_copy(&options.target, &read).await { Ok(copy) => copy, Err(error) => { - return Ok(Some(without_diffs(plan_failure(&format!( - "the synced warehouse copy is unavailable: {error}" - ))))); + return Ok(Some(( + without_diffs(plan_failure(&format!( + "the synced warehouse copy is unavailable: {error}" + ))), + None, + ))); } }; let planned: HeldCopy = match held { @@ -63,8 +66,16 @@ pub(super) async fn plan_location( select: select.clone(), consumption: planned.consumption, }; - if !fresh && let Some(reused) = reused_plan(&PLAN_REUSE, (generation, &planned.version), &key) { - return Ok(Some(reused)); + let planned_at: DateTime = Utc::now(); + if !fresh + && let Some(reused) = reused_plan( + &PLAN_REUSE, + (generation, &planned.version), + &key, + planned_at, + ) + { + return Ok(Some((reused, None))); } let request = json!({"plan": { "argv": argv, @@ -73,14 +84,13 @@ pub(super) async fn plan_location( "version": planned.version, "consumption": planned.consumption, }}); - let plan: Option = run_plan(runtime, request, stopped).await?; - if let Some(plan) = &plan { - remember_plan( - &PLAN_REUSE, - (generation, &planned.version), - key, - (plan, Utc::now()), - ); - } - Ok(plan) + let keep = PlanToKeep { + generation, + version: planned.version, + key, + planned_at, + }; + Ok(run_plan(runtime, request, stopped) + .await? + .map(|plan| (plan, Some(keep)))) } diff --git a/crates/sqlbuild-cli/src/native_commands/main/_location_work.rs b/crates/sqlbuild-cli/src/native_commands/main/_location_work.rs index 7dd5e6c509..32fbf8b992 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/_location_work.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/_location_work.rs @@ -18,11 +18,12 @@ use crate::argv::models::Stream; use crate::native_commands::_helpers::command::paths::location_files; use crate::native_commands::_helpers::command::terminal::write_output; use crate::native_commands::_helpers::location::execution::check_location_connection; +use crate::native_commands::_helpers::location::plan_reuse::keep_plan; use crate::native_commands::_helpers::location::work::work_error; use crate::native_commands::main::_location_plan::plan_location; use crate::native_commands::main::_location_sensor::evaluate_location_sensor; use crate::native_commands::models::{ - LocationRuntimeRequest, LocationServe, LocationWatch, LocationWorkTurn, + LocationRuntimeRequest, LocationServe, LocationWatch, LocationWorkTurn, PlanToKeep, }; /// Retry the exact completed result without rerunning it; a turn's `sync` asks the metadata loop to sync. @@ -74,7 +75,7 @@ pub(super) async fn run_location_work( expires, ) .await?; - let Some((outcome, local_files)) = checked else { + let Some((outcome, local_files, keep)) = checked else { drop(write_output(&[( Stream::Stderr, "Location work stopped before completion; no result reported.\n", @@ -93,6 +94,9 @@ pub(super) async fn run_location_work( let mut excluded = watch.local_files.clone(); excluded.extend(files.clone()); if location_files(&options.project_dir, &watch.rules, &excluded) == expected.0 { + if let LocationWorkOutcome::Plan { document, diffs } = &outcome { + keep_plan(keep, (document, diffs)); + } outcome } else { LocationWorkOutcome::Superseded {} @@ -147,7 +151,7 @@ pub(super) async fn run_location_work( })) } -/// One named operation; `None` means it stopped at shutdown or its lease ended; `sqb dev` never signs in here. +/// One named operation and a plan to keep once checked; `None` when it stopped at shutdown or its lease ended. async fn perform_work( (options, script, (runtime, connected)): ( &LocationServe, @@ -157,7 +161,7 @@ async fn perform_work( work: LocationWork, copy: (&LocationWorkClient, Uuid, i64), expires: impl Future>, -) -> Result)>, ControlError> { +) -> Result, Option)>, ControlError> { let message = match &work { LocationWork::CheckConnection {} => "Checking requested location connection...\n", LocationWork::SyncWarehouse {} => "Synchronizing warehouse history on request...\n", @@ -166,11 +170,15 @@ async fn perform_work( }; drop(write_output(&[(Stream::Stderr, message)])); Ok(match work { - LocationWork::SyncWarehouse {} => Some((LocationWorkOutcome::SyncStarted {}, Vec::new())), + LocationWork::SyncWarehouse {} => { + Some((LocationWorkOutcome::SyncStarted {}, Vec::new(), None)) + } LocationWork::CheckConnection {} => { check_location_connection((options, script, connected), expires) .await? - .map(|(healthy, _, files)| (LocationWorkOutcome::Connection { healthy }, files)) + .map(|(healthy, _, files)| { + (LocationWorkOutcome::Connection { healthy }, files, None) + }) } LocationWork::EvaluateSensor { evaluation } => { evaluate_location_sensor(options, script, &evaluation, expires) @@ -179,6 +187,7 @@ async fn perform_work( ( LocationWorkOutcome::SensorEvaluation { outcome }, Vec::new(), + None, ) }) } @@ -194,6 +203,12 @@ async fn perform_work( expires, ) .await? - .map(|(document, diffs)| (LocationWorkOutcome::Plan { document, diffs }, Vec::new())), + .map(|((document, diffs), keep)| { + ( + LocationWorkOutcome::Plan { document, diffs }, + Vec::new(), + keep, + ) + }), }) } diff --git a/crates/sqlbuild-cli/src/native_commands/models.rs b/crates/sqlbuild-cli/src/native_commands/models.rs index 45469b43b3..46655c1562 100644 --- a/crates/sqlbuild-cli/src/native_commands/models.rs +++ b/crates/sqlbuild-cli/src/native_commands/models.rs @@ -576,23 +576,32 @@ pub(crate) struct PlanKey { pub(crate) consumption: bool, } -/// One finished plan, kept as its JSON text with the diffs that travel beside it. -#[derive(Clone, Debug)] +/// One finished plan, kept as the JSON text of its document and of the diffs beside it. +#[derive(Debug)] pub(crate) struct ReusedPlan { pub(crate) key: PlanKey, pub(crate) document: String, - pub(crate) diffs: - std::collections::BTreeMap, - /// When the plan finished, as RFC 3339 text. - pub(crate) planned_at: String, + pub(crate) diffs: String, + /// When the plan started, which its windows and expiry follow. + pub(crate) planned_at: chrono::DateTime, } -/// The finished plans of one published generation and synced copy, oldest first. -#[derive(Clone, Debug)] +/// The finished plans of one published generation and synced copy, oldest first, and their bytes. +#[derive(Debug)] pub(crate) struct PlanReuse { pub(crate) generation: i64, pub(crate) version: String, - pub(crate) plans: std::collections::VecDeque, + pub(crate) plans: std::collections::VecDeque>, + pub(crate) bytes: usize, +} + +/// A plan the location may keep once its checks pass: its generation, copy, key and start time. +#[derive(Clone, Debug)] +pub(crate) struct PlanToKeep { + pub(crate) generation: i64, + pub(crate) version: String, + pub(crate) key: PlanKey, + pub(crate) planned_at: chrono::DateTime, } /// The copy file the location last read, the mirror version it holds and whether it has consumption. diff --git a/crates/sqlbuild-cli/src/native_commands/tests/helpers.rs b/crates/sqlbuild-cli/src/native_commands/tests/helpers.rs index 866679ba7a..775f6eb238 100644 --- a/crates/sqlbuild-cli/src/native_commands/tests/helpers.rs +++ b/crates/sqlbuild-cli/src/native_commands/tests/helpers.rs @@ -1,7 +1,7 @@ use std::path::Path; use std::sync::Mutex; -use chrono::Utc; +use chrono::{DateTime, TimeZone, Utc}; use serde_json::json; use sqlbuild_orchestration::warehouse_metadata::models::{MetadataBatch, WarehouseStateLocation}; @@ -16,7 +16,7 @@ use crate::native_commands::_helpers::location::execution::{ }; use crate::native_commands::_helpers::location::plan_reuse::remember_plan; use crate::native_commands::models::{ - LocationMetadataPage, LocationMetadataSync, LocationServe, PlanKey, PlanReuse, + LocationMetadataPage, LocationMetadataSync, LocationServe, PlanKey, PlanReuse, PlanToKeep, }; use crate::native_commands::types::LocationPlan; use sqlbuild_orchestration::platform::main::work::connect_location_work::connect_location_work; @@ -163,15 +163,39 @@ pub(super) fn finished_plan(failed: bool) -> LocationPlan { (documents[usize::from(failed)].clone(), Default::default()) } -/// `count` finished plans of `model_` under generation 1 and `copy-1`, oldest first. -pub(super) fn kept_plans(count: usize) -> Mutex> { +/// The bytes `finished_plan(false)` takes when kept: its document's and diffs' JSON text. +pub(super) fn finished_plan_bytes() -> usize { + let (document, diffs) = finished_plan(false); + serde_json::to_string(&document).expect("document").len() + + serde_json::to_string(&diffs).expect("diffs").len() +} + +/// The time the reuse tests plan at. +pub(super) fn reuse_clock() -> DateTime { + Utc.with_ymd_and_hms(2026, 1, 2, 3, 4, 5) + .single() + .expect("time") +} + +/// A plan of `select` to keep under `generation` and `version`, started at `reuse_clock`. +pub(super) fn plan_to_keep(generation: i64, version: &str, select: &[&str]) -> PlanToKeep { + PlanToKeep { + generation, + version: version.to_owned(), + key: reuse_key(select), + planned_at: reuse_clock(), + } +} + +/// `count` finished plans of `model_` under generation 1 and `copy-1`, oldest first, within `limits`. +pub(super) fn kept_plans(count: usize, limits: (usize, usize)) -> Mutex> { let reuse: Mutex> = Mutex::new(None); + let (document, diffs) = finished_plan(false); for index in 0..count { remember_plan( - &reuse, - (1, "copy-1"), - reuse_key(&[format!("model_{index}").as_str()]), - (&finished_plan(false), Utc::now()), + (&reuse, limits), + plan_to_keep(1, "copy-1", &[format!("model_{index}").as_str()]), + (&document, &diffs), ); } reuse diff --git a/crates/sqlbuild-cli/src/native_commands/tests/test_location_plan.rs b/crates/sqlbuild-cli/src/native_commands/tests/test_location_plan.rs index 06f440066b..2f0c5f3083 100644 --- a/crates/sqlbuild-cli/src/native_commands/tests/test_location_plan.rs +++ b/crates/sqlbuild-cli/src/native_commands/tests/test_location_plan.rs @@ -3,18 +3,21 @@ use sqlbuild_orchestration::platform::constants::PLAN_DOCUMENT_BYTES; use std::sync::Mutex; -use chrono::{TimeZone, Utc}; +use chrono::TimeDelta; use crate::native_commands::_helpers::location::plan_document::plan_document; use crate::native_commands::_helpers::location::plan_request::plan_arguments; use crate::native_commands::_helpers::location::plan_reuse::{remember_plan, reused_plan}; use crate::native_commands::_helpers::location::work::next_work_check; use crate::native_commands::constants::LOCATION_WORK_IDLE_POLL; -use crate::native_commands::constants::REUSED_PLANS; +use crate::native_commands::constants::{REUSED_PLAN_BYTES, REUSED_PLANS}; use crate::native_commands::models::LocationServe; use crate::native_commands::models::LocationWorkTurn; use crate::native_commands::models::PlanReuse; -use crate::native_commands::tests::helpers::{finished_plan, kept_plans, plan_output, reuse_key}; +use crate::native_commands::tests::helpers::{ + finished_plan, finished_plan_bytes, kept_plans, plan_output, plan_to_keep, reuse_clock, + reuse_key, +}; use crate::native_commands::tests::test_types::{ PlanArgumentsTestCase, PlanDocumentTestCase, PlanReuseBoundTestCase, PlanReuseTestCase, WorkCheckTestCase, @@ -117,58 +120,70 @@ fn given_a_turn_of_location_work_when_scheduling_the_next_check_then_only_an_idl } #[test] -fn given_a_kept_plan_when_the_same_request_comes_again_then_only_the_same_generation_copy_and_request_reuse_it() +fn given_a_kept_plan_when_the_same_request_comes_again_then_only_the_same_generation_copy_and_request_reuse_it_for_ten_minutes() { let reused_timings = json!({"planned_from": "synced_state", "reused_from": "2026-01-02T03:04:05Z", - "total_seconds": 0.0, - "steps": [{"name": "Reused plan from 2026-01-02T03:04:05Z", "seconds": 0.0}]}); + "total_seconds": 4.2, "steps": [{"name": "Detecting changes", "seconds": 4.2}]}); let test_cases = [ PlanReuseTestCase { description: "the same generation, copy and request reuse the plan and say when it was made", kept: (3, "copy-1", false), asked: (3, "copy-1", &["orders"]), + age: TimeDelta::seconds(599), expected_timings: Some(reused_timings), }, + PlanReuseTestCase { + description: "a plan ten minutes old is not reused, so windows and expiry follow the clock", + kept: (3, "copy-1", false), + asked: (3, "copy-1", &["orders"]), + age: TimeDelta::minutes(10), + expected_timings: None, + }, PlanReuseTestCase { description: "a newer synced copy plans again", kept: (3, "copy-1", false), asked: (3, "copy-2", &["orders"]), + age: TimeDelta::zero(), expected_timings: None, }, PlanReuseTestCase { description: "a newer generation plans again", kept: (3, "copy-1", false), asked: (4, "copy-1", &["orders"]), + age: TimeDelta::zero(), expected_timings: None, }, PlanReuseTestCase { description: "another selection plans again", kept: (3, "copy-1", false), asked: (3, "copy-1", &["customers"]), + age: TimeDelta::zero(), expected_timings: None, }, PlanReuseTestCase { description: "a failed plan is not kept", kept: (3, "copy-1", true), asked: (3, "copy-1", &["orders"]), + age: TimeDelta::zero(), expected_timings: None, }, ]; - let planned_at = Utc - .with_ymd_and_hms(2026, 1, 2, 3, 4, 5) - .single() - .expect("time"); for test_case in test_cases { let reuse: Mutex> = Mutex::new(None); let (generation, version, failed) = test_case.kept; + let (document, diffs) = finished_plan(failed); remember_plan( + (&reuse, (REUSED_PLANS, REUSED_PLAN_BYTES)), + plan_to_keep(generation, version, &["orders"]), + (&document, &diffs), + ); + let (generation, version, select) = test_case.asked; + let reused = reused_plan( &reuse, (generation, version), - reuse_key(&["orders"]), - (&finished_plan(failed), planned_at), + &reuse_key(select), + reuse_clock() + test_case.age, ); - let (generation, version, select) = test_case.asked; - let reused = reused_plan(&reuse, (generation, version), &reuse_key(select)); assert_eq!( reused.map(|(document, _)| document["timings"].clone()), test_case.expected_timings, @@ -181,47 +196,74 @@ fn given_a_kept_plan_when_the_same_request_comes_again_then_only_the_same_genera #[test] fn given_more_finished_plans_than_kept_when_planning_then_the_oldest_is_dropped_and_a_newer_copy_drops_all() { + let plan_bytes: usize = finished_plan_bytes(); let test_cases = [ PlanReuseBoundTestCase { description: "the oldest of one plan too many is dropped", + limits: (REUSED_PLANS, REUSED_PLAN_BYTES, REUSED_PLANS + 1), newer_copy: false, asked: ("model_0", "copy-1"), expected_reused: false, }, PlanReuseBoundTestCase { description: "the next oldest is kept", + limits: (REUSED_PLANS, REUSED_PLAN_BYTES, REUSED_PLANS + 1), + newer_copy: false, + asked: ("model_1", "copy-1"), + expected_reused: true, + }, + PlanReuseBoundTestCase { + description: "past the byte limit the oldest plan is dropped", + limits: (REUSED_PLANS, 2 * plan_bytes, 3), + newer_copy: false, + asked: ("model_0", "copy-1"), + expected_reused: false, + }, + PlanReuseBoundTestCase { + description: "the plans within the byte limit are kept", + limits: (REUSED_PLANS, 2 * plan_bytes, 3), newer_copy: false, asked: ("model_1", "copy-1"), expected_reused: true, }, + PlanReuseBoundTestCase { + description: "a plan larger than the byte limit is not kept", + limits: (REUSED_PLANS, plan_bytes - 1, 1), + newer_copy: false, + asked: ("model_0", "copy-1"), + expected_reused: false, + }, PlanReuseBoundTestCase { description: "a newer synced copy drops every kept plan", + limits: (REUSED_PLANS, REUSED_PLAN_BYTES, REUSED_PLANS + 1), newer_copy: true, asked: ("model_1", "copy-2"), expected_reused: false, }, PlanReuseBoundTestCase { description: "a plan of the newer copy is kept", + limits: (REUSED_PLANS, REUSED_PLAN_BYTES, REUSED_PLANS + 1), newer_copy: true, asked: ("model_new", "copy-2"), expected_reused: true, }, ]; + let (document, diffs) = finished_plan(false); for test_case in test_cases { - let reuse: Mutex> = kept_plans(REUSED_PLANS + 1); + let (most_plans, most_bytes, count) = test_case.limits; + let reuse: Mutex> = kept_plans(count, (most_plans, most_bytes)); let newer: &[(&str, &str)] = [&[][..], &[("model_new", "copy-2")][..]][usize::from(test_case.newer_copy)]; for &(name, version) in newer { remember_plan( - &reuse, - (1, version), - reuse_key(&[name]), - (&finished_plan(false), Utc::now()), + (&reuse, (most_plans, most_bytes)), + plan_to_keep(1, version, &[name]), + (&document, &diffs), ); } let (name, version) = test_case.asked; assert_eq!( - reused_plan(&reuse, (1, version), &reuse_key(&[name])).is_some(), + reused_plan(&reuse, (1, version), &reuse_key(&[name]), reuse_clock()).is_some(), test_case.expected_reused, "{description}", description = test_case.description diff --git a/crates/sqlbuild-cli/src/native_commands/tests/test_types.rs b/crates/sqlbuild-cli/src/native_commands/tests/test_types.rs index 590deafa4c..1d42d8c8d6 100644 --- a/crates/sqlbuild-cli/src/native_commands/tests/test_types.rs +++ b/crates/sqlbuild-cli/src/native_commands/tests/test_types.rs @@ -105,13 +105,17 @@ pub(super) struct PlanReuseTestCase { pub(super) kept: (i64, &'static str, bool), /// The generation, copy version and selected names of the request. pub(super) asked: (i64, &'static str, &'static [&'static str]), + /// How long after the kept plan started the request comes. + pub(super) age: chrono::TimeDelta, /// The reused plan's timings, or `None` when the location plans again. pub(super) expected_timings: Option, } -/// One more finished plan than a location keeps, maybe a plan of a newer copy, then one request. +/// Plans kept within a count and byte limit, maybe a plan of a newer copy, then one request. pub(super) struct PlanReuseBoundTestCase { pub(super) description: &'static str, + /// The most plans and bytes kept, and how many plans are kept first. + pub(super) limits: (usize, usize, usize), pub(super) newer_copy: bool, /// The selected name and copy version of the request. pub(super) asked: (&'static str, &'static str), From c03bf3131a5da02ba1ff988e31a10a510c553f15 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 19 Nov 2026 19:39:56 +0000 Subject: [PATCH 12/15] docs(server): say how long a location reuses a kept plan --- .../.agents/skills/sqlbuild/references/docs/cli/server.md | 5 +++-- website/src/content/docs/docs/cli/server.mdx | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md index d527a8c1e2..255a91828e 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md @@ -893,9 +893,10 @@ These stay unknown until it does: plan takes as declared. A location keeps its last 16 plans for the current publication and synced copy, in memory, and -answers an identical request with the plan it kept instead of planning again. A newer publication +answers an identical request with the plan it kept instead of planning again, for up to 10 +minutes: a plan's incremental windows and expiry follow the time it was made. A newer publication or synced copy drops them all. A kept plan's `timings` say so in `reused_from`, the time it was -made. The page's **Plan again** asks for a fresh plan. +made, and the page shows its age. The page's **Plan again** asks for a fresh plan. Over the API, queue a plan, then read it until `result_json` is set. `fresh` is required: `true` plans again even when the location kept a plan for the same command. diff --git a/website/src/content/docs/docs/cli/server.mdx b/website/src/content/docs/docs/cli/server.mdx index 1be6f9b93f..2cea042e05 100644 --- a/website/src/content/docs/docs/cli/server.mdx +++ b/website/src/content/docs/docs/cli/server.mdx @@ -872,9 +872,10 @@ These stay unknown until it does: plan takes as declared. A location keeps its last 16 plans for the current publication and synced copy, in memory, and -answers an identical request with the plan it kept instead of planning again. A newer publication +answers an identical request with the plan it kept instead of planning again, for up to 10 +minutes: a plan's incremental windows and expiry follow the time it was made. A newer publication or synced copy drops them all. A kept plan's `timings` say so in `reused_from`, the time it was -made. The page's **Plan again** asks for a fresh plan. +made, and the page shows its age. The page's **Plan again** asks for a fresh plan. Over the API, queue a plan, then read it until `result_json` is set. `fresh` is required: `true` plans again even when the location kept a plan for the same command. From c28c6a2215ee9f4c00fac0479e595c61a297b9c5 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 19 Nov 2026 19:39:56 +0000 Subject: [PATCH 13/15] fix(location): plan every SQL test once only for the held project, and say how it went A preview with --vars compiles a project for its plan alone, yet it too planned every SQL test. Only a preview of the compiled project the location holds for its publication plans them all; the rest plan their selection as before. The location's log says once per publication that it planned them, or why that failed and each preview plans its own; an interrupt is raised. --- .../_helpers/plans/plan_output.rs | 29 +++++++++++++++++-- .../_helpers/plans/test_planning.rs | 2 +- .../src/planner_warehouse/models.rs | 11 ++++--- .../src/planner_warehouse/types.rs | 4 +-- .../bindings/_helpers/plan_command/plan.rs | 7 +++-- .../publication/_test_types.py | 1 + .../publication/test_plan_preview.py | 10 +++++++ 7 files changed, 52 insertions(+), 12 deletions(-) diff --git a/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/plan_output.rs b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/plan_output.rs index 112b761f16..452b3f06f4 100644 --- a/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/plan_output.rs +++ b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/plan_output.rs @@ -8,6 +8,7 @@ use crate::planner_entries::models::PlanOutputInputs; use crate::planner_resolution::main::references::resolve_function_sql::resolve_function_sql; use crate::planner_strategy::main::audit_entries::build_selected_audit_entries::build_selected_audit_entries; use crate::planner_strategy::main::model_entries::extract_seed_columns::extract_seed_columns; +use pyo3::exceptions::PyException; use pyo3::prelude::Py; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::pyfunction; @@ -15,6 +16,7 @@ use pyo3::types::PyTupleMethods; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyList, PyTuple}; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::boundary::models::KeywordArguments; +use sqlbuild_python_base::native_host::main::write_console::write_console; use sqlbuild_python_base::native_host::models::Dialect; use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; @@ -44,6 +46,9 @@ use sqlbuild_python_project::sql_test_glue::main::assemble_sql_test_plans::assem use sqlbuild_python_project::sql_test_glue::models::SqlTestAssembly; use std::sync::{Arc, OnceLock}; +/// The stream a location's log reads, told once per publication how planning every SQL test went. +const STDERR: &str = "stderr"; + const LOADER_CONSUMER: &str = "loader"; const HOOK_CONSUMER: &str = "hook"; const MATERIALIZATION_CONSUMER: &str = "custom materialization"; @@ -595,8 +600,28 @@ fn held_selected_test_entries<'py>( held.bind(py).borrow().held_sql_test_results(); if results.get().is_none() { let planned: Option> = match test_plan_results((project, dialect), &tests) { - Ok(planned) => Some(planned.unbind()), - Err(_) => None, + Ok(planned) => { + write_console( + py, + STDERR, + &format!( + "Planned the publication's {} SQL tests once for previews.\n", + tests.len() + ), + )?; + Some(planned.unbind()) + } + Err(error) if error.is_instance_of::(py) => { + write_console( + py, + STDERR, + &format!( + "Planning the publication's SQL tests once failed, so each preview plans its own: {error}\n" + ), + )?; + None + } + Err(error) => return Err(error), }; drop(results.set(planned)); } diff --git a/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/test_planning.rs b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/test_planning.rs index 65798ff518..9f98ce07ef 100644 --- a/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/test_planning.rs +++ b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/test_planning.rs @@ -231,7 +231,7 @@ pub(crate) fn start_scope_planning<'py>( dialect: dialect.unbind(), selected_keys: native_field(scope, "selected_keys")?.unbind(), sql_test_selection: native_field(scope, "sql_test_selection")?.unbind(), - held: reads.get().reads().preview(), + held: reads.get().reads().held_project(), }, enabled, ) diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/models.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/models.rs index 50c131f53c..6b08006c92 100644 --- a/crates/sqlbuild-python-planner/src/planner_warehouse/models.rs +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/models.rs @@ -487,15 +487,18 @@ impl ConsumptionReads for dyn WarehouseReads + '_ { #[derive(Debug)] pub struct SyncedReads { state: Arc, + /// Whether the plan uses the compiled project the location holds for its generation. + held_project: bool, unknowns: Mutex>, windows: Mutex>, } impl SyncedReads { - /// Reads over `state`, which several plans may share. - pub fn new(state: Arc) -> Self { + /// Reads over `state`, which several plans may share, for a plan of the held project or not. + pub fn new(state: Arc, held_project: bool) -> Self { Self { state, + held_project, unknowns: Mutex::new(BTreeSet::new()), windows: Mutex::new(BTreeMap::new()), } @@ -568,8 +571,8 @@ impl SyncedReads { } impl WarehouseReads for SyncedReads { - fn preview(&self) -> bool { - true + fn held_project(&self) -> bool { + self.held_project } fn relations<'py>( diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/types.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/types.rs index 6e8a17dfde..fcee649853 100644 --- a/crates/sqlbuild-python-planner/src/planner_warehouse/types.rs +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/types.rs @@ -32,8 +32,8 @@ pub trait CursorProgress<'py> { /// Everything a plan reads from the warehouse, live or from a synced state that may answer unknown. pub trait WarehouseReads: Send + Sync + std::fmt::Debug { - /// Whether these reads answer a location's preview from its synced state. - fn preview(&self) -> bool { + /// Whether the plan uses the compiled project a location holds for its published generation. + fn held_project(&self) -> bool { false } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs b/crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs index 56da07d274..da57aa5d34 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs @@ -129,7 +129,7 @@ pub(crate) enum PlanProject<'a, 'py> { /// A location preview's reads over the synced `state`, never deferring; other plans read live. fn preview<'py>( - request: &Bound<'py, PyAny>, + (request, project): (&Bound<'py, PyAny>, PlanProject<'_, 'py>), state: Option<&Arc>, ) -> PyResult>> { let py: Python<'py> = request.py(); @@ -146,7 +146,8 @@ fn preview<'py>( }, )); } - let synced: Arc = Arc::new(SyncedReads::new(Arc::clone(state))); + let held_project: bool = matches!(project, PlanProject::Held { .. }); + let synced: Arc = Arc::new(SyncedReads::new(Arc::clone(state), held_project)); let reads: Bound<'py, PlannerReads> = Bound::new(py, PlannerReads::new(synced.clone()))?; Ok(Some(SyncedPlan { synced, reads })) } @@ -225,7 +226,7 @@ fn resolve_invocation<'py>( } else { requested_load }; - let preview: Option> = preview(request, state)?; + let preview: Option> = preview((request, project), state)?; Ok(PlanInvocation { project_dir, discovered, diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py index fe3b1a3fbd..3b0cc10f74 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py @@ -536,6 +536,7 @@ class HeldPlanTestCase: expected_runs: tuple[tuple[str, ...], ...] expected_compiles: int expected_held_plans: int + expected_held_test_plannings: int expected_reused: tuple[bool, ...] diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_plan_preview.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_plan_preview.py index f1fd970c08..4f92c5c88c 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_plan_preview.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_plan_preview.py @@ -145,6 +145,12 @@ "recorded migrations", ] _HELD: str = "Planning from the published compiled project." +_HELD_TESTS: str = "Planned the publication's 1 SQL tests once for previews." +_REFUNDS_TEST: str = ( + "TEST ();\n\nWITH\n" + "__source__raw_refunds AS (SELECT 1 AS refund_id, 100 AS amount_cents),\n" + "__expected__refunds AS (SELECT 1 AS refund_id, 100 AS amount_cents)\n" +) _SYNCED_READING: str = "Reading the synced warehouse state..." _REPLAYED: str = ( "cursor window of daily_orders (the run starts it away from the last build's cursor)" @@ -468,6 +474,7 @@ def changed_build() -> dict[str, list[str]]: expected_runs=((), ("model:order_count",), (), ()), expected_compiles=1, expected_held_plans=3, + expected_held_test_plannings=1, expected_reused=(False, False, False, True), ), ], @@ -482,6 +489,7 @@ def test_given_plan_requests_when_the_location_plans_them_then_only_variables_co "models/order_count.sql": _PROJECT_FILES["models/order_count.sql"].replace( "AS order_count", "AS order_count, @@batch AS batch" ), + "tests/unit/test_refunds.sql": _REFUNDS_TEST, } write_files(project_dir=project, files=files) built: subprocess.CompletedProcess[str] = run_sqb(project, "build", "--target", "development") @@ -497,6 +505,8 @@ def test_given_plan_requests_when_the_location_plans_them_then_only_variables_co assert tuple("reused_from" in plan["timings"] for plan in plans) == test_case.expected_reused assert log.count("Compiling project...") == test_case.expected_compiles, log assert log.count(_HELD) == test_case.expected_held_plans, log + # Only the published project plans every SQL test, once; the plan with variables plans its own. + assert log.count(_HELD_TESTS) == test_case.expected_held_test_plannings, log assert log.count(_SYNCED_READING) == test_case.expected_held_plans + 1, log assert "Warehouse connected" not in log, "a synced copy opens no warehouse connection" From 9da04467d131303a8778c0584a7801d4321f04c8 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 19 Nov 2026 19:46:36 +0000 Subject: [PATCH 14/15] test(ui): give the reused-plan summary the section counts a server sends --- ui/src/lib/api/http-client.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/src/lib/api/http-client.test.ts b/ui/src/lib/api/http-client.test.ts index b29ed14774..aa15550f12 100644 --- a/ui/src/lib/api/http-client.test.ts +++ b/ui/src/lib/api/http-client.test.ts @@ -989,7 +989,7 @@ describe('planning a build', () => { vi.fn(async (request: Request) => request.method === 'POST' ? Response.json({ applied: true }) - : Response.json({ expires_at: '2999-01-01T00:00:00Z', outcome: { kind: 'plan', document: { selected_count: 1, models: [] }, diffs: {} }, timings }) + : Response.json({ expires_at: '2999-01-01T00:00:00Z', outcome: { kind: 'plan', document: { selected_count: 1, models: [], sections: [], migration_steps: 0 }, diffs: {} }, timings }) ) ); const client = createHttpClient({ baseUrl: 'http://ui.test', onUnauthorized: () => {} }); From cac70b1634cdc3f0d57ed6e87ddea4ae046d4ed4 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 19 Nov 2026 20:14:07 +0000 Subject: [PATCH 15/15] test: count only polling waits in the idle dispatcher test, and drop a stale host root The idle dispatcher test counted every Condition wait on other threads, so a busy runner's unrelated waits failed it; it now counts only waits shorter than the idle safety timeout, which polling would add. migration_relation_for_location is no longer a native host entry, so it is no longer a dead-code root; Python code still uses it. --- fensu.toml | 5 ----- .../runtime/event_exporting/classes/_test_types.py | 1 + .../runtime/event_exporting/classes/test_dispatcher.py | 7 ++++++- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/fensu.toml b/fensu.toml index 45cd1f8b9e..01ae72e19a 100644 --- a/fensu.toml +++ b/fensu.toml @@ -467,11 +467,6 @@ modules = ["sqlbuild.executor.load.models"] symbols = ["LoadCallbacks", "LoadExecutionResult", "LoadRuntimeParams"] reason = "Native code resolves these through the sqlbuild._native_host name table." -[[targets.python.dead_code.roots]] -modules = ["sqlbuild.executor.migrations._helpers.relation_for_location"] -symbols = ["migration_relation_for_location"] -reason = "Native code resolves these through the sqlbuild._native_host name table." - [[targets.python.dead_code.roots]] modules = ["sqlbuild.executor.old_name_views._helpers.janitor"] symbols = ["is_compatibility_view"] diff --git a/tests/unit/src/sqlbuild/runtime/event_exporting/classes/_test_types.py b/tests/unit/src/sqlbuild/runtime/event_exporting/classes/_test_types.py index 70c0e22891..025c76a5ad 100644 --- a/tests/unit/src/sqlbuild/runtime/event_exporting/classes/_test_types.py +++ b/tests/unit/src/sqlbuild/runtime/event_exporting/classes/_test_types.py @@ -5,6 +5,7 @@ class IdleDispatcherTestCase: description: str idle_seconds: float + polling_seconds: float expected_max_waits: int expected_max_delivery_seconds: float diff --git a/tests/unit/src/sqlbuild/runtime/event_exporting/classes/test_dispatcher.py b/tests/unit/src/sqlbuild/runtime/event_exporting/classes/test_dispatcher.py index fb20a5471a..f3d6c4c8f2 100644 --- a/tests/unit/src/sqlbuild/runtime/event_exporting/classes/test_dispatcher.py +++ b/tests/unit/src/sqlbuild/runtime/event_exporting/classes/test_dispatcher.py @@ -130,6 +130,7 @@ def publish(*, event: LifecycleEvent) -> None: IdleDispatcherTestCase( description="an idle dispatcher waits for work instead of polling", idle_seconds=0.5, + polling_seconds=0.5, expected_max_waits=10, expected_max_delivery_seconds=0.5, ), @@ -165,7 +166,11 @@ def publish(*, event: LifecycleEvent) -> None: time.sleep(0.05) waits.clear() time.sleep(test_case.idle_seconds) - idle_waits: int = sum(thread != caller_thread for thread, _ in waits) + # Other threads of a busy process wait too; only waits shorter than the idle safety timeout poll. + idle_waits: int = sum( + thread != caller_thread and timeout is not None and timeout < test_case.polling_seconds + for thread, timeout in waits + ) started: float = time.monotonic() dispatcher.enqueue(lifecycle_event())