diff --git a/clients/typescript/src/schema.d.ts b/clients/typescript/src/schema.d.ts index 63288c6dbc..238a3410c8 100644 --- a/clients/typescript/src/schema.d.ts +++ b/clients/typescript/src/schema.d.ts @@ -2311,6 +2311,8 @@ export interface components { kind: "evaluate_sensor"; } | { command: string[]; + /** @description Plan again even when the location kept a plan for the same request. */ + fresh: boolean; /** @enum {string} */ kind: "plan"; selection: string[]; @@ -2812,6 +2814,8 @@ export interface components { /** @description A person's build command to plan on the location; the server freezes it like an ad-hoc run. */ PlanRequest: { command: string[]; + /** @description Plan again instead of returning the plan the location kept for the same request. */ + fresh: boolean; /** Format: uuid */ request_id: string; }; @@ -2846,6 +2850,11 @@ export interface components { PlanTimings: { /** @description `synced_state` for a preview over the synced warehouse state; `live` when it read the warehouse. */ planned_from: string; + /** + * Format: date-time + * @description When the plan the location kept, and answered with again, was made; `None` for a new plan. + */ + reused_from?: string | null; steps: components["schemas"]["PlanTimingStep"][]; /** Format: double */ total_seconds: number; diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/location/mod.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/location/mod.rs index dc88ad9111..ad8a09edea 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/location/mod.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/location/mod.rs @@ -3,6 +3,7 @@ pub(crate) mod execution; pub(crate) mod held_copy; pub(crate) mod plan_document; pub(crate) mod plan_request; +pub(crate) mod plan_reuse; pub(crate) mod sync_outcomes; pub(crate) mod sync_report; pub(crate) mod work; diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/location/plan_reuse.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/location/plan_reuse.rs new file mode 100644 index 0000000000..50fe2ce4e7 --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/location/plan_reuse.rs @@ -0,0 +1,128 @@ +//! The location's last finished plans, in memory, returned again for an identical request. + +use std::collections::{BTreeMap, VecDeque}; +use std::sync::{Arc, Mutex, PoisonError}; + +use chrono::{DateTime, SecondsFormat, Utc}; +use serde_json::{Value, json}; +use sqlbuild_orchestration::platform::models::PlanModelDiff; + +use crate::argv::models::Stream; +use crate::native_commands::_helpers::command::terminal::write_output; +use crate::native_commands::constants::{REUSED_PLAN_AGE, REUSED_PLAN_BYTES, REUSED_PLANS}; +use crate::native_commands::models::{PlanKey, PlanReuse, PlanToKeep, ReusedPlan}; +use crate::native_commands::types::LocationPlan; + +/// The location's kept plans; one location serves one target. +pub(crate) static PLAN_REUSE: Mutex> = Mutex::new(None); + +/// The kept plan for `key` under this generation and copy, if younger than the reuse age at `now`. +pub(crate) fn reused_plan( + reuse: &Mutex>, + (generation, version): (i64, &str), + key: &PlanKey, + now: DateTime, +) -> Option { + let kept: Arc = { + let mut reuse = reuse.lock().unwrap_or_else(PoisonError::into_inner); + let current: &mut PlanReuse = reuse + .as_mut() + .filter(|reuse| reuse.generation == generation && reuse.version == version)?; + while current + .plans + .front() + .is_some_and(|plan| now - plan.planned_at >= REUSED_PLAN_AGE) + { + if let Some(expired) = current.plans.pop_front() { + current.bytes -= plan_bytes(&expired); + } + } + Arc::clone(current.plans.iter().find(|plan| plan.key == *key)?) + }; + let read = serde_json::from_str::(&kept.document).and_then(|document| { + serde_json::from_str::>(&kept.diffs) + .map(|diffs| (document, diffs)) + }); + let (mut document, diffs) = match read { + Ok(read) => read, + Err(error) => { + drop(write_output(&[( + Stream::Stderr, + &format!("A kept plan could not be read, so the location plans again: {error}\n"), + )])); + return None; + } + }; + document["timings"]["reused_from"] = + json!(kept.planned_at.to_rfc3339_opts(SecondsFormat::Secs, true)); + Some((document, diffs)) +} + +/// Keep a checked, successful plan within `(plans, bytes)`, oldest out first; a newer generation or copy replaces all. +pub(crate) fn remember_plan( + (reuse, (most_plans, most_bytes)): (&Mutex>, (usize, usize)), + keep: PlanToKeep, + (document, diffs): (&Value, &BTreeMap), +) { + if !document["error"].is_null() { + return; + } + let (Ok(text), Ok(diff_text)) = ( + serde_json::to_string(document), + serde_json::to_string(diffs), + ) else { + return; + }; + let kept = Arc::new(ReusedPlan { + key: keep.key, + document: text, + diffs: diff_text, + planned_at: keep.planned_at, + }); + if plan_bytes(&kept) > most_bytes { + return; + } + let mut reuse = reuse.lock().unwrap_or_else(PoisonError::into_inner); + let current: &mut PlanReuse = match reuse.as_mut() { + Some(current) + if current.generation == keep.generation && current.version == keep.version => + { + current + } + _ => reuse.insert(PlanReuse { + generation: keep.generation, + version: keep.version, + plans: VecDeque::new(), + bytes: 0, + }), + }; + let replaced: usize = current + .plans + .iter() + .filter(|plan| plan.key == kept.key) + .map(|plan| plan_bytes(plan)) + .sum(); + current.plans.retain(|plan| plan.key != kept.key); + current.bytes = current.bytes - replaced + plan_bytes(&kept); + current.plans.push_back(kept); + while current.plans.len() > most_plans || current.bytes > most_bytes { + if let Some(dropped) = current.plans.pop_front() { + current.bytes -= plan_bytes(&dropped); + } + } +} + +/// Keep a plan for identical requests once the checks after planning passed. +pub(crate) fn keep_plan( + keep: Option, + plan: (&Value, &BTreeMap), +) { + if let Some(keep) = keep { + remember_plan((&PLAN_REUSE, (REUSED_PLANS, REUSED_PLAN_BYTES)), keep, plan); + } +} + +/// The bytes a kept plan's text holds. +fn plan_bytes(plan: &ReusedPlan) -> usize { + plan.document.len() + plan.diffs.len() +} diff --git a/crates/sqlbuild-cli/src/native_commands/constants.rs b/crates/sqlbuild-cli/src/native_commands/constants.rs index caf2ead03a..8f89b1fe35 100644 --- a/crates/sqlbuild-cli/src/native_commands/constants.rs +++ b/crates/sqlbuild-cli/src/native_commands/constants.rs @@ -218,6 +218,15 @@ pub(crate) const CHANGED_BUILD_COMMAND: [&str; 2] = ["build", "--changed"]; /// How long an idle location waits between checks for named work; after work it checks at once. pub(crate) const LOCATION_WORK_IDLE_POLL: std::time::Duration = std::time::Duration::from_secs(2); +/// Finished plans a location keeps for identical requests; a newer generation or synced copy drops them. +pub(crate) const REUSED_PLANS: usize = 16; + +/// The most bytes of plan text a location keeps for reuse; the oldest plans go first. +pub(crate) const REUSED_PLAN_BYTES: usize = 256 * 1024 * 1024; + +/// How long a kept plan may answer a request: its windows and expiry follow the time it planned at. +pub(crate) const REUSED_PLAN_AGE: chrono::TimeDelta = chrono::TimeDelta::minutes(10); + /// What a location reports when `sqb plan` produced no plan. pub(crate) const PLAN_NOT_PRODUCED: &str = "sqb plan did not produce a plan; see the location's log"; diff --git a/crates/sqlbuild-cli/src/native_commands/main/_location_changed_build.rs b/crates/sqlbuild-cli/src/native_commands/main/_location_changed_build.rs index 02ccfba216..2901c1bbae 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/_location_changed_build.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/_location_changed_build.rs @@ -16,6 +16,7 @@ use crate::native_commands::_helpers::command::terminal::write_output; use crate::native_commands::_helpers::location::changed_build::{ changed_build_nodes, changed_build_settings, }; +use crate::native_commands::_helpers::location::plan_reuse::keep_plan; use crate::native_commands::main::_location_plan::plan_location; use crate::native_commands::models::{LocationRuntimeRequest, LocationServe, LocationWatch}; @@ -44,9 +45,9 @@ pub(super) async fn report_changed_build( } }; report("Planning what changed from the synced warehouse copy...\n"); - let Some((document, _)) = plan_location( + let Some(((document, diffs), keep)) = plan_location( (options, runtime), - (&BTreeSet::new(), &settings), + (&BTreeSet::new(), &settings, false), (client, session_id, generation), stopped, ) @@ -58,6 +59,7 @@ pub(super) async fn report_changed_build( report("Project files changed while planning what changed; not reported.\n"); return Ok(Some(false)); } + keep_plan(keep, (&document, &diffs)); if let Some(reason) = document["error"]["message"].as_str() { report(&format!("Could not plan what changed: {reason}\n")); let request = ChangedBuildReport { diff --git a/crates/sqlbuild-cli/src/native_commands/main/_location_plan.rs b/crates/sqlbuild-cli/src/native_commands/main/_location_plan.rs index 0ba6251feb..4443cf2113 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/_location_plan.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/_location_plan.rs @@ -3,6 +3,7 @@ use std::collections::{BTreeMap, BTreeSet}; use std::future::Future; +use chrono::{DateTime, Utc}; use serde_json::{Value, json}; use sqlbuild_orchestration::platform::models::LocationWorkClient; use sqlbuild_orchestration::store::errors::ControlError; @@ -15,21 +16,24 @@ use crate::native_commands::_helpers::location::held_copy::{HELD_COPY, held_copy use crate::native_commands::_helpers::location::plan_request::{ plan_arguments, run_plan, selected_names, }; -use crate::native_commands::models::{HeldCopy, LocationRuntimeRequest, LocationServe}; +use crate::native_commands::_helpers::location::plan_reuse::{PLAN_REUSE, reused_plan}; +use crate::native_commands::models::{ + HeldCopy, LocationRuntimeRequest, LocationServe, PlanKey, PlanToKeep, +}; use crate::native_commands::types::LocationPlan; -/// The plan document with the diffs it left out, an `{"error": ...}` report, or `None` when stopped first. +/// The plan with what to keep once its checks pass, an `{"error": ...}` report, or `None` when stopped first. pub(super) async fn plan_location( (options, runtime): (&LocationServe, &mpsc::Sender), - (selection, settings): (&BTreeSet, &BTreeMap), + (selection, settings, fresh): (&BTreeSet, &BTreeMap, bool), (client, session_id, generation): (&LocationWorkClient, Uuid, i64), stopped: impl Future>, -) -> Result, ControlError> { +) -> Result)>, ControlError> { let (argv, select) = match plan_arguments(options, settings) .and_then(|argv| Ok((argv, selected_names(selection)?))) { Ok(planned) => planned, - Err(message) => return Ok(Some(without_diffs(plan_failure(&message)))), + Err(message) => return Ok(Some((without_diffs(plan_failure(&message)), None))), }; let consumption: bool = settings .get("changed") @@ -45,15 +49,34 @@ pub(super) async fn plan_location( let copy = match client.preview_copy(&options.target, &read).await { Ok(copy) => copy, Err(error) => { - return Ok(Some(without_diffs(plan_failure(&format!( - "the synced warehouse copy is unavailable: {error}" - ))))); + return Ok(Some(( + without_diffs(plan_failure(&format!( + "the synced warehouse copy is unavailable: {error}" + ))), + None, + ))); } }; let planned: HeldCopy = match held { Some(held) if copy.unchanged => held, _ => hold_copy(&HELD_COPY, options, generation, (copy, read_consumption))?, }; + let key: PlanKey = PlanKey { + argv: argv.clone(), + select: select.clone(), + consumption: planned.consumption, + }; + let planned_at: DateTime = Utc::now(); + if !fresh + && let Some(reused) = reused_plan( + &PLAN_REUSE, + (generation, &planned.version), + &key, + planned_at, + ) + { + return Ok(Some((reused, None))); + } let request = json!({"plan": { "argv": argv, "select": select, @@ -61,5 +84,13 @@ pub(super) async fn plan_location( "version": planned.version, "consumption": planned.consumption, }}); - run_plan(runtime, request, stopped).await + let keep = PlanToKeep { + generation, + version: planned.version, + key, + planned_at, + }; + Ok(run_plan(runtime, request, stopped) + .await? + .map(|plan| (plan, Some(keep)))) } diff --git a/crates/sqlbuild-cli/src/native_commands/main/_location_work.rs b/crates/sqlbuild-cli/src/native_commands/main/_location_work.rs index c90cf0c516..32fbf8b992 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/_location_work.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/_location_work.rs @@ -18,11 +18,12 @@ use crate::argv::models::Stream; use crate::native_commands::_helpers::command::paths::location_files; use crate::native_commands::_helpers::command::terminal::write_output; use crate::native_commands::_helpers::location::execution::check_location_connection; +use crate::native_commands::_helpers::location::plan_reuse::keep_plan; use crate::native_commands::_helpers::location::work::work_error; use crate::native_commands::main::_location_plan::plan_location; use crate::native_commands::main::_location_sensor::evaluate_location_sensor; use crate::native_commands::models::{ - LocationRuntimeRequest, LocationServe, LocationWatch, LocationWorkTurn, + LocationRuntimeRequest, LocationServe, LocationWatch, LocationWorkTurn, PlanToKeep, }; /// Retry the exact completed result without rerunning it; a turn's `sync` asks the metadata loop to sync. @@ -74,7 +75,7 @@ pub(super) async fn run_location_work( expires, ) .await?; - let Some((outcome, local_files)) = checked else { + let Some((outcome, local_files, keep)) = checked else { drop(write_output(&[( Stream::Stderr, "Location work stopped before completion; no result reported.\n", @@ -93,6 +94,9 @@ pub(super) async fn run_location_work( let mut excluded = watch.local_files.clone(); excluded.extend(files.clone()); if location_files(&options.project_dir, &watch.rules, &excluded) == expected.0 { + if let LocationWorkOutcome::Plan { document, diffs } = &outcome { + keep_plan(keep, (document, diffs)); + } outcome } else { LocationWorkOutcome::Superseded {} @@ -147,7 +151,7 @@ pub(super) async fn run_location_work( })) } -/// One named operation; `None` means it stopped at shutdown or its lease ended; `sqb dev` never signs in here. +/// One named operation and a plan to keep once checked; `None` when it stopped at shutdown or its lease ended. async fn perform_work( (options, script, (runtime, connected)): ( &LocationServe, @@ -157,7 +161,7 @@ async fn perform_work( work: LocationWork, copy: (&LocationWorkClient, Uuid, i64), expires: impl Future>, -) -> Result)>, ControlError> { +) -> Result, Option)>, ControlError> { let message = match &work { LocationWork::CheckConnection {} => "Checking requested location connection...\n", LocationWork::SyncWarehouse {} => "Synchronizing warehouse history on request...\n", @@ -166,11 +170,15 @@ async fn perform_work( }; drop(write_output(&[(Stream::Stderr, message)])); Ok(match work { - LocationWork::SyncWarehouse {} => Some((LocationWorkOutcome::SyncStarted {}, Vec::new())), + LocationWork::SyncWarehouse {} => { + Some((LocationWorkOutcome::SyncStarted {}, Vec::new(), None)) + } LocationWork::CheckConnection {} => { check_location_connection((options, script, connected), expires) .await? - .map(|(healthy, _, files)| (LocationWorkOutcome::Connection { healthy }, files)) + .map(|(healthy, _, files)| { + (LocationWorkOutcome::Connection { healthy }, files, None) + }) } LocationWork::EvaluateSensor { evaluation } => { evaluate_location_sensor(options, script, &evaluation, expires) @@ -179,15 +187,28 @@ async fn perform_work( ( LocationWorkOutcome::SensorEvaluation { outcome }, Vec::new(), + None, ) }) } LocationWork::Plan { selection, settings, + fresh, .. - } => plan_location((options, runtime), (&selection, &settings), copy, expires) - .await? - .map(|(document, diffs)| (LocationWorkOutcome::Plan { document, diffs }, Vec::new())), + } => plan_location( + (options, runtime), + (&selection, &settings, fresh), + copy, + expires, + ) + .await? + .map(|((document, diffs), keep)| { + ( + LocationWorkOutcome::Plan { document, diffs }, + Vec::new(), + keep, + ) + }), }) } diff --git a/crates/sqlbuild-cli/src/native_commands/models.rs b/crates/sqlbuild-cli/src/native_commands/models.rs index b7ef802894..46655c1562 100644 --- a/crates/sqlbuild-cli/src/native_commands/models.rs +++ b/crates/sqlbuild-cli/src/native_commands/models.rs @@ -565,6 +565,45 @@ pub(crate) enum LostStdout { Raised(String), } +/// What makes two plan requests identical once the generation and synced copy are the same. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct PlanKey { + /// The `sqb plan` arguments the location built from the request's settings. + pub(crate) argv: Vec, + /// The node names the server froze from the request's selection. + pub(crate) select: Vec, + /// Whether the plan read the copy's consumption. + pub(crate) consumption: bool, +} + +/// One finished plan, kept as the JSON text of its document and of the diffs beside it. +#[derive(Debug)] +pub(crate) struct ReusedPlan { + pub(crate) key: PlanKey, + pub(crate) document: String, + pub(crate) diffs: String, + /// When the plan started, which its windows and expiry follow. + pub(crate) planned_at: chrono::DateTime, +} + +/// The finished plans of one published generation and synced copy, oldest first, and their bytes. +#[derive(Debug)] +pub(crate) struct PlanReuse { + pub(crate) generation: i64, + pub(crate) version: String, + pub(crate) plans: std::collections::VecDeque>, + pub(crate) bytes: usize, +} + +/// A plan the location may keep once its checks pass: its generation, copy, key and start time. +#[derive(Clone, Debug)] +pub(crate) struct PlanToKeep { + pub(crate) generation: i64, + pub(crate) version: String, + pub(crate) key: PlanKey, + pub(crate) planned_at: chrono::DateTime, +} + /// The copy file the location last read, the mirror version it holds and whether it has consumption. #[derive(Clone, Debug)] pub(crate) struct HeldCopy { diff --git a/crates/sqlbuild-cli/src/native_commands/tests/helpers.rs b/crates/sqlbuild-cli/src/native_commands/tests/helpers.rs index ec83c94d0a..775f6eb238 100644 --- a/crates/sqlbuild-cli/src/native_commands/tests/helpers.rs +++ b/crates/sqlbuild-cli/src/native_commands/tests/helpers.rs @@ -1,4 +1,8 @@ use std::path::Path; +use std::sync::Mutex; + +use chrono::{DateTime, TimeZone, Utc}; +use serde_json::json; use sqlbuild_orchestration::warehouse_metadata::models::{MetadataBatch, WarehouseStateLocation}; use uuid::Uuid; @@ -10,7 +14,11 @@ use crate::native_commands::_helpers::command::paths::location_watch; use crate::native_commands::_helpers::location::execution::{ acknowledge_metadata_page, settle_publication, }; -use crate::native_commands::models::{LocationMetadataPage, LocationMetadataSync, LocationServe}; +use crate::native_commands::_helpers::location::plan_reuse::remember_plan; +use crate::native_commands::models::{ + LocationMetadataPage, LocationMetadataSync, LocationServe, PlanKey, PlanReuse, PlanToKeep, +}; +use crate::native_commands::types::LocationPlan; use sqlbuild_orchestration::platform::main::work::connect_location_work::connect_location_work; use sqlbuild_orchestration::platform::models::{ ConnectionCheckTrigger, LocationWorkClient, LocationWorkPoll, @@ -132,3 +140,63 @@ pub(super) fn published_location( state.idle = false; ((options, script), (client, poll), state) } + +/// The plan key of a `--changed` plan of `select`. +pub(super) fn reuse_key(select: &[&str]) -> PlanKey { + PlanKey { + argv: vec!["plan".into(), "--json".into(), "--changed".into()], + select: select.iter().map(|name| (*name).to_owned()).collect(), + consumption: true, + } +} + +/// A finished plan document, or a failed plan's error report. +pub(super) fn finished_plan(failed: bool) -> LocationPlan { + let documents = [ + json!({ + "models": [{"name": "orders"}], + "timings": {"planned_from": "synced_state", "total_seconds": 4.2, + "steps": [{"name": "Detecting changes", "seconds": 4.2}]}, + }), + json!({"error": {"code": "plan_failed", "message": "no plan"}}), + ]; + (documents[usize::from(failed)].clone(), Default::default()) +} + +/// The bytes `finished_plan(false)` takes when kept: its document's and diffs' JSON text. +pub(super) fn finished_plan_bytes() -> usize { + let (document, diffs) = finished_plan(false); + serde_json::to_string(&document).expect("document").len() + + serde_json::to_string(&diffs).expect("diffs").len() +} + +/// The time the reuse tests plan at. +pub(super) fn reuse_clock() -> DateTime { + Utc.with_ymd_and_hms(2026, 1, 2, 3, 4, 5) + .single() + .expect("time") +} + +/// A plan of `select` to keep under `generation` and `version`, started at `reuse_clock`. +pub(super) fn plan_to_keep(generation: i64, version: &str, select: &[&str]) -> PlanToKeep { + PlanToKeep { + generation, + version: version.to_owned(), + key: reuse_key(select), + planned_at: reuse_clock(), + } +} + +/// `count` finished plans of `model_` under generation 1 and `copy-1`, oldest first, within `limits`. +pub(super) fn kept_plans(count: usize, limits: (usize, usize)) -> Mutex> { + let reuse: Mutex> = Mutex::new(None); + let (document, diffs) = finished_plan(false); + for index in 0..count { + remember_plan( + (&reuse, limits), + plan_to_keep(1, "copy-1", &[format!("model_{index}").as_str()]), + (&document, &diffs), + ); + } + reuse +} diff --git a/crates/sqlbuild-cli/src/native_commands/tests/test_location_plan.rs b/crates/sqlbuild-cli/src/native_commands/tests/test_location_plan.rs index 7059286714..2f0c5f3083 100644 --- a/crates/sqlbuild-cli/src/native_commands/tests/test_location_plan.rs +++ b/crates/sqlbuild-cli/src/native_commands/tests/test_location_plan.rs @@ -1,12 +1,27 @@ use serde_json::{Value, json}; use sqlbuild_orchestration::platform::constants::PLAN_DOCUMENT_BYTES; +use std::sync::Mutex; + +use chrono::TimeDelta; + use crate::native_commands::_helpers::location::plan_document::plan_document; +use crate::native_commands::_helpers::location::plan_request::plan_arguments; +use crate::native_commands::_helpers::location::plan_reuse::{remember_plan, reused_plan}; use crate::native_commands::_helpers::location::work::next_work_check; use crate::native_commands::constants::LOCATION_WORK_IDLE_POLL; +use crate::native_commands::constants::{REUSED_PLAN_BYTES, REUSED_PLANS}; +use crate::native_commands::models::LocationServe; use crate::native_commands::models::LocationWorkTurn; -use crate::native_commands::tests::helpers::plan_output; -use crate::native_commands::tests::test_types::{PlanDocumentTestCase, WorkCheckTestCase}; +use crate::native_commands::models::PlanReuse; +use crate::native_commands::tests::helpers::{ + finished_plan, finished_plan_bytes, kept_plans, plan_output, plan_to_keep, reuse_clock, + reuse_key, +}; +use crate::native_commands::tests::test_types::{ + PlanArgumentsTestCase, PlanDocumentTestCase, PlanReuseBoundTestCase, PlanReuseTestCase, + WorkCheckTestCase, +}; #[test] fn given_plan_output_when_reading_it_on_a_location_then_diffs_always_travel_beside_the_document() { @@ -103,3 +118,192 @@ fn given_a_turn_of_location_work_when_scheduling_the_next_check_then_only_an_idl ); } } + +#[test] +fn given_a_kept_plan_when_the_same_request_comes_again_then_only_the_same_generation_copy_and_request_reuse_it_for_ten_minutes() + { + let reused_timings = json!({"planned_from": "synced_state", "reused_from": "2026-01-02T03:04:05Z", + "total_seconds": 4.2, "steps": [{"name": "Detecting changes", "seconds": 4.2}]}); + let test_cases = [ + PlanReuseTestCase { + description: "the same generation, copy and request reuse the plan and say when it was made", + kept: (3, "copy-1", false), + asked: (3, "copy-1", &["orders"]), + age: TimeDelta::seconds(599), + expected_timings: Some(reused_timings), + }, + PlanReuseTestCase { + description: "a plan ten minutes old is not reused, so windows and expiry follow the clock", + kept: (3, "copy-1", false), + asked: (3, "copy-1", &["orders"]), + age: TimeDelta::minutes(10), + expected_timings: None, + }, + PlanReuseTestCase { + description: "a newer synced copy plans again", + kept: (3, "copy-1", false), + asked: (3, "copy-2", &["orders"]), + age: TimeDelta::zero(), + expected_timings: None, + }, + PlanReuseTestCase { + description: "a newer generation plans again", + kept: (3, "copy-1", false), + asked: (4, "copy-1", &["orders"]), + age: TimeDelta::zero(), + expected_timings: None, + }, + PlanReuseTestCase { + description: "another selection plans again", + kept: (3, "copy-1", false), + asked: (3, "copy-1", &["customers"]), + age: TimeDelta::zero(), + expected_timings: None, + }, + PlanReuseTestCase { + description: "a failed plan is not kept", + kept: (3, "copy-1", true), + asked: (3, "copy-1", &["orders"]), + age: TimeDelta::zero(), + expected_timings: None, + }, + ]; + for test_case in test_cases { + let reuse: Mutex> = Mutex::new(None); + let (generation, version, failed) = test_case.kept; + let (document, diffs) = finished_plan(failed); + remember_plan( + (&reuse, (REUSED_PLANS, REUSED_PLAN_BYTES)), + plan_to_keep(generation, version, &["orders"]), + (&document, &diffs), + ); + let (generation, version, select) = test_case.asked; + let reused = reused_plan( + &reuse, + (generation, version), + &reuse_key(select), + reuse_clock() + test_case.age, + ); + assert_eq!( + reused.map(|(document, _)| document["timings"].clone()), + test_case.expected_timings, + "{description}", + description = test_case.description + ); + } +} + +#[test] +fn given_more_finished_plans_than_kept_when_planning_then_the_oldest_is_dropped_and_a_newer_copy_drops_all() + { + let plan_bytes: usize = finished_plan_bytes(); + let test_cases = [ + PlanReuseBoundTestCase { + description: "the oldest of one plan too many is dropped", + limits: (REUSED_PLANS, REUSED_PLAN_BYTES, REUSED_PLANS + 1), + newer_copy: false, + asked: ("model_0", "copy-1"), + expected_reused: false, + }, + PlanReuseBoundTestCase { + description: "the next oldest is kept", + limits: (REUSED_PLANS, REUSED_PLAN_BYTES, REUSED_PLANS + 1), + newer_copy: false, + asked: ("model_1", "copy-1"), + expected_reused: true, + }, + PlanReuseBoundTestCase { + description: "past the byte limit the oldest plan is dropped", + limits: (REUSED_PLANS, 2 * plan_bytes, 3), + newer_copy: false, + asked: ("model_0", "copy-1"), + expected_reused: false, + }, + PlanReuseBoundTestCase { + description: "the plans within the byte limit are kept", + limits: (REUSED_PLANS, 2 * plan_bytes, 3), + newer_copy: false, + asked: ("model_1", "copy-1"), + expected_reused: true, + }, + PlanReuseBoundTestCase { + description: "a plan larger than the byte limit is not kept", + limits: (REUSED_PLANS, plan_bytes - 1, 1), + newer_copy: false, + asked: ("model_0", "copy-1"), + expected_reused: false, + }, + PlanReuseBoundTestCase { + description: "a newer synced copy drops every kept plan", + limits: (REUSED_PLANS, REUSED_PLAN_BYTES, REUSED_PLANS + 1), + newer_copy: true, + asked: ("model_1", "copy-2"), + expected_reused: false, + }, + PlanReuseBoundTestCase { + description: "a plan of the newer copy is kept", + limits: (REUSED_PLANS, REUSED_PLAN_BYTES, REUSED_PLANS + 1), + newer_copy: true, + asked: ("model_new", "copy-2"), + expected_reused: true, + }, + ]; + let (document, diffs) = finished_plan(false); + for test_case in test_cases { + let (most_plans, most_bytes, count) = test_case.limits; + let reuse: Mutex> = kept_plans(count, (most_plans, most_bytes)); + let newer: &[(&str, &str)] = + [&[][..], &[("model_new", "copy-2")][..]][usize::from(test_case.newer_copy)]; + for &(name, version) in newer { + remember_plan( + (&reuse, (most_plans, most_bytes)), + plan_to_keep(1, version, &[name]), + (&document, &diffs), + ); + } + let (name, version) = test_case.asked; + assert_eq!( + reused_plan(&reuse, (1, version), &reuse_key(&[name]), reuse_clock()).is_some(), + test_case.expected_reused, + "{description}", + description = test_case.description + ); + } +} + +#[test] +fn given_frozen_plan_settings_when_building_the_plan_arguments_then_run_request_settings_never_vary_a_kept_plan() + { + let test_cases = [ + PlanArgumentsTestCase { + description: "build settings become the plan arguments the kept plan is keyed on", + settings: json!({"changed": true}), + expected_refused: false, + }, + PlanArgumentsTestCase { + description: "--no-job-settings belongs to the run request and never reaches a plan", + settings: json!({"changed": true, "job_settings": false}), + expected_refused: true, + }, + ]; + let options = LocationServe { + project_dir: "/project".into(), + server_url: "http://127.0.0.1:1".into(), + location_id: uuid::Uuid::nil(), + target: "development".into(), + image_id: None, + metadata_reconcile_seconds: 3600, + safety_sync_seconds: 0, + dev_events: false, + }; + for test_case in test_cases { + let settings = serde_json::from_value(test_case.settings).expect("settings"); + let argv: Result, String> = plan_arguments(&options, &settings); + assert_eq!( + argv.is_err(), + test_case.expected_refused, + "{description}: {argv:?}", + description = test_case.description + ); + } +} diff --git a/crates/sqlbuild-cli/src/native_commands/tests/test_types.rs b/crates/sqlbuild-cli/src/native_commands/tests/test_types.rs index 2bf84f524f..1d42d8c8d6 100644 --- a/crates/sqlbuild-cli/src/native_commands/tests/test_types.rs +++ b/crates/sqlbuild-cli/src/native_commands/tests/test_types.rs @@ -90,6 +90,38 @@ pub(super) struct InitFailureTestCase { pub(super) expected_exit_code: i32, } +/// The frozen settings of one plan request. +pub(super) struct PlanArgumentsTestCase { + pub(super) description: &'static str, + pub(super) settings: serde_json::Value, + /// Whether the location refuses the settings instead of planning, and keeping, them. + pub(super) expected_refused: bool, +} + +/// One kept plan, then one request for a plan. +pub(super) struct PlanReuseTestCase { + pub(super) description: &'static str, + /// The kept plan's generation, copy version and whether it failed. + pub(super) kept: (i64, &'static str, bool), + /// The generation, copy version and selected names of the request. + pub(super) asked: (i64, &'static str, &'static [&'static str]), + /// How long after the kept plan started the request comes. + pub(super) age: chrono::TimeDelta, + /// The reused plan's timings, or `None` when the location plans again. + pub(super) expected_timings: Option, +} + +/// Plans kept within a count and byte limit, maybe a plan of a newer copy, then one request. +pub(super) struct PlanReuseBoundTestCase { + pub(super) description: &'static str, + /// The most plans and bytes kept, and how many plans are kept first. + pub(super) limits: (usize, usize, usize), + pub(super) newer_copy: bool, + /// The selected name and copy version of the request. + pub(super) asked: (&'static str, &'static str), + pub(super) expected_reused: bool, +} + /// One `sqb plan --json` output, whether it exited as a plan, and the first model the server receives. pub(super) struct PlanDocumentTestCase { pub(super) description: &'static str, diff --git a/crates/sqlbuild-host/src/entries/_helpers/executor.rs b/crates/sqlbuild-host/src/entries/_helpers/executor.rs index 17302822a0..2041eb81f9 100644 --- a/crates/sqlbuild-host/src/entries/_helpers/executor.rs +++ b/crates/sqlbuild-host/src/entries/_helpers/executor.rs @@ -309,11 +309,6 @@ pub(crate) const ENTRIES: &[HostEntry] = &[ kind: HostKind::Class, edge: HostEdge::Executor, }, - HostEntry { - name: "migration_relation_for_location", - kind: HostKind::Function, - edge: HostEdge::Executor, - }, HostEntry { name: "is_compatibility_view", kind: HostKind::Function, diff --git a/crates/sqlbuild-host/src/entries/models.rs b/crates/sqlbuild-host/src/entries/models.rs index 4c39f96c72..0c11914fac 100644 --- a/crates/sqlbuild-host/src/entries/models.rs +++ b/crates/sqlbuild-host/src/entries/models.rs @@ -268,7 +268,6 @@ pub enum ExecutorHost { LoadCallbacks, LoadExecutionResult, LoadRuntimeParams, - MigrationRelationForLocation, IsCompatibilityView, AuditExecutionError, WarehouseConnectionError, diff --git a/crates/sqlbuild-orchestration/src/platform/main/person_reads/read_person_plan_summary.rs b/crates/sqlbuild-orchestration/src/platform/main/person_reads/read_person_plan_summary.rs index ca6da1caab..805fb8c7d2 100644 --- a/crates/sqlbuild-orchestration/src/platform/main/person_reads/read_person_plan_summary.rs +++ b/crates/sqlbuild-orchestration/src/platform/main/person_reads/read_person_plan_summary.rs @@ -1,7 +1,6 @@ //! A plan request's state with its summary, which the plan page polls instead of the whole document. use serde_json::Value; -use serde_json::error::Category; use sqlbuild_core::plan_display::main::plan_summary::plan_summary; use uuid::Uuid; @@ -43,14 +42,12 @@ pub async fn read_person_plan_summary( })) } -/// The document's planning timings; absent from older locations, left out if of an unknown shape. +/// The document's planning timings; a failed plan's report has none. fn plan_timings(document: &Value) -> Result, ControlError> { let Some(timings) = document.get(PLAN_TIMINGS_KEY) else { return Ok(None); }; - match serde_json::from_value(timings.clone()) { - Ok(timings) => Ok(Some(timings)), - Err(error) if error.classify() == Category::Data => Ok(None), - Err(error) => Err(ControlError::Invalid(format!("plan timings: {error}"))), - } + serde_json::from_value(timings.clone()) + .map(Some) + .map_err(|error| ControlError::Invalid(format!("plan timings: {error}"))) } diff --git a/crates/sqlbuild-orchestration/src/platform/models.rs b/crates/sqlbuild-orchestration/src/platform/models.rs index 4bee656326..68afeb0e15 100644 --- a/crates/sqlbuild-orchestration/src/platform/models.rs +++ b/crates/sqlbuild-orchestration/src/platform/models.rs @@ -214,6 +214,8 @@ pub enum LocationWork { selection: std::collections::BTreeSet, #[schema(value_type = Object)] settings: std::collections::BTreeMap, + /// Plan again even when the location kept a plan for the same request. + fresh: bool, }, } @@ -233,6 +235,8 @@ pub struct LocationWorkRequest { pub struct PlanRequest { pub request_id: Uuid, pub command: Vec, + /// Plan again instead of returning the plan the location kept for the same request. + pub fresh: bool, } /// A person asks a location for one named read-only operation, such as a connection test. @@ -742,7 +746,7 @@ pub struct PlanSummaryRead { pub expires_at: DateTime, /// `None` until the location answers; a plan's document is its summary. pub outcome: Option, - /// How long the location took to plan, step by step; `None` for plans recorded without timings. + /// How long the location took to plan, step by step; `None` until it planned, or when planning failed. #[serde(default, skip_serializing_if = "Option::is_none")] pub timings: Option, } @@ -755,6 +759,9 @@ pub struct PlanTimings { pub steps: Vec, /// `synced_state` for a preview over the synced warehouse state; `live` when it read the warehouse. pub planned_from: String, + /// When the plan the location kept, and answered with again, was made; `None` for a new plan. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub reused_from: Option>, } /// One planning step and its wall time. diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/plan.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/plan.rs index cb0ca4ef81..bafcaff7f1 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/plan.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/plan.rs @@ -37,7 +37,8 @@ pub(in crate::runs) async fn request_plan_in( && previous.record.request_json.location_id == location_id && previous.record.request_json.target == target && matches!(&previous.record.request_json.work, - LocationWork::Plan { command, .. } if *command == request.command); + LocationWork::Plan { command, fresh, .. } + if *command == request.command && *fresh == request.fresh); if !same { return Err(ControlError::Invalid( "location work ID already has different content or actor".into(), @@ -121,6 +122,7 @@ pub(in crate::runs) async fn request_plan_in( .into_iter() .filter(|(name, _)| name != PLAN_IGNORED_SETTING) .collect(), + fresh: request.fresh, }, }; let transaction = enqueue_location_work( diff --git a/crates/sqlbuild-orchestration/tests/api/plans/helpers.rs b/crates/sqlbuild-orchestration/tests/api/plans/helpers.rs index c693b359d4..266d72c524 100644 --- a/crates/sqlbuild-orchestration/tests/api/plans/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/api/plans/helpers.rs @@ -23,6 +23,7 @@ pub(crate) fn plan_request(tokens: &[&str]) -> PlanRequest { PlanRequest { request_id: Uuid::now_v7(), command: tokens.iter().map(|token| (*token).to_owned()).collect(), + fresh: false, } } diff --git a/crates/sqlbuild-orchestration/tests/api/test_person_plans.rs b/crates/sqlbuild-orchestration/tests/api/test_person_plans.rs index 0226a28950..d539033fb4 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_person_plans.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_person_plans.rs @@ -7,7 +7,7 @@ use sqlbuild_core::plan_display::main::detach_model_diffs::detach_model_diffs; use uuid::Uuid; use crate::api::tests::plans_helpers::{ask, person, plan_request, register_instance, run_rows}; -use crate::api::tests::test_types::{PlanDiffTestCase, PlanTestCase}; +use crate::api::tests::test_types::{PlanDiffTestCase, PlanTestCase, PlanTimingsReadTestCase}; use crate::api::tests::work_helpers::execute; use crate::authorization::main::people::bootstrap_user::bootstrap_user; use crate::credentials::models::{AuditContext, BuiltinRole}; @@ -38,12 +38,16 @@ fn given_person_plans_when_requested_claimed_and_read_then_they_stay_location_wo PlanTestCase { description: "SQLite plans", backend: Backend::Sqlite, + fresh: true, + reused_from: None, expected_selection: &["customers", "orders"], expected_warning: "connect failed: db_password=[REDACTED]", }, PlanTestCase { description: "PostgreSQL plans", backend: Backend::Postgres, + fresh: false, + reused_from: Some("2026-01-02T03:04:05Z"), expected_selection: &["customers", "orders"], expected_warning: "connect failed: db_password=[REDACTED]", }, @@ -100,14 +104,17 @@ fn given_person_plans_when_requested_claimed_and_read_then_they_stay_location_wo ) .await; let outsider = person(store, owner, ("orders-outsider", location), &[]).await; - let first = plan_request(&[ - "build", - "customers", - "orders", - "--full-refresh", - "--warehouse", - "WH_PLAN", - ]); + let first = PlanRequest { + fresh: test_case.fresh, + ..plan_request(&[ + "build", + "customers", + "orders", + "--full-refresh", + "--warehouse", + "WH_PLAN", + ]) + }; let silent = ask(store, (&context, viewer), location, &first).await; assert!( @@ -179,6 +186,17 @@ fn given_person_plans_when_requested_claimed_and_read_then_they_stay_location_wo "{description}: a request ID cannot carry another command", description = test_case.description ); + let refreshed = PlanRequest { + fresh: !first.fresh, + ..first.clone() + }; + assert!( + ask(store, (&context, viewer), location, &refreshed) + .await + .is_err(), + "{description}: a request ID cannot change whether it plans again", + description = test_case.description + ); assert!( ask(store, (&context, developer), location, &first) .await @@ -225,6 +243,7 @@ fn given_person_plans_when_requested_claimed_and_read_then_they_stay_location_wo command: vec!["build".into(), "orders".into()], selection: ["orders".to_owned()].into(), settings: Default::default(), + fresh: false, }, }; assert!( @@ -278,6 +297,7 @@ fn given_person_plans_when_requested_claimed_and_read_then_they_stay_location_wo let LocationWork::Plan { selection, settings, + fresh, .. } = &claimed.request.work else { @@ -286,6 +306,12 @@ fn given_person_plans_when_requested_claimed_and_read_then_they_stay_location_wo description = test_case.description ) }; + assert_eq!( + *fresh, + test_case.fresh, + "{description}: the location sees whether to plan again", + description = test_case.description + ); assert_eq!( selection.iter().map(String::as_str).collect::>(), test_case.expected_selection, @@ -343,7 +369,8 @@ fn given_person_plans_when_requested_claimed_and_read_then_they_stay_location_wo "config_change": "full_refresh true → false"}], "warnings": [{"severity": "warning", "message": "connect failed: db_password=orders-pass"}], "python_nodes": [{"name": "orders_export", "identity_diff": {"source_diff": ["-a", "+b"]}}], - "timings": {"total_seconds": 1.5, "steps": [{"name": "Reading warehouse state", "seconds": 1.0}, {"name": "Other", "seconds": 0.5}], "planned_from": "synced_state"}, + "timings": {"total_seconds": 1.5, "steps": [{"name": "Reading warehouse state", "seconds": 1.0}, {"name": "Other", "seconds": 0.5}], "planned_from": "synced_state", + "reused_from": test_case.reused_from}, }); assert!( complete_location_work( @@ -419,8 +446,11 @@ fn given_person_plans_when_requested_claimed_and_read_then_they_stay_location_wo }, ], planned_from: "synced_state".to_owned(), + reused_from: test_case + .reused_from + .map(|at| at.parse().expect("reused_from")), }), - "{description}: typed plan timings", + "{description}: typed plan timings, with when a reused plan was made", description = test_case.description ); let Some(LocationWorkOutcome::Plan { @@ -704,3 +734,74 @@ fn given_a_plan_whose_document_left_diffs_out_when_read_then_pages_count_them_an }); } } + +#[test] +fn given_a_plan_with_timings_when_its_summary_is_read_then_timings_of_another_shape_are_an_error() { + let test_cases = [ + PlanTimingsReadTestCase { + description: "a reused plan's timings say when it was made", + timings: json!({"total_seconds": 1.5, "steps": [], "planned_from": "synced_state", + "reused_from": "2026-01-02T03:04:05Z"}), + expected_read: true, + }, + PlanTimingsReadTestCase { + description: "timings of another shape fail the read rather than vanish", + timings: json!({"total_seconds": 1.5, "steps": [], "planned_from": "synced_state", + "reused": true}), + expected_read: false, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(Backend::Sqlite).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let context = AuditContext { + actor: "control-store-owner".into(), + granting_role: "control-store-owner".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + let admin = bootstrap_user(store, &context, "orders-admin") + .await + .expect("bootstrap"); + let location = create_location(store, "orders", "test-owner") + .await + .expect("location"); + publish_snapshot(store, &snapshot(location)) + .await + .expect("publish"); + let poll = register_instance(store, location).await; + let request = plan_request(&["build", "orders"]); + ask(store, (&context, admin), location, &request) + .await + .expect("plan"); + let claimed = claim_location_work(store, location, &poll) + .await + .expect("claim") + .expect("plan work"); + let result = LocationWorkResult { + claim_id: claimed.claim_id, + session_id: poll.session_id, + outcome: LocationWorkOutcome::Plan { + document: json!({"models": [], "timings": test_case.timings}), + diffs: BTreeMap::new(), + }, + }; + complete_location_work(store, (location, request.request_id), &result) + .await + .expect("complete"); + let plan = (location, "development", request.request_id); + assert_eq!( + read_person_plan_summary(store, (&context, admin), plan) + .await + .is_ok(), + test_case.expected_read, + "{}", + test_case.description + ); + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/api/test_types.rs b/crates/sqlbuild-orchestration/tests/api/test_types.rs index 54c8c21af7..34412fed36 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_types.rs @@ -319,10 +319,21 @@ pub(crate) struct OlderNodeFinished { pub(crate) struct PlanTestCase { pub(crate) description: &'static str, pub(crate) backend: Backend, + /// Whether the person asks to plan again rather than reuse a kept plan. + pub(crate) fresh: bool, + /// When the plan the location answered with was made, for a reused plan. + pub(crate) reused_from: Option<&'static str>, pub(crate) expected_selection: &'static [&'static str], pub(crate) expected_warning: &'static str, } +pub(crate) struct PlanTimingsReadTestCase { + pub(crate) description: &'static str, + pub(crate) timings: serde_json::Value, + /// Whether the plan's summary reads. + pub(crate) expected_read: bool, +} + pub(crate) struct UpcomingTestCase { pub(crate) description: &'static str, pub(crate) backend: Backend, diff --git a/crates/sqlbuild-python-base/src/event_exporting/constants.rs b/crates/sqlbuild-python-base/src/event_exporting/constants.rs index 9a195b54d6..07aaf03de8 100644 --- a/crates/sqlbuild-python-base/src/event_exporting/constants.rs +++ b/crates/sqlbuild-python-base/src/event_exporting/constants.rs @@ -19,6 +19,9 @@ pub const WORKER_ATTRIBUTE: &str = "_sqlbuild_event_exporter_worker"; /// The longest single wait of a polling delivery loop, as the Python dispatcher polled. pub const POLL_SECONDS: f64 = 0.01; +/// The longest an idle dispatcher loop waits for the notification that brings it work. +pub const IDLE_POLL_SECONDS: f64 = 1.0; + /// The most one dispatch round charges a sink call, so the dispatcher's GIL waits are not counted. pub const CHARGE_LIMIT_SECONDS: f64 = 2.0 * POLL_SECONDS; diff --git a/crates/sqlbuild-python-output/src/event_exporting/models.rs b/crates/sqlbuild-python-output/src/event_exporting/models.rs index e8b23c67bd..a7de6b396a 100644 --- a/crates/sqlbuild-python-output/src/event_exporting/models.rs +++ b/crates/sqlbuild-python-output/src/event_exporting/models.rs @@ -47,6 +47,7 @@ use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::event_exporting::constants::DISPATCHER_THREAD_NAME; use sqlbuild_python_base::event_exporting::constants::EXIT_JOIN_SECONDS; use sqlbuild_python_base::event_exporting::constants::HEALTH_THREAD_NAME; +use sqlbuild_python_base::event_exporting::constants::IDLE_POLL_SECONDS; use sqlbuild_python_base::event_exporting::constants::JOIN_GRACE_SECONDS; use sqlbuild_python_base::event_exporting::constants::NO_SECONDS; use sqlbuild_python_base::event_exporting::constants::NOTIFIER_THREAD_NAME; @@ -543,6 +544,7 @@ impl DispatcherWorker { notify |= state.feed(py, round)?; state.stopping && !state.busy() && state.queue.items.is_empty() && state.held.is_empty() }; + let idle: bool = !stopped && Self::idle_round(&state); let blocked: Vec = std::mem::take(&mut state.newly_blocked); drop(state); for name in blocked { @@ -552,11 +554,28 @@ impl DispatcherWorker { shared.notify_all(py)?; } if !stopped { - shared.wait(py, POLL_SECONDS)?; + shared.wait( + py, + if idle { + IDLE_POLL_SECONDS + } else { + POLL_SECONDS + }, + )?; } Ok(!stopped) } + /// Whether a round has nothing to watch: no queued, held or pending work, and no shutdown. + fn idle_round(state: &State) -> bool { + !state.busy() + && state.queue.items.is_empty() + && state.held.is_empty() + && !state.stopping + && !state.force_stop + && state.deadline.is_none() + } + /// Every `(token, sanitized error type)` delivery threads reported since the last round. pub(crate) fn reported(&self, py: Python<'_>) -> PyResult)>> { let results: &Bound<'_, PyAny> = self.shared.results.bind(py); @@ -593,7 +612,7 @@ impl DispatcherWorker { return Ok(Notification::Stop); } drop(state); - shared.wait(py, POLL_SECONDS)?; + shared.wait(py, IDLE_POLL_SECONDS)?; } } diff --git a/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/plan_output.rs b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/plan_output.rs index fb7e92c43d..452b3f06f4 100644 --- a/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/plan_output.rs +++ b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/plan_output.rs @@ -8,18 +8,24 @@ use crate::planner_entries::models::PlanOutputInputs; use crate::planner_resolution::main::references::resolve_function_sql::resolve_function_sql; use crate::planner_strategy::main::audit_entries::build_selected_audit_entries::build_selected_audit_entries; use crate::planner_strategy::main::model_entries::extract_seed_columns::extract_seed_columns; +use pyo3::exceptions::PyException; +use pyo3::prelude::Py; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::pyfunction; +use pyo3::types::PyTupleMethods; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyList, PyTuple}; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::boundary::models::KeywordArguments; +use sqlbuild_python_base::native_host::main::write_console::write_console; use sqlbuild_python_base::native_host::models::Dialect; use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_project::compiled_project::models::NativeCompiledProject; use sqlbuild_python_project::compiled_views::main::project::sql_tests::sql_tests; +use sqlbuild_python_project::compiled_views::models::NativeProjectView; use sqlbuild_python_project::planner_phases::main::object::object; use sqlbuild_python_project::planner_scope::main::build_source_load_entries::build_source_load_entries; use sqlbuild_python_project::planner_scope::main::upstream_loader_dependency_names::upstream_loader_dependency_names; @@ -38,6 +44,10 @@ use sqlbuild_python_project::resource_views::main::seed_reads::schema_entry::sch use sqlbuild_python_project::resource_views::main::seed_reads::seed_file::seed_file; use sqlbuild_python_project::sql_test_glue::main::assemble_sql_test_plans::assemble_sql_test_plans; use sqlbuild_python_project::sql_test_glue::models::SqlTestAssembly; +use std::sync::{Arc, OnceLock}; + +/// The stream a location's log reads, told once per publication how planning every SQL test went. +const STDERR: &str = "stderr"; const LOADER_CONSUMER: &str = "loader"; const HOOK_CONSUMER: &str = "hook"; @@ -506,29 +516,48 @@ pub(crate) fn selected_test_entries<'py>( } selected.push(test); } - let mut entries: Vec> = Vec::new(); - let mut warnings: Vec> = Vec::new(); if selected.is_empty() { - return Ok(SelectedTests { entries, warnings }); + return Ok(SelectedTests { + entries: Vec::new(), + warnings: Vec::new(), + }); } + let results = test_plan_results((project, dialect), &selected)?; + selected_from_results(py, results.try_iter()?.collect::>>()?) +} + +/// One plan result per test in `tests`, in order, with fixture diagnostics in place of failed plans. +fn test_plan_results<'py>( + (project, dialect): (&Bound<'py, PyAny>, &Bound<'py, Dialect>), + tests: &[Bound<'py, PyAny>], +) -> PyResult> { + let py = project.py(); let analysis = native_field(&native_field(project, "settings")?, "sql_analysis")?; let fixture_context = if analysis.is_truthy()? { Bound::new(py, planning_context(project, None)?)?.into_any() } else { none(py) }; - let results = assemble_sql_test_plans( - &selected, + assemble_sql_test_plans( + tests, (project, dialect), &SqlTestAssembly { sql_analysis_enabled: analysis, validate_fixtures: true, fixture_planning_context: fixture_context, }, - )?; + ) +} + +/// The entries and warnings of planned results, failing on invalid fixtures. +fn selected_from_results<'py>( + py: Python<'py>, + results: Vec>, +) -> PyResult> { + let mut entries: Vec> = Vec::new(); + let mut warnings: Vec> = Vec::new(); let mut diagnostics: Vec = Vec::new(); - for result in results.try_iter()? { - let result = result?; + for result in results { let entry = native_field(&result, "entry")?; if entry.is_none() { for diagnostic in native_field(&result, "fixture_diagnostics")?.try_iter()? { @@ -552,15 +581,83 @@ pub(crate) fn selected_test_entries<'py>( Ok(SelectedTests { entries, warnings }) } -/// SQL tests selected by name or overlapping one selection, planned into a reusable result. +/// The selection's entries from every test a held project planned once, or `None` to plan it. +fn held_selected_test_entries<'py>( + (project, dialect): (&Bound<'py, PyAny>, &Bound<'py, Dialect>), + selected_keys: &Bound<'py, PyAny>, + selection: &Bound<'py, PyAny>, +) -> PyResult>> { + let py = project.py(); + let held: Py = match project.cast::() { + Ok(view) => view.get().native_project(py), + Err(_) => match project.cast::() { + Ok(held) => held.clone().unbind(), + Err(_) => return Ok(None), + }, + }; + let tests: Vec> = sql_tests(project)?.try_iter()?.collect::>()?; + let results: Arc>>> = + held.bind(py).borrow().held_sql_test_results(); + if results.get().is_none() { + let planned: Option> = match test_plan_results((project, dialect), &tests) { + Ok(planned) => { + write_console( + py, + STDERR, + &format!( + "Planned the publication's {} SQL tests once for previews.\n", + tests.len() + ), + )?; + Some(planned.unbind()) + } + Err(error) if error.is_instance_of::(py) => { + write_console( + py, + STDERR, + &format!( + "Planning the publication's SQL tests once failed, so each preview plans its own: {error}\n" + ), + )?; + None + } + Err(error) => return Err(error), + }; + drop(results.set(planned)); + } + let Some(Some(planned)) = results.get() else { + return Ok(None); + }; + let planned: &Bound<'py, PyTuple> = planned.bind(py); + if planned.len() != tests.len() { + return Ok(None); + } + let mut selected: Vec> = Vec::new(); + for (position, test) in tests.iter().enumerate() { + if test_selected(test, selected_keys, selection)? { + selected.push(planned.get_item(position)?); + } + } + selected_from_results(py, selected).map(Some) +} + +/// SQL tests selected by name or overlapping one selection; `held` reads a held project's plans. fn planned_sql_tests<'py>( project: &Bound<'py, PyAny>, dialect: &Bound<'py, Dialect>, - selected_keys: &Bound<'py, PyAny>, - selection: &Bound<'py, PyAny>, + (selected_keys, selection): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + held: bool, ) -> PyResult> { let py = project.py(); - let selected = selected_test_entries((project, dialect), selected_keys, selection, &none(py))?; + let reused: Option> = if held { + held_selected_test_entries((project, dialect), selected_keys, selection)? + } else { + None + }; + let selected = match reused { + Some(selected) => selected, + None => selected_test_entries((project, dialect), selected_keys, selection, &none(py))?, + }; native( py, RecordId::PlannedSqlTests, @@ -660,7 +757,7 @@ pub(crate) fn plan_output_fields<'py>( || native_field(&planned, "selected_keys")?.ne(&selected_keys)? || native_field(&planned, "sql_test_selection")?.ne(&selection)? { - planned = planned_sql_tests(&project, &dialect, &selected_keys, &selection)?; + planned = planned_sql_tests(&project, &dialect, (&selected_keys, &selection), false)?; } let test_entries = native_field(&planned, "entries")?; let mut test_keys: Vec> = Vec::new(); @@ -763,10 +860,10 @@ pub(crate) fn plan_output_fields<'py>( pub(crate) fn plan_selected_sql_tests<'py>( project: &Bound<'py, PyAny>, dialect: &Bound<'py, Dialect>, - selected_keys: &Bound<'py, PyAny>, - sql_test_selection: &Bound<'py, PyAny>, + (selected_keys, sql_test_selection): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + held: bool, ) -> PyResult> { - planned_sql_tests(project, dialect, selected_keys, sql_test_selection) + planned_sql_tests(project, dialect, (selected_keys, sql_test_selection), held) } /// Selected SQL test entries and warnings, optionally narrowed to one parameterized case. diff --git a/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/test_planning.rs b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/test_planning.rs index bbaab1bd4e..9f98ce07ef 100644 --- a/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/test_planning.rs +++ b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/test_planning.rs @@ -13,10 +13,12 @@ use pyo3::types::PyDict; use pyo3::{pyclass, pyfunction, pymethods}; use crate::planner_entries::_helpers::plans::plan_output::plan_selected_sql_tests; +use crate::planner_warehouse::models::PlannerReads; use sqlbuild_python_base::boundary::main::compiler_guard::compiler_guard; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::native_threads::main::signal_interpreter_exit::signal_interpreter_exit; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; +use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; const TEST_PLANNING_THREAD: &str = "sqlbuild-test-planning"; @@ -155,6 +157,8 @@ pub(crate) struct PlanningRequest { pub(crate) dialect: Py, pub(crate) selected_keys: Py, pub(crate) sql_test_selection: Py, + /// Plan from the tests a location's held project planned once for its generation. + pub(crate) held: bool, } /// Plan the request's selected SQL tests, reporting a panic as a native compiler error. @@ -163,8 +167,11 @@ fn plan(py: Python<'_>, request: &PlanningRequest) -> Planned { plan_selected_sql_tests( request.project.bind(py), request.dialect.bind(py), - request.selected_keys.bind(py), - request.sql_test_selection.bind(py), + ( + request.selected_keys.bind(py), + request.sql_test_selection.bind(py), + ), + request.held, ) .map(Bound::unbind) }) @@ -211,6 +218,25 @@ pub(crate) fn start_planning( Py::new(py, SqlTestPlanning::of(Some(receiver), None)) } +/// Start planning the scope's SQL tests; a location's previews read its held project's plans. +pub(crate) fn start_scope_planning<'py>( + (project, dialect): (&Bound<'py, PyAny>, Bound<'py, Dialect>), + (scope, reads): (&Bound<'py, PyAny>, &Bound<'py, PlannerReads>), + enabled: bool, +) -> PyResult> { + start_planning( + project.py(), + PlanningRequest { + project: project.clone().unbind(), + dialect: dialect.unbind(), + selected_keys: native_field(scope, "selected_keys")?.unbind(), + sql_test_selection: native_field(scope, "sql_test_selection")?.unbind(), + held: reads.get().reads().held_project(), + }, + enabled, + ) +} + /// Join SQL test planning threads at exit: finalization ending one inside native planning aborts. #[pyfunction] pub(crate) fn join_sql_test_planning_threads(py: Python<'_>) -> PyResult<()> { @@ -249,6 +275,7 @@ pub(crate) fn start_sql_test_planning<'py>( dialect: Dialect::passed(args.value("dialect")?)?.unbind(), selected_keys: args.value("selected_keys")?.unbind(), sql_test_selection: args.value("sql_test_selection")?.unbind(), + held: false, }, args.flag("enabled")?, ) diff --git a/crates/sqlbuild-python-planner/src/planner_entries/main/plans/build_execution_plan.rs b/crates/sqlbuild-python-planner/src/planner_entries/main/plans/build_execution_plan.rs index ff8ea412c4..9cf617e05b 100644 --- a/crates/sqlbuild-python-planner/src/planner_entries/main/plans/build_execution_plan.rs +++ b/crates/sqlbuild-python-planner/src/planner_entries/main/plans/build_execution_plan.rs @@ -2,7 +2,7 @@ use crate::planner_entries::_helpers::plans::execution_plan::planned_output; use crate::planner_entries::_helpers::plans::execution_plan::scopes_and_cursor_overrides; -use crate::planner_entries::_helpers::plans::test_planning::{PlanningRequest, start_planning}; +use crate::planner_entries::_helpers::plans::test_planning::start_scope_planning; use crate::planner_phases::main::inputs::class_field::class_field; use crate::planner_phases::models::PlannerRuntime; use crate::planner_warehouse::models::PlannerReads; @@ -63,14 +63,9 @@ pub fn build_execution_plan<'py>( .into_any(); let (scopes, overrides) = scopes_and_cursor_overrides(&args)?; let inspection = class_field(&scopes, |s: &PlannerScopeResolution| &s.inspection_scope)?; - let test_planning = start_planning( - py, - PlanningRequest { - project: project.clone().unbind(), - dialect: dialect.unbind(), - selected_keys: native_field(&inspection, "selected_keys")?.unbind(), - sql_test_selection: native_field(&inspection, "sql_test_selection")?.unbind(), - }, + let test_planning = start_scope_planning( + (&project, dialect), + (&inspection, &reads), native_field(&args.value("policies")?, "plan_sql_tests")?.is_truthy()?, )?; let planned = reads.get().reads().within_inspection(py, &mut || { diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_entries.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_entries.rs index 436d400ded..7164bdb915 100644 --- a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_entries.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_entries.rs @@ -1,8 +1,8 @@ //! Materialize column decisions from native proofs and live evidence. +use crate::planner_migrations::_helpers::planning::location_relation::location_relation; use crate::planner_phases::main::inputs::runtime_reads::runtime_reads; use crate::planner_warehouse::models::{AdapterRead, Read}; -use sqlbuild_host::entries::models::ExecutorHost; use crate::planner_migrations::_helpers::column::column_candidates::declared_names; use crate::planner_migrations::_helpers::column::column_objects::lower; @@ -19,7 +19,6 @@ use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyListMethods, PyTuple}; use sqlbuild_analysis::planner_migrations::main::column_decision::column_decision; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::keywords::keywords; -use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::construct::enum_from_value::enum_from_value; use sqlbuild_python_base::records::main::construct::enum_member::enum_member; use sqlbuild_python_base::records::main::fields::field::field as native_field; @@ -74,11 +73,11 @@ pub(crate) fn decide<'py>( outputs.push(name.to_lowercase()); } } - let relation = host_entry(py, ExecutorHost::MigrationRelationForLocation)?.call1(( - sqlbuild_python_project::compiled_views::main::compiled_models::destination::destination( + let relation = location_relation( + &sqlbuild_python_project::compiled_views::main::compiled_models::destination::destination( &candidate.model, )?, - ))?; + )?; let mut entries: Vec> = Vec::new(); for (origin_name, destination, discovery) in pairs { let origin_key = lower(&origin_name)?; diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/discovery/discovery_events.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/discovery/discovery_events.rs index ddfd3b7cfa..01acf3c4ce 100644 --- a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/discovery/discovery_events.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/discovery/discovery_events.rs @@ -1,13 +1,12 @@ //! Preserve event and declared-origin precedence at the inspection boundary. -use sqlbuild_host::entries::models::ExecutorHost; +use crate::planner_migrations::_helpers::planning::location_relation::location_relation; use crate::planner_migrations::_helpers::column::column_objects::config; use crate::planner_migrations::models::ModelMigrationDeclaration; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods, PyTuple}; use sqlbuild_python_base::boundary::main::call_with::call_with; -use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_front::state_rows::main::migrations::newest_move::newest_move; use sqlbuild_python_project::compiled_views::main::compiled_models::name_object::name_object; @@ -26,13 +25,12 @@ pub(crate) fn newest<'py>( events: &Bound<'py, PyAny>, location: &Bound<'py, PyAny>, ) -> PyResult> { - let relation = host_entry(location.py(), ExecutorHost::MigrationRelationForLocation)? - .call1((location,))?; + let relation = location_relation(location)?; newest_move(events, &relation) } pub(crate) fn relation<'py>(location: &Bound<'py, PyAny>) -> PyResult> { - host_entry(location.py(), ExecutorHost::MigrationRelationForLocation)?.call1((location,)) + location_relation(location) } pub(crate) fn declared<'py>( diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/model_entries.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/model_entries.rs index 66f9710514..d85bf58fcd 100644 --- a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/model_entries.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/model_entries.rs @@ -1,6 +1,6 @@ //! Assemble model migration decisions from inspected warehouse facts. -use sqlbuild_host::entries::models::ExecutorHost; +use crate::planner_migrations::_helpers::planning::location_relation::location_relation; use crate::planner_migrations::_helpers::column::column_objects::lower; use crate::planner_migrations::_helpers::column::column_objects::model; @@ -16,7 +16,6 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::PyBool; use sqlbuild_analysis::planner_migrations::main::model_decision::model_decision; use sqlbuild_python_base::boundary::main::call_with::call_with; -use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::construct::enum_from_value::enum_from_value; use sqlbuild_python_base::records::main::construct::enum_member::enum_member; use sqlbuild_python_base::records::main::construct::error_class::error_class; @@ -27,7 +26,7 @@ use sqlbuild_python_front::state_rows::main::migrations::newest_move::newest_mov use sqlbuild_python_project::compiled_views::main::compiled_models::name_object::name_object; pub(crate) fn relation<'py>(location: &Bound<'py, PyAny>) -> PyResult> { - host_entry(location.py(), ExecutorHost::MigrationRelationForLocation)?.call1((location,)) + location_relation(location) } fn key(location: &Bound<'_, PyAny>) -> PyResult<(String, String)> { diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/identity_index.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/identity_index.rs new file mode 100644 index 0000000000..500104c890 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/identity_index.rs @@ -0,0 +1,54 @@ +//! Migration relations indexed by schema and name, matched as `MigrationRelation.matches` matches. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use std::collections::HashMap; + +/// A relation's `identity`: `(database.lower() or None, (schema or "").lower(), name.lower())`. +type Identity = (Option, String, String); + +/// Each indexed relation's lowered database and position, in input order. +type Candidates = Vec<(Option, usize)>; + +/// The positions of indexed relations by lowered `(schema, name)`, each list in input order. +#[derive(Debug)] +pub(crate) struct IdentityIndex { + by_name: HashMap<(String, String), Candidates>, +} + +fn identity(relation: &Bound<'_, PyAny>) -> PyResult { + relation.getattr("identity")?.extract() +} + +impl IdentityIndex { + /// Index `relations` by position. + pub(crate) fn of<'py>(relations: &[Bound<'py, PyAny>]) -> PyResult { + let mut by_name: HashMap<(String, String), Candidates> = HashMap::new(); + for (position, relation) in relations.iter().enumerate() { + let (database, schema, name) = identity(relation)?; + by_name + .entry((schema, name)) + .or_default() + .push((database, position)); + } + Ok(Self { by_name }) + } + + /// Positions of the indexed relations `relation.matches(...)` accepts, in input order. + pub(crate) fn matching(&self, relation: &Bound<'_, PyAny>) -> PyResult> { + let (database, schema, name) = identity(relation)?; + let mut positions: Vec = Vec::new(); + let Some(candidates) = self.by_name.get(&(schema, name)) else { + return Ok(positions); + }; + for (candidate, position) in candidates { + let same_database: bool = match (candidate, &database) { + (Some(left), Some(right)) => left == right, + _ => true, + }; + if same_database { + positions.push(*position); + } + } + Ok(positions) + } +} diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/mod.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/mod.rs index 87e886af43..7dc59c904b 100644 --- a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/mod.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/mod.rs @@ -1,3 +1,4 @@ +pub(crate) mod identity_index; pub(crate) mod old_name_entries; pub(crate) mod old_name_objects; pub(crate) mod old_name_planning; diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_name_entries.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_name_entries.rs index 693488e50e..fd444e50f0 100644 --- a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_name_entries.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_name_entries.rs @@ -2,13 +2,13 @@ use crate::planner_migrations::_helpers::column::column_objects::model; use crate::planner_migrations::_helpers::column::column_objects::text; +use crate::planner_migrations::_helpers::old_name::identity_index::IdentityIndex; use crate::planner_migrations::_helpers::old_name::old_name_objects::expires; use crate::planner_migrations::_helpers::old_name::old_name_objects::location; use crate::planner_migrations::_helpers::old_name::old_name_objects::relation; use crate::planner_migrations::_helpers::old_name::old_name_objects::retention; use crate::planner_migrations::_helpers::old_name::old_name_objects::stores_history; use crate::planner_migrations::_helpers::old_name::old_names::declared_column_aliases; -use crate::planner_migrations::_helpers::old_name::old_names::old_name_view_skip_reason; use crate::planner_migrations::constants::PENDING_ARCHIVE; use crate::planner_migrations::constants::PENDING_VIEW; use crate::planner_migrations::models::OldNamePlanner; @@ -16,9 +16,9 @@ use crate::planner_phases::main::inputs::runtime_dialect::runtime_dialect; use crate::planner_phases::main::inputs::runtime_field::runtime_field; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyBool, PyList, PyTuple}; +use sqlbuild_analysis::planner_migrations::main::old_name_skip_reason::old_name_skip_reason; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; -use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::native_host::types::Syntax; use sqlbuild_python_base::records::main::construct::enum_member::enum_member; use sqlbuild_python_base::records::main::construct::new_record::new_record; @@ -28,6 +28,7 @@ use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_project::compiled_views::main::compiled_models::destination::destination; use sqlbuild_python_project::compiled_views::main::compiled_models::name_object::name_object; use sqlbuild_python_project::planner_phases::main::object::object; +use std::cell::OnceCell; impl<'py> OldNamePlanner<'py> { pub(super) fn new( @@ -52,6 +53,51 @@ impl<'py> OldNamePlanner<'py> { retention: native_field(&config, "old_name_views")?, now: runtime_field(&runtime, |runtime| &runtime.invocation_time)?, runtime, + model_index: OnceCell::new(), + view_index: OnceCell::new(), + history_index: OnceCell::new(), + }) + } + + /// `cell`'s index, building it from `relations` the first time. + fn index<'a>( + cell: &'a OnceCell, + relations: impl FnOnce() -> PyResult>>, + ) -> PyResult<&'a IdentityIndex> { + if let Some(index) = cell.get() { + return Ok(index); + } + let built: IdentityIndex = IdentityIndex::of(&relations()?)?; + Ok(cell.get_or_init(|| built)) + } + + /// Model positions by destination relation. + fn model_index(&self) -> PyResult<&IdentityIndex> { + Self::index(&self.model_index, || { + self.models + .iter() + .map(|item| relation(&destination(item)?)) + .collect() + }) + } + + /// Recorded view positions by old relation. + fn view_index(&self) -> PyResult<&IdentityIndex> { + Self::index(&self.view_index, || { + self.views + .try_iter()? + .map(|view| relation(&native_field(&view?, "old")?)) + .collect() + }) + } + + /// History positions by their move's destination relation. + fn history_index(&self) -> PyResult<&IdentityIndex> { + Self::index(&self.history_index, || { + self.histories + .iter() + .map(|history| native_field(&native_field(history, "move")?, "destination")) + .collect() }) } @@ -61,14 +107,9 @@ impl<'py> OldNamePlanner<'py> { origin: Option<&Bound<'py, PyAny>>, ) -> PyResult>> { let mut newest: Option<(Bound<'py, PyTuple>, Bound<'py, PyAny>)> = None; - for history in &self.histories { + for position in self.history_index()?.matching(destination)? { + let history = &self.histories[position]; let movement = native_field(history, "move")?; - if !native_field(&movement, "destination")? - .call_method1("matches", (destination,))? - .is_truthy()? - { - continue; - } if let Some(origin) = origin && !native_field(&movement, "origin")? .call_method1("matches", (origin,))? @@ -145,30 +186,32 @@ impl<'py> OldNamePlanner<'py> { } } + /// Why no old-name view is kept for `origin`: a model reuses its name or a view already serves it. fn skip( &self, origin: &Bound<'py, PyAny>, tracked: Bound<'py, PyAny>, retention: &Bound<'py, PyAny>, ) -> PyResult> { + let py = self.runtime.py(); + if retention.is_none() { + return object(py, old_name_skip_reason(false, None, false, None)); + } + let old = relation(origin)?; + let reuser: Option = match self.model_index()?.matching(&old)?.first() { + Some(position) => Some(name_object(&self.models[*position])?.extract()?), + None => None, + }; + let compatibility_view: bool = + reuser.is_none() && !self.view_index()?.matching(&old)?.is_empty(); object( - self.runtime.py(), - old_name_view_skip_reason( - self.runtime.py(), - Some(keywords( - self.runtime.py(), - &[ - ("origin", origin.clone()), - ("origin_tracked", tracked), - ("retention", retention.clone()), - ( - "models", - PyTuple::new(self.runtime.py(), &self.models)?.into_any(), - ), - ("views", self.views.clone()), - ], - )?), - )?, + py, + old_name_skip_reason( + true, + reuser.as_deref(), + compatibility_view, + tracked.extract()?, + ), ) } diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_name_objects.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_name_objects.rs index cb3a98cf8c..61fe861929 100644 --- a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_name_objects.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_name_objects.rs @@ -1,14 +1,13 @@ //! Old-name planner object conversions and existing native helper calls. +use crate::planner_migrations::_helpers::planning::location_relation::location_relation; use crate::planner_phases::main::inputs::runtime_dialect::runtime_dialect; -use sqlbuild_host::entries::models::ExecutorHost; use sqlbuild_python_base::native_host::types::Syntax; use crate::planner_migrations::_helpers::column::column_objects::config; use crate::planner_migrations::_helpers::column::column_objects::model; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use sqlbuild_python_base::boundary::main::call_with::call_with; -use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; @@ -17,7 +16,7 @@ use sqlbuild_python_front::state_rows::main::migrations::old_name_retention::old use sqlbuild_python_project::compiled_views::main::compiled_models::destination::destination; pub(crate) fn relation<'py>(location: &Bound<'py, PyAny>) -> PyResult> { - host_entry(location.py(), ExecutorHost::MigrationRelationForLocation)?.call1((location,)) + location_relation(location) } pub(crate) fn location<'py>( diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_names.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_names.rs index b870bbb145..ae9bb518cf 100644 --- a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_names.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_names.rs @@ -8,7 +8,6 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PySet, PyTuple}; use sqlbuild_analysis::planner_migrations::main::{ declared_column_aliases as aliases, old_name_claim_message as claim, old_name_schemas, - old_name_skip_reason, }; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::native_host::main::host_entry::host_entry; @@ -132,54 +131,3 @@ pub(crate) fn old_name_claim_errors<'py>( } PyTuple::new(py, errors) } - -pub(crate) fn old_name_view_skip_reason<'py>( - py: Python<'py>, - options: Option>, -) -> PyResult> { - let arguments = KeywordArguments::read( - py, - "old_name_view_skip_reason", - options, - ( - &["origin", "origin_tracked", "retention", "models", "views"], - &[], - ), - )?; - let retained = !arguments.value("retention")?.is_none(); - if !retained { - return Ok(old_name_skip_reason::old_name_skip_reason( - false, None, false, None, - )); - } - let old = relation(&arguments.value("origin")?)?; - let mut reuser: Option = None; - for model in arguments.value("models")?.try_iter()? { - let model = model?; - if relation(&sqlbuild_python_project::compiled_views::main::compiled_models::destination::destination(&model)?)? - .call_method1("matches", (&old,))? - .is_truthy()? - { - reuser = Some(name_object(&model)?.extract()?); - break; - } - } - let mut compatibility_view = false; - if reuser.is_none() { - for view in arguments.value("views")?.try_iter()? { - if relation(&native_field(&view?, "old")?)? - .call_method1("matches", (&old,))? - .is_truthy()? - { - compatibility_view = true; - break; - } - } - } - Ok(old_name_skip_reason::old_name_skip_reason( - true, - reuser.as_deref(), - compatibility_view, - arguments.value("origin_tracked")?.extract()?, - )) -} diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/planning/location_relation.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/planning/location_relation.rs new file mode 100644 index 0000000000..ebb0eedec5 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/planning/location_relation.rs @@ -0,0 +1,19 @@ +//! The migration relation that names one compiled relation location, built natively. + +use pyo3::prelude::{Bound, PyAny, PyResult}; +use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::types::RecordId; + +/// The `MigrationRelation` that names `location`, as `migration_relation_for_location` builds it. +pub(crate) fn location_relation<'py>(location: &Bound<'py, PyAny>) -> PyResult> { + new_record_with( + location.py(), + RecordId::MigrationRelation, + &[ + ("database", native_field(location, "database")?), + ("schema", native_field(location, "schema")?), + ("name", native_field(location, "name")?), + ], + ) +} diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/planning/mod.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/planning/mod.rs index e8f2dc0ad7..fe92772606 100644 --- a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/planning/mod.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/planning/mod.rs @@ -1,4 +1,5 @@ pub(crate) mod compatibility; +pub(crate) mod location_relation; pub(crate) mod matching; pub(crate) mod planning_boundary; pub(crate) mod planning_execution; diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/models.rs b/crates/sqlbuild-python-planner/src/planner_migrations/models.rs index 9dbc21f7eb..1d16a3d952 100644 --- a/crates/sqlbuild-python-planner/src/planner_migrations/models.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/models.rs @@ -7,6 +7,7 @@ use crate::planner_migrations::_helpers::discovery::state_inspection::pending; use crate::planner_migrations::_helpers::discovery::state_inspection::relation_key; use crate::planner_migrations::_helpers::discovery::state_inspection::schema_or_empty; use crate::planner_migrations::_helpers::discovery::state_inspection::tuple_of; +use crate::planner_migrations::_helpers::old_name::identity_index::IdentityIndex; use crate::planner_warehouse::main::inspection::columns_for::columns_for; use crate::planner_warehouse::main::inspection::list_relations::list_relations; use crate::planner_warehouse::models::FingerprintRead; @@ -41,6 +42,7 @@ use sqlbuild_python_front::state_rows::main::migrations::layout::layout; use sqlbuild_python_front::state_rows::main::migrations::old_name_views::old_name_views; use sqlbuild_state::events::main::newest_fingerprint::newest_fingerprint; use sqlbuild_state::events::models::{EventRelation, FingerprintKey}; +use std::cell::OnceCell; use std::collections::HashMap; use std::collections::HashSet; @@ -53,6 +55,12 @@ pub(crate) struct OldNamePlanner<'py> { pub columns: Bound<'py, PyAny>, pub retention: Bound<'py, PyAny>, pub now: Bound<'py, PyAny>, + /// `models` by destination relation, built on first use. + pub model_index: OnceCell, + /// `views` by old relation, built on first use. + pub view_index: OnceCell, + /// `histories` by their move's destination, built on first use. + pub history_index: OnceCell, } #[derive(Debug)] diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/audit_entries.rs b/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/audit_entries.rs index 0e50277e5f..d7296f9142 100644 --- a/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/audit_entries.rs +++ b/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/audit_entries.rs @@ -5,7 +5,7 @@ use sqlbuild_analysis::graph::models::GraphIndexes; use sqlbuild_python_base::native_host::models::Dialect; use crate::planner_strategy::_helpers::audits::{ - attachment, outside_target_lineage, resolve_effective_run_scope, + attachment, outside_target_lineage, outside_target_lineages, resolve_effective_run_scope, }; use pyo3::IntoPyObject; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult}; @@ -114,13 +114,30 @@ pub(crate) fn plan_audit_of<'py>( })), )?, }, + None, ) } -/// One audit's plan entry, rendered against the planning relations. +/// Plan entries of `audits`, walking each target's lineage at most once. +pub(crate) fn plan_audits<'py>( + audits: &[Bound<'py, PyAny>], + planning: &AuditPlanning<'py>, +) -> PyResult>> { + let py = planning.model_locations.py(); + let outside: Vec = + outside_target_lineages(audits, planning.lineage.bind(py).borrow().graph())?; + let mut entries: Vec> = Vec::new(); + for (audit, outside) in audits.iter().zip(outside) { + entries.push(plan_audit(audit, planning, Some(outside))?); + } + Ok(entries) +} + +/// One audit's plan entry, rendered against the planning relations, with `outside_lineage` if known. pub(crate) fn plan_audit<'py>( audit: &Bound<'py, PyAny>, request: &AuditPlanning<'py>, + outside_lineage: Option, ) -> PyResult> { let py = audit.py(); let audit = audit.clone(); @@ -215,9 +232,15 @@ pub(crate) fn plan_audit<'py>( ("attached_target_name", target), ( "reads_outside_target_lineage", - PyBool::new(py, outside_target_lineage(&audit, lineage.graph())?) - .to_owned() - .into_any(), + PyBool::new( + py, + match outside_lineage { + Some(outside) => outside, + None => outside_target_lineage(&audit, lineage.graph())?, + }, + ) + .to_owned() + .into_any(), ), ] { values.set_item(field, value)?; diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/audits.rs b/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/audits.rs index d393d68ca8..8e01a0215e 100644 --- a/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/audits.rs +++ b/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/audits.rs @@ -4,11 +4,13 @@ use crate::planner_strategy::main::values::input_error::input_error; use crate::planner_strategy::main::values::text::text; use std::cell::RefCell; use std::collections::HashMap; +use std::collections::HashSet; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::PyTuple; use pyo3::{IntoPyObject, pyfunction}; +use sqlbuild_analysis::graph::main::edge_keys::edge_keys; use sqlbuild_analysis::graph::main::graph_from_indexes::graph_from_indexes; use sqlbuild_analysis::graph::main::reachable_graph_keys::reachable_graph_keys; use sqlbuild_analysis::graph::models::{EdgeDirection, GraphIndexes, ProjectGraph}; @@ -149,16 +151,72 @@ pub(crate) fn outside_target_lineage( audit: &Bound<'_, PyAny>, graph: &ProjectGraph, ) -> PyResult { + let Some(target) = lineage_target(audit)? else { + return Ok(false); + }; + let lineage = reachable_graph_keys(&[target], graph, EdgeDirection::Upstream); + outside_lineage(audit, &lineage) +} + +/// An audit's position among the selected audits and the keys of what it reads. +type AuditDependencies = (usize, Vec<(String, String)>); + +/// `outside_target_lineage` of each audit, walking a target's lineage once and only when needed. +pub(crate) fn outside_target_lineages( + audits: &[Bound<'_, PyAny>], + graph: &ProjectGraph, +) -> PyResult> { + let mut outside: Vec = vec![false; audits.len()]; + let mut by_target: HashMap<(String, String), Vec> = HashMap::new(); + for (position, audit) in audits.iter().enumerate() { + let Some(target) = lineage_target(audit)? else { + continue; + }; + let dependencies: Vec<(String, String)> = scope_deps(audit)? + .try_iter()? + .map(|dependency| native_key(&dependency?)) + .collect::>()?; + let parents = edge_keys(graph, &target, EdgeDirection::Upstream); + if dependencies + .iter() + .all(|dependency| *dependency == target || parents.contains(dependency)) + { + continue; + } + by_target + .entry(target) + .or_default() + .push((position, dependencies)); + } + for (target, walked) in by_target { + let lineage = reachable_graph_keys(&[target], graph, EdgeDirection::Upstream); + for (position, dependencies) in walked { + outside[position] = dependencies + .iter() + .any(|dependency| !lineage.contains(dependency)); + } + } + Ok(outside) +} + +/// The key an attached audit's target lineage starts from, or `None` for an unattached audit. +fn lineage_target(audit: &Bound<'_, PyAny>) -> PyResult> { let py: Python<'_> = audit.py(); let kind: Bound<'_, PyAny> = attached_target_kind(audit)?; let name: Bound<'_, PyAny> = attached_target_name(audit)?; if kind.is_none() || name.is_none() { - return Ok(false); + return Ok(None); } let kind: String = text(&imported(py, COMPILE_TYPES_MODULE, "AttachedAuditTargetKind")?.call1((kind,))?)?; - let target = (kind, text(&name)?); - let lineage = reachable_graph_keys(&[target], graph, EdgeDirection::Upstream); + Ok(Some((kind, text(&name)?))) +} + +/// Whether the audit reads a dependency outside `lineage`. +fn outside_lineage( + audit: &Bound<'_, PyAny>, + lineage: &HashSet<(String, String)>, +) -> PyResult { for dependency in scope_deps(audit)?.try_iter()? { if !lineage.contains(&native_key(&dependency?)?) { return Ok(true); diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/main/audit_entries/build_selected_audit_entries.rs b/crates/sqlbuild-python-planner/src/planner_strategy/main/audit_entries/build_selected_audit_entries.rs index 69c28222c5..06a2ffbafe 100644 --- a/crates/sqlbuild-python-planner/src/planner_strategy/main/audit_entries/build_selected_audit_entries.rs +++ b/crates/sqlbuild-python-planner/src/planner_strategy/main/audit_entries/build_selected_audit_entries.rs @@ -1,6 +1,6 @@ //! Assemble selected audits in compiled-project order without querying lineage for an empty selection. -use crate::planner_strategy::_helpers::audit_entries::plan_audit; +use crate::planner_strategy::_helpers::audit_entries::plan_audits; use crate::planner_strategy::main::audit_entries::audit_is_selected::audit_is_selected; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyList, PyListMethods}; @@ -39,8 +39,8 @@ pub fn build_selected_audit_entries<'py>( model_materializations: request.get("model_materializations")?, lineage: native_field(&project, "lineage_graph")?.extract()?, }; - for audit in selected { - entries.append(plan_audit(&audit, &planning)?)?; + for entry in plan_audits(&selected, &planning)? { + entries.append(entry)?; } Ok(entries) } diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/models.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/models.rs index bf6b4d06d9..6b08006c92 100644 --- a/crates/sqlbuild-python-planner/src/planner_warehouse/models.rs +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/models.rs @@ -487,15 +487,18 @@ impl ConsumptionReads for dyn WarehouseReads + '_ { #[derive(Debug)] pub struct SyncedReads { state: Arc, + /// Whether the plan uses the compiled project the location holds for its generation. + held_project: bool, unknowns: Mutex>, windows: Mutex>, } impl SyncedReads { - /// Reads over `state`, which several plans may share. - pub fn new(state: Arc) -> Self { + /// Reads over `state`, which several plans may share, for a plan of the held project or not. + pub fn new(state: Arc, held_project: bool) -> Self { Self { state, + held_project, unknowns: Mutex::new(BTreeSet::new()), windows: Mutex::new(BTreeMap::new()), } @@ -568,6 +571,10 @@ impl SyncedReads { } impl WarehouseReads for SyncedReads { + fn held_project(&self) -> bool { + self.held_project + } + fn relations<'py>( &self, py: Python<'py>, diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/types.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/types.rs index cbf790687b..fcee649853 100644 --- a/crates/sqlbuild-python-planner/src/planner_warehouse/types.rs +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/types.rs @@ -32,6 +32,11 @@ pub trait CursorProgress<'py> { /// Everything a plan reads from the warehouse, live or from a synced state that may answer unknown. pub trait WarehouseReads: Send + Sync + std::fmt::Debug { + /// Whether the plan uses the compiled project a location holds for its published generation. + fn held_project(&self) -> bool { + false + } + /// The listed relations of `database` narrowed to `schemas` and `names`, as `RelationInfo`s. fn relations<'py>( &self, diff --git a/crates/sqlbuild-python-project/src/compiled_project/models.rs b/crates/sqlbuild-python-project/src/compiled_project/models.rs index 73eb20c543..4d18108958 100644 --- a/crates/sqlbuild-python-project/src/compiled_project/models.rs +++ b/crates/sqlbuild-python-project/src/compiled_project/models.rs @@ -58,6 +58,7 @@ use pyo3::pymethods; use pyo3::types::PyDict; use pyo3::types::PyDictMethods; use pyo3::types::PyString; +use pyo3::types::PyTuple; use sqlbuild_analysis::compiled_project::main::project_models::project_models; use sqlbuild_analysis::compiled_project::main::record_model_analysis::record_model_analysis; use sqlbuild_analysis::compiled_project::main::record_skipped_analysis::record_skipped_analysis; @@ -75,6 +76,7 @@ use sqlbuild_python_base::cache::models::SqlTestScanStore; use sqlbuild_python_front::render::models::ConstantRendering; use std::collections::HashMap; use std::sync::Mutex; +use std::sync::{Arc, OnceLock}; /// A function's paths, stem, discovered file, header values, body and SQL file contents. #[derive(Debug)] @@ -579,9 +581,16 @@ pub struct NativeCompiledProject { pub(crate) inferred_column_projections: Mutex>>, /// The model loop and its constant rendering, which attach the project's other resources. pub(crate) attachment: Option<(Py, Py)>, + /// Every SQL test's plan result for a location's previews, set once; `None` when planning failed. + pub(crate) held_sql_test_results: Arc>>>, } impl NativeCompiledProject { + /// Every SQL test's plan result for a location's previews, planned once per published project. + pub fn held_sql_test_results(&self) -> Arc>>> { + Arc::clone(&self.held_sql_test_results) + } + /// The facts recorded so far, for native consumers such as the rules request. pub fn facts(&self) -> &CompiledProjectFacts { &self.project.facts diff --git a/crates/sqlbuild-python-project/src/planner_identity/_helpers/hashing.rs b/crates/sqlbuild-python-project/src/planner_identity/_helpers/hashing.rs index 88460a1e35..2eb6a8a87b 100644 --- a/crates/sqlbuild-python-project/src/planner_identity/_helpers/hashing.rs +++ b/crates/sqlbuild-python-project/src/planner_identity/_helpers/hashing.rs @@ -23,8 +23,24 @@ pub(crate) static QUERY_FINGERPRINTS: Mutex> = Mutex:: /// Distinct SQL texts the memo keeps before starting over; a 10K-model project holds about 10K. const QUERY_FINGERPRINTS_LIMIT: usize = 100_000; -/// Fingerprint results by `(dialect, SQL text)`. -pub(crate) type QueryFingerprints = HashMap<(Option, String), Result>; +/// Fingerprint results by dialect, then by SQL text, so a lookup borrows the SQL text. +pub(crate) type QueryFingerprints = Vec<(Option, HashMap>)>; + +/// The remembered fingerprint of `sql` under `dialect`, if any. +fn remembered( + memo: &Mutex>, + sql: &str, + dialect: Option<&str>, +) -> Option> { + memo.lock() + .unwrap_or_else(PoisonError::into_inner) + .as_ref()? + .iter() + .find(|(known, _)| known.as_deref() == dialect)? + .1 + .get(sql) + .cloned() +} /// `compiler_query_fingerprint`, remembered by dialect and SQL text. pub(crate) fn query_fingerprint( @@ -32,22 +48,27 @@ pub(crate) fn query_fingerprint( sql: &str, dialect: Option<&str>, ) -> Result { - let key: (Option, String) = (dialect.map(str::to_owned), sql.to_owned()); - if let Some(known) = memo - .lock() - .unwrap_or_else(PoisonError::into_inner) - .as_ref() - .and_then(|memo| memo.get(&key)) - { - return known.clone(); + if let Some(known) = remembered(memo, sql, dialect) { + return known; } let fingerprint: Result = compiler_query_fingerprint(sql, dialect); let mut memo = memo.lock().unwrap_or_else(PoisonError::into_inner); - let entries: &mut QueryFingerprints = memo.get_or_insert_with(HashMap::new); - if entries.len() > QUERY_FINGERPRINTS_LIMIT { - entries.clear(); + let dialects: &mut QueryFingerprints = memo.get_or_insert_with(Vec::new); + if dialects.iter().map(|(_, texts)| texts.len()).sum::() > QUERY_FINGERPRINTS_LIMIT { + dialects.clear(); + } + match dialects + .iter_mut() + .find(|(known, _)| known.as_deref() == dialect) + { + Some((_, texts)) => { + texts.insert(sql.to_owned(), fingerprint.clone()); + } + None => dialects.push(( + dialect.map(str::to_owned), + HashMap::from([(sql.to_owned(), fingerprint.clone())]), + )), } - entries.insert(key, fingerprint.clone()); fingerprint } @@ -57,8 +78,11 @@ pub(crate) fn model_query_hash( model_name: &str, (query_sql, dialect): (&str, Option<&str>), ) -> PyResult { - py.compiler_detach(|| query_fingerprint(&QUERY_FINGERPRINTS, query_sql, dialect)) - .map_err(|error| fingerprint_error(py, &format!("Model '{model_name}' SQL"), error)) + match remembered(&QUERY_FINGERPRINTS, query_sql, dialect) { + Some(known) => known, + None => py.compiler_detach(|| query_fingerprint(&QUERY_FINGERPRINTS, query_sql, dialect)), + } + .map_err(|error| fingerprint_error(py, &format!("Model '{model_name}' SQL"), error)) } /// `current`, or the fingerprint recorded with a `Fingerprint` row whose definition fingerprints to `current`. @@ -73,6 +97,9 @@ pub(crate) fn recorded_baseline( } let definition: String = native_field(recorded, "definition")?.extract()?; let recorded_hash: String = native_field(recorded, "definition_hash")?.extract()?; + if recorded_hash == current || recorded_hash.is_empty() || definition.is_empty() { + return Ok(current); + } py.compiler_detach(|| { Ok(baseline_fingerprint( current, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs b/crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs index 56da07d274..da57aa5d34 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs @@ -129,7 +129,7 @@ pub(crate) enum PlanProject<'a, 'py> { /// A location preview's reads over the synced `state`, never deferring; other plans read live. fn preview<'py>( - request: &Bound<'py, PyAny>, + (request, project): (&Bound<'py, PyAny>, PlanProject<'_, 'py>), state: Option<&Arc>, ) -> PyResult>> { let py: Python<'py> = request.py(); @@ -146,7 +146,8 @@ fn preview<'py>( }, )); } - let synced: Arc = Arc::new(SyncedReads::new(Arc::clone(state))); + let held_project: bool = matches!(project, PlanProject::Held { .. }); + let synced: Arc = Arc::new(SyncedReads::new(Arc::clone(state), held_project)); let reads: Bound<'py, PlannerReads> = Bound::new(py, PlannerReads::new(synced.clone()))?; Ok(Some(SyncedPlan { synced, reads })) } @@ -225,7 +226,7 @@ fn resolve_invocation<'py>( } else { requested_load }; - let preview: Option> = preview(request, state)?; + let preview: Option> = preview((request, project), state)?; Ok(PlanInvocation { project_dir, discovered, diff --git a/fensu.toml b/fensu.toml index 73dcbfa978..01ae72e19a 100644 --- a/fensu.toml +++ b/fensu.toml @@ -467,11 +467,6 @@ modules = ["sqlbuild.executor.load.models"] symbols = ["LoadCallbacks", "LoadExecutionResult", "LoadRuntimeParams"] reason = "Native code resolves these through the sqlbuild._native_host name table." -[[targets.python.dead_code.roots]] -modules = ["sqlbuild.executor.migrations._helpers.relation_for_location"] -symbols = ["migration_relation_for_location"] -reason = "Native code resolves these through the sqlbuild._native_host name table." - [[targets.python.dead_code.roots]] modules = ["sqlbuild.executor.old_name_views._helpers.janitor"] symbols = ["is_compatibility_view"] @@ -2569,27 +2564,28 @@ pending_budgets = [ "crates/sqlbuild-python-planner/src/planner_identity/main/hook_version_hashes.rs=1", "crates/sqlbuild-python-planner/src/planner_identity/main/merge_recomputed_model_changes.rs=1", "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_candidates.rs=3", - "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_entries.rs=3", + "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_entries.rs=2", "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_objects.rs=1", "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_overlay.rs=8", "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_planning.rs=1", "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/column_warnings.rs=1", "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/column/columns.rs=1", - "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/discovery/discovery_events.rs=6", + "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/discovery/discovery_events.rs=4", "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/discovery/discovery_models.rs=1", "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/discovery/discovery_requests.rs=2", "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/discovery/model_discovery.rs=1", "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/discovery/state_inspection.rs=2", - "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/model_entries.rs=9", + "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/model_entries.rs=8", "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/model_handover.rs=5", "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/model_overlay.rs=4", "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/model_planning.rs=1", "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/model/model_warnings.rs=2", - "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_name_entries.rs=9", - "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_name_objects.rs=4", + "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/identity_index.rs=1", + "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_name_entries.rs=8", + "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_name_objects.rs=3", "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_name_planning.rs=2", "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_name_references.rs=7", - "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_names.rs=7", + "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/old_name/old_names.rs=5", "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/planning/planning_execution.rs=2", "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/planning/planning_origin.rs=6", "crates/sqlbuild-python-planner/src/planner_migrations/_helpers/projections/fingerprint.rs=1", @@ -3017,7 +3013,7 @@ pending_budgets = [ "crates/sqlbuild-python-planner/src/planner_warehouse/models.rs=8", "crates/sqlbuild-python-project/src/analysis/models.rs=9", "crates/sqlbuild-python-project/src/analysis_session/models.rs=1", - "crates/sqlbuild-python-project/src/compiled_project/models.rs=36", + "crates/sqlbuild-python-project/src/compiled_project/models.rs=37", "crates/sqlbuild-python-project/src/compiled_views/_helpers/compiled_models/compiled_model_view.rs=3", "crates/sqlbuild-python-project/src/compiled_views/_helpers/model_inputs/inputs_view.rs=24", "crates/sqlbuild-python-project/src/compiled_views/_helpers/project/project_view.rs=26", diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md index 7e900811e0..255a91828e 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md @@ -892,11 +892,18 @@ These stay unknown until it does: - the columns of an expression source whose SQL changed since it was last described, which the plan takes as declared. -Over the API, queue a plan, then read it until `result_json` is set: +A location keeps its last 16 plans for the current publication and synced copy, in memory, and +answers an identical request with the plan it kept instead of planning again, for up to 10 +minutes: a plan's incremental windows and expiry follow the time it was made. A newer publication +or synced copy drops them all. A kept plan's `timings` say so in `reused_from`, the time it was +made, and the page shows its age. The page's **Plan again** asks for a fresh plan. + +Over the API, queue a plan, then read it until `result_json` is set. `fresh` is required: `true` +plans again even when the location kept a plan for the same command. ```json POST /api/control/locations/{location_id}/targets/{target}/plans -{ "request_id": "0191f0aa-0000-7000-8000-0000000000a1", "command": ["build", "--select", "+orders"] } +{ "request_id": "0191f0aa-0000-7000-8000-0000000000a1", "command": ["build", "--select", "+orders"], "fresh": false } GET /api/control/locations/{location_id}/targets/{target}/plans/{request_id} ``` diff --git a/src/sqlbuild/_native_host.py b/src/sqlbuild/_native_host.py index f215c77928..6b2841f41e 100644 --- a/src/sqlbuild/_native_host.py +++ b/src/sqlbuild/_native_host.py @@ -424,10 +424,6 @@ "LoadCallbacks": ("sqlbuild.executor.load.models", "LoadCallbacks"), "LoadExecutionResult": ("sqlbuild.executor.load.models", "LoadExecutionResult"), "LoadRuntimeParams": ("sqlbuild.executor.load.models", "LoadRuntimeParams"), - "migration_relation_for_location": ( - "sqlbuild.executor.migrations._helpers.relation_for_location", - "migration_relation_for_location", - ), "is_compatibility_view": ( "sqlbuild.executor.old_name_views._helpers.janitor", "is_compatibility_view", diff --git a/src/sqlbuild/_native_host.pyi b/src/sqlbuild/_native_host.pyi index 39099b17ac..6e75602349 100644 --- a/src/sqlbuild/_native_host.pyi +++ b/src/sqlbuild/_native_host.pyi @@ -288,9 +288,6 @@ from sqlbuild.executor.load.main.run import run_load_pipeline as run_load_pipeli from sqlbuild.executor.load.models import LoadCallbacks as LoadCallbacks from sqlbuild.executor.load.models import LoadExecutionResult as LoadExecutionResult from sqlbuild.executor.load.models import LoadRuntimeParams as LoadRuntimeParams -from sqlbuild.executor.migrations._helpers.relation_for_location import ( - migration_relation_for_location as migration_relation_for_location, -) from sqlbuild.executor.old_name_views._helpers.janitor import ( is_compatibility_view as is_compatibility_view, ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/plan/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/plan/helpers.py index 1fd65c5fa5..b203e0b3da 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/plan/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/plan/helpers.py @@ -477,11 +477,18 @@ def page_of(kind: str, records: list[dict[str, Any]]) -> dict[str, Any]: def timed_plan( - served: ServedLocation, selection: list[str], settings: dict[str, Any] + served: ServedLocation, + selection: list[str], + settings: dict[str, Any], + *, + fresh: bool = False, ) -> tuple[float, dict[str, Any]]: - """A person's plan from the moment the server queues it until the location stored it.""" + """A person's plan from the moment the server queues it until the location stored it. + + `fresh` is Plan again, so the location plans anew instead of returning a plan it kept. + """ started: float = time.monotonic() - request_id: str = request_plan(served.control, selection, settings) + request_id: str = request_plan(served.control, selection, settings, fresh=fresh) deadline: float = started + 600.0 result: str | None = None while result is None: diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/plan/test_duckshop_10k_preview_performance.py b/tests/e2e/src/sqlbuild/cli/commands/main/plan/test_duckshop_10k_preview_performance.py index 2c621ff0a7..d204e94573 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/plan/test_duckshop_10k_preview_performance.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/plan/test_duckshop_10k_preview_performance.py @@ -108,8 +108,9 @@ def test_given_a_synced_10k_model_warehouse_when_previewing_a_changed_build_then mirror_synced_state(served, batches) selection: list[str] = published_keys(served.control) cold, _ = timed_plan(served, selection, {"changed": True}) - warm, document = timed_plan(served, selection, {"changed": True}) - _ = wait_for_log_count(served, PREVIEW_DONE, 2, 60.0) + warm, document = timed_plan(served, selection, {"changed": True}, fresh=True) + reused_seconds, reused = timed_plan(served, selection, {"changed": True}) + _ = wait_for_log_count(served, PREVIEW_DONE, 3, 60.0) admin: dict[str, Any] = provision_person( served.control.parents[1], "create-user", "plan-admin" ) @@ -139,7 +140,8 @@ def test_given_a_synced_10k_model_warehouse_when_previewing_a_changed_build_then print( f"duckshop 10k preview ({test_case.description}): cold={cold:.2f}s warm={warm:.2f}s " f"publish={served.published_seconds:.2f}s warm_to_publish={warm_to_publish:.2f} " - f"check_median={check_seconds * 1000:.0f}ms runs={len(reasons)} unbuilt={unbuilt} " + f"reused={reused_seconds:.2f}s check_median={check_seconds * 1000:.0f}ms " + f"runs={len(reasons)} unbuilt={unbuilt} " f"definition_changes={definition_changes} models={len(document['models'])} " f"live={live.seconds:.2f}s live_to_publish={live.seconds / served.published_seconds:.2f} " f"live_entries={live.model_entries} rebinding={live.source_rebinding_analyses} " @@ -161,6 +163,10 @@ def test_given_a_synced_10k_model_warehouse_when_previewing_a_changed_build_then assert warm < test_case.expected_max_warm_seconds, ( "a warm preview of 10K models must stay under ten seconds" ) + assert reused["timings"]["reused_from"] and reused["models"] == document["models"], ( + "the same request again returns the plan the location kept" + ) + assert reused_seconds < warm, "a kept plan comes back without planning again" assert warm_to_publish < test_case.expected_max_warm_to_publish, ( "planning grew faster than compiling: look for a lookup over every model or relation" ) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py index 57a4eabdeb..3b0cc10f74 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py @@ -532,9 +532,12 @@ class PlanPreviewTestCase: class HeldPlanTestCase: description: str settings: tuple[dict[str, object], ...] + fresh: frozenset[int] expected_runs: tuple[tuple[str, ...], ...] expected_compiles: int expected_held_plans: int + expected_held_test_plannings: int + expected_reused: tuple[bool, ...] @dataclass(frozen=True) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/catalog/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/catalog/helpers.py index 0aa5817bea..96ea4148a0 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/catalog/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/catalog/helpers.py @@ -90,9 +90,16 @@ def wait_for_statistics( def request_plan( - control: Path, selection: list[str], settings: dict[str, object] | None = None + control: Path, + selection: list[str], + settings: dict[str, object] | None = None, + *, + fresh: bool = False, ) -> str: - """Queue the plan work a person's Plan creates, frozen to exact node keys and settings.""" + """Queue the plan work a person's Plan creates, frozen to exact node keys and settings. + + `fresh` is Plan again: the location plans anew rather than return a plan it kept. + """ request_id: str = str(uuid4()) now: datetime = datetime.now(UTC) with closing(sqlite3.connect(control)) as database: @@ -101,7 +108,7 @@ def request_plan( "request_json,requested_by,requested_at,expires_at) SELECT ?,c.location_id,c.target," "c.snapshot_id,s.generation,json_object('request_id',?,'location_id',c.location_id," "'target',c.target,'reason','Plan','work',json_object('kind','plan','command'," - "json('[\"build\"]'),'selection',json(?),'settings',json(?)))," + "json('[\"build\"]'),'selection',json(?),'settings',json(?),'fresh',json(?)))," "'test-admin',?,? FROM sqb_current_snapshots c JOIN sqb_snapshots s " "ON s.snapshot_id=c.snapshot_id", ( @@ -109,6 +116,7 @@ def request_plan( request_id, json.dumps(selection), json.dumps(settings or {}), + json.dumps(fresh), now.strftime("%Y-%m-%dT%H:%M:%S.%fZ"), (now + timedelta(minutes=5)).strftime("%Y-%m-%dT%H:%M:%S.%fZ"), ), diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py index fdc1552912..2b646a73be 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py @@ -1886,9 +1886,15 @@ def changed_build_and_plan_from_copy( def plans_from_copy( - tmp_path: Path, project: Path, requests: list[tuple[list[str], dict[str, object]]] + tmp_path: Path, + project: Path, + requests: list[tuple[list[str], dict[str, object]]], + fresh: frozenset[int] = frozenset(), ) -> list[dict[str, Any]]: - """Serve `project` until its first sync completes, then plan each request from the copy.""" + """Serve `project` until its first sync completes, then plan each request from the copy. + + The requests at the `fresh` positions are Plan again, which plans anew instead of reusing. + """ control: Path = tmp_path / "control/.sqlbuild/control.sqlite3" log: Path = tmp_path / "location.log" started: datetime = datetime.now(UTC) @@ -1922,8 +1928,8 @@ def plans_from_copy( log, after=started, ) - for selection, settings in requests: - request_id: str = request_plan(control, selection, settings) + for position, (selection, settings) in enumerate(requests): + request_id: str = request_plan(control, selection, settings, fresh=position in fresh) result: dict[str, Any] = wait_for_location_work(api, request_id, location, log) outcome: dict[str, Any] = result["result_json"]["outcome"] assert outcome["kind"] == "plan", outcome diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_plan_preview.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_plan_preview.py index 793533dc8a..4f92c5c88c 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_plan_preview.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_plan_preview.py @@ -145,6 +145,12 @@ "recorded migrations", ] _HELD: str = "Planning from the published compiled project." +_HELD_TESTS: str = "Planned the publication's 1 SQL tests once for previews." +_REFUNDS_TEST: str = ( + "TEST ();\n\nWITH\n" + "__source__raw_refunds AS (SELECT 1 AS refund_id, 100 AS amount_cents),\n" + "__expected__refunds AS (SELECT 1 AS refund_id, 100 AS amount_cents)\n" +) _SYNCED_READING: str = "Reading the synced warehouse state..." _REPLAYED: str = ( "cursor window of daily_orders (the run starts it away from the last build's cursor)" @@ -455,16 +461,21 @@ def changed_build() -> dict[str, list[str]]: HeldPlanTestCase( description=( "only the plan that sets variables compiles the project again; the other two " - "and the changed-build count plan from the published project" + "and the changed-build count plan from the published project, and the same " + "request again returns the kept plan unless it asks to plan again" ), settings=( {"changed": True}, {"changed": True, "vars": {"batch": "2"}}, {"changed": True}, + {"changed": True}, ), - expected_runs=((), ("model:order_count",), ()), + fresh=frozenset({2}), + expected_runs=((), ("model:order_count",), (), ()), expected_compiles=1, expected_held_plans=3, + expected_held_test_plannings=1, + expected_reused=(False, False, False, True), ), ], ids=lambda case: case.description, @@ -478,6 +489,7 @@ def test_given_plan_requests_when_the_location_plans_them_then_only_variables_co "models/order_count.sql": _PROJECT_FILES["models/order_count.sql"].replace( "AS order_count", "AS order_count, @@batch AS batch" ), + "tests/unit/test_refunds.sql": _REFUNDS_TEST, } write_files(project_dir=project, files=files) built: subprocess.CompletedProcess[str] = run_sqb(project, "build", "--target", "development") @@ -486,12 +498,15 @@ def test_given_plan_requests_when_the_location_plans_them_then_only_variables_co (["model:order_count"], settings) for settings in test_case.settings ] - plans: list[dict[str, Any]] = plans_from_copy(tmp_path, project, requests) + plans: list[dict[str, Any]] = plans_from_copy(tmp_path, project, requests, test_case.fresh) log: str = (tmp_path / "location.log").read_text() assert tuple(map(changed_runs, plans)) == test_case.expected_runs, log + assert tuple("reused_from" in plan["timings"] for plan in plans) == test_case.expected_reused assert log.count("Compiling project...") == test_case.expected_compiles, log assert log.count(_HELD) == test_case.expected_held_plans, log + # Only the published project plans every SQL test, once; the plan with variables plans its own. + assert log.count(_HELD_TESTS) == test_case.expected_held_test_plannings, log assert log.count(_SYNCED_READING) == test_case.expected_held_plans + 1, log assert "Warehouse connected" not in log, "a synced copy opens no warehouse connection" diff --git a/tests/unit/src/sqlbuild/runtime/event_exporting/classes/_test_types.py b/tests/unit/src/sqlbuild/runtime/event_exporting/classes/_test_types.py index 5b534e4ed2..025c76a5ad 100644 --- a/tests/unit/src/sqlbuild/runtime/event_exporting/classes/_test_types.py +++ b/tests/unit/src/sqlbuild/runtime/event_exporting/classes/_test_types.py @@ -1,6 +1,15 @@ from dataclasses import dataclass +@dataclass(frozen=True) +class IdleDispatcherTestCase: + description: str + idle_seconds: float + polling_seconds: float + expected_max_waits: int + expected_max_delivery_seconds: float + + @dataclass(frozen=True) class EventExporterDispatcherTestCase: description: str diff --git a/tests/unit/src/sqlbuild/runtime/event_exporting/classes/test_dispatcher.py b/tests/unit/src/sqlbuild/runtime/event_exporting/classes/test_dispatcher.py index a751b26fbf..f3d6c4c8f2 100644 --- a/tests/unit/src/sqlbuild/runtime/event_exporting/classes/test_dispatcher.py +++ b/tests/unit/src/sqlbuild/runtime/event_exporting/classes/test_dispatcher.py @@ -29,6 +29,7 @@ from tests.unit.src.sqlbuild.runtime.event_exporting.classes._test_types import ( EventExporterDispatcherTestCase, HealthIntervalTestCase, + IdleDispatcherTestCase, OutcomeDeliveryTestCase, ) from tests.unit.src.sqlbuild.runtime.event_exporting.classes.helpers import ( @@ -123,6 +124,62 @@ def publish(*, event: LifecycleEvent) -> None: assert dispatcher.shutdown().delivered == test_case.expected_delivered +@pytest.mark.parametrize( + "test_case", + ( + IdleDispatcherTestCase( + description="an idle dispatcher waits for work instead of polling", + idle_seconds=0.5, + polling_seconds=0.5, + expected_max_waits=10, + expected_max_delivery_seconds=0.5, + ), + ), + ids=lambda case: case.description, +) +def test_given_idle_dispatcher_when_waiting_then_it_does_not_poll_and_still_delivers_promptly( + test_case: IdleDispatcherTestCase, + monkeypatch: pytest.MonkeyPatch, +) -> None: + delivered: list[threading.Event] = [threading.Event(), threading.Event()] + published: list[int] = [] + caller_thread: int = threading.get_ident() + waits: list[tuple[int, float | None]] = [] + condition_wait: Callable[[threading.Condition, float | None], bool] = threading.Condition.wait + + def counted_wait(self: threading.Condition, timeout: float | None = None) -> bool: + waits.append((threading.get_ident(), timeout)) + return condition_wait(self, timeout) + + monkeypatch.setattr(threading.Condition, "wait", counted_wait) + + def publish(*, event: LifecycleEvent) -> None: + del event + delivered[len(published)].set() + published.append(1) + + dispatcher: EventExporterDispatcher = EventExporterDispatcher( + exporters=(BoundEventExporter("publish", publish, {}),), + ) + dispatcher.enqueue(lifecycle_event()) + assert delivered[0].wait(timeout=2.0) + time.sleep(0.05) + waits.clear() + time.sleep(test_case.idle_seconds) + # Other threads of a busy process wait too; only waits shorter than the idle safety timeout poll. + idle_waits: int = sum( + thread != caller_thread and timeout is not None and timeout < test_case.polling_seconds + for thread, timeout in waits + ) + started: float = time.monotonic() + dispatcher.enqueue(lifecycle_event()) + + assert delivered[1].wait(timeout=2.0) + assert time.monotonic() - started < test_case.expected_max_delivery_seconds + assert idle_waits <= test_case.expected_max_waits + assert dispatcher.shutdown().delivered == 2 + + @pytest.mark.parametrize( "test_case", (EventExporterDispatcherTestCase("failure isolation", 1),), diff --git a/ui/src/lib/api/client.ts b/ui/src/lib/api/client.ts index cbd2bf022b..832972e4f8 100644 --- a/ui/src/lib/api/client.ts +++ b/ui/src/lib/api/client.ts @@ -149,8 +149,11 @@ export interface ApiClient extends PlatformApi { /** The server's pre-launch check: resolution, write set and authorization. */ checkAdhocRun(scope: Scope, request: AdhocRunRequest): Promise; launchAdhocRun(scope: Scope, request: AdhocRunRequest): Promise; - /** Asks the location to run `sqb plan` on a build command against the warehouse, and waits for it. */ - planBuild(scope: Scope, command: string[]): Promise; + /** + * Asks the location to run `sqb plan` on a build command against the warehouse, and waits for it. + * The location may answer with the plan it kept for the same command; `fresh` makes it plan again. + */ + planBuild(scope: Scope, command: string[], options?: { fresh?: boolean }): Promise; /** One page of a large section of the plan `requestId` names, which its summary only counts. */ /** One page of a plan section; `name` finds rows whose name contains it, ignoring case. */ planRows(scope: Scope, requestId: string, section: PlanPageSection, offset: number, limit?: number, name?: string): Promise; diff --git a/ui/src/lib/api/http-audit.test.ts b/ui/src/lib/api/http-audit.test.ts index 9c2b905039..7a507556d8 100644 --- a/ui/src/lib/api/http-audit.test.ts +++ b/ui/src/lib/api/http-audit.test.ts @@ -60,7 +60,7 @@ describe('toAuditEntry', () => { const allowed = (action: string, after_json: Record) => toAuditEntry({ ...base, actor: 'principal:p-1', actor_name: 'ana', action, granting_role: 'admin', scope_json: { location_id: 'loc-1', target: 'prod' }, after_json }).summary; const work = (work: Record) => allowed('location.work_requested', { request_id: 'q-1', reason: 'plan', work }); - expect(work({ kind: 'plan', command: ['build', '--select', 'orders'], selection: ['model:orders'], settings: {} })).toBe('Plan sqb build --select orders'); + expect(work({ kind: 'plan', command: ['build', '--select', 'orders'], selection: ['model:orders'], settings: {}, fresh: false })).toBe('Plan sqb build --select orders'); expect(work({ kind: 'sync_warehouse' })).toBe('Sync the warehouse copy'); expect(allowed('error.read', { run_id: 'd36ac8b2-314c-46aa', found: true })).toBe('Run d36ac8b2'); expect(allowed('output.read', { run_id: 'd36ac8b2-314c-46aa', found: false })).toBe('Run d36ac8b2, not found'); diff --git a/ui/src/lib/api/http-client.test.ts b/ui/src/lib/api/http-client.test.ts index c7ed8084dd..aa15550f12 100644 --- a/ui/src/lib/api/http-client.test.ts +++ b/ui/src/lib/api/http-client.test.ts @@ -1,5 +1,6 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { HttpChangeFeed, createHttpClient, feedAllowed } from './http-client'; +import { timingTitle } from '$lib/run/plan-timings'; import type { ChangedSignal, ConnectionState } from './signals'; /** A stand-in `EventSource` the test opens, fails and feeds by hand. */ @@ -952,3 +953,50 @@ describe('listWarehouseConnections', () => { expect(health.filter((p) => p.includes('/development/'))).toHaveLength(2); }); }); + +describe('planning a build', () => { + function server() { + const bodies: Record[] = []; + vi.stubGlobal('document', { cookie: '' }); + vi.stubGlobal( + 'fetch', + vi.fn(async (request: Request) => { + if (request.method === 'POST') { + bodies.push(await request.json()); + return Response.json({ applied: true }); + } + return Response.json({ expires_at: '2999-01-01T00:00:00Z', outcome: { kind: 'plan', document: { error: { code: 'plan_failed', message: 'stop here' } } } }); + }) + ); + return { bodies, client: createHttpClient({ baseUrl: 'http://ui.test', onUnauthorized: () => {} }) }; + } + + it('asks the location to plan again only when the person asks for a fresh plan', async () => { + const { bodies, client } = server(); + for (const options of [undefined, { fresh: true }]) { + const planned = client.planBuild(ORDERS, ['build', '--changed'], options); + await vi.advanceTimersByTimeAsync(1000); + expect((await planned).error).toBe('plan_failed: stop here'); + } + expect(bodies.map((body) => body.fresh)).toEqual([false, true]); + }); + + it("carries a kept plan's age from the server's summary to the plan it returns", async () => { + const timings = { total_seconds: 4.2, steps: [{ name: 'Detecting changes', seconds: 4.2 }], planned_from: 'synced_state', reused_from: '2026-01-02T03:04:05Z' }; + vi.stubGlobal('document', { cookie: '' }); + vi.stubGlobal( + 'fetch', + vi.fn(async (request: Request) => + request.method === 'POST' + ? Response.json({ applied: true }) + : Response.json({ expires_at: '2999-01-01T00:00:00Z', outcome: { kind: 'plan', document: { selected_count: 1, models: [], sections: [], migration_steps: 0 }, diffs: {} }, timings }) + ) + ); + const client = createHttpClient({ baseUrl: 'http://ui.test', onUnauthorized: () => {} }); + const planned = client.planBuild(ORDERS, ['build', '--changed']); + await vi.advanceTimersByTimeAsync(1000); + const { plan } = await planned; + expect(plan?.timings).toEqual(timings); + expect(timingTitle(plan!.timings!, Date.parse('2026-01-02T03:10:05Z'))).toBe('Reused plan from 6m ago'); + }); +}); diff --git a/ui/src/lib/api/http-client.ts b/ui/src/lib/api/http-client.ts index 63b8b3ba3c..251502a20c 100644 --- a/ui/src/lib/api/http-client.ts +++ b/ui/src/lib/api/http-client.ts @@ -599,12 +599,12 @@ export function createHttpClient({ baseUrl = '', onUnauthorized, fetch: answer, * Queues the plan, then waits for the location's summary: changed models in full and the large sections counted. * The first page of each large section loads with it; a plan the location could not make is a result, not a throw. */ - async planBuild(scope: Scope, command: string[]): Promise { + async planBuild(scope: Scope, command: string[], options: { fresh?: boolean } = {}): Promise { const { locationId, target } = parseLocationKey(scope.locationId); const path = { location_id: locationId, target }; const requestId = newRequestId(); const asked = await http.POST('/api/control/locations/{location_id}/targets/{target}/plans', { - body: { request_id: requestId, command }, + body: { request_id: requestId, command, fresh: options.fresh ?? false }, params: { path, header: { 'x-sqb-csrf': csrfToken(document.cookie) } } }); if (asked.response.status === 401) onUnauthorized(); diff --git a/ui/src/lib/api/mock/mock-client.ts b/ui/src/lib/api/mock/mock-client.ts index 0b1881a4ec..4a82d99801 100644 --- a/ui/src/lib/api/mock/mock-client.ts +++ b/ui/src/lib/api/mock/mock-client.ts @@ -371,7 +371,7 @@ export class MockApiClient implements ApiClient { return this.#operate.launchAdhocRun(scope, request); } - async planBuild(scope: Scope, command: string[]): Promise { + async planBuild(scope: Scope, command: string[], _options: { fresh?: boolean } = {}): Promise { this.#location(scope); // `--changed` starts from the nodes whose own definition changed, as the catalog marks them. const changed = command.some((t) => t === '--changed' || t.startsWith('--changed=')) diff --git a/ui/src/lib/api/types/operate.ts b/ui/src/lib/api/types/operate.ts index 4d3dc595c0..3257a30382 100644 --- a/ui/src/lib/api/types/operate.ts +++ b/ui/src/lib/api/types/operate.ts @@ -780,6 +780,8 @@ export type PlanTimings = { steps: { name: string; seconds: number }[]; /** `synced_state` for a location preview over the synced warehouse state; `live` when it read the warehouse. */ planned_from: string; + /** When the plan the location kept, and answered with again, was made; absent for a new plan. */ + reused_from?: string | null; }; /** The server's synced copy of warehouse metadata a location's plan preview planned from. */ diff --git a/ui/src/lib/run/plan-timings.svelte b/ui/src/lib/run/plan-timings.svelte index 96a9784494..065ef90b10 100644 --- a/ui/src/lib/run/plan-timings.svelte +++ b/ui/src/lib/run/plan-timings.svelte @@ -5,15 +5,15 @@ import { cn } from '$lib/utils'; import TimerIcon from '@lucide/svelte/icons/timer'; import type { PlanDocument, PlanTimings } from '$lib/api'; - import { seconds, timingFootnote, timingRows } from './plan-timings'; + import { seconds, timingFootnote, timingRows, timingTitle } from './plan-timings'; - let { plan, timings }: { plan: PlanDocument; timings: PlanTimings } = $props(); + let { plan, timings, now }: { plan: PlanDocument; timings: PlanTimings; now: number } = $props(); const rows = $derived(timingRows(timings)); - Planned in {seconds(timings.total_seconds)} s + {timingTitle(timings, now)}
@@ -32,7 +32,7 @@ Total{seconds(timings.total_seconds)} -

{timingFootnote(plan, timings)}

+

{timingFootnote(plan, timings, now)}

diff --git a/ui/src/lib/run/plan-timings.test.ts b/ui/src/lib/run/plan-timings.test.ts index d7152cd026..208eb6042f 100644 --- a/ui/src/lib/run/plan-timings.test.ts +++ b/ui/src/lib/run/plan-timings.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from 'vitest'; import type { PlanDocument, PlanTimings } from '$lib/api'; -import { timingFootnote, timingRows } from './plan-timings'; +import { timingFootnote, timingRows, timingTitle } from './plan-timings'; const timings: PlanTimings = { total_seconds: 10, @@ -25,7 +25,16 @@ describe('plan timings', () => { it('says where the plan ran and how many nodes it planned', () => { const plan = { selected_count: 2418 } as PlanDocument; - expect(timingFootnote(plan, timings)).toBe('On the location, from the synced warehouse state · 2,418 nodes'); - expect(timingFootnote(plan, { ...timings, planned_from: 'live' })).toBe('On the location, live from the warehouse · 2,418 nodes'); + const now = Date.parse('2026-01-02T03:10:05Z'); + expect(timingFootnote(plan, timings, now)).toBe('On the location, from the synced warehouse state · 2,418 nodes'); + expect(timingFootnote(plan, { ...timings, planned_from: 'live' }, now)).toBe('On the location, live from the warehouse · 2,418 nodes'); + }); + + it("labels a kept plan by its age instead of a planning time", () => { + const plan = { selected_count: 2418 } as PlanDocument; + const now = Date.parse('2026-01-02T03:10:05Z'); + const reused = { ...timings, reused_from: '2026-01-02T03:04:05Z' }; + expect([timingTitle(timings, now), timingTitle(reused, now)]).toEqual(['Planned in 10.0 s', 'Reused plan from 6m ago']); + expect(timingFootnote(plan, reused, now)).toBe("On the location, from the synced warehouse state · 2,418 nodes · the location's plan from 6m ago; Plan again for a new one"); }); }); diff --git a/ui/src/lib/run/plan-timings.ts b/ui/src/lib/run/plan-timings.ts index 4215f659b1..8e953847d0 100644 --- a/ui/src/lib/run/plan-timings.ts +++ b/ui/src/lib/run/plan-timings.ts @@ -2,6 +2,7 @@ import type { PlanDocument, PlanTimings } from '$lib/api'; import { plural } from '$lib/format/plural'; +import { formatAgo } from '$lib/format/time'; export type TimingRow = { name: string; seconds: string; share: number; percent: string }; @@ -19,8 +20,14 @@ export function timingRows(timings: PlanTimings): TimingRow[] { }); } -/** Where the plan ran and from what, with the nodes it planned. */ -export function timingFootnote(plan: PlanDocument, timings: PlanTimings): string { +/** The popover's label: how long planning took, or how old the kept plan the location answered with is. */ +export function timingTitle(timings: PlanTimings, now: number): string { + return timings.reused_from ? `Reused plan from ${formatAgo(timings.reused_from, now)}` : `Planned in ${seconds(timings.total_seconds)} s`; +} + +/** Where the plan ran and from what, with the nodes it planned and, for a kept plan, when it was made. */ +export function timingFootnote(plan: PlanDocument, timings: PlanTimings, now: number): string { const from = timings.planned_from === 'live' ? 'On the location, live from the warehouse' : 'On the location, from the synced warehouse state'; - return `${from} · ${plural(plan.selected_count, 'node')}`; + const reused = timings.reused_from ? ` · the location's plan from ${formatAgo(timings.reused_from, now)}; Plan again for a new one` : ''; + return `${from} · ${plural(plan.selected_count, 'node')}${reused}`; } diff --git a/ui/src/routes/plan/+page.svelte b/ui/src/routes/plan/+page.svelte index 93ca16f293..1ed41efbed 100644 --- a/ui/src/routes/plan/+page.svelte +++ b/ui/src/routes/plan/+page.svelte @@ -133,6 +133,8 @@ let plannedKey = $state(''); let planning = $state(false); let planError = $state<{ key: string; message: string } | null>(null); + /** Set by "Plan again": the next plan skips the plan the location kept for the same command. */ + let planFresh = false; const wantsPlan = $derived(fresh && check?.verb === 'build' && !check.error); $effect(() => { const key = requestKey; @@ -141,7 +143,9 @@ const timer = setTimeout(async () => { planning = true; try { - const result = await app.client.planBuild(scope, request.command); + const replanned = planFresh; + planFresh = false; + const result = await app.client.planBuild(scope, request.command, { fresh: replanned }); if (result.plan) { plan = result.plan; planRequestId = result.requestId ?? null; @@ -179,6 +183,7 @@ return app.client.planDiff(scope, requestId, model); } function replan() { + planFresh = true; plannedKey = ''; planError = null; } @@ -455,7 +460,7 @@ (view = v)} /> {#if shown.verb === 'build' && plan} - {#if plan.timings}{/if} + {#if plan.timings}{/if} {/if} diff --git a/website/src/content/docs/docs/cli/server.mdx b/website/src/content/docs/docs/cli/server.mdx index 2cc6e0df5c..2cea042e05 100644 --- a/website/src/content/docs/docs/cli/server.mdx +++ b/website/src/content/docs/docs/cli/server.mdx @@ -871,11 +871,18 @@ These stay unknown until it does: - the columns of an expression source whose SQL changed since it was last described, which the plan takes as declared. -Over the API, queue a plan, then read it until `result_json` is set: +A location keeps its last 16 plans for the current publication and synced copy, in memory, and +answers an identical request with the plan it kept instead of planning again, for up to 10 +minutes: a plan's incremental windows and expiry follow the time it was made. A newer publication +or synced copy drops them all. A kept plan's `timings` say so in `reused_from`, the time it was +made, and the page shows its age. The page's **Plan again** asks for a fresh plan. + +Over the API, queue a plan, then read it until `result_json` is set. `fresh` is required: `true` +plans again even when the location kept a plan for the same command. ```json POST /api/control/locations/{location_id}/targets/{target}/plans -{ "request_id": "0191f0aa-0000-7000-8000-0000000000a1", "command": ["build", "--select", "+orders"] } +{ "request_id": "0191f0aa-0000-7000-8000-0000000000a1", "command": ["build", "--select", "+orders"], "fresh": false } GET /api/control/locations/{location_id}/targets/{target}/plans/{request_id} ```