From f86ed5fec31d392611a74c246ffb27ba26dfa8e2 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 18 Nov 2026 14:29:08 +0000 Subject: [PATCH 1/7] fix(state): record each run in every schema it writes, not the target's A run appended its started and finished rows to _sqlbuild_runs in the target's default schema, creating that schema when missing. A build that writes only to other schemas then needed rights on a schema it never builds in, and could create a schema solely to hold state. Each run now writes both rows, with the same run ID, to every schema it writes to: the schemas of the models and seeds it runs (plan action not skip), its SQL functions and the sources it loads; for `sqb seed`, its seeds' schemas. The set is fixed from the plan before the first node runs, so each schema gets the pair. The writes run concurrently. As before, the started row sets each schema's runs table up (creating the schema, which the build would create anyway) and the finished row inserts directly, setting the table up again only when an ordinary error stops that insert; an interrupt still stops the run. A failed write warns naming that schema's table and never fails the run; the other schemas still record it. A run whose nodes write no relation records nothing. Readers merge the rows by run ID. --- .../src/native_host/_helpers/mod.rs | 2 + .../src/native_host/_helpers/run_locations.rs | 18 ++ .../src/native_host/_helpers/run_writes.rs | 263 ++++++++++++++++++ .../src/native_host/constants.rs | 4 +- .../native_host/main/build_write_locations.rs | 34 +++ .../src/native_host/main/mod.rs | 2 + .../src/native_host/main/run_recorder.rs | 11 +- .../native_host/main/seed_write_locations.rs | 16 ++ .../src/native_host/models.rs | 177 ++++-------- .../_helpers/build_command/command.rs | 2 + .../bindings/_helpers/seed_command/command.rs | 2 + .../sqlbuild/references/docs/cli/build.md | 20 +- .../sqlbuild/cli/commands/main/_test_types.py | 13 + .../src/sqlbuild/cli/commands/main/helpers.py | 92 +++++- .../cli/commands/main/test_run_records.py | 97 ++++++- website/src/content/docs/docs/cli/build.mdx | 20 +- 16 files changed, 620 insertions(+), 153 deletions(-) create mode 100644 crates/sqlbuild-python-project/src/native_host/_helpers/run_locations.rs create mode 100644 crates/sqlbuild-python-project/src/native_host/_helpers/run_writes.rs create mode 100644 crates/sqlbuild-python-project/src/native_host/main/build_write_locations.rs create mode 100644 crates/sqlbuild-python-project/src/native_host/main/seed_write_locations.rs diff --git a/crates/sqlbuild-python-project/src/native_host/_helpers/mod.rs b/crates/sqlbuild-python-project/src/native_host/_helpers/mod.rs index 9d5bee8dad..e8ecf03a6c 100644 --- a/crates/sqlbuild-python-project/src/native_host/_helpers/mod.rs +++ b/crates/sqlbuild-python-project/src/native_host/_helpers/mod.rs @@ -1,3 +1,5 @@ pub(crate) mod orchestration_triggers; +pub(crate) mod run_locations; +pub(crate) mod run_writes; pub(crate) mod runs_state; pub(crate) mod synced_catalog; diff --git a/crates/sqlbuild-python-project/src/native_host/_helpers/run_locations.rs b/crates/sqlbuild-python-project/src/native_host/_helpers/run_locations.rs new file mode 100644 index 0000000000..d8ecfc71ad --- /dev/null +++ b/crates/sqlbuild-python-project/src/native_host/_helpers/run_locations.rs @@ -0,0 +1,18 @@ +//! The schemas a run writes, where it records its rows in the runs table. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_front::native_host::models::Location; + +/// The schema of a relation the run writes; None for a relation without one. +pub(crate) fn written_schema(destination: &Bound<'_, PyAny>) -> PyResult> { + let schema: Option = native_field(destination, "schema")?.extract()?; + schema + .map(|schema| { + Ok(Location { + database: native_field(destination, "database")?.extract()?, + schema, + }) + }) + .transpose() +} diff --git a/crates/sqlbuild-python-project/src/native_host/_helpers/run_writes.rs b/crates/sqlbuild-python-project/src/native_host/_helpers/run_writes.rs new file mode 100644 index 0000000000..2571e8da65 --- /dev/null +++ b/crates/sqlbuild-python-project/src/native_host/_helpers/run_writes.rs @@ -0,0 +1,263 @@ +//! Writes of one run's rows to the runs table of every schema it writes, over a few connections. + +use crate::native_host::_helpers::runs_state::CREATE_TABLE_METHOD; +use crate::native_host::_helpers::runs_state::WAREHOUSE_USER_METHOD; +use crate::native_host::_helpers::runs_state::row_cells; +use crate::native_host::models::{RunIdentity, RunTable}; +use pyo3::exceptions::{PyException, PyRuntimeError}; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use sqlbuild_core::constants::RUNS_TABLE_NAME; +use sqlbuild_python_base::boundary::main::call_with::call_with; +use sqlbuild_python_base::native_host::main::write_console::write_console; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_front::native_host::main::execute::execute; +use sqlbuild_python_front::native_host::main::literal::literal; +use sqlbuild_state::runs::constants::RUN_COLUMN_TYPES; +use sqlbuild_state::runs::models::RunNodeCounts; +use sqlbuild_state::state_tables::main::values_insert_sql::values_insert_sql; + +/// A row's phase and, for `finished`, its status and node counts. +pub(crate) type RunRow<'a> = (&'a str, Option<(&'a str, &'a RunNodeCounts)>); + +/// How a row reaches a schema's runs table. +#[derive(Clone, Copy, Debug)] +pub(crate) enum RowWrite { + /// Set the schema and table up, then insert: the started row, once per run. + SetUpFirst, + /// Insert, and set the table up and insert again only when an ordinary error stops the insert. + InsertFirst, +} + +/// What one connection's share of schemas came to, by table index, and whether it closed. +pub(crate) struct ShareResult { + pub(crate) written: Vec<(usize, PyResult<()>)>, + pub(crate) closed: PyResult<()>, +} + +/// Write `row` to each share's tables over one connection per share; the first share on this thread. +pub(crate) fn write_shares<'py>( + (adapter, connection_config): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + (tables, identity): (&[RunTable], &RunIdentity), + shares: &[Vec], + (row, mode): (RunRow<'_>, RowWrite), +) -> Vec { + let py: Python<'py> = adapter.py(); + let Some((first, rest)) = shares.split_first() else { + return Vec::new(); + }; + let unbound: (Py, Py) = + (adapter.clone().unbind(), connection_config.clone().unbind()); + std::thread::scope(|scope| { + let writers: Vec<_> = rest + .iter() + .map(|share| { + let (adapter, connection_config) = (&unbound.0, &unbound.1); + scope.spawn(move || { + Python::attach(|py| { + RunWriter { + adapter: adapter.bind(py), + connection_config: connection_config.bind(py), + identity, + } + .write_share(share, tables, (row, mode)) + }) + }) + }) + .collect(); + let mut shared: Vec = vec![ + RunWriter { + adapter, + connection_config, + identity, + } + .write_share(first, tables, (row, mode)), + ]; + for (writer, share) in writers.into_iter().zip(rest) { + shared.push( + py.detach(|| writer.join()) + .unwrap_or_else(|_| panicked_share(share)), + ); + } + shared + }) +} + +/// Warn for each table whose write failed; an interrupt or exit is not a failed write and escapes. +pub(crate) fn warn_on_errors( + py: Python<'_>, + (tables, identity): (&[RunTable], &RunIdentity), + phase: &str, + shared: Vec, +) -> PyResult<()> { + let mut written: Vec<(usize, PyResult<()>)> = Vec::with_capacity(tables.len()); + let mut escaped: Option = None; + for share in shared { + match share.closed { + Err(error) if error.is_instance_of::(py) => { + let _ = write_console( + py, + "stderr", + &format!("Warning: could not close a {RUNS_TABLE_NAME} connection: {error}\n"), + ); + } + Err(error) => { + escaped.get_or_insert(error); + } + Ok(()) => {} + } + written.extend(share.written); + } + written.sort_by_key(|(index, _)| *index); + for (index, result) in written { + match result { + Err(error) if error.is_instance_of::(py) => { + let _ = write_console( + py, + "stderr", + &format!( + "Warning: could not record that run {} {phase} in {}: {error}\n", + identity.run_id, tables[index].table + ), + ); + } + Err(error) => { + escaped.get_or_insert(error); + } + Ok(()) => {} + } + } + escaped.map_or(Ok(()), Err) +} + +/// What a share came to when its writer thread panicked: every one of its tables failed. +fn panicked_share(share: &[usize]) -> ShareResult { + let failed = |index: &usize| { + ( + *index, + Err(PyRuntimeError::new_err("the run record writer panicked")), + ) + }; + ShareResult { + written: share.iter().map(failed).collect(), + closed: Ok(()), + } +} + +/// Writes one run's rows over one connection, on the thread that owns it. +struct RunWriter<'a, 'py> { + adapter: &'a Bound<'py, PyAny>, + connection_config: &'a Bound<'py, PyAny>, + identity: &'a RunIdentity, +} + +impl<'py> RunWriter<'_, 'py> { + /// Write the row to each table of `share` over one connection; a failure stays with its table. + /// + /// Connect for these rows only, so loaders can still lock file warehouses during the build. + fn write_share( + &self, + share: &[usize], + tables: &[RunTable], + (row, mode): (RunRow<'_>, RowWrite), + ) -> ShareResult { + let py: Python<'py> = self.adapter.py(); + let connected: PyResult> = native_field(self.adapter, "connect") + .and_then(|connect| connect.call1((self.connection_config,))); + let connection: Bound<'py, PyAny> = match connected { + Ok(connection) => connection, + Err(error) => { + return ShareResult { + written: share + .iter() + .map(|index| (*index, Err(error.clone_ref(py)))) + .collect(), + closed: Ok(()), + }; + } + }; + let mut written: Vec<(usize, PyResult<()>)> = Vec::with_capacity(share.len()); + for index in share { + let result: PyResult<()> = self.append_row(&connection, &tables[*index], (row, mode)); + let interrupted: bool = + matches!(&result, Err(error) if !error.is_instance_of::(py)); + written.push((*index, result)); + if interrupted { + break; + } + } + let closed: PyResult<()> = native_field(self.adapter, "close") + .and_then(|close| close.call1((&connection,))) + .map(|_| ()); + ShareResult { written, closed } + } + + /// Write one row to one table; a retry after an ordinary error reuses the same connection. + fn append_row( + &self, + connection: &Bound<'py, PyAny>, + table: &RunTable, + (row, mode): (RunRow<'_>, RowWrite), + ) -> PyResult<()> { + match mode { + RowWrite::SetUpFirst => self.append((connection, table), row, true), + RowWrite::InsertFirst => { + self.append((connection, table), row, false) + .or_else(|error| { + if error.is_instance_of::(self.adapter.py()) { + self.append((connection, table), row, true) + } else { + Err(error) + } + }) + } + } + } + + /// The schema is created only because the build writes to it; it never exists just for state. + fn append( + &self, + (connection, table): (&Bound<'py, PyAny>, &RunTable), + row: RunRow<'_>, + ensure_table: bool, + ) -> PyResult<()> { + let py: Python<'py> = self.adapter.py(); + if ensure_table { + let keywords = table.location.keywords(py); + for statement in call_with( + &native_field(self.adapter, "render_create_schema")?, + &keywords, + )? + .try_iter()? + { + execute(self.adapter, connection, &statement?.extract::()?)?; + } + let create: String = + call_with(&native_field(self.adapter, CREATE_TABLE_METHOD)?, &keywords)? + .extract()?; + if !create.is_empty() { + execute(self.adapter, connection, &create)?; + } + } + let recorded_at: String = call_with( + &native_field(self.adapter, "render_current_utc_timestamp")?, + &[], + )? + .extract()?; + let warehouse_user: String = + match call_with(&native_field(self.adapter, WAREHOUSE_USER_METHOD)?, &[])? + .extract::>()? + { + Some(expression) => expression, + None => literal(py, None)?, + }; + let columns: Vec<&str> = RUN_COLUMN_TYPES.iter().map(|(column, _)| *column).collect(); + let cells: Vec = + row_cells(py, self.identity, row, (&warehouse_user, &recorded_at))?; + execute( + self.adapter, + connection, + &values_insert_sql(&table.table, &columns, &[cells]), + ) + .map(|_| ()) + } +} diff --git a/crates/sqlbuild-python-project/src/native_host/constants.rs b/crates/sqlbuild-python-project/src/native_host/constants.rs index 2c58c12343..9ee01d749a 100644 --- a/crates/sqlbuild-python-project/src/native_host/constants.rs +++ b/crates/sqlbuild-python-project/src/native_host/constants.rs @@ -1,4 +1,4 @@ -//! Names a plan from synced state uses to find and attach the target's mirrored catalog. +//! Names a plan from synced state uses to find and attach the target's mirrored catalog, and run records' plan names. /// The `plan_from` source that plans from the target's synced copy of warehouse state. pub(crate) const SYNCED_STATE: &str = "synced_state"; @@ -10,3 +10,5 @@ pub(crate) const CONTROL_STORE_FILE: &str = ".sqlbuild/control.sqlite3"; pub(crate) const RELATION_CHANGES_METHOD: &str = "read_relation_changes"; /// The adapter attribute inspection reads the synced catalog from. pub(crate) const SYNCED_CATALOG_ATTRIBUTE: &str = "synced_catalog"; +/// The plan action of a node the build leaves as it is. +pub(crate) const SKIP_ACTION: &str = "skip"; diff --git a/crates/sqlbuild-python-project/src/native_host/main/build_write_locations.rs b/crates/sqlbuild-python-project/src/native_host/main/build_write_locations.rs new file mode 100644 index 0000000000..3268544366 --- /dev/null +++ b/crates/sqlbuild-python-project/src/native_host/main/build_write_locations.rs @@ -0,0 +1,34 @@ +//! The schemas a build writes, which each record its run. + +use crate::native_host::_helpers::run_locations::written_schema; +use crate::native_host::constants::SKIP_ACTION; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_front::native_host::models::Location; +use std::collections::BTreeSet; + +/// The schemas a build writes: its models' and seeds' that run, its functions' and its loaded sources'. +pub fn build_write_locations(plan: &Bound<'_, PyAny>) -> PyResult> { + let mut schemas: BTreeSet = BTreeSet::new(); + for entries in ["model_entries", "seed_entries"] { + for entry in native_field(plan, entries)?.try_iter()? { + let entry: Bound<'_, PyAny> = entry?; + if !native_field(&entry, "action")?.eq(SKIP_ACTION)? { + schemas.extend(written_schema(&native_field(&entry, "destination")?)?); + } + } + } + for entry in native_field(plan, "function_entries")?.try_iter()? { + schemas.extend(written_schema(&native_field(&entry?, "destination")?)?); + } + let sources: Bound<'_, PyAny> = native_field(plan, "source_map")?; + for entry in native_field(plan, "source_load_entries")?.try_iter()? { + if let Some(source) = sources + .call_method1("get", (native_field(&entry?, "name")?,))? + .extract::>>()? + { + schemas.extend(written_schema(&source)?); + } + } + Ok(schemas.into_iter().collect()) +} diff --git a/crates/sqlbuild-python-project/src/native_host/main/mod.rs b/crates/sqlbuild-python-project/src/native_host/main/mod.rs index 97b1fd6b68..b96ce720f0 100644 --- a/crates/sqlbuild-python-project/src/native_host/main/mod.rs +++ b/crates/sqlbuild-python-project/src/native_host/main/mod.rs @@ -1,6 +1,8 @@ pub mod attach_synced_catalog; +pub mod build_write_locations; pub mod compile_calendars; pub mod compile_sensors; pub mod register_runs_state; pub mod report_synced_catalog; pub mod run_recorder; +pub mod seed_write_locations; diff --git a/crates/sqlbuild-python-project/src/native_host/main/run_recorder.rs b/crates/sqlbuild-python-project/src/native_host/main/run_recorder.rs index a4959440f4..27a5c921b3 100644 --- a/crates/sqlbuild-python-project/src/native_host/main/run_recorder.rs +++ b/crates/sqlbuild-python-project/src/native_host/main/run_recorder.rs @@ -4,20 +4,20 @@ use crate::native_host::_helpers::runs_state::run_identity; use crate::native_host::models::{RunCommandScope, RunIdentity, RunRecorder}; use pyo3::exceptions::PyException; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; -use sqlbuild_analysis::assembly::target_checks::constants::PRESERVE_TARGET_VALUE; use sqlbuild_core::constants::RUNS_TABLE_NAME; use sqlbuild_python_base::native_host::main::write_console::write_console; use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_front::native_host::models::Location; use sqlbuild_state::runs::models::ServerRun; use std::path::PathBuf; -/// A run's recorder with its started row written, or none for a `preserve` schema; only an interrupt errors. +/// A run's recorder with its started row written in each schema it writes; only an interrupt errors. pub fn run_recorder<'py>( (command, sqlbuild_version): (&'static str, &'static str), (request, invocation, project): (&Bound<'py, PyAny>, &Bound<'py, PyAny>, &Bound<'py, PyAny>), + written: impl FnOnce() -> PyResult>, server: Option, ) -> PyResult>> { - let location = |value: Option| value.filter(|value| value != PRESERVE_TARGET_VALUE); let opened = || -> PyResult>> { let project_dir: PathBuf = native_field(invocation, "effective_project_dir")?.extract()?; let settings: Bound<'py, PyAny> = native_field( @@ -47,10 +47,7 @@ pub fn run_recorder<'py>( &native_field(invocation, "adapter")?, &native_field(invocation, "connection_config")?, ), - ( - location(native_field(project, "effective_target_database")?.extract()?), - location(native_field(project, "effective_target_schema")?.extract()?), - ), + written()?, identity, ) }; diff --git a/crates/sqlbuild-python-project/src/native_host/main/seed_write_locations.rs b/crates/sqlbuild-python-project/src/native_host/main/seed_write_locations.rs new file mode 100644 index 0000000000..c0446e5034 --- /dev/null +++ b/crates/sqlbuild-python-project/src/native_host/main/seed_write_locations.rs @@ -0,0 +1,16 @@ +//! The schemas `sqb seed` writes, which each record its run. + +use crate::native_host::_helpers::run_locations::written_schema; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_front::native_host::models::Location; +use std::collections::BTreeSet; + +/// The schemas `sqb seed` writes: those of every seed it loads. +pub fn seed_write_locations(plan: &Bound<'_, PyAny>) -> PyResult> { + let mut schemas: BTreeSet = BTreeSet::new(); + for entry in native_field(plan, "seed_entries")?.try_iter()? { + schemas.extend(written_schema(&native_field(&entry?, "destination")?)?); + } + Ok(schemas.into_iter().collect()) +} diff --git a/crates/sqlbuild-python-project/src/native_host/models.rs b/crates/sqlbuild-python-project/src/native_host/models.rs index c982fe1c73..9845db3acd 100644 --- a/crates/sqlbuild-python-project/src/native_host/models.rs +++ b/crates/sqlbuild-python-project/src/native_host/models.rs @@ -1,13 +1,14 @@ //! Public records of native host. use crate::compiled_views::models::NativeCompiledModel; +use crate::native_host::_helpers::run_writes::{ + RowWrite, RunRow, ShareResult, warn_on_errors, write_shares, +}; use crate::native_host::_helpers::runs_state::CREATE_TABLE_METHOD; -use crate::native_host::_helpers::runs_state::WAREHOUSE_USER_METHOD; -use crate::native_host::_helpers::runs_state::row_cells; use crate::planner_scope::models::ScopeData; use pyo3::IntoPyObject; use pyo3::PyErr; -use pyo3::exceptions::{PyException, PyKeyboardInterrupt}; +use pyo3::exceptions::PyKeyboardInterrupt; use pyo3::prelude::Bound; use pyo3::prelude::Py; use pyo3::prelude::PyAny; @@ -16,18 +17,13 @@ use pyo3::prelude::PyResult; use pyo3::prelude::Python; use sqlbuild_core::constants::RUNS_TABLE_NAME; use sqlbuild_orchestration::declarations::models::TriggerDefinition; -use sqlbuild_python_base::boundary::main::call_with::call_with; -use sqlbuild_python_base::native_host::main::write_console::write_console; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; -use sqlbuild_python_front::native_host::main::execute::execute; -use sqlbuild_python_front::native_host::main::literal::literal; use sqlbuild_python_front::native_host::main::state_table_name::state_table_name; use sqlbuild_python_front::native_host::models::Location; -use sqlbuild_state::runs::constants::{FINISHED_PHASE, RUN_COLUMN_TYPES, STARTED_PHASE}; +use sqlbuild_state::runs::constants::{FINISHED_PHASE, STARTED_PHASE}; use sqlbuild_state::runs::main::run_status::run_status; use sqlbuild_state::runs::models::{RunEnding, RunNodeCounts}; -use sqlbuild_state::state_tables::main::values_insert_sql::values_insert_sql; use std::path::Path; use std::sync::Arc; @@ -175,59 +171,66 @@ pub struct RunCommandScope<'a> { pub sqlbuild_version: &'static str, } -/// A build's two rows in its target's runs table; a failed write warns and never fails the build. +/// The runs table in one schema the run writes to. +#[derive(Debug)] +pub struct RunTable { + pub location: Location, + pub table: String, +} + +/// A run's two rows in the runs table of each schema it writes to; a failed write only warns. #[derive(Debug)] pub struct RunRecorder<'py> { adapter: Bound<'py, PyAny>, connection_config: Bound<'py, PyAny>, - location: Location, - table: String, + tables: Vec, identity: RunIdentity, } impl<'py> RunRecorder<'py> { - /// A recorder when the adapter keeps a runs table and the target has a schema; never connects. + /// A recorder when the adapter keeps a runs table and the run writes a schema; never connects. pub fn open( (adapter, connection_config): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), - (database, schema): (Option, Option), + locations: Vec, identity: RunIdentity, ) -> PyResult> { - let Some(schema) = schema else { - return Ok(None); - }; if !adapter.hasattr(CREATE_TABLE_METHOD)? { return Ok(None); } - let location = Location { database, schema }; - let keywords = location.keywords(adapter.py()); - let Some(table) = state_table_name( - &native_field(adapter, "render_qualified_name")?, - (&keywords[0].1, &keywords[1].1), - RUNS_TABLE_NAME, - )? - else { + let render_qualified_name: Bound<'py, PyAny> = + native_field(adapter, "render_qualified_name")?; + let mut tables: Vec = Vec::with_capacity(locations.len()); + for location in locations { + let keywords = location.keywords(adapter.py()); + if let Some(table) = state_table_name( + &render_qualified_name, + (&keywords[0].1, &keywords[1].1), + RUNS_TABLE_NAME, + )? { + tables.push(RunTable { location, table }); + } + } + if tables.is_empty() { return Ok(None); - }; + } Ok(Some(Self { adapter: adapter.clone(), connection_config: connection_config.clone(), - location, - table, + tables, identity, })) } - /// Set the table up once for the run and record that it started; only an interrupt escapes. + /// Set each schema's table up once for the run and record that it started; only an interrupt escapes. pub fn started(&self) -> PyResult<()> { - self.warn_on_error(STARTED_PHASE, self.write(STARTED_PHASE, None, true)) + self.append_everywhere((STARTED_PHASE, None), RowWrite::SetUpFirst) } /// Record how the run ended and what its nodes did; only an interrupt escapes as an error. pub fn finished(&self, counts: &RunNodeCounts, ending: RunEnding) -> PyResult<()> { - let finish = (run_status(counts, ending), counts); - self.warn_on_error( - FINISHED_PHASE, - self.append_row(FINISHED_PHASE, Some(finish)), + self.append_everywhere( + (FINISHED_PHASE, Some((run_status(counts, ending), counts))), + RowWrite::InsertFirst, ) } @@ -241,98 +244,20 @@ impl<'py> RunRecorder<'py> { let _ = self.finished(&RunNodeCounts::default(), ending); } - /// Insert directly, as the started row set the table up, and set it up again only when that fails. - fn append_row(&self, phase: &str, finish: Option<(&str, &RunNodeCounts)>) -> PyResult<()> { - self.write(phase, finish, false).or_else(|error| { - if error.is_instance_of::(self.adapter.py()) { - self.write(phase, finish, true) - } else { - Err(error) - } - }) - } - - /// Warn on a failed write and carry on; an interrupt is not a failed write, so it propagates. - fn warn_on_error(&self, phase: &str, written: PyResult<()>) -> PyResult<()> { + /// Append the row to every schema's table at once, then warn for each failed write. + /// + /// The first table is written on the calling thread, so an interrupt raised there stops the run + /// as it does for a single schema; an interrupt anywhere else escapes once every write is done. + fn append_everywhere(&self, row: RunRow<'_>, mode: RowWrite) -> PyResult<()> { let py: Python<'py> = self.adapter.py(); - match written { - Err(error) if error.is_instance_of::(py) => { - let _ = write_console( - py, - "stderr", - &format!( - "Warning: could not record that run {} {phase} in {RUNS_TABLE_NAME}: {error}\n", - self.identity.run_id - ), - ); - Ok(()) - } - written => written, - } - } - - /// Connect for this row only, so loaders can still lock file warehouses during the build. - fn write( - &self, - phase: &str, - finish: Option<(&str, &RunNodeCounts)>, - ensure_table: bool, - ) -> PyResult<()> { - let connection: Bound<'py, PyAny> = - native_field(&self.adapter, "connect")?.call1((&self.connection_config,))?; - let written: PyResult<()> = self.append(&connection, (phase, finish), ensure_table); - let closed: PyResult> = - native_field(&self.adapter, "close")?.call1((&connection,)); - written?; - closed.map(|_| ()) - } - - fn append( - &self, - connection: &Bound<'py, PyAny>, - row: (&str, Option<(&str, &RunNodeCounts)>), - ensure_table: bool, - ) -> PyResult<()> { - let py: Python<'py> = self.adapter.py(); - if ensure_table { - let keywords = self.location.keywords(py); - for statement in call_with( - &native_field(&self.adapter, "render_create_schema")?, - &keywords, - )? - .try_iter()? - { - execute(&self.adapter, connection, &statement?.extract::()?)?; - } - let create: String = call_with( - &native_field(&self.adapter, CREATE_TABLE_METHOD)?, - &keywords, - )? - .extract()?; - if !create.is_empty() { - execute(&self.adapter, connection, &create)?; - } - } - let recorded_at: String = call_with( - &native_field(&self.adapter, "render_current_utc_timestamp")?, - &[], - )? - .extract()?; - let warehouse_user: String = - match call_with(&native_field(&self.adapter, WAREHOUSE_USER_METHOD)?, &[])? - .extract::>()? - { - Some(expression) => expression, - None => literal(py, None)?, - }; - let columns: Vec<&str> = RUN_COLUMN_TYPES.iter().map(|(column, _)| *column).collect(); - let cells: Vec = - row_cells(py, &self.identity, row, (&warehouse_user, &recorded_at))?; - execute( - &self.adapter, - connection, - &values_insert_sql(&self.table, &columns, &[cells]), - ) - .map(|_| ()) + let shares: Vec> = (0..self.tables.len()).map(|index| vec![index]).collect(); + let shared: Vec = write_shares( + (&self.adapter, &self.connection_config), + (&self.tables, &self.identity), + &shares, + (row, mode), + ); + warn_on_errors(py, (&self.tables, &self.identity), row.0, shared)?; + py.check_signals() } } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/command.rs index 4ac4bf03ec..f32a4fdd5a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/command.rs @@ -38,6 +38,7 @@ use sqlbuild_python_output::recall::main::write_stderr::write_stderr; use sqlbuild_python_planner::observations::main::arm_source_capture::arm_source_capture; use sqlbuild_python_planner::observations::main::take_source_capture::take_source_capture; use sqlbuild_python_project::graph::main::python_nodes::python_node_graph::python_node_graph; +use sqlbuild_python_project::native_host::main::build_write_locations::build_write_locations; use sqlbuild_python_project::native_host::main::run_recorder::run_recorder; use sqlbuild_python_project::native_host::models::RunRecorder; use sqlbuild_python_project::project_snapshot::models::WorkerBuild; @@ -174,6 +175,7 @@ fn run_captured_build(context: &BuildContext<'_>, providers: &Bound<'_, PyAny>) &context.invocation, &native_field(&pipeline, "project")?, ), + || build_write_locations(&native_field(&pipeline, "plan_output")?), context.worker.as_ref().map(|worker| worker.server_run()), )?; match execute_direct_build(context, &pipeline, providers) { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs index 7cbfff7a89..5700cca7d7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs @@ -42,6 +42,7 @@ use sqlbuild_python_output::result_output::models::ExecutionEventWriter; use sqlbuild_python_planner::planner_entries::main::plans::build_seed_command_plan::build_seed_command_plan; use sqlbuild_python_planner::planner_entries::main::plans::read_selected_seed_fingerprints::read_selected_seed_fingerprints; use sqlbuild_python_project::native_host::main::run_recorder::run_recorder; +use sqlbuild_python_project::native_host::main::seed_write_locations::seed_write_locations; use sqlbuild_python_project::native_host::models::RunRecorder; use sqlbuild_python_project::project_snapshot::models::WorkerBuild; use sqlbuild_python_project::project_snapshot::types::VerifiedRun; @@ -486,6 +487,7 @@ pub(crate) fn run_seed<'py>( let recorder: Option> = run_recorder( (SEED_COMMAND, env!("CARGO_PKG_VERSION")), (request, &invocation, &native_field(&pipeline, "project")?), + || seed_write_locations(&native_field(&pipeline, "plan_output")?), worker.as_ref().map(|worker| worker.server_run()), )?; let started: Instant = Instant::now(); diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/build.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/build.md index ab495b4a1d..de458420b6 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/build.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/build.md @@ -347,18 +347,26 @@ After a successful build, SQLBuild writes version identities to `_sqlbuild_finge ## Run records -Every build that runs nodes, and every `sqb seed`, appends a `started` row to `_sqlbuild_runs` in -the target schema before its first node runs, and a `finished` row with its status and node counts -when it ends. A run with only a `started` row was killed or crashed. A target whose schema is -`preserve` has no schema of its own, so its runs are not recorded. +Every build that runs nodes, and every `sqb seed`, appends a `started` row to `_sqlbuild_runs` +before its first node runs, and a `finished` row with its status and node counts when it ends. It +writes both rows to every schema the run writes to: the schemas of the models and seeds it builds, +its SQL functions and the sources it loads, fixed from the plan before anything runs. The rows in each schema carry +the same run ID, so a role that can build in a schema can also record the run there, with no other +grants. A run never writes to a schema it doesn't otherwise write to, such as the target's default +schema, and never creates a schema just for these rows. A run whose nodes write no relation, such as +one that only runs Python tasks or checks, records nothing. + +A run's status is its `finished` row, in any of its schemas. A run with only `started` rows was +killed or crashed. A synced code location merges the rows of every schema by run ID and shows runs +it did not start. Each row records the command, its selection as typed, the jobs it names, the target, the warehouse user, the `sqb login` identity if you're signed in, the operating-system user, the machine name, the SQLBuild version and the Git commit. Of `--vars` it records only the names, never the values. Set `[settings] record_run_host = false` to leave out the machine name. -Recording a run never fails the build: if a row can't be written, the build prints a warning and -carries on. +Recording a run never fails the build: if a row can't be written to a schema, the build prints a +warning naming that schema's table and carries on, and the run's other schemas still record it. ## Runtime artifacts diff --git a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py index a07f1a708e..3721ffccdb 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py @@ -1475,3 +1475,16 @@ class SeedRunRecordTestCase: description: str # Each row as (phase, command, select_json, named_jobs_json, var_keys_json, status, counts). expected_rows: list[tuple[str, str, str, str, str, str | None, str | None]] + + +@dataclass(frozen=True) +class RunRecordSchemasTestCase: + description: str + command_args: tuple[str, ...] + settings: str + warehouse_setup: str + expected_exit_code: int + # The phases each schema's runs table holds, oldest first. + expected_phases: dict[str, list[str]] + expected_absent_schemas: frozenset[str] + expected_warnings: tuple[str, ...] diff --git a/tests/integration/src/sqlbuild/cli/commands/main/helpers.py b/tests/integration/src/sqlbuild/cli/commands/main/helpers.py index 0be7c535d1..530cb09524 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/helpers.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/helpers.py @@ -8,7 +8,7 @@ import sys import tempfile import time -from collections.abc import Callable +from collections.abc import Callable, Iterable from itertools import takewhile from pathlib import Path from types import ModuleType @@ -1414,11 +1414,45 @@ def write_python_rules_project(*, project_dir: Path, fixture: str, rules_toml: s } -def write_run_record_project(*, project_dir: Path, settings: str) -> None: - """Write the project, with `settings` appended to the project file.""" +RUN_RECORD_MARTS_FILES: dict[str, str] = { + "models/marts/revenue.sql": ( + "MODEL (description 'Revenue in the marts schema.', materialized table);\n" + "SELECT 10 AS amount\n" + ), + "models/marts/daily_events.sql": ( + "MODEL (description 'Loaded events in the marts schema.', materialized table);\n" + 'SELECT event_id FROM __source("landed_events")\n' + ), + "python/loaders/events.py": ( + "from sqlbuild.loaders import loader\n\n\n" + "@loader\n" + "def landed_events(ctx):\n" + ' """Load one event."""\n\n' + ' return [{"event_id": 1}]\n' + ), + "sources/landed.yml": ( + "sources:\n - name: landed_events\n description: Loaded events.\n schema: landing\n" + " managed: true\n write_strategy: table\n columns:\n" + " - name: event_id\n type: INTEGER\n" + ), + "seeds/regions.csv": "region_id\n1\n", + "seeds/regions.yml": ( + "seeds:\n - name: regions\n description: Regions.\n columns:\n" + " - name: region_id\n type: INTEGER\n" + ), +} + +RUN_RECORD_MARTS_SCHEMA: str = '\n[path_defaults.marts]\nschema = "marts"\n' + + +def write_run_record_project( + *, project_dir: Path, settings: str, extra_files: dict[str, str] | None = None +) -> None: + """Write the project with `extra_files`, and `settings` appended after its `dev` target.""" files: dict[str, str] = { **RUN_RECORD_PROJECT_FILES, + **(extra_files or {}), "sqlbuild_project.toml": ( 'name = "demo"\nadapter = "duckdb"\ndefault_target = "dev"\n\n' '[connection]\ndatabase = "demo.duckdb"\n\n[targets.dev]\n' + settings @@ -1430,13 +1464,59 @@ def write_run_record_project(*, project_dir: Path, settings: str) -> None: destination.write_text(contents, encoding="utf-8") -def run_rows(*, project_dir: Path, columns: str) -> list[tuple[object, ...]]: - """`columns` of every runs-table row, oldest first.""" +def run_rows(*, project_dir: Path, columns: str, schema: str = "main") -> list[tuple[object, ...]]: + """`columns` of every row of `schema`'s runs table, oldest first.""" with duckdb.connect(str(project_dir / "demo.duckdb")) as warehouse: return warehouse.sql( - f"SELECT {columns} FROM main._sqlbuild_runs ORDER BY recorded_at, phase DESC" + f"SELECT {columns} FROM {schema}._sqlbuild_runs ORDER BY recorded_at, phase DESC" + ).fetchall() + + +def schema_run_phases(*, project_dir: Path, schemas: Iterable[str]) -> dict[str, list[str]]: + """The phases of each of `schemas`' runs table, oldest first.""" + + phases: dict[str, list[str]] = {} + schema: str + for schema in schemas: + rows: list[tuple[object, ...]] = run_rows( + project_dir=project_dir, columns="phase", schema=schema + ) + phases[schema] = [str(row[0]) for row in rows] + return phases + + +def schema_run_ids(*, project_dir: Path, schemas: Iterable[str]) -> set[object]: + """Every run ID in `schemas`' runs tables.""" + + run_ids: set[object] = set() + schema: str + for schema in schemas: + rows: list[tuple[object, ...]] = run_rows( + project_dir=project_dir, columns="run_id", schema=schema + ) + run_ids.update(row[0] for row in rows) + return run_ids + + +def warehouse_schemas(*, project_dir: Path) -> frozenset[str]: + """The schemas of the project's warehouse.""" + + with duckdb.connect(str(project_dir / "demo.duckdb")) as warehouse: + rows: list[tuple[str]] = warehouse.sql( + "SELECT schema_name FROM duckdb_schemas() WHERE database_name = 'demo'" ).fetchall() + return frozenset(name for (name,) in rows) + + +def runs_tables(*, project_dir: Path) -> frozenset[str]: + """The schemas holding a runs table.""" + + with duckdb.connect(str(project_dir / "demo.duckdb")) as warehouse: + rows: list[tuple[str]] = warehouse.sql( + "SELECT schema_name FROM duckdb_tables() WHERE table_name = '_sqlbuild_runs'" + ).fetchall() + return frozenset(name for (name,) in rows) def built_tables(*, project_dir: Path) -> frozenset[str]: diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_run_records.py b/tests/integration/src/sqlbuild/cli/commands/main/test_run_records.py index 3a20c3a7ad..3836253de9 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_run_records.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_run_records.py @@ -1,4 +1,4 @@ -"""Every build records its start and end in the target's runs table, and a failed write never fails it.""" +"""Every build records its start and end in each schema it writes, and a failed write never fails it.""" from __future__ import annotations @@ -15,17 +15,25 @@ InterruptedRunRecordTestCase, KilledRunRecordTestCase, RunRecordFailureTestCase, + RunRecordSchemasTestCase, RunRecordTestCase, SeedRunRecordTestCase, ) from tests.integration.src.sqlbuild.cli.commands.main.helpers import ( RUN_RECORD_COLUMNS, + RUN_RECORD_MARTS_FILES, + RUN_RECORD_MARTS_SCHEMA, built_tables, prepare_warehouse, run_rows, + runs_tables, + schema_run_ids, + schema_run_phases, + warehouse_schemas, write_run_record_project, ) +_PAIR: list[str] = ["started", "finished"] _SUCCEEDED_TWO: str = '{"succeeded":2,"warned":0,"failed":0,"skipped":0,"not_started":0}' @@ -297,6 +305,7 @@ def test_given_a_killed_build_when_reading_its_runs_then_only_started_remains( "build", "--no-tests", "--select", + "stg_orders", "task:crash", ], capture_output=True, @@ -310,3 +319,89 @@ def test_given_a_killed_build_when_reading_its_runs_then_only_started_remains( assert [row[0] for row in run_rows(project_dir=tmp_path, columns="phase")] == ( test_case.expected_phases ) + + +@pytest.mark.parametrize( + "test_case", + [ + RunRecordSchemasTestCase( + description="a build writing two schemas records its pair in each under one run id", + command_args=("build", "--no-tests", "--select", "stg_orders", "revenue"), + settings=RUN_RECORD_MARTS_SCHEMA, + warehouse_setup="", + expected_exit_code=0, + expected_phases={"main": _PAIR, "marts": _PAIR}, + expected_absent_schemas=frozenset(), + expected_warnings=(), + ), + RunRecordSchemasTestCase( + description="a build writing only a custom schema never creates the target's schema", + command_args=("build", "--no-tests", "--select", "revenue"), + settings='\n[defaults]\nschema = "analytics"\n' + RUN_RECORD_MARTS_SCHEMA, + warehouse_setup="", + expected_exit_code=0, + expected_phases={"marts": _PAIR}, + expected_absent_schemas=frozenset({"analytics"}), + expected_warnings=(), + ), + RunRecordSchemasTestCase( + description="an unwritable schema warns naming it while the other schema records", + command_args=("build", "--no-tests", "--select", "stg_orders", "revenue"), + settings=RUN_RECORD_MARTS_SCHEMA, + warehouse_setup="CREATE VIEW main._sqlbuild_runs AS SELECT 1 AS run_id", + expected_exit_code=0, + expected_phases={"marts": _PAIR}, + expected_absent_schemas=frozenset(), + expected_warnings=( + " started in main._sqlbuild_runs: ", + " finished in main._sqlbuild_runs: ", + ), + ), + RunRecordSchemasTestCase( + description="a build loading a source records in the source's schema too", + command_args=("build", "--no-tests", "--select", "+daily_events"), + settings=RUN_RECORD_MARTS_SCHEMA, + warehouse_setup="", + expected_exit_code=0, + expected_phases={"landing": _PAIR, "marts": _PAIR}, + expected_absent_schemas=frozenset(), + expected_warnings=(), + ), + RunRecordSchemasTestCase( + description="a seed run records in its seeds' schema and not the target's", + command_args=("seed",), + settings='\n[defaults]\nseed_schema = "reference"\n', + warehouse_setup="", + expected_exit_code=0, + expected_phases={"reference": _PAIR}, + expected_absent_schemas=frozenset(), + expected_warnings=(), + ), + ], + ids=lambda case: case.description, +) +def test_given_a_run_writing_schemas_when_it_runs_then_each_schema_records_it( + test_case: RunRecordSchemasTestCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + write_run_record_project( + project_dir=tmp_path, settings=test_case.settings, extra_files=RUN_RECORD_MARTS_FILES + ) + prepare_warehouse(project_dir=tmp_path, sql=test_case.warehouse_setup or "SELECT 1") + + exit_code: int = main(["--project-dir", str(tmp_path), "--no-color", *test_case.command_args]) + captured: str = "".join(capsys.readouterr()) + + assert exit_code == test_case.expected_exit_code, captured + assert runs_tables(project_dir=tmp_path) == frozenset(test_case.expected_phases), captured + assert ( + schema_run_phases(project_dir=tmp_path, schemas=test_case.expected_phases) + == test_case.expected_phases + ) + assert len(schema_run_ids(project_dir=tmp_path, schemas=test_case.expected_phases)) == 1 + assert not test_case.expected_absent_schemas & warehouse_schemas(project_dir=tmp_path) + assert captured.count("Warning: could not record that run") == len( + test_case.expected_warnings + ), captured + assert all(warning in captured for warning in test_case.expected_warnings), captured diff --git a/website/src/content/docs/docs/cli/build.mdx b/website/src/content/docs/docs/cli/build.mdx index 12e7916aca..5c2dadea3e 100644 --- a/website/src/content/docs/docs/cli/build.mdx +++ b/website/src/content/docs/docs/cli/build.mdx @@ -330,18 +330,26 @@ After a successful build, SQLBuild writes version identities to `_sqlbuild_finge ## Run records -Every build that runs nodes, and every `sqb seed`, appends a `started` row to `_sqlbuild_runs` in -the target schema before its first node runs, and a `finished` row with its status and node counts -when it ends. A run with only a `started` row was killed or crashed. A target whose schema is -`preserve` has no schema of its own, so its runs are not recorded. +Every build that runs nodes, and every `sqb seed`, appends a `started` row to `_sqlbuild_runs` +before its first node runs, and a `finished` row with its status and node counts when it ends. It +writes both rows to every schema the run writes to: the schemas of the models and seeds it builds, +its SQL functions and the sources it loads, fixed from the plan before anything runs. The rows in each schema carry +the same run ID, so a role that can build in a schema can also record the run there, with no other +grants. A run never writes to a schema it doesn't otherwise write to, such as the target's default +schema, and never creates a schema just for these rows. A run whose nodes write no relation, such as +one that only runs Python tasks or checks, records nothing. + +A run's status is its `finished` row, in any of its schemas. A run with only `started` rows was +killed or crashed. A synced code location merges the rows of every schema by run ID and shows runs +it did not start. Each row records the command, its selection as typed, the jobs it names, the target, the warehouse user, the `sqb login` identity if you're signed in, the operating-system user, the machine name, the SQLBuild version and the Git commit. Of `--vars` it records only the names, never the values. Set `[settings] record_run_host = false` to leave out the machine name. -Recording a run never fails the build: if a row can't be written, the build prints a warning and -carries on. +Recording a run never fails the build: if a row can't be written to a schema, the build prints a +warning naming that schema's table and carries on, and the run's other schemas still record it. ## Runtime artifacts From 4433272276fc3e6b863c39839d8c262a91de3ab4 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 18 Nov 2026 22:38:18 +0000 Subject: [PATCH 2/7] fix(state): write each run's schema once over a bounded pool, else the default schema Review follow-up to recording runs per schema. One physical schema spelled two ways, such as marts and MARTS on Snowflake or with and without the target's database, was written twice. Schemas now compare as the dialect resolves unquoted names (its native profile) with an unnamed database filled in from the target's, so each runs table gets one pair. Each phase opened a connection per schema, and a second on a first run. It now opens at most four, each writing its share of schemas in turn, and the create-and-retry reuses its connection: one login per phase for one schema, four for five or twenty. A failed schema still warns alone. Every writer runs on its own thread, outside the calling thread's context, so a build stopped by its execution deadline still records its finished row in each schema. An interrupt raised in a writer, or a Ctrl+C while the writers run, still stops the run once they finish. A run that writes no relation, such as only Python tasks or checks, now records in the target's default schema, resolved as audit results resolve theirs, instead of nowhere. Runs that write a relation still write only their own schemas. Tests cover case spellings, a skipped (disabled) model's schema, a function's schema, a build that raises across two schemas, and a task-only run; the killed-build test selects its crashing task alone again. The docs no longer claim the sync merges runs yet. --- .../src/analysis_route/main/mod.rs | 1 + .../analysis_route/main/same_unquoted_name.rs | 17 +++++ .../src/analysis_route/tests.rs | 2 + .../tests/test_same_unquoted_name.rs | 59 ++++++++++++++++ .../src/analysis_route/tests/test_types.rs | 8 +++ .../src/native_host/_helpers/run_locations.rs | 69 ++++++++++++++++++- .../src/native_host/_helpers/run_writes.rs | 48 +++++++------ .../src/native_host/constants.rs | 5 ++ .../native_host/main/build_write_locations.rs | 11 +-- .../native_host/main/seed_write_locations.rs | 11 +-- .../src/native_host/models.rs | 15 ++-- .../_helpers/build_command/command.rs | 7 +- .../bindings/_helpers/seed_command/command.rs | 7 +- .../src/runs/main/distinct_run_schemas.rs | 33 +++++++++ crates/sqlbuild-state/src/runs/main/mod.rs | 1 + crates/sqlbuild-state/src/runs/models.rs | 7 ++ crates/sqlbuild-state/src/runs/tests.rs | 4 ++ .../sqlbuild-state/src/runs/tests/helpers.rs | 16 +++++ .../runs/tests/test_distinct_run_schemas.rs | 49 +++++++++++++ .../src/runs/tests/test_types.rs | 7 ++ .../sqlbuild/references/docs/cli/build.md | 16 +++-- .../sqlbuild/cli/commands/main/_test_types.py | 1 + .../src/sqlbuild/cli/commands/main/helpers.py | 46 ++++++++++++- .../cli/commands/main/test_run_records.py | 59 +++++++++++++++- website/src/content/docs/docs/cli/build.mdx | 16 +++-- 25 files changed, 460 insertions(+), 55 deletions(-) create mode 100644 crates/sqlbuild-analysis/src/analysis_route/main/same_unquoted_name.rs create mode 100644 crates/sqlbuild-analysis/src/analysis_route/tests/test_same_unquoted_name.rs create mode 100644 crates/sqlbuild-state/src/runs/main/distinct_run_schemas.rs create mode 100644 crates/sqlbuild-state/src/runs/tests/helpers.rs create mode 100644 crates/sqlbuild-state/src/runs/tests/test_distinct_run_schemas.rs diff --git a/crates/sqlbuild-analysis/src/analysis_route/main/mod.rs b/crates/sqlbuild-analysis/src/analysis_route/main/mod.rs index 7bf841082f..30cd8c4642 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/main/mod.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/main/mod.rs @@ -6,6 +6,7 @@ pub(crate) mod native_outcomes; pub mod native_revalidation; pub mod query_expression_ranges; pub mod same_column_type; +pub mod same_unquoted_name; pub mod schema_column; pub mod skipped_analysis; pub mod unapplied_nullability_rules; diff --git a/crates/sqlbuild-analysis/src/analysis_route/main/same_unquoted_name.rs b/crates/sqlbuild-analysis/src/analysis_route/main/same_unquoted_name.rs new file mode 100644 index 0000000000..d2762d0482 --- /dev/null +++ b/crates/sqlbuild-analysis/src/analysis_route/main/same_unquoted_name.rs @@ -0,0 +1,17 @@ +//! Whether two unquoted names written in one dialect resolve to the same warehouse object. + +use crate::analysis_route::main::analysis_route::analysis_route; +use crate::analysis_route::models::AnalysisRoute; + +/// Whether unquoted `left` and `right` name one object in `dialect`; exact without its native module. +#[must_use] +pub fn same_unquoted_name(dialect: Option<&str>, left: &str, right: &str) -> bool { + let AnalysisRoute::Native(native) = analysis_route(dialect) else { + return left == right; + }; + let profile = native.profile(); + profile.same_identifier( + &profile.resolved_identifier(left, false), + &profile.resolved_identifier(right, false), + ) +} diff --git a/crates/sqlbuild-analysis/src/analysis_route/tests.rs b/crates/sqlbuild-analysis/src/analysis_route/tests.rs index 4c488bd8ed..31f7734e3a 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/tests.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/tests.rs @@ -26,6 +26,8 @@ mod test_native_revalidation; mod test_postgres_dialect; #[path = "tests/test_same_column_type.rs"] mod test_same_column_type; +#[path = "tests/test_same_unquoted_name.rs"] +mod test_same_unquoted_name; #[path = "tests/test_type_spelling.rs"] mod test_type_spelling; #[path = "tests/test_types.rs"] diff --git a/crates/sqlbuild-analysis/src/analysis_route/tests/test_same_unquoted_name.rs b/crates/sqlbuild-analysis/src/analysis_route/tests/test_same_unquoted_name.rs new file mode 100644 index 0000000000..90eaaefeea --- /dev/null +++ b/crates/sqlbuild-analysis/src/analysis_route/tests/test_same_unquoted_name.rs @@ -0,0 +1,59 @@ +use crate::analysis_route::main::same_unquoted_name::same_unquoted_name; +use crate::analysis_route::tests::test_types::SameUnquotedNameTestCase; + +#[test] +fn given_two_unquoted_names_when_comparing_then_each_dialect_folds_them_as_it_resolves_names() { + let test_cases = [ + SameUnquotedNameTestCase { + description: "Snowflake folds unquoted names to upper case", + dialect: Some("snowflake"), + left: "marts", + right: "MARTS", + expected_same_name: true, + }, + SameUnquotedNameTestCase { + description: "PostgreSQL folds unquoted names to lower case", + dialect: Some("postgres"), + left: "Marts", + right: "marts", + expected_same_name: true, + }, + SameUnquotedNameTestCase { + description: "DuckDB compares names without case", + dialect: Some("duckdb"), + left: "marts", + right: "MARTS", + expected_same_name: true, + }, + SameUnquotedNameTestCase { + description: "ClickHouse names keep their case", + dialect: Some("clickhouse"), + left: "marts", + right: "MARTS", + expected_same_name: false, + }, + SameUnquotedNameTestCase { + description: "different names never match", + dialect: Some("snowflake"), + left: "marts", + right: "staging", + expected_same_name: false, + }, + SameUnquotedNameTestCase { + description: "a dialect without a native module compares exactly", + dialect: None, + left: "marts", + right: "MARTS", + expected_same_name: false, + }, + ]; + + for test_case in test_cases { + assert_eq!( + same_unquoted_name(test_case.dialect, test_case.left, test_case.right), + test_case.expected_same_name, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/analysis_route/tests/test_types.rs b/crates/sqlbuild-analysis/src/analysis_route/tests/test_types.rs index b677fdd610..72b5ded707 100644 --- a/crates/sqlbuild-analysis/src/analysis_route/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/analysis_route/tests/test_types.rs @@ -371,3 +371,11 @@ pub(super) struct RebaseRangesTestCase { /// The text of `member` each lineage row's rebased range covers. pub(super) expected_texts: &'static [Option<&'static str>], } + +pub(super) struct SameUnquotedNameTestCase { + pub(super) description: &'static str, + pub(super) dialect: Option<&'static str>, + pub(super) left: &'static str, + pub(super) right: &'static str, + pub(super) expected_same_name: bool, +} diff --git a/crates/sqlbuild-python-project/src/native_host/_helpers/run_locations.rs b/crates/sqlbuild-python-project/src/native_host/_helpers/run_locations.rs index d8ecfc71ad..c1665d881d 100644 --- a/crates/sqlbuild-python-project/src/native_host/_helpers/run_locations.rs +++ b/crates/sqlbuild-python-project/src/native_host/_helpers/run_locations.rs @@ -1,8 +1,13 @@ -//! The schemas a run writes, where it records its rows in the runs table. +//! The schemas a run writes, where it records its rows in the runs table, each physical schema once. use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_analysis::analysis_route::main::same_unquoted_name::same_unquoted_name; +use sqlbuild_analysis::assembly::target_checks::constants::PRESERVE_TARGET_VALUE; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_front::native_host::models::Location; +use sqlbuild_state::runs::main::distinct_run_schemas::distinct_run_schemas; +use sqlbuild_state::runs::models::RunSchema; +use std::collections::BTreeSet; /// The schema of a relation the run writes; None for a relation without one. pub(crate) fn written_schema(destination: &Bound<'_, PyAny>) -> PyResult> { @@ -16,3 +21,65 @@ pub(crate) fn written_schema(destination: &Bound<'_, PyAny>) -> PyResult, field: &str) -> PyResult> { + Ok(native_field(project, field)? + .extract::>()? + .filter(|value| value != PRESERVE_TARGET_VALUE)) +} + +/// Where a run writing no relation records, as audits do: the target's schema, else the first node's. +fn default_location( + plan: &Bound<'_, PyAny>, + project: &Bound<'_, PyAny>, +) -> PyResult> { + let database: Option = target_value(project, "effective_target_database")?; + if let Some(schema) = target_value(project, "effective_target_schema")? { + return Ok(Some(Location { database, schema })); + } + for locations in ["model_locations", "seed_locations"] { + for location in native_field(plan, locations)? + .call_method0("values")? + .try_iter()? + { + if let Some(found) = written_schema(&location?)? { + return Ok(Some(Location { + database: database.or(found.database), + schema: found.schema, + })); + } + } + } + Ok(None) +} + +/// Each physical schema of `schemas` once, as the dialect resolves names; else the default location. +pub(crate) fn run_locations( + schemas: BTreeSet, + (plan, project): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), +) -> PyResult> { + if schemas.is_empty() { + return Ok(default_location(plan, project)?.into_iter().collect()); + } + let dialect: Option = native_field(project, "sql_analysis_dialect")?.extract()?; + let target_database: Option = target_value(project, "effective_target_database")?; + let same_name = |left: &str, right: &str| same_unquoted_name(dialect.as_deref(), left, right); + Ok(distinct_run_schemas( + schemas + .into_iter() + .map(|location| RunSchema { + database: location.database, + schema: location.schema, + }) + .collect(), + target_database.as_deref(), + &same_name, + ) + .into_iter() + .map(|schema| Location { + database: schema.database, + schema: schema.schema, + }) + .collect()) +} diff --git a/crates/sqlbuild-python-project/src/native_host/_helpers/run_writes.rs b/crates/sqlbuild-python-project/src/native_host/_helpers/run_writes.rs index 2571e8da65..6175a34e83 100644 --- a/crates/sqlbuild-python-project/src/native_host/_helpers/run_writes.rs +++ b/crates/sqlbuild-python-project/src/native_host/_helpers/run_writes.rs @@ -1,9 +1,11 @@ -//! Writes of one run's rows to the runs table of every schema it writes, over a few connections. +//! Writes of one run's rows to the runs table of every schema it writes, over at most a few connections. use crate::native_host::_helpers::runs_state::CREATE_TABLE_METHOD; use crate::native_host::_helpers::runs_state::WAREHOUSE_USER_METHOD; use crate::native_host::_helpers::runs_state::row_cells; +use crate::native_host::constants::RUN_RECORD_SIGNAL_POLL_INTERVAL; use crate::native_host::models::{RunIdentity, RunTable}; +use pyo3::PyErr; use pyo3::exceptions::{PyException, PyRuntimeError}; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; use sqlbuild_core::constants::RUNS_TABLE_NAME; @@ -34,21 +36,22 @@ pub(crate) struct ShareResult { pub(crate) closed: PyResult<()>, } -/// Write `row` to each share's tables over one connection per share; the first share on this thread. +/// Write `row` to each share's tables over one connection per share, each on its own thread. +/// +/// The writers run outside the calling thread's context, so a build's expired deadline does not +/// stop its finished row. A signal such as Ctrl+C that arrives meanwhile is returned once every +/// writer is done. pub(crate) fn write_shares<'py>( (adapter, connection_config): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), (tables, identity): (&[RunTable], &RunIdentity), shares: &[Vec], (row, mode): (RunRow<'_>, RowWrite), -) -> Vec { +) -> (Vec, Option) { let py: Python<'py> = adapter.py(); - let Some((first, rest)) = shares.split_first() else { - return Vec::new(); - }; let unbound: (Py, Py) = (adapter.clone().unbind(), connection_config.clone().unbind()); std::thread::scope(|scope| { - let writers: Vec<_> = rest + let writers: Vec<_> = shares .iter() .map(|share| { let (adapter, connection_config) = (&unbound.0, &unbound.1); @@ -64,21 +67,24 @@ pub(crate) fn write_shares<'py>( }) }) .collect(); - let mut shared: Vec = vec![ - RunWriter { - adapter, - connection_config, - identity, + let mut signal: Option = None; + while !writers.iter().all(|writer| writer.is_finished()) { + if signal.is_none() + && let Err(error) = py.check_signals() + { + signal = Some(error); } - .write_share(first, tables, (row, mode)), - ]; - for (writer, share) in writers.into_iter().zip(rest) { - shared.push( - py.detach(|| writer.join()) - .unwrap_or_else(|_| panicked_share(share)), - ); + py.detach(|| std::thread::sleep(RUN_RECORD_SIGNAL_POLL_INTERVAL)); } - shared + let shared: Vec = writers + .into_iter() + .zip(shares) + .map(|(writer, share)| { + py.detach(|| writer.join()) + .unwrap_or_else(|_| panicked_share(share)) + }) + .collect(); + (shared, signal) }) } @@ -90,7 +96,7 @@ pub(crate) fn warn_on_errors( shared: Vec, ) -> PyResult<()> { let mut written: Vec<(usize, PyResult<()>)> = Vec::with_capacity(tables.len()); - let mut escaped: Option = None; + let mut escaped: Option = None; for share in shared { match share.closed { Err(error) if error.is_instance_of::(py) => { diff --git a/crates/sqlbuild-python-project/src/native_host/constants.rs b/crates/sqlbuild-python-project/src/native_host/constants.rs index 9ee01d749a..7bfdc2f4c8 100644 --- a/crates/sqlbuild-python-project/src/native_host/constants.rs +++ b/crates/sqlbuild-python-project/src/native_host/constants.rs @@ -12,3 +12,8 @@ pub(crate) const RELATION_CHANGES_METHOD: &str = "read_relation_changes"; pub(crate) const SYNCED_CATALOG_ATTRIBUTE: &str = "synced_catalog"; /// The plan action of a node the build leaves as it is. pub(crate) const SKIP_ACTION: &str = "skip"; +/// The most connections one phase of a run record opens; each writes its share of schemas in turn. +pub(crate) const MAX_RUN_RECORD_CONNECTIONS: usize = 4; +/// How often a run record's caller checks for Ctrl+C while its writer threads run. +pub(crate) const RUN_RECORD_SIGNAL_POLL_INTERVAL: std::time::Duration = + std::time::Duration::from_millis(5); diff --git a/crates/sqlbuild-python-project/src/native_host/main/build_write_locations.rs b/crates/sqlbuild-python-project/src/native_host/main/build_write_locations.rs index 3268544366..88c573caa1 100644 --- a/crates/sqlbuild-python-project/src/native_host/main/build_write_locations.rs +++ b/crates/sqlbuild-python-project/src/native_host/main/build_write_locations.rs @@ -1,14 +1,17 @@ //! The schemas a build writes, which each record its run. -use crate::native_host::_helpers::run_locations::written_schema; +use crate::native_host::_helpers::run_locations::{run_locations, written_schema}; use crate::native_host::constants::SKIP_ACTION; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_front::native_host::models::Location; use std::collections::BTreeSet; -/// The schemas a build writes: its models' and seeds' that run, its functions' and its loaded sources'. -pub fn build_write_locations(plan: &Bound<'_, PyAny>) -> PyResult> { +/// Where a build records: its models' and seeds' schemas that run, its functions', its sources'. +pub fn build_write_locations( + plan: &Bound<'_, PyAny>, + project: &Bound<'_, PyAny>, +) -> PyResult> { let mut schemas: BTreeSet = BTreeSet::new(); for entries in ["model_entries", "seed_entries"] { for entry in native_field(plan, entries)?.try_iter()? { @@ -30,5 +33,5 @@ pub fn build_write_locations(plan: &Bound<'_, PyAny>) -> PyResult> schemas.extend(written_schema(&source)?); } } - Ok(schemas.into_iter().collect()) + run_locations(schemas, (plan, project)) } diff --git a/crates/sqlbuild-python-project/src/native_host/main/seed_write_locations.rs b/crates/sqlbuild-python-project/src/native_host/main/seed_write_locations.rs index c0446e5034..fa313e8031 100644 --- a/crates/sqlbuild-python-project/src/native_host/main/seed_write_locations.rs +++ b/crates/sqlbuild-python-project/src/native_host/main/seed_write_locations.rs @@ -1,16 +1,19 @@ //! The schemas `sqb seed` writes, which each record its run. -use crate::native_host::_helpers::run_locations::written_schema; +use crate::native_host::_helpers::run_locations::{run_locations, written_schema}; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_front::native_host::models::Location; use std::collections::BTreeSet; -/// The schemas `sqb seed` writes: those of every seed it loads. -pub fn seed_write_locations(plan: &Bound<'_, PyAny>) -> PyResult> { +/// Where `sqb seed` records: the schemas of every seed it loads. +pub fn seed_write_locations( + plan: &Bound<'_, PyAny>, + project: &Bound<'_, PyAny>, +) -> PyResult> { let mut schemas: BTreeSet = BTreeSet::new(); for entry in native_field(plan, "seed_entries")?.try_iter()? { schemas.extend(written_schema(&native_field(&entry?, "destination")?)?); } - Ok(schemas.into_iter().collect()) + run_locations(schemas, (plan, project)) } diff --git a/crates/sqlbuild-python-project/src/native_host/models.rs b/crates/sqlbuild-python-project/src/native_host/models.rs index 9845db3acd..81257f2392 100644 --- a/crates/sqlbuild-python-project/src/native_host/models.rs +++ b/crates/sqlbuild-python-project/src/native_host/models.rs @@ -5,6 +5,7 @@ use crate::native_host::_helpers::run_writes::{ RowWrite, RunRow, ShareResult, warn_on_errors, write_shares, }; use crate::native_host::_helpers::runs_state::CREATE_TABLE_METHOD; +use crate::native_host::constants::MAX_RUN_RECORD_CONNECTIONS; use crate::planner_scope::models::ScopeData; use pyo3::IntoPyObject; use pyo3::PyErr; @@ -244,20 +245,22 @@ impl<'py> RunRecorder<'py> { let _ = self.finished(&RunNodeCounts::default(), ending); } - /// Append the row to every schema's table at once, then warn for each failed write. + /// Append the row to every schema's table over a few connections, then warn for each failure. /// - /// The first table is written on the calling thread, so an interrupt raised there stops the run - /// as it does for a single schema; an interrupt anywhere else escapes once every write is done. + /// An interrupt, raised in a writer or while waiting for them, escapes once every write is done. fn append_everywhere(&self, row: RunRow<'_>, mode: RowWrite) -> PyResult<()> { let py: Python<'py> = self.adapter.py(); - let shares: Vec> = (0..self.tables.len()).map(|index| vec![index]).collect(); - let shared: Vec = write_shares( + let connections: usize = self.tables.len().min(MAX_RUN_RECORD_CONNECTIONS); + let shares: Vec> = (0..connections) + .map(|first| (first..self.tables.len()).step_by(connections).collect()) + .collect(); + let (shared, signal): (Vec, Option) = write_shares( (&self.adapter, &self.connection_config), (&self.tables, &self.identity), &shares, (row, mode), ); warn_on_errors(py, (&self.tables, &self.identity), row.0, shared)?; - py.check_signals() + signal.map_or(Ok(()), Err) } } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/command.rs index f32a4fdd5a..920773b1b7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/command.rs @@ -175,7 +175,12 @@ fn run_captured_build(context: &BuildContext<'_>, providers: &Bound<'_, PyAny>) &context.invocation, &native_field(&pipeline, "project")?, ), - || build_write_locations(&native_field(&pipeline, "plan_output")?), + || { + build_write_locations( + &native_field(&pipeline, "plan_output")?, + &native_field(&pipeline, "project")?, + ) + }, context.worker.as_ref().map(|worker| worker.server_run()), )?; match execute_direct_build(context, &pipeline, providers) { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs index 5700cca7d7..3465642813 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs @@ -487,7 +487,12 @@ pub(crate) fn run_seed<'py>( let recorder: Option> = run_recorder( (SEED_COMMAND, env!("CARGO_PKG_VERSION")), (request, &invocation, &native_field(&pipeline, "project")?), - || seed_write_locations(&native_field(&pipeline, "plan_output")?), + || { + seed_write_locations( + &native_field(&pipeline, "plan_output")?, + &native_field(&pipeline, "project")?, + ) + }, worker.as_ref().map(|worker| worker.server_run()), )?; let started: Instant = Instant::now(); diff --git a/crates/sqlbuild-state/src/runs/main/distinct_run_schemas.rs b/crates/sqlbuild-state/src/runs/main/distinct_run_schemas.rs new file mode 100644 index 0000000000..0fae52766b --- /dev/null +++ b/crates/sqlbuild-state/src/runs/main/distinct_run_schemas.rs @@ -0,0 +1,33 @@ +//! Each physical schema a run writes once, however its nodes spell it. + +use crate::runs::models::RunSchema; + +/// `schemas`, each physical one once: no database means `default_database`; `same_name` folds names. +#[must_use] +pub fn distinct_run_schemas( + schemas: Vec, + default_database: Option<&str>, + same_name: &dyn Fn(&str, &str) -> bool, +) -> Vec { + let same_database = |left: &Option, right: &Option| match (left, right) { + (Some(left), Some(right)) => same_name(left, right), + (left, right) => left == right, + }; + let mut distinct: Vec = Vec::with_capacity(schemas.len()); + for schema in schemas { + let resolved = RunSchema { + database: schema + .database + .or_else(|| default_database.map(str::to_owned)), + schema: schema.schema, + }; + let seen: bool = distinct.iter().any(|kept| { + same_database(&kept.database, &resolved.database) + && same_name(&kept.schema, &resolved.schema) + }); + if !seen { + distinct.push(resolved); + } + } + distinct +} diff --git a/crates/sqlbuild-state/src/runs/main/mod.rs b/crates/sqlbuild-state/src/runs/main/mod.rs index bf28b9cd60..acb319a8b4 100644 --- a/crates/sqlbuild-state/src/runs/main/mod.rs +++ b/crates/sqlbuild-state/src/runs/main/mod.rs @@ -1 +1,2 @@ +pub mod distinct_run_schemas; pub mod run_status; diff --git a/crates/sqlbuild-state/src/runs/models.rs b/crates/sqlbuild-state/src/runs/models.rs index 740cd08d87..06feacc01d 100644 --- a/crates/sqlbuild-state/src/runs/models.rs +++ b/crates/sqlbuild-state/src/runs/models.rs @@ -54,3 +54,10 @@ pub enum RunEnding { Interrupted, Raised, } + +/// One schema a run writes its rows to; no database means the connection's. +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub struct RunSchema { + pub database: Option, + pub schema: String, +} diff --git a/crates/sqlbuild-state/src/runs/tests.rs b/crates/sqlbuild-state/src/runs/tests.rs index d53d8384e6..816575b8a6 100644 --- a/crates/sqlbuild-state/src/runs/tests.rs +++ b/crates/sqlbuild-state/src/runs/tests.rs @@ -1,3 +1,7 @@ +#[path = "tests/test_distinct_run_schemas.rs"] +mod distinct_run_schemas; +#[path = "tests/helpers.rs"] +mod helpers; #[path = "tests/test_run_status.rs"] mod run_status; #[path = "tests/test_types.rs"] diff --git a/crates/sqlbuild-state/src/runs/tests/helpers.rs b/crates/sqlbuild-state/src/runs/tests/helpers.rs new file mode 100644 index 0000000000..78db384f4c --- /dev/null +++ b/crates/sqlbuild-state/src/runs/tests/helpers.rs @@ -0,0 +1,16 @@ +use crate::runs::models::RunSchema; + +/// Names compared as an upper-case-folding warehouse resolves unquoted names. +pub(super) fn folded(left: &str, right: &str) -> bool { + left.eq_ignore_ascii_case(right) +} + +pub(super) fn run_schemas(pairs: &[(Option<&str>, &str)]) -> Vec { + pairs + .iter() + .map(|(database, schema)| RunSchema { + database: database.map(str::to_owned), + schema: (*schema).to_owned(), + }) + .collect() +} diff --git a/crates/sqlbuild-state/src/runs/tests/test_distinct_run_schemas.rs b/crates/sqlbuild-state/src/runs/tests/test_distinct_run_schemas.rs new file mode 100644 index 0000000000..c16a1beeb7 --- /dev/null +++ b/crates/sqlbuild-state/src/runs/tests/test_distinct_run_schemas.rs @@ -0,0 +1,49 @@ +use crate::runs::main::distinct_run_schemas::distinct_run_schemas; +use crate::runs::models::RunSchema; +use crate::runs::tests::helpers::{folded, run_schemas}; +use crate::runs::tests::test_types::DistinctRunSchemasTestCase; + +#[test] +fn given_spellings_of_schemas_when_deduplicating_then_each_physical_schema_is_written_once() { + let test_cases = [ + DistinctRunSchemasTestCase { + description: "case spellings of one unquoted name are one schema", + schemas: &[(Some("ANALYTICS"), "MARTS"), (Some("ANALYTICS"), "marts")], + default_database: Some("ANALYTICS"), + expected_schemas: &[(Some("ANALYTICS"), "MARTS")], + }, + DistinctRunSchemasTestCase { + description: "an unnamed database is the target's", + schemas: &[(None, "MARTS"), (Some("analytics"), "MARTS")], + default_database: Some("ANALYTICS"), + expected_schemas: &[(Some("ANALYTICS"), "MARTS")], + }, + DistinctRunSchemasTestCase { + description: "the same schema in two databases is two schemas", + schemas: &[(Some("ANALYTICS"), "MARTS"), (Some("RAW"), "MARTS")], + default_database: Some("ANALYTICS"), + expected_schemas: &[(Some("ANALYTICS"), "MARTS"), (Some("RAW"), "MARTS")], + }, + DistinctRunSchemasTestCase { + description: "without a target database an unnamed database stays apart", + schemas: &[(None, "marts"), (Some("demo"), "marts")], + default_database: None, + expected_schemas: &[(None, "marts"), (Some("demo"), "marts")], + }, + ]; + + for test_case in test_cases { + let distinct: Vec = distinct_run_schemas( + run_schemas(test_case.schemas), + test_case.default_database, + &folded, + ); + + assert_eq!( + distinct, + run_schemas(test_case.expected_schemas), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-state/src/runs/tests/test_types.rs b/crates/sqlbuild-state/src/runs/tests/test_types.rs index 6d71eb2c5a..306f24befa 100644 --- a/crates/sqlbuild-state/src/runs/tests/test_types.rs +++ b/crates/sqlbuild-state/src/runs/tests/test_types.rs @@ -6,3 +6,10 @@ pub(crate) struct RunStatusTestCase { pub(crate) ending: RunEnding, pub(crate) expected_status: &'static str, } + +pub(crate) struct DistinctRunSchemasTestCase { + pub(crate) description: &'static str, + pub(crate) schemas: &'static [(Option<&'static str>, &'static str)], + pub(crate) default_database: Option<&'static str>, + pub(crate) expected_schemas: &'static [(Option<&'static str>, &'static str)], +} diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/build.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/build.md index de458420b6..683316d87c 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/build.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/build.md @@ -350,15 +350,17 @@ After a successful build, SQLBuild writes version identities to `_sqlbuild_finge Every build that runs nodes, and every `sqb seed`, appends a `started` row to `_sqlbuild_runs` before its first node runs, and a `finished` row with its status and node counts when it ends. It writes both rows to every schema the run writes to: the schemas of the models and seeds it builds, -its SQL functions and the sources it loads, fixed from the plan before anything runs. The rows in each schema carry -the same run ID, so a role that can build in a schema can also record the run there, with no other -grants. A run never writes to a schema it doesn't otherwise write to, such as the target's default -schema, and never creates a schema just for these rows. A run whose nodes write no relation, such as -one that only runs Python tasks or checks, records nothing. +its SQL functions and the sources it loads, fixed from the plan before anything runs. The rows in +each schema carry the same run ID, so a role that can build in a schema can also record the run +there, with no other grants. A run that writes to any schema never writes these rows anywhere +else, such as the target's default schema. + +A run whose nodes write no relation, such as one that only runs Python tasks or checks, has no +schema of its own. It records in the target's default schema instead, where audit results go. A run's status is its `finished` row, in any of its schemas. A run with only `started` rows was -killed or crashed. A synced code location merges the rows of every schema by run ID and shows runs -it did not start. +killed or crashed. A synced code location will merge the rows of every schema by run ID and show +runs it did not start. Each row records the command, its selection as typed, the jobs it names, the target, the warehouse user, the `sqb login` identity if you're signed in, the operating-system user, the diff --git a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py index 3721ffccdb..fa31557b5f 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py @@ -1488,3 +1488,4 @@ class RunRecordSchemasTestCase: expected_phases: dict[str, list[str]] expected_absent_schemas: frozenset[str] expected_warnings: tuple[str, ...] + expected_finished_status: str = "succeeded" diff --git a/tests/integration/src/sqlbuild/cli/commands/main/helpers.py b/tests/integration/src/sqlbuild/cli/commands/main/helpers.py index 530cb09524..24bda44851 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/helpers.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/helpers.py @@ -1435,6 +1435,33 @@ def write_python_rules_project(*, project_dir: Path, fixture: str, rules_toml: s " managed: true\n write_strategy: table\n columns:\n" " - name: event_id\n type: INTEGER\n" ), + "models/upper/revenue_upper.sql": ( + "MODEL (description 'Revenue under an upper-case schema name.', materialized table);\n" + "SELECT 20 AS amount\n" + ), + "models/marts/doubled.sql": ( + "MODEL (description 'Calls a function in its own schema.', materialized table);\n" + 'SELECT __udf("udf__doubled")(2) AS amount\n' + ), + "functions/sql/udf__doubled.sql": ( + 'FUNCTION (\n description "Twice its argument",\n arguments (value INTEGER),\n' + " returns INTEGER,\n schema udfs,\n);\nvalue * 2\n" + ), + "models/marts/slow_total.sql": ( + "MODEL (description 'Runs past a one-second deadline.', materialized table);\n" + "SELECT COUNT(*) AS total FROM range(100000000000)\n" + ), + "models/archive/old_orders.sql": ( + "MODEL (description 'A disabled model in its own schema.', materialized table,\n" + " enabled false, schema archive);\nSELECT 1 AS order_id\n" + ), + "python/noop.py": ( + '"""A task that writes no relation."""\n\n' + "from sqlbuild.tasks import task\n\n\n" + "@task()\n" + "def noop(ctx):\n" + ' """Do nothing."""\n' + ), "seeds/regions.csv": "region_id\n1\n", "seeds/regions.yml": ( "seeds:\n - name: regions\n description: Regions.\n columns:\n" @@ -1442,7 +1469,9 @@ def write_python_rules_project(*, project_dir: Path, fixture: str, rules_toml: s ), } -RUN_RECORD_MARTS_SCHEMA: str = '\n[path_defaults.marts]\nschema = "marts"\n' +RUN_RECORD_MARTS_SCHEMA: str = ( + '\n[path_defaults.marts]\nschema = "marts"\n\n[path_defaults.upper]\nschema = "MARTS"\n' +) def write_run_record_project( @@ -1499,6 +1528,19 @@ def schema_run_ids(*, project_dir: Path, schemas: Iterable[str]) -> set[object]: return run_ids +def schema_run_statuses(*, project_dir: Path, schemas: Iterable[str]) -> set[object]: + """The status of every row in `schemas`' runs tables; `started` rows have none.""" + + statuses: set[object] = set() + schema: str + for schema in schemas: + rows: list[tuple[object, ...]] = run_rows( + project_dir=project_dir, columns="status", schema=schema + ) + statuses.update(row[0] for row in rows) + return statuses + + def warehouse_schemas(*, project_dir: Path) -> frozenset[str]: """The schemas of the project's warehouse.""" @@ -1516,7 +1558,7 @@ def runs_tables(*, project_dir: Path) -> frozenset[str]: rows: list[tuple[str]] = warehouse.sql( "SELECT schema_name FROM duckdb_tables() WHERE table_name = '_sqlbuild_runs'" ).fetchall() - return frozenset(name for (name,) in rows) + return frozenset(name.lower() for (name,) in rows) def built_tables(*, project_dir: Path) -> frozenset[str]: diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_run_records.py b/tests/integration/src/sqlbuild/cli/commands/main/test_run_records.py index 3836253de9..86d03bb2f2 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_run_records.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_run_records.py @@ -29,6 +29,7 @@ runs_tables, schema_run_ids, schema_run_phases, + schema_run_statuses, warehouse_schemas, write_run_record_project, ) @@ -305,7 +306,6 @@ def test_given_a_killed_build_when_reading_its_runs_then_only_started_remains( "build", "--no-tests", "--select", - "stg_orders", "task:crash", ], capture_output=True, @@ -367,6 +367,59 @@ def test_given_a_killed_build_when_reading_its_runs_then_only_started_remains( expected_absent_schemas=frozenset(), expected_warnings=(), ), + RunRecordSchemasTestCase( + description="case spellings of one schema record once", + command_args=("build", "--no-tests", "--select", "revenue", "revenue_upper"), + settings=RUN_RECORD_MARTS_SCHEMA, + warehouse_setup="", + expected_exit_code=0, + expected_phases={"marts": _PAIR}, + expected_absent_schemas=frozenset(), + expected_warnings=(), + ), + RunRecordSchemasTestCase( + description="a skipped model's schema records nothing", + command_args=("build", "--no-tests", "--select", "stg_orders", "old_orders"), + settings=RUN_RECORD_MARTS_SCHEMA, + warehouse_setup="", + expected_exit_code=0, + expected_phases={"main": _PAIR}, + expected_absent_schemas=frozenset(), + expected_warnings=(), + expected_finished_status="partial", + ), + RunRecordSchemasTestCase( + description="a function's schema records the run that deploys it", + command_args=("build", "--no-tests", "--select", "doubled"), + settings=RUN_RECORD_MARTS_SCHEMA, + warehouse_setup="", + expected_exit_code=0, + expected_phases={"marts": _PAIR, "udfs": _PAIR}, + expected_absent_schemas=frozenset(), + expected_warnings=(), + ), + RunRecordSchemasTestCase( + description="a build that raises finishes failed in each schema", + command_args=("build", "--no-tests", "--select", "stg_orders", "slow_total"), + settings=RUN_RECORD_MARTS_SCHEMA + + '\n[targets.dev.execution_limits]\nmax_duration = "2s"\n', + warehouse_setup="", + expected_exit_code=1, + expected_phases={"main": _PAIR, "marts": _PAIR}, + expected_absent_schemas=frozenset(), + expected_warnings=(), + expected_finished_status="failed", + ), + RunRecordSchemasTestCase( + description="a task-only run records in the target's default schema", + command_args=("build", "--no-tests", "--select", "task:noop"), + settings='\n[defaults]\nschema = "analytics"\n' + RUN_RECORD_MARTS_SCHEMA, + warehouse_setup="", + expected_exit_code=0, + expected_phases={"analytics": _PAIR}, + expected_absent_schemas=frozenset(), + expected_warnings=(), + ), RunRecordSchemasTestCase( description="a seed run records in its seeds' schema and not the target's", command_args=("seed",), @@ -400,6 +453,10 @@ def test_given_a_run_writing_schemas_when_it_runs_then_each_schema_records_it( == test_case.expected_phases ) assert len(schema_run_ids(project_dir=tmp_path, schemas=test_case.expected_phases)) == 1 + assert schema_run_statuses(project_dir=tmp_path, schemas=test_case.expected_phases) == { + None, + test_case.expected_finished_status, + } assert not test_case.expected_absent_schemas & warehouse_schemas(project_dir=tmp_path) assert captured.count("Warning: could not record that run") == len( test_case.expected_warnings diff --git a/website/src/content/docs/docs/cli/build.mdx b/website/src/content/docs/docs/cli/build.mdx index 5c2dadea3e..b4127de505 100644 --- a/website/src/content/docs/docs/cli/build.mdx +++ b/website/src/content/docs/docs/cli/build.mdx @@ -333,15 +333,17 @@ After a successful build, SQLBuild writes version identities to `_sqlbuild_finge Every build that runs nodes, and every `sqb seed`, appends a `started` row to `_sqlbuild_runs` before its first node runs, and a `finished` row with its status and node counts when it ends. It writes both rows to every schema the run writes to: the schemas of the models and seeds it builds, -its SQL functions and the sources it loads, fixed from the plan before anything runs. The rows in each schema carry -the same run ID, so a role that can build in a schema can also record the run there, with no other -grants. A run never writes to a schema it doesn't otherwise write to, such as the target's default -schema, and never creates a schema just for these rows. A run whose nodes write no relation, such as -one that only runs Python tasks or checks, records nothing. +its SQL functions and the sources it loads, fixed from the plan before anything runs. The rows in +each schema carry the same run ID, so a role that can build in a schema can also record the run +there, with no other grants. A run that writes to any schema never writes these rows anywhere +else, such as the target's default schema. + +A run whose nodes write no relation, such as one that only runs Python tasks or checks, has no +schema of its own. It records in the target's default schema instead, where audit results go. A run's status is its `finished` row, in any of its schemas. A run with only `started` rows was -killed or crashed. A synced code location merges the rows of every schema by run ID and shows runs -it did not start. +killed or crashed. A synced code location will merge the rows of every schema by run ID and show +runs it did not start. Each row records the command, its selection as typed, the jobs it names, the target, the warehouse user, the `sqb login` identity if you're signed in, the operating-system user, the From 2c2f99ba8af27267ecdc08122bbddf0c01208f53 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 19 Nov 2026 00:25:07 +0000 Subject: [PATCH 3/7] fix(state): keep run-record writes in the build's context and stop them on Ctrl+C Review follow-up to recording runs per schema. Each run-record writer thread started from an empty context, so its statements left the build's event and recall stream. Writers now take a copy of the caller's context and run with every deadline suspended, through the existing execution_deadlines_suspended scope, now a native host entry. A build stopped by its execution deadline still records its finished row in each schema, and its run-record statements are recalled with the build again. Ctrl+C no longer waits for every writer to finish its share. A caught signal sets a stop flag that each writer checks before its next table and before a retry, and cancels the running statements as SIGTERM does. A further signal while the writers stop cancels again instead of staying pending. Writes cut short that way are not reported as failed, and the interrupt escapes once the writers stop. run_recorder now takes the command's pipeline and picks the build's or seed's schemas itself, which keeps the Python bindings crate within its code-line cap. Tests send a real SIGINT while a run-record insert hangs, and check that the build stops within seconds, raises KeyboardInterrupt and warns of no failed write; and that a build's run-record inserts are among its own recalled statements. --- .../src/entries/_helpers/executor.rs | 5 + crates/sqlbuild-host/src/entries/models.rs | 1 + .../src/native_host/_helpers/run_locations.rs | 46 ++++- .../src/native_host/_helpers/run_writes.rs | 177 +++++++++++++----- .../native_host/main/build_write_locations.rs | 37 ---- .../src/native_host/main/mod.rs | 2 - .../src/native_host/main/run_recorder.rs | 13 +- .../native_host/main/seed_write_locations.rs | 19 -- .../src/native_host/models.rs | 34 +++- .../_helpers/build_command/command.rs | 13 +- .../bindings/_helpers/seed_command/command.rs | 9 +- src/sqlbuild/_native_host.py | 4 + src/sqlbuild/_native_host.pyi | 3 + .../sqlbuild/cli/commands/main/_test_types.py | 18 ++ .../src/sqlbuild/cli/commands/main/helpers.py | 14 ++ .../cli/commands/main/test_run_records.py | 102 ++++++++++ 16 files changed, 357 insertions(+), 140 deletions(-) delete mode 100644 crates/sqlbuild-python-project/src/native_host/main/build_write_locations.rs delete mode 100644 crates/sqlbuild-python-project/src/native_host/main/seed_write_locations.rs diff --git a/crates/sqlbuild-host/src/entries/_helpers/executor.rs b/crates/sqlbuild-host/src/entries/_helpers/executor.rs index 17302822a0..3b0ca7e8fa 100644 --- a/crates/sqlbuild-host/src/entries/_helpers/executor.rs +++ b/crates/sqlbuild-host/src/entries/_helpers/executor.rs @@ -704,6 +704,11 @@ pub(crate) const ENTRIES: &[HostEntry] = &[ kind: HostKind::Function, edge: HostEdge::Executor, }, + HostEntry { + name: "execution_deadlines_suspended", + kind: HostKind::Function, + edge: HostEdge::Executor, + }, HostEntry { name: "audit_evidence_scope", kind: HostKind::Function, diff --git a/crates/sqlbuild-host/src/entries/models.rs b/crates/sqlbuild-host/src/entries/models.rs index 4c39f96c72..2271331f54 100644 --- a/crates/sqlbuild-host/src/entries/models.rs +++ b/crates/sqlbuild-host/src/entries/models.rs @@ -347,6 +347,7 @@ pub enum ExecutorHost { RunStatus, ExecutionDurationLimitError, ExecutionDeadlineScope, + ExecutionDeadlinesSuspended, AuditEvidenceScope, CancelRunningStatements, EventDispatcher, diff --git a/crates/sqlbuild-python-project/src/native_host/_helpers/run_locations.rs b/crates/sqlbuild-python-project/src/native_host/_helpers/run_locations.rs index c1665d881d..58579f59d3 100644 --- a/crates/sqlbuild-python-project/src/native_host/_helpers/run_locations.rs +++ b/crates/sqlbuild-python-project/src/native_host/_helpers/run_locations.rs @@ -1,5 +1,6 @@ -//! The schemas a run writes, where it records its rows in the runs table, each physical schema once. +//! The schemas a build or seed writes, where it records its rows in the runs table, each once. +use crate::native_host::constants::SKIP_ACTION; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use sqlbuild_analysis::analysis_route::main::same_unquoted_name::same_unquoted_name; use sqlbuild_analysis::assembly::target_checks::constants::PRESERVE_TARGET_VALUE; @@ -10,7 +11,7 @@ use sqlbuild_state::runs::models::RunSchema; use std::collections::BTreeSet; /// The schema of a relation the run writes; None for a relation without one. -pub(crate) fn written_schema(destination: &Bound<'_, PyAny>) -> PyResult> { +fn written_schema(destination: &Bound<'_, PyAny>) -> PyResult> { let schema: Option = native_field(destination, "schema")?.extract()?; schema .map(|schema| { @@ -55,7 +56,7 @@ fn default_location( } /// Each physical schema of `schemas` once, as the dialect resolves names; else the default location. -pub(crate) fn run_locations( +fn run_locations( schemas: BTreeSet, (plan, project): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), ) -> PyResult> { @@ -83,3 +84,42 @@ pub(crate) fn run_locations( }) .collect()) } + +/// Where a build records: its models' and seeds' schemas that run, its functions', its sources'. +pub(crate) fn build_write_locations(pipeline: &Bound<'_, PyAny>) -> PyResult> { + let plan: &Bound<'_, PyAny> = &native_field(pipeline, "plan_output")?; + let project: &Bound<'_, PyAny> = &native_field(pipeline, "project")?; + let mut schemas: BTreeSet = BTreeSet::new(); + for entries in ["model_entries", "seed_entries"] { + for entry in native_field(plan, entries)?.try_iter()? { + let entry: Bound<'_, PyAny> = entry?; + if !native_field(&entry, "action")?.eq(SKIP_ACTION)? { + schemas.extend(written_schema(&native_field(&entry, "destination")?)?); + } + } + } + for entry in native_field(plan, "function_entries")?.try_iter()? { + schemas.extend(written_schema(&native_field(&entry?, "destination")?)?); + } + let sources: Bound<'_, PyAny> = native_field(plan, "source_map")?; + for entry in native_field(plan, "source_load_entries")?.try_iter()? { + if let Some(source) = sources + .call_method1("get", (native_field(&entry?, "name")?,))? + .extract::>>()? + { + schemas.extend(written_schema(&source)?); + } + } + run_locations(schemas, (plan, project)) +} + +/// Where `sqb seed` records: the schemas of every seed it loads. +pub(crate) fn seed_write_locations(pipeline: &Bound<'_, PyAny>) -> PyResult> { + let plan: &Bound<'_, PyAny> = &native_field(pipeline, "plan_output")?; + let project: &Bound<'_, PyAny> = &native_field(pipeline, "project")?; + let mut schemas: BTreeSet = BTreeSet::new(); + for entry in native_field(plan, "seed_entries")?.try_iter()? { + schemas.extend(written_schema(&native_field(&entry?, "destination")?)?); + } + run_locations(schemas, (plan, project)) +} diff --git a/crates/sqlbuild-python-project/src/native_host/_helpers/run_writes.rs b/crates/sqlbuild-python-project/src/native_host/_helpers/run_writes.rs index 6175a34e83..310cd6bef0 100644 --- a/crates/sqlbuild-python-project/src/native_host/_helpers/run_writes.rs +++ b/crates/sqlbuild-python-project/src/native_host/_helpers/run_writes.rs @@ -9,7 +9,10 @@ use pyo3::PyErr; use pyo3::exceptions::{PyException, PyRuntimeError}; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; use sqlbuild_core::constants::RUNS_TABLE_NAME; +use sqlbuild_host::entries::models::ExecutorHost; use sqlbuild_python_base::boundary::main::call_with::call_with; +use sqlbuild_python_base::boundary::main::imported::imported; +use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::native_host::main::write_console::write_console; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_front::native_host::main::execute::execute; @@ -17,6 +20,7 @@ use sqlbuild_python_front::native_host::main::literal::literal; use sqlbuild_state::runs::constants::RUN_COLUMN_TYPES; use sqlbuild_state::runs::models::RunNodeCounts; use sqlbuild_state::state_tables::main::values_insert_sql::values_insert_sql; +use std::sync::atomic::{AtomicBool, Ordering}; /// A row's phase and, for `finished`, its status and node counts. pub(crate) type RunRow<'a> = (&'a str, Option<(&'a str, &'a RunNodeCounts)>); @@ -36,43 +40,60 @@ pub(crate) struct ShareResult { pub(crate) closed: PyResult<()>, } -/// Write `row` to each share's tables over one connection per share, each on its own thread. -/// -/// The writers run outside the calling thread's context, so a build's expired deadline does not -/// stop its finished row. A signal such as Ctrl+C that arrives meanwhile is returned once every -/// writer is done. +/// What the writers of one row came to, and whether the run was interrupted while they ran. +pub(crate) struct SharesOutcome { + pub(crate) shared: Vec, + /// The first signal, such as Ctrl+C, caught while waiting for the writers. + pub(crate) signal: Option, + /// Whether the writers were told to stop: a caught signal or an interrupt inside a writer. + pub(crate) stopped: bool, +} + +/// Write `row` to each share's tables on its own thread and connection; Ctrl+C stops and cancels them. pub(crate) fn write_shares<'py>( (adapter, connection_config): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), (tables, identity): (&[RunTable], &RunIdentity), shares: &[Vec], (row, mode): (RunRow<'_>, RowWrite), -) -> (Vec, Option) { +) -> PyResult { let py: Python<'py> = adapter.py(); + let context: Py = imported(py, "contextvars", "copy_context")? + .call0()? + .unbind(); let unbound: (Py, Py) = (adapter.clone().unbind(), connection_config.clone().unbind()); - std::thread::scope(|scope| { + let stop: AtomicBool = AtomicBool::new(false); + Ok(std::thread::scope(|scope| { let writers: Vec<_> = shares .iter() .map(|share| { let (adapter, connection_config) = (&unbound.0, &unbound.1); + let (context, stop) = (&context, &stop); scope.spawn(move || { Python::attach(|py| { RunWriter { adapter: adapter.bind(py), connection_config: connection_config.bind(py), identity, + stop, } - .write_share(share, tables, (row, mode)) + .write_share_in( + context.bind(py), + share, + tables, + (row, mode), + ) }) }) }) .collect(); let mut signal: Option = None; while !writers.iter().all(|writer| writer.is_finished()) { - if signal.is_none() - && let Err(error) = py.check_signals() - { - signal = Some(error); + if let Err(caught) = py.check_signals() { + stop.store(true, Ordering::SeqCst); + let _ = host_entry(py, ExecutorHost::CancelRunningStatements) + .and_then(|cancel| cancel.call0()); + signal.get_or_insert(caught); } py.detach(|| std::thread::sleep(RUN_RECORD_SIGNAL_POLL_INTERVAL)); } @@ -84,39 +105,46 @@ pub(crate) fn write_shares<'py>( .unwrap_or_else(|_| panicked_share(share)) }) .collect(); - (shared, signal) - }) + SharesOutcome { + shared, + signal, + stopped: stop.load(Ordering::SeqCst), + } + })) } -/// Warn for each table whose write failed; an interrupt or exit is not a failed write and escapes. +/// Warn for each failed write unless the run was stopped; return an interrupt to escape. pub(crate) fn warn_on_errors( py: Python<'_>, (tables, identity): (&[RunTable], &RunIdentity), - phase: &str, + (phase, stopped): (&str, bool), shared: Vec, -) -> PyResult<()> { +) -> Option { let mut written: Vec<(usize, PyResult<()>)> = Vec::with_capacity(tables.len()); let mut escaped: Option = None; for share in shared { match share.closed { - Err(error) if error.is_instance_of::(py) => { + Err(error) if !error.is_instance_of::(py) => { + escaped.get_or_insert(error); + } + Err(error) if !stopped => { let _ = write_console( py, "stderr", &format!("Warning: could not close a {RUNS_TABLE_NAME} connection: {error}\n"), ); } - Err(error) => { - escaped.get_or_insert(error); - } - Ok(()) => {} + _ => {} } written.extend(share.written); } written.sort_by_key(|(index, _)| *index); for (index, result) in written { match result { - Err(error) if error.is_instance_of::(py) => { + Err(error) if !error.is_instance_of::(py) => { + escaped.get_or_insert(error); + } + Err(error) if !stopped => { let _ = write_console( py, "stderr", @@ -126,13 +154,19 @@ pub(crate) fn warn_on_errors( ), ); } - Err(error) => { - escaped.get_or_insert(error); - } - Ok(()) => {} + _ => {} } } - escaped.map_or(Ok(()), Err) + escaped +} + +/// Set the caller's context variables, from `context`, in this writer thread's own context. +fn adopt_context(context: &Bound<'_, PyAny>) -> PyResult<()> { + for item in context.call_method0("items")?.try_iter()? { + let (variable, value): (Bound<'_, PyAny>, Bound<'_, PyAny>) = item?.extract()?; + variable.call_method1("set", (value,))?; + } + Ok(()) } /// What a share came to when its writer thread panicked: every one of its tables failed. @@ -154,42 +188,90 @@ struct RunWriter<'a, 'py> { adapter: &'a Bound<'py, PyAny>, connection_config: &'a Bound<'py, PyAny>, identity: &'a RunIdentity, + /// Set once the run is interrupted: no further table is written. + stop: &'a AtomicBool, } impl<'py> RunWriter<'_, 'py> { - /// Write the row to each table of `share` over one connection; a failure stays with its table. - /// - /// Connect for these rows only, so loaders can still lock file warehouses during the build. + /// Write the share in a copy of `context`, the caller's, with every deadline suspended. + fn write_share_in( + &self, + context: &Bound<'py, PyAny>, + share: &[usize], + tables: &[RunTable], + row: (RunRow<'_>, RowWrite), + ) -> ShareResult { + let py: Python<'py> = self.adapter.py(); + let suspended: PyResult> = adopt_context(context).and_then(|()| { + let suspended = host_entry(py, ExecutorHost::ExecutionDeadlinesSuspended)?.call0()?; + suspended.call_method0("__enter__")?; + Ok(suspended) + }); + let suspended: Bound<'py, PyAny> = match suspended { + Ok(suspended) => suspended, + Err(error) => return self.failed_share(share, &error), + }; + let mut result: ShareResult = self.write_share(share, tables, row); + let exited: PyResult<()> = suspended + .call_method1("__exit__", (py.None(), py.None(), py.None())) + .map(|_| ()); + result.closed = result.closed.and(exited); + result + } + + /// Every table of `share` failed with `error`; an interrupt also stops the other writers. + fn failed_share(&self, share: &[usize], error: &PyErr) -> ShareResult { + let py: Python<'py> = self.adapter.py(); + self.note_interrupt(error); + ShareResult { + written: share + .iter() + .map(|index| (*index, Err(error.clone_ref(py)))) + .collect(), + closed: Ok(()), + } + } + + /// Stop the other writers when `error` is an interrupt or exit, not an ordinary failure. + fn note_interrupt(&self, error: &PyErr) { + if !error.is_instance_of::(self.adapter.py()) { + self.stop.store(true, Ordering::SeqCst); + } + } + + fn stopped(&self) -> bool { + self.stop.load(Ordering::SeqCst) + } + + /// Write the row to each table of `share` over one connection, until the run is interrupted. fn write_share( &self, share: &[usize], tables: &[RunTable], (row, mode): (RunRow<'_>, RowWrite), ) -> ShareResult { - let py: Python<'py> = self.adapter.py(); + if self.stopped() { + return ShareResult { + written: Vec::new(), + closed: Ok(()), + }; + } let connected: PyResult> = native_field(self.adapter, "connect") .and_then(|connect| connect.call1((self.connection_config,))); let connection: Bound<'py, PyAny> = match connected { Ok(connection) => connection, - Err(error) => { - return ShareResult { - written: share - .iter() - .map(|index| (*index, Err(error.clone_ref(py)))) - .collect(), - closed: Ok(()), - }; - } + Err(error) => return self.failed_share(share, &error), }; let mut written: Vec<(usize, PyResult<()>)> = Vec::with_capacity(share.len()); for index in share { - let result: PyResult<()> = self.append_row(&connection, &tables[*index], (row, mode)); - let interrupted: bool = - matches!(&result, Err(error) if !error.is_instance_of::(py)); - written.push((*index, result)); - if interrupted { + if self.stopped() { break; } + let result: PyResult<()> = self.append_row(&connection, &tables[*index], (row, mode)); + if let Err(error) = &result { + self.note_interrupt(error); + } + written.push((*index, result)); } let closed: PyResult<()> = native_field(self.adapter, "close") .and_then(|close| close.call1((&connection,))) @@ -209,7 +291,8 @@ impl<'py> RunWriter<'_, 'py> { RowWrite::InsertFirst => { self.append((connection, table), row, false) .or_else(|error| { - if error.is_instance_of::(self.adapter.py()) { + if error.is_instance_of::(self.adapter.py()) && !self.stopped() + { self.append((connection, table), row, true) } else { Err(error) diff --git a/crates/sqlbuild-python-project/src/native_host/main/build_write_locations.rs b/crates/sqlbuild-python-project/src/native_host/main/build_write_locations.rs deleted file mode 100644 index 88c573caa1..0000000000 --- a/crates/sqlbuild-python-project/src/native_host/main/build_write_locations.rs +++ /dev/null @@ -1,37 +0,0 @@ -//! The schemas a build writes, which each record its run. - -use crate::native_host::_helpers::run_locations::{run_locations, written_schema}; -use crate::native_host::constants::SKIP_ACTION; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; -use sqlbuild_python_base::records::main::fields::field::field as native_field; -use sqlbuild_python_front::native_host::models::Location; -use std::collections::BTreeSet; - -/// Where a build records: its models' and seeds' schemas that run, its functions', its sources'. -pub fn build_write_locations( - plan: &Bound<'_, PyAny>, - project: &Bound<'_, PyAny>, -) -> PyResult> { - let mut schemas: BTreeSet = BTreeSet::new(); - for entries in ["model_entries", "seed_entries"] { - for entry in native_field(plan, entries)?.try_iter()? { - let entry: Bound<'_, PyAny> = entry?; - if !native_field(&entry, "action")?.eq(SKIP_ACTION)? { - schemas.extend(written_schema(&native_field(&entry, "destination")?)?); - } - } - } - for entry in native_field(plan, "function_entries")?.try_iter()? { - schemas.extend(written_schema(&native_field(&entry?, "destination")?)?); - } - let sources: Bound<'_, PyAny> = native_field(plan, "source_map")?; - for entry in native_field(plan, "source_load_entries")?.try_iter()? { - if let Some(source) = sources - .call_method1("get", (native_field(&entry?, "name")?,))? - .extract::>>()? - { - schemas.extend(written_schema(&source)?); - } - } - run_locations(schemas, (plan, project)) -} diff --git a/crates/sqlbuild-python-project/src/native_host/main/mod.rs b/crates/sqlbuild-python-project/src/native_host/main/mod.rs index b96ce720f0..97b1fd6b68 100644 --- a/crates/sqlbuild-python-project/src/native_host/main/mod.rs +++ b/crates/sqlbuild-python-project/src/native_host/main/mod.rs @@ -1,8 +1,6 @@ pub mod attach_synced_catalog; -pub mod build_write_locations; pub mod compile_calendars; pub mod compile_sensors; pub mod register_runs_state; pub mod report_synced_catalog; pub mod run_recorder; -pub mod seed_write_locations; diff --git a/crates/sqlbuild-python-project/src/native_host/main/run_recorder.rs b/crates/sqlbuild-python-project/src/native_host/main/run_recorder.rs index 27a5c921b3..968eeffdaf 100644 --- a/crates/sqlbuild-python-project/src/native_host/main/run_recorder.rs +++ b/crates/sqlbuild-python-project/src/native_host/main/run_recorder.rs @@ -1,5 +1,6 @@ //! One run's recorder of the runs table, the same for the CLI and a worker. +use crate::native_host::_helpers::run_locations::{build_write_locations, seed_write_locations}; use crate::native_host::_helpers::runs_state::run_identity; use crate::native_host::models::{RunCommandScope, RunIdentity, RunRecorder}; use pyo3::exceptions::PyException; @@ -7,18 +8,18 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use sqlbuild_core::constants::RUNS_TABLE_NAME; use sqlbuild_python_base::native_host::main::write_console::write_console; use sqlbuild_python_base::records::main::fields::field::field as native_field; -use sqlbuild_python_front::native_host::models::Location; +use sqlbuild_state::runs::constants::SEED_COMMAND; use sqlbuild_state::runs::models::ServerRun; use std::path::PathBuf; /// A run's recorder with its started row written in each schema it writes; only an interrupt errors. pub fn run_recorder<'py>( (command, sqlbuild_version): (&'static str, &'static str), - (request, invocation, project): (&Bound<'py, PyAny>, &Bound<'py, PyAny>, &Bound<'py, PyAny>), - written: impl FnOnce() -> PyResult>, + (request, invocation, pipeline): (&Bound<'py, PyAny>, &Bound<'py, PyAny>, &Bound<'py, PyAny>), server: Option, ) -> PyResult>> { let opened = || -> PyResult>> { + let project: &Bound<'py, PyAny> = &native_field(pipeline, "project")?; let project_dir: PathBuf = native_field(invocation, "effective_project_dir")?.extract()?; let settings: Bound<'py, PyAny> = native_field( &native_field( @@ -47,7 +48,11 @@ pub fn run_recorder<'py>( &native_field(invocation, "adapter")?, &native_field(invocation, "connection_config")?, ), - written()?, + if command == SEED_COMMAND { + seed_write_locations(pipeline)? + } else { + build_write_locations(pipeline)? + }, identity, ) }; diff --git a/crates/sqlbuild-python-project/src/native_host/main/seed_write_locations.rs b/crates/sqlbuild-python-project/src/native_host/main/seed_write_locations.rs deleted file mode 100644 index fa313e8031..0000000000 --- a/crates/sqlbuild-python-project/src/native_host/main/seed_write_locations.rs +++ /dev/null @@ -1,19 +0,0 @@ -//! The schemas `sqb seed` writes, which each record its run. - -use crate::native_host::_helpers::run_locations::{run_locations, written_schema}; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; -use sqlbuild_python_base::records::main::fields::field::field as native_field; -use sqlbuild_python_front::native_host::models::Location; -use std::collections::BTreeSet; - -/// Where `sqb seed` records: the schemas of every seed it loads. -pub fn seed_write_locations( - plan: &Bound<'_, PyAny>, - project: &Bound<'_, PyAny>, -) -> PyResult> { - let mut schemas: BTreeSet = BTreeSet::new(); - for entry in native_field(plan, "seed_entries")?.try_iter()? { - schemas.extend(written_schema(&native_field(&entry?, "destination")?)?); - } - run_locations(schemas, (plan, project)) -} diff --git a/crates/sqlbuild-python-project/src/native_host/models.rs b/crates/sqlbuild-python-project/src/native_host/models.rs index 81257f2392..63cc15a166 100644 --- a/crates/sqlbuild-python-project/src/native_host/models.rs +++ b/crates/sqlbuild-python-project/src/native_host/models.rs @@ -2,14 +2,14 @@ use crate::compiled_views::models::NativeCompiledModel; use crate::native_host::_helpers::run_writes::{ - RowWrite, RunRow, ShareResult, warn_on_errors, write_shares, + RowWrite, RunRow, SharesOutcome, warn_on_errors, write_shares, }; use crate::native_host::_helpers::runs_state::CREATE_TABLE_METHOD; use crate::native_host::constants::MAX_RUN_RECORD_CONNECTIONS; use crate::planner_scope::models::ScopeData; use pyo3::IntoPyObject; use pyo3::PyErr; -use pyo3::exceptions::PyKeyboardInterrupt; +use pyo3::exceptions::{PyException, PyKeyboardInterrupt}; use pyo3::prelude::Bound; use pyo3::prelude::Py; use pyo3::prelude::PyAny; @@ -18,6 +18,7 @@ use pyo3::prelude::PyResult; use pyo3::prelude::Python; use sqlbuild_core::constants::RUNS_TABLE_NAME; use sqlbuild_orchestration::declarations::models::TriggerDefinition; +use sqlbuild_python_base::native_host::main::write_console::write_console; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_front::native_host::main::state_table_name::state_table_name; @@ -245,22 +246,39 @@ impl<'py> RunRecorder<'py> { let _ = self.finished(&RunNodeCounts::default(), ending); } - /// Append the row to every schema's table over a few connections, then warn for each failure. - /// - /// An interrupt, raised in a writer or while waiting for them, escapes once every write is done. + /// Append the row to every schema's table over a few connections; only an interrupt escapes. fn append_everywhere(&self, row: RunRow<'_>, mode: RowWrite) -> PyResult<()> { let py: Python<'py> = self.adapter.py(); let connections: usize = self.tables.len().min(MAX_RUN_RECORD_CONNECTIONS); let shares: Vec> = (0..connections) .map(|first| (first..self.tables.len()).step_by(connections).collect()) .collect(); - let (shared, signal): (Vec, Option) = write_shares( + let outcome: SharesOutcome = match write_shares( (&self.adapter, &self.connection_config), (&self.tables, &self.identity), &shares, (row, mode), + ) { + Ok(outcome) => outcome, + Err(error) if error.is_instance_of::(py) => { + let _ = write_console( + py, + "stderr", + &format!( + "Warning: could not record that run {} {} in {RUNS_TABLE_NAME}: {error}\n", + self.identity.run_id, row.0 + ), + ); + return Ok(()); + } + Err(error) => return Err(error), + }; + let escaped: Option = warn_on_errors( + py, + (&self.tables, &self.identity), + (row.0, outcome.stopped), + outcome.shared, ); - warn_on_errors(py, (&self.tables, &self.identity), row.0, shared)?; - signal.map_or(Ok(()), Err) + outcome.signal.or(escaped).map_or(Ok(()), Err) } } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/command.rs index 920773b1b7..1d8688d836 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/command.rs @@ -38,7 +38,6 @@ use sqlbuild_python_output::recall::main::write_stderr::write_stderr; use sqlbuild_python_planner::observations::main::arm_source_capture::arm_source_capture; use sqlbuild_python_planner::observations::main::take_source_capture::take_source_capture; use sqlbuild_python_project::graph::main::python_nodes::python_node_graph::python_node_graph; -use sqlbuild_python_project::native_host::main::build_write_locations::build_write_locations; use sqlbuild_python_project::native_host::main::run_recorder::run_recorder; use sqlbuild_python_project::native_host::models::RunRecorder; use sqlbuild_python_project::project_snapshot::models::WorkerBuild; @@ -170,17 +169,7 @@ fn run_captured_build(context: &BuildContext<'_>, providers: &Bound<'_, PyAny>) let started_at: Bound<'_, PyAny> = record_pending_cost(context, &pipeline)?; let recorder: Option> = run_recorder( (BUILD_COMMAND, env!("CARGO_PKG_VERSION")), - ( - &context.request, - &context.invocation, - &native_field(&pipeline, "project")?, - ), - || { - build_write_locations( - &native_field(&pipeline, "plan_output")?, - &native_field(&pipeline, "project")?, - ) - }, + (&context.request, &context.invocation, &pipeline), context.worker.as_ref().map(|worker| worker.server_run()), )?; match execute_direct_build(context, &pipeline, providers) { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs index 3465642813..f3e1013527 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs @@ -42,7 +42,6 @@ use sqlbuild_python_output::result_output::models::ExecutionEventWriter; use sqlbuild_python_planner::planner_entries::main::plans::build_seed_command_plan::build_seed_command_plan; use sqlbuild_python_planner::planner_entries::main::plans::read_selected_seed_fingerprints::read_selected_seed_fingerprints; use sqlbuild_python_project::native_host::main::run_recorder::run_recorder; -use sqlbuild_python_project::native_host::main::seed_write_locations::seed_write_locations; use sqlbuild_python_project::native_host::models::RunRecorder; use sqlbuild_python_project::project_snapshot::models::WorkerBuild; use sqlbuild_python_project::project_snapshot::types::VerifiedRun; @@ -486,13 +485,7 @@ pub(crate) fn run_seed<'py>( write_header(&invocation, &pipeline, concurrency)?; let recorder: Option> = run_recorder( (SEED_COMMAND, env!("CARGO_PKG_VERSION")), - (request, &invocation, &native_field(&pipeline, "project")?), - || { - seed_write_locations( - &native_field(&pipeline, "plan_output")?, - &native_field(&pipeline, "project")?, - ) - }, + (request, &invocation, &pipeline), worker.as_ref().map(|worker| worker.server_run()), )?; let started: Instant = Instant::now(); diff --git a/src/sqlbuild/_native_host.py b/src/sqlbuild/_native_host.py index f215c77928..a510e21593 100644 --- a/src/sqlbuild/_native_host.py +++ b/src/sqlbuild/_native_host.py @@ -590,6 +590,10 @@ "sqlbuild.runtime.execution_limits.main.execution_deadline_scope", "execution_deadline_scope", ), + "execution_deadlines_suspended": ( + "sqlbuild.runtime.execution_limits.main.execution_deadlines_suspended", + "execution_deadlines_suspended", + ), "audit_evidence_scope": ( "sqlbuild.executor.auditing.main._audit_evidence_scope", "audit_evidence_scope", diff --git a/src/sqlbuild/_native_host.pyi b/src/sqlbuild/_native_host.pyi index 39099b17ac..bc66b54a07 100644 --- a/src/sqlbuild/_native_host.pyi +++ b/src/sqlbuild/_native_host.pyi @@ -552,6 +552,9 @@ from sqlbuild.runtime.execution_limits.main._cancel_running_statements import ( from sqlbuild.runtime.execution_limits.main.execution_deadline_scope import ( execution_deadline_scope as execution_deadline_scope, ) +from sqlbuild.runtime.execution_limits.main.execution_deadlines_suspended import ( + execution_deadlines_suspended as execution_deadlines_suspended, +) from sqlbuild.runtime.execution_limits.models import RunItemTimeouts as RunItemTimeouts from sqlbuild.runtime.observability.classes.event_dispatcher import ( EventDispatcher as EventDispatcher, diff --git a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py index fa31557b5f..dcb2c14d3d 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py @@ -1464,6 +1464,24 @@ class InterruptedRunRecordTestCase: expected_tables: frozenset[str] +@dataclass(frozen=True) +class SignalledRunRecordTestCase: + description: str + # The warehouse-user SQL that keeps the started row's insert running until it is cancelled. + blocking_warehouse_user_sql: str + signal_after_seconds: float + max_stop_seconds: float + expected_tables: frozenset[str] + + +@dataclass(frozen=True) +class RecalledRunRecordTestCase: + description: str + build_args: tuple[str, ...] + # How many of the build's recalled statements insert into the runs table. + expected_run_inserts: int + + @dataclass(frozen=True) class KilledRunRecordTestCase: description: str diff --git a/tests/integration/src/sqlbuild/cli/commands/main/helpers.py b/tests/integration/src/sqlbuild/cli/commands/main/helpers.py index 24bda44851..713944c1fa 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/helpers.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/helpers.py @@ -4,6 +4,7 @@ import os import shutil import signal +import sqlite3 import subprocess import sys import tempfile @@ -1561,6 +1562,19 @@ def runs_tables(*, project_dir: Path) -> frozenset[str]: return frozenset(name.lower() for (name,) in rows) +def recalled_run_inserts(*, project_dir: Path) -> list[tuple[int, int]]: + """Per recalled invocation, its statement count and how many insert into the runs table.""" + + with sqlite3.connect(project_dir / "logs" / "recall.sqlite3") as store: + return store.execute( + "SELECT COUNT(DISTINCT s.statement_id), " + "COUNT(DISTINCT CASE WHEN a.content LIKE 'INSERT INTO %_sqlbuild_runs%' " + "THEN s.statement_id END) " + "FROM statements AS s LEFT JOIN artifacts AS a ON a.statement_id = s.statement_id " + "GROUP BY s.invocation_id" + ).fetchall() + + def built_tables(*, project_dir: Path) -> frozenset[str]: """The project's own tables in the warehouse, without SQLBuild's state tables.""" diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_run_records.py b/tests/integration/src/sqlbuild/cli/commands/main/test_run_records.py index 86d03bb2f2..11128ce25f 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_run_records.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_run_records.py @@ -3,21 +3,29 @@ from __future__ import annotations import os +import signal import subprocess import sys +import threading +import time from pathlib import Path import pytest from sqlbuild._native import run_cli as main from sqlbuild.adapter.contract.classes.runs_table import RunsTableMixin +from sqlbuild.runtime.execution_limits.main._cancel_running_statements import ( + cancel_running_statements, +) from tests.integration.src.sqlbuild.cli.commands.main._test_types import ( InterruptedRunRecordTestCase, KilledRunRecordTestCase, + RecalledRunRecordTestCase, RunRecordFailureTestCase, RunRecordSchemasTestCase, RunRecordTestCase, SeedRunRecordTestCase, + SignalledRunRecordTestCase, ) from tests.integration.src.sqlbuild.cli.commands.main.helpers import ( RUN_RECORD_COLUMNS, @@ -25,6 +33,7 @@ RUN_RECORD_MARTS_SCHEMA, built_tables, prepare_warehouse, + recalled_run_inserts, run_rows, runs_tables, schema_run_ids, @@ -281,6 +290,99 @@ def interrupt(self: object) -> str | None: assert built_tables(project_dir=tmp_path) == test_case.expected_tables +@pytest.mark.parametrize( + "test_case", + [ + SignalledRunRecordTestCase( + description="a real SIGINT while a run-record insert hangs stops the build promptly", + blocking_warehouse_user_sql=( + "CAST((SELECT SUM(a.range * b.range) " + "FROM range(100000000) AS a, range(100000000) AS b) AS VARCHAR)" + ), + signal_after_seconds=1.0, + max_stop_seconds=15.0, + expected_tables=frozenset(), + ), + ], + ids=lambda case: case.description, +) +def test_given_a_hanging_run_record_when_sigint_arrives_then_the_build_stops_promptly( + test_case: SignalledRunRecordTestCase, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capfd: pytest.CaptureFixture[str], +) -> None: + write_run_record_project(project_dir=tmp_path, settings="") + monkeypatch.setattr( + RunsTableMixin, + "render_current_warehouse_user", + lambda self: test_case.blocking_warehouse_user_sql, + ) + previous = signal.signal(signal.SIGINT, signal.default_int_handler) + signalled_at: list[float] = [] + + def interrupt() -> None: + signalled_at.append(time.monotonic()) + os.kill(os.getpid(), signal.SIGINT) + + sender = threading.Timer(test_case.signal_after_seconds, interrupt) + # A broken stop would hang on the insert forever; cancel it later so the test fails instead. + watchdog = threading.Timer(test_case.max_stop_seconds * 4, cancel_running_statements) + sender.start() + watchdog.start() + try: + with pytest.raises(KeyboardInterrupt): + main( + [ + "--project-dir", + str(tmp_path), + "--no-color", + "build", + "--no-tests", + "-s", + "stg_orders", + ] + ) + stopped_at: float = time.monotonic() + finally: + sender.cancel() + watchdog.cancel() + signal.signal(signal.SIGINT, previous) + captured: str = "".join(capfd.readouterr()) + + assert signalled_at, captured + assert stopped_at - signalled_at[0] < test_case.max_stop_seconds, captured + assert "could not record" not in captured, captured + assert built_tables(project_dir=tmp_path) == test_case.expected_tables + + +@pytest.mark.parametrize( + "test_case", + [ + RecalledRunRecordTestCase( + description="a build's run-record inserts reach its own recalled statements", + build_args=("--select", "stg_orders"), + expected_run_inserts=2, + ), + ], + ids=lambda case: case.description, +) +def test_given_a_recorded_build_when_recalled_then_its_run_record_statements_are_its_own( + test_case: RecalledRunRecordTestCase, + tmp_path: Path, +) -> None: + write_run_record_project(project_dir=tmp_path, settings="") + + exit_code: int = main( + ["--project-dir", str(tmp_path), "--no-color", "build", "--no-tests", *test_case.build_args] + ) + + assert exit_code == 0 + [(statements, run_inserts)] = recalled_run_inserts(project_dir=tmp_path) + assert statements > run_inserts + assert run_inserts == test_case.expected_run_inserts + + @pytest.mark.parametrize( "test_case", [ From 89abf78919d718a5ba4cd5482cd6401c5726edfd Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 19 Nov 2026 17:04:24 +0000 Subject: [PATCH 4/7] refactor(state): share the target location reader with the janitor Run records and janitor planning both read the active target's database or schema as a location, naming none for preserve. They now share native_host's target_location. --- .../src/native_host/_helpers/run_locations.rs | 15 ++++----------- .../src/native_host/main/mod.rs | 1 + .../src/native_host/main/target_location.rs | 12 ++++++++++++ .../bindings/_helpers/janitor/janitor_planning.rs | 9 +-------- 4 files changed, 18 insertions(+), 19 deletions(-) create mode 100644 crates/sqlbuild-python-project/src/native_host/main/target_location.rs diff --git a/crates/sqlbuild-python-project/src/native_host/_helpers/run_locations.rs b/crates/sqlbuild-python-project/src/native_host/_helpers/run_locations.rs index 58579f59d3..598c31c5f3 100644 --- a/crates/sqlbuild-python-project/src/native_host/_helpers/run_locations.rs +++ b/crates/sqlbuild-python-project/src/native_host/_helpers/run_locations.rs @@ -1,9 +1,9 @@ //! The schemas a build or seed writes, where it records its rows in the runs table, each once. use crate::native_host::constants::SKIP_ACTION; +use crate::native_host::main::target_location::target_location; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use sqlbuild_analysis::analysis_route::main::same_unquoted_name::same_unquoted_name; -use sqlbuild_analysis::assembly::target_checks::constants::PRESERVE_TARGET_VALUE; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_front::native_host::models::Location; use sqlbuild_state::runs::main::distinct_run_schemas::distinct_run_schemas; @@ -23,20 +23,13 @@ fn written_schema(destination: &Bound<'_, PyAny>) -> PyResult> .transpose() } -/// A target namespace value, None when the target keeps each resource's own (`preserve`). -fn target_value(project: &Bound<'_, PyAny>, field: &str) -> PyResult> { - Ok(native_field(project, field)? - .extract::>()? - .filter(|value| value != PRESERVE_TARGET_VALUE)) -} - /// Where a run writing no relation records, as audits do: the target's schema, else the first node's. fn default_location( plan: &Bound<'_, PyAny>, project: &Bound<'_, PyAny>, ) -> PyResult> { - let database: Option = target_value(project, "effective_target_database")?; - if let Some(schema) = target_value(project, "effective_target_schema")? { + let database: Option = target_location(project, "effective_target_database")?; + if let Some(schema) = target_location(project, "effective_target_schema")? { return Ok(Some(Location { database, schema })); } for locations in ["model_locations", "seed_locations"] { @@ -64,7 +57,7 @@ fn run_locations( return Ok(default_location(plan, project)?.into_iter().collect()); } let dialect: Option = native_field(project, "sql_analysis_dialect")?.extract()?; - let target_database: Option = target_value(project, "effective_target_database")?; + let target_database: Option = target_location(project, "effective_target_database")?; let same_name = |left: &str, right: &str| same_unquoted_name(dialect.as_deref(), left, right); Ok(distinct_run_schemas( schemas diff --git a/crates/sqlbuild-python-project/src/native_host/main/mod.rs b/crates/sqlbuild-python-project/src/native_host/main/mod.rs index 97b1fd6b68..96201d7349 100644 --- a/crates/sqlbuild-python-project/src/native_host/main/mod.rs +++ b/crates/sqlbuild-python-project/src/native_host/main/mod.rs @@ -4,3 +4,4 @@ pub mod compile_sensors; pub mod register_runs_state; pub mod report_synced_catalog; pub mod run_recorder; +pub mod target_location; diff --git a/crates/sqlbuild-python-project/src/native_host/main/target_location.rs b/crates/sqlbuild-python-project/src/native_host/main/target_location.rs new file mode 100644 index 0000000000..9f4226385f --- /dev/null +++ b/crates/sqlbuild-python-project/src/native_host/main/target_location.rs @@ -0,0 +1,12 @@ +//! A target's database or schema as a location, unless it is `preserve`. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_analysis::assembly::target_checks::constants::PRESERVE_TARGET_VALUE; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// The active target's database or schema as a location: `preserve` names none, since each model keeps its own. +pub fn target_location(project: &Bound<'_, PyAny>, field: &str) -> PyResult> { + Ok(native_field(project, field)? + .extract::>()? + .filter(|value| value != PRESERVE_TARGET_VALUE)) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_planning.rs index 9acfa79849..b78439f47a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_planning.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_planning.rs @@ -6,7 +6,6 @@ use pyo3::IntoPyObjectExt; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::pyfunction; use pyo3::types::{PyBool, PyDict, PyDictMethods}; -use sqlbuild_analysis::assembly::target_checks::constants::PRESERVE_TARGET_VALUE; use sqlbuild_analysis::janitor_planning::constants::OLD_NAME_VIEW_REASON; use sqlbuild_analysis::janitor_planning::main::build_archive_name::build_archive_name; use sqlbuild_analysis::janitor_planning::main::query_diff_artifact_name::query_diff_artifact_name; @@ -48,6 +47,7 @@ use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; +use sqlbuild_python_project::native_host::main::target_location::target_location; use sqlbuild_python_project::relation_naming::main::shared_error::shared_error; const LIFECYCLE: ExecutorHost = ExecutorHost::OperationLifecycle; @@ -139,13 +139,6 @@ fn candidate_records<'py>(candidates: &[Candidate<'py>]) -> PyResult, field: &str) -> PyResult> { - Ok(native_field(project, field)? - .extract::>()? - .filter(|value| value != PRESERVE_TARGET_VALUE)) -} - /// Review schemas whose markers the active target schema holds and that outlived any review run. fn review_schema_candidates<'py>( project: &Bound<'py, PyAny>, From f881d7f6c84e22dd9b29c3261ec6dc6a6d897167 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 19 Nov 2026 17:14:54 +0000 Subject: [PATCH 5/7] test(state): annotate the SIGINT run-record test's locals --- .../sqlbuild/cli/commands/main/test_run_records.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_run_records.py b/tests/integration/src/sqlbuild/cli/commands/main/test_run_records.py index 11128ce25f..93fe10d516 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_run_records.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_run_records.py @@ -8,7 +8,10 @@ import sys import threading import time +from collections.abc import Callable from pathlib import Path +from types import FrameType +from typing import Any import pytest @@ -318,16 +321,20 @@ def test_given_a_hanging_run_record_when_sigint_arrives_then_the_build_stops_pro "render_current_warehouse_user", lambda self: test_case.blocking_warehouse_user_sql, ) - previous = signal.signal(signal.SIGINT, signal.default_int_handler) + previous: Callable[[int, FrameType | None], Any] | int | signal.Handlers | None = signal.signal( + signal.SIGINT, signal.default_int_handler + ) signalled_at: list[float] = [] def interrupt() -> None: signalled_at.append(time.monotonic()) os.kill(os.getpid(), signal.SIGINT) - sender = threading.Timer(test_case.signal_after_seconds, interrupt) + sender: threading.Timer = threading.Timer(test_case.signal_after_seconds, interrupt) # A broken stop would hang on the insert forever; cancel it later so the test fails instead. - watchdog = threading.Timer(test_case.max_stop_seconds * 4, cancel_running_statements) + watchdog: threading.Timer = threading.Timer( + test_case.max_stop_seconds * 4, cancel_running_statements + ) sender.start() watchdog.start() try: From e068c9ce11358f48ec25b38d67e6edf853fea546 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 19 Nov 2026 17:49:47 +0000 Subject: [PATCH 6/7] build(fensu): root the execution_deadlines_suspended host entry Native run-record writers resolve execution_deadlines_suspended through the sqlbuild._native_host name table, so it is a dead-code root like the other host targets. --- fensu.toml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/fensu.toml b/fensu.toml index 73dcbfa978..faa329a5a4 100644 --- a/fensu.toml +++ b/fensu.toml @@ -772,6 +772,11 @@ modules = ["sqlbuild.runtime.execution_limits.main.execution_deadline_scope"] symbols = ["execution_deadline_scope"] reason = "Native code resolves these through the sqlbuild._native_host name table." +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.runtime.execution_limits.main.execution_deadlines_suspended"] +symbols = ["execution_deadlines_suspended"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + [[targets.python.dead_code.roots]] modules = ["sqlbuild.executor.auditing.main._audit_evidence_scope"] symbols = ["audit_evidence_scope"] From f69a51d3a45f2b7e9fe5cdbb96dce802155ea218 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 19 Nov 2026 18:46:05 +0000 Subject: [PATCH 7/7] ci: rerun checks for the fensu root commit