diff --git a/crates/sqlbuild-analysis/src/assembly/maintained_checks/constants.rs b/crates/sqlbuild-analysis/src/assembly/maintained_checks/constants.rs new file mode 100644 index 0000000000..385fec1fd5 --- /dev/null +++ b/crates/sqlbuild-analysis/src/assembly/maintained_checks/constants.rs @@ -0,0 +1,4 @@ +//! Error codes of the project-wide maintained model checks. + +/// A maintained model that nothing would ever refresh. +pub const MAINTAINED_NEVER_REFRESHED_CODE: &str = "A016"; diff --git a/crates/sqlbuild-analysis/src/assembly/maintained_checks/main/mod.rs b/crates/sqlbuild-analysis/src/assembly/maintained_checks/main/mod.rs new file mode 100644 index 0000000000..d43fa0186a --- /dev/null +++ b/crates/sqlbuild-analysis/src/assembly/maintained_checks/main/mod.rs @@ -0,0 +1 @@ +pub mod never_refreshed_maintained; diff --git a/crates/sqlbuild-analysis/src/assembly/maintained_checks/main/never_refreshed_maintained.rs b/crates/sqlbuild-analysis/src/assembly/maintained_checks/main/never_refreshed_maintained.rs new file mode 100644 index 0000000000..078332bc87 --- /dev/null +++ b/crates/sqlbuild-analysis/src/assembly/maintained_checks/main/never_refreshed_maintained.rs @@ -0,0 +1,20 @@ +//! Maintained models with no refresh of their own that no maintained model reads. + +use std::collections::HashSet; + +use crate::assembly::maintained_checks::models::MaintainedModel; + +/// The first by name; without a reader or its own refresh, the warehouse never refreshes it. +pub fn never_refreshed_maintained(models: &[MaintainedModel]) -> Option<&str> { + let maintained: HashSet<&str> = models.iter().map(|model| model.name.as_str()).collect(); + let read: HashSet<&str> = models + .iter() + .flat_map(|model| model.reads.iter().map(String::as_str)) + .filter(|name| maintained.contains(name)) + .collect(); + models + .iter() + .filter(|model| !model.has_refresh && !read.contains(model.name.as_str())) + .map(|model| model.name.as_str()) + .min() +} diff --git a/crates/sqlbuild-analysis/src/assembly/maintained_checks/mod.rs b/crates/sqlbuild-analysis/src/assembly/maintained_checks/mod.rs new file mode 100644 index 0000000000..042268f15e --- /dev/null +++ b/crates/sqlbuild-analysis/src/assembly/maintained_checks/mod.rs @@ -0,0 +1,6 @@ +pub mod constants; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-analysis/src/assembly/maintained_checks/models.rs b/crates/sqlbuild-analysis/src/assembly/maintained_checks/models.rs new file mode 100644 index 0000000000..4724ca32f3 --- /dev/null +++ b/crates/sqlbuild-analysis/src/assembly/maintained_checks/models.rs @@ -0,0 +1,9 @@ +//! What the project-wide maintained model checks read. + +/// One maintained model: whether it sets its own refresh, and the models it reads. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct MaintainedModel { + pub name: String, + pub has_refresh: bool, + pub reads: Vec, +} diff --git a/crates/sqlbuild-analysis/src/assembly/maintained_checks/tests.rs b/crates/sqlbuild-analysis/src/assembly/maintained_checks/tests.rs new file mode 100644 index 0000000000..e40ce2efc9 --- /dev/null +++ b/crates/sqlbuild-analysis/src/assembly/maintained_checks/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_maintained_checks.rs"] +mod maintained_checks; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-analysis/src/assembly/maintained_checks/tests/helpers.rs b/crates/sqlbuild-analysis/src/assembly/maintained_checks/tests/helpers.rs new file mode 100644 index 0000000000..2ad27cec20 --- /dev/null +++ b/crates/sqlbuild-analysis/src/assembly/maintained_checks/tests/helpers.rs @@ -0,0 +1,10 @@ +use crate::assembly::maintained_checks::models::MaintainedModel; + +/// A maintained model named `name` reading `reads`. +pub(crate) fn maintained(name: &str, has_refresh: bool, reads: &[&str]) -> MaintainedModel { + MaintainedModel { + name: name.to_owned(), + has_refresh, + reads: reads.iter().map(|read| (*read).to_owned()).collect(), + } +} diff --git a/crates/sqlbuild-analysis/src/assembly/maintained_checks/tests/test_maintained_checks.rs b/crates/sqlbuild-analysis/src/assembly/maintained_checks/tests/test_maintained_checks.rs new file mode 100644 index 0000000000..f363cfca59 --- /dev/null +++ b/crates/sqlbuild-analysis/src/assembly/maintained_checks/tests/test_maintained_checks.rs @@ -0,0 +1,52 @@ +use crate::assembly::maintained_checks::main::never_refreshed_maintained::never_refreshed_maintained; +use crate::assembly::maintained_checks::tests::helpers::maintained; +use crate::assembly::maintained_checks::tests::test_types::NeverRefreshedTestCase; + +#[test] +fn given_maintained_chains_when_checking_then_unreached_models_without_refresh_are_named() { + let test_cases = [ + NeverRefreshedTestCase { + description: "an upstream model refreshed for its reader", + models: vec![ + maintained("clean_orders", false, &["orders"]), + maintained("daily_totals", true, &["clean_orders"]), + ], + expected_name: None, + }, + NeverRefreshedTestCase { + description: "the last model of a chain without refresh", + models: vec![ + maintained("clean_orders", false, &["orders"]), + maintained("daily_totals", false, &["clean_orders"]), + ], + expected_name: Some("daily_totals"), + }, + NeverRefreshedTestCase { + description: "a model read only by a table, which never asks for a refresh", + models: vec![maintained("clean_orders", false, &["orders"])], + expected_name: Some("clean_orders"), + }, + NeverRefreshedTestCase { + description: "two unread models report the first by name", + models: vec![ + maintained("b_totals", false, &["orders"]), + maintained("a_totals", false, &["orders"]), + ], + expected_name: Some("a_totals"), + }, + NeverRefreshedTestCase { + description: "every model sets its own refresh", + models: vec![maintained("daily_totals", true, &[])], + expected_name: None, + }, + ]; + + for test_case in test_cases { + assert_eq!( + never_refreshed_maintained(&test_case.models), + test_case.expected_name, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/assembly/maintained_checks/tests/test_types.rs b/crates/sqlbuild-analysis/src/assembly/maintained_checks/tests/test_types.rs new file mode 100644 index 0000000000..66f9fbe2f1 --- /dev/null +++ b/crates/sqlbuild-analysis/src/assembly/maintained_checks/tests/test_types.rs @@ -0,0 +1,7 @@ +use crate::assembly::maintained_checks::models::MaintainedModel; + +pub(crate) struct NeverRefreshedTestCase { + pub(crate) description: &'static str, + pub(crate) models: Vec, + pub(crate) expected_name: Option<&'static str>, +} diff --git a/crates/sqlbuild-analysis/src/assembly/mod.rs b/crates/sqlbuild-analysis/src/assembly/mod.rs index 1b4fb3a6af..4e62fc77e8 100644 --- a/crates/sqlbuild-analysis/src/assembly/mod.rs +++ b/crates/sqlbuild-analysis/src/assembly/mod.rs @@ -1,5 +1,6 @@ //! The native model analysis session: catalog, dataflow scheduling, projection and shape publication. pub mod analysis_session; +pub mod maintained_checks; pub mod project; pub mod target_checks; diff --git a/crates/sqlbuild-analysis/src/janitor_planning/constants.rs b/crates/sqlbuild-analysis/src/janitor_planning/constants.rs index 30781736f8..0984a15101 100644 --- a/crates/sqlbuild-analysis/src/janitor_planning/constants.rs +++ b/crates/sqlbuild-analysis/src/janitor_planning/constants.rs @@ -3,7 +3,7 @@ use sqlbuild_core::constants::STATE_TABLE_NAMES; /// SQLBuild state tables the janitor never acts on. -pub const BUILT_IN_EXCLUDE_PATTERNS: [&str; 13] = STATE_TABLE_NAMES; +pub const BUILT_IN_EXCLUDE_PATTERNS: [&str; 14] = STATE_TABLE_NAMES; /// Prefix of every strict janitor archive name. pub const ARCHIVE_NAME_PREFIX: &str = "_SQB_ARCHIVE__"; /// Separator between archive name segments. diff --git a/crates/sqlbuild-analysis/src/lib.rs b/crates/sqlbuild-analysis/src/lib.rs index 5bb2c63eae..303bf144da 100644 --- a/crates/sqlbuild-analysis/src/lib.rs +++ b/crates/sqlbuild-analysis/src/lib.rs @@ -22,6 +22,7 @@ pub mod janitor_planning; pub mod lineage; pub mod lineage_command; pub mod lineage_graph; +pub mod maintained_refresh; pub mod manifest; pub mod node_set_dependencies; pub mod nullability_rules; diff --git a/crates/sqlbuild-analysis/src/maintained_refresh/_helpers/mod.rs b/crates/sqlbuild-analysis/src/maintained_refresh/_helpers/mod.rs new file mode 100644 index 0000000000..f72a321364 --- /dev/null +++ b/crates/sqlbuild-analysis/src/maintained_refresh/_helpers/mod.rs @@ -0,0 +1 @@ +pub(crate) mod target_lag; diff --git a/crates/sqlbuild-analysis/src/maintained_refresh/_helpers/target_lag.rs b/crates/sqlbuild-analysis/src/maintained_refresh/_helpers/target_lag.rs new file mode 100644 index 0000000000..6123f16214 --- /dev/null +++ b/crates/sqlbuild-analysis/src/maintained_refresh/_helpers/target_lag.rs @@ -0,0 +1,24 @@ +//! A dynamic table's listed `target_lag`, such as `DOWNSTREAM` or `2 hours`. + +use crate::maintained_refresh::constants::{DOWNSTREAM_TARGET_LAG, TARGET_LAG_UNITS}; +use crate::maintained_refresh::models::RefreshLag; + +/// The lag `text` lists, or `None` when it is not a lag Snowflake lists. +pub(crate) fn parse_target_lag(text: &str) -> Option { + let text: &str = text.trim(); + if text.eq_ignore_ascii_case(DOWNSTREAM_TARGET_LAG) { + return Some(RefreshLag::Downstream); + } + let (amount, unit) = text.split_once(' ')?; + let amount: u64 = match amount.parse::() { + Ok(amount) => amount, + Err(_) => return None, + }; + let unit: String = unit.trim().to_ascii_lowercase(); + let unit: &str = unit.strip_suffix('s').unwrap_or(&unit); + TARGET_LAG_UNITS + .iter() + .find(|(name, _)| *name == unit) + .and_then(|(_, seconds)| amount.checked_mul(*seconds)) + .map(RefreshLag::Seconds) +} diff --git a/crates/sqlbuild-analysis/src/maintained_refresh/constants.rs b/crates/sqlbuild-analysis/src/maintained_refresh/constants.rs new file mode 100644 index 0000000000..97d74e91a2 --- /dev/null +++ b/crates/sqlbuild-analysis/src/maintained_refresh/constants.rs @@ -0,0 +1,11 @@ +//! Snowflake target lag spellings. + +/// The `target_lag` of a dynamic table refreshed for its readers. +pub const DOWNSTREAM_TARGET_LAG: &str = "DOWNSTREAM"; +/// Target lag units, largest first, with their seconds. +pub(crate) const TARGET_LAG_UNITS: [(&str, u64); 4] = [ + ("day", 86_400), + ("hour", 3_600), + ("minute", 60), + ("second", 1), +]; diff --git a/crates/sqlbuild-analysis/src/maintained_refresh/main/maintained_settings_drift.rs b/crates/sqlbuild-analysis/src/maintained_refresh/main/maintained_settings_drift.rs new file mode 100644 index 0000000000..6c2f11b86c --- /dev/null +++ b/crates/sqlbuild-analysis/src/maintained_refresh/main/maintained_settings_drift.rs @@ -0,0 +1,15 @@ +//! Whether a maintained model's listed refresh settings differ from its declared ones. + +use crate::maintained_refresh::_helpers::target_lag::parse_target_lag; +use crate::maintained_refresh::models::{LiveDynamicTable, MaintainedSettings}; + +/// True when the lag differs or cannot be read, or a declared warehouse differs. +pub fn maintained_settings_drift(desired: &MaintainedSettings, live: &LiveDynamicTable) -> bool { + if parse_target_lag(&live.target_lag) != Some(desired.lag) { + return true; + } + desired + .warehouse + .as_deref() + .is_some_and(|warehouse| !warehouse.eq_ignore_ascii_case(live.warehouse.trim())) +} diff --git a/crates/sqlbuild-analysis/src/maintained_refresh/main/mod.rs b/crates/sqlbuild-analysis/src/maintained_refresh/main/mod.rs new file mode 100644 index 0000000000..a48b621304 --- /dev/null +++ b/crates/sqlbuild-analysis/src/maintained_refresh/main/mod.rs @@ -0,0 +1,3 @@ +pub mod maintained_settings_drift; +pub mod reinitialised_readers; +pub mod render_target_lag; diff --git a/crates/sqlbuild-analysis/src/maintained_refresh/main/reinitialised_readers.rs b/crates/sqlbuild-analysis/src/maintained_refresh/main/reinitialised_readers.rs new file mode 100644 index 0000000000..82f825c8f9 --- /dev/null +++ b/crates/sqlbuild-analysis/src/maintained_refresh/main/reinitialised_readers.rs @@ -0,0 +1,32 @@ +//! Maintained models the plan keeps whose upstream it replaces, so they recompute in full. + +use std::collections::BTreeSet; + +use crate::maintained_refresh::models::MaintainedRead; + +/// Each kept maintained reader by name, with the first replaced upstream it reads by name. +pub fn reinitialised_readers(reads: &[MaintainedRead]) -> Vec<(String, String)> { + let replaced: BTreeSet<&str> = reads + .iter() + .filter(|read| read.replaced) + .map(|read| read.name.as_str()) + .collect(); + let mut readers: Vec<(String, String)> = Vec::new(); + for read in reads + .iter() + .filter(|read| read.maintained && !read.replaced) + { + let mut upstreams: Vec<&str> = read + .reads + .iter() + .map(String::as_str) + .filter(|upstream| replaced.contains(upstream)) + .collect(); + upstreams.sort_unstable(); + if let Some(upstream) = upstreams.first() { + readers.push((read.name.clone(), (*upstream).to_owned())); + } + } + readers.sort(); + readers +} diff --git a/crates/sqlbuild-analysis/src/maintained_refresh/main/render_target_lag.rs b/crates/sqlbuild-analysis/src/maintained_refresh/main/render_target_lag.rs new file mode 100644 index 0000000000..ca61aa4d47 --- /dev/null +++ b/crates/sqlbuild-analysis/src/maintained_refresh/main/render_target_lag.rs @@ -0,0 +1,19 @@ +//! The `TARGET_LAG` value of a refresh lag, in its largest exact unit. + +use crate::maintained_refresh::constants::{DOWNSTREAM_TARGET_LAG, TARGET_LAG_UNITS}; +use crate::maintained_refresh::models::RefreshLag; + +/// `DOWNSTREAM`, or a quoted lag such as `'2 hours'`. +pub fn render_target_lag(lag: RefreshLag) -> String { + let RefreshLag::Seconds(seconds) = lag else { + return DOWNSTREAM_TARGET_LAG.to_owned(); + }; + let (unit, size) = TARGET_LAG_UNITS + .iter() + .find(|(_, size)| seconds % size == 0) + .copied() + .unwrap_or(("second", 1)); + let amount: u64 = seconds / size; + let plural: &str = if amount == 1 { "" } else { "s" }; + format!("'{amount} {unit}{plural}'") +} diff --git a/crates/sqlbuild-analysis/src/maintained_refresh/mod.rs b/crates/sqlbuild-analysis/src/maintained_refresh/mod.rs new file mode 100644 index 0000000000..31a6ce51c4 --- /dev/null +++ b/crates/sqlbuild-analysis/src/maintained_refresh/mod.rs @@ -0,0 +1,9 @@ +//! Maintained refresh: dynamic table lag text, refresh setting drift, and readers a rebuild reinitialises. + +pub(crate) mod _helpers; +pub mod constants; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-analysis/src/maintained_refresh/models.rs b/crates/sqlbuild-analysis/src/maintained_refresh/models.rs new file mode 100644 index 0000000000..057d87a6e7 --- /dev/null +++ b/crates/sqlbuild-analysis/src/maintained_refresh/models.rs @@ -0,0 +1,34 @@ +//! Declared and listed refresh settings of maintained models. + +/// How stale a maintained model may get: refreshed for its readers, or within a fixed lag. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum RefreshLag { + /// Refreshed when the maintained models reading it need it. + Downstream, + /// Refreshed within this many seconds. + Seconds(u64), +} + +/// The refresh settings a maintained model declares. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct MaintainedSettings { + pub lag: RefreshLag, + /// The declared refresh warehouse; unset keeps whichever the object has. + pub warehouse: Option, +} + +/// A maintained model's refresh settings as the warehouse lists them. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct LiveDynamicTable { + pub target_lag: String, + pub warehouse: String, +} + +/// One planned model: whether it is maintained, whether this plan replaces it, and what it reads. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct MaintainedRead { + pub name: String, + pub maintained: bool, + pub replaced: bool, + pub reads: Vec, +} diff --git a/crates/sqlbuild-analysis/src/maintained_refresh/tests.rs b/crates/sqlbuild-analysis/src/maintained_refresh/tests.rs new file mode 100644 index 0000000000..0b70bcf139 --- /dev/null +++ b/crates/sqlbuild-analysis/src/maintained_refresh/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_maintained_refresh.rs"] +mod maintained_refresh; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-analysis/src/maintained_refresh/tests/helpers.rs b/crates/sqlbuild-analysis/src/maintained_refresh/tests/helpers.rs new file mode 100644 index 0000000000..9fa27f7945 --- /dev/null +++ b/crates/sqlbuild-analysis/src/maintained_refresh/tests/helpers.rs @@ -0,0 +1,26 @@ +use crate::maintained_refresh::models::{ + LiveDynamicTable, MaintainedRead, MaintainedSettings, RefreshLag, +}; + +pub(super) fn settings(lag: RefreshLag, warehouse: Option<&str>) -> MaintainedSettings { + MaintainedSettings { + lag, + warehouse: warehouse.map(str::to_owned), + } +} + +pub(super) fn live(target_lag: &str, warehouse: &str) -> LiveDynamicTable { + LiveDynamicTable { + target_lag: target_lag.to_owned(), + warehouse: warehouse.to_owned(), + } +} + +pub(super) fn read(name: &str, maintained: bool, replaced: bool, reads: &[&str]) -> MaintainedRead { + MaintainedRead { + name: name.to_owned(), + maintained, + replaced, + reads: reads.iter().map(|read| (*read).to_owned()).collect(), + } +} diff --git a/crates/sqlbuild-analysis/src/maintained_refresh/tests/test_maintained_refresh.rs b/crates/sqlbuild-analysis/src/maintained_refresh/tests/test_maintained_refresh.rs new file mode 100644 index 0000000000..145cdbf5bd --- /dev/null +++ b/crates/sqlbuild-analysis/src/maintained_refresh/tests/test_maintained_refresh.rs @@ -0,0 +1,209 @@ +use crate::maintained_refresh::_helpers::target_lag::parse_target_lag; +use crate::maintained_refresh::main::maintained_settings_drift::maintained_settings_drift; +use crate::maintained_refresh::main::reinitialised_readers::reinitialised_readers; +use crate::maintained_refresh::main::render_target_lag::render_target_lag; +use crate::maintained_refresh::models::RefreshLag; +use crate::maintained_refresh::tests::helpers::{live, read, settings}; +use crate::maintained_refresh::tests::test_types::{ + DriftTestCase, ReinitialiseTestCase, RenderLagTestCase, TargetLagTestCase, +}; + +#[test] +fn given_snowflake_target_lag_text_when_parsing_then_lag_is_read_in_seconds() { + let test_cases = [ + TargetLagTestCase { + description: "lag set by readers", + text: "DOWNSTREAM", + expected_lag: Some(RefreshLag::Downstream), + }, + TargetLagTestCase { + description: "one minute, singular", + text: "1 minute", + expected_lag: Some(RefreshLag::Seconds(60)), + }, + TargetLagTestCase { + description: "hours, plural and padded", + text: " 2 hours ", + expected_lag: Some(RefreshLag::Seconds(7_200)), + }, + TargetLagTestCase { + description: "seconds", + text: "90 seconds", + expected_lag: Some(RefreshLag::Seconds(90)), + }, + TargetLagTestCase { + description: "days", + text: "1 day", + expected_lag: Some(RefreshLag::Seconds(86_400)), + }, + TargetLagTestCase { + description: "an unknown unit", + text: "1 fortnight", + expected_lag: None, + }, + ]; + for test_case in test_cases { + assert_eq!( + parse_target_lag(test_case.text), + test_case.expected_lag, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_refresh_lag_when_rendering_then_largest_exact_unit_is_used() { + let test_cases = [ + RenderLagTestCase { + description: "lag set by readers", + lag: RefreshLag::Downstream, + expected_sql: "DOWNSTREAM", + }, + RenderLagTestCase { + description: "a whole hour", + lag: RefreshLag::Seconds(3_600), + expected_sql: "'1 hour'", + }, + RenderLagTestCase { + description: "ninety minutes stays in minutes", + lag: RefreshLag::Seconds(5_400), + expected_sql: "'90 minutes'", + }, + RenderLagTestCase { + description: "two days", + lag: RefreshLag::Seconds(172_800), + expected_sql: "'2 days'", + }, + RenderLagTestCase { + description: "seconds that are not whole minutes", + lag: RefreshLag::Seconds(61), + expected_sql: "'61 seconds'", + }, + ]; + for test_case in test_cases { + assert_eq!( + render_target_lag(test_case.lag), + test_case.expected_sql, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_live_dynamic_table_when_comparing_settings_then_only_real_drift_alters() { + let test_cases = [ + DriftTestCase { + description: "the same lag written in another unit", + desired: settings(RefreshLag::Seconds(3_600), None), + live: live("60 minutes", "TRANSFORM_WH"), + expected_drift: false, + }, + DriftTestCase { + description: "a shorter lag", + desired: settings(RefreshLag::Seconds(300), None), + live: live("1 hour", "TRANSFORM_WH"), + expected_drift: true, + }, + DriftTestCase { + description: "a lag now set by readers", + desired: settings(RefreshLag::Downstream, None), + live: live("1 hour", "TRANSFORM_WH"), + expected_drift: true, + }, + DriftTestCase { + description: "an unset refresh warehouse keeps the existing one", + desired: settings(RefreshLag::Downstream, None), + live: live("DOWNSTREAM", "ANY_WH"), + expected_drift: false, + }, + DriftTestCase { + description: "a warehouse matched without case", + desired: settings(RefreshLag::Downstream, Some("transform_wh")), + live: live("DOWNSTREAM", "TRANSFORM_WH"), + expected_drift: false, + }, + DriftTestCase { + description: "a different warehouse", + desired: settings(RefreshLag::Downstream, Some("reporting_wh")), + live: live("DOWNSTREAM", "TRANSFORM_WH"), + expected_drift: true, + }, + DriftTestCase { + description: "an unreadable live lag is set again", + desired: settings(RefreshLag::Seconds(60), None), + live: live("whenever", "TRANSFORM_WH"), + expected_drift: true, + }, + ]; + for test_case in test_cases { + assert_eq!( + maintained_settings_drift(&test_case.desired, &test_case.live), + test_case.expected_drift, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_planned_rebuilds_when_checking_maintained_readers_then_reinitialised_ones_are_named() { + let test_cases = [ + ReinitialiseTestCase { + description: "a maintained model reading a rebuilt table", + reads: vec![ + read("orders", false, true, &[]), + read("daily_totals", true, false, &["orders"]), + ], + expected_readers: vec![("daily_totals", "orders")], + }, + ReinitialiseTestCase { + description: "an upstream kept in place, such as a merged incremental", + reads: vec![ + read("orders", false, false, &[]), + read("daily_totals", true, false, &["orders"]), + ], + expected_readers: vec![], + }, + ReinitialiseTestCase { + description: "a recreated maintained model reinitialises its maintained readers", + reads: vec![ + read("clean_orders", true, true, &["orders"]), + read("daily_totals", true, false, &["clean_orders"]), + ], + expected_readers: vec![("daily_totals", "clean_orders")], + }, + ReinitialiseTestCase { + description: "a recreated model itself is not reported, and a table reader is ignored", + reads: vec![ + read("orders", false, true, &[]), + read("daily_totals", true, true, &["orders"]), + read("report", false, false, &["orders"]), + ], + expected_readers: vec![], + }, + ReinitialiseTestCase { + description: "the first replaced upstream by name is named", + reads: vec![ + read("orders", false, true, &[]), + read("customers", false, true, &[]), + read("daily_totals", true, false, &["orders", "customers"]), + ], + expected_readers: vec![("daily_totals", "customers")], + }, + ]; + for test_case in test_cases { + let readers: Vec<(String, String)> = reinitialised_readers(&test_case.reads); + let named: Vec<(&str, &str)> = readers + .iter() + .map(|(reader, upstream)| (reader.as_str(), upstream.as_str())) + .collect(); + + assert_eq!( + named, test_case.expected_readers, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/maintained_refresh/tests/test_types.rs b/crates/sqlbuild-analysis/src/maintained_refresh/tests/test_types.rs new file mode 100644 index 0000000000..70aedbf086 --- /dev/null +++ b/crates/sqlbuild-analysis/src/maintained_refresh/tests/test_types.rs @@ -0,0 +1,28 @@ +use crate::maintained_refresh::models::{ + LiveDynamicTable, MaintainedRead, MaintainedSettings, RefreshLag, +}; + +pub(super) struct TargetLagTestCase { + pub(super) description: &'static str, + pub(super) text: &'static str, + pub(super) expected_lag: Option, +} + +pub(super) struct RenderLagTestCase { + pub(super) description: &'static str, + pub(super) lag: RefreshLag, + pub(super) expected_sql: &'static str, +} + +pub(super) struct DriftTestCase { + pub(super) description: &'static str, + pub(super) desired: MaintainedSettings, + pub(super) live: LiveDynamicTable, + pub(super) expected_drift: bool, +} + +pub(super) struct ReinitialiseTestCase { + pub(super) description: &'static str, + pub(super) reads: Vec, + pub(super) expected_readers: Vec<(&'static str, &'static str)>, +} diff --git a/crates/sqlbuild-analysis/src/planner_identity/constants.rs b/crates/sqlbuild-analysis/src/planner_identity/constants.rs index 79f8e591ff..fcdf538680 100644 --- a/crates/sqlbuild-analysis/src/planner_identity/constants.rs +++ b/crates/sqlbuild-analysis/src/planner_identity/constants.rs @@ -1,7 +1,7 @@ //! Planner identity constants shared with the Python planner. /// Model config keys whose values participate in version identity, in Python's insertion order after `cursor_inputs`. -pub const VERSION_IDENTITY_CONFIG_KEYS: [&str; 40] = [ +pub const VERSION_IDENTITY_CONFIG_KEYS: [&str; 41] = [ "append_cursor_inclusive", "batch_size", "check_columns", @@ -42,6 +42,7 @@ pub const VERSION_IDENTITY_CONFIG_KEYS: [&str; 40] = [ "partition_by", "primary_key", "table_settings", + "refresh_mode", ]; /// Model config key of cursor inputs, recorded before every other identity key. pub const CURSOR_INPUTS_CONFIG_KEY: &str = "cursor_inputs"; diff --git a/crates/sqlbuild-analysis/src/planner_strategy/main/materialization_type.rs b/crates/sqlbuild-analysis/src/planner_strategy/main/materialization_type.rs index f45163514c..40708d18a5 100644 --- a/crates/sqlbuild-analysis/src/planner_strategy/main/materialization_type.rs +++ b/crates/sqlbuild-analysis/src/planner_strategy/main/materialization_type.rs @@ -10,6 +10,7 @@ pub fn materialization_type(materialized: Option<&str>) -> MaterializationType { Some("incremental") => MaterializationType::Incremental, Some("snapshot") => MaterializationType::Snapshot, Some("seed") => MaterializationType::Seed, + Some("maintained") => MaterializationType::Maintained, Some(_) => MaterializationType::Custom, } } diff --git a/crates/sqlbuild-analysis/src/planner_strategy/main/model_plan_action.rs b/crates/sqlbuild-analysis/src/planner_strategy/main/model_plan_action.rs index 437d66d599..de5c990fc2 100644 --- a/crates/sqlbuild-analysis/src/planner_strategy/main/model_plan_action.rs +++ b/crates/sqlbuild-analysis/src/planner_strategy/main/model_plan_action.rs @@ -41,6 +41,13 @@ pub fn model_plan_action( refreshed(PlanReason::NormalIncremental, false), )); } + MaterializationType::Maintained => { + let reason: PlanReason = refreshed(PlanReason::NoChange, true); + if reason == PlanReason::NoChange { + return Ok((PlanAction::KeepMaintained, reason)); + } + return Ok((PlanAction::CreateMaintained, reason)); + } MaterializationType::Table | MaterializationType::Incremental | MaterializationType::Seed => {} diff --git a/crates/sqlbuild-analysis/src/planner_strategy/models.rs b/crates/sqlbuild-analysis/src/planner_strategy/models.rs index 959f26c038..a760213e79 100644 --- a/crates/sqlbuild-analysis/src/planner_strategy/models.rs +++ b/crates/sqlbuild-analysis/src/planner_strategy/models.rs @@ -11,6 +11,8 @@ pub enum MaterializationType { Snapshot, Seed, Custom, + /// A stored result the warehouse keeps fresh between builds. + Maintained, } /// What the plan does with a model. @@ -24,6 +26,12 @@ pub enum PlanAction { Snapshot, Skip, Custom, + /// Create or replace the warehouse-maintained object. + CreateMaintained, + /// Change its refresh settings in place. + AlterMaintained, + /// Leave the object as it is; builds may still refresh it. + KeepMaintained, } /// Why the plan does it. diff --git a/crates/sqlbuild-analysis/src/planner_strategy/tests/test_planner_strategy.rs b/crates/sqlbuild-analysis/src/planner_strategy/tests/test_planner_strategy.rs index a65170fcaf..66db810b10 100644 --- a/crates/sqlbuild-analysis/src/planner_strategy/tests/test_planner_strategy.rs +++ b/crates/sqlbuild-analysis/src/planner_strategy/tests/test_planner_strategy.rs @@ -102,6 +102,54 @@ fn given_model_change_when_resolving_action_then_matches_python_strategy() { "S202 incremental model 'orders' has unknown strategy 'upsert'".to_owned(), ), }, + ActionTestCase { + description: "maintained first run creates the dynamic table", + materialization: MaterializationType::Maintained, + strategy: None, + change: change(PlanChangeKind::FirstRun), + full_refresh: false, + expected_action: Ok((PlanAction::CreateMaintained, PlanReason::FirstRun)), + }, + ActionTestCase { + description: "maintained query change recreates it", + materialization: MaterializationType::Maintained, + strategy: None, + change: change(PlanChangeKind::QueryChanged), + full_refresh: false, + expected_action: Ok((PlanAction::CreateMaintained, PlanReason::QueryChanged)), + }, + ActionTestCase { + description: "maintained config change such as refresh_mode recreates it", + materialization: MaterializationType::Maintained, + strategy: None, + change: change(PlanChangeKind::ConfigChanged), + full_refresh: false, + expected_action: Ok((PlanAction::CreateMaintained, PlanReason::ConfigChanged)), + }, + ActionTestCase { + description: "maintained rename creates it under the new name", + materialization: MaterializationType::Maintained, + strategy: None, + change: change(PlanChangeKind::Renamed), + full_refresh: false, + expected_action: Ok((PlanAction::CreateMaintained, PlanReason::Renamed)), + }, + ActionTestCase { + description: "maintained full refresh recreates an unchanged model", + materialization: MaterializationType::Maintained, + strategy: None, + change: change(PlanChangeKind::NoChange), + full_refresh: true, + expected_action: Ok((PlanAction::CreateMaintained, PlanReason::FullRefresh)), + }, + ActionTestCase { + description: "unchanged maintained model is kept", + materialization: MaterializationType::Maintained, + strategy: None, + change: change(PlanChangeKind::NoChange), + full_refresh: false, + expected_action: Ok((PlanAction::KeepMaintained, PlanReason::NoChange)), + }, ]; for test_case in test_cases { let model = ModelStrategy { diff --git a/crates/sqlbuild-cli/src/argv/command_table.json b/crates/sqlbuild-cli/src/argv/command_table.json index 0a0e5235bd..80148a54d7 100644 --- a/crates/sqlbuild-cli/src/argv/command_table.json +++ b/crates/sqlbuild-cli/src/argv/command_table.json @@ -2630,6 +2630,17 @@ "default": false, "group": 1 }, + { + "kind": "store_true", + "options": [ + "--ignore-maintained-grace" + ], + "dest": "ignore_maintained_grace", + "nargs": 0, + "const": true, + "default": false, + "group": 1 + }, { "kind": "store", "options": [ diff --git a/crates/sqlbuild-cli/src/argv/constants.rs b/crates/sqlbuild-cli/src/argv/constants.rs index 991c78bae2..14b4b94ec7 100644 --- a/crates/sqlbuild-cli/src/argv/constants.rs +++ b/crates/sqlbuild-cli/src/argv/constants.rs @@ -77,7 +77,7 @@ pub(crate) const INT_CONVERTER: &str = "int"; pub(crate) const PATH_CONVERTER: &str = "Path"; /// Every `CliNamespace` attribute with a default, in declaration order. -pub(crate) const CLI_NAMESPACE_DEFAULTS: [(&str, NamespaceDefault); 192] = [ +pub(crate) const CLI_NAMESPACE_DEFAULTS: [(&str, NamespaceDefault); 193] = [ ("command", NamespaceDefault::None), ("project_dir", NamespaceDefault::None), ("dbt_project_dir", NamespaceDefault::None), @@ -157,6 +157,7 @@ pub(crate) const CLI_NAMESPACE_DEFAULTS: [(&str, NamespaceDefault); 192] = [ "allow_snapshot_schema_change", NamespaceDefault::Bool(false), ), + ("ignore_maintained_grace", NamespaceDefault::Bool(false)), ("concurrency", NamespaceDefault::None), ("verbose", NamespaceDefault::Bool(false)), ("debug", NamespaceDefault::Bool(false)), diff --git a/crates/sqlbuild-cli/src/dispatch/constants.rs b/crates/sqlbuild-cli/src/dispatch/constants.rs index 2910f50607..6b048c7dbb 100644 --- a/crates/sqlbuild-cli/src/dispatch/constants.rs +++ b/crates/sqlbuild-cli/src/dispatch/constants.rs @@ -145,7 +145,7 @@ pub(crate) const PLAN_ROUTE: Route = routed( REQUEST_KEYWORD_SHAPE, ); -const BUILD_FIELDS: [Field; 40] = [ +const BUILD_FIELDS: [Field; 41] = [ PROJECT_DIR, NO_SQL_VALIDATION, same("no_cache"), @@ -169,6 +169,7 @@ const BUILD_FIELDS: [Field; 40] = [ same("allow_retention_decrease"), same("allow_missing_migration_origin"), same("allow_snapshot_schema_change"), + same("ignore_maintained_grace"), same("concurrency"), SELECT, field("selector_files", Source::SelectorFiles), diff --git a/crates/sqlbuild-config/src/effective/constants.rs b/crates/sqlbuild-config/src/effective/constants.rs index 683223e6dd..5508b1565a 100644 --- a/crates/sqlbuild-config/src/effective/constants.rs +++ b/crates/sqlbuild-config/src/effective/constants.rs @@ -238,4 +238,11 @@ pub(crate) const TARGET_FIELDS: &[TargetField] = &[ decided_by: &["plan_from"], }, }, + TargetField { + group: None, + name: "pause_maintained", + rule: MergeRule::Overlay { + decided_by: &["pause_maintained"], + }, + }, ]; diff --git a/crates/sqlbuild-config/src/project/_helpers/documents.rs b/crates/sqlbuild-config/src/project/_helpers/documents.rs index 58d867a89b..bcce7e424e 100644 --- a/crates/sqlbuild-config/src/project/_helpers/documents.rs +++ b/crates/sqlbuild-config/src/project/_helpers/documents.rs @@ -15,8 +15,9 @@ use crate::project::_helpers::help::{ use crate::project::_helpers::keys::Place; use crate::project::_helpers::sections::{ load_changes, load_constants, load_control, load_cost, load_cursors, load_dbt, load_diff, - load_git, load_janitor, load_local_dbt, load_microbatches, load_migrations, load_policy_flag, - load_pools, load_review, load_scenario, load_settings, load_snapshots, setting_overrides, + load_git, load_janitor, load_local_dbt, load_maintained, load_microbatches, load_migrations, + load_policy_flag, load_pools, load_review, load_scenario, load_settings, load_snapshots, + setting_overrides, }; use crate::project::_helpers::sinks::load_sinks; use crate::project::_helpers::targets::{ @@ -30,7 +31,8 @@ use crate::project::constants::{ }; use crate::project::errors::{ConfigFailure, FailureClass, LoadFailure, MessagePiece}; use crate::project::models::{ - Built, ChangesConfig, ConfigFile, ConfigPaths, ControlConfig, PoolLimits, ReviewConfig, + Built, ChangesConfig, ConfigFile, ConfigPaths, ControlConfig, MaintainedConfig, PoolLimits, + ReviewConfig, }; use crate::project::types::{Entries, Fields, Loaded}; use crate::recall::constants::RECALL_SECTION; @@ -297,6 +299,7 @@ fn project_fields(payload: &Entries) -> Loaded { } load_git(get(payload, "git"), PROJECT_FILE)?; load_review(get(payload, "review"), PROJECT_FILE)?; + load_maintained(get(payload, "maintained"), PROJECT_FILE)?; load_control(get(payload, "control"), PROJECT_FILE)?; load_changes(get(payload, "changes"), PROJECT_FILE)?; pool_limits(payload)?; @@ -539,6 +542,11 @@ pub(crate) fn changes_config(paths: &ConfigPaths) -> Result Result { + project_section(paths, "maintained", load_maintained) +} + /// `[review]` of `sqlbuild_project.toml`. pub(crate) fn review_config(paths: &ConfigPaths) -> Result { project_section(paths, "review", load_review) diff --git a/crates/sqlbuild-config/src/project/_helpers/sections.rs b/crates/sqlbuild-config/src/project/_helpers/sections.rs index 7ceafd616b..be75473f5f 100644 --- a/crates/sqlbuild-config/src/project/_helpers/sections.rs +++ b/crates/sqlbuild-config/src/project/_helpers/sections.rs @@ -21,21 +21,23 @@ use crate::project::constants::{ use crate::project::constants::{ AUDIT_SEVERITIES, CHECK_DECLARED_COLUMNS_KEY, COLLECTION_RENDERINGS, COLUMN_CONTRACT_MODES, CONCURRENCY_KEY, DBT_DEFER_CLONE_KEY, DBT_KEYS, DBT_REMOVED_KEYS, DBT_REUSE_KEYS, - DEFAULT_IN_USE_WINDOW, DEFAULT_QUERY_ARTIFACT_TTL, FUTURE_CURSOR_ACTIONS, - LEGACY_COLUMN_CONTRACT_MODE_KEY, LEGACY_CONCURRENCY_KEY, LEGACY_SQL_VALIDATION_KEY, - LOCAL_DBT_KEYS, MAX_BATCHES_KEY, MICROBATCH_LIMIT_ACTIONS, OLD_NAME_VIEWS_KEY, - QUERY_CHANGE_TRACKING_KEY, REQUIRE_SQL_ANALYSIS_KEY, REVIEW_DOWNSTREAM_POLICIES, REVIEW_KEYS, - RUN_NAMESPACE_KEY, RUN_NAMESPACE_MAX_LENGTH, SCENARIO_KEYS, SETTINGS_KEYS, - SNAPSHOT_FULL_REFRESH_POLICIES, SNAPSHOT_KEYS, SNAPSHOT_LIMIT_KEYS, - SNAPSHOT_SCHEMA_CHANGE_POLICIES, SQL_ANALYSIS_KEY, UNACCOUNTED_PARTITION_POLICIES, - USD_PER_CREDIT_KEY, ZERO_FIXED_DURATION, ZERO_FLOAT, + DEFAULT_HEARTBEAT_GRACE_SECONDS, DEFAULT_IN_USE_WINDOW, DEFAULT_QUERY_ARTIFACT_TTL, + FUTURE_CURSOR_ACTIONS, HEARTBEAT_INTERVAL_SECONDS, LEGACY_COLUMN_CONTRACT_MODE_KEY, + LEGACY_CONCURRENCY_KEY, LEGACY_SQL_VALIDATION_KEY, LOCAL_DBT_KEYS, MAINTAINED_KEYS, + MAX_BATCHES_KEY, MICROBATCH_LIMIT_ACTIONS, OLD_NAME_VIEWS_KEY, QUERY_CHANGE_TRACKING_KEY, + REQUIRE_SQL_ANALYSIS_KEY, REVIEW_DOWNSTREAM_POLICIES, REVIEW_KEYS, RUN_NAMESPACE_KEY, + RUN_NAMESPACE_MAX_LENGTH, SCENARIO_KEYS, SETTINGS_KEYS, SNAPSHOT_FULL_REFRESH_POLICIES, + SNAPSHOT_KEYS, SNAPSHOT_LIMIT_KEYS, SNAPSHOT_SCHEMA_CHANGE_POLICIES, SQL_ANALYSIS_KEY, + UNACCOUNTED_PARTITION_POLICIES, USD_PER_CREDIT_KEY, ZERO_FIXED_DURATION, ZERO_FLOAT, }; use crate::project::constants::{ CHANGE_COUNTS, CHANGE_REACHES, CHANGES_KEYS, CONTROL_KEYS, MAX_POOL_LIMIT, MIN_SYNC_INTERVAL_SECONDS, SYNC_INTERVAL_HELP, SYNC_INTERVAL_OFF, }; use crate::project::errors::ConfigFailure; -use crate::project::models::{Built, ChangesConfig, ConfigFile, ControlConfig, ReviewConfig}; +use crate::project::models::{ + Built, ChangesConfig, ConfigFile, ControlConfig, MaintainedConfig, ReviewConfig, +}; use crate::project::types::{Entries, Fields, Loaded}; use sqlbuild_core::constants::ZERO_DAY_DURATION; @@ -626,6 +628,50 @@ pub(crate) fn sync_interval(text: &str) -> Result { Ok(seconds) } +/// `[maintained]`: how long a silent pausing build keeps its maintained models paused. +pub(crate) fn load_maintained( + payload: Option<&ConfigValue>, + file: ConfigFile, +) -> Loaded { + let mapping = coerce_mapping(payload, place(file, "maintained"))?; + validate_allowed_keys(mapping, &MAINTAINED_KEYS, place(file, "maintained"))?; + let heartbeat_grace_seconds: u64 = match get(mapping, "heartbeat_grace") { + None => DEFAULT_HEARTBEAT_GRACE_SECONDS, + Some(ConfigValue::String(text)) => match fixed_duration_seconds(text, false) { + Ok(Some(seconds)) => seconds.parse().map_err(|_| grace_failure(file))?, + Ok(None) | Err(_) => return Err(grace_failure(file)), + }, + Some(_) => return Err(grace_failure(file)), + }; + if heartbeat_grace_seconds <= HEARTBEAT_INTERVAL_SECONDS { + return Err(ConfigFailure::at( + file, + "maintained.heartbeat_grace must be longer than 1m: a build writes a heartbeat every minute, so a shorter grace expires builds that are still running", + )); + } + let allow_ignore_grace: bool = match get(mapping, "allow_ignore_grace") { + None => false, + Some(ConfigValue::Bool(flag)) => *flag, + Some(_) => { + return Err(ConfigFailure::at( + file, + "maintained.allow_ignore_grace must be true or false", + )); + } + }; + Ok(MaintainedConfig { + heartbeat_grace_seconds, + allow_ignore_grace, + }) +} + +fn grace_failure(file: ConfigFile) -> ConfigFailure { + ConfigFailure::at( + file, + "maintained.heartbeat_grace must be a fixed duration such as 30m or 2h", + ) +} + /// `[control]`: where a signed-in build asks for a sync, and how often a location syncs anyway. pub(crate) fn load_control( payload: Option<&ConfigValue>, diff --git a/crates/sqlbuild-config/src/project/_helpers/targets.rs b/crates/sqlbuild-config/src/project/_helpers/targets.rs index e520bb835e..b8bd8df06b 100644 --- a/crates/sqlbuild-config/src/project/_helpers/targets.rs +++ b/crates/sqlbuild-config/src/project/_helpers/targets.rs @@ -16,8 +16,9 @@ use crate::project::_helpers::sections::{fixed_duration_seconds, item_timeout}; use crate::project::_helpers::values::{int_below, is_int}; use crate::project::constants::{ CLONE_KEYS, CONCURRENCY_KEY, CONFIRMATION_POLICIES, EXECUTION_LIMIT_KEYS, - IGNORE_AUDIT_FAILURES_KEY, MAX_FULL_ROWS_KEY, PLAN_FROM_KEY, PLAN_FROM_SOURCES, POOLS_KEY, - RETENTION_MATERIALIZATIONS, TARGET_KEYS, UNLIMITED_DIFF_FULL_ROWS, WAREHOUSE_GROUPS, + IGNORE_AUDIT_FAILURES_KEY, MAX_FULL_ROWS_KEY, PAUSE_MAINTAINED_KEY, PLAN_FROM_KEY, + PLAN_FROM_SOURCES, POOLS_KEY, RETENTION_MATERIALIZATIONS, TARGET_KEYS, + UNLIMITED_DIFF_FULL_ROWS, WAREHOUSE_GROUPS, }; use crate::project::errors::ConfigFailure; use crate::project::models::{Built, ConfigFile}; @@ -433,6 +434,10 @@ fn local_target(payload: &ConfigValue, target: TargetPlace<'_>) -> Loaded Result { + crate::project::_helpers::documents::maintained_config(paths) +} diff --git a/crates/sqlbuild-config/src/project/main/mod.rs b/crates/sqlbuild-config/src/project/main/mod.rs index 52648c7a86..6d54995e7c 100644 --- a/crates/sqlbuild-config/src/project/main/mod.rs +++ b/crates/sqlbuild-config/src/project/main/mod.rs @@ -4,6 +4,7 @@ pub mod load_changes_config; pub mod load_control_config; pub mod load_git_base_ref; pub mod load_local_config; +pub mod load_maintained_config; pub mod load_pool_limits; pub mod load_project_config; pub mod load_project_setting_keys; diff --git a/crates/sqlbuild-config/src/project/models.rs b/crates/sqlbuild-config/src/project/models.rs index 238a914c04..5fab29b4c0 100644 --- a/crates/sqlbuild-config/src/project/models.rs +++ b/crates/sqlbuild-config/src/project/models.rs @@ -91,6 +91,15 @@ pub struct ChangesConfig { pub reach: Option, } +/// `[maintained]` of `sqlbuild_project.toml`: when a build may take over paused maintained models. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct MaintainedConfig { + /// Seconds a pausing build's heartbeat may be silent before another build resumes its models. + pub heartbeat_grace_seconds: u64, + /// Whether `sqb build --ignore-maintained-grace` may take over within the grace. + pub allow_ignore_grace: bool, +} + /// `[review]` of `sqlbuild_project.toml`: the defaults `sqb review` uses when flags are omitted. #[derive(Clone, Debug, Default, PartialEq, Eq)] pub struct ReviewConfig { diff --git a/crates/sqlbuild-config/src/project/tests/test_load_config.rs b/crates/sqlbuild-config/src/project/tests/test_load_config.rs index 8c8a3fc77a..aa14ab21bd 100644 --- a/crates/sqlbuild-config/src/project/tests/test_load_config.rs +++ b/crates/sqlbuild-config/src/project/tests/test_load_config.rs @@ -2,14 +2,16 @@ use crate::project::errors::ConfigFailure; use crate::project::main::load_changes_config::load_changes_config; use crate::project::main::load_control_config::load_control_config; use crate::project::main::load_git_base_ref::load_git_base_ref; +use crate::project::main::load_maintained_config::load_maintained_config; use crate::project::main::load_pool_limits::load_pool_limits; use crate::project::main::load_review_config::load_review_config; use crate::project::models::ConfigFile; -use crate::project::models::{ChangesConfig, ControlConfig, ReviewConfig}; +use crate::project::models::{ChangesConfig, ControlConfig, MaintainedConfig, ReviewConfig}; use crate::project::tests::helpers::{failure_at, first_failure, project_directory}; use crate::project::tests::test_types::{ - ChangesConfigTestCase, ControlConfigTestCase, GitBaseRefTestCase, PoolsErrorTestCase, - PoolsTestCase, ProjectFailureTestCase, ReviewConfigErrorTestCase, ReviewConfigTestCase, + ChangesConfigTestCase, ControlConfigTestCase, GitBaseRefTestCase, + MaintainedConfigErrorTestCase, MaintainedConfigTestCase, PoolsErrorTestCase, PoolsTestCase, + ProjectFailureTestCase, ReviewConfigErrorTestCase, ReviewConfigTestCase, }; const PROJECT: &str = "name = 'orders'\nadapter = 'duckdb'\n"; @@ -62,6 +64,38 @@ fn given_project_files_when_loading_then_rejections_match_python_loader() { "Expected 'plan_from' to be one of: live, synced_state", )), }, + ProjectFailureTestCase { + description: "a target may keep its maintained models paused between builds", + project: "name = 'orders'\nadapter = 'snowflake'\n[targets.dev]\npause_maintained = true\n", + local: "[targets.dev]\npause_maintained = false\n", + expected_failure: None, + }, + ProjectFailureTestCase { + description: "pause_maintained is a boolean", + project: "name = 'orders'\nadapter = 'snowflake'\n[targets.dev]\npause_maintained = 'yes'\n", + local: "", + expected_failure: Some(ConfigFailure::plain( + "Expected 'pause_maintained' to be a boolean when provided", + )), + }, + ProjectFailureTestCase { + description: "an invalid maintained section fails the project load", + project: "name = 'orders'\nadapter = 'snowflake'\n[maintained]\nheartbeat_grace = 'soon'\n", + local: "", + expected_failure: Some(ConfigFailure::at( + ConfigFile::Project, + "maintained.heartbeat_grace must be a fixed duration such as 30m or 2h", + )), + }, + ProjectFailureTestCase { + description: "a grace no longer than the heartbeat interval would expire every build", + project: "name = 'orders'\nadapter = 'snowflake'\n[maintained]\nheartbeat_grace = '1m'\n", + local: "", + expected_failure: Some(ConfigFailure::at( + ConfigFile::Project, + "maintained.heartbeat_grace must be longer than 1m: a build writes a heartbeat every minute, so a shorter grace expires builds that are still running", + )), + }, ProjectFailureTestCase { description: "an unknown key suggests the nearest supported key", project: "name = 'orders'\nadapter = 'duckdb'\n[scopes]\nenforce_placment = true\n", @@ -597,3 +631,90 @@ fn given_invalid_pools_when_loading_then_project_loading_rejects_them() { ); } } + +#[test] +fn given_maintained_section_when_reading_then_returns_its_values_or_defaults() { + let test_cases = [ + MaintainedConfigTestCase { + description: "a longer grace that may be ignored", + project: "name = 'orders'\nadapter = 'snowflake'\n[maintained]\n\ + heartbeat_grace = '45m'\nallow_ignore_grace = true\n", + expected_config: MaintainedConfig { + heartbeat_grace_seconds: 2_700, + allow_ignore_grace: true, + }, + }, + MaintainedConfigTestCase { + description: "any grace longer than the heartbeat interval", + project: "name = 'orders'\nadapter = 'snowflake'\n[maintained]\n\ + heartbeat_grace = '61s'\n", + expected_config: MaintainedConfig { + heartbeat_grace_seconds: 61, + allow_ignore_grace: false, + }, + }, + MaintainedConfigTestCase { + description: "no maintained section", + project: PROJECT, + expected_config: MaintainedConfig { + heartbeat_grace_seconds: 1_800, + allow_ignore_grace: false, + }, + }, + ]; + for test_case in test_cases { + let (_directory, paths) = project_directory(test_case.project, ""); + + let config: MaintainedConfig = load_maintained_config(&paths).expect(test_case.description); + + assert_eq!( + config, test_case.expected_config, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_invalid_maintained_section_when_reading_then_it_is_rejected() { + let test_cases = [ + MaintainedConfigErrorTestCase { + description: "a grace that is not a duration", + project: "name = 'orders'\nadapter = 'snowflake'\n[maintained]\n\ + heartbeat_grace = 'soon'\n", + expected_fragment: "maintained.heartbeat_grace must be a fixed duration such as 30m or 2h", + }, + MaintainedConfigErrorTestCase { + description: "a grace in months", + project: "name = 'orders'\nadapter = 'snowflake'\n[maintained]\n\ + heartbeat_grace = '1mo'\n", + expected_fragment: "maintained.heartbeat_grace must be a fixed duration such as 30m or 2h", + }, + MaintainedConfigErrorTestCase { + description: "allow_ignore_grace written as text", + project: "name = 'orders'\nadapter = 'snowflake'\n[maintained]\n\ + allow_ignore_grace = 'yes'\n", + expected_fragment: "maintained.allow_ignore_grace must be true or false", + }, + MaintainedConfigErrorTestCase { + description: "a misspelt maintained key", + project: "name = 'orders'\nadapter = 'snowflake'\n[maintained]\n\ + heartbeat_grase = '30m'\n", + expected_fragment: "maintained contains unknown key(s): heartbeat_grase", + }, + ]; + for test_case in test_cases { + let (_directory, paths) = project_directory(test_case.project, ""); + + let failure: String = format!( + "{:?}", + load_maintained_config(&paths).expect_err(test_case.description) + ); + + assert!( + failure.contains(test_case.expected_fragment), + "{}: {failure}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-config/src/project/tests/test_types.rs b/crates/sqlbuild-config/src/project/tests/test_types.rs index 413f1af9c0..d0a08468db 100644 --- a/crates/sqlbuild-config/src/project/tests/test_types.rs +++ b/crates/sqlbuild-config/src/project/tests/test_types.rs @@ -1,5 +1,5 @@ use crate::project::errors::ConfigFailure; -use crate::project::models::{ChangesConfig, ControlConfig, ReviewConfig}; +use crate::project::models::{ChangesConfig, ControlConfig, MaintainedConfig, ReviewConfig}; pub(super) struct ProjectFailureTestCase { pub(super) description: &'static str, @@ -58,3 +58,15 @@ pub(crate) struct PoolsErrorTestCase { pub(crate) local: &'static str, pub(crate) expected_fragment: &'static str, } + +pub(crate) struct MaintainedConfigTestCase { + pub(crate) description: &'static str, + pub(crate) project: &'static str, + pub(crate) expected_config: MaintainedConfig, +} + +pub(crate) struct MaintainedConfigErrorTestCase { + pub(crate) description: &'static str, + pub(crate) project: &'static str, + pub(crate) expected_fragment: &'static str, +} diff --git a/crates/sqlbuild-core/src/cli_text/_helpers/plan_sections.rs b/crates/sqlbuild-core/src/cli_text/_helpers/plan_sections.rs index 3a2ce7ce86..03b6cb37ea 100644 --- a/crates/sqlbuild-core/src/cli_text/_helpers/plan_sections.rs +++ b/crates/sqlbuild-core/src/cli_text/_helpers/plan_sections.rs @@ -6,12 +6,13 @@ use crate::cli_text::_helpers::bounded_sections::{ }; use crate::cli_text::_helpers::layout::tree_connector; use crate::cli_text::constants::{ - BOUNDED_BACKFILL, CHANGED_IDENTITY, COLUMN_ADDED, COLUMN_REMOVED, COLUMN_TYPE_CHANGED, - CONFIG_CHANGED_REASON, CREATE_VIEW_ACTION, CUSTOM_ACTION, DIFF_HEADER_MARKER, - EFFECTIVE_BATCH_SIZE_TOKEN, ERROR_SEVERITY_VALUE, FIRST_RUN_REASON, FORWARD_BACKFILL, - FULL_BACKFILL, FULL_REFRESH_REASON, INFO_SEVERITY_VALUE, NO_CHANGE_REASON, - NORMAL_INCREMENTAL_REASON, QUERY_CHANGED_REASON, REASON_GROUPS, SCHEMA_CHANGED_REASON, - SKIP_ACTION, SOURCE_FRESHNESS_UNKNOWN_WARNING_TITLE, STALE_INPUT_WARNING_TITLE, + ALTER_MAINTAINED_ACTION, BOUNDED_BACKFILL, CHANGED_IDENTITY, COLUMN_ADDED, COLUMN_REMOVED, + COLUMN_TYPE_CHANGED, CONFIG_CHANGED_REASON, CREATE_MAINTAINED_ACTION, CREATE_VIEW_ACTION, + CUSTOM_ACTION, DIFF_HEADER_MARKER, EFFECTIVE_BATCH_SIZE_TOKEN, ERROR_SEVERITY_VALUE, + FIRST_RUN_REASON, FORWARD_BACKFILL, FULL_BACKFILL, FULL_REFRESH_REASON, INFO_SEVERITY_VALUE, + NO_CHANGE_REASON, NORMAL_INCREMENTAL_REASON, QUERY_CHANGED_REASON, REASON_GROUPS, + SCHEMA_CHANGED_REASON, SKIP_ACTION, SOURCE_FRESHNESS_UNKNOWN_WARNING_TITLE, + STALE_INPUT_WARNING_TITLE, }; use crate::cli_text::models::{ CliStyle, CursorPlanText, DisplayLimit, FunctionPlanText, ModelPlanText, PlanTextReport, @@ -429,6 +430,16 @@ fn action_text(model: &ModelPlanText) -> String { if model.action == CREATE_VIEW_ACTION { return "recreate view".to_owned(); } + if model.action == CREATE_MAINTAINED_ACTION { + return if model.reason == FIRST_RUN_REASON { + "create".to_owned() + } else { + "recreate, full recompute".to_owned() + }; + } + if model.action == ALTER_MAINTAINED_ACTION { + return "change refresh settings in place".to_owned(); + } if model.action == CUSTOM_ACTION { let materialization: &str = model .custom_materialization_name @@ -454,6 +465,7 @@ fn action_text(model: &ModelPlanText) -> String { fn policy_lines(model: &ModelPlanText) -> Vec { if model.backfill_action == FORWARD_BACKFILL + || model.action == CREATE_MAINTAINED_ACTION || !(model.reason == QUERY_CHANGED_REASON || model.reason == SCHEMA_CHANGED_REASON) { return Vec::new(); @@ -618,6 +630,12 @@ pub(crate) fn routine_model_lines( true, )]; lines.extend(cursor_lines(model, true)); + if let Some(upstream) = &model.reinitialised_by { + lines.push(format!(" recomputes in full: {upstream} is rebuilt")); + } + if model.paused { + lines.push(" paused: scheduled refreshes are suspended".to_owned()); + } lines }) } diff --git a/crates/sqlbuild-core/src/cli_text/constants.rs b/crates/sqlbuild-core/src/cli_text/constants.rs index d1a4378b23..f00f5abc48 100644 --- a/crates/sqlbuild-core/src/cli_text/constants.rs +++ b/crates/sqlbuild-core/src/cli_text/constants.rs @@ -63,6 +63,8 @@ pub(crate) const REASON_GROUPS: [(&str, &str); 7] = [ pub(crate) const SKIP_ACTION: &str = "skip"; pub(crate) const CREATE_VIEW_ACTION: &str = "create_view"; pub(crate) const CUSTOM_ACTION: &str = "custom"; +pub(crate) const CREATE_MAINTAINED_ACTION: &str = "create_maintained"; +pub(crate) const ALTER_MAINTAINED_ACTION: &str = "alter_maintained"; pub(crate) const FULL_REFRESH_REASON: &str = "full_refresh"; pub(crate) const FIRST_RUN_REASON: &str = "first_run"; pub(crate) const QUERY_CHANGED_REASON: &str = "query_changed"; diff --git a/crates/sqlbuild-core/src/cli_text/models.rs b/crates/sqlbuild-core/src/cli_text/models.rs index 34817c0b7b..0274627d1e 100644 --- a/crates/sqlbuild-core/src/cli_text/models.rs +++ b/crates/sqlbuild-core/src/cli_text/models.rs @@ -322,6 +322,10 @@ pub struct ModelPlanText { pub config_diff: Option>, /// The rendered query diff, when the query changed and the previous query is known. pub query_diff: Option>, + /// The rebuilt upstream that makes this kept maintained model recompute in full. + pub reinitialised_by: Option, + /// Whether the warehouse lists this kept maintained model as suspended. + pub paused: bool, } /// One planned function. diff --git a/crates/sqlbuild-core/src/cli_text/tests/test_cli_text.rs b/crates/sqlbuild-core/src/cli_text/tests/test_cli_text.rs index 7b1c772206..51f8d9a1e0 100644 --- a/crates/sqlbuild-core/src/cli_text/tests/test_cli_text.rs +++ b/crates/sqlbuild-core/src/cli_text/tests/test_cli_text.rs @@ -226,6 +226,48 @@ fn given_plan_report_when_rendering_then_groups_models_by_reason() { expected_text: "Plan ready 1 selected\nAudit failures won't stop this build\n\nModels (1)\n\u{2514}\u{2500}\u{2500} orders table", expected_completion: "\u{2713} Plan complete 1 selected", }, + PlanTextTestCase { + description: "maintained models recompute on a query change and alter refresh settings in place", + report: plan_report(vec![ + ModelPlanText { + action: "create_maintained".to_owned(), + materialization_label: "maintained".to_owned(), + ..table_model("daily_totals", "query_changed") + }, + ModelPlanText { + action: "alter_maintained".to_owned(), + materialization_label: "maintained".to_owned(), + ..table_model("clean_orders", "config_changed") + }, + ]), + expected_text: "Plan ready 2 selected\n\nQuery changed (1)\n\u{2514}\u{2500}\u{2500} daily_totals recreate, full recompute\n\nConfig changed (1)\n\u{2514}\u{2500}\u{2500} clean_orders change refresh settings in place", + expected_completion: "\u{2713} Plan complete 2 selected", + }, + PlanTextTestCase { + description: "a kept maintained model whose upstream is rebuilt says it recomputes in full", + report: plan_report(vec![ + table_model("orders", "query_changed"), + ModelPlanText { + action: "keep_maintained".to_owned(), + materialization_label: "maintained".to_owned(), + reinitialised_by: Some("orders".to_owned()), + ..table_model("daily_totals", "no_change") + }, + ]), + expected_text: "Plan ready 2 selected\n\nQuery changed (1)\n\u{2514}\u{2500}\u{2500} orders full rebuild\n \u{2514}\u{2500}\u{2500} policy replay_on_change=full\n\nModels (1)\n\u{2514}\u{2500}\u{2500} daily_totals maintained\n \u{2514}\u{2500}\u{2500} recomputes in full orders is rebuilt", + expected_completion: "\u{2713} Plan complete 2 selected", + }, + PlanTextTestCase { + description: "a kept maintained model that is suspended is reported as paused", + report: plan_report(vec![ModelPlanText { + action: "keep_maintained".to_owned(), + materialization_label: "maintained".to_owned(), + paused: true, + ..table_model("daily_totals", "no_change") + }]), + expected_text: "Plan ready 1 selected\n\nModels (1)\n\u{2514}\u{2500}\u{2500} daily_totals maintained\n \u{2514}\u{2500}\u{2500} paused scheduled refreshes are suspended", + expected_completion: "\u{2713} Plan complete 1 selected", + }, PlanTextTestCase { description: "every active model is a full refresh", report: plan_report(vec![ diff --git a/crates/sqlbuild-core/src/constants.rs b/crates/sqlbuild-core/src/constants.rs index 95966a7de9..82fb646f95 100644 --- a/crates/sqlbuild-core/src/constants.rs +++ b/crates/sqlbuild-core/src/constants.rs @@ -44,8 +44,10 @@ pub const OBSERVATIONS_TABLE_NAME: &str = "_sqlbuild_observations"; pub const RESOURCE_COSTS_TABLE_NAME: &str = "_sqlbuild_resource_costs"; /// State table of each build's start and end, which shows runs the control plane did not start. pub const RUNS_TABLE_NAME: &str = "_sqlbuild_runs"; +/// State table of the maintained models a build paused, and its heartbeats while it holds them. +pub const MAINTAINED_PAUSES_TABLE_NAME: &str = "_sqlbuild_maintained_pauses"; /// Every state table SQLBuild writes in a target's state schema. -pub const STATE_TABLE_NAMES: [&str; 13] = [ +pub const STATE_TABLE_NAMES: [&str; 14] = [ FINGERPRINT_TABLE_NAME, MICROBATCH_TABLE_NAME, SOURCE_FRESHNESS_TABLE_NAME, @@ -59,6 +61,7 @@ pub const STATE_TABLE_NAMES: [&str; 13] = [ OBSERVATIONS_TABLE_NAME, RESOURCE_COSTS_TABLE_NAME, RUNS_TABLE_NAME, + MAINTAINED_PAUSES_TABLE_NAME, ]; /// A zero-length cursor duration. pub const ZERO_DAY_DURATION: &str = "0d"; diff --git a/crates/sqlbuild-discovery/src/constants.rs b/crates/sqlbuild-discovery/src/constants.rs index f6c8b8b807..5991fd2adb 100644 --- a/crates/sqlbuild-discovery/src/constants.rs +++ b/crates/sqlbuild-discovery/src/constants.rs @@ -54,7 +54,7 @@ pub const LOCAL_AUDIT_ROLE_DIRECTORY: &str = "_audits"; /// The singular audit directory below an audit role. pub const SINGULAR_AUDIT_DIRECTORY: &str = "singular"; /// Header keys a `MODEL` statement accepts, sorted by code point. -pub const SQL_MODEL_HEADER_KEYS: [&str; 75] = [ +pub const SQL_MODEL_HEADER_KEYS: [&str; 79] = [ "alias", "append_cursor_inclusive", "audit_factories", @@ -109,6 +109,10 @@ pub const SQL_MODEL_HEADER_KEYS: [&str; 75] = [ "post_hooks", "pre_hooks", "primary_key", + "refresh", + "refresh_mode", + "refresh_on_build", + "refresh_warehouse", "replace_key", "replay_on_change", "row_diff_exclude_columns", diff --git a/crates/sqlbuild-model-config/src/model_validation/_helpers/materialization.rs b/crates/sqlbuild-model-config/src/model_validation/_helpers/materialization.rs index ddc75eeec5..e3633aac1a 100644 --- a/crates/sqlbuild-model-config/src/model_validation/_helpers/materialization.rs +++ b/crates/sqlbuild-model-config/src/model_validation/_helpers/materialization.rs @@ -11,6 +11,7 @@ use crate::model_validation::_helpers::config::python_lower; use crate::model_validation::_helpers::config::{ ConfigView, model_header_help, non_blank_string_check, one_of, sorted_values, text_if_string, }; +use crate::model_validation::_helpers::durations::parse_duration; use crate::model_validation::constants::{ BUILTIN_MATERIALIZATIONS, CONTRACT_POLICIES, CUSTOM_MATERIALIZATION_DISALLOWED_KEYS, HISTORY_MATERIALIZATIONS, INCREMENTAL_MATERIALIZATION, INCREMENTAL_ONLY_KEYS, @@ -26,6 +27,12 @@ use crate::model_validation::constants::{ PHYSICAL_STORAGE_MATERIALIZATIONS, PRIMARY_KEY_KEY, TABLE_SETTINGS_KEY, UNSUPPORTED_ENGINE_PREFIXES, }; +use crate::model_validation::constants::{ + MAINTAINED_ADAPTER_CODE, MAINTAINED_ADAPTERS, MAINTAINED_MATERIALIZATION, MAINTAINED_ONLY_KEYS, + MAINTAINED_REFRESH_FORMS, MAINTAINED_REFRESH_MODES, MAINTAINED_REFRESH_UNSUPPORTED_CODE, + MATERIALIZED_VIEW_ADAPTER_CODE, MATERIALIZED_VIEW_MATERIALIZATION, SNAPSHOT_DISALLOWED_KEYS, + SNOWFLAKE_ADAPTER, SNOWFLAKE_MIN_LAG_SECONDS, +}; use crate::model_validation::models::{ ModelValidationFacts, ProjectValidationFacts, ValidationStop, }; @@ -36,6 +43,11 @@ const MIGRATE_FROM_KEY: &str = "migrate_from"; /// The model key that marks a model whose downstream paths `sqb review` validates. const CRITICAL_KEY: &str = "critical"; const MIGRATE_FORCE_KEY: &str = "migrate_force"; +const REFRESH_KEY: &str = "refresh"; +const LAG_FORM: &str = "lag"; +const REFRESH_MODE_KEY: &str = "refresh_mode"; +const REFRESH_WAREHOUSE_KEY: &str = "refresh_warehouse"; +const REFRESH_ON_BUILD_KEY: &str = "refresh_on_build"; /// Check `validate_microbatch_project_capability`. pub(crate) fn check_project_capability( @@ -609,3 +621,136 @@ fn setting_name(name: &str) -> bool { .chars() .all(|character| character.is_ascii_alphanumeric() || character == '_') } + +/// Check the refresh fields: only maintained models on an adapter that builds them take them. +pub(crate) fn check_maintained( + config: &ConfigView<'_, N>, + project: &ProjectValidationFacts, +) -> Check { + let materialized: Option = config.string("materialized")?; + let adapter: Option<&str> = project.adapter.as_deref(); + if materialized.as_deref() == Some(MATERIALIZED_VIEW_MATERIALIZATION) + && adapter == Some(SNOWFLAKE_ADAPTER) + && !project + .custom_materializations + .contains(MATERIALIZED_VIEW_MATERIALIZATION) + { + return Err(coded( + config, + MATERIALIZED_VIEW_ADAPTER_CODE, + "Snowflake has no materialized_view materialization; use materialized maintained, \ + which builds a dynamic table", + )); + } + if materialized.as_deref() != Some(MAINTAINED_MATERIALIZATION) { + return match config.first_present(&MAINTAINED_ONLY_KEYS) { + Some(key) => Err(config.error(format!("{key} is only valid for maintained models"))), + None => Ok(()), + }; + } + if let Some(adapter) = adapter.filter(|adapter| !one_of(adapter, &MAINTAINED_ADAPTERS)) { + return Err(coded( + config, + MAINTAINED_ADAPTER_CODE, + format!( + "materialized maintained is not supported on the {adapter} adapter; adapters \ + that support it: {}", + MAINTAINED_ADAPTERS.join(", ") + ), + )); + } + if let Some(key) = config.first_present(&SNAPSHOT_DISALLOWED_KEYS) { + return Err(config.error(format!( + "{key} is not valid for maintained models: the warehouse refreshes them from the \ + whole query" + ))); + } + check_refresh(config, adapter)?; + if let Some(mode) = config.get(REFRESH_MODE_KEY) { + let text: Option = text_if_string(mode); + if !text + .as_deref() + .is_some_and(|text| one_of(text, &MAINTAINED_REFRESH_MODES)) + { + return Err(config.error(format!( + "{REFRESH_MODE_KEY} must be one of: {}; got '{}'", + sorted_values(&MAINTAINED_REFRESH_MODES), + text.unwrap_or_default() + ))); + } + } + if config + .get(REFRESH_WAREHOUSE_KEY) + .is_some_and(|warehouse| !non_blank_string_check(warehouse)) + { + return Err(config.error(format!( + "{REFRESH_WAREHOUSE_KEY} must be a non-empty warehouse name" + ))); + } + match config.get(REFRESH_ON_BUILD_KEY).map(AuthoredNode::kind) { + None | Some(NodeKind::Bool(_)) => Ok(()), + Some(_) => Err(config.error(format!("{REFRESH_ON_BUILD_KEY} must be a boolean"))), + } +} + +/// Check that `refresh` is a block with exactly one known form and a valid value. +fn check_refresh(config: &ConfigView<'_, N>, adapter: Option<&str>) -> Check { + let Some(refresh) = config.get(REFRESH_KEY) else { + return Ok(()); + }; + if refresh.kind() != NodeKind::Map { + return Err(config.error(format!( + "{REFRESH_KEY} must be a block such as refresh (lag 1h)" + ))); + } + let block: Vec<(N, N)> = refresh.entries(); + let forms: Vec = block + .iter() + .map(|(name, _)| name.text().unwrap_or_default()) + .collect(); + let [(form, value)] = block.as_slice() else { + return Err(unknown_form(config, &forms)); + }; + if !MAINTAINED_REFRESH_FORMS + .iter() + .any(|known| form.is_text(known)) + { + return Err(unknown_form(config, &forms)); + } + let text: String = text_if_string(value).unwrap_or_default(); + let Ok(Some(lag)) = parse_duration(&text) else { + return Err(not_fixed(config, &text)); + }; + if lag.total_months != 0 { + return Err(not_fixed(config, &text)); + } + if adapter == Some(SNOWFLAKE_ADAPTER) && lag.fixed_seconds < SNOWFLAKE_MIN_LAG_SECONDS { + return Err(coded( + config, + MAINTAINED_REFRESH_UNSUPPORTED_CODE, + format!("{REFRESH_KEY} {LAG_FORM} on Snowflake must be at least 1m; got '{text}'"), + )); + } + Ok(()) +} + +fn not_fixed(config: &ConfigView<'_, N>, text: &str) -> ValidationStop { + config.error(format!( + "{REFRESH_KEY} {LAG_FORM} must be a fixed duration such as 5m, 1h or 1d; got '{text}'" + )) +} + +fn coded( + config: &ConfigView<'_, N>, + code: &'static str, + text: impl std::fmt::Display, +) -> ValidationStop { + ValidationStop::Error(config.config_error(text).with_code(code)) +} + +fn unknown_form(config: &ConfigView<'_, N>, forms: &[String]) -> ValidationStop { + config.error(format!( + "{REFRESH_KEY} takes exactly one form, such as refresh (lag 1h); got: {}", + forms.join(", ") + )) +} diff --git a/crates/sqlbuild-model-config/src/model_validation/constants.rs b/crates/sqlbuild-model-config/src/model_validation/constants.rs index 87b9298bc3..bb0dad808a 100644 --- a/crates/sqlbuild-model-config/src/model_validation/constants.rs +++ b/crates/sqlbuild-model-config/src/model_validation/constants.rs @@ -1,7 +1,8 @@ //! Config keys and vocabularies the Python model validators accept. /// Built-in materializations that never name a custom materialization. -pub const BUILTIN_MATERIALIZATIONS: [&str; 4] = ["view", "table", "incremental", "snapshot"]; +pub const BUILTIN_MATERIALIZATIONS: [&str; 5] = + ["view", "table", "incremental", "snapshot", "maintained"]; /// Materializations that own a physical managed table. pub const TABLE_BACKED_MATERIALIZATIONS: [&str; 3] = ["table", "incremental", "snapshot"]; /// Materializations that keep history across runs. @@ -14,6 +15,33 @@ pub const TABLE_MATERIALIZATION: &str = "table"; pub const INCREMENTAL_MATERIALIZATION: &str = "incremental"; /// The snapshot materialization. pub const SNAPSHOT_MATERIALIZATION: &str = "snapshot"; +/// The materialization whose stored result the warehouse keeps fresh between builds. +pub const MAINTAINED_MATERIALIZATION: &str = "maintained"; +/// Keys only a maintained model accepts. +pub const MAINTAINED_ONLY_KEYS: [&str; 4] = [ + "refresh", + "refresh_mode", + "refresh_warehouse", + "refresh_on_build", +]; +/// The `refresh ( … )` forms any adapter accepts; each adapter accepts only its own. +pub const MAINTAINED_REFRESH_FORMS: [&str; 1] = ["lag"]; +/// `refresh_mode` values. +pub const MAINTAINED_REFRESH_MODES: [&str; 4] = ["incremental", "full", "auto", "adaptive"]; +/// Adapters that build maintained models, sorted. +pub const MAINTAINED_ADAPTERS: [&str; 1] = ["snowflake"]; +/// The Snowflake adapter name. +pub const SNOWFLAKE_ADAPTER: &str = "snowflake"; +/// The materialization name other warehouses use, which Snowflake builds as `maintained`. +pub const MATERIALIZED_VIEW_MATERIALIZATION: &str = "materialized_view"; +/// Snowflake's smallest dynamic table target lag, in seconds. +pub const SNOWFLAKE_MIN_LAG_SECONDS: u128 = 60; +/// The code of a maintained model on an adapter that cannot build one. +pub const MAINTAINED_ADAPTER_CODE: &str = "A013"; +/// The code of `materialized_view` on Snowflake, which builds `maintained` instead. +pub const MATERIALIZED_VIEW_ADAPTER_CODE: &str = "A014"; +/// The code of a refresh form or value the adapter does not accept. +pub const MAINTAINED_REFRESH_UNSUPPORTED_CODE: &str = "A015"; /// `IncrementalStrategy` values. pub const INCREMENTAL_STRATEGIES: [&str; 3] = ["append", "delete_insert", "merge"]; diff --git a/crates/sqlbuild-model-config/src/model_validation/main/validate_model_config.rs b/crates/sqlbuild-model-config/src/model_validation/main/validate_model_config.rs index ea58dfe598..23eb12f947 100644 --- a/crates/sqlbuild-model-config/src/model_validation/main/validate_model_config.rs +++ b/crates/sqlbuild-model-config/src/model_validation/main/validate_model_config.rs @@ -3,9 +3,9 @@ use crate::model_validation::_helpers::config::ConfigView; use crate::model_validation::_helpers::incremental::check_incremental; use crate::model_validation::_helpers::materialization::{ - check_contract, check_custom_materialization, check_execution_limit, check_migration, - check_non_incremental, check_physical_storage, check_placeholders, check_project_capability, - check_review_marker, check_storage_policies, + check_contract, check_custom_materialization, check_execution_limit, check_maintained, + check_migration, check_non_incremental, check_physical_storage, check_placeholders, + check_project_capability, check_review_marker, check_storage_policies, }; use crate::model_validation::_helpers::references::check_references; use crate::model_validation::_helpers::snapshot::check_snapshot; @@ -29,6 +29,7 @@ pub fn validate_model_config( check_review_marker(&config)?; check_non_incremental(&config)?; check_snapshot(&config, facts)?; + check_maintained(&config, project)?; check_custom_materialization(&config, project)?; check_storage_policies(&config, facts)?; check_physical_storage(&config)?; diff --git a/crates/sqlbuild-model-config/src/model_validation/models.rs b/crates/sqlbuild-model-config/src/model_validation/models.rs index c456da27dd..5caa385c4f 100644 --- a/crates/sqlbuild-model-config/src/model_validation/models.rs +++ b/crates/sqlbuild-model-config/src/model_validation/models.rs @@ -21,6 +21,8 @@ pub struct ProjectValidationFacts { pub functions: HashSet, /// Discovered table function names. pub table_functions: HashSet, + /// The effective adapter name, when the project names one. + pub adapter: Option, } /// One extracted SQL reference: its `SqlReferenceKind` value and name. diff --git a/crates/sqlbuild-model-config/src/model_validation/tests/helpers.rs b/crates/sqlbuild-model-config/src/model_validation/tests/helpers.rs index 45308d1651..aabfa836a3 100644 --- a/crates/sqlbuild-model-config/src/model_validation/tests/helpers.rs +++ b/crates/sqlbuild-model-config/src/model_validation/tests/helpers.rs @@ -80,6 +80,7 @@ pub(super) fn project() -> ProjectValidationFacts { sources: names(&["raw.orders"]), functions: names(&["cents", "order_lines"]), table_functions: names(&["order_lines"]), + adapter: Some("snowflake".to_owned()), } } diff --git a/crates/sqlbuild-model-config/src/model_validation/tests/test_validate_model_config.rs b/crates/sqlbuild-model-config/src/model_validation/tests/test_validate_model_config.rs index 24ecccd910..6375dca5f1 100644 --- a/crates/sqlbuild-model-config/src/model_validation/tests/test_validate_model_config.rs +++ b/crates/sqlbuild-model-config/src/model_validation/tests/test_validate_model_config.rs @@ -553,3 +553,151 @@ fn given_effective_configs_when_validating_natively_then_python_outcomes_result( ); } } + +#[test] +fn given_maintained_model_headers_when_validating_then_refresh_fields_are_checked() { + let maintained = |extra: Vec<(&'static str, Value)>| { + let mut config = vec![("materialized", Value::Str("maintained"))]; + config.extend(extra); + config + }; + let test_cases = [ + ValidateTestCase { + description: "a maintained model with a lag, a mode and a warehouse", + config: maintained(vec![ + ("refresh", map(vec![("lag", Value::Str("1h"))])), + ("refresh_mode", Value::Str("adaptive")), + ("refresh_warehouse", Value::Str("transform_wh")), + ("refresh_on_build", Value::Bool(true)), + ]), + references: &["ref:orders"], + query_sql: "select 1", + expected_outcome: "accepted", + }, + ValidateTestCase { + description: "a maintained model refreshed by its readers", + config: maintained(vec![]), + references: &["ref:orders"], + query_sql: "select 1", + expected_outcome: "accepted", + }, + ValidateTestCase { + description: "refresh on a table", + config: vec![ + ("materialized", Value::Str("table")), + ("refresh", map(vec![("lag", Value::Str("1h"))])), + ], + references: &[], + query_sql: "select 1", + expected_outcome: "model 'orders_daily': refresh is only valid for maintained models", + }, + ValidateTestCase { + description: "refresh_on_build on a model with no materialization", + config: vec![("refresh_on_build", Value::Bool(true))], + references: &[], + query_sql: "select 1", + expected_outcome: "model 'orders_daily': refresh_on_build is only valid for maintained models", + }, + ValidateTestCase { + description: "a refresh block with a form no adapter has", + config: maintained(vec![("refresh", map(vec![("every", Value::Str("1h"))]))]), + references: &[], + query_sql: "select 1", + expected_outcome: "model 'orders_daily': refresh takes exactly one form, such as refresh (lag 1h); got: every", + }, + ValidateTestCase { + description: "a refresh block with two forms", + config: maintained(vec![( + "refresh", + map(vec![ + ("lag", Value::Str("1h")), + ("lag_cap", Value::Str("2h")), + ]), + )]), + references: &[], + query_sql: "select 1", + expected_outcome: "model 'orders_daily': refresh takes exactly one form, such as refresh (lag 1h); got: lag, lag_cap", + }, + ValidateTestCase { + description: "refresh written as a plain value", + config: maintained(vec![("refresh", Value::Str("1h"))]), + references: &[], + query_sql: "select 1", + expected_outcome: "model 'orders_daily': refresh must be a block such as refresh (lag 1h)", + }, + ValidateTestCase { + description: "a lag in months", + config: maintained(vec![("refresh", map(vec![("lag", Value::Str("1mo"))]))]), + references: &[], + query_sql: "select 1", + expected_outcome: "model 'orders_daily': refresh lag must be a fixed duration such as 5m, 1h or 1d; got '1mo'", + }, + ValidateTestCase { + description: "a lag that is not a duration", + config: maintained(vec![("refresh", map(vec![("lag", Value::Str("hourly"))]))]), + references: &[], + query_sql: "select 1", + expected_outcome: "model 'orders_daily': refresh lag must be a fixed duration such as 5m, 1h or 1d; got 'hourly'", + }, + ValidateTestCase { + description: "an unknown refresh mode", + config: maintained(vec![("refresh_mode", Value::Str("fast"))]), + references: &[], + query_sql: "select 1", + expected_outcome: "model 'orders_daily': refresh_mode must be one of: adaptive, auto, full, incremental; got 'fast'", + }, + ValidateTestCase { + description: "refresh_on_build that is not a boolean", + config: maintained(vec![("refresh_on_build", Value::Str("yes"))]), + references: &[], + query_sql: "select 1", + expected_outcome: "model 'orders_daily': refresh_on_build must be a boolean", + }, + ValidateTestCase { + description: "a blank refresh warehouse", + config: maintained(vec![("refresh_warehouse", Value::Str(" "))]), + references: &[], + query_sql: "select 1", + expected_outcome: "model 'orders_daily': refresh_warehouse must be a non-empty warehouse name", + }, + ValidateTestCase { + description: "an incremental-only key on a maintained model", + config: maintained(vec![("cursor", Value::Str("updated_at"))]), + references: &[], + query_sql: "select 1", + expected_outcome: "model 'orders_daily': cursor is not valid for maintained models: the warehouse refreshes them from the whole query", + }, + ValidateTestCase { + description: "the smallest lag Snowflake accepts", + config: maintained(vec![("refresh", map(vec![("lag", Value::Str("1m"))]))]), + references: &[], + query_sql: "select 1", + expected_outcome: "accepted", + }, + ValidateTestCase { + description: "a lag one second under Snowflake's minimum", + config: maintained(vec![("refresh", map(vec![("lag", Value::Str("59s"))]))]), + references: &[], + query_sql: "select 1", + expected_outcome: "model 'orders_daily': refresh lag on Snowflake must be at least 1m; got '59s'", + }, + ValidateTestCase { + description: "materialized_view on Snowflake", + config: vec![("materialized", Value::Str("materialized_view"))], + references: &[], + query_sql: "select 1", + expected_outcome: "model 'orders_daily': Snowflake has no materialized_view materialization; use materialized maintained, which builds a dynamic table", + }, + ]; + + for test_case in test_cases { + let outcome = + validation_outcome(test_case.config, test_case.references, test_case.query_sql); + + assert_eq!( + outcome, test_case.expected_outcome, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-orchestration/tests/api/test_warehouse_metadata.rs b/crates/sqlbuild-orchestration/tests/api/test_warehouse_metadata.rs index 6d0b5e4ff7..8df1af7c22 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_warehouse_metadata.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_warehouse_metadata.rs @@ -3,7 +3,7 @@ use chrono::{DateTime, Duration, Utc}; use reqwest::StatusCode; use serde_json::json; -use sqlbuild_core::constants::{RUNS_TABLE_NAME, STATE_TABLE_NAMES}; +use sqlbuild_core::constants::{MAINTAINED_PAUSES_TABLE_NAME, RUNS_TABLE_NAME, STATE_TABLE_NAMES}; use uuid::Uuid; use crate::api::tests::changes_helpers::{connect, expect_topics}; @@ -36,10 +36,11 @@ use crate::warehouse_metadata::models::{MetadataBatch, WarehouseHistoryKind}; #[test] fn given_state_tables_when_classified_then_each_is_mirrored_by_exactly_one_kind() { - // The runs table has no mirror until the control store decides where observed runs live. + // The runs table has no mirror until the control store decides where observed runs live; + // maintained pauses are a lease that builds read from the warehouse, never a mirror. let mut written: Vec<&str> = STATE_TABLE_NAMES .into_iter() - .filter(|table| *table != RUNS_TABLE_NAME) + .filter(|table| ![RUNS_TABLE_NAME, MAINTAINED_PAUSES_TABLE_NAME].contains(table)) .collect(); written.sort_unstable(); let test_cases = [StateTableClassificationTestCase { diff --git a/crates/sqlbuild-python-base/src/records/_helpers/compiler_enums/constants.rs b/crates/sqlbuild-python-base/src/records/_helpers/compiler_enums/constants.rs index 63f2aa3e7b..5b65da0fd2 100644 --- a/crates/sqlbuild-python-base/src/records/_helpers/compiler_enums/constants.rs +++ b/crates/sqlbuild-python-base/src/records/_helpers/compiler_enums/constants.rs @@ -586,6 +586,9 @@ const E_PLAN_ACTION: EnumSpec = EnumSpec { ("LOAD_SEED", "load_seed"), ("SKIP", "skip"), ("CUSTOM", "custom"), + ("CREATE_MAINTAINED", "create_maintained"), + ("ALTER_MAINTAINED", "alter_maintained"), + ("KEEP_MAINTAINED", "keep_maintained"), ], }; const E_PLAN_REASON: EnumSpec = EnumSpec { @@ -683,6 +686,7 @@ const E_MATERIALIZATION_TYPE: EnumSpec = EnumSpec { ("SNAPSHOT", "snapshot"), ("SEED", "seed"), ("CUSTOM", "custom"), + ("MAINTAINED", "maintained"), ], }; const E_SCENARIO_ARTIFACT_KIND: EnumSpec = EnumSpec { diff --git a/crates/sqlbuild-python-base/src/records/_helpers/contract_records/constants.rs b/crates/sqlbuild-python-base/src/records/_helpers/contract_records/constants.rs index 75fe5447c4..46587a7e93 100644 --- a/crates/sqlbuild-python-base/src/records/_helpers/contract_records/constants.rs +++ b/crates/sqlbuild-python-base/src/records/_helpers/contract_records/constants.rs @@ -151,6 +151,7 @@ const R_TARGET_CONFIG: RecordSpec = RecordSpec { FieldSpec::new("catalog_connection", FieldDefault::None), FieldSpec::new("ignore_audit_failures", FieldDefault::None), FieldSpec::new("plan_from", FieldDefault::None), + FieldSpec::new("pause_maintained", FieldDefault::None), ], }; const R_TARGET_SETTINGS: RecordSpec = RecordSpec { @@ -210,6 +211,7 @@ const R_LOCAL_TARGET_CONFIG: RecordSpec = RecordSpec { FieldSpec::new("catalog_connection", FieldDefault::None), FieldSpec::new("ignore_audit_failures", FieldDefault::None), FieldSpec::new("plan_from", FieldDefault::None), + FieldSpec::new("pause_maintained", FieldDefault::None), ], }; const R_SETTINGS_CONFIG: RecordSpec = RecordSpec { diff --git a/crates/sqlbuild-python-base/src/records/_helpers/planner_records/constants.rs b/crates/sqlbuild-python-base/src/records/_helpers/planner_records/constants.rs index bd568fa583..a5bb652b75 100644 --- a/crates/sqlbuild-python-base/src/records/_helpers/planner_records/constants.rs +++ b/crates/sqlbuild-python-base/src/records/_helpers/planner_records/constants.rs @@ -666,6 +666,13 @@ const R_MODEL_PLAN_ENTRY: RecordSpec = RecordSpec { FieldSpec::new("max_duration", FieldDefault::None), FieldSpec::new("max_duration_seconds", FieldDefault::None), FieldSpec::new("max_duration_source", FieldDefault::None), + FieldSpec::new("reinitialised_by", FieldDefault::None), + FieldSpec::new("target_lag", FieldDefault::None), + FieldSpec::new("refresh_mode", FieldDefault::None), + FieldSpec::new("refresh_warehouse", FieldDefault::None), + FieldSpec::new("refresh_on_build", FieldDefault::False), + FieldSpec::new("maintained_readers", FieldDefault::EmptyTuple), + FieldSpec::new("maintained_paused", FieldDefault::False), ], }; const R_CLONE_SOURCE_PLAN_ENTRY: RecordSpec = RecordSpec { diff --git a/crates/sqlbuild-python-base/src/records/constants.rs b/crates/sqlbuild-python-base/src/records/constants.rs index ca6de97cfa..6825d659f6 100644 --- a/crates/sqlbuild-python-base/src/records/constants.rs +++ b/crates/sqlbuild-python-base/src/records/constants.rs @@ -255,6 +255,7 @@ const BUILD_COMMAND_REQUEST: RecordSpec = RecordSpec { FieldSpec::new("allow_retention_decrease", FieldDefault::False), FieldSpec::new("allow_missing_migration_origin", FieldDefault::False), FieldSpec::new("allow_snapshot_schema_change", FieldDefault::False), + FieldSpec::new("ignore_maintained_grace", FieldDefault::False), FieldSpec::new("concurrency", FieldDefault::None), FieldSpec::new("select", FieldDefault::EmptyTuple), FieldSpec::new("exclude", FieldDefault::EmptyTuple), diff --git a/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/maintained.rs b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/maintained.rs new file mode 100644 index 0000000000..cb045e0beb --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/maintained.rs @@ -0,0 +1,68 @@ +//! Project-wide checks of maintained models on the effective adapter. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; + +use sqlbuild_analysis::assembly::maintained_checks::constants::MAINTAINED_NEVER_REFRESHED_CODE; +use sqlbuild_analysis::assembly::maintained_checks::main::never_refreshed_maintained::never_refreshed_maintained; +use sqlbuild_analysis::assembly::maintained_checks::models::MaintainedModel; +use sqlbuild_python_base::boundary::main::optional_text::optional_text; +use sqlbuild_python_base::records::main::construct::enum_member::enum_member; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_front::model_config::main::config::config_text::config_text; +use sqlbuild_python_front::model_config::main::config::config_value::config_value; +use sqlbuild_python_front::model_config::main::config::model_values::model_values; +use sqlbuild_python_front::model_config::types::ConfigValues; + +use crate::compile_driver::_helpers::setup::targets::planner_error; + +const SNOWFLAKE_ADAPTER: &str = "snowflake"; +const MAINTAINED_MATERIALIZATION: &str = "maintained"; +const REFRESH_KEY: &str = "refresh"; + +/// Raise when a Snowflake maintained model has no refresh and no maintained model reads it. +pub(crate) fn maintained_refresh( + adapter_name: &Bound<'_, PyAny>, + project: &Bound<'_, PyAny>, +) -> PyResult<()> { + if optional_text(adapter_name)?.as_deref() != Some(SNOWFLAKE_ADAPTER) { + return Ok(()); + } + let py: Python<'_> = project.py(); + let model_type: Bound<'_, PyAny> = enum_member(py, RecordId::CompiledResourceType, "MODEL")?; + let mut maintained: Vec = Vec::new(); + for model in native_field(project, "models")?.try_iter()? { + let model: Bound<'_, PyAny> = model?; + let values: ConfigValues<'_> = model_values(&model)?; + if config_text(py, &values, "materialized")?.as_deref() != Some(MAINTAINED_MATERIALIZATION) + { + continue; + } + let mut reads: Vec = Vec::new(); + for key in native_field(&model, "deps")?.try_iter()? { + let key: Bound<'_, PyAny> = key?; + if native_field(&key, "resource_type")?.eq(&model_type)? { + reads.push(native_field(&key, "name")?.extract()?); + } + } + maintained.push(MaintainedModel { + name: native_field(&model, "name")?.extract()?, + has_refresh: !config_value(py, &values, REFRESH_KEY)?.is_none(), + reads, + }); + } + let Some(name) = never_refreshed_maintained(&maintained) else { + return Ok(()); + }; + Err(planner_error( + py, + format!( + "maintained model '{name}' has no refresh and no maintained model reads it, so \ + Snowflake would never refresh it; add refresh (lag 1h)" + ), + MAINTAINED_NEVER_REFRESHED_CODE, + Some(format!( + "set how stale '{name}' may get in its MODEL header:\n refresh (lag 1h)," + )), + )) +} diff --git a/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/mod.rs b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/mod.rs index 4f83d12b05..961c650c8f 100644 --- a/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/mod.rs +++ b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/mod.rs @@ -1,4 +1,5 @@ pub(crate) mod inputs; +pub(crate) mod maintained; pub(crate) mod model_free_locations; pub(crate) mod node_results; pub(crate) mod pipeline; diff --git a/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/project.rs b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/project.rs index 5af92903d1..16444bd260 100644 --- a/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/project.rs +++ b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/project.rs @@ -14,6 +14,7 @@ use pyo3::types::{PyBool, PyCFunction, PyDict, PyTuple}; use sqlbuild_host::compile_context::main::record_compile_timing::record_compile_timing; use sqlbuild_python_project::graph::main::python_nodes::resolve_node_sets::resolve_node_sets; +use crate::compile_driver::_helpers::setup::maintained::maintained_refresh; use crate::compile_driver::_helpers::setup::target_defaults::TargetDefaults; use crate::compile_driver::_helpers::setup::target_defaults::apply_target_defaults; use crate::compile_driver::_helpers::setup::targets::managed_loader_target_isolation; @@ -242,6 +243,7 @@ fn targeted_project<'py>( }; let project: Bound<'py, PyAny> = apply_target_defaults(project, &defaults)?; project_targets(&adapter_name, &project)?; + maintained_refresh(&adapter_name, &project)?; let promotion_key: Bound<'py, PyAny> = pyo3::types::PyString::new(py, TABLE_PROMOTION_MODE_SETTING_KEY).into_any(); let settings_file: Bound<'py, PyAny> = diff --git a/crates/sqlbuild-python-front/src/discovery/main/project/configured_maintained.rs b/crates/sqlbuild-python-front/src/discovery/main/project/configured_maintained.rs new file mode 100644 index 0000000000..daf7d1ec7f --- /dev/null +++ b/crates/sqlbuild-python-front/src/discovery/main/project/configured_maintained.rs @@ -0,0 +1,50 @@ +//! A build's maintained-pause runtime settings: the target's `pause_maintained` and `[maintained]`. + +use crate::contracts::main::resolve_target_config::resolve_target_config; +use crate::contracts::main::resolve_target_name::resolve_target_name; +use crate::discovery::_helpers::project::project_config::configured; +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_config::project::main::load_maintained_config::load_maintained_config; +use sqlbuild_config::project::models::MaintainedConfig; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// `BuildRuntimeParams` fields for the request's target and flag, and the project's `[maintained]`. +pub fn configured_maintained<'py>( + discovered: &Bound<'py, PyAny>, + request: &Bound<'py, PyAny>, + project_dir: &Bound<'py, PyAny>, +) -> PyResult)>> { + let py = discovered.py(); + let selected_target: Bound<'py, PyAny> = native_field(request, "selected_target")?; + let project_config: Bound<'py, PyAny> = native_field(discovered, "project_config")?; + let local_config: Bound<'py, PyAny> = native_field(discovered, "local_config")?; + let paused: Option = + match resolve_target_name(py, &project_config, &local_config, &selected_target)? { + Some(name) => native_field( + &resolve_target_config(py, &project_config, &local_config, name.as_any())?, + "pause_maintained", + )? + .extract()?, + None => None, + }; + let config: MaintainedConfig = configured(project_dir, load_maintained_config)?; + Ok(vec![ + ( + "pause_maintained", + paused.unwrap_or(false).into_bound_py_any(py)?, + ), + ( + "maintained_heartbeat_grace_seconds", + config.heartbeat_grace_seconds.into_bound_py_any(py)?, + ), + ( + "allow_ignore_maintained_grace", + config.allow_ignore_grace.into_bound_py_any(py)?, + ), + ( + "ignore_maintained_grace", + native_field(request, "ignore_maintained_grace")?, + ), + ]) +} diff --git a/crates/sqlbuild-python-front/src/discovery/main/project/mod.rs b/crates/sqlbuild-python-front/src/discovery/main/project/mod.rs index 00673e75cd..1f0aeb28e9 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/project/mod.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/project/mod.rs @@ -1,6 +1,7 @@ pub mod column_name; pub mod configured_changes; pub mod configured_git_base_ref; +pub mod configured_maintained; pub mod configured_review; pub mod discover_configured_providers; pub mod discover_connection_configuration; diff --git a/crates/sqlbuild-python-front/src/model_config/models.rs b/crates/sqlbuild-python-front/src/model_config/models.rs index c1b99e21ea..9999245cf3 100644 --- a/crates/sqlbuild-python-front/src/model_config/models.rs +++ b/crates/sqlbuild-python-front/src/model_config/models.rs @@ -3,6 +3,7 @@ use crate::attachments::main::argument_pairs::argument_pairs; use crate::attachments::main::config_argument_value::config_argument_value; use crate::authored_values::main::project_var_text::project_var_text; +use crate::contracts::main::resolve_effective_adapter_name::resolve_effective_adapter_name; use crate::discovery::main::project::each::each; use crate::discovery::models::NativeModelFile; use crate::model_config::_helpers::config::config_classes::built_config; @@ -1356,7 +1357,17 @@ impl NativeModelValidator { })? .into_iter() .collect(); - Self::new(names, custom_materializations, microbatch_concurrency) + let adapter: Option = resolve_effective_adapter_name( + &native_field(&inputs, "project_config")?, + &native_field(&inputs, "local_config")?, + )? + .extract()?; + Self::new( + names, + custom_materializations, + microbatch_concurrency, + adapter, + ) } /// The discovered model names references resolve against. @@ -1365,10 +1376,12 @@ impl NativeModelValidator { } #[new] + #[pyo3(signature = (names, custom_materializations, microbatch_concurrency, adapter=None))] pub(crate) fn new( names: ResourceNames, custom_materializations: HashSet, microbatch_concurrency: bool, + adapter: Option, ) -> PyResult { let ResourceNames { models, @@ -1386,6 +1399,7 @@ impl NativeModelValidator { sources, functions, table_functions, + adapter, }, }) } diff --git a/crates/sqlbuild-python-planner/src/live_reads/models.rs b/crates/sqlbuild-python-planner/src/live_reads/models.rs index 6fb7db6992..3841b31896 100644 --- a/crates/sqlbuild-python-planner/src/live_reads/models.rs +++ b/crates/sqlbuild-python-planner/src/live_reads/models.rs @@ -438,6 +438,10 @@ impl WarehouseReads for LiveReads { "column_rename_unavailable_reason", vec![("destination", destination)], ), + AdapterRead::MaintainedRefresh([database, schema, name]) => ( + "maintained_listing", + vec![("database", database), ("schema", schema), ("name", name)], + ), }; self.known_call(py, method, arguments) } diff --git a/crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_json.rs b/crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_json.rs index 1f1979f407..d92a09036f 100644 --- a/crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_json.rs +++ b/crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_json.rs @@ -230,6 +230,13 @@ pub(crate) fn model_json<'py>(entry: &Bound<'py, PyAny>) -> PyResult = native_field(entry, "reinitialised_by")?; + if !reinitialised_by.is_none() { + model.set_item("reinitialised_by", reinitialised_by)?; + } + if native_field(entry, "maintained_paused")?.is_truthy()? { + model.set_item("paused", true)?; + } let qualified: Bound<'py, PyAny> = native_field(&native_field(entry, "destination")?, "qualified_name")?; if !qualified.is_none() { diff --git a/crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_text.rs b/crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_text.rs index 10a1bfb6ac..e67e1c9a06 100644 --- a/crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_text.rs +++ b/crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_text.rs @@ -250,6 +250,8 @@ fn model_text(entry: &Bound<'_, PyAny>) -> PyResult { rename_hints, config_diff: coloured(model_config_diff(entry, &reason)?), query_diff: coloured(model_query_diff(entry, &reason)?), + reinitialised_by: optional_text(&native_field(entry, "reinitialised_by")?)?, + paused: native_field(entry, "maintained_paused")?.is_truthy()?, name, action, reason, diff --git a/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/display_plan.rs b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/display_plan.rs index 77c64a33bc..cabdb8403f 100644 --- a/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/display_plan.rs +++ b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/display_plan.rs @@ -24,6 +24,7 @@ use sqlbuild_python_project::planner_phases::main::object::object; const VIEW: &str = "view"; const INCREMENTAL: &str = "incremental"; const CUSTOM: &str = "custom"; +const MAINTAINED: &str = "maintained"; fn display_action( materialization_type: &Bound<'_, PyAny>, @@ -39,6 +40,8 @@ fn display_action( } } else if materialization_type.eq(CUSTOM)? { "CUSTOM" + } else if materialization_type.eq(MAINTAINED)? { + "CREATE_MAINTAINED" } else { "CREATE_TABLE" }) diff --git a/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/execution_plan.rs b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/execution_plan.rs index ff6617a080..f33159b252 100644 --- a/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/execution_plan.rs +++ b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/execution_plan.rs @@ -27,6 +27,7 @@ use crate::planner_phases::main::outputs::base_plan_output::base_plan_output; use crate::planner_phases::main::outputs::build_planner_identity_context::build_planner_identity_context; use crate::planner_phases::main::outputs::check_selected_scope_buildability::check_selected_scope_buildability; use crate::planner_phases::main::outputs::detect_stale_warning_changes::detect_stale_warning_changes; +use crate::planner_phases::main::outputs::plan_maintained_refresh::plan_maintained_refresh; use crate::planner_phases::main::outputs::unselected_upstream_models::unselected_upstream_models; use crate::planner_phases::main::rebuilds::build_planner_entry_results::build_planner_entry_results; use crate::planner_phases::main::rebuilds::defer_to_migration_origins::defer_to_migration_origins; @@ -196,6 +197,7 @@ pub(crate) fn planned_output<'py>( (&inspection_scope, &reconciliation), &source_freshness, )?; + let entries = plan_maintained_refresh(runtime, &inspection_scope, entries)?; record_step(BUILDING_ENTRIES, entries_started.elapsed().as_secs_f64()); let planned_sql_tests = timed(TEST_FIXTURES, || test_planning.result(py))?.into_bound(py); let assembling_started = Instant::now(); diff --git a/crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/maintained.rs b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/maintained.rs new file mode 100644 index 0000000000..4f0650b1be --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/maintained.rs @@ -0,0 +1,274 @@ +//! Maintained models whose live refresh settings drifted, kept ones a replaced upstream reinitialises, and who reads what. + +use crate::planner_phases::main::inputs::runtime_reads::runtime_reads; +use crate::planner_phases::models::ModelEntryResults; +use crate::planner_strategy::main::strategy::materialization_of::materialization_of; +use crate::planner_strategy::main::values::text::text; +use crate::planner_warehouse::models::{AdapterRead, PlannerReads, Read}; +use pyo3::IntoPyObject; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyBool, PyString, PyTuple}; +use sqlbuild_analysis::assembly::project::types::ObjectKey; +use sqlbuild_analysis::maintained_refresh::main::maintained_settings_drift::maintained_settings_drift; +use sqlbuild_analysis::maintained_refresh::main::reinitialised_readers::reinitialised_readers; +use sqlbuild_analysis::maintained_refresh::main::render_target_lag::render_target_lag; +use sqlbuild_analysis::maintained_refresh::models::{ + LiveDynamicTable, MaintainedRead, MaintainedSettings, RefreshLag, +}; +use sqlbuild_analysis::planner_scope::constants::MODEL; +use sqlbuild_model_config::model_validation::main::fixed_duration_seconds::fixed_duration_seconds; +use sqlbuild_python_base::records::main::construct::enum_member::enum_member; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::main::fields::replaced::replaced; +use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_front::model_config::main::config::config_text_object::config_text_object; +use sqlbuild_python_front::model_config::main::config::config_value::config_value; +use sqlbuild_python_front::model_config::main::config::model_values::model_values; +use sqlbuild_python_project::graph::main::nodes::native_key::native_key; +use sqlbuild_python_project::native_host::models::ModelLookup; +use sqlbuild_python_project::planner_scope::main::scope_data::scope_data; +use std::collections::{BTreeSet, HashMap}; + +const MAINTAINED: &str = "maintained"; +/// Views pass reads through, so a maintained model reading a view reads the view's inputs. +const VIEW: &str = "view"; +const KEEP_MAINTAINED: &str = "keep_maintained"; +/// The scheduling state Snowflake lists for a dynamic table whose refreshes are paused. +const SUSPENDED: &str = "SUSPENDED"; +/// The refresh mode of a maintained model that declares none, so Snowflake never picks one quietly. +const DEFAULT_REFRESH_MODE: &str = "incremental"; +/// Actions that recreate the relation, which reinitialises every dynamic table reading it. +const REPLACING_ACTIONS: [&str; 3] = ["create_table", "create_view", "create_maintained"]; + +/// Plan maintained refresh settings and reinitialised readers, and name the maintained readers of each entry. +pub(crate) fn plan_maintained_refresh<'py>( + runtime: &Bound<'py, PyAny>, + execution_scope: &Bound<'py, PyAny>, + results: ModelEntryResults<'py>, +) -> PyResult> { + let py: Python<'py> = runtime.py(); + let scope = scope_data(execution_scope)?; + let mut upstream: HashMap<&ObjectKey, Vec> = HashMap::new(); + for (key, deps) in &scope.upstream { + upstream.insert(key, deps.iter().map(|(_, name)| name.clone()).collect()); + } + let downstream: HashMap<&ObjectKey, &Vec> = scope + .downstream + .iter() + .map(|(key, readers)| (key, readers)) + .collect(); + let models = ModelLookup::Native(scope.clone()); + let reads = runtime_reads(runtime)?; + let mut known = KnownReaders { + models: &models, + materializations: HashMap::new(), + through_view: HashMap::new(), + }; + let mut entries: Vec> = Vec::with_capacity(results.entries.len()); + let mut rows: Vec = Vec::with_capacity(results.entries.len()); + for entry in results.entries { + let key: ObjectKey = native_key(&native_field(&entry, "key")?)?; + let is_maintained: bool = + text(&native_field(&entry, "materialization_type")?)? == MAINTAINED; + let entry = if is_maintained { + with_refresh_settings(entry, &models, &reads)? + } else { + entry + }; + let readers: Vec = known.maintained_readers(py, &key, &downstream)?; + let entry = if readers.is_empty() { + entry + } else { + replaced( + &entry, + &[("maintained_readers", PyTuple::new(py, readers)?.into_any())], + )? + }; + let action: String = text(&native_field(&entry, "action")?)?; + rows.push(MaintainedRead { + name: key.1.clone(), + maintained: is_maintained, + replaced: REPLACING_ACTIONS.contains(&action.as_str()), + reads: upstream.remove(&key).unwrap_or_default(), + }); + entries.push(entry); + } + let readers: HashMap = reinitialised_readers(&rows).into_iter().collect(); + for (entry, row) in entries.iter_mut().zip(&rows) { + if let Some(upstream) = readers.get(&row.name) { + *entry = replaced( + entry, + &[("reinitialised_by", PyString::new(py, upstream).into_any())], + )?; + } + } + Ok(ModelEntryResults { + entries, + warnings: results.warnings, + }) +} + +/// Per-plan lookups: each model's materialization, and the maintained readers behind each view. +struct KnownReaders<'a> { + models: &'a ModelLookup, + materializations: HashMap, + through_view: HashMap>, +} + +impl KnownReaders<'_> { + /// The maintained models reading the entry directly or through views, selected or not. + fn maintained_readers( + &mut self, + py: Python<'_>, + key: &ObjectKey, + downstream: &HashMap<&ObjectKey, &Vec>, + ) -> PyResult> { + let mut found: BTreeSet = BTreeSet::new(); + for reader in downstream + .get(key) + .into_iter() + .flat_map(|readers| readers.iter()) + { + let (kind, name) = reader; + if kind != MODEL { + continue; + } + match self.materialization(py, name)?.as_str() { + MAINTAINED => { + found.insert(name.clone()); + } + VIEW => { + if !self.through_view.contains_key(name) { + let through: Vec = + self.maintained_readers(py, reader, downstream)?; + self.through_view.insert(name.clone(), through); + } + found.extend(self.through_view[name].iter().cloned()); + } + _ => {} + } + } + Ok(found.into_iter().collect()) + } + + /// A model's materialization, looked up once per plan. + fn materialization(&mut self, py: Python<'_>, name: &str) -> PyResult { + if let Some(materialization) = self.materializations.get(name) { + return Ok(materialization.clone()); + } + let model = self.models.get(py, name)?; + let materialization: String = if model.is_none() { + String::new() + } else { + text(&materialization_of(&model)?)? + }; + self.materializations + .insert(name.to_owned(), materialization.clone()); + Ok(materialization) + } +} + +/// The entry with its `target_lag`; a kept one alters on drift, or creates when no dynamic table holds its name. +fn with_refresh_settings<'py>( + entry: Bound<'py, PyAny>, + models: &ModelLookup, + reads: &Bound<'py, PlannerReads>, +) -> PyResult> { + let py: Python<'py> = entry.py(); + let model = models.get(py, &native_field(&entry, "name")?.extract::()?)?; + let Some(desired) = declared_settings(&model)? else { + return Ok(entry); + }; + let values = model_values(&model)?; + let mode: String = match config_text_object(py, &values, "refresh_mode")? { + Some(mode) => mode.extract()?, + None => DEFAULT_REFRESH_MODE.to_owned(), + }; + let mut changes: Vec<(&str, Bound<'py, PyAny>)> = vec![ + ( + "target_lag", + PyString::new(py, &render_target_lag(desired.lag)).into_any(), + ), + ("refresh_mode", PyString::new(py, &mode).into_any()), + ( + "refresh_warehouse", + desired.warehouse.as_deref().into_pyobject(py)?.into_any(), + ), + ( + "refresh_on_build", + PyBool::new( + py, + config_value(py, &values, "refresh_on_build")?.is_truthy()?, + ) + .to_owned() + .into_any(), + ), + ]; + if text(&native_field(&entry, "action")?)? == KEEP_MAINTAINED { + let (action, paused) = live_action(&entry, &desired, reads)?; + if let Some((action, reason)) = action { + changes.push(("action", enum_member(py, RecordId::PlanAction, action)?)); + changes.push(("reason", enum_member(py, RecordId::PlanReason, reason)?)); + } + changes.push(( + "maintained_paused", + PyBool::new(py, paused).to_owned().into_any(), + )); + } + replaced(&entry, &changes) +} + +/// The action and reason replacing a kept model's, `None` when its listing matches or is unknown, and whether it is suspended. +fn live_action( + entry: &Bound<'_, PyAny>, + desired: &MaintainedSettings, + reads: &Bound<'_, PlannerReads>, +) -> PyResult<(Option<(&'static str, &'static str)>, bool)> { + let destination = native_field(entry, "destination")?; + let read = reads.get().reads().adapter_read( + entry.py(), + AdapterRead::MaintainedRefresh([ + native_field(&destination, "database")?, + native_field(&destination, "schema")?, + native_field(&destination, "name")?, + ]), + )?; + let listed = match read { + Read::Unknown(_) => return Ok((None, false)), + Read::Known(listed) => listed, + }; + if listed.is_none() { + return Ok((Some(("CREATE_MAINTAINED", "FIRST_RUN")), false)); + } + let (target_lag, warehouse, state): (String, String, String) = listed.extract()?; + let live = LiveDynamicTable { + target_lag, + warehouse, + }; + let action = + maintained_settings_drift(desired, &live).then_some(("ALTER_MAINTAINED", "CONFIG_CHANGED")); + Ok((action, state == SUSPENDED)) +} + +/// The model's declared lag and refresh warehouse, or `None` when its lag does not read. +fn declared_settings(model: &Bound<'_, PyAny>) -> PyResult> { + let py = model.py(); + let values = model_values(model)?; + let refresh = config_value(py, &values, "refresh")?; + let lag: RefreshLag = if refresh.is_none() { + RefreshLag::Downstream + } else { + let text: String = refresh.get_item("lag")?.extract()?; + let Some(seconds) = fixed_duration_seconds(&text) else { + return Ok(None); + }; + match u64::try_from(seconds) { + Ok(seconds) => RefreshLag::Seconds(seconds), + Err(_) => return Ok(None), + } + }; + let warehouse: Option = config_text_object(py, &values, "refresh_warehouse")? + .map(|warehouse| warehouse.extract()) + .transpose()?; + Ok(Some(MaintainedSettings { lag, warehouse })) +} diff --git a/crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/mod.rs b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/mod.rs index fcf2752f38..1295d1283b 100644 --- a/crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/mod.rs +++ b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/mod.rs @@ -1,3 +1,4 @@ pub(crate) mod buildability; pub(crate) mod identities; +pub(crate) mod maintained; pub(crate) mod output_assembly; diff --git a/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/mod.rs b/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/mod.rs index 69b6f2dbe9..28947d1f49 100644 --- a/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/mod.rs +++ b/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/mod.rs @@ -7,5 +7,6 @@ pub mod build_planner_identity_context; pub mod check_selected_scope_buildability; pub mod detect_stale_warning_changes; pub mod detect_stale_warning_changes_of_function; +pub mod plan_maintained_refresh; pub mod planner_identity_work_info_function; pub mod unselected_upstream_models; diff --git a/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/plan_maintained_refresh.rs b/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/plan_maintained_refresh.rs new file mode 100644 index 0000000000..2b789a1dc5 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/plan_maintained_refresh.rs @@ -0,0 +1,15 @@ +//! Alter drifted maintained models, set their target lag, and mark kept readers of a replaced upstream. + +use crate::planner_phases::_helpers::outputs::maintained; +use crate::planner_phases::models::ModelEntryResults; +use pyo3::prelude::{Bound, PyAny, PyResult}; + +/// Alter drifted maintained models, set their target lag, and mark kept readers of a replaced upstream. +#[inline] +pub fn plan_maintained_refresh<'py>( + runtime: &Bound<'py, PyAny>, + execution_scope: &Bound<'py, PyAny>, + entries: ModelEntryResults<'py>, +) -> PyResult> { + maintained::plan_maintained_refresh(runtime, execution_scope, entries) +} diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/strategy.rs b/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/strategy.rs index b620572c5d..89cc09cce8 100644 --- a/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/strategy.rs +++ b/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/strategy.rs @@ -30,6 +30,9 @@ pub(crate) fn action_value(action: PlanAction) -> &'static str { PlanAction::Snapshot => "snapshot", PlanAction::Skip => "skip", PlanAction::Custom => "custom", + PlanAction::CreateMaintained => "create_maintained", + PlanAction::AlterMaintained => "alter_maintained", + PlanAction::KeepMaintained => "keep_maintained", } } diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/values.rs b/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/values.rs index d910fd16b3..284859ce87 100644 --- a/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/values.rs +++ b/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/values.rs @@ -55,6 +55,7 @@ pub(crate) fn materialization_value(materialization: MaterializationType) -> &'s MaterializationType::Snapshot => "snapshot", MaterializationType::Seed => "seed", MaterializationType::Custom => "custom", + MaterializationType::Maintained => "maintained", } } diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/models.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/models.rs index da5c4f3d27..b09c27b27d 100644 --- a/crates/sqlbuild-python-planner/src/planner_warehouse/models.rs +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/models.rs @@ -383,6 +383,8 @@ pub enum AdapterRead<'py> { FreshnessQuery(Bound<'py, PyAny>), /// Why a destination cannot rename columns, `None` when it can. ColumnRenameUnavailable(Bound<'py, PyAny>), + /// One maintained model's `[database, schema, name]` listed target lag and refresh warehouse. + MaintainedRefresh([Bound<'py, PyAny>; 3]), } /// A read of what the warehouse state tables recorded. @@ -754,6 +756,9 @@ impl WarehouseReads for SyncedReads { AdapterRead::ColumnRenameUnavailable(_) => { return Ok(unknown(self, "column rename support")); } + AdapterRead::MaintainedRefresh(_) => { + return Ok(unknown(self, "maintained model listings")); + } }; let unavailable = host_entry(py, AdapterHost::TableFreshnessMetadata)?.getattr("unavailable")?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs index 7186236739..5a0a7a7829 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs @@ -49,6 +49,7 @@ use sqlbuild_python_compile::compile_driver::models::TargetSettingsValues; use sqlbuild_python_front::cost_context::models::CostContext; use sqlbuild_python_front::cursor::main::overrides::cursor_integer::cursor_integer; use sqlbuild_python_front::cursor::main::overrides::cursor_timestamp::cursor_timestamp; +use sqlbuild_python_front::discovery::main::project::configured_maintained::configured_maintained; use sqlbuild_python_front::native_host::main::recorder_callbacks::recorder_callbacks; use sqlbuild_python_output::build_progress::main::batch_complete_callback::batch_complete_callback; use sqlbuild_python_output::build_progress::main::check_callback::check_callback; @@ -336,77 +337,71 @@ pub(crate) fn execute_build_plan<'py>( let lifecycle: &DirectPythonLifecycle = preparation.lifecycle.get(); let [start_ts, end_ts, start_int, end_int] = &preparation.cursors; let request = |name: &str| native_field(&context.request, name); - let target_config: TargetSettingsValues = project_target_settings( - &context.get("discovered_inputs")?, - &project, - (&connection_config, &adapter), - )?; + let discovered: Bound<'py, PyAny> = context.get("discovered_inputs")?; + let project_dir: Bound<'py, PyAny> = context.get("effective_project_dir")?; + let target_config: TargetSettingsValues = + project_target_settings(&discovered, &project, (&connection_config, &adapter))?; let (default_database, default_schema) = target_config.node_locations(&adapter)?; + let maintained = configured_maintained(&discovered, &context.request, &project_dir)?; let runtime: Bound<'py, PyAny> = call_with( &host_entry(py, ExecutorHost::BuildRuntimeParams)?, &[ - ("run_id", native_field(&project, "run_id")?), - ( - "runtime_dir", - joined(&context.get("effective_project_dir")?, &["target"])?, - ), - ( - "snapshots", - context - .get("discovered_inputs")? - .getattr("project_config")? - .getattr("snapshots")?, - ), - ( - "allow_snapshot_schema_change", - request("allow_snapshot_schema_change")?, - ), - ("run_tests", request("run_tests")?), - ("run_audits", request("run_audits")?), - ("fail_fast", request("fail_fast")?), - ( - "max_concurrency", - preparation.concurrency.into_pyobject(py)?.into_any(), - ), - ("loader_is_reload", request("reload_sources")?), - ("target_config", target_config.record(py)?), - ("default_database", default_database), - ("default_schema", default_schema), - ("start_cursor_ts", start_ts.clone()), - ("end_cursor_ts", end_ts.clone()), - ("start_cursor_int", start_int.clone()), - ("end_cursor_int", end_int.clone()), - ("use_color", boolean(py, context.use_color)), - ("providers", providers.clone()), - ( - "test_timeouts", - timeout_record( - py, - &resolve_run_timeout( - (&project, &context.get("discovered_inputs")?), - request("test_timeout")?.extract()?, - TimeoutKind::Test, + &maintained[..], + &[ + ("run_id", native_field(&project, "run_id")?), + ("runtime_dir", joined(&project_dir, &["target"])?), + ( + "snapshots", + discovered.getattr("project_config")?.getattr("snapshots")?, + ), + ( + "allow_snapshot_schema_change", + request("allow_snapshot_schema_change")?, + ), + ("run_tests", request("run_tests")?), + ("run_audits", request("run_audits")?), + ("fail_fast", request("fail_fast")?), + ( + "max_concurrency", + preparation.concurrency.into_pyobject(py)?.into_any(), + ), + ("loader_is_reload", request("reload_sources")?), + ("target_config", target_config.record(py)?), + ("default_database", default_database), + ("default_schema", default_schema), + ("start_cursor_ts", start_ts.clone()), + ("end_cursor_ts", end_ts.clone()), + ("start_cursor_int", start_int.clone()), + ("end_cursor_int", end_int.clone()), + ("use_color", boolean(py, context.use_color)), + ("providers", providers.clone()), + ( + "test_timeouts", + timeout_record( + py, + &resolve_run_timeout( + (&project, &discovered), + request("test_timeout")?.extract()?, + TimeoutKind::Test, + )?, )?, - )?, - ), - ( - "project_relations", - build_project_relation_targets( - py, - Some(keywords( + ), + ( + "project_relations", + build_project_relation_targets( py, - &[("adapter", adapter.clone()), ("plan_output", plan.clone())], - )?), - ) - .and_then(|value| pyo3::IntoPyObjectExt::into_bound_py_any(value, py))?, - ), - ], - )?; - let policy: Option = changed_policy( - &context.get("effective_project_dir")?, - &context.request, - &project, + Some(keywords( + py, + &[("adapter", adapter.clone()), ("plan_output", plan.clone())], + )?), + ) + .and_then(|value| pyo3::IntoPyObjectExt::into_bound_py_any(value, py))?, + ), + ], + ] + .concat(), )?; + let policy: Option = changed_policy(&project_dir, &context.request, &project)?; let prepared: Prepared<'py> = prepare_consumption( (&plan, &project), (&adapter, &connection_config), @@ -423,7 +418,7 @@ pub(crate) fn execute_build_plan<'py>( let build_callbacks: Bound<'py, PyAny> = call_with( &host_entry(py, ExecutorHost::BuildCallbacks)?, &[ - ("on_migration_progress", line_writer(&context.stream)?), + ("on_lifecycle_progress", line_writer(&context.stream)?), ( "on_node_start", keyword_relay(native_field(callbacks, "on_node_start")?, &NODE_START)?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/constants.rs index f63a0244ed..a0c36d1e5c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/constants.rs @@ -19,10 +19,10 @@ use sqlbuild_core::command_output::constants::{ }; use sqlbuild_core::constants::{ AUDIT_RESULTS_TABLE_NAME, COLUMN_MIGRATION_TABLE_NAME, CONSUMPTION_TABLE_NAME, - FINGERPRINT_TABLE_NAME, JANITOR_EVENTS_TABLE_NAME, MICROBATCH_TABLE_NAME, MIGRATION_TABLE_NAME, - NODE_RESULTS_TABLE_NAME, OBSERVATIONS_TABLE_NAME, OLD_NAME_VIEW_TABLE_NAME, - PROJECT_CONFIG_FILENAME, PYTHON_INIT_MODULE_STEM, PYTHON_NODE_ROOT, RUNS_TABLE_NAME, - SETTINGS_SECTION, SOURCE_FRESHNESS_TABLE_NAME, STAGED_PROMOTION_MODE, + FINGERPRINT_TABLE_NAME, JANITOR_EVENTS_TABLE_NAME, MAINTAINED_PAUSES_TABLE_NAME, + MICROBATCH_TABLE_NAME, MIGRATION_TABLE_NAME, NODE_RESULTS_TABLE_NAME, OBSERVATIONS_TABLE_NAME, + OLD_NAME_VIEW_TABLE_NAME, PROJECT_CONFIG_FILENAME, PYTHON_INIT_MODULE_STEM, PYTHON_NODE_ROOT, + RUNS_TABLE_NAME, SETTINGS_SECTION, SOURCE_FRESHNESS_TABLE_NAME, STAGED_PROMOTION_MODE, TABLE_PROMOTION_MODE_SETTING_KEY, ZERO_DAY_DURATION, }; use sqlbuild_core::diagnostic_routing::constants::{ @@ -95,6 +95,7 @@ pub(crate) const EXPORTED_TEXTS: &[(&str, &str)] = &[ ("JANITOR_EVENTS_TABLE_NAME", JANITOR_EVENTS_TABLE_NAME), ("OBSERVATIONS_TABLE_NAME", OBSERVATIONS_TABLE_NAME), ("RUNS_TABLE_NAME", RUNS_TABLE_NAME), + ("MAINTAINED_PAUSES_TABLE_NAME", MAINTAINED_PAUSES_TABLE_NAME), ("COLUMN_NODE_TYPE", COLUMN_NODE_TYPE), ("COLUMN_NODE_NAME", COLUMN_NODE_NAME), ("COLUMN_TARGET_DATABASE", COLUMN_TARGET_DATABASE), diff --git a/fensu.toml b/fensu.toml index 9fec3e7dee..2a1becc261 100644 --- a/fensu.toml +++ b/fensu.toml @@ -1351,6 +1351,10 @@ rule = "FFR601" path = "src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py" reason = "The SQL Server adapter intentionally keeps overrideable backend behavior on one class." +[[dupes.allowlist]] +paths = ["scripts/native_layering/_helpers/layering.py", "src/sqlbuild/executor/maintained_pauses/_helpers/scope.py"] +reason = "A ten-line depth-first walk in two packages that must not import each other: the dependency-layering tool walks Cargo.lock packages, the build's maintained-model hold walks plan edges between models." + [[dupes.allowlist]] paths = ["crates/sqlbuild-python-planner/src/planner_changes/_helpers/detect.rs", "crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/python_selection.rs"] reason = "One reads a column's name and type for schema-change detection, the other the kind and name of each SQL resource a Python node reads; only the two-field record loop is alike." diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/build.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/build.md index 129b506831..f006832b56 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/build.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/build.md @@ -66,6 +66,7 @@ own option. `--version` works only directly after `sqb`, as in `sqb --version`: | `--defer-sources-to` | Read managed source data from another target | | `--plan-from` | Read warehouse state `live`, or from the target's `synced_state` checked against the warehouse first; overrides the target's `plan_from`. See [Planning from synced state](../concepts/project-configuration.md#planning-from-synced-state) | | `--fail-fast` | Stop on first failure and skip remaining nodes | +| `--ignore-maintained-grace` | Take over [maintained models](../concepts/models/materializations.md#during-a-build) another build holds, without waiting for `[maintained] heartbeat_grace`. Needs `allow_ignore_grace = true`, else fails with `A719` | | `--concurrency` | Number of worker connections (default: 1) | | `--test-timeout ` | Time limit for each unit test the build runs, such as `30s`, or `disabled`; see [Time limits](../concepts/testing-reference.md#time-limits) | | `--verbose`, `-v` | Show lifecycle SQL inline after each model | diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/dag.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/dag.md index dd9ae1bcfd..c87285ddeb 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/dag.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/dag.md @@ -105,7 +105,7 @@ Each node represents a source, seed, model, or function: | `description` | Model or source description, if declared | | `tags` | Model tags | | `columns` | Column metadata (name, type, nullable, description) | -| `materialization_type` | For models: `view`, `table`, `incremental`, or custom name | +| `materialization_type` | For models: `view`, `table`, `incremental`, `snapshot`, `maintained`, or custom name | | `language` | For functions: `sql` or `python` | | `return_kind` | For functions: `scalar` or `table` | | `arguments` | For functions: argument name and type pairs | diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/snowflake.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/snowflake.md index dcedd89b3c..3074e3168e 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/snowflake.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/snowflake.md @@ -14,6 +14,7 @@ Online: https://sqlbuild.com/docs/concepts/adapters/snowflake/ - Case-sensitive column names - Metadata reads - State tables +- Maintained models - Shared connections across targets - Cost estimates - Functions in unit tests @@ -165,6 +166,28 @@ SQLBuild records its state in `_sqlbuild_*` tables. On Snowflake it creates them with 90 days of time travel, or 1 day on editions that allow only that. It never alters an existing state table. Models are not affected: their table type follows the model's `table_type` setting. +## Maintained models + +A [maintained model](../models/materializations.md#maintained) is a Snowflake dynamic +table. `materialized materialized_view` fails with `A014`; use `materialized maintained`. + +The build role needs these grants: + +| Grant | Why | +|-------|-----| +| `CREATE DYNAMIC TABLE` on the model's schema | Create and recreate the model | +| Ownership or `OPERATE` on the dynamic table | Suspend, resume, refresh and alter it during builds | +| `USAGE` on the `refresh_warehouse` | Run the scheduled refreshes | + +Dynamic tables refresh from changes, which needs change tracking on every table they read: + +- A table model that a maintained model reads, directly or through views, is built with change tracking on. +- For a source table, Snowflake turns change tracking on when the build role owns the table. + Otherwise its owner runs `ALTER TABLE SET CHANGE_TRACKING = TRUE`. + +Builds record the models they pause in the `_sqlbuild_maintained_pauses` state table, so another +build can tell whether they are still held. + ## Shared connections across targets Use multiple targets to reuse one Snowflake connection while selecting different authoritative diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/configuration.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/configuration.md index 94cda6a657..00a9c2512f 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/configuration.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/configuration.md @@ -36,7 +36,7 @@ which accepts any keys. | Field | Description | |-------|-------------| -| `materialized` | `view`, `table`, `incremental`, `snapshot`, or a custom materialization name | +| `materialized` | `view`, `table`, `incremental`, `snapshot`, `maintained`, or a custom materialization name; see [Materializations](materializations.md) | | `tags` | Tags used by selectors | | `critical` | `true` marks the model as critical, so [`sqb review`](../../cli/review.md) builds and compares it whenever something upstream of it changes (default `false`) | | `description` | Required human-readable model description; see [Descriptions](#descriptions) | diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/materializations.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/materializations.md index 3baa7ed8c0..a6c8c4d203 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/materializations.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/materializations.md @@ -12,6 +12,7 @@ Online: https://sqlbuild.com/docs/concepts/models/materializations/ - Table - Incremental - Snapshot +- Maintained - Custom - When a build fails @@ -83,6 +84,78 @@ FROM __source("customers") See [Snapshots](../snapshots.md) for timestamp and check strategies, historical inputs, and full-refresh policies. +## Maintained + +The warehouse keeps a maintained model up to date on a schedule you set, between builds. On Snowflake it is a dynamic table; a build creates it, changes it when its definition changes, and otherwise leaves the refreshing to Snowflake. + +```sql +MODEL ( + description "One row per customer per order date", + materialized maintained, + refresh (lag 1h), +); + +SELECT customer_id, CAST(ordered_at AS DATE) AS order_date, SUM(amount) AS revenue +FROM __ref("stg_orders") +GROUP BY customer_id, CAST(ordered_at AS DATE) +``` + +| Field | Default | Description | +|-------|---------|-------------| +| `refresh (lag )` | none | How far behind its inputs the model may fall: a fixed duration such as `5m`, `1h` or `1d`, at least `1m` on Snowflake and no shorter than the lag of a maintained model it reads. Without it, the model refreshes when a maintained model that reads it refreshes | +| `refresh_mode` | `incremental` | `incremental`, `full`, `auto` or `adaptive`. Snowflake rejects `incremental` for a query it can't refresh incrementally; use `full` for that query | +| `refresh_warehouse` | the connection's warehouse | Warehouse the scheduled refreshes run on | +| `refresh_on_build` | `false` | Refresh the model in every build that selects it, not only on its schedule | + +Only Snowflake supports maintained models. See [Snowflake](../adapters/snowflake.md#maintained-models) for the grants and change tracking they need. + +### What the plan shows + +| Change | Plan | What the build does | +|--------|------|---------------------| +| First build, or the dynamic table is missing | `create` | Creates it and fills it | +| Query or `refresh_mode` changed | `recreate, full recompute` | Replaces it; every row is computed again | +| `refresh` lag or `refresh_warehouse` differs from the warehouse | `change refresh settings in place` | Alters the schedule; rows stay | +| Nothing | `maintained` | Leaves it alone | + +A model the plan keeps can carry one more line: + +- `recomputes in full: stg_orders is rebuilt`: the build replaces a model it reads, so its next refresh computes every row again. +- `paused: scheduled refreshes are suspended`: Snowflake lists it as suspended, so it is not refreshing. + +### During a build + +A build pauses the maintained models it changes, and those that read a model it rebuilds, directly or through views, so they never refresh from half-built inputs. It resumes them when it ends, whether it succeeds or fails. + +- Dynamic tables outside the project that read a paused model pause too, until the build resumes it. One that someone had suspended before stays suspended. +- A model that was already suspended when the build started stays suspended, and the build warns about it. +- If someone suspends a model during the build, the build's resume undoes that suspend. +- A kept model is refreshed during the build when a model built later in the same build reads it, so that model reads current rows. `refresh_on_build` refreshes it in every build. +- A target with `pause_maintained = true` leaves its maintained models suspended between builds and refreshes them only in builds; see [Targets](../project-configuration.md#targets). + +Two builds can't pause the same models at once. A build that needs a model another build holds fails with `A718`, naming that run. When two builds pause a model at the same moment, the one that recorded its pause first keeps it, and the other fails with `A718` without suspending or resuming anything. A build that stops without resuming, for example because its process was killed, leaves its models suspended. The next build that pauses maintained models on the target resumes them once that build has been silent for `[maintained] heartbeat_grace` (see [Maintained](../project-configuration.md#maintained)). Until then, resume them in Snowflake by hand, or run `sqb build --ignore-maintained-grace` where the project allows it. + +### Error codes + +| Code | When | Fix | +|------|------|-----| +| `A013` | `materialized maintained` on an adapter other than Snowflake | Use `table` or `incremental` on that adapter | +| `A014` | `materialized materialized_view` on Snowflake | Use `materialized maintained` | +| `A015` | A `refresh` lag under `1m` on Snowflake | Set a lag of at least `1m` | +| `A016` | A model with no `refresh` that no maintained model reads, so it would never refresh | Add `refresh (lag 1h)` or another lag | +| `A716` | The adapter can't build a maintained model | Use an adapter that supports them | +| `A717` | No warehouse to refresh with | Set `refresh_warehouse`, or a `warehouse` on the connection | +| `A718` | Another build holds the models this build needs | Wait for it to finish, or for its hold to pass `heartbeat_grace` | +| `A719` | `--ignore-maintained-grace` without `allow_ignore_grace = true` | Set `[maintained] allow_ignore_grace = true`, or wait for the grace to pass | + +### Limitations + +- Writing out the default `refresh_mode incremental` changes the model's definition, so the next build recreates it once. +- A custom materialization named `maintained` is not used; the built-in one takes the name. +- Unit tests and scenarios treat a maintained model as a table. +- On a target with `schema = "preserve"`, builds see each other's pauses only when their maintained models share a schema. +- A snapshot rebuilt in full is created without change tracking. Before a maintained model reads it again, run `ALTER TABLE SET CHANGE_TRACKING = TRUE`. + ## Custom A project-local Python materialization can manage specialized persistence with adapter access, schema findings, query-change state, declared columns, and `ctx.run_audits`. @@ -120,4 +193,5 @@ The custom function owns staging, runtime type or contract enforcement, audit ti | Table (`immediate`) | The new table. Audits run after it is replaced. | | Incremental | The old table if a delta audit fails. Rows already written stay if a final audit or a later microbatch fails. | | Microbatch full rebuild | A partly rebuilt table; the old one was dropped first. | +| Maintained | The created or altered dynamic table, resumed. Audits run after it changes. | | Custom | Whatever the custom materialization does. | diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md index 6b7b5893af..38ca3a22f6 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md @@ -20,6 +20,7 @@ Online: https://sqlbuild.com/docs/concepts/project-configuration/ - Project variables - Janitor - Migrations +- Maintained - Scenario - Git - Review @@ -189,6 +190,7 @@ A target is a named build context - the database and schema you build into, plus | `concurrency` | Worker limit for commands run against this target, replacing `[settings] concurrency` (see below) | | `ignore_audit_failures` | Whether builds on this target let audit failures through, replacing `[settings] ignore_audit_failures`; see [Ignoring audit failures](audits.md#ignoring-audit-failures) | | `plan_from` | Where `sqb plan` and `sqb build` read warehouse state: `live` (default) or `synced_state`; see [Planning from synced state](#planning-from-synced-state) | +| `pause_maintained` | `true` keeps this target's [maintained models](models/materializations.md#maintained) suspended between builds; each build refreshes the ones it selects. Default `false` | ```toml [connections.warehouse] @@ -695,6 +697,21 @@ old_name_views = "30d" |-------|---------|-------------| | `old_name_views` | `"30d"` | How long a renamed model's old name keeps working through a [compatibility view](models/migrations.md#old-names) before janitor drops it. `false` keeps the old relation in place instead. A model's `old_name_view` header overrides it. | +## Maintained + +How builds pause [maintained models](models/materializations.md#maintained) while they run: + +```toml +[maintained] +heartbeat_grace = "30m" +allow_ignore_grace = false +``` + +| Field | Default | Description | +|-------|---------|-------------| +| `heartbeat_grace` | `"30m"` | How long a build that paused maintained models may go silent before another build resumes them or takes them over. A fixed duration longer than `"1m"`, such as `"30m"` or `"2h"`; a build writes a heartbeat every minute | +| `allow_ignore_grace` | `false` | Allow `sqb build --ignore-maintained-grace`, which takes over models another build holds without waiting for the grace. Without it the flag fails with `A719` | + ## Scenario Configuration for scenario run isolation, snapshot capture safety limits, and local type overrides: diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index fe94cec490..eced41585a 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -192,6 +192,7 @@ MICROBATCH_TABLE_NAME: str NODE_RESULTS_TABLE_NAME: str OBSERVATIONS_TABLE_NAME: str RUNS_TABLE_NAME: str +MAINTAINED_PAUSES_TABLE_NAME: str AUDIT_RESULTS_TABLE_NAME: str JANITOR_EVENTS_TABLE_NAME: str COLUMN_NODE_TYPE: str @@ -4316,6 +4317,7 @@ class NativeModelValidator: names: tuple[set[str], set[str], set[str], set[str], set[str]], custom_materializations: set[str], microbatch_concurrency: bool, + adapter: str | None = None, ) -> None: ... @staticmethod def from_inputs( @@ -6591,6 +6593,7 @@ class BuildCommandRequest(NativeRecord): allow_retention_decrease: bool allow_missing_migration_origin: bool allow_snapshot_schema_change: bool + ignore_maintained_grace: bool concurrency: int | None select: tuple[str, ...] exclude: tuple[str, ...] @@ -6635,6 +6638,7 @@ class BuildCommandRequest(NativeRecord): allow_retention_decrease: bool = False, allow_missing_migration_origin: bool = False, allow_snapshot_schema_change: bool = False, + ignore_maintained_grace: bool = False, concurrency: int | None = None, select: tuple[str, ...] = (), exclude: tuple[str, ...] = (), @@ -12014,6 +12018,7 @@ class TargetConfig(NativeRecord): catalog_connection: str | None ignore_audit_failures: bool | None plan_from: str | None + pause_maintained: bool | None def __init__( self, connection: dict[str, object] = ..., @@ -12040,6 +12045,7 @@ class TargetConfig(NativeRecord): catalog_connection: str | None = ..., ignore_audit_failures: bool | None = ..., plan_from: str | None = ..., + pause_maintained: bool | None = ..., ) -> None: ... class LocalTargetConfig(NativeRecord): @@ -12069,6 +12075,7 @@ class LocalTargetConfig(NativeRecord): catalog_connection: str | None ignore_audit_failures: bool | None plan_from: str | None + pause_maintained: bool | None def __init__( self, connection: dict[str, object] = ..., @@ -12096,6 +12103,7 @@ class LocalTargetConfig(NativeRecord): catalog_connection: str | None = ..., ignore_audit_failures: bool | None = ..., plan_from: str | None = ..., + pause_maintained: bool | None = ..., ) -> None: ... class SettingsConfig(NativeRecord): @@ -13522,6 +13530,9 @@ class PlanAction(StrEnum): LOAD_SEED = "load_seed" SKIP = "skip" CUSTOM = "custom" + CREATE_MAINTAINED = "create_maintained" + ALTER_MAINTAINED = "alter_maintained" + KEEP_MAINTAINED = "keep_maintained" class PlanReason(StrEnum): FIRST_RUN = "first_run" @@ -13589,6 +13600,7 @@ class MaterializationType(StrEnum): SNAPSHOT = "snapshot" SEED = "seed" CUSTOM = "custom" + MAINTAINED = "maintained" @classmethod def is_table_backed(cls, *, materialized: object | None) -> bool: ... @@ -14110,6 +14122,13 @@ class ModelPlanEntry(NativeRecord): max_duration: str | None max_duration_seconds: int | None max_duration_source: str | None + reinitialised_by: str | None + target_lag: str | None + refresh_mode: str | None + refresh_warehouse: str | None + refresh_on_build: bool + maintained_readers: tuple[str, ...] + maintained_paused: bool def __init__( self, key: CompiledObjectKey, @@ -14199,6 +14218,13 @@ class ModelPlanEntry(NativeRecord): max_duration: str | None = ..., max_duration_seconds: int | None = ..., max_duration_source: str | None = ..., + reinitialised_by: str | None = ..., + target_lag: str | None = ..., + refresh_mode: str | None = ..., + refresh_warehouse: str | None = ..., + refresh_on_build: bool = ..., + maintained_readers: tuple[str, ...] = ..., + maintained_paused: bool = ..., ) -> None: ... class CloneSourcePlanEntry(NativeRecord): diff --git a/src/sqlbuild/adapter/contract/classes/strict_adapter.py b/src/sqlbuild/adapter/contract/classes/strict_adapter.py index 19023aa933..06cc06a89e 100644 --- a/src/sqlbuild/adapter/contract/classes/strict_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/strict_adapter.py @@ -33,6 +33,8 @@ from sqlbuild.adapter.contract.types import ( FrameworkType, LoaderLogicalType, + MaintainedListing, + MaintainedOperation, PromotionStrategy, StatementSizeLimit, TablePromotionMode, @@ -321,6 +323,51 @@ def render_drop_schema( help="sqb review needs a warehouse that can drop a schema it created", ) + def session_warehouse(self, *, connection: Any) -> str | None: + """Optional: the warehouse the session runs on, or None when the adapter has none.""" + + del connection + return None + + def render_create_maintained( + self, *, destination: str, sql: str, target_lag: str, refresh_mode: str, warehouse: str + ) -> tuple[str, ...]: + """Optional: SQL that creates or replaces a maintained model; unsupported by default.""" + + del sql, target_lag, refresh_mode, warehouse + raise self._maintained_unsupported(destination) + + def render_alter_maintained( + self, *, destination: str, target_lag: str, warehouse: str | None + ) -> tuple[str, ...]: + """Optional: SQL that changes a maintained model's refresh settings in place.""" + + del target_lag, warehouse + raise self._maintained_unsupported(destination) + + def render_maintained_operation( + self, *, destination: str, operation: MaintainedOperation + ) -> tuple[str, ...]: + """Optional: SQL that suspends, resumes or refreshes a maintained model.""" + + del operation + raise self._maintained_unsupported(destination) + + def _maintained_unsupported(self, destination: str) -> AdapterUserError: + return AdapterUserError( + f"the {self.adapter_name} adapter cannot build maintained model '{destination}'", + code="A716", + help="materialized maintained needs an adapter whose warehouse refreshes it", + ) + + def maintained_listing( + self, *, connection: Any, database: str | None, schema: str | None, name: str + ) -> MaintainedListing | None: + """Optional: how the warehouse lists a maintained model, or None when it lists none.""" + + del connection, database, schema, name + return None + @abstractmethod def render_create_table_as(self, *, destination: str, sql: str) -> tuple[str, ...]: """Render SQL statements that create or replace a table from a query.""" diff --git a/src/sqlbuild/adapter/contract/types.py b/src/sqlbuild/adapter/contract/types.py index 0ef3963263..cf88c3cb4e 100644 --- a/src/sqlbuild/adapter/contract/types.py +++ b/src/sqlbuild/adapter/contract/types.py @@ -4,7 +4,7 @@ from collections.abc import Callable from enum import StrEnum -from typing import Literal, Protocol +from typing import Literal, NamedTuple, Protocol from sqlbuild._native import InferredNullability @@ -82,6 +82,14 @@ class RetentionChangePhase(StrEnum): FINALIZE = "finalize" +class MaintainedOperation(StrEnum): + """What a build does to a maintained model's scheduled refreshes.""" + + SUSPEND = "suspend" + RESUME = "resume" + REFRESH = "refresh" + + class RelationType(StrEnum): TABLE = "table" VIEW = "view" @@ -164,3 +172,11 @@ class HistoricalSnapshotInsertStyle(StrEnum): WITH_INSERT = "with_insert" INSERT_WITH = "insert_with" TSQL = "tsql" + + +class MaintainedListing(NamedTuple): + """A maintained model as the warehouse lists it: lag, refresh warehouse and schedule state.""" + + target_lag: str + warehouse: str + scheduling_state: str diff --git a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py index ed578f251e..19dc7d7195 100644 --- a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py +++ b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py @@ -6,6 +6,7 @@ import json import logging import os +import re import time from collections.abc import Callable from dataclasses import replace @@ -98,6 +99,8 @@ HistoricalSnapshotCloseStyle, HistoricalSnapshotInsertStyle, LoaderLogicalType, + MaintainedListing, + MaintainedOperation, MigrationTransfer, PromotionStrategy, RelationReadStatus, @@ -204,6 +207,7 @@ "table_name, table_schema, table_type, is_transient, created, last_altered, retention_time" ) _BULK_COLUMN_RELATION_CHUNK_SIZE: int = 2_000 +_PLAIN_WAREHOUSE_NAME: re.Pattern[str] = re.compile(r"[A-Za-z_][A-Za-z0-9_$]*") _RELATION_CHANGE_COLUMNS: str = ( "table_name, table_schema, table_type, is_transient, created, last_altered, last_ddl, " "retention_time" @@ -1158,10 +1162,11 @@ def render_create_table_as(self, *, destination: str, sql: str) -> tuple[str, .. ) def _render_create_table_as_type( - self, *, destination: str, sql: str, table_type: str + self, *, destination: str, sql: str, table_type: str, change_tracking: bool = False ) -> tuple[str, ...]: table_kind: str = "TABLE" if table_type == TableType.PERMANENT else "TRANSIENT TABLE" - return (f"CREATE OR REPLACE {table_kind} {destination} AS {sql}",) + tracking: str = " CHANGE_TRACKING = TRUE" if change_tracking else "" + return (f"CREATE OR REPLACE {table_kind} {destination}{tracking} AS {sql}",) def render_create_view_as(self, *, destination: str, sql: str) -> tuple[str, ...]: return (f"CREATE OR REPLACE VIEW {destination} AS {sql}",) @@ -1329,7 +1334,10 @@ def create_table_as( ) -> None: table_type: str = str((config or {}).get("table_type", "transient")) statements: tuple[str, ...] = self._render_create_table_as_type( - destination=destination, sql=sql, table_type=table_type + destination=destination, + sql=sql, + table_type=table_type, + change_tracking=bool((config or {}).get("change_tracking", False)), ) statement_recorder.record_many(statements) stmt: str @@ -3665,6 +3673,102 @@ def column_rename_unavailable_reason(self, *, connection: Any, destination: str) del connection, destination return None + def session_warehouse(self, *, connection: Any) -> str | None: + """The session's warehouse, which refreshes a maintained model that names none.""" + + rows: tuple[tuple[object, ...], ...] = self.query( + connection=connection, sql="SELECT CURRENT_WAREHOUSE()", limit=None + ).rows + return None if not rows or rows[0][0] is None else str(rows[0][0]) + + def render_create_maintained( + self, *, destination: str, sql: str, target_lag: str, refresh_mode: str, warehouse: str + ) -> tuple[str, ...]: + """A dynamic table with an explicit mode and ON_CREATE, so it is readable at once.""" + + return ( + f"CREATE OR REPLACE DYNAMIC TABLE {destination} TARGET_LAG = {target_lag} " + f"WAREHOUSE = {self._warehouse_name(warehouse)} " + f"REFRESH_MODE = {refresh_mode.upper()} INITIALIZE = ON_CREATE AS {sql}", + ) + + def render_alter_maintained( + self, *, destination: str, target_lag: str, warehouse: str | None + ) -> tuple[str, ...]: + """The lag, and a declared warehouse, set in place.""" + + statements: list[str] = [f"ALTER DYNAMIC TABLE {destination} SET TARGET_LAG = {target_lag}"] + if warehouse is not None: + statements.append( + f"ALTER DYNAMIC TABLE {destination} " + f"SET WAREHOUSE = {self._warehouse_name(warehouse)}" + ) + return tuple(statements) + + def render_maintained_operation( + self, *, destination: str, operation: MaintainedOperation + ) -> tuple[str, ...]: + return (f"ALTER DYNAMIC TABLE {destination} {operation.value.upper()}",) + + def _warehouse_name(self, warehouse: str) -> str: + """A plain warehouse name as written, resolved without case; any other quoted exactly.""" + + if _PLAIN_WAREHOUSE_NAME.fullmatch(warehouse): + return warehouse + return '"' + warehouse.replace('"', '""') + '"' + + def maintained_listing( + self, *, connection: Any, database: str | None, schema: str | None, name: str + ) -> MaintainedListing | None: + """The dynamic table's lag, warehouse and scheduling state from SHOW DYNAMIC TABLES.""" + + row: dict[str, object] | None = self._listed_dynamic_table( + connection=connection, database=database, schema=schema, name=name + ) + if row is None: + return None + return MaintainedListing( + target_lag=str(row.get("target_lag")), + warehouse=str(row.get("warehouse")), + scheduling_state=str(row.get("scheduling_state")), + ) + + def _listed_dynamic_table( + self, *, connection: Any, database: str | None, schema: str | None, name: str + ) -> dict[str, object] | None: + if schema is None: + return None + return self._show_named_row( + connection=connection, + listing="DYNAMIC TABLES", + database=database, + schema=schema, + name=name, + ) + + def _show_named_row( + self, *, connection: Any, listing: str, database: str | None, schema: str, name: str + ) -> dict[str, object] | None: + """The SHOW row for exactly this name in the schema, if it lists one.""" + + stored_name: str = self._information_schema_identifier(name) + rows: list[dict[str, object]] = self._fetch_schema_show_rows( + connection=connection, + query=( + f"SHOW {listing} LIKE '{show_like_pattern(stored_name)}' IN SCHEMA " + + self._show_schema_scope(connection=connection, database=database, schema=schema) + ), + show_database=self._show_database( + connection=connection, database=database, schema=schema + ), + schema=schema, + ) + row: dict[str, object] + for row in rows: + if str(row.get("name")) == stored_name: + return row + return None + def widen_target_nullability( self, *, @@ -3797,27 +3901,14 @@ def _show_object_row_count( ) -> tuple[Any, Any] | None: """Rows and bytes of one table; a view or other object lists neither.""" - stored_name: str = self._information_schema_identifier(name) - rows: list[dict[str, object]] = self._fetch_schema_show_rows( - connection=connection, - query=( - f"SHOW OBJECTS LIKE '{show_like_pattern(stored_name)}' IN SCHEMA " - + self._show_schema_scope(connection=connection, database=database, schema=schema) - ), - show_database=self._show_database( - connection=connection, database=database, schema=schema - ), - schema=schema, + row: dict[str, object] | None = self._show_named_row( + connection=connection, listing="OBJECTS", database=database, schema=schema, name=name ) - row: dict[str, object] - for row in rows: - if str(row.get("name")) == stored_name: - return ( - (row.get("rows"), row.get("bytes")) - if str(row.get("kind")) == SHOW_OBJECTS_TABLE_KIND - else (None, None) - ) - return None + if row is None: + return None + if str(row.get("kind")) != SHOW_OBJECTS_TABLE_KIND: + return (None, None) + return (row.get("rows"), row.get("bytes")) def inspect_row_diff_coverage( self, diff --git a/src/sqlbuild/executor/build/_helpers/scheduler.py b/src/sqlbuild/executor/build/_helpers/scheduler.py index 727e48f49e..527dd72fff 100644 --- a/src/sqlbuild/executor/build/_helpers/scheduler.py +++ b/src/sqlbuild/executor/build/_helpers/scheduler.py @@ -25,6 +25,7 @@ BUILD_CUSTOM_MATERIALIZATION_MISSING_CODE, BUILD_SOURCE_FRESHNESS_BLOCKED_CODE, INCREMENTAL_ACTIONS, + MAINTAINED_ACTIONS, ) from sqlbuild.executor.build.models import ( FunctionExecutionResult, @@ -36,6 +37,7 @@ from sqlbuild.executor.run.main._execute import ( execute_custom_entry, execute_incremental_entry, + execute_maintained_entry, execute_microbatch_entry, execute_snapshot_entry, execute_table_entry, @@ -163,6 +165,8 @@ def _dispatch_model_entry( context=context, declared_columns=entry.declared_columns, ) + if entry.action in MAINTAINED_ACTIONS: + return execute_maintained_entry(context=context) if entry.action == PlanAction.CREATE_VIEW: return execute_view_entry(context=context) if entry.action == PlanAction.SNAPSHOT: diff --git a/src/sqlbuild/executor/build/classes/build_scheduler.py b/src/sqlbuild/executor/build/classes/build_scheduler.py index 5790b24ac8..b8c628ecfd 100644 --- a/src/sqlbuild/executor/build/classes/build_scheduler.py +++ b/src/sqlbuild/executor/build/classes/build_scheduler.py @@ -1256,6 +1256,7 @@ def _execute_model_node( watermark_resolver=self._watermark_resolver, bound_views=bound_views, fingerprint_writes=self._fingerprint_writes, + maintained_hold=self._runtime.maintained_hold, ), promotion_mode=self._promotion_mode, snapshots=self._snapshots, diff --git a/src/sqlbuild/executor/build/constants.py b/src/sqlbuild/executor/build/constants.py index f7109ec129..f4d7636187 100644 --- a/src/sqlbuild/executor/build/constants.py +++ b/src/sqlbuild/executor/build/constants.py @@ -11,6 +11,13 @@ PlanAction.INCREMENTAL_MERGE, } ) +MAINTAINED_ACTIONS: frozenset[PlanAction] = frozenset( + { + PlanAction.CREATE_MAINTAINED, + PlanAction.ALTER_MAINTAINED, + PlanAction.KEEP_MAINTAINED, + } +) BUILD_WORKER_FAILED_CODE: str = "B001" BUILD_UNKNOWN_RESOURCE_FAILED_CODE: str = "B002" diff --git a/src/sqlbuild/executor/build/main/_execute.py b/src/sqlbuild/executor/build/main/_execute.py index 87f8f4bdc1..1d48b1c2f0 100644 --- a/src/sqlbuild/executor/build/main/_execute.py +++ b/src/sqlbuild/executor/build/main/_execute.py @@ -2,6 +2,7 @@ from __future__ import annotations +from contextlib import AbstractContextManager, nullcontext from dataclasses import replace from typing import Any @@ -35,6 +36,10 @@ ) from sqlbuild.executor.build.types import BuildStatus from sqlbuild.executor.load.models import LoadExecutionResult +from sqlbuild.executor.maintained_pauses.classes.maintained_hold import MaintainedHold +from sqlbuild.executor.maintained_pauses.main.hold_maintained_models import ( + hold_maintained_models, +) from sqlbuild.executor.migrations.main._apply import apply_model_migrations from sqlbuild.executor.migrations.main._apply_columns import apply_column_migrations from sqlbuild.executor.run.models import ModelExecutionResult @@ -54,16 +59,27 @@ def execute_build_plan( initial_state: BuildInitialState | None = None, schema_prepared: bool = False, ) -> BuildExecutionResult: - """Execute a full build plan using the DAG scheduler.""" + """Execute a build plan with the DAG scheduler, holding the maintained models it touches.""" - with relation_metadata_cache_scope(adapter=adapter): + hold: MaintainedHold | None = hold_maintained_models( + plan=plan, adapter=adapter, runtime=runtime, connection_config=connection_config + ) + held: AbstractContextManager[object] = ( + nullcontext() + if hold is None + else hold.holding( + connection=scheduler_connection, + on_progress=None if callbacks is None else callbacks.on_lifecycle_progress, + ) + ) + with relation_metadata_cache_scope(adapter=adapter), held: return _execute_build_plan( plan=plan, adapter=adapter, connection_config=connection_config, connections=connections, scheduler_connection=scheduler_connection, - runtime=runtime, + runtime=replace(runtime, maintained_hold=hold), callbacks=callbacks, customizations=customizations, initial_state=initial_state, @@ -109,14 +125,14 @@ def _execute_build_plan( adapter=adapter, connection=scheduler_connection, run_id=runtime.run_id, - on_progress=resolved_callbacks.on_migration_progress, + on_progress=resolved_callbacks.on_lifecycle_progress, ) _ = apply_column_migrations( plan=plan, adapter=adapter, connection=scheduler_connection, run_id=runtime.run_id, - on_progress=resolved_callbacks.on_migration_progress, + on_progress=resolved_callbacks.on_lifecycle_progress, ) _ = apply_retention_phase( plan=plan, diff --git a/src/sqlbuild/executor/build/models.py b/src/sqlbuild/executor/build/models.py index 2883e40c8f..d0c19dfe0a 100644 --- a/src/sqlbuild/executor/build/models.py +++ b/src/sqlbuild/executor/build/models.py @@ -28,6 +28,7 @@ from sqlbuild.executor.build.types import BeforeNodeWriteCallback, BuildStatus from sqlbuild.executor.custom.models import MaterializationResult from sqlbuild.executor.load.models import LoadExecutionResult +from sqlbuild.executor.maintained_pauses.classes.maintained_hold import MaintainedHold from sqlbuild.executor.python_nodes.types import PythonIdentityRecorder from sqlbuild.executor.run.models import MicrobatchBatchProgress, ModelExecutionResult from sqlbuild.executor.scheduling.types import ExecutionStatus @@ -101,6 +102,11 @@ class BuildRuntimeParams: microbatch_lease_check: Callable[[], None] | None = None project_relations: Mapping[SqlResourceRef, str] | None = None test_timeouts: RunItemTimeouts | None = None + pause_maintained: bool = False + maintained_heartbeat_grace_seconds: int = 1800 + allow_ignore_maintained_grace: bool = False + ignore_maintained_grace: bool = False + maintained_hold: MaintainedHold | None = None @dataclass(frozen=True) @@ -119,7 +125,7 @@ class BuildCallbacks: python_identity_recorder: PythonIdentityRecorder | None = None on_scheduler_state: Callable[[SchedulerState], None] | None = None on_statement_complete: Callable[[StatementExecutionTelemetry], None] | None = None - on_migration_progress: Callable[[str], None] | None = None + on_lifecycle_progress: Callable[[str], None] | None = None @dataclass(frozen=True) diff --git a/src/sqlbuild/executor/maintained_pauses/__init__.py b/src/sqlbuild/executor/maintained_pauses/__init__.py new file mode 100644 index 0000000000..9aae34706c --- /dev/null +++ b/src/sqlbuild/executor/maintained_pauses/__init__.py @@ -0,0 +1 @@ +"""Maintained models a build holds paused, and the append-only lease that records it.""" diff --git a/src/sqlbuild/executor/maintained_pauses/_helpers/__init__.py b/src/sqlbuild/executor/maintained_pauses/_helpers/__init__.py new file mode 100644 index 0000000000..928816ab73 --- /dev/null +++ b/src/sqlbuild/executor/maintained_pauses/_helpers/__init__.py @@ -0,0 +1 @@ +"""Maintained pause helpers.""" diff --git a/src/sqlbuild/executor/maintained_pauses/_helpers/scope.py b/src/sqlbuild/executor/maintained_pauses/_helpers/scope.py new file mode 100644 index 0000000000..eff45d123d --- /dev/null +++ b/src/sqlbuild/executor/maintained_pauses/_helpers/scope.py @@ -0,0 +1,92 @@ +"""Which maintained models a build holds, upstream first, and which ones it refreshes by hand.""" + +from __future__ import annotations + +from sqlbuild._native import ( + CompiledObjectKey, + CompiledResourceType, + MaterializationType, + ModelPlanEntry, + PlanAction, + PlanOutput, +) + +_UNWRITTEN: frozenset[PlanAction] = frozenset( + {PlanAction.SKIP, PlanAction.KEEP_MAINTAINED, PlanAction.ALTER_MAINTAINED} +) + + +def held_names(*, plan: PlanOutput, pause_maintained: bool) -> tuple[str, ...]: + """Maintained models built, or reading what is written (every planned one when paused).""" + + names: set[str] = set() + entry: ModelPlanEntry + for entry in plan.model_entries: + maintained: bool = entry.materialization_type == MaterializationType.MAINTAINED + if maintained and (pause_maintained or entry.action not in _UNWRITTEN): + names.add(entry.name) + if entry.action not in _UNWRITTEN: + names.update(entry.maintained_readers) + return _upstream_first(names=frozenset(names), plan=plan) + + +def refreshed_for_readers(*, plan: PlanOutput) -> frozenset[str]: + """Maintained models that a model built later in this build reads, directly or not.""" + + entries: dict[str, ModelPlanEntry] = {entry.name: entry for entry in plan.model_entries} + readers: dict[str, tuple[str, ...]] = _model_edges(edges=plan.downstream_deps) + found: set[str] = set() + name: str + for name, entry in entries.items(): + if entry.materialization_type != MaterializationType.MAINTAINED: + continue + reached: set[str] = _reachable(start=name, edges=readers) + if any(_built_table(entry=entries.get(reader)) for reader in reached): + found.add(name) + return frozenset(found) + + +def _built_table(*, entry: ModelPlanEntry | None) -> bool: + return ( + entry is not None + and entry.materialization_type != MaterializationType.MAINTAINED + and entry.action != PlanAction.SKIP + ) + + +def _upstream_first(*, names: frozenset[str], plan: PlanOutput) -> tuple[str, ...]: + """A model upstream of another has strictly fewer held ancestors, so it sorts first.""" + + upstream: dict[str, tuple[str, ...]] = _model_edges(edges=plan.upstream_deps) + held_ancestors: dict[str, int] = { + name: len(_reachable(start=name, edges=upstream) & names) for name in names + } + return tuple(sorted(names, key=lambda name: (held_ancestors[name], name))) + + +def _model_edges( + *, edges: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]] +) -> dict[str, tuple[str, ...]]: + models: dict[str, tuple[str, ...]] = {} + key: CompiledObjectKey + others: tuple[CompiledObjectKey, ...] + for key, others in edges.items(): + if key.resource_type == CompiledResourceType.MODEL: + models[key.name] = _model_names(keys=others) + return models + + +def _model_names(*, keys: tuple[CompiledObjectKey, ...]) -> tuple[str, ...]: + return tuple(key.name for key in keys if key.resource_type == CompiledResourceType.MODEL) + + +def _reachable(*, start: str, edges: dict[str, tuple[str, ...]]) -> set[str]: + seen: set[str] = set() + pending: list[str] = list(edges.get(start, ())) + while pending: + name: str = pending.pop() + if name in seen: + continue + seen.add(name) + pending.extend(edges.get(name, ())) + return seen diff --git a/src/sqlbuild/executor/maintained_pauses/_helpers/sql.py b/src/sqlbuild/executor/maintained_pauses/_helpers/sql.py new file mode 100644 index 0000000000..1a83fb5548 --- /dev/null +++ b/src/sqlbuild/executor/maintained_pauses/_helpers/sql.py @@ -0,0 +1,158 @@ +"""Portable SQL for the append-only maintained pause table, timed by the warehouse clock.""" + +from __future__ import annotations + +import hashlib +import json +from datetime import datetime +from typing import Any + +from sqlbuild._native import ( + MAINTAINED_PAUSES_TABLE_NAME, + CompiledRelationLocation, + render_state_sql_literal, + render_state_table_create_sql, +) +from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter +from sqlbuild.errors.contracts.exceptions import ExecutorInputError +from sqlbuild.executor.maintained_pauses.constants import ( + COLUMN_EVENT_ID, + COLUMN_EVENT_TYPE, + COLUMN_RECORDED_AT, + COLUMN_RECORDED_BY, + COLUMN_RELATION_DATABASE, + COLUMN_RELATION_NAME, + COLUMN_RELATION_SCHEMA, + COLUMN_RUN_ID, + COLUMN_SCHEMA_VERSION, + MAINTAINED_PAUSE_COLUMN_TYPES, + MAINTAINED_PAUSE_COLUMNS, + MAINTAINED_PAUSE_REQUIRED_COLUMNS, + MAINTAINED_PAUSE_SCHEMA_VERSION, +) +from sqlbuild.executor.maintained_pauses.models import MaintainedPauseEvent, OpenHold +from sqlbuild.executor.maintained_pauses.types import MaintainedPauseEventType, StateLocation + + +def pause_table_name(*, adapter: BaseAdapter, location: StateLocation) -> str: + qualified: str | None = adapter.render_qualified_name( + database=location[0], schema=location[1], name=MAINTAINED_PAUSES_TABLE_NAME + ) + if qualified is None: + raise ExecutorInputError("the maintained pause table needs a schema") + return qualified + + +def setup_statements(*, adapter: BaseAdapter, location: StateLocation) -> tuple[str, ...]: + """Create the schema and the pause table when they are missing.""" + + return ( + *adapter.render_create_schema(database=location[0], schema=location[1]), + render_state_table_create_sql( + qualified_name=pause_table_name(adapter=adapter, location=location), + columns=MAINTAINED_PAUSE_COLUMNS, + column_types=MAINTAINED_PAUSE_COLUMN_TYPES, + required_columns=MAINTAINED_PAUSE_REQUIRED_COLUMNS, + render_framework_type=adapter.render_framework_type, + ), + ) + + +def insert_sql( + *, adapter: BaseAdapter, location: StateLocation, events: tuple[MaintainedPauseEvent, ...] +) -> str: + """Append every event in one statement, so a hand-over closes and opens holds together.""" + + now: str = adapter.render_current_utc_timestamp() + rows: str = " UNION ALL ".join( + f"SELECT {_row_literals(event=event)}, {now}" for event in events + ) + return ( + f"INSERT INTO {pause_table_name(adapter=adapter, location=location)} " + f"({', '.join(MAINTAINED_PAUSE_COLUMNS)}) {rows}" + ) + + +def open_holds_sql(*, adapter: BaseAdapter, location: StateLocation) -> str: + """Paused models nobody resumed, with when their run last wrote and the warehouse's now.""" + + table: str = pause_table_name(adapter=adapter, location=location) + now: str = adapter.render_current_utc_timestamp() + same_relation: str = " AND ".join( + f"COALESCE(r.{column}, '') = COALESCE(p.{column}, '')" + for column in (COLUMN_RELATION_DATABASE, COLUMN_RELATION_SCHEMA, COLUMN_RELATION_NAME) + ) + return ( + f"SELECT p.{COLUMN_RUN_ID}, p.{COLUMN_RELATION_DATABASE}, p.{COLUMN_RELATION_SCHEMA}, " + f"p.{COLUMN_RELATION_NAME}, CAST(h.last_seen AS TIMESTAMP), CAST({now} AS TIMESTAMP), " + f"CAST(p.{COLUMN_RECORDED_AT} AS TIMESTAMP) " + f"FROM {table} p JOIN (SELECT {COLUMN_RUN_ID}, MAX({COLUMN_RECORDED_AT}) AS last_seen " + f"FROM {table} WHERE {COLUMN_RECORDED_BY} = {COLUMN_RUN_ID} GROUP BY {COLUMN_RUN_ID}) h " + f"ON h.{COLUMN_RUN_ID} = p.{COLUMN_RUN_ID} " + f"WHERE p.{COLUMN_EVENT_TYPE} = '{MaintainedPauseEventType.PAUSED.value}' " + f"AND NOT EXISTS (SELECT 1 FROM {table} r " + f"WHERE r.{COLUMN_EVENT_TYPE} = '{MaintainedPauseEventType.RESUMED.value}' " + f"AND r.{COLUMN_RUN_ID} = p.{COLUMN_RUN_ID} AND {same_relation})" + ) + + +def open_holds(*, rows: tuple[tuple[Any, ...], ...]) -> tuple[OpenHold, ...]: + """The open holds of `open_holds_sql` rows.""" + + return tuple( + OpenHold( + run_id=str(row[0]), + relation=CompiledRelationLocation( + database=_optional(row[1]), + schema=_optional(row[2]), + name=str(row[3]), + qualified_name=None, + ), + idle_seconds=_seconds_between(start=row[4], end=row[5]), + paused_at=row[6].replace(tzinfo=None), + ) + for row in rows + ) + + +def _seconds_between(*, start: datetime, end: datetime) -> float: + return (end.replace(tzinfo=None) - start.replace(tzinfo=None)).total_seconds() + + +def _optional(value: object) -> str | None: + return None if value is None else str(value) + + +def _row_literals(*, event: MaintainedPauseEvent) -> str: + relation: CompiledRelationLocation | None = event.relation + values: dict[str, object | None] = { + COLUMN_EVENT_ID: _event_id(event=event), + COLUMN_SCHEMA_VERSION: MAINTAINED_PAUSE_SCHEMA_VERSION, + COLUMN_EVENT_TYPE: event.event_type.value, + COLUMN_RUN_ID: event.run_id, + COLUMN_RELATION_DATABASE: None if relation is None else relation.database, + COLUMN_RELATION_SCHEMA: None if relation is None else relation.schema, + COLUMN_RELATION_NAME: None if relation is None else relation.name, + COLUMN_RECORDED_BY: event.recorded_by, + } + return ", ".join( + render_state_sql_literal( + value=values[column], declared_type=MAINTAINED_PAUSE_COLUMN_TYPES[column] + ) + for column in MAINTAINED_PAUSE_COLUMNS + if column != COLUMN_RECORDED_AT + ) + + +def _event_id(*, event: MaintainedPauseEvent) -> str: + """Content-addressed: one pause and one resume per run and model, one ID per heartbeat.""" + + relation: CompiledRelationLocation | None = event.relation + identity: tuple[object, ...] = ( + event.event_type.value, + event.run_id, + None if relation is None else (relation.database, relation.schema, relation.name), + event.sequence, + ) + encoded: str = json.dumps(identity, separators=(",", ":"), ensure_ascii=True) + return hashlib.sha256(encoded.encode("utf-8")).hexdigest() diff --git a/src/sqlbuild/executor/maintained_pauses/classes/__init__.py b/src/sqlbuild/executor/maintained_pauses/classes/__init__.py new file mode 100644 index 0000000000..4dbf76c0aa --- /dev/null +++ b/src/sqlbuild/executor/maintained_pauses/classes/__init__.py @@ -0,0 +1 @@ +"""Maintained pause classes.""" diff --git a/src/sqlbuild/executor/maintained_pauses/classes/maintained_heartbeat.py b/src/sqlbuild/executor/maintained_pauses/classes/maintained_heartbeat.py new file mode 100644 index 0000000000..a689c16004 --- /dev/null +++ b/src/sqlbuild/executor/maintained_pauses/classes/maintained_heartbeat.py @@ -0,0 +1,64 @@ +"""A thread that tells other builds, through the warehouse, that a build still holds its models.""" + +from __future__ import annotations + +import threading +from collections.abc import Callable +from typing import Any + +from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter + + +class MaintainedHeartbeat: + """Write a heartbeat every interval on its own connection, which a busy build never blocks.""" + + def __init__( + self, + *, + adapter: BaseAdapter, + connection_config: dict[str, object], + write: Callable[..., None], + interval_seconds: float, + ) -> None: + self._adapter: BaseAdapter = adapter + self._connection_config: dict[str, object] = connection_config + self._write: Callable[..., None] = write + self._interval_seconds: float = interval_seconds + self._stopped: threading.Event = threading.Event() + self._thread: threading.Thread = threading.Thread( + target=self._beat, name="sqlbuild-maintained-heartbeat", daemon=True + ) + self.failure: Exception | None = None + + def start(self) -> None: + self._thread.start() + + def stop(self) -> None: + self._stopped.set() + self._thread.join() + + def _beat(self) -> None: + """Beat at once, then every interval, so no gap exceeds one interval plus a write.""" + + connection: Any = None + sequence: int = 0 + stopped: bool = False + while not stopped: + sequence += 1 + try: + if connection is None: + connection = self._adapter.connect(dict(self._connection_config)) + self._write(connection=connection, sequence=sequence) + except Exception as error: + self.failure = error + connection = self._closed(connection=connection) + stopped = self._stopped.wait(self._interval_seconds) + _ = self._closed(connection=connection) + + def _closed(self, *, connection: Any) -> None: + if connection is not None: + try: + self._adapter.close(connection) + except Exception as error: + self.failure = error + return None diff --git a/src/sqlbuild/executor/maintained_pauses/classes/maintained_hold.py b/src/sqlbuild/executor/maintained_pauses/classes/maintained_hold.py new file mode 100644 index 0000000000..9a5c14654b --- /dev/null +++ b/src/sqlbuild/executor/maintained_pauses/classes/maintained_hold.py @@ -0,0 +1,414 @@ +"""One build's hold on the maintained models it touches: pause, record, beat, then resume.""" + +from __future__ import annotations + +import threading +from collections.abc import Callable, Iterator +from contextlib import contextmanager +from typing import Any + +from sqlbuild._native import CompiledRelationLocation +from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter +from sqlbuild.adapter.contract.exceptions import AdapterUserError +from sqlbuild.adapter.contract.types import MaintainedListing, MaintainedOperation +from sqlbuild.executor.maintained_pauses._helpers.sql import ( + insert_sql, + open_holds, + open_holds_sql, + setup_statements, +) +from sqlbuild.executor.maintained_pauses.classes.maintained_heartbeat import ( + MaintainedHeartbeat, +) +from sqlbuild.executor.maintained_pauses.constants import ( + ACTIVE_STATE, + HELD_ELSEWHERE_CODE, + SUSPENDED_STATE, +) +from sqlbuild.executor.maintained_pauses.models import MaintainedPauseEvent, OpenHold +from sqlbuild.executor.maintained_pauses.types import MaintainedPauseEventType, StateLocation + +type _RelationKey = tuple[str | None, str | None, str] + + +class MaintainedHold: + """Pause the held models a build found active, record it, and resume only those at the end.""" + + def __init__( + self, + *, + adapter: BaseAdapter, + held: dict[str, CompiledRelationLocation], + refreshed: frozenset[str], + state_location: StateLocation | None, + settings: tuple[str, bool, float, bool], + heartbeat: Callable[[Callable[..., None]], MaintainedHeartbeat], + ) -> None: + self._adapter: BaseAdapter = adapter + self._held: dict[str, CompiledRelationLocation] = held + self._refreshed: frozenset[str] = refreshed + self._state_location: StateLocation | None = state_location + self._run_id: str + self._pause_maintained: bool + self._grace_seconds: float + self._ignore_grace: bool + self._run_id, self._pause_maintained, self._grace_seconds, self._ignore_grace = settings + self._new_heartbeat: Callable[[Callable[..., None]], MaintainedHeartbeat] = heartbeat + self._heartbeat: MaintainedHeartbeat | None = None + self._paused: list[str] = [] + self._left_paused: frozenset[str] = frozenset() + self._lock: threading.Lock = threading.Lock() + self._on_progress: Callable[[str], None] | None = None + + @contextmanager + def holding( + self, *, connection: Any, on_progress: Callable[[str], None] | None + ) -> Iterator[None]: + """Hold for the build; after a failure, resume without hiding its error.""" + + try: + self.acquire(connection=connection, on_progress=on_progress) + yield + except BaseException: + try: + self.release(connection=connection) + except Exception as error: + self._warn(f"could not resume the maintained models this build paused: {error}") + raise + self.release(connection=connection) + + def refreshes(self, *, name: str) -> bool: + """Whether a kept or altered model is refreshed by hand once its inputs are built.""" + + return self._pause_maintained or name in self._refreshed + + def acquire(self, *, connection: Any, on_progress: Callable[[str], None] | None) -> None: + """Take over expired holds, refuse live ones, then pause the held models found active.""" + + self._on_progress = on_progress + if self._pause_maintained: + self._suspend_active(connection=connection, names=tuple(self._held)) + return + location: StateLocation | None = self._state_location + if location is None: + return + for statement in setup_statements(adapter=self._adapter, location=location): + self._adapter.execute(connection=connection, sql=statement) + taken: tuple[OpenHold, ...] = self._take_over(connection=connection, location=location) + taken_keys: set[_RelationKey] = {_key(hold.relation) for hold in taken} + states: dict[str, str | None] = { + name: self._state(connection=connection, name=name) for name in self._held + } + self._paused = [ + name + for name in self._held + if states[name] == ACTIVE_STATE or _key(self._held[name]) in taken_keys + ] + self._left_paused = frozenset( + name + for name in self._held + if states[name] == SUSPENDED_STATE and name not in self._paused + ) + for name in sorted(self._left_paused): + self._warn(f"maintained model {name} was paused outside SQLBuild; it stays paused") + self._record( + connection=connection, + events=( + *( + self._event(event_type=MaintainedPauseEventType.RESUMED, hold=hold) + for hold in taken + ), + *( + self._own(event_type=MaintainedPauseEventType.PAUSED, name=name) + for name in self._paused + ), + ), + ) + self._yield_to_earlier(connection=connection, location=location) + self._suspend_active(connection=connection, names=tuple(self._paused), states=states) + self._heartbeat = self._new_heartbeat(self._write_heartbeat) + self._heartbeat.start() + + def after_create(self, *, connection: Any, name: str) -> None: + """Suspend a held model that CREATE OR REPLACE made active; resume it only if ours.""" + + if name not in self._held: + return + with self._lock: + newly_paused: bool = not ( + self._pause_maintained or name in self._paused or name in self._left_paused + ) + if newly_paused: + self._paused.append(name) + if newly_paused: + self._record( + connection=connection, + events=(self._own(event_type=MaintainedPauseEventType.PAUSED, name=name),), + ) + self._operate(connection=connection, name=name, operation=MaintainedOperation.SUSPEND) + + def release(self, *, connection: Any) -> None: + """Stop the heartbeat and resume what this build paused, upstream first.""" + + if self._heartbeat is not None: + self._heartbeat.stop() + self._report_heartbeat_failure(failure=self._heartbeat.failure) + order: list[str] = list(self._held) + still_held: frozenset[_RelationKey] | None = self._still_held(connection=connection) + resumed: list[str] = [] + failures: list[Exception] = [] + for name in sorted(self._paused, key=order.index): + if still_held is not None and _key(self._held[name]) not in still_held: + continue + try: + self._operate( + connection=connection, name=name, operation=MaintainedOperation.RESUME + ) + resumed.append(name) + except Exception as error: + failures.append(error) + self._record( + connection=connection, + events=tuple( + self._own(event_type=MaintainedPauseEventType.RESUMED, name=name) + for name in resumed + ), + ) + if failures: + raise failures[0] + + def _take_over(self, *, connection: Any, location: StateLocation) -> tuple[OpenHold, ...]: + """Resume expired holds on other models now; return the held models' holds to take over.""" + + rows: tuple[tuple[Any, ...], ...] = tuple( + self._adapter.query( + connection=connection, + sql=open_holds_sql(adapter=self._adapter, location=location), + limit=None, + ).rows + ) + held_keys: set[_RelationKey] = {_key(location) for location in self._held.values()} + holds: tuple[OpenHold, ...] = tuple( + hold for hold in open_holds(rows=rows) if hold.run_id != self._run_id + ) + live: tuple[OpenHold, ...] = tuple( + hold + for hold in holds + if hold.idle_seconds < self._grace_seconds and _key(hold.relation) in held_keys + ) + if live and not self._ignore_grace: + raise _held_elsewhere(holds=live) + expired: tuple[OpenHold, ...] = tuple( + hold for hold in holds if hold.idle_seconds >= self._grace_seconds + ) + elsewhere: tuple[OpenHold, ...] = tuple( + hold for hold in expired if _key(hold.relation) not in held_keys + ) + for hold in elsewhere: + self._resume_found(connection=connection, relation=hold.relation) + self._record( + connection=connection, + events=tuple( + self._event(event_type=MaintainedPauseEventType.RESUMED, hold=hold) + for hold in elsewhere + ), + ) + taken: tuple[OpenHold, ...] = (*live, *(h for h in expired if h not in elsewhere)) + self._report(holds=elsewhere, verb="Resumed") + self._report(holds=taken, verb="Took over") + return taken + + def _yield_to_earlier(self, *, connection: Any, location: StateLocation) -> None: + """Read the pauses back; if another live hold on a model came first, close ours and stop.""" + + mine: set[_RelationKey] = {_key(self._held[name]) for name in self._paused} + if not mine: + return + rows: tuple[tuple[Any, ...], ...] = tuple( + self._adapter.query( + connection=connection, + sql=open_holds_sql(adapter=self._adapter, location=location), + limit=None, + ).rows + ) + first: dict[_RelationKey, OpenHold] = {} + for hold in sorted(open_holds(rows=rows), key=lambda hold: (hold.paused_at, hold.run_id)): + if hold.idle_seconds < self._grace_seconds and _key(hold.relation) in mine: + first.setdefault(_key(hold.relation), hold) + winners: tuple[OpenHold, ...] = tuple( + hold for hold in first.values() if hold.run_id != self._run_id + ) + if not winners: + return + self._record( + connection=connection, + events=tuple( + self._own(event_type=MaintainedPauseEventType.RESUMED, name=name) + for name in self._paused + ), + ) + self._paused = [] + raise _held_elsewhere(holds=winners) + + def _still_held(self, *, connection: Any) -> frozenset[_RelationKey] | None: + """This run's holds nobody took over, or None when the pause table can't say.""" + + location: StateLocation | None = self._state_location + if location is None or self._pause_maintained: + return None + try: + rows: tuple[tuple[Any, ...], ...] = tuple( + self._adapter.query( + connection=connection, + sql=open_holds_sql(adapter=self._adapter, location=location), + limit=None, + ).rows + ) + except Exception as error: + self._warn(f"could not read which maintained models this build still holds: {error}") + return None + return frozenset( + _key(hold.relation) for hold in open_holds(rows=rows) if hold.run_id == self._run_id + ) + + def _resume_found(self, *, connection: Any, relation: CompiledRelationLocation) -> None: + if _scheduling_state(adapter=self._adapter, connection=connection, relation=relation) == ( + SUSPENDED_STATE + ): + self._run( + connection=connection, + destination=self._destination(relation), + operation=MaintainedOperation.RESUME, + ) + + def _suspend_active( + self, + *, + connection: Any, + names: tuple[str, ...], + states: dict[str, str | None] | None = None, + ) -> None: + for name in names: + state: str | None = ( + states[name] + if states is not None + else self._state(connection=connection, name=name) + ) + if state == ACTIVE_STATE: + self._operate( + connection=connection, name=name, operation=MaintainedOperation.SUSPEND + ) + + def _state(self, *, connection: Any, name: str) -> str | None: + return _scheduling_state( + adapter=self._adapter, connection=connection, relation=self._held[name] + ) + + def _operate(self, *, connection: Any, name: str, operation: MaintainedOperation) -> None: + self._run( + connection=connection, + destination=self._destination(self._held[name]), + operation=operation, + ) + + def _run(self, *, connection: Any, destination: str, operation: MaintainedOperation) -> None: + for statement in self._adapter.render_maintained_operation( + destination=destination, operation=operation + ): + self._adapter.execute(connection=connection, sql=statement) + + def _destination(self, location: CompiledRelationLocation) -> str: + return location.qualified_name or str( + self._adapter.render_qualified_name( + database=location.database, schema=location.schema, name=location.name + ) + ) + + def _record(self, *, connection: Any, events: tuple[MaintainedPauseEvent, ...]) -> None: + location: StateLocation | None = self._state_location + if not events or location is None: + return + self._adapter.execute( + connection=connection, + sql=insert_sql(adapter=self._adapter, location=location, events=events), + ) + + def _write_heartbeat(self, *, connection: Any, sequence: int) -> None: + self._record( + connection=connection, + events=( + MaintainedPauseEvent( + event_type=MaintainedPauseEventType.HEARTBEAT, + run_id=self._run_id, + recorded_by=self._run_id, + relation=None, + sequence=sequence, + ), + ), + ) + + def _own(self, *, event_type: MaintainedPauseEventType, name: str) -> MaintainedPauseEvent: + return MaintainedPauseEvent( + event_type=event_type, + run_id=self._run_id, + recorded_by=self._run_id, + relation=self._held[name], + ) + + def _event( + self, *, event_type: MaintainedPauseEventType, hold: OpenHold + ) -> MaintainedPauseEvent: + return MaintainedPauseEvent( + event_type=event_type, + run_id=hold.run_id, + recorded_by=self._run_id, + relation=hold.relation, + ) + + def _report(self, *, holds: tuple[OpenHold, ...], verb: str) -> None: + if self._on_progress is None: + return + for hold in holds: + self._on_progress( + f"{verb} maintained model {hold.relation.name}, paused by run {hold.run_id} " + f"whose heartbeat has been silent for {int(hold.idle_seconds)}s" + ) + + def _report_heartbeat_failure(self, *, failure: Exception | None) -> None: + if failure is not None: + self._warn(f"a maintained pause heartbeat failed: {failure}") + + def _warn(self, message: str) -> None: + if self._on_progress is not None: + self._on_progress(f"Warning: {message}") + + +def _scheduling_state( + *, adapter: BaseAdapter, connection: Any, relation: CompiledRelationLocation +) -> str | None: + listing: MaintainedListing | None = adapter.maintained_listing( + connection=connection, + database=relation.database, + schema=relation.schema, + name=relation.name, + ) + return None if listing is None else listing.scheduling_state + + +def _key(location: CompiledRelationLocation) -> _RelationKey: + return (location.database, location.schema, location.name) + + +def _held_elsewhere(*, holds: tuple[OpenHold, ...]) -> AdapterUserError: + first: OpenHold = holds[0] + names: str = ", ".join(sorted({hold.relation.name for hold in holds})) + return AdapterUserError( + f"run {first.run_id} holds maintained models this build needs ({names}); " + f"its last heartbeat was {int(first.idle_seconds)}s ago", + code=HELD_ELSEWHERE_CODE, + help=( + "wait for that build to finish, or for its hold to pass [maintained] " + "heartbeat_grace; with allow_ignore_grace = true, " + "sqb build --ignore-maintained-grace takes over at once" + ), + ) diff --git a/src/sqlbuild/executor/maintained_pauses/constants.py b/src/sqlbuild/executor/maintained_pauses/constants.py new file mode 100644 index 0000000000..22fd73f5fa --- /dev/null +++ b/src/sqlbuild/executor/maintained_pauses/constants.py @@ -0,0 +1,49 @@ +"""Maintained pause table columns, timings and error codes.""" + +from sqlbuild._native import StateSqlValueType + +MAINTAINED_PAUSE_SCHEMA_VERSION: int = 1 +HEARTBEAT_SECONDS: float = 60.0 +ACTIVE_STATE: str = "ACTIVE" +SUSPENDED_STATE: str = "SUSPENDED" +HELD_ELSEWHERE_CODE: str = "A718" +IGNORE_GRACE_REFUSED_CODE: str = "A719" + +COLUMN_EVENT_ID: str = "event_id" +COLUMN_SCHEMA_VERSION: str = "schema_version" +COLUMN_EVENT_TYPE: str = "event_type" +COLUMN_RUN_ID: str = "run_id" +COLUMN_RELATION_DATABASE: str = "relation_database" +COLUMN_RELATION_SCHEMA: str = "relation_schema" +COLUMN_RELATION_NAME: str = "relation_name" +COLUMN_RECORDED_BY: str = "recorded_by" +COLUMN_RECORDED_AT: str = "recorded_at" + +MAINTAINED_PAUSE_COLUMNS: tuple[str, ...] = ( + COLUMN_EVENT_ID, + COLUMN_SCHEMA_VERSION, + COLUMN_EVENT_TYPE, + COLUMN_RUN_ID, + COLUMN_RELATION_DATABASE, + COLUMN_RELATION_SCHEMA, + COLUMN_RELATION_NAME, + COLUMN_RECORDED_BY, + COLUMN_RECORDED_AT, +) + +MAINTAINED_PAUSE_REQUIRED_COLUMNS: frozenset[str] = frozenset( + { + COLUMN_EVENT_ID, + COLUMN_SCHEMA_VERSION, + COLUMN_EVENT_TYPE, + COLUMN_RUN_ID, + COLUMN_RECORDED_BY, + COLUMN_RECORDED_AT, + } +) + +MAINTAINED_PAUSE_COLUMN_TYPES: dict[str, StateSqlValueType] = { + **{column: StateSqlValueType.STRING for column in MAINTAINED_PAUSE_COLUMNS}, + COLUMN_SCHEMA_VERSION: StateSqlValueType.INTEGER, + COLUMN_RECORDED_AT: StateSqlValueType.TEXT_TIMESTAMP, +} diff --git a/src/sqlbuild/executor/maintained_pauses/main/__init__.py b/src/sqlbuild/executor/maintained_pauses/main/__init__.py new file mode 100644 index 0000000000..36ef4cff68 --- /dev/null +++ b/src/sqlbuild/executor/maintained_pauses/main/__init__.py @@ -0,0 +1 @@ +"""Maintained pause entrypoints.""" diff --git a/src/sqlbuild/executor/maintained_pauses/main/hold_maintained_models.py b/src/sqlbuild/executor/maintained_pauses/main/hold_maintained_models.py new file mode 100644 index 0000000000..dc8045be4d --- /dev/null +++ b/src/sqlbuild/executor/maintained_pauses/main/hold_maintained_models.py @@ -0,0 +1,71 @@ +"""The hold a build keeps on the maintained models it touches, or None when it touches none.""" + +from __future__ import annotations + +from sqlbuild._native import CompiledRelationLocation, PlanOutput, TargetSettings +from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter +from sqlbuild.adapter.contract.exceptions import AdapterUserError +from sqlbuild.executor.build.models import BuildRuntimeParams +from sqlbuild.executor.maintained_pauses._helpers.scope import held_names, refreshed_for_readers +from sqlbuild.executor.maintained_pauses.classes.maintained_heartbeat import ( + MaintainedHeartbeat, +) +from sqlbuild.executor.maintained_pauses.classes.maintained_hold import MaintainedHold +from sqlbuild.executor.maintained_pauses.constants import ( + HEARTBEAT_SECONDS, + IGNORE_GRACE_REFUSED_CODE, +) +from sqlbuild.executor.maintained_pauses.types import StateLocation + + +def hold_maintained_models( + *, + plan: PlanOutput, + adapter: BaseAdapter, + runtime: BuildRuntimeParams, + connection_config: dict[str, object], + heartbeat_seconds: float = HEARTBEAT_SECONDS, +) -> MaintainedHold | None: + """The build's hold, unacquired; `--ignore-maintained-grace` needs `allow_ignore_grace`.""" + + if runtime.ignore_maintained_grace and not runtime.allow_ignore_maintained_grace: + raise AdapterUserError( + "--ignore-maintained-grace needs [maintained] allow_ignore_grace = true", + code=IGNORE_GRACE_REFUSED_CODE, + help="set allow_ignore_grace = true under [maintained] in sqlbuild_project.toml", + ) + names: tuple[str, ...] = held_names(plan=plan, pause_maintained=runtime.pause_maintained) + refreshed: frozenset[str] = refreshed_for_readers(plan=plan) + if not names and not refreshed: + return None + held: dict[str, CompiledRelationLocation] = {name: plan.model_locations[name] for name in names} + return MaintainedHold( + adapter=adapter, + held=held, + refreshed=refreshed, + state_location=_state_location(runtime=runtime, held=held) if held else None, + settings=( + runtime.run_id, + runtime.pause_maintained, + float(runtime.maintained_heartbeat_grace_seconds), + runtime.ignore_maintained_grace, + ), + heartbeat=lambda write: MaintainedHeartbeat( + adapter=adapter, + connection_config=connection_config, + write=write, + interval_seconds=heartbeat_seconds, + ), + ) + + +def _state_location( + *, runtime: BuildRuntimeParams, held: dict[str, CompiledRelationLocation] +) -> StateLocation: + """The target's schema, next to the runs table; the first held model's for a preserve schema.""" + + target: TargetSettings | None = runtime.target_config + if target is not None and target.schema is not None and not target.schema_preserved: + return (target.database, target.schema) + first: CompiledRelationLocation = next(iter(held.values())) + return (first.database, str(first.schema)) diff --git a/src/sqlbuild/executor/maintained_pauses/models.py b/src/sqlbuild/executor/maintained_pauses/models.py new file mode 100644 index 0000000000..f76cca16b9 --- /dev/null +++ b/src/sqlbuild/executor/maintained_pauses/models.py @@ -0,0 +1,30 @@ +"""Records of the maintained pause lease.""" + +from __future__ import annotations + +from dataclasses import dataclass +from datetime import datetime + +from sqlbuild._native import CompiledRelationLocation +from sqlbuild.executor.maintained_pauses.types import MaintainedPauseEventType + + +@dataclass(frozen=True) +class MaintainedPauseEvent: + """One row to append: a pause, a resume, or a heartbeat of the run holding models.""" + + event_type: MaintainedPauseEventType + run_id: str + recorded_by: str + relation: CompiledRelationLocation | None + sequence: int = 0 + + +@dataclass(frozen=True) +class OpenHold: + """A model a run paused and nobody resumed, when it paused, and how long the run is silent.""" + + run_id: str + relation: CompiledRelationLocation + idle_seconds: float + paused_at: datetime diff --git a/src/sqlbuild/executor/maintained_pauses/types.py b/src/sqlbuild/executor/maintained_pauses/types.py new file mode 100644 index 0000000000..c199bdd358 --- /dev/null +++ b/src/sqlbuild/executor/maintained_pauses/types.py @@ -0,0 +1,15 @@ +"""Maintained pause event types.""" + +from __future__ import annotations + +from enum import StrEnum + +type StateLocation = tuple[str | None, str] + + +class MaintainedPauseEventType(StrEnum): + """What one row of the pause table records.""" + + PAUSED = "paused" + RESUMED = "resumed" + HEARTBEAT = "heartbeat" diff --git a/src/sqlbuild/executor/run/_helpers/execution/table_config.py b/src/sqlbuild/executor/run/_helpers/execution/table_config.py index 7f71b3f645..ceeee3e044 100644 --- a/src/sqlbuild/executor/run/_helpers/execution/table_config.py +++ b/src/sqlbuild/executor/run/_helpers/execution/table_config.py @@ -17,10 +17,11 @@ def model_output_config(entry: ModelPlanEntry) -> dict[str, object]: def model_table_config(entry: ModelPlanEntry) -> dict[str, object]: - """The model's table type, physical layout and expected output columns.""" + """The model's table type, physical layout, change tracking and expected output columns.""" return { "table_type": entry.table_type, + "change_tracking": bool(entry.maintained_readers), "physical_storage": entry.physical_storage, **model_output_config(entry), } diff --git a/src/sqlbuild/executor/run/_helpers/materializations/incremental.py b/src/sqlbuild/executor/run/_helpers/materializations/incremental.py index 01b9fa6f56..700add7c56 100644 --- a/src/sqlbuild/executor/run/_helpers/materializations/incremental.py +++ b/src/sqlbuild/executor/run/_helpers/materializations/incremental.py @@ -689,7 +689,7 @@ def _prepare_incremental_delta_types( staging_schema=inputs.target_schema, staging_table=inputs.delta_table, declared_columns=tuple(desired_columns), - table_type=TableType.TRANSIENT, + table_config={"table_type": TableType.TRANSIENT, "physical_storage": {}}, statement_recorder=statement_recorder, ) diff --git a/src/sqlbuild/executor/run/_helpers/materializations/maintained.py b/src/sqlbuild/executor/run/_helpers/materializations/maintained.py new file mode 100644 index 0000000000..55821b6928 --- /dev/null +++ b/src/sqlbuild/executor/run/_helpers/materializations/maintained.py @@ -0,0 +1,196 @@ +"""Maintained model lifecycle: the warehouse refreshes the model; a build creates or alters it.""" + +from __future__ import annotations + +from collections.abc import Callable +from typing import Any + +from sqlbuild._native import ( + CompiledRelationLocation, + ModelPlanEntry, + PlanAction, + error_code, + error_message, + fit_auxiliary_relation_name, +) +from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter +from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder +from sqlbuild.adapter.contract.exceptions import AdapterUserError +from sqlbuild.adapter.contract.types import MaintainedListing, MaintainedOperation, RelationType +from sqlbuild.adapter.relations.main.cached_relation_type import cached_relation_type +from sqlbuild.adapter.relations.main.resolve_qualified_name_parts import ( + resolve_qualified_name_parts, +) +from sqlbuild.executor.maintained_pauses.classes.maintained_hold import MaintainedHold +from sqlbuild.executor.run._helpers.materializations.view import execute_in_place_entry +from sqlbuild.executor.run.constants import REPLACED_RELATION_SUFFIX, RUN_STAGING_FAILED_CODE +from sqlbuild.executor.run.models import ModelExecutionResult, ModelMaterializationContext + +_NO_WAREHOUSE_CODE: str = "A717" + + +def execute_maintained_entry(*, context: ModelMaterializationContext) -> ModelExecutionResult: + """Create, alter or keep one maintained model, refresh a kept one when asked, then audit it.""" + + def materialize(*, target_qualified: str, statement_recorder: StatementRecorder) -> None: + entry: ModelPlanEntry = context.entry + hold: MaintainedHold | None = context.maintained_hold + if entry.action == PlanAction.CREATE_MAINTAINED: + _create(context=context, destination=target_qualified, recorder=statement_recorder) + if hold is not None: + hold.after_create(connection=context.connection, name=entry.name) + return + if entry.action == PlanAction.ALTER_MAINTAINED: + _run( + adapter=context.adapter, + connection=context.connection, + statements=context.adapter.render_alter_maintained( + destination=target_qualified, + target_lag=str(entry.target_lag), + warehouse=entry.refresh_warehouse, + ), + recorder=statement_recorder, + ) + if entry.refresh_on_build or (hold is not None and hold.refreshes(name=entry.name)): + _run( + adapter=context.adapter, + connection=context.connection, + statements=context.adapter.render_maintained_operation( + destination=target_qualified, operation=MaintainedOperation.REFRESH + ), + recorder=statement_recorder, + ) + + return execute_in_place_entry( + context=context, + materialize=materialize, + action_name=str(context.entry.action), + built="the maintained model's refresh", + ) + + +def _create( + *, context: ModelMaterializationContext, destination: str, recorder: StatementRecorder +) -> None: + """Create or replace; a table or view the model was before is swapped out, never a DT.""" + + entry: ModelPlanEntry = context.entry + adapter: BaseAdapter = context.adapter + connection: Any = context.connection + warehouse: str | None = entry.refresh_warehouse or adapter.session_warehouse( + connection=connection + ) + if warehouse is None: + raise AdapterUserError( + f"maintained model '{entry.name}' has no warehouse to refresh it", + code=_NO_WAREHOUSE_CODE, + help="set refresh_warehouse on the model, or a warehouse on the target's connection", + ) + statements: tuple[str, ...] = adapter.render_create_maintained( + destination=destination, + sql=entry.resolved_sql, + target_lag=str(entry.target_lag), + refresh_mode=str(entry.refresh_mode), + warehouse=warehouse, + ) + try: + _run(adapter=adapter, connection=connection, statements=statements, recorder=recorder) + except Exception as error: + former: RelationType | None = _former_relation( + adapter=adapter, connection=connection, entry=entry + ) + if former is None: + raise error + _replace_former( + context=context, + destination=destination, + statements=statements, + former=former, + recorder=recorder, + ) + + +def _replace_former( + *, + context: ModelMaterializationContext, + destination: str, + statements: tuple[str, ...], + former: RelationType, + recorder: StatementRecorder, +) -> None: + """Move the former table or view aside, create, then drop it; a failed create restores it.""" + + adapter: BaseAdapter = context.adapter + connection: Any = context.connection + location: CompiledRelationLocation = context.entry.destination + aside: str = resolve_qualified_name_parts( + adapter=adapter, + database=location.database, + schema=location.schema, + name=fit_auxiliary_relation_name( + base_name=location.name, + suffix=REPLACED_RELATION_SUFFIX, + identifier_limit=adapter.maximum_identifier_length(), + ), + ) + view: bool = former == RelationType.VIEW + rename: Callable[..., None] = adapter.rename_view if view else adapter.rename + drop: Callable[..., None] = adapter.drop_view if view else adapter.drop + rename( + connection=connection, origin=destination, destination=aside, statement_recorder=recorder + ) + try: + _run(adapter=adapter, connection=connection, statements=statements, recorder=recorder) + except Exception as create_error: + try: + rename( + connection=connection, + origin=aside, + destination=destination, + statement_recorder=recorder, + ) + except Exception as restore_error: + raise AdapterUserError( + f"{error_message(create_error)}; restoring the former {former.value} " + f"also failed, so it is left at {aside}: {error_message(restore_error)}", + code=error_code(error=create_error, fallback_code=RUN_STAGING_FAILED_CODE), + help=f"rename {aside} back to {destination}, then build the model again", + ) from create_error + raise + drop(connection=connection, destination=aside, if_exists=True, statement_recorder=recorder) + + +def _former_relation( + *, adapter: BaseAdapter, connection: Any, entry: ModelPlanEntry +) -> RelationType | None: + """The type of a relation at the destination the warehouse does not list as maintained.""" + + location: CompiledRelationLocation = entry.destination + existing: RelationType | None = cached_relation_type( + adapter=adapter, + connection=connection, + database=location.database, + schema=location.schema, + name=location.name, + ) + if existing is None: + return None + listed: MaintainedListing | None = adapter.maintained_listing( + connection=connection, + database=location.database, + schema=location.schema, + name=location.name, + ) + return existing if listed is None else None + + +def _run( + *, + adapter: BaseAdapter, + connection: Any, + statements: tuple[str, ...], + recorder: StatementRecorder, +) -> None: + recorder.record_many(statements) + for statement in statements: + adapter.execute(connection=connection, sql=statement) diff --git a/src/sqlbuild/executor/run/_helpers/materializations/microbatch.py b/src/sqlbuild/executor/run/_helpers/materializations/microbatch.py index d6f3a5e9ce..d3389b7b36 100644 --- a/src/sqlbuild/executor/run/_helpers/materializations/microbatch.py +++ b/src/sqlbuild/executor/run/_helpers/materializations/microbatch.py @@ -2957,7 +2957,7 @@ def _enforce_microbatch_types( staging_schema=targets.target_schema, staging_table=targets.delta_table, declared_columns=declared_columns, - table_type=TableType.TRANSIENT, + table_config={"table_type": TableType.TRANSIENT, "physical_storage": {}}, statement_recorder=state.statement_recorder, ) except Exception as exc: @@ -3057,6 +3057,7 @@ def _apply_microbatch_dml( config={ "table_type": context.entry.table_type, "physical_storage": context.entry.physical_storage, + "change_tracking": bool(context.entry.maintained_readers), }, statement_recorder=state.statement_recorder, ) diff --git a/src/sqlbuild/executor/run/_helpers/materializations/view.py b/src/sqlbuild/executor/run/_helpers/materializations/view.py index 1ab396dd2c..b266b0b004 100644 --- a/src/sqlbuild/executor/run/_helpers/materializations/view.py +++ b/src/sqlbuild/executor/run/_helpers/materializations/view.py @@ -2,7 +2,7 @@ from __future__ import annotations -from collections.abc import Mapping +from collections.abc import Callable, Mapping from typing import Any from sqlbuild._native import ( @@ -45,6 +45,29 @@ def execute_view_entry( ) -> ModelExecutionResult: """Execute one view model through its full materialization lifecycle.""" + def materialize(*, target_qualified: str, statement_recorder: StatementRecorder) -> None: + _create_view_replacing_table( + adapter=context.adapter, + connection=context.connection, + entry=context.entry, + target_qualified=target_qualified, + statement_recorder=statement_recorder, + ) + + return execute_in_place_entry( + context=context, materialize=materialize, action_name="create_view", built="view creation" + ) + + +def execute_in_place_entry( + *, + context: ModelMaterializationContext, + materialize: Callable[..., None], + action_name: str, + built: str, +) -> ModelExecutionResult: + """Hooks, schema, `materialize` on the destination, final audits and the fingerprint.""" + entry: ModelPlanEntry = context.entry adapter: BaseAdapter = context.adapter connection: Any = context.connection @@ -112,14 +135,8 @@ def execute_view_entry( schema=target_schema, statement_recorder=statement_recorder, ) - with diagnostics_context(sqlbuild_phase="materialize", sqlbuild_action_name="create_view"): - _create_view_replacing_table( - adapter=adapter, - connection=connection, - entry=entry, - target_qualified=target_qualified, - statement_recorder=statement_recorder, - ) + with diagnostics_context(sqlbuild_phase="materialize", sqlbuild_action_name=action_name): + materialize(target_qualified=target_qualified, statement_recorder=statement_recorder) except Exception as exc: return build_failed_result( entry=entry, @@ -154,8 +171,7 @@ def execute_view_entry( entry=entry, phase=ExecutionPhase.AUDIT, error=( - f"final audit for '{entry.name}' failed after view creation " - "with severity level: error" + f"final audit for '{entry.name}' failed after {built} with severity level: error" ), promoted_relation=target_qualified, warnings=warnings, diff --git a/src/sqlbuild/executor/run/_helpers/validation/type_enforcement.py b/src/sqlbuild/executor/run/_helpers/validation/type_enforcement.py index 09649f37a5..9da1bd78c0 100644 --- a/src/sqlbuild/executor/run/_helpers/validation/type_enforcement.py +++ b/src/sqlbuild/executor/run/_helpers/validation/type_enforcement.py @@ -2,6 +2,7 @@ from __future__ import annotations +from collections.abc import Mapping from typing import Any from sqlbuild._native import fit_auxiliary_relation_name @@ -26,9 +27,8 @@ def enforce_types_staged( staging_schema: str | None, staging_table: str, declared_columns: tuple[ColumnInfo, ...], - table_type: str, + table_config: Mapping[str, object], statement_recorder: StatementRecorder, - physical_storage: dict[str, object] | None = None, ) -> None: """Inspect staging columns and rebuild with casts for declared types.""" @@ -76,11 +76,7 @@ def enforce_types_staged( connection=connection, destination=enforced_qualified, sql=f"SELECT {projection_sql} FROM {staging_qualified}", - config={ - "table_type": table_type, - "column_types": declared_map, - "physical_storage": physical_storage or {}, - }, + config={**table_config, "column_types": declared_map}, statement_recorder=statement_recorder, ) adapter.drop_table( diff --git a/src/sqlbuild/executor/run/constants.py b/src/sqlbuild/executor/run/constants.py index d5486ad907..397bb1587e 100644 --- a/src/sqlbuild/executor/run/constants.py +++ b/src/sqlbuild/executor/run/constants.py @@ -21,6 +21,7 @@ SNAPSHOT_ALL_CHECK_COLUMNS: tuple[str, ...] = ("*",) STAGING_RELATION_SUFFIX: str = "__staging" +REPLACED_RELATION_SUFFIX: str = "__replaced" TYPE_ENFORCEMENT_RELATION_SUFFIX: str = "__enforced" DELTA_RELATION_SUFFIX: str = "__delta" SNAPSHOT_DELTA_RELATION_SUFFIX: str = "__snapshot_delta" diff --git a/src/sqlbuild/executor/run/main/_execute.py b/src/sqlbuild/executor/run/main/_execute.py index 7089ac50e8..1d8584e21b 100644 --- a/src/sqlbuild/executor/run/main/_execute.py +++ b/src/sqlbuild/executor/run/main/_execute.py @@ -44,6 +44,9 @@ from sqlbuild.executor.run._helpers.materializations.incremental import ( execute_incremental_entry as execute_incremental_entry, ) +from sqlbuild.executor.run._helpers.materializations.maintained import ( + execute_maintained_entry as execute_maintained_entry, +) from sqlbuild.executor.run._helpers.materializations.microbatch import ( execute_microbatch_entry as execute_microbatch_entry, ) @@ -243,8 +246,11 @@ def _staged_lifecycle( staging_schema=targets.target_schema, staging_table=targets.staging_table, declared_columns=declared_columns, - table_type=entry.table_type, - physical_storage=entry.physical_storage, + table_config={ + "table_type": entry.table_type, + "physical_storage": entry.physical_storage, + "change_tracking": bool(entry.maintained_readers), + }, statement_recorder=statement_recorder, ) except Exception as exc: diff --git a/src/sqlbuild/executor/run/models.py b/src/sqlbuild/executor/run/models.py index 674a5fba62..84bec90f9f 100644 --- a/src/sqlbuild/executor/run/models.py +++ b/src/sqlbuild/executor/run/models.py @@ -40,6 +40,7 @@ from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.auditing.models import AuditExecutionResult from sqlbuild.executor.custom.models import MaterializationResult +from sqlbuild.executor.maintained_pauses.classes.maintained_hold import MaintainedHold from sqlbuild.executor.python_nodes.classes.runtime_relation_guard import RuntimeRelationGuard from sqlbuild.executor.python_nodes.types import PythonIdentityRecorder from sqlbuild.executor.run.classes.fingerprint_writes import FingerprintWrites @@ -333,6 +334,7 @@ class ModelMaterializationContext: enforce_explicit_references: bool = False bound_views: BoundViewGuard = field(default_factory=BoundViewGuard) fingerprint_writes: FingerprintWrites | None = None + maintained_hold: MaintainedHold | None = None @dataclass(frozen=True, kw_only=True, init=False) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py index ba09c7b216..639178b99d 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py @@ -1461,3 +1461,14 @@ class JobGroupCyclesCase: description: str fixed_group_toml: str expected_messages: tuple[str, ...] + + +@dataclass(frozen=True) +class MaintainedCompileE2ETestCase: + """A project with maintained models, its exit code and text its compile output contains.""" + + description: str + adapter: str + daily_totals_header: str + expected_exit_code: int + expected_fragments: tuple[str, ...] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_maintained_models.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_maintained_models.py new file mode 100644 index 0000000000..da2449debf --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_maintained_models.py @@ -0,0 +1,115 @@ +"""E2E coverage of maintained models: which adapters and headers compile, and the errors they get.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import ( + MaintainedCompileE2ETestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import prepare_inline_project, run_sqb + +_PROJECT_TOML: dict[str, str] = { + "snowflake": ( + 'name = "shop"\nadapter = "snowflake"\n\n[connection]\naccount = "example"\n' + 'user = "builder"\n\n[defaults]\ndatabase = "ANALYTICS"\nschema = "MARTS"\n\n' + "[scopes]\nenforce_placement = false\n" + ), + "duckdb": ( + 'name = "shop"\nadapter = "duckdb"\n\n[connection]\npath = "shop.duckdb"\n\n' + "[scopes]\nenforce_placement = false\n" + ), +} +_ORDERS: str = ( + "MODEL (description 'Orders.', materialized table);\nSELECT 1 AS order_id, 10 AS amount\n" +) +_CLEAN_ORDERS: str = ( + "MODEL (description 'Orders with an amount.', materialized maintained);\n" + 'SELECT order_id, amount FROM __ref("orders") WHERE amount > 0\n' +) +_DAILY_TOTALS_QUERY: str = 'SELECT SUM(amount) AS total FROM __ref("clean_orders")\n' + + +@pytest.mark.parametrize( + "test_case", + [ + MaintainedCompileE2ETestCase( + description="a chain refreshed by the lag of its last model compiles on Snowflake", + adapter="snowflake", + daily_totals_header="materialized maintained, refresh (lag 1h), refresh_mode full", + expected_exit_code=0, + expected_fragments=("Project compiled 3 models",), + ), + MaintainedCompileE2ETestCase( + description="maintained models are refused on DuckDB", + adapter="duckdb", + daily_totals_header="materialized maintained, refresh (lag 1h)", + expected_exit_code=1, + expected_fragments=( + "[A013]", + "materialized maintained is not supported on the duckdb adapter; " + "adapters that support it: snowflake", + ), + ), + MaintainedCompileE2ETestCase( + description="materialized_view on Snowflake points to maintained", + adapter="snowflake", + daily_totals_header="materialized materialized_view", + expected_exit_code=1, + expected_fragments=( + "[A014]", + "Snowflake has no materialized_view materialization; use materialized " + "maintained, which builds a dynamic table", + ), + ), + MaintainedCompileE2ETestCase( + description="a lag below Snowflake's minimum", + adapter="snowflake", + daily_totals_header="materialized maintained, refresh (lag 30s)", + expected_exit_code=1, + expected_fragments=( + "[A015]", + "refresh lag on Snowflake must be at least 1m; got '30s'", + ), + ), + MaintainedCompileE2ETestCase( + description="a last maintained model with no refresh would never refresh", + adapter="snowflake", + daily_totals_header="materialized maintained", + expected_exit_code=1, + expected_fragments=( + "[A016]", + "maintained model 'daily_totals' has no refresh and no maintained model reads " + "it, so Snowflake would never refresh it; add refresh (lag 1h)", + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_maintained_models_when_compiling_then_adapter_rules_apply( + test_case: MaintainedCompileE2ETestCase, tmp_path: Path +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="shop", + repo_files={ + "sqlbuild_project.toml": _PROJECT_TOML[test_case.adapter], + "models/orders.sql": _ORDERS, + "models/clean_orders.sql": _CLEAN_ORDERS, + "models/daily_totals.sql": ( + f"MODEL (description 'Daily totals.', {test_case.daily_totals_header});\n" + f"{_DAILY_TOTALS_QUERY}" + ), + }, + ) + + compiled: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "compile"), project_dir=project_dir + ) + + output: str = compiled.stdout + compiled.stderr + assert compiled.returncode == test_case.expected_exit_code, output + assert all(map(output.__contains__, test_case.expected_fragments)), output diff --git a/tests/goldens/compiler/failure/target-unknown-key.json b/tests/goldens/compiler/failure/target-unknown-key.json index 18a234ac3f..9d222e2426 100644 --- a/tests/goldens/compiler/failure/target-unknown-key.json +++ b/tests/goldens/compiler/failure/target-unknown-key.json @@ -8,7 +8,7 @@ "phase": "compile", "severity": "error", "code": "D001", - "message": "/sqlbuild_project.toml targets.ci contains unknown key(s): warehouse_size. Allowed keys: catalog_connection, clone, compile_cache, concurrency, connection, database, default_table_type, defer_clone_from, defer_sources_to, diff, execution_limits, ignore_audit_failures, loader_schema, missing_migration_origin, owns_time_travel_retention_namespace, plan_from, pools, recall, schema, table_type_downgrade, time_travel_retention, time_travel_retention_decrease, vars, warehouses", + "message": "/sqlbuild_project.toml targets.ci contains unknown key(s): warehouse_size. Allowed keys: catalog_connection, clone, compile_cache, concurrency, connection, database, default_table_type, defer_clone_from, defer_sources_to, diff, execution_limits, ignore_audit_failures, loader_schema, missing_migration_origin, owns_time_travel_retention_namespace, pause_maintained, plan_from, pools, recall, schema, table_type_downgrade, time_travel_retention, time_travel_retention_decrease, vars, warehouses", "help": "did you mean 'warehouses'?" } ] diff --git a/tests/integration/src/sqlbuild/adapters/sqlite/helpers.py b/tests/integration/src/sqlbuild/adapters/sqlite/helpers.py index 0584f027d5..2188e2a42c 100644 --- a/tests/integration/src/sqlbuild/adapters/sqlite/helpers.py +++ b/tests/integration/src/sqlbuild/adapters/sqlite/helpers.py @@ -54,7 +54,7 @@ def stage_contract_relation( ColumnInfo("payment_id", "INTEGER"), ColumnInfo("amount", "DECIMAL(10,2)"), ), - table_type="table", + table_config={"table_type": "table", "physical_storage": {}}, statement_recorder=recorder(), ) diff --git a/tests/integration/src/sqlbuild/compiler/planner/main/_test_types.py b/tests/integration/src/sqlbuild/compiler/planner/main/_test_types.py index 7c19d1c40c..ac3e1abfbe 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/main/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/planner/main/_test_types.py @@ -7,6 +7,7 @@ PlanReason, WarningSeverity, ) +from sqlbuild.adapter.contract.types import MaintainedListing from sqlbuild.compiler.compile.types import FunctionLanguage @@ -205,3 +206,24 @@ class SyncedScanTestCase: last_ddl_after_scan_start: timedelta expected_live_describe: bool location_clock_ahead: timedelta = timedelta() + + +@dataclass(frozen=True) +class MaintainedRefreshPlanTestCase: + description: str + listed_settings: MaintainedListing | None + upstream_query: str + expected_upstream_action: PlanAction + expected_action: PlanAction + expected_reason: PlanReason + expected_reinitialised_by: str | None + expected_read_names: tuple[str, ...] + expected_paused: bool = False + + +@dataclass(frozen=True) +class ChangeTrackingPlanTestCase: + description: str + select: tuple[str, ...] + expected_maintained_readers: dict[str, tuple[str, ...]] + expected_refresh_on_build: dict[str, bool] diff --git a/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py b/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py index 9c63c10938..b13f754f11 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py @@ -56,7 +56,12 @@ run_compile_pipeline, write_fingerprint, ) -from sqlbuild.adapter.contract.models import ColumnInfo, RelationInfo, RelationReadProbe +from sqlbuild.adapter.contract.models import ( + ColumnInfo, + RelationInfo, + RelationReadProbe, +) +from sqlbuild.adapter.contract.types import MaintainedListing from sqlbuild.adapter.relations.classes.inspection_catalog import InspectionCatalog from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.types import ( @@ -827,3 +832,90 @@ def planned_entries(result: CompilePipelineResult) -> list[tuple[str, str]]: """Each planned model's name and reason, in name order.""" return sorted((entry.name, str(entry.reason)) for entry in result.plan_output.model_entries) + + +CLEAN_ORDERS_QUERY: str = "SELECT 1 AS order_id" +_MAINTAINED: dict[str, object] = { + "materialized": "maintained", + "refresh": {"lag": "1h"}, + "refresh_warehouse": "transform_wh", +} + + +class ListingDuckDbAdapter(DuckDbAdapter): + """DuckDB that lists every dynamic table the same way, as Snowflake's SHOW would.""" + + def __init__(self, listed: MaintainedListing | None) -> None: + super().__init__() + self.listed: MaintainedListing | None = listed + self.read_names: list[str] = [] + + def maintained_listing( + self, *, connection: Any, database: str | None, schema: str | None, name: str + ) -> MaintainedListing | None: + del connection, database, schema + self.read_names.append(name) + return self.listed + + +def maintained_project_case(upstream_query: str) -> BuildExecutionPlanTestCase: + """Clean orders and the daily totals maintained from it, both refreshed within an hour.""" + + return BuildExecutionPlanTestCase( + description="clean orders and the daily totals maintained from it", + setup_sql=(), + model_locations={"clean_orders": "staging", "daily_totals": "staging"}, + model_configs={"clean_orders": _MAINTAINED, "daily_totals": _MAINTAINED}, + model_queries={ + "clean_orders": upstream_query, + "daily_totals": "SELECT count(*) AS orders FROM staging.clean_orders", + }, + model_deps={"daily_totals": ("clean_orders",)}, + full_refresh=False, + expected_action={}, + expected_reason={}, + ) + + +def change_tracking_project_case() -> BuildExecutionPlanTestCase: + """Daily totals is maintained from orders, and from payments through a view; not customers.""" + + return BuildExecutionPlanTestCase( + description="tables a maintained model reads, directly or via a view, and one it does not", + setup_sql=(), + model_locations={ + "orders": "staging", + "customers": "staging", + "payments": "staging", + "settled_payments": "staging", + "daily_totals": "staging", + }, + model_configs={ + "orders": {"materialized": "table"}, + "customers": {"materialized": "table"}, + "payments": {"materialized": "table"}, + "settled_payments": {"materialized": "view"}, + "daily_totals": { + "materialized": "maintained", + "refresh": {"lag": "1h"}, + "refresh_on_build": True, + }, + }, + model_queries={ + "orders": "SELECT 1 AS order_id", + "customers": "SELECT 1 AS customer_id", + "payments": "SELECT 1 AS order_id", + "settled_payments": "SELECT order_id FROM staging.payments", + "daily_totals": ( + "SELECT count(*) AS orders FROM staging.orders " + "JOIN staging.settled_payments USING (order_id)" + ), + }, + model_deps={ + "daily_totals": ("orders", "settled_payments"), + "settled_payments": ("payments",), + }, + full_refresh=False, + expected_action={}, + expected_reason={}, + ) diff --git a/tests/integration/src/sqlbuild/compiler/planner/main/test_maintained_refresh_plan.py b/tests/integration/src/sqlbuild/compiler/planner/main/test_maintained_refresh_plan.py new file mode 100644 index 0000000000..351c65275e --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/planner/main/test_maintained_refresh_plan.py @@ -0,0 +1,181 @@ +"""A replan of a maintained model reads its live refresh settings and marks rebuilt upstreams.""" + +from __future__ import annotations + +from typing import Any + +import pytest + +from sqlbuild._native import ModelPlanEntry, PlanAction, PlanOutput, PlanReason +from sqlbuild.adapter.contract.types import MaintainedListing +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from sqlbuild.executor.run._helpers.reuse.fingerprinting import try_write_fingerprint +from tests.integration.src.sqlbuild.compiler.planner.main._test_types import ( + ChangeTrackingPlanTestCase, + MaintainedRefreshPlanTestCase, +) +from tests.integration.src.sqlbuild.compiler.planner.main.helpers import ( + CLEAN_ORDERS_QUERY, + ListingDuckDbAdapter, + build_execution_plan_from_kwargs, + build_project_from_test_case, + change_tracking_project_case, + maintained_project_case, +) + + +@pytest.mark.parametrize( + "test_case", + [ + MaintainedRefreshPlanTestCase( + description="listed settings that match keep both models", + listed_settings=MaintainedListing("60 minutes", "TRANSFORM_WH", "ACTIVE"), + upstream_query=CLEAN_ORDERS_QUERY, + expected_upstream_action=PlanAction.KEEP_MAINTAINED, + expected_action=PlanAction.KEEP_MAINTAINED, + expected_reason=PlanReason.NO_CHANGE, + expected_reinitialised_by=None, + expected_read_names=("clean_orders", "daily_totals"), + ), + MaintainedRefreshPlanTestCase( + description="a kept model listed as suspended is reported as paused", + listed_settings=MaintainedListing("60 minutes", "TRANSFORM_WH", "SUSPENDED"), + upstream_query=CLEAN_ORDERS_QUERY, + expected_upstream_action=PlanAction.KEEP_MAINTAINED, + expected_action=PlanAction.KEEP_MAINTAINED, + expected_reason=PlanReason.NO_CHANGE, + expected_reinitialised_by=None, + expected_read_names=("clean_orders", "daily_totals"), + expected_paused=True, + ), + MaintainedRefreshPlanTestCase( + description="a lag changed outside the project is altered in place", + listed_settings=MaintainedListing("2 hours", "TRANSFORM_WH", "ACTIVE"), + upstream_query=CLEAN_ORDERS_QUERY, + expected_upstream_action=PlanAction.ALTER_MAINTAINED, + expected_action=PlanAction.ALTER_MAINTAINED, + expected_reason=PlanReason.CONFIG_CHANGED, + expected_reinitialised_by=None, + expected_read_names=("clean_orders", "daily_totals"), + ), + MaintainedRefreshPlanTestCase( + description="a name that holds no dynamic table is created", + listed_settings=None, + upstream_query=CLEAN_ORDERS_QUERY, + expected_upstream_action=PlanAction.CREATE_MAINTAINED, + expected_action=PlanAction.CREATE_MAINTAINED, + expected_reason=PlanReason.FIRST_RUN, + expected_reinitialised_by=None, + expected_read_names=("clean_orders", "daily_totals"), + ), + MaintainedRefreshPlanTestCase( + description="a recreated upstream marks the kept reader as recomputing in full", + listed_settings=MaintainedListing("1 hour", "TRANSFORM_WH", "ACTIVE"), + upstream_query="SELECT 2 AS order_id", + expected_upstream_action=PlanAction.CREATE_MAINTAINED, + expected_action=PlanAction.KEEP_MAINTAINED, + expected_reason=PlanReason.NO_CHANGE, + expected_reinitialised_by="clean_orders", + expected_read_names=("daily_totals",), + ), + ], + ids=lambda case: case.description, +) +def test_given_built_maintained_models_when_replanning_then_live_settings_and_rebuilds_decide( + test_case: MaintainedRefreshPlanTestCase, +) -> None: + adapter: ListingDuckDbAdapter = ListingDuckDbAdapter(test_case.listed_settings) + connection: Any = adapter.connect({"database": ":memory:"}) + connection.execute("CREATE SCHEMA staging") + first: PlanOutput = build_execution_plan_from_kwargs( + project=build_project_from_test_case(maintained_project_case(CLEAN_ORDERS_QUERY)), + adapter=adapter, + connection=connection, + ) + connection.execute("CREATE TABLE staging.clean_orders AS SELECT 1 AS order_id") + connection.execute("CREATE TABLE staging.daily_totals AS SELECT 1 AS orders") + warnings: list[str] = [] + for entry in first.model_entries: + warnings.extend( + try_write_fingerprint( + entry=entry, + adapter=adapter, + connection=connection, + run_id="first_build", + query_change_tracking=True, + ) + ) + + plan: PlanOutput = build_execution_plan_from_kwargs( + project=build_project_from_test_case(maintained_project_case(test_case.upstream_query)), + adapter=adapter, + connection=connection, + ) + + entries: dict[str, ModelPlanEntry] = {entry.name: entry for entry in plan.model_entries} + assert warnings == [] + assert entries["clean_orders"].action == test_case.expected_upstream_action + assert entries["daily_totals"].action == test_case.expected_action + assert entries["daily_totals"].reason == test_case.expected_reason + assert entries["daily_totals"].reinitialised_by == test_case.expected_reinitialised_by + assert entries["daily_totals"].maintained_paused == test_case.expected_paused + assert entries["clean_orders"].reinitialised_by is None + assert entries["daily_totals"].target_lag == "'1 hour'" + assert entries["daily_totals"].refresh_mode == "incremental" + assert entries["daily_totals"].refresh_warehouse == "transform_wh" + assert tuple(sorted(adapter.read_names)) == test_case.expected_read_names + adapter.close(connection) + + +@pytest.mark.parametrize( + "test_case", + [ + ChangeTrackingPlanTestCase( + description="a table a maintained model reads, directly or through a view, is tracked", + select=(), + expected_maintained_readers={ + "orders": ("daily_totals",), + "customers": (), + "payments": ("daily_totals",), + "settled_payments": ("daily_totals",), + "daily_totals": (), + }, + expected_refresh_on_build={ + "orders": False, + "customers": False, + "payments": False, + "settled_payments": False, + "daily_totals": True, + }, + ), + ChangeTrackingPlanTestCase( + description="the reader need not be selected for its upstream to be tracked", + select=("orders", "customers", "payments"), + expected_maintained_readers={ + "orders": ("daily_totals",), + "customers": (), + "payments": ("daily_totals",), + }, + expected_refresh_on_build={"orders": False, "customers": False, "payments": False}, + ), + ], + ids=lambda case: case.description, +) +def test_given_table_read_by_maintained_model_when_planning_then_it_names_its_maintained_readers( + test_case: ChangeTrackingPlanTestCase, adapter: DuckDbAdapter, connection: Any +) -> None: + plan: PlanOutput = build_execution_plan_from_kwargs( + project=build_project_from_test_case(change_tracking_project_case()), + adapter=adapter, + connection=connection, + select=test_case.select, + ) + + readers: dict[str, tuple[str, ...]] = { + entry.name: entry.maintained_readers for entry in plan.model_entries + } + assert readers == test_case.expected_maintained_readers + assert { + entry.name: entry.refresh_on_build for entry in plan.model_entries + } == test_case.expected_refresh_on_build + assert plan.model_locations["daily_totals"].name == "daily_totals" diff --git a/tests/integration/src/sqlbuild/executor/maintained_pauses/__init__.py b/tests/integration/src/sqlbuild/executor/maintained_pauses/__init__.py new file mode 100644 index 0000000000..11f5a11f26 --- /dev/null +++ b/tests/integration/src/sqlbuild/executor/maintained_pauses/__init__.py @@ -0,0 +1 @@ +"""Maintained model pause lease integration tests.""" diff --git a/tests/integration/src/sqlbuild/executor/maintained_pauses/_test_types.py b/tests/integration/src/sqlbuild/executor/maintained_pauses/_test_types.py new file mode 100644 index 0000000000..01cd40ee39 --- /dev/null +++ b/tests/integration/src/sqlbuild/executor/maintained_pauses/_test_types.py @@ -0,0 +1,76 @@ +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class PauseLeaseTestCase: + description: str + second_select: tuple[str, ...] + second_orders_query: str + second_totals_query: str + report_locations: dict[str, str] + refresh_on_build: bool + states_before_second: dict[str, str] + expected_operations: tuple[tuple[str, str], ...] + expected_events: tuple[tuple[str, str, str, str], ...] + + +@dataclass(frozen=True) +class HoldTakeoverTestCase: + description: str + grace_seconds: int + ignore_grace: bool + allow_ignore_grace: bool + expected_operations: tuple[tuple[str, str], ...] + expected_events: tuple[tuple[str, str, str, str], ...] + + +@dataclass(frozen=True) +class HoldRefusalTestCase: + description: str + grace_seconds: int + ignore_grace: bool + allow_ignore_grace: bool + expected_code: str + expected_message_fragment: str + + +@dataclass(frozen=True) +class FirstBuildTestCase: + description: str + pause_maintained: bool + expected_operations: tuple[tuple[str, str], ...] + expected_states: dict[str, str] + + +@dataclass(frozen=True) +class TakenOverReleaseTestCase: + description: str + expected_release_operations: tuple[tuple[str, str], ...] + expected_states: dict[str, str] + expected_events: tuple[tuple[str, str, str, str], ...] + + +@dataclass(frozen=True) +class ConcurrentHoldTestCase: + description: str + expected_code: str + expected_winner: str + expected_states: dict[str, str] + expected_events: tuple[tuple[str, str, str, str], ...] + + +@dataclass(frozen=True) +class LifecycleLineTestCase: + description: str + states_before_second: dict[str, str] + expected_line: str + + +@dataclass(frozen=True) +class HeartbeatTestCase: + description: str + heartbeat_seconds: float + wait_seconds: float + expected_min_beats: int diff --git a/tests/integration/src/sqlbuild/executor/maintained_pauses/conftest.py b/tests/integration/src/sqlbuild/executor/maintained_pauses/conftest.py new file mode 100644 index 0000000000..1634ea5edb --- /dev/null +++ b/tests/integration/src/sqlbuild/executor/maintained_pauses/conftest.py @@ -0,0 +1,31 @@ +"""A file-backed DuckDB with a staging schema, so a heartbeat can open its own connection.""" + +from __future__ import annotations + +from collections.abc import Iterator +from pathlib import Path +from typing import Any + +import pytest + +from tests.integration.src.sqlbuild.executor.maintained_pauses.helpers import ( + PausingDuckDbAdapter, +) + + +@pytest.fixture +def database_path(tmp_path: Path) -> Path: + return tmp_path / "lease.duckdb" + + +@pytest.fixture +def adapter() -> PausingDuckDbAdapter: + return PausingDuckDbAdapter() + + +@pytest.fixture +def connection(adapter: PausingDuckDbAdapter, database_path: Path) -> Iterator[Any]: + opened: Any = adapter.connect({"database": str(database_path)}) + opened.execute("CREATE SCHEMA staging") + yield opened + adapter.close(opened) diff --git a/tests/integration/src/sqlbuild/executor/maintained_pauses/helpers.py b/tests/integration/src/sqlbuild/executor/maintained_pauses/helpers.py new file mode 100644 index 0000000000..f956db03e5 --- /dev/null +++ b/tests/integration/src/sqlbuild/executor/maintained_pauses/helpers.py @@ -0,0 +1,176 @@ +"""A DuckDB adapter that tracks dynamic-table scheduling, and the plans the lease tests build.""" + +from __future__ import annotations + +from dataclasses import replace +from pathlib import Path +from typing import Any + +from sqlbuild._native import PlanOutput +from sqlbuild.adapter.contract.types import ( + MaintainedListing, + MaintainedOperation, + TablePromotionMode, +) +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from sqlbuild.executor.build.main._execute import execute_build_plan +from sqlbuild.executor.build.models import ( + BuildCallbacks, + BuildExecutionResult, + BuildRuntimeParams, +) +from tests.integration.src.sqlbuild.compiler.planner.main._test_types import ( + BuildExecutionPlanTestCase, +) +from tests.integration.src.sqlbuild.compiler.planner.main.helpers import ( + build_execution_plan_from_kwargs, + build_project_from_test_case, +) + +type LeaseProject = BuildExecutionPlanTestCase + +_MAINTAINED: dict[str, object] = { + "materialized": "maintained", + "refresh": {"lag": "1h"}, + "refresh_warehouse": "transform_wh", +} +_CREATED_STATE: str = "ACTIVE" +TOTALS_QUERY: str = "SELECT count(*) AS orders FROM staging.orders" +_STATES: dict[MaintainedOperation, str] = { + MaintainedOperation.SUSPEND: "SUSPENDED", + MaintainedOperation.RESUME: "ACTIVE", +} + + +class PausingDuckDbAdapter(DuckDbAdapter): + """Builds maintained models as tables and keeps their scheduling state as Snowflake would.""" + + def __init__(self) -> None: + super().__init__() + self.states: dict[str, str] = {} + self.operations: list[tuple[str, str]] = [] + + def session_warehouse(self, *, connection: Any) -> str | None: + del connection + return "SESSION_WH" + + def maintained_listing( + self, *, connection: Any, database: str | None, schema: str | None, name: str + ) -> MaintainedListing | None: + del connection, database, schema + listings: dict[str, MaintainedListing] = { + listed: MaintainedListing("1 hour", "TRANSFORM_WH", state) + for listed, state in self.states.items() + } + return listings.get(name) + + def render_create_maintained( + self, *, destination: str, sql: str, target_lag: str, refresh_mode: str, warehouse: str + ) -> tuple[str, ...]: + del target_lag, refresh_mode, warehouse + name: str = destination.rsplit(".", 1)[-1] + self.states[name] = _CREATED_STATE + self.operations.append(("create", name)) + return (f"CREATE OR REPLACE TABLE {destination} AS {sql}",) + + def render_maintained_operation( + self, *, destination: str, operation: MaintainedOperation + ) -> tuple[str, ...]: + name: str = destination.rsplit(".", 1)[-1] + self.states[name] = _STATES.get(operation, self.states[name]) + self.operations.append((operation.value, name)) + return ("SELECT 1",) + + +def lease_project_case( + *, + orders_query: str, + report_locations: dict[str, str], + refresh_on_build: bool, + totals_query: str = TOTALS_QUERY, +) -> BuildExecutionPlanTestCase: + """Orders, daily totals maintained from them, and any report table located to read those.""" + + maintained: dict[str, object] = {**_MAINTAINED, "refresh_on_build": refresh_on_build} + return BuildExecutionPlanTestCase( + description="orders and the daily totals maintained from them", + setup_sql=(), + model_locations={"orders": "staging", "daily_totals": "staging", **report_locations}, + model_configs={ + "orders": {"materialized": "table"}, + "daily_totals": maintained, + "order_report": {"materialized": "table"}, + }, + model_queries={ + "orders": orders_query, + "daily_totals": totals_query, + "order_report": "SELECT orders FROM staging.daily_totals", + }, + model_deps={"daily_totals": ("orders",), "order_report": ("daily_totals",)}, + full_refresh=False, + expected_action={}, + expected_reason={}, + ) + + +def plan_lease_project( + *, + adapter: PausingDuckDbAdapter, + connection: Any, + case: BuildExecutionPlanTestCase, + select: tuple[str, ...] = (), +) -> PlanOutput: + """Plan the selected models against the warehouse as it stands.""" + + return build_execution_plan_from_kwargs( + project=build_project_from_test_case(case), + adapter=adapter, + connection=connection, + select=select, + ) + + +def build_lease_project( + *, + adapter: PausingDuckDbAdapter, + connection: Any, + case: BuildExecutionPlanTestCase, + runtime: BuildRuntimeParams, + database_path: Path, + select: tuple[str, ...] = (), + callbacks: BuildCallbacks | None = None, +) -> BuildExecutionResult: + """Plan the selected models, then build them with `runtime` and `callbacks`.""" + + return execute_build_plan( + plan=plan_lease_project(adapter=adapter, connection=connection, case=case, select=select), + adapter=adapter, + connection_config={"database": str(database_path)}, + connections=(connection,), + scheduler_connection=connection, + runtime=replace( + runtime, + promotion_mode=TablePromotionMode(adapter.default_table_promotion_mode()), + ), + callbacks=callbacks, + ) + + +def pause_events(*, connection: Any) -> list[tuple[str, str, str, str]]: + """Each recorded pause event's type, holding run, recording run and model, sorted.""" + + rows: list[tuple[Any, ...]] = connection.execute( + "SELECT event_type, run_id, recorded_by, relation_name " + "FROM staging._sqlbuild_maintained_pauses WHERE event_type <> 'heartbeat' " + "ORDER BY 1, 2, 3, 4" + ).fetchall() + return [(str(row[0]), str(row[1]), str(row[2]), str(row[3])) for row in rows] + + +def pause_table_names(*, connection: Any) -> list[tuple[Any, ...]]: + """The pause state table, when a build created one in staging.""" + + return connection.execute( + "SELECT table_name FROM information_schema.tables " + "WHERE table_schema = 'staging' AND table_name = '_sqlbuild_maintained_pauses'" + ).fetchall() diff --git a/tests/integration/src/sqlbuild/executor/maintained_pauses/test_maintained_pause_lease.py b/tests/integration/src/sqlbuild/executor/maintained_pauses/test_maintained_pause_lease.py new file mode 100644 index 0000000000..0bb8cbd9e6 --- /dev/null +++ b/tests/integration/src/sqlbuild/executor/maintained_pauses/test_maintained_pause_lease.py @@ -0,0 +1,638 @@ +"""A build pauses the maintained models it touches, records its hold, and resumes them at the end.""" + +from __future__ import annotations + +import time +from collections.abc import Callable +from pathlib import Path +from typing import Any + +import pytest + +from sqlbuild._native import PlanOutput +from sqlbuild.adapter.contract.exceptions import AdapterUserError +from sqlbuild.executor.build.models import ( + BuildCallbacks, + BuildExecutionResult, + BuildRuntimeParams, +) +from sqlbuild.executor.build.types import BuildStatus +from sqlbuild.executor.maintained_pauses.classes.maintained_hold import MaintainedHold +from sqlbuild.executor.maintained_pauses.main.hold_maintained_models import ( + hold_maintained_models, +) +from tests.integration.src.sqlbuild.executor.maintained_pauses._test_types import ( + ConcurrentHoldTestCase, + FirstBuildTestCase, + HeartbeatTestCase, + HoldRefusalTestCase, + HoldTakeoverTestCase, + LifecycleLineTestCase, + PauseLeaseTestCase, + TakenOverReleaseTestCase, +) +from tests.integration.src.sqlbuild.executor.maintained_pauses.helpers import ( + TOTALS_QUERY, + LeaseProject, + PausingDuckDbAdapter, + build_lease_project, + lease_project_case, + pause_events, + pause_table_names, + plan_lease_project, +) + +_ORDERS: str = "SELECT 1 AS order_id" +_MORE_ORDERS: str = "SELECT 1 AS order_id UNION ALL SELECT 2" +_REPORT: dict[str, str] = {"order_report": "staging"} +_FIRST_HOLD: tuple[tuple[str, str, str, str], ...] = ( + ("paused", "first_run", "first_run", "daily_totals"), + ("resumed", "first_run", "first_run", "daily_totals"), +) +_SECOND_HOLD: tuple[tuple[str, str, str, str], ...] = ( + ("paused", "second_run", "second_run", "daily_totals"), + ("resumed", "second_run", "second_run", "daily_totals"), +) +_TAKEN_OVER: tuple[tuple[str, str, str, str], ...] = ( + ("paused", "dead_run", "dead_run", "daily_totals"), + ("paused", "first_run", "first_run", "daily_totals"), + ("paused", "second_run", "second_run", "daily_totals"), + ("resumed", "dead_run", "second_run", "daily_totals"), + ("resumed", "first_run", "first_run", "daily_totals"), + ("resumed", "second_run", "second_run", "daily_totals"), +) + + +@pytest.mark.parametrize( + "test_case", + [ + FirstBuildTestCase( + description="a created model is paused for the build and resumed at its end", + pause_maintained=False, + expected_operations=( + ("create", "daily_totals"), + ("suspend", "daily_totals"), + ("resume", "daily_totals"), + ), + expected_states={"daily_totals": "ACTIVE"}, + ), + ], + ids=lambda case: case.description, +) +def test_given_first_build_when_creating_maintained_model_then_pauses_it_until_the_end( + test_case: FirstBuildTestCase, + adapter: PausingDuckDbAdapter, + connection: Any, + database_path: Path, +) -> None: + result: BuildExecutionResult = build_lease_project( + adapter=adapter, + connection=connection, + case=lease_project_case(orders_query=_ORDERS, report_locations={}, refresh_on_build=False), + runtime=BuildRuntimeParams(run_id="first_run", pause_maintained=test_case.pause_maintained), + database_path=database_path, + ) + + assert result.status == BuildStatus.SUCCESS + assert tuple(adapter.operations) == test_case.expected_operations + assert adapter.states == test_case.expected_states + assert pause_events(connection=connection) == sorted(_FIRST_HOLD) + + +@pytest.mark.parametrize( + "test_case", + [ + PauseLeaseTestCase( + description="a build of the maintained model alone writes nothing it reads", + second_select=("daily_totals",), + second_orders_query=_ORDERS, + second_totals_query=TOTALS_QUERY, + report_locations={}, + refresh_on_build=False, + states_before_second={}, + expected_operations=(), + expected_events=_FIRST_HOLD, + ), + PauseLeaseTestCase( + description="a rebuilt upstream pauses the model for the build", + second_select=(), + second_orders_query=_MORE_ORDERS, + second_totals_query=TOTALS_QUERY, + report_locations={}, + refresh_on_build=False, + states_before_second={}, + expected_operations=(("suspend", "daily_totals"), ("resume", "daily_totals")), + expected_events=(*_FIRST_HOLD, *_SECOND_HOLD), + ), + PauseLeaseTestCase( + description="a model a person suspended is neither paused nor resumed", + second_select=(), + second_orders_query=_MORE_ORDERS, + second_totals_query=TOTALS_QUERY, + report_locations={}, + refresh_on_build=False, + states_before_second={"daily_totals": "SUSPENDED"}, + expected_operations=(), + expected_events=_FIRST_HOLD, + ), + PauseLeaseTestCase( + description="a model a person suspended stays suspended when it is recreated", + second_select=(), + second_orders_query=_ORDERS, + second_totals_query="SELECT count(*) + 0 AS orders FROM staging.orders", + report_locations={}, + refresh_on_build=False, + states_before_second={"daily_totals": "SUSPENDED"}, + expected_operations=(("create", "daily_totals"), ("suspend", "daily_totals")), + expected_events=_FIRST_HOLD, + ), + PauseLeaseTestCase( + description="a table built from the model refreshes it first", + second_select=(), + second_orders_query=_MORE_ORDERS, + second_totals_query=TOTALS_QUERY, + report_locations=_REPORT, + refresh_on_build=False, + states_before_second={}, + expected_operations=( + ("suspend", "daily_totals"), + ("refresh", "daily_totals"), + ("resume", "daily_totals"), + ), + expected_events=(*_FIRST_HOLD, *_SECOND_HOLD), + ), + PauseLeaseTestCase( + description="refresh_on_build refreshes a kept model", + second_select=(), + second_orders_query=_MORE_ORDERS, + second_totals_query=TOTALS_QUERY, + report_locations={}, + refresh_on_build=True, + states_before_second={}, + expected_operations=( + ("suspend", "daily_totals"), + ("refresh", "daily_totals"), + ("resume", "daily_totals"), + ), + expected_events=(*_FIRST_HOLD, *_SECOND_HOLD), + ), + ], + ids=lambda case: case.description, +) +def test_given_built_maintained_model_when_building_again_then_pauses_only_what_it_touches( + test_case: PauseLeaseTestCase, + adapter: PausingDuckDbAdapter, + connection: Any, + database_path: Path, +) -> None: + _ = build_lease_project( + adapter=adapter, + connection=connection, + case=lease_project_case( + orders_query=_ORDERS, + report_locations=test_case.report_locations, + refresh_on_build=test_case.refresh_on_build, + ), + runtime=BuildRuntimeParams(run_id="first_run"), + database_path=database_path, + ) + adapter.operations.clear() + adapter.states.update(test_case.states_before_second) + + result: BuildExecutionResult = build_lease_project( + adapter=adapter, + connection=connection, + case=lease_project_case( + orders_query=test_case.second_orders_query, + report_locations=test_case.report_locations, + refresh_on_build=test_case.refresh_on_build, + totals_query=test_case.second_totals_query, + ), + runtime=BuildRuntimeParams(run_id="second_run"), + database_path=database_path, + select=test_case.second_select, + ) + + assert result.status == BuildStatus.SUCCESS + assert tuple(adapter.operations) == test_case.expected_operations + assert pause_events(connection=connection) == sorted(test_case.expected_events) + + +@pytest.mark.parametrize( + "test_case", + [ + FirstBuildTestCase( + description="a pause_maintained target leaves models suspended and records nothing", + pause_maintained=True, + expected_operations=(("refresh", "daily_totals"),), + expected_states={"daily_totals": "SUSPENDED"}, + ), + ], + ids=lambda case: case.description, +) +def test_given_pause_maintained_target_when_building_then_models_stay_suspended_and_refresh( + test_case: FirstBuildTestCase, + adapter: PausingDuckDbAdapter, + connection: Any, + database_path: Path, +) -> None: + paused: BuildRuntimeParams = BuildRuntimeParams( + run_id="first_run", pause_maintained=test_case.pause_maintained + ) + _ = build_lease_project( + adapter=adapter, + connection=connection, + case=lease_project_case(orders_query=_ORDERS, report_locations={}, refresh_on_build=False), + runtime=paused, + database_path=database_path, + ) + first: list[tuple[str, str]] = list(adapter.operations) + adapter.operations.clear() + + result: BuildExecutionResult = build_lease_project( + adapter=adapter, + connection=connection, + case=lease_project_case( + orders_query=_MORE_ORDERS, report_locations={}, refresh_on_build=False + ), + runtime=paused, + database_path=database_path, + ) + + assert result.status == BuildStatus.SUCCESS + assert first == [("create", "daily_totals"), ("suspend", "daily_totals")] + assert tuple(adapter.operations) == test_case.expected_operations + assert adapter.states == test_case.expected_states + assert pause_table_names(connection=connection) == [] + + +@pytest.mark.parametrize( + "test_case", + [ + HoldTakeoverTestCase( + description="a hold silent past the grace is taken over", + grace_seconds=0, + ignore_grace=False, + allow_ignore_grace=False, + expected_operations=(("resume", "daily_totals"),), + expected_events=_TAKEN_OVER, + ), + HoldTakeoverTestCase( + description="--ignore-maintained-grace takes over a live hold when the project allows", + grace_seconds=1800, + ignore_grace=True, + allow_ignore_grace=True, + expected_operations=(("resume", "daily_totals"),), + expected_events=_TAKEN_OVER, + ), + ], + ids=lambda case: case.description, +) +def test_given_hold_of_a_stopped_build_when_building_then_takes_it_over_and_resumes_at_the_end( + test_case: HoldTakeoverTestCase, + adapter: PausingDuckDbAdapter, + connection: Any, + database_path: Path, +) -> None: + _ = build_lease_project( + adapter=adapter, + connection=connection, + case=lease_project_case(orders_query=_ORDERS, report_locations={}, refresh_on_build=False), + runtime=BuildRuntimeParams(run_id="first_run"), + database_path=database_path, + ) + changed: LeaseProject = lease_project_case( + orders_query=_MORE_ORDERS, report_locations={}, refresh_on_build=False + ) + dead: MaintainedHold | None = hold_maintained_models( + plan=plan_lease_project(adapter=adapter, connection=connection, case=changed), + adapter=adapter, + runtime=BuildRuntimeParams(run_id="dead_run"), + connection_config={"database": str(database_path)}, + ) + assert dead is not None + dead.acquire(connection=connection, on_progress=None) + adapter.operations.clear() + + result: BuildExecutionResult = build_lease_project( + adapter=adapter, + connection=connection, + case=changed, + runtime=BuildRuntimeParams( + run_id="second_run", + maintained_heartbeat_grace_seconds=test_case.grace_seconds, + ignore_maintained_grace=test_case.ignore_grace, + allow_ignore_maintained_grace=test_case.allow_ignore_grace, + ), + database_path=database_path, + ) + + assert result.status == BuildStatus.SUCCESS + assert tuple(adapter.operations) == test_case.expected_operations + assert adapter.states == {"daily_totals": "ACTIVE"} + assert pause_events(connection=connection) == sorted(test_case.expected_events) + + +@pytest.mark.parametrize( + "test_case", + [ + HoldRefusalTestCase( + description="a live hold stops the build and names the run holding it", + grace_seconds=1800, + ignore_grace=False, + allow_ignore_grace=False, + expected_code="A718", + expected_message_fragment="dead_run", + ), + HoldRefusalTestCase( + description="--ignore-maintained-grace needs allow_ignore_grace", + grace_seconds=1800, + ignore_grace=True, + allow_ignore_grace=False, + expected_code="A719", + expected_message_fragment="allow_ignore_grace", + ), + ], + ids=lambda case: case.description, +) +def test_given_live_hold_when_building_then_stops_before_any_change( + test_case: HoldRefusalTestCase, + adapter: PausingDuckDbAdapter, + connection: Any, + database_path: Path, +) -> None: + _ = build_lease_project( + adapter=adapter, + connection=connection, + case=lease_project_case(orders_query=_ORDERS, report_locations={}, refresh_on_build=False), + runtime=BuildRuntimeParams(run_id="first_run"), + database_path=database_path, + ) + changed: LeaseProject = lease_project_case( + orders_query=_MORE_ORDERS, report_locations={}, refresh_on_build=False + ) + dead: MaintainedHold | None = hold_maintained_models( + plan=plan_lease_project(adapter=adapter, connection=connection, case=changed), + adapter=adapter, + runtime=BuildRuntimeParams(run_id="dead_run"), + connection_config={"database": str(database_path)}, + ) + assert dead is not None + dead.acquire(connection=connection, on_progress=None) + adapter.operations.clear() + + with pytest.raises(AdapterUserError) as raised: + _ = build_lease_project( + adapter=adapter, + connection=connection, + case=changed, + runtime=BuildRuntimeParams( + run_id="second_run", + maintained_heartbeat_grace_seconds=test_case.grace_seconds, + ignore_maintained_grace=test_case.ignore_grace, + allow_ignore_maintained_grace=test_case.allow_ignore_grace, + ), + database_path=database_path, + ) + + assert raised.value.code == test_case.expected_code + assert test_case.expected_message_fragment in str(raised.value) + assert adapter.operations == [] + assert connection.execute("SELECT count(*) FROM staging.orders").fetchall() == [(1,)] + + +@pytest.mark.parametrize( + "test_case", + [ + HeartbeatTestCase( + description="a hold beats on its own connection while the build runs", + heartbeat_seconds=0.05, + wait_seconds=0.4, + expected_min_beats=2, + ), + HeartbeatTestCase( + description="a hold beats as soon as it is taken, not one interval later", + heartbeat_seconds=30.0, + wait_seconds=0.3, + expected_min_beats=1, + ), + ], + ids=lambda case: case.description, +) +def test_given_held_models_when_build_runs_then_heartbeat_writes_on_its_own_connection( + test_case: HeartbeatTestCase, + adapter: PausingDuckDbAdapter, + connection: Any, + database_path: Path, +) -> None: + _ = build_lease_project( + adapter=adapter, + connection=connection, + case=lease_project_case(orders_query=_ORDERS, report_locations={}, refresh_on_build=False), + runtime=BuildRuntimeParams(run_id="first_run"), + database_path=database_path, + ) + changed: LeaseProject = lease_project_case( + orders_query=_MORE_ORDERS, report_locations={}, refresh_on_build=False + ) + hold: MaintainedHold | None = hold_maintained_models( + plan=plan_lease_project(adapter=adapter, connection=connection, case=changed), + adapter=adapter, + runtime=BuildRuntimeParams(run_id="beating_run"), + connection_config={"database": str(database_path)}, + heartbeat_seconds=test_case.heartbeat_seconds, + ) + assert hold is not None + + hold.acquire(connection=connection, on_progress=None) + time.sleep(test_case.wait_seconds) + hold.release(connection=connection) + + beats: list[tuple[Any, ...]] = connection.execute( + "SELECT count(*) FROM staging._sqlbuild_maintained_pauses " + "WHERE event_type = 'heartbeat' AND run_id = 'beating_run'" + ).fetchall() + assert beats[0][0] >= test_case.expected_min_beats + + +@pytest.mark.parametrize( + "test_case", + [ + LifecycleLineTestCase( + description="a model suspended outside SQLBuild is reported on the build's output", + states_before_second={"daily_totals": "SUSPENDED"}, + expected_line=( + "Warning: maintained model daily_totals was paused outside SQLBuild; " + "it stays paused" + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_build_output_when_hold_reports_then_line_reaches_the_lifecycle_writer( + test_case: LifecycleLineTestCase, + adapter: PausingDuckDbAdapter, + connection: Any, + database_path: Path, +) -> None: + _ = build_lease_project( + adapter=adapter, + connection=connection, + case=lease_project_case(orders_query=_ORDERS, report_locations={}, refresh_on_build=False), + runtime=BuildRuntimeParams(run_id="first_run"), + database_path=database_path, + ) + adapter.states.update(test_case.states_before_second) + lines: list[str] = [] + + result: BuildExecutionResult = build_lease_project( + adapter=adapter, + connection=connection, + case=lease_project_case( + orders_query=_MORE_ORDERS, report_locations={}, refresh_on_build=False + ), + runtime=BuildRuntimeParams(run_id="second_run"), + database_path=database_path, + callbacks=BuildCallbacks(on_lifecycle_progress=lines.append), + ) + + assert result.status == BuildStatus.SUCCESS + assert test_case.expected_line in lines + + +@pytest.mark.parametrize( + "test_case", + [ + TakenOverReleaseTestCase( + description="a build whose hold was taken over leaves the model to the new holder", + expected_release_operations=(), + expected_states={"daily_totals": "SUSPENDED"}, + expected_events=( + ("paused", "first_run", "first_run", "daily_totals"), + ("paused", "second_run", "second_run", "daily_totals"), + ("paused", "slow_run", "slow_run", "daily_totals"), + ("resumed", "first_run", "first_run", "daily_totals"), + ("resumed", "slow_run", "second_run", "daily_totals"), + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_hold_taken_over_when_first_build_releases_then_it_resumes_nothing( + test_case: TakenOverReleaseTestCase, + adapter: PausingDuckDbAdapter, + connection: Any, + database_path: Path, +) -> None: + _ = build_lease_project( + adapter=adapter, + connection=connection, + case=lease_project_case(orders_query=_ORDERS, report_locations={}, refresh_on_build=False), + runtime=BuildRuntimeParams(run_id="first_run"), + database_path=database_path, + ) + changed: LeaseProject = lease_project_case( + orders_query=_MORE_ORDERS, report_locations={}, refresh_on_build=False + ) + plan: PlanOutput = plan_lease_project(adapter=adapter, connection=connection, case=changed) + holds: list[MaintainedHold] = [] + for runtime in ( + BuildRuntimeParams(run_id="slow_run"), + BuildRuntimeParams( + run_id="second_run", + ignore_maintained_grace=True, + allow_ignore_maintained_grace=True, + ), + ): + hold: MaintainedHold | None = hold_maintained_models( + plan=plan, + adapter=adapter, + runtime=runtime, + connection_config={"database": str(database_path)}, + ) + assert hold is not None + hold.acquire(connection=connection, on_progress=None) + holds.append(hold) + adapter.operations.clear() + + holds[0].release(connection=connection) + + assert tuple(adapter.operations) == test_case.expected_release_operations + assert adapter.states == test_case.expected_states + assert pause_events(connection=connection) == sorted(test_case.expected_events) + holds[1].release(connection=connection) + + +@pytest.mark.parametrize( + "test_case", + [ + ConcurrentHoldTestCase( + description="of two builds pausing a model at once, the later one backs out", + expected_code="A718", + expected_winner="early_run", + expected_states={"daily_totals": "SUSPENDED"}, + expected_events=( + ("paused", "early_run", "early_run", "daily_totals"), + ("paused", "first_run", "first_run", "daily_totals"), + ("paused", "late_run", "late_run", "daily_totals"), + ("resumed", "first_run", "first_run", "daily_totals"), + ("resumed", "late_run", "late_run", "daily_totals"), + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_two_builds_pausing_together_when_both_record_a_pause_then_only_the_earliest_holds( + test_case: ConcurrentHoldTestCase, + adapter: PausingDuckDbAdapter, + connection: Any, + database_path: Path, +) -> None: + _ = build_lease_project( + adapter=adapter, + connection=connection, + case=lease_project_case(orders_query=_ORDERS, report_locations={}, refresh_on_build=False), + runtime=BuildRuntimeParams(run_id="first_run"), + database_path=database_path, + ) + plan: PlanOutput = plan_lease_project( + adapter=adapter, + connection=connection, + case=lease_project_case( + orders_query=_MORE_ORDERS, report_locations={}, refresh_on_build=False + ), + ) + early: MaintainedHold | None = hold_maintained_models( + plan=plan, + adapter=adapter, + runtime=BuildRuntimeParams(run_id="early_run"), + connection_config={"database": str(database_path)}, + ) + late: MaintainedHold | None = hold_maintained_models( + plan=plan, + adapter=adapter, + runtime=BuildRuntimeParams(run_id="late_run"), + connection_config={"database": str(database_path)}, + ) + assert early is not None + assert late is not None + read_state: Callable[..., str | None] = late._state + + def state_then_rival_acquires(*, connection: Any, name: str) -> str | None: + state: str | None = read_state(connection=connection, name=name) + early.acquire(connection=connection, on_progress=None) + adapter.operations.clear() + return state + + late._state = state_then_rival_acquires # ty: ignore[invalid-assignment] + + with pytest.raises(AdapterUserError) as raised: + with late.holding(connection=connection, on_progress=None): + pass + + assert raised.value.code == test_case.expected_code + assert test_case.expected_winner in str(raised.value) + assert adapter.operations == [] + assert adapter.states == test_case.expected_states + assert pause_events(connection=connection) == sorted(test_case.expected_events) + early.release(connection=connection) diff --git a/tests/integration/src/sqlbuild/executor/run/maintained/__init__.py b/tests/integration/src/sqlbuild/executor/run/maintained/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/integration/src/sqlbuild/executor/run/maintained/_test_types.py b/tests/integration/src/sqlbuild/executor/run/maintained/_test_types.py new file mode 100644 index 0000000000..3958be16a0 --- /dev/null +++ b/tests/integration/src/sqlbuild/executor/run/maintained/_test_types.py @@ -0,0 +1,48 @@ +from __future__ import annotations + +from dataclasses import dataclass + +from sqlbuild._native import PlanAction +from sqlbuild.adapter.contract.types import MaintainedListing, TablePromotionMode + + +@dataclass(frozen=True) +class MaintainedSuccessTestCase: + description: str + action: PlanAction + setup_sql: tuple[str, ...] + refresh_warehouse: str | None + expected_created: tuple[tuple[str, str, str], ...] + expected_altered: tuple[tuple[str, str | None], ...] + expected_rows: tuple[tuple[int, ...], ...] + + +@dataclass(frozen=True) +class MaintainedFailureTestCase: + description: str + setup_sql: tuple[str, ...] + session_warehouse: str | None + listed_settings: MaintainedListing | None + failing_query: str | None + expected_error_code: str + expected_listed_types: tuple[tuple[str], ...] + + +@dataclass(frozen=True) +class FailedRestoreTestCase: + description: str + setup_sql: tuple[str, ...] + failing_query: str + failing_renames: dict[str, str] + expected_error_code: str + expected_fragments: tuple[str, ...] + expected_relations: tuple[tuple[str, str], ...] + + +@dataclass(frozen=True) +class ChangeTrackingTableTestCase: + description: str + maintained_readers: tuple[str, ...] + promotion_mode: TablePromotionMode + declared_columns: tuple[tuple[str, str], ...] + expected_tracking: tuple[bool, ...] diff --git a/tests/integration/src/sqlbuild/executor/run/maintained/helpers.py b/tests/integration/src/sqlbuild/executor/run/maintained/helpers.py new file mode 100644 index 0000000000..1f2a9d098b --- /dev/null +++ b/tests/integration/src/sqlbuild/executor/run/maintained/helpers.py @@ -0,0 +1,150 @@ +"""A DuckDB adapter that builds maintained models as tables, for executor lifecycle tests.""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +from sqlbuild._native import ( + CompiledObjectKey, + CompiledRelationLocation, + CompiledResourceType, + MaterializationType, + ModelPlanEntry, + PlanAction, + PlanReason, +) +from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder +from sqlbuild.adapter.contract.types import MaintainedListing +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from sqlbuild.executor.run.models import ModelMaterializationContext + +DAILY_TOTALS_SQL: str = "SELECT 3 AS orders" + + +class MaintainedDuckDbAdapter(DuckDbAdapter): + """Builds a maintained model as a table that, like a dynamic table, replaces nothing else.""" + + def __init__( + self, + *, + session_warehouse: str | None, + listed_settings: MaintainedListing | None = None, + failing_query: str | None = None, + failing_renames: dict[str, str] | None = None, + ) -> None: + super().__init__() + self.warehouse: str | None = session_warehouse + self.listed_settings: MaintainedListing | None = listed_settings + self.failing_query: str | None = failing_query + self.failing_renames: dict[str, str] = dict(failing_renames or {}) + self.created: list[tuple[str, str, str]] = [] + self.altered: list[tuple[str, str | None]] = [] + + def session_warehouse(self, *, connection: Any) -> str | None: + del connection + return self.warehouse + + def maintained_listing( + self, *, connection: Any, database: str | None, schema: str | None, name: str + ) -> MaintainedListing | None: + del connection, database, schema, name + return self.listed_settings + + def render_create_maintained( + self, *, destination: str, sql: str, target_lag: str, refresh_mode: str, warehouse: str + ) -> tuple[str, ...]: + self.created.append((target_lag, refresh_mode, warehouse)) + return (f"CREATE TABLE {destination} AS {self.failing_query or sql}",) + + def render_rename(self, *, origin: str, destination: str) -> tuple[str, ...]: + renamed: str = f"ALTER TABLE {origin} RENAME TO {destination.split('.')[-1]}" + return (self.failing_renames.get(origin, renamed),) + + def render_rename_view(self, *, origin: str, destination: str) -> tuple[str, ...]: + return (f"ALTER VIEW {origin} RENAME TO {destination.split('.')[-1]}",) + + def render_alter_maintained( + self, *, destination: str, target_lag: str, warehouse: str | None + ) -> tuple[str, ...]: + self.altered.append((target_lag, warehouse)) + lag: str = target_lag.replace("'", "''") + return (f"COMMENT ON TABLE {destination} IS 'refreshed within {lag}'",) + + +def maintained_context( + *, + adapter: MaintainedDuckDbAdapter, + connection: Any, + action: PlanAction, + refresh_warehouse: str | None, +) -> ModelMaterializationContext: + """Daily totals, maintained within an hour, planned with `action`.""" + + entry: ModelPlanEntry = ModelPlanEntry( + key=CompiledObjectKey(resource_type=CompiledResourceType.MODEL, name="daily_totals"), + name="daily_totals", + relative_path=Path("models/daily_totals.sql"), + materialization_type=MaterializationType.MAINTAINED, + action=action, + reason=PlanReason.FIRST_RUN, + destination=CompiledRelationLocation( + database=None, + schema="staging", + name="daily_totals", + qualified_name="staging.daily_totals", + ), + fingerprint_query_sql=DAILY_TOTALS_SQL, + resolved_sql=DAILY_TOTALS_SQL, + logical_ddl="", + target_lag="'1 hour'", + refresh_mode="incremental", + refresh_warehouse=refresh_warehouse, + ) + return ModelMaterializationContext( + entry=entry, + adapter=adapter, + connection=connection, + model_locations={}, + seed_locations={}, + source_map={}, + python_source_read_map={}, + model_audits=(), + run_id="maintained_run", + query_change_tracking=True, + ) + + +def listed_types(*, connection: Any) -> list[tuple[Any, ...]]: + """The relation types daily_totals is listed as; empty when nothing holds the name.""" + + return connection.execute( + "SELECT table_type FROM information_schema.tables " + "WHERE table_schema = 'staging' AND table_name = 'daily_totals'" + ).fetchall() + + +class TrackingDuckDbAdapter(DuckDbAdapter): + """Records whether each table it creates was asked to track changes.""" + + def __init__(self) -> None: + super().__init__() + self.tracking: list[bool] = [] + + def create_table_as( + self, + *, + connection: Any, + destination: str, + sql: str, + config: dict[str, Any] | None = None, + statement_recorder: StatementRecorder, + ) -> None: + self.tracking.append(bool((config or {}).get("change_tracking"))) + super().create_table_as( + connection=connection, + destination=destination, + sql=sql, + config=config, + statement_recorder=statement_recorder, + ) diff --git a/tests/integration/src/sqlbuild/executor/run/maintained/test_change_tracking.py b/tests/integration/src/sqlbuild/executor/run/maintained/test_change_tracking.py new file mode 100644 index 0000000000..4321614ce8 --- /dev/null +++ b/tests/integration/src/sqlbuild/executor/run/maintained/test_change_tracking.py @@ -0,0 +1,85 @@ +"""A table a maintained model reads is created with change tracking on every build path.""" + +from __future__ import annotations + +from dataclasses import replace +from typing import Any + +import pytest + +from sqlbuild._native import ModelPlanEntry +from sqlbuild.adapter.contract.types import TablePromotionMode +from sqlbuild.executor.run.main._execute import execute_table_entry +from sqlbuild.executor.run.models import ModelExecutionResult, ModelMaterializationContext +from sqlbuild.executor.scheduling.types import ExecutionStatus +from tests.integration.src.sqlbuild.executor.run.helpers import ( + build_declared_columns, + build_table_plan_entry, +) +from tests.integration.src.sqlbuild.executor.run.maintained._test_types import ( + ChangeTrackingTableTestCase, +) +from tests.integration.src.sqlbuild.executor.run.maintained.helpers import TrackingDuckDbAdapter + + +@pytest.mark.parametrize( + "test_case", + [ + ChangeTrackingTableTestCase( + description="a staged build tracks changes on the stage it swaps in", + maintained_readers=("daily_totals",), + promotion_mode=TablePromotionMode.STAGED, + declared_columns=(), + expected_tracking=(True,), + ), + ChangeTrackingTableTestCase( + description="type enforcement rebuilds the stage with change tracking too", + maintained_readers=("daily_totals",), + promotion_mode=TablePromotionMode.STAGED, + declared_columns=(("order_id", "BIGINT"),), + expected_tracking=(True, True), + ), + ChangeTrackingTableTestCase( + description="a table no maintained model reads is created as before", + maintained_readers=(), + promotion_mode=TablePromotionMode.STAGED, + declared_columns=(("order_id", "BIGINT"),), + expected_tracking=(False, False), + ), + ], + ids=lambda case: case.description, +) +def test_given_table_read_by_maintained_model_when_building_then_every_create_tracks_changes( + test_case: ChangeTrackingTableTestCase, connection: Any +) -> None: + adapter: TrackingDuckDbAdapter = TrackingDuckDbAdapter() + entry: ModelPlanEntry = replace( + build_table_plan_entry( + name="orders", + sql="SELECT 1::INTEGER AS order_id", + target_schema="staging", + target_name="orders", + type_enforcement=bool(test_case.declared_columns), + ), + maintained_readers=test_case.maintained_readers, + ) + + result: ModelExecutionResult = execute_table_entry( + context=ModelMaterializationContext( + entry=entry, + adapter=adapter, + connection=connection, + model_locations={}, + seed_locations={}, + source_map={}, + python_source_read_map={}, + model_audits=(), + run_id="tracking_run", + query_change_tracking=False, + ), + declared_columns=build_declared_columns(test_case.declared_columns), + promotion_mode=test_case.promotion_mode, + ) + + assert result.status == ExecutionStatus.SUCCESS, result.error_message + assert tuple(adapter.tracking) == test_case.expected_tracking diff --git a/tests/integration/src/sqlbuild/executor/run/maintained/test_maintained_execution.py b/tests/integration/src/sqlbuild/executor/run/maintained/test_maintained_execution.py new file mode 100644 index 0000000000..1df742d105 --- /dev/null +++ b/tests/integration/src/sqlbuild/executor/run/maintained/test_maintained_execution.py @@ -0,0 +1,219 @@ +"""Integration tests for the maintained model lifecycle: create, alter and keep.""" + +from __future__ import annotations + +from typing import Any + +import pytest + +from sqlbuild._native import PlanAction +from sqlbuild.adapter.contract.types import MaintainedListing +from sqlbuild.executor.run.main._execute import execute_maintained_entry +from sqlbuild.executor.run.models import ModelExecutionResult +from sqlbuild.executor.scheduling.types import ExecutionStatus +from tests.integration.src.sqlbuild.executor.run.maintained._test_types import ( + FailedRestoreTestCase, + MaintainedFailureTestCase, + MaintainedSuccessTestCase, +) +from tests.integration.src.sqlbuild.executor.run.maintained.helpers import ( + MaintainedDuckDbAdapter, + listed_types, + maintained_context, +) + +_BUILT: tuple[str, ...] = ("CREATE TABLE staging.daily_totals AS SELECT 1 AS orders",) + + +@pytest.mark.parametrize( + "test_case", + [ + MaintainedSuccessTestCase( + description="a first run creates on the session warehouse", + action=PlanAction.CREATE_MAINTAINED, + setup_sql=(), + refresh_warehouse=None, + expected_created=(("'1 hour'", "incremental", "SESSION_WH"),), + expected_altered=(), + expected_rows=((3,),), + ), + MaintainedSuccessTestCase( + description="a declared refresh warehouse is used over the session's", + action=PlanAction.CREATE_MAINTAINED, + setup_sql=(), + refresh_warehouse="transform_wh", + expected_created=(("'1 hour'", "incremental", "transform_wh"),), + expected_altered=(), + expected_rows=((3,),), + ), + MaintainedSuccessTestCase( + description="a view the model was before is dropped and replaced", + action=PlanAction.CREATE_MAINTAINED, + setup_sql=("CREATE VIEW staging.daily_totals AS SELECT 1 AS orders",), + refresh_warehouse=None, + expected_created=(("'1 hour'", "incremental", "SESSION_WH"),), + expected_altered=(), + expected_rows=((3,),), + ), + MaintainedSuccessTestCase( + description="drifted settings alter in place and keep the data", + action=PlanAction.ALTER_MAINTAINED, + setup_sql=_BUILT, + refresh_warehouse="transform_wh", + expected_created=(), + expected_altered=(("'1 hour'", "transform_wh"),), + expected_rows=((1,),), + ), + MaintainedSuccessTestCase( + description="a kept model runs no refresh statement", + action=PlanAction.KEEP_MAINTAINED, + setup_sql=_BUILT, + refresh_warehouse=None, + expected_created=(), + expected_altered=(), + expected_rows=((1,),), + ), + ], + ids=lambda case: case.description, +) +def test_given_maintained_entry_when_executing_then_settings_reach_the_warehouse( + test_case: MaintainedSuccessTestCase, connection: Any +) -> None: + adapter: MaintainedDuckDbAdapter = MaintainedDuckDbAdapter(session_warehouse="SESSION_WH") + for sql in test_case.setup_sql: + connection.execute(sql) + + result: ModelExecutionResult = execute_maintained_entry( + context=maintained_context( + adapter=adapter, + connection=connection, + action=test_case.action, + refresh_warehouse=test_case.refresh_warehouse, + ) + ) + + assert result.status == ExecutionStatus.SUCCESS, result.error_message + assert tuple(adapter.created) == test_case.expected_created + assert tuple(adapter.altered) == test_case.expected_altered + assert connection.execute("SELECT orders FROM staging.daily_totals").fetchall() == list( + test_case.expected_rows + ) + assert connection.execute( + "SELECT node_name FROM staging._sqlbuild_fingerprints" + ).fetchall() == [("daily_totals",)] + + +@pytest.mark.parametrize( + "test_case", + [ + MaintainedFailureTestCase( + description="no declared or session warehouse fails before any statement", + setup_sql=(), + session_warehouse=None, + listed_settings=None, + failing_query=None, + expected_error_code="A717", + expected_listed_types=(), + ), + MaintainedFailureTestCase( + description="a failed create never drops a relation listed as a dynamic table", + setup_sql=("CREATE VIEW staging.daily_totals AS SELECT 1 AS orders",), + session_warehouse="SESSION_WH", + listed_settings=MaintainedListing("1 hour", "SESSION_WH", "ACTIVE"), + failing_query=None, + expected_error_code="R002", + expected_listed_types=(("VIEW",),), + ), + MaintainedFailureTestCase( + description="a table the model was before is restored when the create keeps failing", + setup_sql=("CREATE TABLE staging.daily_totals AS SELECT 1 AS orders",), + session_warehouse="SESSION_WH", + listed_settings=None, + failing_query="SELECT missing_column FROM staging.no_such_table", + expected_error_code="R002", + expected_listed_types=(("BASE TABLE",),), + ), + MaintainedFailureTestCase( + description="a view the model was before is restored when the create keeps failing", + setup_sql=("CREATE VIEW staging.daily_totals AS SELECT 1 AS orders",), + session_warehouse="SESSION_WH", + listed_settings=None, + failing_query="SELECT missing_column FROM staging.no_such_table", + expected_error_code="R002", + expected_listed_types=(("VIEW",),), + ), + ], + ids=lambda case: case.description, +) +def test_given_unbuildable_maintained_entry_when_executing_then_fails_keeping_the_relation( + test_case: MaintainedFailureTestCase, connection: Any +) -> None: + adapter: MaintainedDuckDbAdapter = MaintainedDuckDbAdapter( + session_warehouse=test_case.session_warehouse, + listed_settings=test_case.listed_settings, + failing_query=test_case.failing_query, + ) + for sql in test_case.setup_sql: + connection.execute(sql) + + result: ModelExecutionResult = execute_maintained_entry( + context=maintained_context( + adapter=adapter, + connection=connection, + action=PlanAction.CREATE_MAINTAINED, + refresh_warehouse=None, + ) + ) + + assert result.status == ExecutionStatus.FAILED + assert result.error_code == test_case.expected_error_code + assert listed_types(connection=connection) == list(test_case.expected_listed_types) + + +@pytest.mark.parametrize( + "test_case", + [ + FailedRestoreTestCase( + description="a failed create and a failed restore are both reported", + setup_sql=("CREATE TABLE staging.daily_totals AS SELECT 1 AS orders",), + failing_query="SELECT missing_column FROM staging.no_such_table", + failing_renames={ + "staging.daily_totals__replaced": "SELECT restore_column FROM staging.no_restore" + }, + expected_error_code="R002", + expected_fragments=("no_such_table", "no_restore", "daily_totals__replaced"), + expected_relations=(("daily_totals__replaced", "BASE TABLE"),), + ), + ], + ids=lambda case: case.description, +) +def test_given_failed_create_and_restore_when_executing_then_reports_both_errors( + test_case: FailedRestoreTestCase, connection: Any +) -> None: + adapter: MaintainedDuckDbAdapter = MaintainedDuckDbAdapter( + session_warehouse="SESSION_WH", + failing_query=test_case.failing_query, + failing_renames=test_case.failing_renames, + ) + for sql in test_case.setup_sql: + connection.execute(sql) + + result: ModelExecutionResult = execute_maintained_entry( + context=maintained_context( + adapter=adapter, + connection=connection, + action=PlanAction.CREATE_MAINTAINED, + refresh_warehouse=None, + ) + ) + + assert result.status == ExecutionStatus.FAILED + assert result.error_code == test_case.expected_error_code + message: str = str(result.error_message) + assert [fragment in message for fragment in test_case.expected_fragments] == [True] * len( + test_case.expected_fragments + ) + assert connection.execute( + "SELECT table_name, table_type FROM information_schema.tables " + "WHERE table_schema = 'staging' AND table_name LIKE 'daily_totals%' ORDER BY 1" + ).fetchall() == list(test_case.expected_relations) diff --git a/tests/unit/src/sqlbuild/adapter/contract/classes/strict_adapter/test_strict_adapter.py b/tests/unit/src/sqlbuild/adapter/contract/classes/strict_adapter/test_strict_adapter.py index faad1a7145..301ac4504f 100644 --- a/tests/unit/src/sqlbuild/adapter/contract/classes/strict_adapter/test_strict_adapter.py +++ b/tests/unit/src/sqlbuild/adapter/contract/classes/strict_adapter/test_strict_adapter.py @@ -89,8 +89,13 @@ def test_given_first_class_adapters_when_checking_contract_then_no_method_comes_ { "create_sql_test_output", "drop_sql_test_output", + "maintained_listing", + "render_alter_maintained", + "render_create_maintained", "render_create_temporary_function", "render_drop_schema", + "render_maintained_operation", + "session_warehouse", } ), expected_output_relation=None, @@ -124,6 +129,16 @@ def test_given_strict_adapter_when_reading_optional_methods_then_defaults_keep_i assert relation is test_case.expected_output_relation assert temporary is test_case.expected_temporary_function assert drop_error.value.code == "A714" + assert adapter.session_warehouse(connection=None) is None + with pytest.raises(AdapterUserError) as maintained_error: + adapter.render_create_maintained( + destination="main.daily_totals", + sql="SELECT 1", + target_lag="DOWNSTREAM", + refresh_mode="incremental", + warehouse="transform_wh", + ) + assert maintained_error.value.code == "A716" @pytest.mark.parametrize( diff --git a/tests/unit/src/sqlbuild/adapters/snowflake/_test_types.py b/tests/unit/src/sqlbuild/adapters/snowflake/_test_types.py index 0fb684cdf3..e73f7229f2 100644 --- a/tests/unit/src/sqlbuild/adapters/snowflake/_test_types.py +++ b/tests/unit/src/sqlbuild/adapters/snowflake/_test_types.py @@ -3,8 +3,16 @@ from sqlbuild._native import InferredNullability from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.adapter.contract.models import FunctionInfo, RelationUsageReport, SchemaDiffResult -from sqlbuild.adapter.contract.types import TableFreshnessStatus +from sqlbuild.adapter.contract.models import ( + FunctionInfo, + RelationUsageReport, + SchemaDiffResult, +) +from sqlbuild.adapter.contract.types import ( + MaintainedListing, + MaintainedOperation, + TableFreshnessStatus, +) from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter from sqlbuild.cost.types import CostStatus from tests.unit.src.sqlbuild.adapters.snowflake.inspection.helpers import FakeRelation @@ -453,3 +461,41 @@ class SnowflakeKeyedDeleteInsertFailureTestCase: class SnowflakeWarehouseUserTestCase: description: str expected_expression: str + + +@dataclass(frozen=True) +class SnowflakeMaintainedRefreshTestCase: + description: str + rows: tuple[tuple[object, ...], ...] + expected_settings: MaintainedListing | None + + +@dataclass(frozen=True) +class SnowflakeMaintainedCreateTestCase: + description: str + target_lag: str + refresh_mode: str + warehouse: str + expected_sql: str + + +@dataclass(frozen=True) +class SnowflakeMaintainedAlterTestCase: + description: str + target_lag: str + warehouse: str | None + expected_sql: tuple[str, ...] + + +@dataclass(frozen=True) +class SnowflakeMaintainedOperationTestCase: + description: str + operation: MaintainedOperation + expected_sql: str + + +@dataclass(frozen=True) +class SnowflakeChangeTrackingTestCase: + description: str + config: dict[str, object] + expected_sql: str diff --git a/tests/unit/src/sqlbuild/adapters/snowflake/test_maintained_models.py b/tests/unit/src/sqlbuild/adapters/snowflake/test_maintained_models.py new file mode 100644 index 0000000000..a050065022 --- /dev/null +++ b/tests/unit/src/sqlbuild/adapters/snowflake/test_maintained_models.py @@ -0,0 +1,219 @@ +import pytest + +from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder +from sqlbuild.adapter.contract.types import MaintainedListing, MaintainedOperation +from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter +from tests.unit.src.sqlbuild.adapters.helpers import MetadataConnection, execute_through_cursor +from tests.unit.src.sqlbuild.adapters.snowflake._test_types import ( + SnowflakeChangeTrackingTestCase, + SnowflakeMaintainedAlterTestCase, + SnowflakeMaintainedCreateTestCase, + SnowflakeMaintainedOperationTestCase, + SnowflakeMaintainedRefreshTestCase, +) + +_SHOW_DYNAMIC_TABLES_COLUMNS: tuple[str, ...] = ( + "name", + "target_lag", + "warehouse", + "scheduling_state", +) + + +@pytest.mark.parametrize( + "test_case", + [ + SnowflakeMaintainedRefreshTestCase( + description="a dynamic table lists its lag, refresh warehouse and schedule state", + rows=(("DAILY_TOTALS", "1 hour", "TRANSFORM_WH", "SUSPENDED"),), + expected_settings=MaintainedListing("1 hour", "TRANSFORM_WH", "SUSPENDED"), + ), + SnowflakeMaintainedRefreshTestCase( + description="a neighbour matched by the LIKE wildcard is not the model", + rows=(("DAILY_TOTALS_ARCHIVE", "DOWNSTREAM", "TRANSFORM_WH", "ACTIVE"),), + expected_settings=None, + ), + SnowflakeMaintainedRefreshTestCase( + description="a relation that is no dynamic table lists nothing", + rows=(), + expected_settings=None, + ), + ], + ids=lambda case: case.description, +) +def test_given_dynamic_table_listing_when_reading_it_then_reads_one_show( + test_case: SnowflakeMaintainedRefreshTestCase, monkeypatch: pytest.MonkeyPatch +) -> None: + adapter: SnowflakeAdapter = SnowflakeAdapter() + connection: MetadataConnection = MetadataConnection( + test_case.rows, _SHOW_DYNAMIC_TABLES_COLUMNS + ) + monkeypatch.setattr(adapter, "execute", execute_through_cursor) + + settings: MaintainedListing | None = adapter.maintained_listing( + connection=connection, database="ANALYTICS", schema="MARTS", name="daily_totals" + ) + + assert settings == test_case.expected_settings + assert len(connection.executed) == 1 + assert connection.executed[0].startswith( + 'SHOW DYNAMIC TABLES LIKE \'DAILY_TOTALS\' IN SCHEMA "ANALYTICS"."MARTS"' + ) + + +@pytest.mark.parametrize( + "test_case", + [ + SnowflakeMaintainedCreateTestCase( + description="a lag, the default incremental mode and an unquoted warehouse", + target_lag="'1 hour'", + refresh_mode="incremental", + warehouse="transform_wh", + expected_sql=( + "CREATE OR REPLACE DYNAMIC TABLE ANALYTICS.MARTS.DAILY_TOTALS " + "TARGET_LAG = '1 hour' WAREHOUSE = transform_wh REFRESH_MODE = INCREMENTAL " + "INITIALIZE = ON_CREATE AS SELECT 1 AS orders" + ), + ), + SnowflakeMaintainedCreateTestCase( + description="refreshed for its readers, in adaptive mode", + target_lag="DOWNSTREAM", + refresh_mode="adaptive", + warehouse="TRANSFORM_WH", + expected_sql=( + "CREATE OR REPLACE DYNAMIC TABLE ANALYTICS.MARTS.DAILY_TOTALS " + "TARGET_LAG = DOWNSTREAM WAREHOUSE = TRANSFORM_WH REFRESH_MODE = ADAPTIVE " + "INITIALIZE = ON_CREATE AS SELECT 1 AS orders" + ), + ), + SnowflakeMaintainedCreateTestCase( + description="a warehouse name that needs quoting", + target_lag="'5 minutes'", + refresh_mode="full", + warehouse="transform-wh", + expected_sql=( + "CREATE OR REPLACE DYNAMIC TABLE ANALYTICS.MARTS.DAILY_TOTALS " + "TARGET_LAG = '5 minutes' WAREHOUSE = \"transform-wh\" REFRESH_MODE = FULL " + "INITIALIZE = ON_CREATE AS SELECT 1 AS orders" + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_maintained_settings_when_rendering_create_then_mode_and_initialize_are_explicit( + test_case: SnowflakeMaintainedCreateTestCase, +) -> None: + statements: tuple[str, ...] = SnowflakeAdapter().render_create_maintained( + destination="ANALYTICS.MARTS.DAILY_TOTALS", + sql="SELECT 1 AS orders", + target_lag=test_case.target_lag, + refresh_mode=test_case.refresh_mode, + warehouse=test_case.warehouse, + ) + + assert statements == (test_case.expected_sql,) + + +@pytest.mark.parametrize( + "test_case", + [ + SnowflakeMaintainedAlterTestCase( + description="an unset refresh warehouse alters only the lag", + target_lag="'2 hours'", + warehouse=None, + expected_sql=( + "ALTER DYNAMIC TABLE ANALYTICS.MARTS.DAILY_TOTALS SET TARGET_LAG = '2 hours'", + ), + ), + SnowflakeMaintainedAlterTestCase( + description="a declared refresh warehouse is set too", + target_lag="DOWNSTREAM", + warehouse="reporting_wh", + expected_sql=( + "ALTER DYNAMIC TABLE ANALYTICS.MARTS.DAILY_TOTALS SET TARGET_LAG = DOWNSTREAM", + "ALTER DYNAMIC TABLE ANALYTICS.MARTS.DAILY_TOTALS SET WAREHOUSE = reporting_wh", + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_drifted_settings_when_rendering_alter_then_settings_change_in_place( + test_case: SnowflakeMaintainedAlterTestCase, +) -> None: + statements: tuple[str, ...] = SnowflakeAdapter().render_alter_maintained( + destination="ANALYTICS.MARTS.DAILY_TOTALS", + target_lag=test_case.target_lag, + warehouse=test_case.warehouse, + ) + + assert statements == test_case.expected_sql + + +@pytest.mark.parametrize( + "test_case", + [ + SnowflakeMaintainedOperationTestCase( + description="suspend", + operation=MaintainedOperation.SUSPEND, + expected_sql="ALTER DYNAMIC TABLE ANALYTICS.MARTS.DAILY_TOTALS SUSPEND", + ), + SnowflakeMaintainedOperationTestCase( + description="resume", + operation=MaintainedOperation.RESUME, + expected_sql="ALTER DYNAMIC TABLE ANALYTICS.MARTS.DAILY_TOTALS RESUME", + ), + SnowflakeMaintainedOperationTestCase( + description="refresh", + operation=MaintainedOperation.REFRESH, + expected_sql="ALTER DYNAMIC TABLE ANALYTICS.MARTS.DAILY_TOTALS REFRESH", + ), + ], + ids=lambda case: case.description, +) +def test_given_operation_when_rendering_then_one_alter_dynamic_table_statement( + test_case: SnowflakeMaintainedOperationTestCase, +) -> None: + statements: tuple[str, ...] = SnowflakeAdapter().render_maintained_operation( + destination="ANALYTICS.MARTS.DAILY_TOTALS", operation=test_case.operation + ) + + assert statements == (test_case.expected_sql,) + + +@pytest.mark.parametrize( + "test_case", + [ + SnowflakeChangeTrackingTestCase( + description="a table a maintained model reads tracks changes from creation", + config={"table_type": "transient", "change_tracking": True}, + expected_sql=( + "CREATE OR REPLACE TRANSIENT TABLE ANALYTICS.STAGING.ORDERS " + "CHANGE_TRACKING = TRUE AS SELECT 1 AS order_id" + ), + ), + SnowflakeChangeTrackingTestCase( + description="a permanent table read by none keeps the plain form", + config={"table_type": "permanent", "change_tracking": False}, + expected_sql=( + "CREATE OR REPLACE TABLE ANALYTICS.STAGING.ORDERS AS SELECT 1 AS order_id" + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_table_config_when_creating_table_then_change_tracking_follows_it( + test_case: SnowflakeChangeTrackingTestCase, monkeypatch: pytest.MonkeyPatch +) -> None: + adapter: SnowflakeAdapter = SnowflakeAdapter() + connection: MetadataConnection = MetadataConnection(()) + monkeypatch.setattr(adapter, "execute", execute_through_cursor) + + adapter.create_table_as( + connection=connection, + destination="ANALYTICS.STAGING.ORDERS", + sql="SELECT 1 AS order_id", + config=test_case.config, + statement_recorder=StatementRecorder(), + ) + + assert [sql.removesuffix(" ()") for sql in connection.executed] == [test_case.expected_sql] diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_detect.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_detect.py index 8187638751..6edecd18ea 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_detect.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_detect.py @@ -697,6 +697,35 @@ def test_given_model_identity_metadata_when_detecting_changes_then_uses_aligned_ expected_config_changed=True, expected_backfill=BackfillResult(action=BackfillAction.FORWARD_ONLY), ), + DetectFunctionCallerChangeTestCase( + description="a maintained model's refresh_mode change is a config change", + metadata_case=build_function_caller_metadata_case( + config_values={"materialized": "maintained", "refresh_mode": "full"}, + function_local_hashes={}, + previous_function_hashes={}, + previous_config_values={ + "materialized": "maintained", + "refresh_mode": "incremental", + }, + ), + query_change_tracking=True, + expected_changed_functions=(), + expected_config_changed=True, + expected_backfill=BackfillResult(action=BackfillAction.FORWARD_ONLY), + ), + DetectFunctionCallerChangeTestCase( + description="a maintained model's lag change is altered in place, not a config change", + metadata_case=build_function_caller_metadata_case( + config_values={"materialized": "maintained", "refresh": {"lag": "5m"}}, + function_local_hashes={}, + previous_function_hashes={}, + previous_config_values={"materialized": "maintained", "refresh": {"lag": "1h"}}, + ), + query_change_tracking=True, + expected_changed_functions=(), + expected_config_changed=False, + expected_backfill=BackfillResult(action=BackfillAction.FORWARD_ONLY), + ), ], ids=lambda case: case.description, ) diff --git a/ui/src/lib/api/types/operate.ts b/ui/src/lib/api/types/operate.ts index e6ae455140..38eeff8328 100644 --- a/ui/src/lib/api/types/operate.ts +++ b/ui/src/lib/api/types/operate.ts @@ -656,6 +656,10 @@ export type PlanModel = { backfill: { action: string; duration: string | null }; qualified_name?: string; changed_functions?: string[]; + /** The rebuilt upstream that makes this kept maintained model recompute in full. */ + reinitialised_by?: string; + /** Whether the warehouse lists this kept maintained model as suspended. */ + paused?: boolean; cursor_type?: string; cursor?: { requested_bounds: { start: string | null; end: string | null }; resolved_bounds: { start: string | null; end: string | null } | null } | null; /** Unified diff lines of the query since the model's last build, as `sqb plan` prints them. */ diff --git a/ui/src/lib/run/plan-text.test.ts b/ui/src/lib/run/plan-text.test.ts index 86723bb729..e80b045adc 100644 --- a/ui/src/lib/run/plan-text.test.ts +++ b/ui/src/lib/run/plan-text.test.ts @@ -35,6 +35,27 @@ describe('plan text, as sqb plan prints it', () => { expect(actionText(model('a', { reason: 'first_run', backfill: { action: 'full', duration: null } }))).toBe('continue forward'); }); + it('words maintained models as the CLI does, with no replay policy', () => { + const recreated = model('a', { action: 'create_maintained', reason: 'query_changed', backfill: { action: 'full', duration: null } }); + expect(actionText(recreated)).toBe('recreate, full recompute'); + expect(actionText(model('a', { action: 'create_maintained', reason: 'first_run' }))).toBe('create'); + expect(actionText(model('a', { action: 'alter_maintained', reason: 'config_changed' }))).toBe('change refresh settings in place'); + const [section] = planSections(plan([recreated])); + expect(section.entries[0].detail).toEqual([]); + }); + + it('says a kept maintained model recomputes in full when an upstream it reads is rebuilt', () => { + const kept = model('daily_totals', { materialization_type: 'maintained', action: 'keep_maintained', reinitialised_by: 'orders' }); + const [routine] = planSections(plan([kept])); + expect(routine.entries).toEqual([{ name: 'daily_totals', value: 'maintained', detail: ['recomputes in full: orders is rebuilt'] }]); + }); + + it('says a kept maintained model is paused when the warehouse lists it as suspended', () => { + const kept = model('daily_totals', { materialization_type: 'maintained', action: 'keep_maintained', paused: true }); + const [routine] = planSections(plan([kept])); + expect(routine.entries).toEqual([{ name: 'daily_totals', value: 'maintained', detail: ['paused: scheduled refreshes are suspended'] }]); + }); + it('puts each change in its own section in reason order, then the routine models, and leaves skipped ones out', () => { const sections = planSections( plan([ diff --git a/ui/src/lib/run/plan-text.ts b/ui/src/lib/run/plan-text.ts index a9180cf2ea..97825f856f 100644 --- a/ui/src/lib/run/plan-text.ts +++ b/ui/src/lib/run/plan-text.ts @@ -97,6 +97,8 @@ export function materializationLabel(m: PlanModel): string { export function actionText(m: PlanModel): string { if (m.action === 'create_view') return 'recreate view'; if (m.action === 'custom') return 'run custom materialization'; + if (m.action === 'create_maintained') return m.reason === 'first_run' ? 'create' : 'recreate, full recompute'; + if (m.action === 'alter_maintained') return 'change refresh settings in place'; if (m.backfill.action === 'bounded' && m.backfill.duration) return `rebuild last ${m.backfill.duration}`; if (m.backfill.action === 'full' && m.reason !== 'first_run') return 'full rebuild'; return 'continue forward'; @@ -116,7 +118,7 @@ function cursorLines(m: PlanModel, showRange: boolean, windowStart?: string): st } function policyLine(m: PlanModel): string[] { - if (m.backfill.action === 'forward' || !(m.reason === 'query_changed' || m.reason === 'schema_changed')) return []; + if (m.backfill.action === 'forward' || m.action === 'create_maintained' || !(m.reason === 'query_changed' || m.reason === 'schema_changed')) return []; const value = m.backfill.action === 'bounded' ? `bounded-${m.backfill.duration ?? 'full'}` : m.backfill.action; return [`policy: replay_on_change=${value}`]; } @@ -187,7 +189,7 @@ export function planSections(plan: PlanDocument): PlanSection[] { const why = changedRuns(plan); add('Models', active.filter((m) => ROUTINE.has(m.reason)).map((m) => { const run = why.get(m.name); - return { name: m.name, value: materializationLabel(m), detail: [...cursorLines(m, true, starts[m.name]), ...(run ? [run.downstream_of ? `reads ${keyName(run.downstream_of)}` : run.reason] : [])] }; + return { name: m.name, value: materializationLabel(m), detail: [...cursorLines(m, true, starts[m.name]), ...(m.reinitialised_by ? [`recomputes in full: ${m.reinitialised_by} is rebuilt`] : []), ...(m.paused ? ['paused: scheduled refreshes are suspended'] : []), ...(run ? [run.downstream_of ? `reads ${keyName(run.downstream_of)}` : run.reason] : [])] }; }), false, 'routine_models'); } add('Functions', plan.functions.filter((f) => !f.reason || f.reason === 'no_change').map(functionEntry), false, 'functions'); diff --git a/website/src/content/docs/docs/cli/build.mdx b/website/src/content/docs/docs/cli/build.mdx index 195c93647a..31fed956bc 100644 --- a/website/src/content/docs/docs/cli/build.mdx +++ b/website/src/content/docs/docs/cli/build.mdx @@ -49,6 +49,7 @@ own option. `--version` works only directly after `sqb`, as in `sqb --version`: | `--defer-sources-to` | Read managed source data from another target | | `--plan-from` | Read warehouse state `live`, or from the target's `synced_state` checked against the warehouse first; overrides the target's `plan_from`. See [Planning from synced state](/docs/concepts/project-configuration#planning-from-synced-state) | | `--fail-fast` | Stop on first failure and skip remaining nodes | +| `--ignore-maintained-grace` | Take over [maintained models](/docs/concepts/models/materializations#during-a-build) another build holds, without waiting for `[maintained] heartbeat_grace`. Needs `allow_ignore_grace = true`, else fails with `A719` | | `--concurrency` | Number of worker connections (default: 1) | | `--test-timeout ` | Time limit for each unit test the build runs, such as `30s`, or `disabled`; see [Time limits](/docs/concepts/testing-reference#time-limits) | | `--verbose`, `-v` | Show lifecycle SQL inline after each model | diff --git a/website/src/content/docs/docs/cli/dag.mdx b/website/src/content/docs/docs/cli/dag.mdx index dbefae2d51..cd8a98262e 100644 --- a/website/src/content/docs/docs/cli/dag.mdx +++ b/website/src/content/docs/docs/cli/dag.mdx @@ -96,7 +96,7 @@ Each node represents a source, seed, model, or function: | `description` | Model or source description, if declared | | `tags` | Model tags | | `columns` | Column metadata (name, type, nullable, description) | -| `materialization_type` | For models: `view`, `table`, `incremental`, or custom name | +| `materialization_type` | For models: `view`, `table`, `incremental`, `snapshot`, `maintained`, or custom name | | `language` | For functions: `sql` or `python` | | `return_kind` | For functions: `scalar` or `table` | | `arguments` | For functions: argument name and type pairs | diff --git a/website/src/content/docs/docs/concepts/adapters/snowflake.mdx b/website/src/content/docs/docs/concepts/adapters/snowflake.mdx index c0c1319e5b..8dddc56ef3 100644 --- a/website/src/content/docs/docs/concepts/adapters/snowflake.mdx +++ b/website/src/content/docs/docs/concepts/adapters/snowflake.mdx @@ -150,6 +150,28 @@ SQLBuild records its state in `_sqlbuild_*` tables. On Snowflake it creates them with 90 days of time travel, or 1 day on editions that allow only that. It never alters an existing state table. Models are not affected: their table type follows the model's `table_type` setting. +## Maintained models + +A [maintained model](/docs/concepts/models/materializations#maintained) is a Snowflake dynamic +table. `materialized materialized_view` fails with `A014`; use `materialized maintained`. + +The build role needs these grants: + +| Grant | Why | +|-------|-----| +| `CREATE DYNAMIC TABLE` on the model's schema | Create and recreate the model | +| Ownership or `OPERATE` on the dynamic table | Suspend, resume, refresh and alter it during builds | +| `USAGE` on the `refresh_warehouse` | Run the scheduled refreshes | + +Dynamic tables refresh from changes, which needs change tracking on every table they read: + +- A table model that a maintained model reads, directly or through views, is built with change tracking on. +- For a source table, Snowflake turns change tracking on when the build role owns the table. + Otherwise its owner runs `ALTER TABLE SET CHANGE_TRACKING = TRUE`. + +Builds record the models they pause in the `_sqlbuild_maintained_pauses` state table, so another +build can tell whether they are still held. + ## Shared connections across targets Use multiple targets to reuse one Snowflake connection while selecting different authoritative diff --git a/website/src/content/docs/docs/concepts/models/configuration.mdx b/website/src/content/docs/docs/concepts/models/configuration.mdx index ce135807da..c72e3487cf 100644 --- a/website/src/content/docs/docs/concepts/models/configuration.mdx +++ b/website/src/content/docs/docs/concepts/models/configuration.mdx @@ -20,7 +20,7 @@ which accepts any keys. | Field | Description | |-------|-------------| -| `materialized` | `view`, `table`, `incremental`, `snapshot`, or a custom materialization name | +| `materialized` | `view`, `table`, `incremental`, `snapshot`, `maintained`, or a custom materialization name; see [Materializations](/docs/concepts/models/materializations) | | `tags` | Tags used by selectors | | `critical` | `true` marks the model as critical, so [`sqb review`](/docs/cli/review) builds and compares it whenever something upstream of it changes (default `false`) | | `description` | Required human-readable model description; see [Descriptions](#descriptions) | diff --git a/website/src/content/docs/docs/concepts/models/materializations.mdx b/website/src/content/docs/docs/concepts/models/materializations.mdx index 14e18f9791..9fa53d5d07 100644 --- a/website/src/content/docs/docs/concepts/models/materializations.mdx +++ b/website/src/content/docs/docs/concepts/models/materializations.mdx @@ -71,6 +71,78 @@ FROM __source("customers") See [Snapshots](/docs/concepts/snapshots) for timestamp and check strategies, historical inputs, and full-refresh policies. +## Maintained + +The warehouse keeps a maintained model up to date on a schedule you set, between builds. On Snowflake it is a dynamic table; a build creates it, changes it when its definition changes, and otherwise leaves the refreshing to Snowflake. + +```sql +MODEL ( + description "One row per customer per order date", + materialized maintained, + refresh (lag 1h), +); + +SELECT customer_id, CAST(ordered_at AS DATE) AS order_date, SUM(amount) AS revenue +FROM __ref("stg_orders") +GROUP BY customer_id, CAST(ordered_at AS DATE) +``` + +| Field | Default | Description | +|-------|---------|-------------| +| `refresh (lag )` | none | How far behind its inputs the model may fall: a fixed duration such as `5m`, `1h` or `1d`, at least `1m` on Snowflake and no shorter than the lag of a maintained model it reads. Without it, the model refreshes when a maintained model that reads it refreshes | +| `refresh_mode` | `incremental` | `incremental`, `full`, `auto` or `adaptive`. Snowflake rejects `incremental` for a query it can't refresh incrementally; use `full` for that query | +| `refresh_warehouse` | the connection's warehouse | Warehouse the scheduled refreshes run on | +| `refresh_on_build` | `false` | Refresh the model in every build that selects it, not only on its schedule | + +Only Snowflake supports maintained models. See [Snowflake](/docs/concepts/adapters/snowflake#maintained-models) for the grants and change tracking they need. + +### What the plan shows + +| Change | Plan | What the build does | +|--------|------|---------------------| +| First build, or the dynamic table is missing | `create` | Creates it and fills it | +| Query or `refresh_mode` changed | `recreate, full recompute` | Replaces it; every row is computed again | +| `refresh` lag or `refresh_warehouse` differs from the warehouse | `change refresh settings in place` | Alters the schedule; rows stay | +| Nothing | `maintained` | Leaves it alone | + +A model the plan keeps can carry one more line: + +- `recomputes in full: stg_orders is rebuilt`: the build replaces a model it reads, so its next refresh computes every row again. +- `paused: scheduled refreshes are suspended`: Snowflake lists it as suspended, so it is not refreshing. + +### During a build + +A build pauses the maintained models it changes, and those that read a model it rebuilds, directly or through views, so they never refresh from half-built inputs. It resumes them when it ends, whether it succeeds or fails. + +- Dynamic tables outside the project that read a paused model pause too, until the build resumes it. One that someone had suspended before stays suspended. +- A model that was already suspended when the build started stays suspended, and the build warns about it. +- If someone suspends a model during the build, the build's resume undoes that suspend. +- A kept model is refreshed during the build when a model built later in the same build reads it, so that model reads current rows. `refresh_on_build` refreshes it in every build. +- A target with `pause_maintained = true` leaves its maintained models suspended between builds and refreshes them only in builds; see [Targets](/docs/concepts/project-configuration#targets). + +Two builds can't pause the same models at once. A build that needs a model another build holds fails with `A718`, naming that run. When two builds pause a model at the same moment, the one that recorded its pause first keeps it, and the other fails with `A718` without suspending or resuming anything. A build that stops without resuming, for example because its process was killed, leaves its models suspended. The next build that pauses maintained models on the target resumes them once that build has been silent for `[maintained] heartbeat_grace` (see [Maintained](/docs/concepts/project-configuration#maintained)). Until then, resume them in Snowflake by hand, or run `sqb build --ignore-maintained-grace` where the project allows it. + +### Error codes + +| Code | When | Fix | +|------|------|-----| +| `A013` | `materialized maintained` on an adapter other than Snowflake | Use `table` or `incremental` on that adapter | +| `A014` | `materialized materialized_view` on Snowflake | Use `materialized maintained` | +| `A015` | A `refresh` lag under `1m` on Snowflake | Set a lag of at least `1m` | +| `A016` | A model with no `refresh` that no maintained model reads, so it would never refresh | Add `refresh (lag 1h)` or another lag | +| `A716` | The adapter can't build a maintained model | Use an adapter that supports them | +| `A717` | No warehouse to refresh with | Set `refresh_warehouse`, or a `warehouse` on the connection | +| `A718` | Another build holds the models this build needs | Wait for it to finish, or for its hold to pass `heartbeat_grace` | +| `A719` | `--ignore-maintained-grace` without `allow_ignore_grace = true` | Set `[maintained] allow_ignore_grace = true`, or wait for the grace to pass | + +### Limitations + +- Writing out the default `refresh_mode incremental` changes the model's definition, so the next build recreates it once. +- A custom materialization named `maintained` is not used; the built-in one takes the name. +- Unit tests and scenarios treat a maintained model as a table. +- On a target with `schema = "preserve"`, builds see each other's pauses only when their maintained models share a schema. +- A snapshot rebuilt in full is created without change tracking. Before a maintained model reads it again, run `ALTER TABLE SET CHANGE_TRACKING = TRUE`. + ## Custom A project-local Python materialization can manage specialized persistence with adapter access, schema findings, query-change state, declared columns, and `ctx.run_audits`. @@ -108,4 +180,5 @@ The custom function owns staging, runtime type or contract enforcement, audit ti | Table (`immediate`) | The new table. Audits run after it is replaced. | | Incremental | The old table if a delta audit fails. Rows already written stay if a final audit or a later microbatch fails. | | Microbatch full rebuild | A partly rebuilt table; the old one was dropped first. | +| Maintained | The created or altered dynamic table, resumed. Audits run after it changes. | | Custom | Whatever the custom materialization does. | diff --git a/website/src/content/docs/docs/concepts/project-configuration.mdx b/website/src/content/docs/docs/concepts/project-configuration.mdx index 30f05b29f4..2d11732db2 100644 --- a/website/src/content/docs/docs/concepts/project-configuration.mdx +++ b/website/src/content/docs/docs/concepts/project-configuration.mdx @@ -163,6 +163,7 @@ A target is a named build context - the database and schema you build into, plus | `concurrency` | Worker limit for commands run against this target, replacing `[settings] concurrency` (see below) | | `ignore_audit_failures` | Whether builds on this target let audit failures through, replacing `[settings] ignore_audit_failures`; see [Ignoring audit failures](/docs/concepts/audits#ignoring-audit-failures) | | `plan_from` | Where `sqb plan` and `sqb build` read warehouse state: `live` (default) or `synced_state`; see [Planning from synced state](#planning-from-synced-state) | +| `pause_maintained` | `true` keeps this target's [maintained models](/docs/concepts/models/materializations#maintained) suspended between builds; each build refreshes the ones it selects. Default `false` | ```toml [connections.warehouse] @@ -669,6 +670,21 @@ old_name_views = "30d" |-------|---------|-------------| | `old_name_views` | `"30d"` | How long a renamed model's old name keeps working through a [compatibility view](/docs/concepts/models/migrations#old-names) before janitor drops it. `false` keeps the old relation in place instead. A model's `old_name_view` header overrides it. | +## Maintained + +How builds pause [maintained models](/docs/concepts/models/materializations#maintained) while they run: + +```toml +[maintained] +heartbeat_grace = "30m" +allow_ignore_grace = false +``` + +| Field | Default | Description | +|-------|---------|-------------| +| `heartbeat_grace` | `"30m"` | How long a build that paused maintained models may go silent before another build resumes them or takes them over. A fixed duration longer than `"1m"`, such as `"30m"` or `"2h"`; a build writes a heartbeat every minute | +| `allow_ignore_grace` | `false` | Allow `sqb build --ignore-maintained-grace`, which takes over models another build holds without waiting for the grace. Without it the flag fails with `A719` | + ## Scenario Configuration for scenario run isolation, snapshot capture safety limits, and local type overrides: