diff --git a/crates/sqlbuild-analysis/src/assembly/dynamic_checks/constants.rs b/crates/sqlbuild-analysis/src/assembly/dynamic_checks/constants.rs new file mode 100644 index 0000000000..f3d94893ac --- /dev/null +++ b/crates/sqlbuild-analysis/src/assembly/dynamic_checks/constants.rs @@ -0,0 +1,4 @@ +//! Error codes of the project-wide dynamic model checks. + +/// A dynamic model that nothing would ever refresh. +pub const DYNAMIC_NEVER_REFRESHED_CODE: &str = "A016"; diff --git a/crates/sqlbuild-analysis/src/assembly/dynamic_checks/main/mod.rs b/crates/sqlbuild-analysis/src/assembly/dynamic_checks/main/mod.rs new file mode 100644 index 0000000000..cb3c63689b --- /dev/null +++ b/crates/sqlbuild-analysis/src/assembly/dynamic_checks/main/mod.rs @@ -0,0 +1 @@ +pub mod never_refreshed_dynamic; diff --git a/crates/sqlbuild-analysis/src/assembly/maintained_checks/main/never_refreshed_maintained.rs b/crates/sqlbuild-analysis/src/assembly/dynamic_checks/main/never_refreshed_dynamic.rs similarity index 53% rename from crates/sqlbuild-analysis/src/assembly/maintained_checks/main/never_refreshed_maintained.rs rename to crates/sqlbuild-analysis/src/assembly/dynamic_checks/main/never_refreshed_dynamic.rs index 078332bc87..74ecb201d2 100644 --- a/crates/sqlbuild-analysis/src/assembly/maintained_checks/main/never_refreshed_maintained.rs +++ b/crates/sqlbuild-analysis/src/assembly/dynamic_checks/main/never_refreshed_dynamic.rs @@ -1,16 +1,16 @@ -//! Maintained models with no refresh of their own that no maintained model reads. +//! Dynamic models with no refresh of their own that no dynamic model reads. use std::collections::HashSet; -use crate::assembly::maintained_checks::models::MaintainedModel; +use crate::assembly::dynamic_checks::models::DynamicModel; /// The first by name; without a reader or its own refresh, the warehouse never refreshes it. -pub fn never_refreshed_maintained(models: &[MaintainedModel]) -> Option<&str> { - let maintained: HashSet<&str> = models.iter().map(|model| model.name.as_str()).collect(); +pub fn never_refreshed_dynamic(models: &[DynamicModel]) -> Option<&str> { + let dynamic: HashSet<&str> = models.iter().map(|model| model.name.as_str()).collect(); let read: HashSet<&str> = models .iter() .flat_map(|model| model.reads.iter().map(String::as_str)) - .filter(|name| maintained.contains(name)) + .filter(|name| dynamic.contains(name)) .collect(); models .iter() diff --git a/crates/sqlbuild-analysis/src/assembly/maintained_checks/mod.rs b/crates/sqlbuild-analysis/src/assembly/dynamic_checks/mod.rs similarity index 100% rename from crates/sqlbuild-analysis/src/assembly/maintained_checks/mod.rs rename to crates/sqlbuild-analysis/src/assembly/dynamic_checks/mod.rs diff --git a/crates/sqlbuild-analysis/src/assembly/dynamic_checks/models.rs b/crates/sqlbuild-analysis/src/assembly/dynamic_checks/models.rs new file mode 100644 index 0000000000..9d23af7451 --- /dev/null +++ b/crates/sqlbuild-analysis/src/assembly/dynamic_checks/models.rs @@ -0,0 +1,9 @@ +//! What the project-wide dynamic model checks read. + +/// One dynamic model: whether it sets its own refresh, and the models it reads. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct DynamicModel { + pub name: String, + pub has_refresh: bool, + pub reads: Vec, +} diff --git a/crates/sqlbuild-analysis/src/assembly/maintained_checks/tests.rs b/crates/sqlbuild-analysis/src/assembly/dynamic_checks/tests.rs similarity index 57% rename from crates/sqlbuild-analysis/src/assembly/maintained_checks/tests.rs rename to crates/sqlbuild-analysis/src/assembly/dynamic_checks/tests.rs index e40ce2efc9..66cecbeac6 100644 --- a/crates/sqlbuild-analysis/src/assembly/maintained_checks/tests.rs +++ b/crates/sqlbuild-analysis/src/assembly/dynamic_checks/tests.rs @@ -1,6 +1,6 @@ +#[path = "tests/test_dynamic_checks.rs"] +mod dynamic_checks; #[path = "tests/helpers.rs"] mod helpers; -#[path = "tests/test_maintained_checks.rs"] -mod maintained_checks; #[path = "tests/test_types.rs"] mod test_types; diff --git a/crates/sqlbuild-analysis/src/assembly/dynamic_checks/tests/helpers.rs b/crates/sqlbuild-analysis/src/assembly/dynamic_checks/tests/helpers.rs new file mode 100644 index 0000000000..b1d8b22730 --- /dev/null +++ b/crates/sqlbuild-analysis/src/assembly/dynamic_checks/tests/helpers.rs @@ -0,0 +1,10 @@ +use crate::assembly::dynamic_checks::models::DynamicModel; + +/// A dynamic model named `name` reading `reads`. +pub(crate) fn dynamic(name: &str, has_refresh: bool, reads: &[&str]) -> DynamicModel { + DynamicModel { + name: name.to_owned(), + has_refresh, + reads: reads.iter().map(|read| (*read).to_owned()).collect(), + } +} diff --git a/crates/sqlbuild-analysis/src/assembly/maintained_checks/tests/test_maintained_checks.rs b/crates/sqlbuild-analysis/src/assembly/dynamic_checks/tests/test_dynamic_checks.rs similarity index 54% rename from crates/sqlbuild-analysis/src/assembly/maintained_checks/tests/test_maintained_checks.rs rename to crates/sqlbuild-analysis/src/assembly/dynamic_checks/tests/test_dynamic_checks.rs index f363cfca59..577186ee4e 100644 --- a/crates/sqlbuild-analysis/src/assembly/maintained_checks/tests/test_maintained_checks.rs +++ b/crates/sqlbuild-analysis/src/assembly/dynamic_checks/tests/test_dynamic_checks.rs @@ -1,49 +1,49 @@ -use crate::assembly::maintained_checks::main::never_refreshed_maintained::never_refreshed_maintained; -use crate::assembly::maintained_checks::tests::helpers::maintained; -use crate::assembly::maintained_checks::tests::test_types::NeverRefreshedTestCase; +use crate::assembly::dynamic_checks::main::never_refreshed_dynamic::never_refreshed_dynamic; +use crate::assembly::dynamic_checks::tests::helpers::dynamic; +use crate::assembly::dynamic_checks::tests::test_types::NeverRefreshedTestCase; #[test] -fn given_maintained_chains_when_checking_then_unreached_models_without_refresh_are_named() { +fn given_dynamic_chains_when_checking_then_unreached_models_without_refresh_are_named() { let test_cases = [ NeverRefreshedTestCase { description: "an upstream model refreshed for its reader", models: vec![ - maintained("clean_orders", false, &["orders"]), - maintained("daily_totals", true, &["clean_orders"]), + dynamic("clean_orders", false, &["orders"]), + dynamic("daily_totals", true, &["clean_orders"]), ], expected_name: None, }, NeverRefreshedTestCase { description: "the last model of a chain without refresh", models: vec![ - maintained("clean_orders", false, &["orders"]), - maintained("daily_totals", false, &["clean_orders"]), + dynamic("clean_orders", false, &["orders"]), + dynamic("daily_totals", false, &["clean_orders"]), ], expected_name: Some("daily_totals"), }, NeverRefreshedTestCase { description: "a model read only by a table, which never asks for a refresh", - models: vec![maintained("clean_orders", false, &["orders"])], + models: vec![dynamic("clean_orders", false, &["orders"])], expected_name: Some("clean_orders"), }, NeverRefreshedTestCase { description: "two unread models report the first by name", models: vec![ - maintained("b_totals", false, &["orders"]), - maintained("a_totals", false, &["orders"]), + dynamic("b_totals", false, &["orders"]), + dynamic("a_totals", false, &["orders"]), ], expected_name: Some("a_totals"), }, NeverRefreshedTestCase { description: "every model sets its own refresh", - models: vec![maintained("daily_totals", true, &[])], + models: vec![dynamic("daily_totals", true, &[])], expected_name: None, }, ]; for test_case in test_cases { assert_eq!( - never_refreshed_maintained(&test_case.models), + never_refreshed_dynamic(&test_case.models), test_case.expected_name, "{}", test_case.description diff --git a/crates/sqlbuild-analysis/src/assembly/maintained_checks/tests/test_types.rs b/crates/sqlbuild-analysis/src/assembly/dynamic_checks/tests/test_types.rs similarity index 56% rename from crates/sqlbuild-analysis/src/assembly/maintained_checks/tests/test_types.rs rename to crates/sqlbuild-analysis/src/assembly/dynamic_checks/tests/test_types.rs index 66f9fbe2f1..0bd0f37594 100644 --- a/crates/sqlbuild-analysis/src/assembly/maintained_checks/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/assembly/dynamic_checks/tests/test_types.rs @@ -1,7 +1,7 @@ -use crate::assembly::maintained_checks::models::MaintainedModel; +use crate::assembly::dynamic_checks::models::DynamicModel; pub(crate) struct NeverRefreshedTestCase { pub(crate) description: &'static str, - pub(crate) models: Vec, + pub(crate) models: Vec, pub(crate) expected_name: Option<&'static str>, } diff --git a/crates/sqlbuild-analysis/src/assembly/maintained_checks/constants.rs b/crates/sqlbuild-analysis/src/assembly/maintained_checks/constants.rs deleted file mode 100644 index 385fec1fd5..0000000000 --- a/crates/sqlbuild-analysis/src/assembly/maintained_checks/constants.rs +++ /dev/null @@ -1,4 +0,0 @@ -//! Error codes of the project-wide maintained model checks. - -/// A maintained model that nothing would ever refresh. -pub const MAINTAINED_NEVER_REFRESHED_CODE: &str = "A016"; diff --git a/crates/sqlbuild-analysis/src/assembly/maintained_checks/main/mod.rs b/crates/sqlbuild-analysis/src/assembly/maintained_checks/main/mod.rs deleted file mode 100644 index d43fa0186a..0000000000 --- a/crates/sqlbuild-analysis/src/assembly/maintained_checks/main/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub mod never_refreshed_maintained; diff --git a/crates/sqlbuild-analysis/src/assembly/maintained_checks/models.rs b/crates/sqlbuild-analysis/src/assembly/maintained_checks/models.rs deleted file mode 100644 index 4724ca32f3..0000000000 --- a/crates/sqlbuild-analysis/src/assembly/maintained_checks/models.rs +++ /dev/null @@ -1,9 +0,0 @@ -//! What the project-wide maintained model checks read. - -/// One maintained model: whether it sets its own refresh, and the models it reads. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct MaintainedModel { - pub name: String, - pub has_refresh: bool, - pub reads: Vec, -} diff --git a/crates/sqlbuild-analysis/src/assembly/maintained_checks/tests/helpers.rs b/crates/sqlbuild-analysis/src/assembly/maintained_checks/tests/helpers.rs deleted file mode 100644 index 2ad27cec20..0000000000 --- a/crates/sqlbuild-analysis/src/assembly/maintained_checks/tests/helpers.rs +++ /dev/null @@ -1,10 +0,0 @@ -use crate::assembly::maintained_checks::models::MaintainedModel; - -/// A maintained model named `name` reading `reads`. -pub(crate) fn maintained(name: &str, has_refresh: bool, reads: &[&str]) -> MaintainedModel { - MaintainedModel { - name: name.to_owned(), - has_refresh, - reads: reads.iter().map(|read| (*read).to_owned()).collect(), - } -} diff --git a/crates/sqlbuild-analysis/src/assembly/mod.rs b/crates/sqlbuild-analysis/src/assembly/mod.rs index 4e62fc77e8..4405d0dfa9 100644 --- a/crates/sqlbuild-analysis/src/assembly/mod.rs +++ b/crates/sqlbuild-analysis/src/assembly/mod.rs @@ -1,6 +1,6 @@ //! The native model analysis session: catalog, dataflow scheduling, projection and shape publication. pub mod analysis_session; -pub mod maintained_checks; +pub mod dynamic_checks; pub mod project; pub mod target_checks; diff --git a/crates/sqlbuild-analysis/src/maintained_refresh/_helpers/mod.rs b/crates/sqlbuild-analysis/src/dynamic_refresh/_helpers/mod.rs similarity index 100% rename from crates/sqlbuild-analysis/src/maintained_refresh/_helpers/mod.rs rename to crates/sqlbuild-analysis/src/dynamic_refresh/_helpers/mod.rs diff --git a/crates/sqlbuild-analysis/src/maintained_refresh/_helpers/target_lag.rs b/crates/sqlbuild-analysis/src/dynamic_refresh/_helpers/target_lag.rs similarity index 85% rename from crates/sqlbuild-analysis/src/maintained_refresh/_helpers/target_lag.rs rename to crates/sqlbuild-analysis/src/dynamic_refresh/_helpers/target_lag.rs index 6123f16214..764a87e693 100644 --- a/crates/sqlbuild-analysis/src/maintained_refresh/_helpers/target_lag.rs +++ b/crates/sqlbuild-analysis/src/dynamic_refresh/_helpers/target_lag.rs @@ -1,7 +1,7 @@ //! A dynamic table's listed `target_lag`, such as `DOWNSTREAM` or `2 hours`. -use crate::maintained_refresh::constants::{DOWNSTREAM_TARGET_LAG, TARGET_LAG_UNITS}; -use crate::maintained_refresh::models::RefreshLag; +use crate::dynamic_refresh::constants::{DOWNSTREAM_TARGET_LAG, TARGET_LAG_UNITS}; +use crate::dynamic_refresh::models::RefreshLag; /// The lag `text` lists, or `None` when it is not a lag Snowflake lists. pub(crate) fn parse_target_lag(text: &str) -> Option { diff --git a/crates/sqlbuild-analysis/src/maintained_refresh/constants.rs b/crates/sqlbuild-analysis/src/dynamic_refresh/constants.rs similarity index 100% rename from crates/sqlbuild-analysis/src/maintained_refresh/constants.rs rename to crates/sqlbuild-analysis/src/dynamic_refresh/constants.rs diff --git a/crates/sqlbuild-analysis/src/dynamic_refresh/main/dynamic_settings_drift.rs b/crates/sqlbuild-analysis/src/dynamic_refresh/main/dynamic_settings_drift.rs new file mode 100644 index 0000000000..db0191481c --- /dev/null +++ b/crates/sqlbuild-analysis/src/dynamic_refresh/main/dynamic_settings_drift.rs @@ -0,0 +1,15 @@ +//! Whether a dynamic model's listed refresh settings differ from its declared ones. + +use crate::dynamic_refresh::_helpers::target_lag::parse_target_lag; +use crate::dynamic_refresh::models::{DynamicSettings, LiveDynamicTable}; + +/// True when the lag differs or cannot be read, or a declared warehouse differs. +pub fn dynamic_settings_drift(desired: &DynamicSettings, live: &LiveDynamicTable) -> bool { + if parse_target_lag(&live.target_lag) != Some(desired.lag) { + return true; + } + desired + .warehouse + .as_deref() + .is_some_and(|warehouse| !warehouse.eq_ignore_ascii_case(live.warehouse.trim())) +} diff --git a/crates/sqlbuild-analysis/src/maintained_refresh/main/mod.rs b/crates/sqlbuild-analysis/src/dynamic_refresh/main/mod.rs similarity index 62% rename from crates/sqlbuild-analysis/src/maintained_refresh/main/mod.rs rename to crates/sqlbuild-analysis/src/dynamic_refresh/main/mod.rs index a48b621304..d9a08bc028 100644 --- a/crates/sqlbuild-analysis/src/maintained_refresh/main/mod.rs +++ b/crates/sqlbuild-analysis/src/dynamic_refresh/main/mod.rs @@ -1,3 +1,3 @@ -pub mod maintained_settings_drift; +pub mod dynamic_settings_drift; pub mod reinitialised_readers; pub mod render_target_lag; diff --git a/crates/sqlbuild-analysis/src/maintained_refresh/main/reinitialised_readers.rs b/crates/sqlbuild-analysis/src/dynamic_refresh/main/reinitialised_readers.rs similarity index 60% rename from crates/sqlbuild-analysis/src/maintained_refresh/main/reinitialised_readers.rs rename to crates/sqlbuild-analysis/src/dynamic_refresh/main/reinitialised_readers.rs index 82f825c8f9..da5b210da9 100644 --- a/crates/sqlbuild-analysis/src/maintained_refresh/main/reinitialised_readers.rs +++ b/crates/sqlbuild-analysis/src/dynamic_refresh/main/reinitialised_readers.rs @@ -1,21 +1,18 @@ -//! Maintained models the plan keeps whose upstream it replaces, so they recompute in full. +//! Dynamic models the plan keeps whose upstream it replaces, so they recompute in full. use std::collections::BTreeSet; -use crate::maintained_refresh::models::MaintainedRead; +use crate::dynamic_refresh::models::DynamicRead; -/// Each kept maintained reader by name, with the first replaced upstream it reads by name. -pub fn reinitialised_readers(reads: &[MaintainedRead]) -> Vec<(String, String)> { +/// Each kept dynamic reader by name, with the first replaced upstream it reads by name. +pub fn reinitialised_readers(reads: &[DynamicRead]) -> Vec<(String, String)> { let replaced: BTreeSet<&str> = reads .iter() .filter(|read| read.replaced) .map(|read| read.name.as_str()) .collect(); let mut readers: Vec<(String, String)> = Vec::new(); - for read in reads - .iter() - .filter(|read| read.maintained && !read.replaced) - { + for read in reads.iter().filter(|read| read.dynamic && !read.replaced) { let mut upstreams: Vec<&str> = read .reads .iter() diff --git a/crates/sqlbuild-analysis/src/maintained_refresh/main/render_target_lag.rs b/crates/sqlbuild-analysis/src/dynamic_refresh/main/render_target_lag.rs similarity index 81% rename from crates/sqlbuild-analysis/src/maintained_refresh/main/render_target_lag.rs rename to crates/sqlbuild-analysis/src/dynamic_refresh/main/render_target_lag.rs index ca61aa4d47..c76758be3e 100644 --- a/crates/sqlbuild-analysis/src/maintained_refresh/main/render_target_lag.rs +++ b/crates/sqlbuild-analysis/src/dynamic_refresh/main/render_target_lag.rs @@ -1,7 +1,7 @@ //! The `TARGET_LAG` value of a refresh lag, in its largest exact unit. -use crate::maintained_refresh::constants::{DOWNSTREAM_TARGET_LAG, TARGET_LAG_UNITS}; -use crate::maintained_refresh::models::RefreshLag; +use crate::dynamic_refresh::constants::{DOWNSTREAM_TARGET_LAG, TARGET_LAG_UNITS}; +use crate::dynamic_refresh::models::RefreshLag; /// `DOWNSTREAM`, or a quoted lag such as `'2 hours'`. pub fn render_target_lag(lag: RefreshLag) -> String { diff --git a/crates/sqlbuild-analysis/src/dynamic_refresh/mod.rs b/crates/sqlbuild-analysis/src/dynamic_refresh/mod.rs new file mode 100644 index 0000000000..2c9f901902 --- /dev/null +++ b/crates/sqlbuild-analysis/src/dynamic_refresh/mod.rs @@ -0,0 +1,9 @@ +//! Dynamic refresh: dynamic table lag text, refresh setting drift, and readers a rebuild reinitialises. + +pub(crate) mod _helpers; +pub mod constants; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-analysis/src/maintained_refresh/models.rs b/crates/sqlbuild-analysis/src/dynamic_refresh/models.rs similarity index 51% rename from crates/sqlbuild-analysis/src/maintained_refresh/models.rs rename to crates/sqlbuild-analysis/src/dynamic_refresh/models.rs index 057d87a6e7..68b0d154a9 100644 --- a/crates/sqlbuild-analysis/src/maintained_refresh/models.rs +++ b/crates/sqlbuild-analysis/src/dynamic_refresh/models.rs @@ -1,34 +1,34 @@ -//! Declared and listed refresh settings of maintained models. +//! Declared and listed refresh settings of dynamic models. -/// How stale a maintained model may get: refreshed for its readers, or within a fixed lag. +/// How stale a dynamic model may get: refreshed for its readers, or within a fixed lag. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum RefreshLag { - /// Refreshed when the maintained models reading it need it. + /// Refreshed when the dynamic models reading it need it. Downstream, /// Refreshed within this many seconds. Seconds(u64), } -/// The refresh settings a maintained model declares. +/// The refresh settings a dynamic model declares. #[derive(Clone, Debug, PartialEq, Eq)] -pub struct MaintainedSettings { +pub struct DynamicSettings { pub lag: RefreshLag, /// The declared refresh warehouse; unset keeps whichever the object has. pub warehouse: Option, } -/// A maintained model's refresh settings as the warehouse lists them. +/// A dynamic model's refresh settings as the warehouse lists them. #[derive(Clone, Debug, PartialEq, Eq)] pub struct LiveDynamicTable { pub target_lag: String, pub warehouse: String, } -/// One planned model: whether it is maintained, whether this plan replaces it, and what it reads. +/// One planned model: whether it is dynamic, whether this plan replaces it, and what it reads. #[derive(Clone, Debug, PartialEq, Eq)] -pub struct MaintainedRead { +pub struct DynamicRead { pub name: String, - pub maintained: bool, + pub dynamic: bool, pub replaced: bool, pub reads: Vec, } diff --git a/crates/sqlbuild-analysis/src/maintained_refresh/tests.rs b/crates/sqlbuild-analysis/src/dynamic_refresh/tests.rs similarity index 56% rename from crates/sqlbuild-analysis/src/maintained_refresh/tests.rs rename to crates/sqlbuild-analysis/src/dynamic_refresh/tests.rs index 0b70bcf139..141538094c 100644 --- a/crates/sqlbuild-analysis/src/maintained_refresh/tests.rs +++ b/crates/sqlbuild-analysis/src/dynamic_refresh/tests.rs @@ -1,6 +1,6 @@ +#[path = "tests/test_dynamic_refresh.rs"] +mod dynamic_refresh; #[path = "tests/helpers.rs"] mod helpers; -#[path = "tests/test_maintained_refresh.rs"] -mod maintained_refresh; #[path = "tests/test_types.rs"] mod test_types; diff --git a/crates/sqlbuild-analysis/src/maintained_refresh/tests/helpers.rs b/crates/sqlbuild-analysis/src/dynamic_refresh/tests/helpers.rs similarity index 59% rename from crates/sqlbuild-analysis/src/maintained_refresh/tests/helpers.rs rename to crates/sqlbuild-analysis/src/dynamic_refresh/tests/helpers.rs index 9fa27f7945..3f10885306 100644 --- a/crates/sqlbuild-analysis/src/maintained_refresh/tests/helpers.rs +++ b/crates/sqlbuild-analysis/src/dynamic_refresh/tests/helpers.rs @@ -1,9 +1,7 @@ -use crate::maintained_refresh::models::{ - LiveDynamicTable, MaintainedRead, MaintainedSettings, RefreshLag, -}; +use crate::dynamic_refresh::models::{DynamicRead, DynamicSettings, LiveDynamicTable, RefreshLag}; -pub(super) fn settings(lag: RefreshLag, warehouse: Option<&str>) -> MaintainedSettings { - MaintainedSettings { +pub(super) fn settings(lag: RefreshLag, warehouse: Option<&str>) -> DynamicSettings { + DynamicSettings { lag, warehouse: warehouse.map(str::to_owned), } @@ -16,10 +14,10 @@ pub(super) fn live(target_lag: &str, warehouse: &str) -> LiveDynamicTable { } } -pub(super) fn read(name: &str, maintained: bool, replaced: bool, reads: &[&str]) -> MaintainedRead { - MaintainedRead { +pub(super) fn read(name: &str, dynamic: bool, replaced: bool, reads: &[&str]) -> DynamicRead { + DynamicRead { name: name.to_owned(), - maintained, + dynamic, replaced, reads: reads.iter().map(|read| (*read).to_owned()).collect(), } diff --git a/crates/sqlbuild-analysis/src/maintained_refresh/tests/test_maintained_refresh.rs b/crates/sqlbuild-analysis/src/dynamic_refresh/tests/test_dynamic_refresh.rs similarity index 88% rename from crates/sqlbuild-analysis/src/maintained_refresh/tests/test_maintained_refresh.rs rename to crates/sqlbuild-analysis/src/dynamic_refresh/tests/test_dynamic_refresh.rs index 145cdbf5bd..6938568355 100644 --- a/crates/sqlbuild-analysis/src/maintained_refresh/tests/test_maintained_refresh.rs +++ b/crates/sqlbuild-analysis/src/dynamic_refresh/tests/test_dynamic_refresh.rs @@ -1,10 +1,10 @@ -use crate::maintained_refresh::_helpers::target_lag::parse_target_lag; -use crate::maintained_refresh::main::maintained_settings_drift::maintained_settings_drift; -use crate::maintained_refresh::main::reinitialised_readers::reinitialised_readers; -use crate::maintained_refresh::main::render_target_lag::render_target_lag; -use crate::maintained_refresh::models::RefreshLag; -use crate::maintained_refresh::tests::helpers::{live, read, settings}; -use crate::maintained_refresh::tests::test_types::{ +use crate::dynamic_refresh::_helpers::target_lag::parse_target_lag; +use crate::dynamic_refresh::main::dynamic_settings_drift::dynamic_settings_drift; +use crate::dynamic_refresh::main::reinitialised_readers::reinitialised_readers; +use crate::dynamic_refresh::main::render_target_lag::render_target_lag; +use crate::dynamic_refresh::models::RefreshLag; +use crate::dynamic_refresh::tests::helpers::{live, read, settings}; +use crate::dynamic_refresh::tests::test_types::{ DriftTestCase, ReinitialiseTestCase, RenderLagTestCase, TargetLagTestCase, }; @@ -139,7 +139,7 @@ fn given_live_dynamic_table_when_comparing_settings_then_only_real_drift_alters( ]; for test_case in test_cases { assert_eq!( - maintained_settings_drift(&test_case.desired, &test_case.live), + dynamic_settings_drift(&test_case.desired, &test_case.live), test_case.expected_drift, "{}", test_case.description @@ -148,10 +148,10 @@ fn given_live_dynamic_table_when_comparing_settings_then_only_real_drift_alters( } #[test] -fn given_planned_rebuilds_when_checking_maintained_readers_then_reinitialised_ones_are_named() { +fn given_planned_rebuilds_when_checking_dynamic_readers_then_reinitialised_ones_are_named() { let test_cases = [ ReinitialiseTestCase { - description: "a maintained model reading a rebuilt table", + description: "a dynamic model reading a rebuilt table", reads: vec![ read("orders", false, true, &[]), read("daily_totals", true, false, &["orders"]), @@ -167,7 +167,7 @@ fn given_planned_rebuilds_when_checking_maintained_readers_then_reinitialised_on expected_readers: vec![], }, ReinitialiseTestCase { - description: "a recreated maintained model reinitialises its maintained readers", + description: "a recreated dynamic model reinitialises its dynamic readers", reads: vec![ read("clean_orders", true, true, &["orders"]), read("daily_totals", true, false, &["clean_orders"]), diff --git a/crates/sqlbuild-analysis/src/maintained_refresh/tests/test_types.rs b/crates/sqlbuild-analysis/src/dynamic_refresh/tests/test_types.rs similarity index 75% rename from crates/sqlbuild-analysis/src/maintained_refresh/tests/test_types.rs rename to crates/sqlbuild-analysis/src/dynamic_refresh/tests/test_types.rs index 70aedbf086..bf88ee50a8 100644 --- a/crates/sqlbuild-analysis/src/maintained_refresh/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/dynamic_refresh/tests/test_types.rs @@ -1,6 +1,4 @@ -use crate::maintained_refresh::models::{ - LiveDynamicTable, MaintainedRead, MaintainedSettings, RefreshLag, -}; +use crate::dynamic_refresh::models::{DynamicRead, DynamicSettings, LiveDynamicTable, RefreshLag}; pub(super) struct TargetLagTestCase { pub(super) description: &'static str, @@ -16,13 +14,13 @@ pub(super) struct RenderLagTestCase { pub(super) struct DriftTestCase { pub(super) description: &'static str, - pub(super) desired: MaintainedSettings, + pub(super) desired: DynamicSettings, pub(super) live: LiveDynamicTable, pub(super) expected_drift: bool, } pub(super) struct ReinitialiseTestCase { pub(super) description: &'static str, - pub(super) reads: Vec, + pub(super) reads: Vec, pub(super) expected_readers: Vec<(&'static str, &'static str)>, } diff --git a/crates/sqlbuild-analysis/src/lib.rs b/crates/sqlbuild-analysis/src/lib.rs index 303bf144da..0f0ca871f0 100644 --- a/crates/sqlbuild-analysis/src/lib.rs +++ b/crates/sqlbuild-analysis/src/lib.rs @@ -14,6 +14,7 @@ pub mod contracts; pub mod dag_artifact; pub mod definition_changes; pub mod diff_selection; +pub mod dynamic_refresh; pub mod explicit_references; pub mod fingerprint_storage; pub mod graph; @@ -22,7 +23,6 @@ pub mod janitor_planning; pub mod lineage; pub mod lineage_command; pub mod lineage_graph; -pub mod maintained_refresh; pub mod manifest; pub mod node_set_dependencies; pub mod nullability_rules; diff --git a/crates/sqlbuild-analysis/src/maintained_refresh/main/maintained_settings_drift.rs b/crates/sqlbuild-analysis/src/maintained_refresh/main/maintained_settings_drift.rs deleted file mode 100644 index 6c2f11b86c..0000000000 --- a/crates/sqlbuild-analysis/src/maintained_refresh/main/maintained_settings_drift.rs +++ /dev/null @@ -1,15 +0,0 @@ -//! Whether a maintained model's listed refresh settings differ from its declared ones. - -use crate::maintained_refresh::_helpers::target_lag::parse_target_lag; -use crate::maintained_refresh::models::{LiveDynamicTable, MaintainedSettings}; - -/// True when the lag differs or cannot be read, or a declared warehouse differs. -pub fn maintained_settings_drift(desired: &MaintainedSettings, live: &LiveDynamicTable) -> bool { - if parse_target_lag(&live.target_lag) != Some(desired.lag) { - return true; - } - desired - .warehouse - .as_deref() - .is_some_and(|warehouse| !warehouse.eq_ignore_ascii_case(live.warehouse.trim())) -} diff --git a/crates/sqlbuild-analysis/src/maintained_refresh/mod.rs b/crates/sqlbuild-analysis/src/maintained_refresh/mod.rs deleted file mode 100644 index 31a6ce51c4..0000000000 --- a/crates/sqlbuild-analysis/src/maintained_refresh/mod.rs +++ /dev/null @@ -1,9 +0,0 @@ -//! Maintained refresh: dynamic table lag text, refresh setting drift, and readers a rebuild reinitialises. - -pub(crate) mod _helpers; -pub mod constants; -pub mod main; -pub mod models; - -#[cfg(test)] -mod tests; diff --git a/crates/sqlbuild-analysis/src/planner_changes/main/version_identity_payload.rs b/crates/sqlbuild-analysis/src/planner_changes/main/version_identity_payload.rs index 62976bbb3b..155921eda4 100644 --- a/crates/sqlbuild-analysis/src/planner_changes/main/version_identity_payload.rs +++ b/crates/sqlbuild-analysis/src/planner_changes/main/version_identity_payload.rs @@ -3,8 +3,9 @@ use crate::planner_changes::models::IdentityPayload; use sqlbuild_core::fingerprint_identity::main::decoded_object::decoded_object; use sqlbuild_core::fingerprint_identity::main::normalize_version_identity_metadata::normalize_version_identity_metadata; +use sqlbuild_core::fingerprint_identity::main::without_default_refresh_mode::without_default_refresh_mode; -/// The metadata without audit-gate, migration and declared-column entries; with `without_functions`, also without function hashes. +/// The metadata without audit-gate, migration, declared-column and default refresh-mode entries; with `without_functions`, also without function hashes. pub fn version_identity_payload( metadata_json: Option<&str>, without_functions: bool, @@ -17,7 +18,7 @@ pub fn version_identity_payload( Ok(None) => IdentityPayload::Absent, Ok(Some(mut object)) => { normalize_version_identity_metadata(&mut object, without_functions); - IdentityPayload::Object(object) + IdentityPayload::Object(without_default_refresh_mode(object)) } } } diff --git a/crates/sqlbuild-analysis/src/planner_strategy/main/materialization_type.rs b/crates/sqlbuild-analysis/src/planner_strategy/main/materialization_type.rs index 40708d18a5..80ac505416 100644 --- a/crates/sqlbuild-analysis/src/planner_strategy/main/materialization_type.rs +++ b/crates/sqlbuild-analysis/src/planner_strategy/main/materialization_type.rs @@ -10,7 +10,7 @@ pub fn materialization_type(materialized: Option<&str>) -> MaterializationType { Some("incremental") => MaterializationType::Incremental, Some("snapshot") => MaterializationType::Snapshot, Some("seed") => MaterializationType::Seed, - Some("maintained") => MaterializationType::Maintained, + Some("dynamic") => MaterializationType::Dynamic, Some(_) => MaterializationType::Custom, } } diff --git a/crates/sqlbuild-analysis/src/planner_strategy/main/model_plan_action.rs b/crates/sqlbuild-analysis/src/planner_strategy/main/model_plan_action.rs index de5c990fc2..8612eca45d 100644 --- a/crates/sqlbuild-analysis/src/planner_strategy/main/model_plan_action.rs +++ b/crates/sqlbuild-analysis/src/planner_strategy/main/model_plan_action.rs @@ -41,12 +41,12 @@ pub fn model_plan_action( refreshed(PlanReason::NormalIncremental, false), )); } - MaterializationType::Maintained => { + MaterializationType::Dynamic => { let reason: PlanReason = refreshed(PlanReason::NoChange, true); if reason == PlanReason::NoChange { - return Ok((PlanAction::KeepMaintained, reason)); + return Ok((PlanAction::KeepDynamic, reason)); } - return Ok((PlanAction::CreateMaintained, reason)); + return Ok((PlanAction::CreateDynamic, reason)); } MaterializationType::Table | MaterializationType::Incremental diff --git a/crates/sqlbuild-analysis/src/planner_strategy/models.rs b/crates/sqlbuild-analysis/src/planner_strategy/models.rs index a760213e79..d72cc734cb 100644 --- a/crates/sqlbuild-analysis/src/planner_strategy/models.rs +++ b/crates/sqlbuild-analysis/src/planner_strategy/models.rs @@ -12,7 +12,7 @@ pub enum MaterializationType { Seed, Custom, /// A stored result the warehouse keeps fresh between builds. - Maintained, + Dynamic, } /// What the plan does with a model. @@ -26,12 +26,12 @@ pub enum PlanAction { Snapshot, Skip, Custom, - /// Create or replace the warehouse-maintained object. - CreateMaintained, + /// Create or replace the warehouse-dynamic object. + CreateDynamic, /// Change its refresh settings in place. - AlterMaintained, + AlterDynamic, /// Leave the object as it is; builds may still refresh it. - KeepMaintained, + KeepDynamic, } /// Why the plan does it. diff --git a/crates/sqlbuild-analysis/src/planner_strategy/tests/test_planner_strategy.rs b/crates/sqlbuild-analysis/src/planner_strategy/tests/test_planner_strategy.rs index 66db810b10..d980633888 100644 --- a/crates/sqlbuild-analysis/src/planner_strategy/tests/test_planner_strategy.rs +++ b/crates/sqlbuild-analysis/src/planner_strategy/tests/test_planner_strategy.rs @@ -103,52 +103,52 @@ fn given_model_change_when_resolving_action_then_matches_python_strategy() { ), }, ActionTestCase { - description: "maintained first run creates the dynamic table", - materialization: MaterializationType::Maintained, + description: "dynamic first run creates the dynamic table", + materialization: MaterializationType::Dynamic, strategy: None, change: change(PlanChangeKind::FirstRun), full_refresh: false, - expected_action: Ok((PlanAction::CreateMaintained, PlanReason::FirstRun)), + expected_action: Ok((PlanAction::CreateDynamic, PlanReason::FirstRun)), }, ActionTestCase { - description: "maintained query change recreates it", - materialization: MaterializationType::Maintained, + description: "dynamic query change recreates it", + materialization: MaterializationType::Dynamic, strategy: None, change: change(PlanChangeKind::QueryChanged), full_refresh: false, - expected_action: Ok((PlanAction::CreateMaintained, PlanReason::QueryChanged)), + expected_action: Ok((PlanAction::CreateDynamic, PlanReason::QueryChanged)), }, ActionTestCase { - description: "maintained config change such as refresh_mode recreates it", - materialization: MaterializationType::Maintained, + description: "dynamic config change such as refresh_mode recreates it", + materialization: MaterializationType::Dynamic, strategy: None, change: change(PlanChangeKind::ConfigChanged), full_refresh: false, - expected_action: Ok((PlanAction::CreateMaintained, PlanReason::ConfigChanged)), + expected_action: Ok((PlanAction::CreateDynamic, PlanReason::ConfigChanged)), }, ActionTestCase { - description: "maintained rename creates it under the new name", - materialization: MaterializationType::Maintained, + description: "dynamic rename creates it under the new name", + materialization: MaterializationType::Dynamic, strategy: None, change: change(PlanChangeKind::Renamed), full_refresh: false, - expected_action: Ok((PlanAction::CreateMaintained, PlanReason::Renamed)), + expected_action: Ok((PlanAction::CreateDynamic, PlanReason::Renamed)), }, ActionTestCase { - description: "maintained full refresh recreates an unchanged model", - materialization: MaterializationType::Maintained, + description: "dynamic full refresh recreates an unchanged model", + materialization: MaterializationType::Dynamic, strategy: None, change: change(PlanChangeKind::NoChange), full_refresh: true, - expected_action: Ok((PlanAction::CreateMaintained, PlanReason::FullRefresh)), + expected_action: Ok((PlanAction::CreateDynamic, PlanReason::FullRefresh)), }, ActionTestCase { - description: "unchanged maintained model is kept", - materialization: MaterializationType::Maintained, + description: "unchanged dynamic model is kept", + materialization: MaterializationType::Dynamic, strategy: None, change: change(PlanChangeKind::NoChange), full_refresh: false, - expected_action: Ok((PlanAction::KeepMaintained, PlanReason::NoChange)), + expected_action: Ok((PlanAction::KeepDynamic, PlanReason::NoChange)), }, ]; for test_case in test_cases { diff --git a/crates/sqlbuild-cli/src/argv/command_table.json b/crates/sqlbuild-cli/src/argv/command_table.json index 80148a54d7..6f19fa1d5d 100644 --- a/crates/sqlbuild-cli/src/argv/command_table.json +++ b/crates/sqlbuild-cli/src/argv/command_table.json @@ -2633,9 +2633,9 @@ { "kind": "store_true", "options": [ - "--ignore-maintained-grace" + "--ignore-dynamic-grace" ], - "dest": "ignore_maintained_grace", + "dest": "ignore_dynamic_grace", "nargs": 0, "const": true, "default": false, diff --git a/crates/sqlbuild-cli/src/argv/constants.rs b/crates/sqlbuild-cli/src/argv/constants.rs index 14b4b94ec7..c99a0c09a5 100644 --- a/crates/sqlbuild-cli/src/argv/constants.rs +++ b/crates/sqlbuild-cli/src/argv/constants.rs @@ -157,7 +157,7 @@ pub(crate) const CLI_NAMESPACE_DEFAULTS: [(&str, NamespaceDefault); 193] = [ "allow_snapshot_schema_change", NamespaceDefault::Bool(false), ), - ("ignore_maintained_grace", NamespaceDefault::Bool(false)), + ("ignore_dynamic_grace", NamespaceDefault::Bool(false)), ("concurrency", NamespaceDefault::None), ("verbose", NamespaceDefault::Bool(false)), ("debug", NamespaceDefault::Bool(false)), diff --git a/crates/sqlbuild-cli/src/dispatch/constants.rs b/crates/sqlbuild-cli/src/dispatch/constants.rs index 6b048c7dbb..953ee7474c 100644 --- a/crates/sqlbuild-cli/src/dispatch/constants.rs +++ b/crates/sqlbuild-cli/src/dispatch/constants.rs @@ -169,7 +169,7 @@ const BUILD_FIELDS: [Field; 41] = [ same("allow_retention_decrease"), same("allow_missing_migration_origin"), same("allow_snapshot_schema_change"), - same("ignore_maintained_grace"), + same("ignore_dynamic_grace"), same("concurrency"), SELECT, field("selector_files", Source::SelectorFiles), diff --git a/crates/sqlbuild-config/src/effective/constants.rs b/crates/sqlbuild-config/src/effective/constants.rs index 5508b1565a..18b48d3613 100644 --- a/crates/sqlbuild-config/src/effective/constants.rs +++ b/crates/sqlbuild-config/src/effective/constants.rs @@ -240,9 +240,9 @@ pub(crate) const TARGET_FIELDS: &[TargetField] = &[ }, TargetField { group: None, - name: "pause_maintained", + name: "pause_dynamic", rule: MergeRule::Overlay { - decided_by: &["pause_maintained"], + decided_by: &["pause_dynamic"], }, }, ]; diff --git a/crates/sqlbuild-config/src/project/_helpers/documents.rs b/crates/sqlbuild-config/src/project/_helpers/documents.rs index bcce7e424e..9b85455043 100644 --- a/crates/sqlbuild-config/src/project/_helpers/documents.rs +++ b/crates/sqlbuild-config/src/project/_helpers/documents.rs @@ -15,7 +15,7 @@ use crate::project::_helpers::help::{ use crate::project::_helpers::keys::Place; use crate::project::_helpers::sections::{ load_changes, load_constants, load_control, load_cost, load_cursors, load_dbt, load_diff, - load_git, load_janitor, load_local_dbt, load_maintained, load_microbatches, load_migrations, + load_dynamic, load_git, load_janitor, load_local_dbt, load_microbatches, load_migrations, load_policy_flag, load_pools, load_review, load_scenario, load_settings, load_snapshots, setting_overrides, }; @@ -31,7 +31,7 @@ use crate::project::constants::{ }; use crate::project::errors::{ConfigFailure, FailureClass, LoadFailure, MessagePiece}; use crate::project::models::{ - Built, ChangesConfig, ConfigFile, ConfigPaths, ControlConfig, MaintainedConfig, PoolLimits, + Built, ChangesConfig, ConfigFile, ConfigPaths, ControlConfig, DynamicConfig, PoolLimits, ReviewConfig, }; use crate::project::types::{Entries, Fields, Loaded}; @@ -299,7 +299,7 @@ fn project_fields(payload: &Entries) -> Loaded { } load_git(get(payload, "git"), PROJECT_FILE)?; load_review(get(payload, "review"), PROJECT_FILE)?; - load_maintained(get(payload, "maintained"), PROJECT_FILE)?; + load_dynamic(get(payload, "dynamic"), PROJECT_FILE)?; load_control(get(payload, "control"), PROJECT_FILE)?; load_changes(get(payload, "changes"), PROJECT_FILE)?; pool_limits(payload)?; @@ -542,9 +542,9 @@ pub(crate) fn changes_config(paths: &ConfigPaths) -> Result Result { - project_section(paths, "maintained", load_maintained) +/// `[dynamic]` of `sqlbuild_project.toml`. +pub(crate) fn dynamic_config(paths: &ConfigPaths) -> Result { + project_section(paths, "dynamic", load_dynamic) } /// `[review]` of `sqlbuild_project.toml`. diff --git a/crates/sqlbuild-config/src/project/_helpers/sections.rs b/crates/sqlbuild-config/src/project/_helpers/sections.rs index be75473f5f..c8c1f739b2 100644 --- a/crates/sqlbuild-config/src/project/_helpers/sections.rs +++ b/crates/sqlbuild-config/src/project/_helpers/sections.rs @@ -22,13 +22,14 @@ use crate::project::constants::{ AUDIT_SEVERITIES, CHECK_DECLARED_COLUMNS_KEY, COLLECTION_RENDERINGS, COLUMN_CONTRACT_MODES, CONCURRENCY_KEY, DBT_DEFER_CLONE_KEY, DBT_KEYS, DBT_REMOVED_KEYS, DBT_REUSE_KEYS, DEFAULT_HEARTBEAT_GRACE_SECONDS, DEFAULT_IN_USE_WINDOW, DEFAULT_QUERY_ARTIFACT_TTL, - FUTURE_CURSOR_ACTIONS, HEARTBEAT_INTERVAL_SECONDS, LEGACY_COLUMN_CONTRACT_MODE_KEY, - LEGACY_CONCURRENCY_KEY, LEGACY_SQL_VALIDATION_KEY, LOCAL_DBT_KEYS, MAINTAINED_KEYS, - MAX_BATCHES_KEY, MICROBATCH_LIMIT_ACTIONS, OLD_NAME_VIEWS_KEY, QUERY_CHANGE_TRACKING_KEY, - REQUIRE_SQL_ANALYSIS_KEY, REVIEW_DOWNSTREAM_POLICIES, REVIEW_KEYS, RUN_NAMESPACE_KEY, - RUN_NAMESPACE_MAX_LENGTH, SCENARIO_KEYS, SETTINGS_KEYS, SNAPSHOT_FULL_REFRESH_POLICIES, - SNAPSHOT_KEYS, SNAPSHOT_LIMIT_KEYS, SNAPSHOT_SCHEMA_CHANGE_POLICIES, SQL_ANALYSIS_KEY, - UNACCOUNTED_PARTITION_POLICIES, USD_PER_CREDIT_KEY, ZERO_FIXED_DURATION, ZERO_FLOAT, + DYNAMIC_KEYS, FUTURE_CURSOR_ACTIONS, HEARTBEAT_INTERVAL_SECONDS, + LEGACY_COLUMN_CONTRACT_MODE_KEY, LEGACY_CONCURRENCY_KEY, LEGACY_SQL_VALIDATION_KEY, + LOCAL_DBT_KEYS, MAX_BATCHES_KEY, MICROBATCH_LIMIT_ACTIONS, OLD_NAME_VIEWS_KEY, + QUERY_CHANGE_TRACKING_KEY, REQUIRE_SQL_ANALYSIS_KEY, REVIEW_DOWNSTREAM_POLICIES, REVIEW_KEYS, + RUN_NAMESPACE_KEY, RUN_NAMESPACE_MAX_LENGTH, SCENARIO_KEYS, SETTINGS_KEYS, + SNAPSHOT_FULL_REFRESH_POLICIES, SNAPSHOT_KEYS, SNAPSHOT_LIMIT_KEYS, + SNAPSHOT_SCHEMA_CHANGE_POLICIES, SQL_ANALYSIS_KEY, UNACCOUNTED_PARTITION_POLICIES, + USD_PER_CREDIT_KEY, ZERO_FIXED_DURATION, ZERO_FLOAT, }; use crate::project::constants::{ CHANGE_COUNTS, CHANGE_REACHES, CHANGES_KEYS, CONTROL_KEYS, MAX_POOL_LIMIT, @@ -36,7 +37,7 @@ use crate::project::constants::{ }; use crate::project::errors::ConfigFailure; use crate::project::models::{ - Built, ChangesConfig, ConfigFile, ControlConfig, MaintainedConfig, ReviewConfig, + Built, ChangesConfig, ConfigFile, ControlConfig, DynamicConfig, ReviewConfig, }; use crate::project::types::{Entries, Fields, Loaded}; use sqlbuild_core::constants::ZERO_DAY_DURATION; @@ -628,13 +629,13 @@ pub(crate) fn sync_interval(text: &str) -> Result { Ok(seconds) } -/// `[maintained]`: how long a silent pausing build keeps its maintained models paused. -pub(crate) fn load_maintained( +/// `[dynamic]`: how long a silent pausing build keeps its dynamic models paused. +pub(crate) fn load_dynamic( payload: Option<&ConfigValue>, file: ConfigFile, -) -> Loaded { - let mapping = coerce_mapping(payload, place(file, "maintained"))?; - validate_allowed_keys(mapping, &MAINTAINED_KEYS, place(file, "maintained"))?; +) -> Loaded { + let mapping = coerce_mapping(payload, place(file, "dynamic"))?; + validate_allowed_keys(mapping, &DYNAMIC_KEYS, place(file, "dynamic"))?; let heartbeat_grace_seconds: u64 = match get(mapping, "heartbeat_grace") { None => DEFAULT_HEARTBEAT_GRACE_SECONDS, Some(ConfigValue::String(text)) => match fixed_duration_seconds(text, false) { @@ -646,7 +647,7 @@ pub(crate) fn load_maintained( if heartbeat_grace_seconds <= HEARTBEAT_INTERVAL_SECONDS { return Err(ConfigFailure::at( file, - "maintained.heartbeat_grace must be longer than 1m: a build writes a heartbeat every minute, so a shorter grace expires builds that are still running", + "dynamic.heartbeat_grace must be longer than 1m: a build writes a heartbeat every minute, so a shorter grace expires builds that are still running", )); } let allow_ignore_grace: bool = match get(mapping, "allow_ignore_grace") { @@ -655,11 +656,11 @@ pub(crate) fn load_maintained( Some(_) => { return Err(ConfigFailure::at( file, - "maintained.allow_ignore_grace must be true or false", + "dynamic.allow_ignore_grace must be true or false", )); } }; - Ok(MaintainedConfig { + Ok(DynamicConfig { heartbeat_grace_seconds, allow_ignore_grace, }) @@ -668,7 +669,7 @@ pub(crate) fn load_maintained( fn grace_failure(file: ConfigFile) -> ConfigFailure { ConfigFailure::at( file, - "maintained.heartbeat_grace must be a fixed duration such as 30m or 2h", + "dynamic.heartbeat_grace must be a fixed duration such as 30m or 2h", ) } diff --git a/crates/sqlbuild-config/src/project/_helpers/targets.rs b/crates/sqlbuild-config/src/project/_helpers/targets.rs index b8bd8df06b..73b6fd74f9 100644 --- a/crates/sqlbuild-config/src/project/_helpers/targets.rs +++ b/crates/sqlbuild-config/src/project/_helpers/targets.rs @@ -16,7 +16,7 @@ use crate::project::_helpers::sections::{fixed_duration_seconds, item_timeout}; use crate::project::_helpers::values::{int_below, is_int}; use crate::project::constants::{ CLONE_KEYS, CONCURRENCY_KEY, CONFIRMATION_POLICIES, EXECUTION_LIMIT_KEYS, - IGNORE_AUDIT_FAILURES_KEY, MAX_FULL_ROWS_KEY, PAUSE_MAINTAINED_KEY, PLAN_FROM_KEY, + IGNORE_AUDIT_FAILURES_KEY, MAX_FULL_ROWS_KEY, PAUSE_DYNAMIC_KEY, PLAN_FROM_KEY, PLAN_FROM_SOURCES, POOLS_KEY, RETENTION_MATERIALIZATIONS, TARGET_KEYS, UNLIMITED_DIFF_FULL_ROWS, WAREHOUSE_GROUPS, }; @@ -435,8 +435,8 @@ fn local_target(payload: &ConfigValue, target: TargetPlace<'_>) -> Loaded Result { + crate::project::_helpers::documents::dynamic_config(paths) +} diff --git a/crates/sqlbuild-config/src/project/main/load_maintained_config.rs b/crates/sqlbuild-config/src/project/main/load_maintained_config.rs deleted file mode 100644 index b5740f203d..0000000000 --- a/crates/sqlbuild-config/src/project/main/load_maintained_config.rs +++ /dev/null @@ -1,9 +0,0 @@ -//! Read `[maintained]` from `sqlbuild_project.toml`. - -use crate::project::errors::LoadFailure; -use crate::project::models::{ConfigPaths, MaintainedConfig}; - -/// The maintained-model pause settings, with their defaults when `[maintained]` sets none. -pub fn load_maintained_config(paths: &ConfigPaths) -> Result { - crate::project::_helpers::documents::maintained_config(paths) -} diff --git a/crates/sqlbuild-config/src/project/main/mod.rs b/crates/sqlbuild-config/src/project/main/mod.rs index 6d54995e7c..abf7b235dc 100644 --- a/crates/sqlbuild-config/src/project/main/mod.rs +++ b/crates/sqlbuild-config/src/project/main/mod.rs @@ -2,9 +2,9 @@ pub mod deprecated_setting_warnings; pub mod explicit_references_help; pub mod load_changes_config; pub mod load_control_config; +pub mod load_dynamic_config; pub mod load_git_base_ref; pub mod load_local_config; -pub mod load_maintained_config; pub mod load_pool_limits; pub mod load_project_config; pub mod load_project_setting_keys; diff --git a/crates/sqlbuild-config/src/project/models.rs b/crates/sqlbuild-config/src/project/models.rs index 5fab29b4c0..6a6f921209 100644 --- a/crates/sqlbuild-config/src/project/models.rs +++ b/crates/sqlbuild-config/src/project/models.rs @@ -91,12 +91,12 @@ pub struct ChangesConfig { pub reach: Option, } -/// `[maintained]` of `sqlbuild_project.toml`: when a build may take over paused maintained models. +/// `[dynamic]` of `sqlbuild_project.toml`: when a build may take over paused dynamic models. #[derive(Clone, Debug, PartialEq, Eq)] -pub struct MaintainedConfig { +pub struct DynamicConfig { /// Seconds a pausing build's heartbeat may be silent before another build resumes its models. pub heartbeat_grace_seconds: u64, - /// Whether `sqb build --ignore-maintained-grace` may take over within the grace. + /// Whether `sqb build --ignore-dynamic-grace` may take over within the grace. pub allow_ignore_grace: bool, } diff --git a/crates/sqlbuild-config/src/project/tests/test_load_config.rs b/crates/sqlbuild-config/src/project/tests/test_load_config.rs index aa14ab21bd..84dbfb4f22 100644 --- a/crates/sqlbuild-config/src/project/tests/test_load_config.rs +++ b/crates/sqlbuild-config/src/project/tests/test_load_config.rs @@ -1,16 +1,16 @@ use crate::project::errors::ConfigFailure; use crate::project::main::load_changes_config::load_changes_config; use crate::project::main::load_control_config::load_control_config; +use crate::project::main::load_dynamic_config::load_dynamic_config; use crate::project::main::load_git_base_ref::load_git_base_ref; -use crate::project::main::load_maintained_config::load_maintained_config; use crate::project::main::load_pool_limits::load_pool_limits; use crate::project::main::load_review_config::load_review_config; use crate::project::models::ConfigFile; -use crate::project::models::{ChangesConfig, ControlConfig, MaintainedConfig, ReviewConfig}; +use crate::project::models::{ChangesConfig, ControlConfig, DynamicConfig, ReviewConfig}; use crate::project::tests::helpers::{failure_at, first_failure, project_directory}; use crate::project::tests::test_types::{ - ChangesConfigTestCase, ControlConfigTestCase, GitBaseRefTestCase, - MaintainedConfigErrorTestCase, MaintainedConfigTestCase, PoolsErrorTestCase, PoolsTestCase, + ChangesConfigTestCase, ControlConfigTestCase, DynamicConfigErrorTestCase, + DynamicConfigTestCase, GitBaseRefTestCase, PoolsErrorTestCase, PoolsTestCase, ProjectFailureTestCase, ReviewConfigErrorTestCase, ReviewConfigTestCase, }; @@ -65,35 +65,35 @@ fn given_project_files_when_loading_then_rejections_match_python_loader() { )), }, ProjectFailureTestCase { - description: "a target may keep its maintained models paused between builds", - project: "name = 'orders'\nadapter = 'snowflake'\n[targets.dev]\npause_maintained = true\n", - local: "[targets.dev]\npause_maintained = false\n", + description: "a target may keep its dynamic models paused between builds", + project: "name = 'orders'\nadapter = 'snowflake'\n[targets.dev]\npause_dynamic = true\n", + local: "[targets.dev]\npause_dynamic = false\n", expected_failure: None, }, ProjectFailureTestCase { - description: "pause_maintained is a boolean", - project: "name = 'orders'\nadapter = 'snowflake'\n[targets.dev]\npause_maintained = 'yes'\n", + description: "pause_dynamic is a boolean", + project: "name = 'orders'\nadapter = 'snowflake'\n[targets.dev]\npause_dynamic = 'yes'\n", local: "", expected_failure: Some(ConfigFailure::plain( - "Expected 'pause_maintained' to be a boolean when provided", + "Expected 'pause_dynamic' to be a boolean when provided", )), }, ProjectFailureTestCase { - description: "an invalid maintained section fails the project load", - project: "name = 'orders'\nadapter = 'snowflake'\n[maintained]\nheartbeat_grace = 'soon'\n", + description: "an invalid dynamic section fails the project load", + project: "name = 'orders'\nadapter = 'snowflake'\n[dynamic]\nheartbeat_grace = 'soon'\n", local: "", expected_failure: Some(ConfigFailure::at( ConfigFile::Project, - "maintained.heartbeat_grace must be a fixed duration such as 30m or 2h", + "dynamic.heartbeat_grace must be a fixed duration such as 30m or 2h", )), }, ProjectFailureTestCase { description: "a grace no longer than the heartbeat interval would expire every build", - project: "name = 'orders'\nadapter = 'snowflake'\n[maintained]\nheartbeat_grace = '1m'\n", + project: "name = 'orders'\nadapter = 'snowflake'\n[dynamic]\nheartbeat_grace = '1m'\n", local: "", expected_failure: Some(ConfigFailure::at( ConfigFile::Project, - "maintained.heartbeat_grace must be longer than 1m: a build writes a heartbeat every minute, so a shorter grace expires builds that are still running", + "dynamic.heartbeat_grace must be longer than 1m: a build writes a heartbeat every minute, so a shorter grace expires builds that are still running", )), }, ProjectFailureTestCase { @@ -633,30 +633,30 @@ fn given_invalid_pools_when_loading_then_project_loading_rejects_them() { } #[test] -fn given_maintained_section_when_reading_then_returns_its_values_or_defaults() { +fn given_dynamic_section_when_reading_then_returns_its_values_or_defaults() { let test_cases = [ - MaintainedConfigTestCase { + DynamicConfigTestCase { description: "a longer grace that may be ignored", - project: "name = 'orders'\nadapter = 'snowflake'\n[maintained]\n\ + project: "name = 'orders'\nadapter = 'snowflake'\n[dynamic]\n\ heartbeat_grace = '45m'\nallow_ignore_grace = true\n", - expected_config: MaintainedConfig { + expected_config: DynamicConfig { heartbeat_grace_seconds: 2_700, allow_ignore_grace: true, }, }, - MaintainedConfigTestCase { + DynamicConfigTestCase { description: "any grace longer than the heartbeat interval", - project: "name = 'orders'\nadapter = 'snowflake'\n[maintained]\n\ + project: "name = 'orders'\nadapter = 'snowflake'\n[dynamic]\n\ heartbeat_grace = '61s'\n", - expected_config: MaintainedConfig { + expected_config: DynamicConfig { heartbeat_grace_seconds: 61, allow_ignore_grace: false, }, }, - MaintainedConfigTestCase { - description: "no maintained section", + DynamicConfigTestCase { + description: "no dynamic section", project: PROJECT, - expected_config: MaintainedConfig { + expected_config: DynamicConfig { heartbeat_grace_seconds: 1_800, allow_ignore_grace: false, }, @@ -665,7 +665,7 @@ fn given_maintained_section_when_reading_then_returns_its_values_or_defaults() { for test_case in test_cases { let (_directory, paths) = project_directory(test_case.project, ""); - let config: MaintainedConfig = load_maintained_config(&paths).expect(test_case.description); + let config: DynamicConfig = load_dynamic_config(&paths).expect(test_case.description); assert_eq!( config, test_case.expected_config, @@ -676,31 +676,31 @@ fn given_maintained_section_when_reading_then_returns_its_values_or_defaults() { } #[test] -fn given_invalid_maintained_section_when_reading_then_it_is_rejected() { +fn given_invalid_dynamic_section_when_reading_then_it_is_rejected() { let test_cases = [ - MaintainedConfigErrorTestCase { + DynamicConfigErrorTestCase { description: "a grace that is not a duration", - project: "name = 'orders'\nadapter = 'snowflake'\n[maintained]\n\ + project: "name = 'orders'\nadapter = 'snowflake'\n[dynamic]\n\ heartbeat_grace = 'soon'\n", - expected_fragment: "maintained.heartbeat_grace must be a fixed duration such as 30m or 2h", + expected_fragment: "dynamic.heartbeat_grace must be a fixed duration such as 30m or 2h", }, - MaintainedConfigErrorTestCase { + DynamicConfigErrorTestCase { description: "a grace in months", - project: "name = 'orders'\nadapter = 'snowflake'\n[maintained]\n\ + project: "name = 'orders'\nadapter = 'snowflake'\n[dynamic]\n\ heartbeat_grace = '1mo'\n", - expected_fragment: "maintained.heartbeat_grace must be a fixed duration such as 30m or 2h", + expected_fragment: "dynamic.heartbeat_grace must be a fixed duration such as 30m or 2h", }, - MaintainedConfigErrorTestCase { + DynamicConfigErrorTestCase { description: "allow_ignore_grace written as text", - project: "name = 'orders'\nadapter = 'snowflake'\n[maintained]\n\ + project: "name = 'orders'\nadapter = 'snowflake'\n[dynamic]\n\ allow_ignore_grace = 'yes'\n", - expected_fragment: "maintained.allow_ignore_grace must be true or false", + expected_fragment: "dynamic.allow_ignore_grace must be true or false", }, - MaintainedConfigErrorTestCase { - description: "a misspelt maintained key", - project: "name = 'orders'\nadapter = 'snowflake'\n[maintained]\n\ + DynamicConfigErrorTestCase { + description: "a misspelt dynamic key", + project: "name = 'orders'\nadapter = 'snowflake'\n[dynamic]\n\ heartbeat_grase = '30m'\n", - expected_fragment: "maintained contains unknown key(s): heartbeat_grase", + expected_fragment: "dynamic contains unknown key(s): heartbeat_grase", }, ]; for test_case in test_cases { @@ -708,7 +708,7 @@ fn given_invalid_maintained_section_when_reading_then_it_is_rejected() { let failure: String = format!( "{:?}", - load_maintained_config(&paths).expect_err(test_case.description) + load_dynamic_config(&paths).expect_err(test_case.description) ); assert!( diff --git a/crates/sqlbuild-config/src/project/tests/test_types.rs b/crates/sqlbuild-config/src/project/tests/test_types.rs index d0a08468db..2189e764b3 100644 --- a/crates/sqlbuild-config/src/project/tests/test_types.rs +++ b/crates/sqlbuild-config/src/project/tests/test_types.rs @@ -1,5 +1,5 @@ use crate::project::errors::ConfigFailure; -use crate::project::models::{ChangesConfig, ControlConfig, MaintainedConfig, ReviewConfig}; +use crate::project::models::{ChangesConfig, ControlConfig, DynamicConfig, ReviewConfig}; pub(super) struct ProjectFailureTestCase { pub(super) description: &'static str, @@ -59,13 +59,13 @@ pub(crate) struct PoolsErrorTestCase { pub(crate) expected_fragment: &'static str, } -pub(crate) struct MaintainedConfigTestCase { +pub(crate) struct DynamicConfigTestCase { pub(crate) description: &'static str, pub(crate) project: &'static str, - pub(crate) expected_config: MaintainedConfig, + pub(crate) expected_config: DynamicConfig, } -pub(crate) struct MaintainedConfigErrorTestCase { +pub(crate) struct DynamicConfigErrorTestCase { pub(crate) description: &'static str, pub(crate) project: &'static str, pub(crate) expected_fragment: &'static str, diff --git a/crates/sqlbuild-core/src/cli_text/_helpers/plan_sections.rs b/crates/sqlbuild-core/src/cli_text/_helpers/plan_sections.rs index 03b6cb37ea..11cccfe5ed 100644 --- a/crates/sqlbuild-core/src/cli_text/_helpers/plan_sections.rs +++ b/crates/sqlbuild-core/src/cli_text/_helpers/plan_sections.rs @@ -6,8 +6,8 @@ use crate::cli_text::_helpers::bounded_sections::{ }; use crate::cli_text::_helpers::layout::tree_connector; use crate::cli_text::constants::{ - ALTER_MAINTAINED_ACTION, BOUNDED_BACKFILL, CHANGED_IDENTITY, COLUMN_ADDED, COLUMN_REMOVED, - COLUMN_TYPE_CHANGED, CONFIG_CHANGED_REASON, CREATE_MAINTAINED_ACTION, CREATE_VIEW_ACTION, + ALTER_DYNAMIC_ACTION, BOUNDED_BACKFILL, CHANGED_IDENTITY, COLUMN_ADDED, COLUMN_REMOVED, + COLUMN_TYPE_CHANGED, CONFIG_CHANGED_REASON, CREATE_DYNAMIC_ACTION, CREATE_VIEW_ACTION, CUSTOM_ACTION, DIFF_HEADER_MARKER, EFFECTIVE_BATCH_SIZE_TOKEN, ERROR_SEVERITY_VALUE, FIRST_RUN_REASON, FORWARD_BACKFILL, FULL_BACKFILL, FULL_REFRESH_REASON, INFO_SEVERITY_VALUE, NO_CHANGE_REASON, NORMAL_INCREMENTAL_REASON, QUERY_CHANGED_REASON, REASON_GROUPS, @@ -430,14 +430,14 @@ fn action_text(model: &ModelPlanText) -> String { if model.action == CREATE_VIEW_ACTION { return "recreate view".to_owned(); } - if model.action == CREATE_MAINTAINED_ACTION { + if model.action == CREATE_DYNAMIC_ACTION { return if model.reason == FIRST_RUN_REASON { "create".to_owned() } else { "recreate, full recompute".to_owned() }; } - if model.action == ALTER_MAINTAINED_ACTION { + if model.action == ALTER_DYNAMIC_ACTION { return "change refresh settings in place".to_owned(); } if model.action == CUSTOM_ACTION { @@ -465,7 +465,7 @@ fn action_text(model: &ModelPlanText) -> String { fn policy_lines(model: &ModelPlanText) -> Vec { if model.backfill_action == FORWARD_BACKFILL - || model.action == CREATE_MAINTAINED_ACTION + || model.action == CREATE_DYNAMIC_ACTION || !(model.reason == QUERY_CHANGED_REASON || model.reason == SCHEMA_CHANGED_REASON) { return Vec::new(); diff --git a/crates/sqlbuild-core/src/cli_text/constants.rs b/crates/sqlbuild-core/src/cli_text/constants.rs index f00f5abc48..bd63958abd 100644 --- a/crates/sqlbuild-core/src/cli_text/constants.rs +++ b/crates/sqlbuild-core/src/cli_text/constants.rs @@ -63,8 +63,8 @@ pub(crate) const REASON_GROUPS: [(&str, &str); 7] = [ pub(crate) const SKIP_ACTION: &str = "skip"; pub(crate) const CREATE_VIEW_ACTION: &str = "create_view"; pub(crate) const CUSTOM_ACTION: &str = "custom"; -pub(crate) const CREATE_MAINTAINED_ACTION: &str = "create_maintained"; -pub(crate) const ALTER_MAINTAINED_ACTION: &str = "alter_maintained"; +pub(crate) const CREATE_DYNAMIC_ACTION: &str = "create_dynamic"; +pub(crate) const ALTER_DYNAMIC_ACTION: &str = "alter_dynamic"; pub(crate) const FULL_REFRESH_REASON: &str = "full_refresh"; pub(crate) const FIRST_RUN_REASON: &str = "first_run"; pub(crate) const QUERY_CHANGED_REASON: &str = "query_changed"; diff --git a/crates/sqlbuild-core/src/cli_text/models.rs b/crates/sqlbuild-core/src/cli_text/models.rs index 0274627d1e..189916b4e5 100644 --- a/crates/sqlbuild-core/src/cli_text/models.rs +++ b/crates/sqlbuild-core/src/cli_text/models.rs @@ -322,9 +322,9 @@ pub struct ModelPlanText { pub config_diff: Option>, /// The rendered query diff, when the query changed and the previous query is known. pub query_diff: Option>, - /// The rebuilt upstream that makes this kept maintained model recompute in full. + /// The rebuilt upstream that makes this kept dynamic model recompute in full. pub reinitialised_by: Option, - /// Whether the warehouse lists this kept maintained model as suspended. + /// Whether the warehouse lists this kept dynamic model as suspended. pub paused: bool, } diff --git a/crates/sqlbuild-core/src/cli_text/tests/test_cli_text.rs b/crates/sqlbuild-core/src/cli_text/tests/test_cli_text.rs index 51f8d9a1e0..511bd83618 100644 --- a/crates/sqlbuild-core/src/cli_text/tests/test_cli_text.rs +++ b/crates/sqlbuild-core/src/cli_text/tests/test_cli_text.rs @@ -227,16 +227,16 @@ fn given_plan_report_when_rendering_then_groups_models_by_reason() { expected_completion: "\u{2713} Plan complete 1 selected", }, PlanTextTestCase { - description: "maintained models recompute on a query change and alter refresh settings in place", + description: "dynamic models recompute on a query change and alter refresh settings in place", report: plan_report(vec![ ModelPlanText { - action: "create_maintained".to_owned(), - materialization_label: "maintained".to_owned(), + action: "create_dynamic".to_owned(), + materialization_label: "dynamic".to_owned(), ..table_model("daily_totals", "query_changed") }, ModelPlanText { - action: "alter_maintained".to_owned(), - materialization_label: "maintained".to_owned(), + action: "alter_dynamic".to_owned(), + materialization_label: "dynamic".to_owned(), ..table_model("clean_orders", "config_changed") }, ]), @@ -244,28 +244,28 @@ fn given_plan_report_when_rendering_then_groups_models_by_reason() { expected_completion: "\u{2713} Plan complete 2 selected", }, PlanTextTestCase { - description: "a kept maintained model whose upstream is rebuilt says it recomputes in full", + description: "a kept dynamic model whose upstream is rebuilt says it recomputes in full", report: plan_report(vec![ table_model("orders", "query_changed"), ModelPlanText { - action: "keep_maintained".to_owned(), - materialization_label: "maintained".to_owned(), + action: "keep_dynamic".to_owned(), + materialization_label: "dynamic".to_owned(), reinitialised_by: Some("orders".to_owned()), ..table_model("daily_totals", "no_change") }, ]), - expected_text: "Plan ready 2 selected\n\nQuery changed (1)\n\u{2514}\u{2500}\u{2500} orders full rebuild\n \u{2514}\u{2500}\u{2500} policy replay_on_change=full\n\nModels (1)\n\u{2514}\u{2500}\u{2500} daily_totals maintained\n \u{2514}\u{2500}\u{2500} recomputes in full orders is rebuilt", + expected_text: "Plan ready 2 selected\n\nQuery changed (1)\n\u{2514}\u{2500}\u{2500} orders full rebuild\n \u{2514}\u{2500}\u{2500} policy replay_on_change=full\n\nModels (1)\n\u{2514}\u{2500}\u{2500} daily_totals dynamic\n \u{2514}\u{2500}\u{2500} recomputes in full orders is rebuilt", expected_completion: "\u{2713} Plan complete 2 selected", }, PlanTextTestCase { - description: "a kept maintained model that is suspended is reported as paused", + description: "a kept dynamic model that is suspended is reported as paused", report: plan_report(vec![ModelPlanText { - action: "keep_maintained".to_owned(), - materialization_label: "maintained".to_owned(), + action: "keep_dynamic".to_owned(), + materialization_label: "dynamic".to_owned(), paused: true, ..table_model("daily_totals", "no_change") }]), - expected_text: "Plan ready 1 selected\n\nModels (1)\n\u{2514}\u{2500}\u{2500} daily_totals maintained\n \u{2514}\u{2500}\u{2500} paused scheduled refreshes are suspended", + expected_text: "Plan ready 1 selected\n\nModels (1)\n\u{2514}\u{2500}\u{2500} daily_totals dynamic\n \u{2514}\u{2500}\u{2500} paused scheduled refreshes are suspended", expected_completion: "\u{2713} Plan complete 1 selected", }, PlanTextTestCase { diff --git a/crates/sqlbuild-core/src/constants.rs b/crates/sqlbuild-core/src/constants.rs index 82fb646f95..a6629f8912 100644 --- a/crates/sqlbuild-core/src/constants.rs +++ b/crates/sqlbuild-core/src/constants.rs @@ -44,8 +44,8 @@ pub const OBSERVATIONS_TABLE_NAME: &str = "_sqlbuild_observations"; pub const RESOURCE_COSTS_TABLE_NAME: &str = "_sqlbuild_resource_costs"; /// State table of each build's start and end, which shows runs the control plane did not start. pub const RUNS_TABLE_NAME: &str = "_sqlbuild_runs"; -/// State table of the maintained models a build paused, and its heartbeats while it holds them. -pub const MAINTAINED_PAUSES_TABLE_NAME: &str = "_sqlbuild_maintained_pauses"; +/// State table of the dynamic models a build paused, and its heartbeats while it holds them. +pub const DYNAMIC_PAUSES_TABLE_NAME: &str = "_sqlbuild_dynamic_pauses"; /// Every state table SQLBuild writes in a target's state schema. pub const STATE_TABLE_NAMES: [&str; 14] = [ FINGERPRINT_TABLE_NAME, @@ -61,7 +61,7 @@ pub const STATE_TABLE_NAMES: [&str; 14] = [ OBSERVATIONS_TABLE_NAME, RESOURCE_COSTS_TABLE_NAME, RUNS_TABLE_NAME, - MAINTAINED_PAUSES_TABLE_NAME, + DYNAMIC_PAUSES_TABLE_NAME, ]; /// A zero-length cursor duration. pub const ZERO_DAY_DURATION: &str = "0d"; diff --git a/crates/sqlbuild-core/src/fingerprint_identity/constants.rs b/crates/sqlbuild-core/src/fingerprint_identity/constants.rs index c32ef975c0..ac688e2b22 100644 --- a/crates/sqlbuild-core/src/fingerprint_identity/constants.rs +++ b/crates/sqlbuild-core/src/fingerprint_identity/constants.rs @@ -4,6 +4,12 @@ pub const AUDIT_GATE_METADATA_KEY: &str = "audit_gate"; pub const MIGRATION_FINGERPRINT_METADATA_KEY: &str = "migration_fingerprint"; pub const DECLARED_COLUMNS_METADATA_KEY: &str = "declared_columns_hash"; pub const LOCAL_FUNCTION_HASHES_METADATA_KEY: &str = "local_function_hashes"; +/// Metadata key of the model config entries in version identity. +pub const CONFIG_METADATA_KEY: &str = "config"; +/// Config key of a dynamic model's refresh mode. +pub const REFRESH_MODE_CONFIG_KEY: &str = "refresh_mode"; +/// The refresh mode of a dynamic model that declares none; writing it out changes nothing. +pub const DEFAULT_REFRESH_MODE: &str = "incremental"; /// The fingerprint dialect when a project names none. pub(crate) const GENERIC_FINGERPRINT_DIALECT: &str = "generic"; /// Python's `NATIVE_DIALECT_ALIASES`. diff --git a/crates/sqlbuild-core/src/fingerprint_identity/main/mod.rs b/crates/sqlbuild-core/src/fingerprint_identity/main/mod.rs index 86131ac87e..e5bc381e33 100644 --- a/crates/sqlbuild-core/src/fingerprint_identity/main/mod.rs +++ b/crates/sqlbuild-core/src/fingerprint_identity/main/mod.rs @@ -6,3 +6,4 @@ pub mod normalize_version_identity_metadata; pub mod python_equal; pub mod recorded_model_local_hash; pub mod stable_identity_hash; +pub mod without_default_refresh_mode; diff --git a/crates/sqlbuild-core/src/fingerprint_identity/main/without_default_refresh_mode.rs b/crates/sqlbuild-core/src/fingerprint_identity/main/without_default_refresh_mode.rs new file mode 100644 index 0000000000..43d32c10a6 --- /dev/null +++ b/crates/sqlbuild-core/src/fingerprint_identity/main/without_default_refresh_mode.rs @@ -0,0 +1,16 @@ +//! Drop a written-out default refresh mode, which plans the same as none. + +use serde_json::{Map, Value}; + +use crate::fingerprint_identity::constants::{ + CONFIG_METADATA_KEY, DEFAULT_REFRESH_MODE, REFRESH_MODE_CONFIG_KEY, +}; + +pub fn without_default_refresh_mode(mut object: Map) -> Map { + if let Some(Value::Object(config)) = object.get_mut(CONFIG_METADATA_KEY) + && config.get(REFRESH_MODE_CONFIG_KEY) == Some(&Value::from(DEFAULT_REFRESH_MODE)) + { + config.shift_remove(REFRESH_MODE_CONFIG_KEY); + } + object +} diff --git a/crates/sqlbuild-core/src/fingerprint_identity/mod.rs b/crates/sqlbuild-core/src/fingerprint_identity/mod.rs index 919648bfd1..0cc81d3427 100644 --- a/crates/sqlbuild-core/src/fingerprint_identity/mod.rs +++ b/crates/sqlbuild-core/src/fingerprint_identity/mod.rs @@ -2,3 +2,6 @@ pub mod constants; pub mod main; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-core/src/fingerprint_identity/tests.rs b/crates/sqlbuild-core/src/fingerprint_identity/tests.rs new file mode 100644 index 0000000000..9f95f97207 --- /dev/null +++ b/crates/sqlbuild-core/src/fingerprint_identity/tests.rs @@ -0,0 +1,4 @@ +#[path = "tests/test_fingerprint_identity.rs"] +mod fingerprint_identity; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-core/src/fingerprint_identity/tests/test_fingerprint_identity.rs b/crates/sqlbuild-core/src/fingerprint_identity/tests/test_fingerprint_identity.rs new file mode 100644 index 0000000000..dded25744f --- /dev/null +++ b/crates/sqlbuild-core/src/fingerprint_identity/tests/test_fingerprint_identity.rs @@ -0,0 +1,37 @@ +use crate::fingerprint_identity::main::canonical_text::canonical_text; +use crate::fingerprint_identity::main::decoded_object::decoded_object; +use crate::fingerprint_identity::main::model_local_identity_hash::model_local_identity_hash; +use crate::fingerprint_identity::main::recorded_model_local_hash::recorded_model_local_hash; +use crate::fingerprint_identity::tests::test_types::RecordedLocalHashTestCase; + +#[test] +fn given_recorded_metadata_when_hashing_then_it_matches_the_config_as_written() { + let test_cases = [ + RecordedLocalHashTestCase { + description: "a written-out default refresh mode stays in the hash", + metadata_json: r#"{"config":{"materialized":"dynamic","refresh_mode":"incremental"},"execution_signature":{},"local_function_hashes":{},"model_name":"daily_totals"}"#, + expected_hashed_json: r#"{"config":{"materialized":"dynamic","refresh_mode":"incremental"},"execution_signature":{},"local_function_hashes":{},"model_name":"daily_totals"}"#, + }, + RecordedLocalHashTestCase { + description: "a model that declares no refresh mode", + metadata_json: r#"{"config":{"materialized":"dynamic"},"execution_signature":{},"local_function_hashes":{},"model_name":"daily_totals"}"#, + expected_hashed_json: r#"{"config":{"materialized":"dynamic"},"execution_signature":{},"local_function_hashes":{},"model_name":"daily_totals"}"#, + }, + ]; + + for test_case in test_cases { + assert_eq!( + recorded_model_local_hash("query_hash", test_case.metadata_json), + Some(model_local_identity_hash( + "query_hash", + &canonical_text( + decoded_object(test_case.expected_hashed_json) + .expect("metadata is JSON") + .expect("metadata is an object") + ) + )), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-core/src/fingerprint_identity/tests/test_types.rs b/crates/sqlbuild-core/src/fingerprint_identity/tests/test_types.rs new file mode 100644 index 0000000000..669e385090 --- /dev/null +++ b/crates/sqlbuild-core/src/fingerprint_identity/tests/test_types.rs @@ -0,0 +1,5 @@ +pub(super) struct RecordedLocalHashTestCase { + pub(super) description: &'static str, + pub(super) metadata_json: &'static str, + pub(super) expected_hashed_json: &'static str, +} diff --git a/crates/sqlbuild-model-config/src/model_validation/_helpers/materialization.rs b/crates/sqlbuild-model-config/src/model_validation/_helpers/materialization.rs index e3633aac1a..6e61dd7b8e 100644 --- a/crates/sqlbuild-model-config/src/model_validation/_helpers/materialization.rs +++ b/crates/sqlbuild-model-config/src/model_validation/_helpers/materialization.rs @@ -22,17 +22,17 @@ use crate::model_validation::constants::{ CHAINED_HELP, CLAIMED_HELP, MATERIALIZATION_HELP, MIGRATABLE_MATERIALIZATIONS, ORIGIN_HELP, QUALIFIED_NAME_SEPARATOR, }; +use crate::model_validation::constants::{ + DYNAMIC_ADAPTER_CODE, DYNAMIC_ADAPTERS, DYNAMIC_MATERIALIZATION, DYNAMIC_ONLY_KEYS, + DYNAMIC_REFRESH_FORMS, DYNAMIC_REFRESH_MODES, DYNAMIC_REFRESH_UNSUPPORTED_CODE, + MATERIALIZED_VIEW_ADAPTER_CODE, MATERIALIZED_VIEW_MATERIALIZATION, SNAPSHOT_DISALLOWED_KEYS, + SNOWFLAKE_ADAPTER, SNOWFLAKE_MIN_LAG_SECONDS, +}; use crate::model_validation::constants::{ ENGINE_FAMILY_SUFFIX, ENGINE_KEY, ORDER_BY_KEY, PARTITION_BY_KEY, PHYSICAL_STORAGE_KEYS, PHYSICAL_STORAGE_MATERIALIZATIONS, PRIMARY_KEY_KEY, TABLE_SETTINGS_KEY, UNSUPPORTED_ENGINE_PREFIXES, }; -use crate::model_validation::constants::{ - MAINTAINED_ADAPTER_CODE, MAINTAINED_ADAPTERS, MAINTAINED_MATERIALIZATION, MAINTAINED_ONLY_KEYS, - MAINTAINED_REFRESH_FORMS, MAINTAINED_REFRESH_MODES, MAINTAINED_REFRESH_UNSUPPORTED_CODE, - MATERIALIZED_VIEW_ADAPTER_CODE, MATERIALIZED_VIEW_MATERIALIZATION, SNAPSHOT_DISALLOWED_KEYS, - SNOWFLAKE_ADAPTER, SNOWFLAKE_MIN_LAG_SECONDS, -}; use crate::model_validation::models::{ ModelValidationFacts, ProjectValidationFacts, ValidationStop, }; @@ -622,8 +622,8 @@ fn setting_name(name: &str) -> bool { .all(|character| character.is_ascii_alphanumeric() || character == '_') } -/// Check the refresh fields: only maintained models on an adapter that builds them take them. -pub(crate) fn check_maintained( +/// Check the refresh fields: only dynamic models on an adapter that builds them take them. +pub(crate) fn check_dynamic( config: &ConfigView<'_, N>, project: &ProjectValidationFacts, ) -> Check { @@ -638,30 +638,30 @@ pub(crate) fn check_maintained( return Err(coded( config, MATERIALIZED_VIEW_ADAPTER_CODE, - "Snowflake has no materialized_view materialization; use materialized maintained, \ + "Snowflake has no materialized_view materialization; use materialized dynamic, \ which builds a dynamic table", )); } - if materialized.as_deref() != Some(MAINTAINED_MATERIALIZATION) { - return match config.first_present(&MAINTAINED_ONLY_KEYS) { - Some(key) => Err(config.error(format!("{key} is only valid for maintained models"))), + if materialized.as_deref() != Some(DYNAMIC_MATERIALIZATION) { + return match config.first_present(&DYNAMIC_ONLY_KEYS) { + Some(key) => Err(config.error(format!("{key} is only valid for dynamic models"))), None => Ok(()), }; } - if let Some(adapter) = adapter.filter(|adapter| !one_of(adapter, &MAINTAINED_ADAPTERS)) { + if let Some(adapter) = adapter.filter(|adapter| !one_of(adapter, &DYNAMIC_ADAPTERS)) { return Err(coded( config, - MAINTAINED_ADAPTER_CODE, + DYNAMIC_ADAPTER_CODE, format!( - "materialized maintained is not supported on the {adapter} adapter; adapters \ + "materialized dynamic is not supported on the {adapter} adapter; adapters \ that support it: {}", - MAINTAINED_ADAPTERS.join(", ") + DYNAMIC_ADAPTERS.join(", ") ), )); } if let Some(key) = config.first_present(&SNAPSHOT_DISALLOWED_KEYS) { return Err(config.error(format!( - "{key} is not valid for maintained models: the warehouse refreshes them from the \ + "{key} is not valid for dynamic models: the warehouse refreshes them from the \ whole query" ))); } @@ -670,11 +670,11 @@ pub(crate) fn check_maintained( let text: Option = text_if_string(mode); if !text .as_deref() - .is_some_and(|text| one_of(text, &MAINTAINED_REFRESH_MODES)) + .is_some_and(|text| one_of(text, &DYNAMIC_REFRESH_MODES)) { return Err(config.error(format!( "{REFRESH_MODE_KEY} must be one of: {}; got '{}'", - sorted_values(&MAINTAINED_REFRESH_MODES), + sorted_values(&DYNAMIC_REFRESH_MODES), text.unwrap_or_default() ))); } @@ -711,7 +711,7 @@ fn check_refresh(config: &ConfigView<'_, N>, adapter: Option<&s let [(form, value)] = block.as_slice() else { return Err(unknown_form(config, &forms)); }; - if !MAINTAINED_REFRESH_FORMS + if !DYNAMIC_REFRESH_FORMS .iter() .any(|known| form.is_text(known)) { @@ -727,7 +727,7 @@ fn check_refresh(config: &ConfigView<'_, N>, adapter: Option<&s if adapter == Some(SNOWFLAKE_ADAPTER) && lag.fixed_seconds < SNOWFLAKE_MIN_LAG_SECONDS { return Err(coded( config, - MAINTAINED_REFRESH_UNSUPPORTED_CODE, + DYNAMIC_REFRESH_UNSUPPORTED_CODE, format!("{REFRESH_KEY} {LAG_FORM} on Snowflake must be at least 1m; got '{text}'"), )); } diff --git a/crates/sqlbuild-model-config/src/model_validation/constants.rs b/crates/sqlbuild-model-config/src/model_validation/constants.rs index bb0dad808a..c0749a7410 100644 --- a/crates/sqlbuild-model-config/src/model_validation/constants.rs +++ b/crates/sqlbuild-model-config/src/model_validation/constants.rs @@ -2,7 +2,7 @@ /// Built-in materializations that never name a custom materialization. pub const BUILTIN_MATERIALIZATIONS: [&str; 5] = - ["view", "table", "incremental", "snapshot", "maintained"]; + ["view", "table", "incremental", "snapshot", "dynamic"]; /// Materializations that own a physical managed table. pub const TABLE_BACKED_MATERIALIZATIONS: [&str; 3] = ["table", "incremental", "snapshot"]; /// Materializations that keep history across runs. @@ -16,32 +16,32 @@ pub const INCREMENTAL_MATERIALIZATION: &str = "incremental"; /// The snapshot materialization. pub const SNAPSHOT_MATERIALIZATION: &str = "snapshot"; /// The materialization whose stored result the warehouse keeps fresh between builds. -pub const MAINTAINED_MATERIALIZATION: &str = "maintained"; -/// Keys only a maintained model accepts. -pub const MAINTAINED_ONLY_KEYS: [&str; 4] = [ +pub const DYNAMIC_MATERIALIZATION: &str = "dynamic"; +/// Keys only a dynamic model accepts. +pub const DYNAMIC_ONLY_KEYS: [&str; 4] = [ "refresh", "refresh_mode", "refresh_warehouse", "refresh_on_build", ]; /// The `refresh ( … )` forms any adapter accepts; each adapter accepts only its own. -pub const MAINTAINED_REFRESH_FORMS: [&str; 1] = ["lag"]; +pub const DYNAMIC_REFRESH_FORMS: [&str; 1] = ["lag"]; /// `refresh_mode` values. -pub const MAINTAINED_REFRESH_MODES: [&str; 4] = ["incremental", "full", "auto", "adaptive"]; -/// Adapters that build maintained models, sorted. -pub const MAINTAINED_ADAPTERS: [&str; 1] = ["snowflake"]; +pub const DYNAMIC_REFRESH_MODES: [&str; 4] = ["incremental", "full", "auto", "adaptive"]; +/// Adapters that build dynamic models, sorted. +pub const DYNAMIC_ADAPTERS: [&str; 1] = ["snowflake"]; /// The Snowflake adapter name. pub const SNOWFLAKE_ADAPTER: &str = "snowflake"; -/// The materialization name other warehouses use, which Snowflake builds as `maintained`. +/// The materialization name other warehouses use, which Snowflake builds as `dynamic`. pub const MATERIALIZED_VIEW_MATERIALIZATION: &str = "materialized_view"; /// Snowflake's smallest dynamic table target lag, in seconds. pub const SNOWFLAKE_MIN_LAG_SECONDS: u128 = 60; -/// The code of a maintained model on an adapter that cannot build one. -pub const MAINTAINED_ADAPTER_CODE: &str = "A013"; -/// The code of `materialized_view` on Snowflake, which builds `maintained` instead. +/// The code of a dynamic model on an adapter that cannot build one. +pub const DYNAMIC_ADAPTER_CODE: &str = "A013"; +/// The code of `materialized_view` on Snowflake, which builds `dynamic` instead. pub const MATERIALIZED_VIEW_ADAPTER_CODE: &str = "A014"; /// The code of a refresh form or value the adapter does not accept. -pub const MAINTAINED_REFRESH_UNSUPPORTED_CODE: &str = "A015"; +pub const DYNAMIC_REFRESH_UNSUPPORTED_CODE: &str = "A015"; /// `IncrementalStrategy` values. pub const INCREMENTAL_STRATEGIES: [&str; 3] = ["append", "delete_insert", "merge"]; diff --git a/crates/sqlbuild-model-config/src/model_validation/main/validate_model_config.rs b/crates/sqlbuild-model-config/src/model_validation/main/validate_model_config.rs index 23eb12f947..85fad21212 100644 --- a/crates/sqlbuild-model-config/src/model_validation/main/validate_model_config.rs +++ b/crates/sqlbuild-model-config/src/model_validation/main/validate_model_config.rs @@ -3,7 +3,7 @@ use crate::model_validation::_helpers::config::ConfigView; use crate::model_validation::_helpers::incremental::check_incremental; use crate::model_validation::_helpers::materialization::{ - check_contract, check_custom_materialization, check_execution_limit, check_maintained, + check_contract, check_custom_materialization, check_dynamic, check_execution_limit, check_migration, check_non_incremental, check_physical_storage, check_placeholders, check_project_capability, check_review_marker, check_storage_policies, }; @@ -29,7 +29,7 @@ pub fn validate_model_config( check_review_marker(&config)?; check_non_incremental(&config)?; check_snapshot(&config, facts)?; - check_maintained(&config, project)?; + check_dynamic(&config, project)?; check_custom_materialization(&config, project)?; check_storage_policies(&config, facts)?; check_physical_storage(&config)?; diff --git a/crates/sqlbuild-model-config/src/model_validation/tests/test_validate_model_config.rs b/crates/sqlbuild-model-config/src/model_validation/tests/test_validate_model_config.rs index 6375dca5f1..6362687338 100644 --- a/crates/sqlbuild-model-config/src/model_validation/tests/test_validate_model_config.rs +++ b/crates/sqlbuild-model-config/src/model_validation/tests/test_validate_model_config.rs @@ -555,16 +555,16 @@ fn given_effective_configs_when_validating_natively_then_python_outcomes_result( } #[test] -fn given_maintained_model_headers_when_validating_then_refresh_fields_are_checked() { - let maintained = |extra: Vec<(&'static str, Value)>| { - let mut config = vec![("materialized", Value::Str("maintained"))]; +fn given_dynamic_model_headers_when_validating_then_refresh_fields_are_checked() { + let dynamic = |extra: Vec<(&'static str, Value)>| { + let mut config = vec![("materialized", Value::Str("dynamic"))]; config.extend(extra); config }; let test_cases = [ ValidateTestCase { - description: "a maintained model with a lag, a mode and a warehouse", - config: maintained(vec![ + description: "a dynamic model with a lag, a mode and a warehouse", + config: dynamic(vec![ ("refresh", map(vec![("lag", Value::Str("1h"))])), ("refresh_mode", Value::Str("adaptive")), ("refresh_warehouse", Value::Str("transform_wh")), @@ -575,8 +575,8 @@ fn given_maintained_model_headers_when_validating_then_refresh_fields_are_checke expected_outcome: "accepted", }, ValidateTestCase { - description: "a maintained model refreshed by its readers", - config: maintained(vec![]), + description: "a dynamic model refreshed by its readers", + config: dynamic(vec![]), references: &["ref:orders"], query_sql: "select 1", expected_outcome: "accepted", @@ -589,25 +589,25 @@ fn given_maintained_model_headers_when_validating_then_refresh_fields_are_checke ], references: &[], query_sql: "select 1", - expected_outcome: "model 'orders_daily': refresh is only valid for maintained models", + expected_outcome: "model 'orders_daily': refresh is only valid for dynamic models", }, ValidateTestCase { description: "refresh_on_build on a model with no materialization", config: vec![("refresh_on_build", Value::Bool(true))], references: &[], query_sql: "select 1", - expected_outcome: "model 'orders_daily': refresh_on_build is only valid for maintained models", + expected_outcome: "model 'orders_daily': refresh_on_build is only valid for dynamic models", }, ValidateTestCase { description: "a refresh block with a form no adapter has", - config: maintained(vec![("refresh", map(vec![("every", Value::Str("1h"))]))]), + config: dynamic(vec![("refresh", map(vec![("every", Value::Str("1h"))]))]), references: &[], query_sql: "select 1", expected_outcome: "model 'orders_daily': refresh takes exactly one form, such as refresh (lag 1h); got: every", }, ValidateTestCase { description: "a refresh block with two forms", - config: maintained(vec![( + config: dynamic(vec![( "refresh", map(vec![ ("lag", Value::Str("1h")), @@ -620,63 +620,63 @@ fn given_maintained_model_headers_when_validating_then_refresh_fields_are_checke }, ValidateTestCase { description: "refresh written as a plain value", - config: maintained(vec![("refresh", Value::Str("1h"))]), + config: dynamic(vec![("refresh", Value::Str("1h"))]), references: &[], query_sql: "select 1", expected_outcome: "model 'orders_daily': refresh must be a block such as refresh (lag 1h)", }, ValidateTestCase { description: "a lag in months", - config: maintained(vec![("refresh", map(vec![("lag", Value::Str("1mo"))]))]), + config: dynamic(vec![("refresh", map(vec![("lag", Value::Str("1mo"))]))]), references: &[], query_sql: "select 1", expected_outcome: "model 'orders_daily': refresh lag must be a fixed duration such as 5m, 1h or 1d; got '1mo'", }, ValidateTestCase { description: "a lag that is not a duration", - config: maintained(vec![("refresh", map(vec![("lag", Value::Str("hourly"))]))]), + config: dynamic(vec![("refresh", map(vec![("lag", Value::Str("hourly"))]))]), references: &[], query_sql: "select 1", expected_outcome: "model 'orders_daily': refresh lag must be a fixed duration such as 5m, 1h or 1d; got 'hourly'", }, ValidateTestCase { description: "an unknown refresh mode", - config: maintained(vec![("refresh_mode", Value::Str("fast"))]), + config: dynamic(vec![("refresh_mode", Value::Str("fast"))]), references: &[], query_sql: "select 1", expected_outcome: "model 'orders_daily': refresh_mode must be one of: adaptive, auto, full, incremental; got 'fast'", }, ValidateTestCase { description: "refresh_on_build that is not a boolean", - config: maintained(vec![("refresh_on_build", Value::Str("yes"))]), + config: dynamic(vec![("refresh_on_build", Value::Str("yes"))]), references: &[], query_sql: "select 1", expected_outcome: "model 'orders_daily': refresh_on_build must be a boolean", }, ValidateTestCase { description: "a blank refresh warehouse", - config: maintained(vec![("refresh_warehouse", Value::Str(" "))]), + config: dynamic(vec![("refresh_warehouse", Value::Str(" "))]), references: &[], query_sql: "select 1", expected_outcome: "model 'orders_daily': refresh_warehouse must be a non-empty warehouse name", }, ValidateTestCase { - description: "an incremental-only key on a maintained model", - config: maintained(vec![("cursor", Value::Str("updated_at"))]), + description: "an incremental-only key on a dynamic model", + config: dynamic(vec![("cursor", Value::Str("updated_at"))]), references: &[], query_sql: "select 1", - expected_outcome: "model 'orders_daily': cursor is not valid for maintained models: the warehouse refreshes them from the whole query", + expected_outcome: "model 'orders_daily': cursor is not valid for dynamic models: the warehouse refreshes them from the whole query", }, ValidateTestCase { description: "the smallest lag Snowflake accepts", - config: maintained(vec![("refresh", map(vec![("lag", Value::Str("1m"))]))]), + config: dynamic(vec![("refresh", map(vec![("lag", Value::Str("1m"))]))]), references: &[], query_sql: "select 1", expected_outcome: "accepted", }, ValidateTestCase { description: "a lag one second under Snowflake's minimum", - config: maintained(vec![("refresh", map(vec![("lag", Value::Str("59s"))]))]), + config: dynamic(vec![("refresh", map(vec![("lag", Value::Str("59s"))]))]), references: &[], query_sql: "select 1", expected_outcome: "model 'orders_daily': refresh lag on Snowflake must be at least 1m; got '59s'", @@ -686,7 +686,7 @@ fn given_maintained_model_headers_when_validating_then_refresh_fields_are_checke config: vec![("materialized", Value::Str("materialized_view"))], references: &[], query_sql: "select 1", - expected_outcome: "model 'orders_daily': Snowflake has no materialized_view materialization; use materialized maintained, which builds a dynamic table", + expected_outcome: "model 'orders_daily': Snowflake has no materialized_view materialization; use materialized dynamic, which builds a dynamic table", }, ]; diff --git a/crates/sqlbuild-orchestration/tests/api/test_warehouse_metadata.rs b/crates/sqlbuild-orchestration/tests/api/test_warehouse_metadata.rs index 8df1af7c22..901f154c74 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_warehouse_metadata.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_warehouse_metadata.rs @@ -3,7 +3,7 @@ use chrono::{DateTime, Duration, Utc}; use reqwest::StatusCode; use serde_json::json; -use sqlbuild_core::constants::{MAINTAINED_PAUSES_TABLE_NAME, RUNS_TABLE_NAME, STATE_TABLE_NAMES}; +use sqlbuild_core::constants::{DYNAMIC_PAUSES_TABLE_NAME, RUNS_TABLE_NAME, STATE_TABLE_NAMES}; use uuid::Uuid; use crate::api::tests::changes_helpers::{connect, expect_topics}; @@ -37,10 +37,10 @@ use crate::warehouse_metadata::models::{MetadataBatch, WarehouseHistoryKind}; #[test] fn given_state_tables_when_classified_then_each_is_mirrored_by_exactly_one_kind() { // The runs table has no mirror until the control store decides where observed runs live; - // maintained pauses are a lease that builds read from the warehouse, never a mirror. + // dynamic pauses are a lease that builds read from the warehouse, never a mirror. let mut written: Vec<&str> = STATE_TABLE_NAMES .into_iter() - .filter(|table| ![RUNS_TABLE_NAME, MAINTAINED_PAUSES_TABLE_NAME].contains(table)) + .filter(|table| ![RUNS_TABLE_NAME, DYNAMIC_PAUSES_TABLE_NAME].contains(table)) .collect(); written.sort_unstable(); let test_cases = [StateTableClassificationTestCase { diff --git a/crates/sqlbuild-python-base/src/records/_helpers/compiler_enums/constants.rs b/crates/sqlbuild-python-base/src/records/_helpers/compiler_enums/constants.rs index 5b65da0fd2..8621d90ffb 100644 --- a/crates/sqlbuild-python-base/src/records/_helpers/compiler_enums/constants.rs +++ b/crates/sqlbuild-python-base/src/records/_helpers/compiler_enums/constants.rs @@ -586,9 +586,9 @@ const E_PLAN_ACTION: EnumSpec = EnumSpec { ("LOAD_SEED", "load_seed"), ("SKIP", "skip"), ("CUSTOM", "custom"), - ("CREATE_MAINTAINED", "create_maintained"), - ("ALTER_MAINTAINED", "alter_maintained"), - ("KEEP_MAINTAINED", "keep_maintained"), + ("CREATE_DYNAMIC", "create_dynamic"), + ("ALTER_DYNAMIC", "alter_dynamic"), + ("KEEP_DYNAMIC", "keep_dynamic"), ], }; const E_PLAN_REASON: EnumSpec = EnumSpec { @@ -686,7 +686,7 @@ const E_MATERIALIZATION_TYPE: EnumSpec = EnumSpec { ("SNAPSHOT", "snapshot"), ("SEED", "seed"), ("CUSTOM", "custom"), - ("MAINTAINED", "maintained"), + ("DYNAMIC", "dynamic"), ], }; const E_SCENARIO_ARTIFACT_KIND: EnumSpec = EnumSpec { diff --git a/crates/sqlbuild-python-base/src/records/_helpers/contract_records/constants.rs b/crates/sqlbuild-python-base/src/records/_helpers/contract_records/constants.rs index 46587a7e93..58bc8be8fa 100644 --- a/crates/sqlbuild-python-base/src/records/_helpers/contract_records/constants.rs +++ b/crates/sqlbuild-python-base/src/records/_helpers/contract_records/constants.rs @@ -151,7 +151,7 @@ const R_TARGET_CONFIG: RecordSpec = RecordSpec { FieldSpec::new("catalog_connection", FieldDefault::None), FieldSpec::new("ignore_audit_failures", FieldDefault::None), FieldSpec::new("plan_from", FieldDefault::None), - FieldSpec::new("pause_maintained", FieldDefault::None), + FieldSpec::new("pause_dynamic", FieldDefault::None), ], }; const R_TARGET_SETTINGS: RecordSpec = RecordSpec { @@ -211,7 +211,7 @@ const R_LOCAL_TARGET_CONFIG: RecordSpec = RecordSpec { FieldSpec::new("catalog_connection", FieldDefault::None), FieldSpec::new("ignore_audit_failures", FieldDefault::None), FieldSpec::new("plan_from", FieldDefault::None), - FieldSpec::new("pause_maintained", FieldDefault::None), + FieldSpec::new("pause_dynamic", FieldDefault::None), ], }; const R_SETTINGS_CONFIG: RecordSpec = RecordSpec { diff --git a/crates/sqlbuild-python-base/src/records/_helpers/planner_records/constants.rs b/crates/sqlbuild-python-base/src/records/_helpers/planner_records/constants.rs index a5bb652b75..be7daa312e 100644 --- a/crates/sqlbuild-python-base/src/records/_helpers/planner_records/constants.rs +++ b/crates/sqlbuild-python-base/src/records/_helpers/planner_records/constants.rs @@ -671,8 +671,8 @@ const R_MODEL_PLAN_ENTRY: RecordSpec = RecordSpec { FieldSpec::new("refresh_mode", FieldDefault::None), FieldSpec::new("refresh_warehouse", FieldDefault::None), FieldSpec::new("refresh_on_build", FieldDefault::False), - FieldSpec::new("maintained_readers", FieldDefault::EmptyTuple), - FieldSpec::new("maintained_paused", FieldDefault::False), + FieldSpec::new("dynamic_readers", FieldDefault::EmptyTuple), + FieldSpec::new("dynamic_paused", FieldDefault::False), ], }; const R_CLONE_SOURCE_PLAN_ENTRY: RecordSpec = RecordSpec { diff --git a/crates/sqlbuild-python-base/src/records/constants.rs b/crates/sqlbuild-python-base/src/records/constants.rs index 6825d659f6..d2b0e21e1b 100644 --- a/crates/sqlbuild-python-base/src/records/constants.rs +++ b/crates/sqlbuild-python-base/src/records/constants.rs @@ -255,7 +255,7 @@ const BUILD_COMMAND_REQUEST: RecordSpec = RecordSpec { FieldSpec::new("allow_retention_decrease", FieldDefault::False), FieldSpec::new("allow_missing_migration_origin", FieldDefault::False), FieldSpec::new("allow_snapshot_schema_change", FieldDefault::False), - FieldSpec::new("ignore_maintained_grace", FieldDefault::False), + FieldSpec::new("ignore_dynamic_grace", FieldDefault::False), FieldSpec::new("concurrency", FieldDefault::None), FieldSpec::new("select", FieldDefault::EmptyTuple), FieldSpec::new("exclude", FieldDefault::EmptyTuple), diff --git a/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/maintained.rs b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/dynamic.rs similarity index 71% rename from crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/maintained.rs rename to crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/dynamic.rs index cb045e0beb..76f6016522 100644 --- a/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/maintained.rs +++ b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/dynamic.rs @@ -1,10 +1,10 @@ -//! Project-wide checks of maintained models on the effective adapter. +//! Project-wide checks of dynamic models on the effective adapter. use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; -use sqlbuild_analysis::assembly::maintained_checks::constants::MAINTAINED_NEVER_REFRESHED_CODE; -use sqlbuild_analysis::assembly::maintained_checks::main::never_refreshed_maintained::never_refreshed_maintained; -use sqlbuild_analysis::assembly::maintained_checks::models::MaintainedModel; +use sqlbuild_analysis::assembly::dynamic_checks::constants::DYNAMIC_NEVER_REFRESHED_CODE; +use sqlbuild_analysis::assembly::dynamic_checks::main::never_refreshed_dynamic::never_refreshed_dynamic; +use sqlbuild_analysis::assembly::dynamic_checks::models::DynamicModel; use sqlbuild_python_base::boundary::main::optional_text::optional_text; use sqlbuild_python_base::records::main::construct::enum_member::enum_member; use sqlbuild_python_base::records::main::fields::field::field as native_field; @@ -17,11 +17,11 @@ use sqlbuild_python_front::model_config::types::ConfigValues; use crate::compile_driver::_helpers::setup::targets::planner_error; const SNOWFLAKE_ADAPTER: &str = "snowflake"; -const MAINTAINED_MATERIALIZATION: &str = "maintained"; +const DYNAMIC_MATERIALIZATION: &str = "dynamic"; const REFRESH_KEY: &str = "refresh"; -/// Raise when a Snowflake maintained model has no refresh and no maintained model reads it. -pub(crate) fn maintained_refresh( +/// Raise when a Snowflake dynamic model has no refresh and no dynamic model reads it. +pub(crate) fn dynamic_refresh( adapter_name: &Bound<'_, PyAny>, project: &Bound<'_, PyAny>, ) -> PyResult<()> { @@ -30,12 +30,11 @@ pub(crate) fn maintained_refresh( } let py: Python<'_> = project.py(); let model_type: Bound<'_, PyAny> = enum_member(py, RecordId::CompiledResourceType, "MODEL")?; - let mut maintained: Vec = Vec::new(); + let mut dynamic: Vec = Vec::new(); for model in native_field(project, "models")?.try_iter()? { let model: Bound<'_, PyAny> = model?; let values: ConfigValues<'_> = model_values(&model)?; - if config_text(py, &values, "materialized")?.as_deref() != Some(MAINTAINED_MATERIALIZATION) - { + if config_text(py, &values, "materialized")?.as_deref() != Some(DYNAMIC_MATERIALIZATION) { continue; } let mut reads: Vec = Vec::new(); @@ -45,22 +44,22 @@ pub(crate) fn maintained_refresh( reads.push(native_field(&key, "name")?.extract()?); } } - maintained.push(MaintainedModel { + dynamic.push(DynamicModel { name: native_field(&model, "name")?.extract()?, has_refresh: !config_value(py, &values, REFRESH_KEY)?.is_none(), reads, }); } - let Some(name) = never_refreshed_maintained(&maintained) else { + let Some(name) = never_refreshed_dynamic(&dynamic) else { return Ok(()); }; Err(planner_error( py, format!( - "maintained model '{name}' has no refresh and no maintained model reads it, so \ + "dynamic model '{name}' has no refresh and no dynamic model reads it, so \ Snowflake would never refresh it; add refresh (lag 1h)" ), - MAINTAINED_NEVER_REFRESHED_CODE, + DYNAMIC_NEVER_REFRESHED_CODE, Some(format!( "set how stale '{name}' may get in its MODEL header:\n refresh (lag 1h)," )), diff --git a/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/mod.rs b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/mod.rs index 961c650c8f..7c158b10a8 100644 --- a/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/mod.rs +++ b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/mod.rs @@ -1,5 +1,5 @@ +pub(crate) mod dynamic; pub(crate) mod inputs; -pub(crate) mod maintained; pub(crate) mod model_free_locations; pub(crate) mod node_results; pub(crate) mod pipeline; diff --git a/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/project.rs b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/project.rs index 16444bd260..4f640436b0 100644 --- a/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/project.rs +++ b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/project.rs @@ -14,7 +14,7 @@ use pyo3::types::{PyBool, PyCFunction, PyDict, PyTuple}; use sqlbuild_host::compile_context::main::record_compile_timing::record_compile_timing; use sqlbuild_python_project::graph::main::python_nodes::resolve_node_sets::resolve_node_sets; -use crate::compile_driver::_helpers::setup::maintained::maintained_refresh; +use crate::compile_driver::_helpers::setup::dynamic::dynamic_refresh; use crate::compile_driver::_helpers::setup::target_defaults::TargetDefaults; use crate::compile_driver::_helpers::setup::target_defaults::apply_target_defaults; use crate::compile_driver::_helpers::setup::targets::managed_loader_target_isolation; @@ -243,7 +243,7 @@ fn targeted_project<'py>( }; let project: Bound<'py, PyAny> = apply_target_defaults(project, &defaults)?; project_targets(&adapter_name, &project)?; - maintained_refresh(&adapter_name, &project)?; + dynamic_refresh(&adapter_name, &project)?; let promotion_key: Bound<'py, PyAny> = pyo3::types::PyString::new(py, TABLE_PROMOTION_MODE_SETTING_KEY).into_any(); let settings_file: Bound<'py, PyAny> = diff --git a/crates/sqlbuild-python-front/src/discovery/main/project/configured_maintained.rs b/crates/sqlbuild-python-front/src/discovery/main/project/configured_dynamic.rs similarity index 69% rename from crates/sqlbuild-python-front/src/discovery/main/project/configured_maintained.rs rename to crates/sqlbuild-python-front/src/discovery/main/project/configured_dynamic.rs index daf7d1ec7f..71b2570941 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/project/configured_maintained.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/project/configured_dynamic.rs @@ -1,16 +1,16 @@ -//! A build's maintained-pause runtime settings: the target's `pause_maintained` and `[maintained]`. +//! A build's dynamic-pause runtime settings: the target's `pause_dynamic` and `[dynamic]`. use crate::contracts::main::resolve_target_config::resolve_target_config; use crate::contracts::main::resolve_target_name::resolve_target_name; use crate::discovery::_helpers::project::project_config::configured; use pyo3::IntoPyObjectExt; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; -use sqlbuild_config::project::main::load_maintained_config::load_maintained_config; -use sqlbuild_config::project::models::MaintainedConfig; +use sqlbuild_config::project::main::load_dynamic_config::load_dynamic_config; +use sqlbuild_config::project::models::DynamicConfig; use sqlbuild_python_base::records::main::fields::field::field as native_field; -/// `BuildRuntimeParams` fields for the request's target and flag, and the project's `[maintained]`. -pub fn configured_maintained<'py>( +/// `BuildRuntimeParams` fields for the request's target and flag, and the project's `[dynamic]`. +pub fn configured_dynamic<'py>( discovered: &Bound<'py, PyAny>, request: &Bound<'py, PyAny>, project_dir: &Bound<'py, PyAny>, @@ -23,28 +23,28 @@ pub fn configured_maintained<'py>( match resolve_target_name(py, &project_config, &local_config, &selected_target)? { Some(name) => native_field( &resolve_target_config(py, &project_config, &local_config, name.as_any())?, - "pause_maintained", + "pause_dynamic", )? .extract()?, None => None, }; - let config: MaintainedConfig = configured(project_dir, load_maintained_config)?; + let config: DynamicConfig = configured(project_dir, load_dynamic_config)?; Ok(vec![ ( - "pause_maintained", + "pause_dynamic", paused.unwrap_or(false).into_bound_py_any(py)?, ), ( - "maintained_heartbeat_grace_seconds", + "dynamic_heartbeat_grace_seconds", config.heartbeat_grace_seconds.into_bound_py_any(py)?, ), ( - "allow_ignore_maintained_grace", + "allow_ignore_dynamic_grace", config.allow_ignore_grace.into_bound_py_any(py)?, ), ( - "ignore_maintained_grace", - native_field(request, "ignore_maintained_grace")?, + "ignore_dynamic_grace", + native_field(request, "ignore_dynamic_grace")?, ), ]) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/project/mod.rs b/crates/sqlbuild-python-front/src/discovery/main/project/mod.rs index 1f0aeb28e9..85eb080380 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/project/mod.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/project/mod.rs @@ -1,7 +1,7 @@ pub mod column_name; pub mod configured_changes; +pub mod configured_dynamic; pub mod configured_git_base_ref; -pub mod configured_maintained; pub mod configured_review; pub mod discover_configured_providers; pub mod discover_connection_configuration; diff --git a/crates/sqlbuild-python-planner/src/live_reads/models.rs b/crates/sqlbuild-python-planner/src/live_reads/models.rs index 3841b31896..06110c27ea 100644 --- a/crates/sqlbuild-python-planner/src/live_reads/models.rs +++ b/crates/sqlbuild-python-planner/src/live_reads/models.rs @@ -438,8 +438,8 @@ impl WarehouseReads for LiveReads { "column_rename_unavailable_reason", vec![("destination", destination)], ), - AdapterRead::MaintainedRefresh([database, schema, name]) => ( - "maintained_listing", + AdapterRead::DynamicRefresh([database, schema, name]) => ( + "dynamic_listing", vec![("database", database), ("schema", schema), ("name", name)], ), }; diff --git a/crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_json.rs b/crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_json.rs index d92a09036f..186f30f55c 100644 --- a/crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_json.rs +++ b/crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_json.rs @@ -234,7 +234,7 @@ pub(crate) fn model_json<'py>(entry: &Bound<'py, PyAny>) -> PyResult = diff --git a/crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_text.rs b/crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_text.rs index e67e1c9a06..9dcef5903c 100644 --- a/crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_text.rs +++ b/crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_text.rs @@ -251,7 +251,7 @@ fn model_text(entry: &Bound<'_, PyAny>) -> PyResult { config_diff: coloured(model_config_diff(entry, &reason)?), query_diff: coloured(model_query_diff(entry, &reason)?), reinitialised_by: optional_text(&native_field(entry, "reinitialised_by")?)?, - paused: native_field(entry, "maintained_paused")?.is_truthy()?, + paused: native_field(entry, "dynamic_paused")?.is_truthy()?, name, action, reason, diff --git a/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/display_plan.rs b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/display_plan.rs index cabdb8403f..c26c222219 100644 --- a/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/display_plan.rs +++ b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/display_plan.rs @@ -24,7 +24,7 @@ use sqlbuild_python_project::planner_phases::main::object::object; const VIEW: &str = "view"; const INCREMENTAL: &str = "incremental"; const CUSTOM: &str = "custom"; -const MAINTAINED: &str = "maintained"; +const DYNAMIC: &str = "dynamic"; fn display_action( materialization_type: &Bound<'_, PyAny>, @@ -40,8 +40,8 @@ fn display_action( } } else if materialization_type.eq(CUSTOM)? { "CUSTOM" - } else if materialization_type.eq(MAINTAINED)? { - "CREATE_MAINTAINED" + } else if materialization_type.eq(DYNAMIC)? { + "CREATE_DYNAMIC" } else { "CREATE_TABLE" }) diff --git a/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/execution_plan.rs b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/execution_plan.rs index f33159b252..a36dd6b897 100644 --- a/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/execution_plan.rs +++ b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/execution_plan.rs @@ -27,7 +27,7 @@ use crate::planner_phases::main::outputs::base_plan_output::base_plan_output; use crate::planner_phases::main::outputs::build_planner_identity_context::build_planner_identity_context; use crate::planner_phases::main::outputs::check_selected_scope_buildability::check_selected_scope_buildability; use crate::planner_phases::main::outputs::detect_stale_warning_changes::detect_stale_warning_changes; -use crate::planner_phases::main::outputs::plan_maintained_refresh::plan_maintained_refresh; +use crate::planner_phases::main::outputs::plan_dynamic_refresh::plan_dynamic_refresh; use crate::planner_phases::main::outputs::unselected_upstream_models::unselected_upstream_models; use crate::planner_phases::main::rebuilds::build_planner_entry_results::build_planner_entry_results; use crate::planner_phases::main::rebuilds::defer_to_migration_origins::defer_to_migration_origins; @@ -197,7 +197,7 @@ pub(crate) fn planned_output<'py>( (&inspection_scope, &reconciliation), &source_freshness, )?; - let entries = plan_maintained_refresh(runtime, &inspection_scope, entries)?; + let entries = plan_dynamic_refresh(runtime, &inspection_scope, entries)?; record_step(BUILDING_ENTRIES, entries_started.elapsed().as_secs_f64()); let planned_sql_tests = timed(TEST_FIXTURES, || test_planning.result(py))?.into_bound(py); let assembling_started = Instant::now(); diff --git a/crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/maintained.rs b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/dynamic.rs similarity index 79% rename from crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/maintained.rs rename to crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/dynamic.rs index 4f0650b1be..d073a6e2e2 100644 --- a/crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/maintained.rs +++ b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/dynamic.rs @@ -1,4 +1,4 @@ -//! Maintained models whose live refresh settings drifted, kept ones a replaced upstream reinitialises, and who reads what. +//! Dynamic models whose live refresh settings drifted, kept ones a replaced upstream reinitialises, and who reads what. use crate::planner_phases::main::inputs::runtime_reads::runtime_reads; use crate::planner_phases::models::ModelEntryResults; @@ -9,13 +9,14 @@ use pyo3::IntoPyObject; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyBool, PyString, PyTuple}; use sqlbuild_analysis::assembly::project::types::ObjectKey; -use sqlbuild_analysis::maintained_refresh::main::maintained_settings_drift::maintained_settings_drift; -use sqlbuild_analysis::maintained_refresh::main::reinitialised_readers::reinitialised_readers; -use sqlbuild_analysis::maintained_refresh::main::render_target_lag::render_target_lag; -use sqlbuild_analysis::maintained_refresh::models::{ - LiveDynamicTable, MaintainedRead, MaintainedSettings, RefreshLag, +use sqlbuild_analysis::dynamic_refresh::main::dynamic_settings_drift::dynamic_settings_drift; +use sqlbuild_analysis::dynamic_refresh::main::reinitialised_readers::reinitialised_readers; +use sqlbuild_analysis::dynamic_refresh::main::render_target_lag::render_target_lag; +use sqlbuild_analysis::dynamic_refresh::models::{ + DynamicRead, DynamicSettings, LiveDynamicTable, RefreshLag, }; use sqlbuild_analysis::planner_scope::constants::MODEL; +use sqlbuild_core::fingerprint_identity::constants::DEFAULT_REFRESH_MODE; use sqlbuild_model_config::model_validation::main::fixed_duration_seconds::fixed_duration_seconds; use sqlbuild_python_base::records::main::construct::enum_member::enum_member; use sqlbuild_python_base::records::main::fields::field::field as native_field; @@ -29,19 +30,17 @@ use sqlbuild_python_project::native_host::models::ModelLookup; use sqlbuild_python_project::planner_scope::main::scope_data::scope_data; use std::collections::{BTreeSet, HashMap}; -const MAINTAINED: &str = "maintained"; -/// Views pass reads through, so a maintained model reading a view reads the view's inputs. +const DYNAMIC: &str = "dynamic"; +/// Views pass reads through, so a dynamic model reading a view reads the view's inputs. const VIEW: &str = "view"; -const KEEP_MAINTAINED: &str = "keep_maintained"; +const KEEP_DYNAMIC: &str = "keep_dynamic"; /// The scheduling state Snowflake lists for a dynamic table whose refreshes are paused. const SUSPENDED: &str = "SUSPENDED"; -/// The refresh mode of a maintained model that declares none, so Snowflake never picks one quietly. -const DEFAULT_REFRESH_MODE: &str = "incremental"; /// Actions that recreate the relation, which reinitialises every dynamic table reading it. -const REPLACING_ACTIONS: [&str; 3] = ["create_table", "create_view", "create_maintained"]; +const REPLACING_ACTIONS: [&str; 3] = ["create_table", "create_view", "create_dynamic"]; -/// Plan maintained refresh settings and reinitialised readers, and name the maintained readers of each entry. -pub(crate) fn plan_maintained_refresh<'py>( +/// Plan dynamic refresh settings and reinitialised readers, and name the dynamic readers of each entry. +pub(crate) fn plan_dynamic_refresh<'py>( runtime: &Bound<'py, PyAny>, execution_scope: &Bound<'py, PyAny>, results: ModelEntryResults<'py>, @@ -65,29 +64,28 @@ pub(crate) fn plan_maintained_refresh<'py>( through_view: HashMap::new(), }; let mut entries: Vec> = Vec::with_capacity(results.entries.len()); - let mut rows: Vec = Vec::with_capacity(results.entries.len()); + let mut rows: Vec = Vec::with_capacity(results.entries.len()); for entry in results.entries { let key: ObjectKey = native_key(&native_field(&entry, "key")?)?; - let is_maintained: bool = - text(&native_field(&entry, "materialization_type")?)? == MAINTAINED; - let entry = if is_maintained { + let is_dynamic: bool = text(&native_field(&entry, "materialization_type")?)? == DYNAMIC; + let entry = if is_dynamic { with_refresh_settings(entry, &models, &reads)? } else { entry }; - let readers: Vec = known.maintained_readers(py, &key, &downstream)?; + let readers: Vec = known.dynamic_readers(py, &key, &downstream)?; let entry = if readers.is_empty() { entry } else { replaced( &entry, - &[("maintained_readers", PyTuple::new(py, readers)?.into_any())], + &[("dynamic_readers", PyTuple::new(py, readers)?.into_any())], )? }; let action: String = text(&native_field(&entry, "action")?)?; - rows.push(MaintainedRead { + rows.push(DynamicRead { name: key.1.clone(), - maintained: is_maintained, + dynamic: is_dynamic, replaced: REPLACING_ACTIONS.contains(&action.as_str()), reads: upstream.remove(&key).unwrap_or_default(), }); @@ -108,7 +106,7 @@ pub(crate) fn plan_maintained_refresh<'py>( }) } -/// Per-plan lookups: each model's materialization, and the maintained readers behind each view. +/// Per-plan lookups: each model's materialization, and the dynamic readers behind each view. struct KnownReaders<'a> { models: &'a ModelLookup, materializations: HashMap, @@ -116,8 +114,8 @@ struct KnownReaders<'a> { } impl KnownReaders<'_> { - /// The maintained models reading the entry directly or through views, selected or not. - fn maintained_readers( + /// The dynamic models reading the entry directly or through views, selected or not. + fn dynamic_readers( &mut self, py: Python<'_>, key: &ObjectKey, @@ -134,13 +132,12 @@ impl KnownReaders<'_> { continue; } match self.materialization(py, name)?.as_str() { - MAINTAINED => { + DYNAMIC => { found.insert(name.clone()); } VIEW => { if !self.through_view.contains_key(name) { - let through: Vec = - self.maintained_readers(py, reader, downstream)?; + let through: Vec = self.dynamic_readers(py, reader, downstream)?; self.through_view.insert(name.clone(), through); } found.extend(self.through_view[name].iter().cloned()); @@ -204,14 +201,14 @@ fn with_refresh_settings<'py>( .into_any(), ), ]; - if text(&native_field(&entry, "action")?)? == KEEP_MAINTAINED { + if text(&native_field(&entry, "action")?)? == KEEP_DYNAMIC { let (action, paused) = live_action(&entry, &desired, reads)?; if let Some((action, reason)) = action { changes.push(("action", enum_member(py, RecordId::PlanAction, action)?)); changes.push(("reason", enum_member(py, RecordId::PlanReason, reason)?)); } changes.push(( - "maintained_paused", + "dynamic_paused", PyBool::new(py, paused).to_owned().into_any(), )); } @@ -221,13 +218,13 @@ fn with_refresh_settings<'py>( /// The action and reason replacing a kept model's, `None` when its listing matches or is unknown, and whether it is suspended. fn live_action( entry: &Bound<'_, PyAny>, - desired: &MaintainedSettings, + desired: &DynamicSettings, reads: &Bound<'_, PlannerReads>, ) -> PyResult<(Option<(&'static str, &'static str)>, bool)> { let destination = native_field(entry, "destination")?; let read = reads.get().reads().adapter_read( entry.py(), - AdapterRead::MaintainedRefresh([ + AdapterRead::DynamicRefresh([ native_field(&destination, "database")?, native_field(&destination, "schema")?, native_field(&destination, "name")?, @@ -238,7 +235,7 @@ fn live_action( Read::Known(listed) => listed, }; if listed.is_none() { - return Ok((Some(("CREATE_MAINTAINED", "FIRST_RUN")), false)); + return Ok((Some(("CREATE_DYNAMIC", "FIRST_RUN")), false)); } let (target_lag, warehouse, state): (String, String, String) = listed.extract()?; let live = LiveDynamicTable { @@ -246,12 +243,12 @@ fn live_action( warehouse, }; let action = - maintained_settings_drift(desired, &live).then_some(("ALTER_MAINTAINED", "CONFIG_CHANGED")); + dynamic_settings_drift(desired, &live).then_some(("ALTER_DYNAMIC", "CONFIG_CHANGED")); Ok((action, state == SUSPENDED)) } /// The model's declared lag and refresh warehouse, or `None` when its lag does not read. -fn declared_settings(model: &Bound<'_, PyAny>) -> PyResult> { +fn declared_settings(model: &Bound<'_, PyAny>) -> PyResult> { let py = model.py(); let values = model_values(model)?; let refresh = config_value(py, &values, "refresh")?; @@ -270,5 +267,5 @@ fn declared_settings(model: &Bound<'_, PyAny>) -> PyResult = config_text_object(py, &values, "refresh_warehouse")? .map(|warehouse| warehouse.extract()) .transpose()?; - Ok(Some(MaintainedSettings { lag, warehouse })) + Ok(Some(DynamicSettings { lag, warehouse })) } diff --git a/crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/mod.rs b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/mod.rs index 1295d1283b..b1dc8c7e1a 100644 --- a/crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/mod.rs +++ b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/mod.rs @@ -1,4 +1,4 @@ pub(crate) mod buildability; +pub(crate) mod dynamic; pub(crate) mod identities; -pub(crate) mod maintained; pub(crate) mod output_assembly; diff --git a/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/mod.rs b/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/mod.rs index 28947d1f49..64d9efc5fb 100644 --- a/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/mod.rs +++ b/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/mod.rs @@ -7,6 +7,6 @@ pub mod build_planner_identity_context; pub mod check_selected_scope_buildability; pub mod detect_stale_warning_changes; pub mod detect_stale_warning_changes_of_function; -pub mod plan_maintained_refresh; +pub mod plan_dynamic_refresh; pub mod planner_identity_work_info_function; pub mod unselected_upstream_models; diff --git a/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/plan_dynamic_refresh.rs b/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/plan_dynamic_refresh.rs new file mode 100644 index 0000000000..b0cc4aef7b --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/plan_dynamic_refresh.rs @@ -0,0 +1,15 @@ +//! Alter drifted dynamic models, set their target lag, and mark kept readers of a replaced upstream. + +use crate::planner_phases::_helpers::outputs::dynamic; +use crate::planner_phases::models::ModelEntryResults; +use pyo3::prelude::{Bound, PyAny, PyResult}; + +/// Alter drifted dynamic models, set their target lag, and mark kept readers of a replaced upstream. +#[inline] +pub fn plan_dynamic_refresh<'py>( + runtime: &Bound<'py, PyAny>, + execution_scope: &Bound<'py, PyAny>, + entries: ModelEntryResults<'py>, +) -> PyResult> { + dynamic::plan_dynamic_refresh(runtime, execution_scope, entries) +} diff --git a/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/plan_maintained_refresh.rs b/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/plan_maintained_refresh.rs deleted file mode 100644 index 2b789a1dc5..0000000000 --- a/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/plan_maintained_refresh.rs +++ /dev/null @@ -1,15 +0,0 @@ -//! Alter drifted maintained models, set their target lag, and mark kept readers of a replaced upstream. - -use crate::planner_phases::_helpers::outputs::maintained; -use crate::planner_phases::models::ModelEntryResults; -use pyo3::prelude::{Bound, PyAny, PyResult}; - -/// Alter drifted maintained models, set their target lag, and mark kept readers of a replaced upstream. -#[inline] -pub fn plan_maintained_refresh<'py>( - runtime: &Bound<'py, PyAny>, - execution_scope: &Bound<'py, PyAny>, - entries: ModelEntryResults<'py>, -) -> PyResult> { - maintained::plan_maintained_refresh(runtime, execution_scope, entries) -} diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/strategy.rs b/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/strategy.rs index 89cc09cce8..271f327da0 100644 --- a/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/strategy.rs +++ b/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/strategy.rs @@ -30,9 +30,9 @@ pub(crate) fn action_value(action: PlanAction) -> &'static str { PlanAction::Snapshot => "snapshot", PlanAction::Skip => "skip", PlanAction::Custom => "custom", - PlanAction::CreateMaintained => "create_maintained", - PlanAction::AlterMaintained => "alter_maintained", - PlanAction::KeepMaintained => "keep_maintained", + PlanAction::CreateDynamic => "create_dynamic", + PlanAction::AlterDynamic => "alter_dynamic", + PlanAction::KeepDynamic => "keep_dynamic", } } diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/values.rs b/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/values.rs index 284859ce87..5a67e42a1c 100644 --- a/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/values.rs +++ b/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/values.rs @@ -55,7 +55,7 @@ pub(crate) fn materialization_value(materialization: MaterializationType) -> &'s MaterializationType::Snapshot => "snapshot", MaterializationType::Seed => "seed", MaterializationType::Custom => "custom", - MaterializationType::Maintained => "maintained", + MaterializationType::Dynamic => "dynamic", } } diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/models.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/models.rs index b09c27b27d..bf6b4d06d9 100644 --- a/crates/sqlbuild-python-planner/src/planner_warehouse/models.rs +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/models.rs @@ -383,8 +383,8 @@ pub enum AdapterRead<'py> { FreshnessQuery(Bound<'py, PyAny>), /// Why a destination cannot rename columns, `None` when it can. ColumnRenameUnavailable(Bound<'py, PyAny>), - /// One maintained model's `[database, schema, name]` listed target lag and refresh warehouse. - MaintainedRefresh([Bound<'py, PyAny>; 3]), + /// One dynamic model's `[database, schema, name]` listed target lag and refresh warehouse. + DynamicRefresh([Bound<'py, PyAny>; 3]), } /// A read of what the warehouse state tables recorded. @@ -756,8 +756,8 @@ impl WarehouseReads for SyncedReads { AdapterRead::ColumnRenameUnavailable(_) => { return Ok(unknown(self, "column rename support")); } - AdapterRead::MaintainedRefresh(_) => { - return Ok(unknown(self, "maintained model listings")); + AdapterRead::DynamicRefresh(_) => { + return Ok(unknown(self, "dynamic model listings")); } }; let unavailable = diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs index 5a0a7a7829..cba13215c6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs @@ -49,7 +49,7 @@ use sqlbuild_python_compile::compile_driver::models::TargetSettingsValues; use sqlbuild_python_front::cost_context::models::CostContext; use sqlbuild_python_front::cursor::main::overrides::cursor_integer::cursor_integer; use sqlbuild_python_front::cursor::main::overrides::cursor_timestamp::cursor_timestamp; -use sqlbuild_python_front::discovery::main::project::configured_maintained::configured_maintained; +use sqlbuild_python_front::discovery::main::project::configured_dynamic::configured_dynamic; use sqlbuild_python_front::native_host::main::recorder_callbacks::recorder_callbacks; use sqlbuild_python_output::build_progress::main::batch_complete_callback::batch_complete_callback; use sqlbuild_python_output::build_progress::main::check_callback::check_callback; @@ -342,11 +342,11 @@ pub(crate) fn execute_build_plan<'py>( let target_config: TargetSettingsValues = project_target_settings(&discovered, &project, (&connection_config, &adapter))?; let (default_database, default_schema) = target_config.node_locations(&adapter)?; - let maintained = configured_maintained(&discovered, &context.request, &project_dir)?; + let dynamic = configured_dynamic(&discovered, &context.request, &project_dir)?; let runtime: Bound<'py, PyAny> = call_with( &host_entry(py, ExecutorHost::BuildRuntimeParams)?, &[ - &maintained[..], + &dynamic[..], &[ ("run_id", native_field(&project, "run_id")?), ("runtime_dir", joined(&project_dir, &["target"])?), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/constants.rs index a0c36d1e5c..510cd06cc0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/constants.rs @@ -19,7 +19,7 @@ use sqlbuild_core::command_output::constants::{ }; use sqlbuild_core::constants::{ AUDIT_RESULTS_TABLE_NAME, COLUMN_MIGRATION_TABLE_NAME, CONSUMPTION_TABLE_NAME, - FINGERPRINT_TABLE_NAME, JANITOR_EVENTS_TABLE_NAME, MAINTAINED_PAUSES_TABLE_NAME, + DYNAMIC_PAUSES_TABLE_NAME, FINGERPRINT_TABLE_NAME, JANITOR_EVENTS_TABLE_NAME, MICROBATCH_TABLE_NAME, MIGRATION_TABLE_NAME, NODE_RESULTS_TABLE_NAME, OBSERVATIONS_TABLE_NAME, OLD_NAME_VIEW_TABLE_NAME, PROJECT_CONFIG_FILENAME, PYTHON_INIT_MODULE_STEM, PYTHON_NODE_ROOT, RUNS_TABLE_NAME, SETTINGS_SECTION, SOURCE_FRESHNESS_TABLE_NAME, STAGED_PROMOTION_MODE, @@ -95,7 +95,7 @@ pub(crate) const EXPORTED_TEXTS: &[(&str, &str)] = &[ ("JANITOR_EVENTS_TABLE_NAME", JANITOR_EVENTS_TABLE_NAME), ("OBSERVATIONS_TABLE_NAME", OBSERVATIONS_TABLE_NAME), ("RUNS_TABLE_NAME", RUNS_TABLE_NAME), - ("MAINTAINED_PAUSES_TABLE_NAME", MAINTAINED_PAUSES_TABLE_NAME), + ("DYNAMIC_PAUSES_TABLE_NAME", DYNAMIC_PAUSES_TABLE_NAME), ("COLUMN_NODE_TYPE", COLUMN_NODE_TYPE), ("COLUMN_NODE_NAME", COLUMN_NODE_NAME), ("COLUMN_TARGET_DATABASE", COLUMN_TARGET_DATABASE), diff --git a/fensu.toml b/fensu.toml index 2a1becc261..73dcbfa978 100644 --- a/fensu.toml +++ b/fensu.toml @@ -1352,8 +1352,8 @@ path = "src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py" reason = "The SQL Server adapter intentionally keeps overrideable backend behavior on one class." [[dupes.allowlist]] -paths = ["scripts/native_layering/_helpers/layering.py", "src/sqlbuild/executor/maintained_pauses/_helpers/scope.py"] -reason = "A ten-line depth-first walk in two packages that must not import each other: the dependency-layering tool walks Cargo.lock packages, the build's maintained-model hold walks plan edges between models." +paths = ["scripts/native_layering/_helpers/layering.py", "src/sqlbuild/executor/dynamic_pauses/_helpers/scope.py"] +reason = "A ten-line depth-first walk in two packages that must not import each other: the dependency-layering tool walks Cargo.lock packages, the build's dynamic-model hold walks plan edges between models." [[dupes.allowlist]] paths = ["crates/sqlbuild-python-planner/src/planner_changes/_helpers/detect.rs", "crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/python_selection.rs"] diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/CONTENTS.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/CONTENTS.md index 484e052615..47720be1f4 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/CONTENTS.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/CONTENTS.md @@ -24,7 +24,7 @@ Bundled copies of every page on the SQLBuild documentation site, matching the in - [Seeds](concepts/seeds.md) (`concepts/seeds`) - Load static CSV data into your pipeline as tables. - [Incremental Models](concepts/incremental.md) (`concepts/incremental`) - Cursor-based incremental strategies, microbatch execution, and backfill policies. - [Snapshots (SCD Type 2)](concepts/snapshots.md) (`concepts/snapshots`) - Preserve row history over time using SCD Type 2 semantics with timestamp or check-based change detection. -- [Maintained Models](concepts/models/maintained.md) (`concepts/models/maintained`) - Let the warehouse keep a model up to date on a schedule, paused safely while builds run. +- [Dynamic models](concepts/models/dynamic.md) (`concepts/models/dynamic`) - Let the warehouse keep a model up to date on a schedule, paused safely while builds run. - [Functions](concepts/functions.md) (`concepts/functions`) - User-defined functions and table functions managed as part of your project. - [Macros](concepts/macros.md) (`concepts/macros`) - Reuse SQL with typed SQL macros, or generate it with Python macros when it needs logic. - [SQL Macros](concepts/macros/sql-macros.md) (`concepts/macros/sql-macros`) - Write typed, logic-free SQL fragments that the compiler checks and sees inside. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/build.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/build.md index cc013924c9..ab495b4a1d 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/build.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/build.md @@ -66,7 +66,7 @@ own option. `--version` works only directly after `sqb`, as in `sqb --version`: | `--defer-sources-to` | Read managed source data from another target | | `--plan-from` | Read warehouse state `live`, or from the target's `synced_state` checked against the warehouse first; overrides the target's `plan_from`. See [Planning from synced state](../concepts/project-configuration.md#planning-from-synced-state) | | `--fail-fast` | Stop on first failure and skip remaining nodes | -| `--ignore-maintained-grace` | Take over [maintained models](../concepts/models/maintained.md#when-a-build-dies) another build holds, without waiting for `[maintained] heartbeat_grace`. Needs `allow_ignore_grace = true`, else fails with `A719` | +| `--ignore-dynamic-grace` | Take over [dynamic models](../concepts/models/dynamic.md#when-a-build-dies) another build holds, without waiting for `[dynamic] heartbeat_grace`. Needs `allow_ignore_grace = true`, else fails with `A719` | | `--concurrency` | Number of worker connections (default: 1) | | `--test-timeout ` | Time limit for each unit test the build runs, such as `30s`, or `disabled`; see [Time limits](../concepts/testing-reference.md#time-limits) | | `--verbose`, `-v` | Show lifecycle SQL inline after each model | diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/dag.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/dag.md index c87285ddeb..3b1f83954c 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/dag.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/dag.md @@ -105,7 +105,7 @@ Each node represents a source, seed, model, or function: | `description` | Model or source description, if declared | | `tags` | Model tags | | `columns` | Column metadata (name, type, nullable, description) | -| `materialization_type` | For models: `view`, `table`, `incremental`, `snapshot`, `maintained`, or custom name | +| `materialization_type` | For models: `view`, `table`, `incremental`, `snapshot`, `dynamic`, or custom name | | `language` | For functions: `sql` or `python` | | `return_kind` | For functions: `scalar` or `table` | | `arguments` | For functions: argument name and type pairs | diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/snowflake.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/snowflake.md index fa4c878d51..85c44a8187 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/snowflake.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/snowflake.md @@ -14,7 +14,7 @@ Online: https://sqlbuild.com/docs/concepts/adapters/snowflake/ - Case-sensitive column names - Metadata reads - State tables -- Maintained models +- Dynamic models - Shared connections across targets - Cost estimates - Functions in unit tests @@ -166,10 +166,10 @@ SQLBuild records its state in `_sqlbuild_*` tables. On Snowflake it creates them with 90 days of time travel, or 1 day on editions that allow only that. It never alters an existing state table. Models are not affected: their table type follows the model's `table_type` setting. -## Maintained models +## Dynamic models -A [maintained model](../models/maintained.md) is a Snowflake dynamic -table. `materialized materialized_view` fails with `A014`; use `materialized maintained`. +A [dynamic model](../models/dynamic.md) is a Snowflake dynamic +table. `materialized materialized_view` fails with `A014`; use `materialized dynamic`. The build role needs these grants: @@ -181,11 +181,12 @@ The build role needs these grants: Dynamic tables refresh from changes, which needs change tracking on every table they read: -- A table model that a maintained model reads, directly or through views, is built with change tracking on. +- A table or snapshot model that a dynamic model reads, directly or through views, is built with + change tracking on. - For a source table, Snowflake turns change tracking on when the build role owns the table. Otherwise its owner runs `ALTER TABLE SET CHANGE_TRACKING = TRUE`. -Builds record the models they pause in the `_sqlbuild_maintained_pauses` state table, so another +Builds record the models they pause in the `_sqlbuild_dynamic_pauses` state table, so another build can tell whether they are still held. ## Shared connections across targets diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/configuration.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/configuration.md index 00a9c2512f..754c10adf1 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/configuration.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/configuration.md @@ -36,7 +36,7 @@ which accepts any keys. | Field | Description | |-------|-------------| -| `materialized` | `view`, `table`, `incremental`, `snapshot`, `maintained`, or a custom materialization name; see [Materializations](materializations.md) | +| `materialized` | `view`, `table`, `incremental`, `snapshot`, `dynamic`, or a custom materialization name; see [Materializations](materializations.md) | | `tags` | Tags used by selectors | | `critical` | `true` marks the model as critical, so [`sqb review`](../../cli/review.md) builds and compares it whenever something upstream of it changes (default `false`) | | `description` | Required human-readable model description; see [Descriptions](#descriptions) | diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/maintained.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/dynamic.md similarity index 62% rename from src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/maintained.md rename to src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/dynamic.md index 777c4de3c5..53cc1d2d88 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/maintained.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/dynamic.md @@ -1,19 +1,19 @@ -# Maintained Models +# Dynamic models > Let the warehouse keep a model up to date on a schedule, paused safely while builds run. -Online: https://sqlbuild.com/docs/concepts/models/maintained/ +Online: https://sqlbuild.com/docs/concepts/models/dynamic/ -A maintained model is one the warehouse keeps up to date on a schedule you set, between builds. A +A dynamic model is one the warehouse keeps up to date on a schedule you set, between builds. A build creates it, changes it when its definition changes, and otherwise leaves the refreshing to the warehouse; on Snowflake it is a dynamic table. ```sql MODEL ( description "One row per customer per order date", - materialized maintained, + materialized dynamic, refresh (lag 1h), ); @@ -22,14 +22,14 @@ FROM __ref("stg_orders") GROUP BY customer_id, CAST(ordered_at AS DATE) ``` -Only Snowflake supports maintained models. See [Snowflake](../adapters/snowflake.md#maintained-models) +Only Snowflake supports dynamic models. See [Snowflake](../adapters/snowflake.md#dynamic-models) for the grants and change tracking they need. ## Settings | Field | Default | Description | |-------|---------|-------------| -| `refresh (lag )` | none | How far behind its inputs the model may fall: a fixed duration such as `5m`, `1h` or `1d`, at least `1m` on Snowflake and no shorter than the lag of a maintained model it reads. Without it, the model refreshes when a maintained model that reads it refreshes | +| `refresh (lag )` | none | How far behind its inputs the model may fall: a fixed duration such as `5m`, `1h` or `1d`, at least `1m` on Snowflake and no shorter than the lag of a dynamic model it reads. Without it, the model refreshes when a dynamic model that reads it refreshes | | `refresh_mode` | `incremental` | `incremental`, `full`, `auto` or `adaptive`. Snowflake rejects `incremental` for a query it can't refresh incrementally; use `full` for that query | | `refresh_warehouse` | the connection's warehouse | Warehouse the scheduled refreshes run on | | `refresh_on_build` | `false` | Refresh the model in every build that selects it, not only on its schedule | @@ -41,7 +41,7 @@ for the grants and change tracking they need. | First build, or the dynamic table is missing | `create` | Creates it and fills it | | Query or `refresh_mode` changed | `recreate, full recompute` | Replaces it; every row is computed again | | `refresh` lag or `refresh_warehouse` differs from the warehouse | `change refresh settings in place` | Alters the schedule; rows stay | -| Nothing | `maintained` | Leaves it alone | +| Nothing | `dynamic` | Leaves it alone | A model the plan keeps can carry one more line: @@ -50,7 +50,7 @@ A model the plan keeps can carry one more line: ## During a build -A build pauses the maintained models it changes, and those that read a model it rebuilds, directly +A build pauses the dynamic models it changes, and those that read a model it rebuilds, directly or through views, so they never refresh from half-built inputs. It resumes them when it ends, whether it succeeds or fails. @@ -59,24 +59,23 @@ whether it succeeds or fails. - A suspend made by someone else during the build is undone when the build resumes the model. - A kept model refreshes during the build when a model built later in the same build reads it. - `refresh_on_build` refreshes the model in every build that selects it. -- A target with `pause_maintained = true` leaves its maintained models suspended between builds; see [Targets](../project-configuration.md#targets). +- A target with `pause_dynamic = true` leaves its dynamic models suspended between builds; see [Targets](../project-configuration.md#targets). ## When a build dies -- **One build at a time.** A build that needs a maintained model another build holds fails with - `A718`, naming that run. Wait for it to finish. +- **One build at a time.** A build that needs a dynamic model another build holds fails with + `A718`, naming that run. Wait for it to finish. Two builds that start at the same moment can + both fail this way; run one of them again. - **A stopped build leaves its models suspended.** A build killed before it resumes them, for example by `SIGKILL`, can't resume them itself. -- **The next build resumes them.** The next build that pauses maintained models on the target - resumes them once the stopped build has been silent for `[maintained] heartbeat_grace`, 30 - minutes by default; see [Maintained](../project-configuration.md#maintained). -- **Taking over at once.** Run `sqb build --ignore-maintained-grace` to take the models over - without waiting. It needs `[maintained] allow_ignore_grace = true`; without it the flag fails +- **The next build resumes them.** The next build that pauses dynamic models on the target + resumes them once the stopped build has been silent for `[dynamic] heartbeat_grace`, 30 + minutes by default; see [Dynamic](../project-configuration.md#dynamic). +- **Taking over at once.** Run `sqb build --ignore-dynamic-grace` to take the models over + without waiting. It needs `[dynamic] allow_ignore_grace = true`; without it the flag fails with `A719`. You can also resume the models in Snowflake by hand. ## Limitations -- Writing out the default `refresh_mode incremental` recreates the model once. -- A snapshot rebuilt in full loses change tracking; run `ALTER TABLE SET CHANGE_TRACKING = TRUE` before a maintained model reads it again. -- Unit tests and scenarios treat a maintained model as a table. -- On a target with `schema = "preserve"`, builds see each other's pauses only when their maintained models share a schema. +- Unit tests and scenarios treat a dynamic model as a table. +- On a target with `schema = "preserve"`, builds see each other's pauses only when their dynamic models share a schema. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/materializations.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/materializations.md index 5ca09cd193..2aa4141724 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/materializations.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/materializations.md @@ -12,7 +12,7 @@ Online: https://sqlbuild.com/docs/concepts/models/materializations/ - Table - Incremental - Snapshot -- Maintained +- Dynamic - Custom - When a build fails @@ -84,19 +84,19 @@ FROM __source("customers") See [Snapshots](../snapshots.md) for timestamp and check strategies, historical inputs, and full-refresh policies. -## Maintained +## Dynamic The warehouse keeps the model up to date on a schedule you set, and builds pause it while they change its inputs: ```sql -MODEL (description "One row per customer per order date", materialized maintained, refresh (lag 1h)); +MODEL (description "One row per customer per order date", materialized dynamic, refresh (lag 1h)); SELECT customer_id, CAST(ordered_at AS DATE) AS order_date, SUM(amount) AS revenue FROM __ref("stg_orders") GROUP BY customer_id, CAST(ordered_at AS DATE) ``` -See [Maintained models](maintained.md) for its settings, what the plan shows and what happens when a build dies. +See [Dynamic models](dynamic.md) for its settings, what the plan shows and what happens when a build dies. ## Custom @@ -135,5 +135,5 @@ The custom function owns staging, runtime type or contract enforcement, audit ti | Table (`immediate`) | The new table. Audits run after it is replaced. | | Incremental | The old table if a delta audit fails. Rows already written stay if a final audit or a later microbatch fails. | | Microbatch full rebuild | A partly rebuilt table; the old one was dropped first. | -| Maintained | The created or altered dynamic table, resumed. Audits run after it changes. | +| Dynamic | The created or altered dynamic table, resumed. Audits run after it changes. | | Custom | Whatever the custom materialization does. | diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md index 9a40443f25..539bf642a3 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md @@ -20,7 +20,7 @@ Online: https://sqlbuild.com/docs/concepts/project-configuration/ - Project variables - Janitor - Migrations -- Maintained +- Dynamic - Scenario - Git - Review @@ -190,7 +190,7 @@ A target is a named build context - the database and schema you build into, plus | `concurrency` | Worker limit for commands run against this target, replacing `[settings] concurrency` (see below) | | `ignore_audit_failures` | Whether builds on this target let audit failures through, replacing `[settings] ignore_audit_failures`; see [Ignoring audit failures](audits.md#ignoring-audit-failures) | | `plan_from` | Where `sqb plan` and `sqb build` read warehouse state: `live` (default) or `synced_state`; see [Planning from synced state](#planning-from-synced-state) | -| `pause_maintained` | `true` keeps this target's [maintained models](models/maintained.md) suspended between builds; each build refreshes the ones it selects. Default `false` | +| `pause_dynamic` | `true` keeps this target's [dynamic models](models/dynamic.md) suspended between builds; each build refreshes the ones it selects. Default `false` | ```toml [connections.warehouse] @@ -697,20 +697,20 @@ old_name_views = "30d" |-------|---------|-------------| | `old_name_views` | `"30d"` | How long a renamed model's old name keeps working through a [compatibility view](models/migrations.md#old-names) before janitor drops it. `false` keeps the old relation in place instead. A model's `old_name_view` header overrides it. | -## Maintained +## Dynamic -How builds pause [maintained models](models/maintained.md) while they run: +How builds pause [dynamic models](models/dynamic.md) while they run: ```toml -[maintained] +[dynamic] heartbeat_grace = "30m" allow_ignore_grace = false ``` | Field | Default | Description | |-------|---------|-------------| -| `heartbeat_grace` | `"30m"` | How long a build that paused maintained models may go silent before another build resumes them or takes them over. A fixed duration longer than `"1m"`, such as `"30m"` or `"2h"`; a build writes a heartbeat every minute | -| `allow_ignore_grace` | `false` | Allow `sqb build --ignore-maintained-grace`, which takes over models another build holds without waiting for the grace. Without it the flag fails with `A719` | +| `heartbeat_grace` | `"30m"` | How long a build that paused dynamic models may go silent before another build resumes them or takes them over. A fixed duration longer than `"1m"`, such as `"30m"` or `"2h"`; a build writes a heartbeat every minute | +| `allow_ignore_grace` | `false` | Allow `sqb build --ignore-dynamic-grace`, which takes over models another build holds without waiting for the grace. Without it the flag fails with `A719` | ## Scenario diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index eced41585a..47b8971951 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -192,7 +192,7 @@ MICROBATCH_TABLE_NAME: str NODE_RESULTS_TABLE_NAME: str OBSERVATIONS_TABLE_NAME: str RUNS_TABLE_NAME: str -MAINTAINED_PAUSES_TABLE_NAME: str +DYNAMIC_PAUSES_TABLE_NAME: str AUDIT_RESULTS_TABLE_NAME: str JANITOR_EVENTS_TABLE_NAME: str COLUMN_NODE_TYPE: str @@ -6593,7 +6593,7 @@ class BuildCommandRequest(NativeRecord): allow_retention_decrease: bool allow_missing_migration_origin: bool allow_snapshot_schema_change: bool - ignore_maintained_grace: bool + ignore_dynamic_grace: bool concurrency: int | None select: tuple[str, ...] exclude: tuple[str, ...] @@ -6638,7 +6638,7 @@ class BuildCommandRequest(NativeRecord): allow_retention_decrease: bool = False, allow_missing_migration_origin: bool = False, allow_snapshot_schema_change: bool = False, - ignore_maintained_grace: bool = False, + ignore_dynamic_grace: bool = False, concurrency: int | None = None, select: tuple[str, ...] = (), exclude: tuple[str, ...] = (), @@ -12018,7 +12018,7 @@ class TargetConfig(NativeRecord): catalog_connection: str | None ignore_audit_failures: bool | None plan_from: str | None - pause_maintained: bool | None + pause_dynamic: bool | None def __init__( self, connection: dict[str, object] = ..., @@ -12045,7 +12045,7 @@ class TargetConfig(NativeRecord): catalog_connection: str | None = ..., ignore_audit_failures: bool | None = ..., plan_from: str | None = ..., - pause_maintained: bool | None = ..., + pause_dynamic: bool | None = ..., ) -> None: ... class LocalTargetConfig(NativeRecord): @@ -12075,7 +12075,7 @@ class LocalTargetConfig(NativeRecord): catalog_connection: str | None ignore_audit_failures: bool | None plan_from: str | None - pause_maintained: bool | None + pause_dynamic: bool | None def __init__( self, connection: dict[str, object] = ..., @@ -12103,7 +12103,7 @@ class LocalTargetConfig(NativeRecord): catalog_connection: str | None = ..., ignore_audit_failures: bool | None = ..., plan_from: str | None = ..., - pause_maintained: bool | None = ..., + pause_dynamic: bool | None = ..., ) -> None: ... class SettingsConfig(NativeRecord): @@ -13530,9 +13530,9 @@ class PlanAction(StrEnum): LOAD_SEED = "load_seed" SKIP = "skip" CUSTOM = "custom" - CREATE_MAINTAINED = "create_maintained" - ALTER_MAINTAINED = "alter_maintained" - KEEP_MAINTAINED = "keep_maintained" + CREATE_DYNAMIC = "create_dynamic" + ALTER_DYNAMIC = "alter_dynamic" + KEEP_DYNAMIC = "keep_dynamic" class PlanReason(StrEnum): FIRST_RUN = "first_run" @@ -13600,7 +13600,7 @@ class MaterializationType(StrEnum): SNAPSHOT = "snapshot" SEED = "seed" CUSTOM = "custom" - MAINTAINED = "maintained" + DYNAMIC = "dynamic" @classmethod def is_table_backed(cls, *, materialized: object | None) -> bool: ... @@ -14127,8 +14127,8 @@ class ModelPlanEntry(NativeRecord): refresh_mode: str | None refresh_warehouse: str | None refresh_on_build: bool - maintained_readers: tuple[str, ...] - maintained_paused: bool + dynamic_readers: tuple[str, ...] + dynamic_paused: bool def __init__( self, key: CompiledObjectKey, @@ -14223,8 +14223,8 @@ class ModelPlanEntry(NativeRecord): refresh_mode: str | None = ..., refresh_warehouse: str | None = ..., refresh_on_build: bool = ..., - maintained_readers: tuple[str, ...] = ..., - maintained_paused: bool = ..., + dynamic_readers: tuple[str, ...] = ..., + dynamic_paused: bool = ..., ) -> None: ... class CloneSourcePlanEntry(NativeRecord): diff --git a/src/sqlbuild/adapter/contract/classes/strict_adapter.py b/src/sqlbuild/adapter/contract/classes/strict_adapter.py index 06cc06a89e..89c03cf168 100644 --- a/src/sqlbuild/adapter/contract/classes/strict_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/strict_adapter.py @@ -31,10 +31,10 @@ TemporaryFunction, ) from sqlbuild.adapter.contract.types import ( + DynamicListing, + DynamicOperation, FrameworkType, LoaderLogicalType, - MaintainedListing, - MaintainedOperation, PromotionStrategy, StatementSizeLimit, TablePromotionMode, @@ -329,41 +329,47 @@ def session_warehouse(self, *, connection: Any) -> str | None: del connection return None - def render_create_maintained( + def render_create_dynamic( self, *, destination: str, sql: str, target_lag: str, refresh_mode: str, warehouse: str ) -> tuple[str, ...]: - """Optional: SQL that creates or replaces a maintained model; unsupported by default.""" + """Optional: SQL that creates or replaces a dynamic model; unsupported by default.""" del sql, target_lag, refresh_mode, warehouse - raise self._maintained_unsupported(destination) + raise self._dynamic_unsupported(destination) - def render_alter_maintained( + def render_alter_dynamic( self, *, destination: str, target_lag: str, warehouse: str | None ) -> tuple[str, ...]: - """Optional: SQL that changes a maintained model's refresh settings in place.""" + """Optional: SQL that changes a dynamic model's refresh settings in place.""" del target_lag, warehouse - raise self._maintained_unsupported(destination) + raise self._dynamic_unsupported(destination) - def render_maintained_operation( - self, *, destination: str, operation: MaintainedOperation + def render_dynamic_operation( + self, *, destination: str, operation: DynamicOperation ) -> tuple[str, ...]: - """Optional: SQL that suspends, resumes or refreshes a maintained model.""" + """Optional: SQL that suspends, resumes or refreshes a dynamic model.""" del operation - raise self._maintained_unsupported(destination) + raise self._dynamic_unsupported(destination) - def _maintained_unsupported(self, destination: str) -> AdapterUserError: + def render_track_changes(self, *, destination: str) -> tuple[str, ...]: + """Optional: SQL that turns on change tracking for a table a dynamic model reads.""" + + del destination + return () + + def _dynamic_unsupported(self, destination: str) -> AdapterUserError: return AdapterUserError( - f"the {self.adapter_name} adapter cannot build maintained model '{destination}'", + f"the {self.adapter_name} adapter cannot build dynamic model '{destination}'", code="A716", - help="materialized maintained needs an adapter whose warehouse refreshes it", + help="materialized dynamic needs an adapter whose warehouse refreshes it", ) - def maintained_listing( + def dynamic_listing( self, *, connection: Any, database: str | None, schema: str | None, name: str - ) -> MaintainedListing | None: - """Optional: how the warehouse lists a maintained model, or None when it lists none.""" + ) -> DynamicListing | None: + """Optional: how the warehouse lists a dynamic model, or None when it lists none.""" del connection, database, schema, name return None diff --git a/src/sqlbuild/adapter/contract/types.py b/src/sqlbuild/adapter/contract/types.py index cf88c3cb4e..80d52e02f0 100644 --- a/src/sqlbuild/adapter/contract/types.py +++ b/src/sqlbuild/adapter/contract/types.py @@ -82,8 +82,8 @@ class RetentionChangePhase(StrEnum): FINALIZE = "finalize" -class MaintainedOperation(StrEnum): - """What a build does to a maintained model's scheduled refreshes.""" +class DynamicOperation(StrEnum): + """What a build does to a dynamic model's scheduled refreshes.""" SUSPEND = "suspend" RESUME = "resume" @@ -174,8 +174,8 @@ class HistoricalSnapshotInsertStyle(StrEnum): TSQL = "tsql" -class MaintainedListing(NamedTuple): - """A maintained model as the warehouse lists it: lag, refresh warehouse and schedule state.""" +class DynamicListing(NamedTuple): + """A dynamic model as the warehouse lists it: lag, refresh warehouse and schedule state.""" target_lag: str warehouse: str diff --git a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py index 19dc7d7195..6d62b24af0 100644 --- a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py +++ b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py @@ -95,12 +95,12 @@ from sqlbuild.adapter.contract.types import ( BuiltinAdapter, CursorKind, + DynamicListing, + DynamicOperation, FrameworkType, HistoricalSnapshotCloseStyle, HistoricalSnapshotInsertStyle, LoaderLogicalType, - MaintainedListing, - MaintainedOperation, MigrationTransfer, PromotionStrategy, RelationReadStatus, @@ -3674,14 +3674,14 @@ def column_rename_unavailable_reason(self, *, connection: Any, destination: str) return None def session_warehouse(self, *, connection: Any) -> str | None: - """The session's warehouse, which refreshes a maintained model that names none.""" + """The session's warehouse, which refreshes a dynamic model that names none.""" rows: tuple[tuple[object, ...], ...] = self.query( connection=connection, sql="SELECT CURRENT_WAREHOUSE()", limit=None ).rows return None if not rows or rows[0][0] is None else str(rows[0][0]) - def render_create_maintained( + def render_create_dynamic( self, *, destination: str, sql: str, target_lag: str, refresh_mode: str, warehouse: str ) -> tuple[str, ...]: """A dynamic table with an explicit mode and ON_CREATE, so it is readable at once.""" @@ -3692,7 +3692,7 @@ def render_create_maintained( f"REFRESH_MODE = {refresh_mode.upper()} INITIALIZE = ON_CREATE AS {sql}", ) - def render_alter_maintained( + def render_alter_dynamic( self, *, destination: str, target_lag: str, warehouse: str | None ) -> tuple[str, ...]: """The lag, and a declared warehouse, set in place.""" @@ -3705,11 +3705,14 @@ def render_alter_maintained( ) return tuple(statements) - def render_maintained_operation( - self, *, destination: str, operation: MaintainedOperation + def render_dynamic_operation( + self, *, destination: str, operation: DynamicOperation ) -> tuple[str, ...]: return (f"ALTER DYNAMIC TABLE {destination} {operation.value.upper()}",) + def render_track_changes(self, *, destination: str) -> tuple[str, ...]: + return (f"ALTER TABLE {destination} SET CHANGE_TRACKING = TRUE",) + def _warehouse_name(self, warehouse: str) -> str: """A plain warehouse name as written, resolved without case; any other quoted exactly.""" @@ -3717,9 +3720,9 @@ def _warehouse_name(self, warehouse: str) -> str: return warehouse return '"' + warehouse.replace('"', '""') + '"' - def maintained_listing( + def dynamic_listing( self, *, connection: Any, database: str | None, schema: str | None, name: str - ) -> MaintainedListing | None: + ) -> DynamicListing | None: """The dynamic table's lag, warehouse and scheduling state from SHOW DYNAMIC TABLES.""" row: dict[str, object] | None = self._listed_dynamic_table( @@ -3727,7 +3730,7 @@ def maintained_listing( ) if row is None: return None - return MaintainedListing( + return DynamicListing( target_lag=str(row.get("target_lag")), warehouse=str(row.get("warehouse")), scheduling_state=str(row.get("scheduling_state")), diff --git a/src/sqlbuild/executor/build/_helpers/scheduler.py b/src/sqlbuild/executor/build/_helpers/scheduler.py index 527dd72fff..3e03ac59d4 100644 --- a/src/sqlbuild/executor/build/_helpers/scheduler.py +++ b/src/sqlbuild/executor/build/_helpers/scheduler.py @@ -24,8 +24,8 @@ from sqlbuild.executor.build.constants import ( BUILD_CUSTOM_MATERIALIZATION_MISSING_CODE, BUILD_SOURCE_FRESHNESS_BLOCKED_CODE, + DYNAMIC_ACTIONS, INCREMENTAL_ACTIONS, - MAINTAINED_ACTIONS, ) from sqlbuild.executor.build.models import ( FunctionExecutionResult, @@ -36,8 +36,8 @@ from sqlbuild.executor.load.models import LoadExecutionResult from sqlbuild.executor.run.main._execute import ( execute_custom_entry, + execute_dynamic_entry, execute_incremental_entry, - execute_maintained_entry, execute_microbatch_entry, execute_snapshot_entry, execute_table_entry, @@ -165,8 +165,8 @@ def _dispatch_model_entry( context=context, declared_columns=entry.declared_columns, ) - if entry.action in MAINTAINED_ACTIONS: - return execute_maintained_entry(context=context) + if entry.action in DYNAMIC_ACTIONS: + return execute_dynamic_entry(context=context) if entry.action == PlanAction.CREATE_VIEW: return execute_view_entry(context=context) if entry.action == PlanAction.SNAPSHOT: diff --git a/src/sqlbuild/executor/build/classes/build_scheduler.py b/src/sqlbuild/executor/build/classes/build_scheduler.py index b8c628ecfd..395f78de8b 100644 --- a/src/sqlbuild/executor/build/classes/build_scheduler.py +++ b/src/sqlbuild/executor/build/classes/build_scheduler.py @@ -1256,7 +1256,7 @@ def _execute_model_node( watermark_resolver=self._watermark_resolver, bound_views=bound_views, fingerprint_writes=self._fingerprint_writes, - maintained_hold=self._runtime.maintained_hold, + dynamic_hold=self._runtime.dynamic_hold, ), promotion_mode=self._promotion_mode, snapshots=self._snapshots, diff --git a/src/sqlbuild/executor/build/constants.py b/src/sqlbuild/executor/build/constants.py index f4d7636187..ef38d54f7a 100644 --- a/src/sqlbuild/executor/build/constants.py +++ b/src/sqlbuild/executor/build/constants.py @@ -11,11 +11,11 @@ PlanAction.INCREMENTAL_MERGE, } ) -MAINTAINED_ACTIONS: frozenset[PlanAction] = frozenset( +DYNAMIC_ACTIONS: frozenset[PlanAction] = frozenset( { - PlanAction.CREATE_MAINTAINED, - PlanAction.ALTER_MAINTAINED, - PlanAction.KEEP_MAINTAINED, + PlanAction.CREATE_DYNAMIC, + PlanAction.ALTER_DYNAMIC, + PlanAction.KEEP_DYNAMIC, } ) diff --git a/src/sqlbuild/executor/build/main/_execute.py b/src/sqlbuild/executor/build/main/_execute.py index 1d48b1c2f0..b0191f6d04 100644 --- a/src/sqlbuild/executor/build/main/_execute.py +++ b/src/sqlbuild/executor/build/main/_execute.py @@ -35,11 +35,11 @@ SeedExecutionResult, ) from sqlbuild.executor.build.types import BuildStatus -from sqlbuild.executor.load.models import LoadExecutionResult -from sqlbuild.executor.maintained_pauses.classes.maintained_hold import MaintainedHold -from sqlbuild.executor.maintained_pauses.main.hold_maintained_models import ( - hold_maintained_models, +from sqlbuild.executor.dynamic_pauses.classes.dynamic_hold import DynamicHold +from sqlbuild.executor.dynamic_pauses.main.hold_dynamic_models import ( + hold_dynamic_models, ) +from sqlbuild.executor.load.models import LoadExecutionResult from sqlbuild.executor.migrations.main._apply import apply_model_migrations from sqlbuild.executor.migrations.main._apply_columns import apply_column_migrations from sqlbuild.executor.run.models import ModelExecutionResult @@ -59,9 +59,9 @@ def execute_build_plan( initial_state: BuildInitialState | None = None, schema_prepared: bool = False, ) -> BuildExecutionResult: - """Execute a build plan with the DAG scheduler, holding the maintained models it touches.""" + """Execute a build plan with the DAG scheduler, holding the dynamic models it touches.""" - hold: MaintainedHold | None = hold_maintained_models( + hold: DynamicHold | None = hold_dynamic_models( plan=plan, adapter=adapter, runtime=runtime, connection_config=connection_config ) held: AbstractContextManager[object] = ( @@ -79,7 +79,7 @@ def execute_build_plan( connection_config=connection_config, connections=connections, scheduler_connection=scheduler_connection, - runtime=replace(runtime, maintained_hold=hold), + runtime=replace(runtime, dynamic_hold=hold), callbacks=callbacks, customizations=customizations, initial_state=initial_state, diff --git a/src/sqlbuild/executor/build/models.py b/src/sqlbuild/executor/build/models.py index d0c19dfe0a..d245ce43a3 100644 --- a/src/sqlbuild/executor/build/models.py +++ b/src/sqlbuild/executor/build/models.py @@ -27,8 +27,8 @@ from sqlbuild.executor.auditing.models import AuditExecutionResult from sqlbuild.executor.build.types import BeforeNodeWriteCallback, BuildStatus from sqlbuild.executor.custom.models import MaterializationResult +from sqlbuild.executor.dynamic_pauses.classes.dynamic_hold import DynamicHold from sqlbuild.executor.load.models import LoadExecutionResult -from sqlbuild.executor.maintained_pauses.classes.maintained_hold import MaintainedHold from sqlbuild.executor.python_nodes.types import PythonIdentityRecorder from sqlbuild.executor.run.models import MicrobatchBatchProgress, ModelExecutionResult from sqlbuild.executor.scheduling.types import ExecutionStatus @@ -102,11 +102,11 @@ class BuildRuntimeParams: microbatch_lease_check: Callable[[], None] | None = None project_relations: Mapping[SqlResourceRef, str] | None = None test_timeouts: RunItemTimeouts | None = None - pause_maintained: bool = False - maintained_heartbeat_grace_seconds: int = 1800 - allow_ignore_maintained_grace: bool = False - ignore_maintained_grace: bool = False - maintained_hold: MaintainedHold | None = None + pause_dynamic: bool = False + dynamic_heartbeat_grace_seconds: int = 1800 + allow_ignore_dynamic_grace: bool = False + ignore_dynamic_grace: bool = False + dynamic_hold: DynamicHold | None = None @dataclass(frozen=True) diff --git a/src/sqlbuild/executor/dynamic_pauses/__init__.py b/src/sqlbuild/executor/dynamic_pauses/__init__.py new file mode 100644 index 0000000000..22d49862b2 --- /dev/null +++ b/src/sqlbuild/executor/dynamic_pauses/__init__.py @@ -0,0 +1 @@ +"""Dynamic models a build holds paused, and the append-only lease that records it.""" diff --git a/src/sqlbuild/executor/dynamic_pauses/_helpers/__init__.py b/src/sqlbuild/executor/dynamic_pauses/_helpers/__init__.py new file mode 100644 index 0000000000..b14a823fbc --- /dev/null +++ b/src/sqlbuild/executor/dynamic_pauses/_helpers/__init__.py @@ -0,0 +1 @@ +"""Dynamic pause helpers.""" diff --git a/src/sqlbuild/executor/maintained_pauses/_helpers/scope.py b/src/sqlbuild/executor/dynamic_pauses/_helpers/scope.py similarity index 76% rename from src/sqlbuild/executor/maintained_pauses/_helpers/scope.py rename to src/sqlbuild/executor/dynamic_pauses/_helpers/scope.py index eff45d123d..6cafffdb03 100644 --- a/src/sqlbuild/executor/maintained_pauses/_helpers/scope.py +++ b/src/sqlbuild/executor/dynamic_pauses/_helpers/scope.py @@ -1,4 +1,4 @@ -"""Which maintained models a build holds, upstream first, and which ones it refreshes by hand.""" +"""Which dynamic models a build holds, upstream first, and which ones it refreshes by hand.""" from __future__ import annotations @@ -12,33 +12,33 @@ ) _UNWRITTEN: frozenset[PlanAction] = frozenset( - {PlanAction.SKIP, PlanAction.KEEP_MAINTAINED, PlanAction.ALTER_MAINTAINED} + {PlanAction.SKIP, PlanAction.KEEP_DYNAMIC, PlanAction.ALTER_DYNAMIC} ) -def held_names(*, plan: PlanOutput, pause_maintained: bool) -> tuple[str, ...]: - """Maintained models built, or reading what is written (every planned one when paused).""" +def held_names(*, plan: PlanOutput, pause_dynamic: bool) -> tuple[str, ...]: + """Dynamic models built, or reading what is written (every planned one when paused).""" names: set[str] = set() entry: ModelPlanEntry for entry in plan.model_entries: - maintained: bool = entry.materialization_type == MaterializationType.MAINTAINED - if maintained and (pause_maintained or entry.action not in _UNWRITTEN): + dynamic: bool = entry.materialization_type == MaterializationType.DYNAMIC + if dynamic and (pause_dynamic or entry.action not in _UNWRITTEN): names.add(entry.name) if entry.action not in _UNWRITTEN: - names.update(entry.maintained_readers) + names.update(entry.dynamic_readers) return _upstream_first(names=frozenset(names), plan=plan) def refreshed_for_readers(*, plan: PlanOutput) -> frozenset[str]: - """Maintained models that a model built later in this build reads, directly or not.""" + """Dynamic models that a model built later in this build reads, directly or not.""" entries: dict[str, ModelPlanEntry] = {entry.name: entry for entry in plan.model_entries} readers: dict[str, tuple[str, ...]] = _model_edges(edges=plan.downstream_deps) found: set[str] = set() name: str for name, entry in entries.items(): - if entry.materialization_type != MaterializationType.MAINTAINED: + if entry.materialization_type != MaterializationType.DYNAMIC: continue reached: set[str] = _reachable(start=name, edges=readers) if any(_built_table(entry=entries.get(reader)) for reader in reached): @@ -49,7 +49,7 @@ def refreshed_for_readers(*, plan: PlanOutput) -> frozenset[str]: def _built_table(*, entry: ModelPlanEntry | None) -> bool: return ( entry is not None - and entry.materialization_type != MaterializationType.MAINTAINED + and entry.materialization_type != MaterializationType.DYNAMIC and entry.action != PlanAction.SKIP ) diff --git a/src/sqlbuild/executor/maintained_pauses/_helpers/sql.py b/src/sqlbuild/executor/dynamic_pauses/_helpers/sql.py similarity index 78% rename from src/sqlbuild/executor/maintained_pauses/_helpers/sql.py rename to src/sqlbuild/executor/dynamic_pauses/_helpers/sql.py index 1a83fb5548..1870e75994 100644 --- a/src/sqlbuild/executor/maintained_pauses/_helpers/sql.py +++ b/src/sqlbuild/executor/dynamic_pauses/_helpers/sql.py @@ -1,4 +1,4 @@ -"""Portable SQL for the append-only maintained pause table, timed by the warehouse clock.""" +"""Portable SQL for the append-only dynamic pause table, timed by the warehouse clock.""" from __future__ import annotations @@ -8,14 +8,14 @@ from typing import Any from sqlbuild._native import ( - MAINTAINED_PAUSES_TABLE_NAME, + DYNAMIC_PAUSES_TABLE_NAME, CompiledRelationLocation, render_state_sql_literal, render_state_table_create_sql, ) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.errors.contracts.exceptions import ExecutorInputError -from sqlbuild.executor.maintained_pauses.constants import ( +from sqlbuild.executor.dynamic_pauses.constants import ( COLUMN_EVENT_ID, COLUMN_EVENT_TYPE, COLUMN_RECORDED_AT, @@ -25,21 +25,21 @@ COLUMN_RELATION_SCHEMA, COLUMN_RUN_ID, COLUMN_SCHEMA_VERSION, - MAINTAINED_PAUSE_COLUMN_TYPES, - MAINTAINED_PAUSE_COLUMNS, - MAINTAINED_PAUSE_REQUIRED_COLUMNS, - MAINTAINED_PAUSE_SCHEMA_VERSION, + DYNAMIC_PAUSE_COLUMN_TYPES, + DYNAMIC_PAUSE_COLUMNS, + DYNAMIC_PAUSE_REQUIRED_COLUMNS, + DYNAMIC_PAUSE_SCHEMA_VERSION, ) -from sqlbuild.executor.maintained_pauses.models import MaintainedPauseEvent, OpenHold -from sqlbuild.executor.maintained_pauses.types import MaintainedPauseEventType, StateLocation +from sqlbuild.executor.dynamic_pauses.models import DynamicPauseEvent, OpenHold +from sqlbuild.executor.dynamic_pauses.types import DynamicPauseEventType, StateLocation def pause_table_name(*, adapter: BaseAdapter, location: StateLocation) -> str: qualified: str | None = adapter.render_qualified_name( - database=location[0], schema=location[1], name=MAINTAINED_PAUSES_TABLE_NAME + database=location[0], schema=location[1], name=DYNAMIC_PAUSES_TABLE_NAME ) if qualified is None: - raise ExecutorInputError("the maintained pause table needs a schema") + raise ExecutorInputError("the dynamic pause table needs a schema") return qualified @@ -50,16 +50,16 @@ def setup_statements(*, adapter: BaseAdapter, location: StateLocation) -> tuple[ *adapter.render_create_schema(database=location[0], schema=location[1]), render_state_table_create_sql( qualified_name=pause_table_name(adapter=adapter, location=location), - columns=MAINTAINED_PAUSE_COLUMNS, - column_types=MAINTAINED_PAUSE_COLUMN_TYPES, - required_columns=MAINTAINED_PAUSE_REQUIRED_COLUMNS, + columns=DYNAMIC_PAUSE_COLUMNS, + column_types=DYNAMIC_PAUSE_COLUMN_TYPES, + required_columns=DYNAMIC_PAUSE_REQUIRED_COLUMNS, render_framework_type=adapter.render_framework_type, ), ) def insert_sql( - *, adapter: BaseAdapter, location: StateLocation, events: tuple[MaintainedPauseEvent, ...] + *, adapter: BaseAdapter, location: StateLocation, events: tuple[DynamicPauseEvent, ...] ) -> str: """Append every event in one statement, so a hand-over closes and opens holds together.""" @@ -69,7 +69,7 @@ def insert_sql( ) return ( f"INSERT INTO {pause_table_name(adapter=adapter, location=location)} " - f"({', '.join(MAINTAINED_PAUSE_COLUMNS)}) {rows}" + f"({', '.join(DYNAMIC_PAUSE_COLUMNS)}) {rows}" ) @@ -89,9 +89,9 @@ def open_holds_sql(*, adapter: BaseAdapter, location: StateLocation) -> str: f"FROM {table} p JOIN (SELECT {COLUMN_RUN_ID}, MAX({COLUMN_RECORDED_AT}) AS last_seen " f"FROM {table} WHERE {COLUMN_RECORDED_BY} = {COLUMN_RUN_ID} GROUP BY {COLUMN_RUN_ID}) h " f"ON h.{COLUMN_RUN_ID} = p.{COLUMN_RUN_ID} " - f"WHERE p.{COLUMN_EVENT_TYPE} = '{MaintainedPauseEventType.PAUSED.value}' " + f"WHERE p.{COLUMN_EVENT_TYPE} = '{DynamicPauseEventType.PAUSED.value}' " f"AND NOT EXISTS (SELECT 1 FROM {table} r " - f"WHERE r.{COLUMN_EVENT_TYPE} = '{MaintainedPauseEventType.RESUMED.value}' " + f"WHERE r.{COLUMN_EVENT_TYPE} = '{DynamicPauseEventType.RESUMED.value}' " f"AND r.{COLUMN_RUN_ID} = p.{COLUMN_RUN_ID} AND {same_relation})" ) @@ -123,11 +123,11 @@ def _optional(value: object) -> str | None: return None if value is None else str(value) -def _row_literals(*, event: MaintainedPauseEvent) -> str: +def _row_literals(*, event: DynamicPauseEvent) -> str: relation: CompiledRelationLocation | None = event.relation values: dict[str, object | None] = { COLUMN_EVENT_ID: _event_id(event=event), - COLUMN_SCHEMA_VERSION: MAINTAINED_PAUSE_SCHEMA_VERSION, + COLUMN_SCHEMA_VERSION: DYNAMIC_PAUSE_SCHEMA_VERSION, COLUMN_EVENT_TYPE: event.event_type.value, COLUMN_RUN_ID: event.run_id, COLUMN_RELATION_DATABASE: None if relation is None else relation.database, @@ -137,14 +137,14 @@ def _row_literals(*, event: MaintainedPauseEvent) -> str: } return ", ".join( render_state_sql_literal( - value=values[column], declared_type=MAINTAINED_PAUSE_COLUMN_TYPES[column] + value=values[column], declared_type=DYNAMIC_PAUSE_COLUMN_TYPES[column] ) - for column in MAINTAINED_PAUSE_COLUMNS + for column in DYNAMIC_PAUSE_COLUMNS if column != COLUMN_RECORDED_AT ) -def _event_id(*, event: MaintainedPauseEvent) -> str: +def _event_id(*, event: DynamicPauseEvent) -> str: """Content-addressed: one pause and one resume per run and model, one ID per heartbeat.""" relation: CompiledRelationLocation | None = event.relation diff --git a/src/sqlbuild/executor/dynamic_pauses/classes/__init__.py b/src/sqlbuild/executor/dynamic_pauses/classes/__init__.py new file mode 100644 index 0000000000..128dde7b94 --- /dev/null +++ b/src/sqlbuild/executor/dynamic_pauses/classes/__init__.py @@ -0,0 +1 @@ +"""Dynamic pause classes.""" diff --git a/src/sqlbuild/executor/maintained_pauses/classes/maintained_heartbeat.py b/src/sqlbuild/executor/dynamic_pauses/classes/dynamic_heartbeat.py similarity index 95% rename from src/sqlbuild/executor/maintained_pauses/classes/maintained_heartbeat.py rename to src/sqlbuild/executor/dynamic_pauses/classes/dynamic_heartbeat.py index a689c16004..1e9da2861e 100644 --- a/src/sqlbuild/executor/maintained_pauses/classes/maintained_heartbeat.py +++ b/src/sqlbuild/executor/dynamic_pauses/classes/dynamic_heartbeat.py @@ -9,7 +9,7 @@ from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -class MaintainedHeartbeat: +class DynamicHeartbeat: """Write a heartbeat every interval on its own connection, which a busy build never blocks.""" def __init__( @@ -26,7 +26,7 @@ def __init__( self._interval_seconds: float = interval_seconds self._stopped: threading.Event = threading.Event() self._thread: threading.Thread = threading.Thread( - target=self._beat, name="sqlbuild-maintained-heartbeat", daemon=True + target=self._beat, name="sqlbuild-dynamic-heartbeat", daemon=True ) self.failure: Exception | None = None diff --git a/src/sqlbuild/executor/maintained_pauses/classes/maintained_hold.py b/src/sqlbuild/executor/dynamic_pauses/classes/dynamic_hold.py similarity index 60% rename from src/sqlbuild/executor/maintained_pauses/classes/maintained_hold.py rename to src/sqlbuild/executor/dynamic_pauses/classes/dynamic_hold.py index 9a5c14654b..9db3018d4c 100644 --- a/src/sqlbuild/executor/maintained_pauses/classes/maintained_hold.py +++ b/src/sqlbuild/executor/dynamic_pauses/classes/dynamic_hold.py @@ -1,37 +1,39 @@ -"""One build's hold on the maintained models it touches: pause, record, beat, then resume.""" +"""One build's hold on the dynamic models it touches: pause, record, beat, then resume.""" from __future__ import annotations import threading +import time from collections.abc import Callable, Iterator from contextlib import contextmanager +from datetime import datetime, timedelta from typing import Any from sqlbuild._native import CompiledRelationLocation from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.exceptions import AdapterUserError -from sqlbuild.adapter.contract.types import MaintainedListing, MaintainedOperation -from sqlbuild.executor.maintained_pauses._helpers.sql import ( +from sqlbuild.adapter.contract.types import DynamicListing, DynamicOperation +from sqlbuild.executor.dynamic_pauses._helpers.sql import ( insert_sql, open_holds, open_holds_sql, setup_statements, ) -from sqlbuild.executor.maintained_pauses.classes.maintained_heartbeat import ( - MaintainedHeartbeat, +from sqlbuild.executor.dynamic_pauses.classes.dynamic_heartbeat import ( + DynamicHeartbeat, ) -from sqlbuild.executor.maintained_pauses.constants import ( +from sqlbuild.executor.dynamic_pauses.constants import ( ACTIVE_STATE, HELD_ELSEWHERE_CODE, SUSPENDED_STATE, ) -from sqlbuild.executor.maintained_pauses.models import MaintainedPauseEvent, OpenHold -from sqlbuild.executor.maintained_pauses.types import MaintainedPauseEventType, StateLocation +from sqlbuild.executor.dynamic_pauses.models import DynamicPauseEvent, OpenHold +from sqlbuild.executor.dynamic_pauses.types import DynamicPauseEventType, StateLocation type _RelationKey = tuple[str | None, str | None, str] -class MaintainedHold: +class DynamicHold: """Pause the held models a build found active, record it, and resume only those at the end.""" def __init__( @@ -42,21 +44,22 @@ def __init__( refreshed: frozenset[str], state_location: StateLocation | None, settings: tuple[str, bool, float, bool], - heartbeat: Callable[[Callable[..., None]], MaintainedHeartbeat], + heartbeat: Callable[[Callable[..., None]], DynamicHeartbeat], ) -> None: self._adapter: BaseAdapter = adapter self._held: dict[str, CompiledRelationLocation] = held self._refreshed: frozenset[str] = refreshed self._state_location: StateLocation | None = state_location self._run_id: str - self._pause_maintained: bool + self._pause_dynamic: bool self._grace_seconds: float self._ignore_grace: bool - self._run_id, self._pause_maintained, self._grace_seconds, self._ignore_grace = settings - self._new_heartbeat: Callable[[Callable[..., None]], MaintainedHeartbeat] = heartbeat - self._heartbeat: MaintainedHeartbeat | None = None + self._run_id, self._pause_dynamic, self._grace_seconds, self._ignore_grace = settings + self._new_heartbeat: Callable[[Callable[..., None]], DynamicHeartbeat] = heartbeat + self._heartbeat: DynamicHeartbeat | None = None self._paused: list[str] = [] self._left_paused: frozenset[str] = frozenset() + self._suspended_unheld: list[str] = [] self._lock: threading.Lock = threading.Lock() self._on_progress: Callable[[str], None] | None = None @@ -73,20 +76,20 @@ def holding( try: self.release(connection=connection) except Exception as error: - self._warn(f"could not resume the maintained models this build paused: {error}") + self._warn(f"could not resume the dynamic models this build paused: {error}") raise self.release(connection=connection) def refreshes(self, *, name: str) -> bool: """Whether a kept or altered model is refreshed by hand once its inputs are built.""" - return self._pause_maintained or name in self._refreshed + return self._pause_dynamic or name in self._refreshed def acquire(self, *, connection: Any, on_progress: Callable[[str], None] | None) -> None: """Take over expired holds, refuse live ones, then pause the held models found active.""" self._on_progress = on_progress - if self._pause_maintained: + if self._pause_dynamic: self._suspend_active(connection=connection, names=tuple(self._held)) return location: StateLocation | None = self._state_location @@ -110,42 +113,55 @@ def acquire(self, *, connection: Any, on_progress: Callable[[str], None] | None) if states[name] == SUSPENDED_STATE and name not in self._paused ) for name in sorted(self._left_paused): - self._warn(f"maintained model {name} was paused outside SQLBuild; it stays paused") - self._record( + self._warn(f"dynamic model {name} was paused outside SQLBuild; it stays paused") + window: timedelta = self._timed_record( connection=connection, events=( *( - self._event(event_type=MaintainedPauseEventType.RESUMED, hold=hold) + self._event(event_type=DynamicPauseEventType.RESUMED, hold=hold) for hold in taken ), *( - self._own(event_type=MaintainedPauseEventType.PAUSED, name=name) + self._own(event_type=DynamicPauseEventType.PAUSED, name=name) for name in self._paused ), ), ) - self._yield_to_earlier(connection=connection, location=location) + self._yield_to_rivals( + connection=connection, location=location, names=tuple(self._paused), window=window + ) self._suspend_active(connection=connection, names=tuple(self._paused), states=states) self._heartbeat = self._new_heartbeat(self._write_heartbeat) self._heartbeat.start() def after_create(self, *, connection: Any, name: str) -> None: - """Suspend a held model that CREATE OR REPLACE made active; resume it only if ours.""" + """Suspend a model CREATE OR REPLACE made active; check its new hold as acquire does.""" if name not in self._held: return with self._lock: newly_paused: bool = not ( - self._pause_maintained or name in self._paused or name in self._left_paused + self._pause_dynamic or name in self._paused or name in self._left_paused ) if newly_paused: self._paused.append(name) + window: timedelta = timedelta() if newly_paused: - self._record( + window = self._timed_record( connection=connection, - events=(self._own(event_type=MaintainedPauseEventType.PAUSED, name=name),), + events=(self._own(event_type=DynamicPauseEventType.PAUSED, name=name),), ) - self._operate(connection=connection, name=name, operation=MaintainedOperation.SUSPEND) + self._operate(connection=connection, name=name, operation=DynamicOperation.SUSPEND) + location: StateLocation | None = self._state_location + if newly_paused and location is not None: + try: + self._yield_to_rivals( + connection=connection, location=location, names=(name,), window=window + ) + except AdapterUserError: + with self._lock: + self._suspended_unheld.append(name) + raise def release(self, *, connection: Any) -> None: """Stop the heartbeat and resume what this build paused, upstream first.""" @@ -154,24 +170,26 @@ def release(self, *, connection: Any) -> None: self._heartbeat.stop() self._report_heartbeat_failure(failure=self._heartbeat.failure) order: list[str] = list(self._held) - still_held: frozenset[_RelationKey] | None = self._still_held(connection=connection) + holds: tuple[OpenHold, ...] | None = self._open_holds(connection=connection) + still_held: frozenset[_RelationKey] | None = ( + None + if holds is None + else frozenset(_key(hold.relation) for hold in holds if hold.run_id == self._run_id) + ) resumed: list[str] = [] - failures: list[Exception] = [] + failures: list[Exception] = self._resume_unheld(connection=connection, holds=holds) for name in sorted(self._paused, key=order.index): if still_held is not None and _key(self._held[name]) not in still_held: continue try: - self._operate( - connection=connection, name=name, operation=MaintainedOperation.RESUME - ) + self._operate(connection=connection, name=name, operation=DynamicOperation.RESUME) resumed.append(name) except Exception as error: failures.append(error) self._record( connection=connection, events=tuple( - self._own(event_type=MaintainedPauseEventType.RESUMED, name=name) - for name in resumed + self._own(event_type=DynamicPauseEventType.RESUMED, name=name) for name in resumed ), ) if failures: @@ -209,7 +227,7 @@ def _take_over(self, *, connection: Any, location: StateLocation) -> tuple[OpenH self._record( connection=connection, events=tuple( - self._event(event_type=MaintainedPauseEventType.RESUMED, hold=hold) + self._event(event_type=DynamicPauseEventType.RESUMED, hold=hold) for hold in elsewhere ), ) @@ -218,10 +236,17 @@ def _take_over(self, *, connection: Any, location: StateLocation) -> tuple[OpenH self._report(holds=taken, verb="Took over") return taken - def _yield_to_earlier(self, *, connection: Any, location: StateLocation) -> None: - """Read the pauses back; if another live hold on a model came first, close ours and stop.""" + def _yield_to_rivals( + self, + *, + connection: Any, + location: StateLocation, + names: tuple[str, ...], + window: timedelta, + ) -> None: + """Read the pauses back; if another live hold may have come first, close ours and stop.""" - mine: set[_RelationKey] = {_key(self._held[name]) for name in self._paused} + mine: set[_RelationKey] = {_key(self._held[name]) for name in names} if not mine: return rows: tuple[tuple[Any, ...], ...] = tuple( @@ -231,30 +256,67 @@ def _yield_to_earlier(self, *, connection: Any, location: StateLocation) -> None limit=None, ).rows ) - first: dict[_RelationKey, OpenHold] = {} - for hold in sorted(open_holds(rows=rows), key=lambda hold: (hold.paused_at, hold.run_id)): - if hold.idle_seconds < self._grace_seconds and _key(hold.relation) in mine: - first.setdefault(_key(hold.relation), hold) - winners: tuple[OpenHold, ...] = tuple( - hold for hold in first.values() if hold.run_id != self._run_id + holds: tuple[OpenHold, ...] = open_holds(rows=rows) + paused_at: dict[_RelationKey, datetime] = { + _key(hold.relation): hold.paused_at + for hold in holds + if hold.run_id == self._run_id and _key(hold.relation) in mine + } + rivals: list[OpenHold] = sorted( + ( + hold + for hold in holds + if hold.run_id != self._run_id + and hold.idle_seconds < self._grace_seconds + and _key(hold.relation) in mine + and ( + _key(hold.relation) not in paused_at + or hold.paused_at <= paused_at[_key(hold.relation)] + window + ) + ), + key=lambda hold: (hold.paused_at, hold.run_id), ) - if not winners: + if not rivals: return - self._record( - connection=connection, - events=tuple( - self._own(event_type=MaintainedPauseEventType.RESUMED, name=name) - for name in self._paused - ), + with self._lock: + self._paused = [name for name in self._paused if name not in names] + try: + self._record( + connection=connection, + events=tuple( + self._own(event_type=DynamicPauseEventType.RESUMED, name=name) for name in names + ), + ) + except Exception as error: + raise _held_elsewhere(holds=tuple(rivals), close_failure=error) from error + raise _held_elsewhere(holds=tuple(rivals)) + + def _resume_unheld( + self, *, connection: Any, holds: tuple[OpenHold, ...] | None + ) -> list[Exception]: + """Resume the models this build suspended after creating them, then gave up, if unheld.""" + + held_elsewhere: frozenset[_RelationKey] = frozenset( + _key(hold.relation) for hold in holds or () if hold.run_id != self._run_id ) - self._paused = [] - raise _held_elsewhere(holds=winners) + failures: list[Exception] = [] + for name in self._suspended_unheld: + if holds is None or _key(self._held[name]) in held_elsewhere: + continue + try: + if self._state(connection=connection, name=name) == SUSPENDED_STATE: + self._operate( + connection=connection, name=name, operation=DynamicOperation.RESUME + ) + except Exception as error: + failures.append(error) + return failures - def _still_held(self, *, connection: Any) -> frozenset[_RelationKey] | None: - """This run's holds nobody took over, or None when the pause table can't say.""" + def _open_holds(self, *, connection: Any) -> tuple[OpenHold, ...] | None: + """Every open hold, or None when the pause table can't say.""" location: StateLocation | None = self._state_location - if location is None or self._pause_maintained: + if location is None or self._pause_dynamic: return None try: rows: tuple[tuple[Any, ...], ...] = tuple( @@ -265,11 +327,9 @@ def _still_held(self, *, connection: Any) -> frozenset[_RelationKey] | None: ).rows ) except Exception as error: - self._warn(f"could not read which maintained models this build still holds: {error}") + self._warn(f"could not read which dynamic models this build still holds: {error}") return None - return frozenset( - _key(hold.relation) for hold in open_holds(rows=rows) if hold.run_id == self._run_id - ) + return open_holds(rows=rows) def _resume_found(self, *, connection: Any, relation: CompiledRelationLocation) -> None: if _scheduling_state(adapter=self._adapter, connection=connection, relation=relation) == ( @@ -278,7 +338,7 @@ def _resume_found(self, *, connection: Any, relation: CompiledRelationLocation) self._run( connection=connection, destination=self._destination(relation), - operation=MaintainedOperation.RESUME, + operation=DynamicOperation.RESUME, ) def _suspend_active( @@ -295,24 +355,22 @@ def _suspend_active( else self._state(connection=connection, name=name) ) if state == ACTIVE_STATE: - self._operate( - connection=connection, name=name, operation=MaintainedOperation.SUSPEND - ) + self._operate(connection=connection, name=name, operation=DynamicOperation.SUSPEND) def _state(self, *, connection: Any, name: str) -> str | None: return _scheduling_state( adapter=self._adapter, connection=connection, relation=self._held[name] ) - def _operate(self, *, connection: Any, name: str, operation: MaintainedOperation) -> None: + def _operate(self, *, connection: Any, name: str, operation: DynamicOperation) -> None: self._run( connection=connection, destination=self._destination(self._held[name]), operation=operation, ) - def _run(self, *, connection: Any, destination: str, operation: MaintainedOperation) -> None: - for statement in self._adapter.render_maintained_operation( + def _run(self, *, connection: Any, destination: str, operation: DynamicOperation) -> None: + for statement in self._adapter.render_dynamic_operation( destination=destination, operation=operation ): self._adapter.execute(connection=connection, sql=statement) @@ -324,7 +382,14 @@ def _destination(self, location: CompiledRelationLocation) -> str: ) ) - def _record(self, *, connection: Any, events: tuple[MaintainedPauseEvent, ...]) -> None: + def _timed_record(self, *, connection: Any, events: tuple[DynamicPauseEvent, ...]) -> timedelta: + """Record, and return how long the insert took: an upper bound on its commit delay.""" + + started: float = time.monotonic() + self._record(connection=connection, events=events) + return timedelta(seconds=time.monotonic() - started) + + def _record(self, *, connection: Any, events: tuple[DynamicPauseEvent, ...]) -> None: location: StateLocation | None = self._state_location if not events or location is None: return @@ -337,8 +402,8 @@ def _write_heartbeat(self, *, connection: Any, sequence: int) -> None: self._record( connection=connection, events=( - MaintainedPauseEvent( - event_type=MaintainedPauseEventType.HEARTBEAT, + DynamicPauseEvent( + event_type=DynamicPauseEventType.HEARTBEAT, run_id=self._run_id, recorded_by=self._run_id, relation=None, @@ -347,18 +412,16 @@ def _write_heartbeat(self, *, connection: Any, sequence: int) -> None: ), ) - def _own(self, *, event_type: MaintainedPauseEventType, name: str) -> MaintainedPauseEvent: - return MaintainedPauseEvent( + def _own(self, *, event_type: DynamicPauseEventType, name: str) -> DynamicPauseEvent: + return DynamicPauseEvent( event_type=event_type, run_id=self._run_id, recorded_by=self._run_id, relation=self._held[name], ) - def _event( - self, *, event_type: MaintainedPauseEventType, hold: OpenHold - ) -> MaintainedPauseEvent: - return MaintainedPauseEvent( + def _event(self, *, event_type: DynamicPauseEventType, hold: OpenHold) -> DynamicPauseEvent: + return DynamicPauseEvent( event_type=event_type, run_id=hold.run_id, recorded_by=self._run_id, @@ -370,13 +433,13 @@ def _report(self, *, holds: tuple[OpenHold, ...], verb: str) -> None: return for hold in holds: self._on_progress( - f"{verb} maintained model {hold.relation.name}, paused by run {hold.run_id} " + f"{verb} dynamic model {hold.relation.name}, paused by run {hold.run_id} " f"whose heartbeat has been silent for {int(hold.idle_seconds)}s" ) def _report_heartbeat_failure(self, *, failure: Exception | None) -> None: if failure is not None: - self._warn(f"a maintained pause heartbeat failed: {failure}") + self._warn(f"a dynamic pause heartbeat failed: {failure}") def _warn(self, message: str) -> None: if self._on_progress is not None: @@ -386,7 +449,7 @@ def _warn(self, message: str) -> None: def _scheduling_state( *, adapter: BaseAdapter, connection: Any, relation: CompiledRelationLocation ) -> str | None: - listing: MaintainedListing | None = adapter.maintained_listing( + listing: DynamicListing | None = adapter.dynamic_listing( connection=connection, database=relation.database, schema=relation.schema, @@ -399,16 +462,23 @@ def _key(location: CompiledRelationLocation) -> _RelationKey: return (location.database, location.schema, location.name) -def _held_elsewhere(*, holds: tuple[OpenHold, ...]) -> AdapterUserError: +def _held_elsewhere( + *, holds: tuple[OpenHold, ...], close_failure: Exception | None = None +) -> AdapterUserError: first: OpenHold = holds[0] names: str = ", ".join(sorted({hold.relation.name for hold in holds})) + closing: str = ( + "" + if close_failure is None + else f"; closing this build's own pause also failed: {close_failure}" + ) return AdapterUserError( - f"run {first.run_id} holds maintained models this build needs ({names}); " - f"its last heartbeat was {int(first.idle_seconds)}s ago", + f"run {first.run_id} holds dynamic models this build needs ({names}); " + f"its last heartbeat was {int(first.idle_seconds)}s ago{closing}", code=HELD_ELSEWHERE_CODE, help=( - "wait for that build to finish, or for its hold to pass [maintained] " + "wait for that build to finish, or for its hold to pass [dynamic] " "heartbeat_grace; with allow_ignore_grace = true, " - "sqb build --ignore-maintained-grace takes over at once" + "sqb build --ignore-dynamic-grace takes over at once" ), ) diff --git a/src/sqlbuild/executor/maintained_pauses/constants.py b/src/sqlbuild/executor/dynamic_pauses/constants.py similarity index 75% rename from src/sqlbuild/executor/maintained_pauses/constants.py rename to src/sqlbuild/executor/dynamic_pauses/constants.py index 22fd73f5fa..1abaac9786 100644 --- a/src/sqlbuild/executor/maintained_pauses/constants.py +++ b/src/sqlbuild/executor/dynamic_pauses/constants.py @@ -1,8 +1,8 @@ -"""Maintained pause table columns, timings and error codes.""" +"""Dynamic pause table columns, timings and error codes.""" from sqlbuild._native import StateSqlValueType -MAINTAINED_PAUSE_SCHEMA_VERSION: int = 1 +DYNAMIC_PAUSE_SCHEMA_VERSION: int = 1 HEARTBEAT_SECONDS: float = 60.0 ACTIVE_STATE: str = "ACTIVE" SUSPENDED_STATE: str = "SUSPENDED" @@ -19,7 +19,7 @@ COLUMN_RECORDED_BY: str = "recorded_by" COLUMN_RECORDED_AT: str = "recorded_at" -MAINTAINED_PAUSE_COLUMNS: tuple[str, ...] = ( +DYNAMIC_PAUSE_COLUMNS: tuple[str, ...] = ( COLUMN_EVENT_ID, COLUMN_SCHEMA_VERSION, COLUMN_EVENT_TYPE, @@ -31,7 +31,7 @@ COLUMN_RECORDED_AT, ) -MAINTAINED_PAUSE_REQUIRED_COLUMNS: frozenset[str] = frozenset( +DYNAMIC_PAUSE_REQUIRED_COLUMNS: frozenset[str] = frozenset( { COLUMN_EVENT_ID, COLUMN_SCHEMA_VERSION, @@ -42,8 +42,8 @@ } ) -MAINTAINED_PAUSE_COLUMN_TYPES: dict[str, StateSqlValueType] = { - **{column: StateSqlValueType.STRING for column in MAINTAINED_PAUSE_COLUMNS}, +DYNAMIC_PAUSE_COLUMN_TYPES: dict[str, StateSqlValueType] = { + **{column: StateSqlValueType.STRING for column in DYNAMIC_PAUSE_COLUMNS}, COLUMN_SCHEMA_VERSION: StateSqlValueType.INTEGER, COLUMN_RECORDED_AT: StateSqlValueType.TEXT_TIMESTAMP, } diff --git a/src/sqlbuild/executor/dynamic_pauses/main/__init__.py b/src/sqlbuild/executor/dynamic_pauses/main/__init__.py new file mode 100644 index 0000000000..3fa138e58c --- /dev/null +++ b/src/sqlbuild/executor/dynamic_pauses/main/__init__.py @@ -0,0 +1 @@ +"""Dynamic pause entrypoints.""" diff --git a/src/sqlbuild/executor/maintained_pauses/main/hold_maintained_models.py b/src/sqlbuild/executor/dynamic_pauses/main/hold_dynamic_models.py similarity index 58% rename from src/sqlbuild/executor/maintained_pauses/main/hold_maintained_models.py rename to src/sqlbuild/executor/dynamic_pauses/main/hold_dynamic_models.py index dc8045be4d..f6a3112bc0 100644 --- a/src/sqlbuild/executor/maintained_pauses/main/hold_maintained_models.py +++ b/src/sqlbuild/executor/dynamic_pauses/main/hold_dynamic_models.py @@ -1,4 +1,4 @@ -"""The hold a build keeps on the maintained models it touches, or None when it touches none.""" +"""The hold a build keeps on the dynamic models it touches, or None when it touches none.""" from __future__ import annotations @@ -6,51 +6,51 @@ from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.exceptions import AdapterUserError from sqlbuild.executor.build.models import BuildRuntimeParams -from sqlbuild.executor.maintained_pauses._helpers.scope import held_names, refreshed_for_readers -from sqlbuild.executor.maintained_pauses.classes.maintained_heartbeat import ( - MaintainedHeartbeat, +from sqlbuild.executor.dynamic_pauses._helpers.scope import held_names, refreshed_for_readers +from sqlbuild.executor.dynamic_pauses.classes.dynamic_heartbeat import ( + DynamicHeartbeat, ) -from sqlbuild.executor.maintained_pauses.classes.maintained_hold import MaintainedHold -from sqlbuild.executor.maintained_pauses.constants import ( +from sqlbuild.executor.dynamic_pauses.classes.dynamic_hold import DynamicHold +from sqlbuild.executor.dynamic_pauses.constants import ( HEARTBEAT_SECONDS, IGNORE_GRACE_REFUSED_CODE, ) -from sqlbuild.executor.maintained_pauses.types import StateLocation +from sqlbuild.executor.dynamic_pauses.types import StateLocation -def hold_maintained_models( +def hold_dynamic_models( *, plan: PlanOutput, adapter: BaseAdapter, runtime: BuildRuntimeParams, connection_config: dict[str, object], heartbeat_seconds: float = HEARTBEAT_SECONDS, -) -> MaintainedHold | None: - """The build's hold, unacquired; `--ignore-maintained-grace` needs `allow_ignore_grace`.""" +) -> DynamicHold | None: + """The build's hold, unacquired; `--ignore-dynamic-grace` needs `allow_ignore_grace`.""" - if runtime.ignore_maintained_grace and not runtime.allow_ignore_maintained_grace: + if runtime.ignore_dynamic_grace and not runtime.allow_ignore_dynamic_grace: raise AdapterUserError( - "--ignore-maintained-grace needs [maintained] allow_ignore_grace = true", + "--ignore-dynamic-grace needs [dynamic] allow_ignore_grace = true", code=IGNORE_GRACE_REFUSED_CODE, - help="set allow_ignore_grace = true under [maintained] in sqlbuild_project.toml", + help="set allow_ignore_grace = true under [dynamic] in sqlbuild_project.toml", ) - names: tuple[str, ...] = held_names(plan=plan, pause_maintained=runtime.pause_maintained) + names: tuple[str, ...] = held_names(plan=plan, pause_dynamic=runtime.pause_dynamic) refreshed: frozenset[str] = refreshed_for_readers(plan=plan) if not names and not refreshed: return None held: dict[str, CompiledRelationLocation] = {name: plan.model_locations[name] for name in names} - return MaintainedHold( + return DynamicHold( adapter=adapter, held=held, refreshed=refreshed, state_location=_state_location(runtime=runtime, held=held) if held else None, settings=( runtime.run_id, - runtime.pause_maintained, - float(runtime.maintained_heartbeat_grace_seconds), - runtime.ignore_maintained_grace, + runtime.pause_dynamic, + float(runtime.dynamic_heartbeat_grace_seconds), + runtime.ignore_dynamic_grace, ), - heartbeat=lambda write: MaintainedHeartbeat( + heartbeat=lambda write: DynamicHeartbeat( adapter=adapter, connection_config=connection_config, write=write, diff --git a/src/sqlbuild/executor/maintained_pauses/models.py b/src/sqlbuild/executor/dynamic_pauses/models.py similarity index 76% rename from src/sqlbuild/executor/maintained_pauses/models.py rename to src/sqlbuild/executor/dynamic_pauses/models.py index f76cca16b9..62f103abc2 100644 --- a/src/sqlbuild/executor/maintained_pauses/models.py +++ b/src/sqlbuild/executor/dynamic_pauses/models.py @@ -1,4 +1,4 @@ -"""Records of the maintained pause lease.""" +"""Records of the dynamic pause lease.""" from __future__ import annotations @@ -6,14 +6,14 @@ from datetime import datetime from sqlbuild._native import CompiledRelationLocation -from sqlbuild.executor.maintained_pauses.types import MaintainedPauseEventType +from sqlbuild.executor.dynamic_pauses.types import DynamicPauseEventType @dataclass(frozen=True) -class MaintainedPauseEvent: +class DynamicPauseEvent: """One row to append: a pause, a resume, or a heartbeat of the run holding models.""" - event_type: MaintainedPauseEventType + event_type: DynamicPauseEventType run_id: str recorded_by: str relation: CompiledRelationLocation | None diff --git a/src/sqlbuild/executor/maintained_pauses/types.py b/src/sqlbuild/executor/dynamic_pauses/types.py similarity index 75% rename from src/sqlbuild/executor/maintained_pauses/types.py rename to src/sqlbuild/executor/dynamic_pauses/types.py index c199bdd358..11f347b513 100644 --- a/src/sqlbuild/executor/maintained_pauses/types.py +++ b/src/sqlbuild/executor/dynamic_pauses/types.py @@ -1,4 +1,4 @@ -"""Maintained pause event types.""" +"""Dynamic pause event types.""" from __future__ import annotations @@ -7,7 +7,7 @@ type StateLocation = tuple[str | None, str] -class MaintainedPauseEventType(StrEnum): +class DynamicPauseEventType(StrEnum): """What one row of the pause table records.""" PAUSED = "paused" diff --git a/src/sqlbuild/executor/maintained_pauses/__init__.py b/src/sqlbuild/executor/maintained_pauses/__init__.py deleted file mode 100644 index 9aae34706c..0000000000 --- a/src/sqlbuild/executor/maintained_pauses/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Maintained models a build holds paused, and the append-only lease that records it.""" diff --git a/src/sqlbuild/executor/maintained_pauses/_helpers/__init__.py b/src/sqlbuild/executor/maintained_pauses/_helpers/__init__.py deleted file mode 100644 index 928816ab73..0000000000 --- a/src/sqlbuild/executor/maintained_pauses/_helpers/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Maintained pause helpers.""" diff --git a/src/sqlbuild/executor/maintained_pauses/classes/__init__.py b/src/sqlbuild/executor/maintained_pauses/classes/__init__.py deleted file mode 100644 index 4dbf76c0aa..0000000000 --- a/src/sqlbuild/executor/maintained_pauses/classes/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Maintained pause classes.""" diff --git a/src/sqlbuild/executor/maintained_pauses/main/__init__.py b/src/sqlbuild/executor/maintained_pauses/main/__init__.py deleted file mode 100644 index 36ef4cff68..0000000000 --- a/src/sqlbuild/executor/maintained_pauses/main/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Maintained pause entrypoints.""" diff --git a/src/sqlbuild/executor/run/_helpers/execution/table_config.py b/src/sqlbuild/executor/run/_helpers/execution/table_config.py index ceeee3e044..cd4c5ede42 100644 --- a/src/sqlbuild/executor/run/_helpers/execution/table_config.py +++ b/src/sqlbuild/executor/run/_helpers/execution/table_config.py @@ -21,7 +21,7 @@ def model_table_config(entry: ModelPlanEntry) -> dict[str, object]: return { "table_type": entry.table_type, - "change_tracking": bool(entry.maintained_readers), + "change_tracking": bool(entry.dynamic_readers), "physical_storage": entry.physical_storage, **model_output_config(entry), } diff --git a/src/sqlbuild/executor/run/_helpers/materializations/maintained.py b/src/sqlbuild/executor/run/_helpers/materializations/dynamic.py similarity index 84% rename from src/sqlbuild/executor/run/_helpers/materializations/maintained.py rename to src/sqlbuild/executor/run/_helpers/materializations/dynamic.py index 55821b6928..7d104dcbe9 100644 --- a/src/sqlbuild/executor/run/_helpers/materializations/maintained.py +++ b/src/sqlbuild/executor/run/_helpers/materializations/dynamic.py @@ -1,4 +1,4 @@ -"""Maintained model lifecycle: the warehouse refreshes the model; a build creates or alters it.""" +"""Dynamic model lifecycle: the warehouse refreshes the model; a build creates or alters it.""" from __future__ import annotations @@ -16,12 +16,12 @@ from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.exceptions import AdapterUserError -from sqlbuild.adapter.contract.types import MaintainedListing, MaintainedOperation, RelationType +from sqlbuild.adapter.contract.types import DynamicListing, DynamicOperation, RelationType from sqlbuild.adapter.relations.main.cached_relation_type import cached_relation_type from sqlbuild.adapter.relations.main.resolve_qualified_name_parts import ( resolve_qualified_name_parts, ) -from sqlbuild.executor.maintained_pauses.classes.maintained_hold import MaintainedHold +from sqlbuild.executor.dynamic_pauses.classes.dynamic_hold import DynamicHold from sqlbuild.executor.run._helpers.materializations.view import execute_in_place_entry from sqlbuild.executor.run.constants import REPLACED_RELATION_SUFFIX, RUN_STAGING_FAILED_CODE from sqlbuild.executor.run.models import ModelExecutionResult, ModelMaterializationContext @@ -29,22 +29,22 @@ _NO_WAREHOUSE_CODE: str = "A717" -def execute_maintained_entry(*, context: ModelMaterializationContext) -> ModelExecutionResult: - """Create, alter or keep one maintained model, refresh a kept one when asked, then audit it.""" +def execute_dynamic_entry(*, context: ModelMaterializationContext) -> ModelExecutionResult: + """Create, alter or keep one dynamic model, refresh a kept one when asked, then audit it.""" def materialize(*, target_qualified: str, statement_recorder: StatementRecorder) -> None: entry: ModelPlanEntry = context.entry - hold: MaintainedHold | None = context.maintained_hold - if entry.action == PlanAction.CREATE_MAINTAINED: + hold: DynamicHold | None = context.dynamic_hold + if entry.action == PlanAction.CREATE_DYNAMIC: _create(context=context, destination=target_qualified, recorder=statement_recorder) if hold is not None: hold.after_create(connection=context.connection, name=entry.name) return - if entry.action == PlanAction.ALTER_MAINTAINED: + if entry.action == PlanAction.ALTER_DYNAMIC: _run( adapter=context.adapter, connection=context.connection, - statements=context.adapter.render_alter_maintained( + statements=context.adapter.render_alter_dynamic( destination=target_qualified, target_lag=str(entry.target_lag), warehouse=entry.refresh_warehouse, @@ -55,8 +55,8 @@ def materialize(*, target_qualified: str, statement_recorder: StatementRecorder) _run( adapter=context.adapter, connection=context.connection, - statements=context.adapter.render_maintained_operation( - destination=target_qualified, operation=MaintainedOperation.REFRESH + statements=context.adapter.render_dynamic_operation( + destination=target_qualified, operation=DynamicOperation.REFRESH ), recorder=statement_recorder, ) @@ -65,7 +65,7 @@ def materialize(*, target_qualified: str, statement_recorder: StatementRecorder) context=context, materialize=materialize, action_name=str(context.entry.action), - built="the maintained model's refresh", + built="the dynamic model's refresh", ) @@ -82,11 +82,11 @@ def _create( ) if warehouse is None: raise AdapterUserError( - f"maintained model '{entry.name}' has no warehouse to refresh it", + f"dynamic model '{entry.name}' has no warehouse to refresh it", code=_NO_WAREHOUSE_CODE, help="set refresh_warehouse on the model, or a warehouse on the target's connection", ) - statements: tuple[str, ...] = adapter.render_create_maintained( + statements: tuple[str, ...] = adapter.render_create_dynamic( destination=destination, sql=entry.resolved_sql, target_lag=str(entry.target_lag), @@ -163,7 +163,7 @@ def _replace_former( def _former_relation( *, adapter: BaseAdapter, connection: Any, entry: ModelPlanEntry ) -> RelationType | None: - """The type of a relation at the destination the warehouse does not list as maintained.""" + """The type of a relation at the destination the warehouse does not list as dynamic.""" location: CompiledRelationLocation = entry.destination existing: RelationType | None = cached_relation_type( @@ -175,7 +175,7 @@ def _former_relation( ) if existing is None: return None - listed: MaintainedListing | None = adapter.maintained_listing( + listed: DynamicListing | None = adapter.dynamic_listing( connection=connection, database=location.database, schema=location.schema, diff --git a/src/sqlbuild/executor/run/_helpers/materializations/microbatch.py b/src/sqlbuild/executor/run/_helpers/materializations/microbatch.py index d3389b7b36..d9886af16f 100644 --- a/src/sqlbuild/executor/run/_helpers/materializations/microbatch.py +++ b/src/sqlbuild/executor/run/_helpers/materializations/microbatch.py @@ -3057,7 +3057,7 @@ def _apply_microbatch_dml( config={ "table_type": context.entry.table_type, "physical_storage": context.entry.physical_storage, - "change_tracking": bool(context.entry.maintained_readers), + "change_tracking": bool(context.entry.dynamic_readers), }, statement_recorder=state.statement_recorder, ) diff --git a/src/sqlbuild/executor/run/_helpers/materializations/snapshot.py b/src/sqlbuild/executor/run/_helpers/materializations/snapshot.py index cf3b7bf4e7..4243567e3f 100644 --- a/src/sqlbuild/executor/run/_helpers/materializations/snapshot.py +++ b/src/sqlbuild/executor/run/_helpers/materializations/snapshot.py @@ -800,6 +800,8 @@ def _create_initial_snapshot_target( valid_to_column=valid_to_column, initial_valid_from=entry.initial_valid_from, ) + if entry.dynamic_readers: + statements = (*statements, *adapter.render_track_changes(destination=target_qualified)) statement_recorder.record_many(statements) statement: str for statement in statements: diff --git a/src/sqlbuild/executor/run/main/_execute.py b/src/sqlbuild/executor/run/main/_execute.py index 1d8584e21b..fe13cbb17c 100644 --- a/src/sqlbuild/executor/run/main/_execute.py +++ b/src/sqlbuild/executor/run/main/_execute.py @@ -41,12 +41,12 @@ from sqlbuild.executor.run._helpers.materializations.custom import ( execute_custom_entry as execute_custom_entry, ) +from sqlbuild.executor.run._helpers.materializations.dynamic import ( + execute_dynamic_entry as execute_dynamic_entry, +) from sqlbuild.executor.run._helpers.materializations.incremental import ( execute_incremental_entry as execute_incremental_entry, ) -from sqlbuild.executor.run._helpers.materializations.maintained import ( - execute_maintained_entry as execute_maintained_entry, -) from sqlbuild.executor.run._helpers.materializations.microbatch import ( execute_microbatch_entry as execute_microbatch_entry, ) @@ -249,7 +249,7 @@ def _staged_lifecycle( table_config={ "table_type": entry.table_type, "physical_storage": entry.physical_storage, - "change_tracking": bool(entry.maintained_readers), + "change_tracking": bool(entry.dynamic_readers), }, statement_recorder=statement_recorder, ) diff --git a/src/sqlbuild/executor/run/models.py b/src/sqlbuild/executor/run/models.py index 84bec90f9f..fc47787c66 100644 --- a/src/sqlbuild/executor/run/models.py +++ b/src/sqlbuild/executor/run/models.py @@ -40,7 +40,7 @@ from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.auditing.models import AuditExecutionResult from sqlbuild.executor.custom.models import MaterializationResult -from sqlbuild.executor.maintained_pauses.classes.maintained_hold import MaintainedHold +from sqlbuild.executor.dynamic_pauses.classes.dynamic_hold import DynamicHold from sqlbuild.executor.python_nodes.classes.runtime_relation_guard import RuntimeRelationGuard from sqlbuild.executor.python_nodes.types import PythonIdentityRecorder from sqlbuild.executor.run.classes.fingerprint_writes import FingerprintWrites @@ -334,7 +334,7 @@ class ModelMaterializationContext: enforce_explicit_references: bool = False bound_views: BoundViewGuard = field(default_factory=BoundViewGuard) fingerprint_writes: FingerprintWrites | None = None - maintained_hold: MaintainedHold | None = None + dynamic_hold: DynamicHold | None = None @dataclass(frozen=True, kw_only=True, init=False) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py index 639178b99d..c50594341d 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py @@ -1464,8 +1464,8 @@ class JobGroupCyclesCase: @dataclass(frozen=True) -class MaintainedCompileE2ETestCase: - """A project with maintained models, its exit code and text its compile output contains.""" +class DynamicCompileE2ETestCase: + """A project with dynamic models, its exit code and text its compile output contains.""" description: str adapter: str diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_maintained_models.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dynamic_models.py similarity index 72% rename from tests/e2e/src/sqlbuild/cli/commands/main/compile/test_maintained_models.py rename to tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dynamic_models.py index da2449debf..ca307d8ad0 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_maintained_models.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dynamic_models.py @@ -1,4 +1,4 @@ -"""E2E coverage of maintained models: which adapters and headers compile, and the errors they get.""" +"""E2E coverage of dynamic models: which adapters and headers compile, and the errors they get.""" from __future__ import annotations @@ -8,7 +8,7 @@ import pytest from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import ( - MaintainedCompileE2ETestCase, + DynamicCompileE2ETestCase, ) from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import prepare_inline_project, run_sqb @@ -27,7 +27,7 @@ "MODEL (description 'Orders.', materialized table);\nSELECT 1 AS order_id, 10 AS amount\n" ) _CLEAN_ORDERS: str = ( - "MODEL (description 'Orders with an amount.', materialized maintained);\n" + "MODEL (description 'Orders with an amount.', materialized dynamic);\n" 'SELECT order_id, amount FROM __ref("orders") WHERE amount > 0\n' ) _DAILY_TOTALS_QUERY: str = 'SELECT SUM(amount) AS total FROM __ref("clean_orders")\n' @@ -36,61 +36,61 @@ @pytest.mark.parametrize( "test_case", [ - MaintainedCompileE2ETestCase( + DynamicCompileE2ETestCase( description="a chain refreshed by the lag of its last model compiles on Snowflake", adapter="snowflake", - daily_totals_header="materialized maintained, refresh (lag 1h), refresh_mode full", + daily_totals_header="materialized dynamic, refresh (lag 1h), refresh_mode full", expected_exit_code=0, expected_fragments=("Project compiled 3 models",), ), - MaintainedCompileE2ETestCase( - description="maintained models are refused on DuckDB", + DynamicCompileE2ETestCase( + description="dynamic models are refused on DuckDB", adapter="duckdb", - daily_totals_header="materialized maintained, refresh (lag 1h)", + daily_totals_header="materialized dynamic, refresh (lag 1h)", expected_exit_code=1, expected_fragments=( "[A013]", - "materialized maintained is not supported on the duckdb adapter; " + "materialized dynamic is not supported on the duckdb adapter; " "adapters that support it: snowflake", ), ), - MaintainedCompileE2ETestCase( - description="materialized_view on Snowflake points to maintained", + DynamicCompileE2ETestCase( + description="materialized_view on Snowflake points to dynamic", adapter="snowflake", daily_totals_header="materialized materialized_view", expected_exit_code=1, expected_fragments=( "[A014]", "Snowflake has no materialized_view materialization; use materialized " - "maintained, which builds a dynamic table", + "dynamic, which builds a dynamic table", ), ), - MaintainedCompileE2ETestCase( + DynamicCompileE2ETestCase( description="a lag below Snowflake's minimum", adapter="snowflake", - daily_totals_header="materialized maintained, refresh (lag 30s)", + daily_totals_header="materialized dynamic, refresh (lag 30s)", expected_exit_code=1, expected_fragments=( "[A015]", "refresh lag on Snowflake must be at least 1m; got '30s'", ), ), - MaintainedCompileE2ETestCase( - description="a last maintained model with no refresh would never refresh", + DynamicCompileE2ETestCase( + description="a last dynamic model with no refresh would never refresh", adapter="snowflake", - daily_totals_header="materialized maintained", + daily_totals_header="materialized dynamic", expected_exit_code=1, expected_fragments=( "[A016]", - "maintained model 'daily_totals' has no refresh and no maintained model reads " + "dynamic model 'daily_totals' has no refresh and no dynamic model reads " "it, so Snowflake would never refresh it; add refresh (lag 1h)", ), ), ], ids=lambda case: case.description, ) -def test_given_maintained_models_when_compiling_then_adapter_rules_apply( - test_case: MaintainedCompileE2ETestCase, tmp_path: Path +def test_given_dynamic_models_when_compiling_then_adapter_rules_apply( + test_case: DynamicCompileE2ETestCase, tmp_path: Path ) -> None: project_dir: Path = prepare_inline_project( tmp_path=tmp_path, diff --git a/tests/goldens/compiler/failure/target-unknown-key.json b/tests/goldens/compiler/failure/target-unknown-key.json index 9d222e2426..fc7580ddd4 100644 --- a/tests/goldens/compiler/failure/target-unknown-key.json +++ b/tests/goldens/compiler/failure/target-unknown-key.json @@ -8,7 +8,7 @@ "phase": "compile", "severity": "error", "code": "D001", - "message": "/sqlbuild_project.toml targets.ci contains unknown key(s): warehouse_size. Allowed keys: catalog_connection, clone, compile_cache, concurrency, connection, database, default_table_type, defer_clone_from, defer_sources_to, diff, execution_limits, ignore_audit_failures, loader_schema, missing_migration_origin, owns_time_travel_retention_namespace, pause_maintained, plan_from, pools, recall, schema, table_type_downgrade, time_travel_retention, time_travel_retention_decrease, vars, warehouses", + "message": "/sqlbuild_project.toml targets.ci contains unknown key(s): warehouse_size. Allowed keys: catalog_connection, clone, compile_cache, concurrency, connection, database, default_table_type, defer_clone_from, defer_sources_to, diff, execution_limits, ignore_audit_failures, loader_schema, missing_migration_origin, owns_time_travel_retention_namespace, pause_dynamic, plan_from, pools, recall, schema, table_type_downgrade, time_travel_retention, time_travel_retention_decrease, vars, warehouses", "help": "did you mean 'warehouses'?" } ] diff --git a/tests/integration/src/sqlbuild/compiler/planner/main/_test_types.py b/tests/integration/src/sqlbuild/compiler/planner/main/_test_types.py index ac3e1abfbe..2add956198 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/main/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/planner/main/_test_types.py @@ -7,7 +7,7 @@ PlanReason, WarningSeverity, ) -from sqlbuild.adapter.contract.types import MaintainedListing +from sqlbuild.adapter.contract.types import DynamicListing from sqlbuild.compiler.compile.types import FunctionLanguage @@ -209,9 +209,9 @@ class SyncedScanTestCase: @dataclass(frozen=True) -class MaintainedRefreshPlanTestCase: +class DynamicRefreshPlanTestCase: description: str - listed_settings: MaintainedListing | None + listed_settings: DynamicListing | None upstream_query: str expected_upstream_action: PlanAction expected_action: PlanAction @@ -219,11 +219,12 @@ class MaintainedRefreshPlanTestCase: expected_reinitialised_by: str | None expected_read_names: tuple[str, ...] expected_paused: bool = False + totals_config: dict[str, object] | None = None @dataclass(frozen=True) class ChangeTrackingPlanTestCase: description: str select: tuple[str, ...] - expected_maintained_readers: dict[str, tuple[str, ...]] + expected_dynamic_readers: dict[str, tuple[str, ...]] expected_refresh_on_build: dict[str, bool] diff --git a/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py b/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py index b13f754f11..9a963f36b6 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py @@ -61,7 +61,7 @@ RelationInfo, RelationReadProbe, ) -from sqlbuild.adapter.contract.types import MaintainedListing +from sqlbuild.adapter.contract.types import DynamicListing from sqlbuild.adapter.relations.classes.inspection_catalog import InspectionCatalog from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.types import ( @@ -835,8 +835,8 @@ def planned_entries(result: CompilePipelineResult) -> list[tuple[str, str]]: CLEAN_ORDERS_QUERY: str = "SELECT 1 AS order_id" -_MAINTAINED: dict[str, object] = { - "materialized": "maintained", +_DYNAMIC: dict[str, object] = { + "materialized": "dynamic", "refresh": {"lag": "1h"}, "refresh_warehouse": "transform_wh", } @@ -845,27 +845,32 @@ def planned_entries(result: CompilePipelineResult) -> list[tuple[str, str]]: class ListingDuckDbAdapter(DuckDbAdapter): """DuckDB that lists every dynamic table the same way, as Snowflake's SHOW would.""" - def __init__(self, listed: MaintainedListing | None) -> None: + def __init__(self, listed: DynamicListing | None) -> None: super().__init__() - self.listed: MaintainedListing | None = listed + self.listed: DynamicListing | None = listed self.read_names: list[str] = [] - def maintained_listing( + def dynamic_listing( self, *, connection: Any, database: str | None, schema: str | None, name: str - ) -> MaintainedListing | None: + ) -> DynamicListing | None: del connection, database, schema self.read_names.append(name) return self.listed -def maintained_project_case(upstream_query: str) -> BuildExecutionPlanTestCase: - """Clean orders and the daily totals maintained from it, both refreshed within an hour.""" +def dynamic_project_case( + upstream_query: str, totals_config: dict[str, object] | None = None +) -> BuildExecutionPlanTestCase: + """Clean orders and dynamic daily totals built from it, both refreshed within an hour.""" return BuildExecutionPlanTestCase( - description="clean orders and the daily totals maintained from it", + description="clean orders and the daily totals built from it as a dynamic model", setup_sql=(), model_locations={"clean_orders": "staging", "daily_totals": "staging"}, - model_configs={"clean_orders": _MAINTAINED, "daily_totals": _MAINTAINED}, + model_configs={ + "clean_orders": _DYNAMIC, + "daily_totals": {**_DYNAMIC, **(totals_config or {})}, + }, model_queries={ "clean_orders": upstream_query, "daily_totals": "SELECT count(*) AS orders FROM staging.clean_orders", @@ -878,10 +883,10 @@ def maintained_project_case(upstream_query: str) -> BuildExecutionPlanTestCase: def change_tracking_project_case() -> BuildExecutionPlanTestCase: - """Daily totals is maintained from orders, and from payments through a view; not customers.""" + """Dynamic daily totals read orders, and payments through a view; not customers.""" return BuildExecutionPlanTestCase( - description="tables a maintained model reads, directly or via a view, and one it does not", + description="tables a dynamic model reads, directly or via a view, and one it does not", setup_sql=(), model_locations={ "orders": "staging", @@ -896,7 +901,7 @@ def change_tracking_project_case() -> BuildExecutionPlanTestCase: "payments": {"materialized": "table"}, "settled_payments": {"materialized": "view"}, "daily_totals": { - "materialized": "maintained", + "materialized": "dynamic", "refresh": {"lag": "1h"}, "refresh_on_build": True, }, diff --git a/tests/integration/src/sqlbuild/compiler/planner/main/test_maintained_refresh_plan.py b/tests/integration/src/sqlbuild/compiler/planner/main/test_dynamic_refresh_plan.py similarity index 67% rename from tests/integration/src/sqlbuild/compiler/planner/main/test_maintained_refresh_plan.py rename to tests/integration/src/sqlbuild/compiler/planner/main/test_dynamic_refresh_plan.py index 351c65275e..1ccff4c8e3 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/main/test_maintained_refresh_plan.py +++ b/tests/integration/src/sqlbuild/compiler/planner/main/test_dynamic_refresh_plan.py @@ -1,4 +1,4 @@ -"""A replan of a maintained model reads its live refresh settings and marks rebuilt upstreams.""" +"""A replan of a dynamic model reads its live refresh settings and marks rebuilt upstreams.""" from __future__ import annotations @@ -7,12 +7,12 @@ import pytest from sqlbuild._native import ModelPlanEntry, PlanAction, PlanOutput, PlanReason -from sqlbuild.adapter.contract.types import MaintainedListing +from sqlbuild.adapter.contract.types import DynamicListing from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.executor.run._helpers.reuse.fingerprinting import try_write_fingerprint from tests.integration.src.sqlbuild.compiler.planner.main._test_types import ( ChangeTrackingPlanTestCase, - MaintainedRefreshPlanTestCase, + DynamicRefreshPlanTestCase, ) from tests.integration.src.sqlbuild.compiler.planner.main.helpers import ( CLEAN_ORDERS_QUERY, @@ -20,60 +20,71 @@ build_execution_plan_from_kwargs, build_project_from_test_case, change_tracking_project_case, - maintained_project_case, + dynamic_project_case, ) @pytest.mark.parametrize( "test_case", [ - MaintainedRefreshPlanTestCase( + DynamicRefreshPlanTestCase( description="listed settings that match keep both models", - listed_settings=MaintainedListing("60 minutes", "TRANSFORM_WH", "ACTIVE"), + listed_settings=DynamicListing("60 minutes", "TRANSFORM_WH", "ACTIVE"), upstream_query=CLEAN_ORDERS_QUERY, - expected_upstream_action=PlanAction.KEEP_MAINTAINED, - expected_action=PlanAction.KEEP_MAINTAINED, + expected_upstream_action=PlanAction.KEEP_DYNAMIC, + expected_action=PlanAction.KEEP_DYNAMIC, expected_reason=PlanReason.NO_CHANGE, expected_reinitialised_by=None, expected_read_names=("clean_orders", "daily_totals"), ), - MaintainedRefreshPlanTestCase( + DynamicRefreshPlanTestCase( + description="writing out the default refresh_mode keeps the model", + listed_settings=DynamicListing("60 minutes", "TRANSFORM_WH", "ACTIVE"), + upstream_query=CLEAN_ORDERS_QUERY, + expected_upstream_action=PlanAction.KEEP_DYNAMIC, + expected_action=PlanAction.KEEP_DYNAMIC, + expected_reason=PlanReason.NO_CHANGE, + expected_reinitialised_by=None, + expected_read_names=("clean_orders", "daily_totals"), + totals_config={"refresh_mode": "incremental"}, + ), + DynamicRefreshPlanTestCase( description="a kept model listed as suspended is reported as paused", - listed_settings=MaintainedListing("60 minutes", "TRANSFORM_WH", "SUSPENDED"), + listed_settings=DynamicListing("60 minutes", "TRANSFORM_WH", "SUSPENDED"), upstream_query=CLEAN_ORDERS_QUERY, - expected_upstream_action=PlanAction.KEEP_MAINTAINED, - expected_action=PlanAction.KEEP_MAINTAINED, + expected_upstream_action=PlanAction.KEEP_DYNAMIC, + expected_action=PlanAction.KEEP_DYNAMIC, expected_reason=PlanReason.NO_CHANGE, expected_reinitialised_by=None, expected_read_names=("clean_orders", "daily_totals"), expected_paused=True, ), - MaintainedRefreshPlanTestCase( + DynamicRefreshPlanTestCase( description="a lag changed outside the project is altered in place", - listed_settings=MaintainedListing("2 hours", "TRANSFORM_WH", "ACTIVE"), + listed_settings=DynamicListing("2 hours", "TRANSFORM_WH", "ACTIVE"), upstream_query=CLEAN_ORDERS_QUERY, - expected_upstream_action=PlanAction.ALTER_MAINTAINED, - expected_action=PlanAction.ALTER_MAINTAINED, + expected_upstream_action=PlanAction.ALTER_DYNAMIC, + expected_action=PlanAction.ALTER_DYNAMIC, expected_reason=PlanReason.CONFIG_CHANGED, expected_reinitialised_by=None, expected_read_names=("clean_orders", "daily_totals"), ), - MaintainedRefreshPlanTestCase( + DynamicRefreshPlanTestCase( description="a name that holds no dynamic table is created", listed_settings=None, upstream_query=CLEAN_ORDERS_QUERY, - expected_upstream_action=PlanAction.CREATE_MAINTAINED, - expected_action=PlanAction.CREATE_MAINTAINED, + expected_upstream_action=PlanAction.CREATE_DYNAMIC, + expected_action=PlanAction.CREATE_DYNAMIC, expected_reason=PlanReason.FIRST_RUN, expected_reinitialised_by=None, expected_read_names=("clean_orders", "daily_totals"), ), - MaintainedRefreshPlanTestCase( + DynamicRefreshPlanTestCase( description="a recreated upstream marks the kept reader as recomputing in full", - listed_settings=MaintainedListing("1 hour", "TRANSFORM_WH", "ACTIVE"), + listed_settings=DynamicListing("1 hour", "TRANSFORM_WH", "ACTIVE"), upstream_query="SELECT 2 AS order_id", - expected_upstream_action=PlanAction.CREATE_MAINTAINED, - expected_action=PlanAction.KEEP_MAINTAINED, + expected_upstream_action=PlanAction.CREATE_DYNAMIC, + expected_action=PlanAction.KEEP_DYNAMIC, expected_reason=PlanReason.NO_CHANGE, expected_reinitialised_by="clean_orders", expected_read_names=("daily_totals",), @@ -81,14 +92,14 @@ ], ids=lambda case: case.description, ) -def test_given_built_maintained_models_when_replanning_then_live_settings_and_rebuilds_decide( - test_case: MaintainedRefreshPlanTestCase, +def test_given_built_dynamic_models_when_replanning_then_live_settings_and_rebuilds_decide( + test_case: DynamicRefreshPlanTestCase, ) -> None: adapter: ListingDuckDbAdapter = ListingDuckDbAdapter(test_case.listed_settings) connection: Any = adapter.connect({"database": ":memory:"}) connection.execute("CREATE SCHEMA staging") first: PlanOutput = build_execution_plan_from_kwargs( - project=build_project_from_test_case(maintained_project_case(CLEAN_ORDERS_QUERY)), + project=build_project_from_test_case(dynamic_project_case(CLEAN_ORDERS_QUERY)), adapter=adapter, connection=connection, ) @@ -107,7 +118,9 @@ def test_given_built_maintained_models_when_replanning_then_live_settings_and_re ) plan: PlanOutput = build_execution_plan_from_kwargs( - project=build_project_from_test_case(maintained_project_case(test_case.upstream_query)), + project=build_project_from_test_case( + dynamic_project_case(test_case.upstream_query, test_case.totals_config) + ), adapter=adapter, connection=connection, ) @@ -118,7 +131,7 @@ def test_given_built_maintained_models_when_replanning_then_live_settings_and_re assert entries["daily_totals"].action == test_case.expected_action assert entries["daily_totals"].reason == test_case.expected_reason assert entries["daily_totals"].reinitialised_by == test_case.expected_reinitialised_by - assert entries["daily_totals"].maintained_paused == test_case.expected_paused + assert entries["daily_totals"].dynamic_paused == test_case.expected_paused assert entries["clean_orders"].reinitialised_by is None assert entries["daily_totals"].target_lag == "'1 hour'" assert entries["daily_totals"].refresh_mode == "incremental" @@ -131,9 +144,9 @@ def test_given_built_maintained_models_when_replanning_then_live_settings_and_re "test_case", [ ChangeTrackingPlanTestCase( - description="a table a maintained model reads, directly or through a view, is tracked", + description="a table a dynamic model reads, directly or through a view, is tracked", select=(), - expected_maintained_readers={ + expected_dynamic_readers={ "orders": ("daily_totals",), "customers": (), "payments": ("daily_totals",), @@ -151,7 +164,7 @@ def test_given_built_maintained_models_when_replanning_then_live_settings_and_re ChangeTrackingPlanTestCase( description="the reader need not be selected for its upstream to be tracked", select=("orders", "customers", "payments"), - expected_maintained_readers={ + expected_dynamic_readers={ "orders": ("daily_totals",), "customers": (), "payments": ("daily_totals",), @@ -161,7 +174,7 @@ def test_given_built_maintained_models_when_replanning_then_live_settings_and_re ], ids=lambda case: case.description, ) -def test_given_table_read_by_maintained_model_when_planning_then_it_names_its_maintained_readers( +def test_given_table_read_by_dynamic_model_when_planning_then_it_names_its_dynamic_readers( test_case: ChangeTrackingPlanTestCase, adapter: DuckDbAdapter, connection: Any ) -> None: plan: PlanOutput = build_execution_plan_from_kwargs( @@ -172,9 +185,9 @@ def test_given_table_read_by_maintained_model_when_planning_then_it_names_its_ma ) readers: dict[str, tuple[str, ...]] = { - entry.name: entry.maintained_readers for entry in plan.model_entries + entry.name: entry.dynamic_readers for entry in plan.model_entries } - assert readers == test_case.expected_maintained_readers + assert readers == test_case.expected_dynamic_readers assert { entry.name: entry.refresh_on_build for entry in plan.model_entries } == test_case.expected_refresh_on_build diff --git a/tests/integration/src/sqlbuild/executor/dynamic_pauses/__init__.py b/tests/integration/src/sqlbuild/executor/dynamic_pauses/__init__.py new file mode 100644 index 0000000000..2097ead2ba --- /dev/null +++ b/tests/integration/src/sqlbuild/executor/dynamic_pauses/__init__.py @@ -0,0 +1 @@ +"""Dynamic model pause lease integration tests.""" diff --git a/tests/integration/src/sqlbuild/executor/maintained_pauses/_test_types.py b/tests/integration/src/sqlbuild/executor/dynamic_pauses/_test_types.py similarity index 66% rename from tests/integration/src/sqlbuild/executor/maintained_pauses/_test_types.py rename to tests/integration/src/sqlbuild/executor/dynamic_pauses/_test_types.py index 01cd40ee39..f6b867c312 100644 --- a/tests/integration/src/sqlbuild/executor/maintained_pauses/_test_types.py +++ b/tests/integration/src/sqlbuild/executor/dynamic_pauses/_test_types.py @@ -2,6 +2,8 @@ from dataclasses import dataclass +from sqlbuild.executor.dynamic_pauses.types import DynamicPauseEventType + @dataclass(frozen=True) class PauseLeaseTestCase: @@ -39,7 +41,7 @@ class HoldRefusalTestCase: @dataclass(frozen=True) class FirstBuildTestCase: description: str - pause_maintained: bool + pause_dynamic: bool expected_operations: tuple[tuple[str, str], ...] expected_states: dict[str, str] @@ -74,3 +76,33 @@ class HeartbeatTestCase: heartbeat_seconds: float wait_seconds: float expected_min_beats: int + + +@dataclass(frozen=True) +class CreatedModelHoldTestCase: + description: str + refused_events: tuple[DynamicPauseEventType, ...] + rival_closes: tuple[str, ...] + expected_release_operations: tuple[tuple[str, str], ...] + expected_code: str + expected_message_fragments: tuple[str, ...] + expected_states: dict[str, str] + expected_events: tuple[tuple[str, str, str, str], ...] + + +@dataclass(frozen=True) +class LateCommitTestCase: + description: str + expected_code: str + expected_rival: str + expected_states: dict[str, str] + expected_events: tuple[tuple[str, str, str, str], ...] + + +@dataclass(frozen=True) +class TakenOverPauseTestCase: + description: str + expected_code: str + expected_rival: str + expected_states: dict[str, str] + expected_events: tuple[tuple[str, str, str, str], ...] diff --git a/tests/integration/src/sqlbuild/executor/maintained_pauses/conftest.py b/tests/integration/src/sqlbuild/executor/dynamic_pauses/conftest.py similarity index 89% rename from tests/integration/src/sqlbuild/executor/maintained_pauses/conftest.py rename to tests/integration/src/sqlbuild/executor/dynamic_pauses/conftest.py index 1634ea5edb..a575688845 100644 --- a/tests/integration/src/sqlbuild/executor/maintained_pauses/conftest.py +++ b/tests/integration/src/sqlbuild/executor/dynamic_pauses/conftest.py @@ -8,7 +8,7 @@ import pytest -from tests.integration.src.sqlbuild.executor.maintained_pauses.helpers import ( +from tests.integration.src.sqlbuild.executor.dynamic_pauses.helpers import ( PausingDuckDbAdapter, ) diff --git a/tests/integration/src/sqlbuild/executor/maintained_pauses/helpers.py b/tests/integration/src/sqlbuild/executor/dynamic_pauses/helpers.py similarity index 79% rename from tests/integration/src/sqlbuild/executor/maintained_pauses/helpers.py rename to tests/integration/src/sqlbuild/executor/dynamic_pauses/helpers.py index f956db03e5..f430e65aef 100644 --- a/tests/integration/src/sqlbuild/executor/maintained_pauses/helpers.py +++ b/tests/integration/src/sqlbuild/executor/dynamic_pauses/helpers.py @@ -8,8 +8,8 @@ from sqlbuild._native import PlanOutput from sqlbuild.adapter.contract.types import ( - MaintainedListing, - MaintainedOperation, + DynamicListing, + DynamicOperation, TablePromotionMode, ) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter @@ -19,6 +19,7 @@ BuildExecutionResult, BuildRuntimeParams, ) +from sqlbuild.executor.dynamic_pauses.models import DynamicPauseEvent from tests.integration.src.sqlbuild.compiler.planner.main._test_types import ( BuildExecutionPlanTestCase, ) @@ -29,21 +30,21 @@ type LeaseProject = BuildExecutionPlanTestCase -_MAINTAINED: dict[str, object] = { - "materialized": "maintained", +_DYNAMIC: dict[str, object] = { + "materialized": "dynamic", "refresh": {"lag": "1h"}, "refresh_warehouse": "transform_wh", } _CREATED_STATE: str = "ACTIVE" TOTALS_QUERY: str = "SELECT count(*) AS orders FROM staging.orders" -_STATES: dict[MaintainedOperation, str] = { - MaintainedOperation.SUSPEND: "SUSPENDED", - MaintainedOperation.RESUME: "ACTIVE", +_STATES: dict[DynamicOperation, str] = { + DynamicOperation.SUSPEND: "SUSPENDED", + DynamicOperation.RESUME: "ACTIVE", } class PausingDuckDbAdapter(DuckDbAdapter): - """Builds maintained models as tables and keeps their scheduling state as Snowflake would.""" + """Builds dynamic models as tables and keeps their scheduling state as Snowflake would.""" def __init__(self) -> None: super().__init__() @@ -54,17 +55,17 @@ def session_warehouse(self, *, connection: Any) -> str | None: del connection return "SESSION_WH" - def maintained_listing( + def dynamic_listing( self, *, connection: Any, database: str | None, schema: str | None, name: str - ) -> MaintainedListing | None: + ) -> DynamicListing | None: del connection, database, schema - listings: dict[str, MaintainedListing] = { - listed: MaintainedListing("1 hour", "TRANSFORM_WH", state) + listings: dict[str, DynamicListing] = { + listed: DynamicListing("1 hour", "TRANSFORM_WH", state) for listed, state in self.states.items() } return listings.get(name) - def render_create_maintained( + def render_create_dynamic( self, *, destination: str, sql: str, target_lag: str, refresh_mode: str, warehouse: str ) -> tuple[str, ...]: del target_lag, refresh_mode, warehouse @@ -73,8 +74,8 @@ def render_create_maintained( self.operations.append(("create", name)) return (f"CREATE OR REPLACE TABLE {destination} AS {sql}",) - def render_maintained_operation( - self, *, destination: str, operation: MaintainedOperation + def render_dynamic_operation( + self, *, destination: str, operation: DynamicOperation ) -> tuple[str, ...]: name: str = destination.rsplit(".", 1)[-1] self.states[name] = _STATES.get(operation, self.states[name]) @@ -89,16 +90,16 @@ def lease_project_case( refresh_on_build: bool, totals_query: str = TOTALS_QUERY, ) -> BuildExecutionPlanTestCase: - """Orders, daily totals maintained from them, and any report table located to read those.""" + """Orders, dynamic daily totals built from them, and any report table that reads those.""" - maintained: dict[str, object] = {**_MAINTAINED, "refresh_on_build": refresh_on_build} + dynamic: dict[str, object] = {**_DYNAMIC, "refresh_on_build": refresh_on_build} return BuildExecutionPlanTestCase( - description="orders and the daily totals maintained from them", + description="orders and the daily totals built from them as a dynamic model", setup_sql=(), model_locations={"orders": "staging", "daily_totals": "staging", **report_locations}, model_configs={ "orders": {"materialized": "table"}, - "daily_totals": maintained, + "daily_totals": dynamic, "order_report": {"materialized": "table"}, }, model_queries={ @@ -161,7 +162,7 @@ def pause_events(*, connection: Any) -> list[tuple[str, str, str, str]]: rows: list[tuple[Any, ...]] = connection.execute( "SELECT event_type, run_id, recorded_by, relation_name " - "FROM staging._sqlbuild_maintained_pauses WHERE event_type <> 'heartbeat' " + "FROM staging._sqlbuild_dynamic_pauses WHERE event_type <> 'heartbeat' " "ORDER BY 1, 2, 3, 4" ).fetchall() return [(str(row[0]), str(row[1]), str(row[2]), str(row[3])) for row in rows] @@ -172,5 +173,12 @@ def pause_table_names(*, connection: Any) -> list[tuple[Any, ...]]: return connection.execute( "SELECT table_name FROM information_schema.tables " - "WHERE table_schema = 'staging' AND table_name = '_sqlbuild_maintained_pauses'" + "WHERE table_schema = 'staging' AND table_name = '_sqlbuild_dynamic_pauses'" ).fetchall() + + +def refuse_insert(*, connection: Any, events: tuple[DynamicPauseEvent, ...]) -> None: + """Stand in for a pause-table insert the warehouse refuses.""" + + del connection, events + raise RuntimeError("pause table is read-only") diff --git a/tests/integration/src/sqlbuild/executor/maintained_pauses/test_maintained_pause_lease.py b/tests/integration/src/sqlbuild/executor/dynamic_pauses/test_dynamic_pause_lease.py similarity index 60% rename from tests/integration/src/sqlbuild/executor/maintained_pauses/test_maintained_pause_lease.py rename to tests/integration/src/sqlbuild/executor/dynamic_pauses/test_dynamic_pause_lease.py index 0bb8cbd9e6..ebb12aa730 100644 --- a/tests/integration/src/sqlbuild/executor/maintained_pauses/test_maintained_pause_lease.py +++ b/tests/integration/src/sqlbuild/executor/dynamic_pauses/test_dynamic_pause_lease.py @@ -1,4 +1,4 @@ -"""A build pauses the maintained models it touches, records its hold, and resumes them at the end.""" +"""A build pauses the dynamic models it touches, records its hold, and resumes them at the end.""" from __future__ import annotations @@ -17,21 +17,27 @@ BuildRuntimeParams, ) from sqlbuild.executor.build.types import BuildStatus -from sqlbuild.executor.maintained_pauses.classes.maintained_hold import MaintainedHold -from sqlbuild.executor.maintained_pauses.main.hold_maintained_models import ( - hold_maintained_models, +from sqlbuild.executor.dynamic_pauses._helpers.sql import insert_sql +from sqlbuild.executor.dynamic_pauses.classes.dynamic_hold import DynamicHold +from sqlbuild.executor.dynamic_pauses.main.hold_dynamic_models import ( + hold_dynamic_models, ) -from tests.integration.src.sqlbuild.executor.maintained_pauses._test_types import ( +from sqlbuild.executor.dynamic_pauses.models import DynamicPauseEvent +from sqlbuild.executor.dynamic_pauses.types import DynamicPauseEventType, StateLocation +from tests.integration.src.sqlbuild.executor.dynamic_pauses._test_types import ( ConcurrentHoldTestCase, + CreatedModelHoldTestCase, FirstBuildTestCase, HeartbeatTestCase, HoldRefusalTestCase, HoldTakeoverTestCase, + LateCommitTestCase, LifecycleLineTestCase, PauseLeaseTestCase, + TakenOverPauseTestCase, TakenOverReleaseTestCase, ) -from tests.integration.src.sqlbuild.executor.maintained_pauses.helpers import ( +from tests.integration.src.sqlbuild.executor.dynamic_pauses.helpers import ( TOTALS_QUERY, LeaseProject, PausingDuckDbAdapter, @@ -40,6 +46,7 @@ pause_events, pause_table_names, plan_lease_project, + refuse_insert, ) _ORDERS: str = "SELECT 1 AS order_id" @@ -68,7 +75,7 @@ [ FirstBuildTestCase( description="a created model is paused for the build and resumed at its end", - pause_maintained=False, + pause_dynamic=False, expected_operations=( ("create", "daily_totals"), ("suspend", "daily_totals"), @@ -79,7 +86,7 @@ ], ids=lambda case: case.description, ) -def test_given_first_build_when_creating_maintained_model_then_pauses_it_until_the_end( +def test_given_first_build_when_creating_dynamic_model_then_pauses_it_until_the_end( test_case: FirstBuildTestCase, adapter: PausingDuckDbAdapter, connection: Any, @@ -89,7 +96,7 @@ def test_given_first_build_when_creating_maintained_model_then_pauses_it_until_t adapter=adapter, connection=connection, case=lease_project_case(orders_query=_ORDERS, report_locations={}, refresh_on_build=False), - runtime=BuildRuntimeParams(run_id="first_run", pause_maintained=test_case.pause_maintained), + runtime=BuildRuntimeParams(run_id="first_run", pause_dynamic=test_case.pause_dynamic), database_path=database_path, ) @@ -103,7 +110,7 @@ def test_given_first_build_when_creating_maintained_model_then_pauses_it_until_t "test_case", [ PauseLeaseTestCase( - description="a build of the maintained model alone writes nothing it reads", + description="a build of the dynamic model alone writes nothing it reads", second_select=("daily_totals",), second_orders_query=_ORDERS, second_totals_query=TOTALS_QUERY, @@ -179,7 +186,7 @@ def test_given_first_build_when_creating_maintained_model_then_pauses_it_until_t ], ids=lambda case: case.description, ) -def test_given_built_maintained_model_when_building_again_then_pauses_only_what_it_touches( +def test_given_built_dynamic_model_when_building_again_then_pauses_only_what_it_touches( test_case: PauseLeaseTestCase, adapter: PausingDuckDbAdapter, connection: Any, @@ -222,22 +229,22 @@ def test_given_built_maintained_model_when_building_again_then_pauses_only_what_ "test_case", [ FirstBuildTestCase( - description="a pause_maintained target leaves models suspended and records nothing", - pause_maintained=True, + description="a pause_dynamic target leaves models suspended and records nothing", + pause_dynamic=True, expected_operations=(("refresh", "daily_totals"),), expected_states={"daily_totals": "SUSPENDED"}, ), ], ids=lambda case: case.description, ) -def test_given_pause_maintained_target_when_building_then_models_stay_suspended_and_refresh( +def test_given_pause_dynamic_target_when_building_then_models_stay_suspended_and_refresh( test_case: FirstBuildTestCase, adapter: PausingDuckDbAdapter, connection: Any, database_path: Path, ) -> None: paused: BuildRuntimeParams = BuildRuntimeParams( - run_id="first_run", pause_maintained=test_case.pause_maintained + run_id="first_run", pause_dynamic=test_case.pause_dynamic ) _ = build_lease_project( adapter=adapter, @@ -278,7 +285,7 @@ def test_given_pause_maintained_target_when_building_then_models_stay_suspended_ expected_events=_TAKEN_OVER, ), HoldTakeoverTestCase( - description="--ignore-maintained-grace takes over a live hold when the project allows", + description="--ignore-dynamic-grace takes over a live hold when the project allows", grace_seconds=1800, ignore_grace=True, allow_ignore_grace=True, @@ -304,7 +311,7 @@ def test_given_hold_of_a_stopped_build_when_building_then_takes_it_over_and_resu changed: LeaseProject = lease_project_case( orders_query=_MORE_ORDERS, report_locations={}, refresh_on_build=False ) - dead: MaintainedHold | None = hold_maintained_models( + dead: DynamicHold | None = hold_dynamic_models( plan=plan_lease_project(adapter=adapter, connection=connection, case=changed), adapter=adapter, runtime=BuildRuntimeParams(run_id="dead_run"), @@ -320,9 +327,9 @@ def test_given_hold_of_a_stopped_build_when_building_then_takes_it_over_and_resu case=changed, runtime=BuildRuntimeParams( run_id="second_run", - maintained_heartbeat_grace_seconds=test_case.grace_seconds, - ignore_maintained_grace=test_case.ignore_grace, - allow_ignore_maintained_grace=test_case.allow_ignore_grace, + dynamic_heartbeat_grace_seconds=test_case.grace_seconds, + ignore_dynamic_grace=test_case.ignore_grace, + allow_ignore_dynamic_grace=test_case.allow_ignore_grace, ), database_path=database_path, ) @@ -345,7 +352,7 @@ def test_given_hold_of_a_stopped_build_when_building_then_takes_it_over_and_resu expected_message_fragment="dead_run", ), HoldRefusalTestCase( - description="--ignore-maintained-grace needs allow_ignore_grace", + description="--ignore-dynamic-grace needs allow_ignore_grace", grace_seconds=1800, ignore_grace=True, allow_ignore_grace=False, @@ -371,7 +378,7 @@ def test_given_live_hold_when_building_then_stops_before_any_change( changed: LeaseProject = lease_project_case( orders_query=_MORE_ORDERS, report_locations={}, refresh_on_build=False ) - dead: MaintainedHold | None = hold_maintained_models( + dead: DynamicHold | None = hold_dynamic_models( plan=plan_lease_project(adapter=adapter, connection=connection, case=changed), adapter=adapter, runtime=BuildRuntimeParams(run_id="dead_run"), @@ -388,9 +395,9 @@ def test_given_live_hold_when_building_then_stops_before_any_change( case=changed, runtime=BuildRuntimeParams( run_id="second_run", - maintained_heartbeat_grace_seconds=test_case.grace_seconds, - ignore_maintained_grace=test_case.ignore_grace, - allow_ignore_maintained_grace=test_case.allow_ignore_grace, + dynamic_heartbeat_grace_seconds=test_case.grace_seconds, + ignore_dynamic_grace=test_case.ignore_grace, + allow_ignore_dynamic_grace=test_case.allow_ignore_grace, ), database_path=database_path, ) @@ -435,7 +442,7 @@ def test_given_held_models_when_build_runs_then_heartbeat_writes_on_its_own_conn changed: LeaseProject = lease_project_case( orders_query=_MORE_ORDERS, report_locations={}, refresh_on_build=False ) - hold: MaintainedHold | None = hold_maintained_models( + hold: DynamicHold | None = hold_dynamic_models( plan=plan_lease_project(adapter=adapter, connection=connection, case=changed), adapter=adapter, runtime=BuildRuntimeParams(run_id="beating_run"), @@ -449,7 +456,7 @@ def test_given_held_models_when_build_runs_then_heartbeat_writes_on_its_own_conn hold.release(connection=connection) beats: list[tuple[Any, ...]] = connection.execute( - "SELECT count(*) FROM staging._sqlbuild_maintained_pauses " + "SELECT count(*) FROM staging._sqlbuild_dynamic_pauses " "WHERE event_type = 'heartbeat' AND run_id = 'beating_run'" ).fetchall() assert beats[0][0] >= test_case.expected_min_beats @@ -462,8 +469,7 @@ def test_given_held_models_when_build_runs_then_heartbeat_writes_on_its_own_conn description="a model suspended outside SQLBuild is reported on the build's output", states_before_second={"daily_totals": "SUSPENDED"}, expected_line=( - "Warning: maintained model daily_totals was paused outside SQLBuild; " - "it stays paused" + "Warning: dynamic model daily_totals was paused outside SQLBuild; it stays paused" ), ), ], @@ -535,16 +541,16 @@ def test_given_hold_taken_over_when_first_build_releases_then_it_resumes_nothing orders_query=_MORE_ORDERS, report_locations={}, refresh_on_build=False ) plan: PlanOutput = plan_lease_project(adapter=adapter, connection=connection, case=changed) - holds: list[MaintainedHold] = [] + holds: list[DynamicHold] = [] for runtime in ( BuildRuntimeParams(run_id="slow_run"), BuildRuntimeParams( run_id="second_run", - ignore_maintained_grace=True, - allow_ignore_maintained_grace=True, + ignore_dynamic_grace=True, + allow_ignore_dynamic_grace=True, ), ): - hold: MaintainedHold | None = hold_maintained_models( + hold: DynamicHold | None = hold_dynamic_models( plan=plan, adapter=adapter, runtime=runtime, @@ -602,13 +608,13 @@ def test_given_two_builds_pausing_together_when_both_record_a_pause_then_only_th orders_query=_MORE_ORDERS, report_locations={}, refresh_on_build=False ), ) - early: MaintainedHold | None = hold_maintained_models( + early: DynamicHold | None = hold_dynamic_models( plan=plan, adapter=adapter, runtime=BuildRuntimeParams(run_id="early_run"), connection_config={"database": str(database_path)}, ) - late: MaintainedHold | None = hold_maintained_models( + late: DynamicHold | None = hold_dynamic_models( plan=plan, adapter=adapter, runtime=BuildRuntimeParams(run_id="late_run"), @@ -636,3 +642,280 @@ def state_then_rival_acquires(*, connection: Any, name: str) -> str | None: assert adapter.states == test_case.expected_states assert pause_events(connection=connection) == sorted(test_case.expected_events) early.release(connection=connection) + + +@pytest.mark.parametrize( + "test_case", + [ + CreatedModelHoldTestCase( + description="of two builds creating a model, the later one backs out", + refused_events=(), + rival_closes=(), + expected_release_operations=(), + expected_code="A718", + expected_message_fragments=("early_run",), + expected_states={"daily_totals": "SUSPENDED"}, + expected_events=( + ("paused", "early_run", "early_run", "daily_totals"), + ("paused", "late_run", "late_run", "daily_totals"), + ("resumed", "late_run", "late_run", "daily_totals"), + ), + ), + CreatedModelHoldTestCase( + description="a failed close of the later pause is reported with the hold", + refused_events=(DynamicPauseEventType.RESUMED,), + rival_closes=(), + expected_release_operations=(), + expected_code="A718", + expected_message_fragments=("early_run", "pause table is read-only"), + expected_states={"daily_totals": "SUSPENDED"}, + expected_events=( + ("paused", "early_run", "early_run", "daily_totals"), + ("paused", "late_run", "late_run", "daily_totals"), + ), + ), + CreatedModelHoldTestCase( + description="when the earlier build backs out too, the later one resumes the model", + refused_events=(), + rival_closes=("daily_totals",), + expected_release_operations=(("resume", "daily_totals"),), + expected_code="A718", + expected_message_fragments=("early_run",), + expected_states={"daily_totals": "ACTIVE"}, + expected_events=( + ("paused", "early_run", "early_run", "daily_totals"), + ("paused", "late_run", "late_run", "daily_totals"), + ("resumed", "early_run", "early_run", "daily_totals"), + ("resumed", "late_run", "late_run", "daily_totals"), + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_two_builds_creating_a_model_when_both_pause_it_then_only_the_earliest_holds( + test_case: CreatedModelHoldTestCase, + adapter: PausingDuckDbAdapter, + connection: Any, + database_path: Path, +) -> None: + plan: PlanOutput = plan_lease_project( + adapter=adapter, + connection=connection, + case=lease_project_case(orders_query=_ORDERS, report_locations={}, refresh_on_build=False), + ) + holds: dict[str, DynamicHold] = {} + for run_id in ("early_run", "late_run"): + hold: DynamicHold | None = hold_dynamic_models( + plan=plan, + adapter=adapter, + runtime=BuildRuntimeParams(run_id=run_id), + connection_config={"database": str(database_path)}, + ) + assert hold is not None + hold.acquire(connection=connection, on_progress=None) + holds[run_id] = hold + late: DynamicHold = holds["late_run"] + recorders: dict[tuple[DynamicPauseEventType, ...], Callable[..., None]] = { + (): late._record, + **{(event_type,): late._record for event_type in DynamicPauseEventType}, + **{(event_type,): refuse_insert for event_type in test_case.refused_events}, + } + + def record_or_refuse(*, connection: Any, events: tuple[DynamicPauseEvent, ...]) -> None: + recorders[tuple(event.event_type for event in events)](connection=connection, events=events) + + late._record = record_or_refuse # ty: ignore[invalid-assignment] + adapter.states["daily_totals"] = "ACTIVE" + holds["early_run"].after_create(connection=connection, name="daily_totals") + adapter.states["daily_totals"] = "ACTIVE" + + with pytest.raises(AdapterUserError) as raised: + late.after_create(connection=connection, name="daily_totals") + early: DynamicHold = holds["early_run"] + early._record( + connection=connection, + events=tuple( + early._own(event_type=DynamicPauseEventType.RESUMED, name=name) + for name in test_case.rival_closes + ), + ) + adapter.operations.clear() + late.release(connection=connection) + + assert raised.value.code == test_case.expected_code + for fragment in test_case.expected_message_fragments: + assert fragment in str(raised.value) + assert tuple(adapter.operations) == test_case.expected_release_operations + assert adapter.states == test_case.expected_states + assert pause_events(connection=connection) == sorted(test_case.expected_events) + holds["early_run"].release(connection=connection) + + +@pytest.mark.parametrize( + "test_case", + [ + LateCommitTestCase( + description="a build whose pause commits after a rival read the table backs out", + expected_code="A718", + expected_rival="late_run", + expected_states={"daily_totals": "SUSPENDED"}, + expected_events=( + ("paused", "early_run", "early_run", "daily_totals"), + ("paused", "first_run", "first_run", "daily_totals"), + ("paused", "late_run", "late_run", "daily_totals"), + ("resumed", "early_run", "early_run", "daily_totals"), + ("resumed", "first_run", "first_run", "daily_totals"), + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_pause_committing_late_when_rival_paused_meanwhile_then_the_late_one_backs_out( + test_case: LateCommitTestCase, + adapter: PausingDuckDbAdapter, + connection: Any, + database_path: Path, +) -> None: + _ = build_lease_project( + adapter=adapter, + connection=connection, + case=lease_project_case(orders_query=_ORDERS, report_locations={}, refresh_on_build=False), + runtime=BuildRuntimeParams(run_id="first_run"), + database_path=database_path, + ) + plan: PlanOutput = plan_lease_project( + adapter=adapter, + connection=connection, + case=lease_project_case( + orders_query=_MORE_ORDERS, report_locations={}, refresh_on_build=False + ), + ) + holds: dict[str, DynamicHold] = {} + for run_id in ("early_run", "late_run"): + hold: DynamicHold | None = hold_dynamic_models( + plan=plan, + adapter=adapter, + runtime=BuildRuntimeParams(run_id=run_id), + connection_config={"database": str(database_path)}, + ) + assert hold is not None + holds[run_id] = hold + early: DynamicHold = holds["early_run"] + location: StateLocation | None = early._state_location + assert location is not None + now: str = adapter.render_current_utc_timestamp() + + def commit_after_rival(*, connection: Any, events: tuple[DynamicPauseEvent, ...]) -> None: + started: Any = connection.execute(f"SELECT {now}").fetchone()[0] + holds["late_run"].acquire(connection=connection, on_progress=None) + adapter.execute( + connection=connection, + sql=insert_sql(adapter=adapter, location=location, events=events).replace( + now, f"TIMESTAMP '{started.isoformat()}'" + ), + ) + + recorders: dict[tuple[DynamicPauseEventType, ...], Callable[..., None]] = { + (): early._record, + (DynamicPauseEventType.PAUSED,): commit_after_rival, + (DynamicPauseEventType.RESUMED,): early._record, + } + + def record_committing_late(*, connection: Any, events: tuple[DynamicPauseEvent, ...]) -> None: + recorders[tuple(event.event_type for event in events)](connection=connection, events=events) + + early._record = record_committing_late # ty: ignore[invalid-assignment] + + with pytest.raises(AdapterUserError) as raised: + early.acquire(connection=connection, on_progress=None) + + assert raised.value.code == test_case.expected_code + assert test_case.expected_rival in str(raised.value) + assert adapter.states == test_case.expected_states + assert pause_events(connection=connection) == sorted(test_case.expected_events) + holds["late_run"].release(connection=connection) + + +@pytest.mark.parametrize( + "test_case", + [ + TakenOverPauseTestCase( + description="a build whose pause is taken over before it reads back backs out", + expected_code="A718", + expected_rival="taker_run", + expected_states={"daily_totals": "SUSPENDED"}, + expected_events=( + ("paused", "early_run", "early_run", "daily_totals"), + ("paused", "first_run", "first_run", "daily_totals"), + ("paused", "taker_run", "taker_run", "daily_totals"), + ("resumed", "early_run", "early_run", "daily_totals"), + ("resumed", "early_run", "taker_run", "daily_totals"), + ("resumed", "first_run", "first_run", "daily_totals"), + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_pause_taken_over_before_read_back_when_acquiring_then_the_build_backs_out( + test_case: TakenOverPauseTestCase, + adapter: PausingDuckDbAdapter, + connection: Any, + database_path: Path, +) -> None: + _ = build_lease_project( + adapter=adapter, + connection=connection, + case=lease_project_case(orders_query=_ORDERS, report_locations={}, refresh_on_build=False), + runtime=BuildRuntimeParams(run_id="first_run"), + database_path=database_path, + ) + plan: PlanOutput = plan_lease_project( + adapter=adapter, + connection=connection, + case=lease_project_case( + orders_query=_MORE_ORDERS, report_locations={}, refresh_on_build=False + ), + ) + holds: dict[str, DynamicHold] = {} + for runtime in ( + BuildRuntimeParams(run_id="early_run"), + BuildRuntimeParams( + run_id="taker_run", ignore_dynamic_grace=True, allow_ignore_dynamic_grace=True + ), + ): + hold: DynamicHold | None = hold_dynamic_models( + plan=plan, + adapter=adapter, + runtime=runtime, + connection_config={"database": str(database_path)}, + ) + assert hold is not None + holds[runtime.run_id] = hold + early: DynamicHold = holds["early_run"] + record: Callable[..., None] = early._record + + def record_then_rival_takes_over( + *, connection: Any, events: tuple[DynamicPauseEvent, ...] + ) -> None: + record(connection=connection, events=events) + holds["taker_run"].acquire(connection=connection, on_progress=None) + + recorders: dict[tuple[DynamicPauseEventType, ...], Callable[..., None]] = { + (): record, + (DynamicPauseEventType.PAUSED,): record_then_rival_takes_over, + (DynamicPauseEventType.RESUMED,): record, + } + + def record_or_take_over(*, connection: Any, events: tuple[DynamicPauseEvent, ...]) -> None: + recorders[tuple(event.event_type for event in events)](connection=connection, events=events) + + early._record = record_or_take_over # ty: ignore[invalid-assignment] + + with pytest.raises(AdapterUserError) as raised: + early.acquire(connection=connection, on_progress=None) + + assert raised.value.code == test_case.expected_code + assert test_case.expected_rival in str(raised.value) + assert adapter.states == test_case.expected_states + assert pause_events(connection=connection) == sorted(test_case.expected_events) + holds["taker_run"].release(connection=connection) diff --git a/tests/integration/src/sqlbuild/executor/maintained_pauses/__init__.py b/tests/integration/src/sqlbuild/executor/maintained_pauses/__init__.py deleted file mode 100644 index 11f5a11f26..0000000000 --- a/tests/integration/src/sqlbuild/executor/maintained_pauses/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Maintained model pause lease integration tests.""" diff --git a/tests/integration/src/sqlbuild/executor/run/maintained/__init__.py b/tests/integration/src/sqlbuild/executor/run/dynamic/__init__.py similarity index 100% rename from tests/integration/src/sqlbuild/executor/run/maintained/__init__.py rename to tests/integration/src/sqlbuild/executor/run/dynamic/__init__.py diff --git a/tests/integration/src/sqlbuild/executor/run/maintained/_test_types.py b/tests/integration/src/sqlbuild/executor/run/dynamic/_test_types.py similarity index 74% rename from tests/integration/src/sqlbuild/executor/run/maintained/_test_types.py rename to tests/integration/src/sqlbuild/executor/run/dynamic/_test_types.py index 3958be16a0..5d3a14525c 100644 --- a/tests/integration/src/sqlbuild/executor/run/maintained/_test_types.py +++ b/tests/integration/src/sqlbuild/executor/run/dynamic/_test_types.py @@ -3,11 +3,11 @@ from dataclasses import dataclass from sqlbuild._native import PlanAction -from sqlbuild.adapter.contract.types import MaintainedListing, TablePromotionMode +from sqlbuild.adapter.contract.types import DynamicListing, TablePromotionMode @dataclass(frozen=True) -class MaintainedSuccessTestCase: +class DynamicSuccessTestCase: description: str action: PlanAction setup_sql: tuple[str, ...] @@ -18,11 +18,11 @@ class MaintainedSuccessTestCase: @dataclass(frozen=True) -class MaintainedFailureTestCase: +class DynamicFailureTestCase: description: str setup_sql: tuple[str, ...] session_warehouse: str | None - listed_settings: MaintainedListing | None + listed_settings: DynamicListing | None failing_query: str | None expected_error_code: str expected_listed_types: tuple[tuple[str], ...] @@ -42,7 +42,14 @@ class FailedRestoreTestCase: @dataclass(frozen=True) class ChangeTrackingTableTestCase: description: str - maintained_readers: tuple[str, ...] + dynamic_readers: tuple[str, ...] promotion_mode: TablePromotionMode declared_columns: tuple[tuple[str, str], ...] expected_tracking: tuple[bool, ...] + + +@dataclass(frozen=True) +class ChangeTrackingSnapshotTestCase: + description: str + dynamic_readers: tuple[str, ...] + expected_tracked: tuple[str, ...] diff --git a/tests/integration/src/sqlbuild/executor/run/maintained/helpers.py b/tests/integration/src/sqlbuild/executor/run/dynamic/helpers.py similarity index 80% rename from tests/integration/src/sqlbuild/executor/run/maintained/helpers.py rename to tests/integration/src/sqlbuild/executor/run/dynamic/helpers.py index 1f2a9d098b..146f816ebe 100644 --- a/tests/integration/src/sqlbuild/executor/run/maintained/helpers.py +++ b/tests/integration/src/sqlbuild/executor/run/dynamic/helpers.py @@ -1,4 +1,4 @@ -"""A DuckDB adapter that builds maintained models as tables, for executor lifecycle tests.""" +"""A DuckDB adapter that builds dynamic models as tables, for executor lifecycle tests.""" from __future__ import annotations @@ -15,27 +15,27 @@ PlanReason, ) from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder -from sqlbuild.adapter.contract.types import MaintainedListing +from sqlbuild.adapter.contract.types import DynamicListing from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.executor.run.models import ModelMaterializationContext DAILY_TOTALS_SQL: str = "SELECT 3 AS orders" -class MaintainedDuckDbAdapter(DuckDbAdapter): - """Builds a maintained model as a table that, like a dynamic table, replaces nothing else.""" +class DynamicDuckDbAdapter(DuckDbAdapter): + """Builds a dynamic model as a table that, like a dynamic table, replaces nothing else.""" def __init__( self, *, session_warehouse: str | None, - listed_settings: MaintainedListing | None = None, + listed_settings: DynamicListing | None = None, failing_query: str | None = None, failing_renames: dict[str, str] | None = None, ) -> None: super().__init__() self.warehouse: str | None = session_warehouse - self.listed_settings: MaintainedListing | None = listed_settings + self.listed_settings: DynamicListing | None = listed_settings self.failing_query: str | None = failing_query self.failing_renames: dict[str, str] = dict(failing_renames or {}) self.created: list[tuple[str, str, str]] = [] @@ -45,13 +45,13 @@ def session_warehouse(self, *, connection: Any) -> str | None: del connection return self.warehouse - def maintained_listing( + def dynamic_listing( self, *, connection: Any, database: str | None, schema: str | None, name: str - ) -> MaintainedListing | None: + ) -> DynamicListing | None: del connection, database, schema, name return self.listed_settings - def render_create_maintained( + def render_create_dynamic( self, *, destination: str, sql: str, target_lag: str, refresh_mode: str, warehouse: str ) -> tuple[str, ...]: self.created.append((target_lag, refresh_mode, warehouse)) @@ -64,7 +64,7 @@ def render_rename(self, *, origin: str, destination: str) -> tuple[str, ...]: def render_rename_view(self, *, origin: str, destination: str) -> tuple[str, ...]: return (f"ALTER VIEW {origin} RENAME TO {destination.split('.')[-1]}",) - def render_alter_maintained( + def render_alter_dynamic( self, *, destination: str, target_lag: str, warehouse: str | None ) -> tuple[str, ...]: self.altered.append((target_lag, warehouse)) @@ -72,20 +72,20 @@ def render_alter_maintained( return (f"COMMENT ON TABLE {destination} IS 'refreshed within {lag}'",) -def maintained_context( +def dynamic_context( *, - adapter: MaintainedDuckDbAdapter, + adapter: DynamicDuckDbAdapter, connection: Any, action: PlanAction, refresh_warehouse: str | None, ) -> ModelMaterializationContext: - """Daily totals, maintained within an hour, planned with `action`.""" + """Daily totals, dynamic within an hour, planned with `action`.""" entry: ModelPlanEntry = ModelPlanEntry( key=CompiledObjectKey(resource_type=CompiledResourceType.MODEL, name="daily_totals"), name="daily_totals", relative_path=Path("models/daily_totals.sql"), - materialization_type=MaterializationType.MAINTAINED, + materialization_type=MaterializationType.DYNAMIC, action=action, reason=PlanReason.FIRST_RUN, destination=CompiledRelationLocation( @@ -110,7 +110,7 @@ def maintained_context( source_map={}, python_source_read_map={}, model_audits=(), - run_id="maintained_run", + run_id="dynamic_run", query_change_tracking=True, ) @@ -148,3 +148,15 @@ def create_table_as( config=config, statement_recorder=statement_recorder, ) + + +class TrackedSnapshotDuckDbAdapter(DuckDbAdapter): + """Records each table it is asked to track changes on after creating it.""" + + def __init__(self) -> None: + super().__init__() + self.tracked: list[str] = [] + + def render_track_changes(self, *, destination: str) -> tuple[str, ...]: + self.tracked.append(destination) + return () diff --git a/tests/integration/src/sqlbuild/executor/run/dynamic/test_change_tracking.py b/tests/integration/src/sqlbuild/executor/run/dynamic/test_change_tracking.py new file mode 100644 index 0000000000..516ca26749 --- /dev/null +++ b/tests/integration/src/sqlbuild/executor/run/dynamic/test_change_tracking.py @@ -0,0 +1,146 @@ +"""A table a dynamic model reads is created with change tracking on every build path.""" + +from __future__ import annotations + +from dataclasses import replace +from typing import Any + +import pytest + +from sqlbuild._native import ModelPlanEntry, PlanReason +from sqlbuild.adapter.contract.types import TablePromotionMode +from sqlbuild.executor.run._helpers.materializations.snapshot import execute_snapshot_entry +from sqlbuild.executor.run.main._execute import execute_table_entry +from sqlbuild.executor.run.models import ModelExecutionResult, ModelMaterializationContext +from sqlbuild.executor.scheduling.types import ExecutionStatus +from tests.integration.src.sqlbuild.executor.run.dynamic._test_types import ( + ChangeTrackingSnapshotTestCase, + ChangeTrackingTableTestCase, +) +from tests.integration.src.sqlbuild.executor.run.dynamic.helpers import ( + TrackedSnapshotDuckDbAdapter, + TrackingDuckDbAdapter, +) +from tests.integration.src.sqlbuild.executor.run.helpers import ( + build_declared_columns, + build_table_plan_entry, +) +from tests.unit.src.sqlbuild.executor.run._helpers.helpers import ( + build_snapshot_execution_plan_entry, +) + + +@pytest.mark.parametrize( + "test_case", + [ + ChangeTrackingTableTestCase( + description="a staged build tracks changes on the stage it swaps in", + dynamic_readers=("daily_totals",), + promotion_mode=TablePromotionMode.STAGED, + declared_columns=(), + expected_tracking=(True,), + ), + ChangeTrackingTableTestCase( + description="type enforcement rebuilds the stage with change tracking too", + dynamic_readers=("daily_totals",), + promotion_mode=TablePromotionMode.STAGED, + declared_columns=(("order_id", "BIGINT"),), + expected_tracking=(True, True), + ), + ChangeTrackingTableTestCase( + description="a table no dynamic model reads is created as before", + dynamic_readers=(), + promotion_mode=TablePromotionMode.STAGED, + declared_columns=(("order_id", "BIGINT"),), + expected_tracking=(False, False), + ), + ], + ids=lambda case: case.description, +) +def test_given_table_read_by_dynamic_model_when_building_then_every_create_tracks_changes( + test_case: ChangeTrackingTableTestCase, connection: Any +) -> None: + adapter: TrackingDuckDbAdapter = TrackingDuckDbAdapter() + entry: ModelPlanEntry = replace( + build_table_plan_entry( + name="orders", + sql="SELECT 1::INTEGER AS order_id", + target_schema="staging", + target_name="orders", + type_enforcement=bool(test_case.declared_columns), + ), + dynamic_readers=test_case.dynamic_readers, + ) + + result: ModelExecutionResult = execute_table_entry( + context=ModelMaterializationContext( + entry=entry, + adapter=adapter, + connection=connection, + model_locations={}, + seed_locations={}, + source_map={}, + python_source_read_map={}, + model_audits=(), + run_id="tracking_run", + query_change_tracking=False, + ), + declared_columns=build_declared_columns(test_case.declared_columns), + promotion_mode=test_case.promotion_mode, + ) + + assert result.status == ExecutionStatus.SUCCESS, result.error_message + assert tuple(adapter.tracking) == test_case.expected_tracking + + +@pytest.mark.parametrize( + "test_case", + [ + ChangeTrackingSnapshotTestCase( + description="a snapshot's first build and its full rebuild both track changes", + dynamic_readers=("daily_totals",), + expected_tracked=( + "main.customer_snapshot", + "main.__sqb_rebuild__customer_snapshot", + ), + ), + ChangeTrackingSnapshotTestCase( + description="a snapshot no dynamic model reads is built as before", + dynamic_readers=(), + expected_tracked=(), + ), + ], + ids=lambda case: case.description, +) +def test_given_snapshot_read_by_dynamic_model_when_rebuilding_then_it_tracks_changes( + test_case: ChangeTrackingSnapshotTestCase, +) -> None: + adapter: TrackedSnapshotDuckDbAdapter = TrackedSnapshotDuckDbAdapter() + connection: Any = adapter.connect({"database": ":memory:"}) + entry: ModelPlanEntry = replace( + build_snapshot_execution_plan_entry(), dynamic_readers=test_case.dynamic_readers + ) + + results: list[ModelExecutionResult] = [ + execute_snapshot_entry( + context=ModelMaterializationContext( + entry=replace(entry, reason=reason), + adapter=adapter, + connection=connection, + model_locations={}, + seed_locations={}, + source_map={}, + python_source_read_map={}, + model_audits=(), + run_id="tracking_run", + query_change_tracking=False, + ) + ) + for reason in (PlanReason.FIRST_RUN, PlanReason.FULL_REFRESH) + ] + + assert [result.status for result in results] == [ExecutionStatus.SUCCESS] * 2, [ + result.error_message for result in results + ] + assert tuple(adapter.tracked) == test_case.expected_tracked + adapter.close(connection) diff --git a/tests/integration/src/sqlbuild/executor/run/maintained/test_maintained_execution.py b/tests/integration/src/sqlbuild/executor/run/dynamic/test_dynamic_execution.py similarity index 76% rename from tests/integration/src/sqlbuild/executor/run/maintained/test_maintained_execution.py rename to tests/integration/src/sqlbuild/executor/run/dynamic/test_dynamic_execution.py index 1df742d105..3dd65a043b 100644 --- a/tests/integration/src/sqlbuild/executor/run/maintained/test_maintained_execution.py +++ b/tests/integration/src/sqlbuild/executor/run/dynamic/test_dynamic_execution.py @@ -1,4 +1,4 @@ -"""Integration tests for the maintained model lifecycle: create, alter and keep.""" +"""Integration tests for the dynamic model lifecycle: create, alter and keep.""" from __future__ import annotations @@ -7,19 +7,19 @@ import pytest from sqlbuild._native import PlanAction -from sqlbuild.adapter.contract.types import MaintainedListing -from sqlbuild.executor.run.main._execute import execute_maintained_entry +from sqlbuild.adapter.contract.types import DynamicListing +from sqlbuild.executor.run.main._execute import execute_dynamic_entry from sqlbuild.executor.run.models import ModelExecutionResult from sqlbuild.executor.scheduling.types import ExecutionStatus -from tests.integration.src.sqlbuild.executor.run.maintained._test_types import ( +from tests.integration.src.sqlbuild.executor.run.dynamic._test_types import ( + DynamicFailureTestCase, + DynamicSuccessTestCase, FailedRestoreTestCase, - MaintainedFailureTestCase, - MaintainedSuccessTestCase, ) -from tests.integration.src.sqlbuild.executor.run.maintained.helpers import ( - MaintainedDuckDbAdapter, +from tests.integration.src.sqlbuild.executor.run.dynamic.helpers import ( + DynamicDuckDbAdapter, + dynamic_context, listed_types, - maintained_context, ) _BUILT: tuple[str, ...] = ("CREATE TABLE staging.daily_totals AS SELECT 1 AS orders",) @@ -28,45 +28,45 @@ @pytest.mark.parametrize( "test_case", [ - MaintainedSuccessTestCase( + DynamicSuccessTestCase( description="a first run creates on the session warehouse", - action=PlanAction.CREATE_MAINTAINED, + action=PlanAction.CREATE_DYNAMIC, setup_sql=(), refresh_warehouse=None, expected_created=(("'1 hour'", "incremental", "SESSION_WH"),), expected_altered=(), expected_rows=((3,),), ), - MaintainedSuccessTestCase( + DynamicSuccessTestCase( description="a declared refresh warehouse is used over the session's", - action=PlanAction.CREATE_MAINTAINED, + action=PlanAction.CREATE_DYNAMIC, setup_sql=(), refresh_warehouse="transform_wh", expected_created=(("'1 hour'", "incremental", "transform_wh"),), expected_altered=(), expected_rows=((3,),), ), - MaintainedSuccessTestCase( + DynamicSuccessTestCase( description="a view the model was before is dropped and replaced", - action=PlanAction.CREATE_MAINTAINED, + action=PlanAction.CREATE_DYNAMIC, setup_sql=("CREATE VIEW staging.daily_totals AS SELECT 1 AS orders",), refresh_warehouse=None, expected_created=(("'1 hour'", "incremental", "SESSION_WH"),), expected_altered=(), expected_rows=((3,),), ), - MaintainedSuccessTestCase( + DynamicSuccessTestCase( description="drifted settings alter in place and keep the data", - action=PlanAction.ALTER_MAINTAINED, + action=PlanAction.ALTER_DYNAMIC, setup_sql=_BUILT, refresh_warehouse="transform_wh", expected_created=(), expected_altered=(("'1 hour'", "transform_wh"),), expected_rows=((1,),), ), - MaintainedSuccessTestCase( + DynamicSuccessTestCase( description="a kept model runs no refresh statement", - action=PlanAction.KEEP_MAINTAINED, + action=PlanAction.KEEP_DYNAMIC, setup_sql=_BUILT, refresh_warehouse=None, expected_created=(), @@ -76,15 +76,15 @@ ], ids=lambda case: case.description, ) -def test_given_maintained_entry_when_executing_then_settings_reach_the_warehouse( - test_case: MaintainedSuccessTestCase, connection: Any +def test_given_dynamic_entry_when_executing_then_settings_reach_the_warehouse( + test_case: DynamicSuccessTestCase, connection: Any ) -> None: - adapter: MaintainedDuckDbAdapter = MaintainedDuckDbAdapter(session_warehouse="SESSION_WH") + adapter: DynamicDuckDbAdapter = DynamicDuckDbAdapter(session_warehouse="SESSION_WH") for sql in test_case.setup_sql: connection.execute(sql) - result: ModelExecutionResult = execute_maintained_entry( - context=maintained_context( + result: ModelExecutionResult = execute_dynamic_entry( + context=dynamic_context( adapter=adapter, connection=connection, action=test_case.action, @@ -106,7 +106,7 @@ def test_given_maintained_entry_when_executing_then_settings_reach_the_warehouse @pytest.mark.parametrize( "test_case", [ - MaintainedFailureTestCase( + DynamicFailureTestCase( description="no declared or session warehouse fails before any statement", setup_sql=(), session_warehouse=None, @@ -115,16 +115,16 @@ def test_given_maintained_entry_when_executing_then_settings_reach_the_warehouse expected_error_code="A717", expected_listed_types=(), ), - MaintainedFailureTestCase( + DynamicFailureTestCase( description="a failed create never drops a relation listed as a dynamic table", setup_sql=("CREATE VIEW staging.daily_totals AS SELECT 1 AS orders",), session_warehouse="SESSION_WH", - listed_settings=MaintainedListing("1 hour", "SESSION_WH", "ACTIVE"), + listed_settings=DynamicListing("1 hour", "SESSION_WH", "ACTIVE"), failing_query=None, expected_error_code="R002", expected_listed_types=(("VIEW",),), ), - MaintainedFailureTestCase( + DynamicFailureTestCase( description="a table the model was before is restored when the create keeps failing", setup_sql=("CREATE TABLE staging.daily_totals AS SELECT 1 AS orders",), session_warehouse="SESSION_WH", @@ -133,7 +133,7 @@ def test_given_maintained_entry_when_executing_then_settings_reach_the_warehouse expected_error_code="R002", expected_listed_types=(("BASE TABLE",),), ), - MaintainedFailureTestCase( + DynamicFailureTestCase( description="a view the model was before is restored when the create keeps failing", setup_sql=("CREATE VIEW staging.daily_totals AS SELECT 1 AS orders",), session_warehouse="SESSION_WH", @@ -145,10 +145,10 @@ def test_given_maintained_entry_when_executing_then_settings_reach_the_warehouse ], ids=lambda case: case.description, ) -def test_given_unbuildable_maintained_entry_when_executing_then_fails_keeping_the_relation( - test_case: MaintainedFailureTestCase, connection: Any +def test_given_unbuildable_dynamic_entry_when_executing_then_fails_keeping_the_relation( + test_case: DynamicFailureTestCase, connection: Any ) -> None: - adapter: MaintainedDuckDbAdapter = MaintainedDuckDbAdapter( + adapter: DynamicDuckDbAdapter = DynamicDuckDbAdapter( session_warehouse=test_case.session_warehouse, listed_settings=test_case.listed_settings, failing_query=test_case.failing_query, @@ -156,11 +156,11 @@ def test_given_unbuildable_maintained_entry_when_executing_then_fails_keeping_th for sql in test_case.setup_sql: connection.execute(sql) - result: ModelExecutionResult = execute_maintained_entry( - context=maintained_context( + result: ModelExecutionResult = execute_dynamic_entry( + context=dynamic_context( adapter=adapter, connection=connection, - action=PlanAction.CREATE_MAINTAINED, + action=PlanAction.CREATE_DYNAMIC, refresh_warehouse=None, ) ) @@ -190,7 +190,7 @@ def test_given_unbuildable_maintained_entry_when_executing_then_fails_keeping_th def test_given_failed_create_and_restore_when_executing_then_reports_both_errors( test_case: FailedRestoreTestCase, connection: Any ) -> None: - adapter: MaintainedDuckDbAdapter = MaintainedDuckDbAdapter( + adapter: DynamicDuckDbAdapter = DynamicDuckDbAdapter( session_warehouse="SESSION_WH", failing_query=test_case.failing_query, failing_renames=test_case.failing_renames, @@ -198,11 +198,11 @@ def test_given_failed_create_and_restore_when_executing_then_reports_both_errors for sql in test_case.setup_sql: connection.execute(sql) - result: ModelExecutionResult = execute_maintained_entry( - context=maintained_context( + result: ModelExecutionResult = execute_dynamic_entry( + context=dynamic_context( adapter=adapter, connection=connection, - action=PlanAction.CREATE_MAINTAINED, + action=PlanAction.CREATE_DYNAMIC, refresh_warehouse=None, ) ) diff --git a/tests/integration/src/sqlbuild/executor/run/maintained/test_change_tracking.py b/tests/integration/src/sqlbuild/executor/run/maintained/test_change_tracking.py deleted file mode 100644 index 4321614ce8..0000000000 --- a/tests/integration/src/sqlbuild/executor/run/maintained/test_change_tracking.py +++ /dev/null @@ -1,85 +0,0 @@ -"""A table a maintained model reads is created with change tracking on every build path.""" - -from __future__ import annotations - -from dataclasses import replace -from typing import Any - -import pytest - -from sqlbuild._native import ModelPlanEntry -from sqlbuild.adapter.contract.types import TablePromotionMode -from sqlbuild.executor.run.main._execute import execute_table_entry -from sqlbuild.executor.run.models import ModelExecutionResult, ModelMaterializationContext -from sqlbuild.executor.scheduling.types import ExecutionStatus -from tests.integration.src.sqlbuild.executor.run.helpers import ( - build_declared_columns, - build_table_plan_entry, -) -from tests.integration.src.sqlbuild.executor.run.maintained._test_types import ( - ChangeTrackingTableTestCase, -) -from tests.integration.src.sqlbuild.executor.run.maintained.helpers import TrackingDuckDbAdapter - - -@pytest.mark.parametrize( - "test_case", - [ - ChangeTrackingTableTestCase( - description="a staged build tracks changes on the stage it swaps in", - maintained_readers=("daily_totals",), - promotion_mode=TablePromotionMode.STAGED, - declared_columns=(), - expected_tracking=(True,), - ), - ChangeTrackingTableTestCase( - description="type enforcement rebuilds the stage with change tracking too", - maintained_readers=("daily_totals",), - promotion_mode=TablePromotionMode.STAGED, - declared_columns=(("order_id", "BIGINT"),), - expected_tracking=(True, True), - ), - ChangeTrackingTableTestCase( - description="a table no maintained model reads is created as before", - maintained_readers=(), - promotion_mode=TablePromotionMode.STAGED, - declared_columns=(("order_id", "BIGINT"),), - expected_tracking=(False, False), - ), - ], - ids=lambda case: case.description, -) -def test_given_table_read_by_maintained_model_when_building_then_every_create_tracks_changes( - test_case: ChangeTrackingTableTestCase, connection: Any -) -> None: - adapter: TrackingDuckDbAdapter = TrackingDuckDbAdapter() - entry: ModelPlanEntry = replace( - build_table_plan_entry( - name="orders", - sql="SELECT 1::INTEGER AS order_id", - target_schema="staging", - target_name="orders", - type_enforcement=bool(test_case.declared_columns), - ), - maintained_readers=test_case.maintained_readers, - ) - - result: ModelExecutionResult = execute_table_entry( - context=ModelMaterializationContext( - entry=entry, - adapter=adapter, - connection=connection, - model_locations={}, - seed_locations={}, - source_map={}, - python_source_read_map={}, - model_audits=(), - run_id="tracking_run", - query_change_tracking=False, - ), - declared_columns=build_declared_columns(test_case.declared_columns), - promotion_mode=test_case.promotion_mode, - ) - - assert result.status == ExecutionStatus.SUCCESS, result.error_message - assert tuple(adapter.tracking) == test_case.expected_tracking diff --git a/tests/unit/src/sqlbuild/adapter/contract/classes/strict_adapter/test_strict_adapter.py b/tests/unit/src/sqlbuild/adapter/contract/classes/strict_adapter/test_strict_adapter.py index 301ac4504f..6605bc233c 100644 --- a/tests/unit/src/sqlbuild/adapter/contract/classes/strict_adapter/test_strict_adapter.py +++ b/tests/unit/src/sqlbuild/adapter/contract/classes/strict_adapter/test_strict_adapter.py @@ -89,12 +89,13 @@ def test_given_first_class_adapters_when_checking_contract_then_no_method_comes_ { "create_sql_test_output", "drop_sql_test_output", - "maintained_listing", - "render_alter_maintained", - "render_create_maintained", + "dynamic_listing", + "render_alter_dynamic", + "render_create_dynamic", "render_create_temporary_function", "render_drop_schema", - "render_maintained_operation", + "render_dynamic_operation", + "render_track_changes", "session_warehouse", } ), @@ -130,15 +131,15 @@ def test_given_strict_adapter_when_reading_optional_methods_then_defaults_keep_i assert temporary is test_case.expected_temporary_function assert drop_error.value.code == "A714" assert adapter.session_warehouse(connection=None) is None - with pytest.raises(AdapterUserError) as maintained_error: - adapter.render_create_maintained( + with pytest.raises(AdapterUserError) as dynamic_error: + adapter.render_create_dynamic( destination="main.daily_totals", sql="SELECT 1", target_lag="DOWNSTREAM", refresh_mode="incremental", warehouse="transform_wh", ) - assert maintained_error.value.code == "A716" + assert dynamic_error.value.code == "A716" @pytest.mark.parametrize( diff --git a/tests/unit/src/sqlbuild/adapters/snowflake/_test_types.py b/tests/unit/src/sqlbuild/adapters/snowflake/_test_types.py index e73f7229f2..6065bcc473 100644 --- a/tests/unit/src/sqlbuild/adapters/snowflake/_test_types.py +++ b/tests/unit/src/sqlbuild/adapters/snowflake/_test_types.py @@ -9,8 +9,8 @@ SchemaDiffResult, ) from sqlbuild.adapter.contract.types import ( - MaintainedListing, - MaintainedOperation, + DynamicListing, + DynamicOperation, TableFreshnessStatus, ) from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter @@ -464,14 +464,14 @@ class SnowflakeWarehouseUserTestCase: @dataclass(frozen=True) -class SnowflakeMaintainedRefreshTestCase: +class SnowflakeDynamicRefreshTestCase: description: str rows: tuple[tuple[object, ...], ...] - expected_settings: MaintainedListing | None + expected_settings: DynamicListing | None @dataclass(frozen=True) -class SnowflakeMaintainedCreateTestCase: +class SnowflakeDynamicCreateTestCase: description: str target_lag: str refresh_mode: str @@ -480,7 +480,7 @@ class SnowflakeMaintainedCreateTestCase: @dataclass(frozen=True) -class SnowflakeMaintainedAlterTestCase: +class SnowflakeDynamicAlterTestCase: description: str target_lag: str warehouse: str | None @@ -488,9 +488,9 @@ class SnowflakeMaintainedAlterTestCase: @dataclass(frozen=True) -class SnowflakeMaintainedOperationTestCase: +class SnowflakeDynamicOperationTestCase: description: str - operation: MaintainedOperation + operation: DynamicOperation expected_sql: str diff --git a/tests/unit/src/sqlbuild/adapters/snowflake/test_maintained_models.py b/tests/unit/src/sqlbuild/adapters/snowflake/test_dynamic_models.py similarity index 81% rename from tests/unit/src/sqlbuild/adapters/snowflake/test_maintained_models.py rename to tests/unit/src/sqlbuild/adapters/snowflake/test_dynamic_models.py index a050065022..8790a71a29 100644 --- a/tests/unit/src/sqlbuild/adapters/snowflake/test_maintained_models.py +++ b/tests/unit/src/sqlbuild/adapters/snowflake/test_dynamic_models.py @@ -1,15 +1,15 @@ import pytest from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder -from sqlbuild.adapter.contract.types import MaintainedListing, MaintainedOperation +from sqlbuild.adapter.contract.types import DynamicListing, DynamicOperation from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter from tests.unit.src.sqlbuild.adapters.helpers import MetadataConnection, execute_through_cursor from tests.unit.src.sqlbuild.adapters.snowflake._test_types import ( SnowflakeChangeTrackingTestCase, - SnowflakeMaintainedAlterTestCase, - SnowflakeMaintainedCreateTestCase, - SnowflakeMaintainedOperationTestCase, - SnowflakeMaintainedRefreshTestCase, + SnowflakeDynamicAlterTestCase, + SnowflakeDynamicCreateTestCase, + SnowflakeDynamicOperationTestCase, + SnowflakeDynamicRefreshTestCase, ) _SHOW_DYNAMIC_TABLES_COLUMNS: tuple[str, ...] = ( @@ -23,17 +23,17 @@ @pytest.mark.parametrize( "test_case", [ - SnowflakeMaintainedRefreshTestCase( + SnowflakeDynamicRefreshTestCase( description="a dynamic table lists its lag, refresh warehouse and schedule state", rows=(("DAILY_TOTALS", "1 hour", "TRANSFORM_WH", "SUSPENDED"),), - expected_settings=MaintainedListing("1 hour", "TRANSFORM_WH", "SUSPENDED"), + expected_settings=DynamicListing("1 hour", "TRANSFORM_WH", "SUSPENDED"), ), - SnowflakeMaintainedRefreshTestCase( + SnowflakeDynamicRefreshTestCase( description="a neighbour matched by the LIKE wildcard is not the model", rows=(("DAILY_TOTALS_ARCHIVE", "DOWNSTREAM", "TRANSFORM_WH", "ACTIVE"),), expected_settings=None, ), - SnowflakeMaintainedRefreshTestCase( + SnowflakeDynamicRefreshTestCase( description="a relation that is no dynamic table lists nothing", rows=(), expected_settings=None, @@ -42,7 +42,7 @@ ids=lambda case: case.description, ) def test_given_dynamic_table_listing_when_reading_it_then_reads_one_show( - test_case: SnowflakeMaintainedRefreshTestCase, monkeypatch: pytest.MonkeyPatch + test_case: SnowflakeDynamicRefreshTestCase, monkeypatch: pytest.MonkeyPatch ) -> None: adapter: SnowflakeAdapter = SnowflakeAdapter() connection: MetadataConnection = MetadataConnection( @@ -50,7 +50,7 @@ def test_given_dynamic_table_listing_when_reading_it_then_reads_one_show( ) monkeypatch.setattr(adapter, "execute", execute_through_cursor) - settings: MaintainedListing | None = adapter.maintained_listing( + settings: DynamicListing | None = adapter.dynamic_listing( connection=connection, database="ANALYTICS", schema="MARTS", name="daily_totals" ) @@ -64,7 +64,7 @@ def test_given_dynamic_table_listing_when_reading_it_then_reads_one_show( @pytest.mark.parametrize( "test_case", [ - SnowflakeMaintainedCreateTestCase( + SnowflakeDynamicCreateTestCase( description="a lag, the default incremental mode and an unquoted warehouse", target_lag="'1 hour'", refresh_mode="incremental", @@ -75,7 +75,7 @@ def test_given_dynamic_table_listing_when_reading_it_then_reads_one_show( "INITIALIZE = ON_CREATE AS SELECT 1 AS orders" ), ), - SnowflakeMaintainedCreateTestCase( + SnowflakeDynamicCreateTestCase( description="refreshed for its readers, in adaptive mode", target_lag="DOWNSTREAM", refresh_mode="adaptive", @@ -86,7 +86,7 @@ def test_given_dynamic_table_listing_when_reading_it_then_reads_one_show( "INITIALIZE = ON_CREATE AS SELECT 1 AS orders" ), ), - SnowflakeMaintainedCreateTestCase( + SnowflakeDynamicCreateTestCase( description="a warehouse name that needs quoting", target_lag="'5 minutes'", refresh_mode="full", @@ -100,10 +100,10 @@ def test_given_dynamic_table_listing_when_reading_it_then_reads_one_show( ], ids=lambda case: case.description, ) -def test_given_maintained_settings_when_rendering_create_then_mode_and_initialize_are_explicit( - test_case: SnowflakeMaintainedCreateTestCase, +def test_given_dynamic_settings_when_rendering_create_then_mode_and_initialize_are_explicit( + test_case: SnowflakeDynamicCreateTestCase, ) -> None: - statements: tuple[str, ...] = SnowflakeAdapter().render_create_maintained( + statements: tuple[str, ...] = SnowflakeAdapter().render_create_dynamic( destination="ANALYTICS.MARTS.DAILY_TOTALS", sql="SELECT 1 AS orders", target_lag=test_case.target_lag, @@ -117,7 +117,7 @@ def test_given_maintained_settings_when_rendering_create_then_mode_and_initializ @pytest.mark.parametrize( "test_case", [ - SnowflakeMaintainedAlterTestCase( + SnowflakeDynamicAlterTestCase( description="an unset refresh warehouse alters only the lag", target_lag="'2 hours'", warehouse=None, @@ -125,7 +125,7 @@ def test_given_maintained_settings_when_rendering_create_then_mode_and_initializ "ALTER DYNAMIC TABLE ANALYTICS.MARTS.DAILY_TOTALS SET TARGET_LAG = '2 hours'", ), ), - SnowflakeMaintainedAlterTestCase( + SnowflakeDynamicAlterTestCase( description="a declared refresh warehouse is set too", target_lag="DOWNSTREAM", warehouse="reporting_wh", @@ -138,9 +138,9 @@ def test_given_maintained_settings_when_rendering_create_then_mode_and_initializ ids=lambda case: case.description, ) def test_given_drifted_settings_when_rendering_alter_then_settings_change_in_place( - test_case: SnowflakeMaintainedAlterTestCase, + test_case: SnowflakeDynamicAlterTestCase, ) -> None: - statements: tuple[str, ...] = SnowflakeAdapter().render_alter_maintained( + statements: tuple[str, ...] = SnowflakeAdapter().render_alter_dynamic( destination="ANALYTICS.MARTS.DAILY_TOTALS", target_lag=test_case.target_lag, warehouse=test_case.warehouse, @@ -152,28 +152,28 @@ def test_given_drifted_settings_when_rendering_alter_then_settings_change_in_pla @pytest.mark.parametrize( "test_case", [ - SnowflakeMaintainedOperationTestCase( + SnowflakeDynamicOperationTestCase( description="suspend", - operation=MaintainedOperation.SUSPEND, + operation=DynamicOperation.SUSPEND, expected_sql="ALTER DYNAMIC TABLE ANALYTICS.MARTS.DAILY_TOTALS SUSPEND", ), - SnowflakeMaintainedOperationTestCase( + SnowflakeDynamicOperationTestCase( description="resume", - operation=MaintainedOperation.RESUME, + operation=DynamicOperation.RESUME, expected_sql="ALTER DYNAMIC TABLE ANALYTICS.MARTS.DAILY_TOTALS RESUME", ), - SnowflakeMaintainedOperationTestCase( + SnowflakeDynamicOperationTestCase( description="refresh", - operation=MaintainedOperation.REFRESH, + operation=DynamicOperation.REFRESH, expected_sql="ALTER DYNAMIC TABLE ANALYTICS.MARTS.DAILY_TOTALS REFRESH", ), ], ids=lambda case: case.description, ) def test_given_operation_when_rendering_then_one_alter_dynamic_table_statement( - test_case: SnowflakeMaintainedOperationTestCase, + test_case: SnowflakeDynamicOperationTestCase, ) -> None: - statements: tuple[str, ...] = SnowflakeAdapter().render_maintained_operation( + statements: tuple[str, ...] = SnowflakeAdapter().render_dynamic_operation( destination="ANALYTICS.MARTS.DAILY_TOTALS", operation=test_case.operation ) @@ -184,7 +184,7 @@ def test_given_operation_when_rendering_then_one_alter_dynamic_table_statement( "test_case", [ SnowflakeChangeTrackingTestCase( - description="a table a maintained model reads tracks changes from creation", + description="a table a dynamic model reads tracks changes from creation", config={"table_type": "transient", "change_tracking": True}, expected_sql=( "CREATE OR REPLACE TRANSIENT TABLE ANALYTICS.STAGING.ORDERS " diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_detect.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_detect.py index 6edecd18ea..9a3c6b2bee 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_detect.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_detect.py @@ -590,6 +590,32 @@ def test_given_dropped_relation_with_recorded_build_when_planning_then_plans_fir '"post_hooks":[{"statement":"INSERT INTO audit_log SELECT \'new\'","type":"sql"}]', ), ), + DetectModelMetadataTestCase( + description="writing out the default refresh_mode is no change", + config_values={"materialized": "dynamic", "refresh_mode": "incremental"}, + schema_columns=(), + deps=(), + function_local_hashes={}, + previous_metadata_json=build_version_identity_metadata_json( + model_name="orders", + config_values={"materialized": "dynamic"}, + execution_signature={}, + ), + expected_change_kind=ChangeKind.NO_CHANGE, + ), + DetectModelMetadataTestCase( + description="removing a written-out default refresh_mode is no change", + config_values={"materialized": "dynamic"}, + schema_columns=(), + deps=(), + function_local_hashes={}, + previous_metadata_json=build_version_identity_metadata_json( + model_name="orders", + config_values={"materialized": "dynamic", "refresh_mode": "incremental"}, + execution_signature={}, + ), + expected_change_kind=ChangeKind.NO_CHANGE, + ), DetectModelMetadataTestCase( description="ignores runtime audit gate proof when version identity matches", config_values={"materialized": "table"}, @@ -698,13 +724,13 @@ def test_given_model_identity_metadata_when_detecting_changes_then_uses_aligned_ expected_backfill=BackfillResult(action=BackfillAction.FORWARD_ONLY), ), DetectFunctionCallerChangeTestCase( - description="a maintained model's refresh_mode change is a config change", + description="a dynamic model's refresh_mode change is a config change", metadata_case=build_function_caller_metadata_case( - config_values={"materialized": "maintained", "refresh_mode": "full"}, + config_values={"materialized": "dynamic", "refresh_mode": "full"}, function_local_hashes={}, previous_function_hashes={}, previous_config_values={ - "materialized": "maintained", + "materialized": "dynamic", "refresh_mode": "incremental", }, ), @@ -714,12 +740,12 @@ def test_given_model_identity_metadata_when_detecting_changes_then_uses_aligned_ expected_backfill=BackfillResult(action=BackfillAction.FORWARD_ONLY), ), DetectFunctionCallerChangeTestCase( - description="a maintained model's lag change is altered in place, not a config change", + description="a dynamic model's lag change is altered in place, not a config change", metadata_case=build_function_caller_metadata_case( - config_values={"materialized": "maintained", "refresh": {"lag": "5m"}}, + config_values={"materialized": "dynamic", "refresh": {"lag": "5m"}}, function_local_hashes={}, previous_function_hashes={}, - previous_config_values={"materialized": "maintained", "refresh": {"lag": "1h"}}, + previous_config_values={"materialized": "dynamic", "refresh": {"lag": "1h"}}, ), query_change_tracking=True, expected_changed_functions=(), diff --git a/ui/src/lib/api/types/operate.ts b/ui/src/lib/api/types/operate.ts index 38eeff8328..4d3dc595c0 100644 --- a/ui/src/lib/api/types/operate.ts +++ b/ui/src/lib/api/types/operate.ts @@ -656,9 +656,9 @@ export type PlanModel = { backfill: { action: string; duration: string | null }; qualified_name?: string; changed_functions?: string[]; - /** The rebuilt upstream that makes this kept maintained model recompute in full. */ + /** The rebuilt upstream that makes this kept dynamic model recompute in full. */ reinitialised_by?: string; - /** Whether the warehouse lists this kept maintained model as suspended. */ + /** Whether the warehouse lists this kept dynamic model as suspended. */ paused?: boolean; cursor_type?: string; cursor?: { requested_bounds: { start: string | null; end: string | null }; resolved_bounds: { start: string | null; end: string | null } | null } | null; diff --git a/ui/src/lib/run/plan-text.test.ts b/ui/src/lib/run/plan-text.test.ts index e80b045adc..45e6d4f4cd 100644 --- a/ui/src/lib/run/plan-text.test.ts +++ b/ui/src/lib/run/plan-text.test.ts @@ -35,25 +35,25 @@ describe('plan text, as sqb plan prints it', () => { expect(actionText(model('a', { reason: 'first_run', backfill: { action: 'full', duration: null } }))).toBe('continue forward'); }); - it('words maintained models as the CLI does, with no replay policy', () => { - const recreated = model('a', { action: 'create_maintained', reason: 'query_changed', backfill: { action: 'full', duration: null } }); + it('words dynamic models as the CLI does, with no replay policy', () => { + const recreated = model('a', { action: 'create_dynamic', reason: 'query_changed', backfill: { action: 'full', duration: null } }); expect(actionText(recreated)).toBe('recreate, full recompute'); - expect(actionText(model('a', { action: 'create_maintained', reason: 'first_run' }))).toBe('create'); - expect(actionText(model('a', { action: 'alter_maintained', reason: 'config_changed' }))).toBe('change refresh settings in place'); + expect(actionText(model('a', { action: 'create_dynamic', reason: 'first_run' }))).toBe('create'); + expect(actionText(model('a', { action: 'alter_dynamic', reason: 'config_changed' }))).toBe('change refresh settings in place'); const [section] = planSections(plan([recreated])); expect(section.entries[0].detail).toEqual([]); }); - it('says a kept maintained model recomputes in full when an upstream it reads is rebuilt', () => { - const kept = model('daily_totals', { materialization_type: 'maintained', action: 'keep_maintained', reinitialised_by: 'orders' }); + it('says a kept dynamic model recomputes in full when an upstream it reads is rebuilt', () => { + const kept = model('daily_totals', { materialization_type: 'dynamic', action: 'keep_dynamic', reinitialised_by: 'orders' }); const [routine] = planSections(plan([kept])); - expect(routine.entries).toEqual([{ name: 'daily_totals', value: 'maintained', detail: ['recomputes in full: orders is rebuilt'] }]); + expect(routine.entries).toEqual([{ name: 'daily_totals', value: 'dynamic', detail: ['recomputes in full: orders is rebuilt'] }]); }); - it('says a kept maintained model is paused when the warehouse lists it as suspended', () => { - const kept = model('daily_totals', { materialization_type: 'maintained', action: 'keep_maintained', paused: true }); + it('says a kept dynamic model is paused when the warehouse lists it as suspended', () => { + const kept = model('daily_totals', { materialization_type: 'dynamic', action: 'keep_dynamic', paused: true }); const [routine] = planSections(plan([kept])); - expect(routine.entries).toEqual([{ name: 'daily_totals', value: 'maintained', detail: ['paused: scheduled refreshes are suspended'] }]); + expect(routine.entries).toEqual([{ name: 'daily_totals', value: 'dynamic', detail: ['paused: scheduled refreshes are suspended'] }]); }); it('puts each change in its own section in reason order, then the routine models, and leaves skipped ones out', () => { diff --git a/ui/src/lib/run/plan-text.ts b/ui/src/lib/run/plan-text.ts index 97825f856f..6b20698164 100644 --- a/ui/src/lib/run/plan-text.ts +++ b/ui/src/lib/run/plan-text.ts @@ -97,8 +97,8 @@ export function materializationLabel(m: PlanModel): string { export function actionText(m: PlanModel): string { if (m.action === 'create_view') return 'recreate view'; if (m.action === 'custom') return 'run custom materialization'; - if (m.action === 'create_maintained') return m.reason === 'first_run' ? 'create' : 'recreate, full recompute'; - if (m.action === 'alter_maintained') return 'change refresh settings in place'; + if (m.action === 'create_dynamic') return m.reason === 'first_run' ? 'create' : 'recreate, full recompute'; + if (m.action === 'alter_dynamic') return 'change refresh settings in place'; if (m.backfill.action === 'bounded' && m.backfill.duration) return `rebuild last ${m.backfill.duration}`; if (m.backfill.action === 'full' && m.reason !== 'first_run') return 'full rebuild'; return 'continue forward'; @@ -118,7 +118,7 @@ function cursorLines(m: PlanModel, showRange: boolean, windowStart?: string): st } function policyLine(m: PlanModel): string[] { - if (m.backfill.action === 'forward' || m.action === 'create_maintained' || !(m.reason === 'query_changed' || m.reason === 'schema_changed')) return []; + if (m.backfill.action === 'forward' || m.action === 'create_dynamic' || !(m.reason === 'query_changed' || m.reason === 'schema_changed')) return []; const value = m.backfill.action === 'bounded' ? `bounded-${m.backfill.duration ?? 'full'}` : m.backfill.action; return [`policy: replay_on_change=${value}`]; } diff --git a/website/src/content/docs/docs/cli/build.mdx b/website/src/content/docs/docs/cli/build.mdx index 2a69160633..12e7916aca 100644 --- a/website/src/content/docs/docs/cli/build.mdx +++ b/website/src/content/docs/docs/cli/build.mdx @@ -49,7 +49,7 @@ own option. `--version` works only directly after `sqb`, as in `sqb --version`: | `--defer-sources-to` | Read managed source data from another target | | `--plan-from` | Read warehouse state `live`, or from the target's `synced_state` checked against the warehouse first; overrides the target's `plan_from`. See [Planning from synced state](/docs/concepts/project-configuration#planning-from-synced-state) | | `--fail-fast` | Stop on first failure and skip remaining nodes | -| `--ignore-maintained-grace` | Take over [maintained models](/docs/concepts/models/maintained#when-a-build-dies) another build holds, without waiting for `[maintained] heartbeat_grace`. Needs `allow_ignore_grace = true`, else fails with `A719` | +| `--ignore-dynamic-grace` | Take over [dynamic models](/docs/concepts/models/dynamic#when-a-build-dies) another build holds, without waiting for `[dynamic] heartbeat_grace`. Needs `allow_ignore_grace = true`, else fails with `A719` | | `--concurrency` | Number of worker connections (default: 1) | | `--test-timeout ` | Time limit for each unit test the build runs, such as `30s`, or `disabled`; see [Time limits](/docs/concepts/testing-reference#time-limits) | | `--verbose`, `-v` | Show lifecycle SQL inline after each model | diff --git a/website/src/content/docs/docs/cli/dag.mdx b/website/src/content/docs/docs/cli/dag.mdx index cd8a98262e..f945007864 100644 --- a/website/src/content/docs/docs/cli/dag.mdx +++ b/website/src/content/docs/docs/cli/dag.mdx @@ -96,7 +96,7 @@ Each node represents a source, seed, model, or function: | `description` | Model or source description, if declared | | `tags` | Model tags | | `columns` | Column metadata (name, type, nullable, description) | -| `materialization_type` | For models: `view`, `table`, `incremental`, `snapshot`, `maintained`, or custom name | +| `materialization_type` | For models: `view`, `table`, `incremental`, `snapshot`, `dynamic`, or custom name | | `language` | For functions: `sql` or `python` | | `return_kind` | For functions: `scalar` or `table` | | `arguments` | For functions: argument name and type pairs | diff --git a/website/src/content/docs/docs/concepts/adapters/snowflake.mdx b/website/src/content/docs/docs/concepts/adapters/snowflake.mdx index f229f05121..29194563a5 100644 --- a/website/src/content/docs/docs/concepts/adapters/snowflake.mdx +++ b/website/src/content/docs/docs/concepts/adapters/snowflake.mdx @@ -150,10 +150,10 @@ SQLBuild records its state in `_sqlbuild_*` tables. On Snowflake it creates them with 90 days of time travel, or 1 day on editions that allow only that. It never alters an existing state table. Models are not affected: their table type follows the model's `table_type` setting. -## Maintained models +## Dynamic models -A [maintained model](/docs/concepts/models/maintained) is a Snowflake dynamic -table. `materialized materialized_view` fails with `A014`; use `materialized maintained`. +A [dynamic model](/docs/concepts/models/dynamic) is a Snowflake dynamic +table. `materialized materialized_view` fails with `A014`; use `materialized dynamic`. The build role needs these grants: @@ -165,11 +165,12 @@ The build role needs these grants: Dynamic tables refresh from changes, which needs change tracking on every table they read: -- A table model that a maintained model reads, directly or through views, is built with change tracking on. +- A table or snapshot model that a dynamic model reads, directly or through views, is built with + change tracking on. - For a source table, Snowflake turns change tracking on when the build role owns the table. Otherwise its owner runs `ALTER TABLE SET CHANGE_TRACKING = TRUE`. -Builds record the models they pause in the `_sqlbuild_maintained_pauses` state table, so another +Builds record the models they pause in the `_sqlbuild_dynamic_pauses` state table, so another build can tell whether they are still held. ## Shared connections across targets diff --git a/website/src/content/docs/docs/concepts/models/configuration.mdx b/website/src/content/docs/docs/concepts/models/configuration.mdx index c72e3487cf..8755d418db 100644 --- a/website/src/content/docs/docs/concepts/models/configuration.mdx +++ b/website/src/content/docs/docs/concepts/models/configuration.mdx @@ -20,7 +20,7 @@ which accepts any keys. | Field | Description | |-------|-------------| -| `materialized` | `view`, `table`, `incremental`, `snapshot`, `maintained`, or a custom materialization name; see [Materializations](/docs/concepts/models/materializations) | +| `materialized` | `view`, `table`, `incremental`, `snapshot`, `dynamic`, or a custom materialization name; see [Materializations](/docs/concepts/models/materializations) | | `tags` | Tags used by selectors | | `critical` | `true` marks the model as critical, so [`sqb review`](/docs/cli/review) builds and compares it whenever something upstream of it changes (default `false`) | | `description` | Required human-readable model description; see [Descriptions](#descriptions) | diff --git a/website/src/content/docs/docs/concepts/models/maintained.mdx b/website/src/content/docs/docs/concepts/models/dynamic.mdx similarity index 62% rename from website/src/content/docs/docs/concepts/models/maintained.mdx rename to website/src/content/docs/docs/concepts/models/dynamic.mdx index e825a6db5e..9cb53ce8ff 100644 --- a/website/src/content/docs/docs/concepts/models/maintained.mdx +++ b/website/src/content/docs/docs/concepts/models/dynamic.mdx @@ -1,16 +1,16 @@ --- -title: "Maintained Models" +title: "Dynamic models" description: "Let the warehouse keep a model up to date on a schedule, paused safely while builds run." --- -A maintained model is one the warehouse keeps up to date on a schedule you set, between builds. A +A dynamic model is one the warehouse keeps up to date on a schedule you set, between builds. A build creates it, changes it when its definition changes, and otherwise leaves the refreshing to the warehouse; on Snowflake it is a dynamic table. ```sql MODEL ( description "One row per customer per order date", - materialized maintained, + materialized dynamic, refresh (lag 1h), ); @@ -19,14 +19,14 @@ FROM __ref("stg_orders") GROUP BY customer_id, CAST(ordered_at AS DATE) ``` -Only Snowflake supports maintained models. See [Snowflake](/docs/concepts/adapters/snowflake#maintained-models) +Only Snowflake supports dynamic models. See [Snowflake](/docs/concepts/adapters/snowflake#dynamic-models) for the grants and change tracking they need. ## Settings | Field | Default | Description | |-------|---------|-------------| -| `refresh (lag )` | none | How far behind its inputs the model may fall: a fixed duration such as `5m`, `1h` or `1d`, at least `1m` on Snowflake and no shorter than the lag of a maintained model it reads. Without it, the model refreshes when a maintained model that reads it refreshes | +| `refresh (lag )` | none | How far behind its inputs the model may fall: a fixed duration such as `5m`, `1h` or `1d`, at least `1m` on Snowflake and no shorter than the lag of a dynamic model it reads. Without it, the model refreshes when a dynamic model that reads it refreshes | | `refresh_mode` | `incremental` | `incremental`, `full`, `auto` or `adaptive`. Snowflake rejects `incremental` for a query it can't refresh incrementally; use `full` for that query | | `refresh_warehouse` | the connection's warehouse | Warehouse the scheduled refreshes run on | | `refresh_on_build` | `false` | Refresh the model in every build that selects it, not only on its schedule | @@ -38,7 +38,7 @@ for the grants and change tracking they need. | First build, or the dynamic table is missing | `create` | Creates it and fills it | | Query or `refresh_mode` changed | `recreate, full recompute` | Replaces it; every row is computed again | | `refresh` lag or `refresh_warehouse` differs from the warehouse | `change refresh settings in place` | Alters the schedule; rows stay | -| Nothing | `maintained` | Leaves it alone | +| Nothing | `dynamic` | Leaves it alone | A model the plan keeps can carry one more line: @@ -47,7 +47,7 @@ A model the plan keeps can carry one more line: ## During a build -A build pauses the maintained models it changes, and those that read a model it rebuilds, directly +A build pauses the dynamic models it changes, and those that read a model it rebuilds, directly or through views, so they never refresh from half-built inputs. It resumes them when it ends, whether it succeeds or fails. @@ -56,24 +56,23 @@ whether it succeeds or fails. - A suspend made by someone else during the build is undone when the build resumes the model. - A kept model refreshes during the build when a model built later in the same build reads it. - `refresh_on_build` refreshes the model in every build that selects it. -- A target with `pause_maintained = true` leaves its maintained models suspended between builds; see [Targets](/docs/concepts/project-configuration#targets). +- A target with `pause_dynamic = true` leaves its dynamic models suspended between builds; see [Targets](/docs/concepts/project-configuration#targets). ## When a build dies -- **One build at a time.** A build that needs a maintained model another build holds fails with - `A718`, naming that run. Wait for it to finish. +- **One build at a time.** A build that needs a dynamic model another build holds fails with + `A718`, naming that run. Wait for it to finish. Two builds that start at the same moment can + both fail this way; run one of them again. - **A stopped build leaves its models suspended.** A build killed before it resumes them, for example by `SIGKILL`, can't resume them itself. -- **The next build resumes them.** The next build that pauses maintained models on the target - resumes them once the stopped build has been silent for `[maintained] heartbeat_grace`, 30 - minutes by default; see [Maintained](/docs/concepts/project-configuration#maintained). -- **Taking over at once.** Run `sqb build --ignore-maintained-grace` to take the models over - without waiting. It needs `[maintained] allow_ignore_grace = true`; without it the flag fails +- **The next build resumes them.** The next build that pauses dynamic models on the target + resumes them once the stopped build has been silent for `[dynamic] heartbeat_grace`, 30 + minutes by default; see [Dynamic](/docs/concepts/project-configuration#dynamic). +- **Taking over at once.** Run `sqb build --ignore-dynamic-grace` to take the models over + without waiting. It needs `[dynamic] allow_ignore_grace = true`; without it the flag fails with `A719`. You can also resume the models in Snowflake by hand. ## Limitations -- Writing out the default `refresh_mode incremental` recreates the model once. -- A snapshot rebuilt in full loses change tracking; run `ALTER TABLE SET CHANGE_TRACKING = TRUE` before a maintained model reads it again. -- Unit tests and scenarios treat a maintained model as a table. -- On a target with `schema = "preserve"`, builds see each other's pauses only when their maintained models share a schema. +- Unit tests and scenarios treat a dynamic model as a table. +- On a target with `schema = "preserve"`, builds see each other's pauses only when their dynamic models share a schema. diff --git a/website/src/content/docs/docs/concepts/models/materializations.mdx b/website/src/content/docs/docs/concepts/models/materializations.mdx index 827b8616ed..f18aef09bb 100644 --- a/website/src/content/docs/docs/concepts/models/materializations.mdx +++ b/website/src/content/docs/docs/concepts/models/materializations.mdx @@ -71,19 +71,19 @@ FROM __source("customers") See [Snapshots](/docs/concepts/snapshots) for timestamp and check strategies, historical inputs, and full-refresh policies. -## Maintained +## Dynamic The warehouse keeps the model up to date on a schedule you set, and builds pause it while they change its inputs: ```sql -MODEL (description "One row per customer per order date", materialized maintained, refresh (lag 1h)); +MODEL (description "One row per customer per order date", materialized dynamic, refresh (lag 1h)); SELECT customer_id, CAST(ordered_at AS DATE) AS order_date, SUM(amount) AS revenue FROM __ref("stg_orders") GROUP BY customer_id, CAST(ordered_at AS DATE) ``` -See [Maintained models](/docs/concepts/models/maintained) for its settings, what the plan shows and what happens when a build dies. +See [Dynamic models](/docs/concepts/models/dynamic) for its settings, what the plan shows and what happens when a build dies. ## Custom @@ -122,5 +122,5 @@ The custom function owns staging, runtime type or contract enforcement, audit ti | Table (`immediate`) | The new table. Audits run after it is replaced. | | Incremental | The old table if a delta audit fails. Rows already written stay if a final audit or a later microbatch fails. | | Microbatch full rebuild | A partly rebuilt table; the old one was dropped first. | -| Maintained | The created or altered dynamic table, resumed. Audits run after it changes. | +| Dynamic | The created or altered dynamic table, resumed. Audits run after it changes. | | Custom | Whatever the custom materialization does. | diff --git a/website/src/content/docs/docs/concepts/project-configuration.mdx b/website/src/content/docs/docs/concepts/project-configuration.mdx index 52fb28a2c0..827f300ec7 100644 --- a/website/src/content/docs/docs/concepts/project-configuration.mdx +++ b/website/src/content/docs/docs/concepts/project-configuration.mdx @@ -163,7 +163,7 @@ A target is a named build context - the database and schema you build into, plus | `concurrency` | Worker limit for commands run against this target, replacing `[settings] concurrency` (see below) | | `ignore_audit_failures` | Whether builds on this target let audit failures through, replacing `[settings] ignore_audit_failures`; see [Ignoring audit failures](/docs/concepts/audits#ignoring-audit-failures) | | `plan_from` | Where `sqb plan` and `sqb build` read warehouse state: `live` (default) or `synced_state`; see [Planning from synced state](#planning-from-synced-state) | -| `pause_maintained` | `true` keeps this target's [maintained models](/docs/concepts/models/maintained) suspended between builds; each build refreshes the ones it selects. Default `false` | +| `pause_dynamic` | `true` keeps this target's [dynamic models](/docs/concepts/models/dynamic) suspended between builds; each build refreshes the ones it selects. Default `false` | ```toml [connections.warehouse] @@ -670,20 +670,20 @@ old_name_views = "30d" |-------|---------|-------------| | `old_name_views` | `"30d"` | How long a renamed model's old name keeps working through a [compatibility view](/docs/concepts/models/migrations#old-names) before janitor drops it. `false` keeps the old relation in place instead. A model's `old_name_view` header overrides it. | -## Maintained +## Dynamic -How builds pause [maintained models](/docs/concepts/models/maintained) while they run: +How builds pause [dynamic models](/docs/concepts/models/dynamic) while they run: ```toml -[maintained] +[dynamic] heartbeat_grace = "30m" allow_ignore_grace = false ``` | Field | Default | Description | |-------|---------|-------------| -| `heartbeat_grace` | `"30m"` | How long a build that paused maintained models may go silent before another build resumes them or takes them over. A fixed duration longer than `"1m"`, such as `"30m"` or `"2h"`; a build writes a heartbeat every minute | -| `allow_ignore_grace` | `false` | Allow `sqb build --ignore-maintained-grace`, which takes over models another build holds without waiting for the grace. Without it the flag fails with `A719` | +| `heartbeat_grace` | `"30m"` | How long a build that paused dynamic models may go silent before another build resumes them or takes them over. A fixed duration longer than `"1m"`, such as `"30m"` or `"2h"`; a build writes a heartbeat every minute | +| `allow_ignore_grace` | `false` | Allow `sqb build --ignore-dynamic-grace`, which takes over models another build holds without waiting for the grace. Without it the flag fails with `A719` | ## Scenario diff --git a/website/src/sidebar.json b/website/src/sidebar.json index 0be8c87a80..a2d01907f1 100644 --- a/website/src/sidebar.json +++ b/website/src/sidebar.json @@ -31,7 +31,7 @@ "docs/concepts/seeds", "docs/concepts/incremental", "docs/concepts/snapshots", - "docs/concepts/models/maintained", + "docs/concepts/models/dynamic", "docs/concepts/functions", { "label": "Macros",