From 7c39a06fbb2f7763946634573c35d926b0a00b7c Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 18 Nov 2026 22:10:36 +0000 Subject: [PATCH 01/17] feat: accept maintained models in MODEL headers with offline adapter checks Work in progress. Adds materialized maintained with refresh (lag ...), refresh_mode, refresh_warehouse and refresh_on_build, validated offline. Coded errors: A013 maintained on an adapter that cannot build it, A014 materialized_view on Snowflake, A015 a lag under Snowflake's 1m minimum, A016 a maintained model nothing would ever refresh. Validated: sqlbuild-model-config and maintained_checks unit tests, the maintained-model compile E2E (5 cases), native model config, unknown key and compiler differential suites, fensu check. Not yet built: plan, build execution, pause lease, project settings, docs. --- .../assembly/maintained_checks/constants.rs | 4 + .../assembly/maintained_checks/main/mod.rs | 1 + .../main/never_refreshed_maintained.rs | 20 +++ .../src/assembly/maintained_checks/mod.rs | 6 + .../src/assembly/maintained_checks/models.rs | 9 ++ .../src/assembly/maintained_checks/tests.rs | 6 + .../maintained_checks/tests/helpers.rs | 10 ++ .../tests/test_maintained_checks.rs | 52 ++++++ .../maintained_checks/tests/test_types.rs | 7 + crates/sqlbuild-analysis/src/assembly/mod.rs | 1 + crates/sqlbuild-discovery/src/constants.rs | 6 +- .../_helpers/materialization.rs | 145 +++++++++++++++++ .../src/model_validation/constants.rs | 30 +++- .../main/validate_model_config.rs | 7 +- .../src/model_validation/models.rs | 2 + .../src/model_validation/tests/helpers.rs | 1 + .../tests/test_validate_model_config.rs | 148 ++++++++++++++++++ .../_helpers/setup/maintained.rs | 68 ++++++++ .../src/compile_driver/_helpers/setup/mod.rs | 1 + .../compile_driver/_helpers/setup/project.rs | 2 + .../src/model_config/models.rs | 16 +- src/sqlbuild/_native.pyi | 1 + .../cli/commands/main/compile/_test_types.py | 11 ++ .../main/compile/test_maintained_models.py | 115 ++++++++++++++ 24 files changed, 663 insertions(+), 6 deletions(-) create mode 100644 crates/sqlbuild-analysis/src/assembly/maintained_checks/constants.rs create mode 100644 crates/sqlbuild-analysis/src/assembly/maintained_checks/main/mod.rs create mode 100644 crates/sqlbuild-analysis/src/assembly/maintained_checks/main/never_refreshed_maintained.rs create mode 100644 crates/sqlbuild-analysis/src/assembly/maintained_checks/mod.rs create mode 100644 crates/sqlbuild-analysis/src/assembly/maintained_checks/models.rs create mode 100644 crates/sqlbuild-analysis/src/assembly/maintained_checks/tests.rs create mode 100644 crates/sqlbuild-analysis/src/assembly/maintained_checks/tests/helpers.rs create mode 100644 crates/sqlbuild-analysis/src/assembly/maintained_checks/tests/test_maintained_checks.rs create mode 100644 crates/sqlbuild-analysis/src/assembly/maintained_checks/tests/test_types.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/maintained.rs create mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/compile/test_maintained_models.py diff --git a/crates/sqlbuild-analysis/src/assembly/maintained_checks/constants.rs b/crates/sqlbuild-analysis/src/assembly/maintained_checks/constants.rs new file mode 100644 index 0000000000..385fec1fd5 --- /dev/null +++ b/crates/sqlbuild-analysis/src/assembly/maintained_checks/constants.rs @@ -0,0 +1,4 @@ +//! Error codes of the project-wide maintained model checks. + +/// A maintained model that nothing would ever refresh. +pub const MAINTAINED_NEVER_REFRESHED_CODE: &str = "A016"; diff --git a/crates/sqlbuild-analysis/src/assembly/maintained_checks/main/mod.rs b/crates/sqlbuild-analysis/src/assembly/maintained_checks/main/mod.rs new file mode 100644 index 0000000000..d43fa0186a --- /dev/null +++ b/crates/sqlbuild-analysis/src/assembly/maintained_checks/main/mod.rs @@ -0,0 +1 @@ +pub mod never_refreshed_maintained; diff --git a/crates/sqlbuild-analysis/src/assembly/maintained_checks/main/never_refreshed_maintained.rs b/crates/sqlbuild-analysis/src/assembly/maintained_checks/main/never_refreshed_maintained.rs new file mode 100644 index 0000000000..078332bc87 --- /dev/null +++ b/crates/sqlbuild-analysis/src/assembly/maintained_checks/main/never_refreshed_maintained.rs @@ -0,0 +1,20 @@ +//! Maintained models with no refresh of their own that no maintained model reads. + +use std::collections::HashSet; + +use crate::assembly::maintained_checks::models::MaintainedModel; + +/// The first by name; without a reader or its own refresh, the warehouse never refreshes it. +pub fn never_refreshed_maintained(models: &[MaintainedModel]) -> Option<&str> { + let maintained: HashSet<&str> = models.iter().map(|model| model.name.as_str()).collect(); + let read: HashSet<&str> = models + .iter() + .flat_map(|model| model.reads.iter().map(String::as_str)) + .filter(|name| maintained.contains(name)) + .collect(); + models + .iter() + .filter(|model| !model.has_refresh && !read.contains(model.name.as_str())) + .map(|model| model.name.as_str()) + .min() +} diff --git a/crates/sqlbuild-analysis/src/assembly/maintained_checks/mod.rs b/crates/sqlbuild-analysis/src/assembly/maintained_checks/mod.rs new file mode 100644 index 0000000000..042268f15e --- /dev/null +++ b/crates/sqlbuild-analysis/src/assembly/maintained_checks/mod.rs @@ -0,0 +1,6 @@ +pub mod constants; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-analysis/src/assembly/maintained_checks/models.rs b/crates/sqlbuild-analysis/src/assembly/maintained_checks/models.rs new file mode 100644 index 0000000000..4724ca32f3 --- /dev/null +++ b/crates/sqlbuild-analysis/src/assembly/maintained_checks/models.rs @@ -0,0 +1,9 @@ +//! What the project-wide maintained model checks read. + +/// One maintained model: whether it sets its own refresh, and the models it reads. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct MaintainedModel { + pub name: String, + pub has_refresh: bool, + pub reads: Vec, +} diff --git a/crates/sqlbuild-analysis/src/assembly/maintained_checks/tests.rs b/crates/sqlbuild-analysis/src/assembly/maintained_checks/tests.rs new file mode 100644 index 0000000000..e40ce2efc9 --- /dev/null +++ b/crates/sqlbuild-analysis/src/assembly/maintained_checks/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_maintained_checks.rs"] +mod maintained_checks; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-analysis/src/assembly/maintained_checks/tests/helpers.rs b/crates/sqlbuild-analysis/src/assembly/maintained_checks/tests/helpers.rs new file mode 100644 index 0000000000..2ad27cec20 --- /dev/null +++ b/crates/sqlbuild-analysis/src/assembly/maintained_checks/tests/helpers.rs @@ -0,0 +1,10 @@ +use crate::assembly::maintained_checks::models::MaintainedModel; + +/// A maintained model named `name` reading `reads`. +pub(crate) fn maintained(name: &str, has_refresh: bool, reads: &[&str]) -> MaintainedModel { + MaintainedModel { + name: name.to_owned(), + has_refresh, + reads: reads.iter().map(|read| (*read).to_owned()).collect(), + } +} diff --git a/crates/sqlbuild-analysis/src/assembly/maintained_checks/tests/test_maintained_checks.rs b/crates/sqlbuild-analysis/src/assembly/maintained_checks/tests/test_maintained_checks.rs new file mode 100644 index 0000000000..f363cfca59 --- /dev/null +++ b/crates/sqlbuild-analysis/src/assembly/maintained_checks/tests/test_maintained_checks.rs @@ -0,0 +1,52 @@ +use crate::assembly::maintained_checks::main::never_refreshed_maintained::never_refreshed_maintained; +use crate::assembly::maintained_checks::tests::helpers::maintained; +use crate::assembly::maintained_checks::tests::test_types::NeverRefreshedTestCase; + +#[test] +fn given_maintained_chains_when_checking_then_unreached_models_without_refresh_are_named() { + let test_cases = [ + NeverRefreshedTestCase { + description: "an upstream model refreshed for its reader", + models: vec![ + maintained("clean_orders", false, &["orders"]), + maintained("daily_totals", true, &["clean_orders"]), + ], + expected_name: None, + }, + NeverRefreshedTestCase { + description: "the last model of a chain without refresh", + models: vec![ + maintained("clean_orders", false, &["orders"]), + maintained("daily_totals", false, &["clean_orders"]), + ], + expected_name: Some("daily_totals"), + }, + NeverRefreshedTestCase { + description: "a model read only by a table, which never asks for a refresh", + models: vec![maintained("clean_orders", false, &["orders"])], + expected_name: Some("clean_orders"), + }, + NeverRefreshedTestCase { + description: "two unread models report the first by name", + models: vec![ + maintained("b_totals", false, &["orders"]), + maintained("a_totals", false, &["orders"]), + ], + expected_name: Some("a_totals"), + }, + NeverRefreshedTestCase { + description: "every model sets its own refresh", + models: vec![maintained("daily_totals", true, &[])], + expected_name: None, + }, + ]; + + for test_case in test_cases { + assert_eq!( + never_refreshed_maintained(&test_case.models), + test_case.expected_name, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/assembly/maintained_checks/tests/test_types.rs b/crates/sqlbuild-analysis/src/assembly/maintained_checks/tests/test_types.rs new file mode 100644 index 0000000000..66f9fbe2f1 --- /dev/null +++ b/crates/sqlbuild-analysis/src/assembly/maintained_checks/tests/test_types.rs @@ -0,0 +1,7 @@ +use crate::assembly::maintained_checks::models::MaintainedModel; + +pub(crate) struct NeverRefreshedTestCase { + pub(crate) description: &'static str, + pub(crate) models: Vec, + pub(crate) expected_name: Option<&'static str>, +} diff --git a/crates/sqlbuild-analysis/src/assembly/mod.rs b/crates/sqlbuild-analysis/src/assembly/mod.rs index 1b4fb3a6af..4e62fc77e8 100644 --- a/crates/sqlbuild-analysis/src/assembly/mod.rs +++ b/crates/sqlbuild-analysis/src/assembly/mod.rs @@ -1,5 +1,6 @@ //! The native model analysis session: catalog, dataflow scheduling, projection and shape publication. pub mod analysis_session; +pub mod maintained_checks; pub mod project; pub mod target_checks; diff --git a/crates/sqlbuild-discovery/src/constants.rs b/crates/sqlbuild-discovery/src/constants.rs index f6c8b8b807..5991fd2adb 100644 --- a/crates/sqlbuild-discovery/src/constants.rs +++ b/crates/sqlbuild-discovery/src/constants.rs @@ -54,7 +54,7 @@ pub const LOCAL_AUDIT_ROLE_DIRECTORY: &str = "_audits"; /// The singular audit directory below an audit role. pub const SINGULAR_AUDIT_DIRECTORY: &str = "singular"; /// Header keys a `MODEL` statement accepts, sorted by code point. -pub const SQL_MODEL_HEADER_KEYS: [&str; 75] = [ +pub const SQL_MODEL_HEADER_KEYS: [&str; 79] = [ "alias", "append_cursor_inclusive", "audit_factories", @@ -109,6 +109,10 @@ pub const SQL_MODEL_HEADER_KEYS: [&str; 75] = [ "post_hooks", "pre_hooks", "primary_key", + "refresh", + "refresh_mode", + "refresh_on_build", + "refresh_warehouse", "replace_key", "replay_on_change", "row_diff_exclude_columns", diff --git a/crates/sqlbuild-model-config/src/model_validation/_helpers/materialization.rs b/crates/sqlbuild-model-config/src/model_validation/_helpers/materialization.rs index ddc75eeec5..e3633aac1a 100644 --- a/crates/sqlbuild-model-config/src/model_validation/_helpers/materialization.rs +++ b/crates/sqlbuild-model-config/src/model_validation/_helpers/materialization.rs @@ -11,6 +11,7 @@ use crate::model_validation::_helpers::config::python_lower; use crate::model_validation::_helpers::config::{ ConfigView, model_header_help, non_blank_string_check, one_of, sorted_values, text_if_string, }; +use crate::model_validation::_helpers::durations::parse_duration; use crate::model_validation::constants::{ BUILTIN_MATERIALIZATIONS, CONTRACT_POLICIES, CUSTOM_MATERIALIZATION_DISALLOWED_KEYS, HISTORY_MATERIALIZATIONS, INCREMENTAL_MATERIALIZATION, INCREMENTAL_ONLY_KEYS, @@ -26,6 +27,12 @@ use crate::model_validation::constants::{ PHYSICAL_STORAGE_MATERIALIZATIONS, PRIMARY_KEY_KEY, TABLE_SETTINGS_KEY, UNSUPPORTED_ENGINE_PREFIXES, }; +use crate::model_validation::constants::{ + MAINTAINED_ADAPTER_CODE, MAINTAINED_ADAPTERS, MAINTAINED_MATERIALIZATION, MAINTAINED_ONLY_KEYS, + MAINTAINED_REFRESH_FORMS, MAINTAINED_REFRESH_MODES, MAINTAINED_REFRESH_UNSUPPORTED_CODE, + MATERIALIZED_VIEW_ADAPTER_CODE, MATERIALIZED_VIEW_MATERIALIZATION, SNAPSHOT_DISALLOWED_KEYS, + SNOWFLAKE_ADAPTER, SNOWFLAKE_MIN_LAG_SECONDS, +}; use crate::model_validation::models::{ ModelValidationFacts, ProjectValidationFacts, ValidationStop, }; @@ -36,6 +43,11 @@ const MIGRATE_FROM_KEY: &str = "migrate_from"; /// The model key that marks a model whose downstream paths `sqb review` validates. const CRITICAL_KEY: &str = "critical"; const MIGRATE_FORCE_KEY: &str = "migrate_force"; +const REFRESH_KEY: &str = "refresh"; +const LAG_FORM: &str = "lag"; +const REFRESH_MODE_KEY: &str = "refresh_mode"; +const REFRESH_WAREHOUSE_KEY: &str = "refresh_warehouse"; +const REFRESH_ON_BUILD_KEY: &str = "refresh_on_build"; /// Check `validate_microbatch_project_capability`. pub(crate) fn check_project_capability( @@ -609,3 +621,136 @@ fn setting_name(name: &str) -> bool { .chars() .all(|character| character.is_ascii_alphanumeric() || character == '_') } + +/// Check the refresh fields: only maintained models on an adapter that builds them take them. +pub(crate) fn check_maintained( + config: &ConfigView<'_, N>, + project: &ProjectValidationFacts, +) -> Check { + let materialized: Option = config.string("materialized")?; + let adapter: Option<&str> = project.adapter.as_deref(); + if materialized.as_deref() == Some(MATERIALIZED_VIEW_MATERIALIZATION) + && adapter == Some(SNOWFLAKE_ADAPTER) + && !project + .custom_materializations + .contains(MATERIALIZED_VIEW_MATERIALIZATION) + { + return Err(coded( + config, + MATERIALIZED_VIEW_ADAPTER_CODE, + "Snowflake has no materialized_view materialization; use materialized maintained, \ + which builds a dynamic table", + )); + } + if materialized.as_deref() != Some(MAINTAINED_MATERIALIZATION) { + return match config.first_present(&MAINTAINED_ONLY_KEYS) { + Some(key) => Err(config.error(format!("{key} is only valid for maintained models"))), + None => Ok(()), + }; + } + if let Some(adapter) = adapter.filter(|adapter| !one_of(adapter, &MAINTAINED_ADAPTERS)) { + return Err(coded( + config, + MAINTAINED_ADAPTER_CODE, + format!( + "materialized maintained is not supported on the {adapter} adapter; adapters \ + that support it: {}", + MAINTAINED_ADAPTERS.join(", ") + ), + )); + } + if let Some(key) = config.first_present(&SNAPSHOT_DISALLOWED_KEYS) { + return Err(config.error(format!( + "{key} is not valid for maintained models: the warehouse refreshes them from the \ + whole query" + ))); + } + check_refresh(config, adapter)?; + if let Some(mode) = config.get(REFRESH_MODE_KEY) { + let text: Option = text_if_string(mode); + if !text + .as_deref() + .is_some_and(|text| one_of(text, &MAINTAINED_REFRESH_MODES)) + { + return Err(config.error(format!( + "{REFRESH_MODE_KEY} must be one of: {}; got '{}'", + sorted_values(&MAINTAINED_REFRESH_MODES), + text.unwrap_or_default() + ))); + } + } + if config + .get(REFRESH_WAREHOUSE_KEY) + .is_some_and(|warehouse| !non_blank_string_check(warehouse)) + { + return Err(config.error(format!( + "{REFRESH_WAREHOUSE_KEY} must be a non-empty warehouse name" + ))); + } + match config.get(REFRESH_ON_BUILD_KEY).map(AuthoredNode::kind) { + None | Some(NodeKind::Bool(_)) => Ok(()), + Some(_) => Err(config.error(format!("{REFRESH_ON_BUILD_KEY} must be a boolean"))), + } +} + +/// Check that `refresh` is a block with exactly one known form and a valid value. +fn check_refresh(config: &ConfigView<'_, N>, adapter: Option<&str>) -> Check { + let Some(refresh) = config.get(REFRESH_KEY) else { + return Ok(()); + }; + if refresh.kind() != NodeKind::Map { + return Err(config.error(format!( + "{REFRESH_KEY} must be a block such as refresh (lag 1h)" + ))); + } + let block: Vec<(N, N)> = refresh.entries(); + let forms: Vec = block + .iter() + .map(|(name, _)| name.text().unwrap_or_default()) + .collect(); + let [(form, value)] = block.as_slice() else { + return Err(unknown_form(config, &forms)); + }; + if !MAINTAINED_REFRESH_FORMS + .iter() + .any(|known| form.is_text(known)) + { + return Err(unknown_form(config, &forms)); + } + let text: String = text_if_string(value).unwrap_or_default(); + let Ok(Some(lag)) = parse_duration(&text) else { + return Err(not_fixed(config, &text)); + }; + if lag.total_months != 0 { + return Err(not_fixed(config, &text)); + } + if adapter == Some(SNOWFLAKE_ADAPTER) && lag.fixed_seconds < SNOWFLAKE_MIN_LAG_SECONDS { + return Err(coded( + config, + MAINTAINED_REFRESH_UNSUPPORTED_CODE, + format!("{REFRESH_KEY} {LAG_FORM} on Snowflake must be at least 1m; got '{text}'"), + )); + } + Ok(()) +} + +fn not_fixed(config: &ConfigView<'_, N>, text: &str) -> ValidationStop { + config.error(format!( + "{REFRESH_KEY} {LAG_FORM} must be a fixed duration such as 5m, 1h or 1d; got '{text}'" + )) +} + +fn coded( + config: &ConfigView<'_, N>, + code: &'static str, + text: impl std::fmt::Display, +) -> ValidationStop { + ValidationStop::Error(config.config_error(text).with_code(code)) +} + +fn unknown_form(config: &ConfigView<'_, N>, forms: &[String]) -> ValidationStop { + config.error(format!( + "{REFRESH_KEY} takes exactly one form, such as refresh (lag 1h); got: {}", + forms.join(", ") + )) +} diff --git a/crates/sqlbuild-model-config/src/model_validation/constants.rs b/crates/sqlbuild-model-config/src/model_validation/constants.rs index 87b9298bc3..bb0dad808a 100644 --- a/crates/sqlbuild-model-config/src/model_validation/constants.rs +++ b/crates/sqlbuild-model-config/src/model_validation/constants.rs @@ -1,7 +1,8 @@ //! Config keys and vocabularies the Python model validators accept. /// Built-in materializations that never name a custom materialization. -pub const BUILTIN_MATERIALIZATIONS: [&str; 4] = ["view", "table", "incremental", "snapshot"]; +pub const BUILTIN_MATERIALIZATIONS: [&str; 5] = + ["view", "table", "incremental", "snapshot", "maintained"]; /// Materializations that own a physical managed table. pub const TABLE_BACKED_MATERIALIZATIONS: [&str; 3] = ["table", "incremental", "snapshot"]; /// Materializations that keep history across runs. @@ -14,6 +15,33 @@ pub const TABLE_MATERIALIZATION: &str = "table"; pub const INCREMENTAL_MATERIALIZATION: &str = "incremental"; /// The snapshot materialization. pub const SNAPSHOT_MATERIALIZATION: &str = "snapshot"; +/// The materialization whose stored result the warehouse keeps fresh between builds. +pub const MAINTAINED_MATERIALIZATION: &str = "maintained"; +/// Keys only a maintained model accepts. +pub const MAINTAINED_ONLY_KEYS: [&str; 4] = [ + "refresh", + "refresh_mode", + "refresh_warehouse", + "refresh_on_build", +]; +/// The `refresh ( … )` forms any adapter accepts; each adapter accepts only its own. +pub const MAINTAINED_REFRESH_FORMS: [&str; 1] = ["lag"]; +/// `refresh_mode` values. +pub const MAINTAINED_REFRESH_MODES: [&str; 4] = ["incremental", "full", "auto", "adaptive"]; +/// Adapters that build maintained models, sorted. +pub const MAINTAINED_ADAPTERS: [&str; 1] = ["snowflake"]; +/// The Snowflake adapter name. +pub const SNOWFLAKE_ADAPTER: &str = "snowflake"; +/// The materialization name other warehouses use, which Snowflake builds as `maintained`. +pub const MATERIALIZED_VIEW_MATERIALIZATION: &str = "materialized_view"; +/// Snowflake's smallest dynamic table target lag, in seconds. +pub const SNOWFLAKE_MIN_LAG_SECONDS: u128 = 60; +/// The code of a maintained model on an adapter that cannot build one. +pub const MAINTAINED_ADAPTER_CODE: &str = "A013"; +/// The code of `materialized_view` on Snowflake, which builds `maintained` instead. +pub const MATERIALIZED_VIEW_ADAPTER_CODE: &str = "A014"; +/// The code of a refresh form or value the adapter does not accept. +pub const MAINTAINED_REFRESH_UNSUPPORTED_CODE: &str = "A015"; /// `IncrementalStrategy` values. pub const INCREMENTAL_STRATEGIES: [&str; 3] = ["append", "delete_insert", "merge"]; diff --git a/crates/sqlbuild-model-config/src/model_validation/main/validate_model_config.rs b/crates/sqlbuild-model-config/src/model_validation/main/validate_model_config.rs index ea58dfe598..23eb12f947 100644 --- a/crates/sqlbuild-model-config/src/model_validation/main/validate_model_config.rs +++ b/crates/sqlbuild-model-config/src/model_validation/main/validate_model_config.rs @@ -3,9 +3,9 @@ use crate::model_validation::_helpers::config::ConfigView; use crate::model_validation::_helpers::incremental::check_incremental; use crate::model_validation::_helpers::materialization::{ - check_contract, check_custom_materialization, check_execution_limit, check_migration, - check_non_incremental, check_physical_storage, check_placeholders, check_project_capability, - check_review_marker, check_storage_policies, + check_contract, check_custom_materialization, check_execution_limit, check_maintained, + check_migration, check_non_incremental, check_physical_storage, check_placeholders, + check_project_capability, check_review_marker, check_storage_policies, }; use crate::model_validation::_helpers::references::check_references; use crate::model_validation::_helpers::snapshot::check_snapshot; @@ -29,6 +29,7 @@ pub fn validate_model_config( check_review_marker(&config)?; check_non_incremental(&config)?; check_snapshot(&config, facts)?; + check_maintained(&config, project)?; check_custom_materialization(&config, project)?; check_storage_policies(&config, facts)?; check_physical_storage(&config)?; diff --git a/crates/sqlbuild-model-config/src/model_validation/models.rs b/crates/sqlbuild-model-config/src/model_validation/models.rs index c456da27dd..5caa385c4f 100644 --- a/crates/sqlbuild-model-config/src/model_validation/models.rs +++ b/crates/sqlbuild-model-config/src/model_validation/models.rs @@ -21,6 +21,8 @@ pub struct ProjectValidationFacts { pub functions: HashSet, /// Discovered table function names. pub table_functions: HashSet, + /// The effective adapter name, when the project names one. + pub adapter: Option, } /// One extracted SQL reference: its `SqlReferenceKind` value and name. diff --git a/crates/sqlbuild-model-config/src/model_validation/tests/helpers.rs b/crates/sqlbuild-model-config/src/model_validation/tests/helpers.rs index 45308d1651..aabfa836a3 100644 --- a/crates/sqlbuild-model-config/src/model_validation/tests/helpers.rs +++ b/crates/sqlbuild-model-config/src/model_validation/tests/helpers.rs @@ -80,6 +80,7 @@ pub(super) fn project() -> ProjectValidationFacts { sources: names(&["raw.orders"]), functions: names(&["cents", "order_lines"]), table_functions: names(&["order_lines"]), + adapter: Some("snowflake".to_owned()), } } diff --git a/crates/sqlbuild-model-config/src/model_validation/tests/test_validate_model_config.rs b/crates/sqlbuild-model-config/src/model_validation/tests/test_validate_model_config.rs index 24ecccd910..6375dca5f1 100644 --- a/crates/sqlbuild-model-config/src/model_validation/tests/test_validate_model_config.rs +++ b/crates/sqlbuild-model-config/src/model_validation/tests/test_validate_model_config.rs @@ -553,3 +553,151 @@ fn given_effective_configs_when_validating_natively_then_python_outcomes_result( ); } } + +#[test] +fn given_maintained_model_headers_when_validating_then_refresh_fields_are_checked() { + let maintained = |extra: Vec<(&'static str, Value)>| { + let mut config = vec![("materialized", Value::Str("maintained"))]; + config.extend(extra); + config + }; + let test_cases = [ + ValidateTestCase { + description: "a maintained model with a lag, a mode and a warehouse", + config: maintained(vec![ + ("refresh", map(vec![("lag", Value::Str("1h"))])), + ("refresh_mode", Value::Str("adaptive")), + ("refresh_warehouse", Value::Str("transform_wh")), + ("refresh_on_build", Value::Bool(true)), + ]), + references: &["ref:orders"], + query_sql: "select 1", + expected_outcome: "accepted", + }, + ValidateTestCase { + description: "a maintained model refreshed by its readers", + config: maintained(vec![]), + references: &["ref:orders"], + query_sql: "select 1", + expected_outcome: "accepted", + }, + ValidateTestCase { + description: "refresh on a table", + config: vec![ + ("materialized", Value::Str("table")), + ("refresh", map(vec![("lag", Value::Str("1h"))])), + ], + references: &[], + query_sql: "select 1", + expected_outcome: "model 'orders_daily': refresh is only valid for maintained models", + }, + ValidateTestCase { + description: "refresh_on_build on a model with no materialization", + config: vec![("refresh_on_build", Value::Bool(true))], + references: &[], + query_sql: "select 1", + expected_outcome: "model 'orders_daily': refresh_on_build is only valid for maintained models", + }, + ValidateTestCase { + description: "a refresh block with a form no adapter has", + config: maintained(vec![("refresh", map(vec![("every", Value::Str("1h"))]))]), + references: &[], + query_sql: "select 1", + expected_outcome: "model 'orders_daily': refresh takes exactly one form, such as refresh (lag 1h); got: every", + }, + ValidateTestCase { + description: "a refresh block with two forms", + config: maintained(vec![( + "refresh", + map(vec![ + ("lag", Value::Str("1h")), + ("lag_cap", Value::Str("2h")), + ]), + )]), + references: &[], + query_sql: "select 1", + expected_outcome: "model 'orders_daily': refresh takes exactly one form, such as refresh (lag 1h); got: lag, lag_cap", + }, + ValidateTestCase { + description: "refresh written as a plain value", + config: maintained(vec![("refresh", Value::Str("1h"))]), + references: &[], + query_sql: "select 1", + expected_outcome: "model 'orders_daily': refresh must be a block such as refresh (lag 1h)", + }, + ValidateTestCase { + description: "a lag in months", + config: maintained(vec![("refresh", map(vec![("lag", Value::Str("1mo"))]))]), + references: &[], + query_sql: "select 1", + expected_outcome: "model 'orders_daily': refresh lag must be a fixed duration such as 5m, 1h or 1d; got '1mo'", + }, + ValidateTestCase { + description: "a lag that is not a duration", + config: maintained(vec![("refresh", map(vec![("lag", Value::Str("hourly"))]))]), + references: &[], + query_sql: "select 1", + expected_outcome: "model 'orders_daily': refresh lag must be a fixed duration such as 5m, 1h or 1d; got 'hourly'", + }, + ValidateTestCase { + description: "an unknown refresh mode", + config: maintained(vec![("refresh_mode", Value::Str("fast"))]), + references: &[], + query_sql: "select 1", + expected_outcome: "model 'orders_daily': refresh_mode must be one of: adaptive, auto, full, incremental; got 'fast'", + }, + ValidateTestCase { + description: "refresh_on_build that is not a boolean", + config: maintained(vec![("refresh_on_build", Value::Str("yes"))]), + references: &[], + query_sql: "select 1", + expected_outcome: "model 'orders_daily': refresh_on_build must be a boolean", + }, + ValidateTestCase { + description: "a blank refresh warehouse", + config: maintained(vec![("refresh_warehouse", Value::Str(" "))]), + references: &[], + query_sql: "select 1", + expected_outcome: "model 'orders_daily': refresh_warehouse must be a non-empty warehouse name", + }, + ValidateTestCase { + description: "an incremental-only key on a maintained model", + config: maintained(vec![("cursor", Value::Str("updated_at"))]), + references: &[], + query_sql: "select 1", + expected_outcome: "model 'orders_daily': cursor is not valid for maintained models: the warehouse refreshes them from the whole query", + }, + ValidateTestCase { + description: "the smallest lag Snowflake accepts", + config: maintained(vec![("refresh", map(vec![("lag", Value::Str("1m"))]))]), + references: &[], + query_sql: "select 1", + expected_outcome: "accepted", + }, + ValidateTestCase { + description: "a lag one second under Snowflake's minimum", + config: maintained(vec![("refresh", map(vec![("lag", Value::Str("59s"))]))]), + references: &[], + query_sql: "select 1", + expected_outcome: "model 'orders_daily': refresh lag on Snowflake must be at least 1m; got '59s'", + }, + ValidateTestCase { + description: "materialized_view on Snowflake", + config: vec![("materialized", Value::Str("materialized_view"))], + references: &[], + query_sql: "select 1", + expected_outcome: "model 'orders_daily': Snowflake has no materialized_view materialization; use materialized maintained, which builds a dynamic table", + }, + ]; + + for test_case in test_cases { + let outcome = + validation_outcome(test_case.config, test_case.references, test_case.query_sql); + + assert_eq!( + outcome, test_case.expected_outcome, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/maintained.rs b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/maintained.rs new file mode 100644 index 0000000000..cb045e0beb --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/maintained.rs @@ -0,0 +1,68 @@ +//! Project-wide checks of maintained models on the effective adapter. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; + +use sqlbuild_analysis::assembly::maintained_checks::constants::MAINTAINED_NEVER_REFRESHED_CODE; +use sqlbuild_analysis::assembly::maintained_checks::main::never_refreshed_maintained::never_refreshed_maintained; +use sqlbuild_analysis::assembly::maintained_checks::models::MaintainedModel; +use sqlbuild_python_base::boundary::main::optional_text::optional_text; +use sqlbuild_python_base::records::main::construct::enum_member::enum_member; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_front::model_config::main::config::config_text::config_text; +use sqlbuild_python_front::model_config::main::config::config_value::config_value; +use sqlbuild_python_front::model_config::main::config::model_values::model_values; +use sqlbuild_python_front::model_config::types::ConfigValues; + +use crate::compile_driver::_helpers::setup::targets::planner_error; + +const SNOWFLAKE_ADAPTER: &str = "snowflake"; +const MAINTAINED_MATERIALIZATION: &str = "maintained"; +const REFRESH_KEY: &str = "refresh"; + +/// Raise when a Snowflake maintained model has no refresh and no maintained model reads it. +pub(crate) fn maintained_refresh( + adapter_name: &Bound<'_, PyAny>, + project: &Bound<'_, PyAny>, +) -> PyResult<()> { + if optional_text(adapter_name)?.as_deref() != Some(SNOWFLAKE_ADAPTER) { + return Ok(()); + } + let py: Python<'_> = project.py(); + let model_type: Bound<'_, PyAny> = enum_member(py, RecordId::CompiledResourceType, "MODEL")?; + let mut maintained: Vec = Vec::new(); + for model in native_field(project, "models")?.try_iter()? { + let model: Bound<'_, PyAny> = model?; + let values: ConfigValues<'_> = model_values(&model)?; + if config_text(py, &values, "materialized")?.as_deref() != Some(MAINTAINED_MATERIALIZATION) + { + continue; + } + let mut reads: Vec = Vec::new(); + for key in native_field(&model, "deps")?.try_iter()? { + let key: Bound<'_, PyAny> = key?; + if native_field(&key, "resource_type")?.eq(&model_type)? { + reads.push(native_field(&key, "name")?.extract()?); + } + } + maintained.push(MaintainedModel { + name: native_field(&model, "name")?.extract()?, + has_refresh: !config_value(py, &values, REFRESH_KEY)?.is_none(), + reads, + }); + } + let Some(name) = never_refreshed_maintained(&maintained) else { + return Ok(()); + }; + Err(planner_error( + py, + format!( + "maintained model '{name}' has no refresh and no maintained model reads it, so \ + Snowflake would never refresh it; add refresh (lag 1h)" + ), + MAINTAINED_NEVER_REFRESHED_CODE, + Some(format!( + "set how stale '{name}' may get in its MODEL header:\n refresh (lag 1h)," + )), + )) +} diff --git a/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/mod.rs b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/mod.rs index 4f83d12b05..961c650c8f 100644 --- a/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/mod.rs +++ b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/mod.rs @@ -1,4 +1,5 @@ pub(crate) mod inputs; +pub(crate) mod maintained; pub(crate) mod model_free_locations; pub(crate) mod node_results; pub(crate) mod pipeline; diff --git a/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/project.rs b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/project.rs index 5af92903d1..16444bd260 100644 --- a/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/project.rs +++ b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/project.rs @@ -14,6 +14,7 @@ use pyo3::types::{PyBool, PyCFunction, PyDict, PyTuple}; use sqlbuild_host::compile_context::main::record_compile_timing::record_compile_timing; use sqlbuild_python_project::graph::main::python_nodes::resolve_node_sets::resolve_node_sets; +use crate::compile_driver::_helpers::setup::maintained::maintained_refresh; use crate::compile_driver::_helpers::setup::target_defaults::TargetDefaults; use crate::compile_driver::_helpers::setup::target_defaults::apply_target_defaults; use crate::compile_driver::_helpers::setup::targets::managed_loader_target_isolation; @@ -242,6 +243,7 @@ fn targeted_project<'py>( }; let project: Bound<'py, PyAny> = apply_target_defaults(project, &defaults)?; project_targets(&adapter_name, &project)?; + maintained_refresh(&adapter_name, &project)?; let promotion_key: Bound<'py, PyAny> = pyo3::types::PyString::new(py, TABLE_PROMOTION_MODE_SETTING_KEY).into_any(); let settings_file: Bound<'py, PyAny> = diff --git a/crates/sqlbuild-python-front/src/model_config/models.rs b/crates/sqlbuild-python-front/src/model_config/models.rs index c1b99e21ea..9999245cf3 100644 --- a/crates/sqlbuild-python-front/src/model_config/models.rs +++ b/crates/sqlbuild-python-front/src/model_config/models.rs @@ -3,6 +3,7 @@ use crate::attachments::main::argument_pairs::argument_pairs; use crate::attachments::main::config_argument_value::config_argument_value; use crate::authored_values::main::project_var_text::project_var_text; +use crate::contracts::main::resolve_effective_adapter_name::resolve_effective_adapter_name; use crate::discovery::main::project::each::each; use crate::discovery::models::NativeModelFile; use crate::model_config::_helpers::config::config_classes::built_config; @@ -1356,7 +1357,17 @@ impl NativeModelValidator { })? .into_iter() .collect(); - Self::new(names, custom_materializations, microbatch_concurrency) + let adapter: Option = resolve_effective_adapter_name( + &native_field(&inputs, "project_config")?, + &native_field(&inputs, "local_config")?, + )? + .extract()?; + Self::new( + names, + custom_materializations, + microbatch_concurrency, + adapter, + ) } /// The discovered model names references resolve against. @@ -1365,10 +1376,12 @@ impl NativeModelValidator { } #[new] + #[pyo3(signature = (names, custom_materializations, microbatch_concurrency, adapter=None))] pub(crate) fn new( names: ResourceNames, custom_materializations: HashSet, microbatch_concurrency: bool, + adapter: Option, ) -> PyResult { let ResourceNames { models, @@ -1386,6 +1399,7 @@ impl NativeModelValidator { sources, functions, table_functions, + adapter, }, }) } diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index fe94cec490..949d974f84 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -4316,6 +4316,7 @@ class NativeModelValidator: names: tuple[set[str], set[str], set[str], set[str], set[str]], custom_materializations: set[str], microbatch_concurrency: bool, + adapter: str | None = None, ) -> None: ... @staticmethod def from_inputs( diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py index ba09c7b216..639178b99d 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py @@ -1461,3 +1461,14 @@ class JobGroupCyclesCase: description: str fixed_group_toml: str expected_messages: tuple[str, ...] + + +@dataclass(frozen=True) +class MaintainedCompileE2ETestCase: + """A project with maintained models, its exit code and text its compile output contains.""" + + description: str + adapter: str + daily_totals_header: str + expected_exit_code: int + expected_fragments: tuple[str, ...] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_maintained_models.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_maintained_models.py new file mode 100644 index 0000000000..da2449debf --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_maintained_models.py @@ -0,0 +1,115 @@ +"""E2E coverage of maintained models: which adapters and headers compile, and the errors they get.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import ( + MaintainedCompileE2ETestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import prepare_inline_project, run_sqb + +_PROJECT_TOML: dict[str, str] = { + "snowflake": ( + 'name = "shop"\nadapter = "snowflake"\n\n[connection]\naccount = "example"\n' + 'user = "builder"\n\n[defaults]\ndatabase = "ANALYTICS"\nschema = "MARTS"\n\n' + "[scopes]\nenforce_placement = false\n" + ), + "duckdb": ( + 'name = "shop"\nadapter = "duckdb"\n\n[connection]\npath = "shop.duckdb"\n\n' + "[scopes]\nenforce_placement = false\n" + ), +} +_ORDERS: str = ( + "MODEL (description 'Orders.', materialized table);\nSELECT 1 AS order_id, 10 AS amount\n" +) +_CLEAN_ORDERS: str = ( + "MODEL (description 'Orders with an amount.', materialized maintained);\n" + 'SELECT order_id, amount FROM __ref("orders") WHERE amount > 0\n' +) +_DAILY_TOTALS_QUERY: str = 'SELECT SUM(amount) AS total FROM __ref("clean_orders")\n' + + +@pytest.mark.parametrize( + "test_case", + [ + MaintainedCompileE2ETestCase( + description="a chain refreshed by the lag of its last model compiles on Snowflake", + adapter="snowflake", + daily_totals_header="materialized maintained, refresh (lag 1h), refresh_mode full", + expected_exit_code=0, + expected_fragments=("Project compiled 3 models",), + ), + MaintainedCompileE2ETestCase( + description="maintained models are refused on DuckDB", + adapter="duckdb", + daily_totals_header="materialized maintained, refresh (lag 1h)", + expected_exit_code=1, + expected_fragments=( + "[A013]", + "materialized maintained is not supported on the duckdb adapter; " + "adapters that support it: snowflake", + ), + ), + MaintainedCompileE2ETestCase( + description="materialized_view on Snowflake points to maintained", + adapter="snowflake", + daily_totals_header="materialized materialized_view", + expected_exit_code=1, + expected_fragments=( + "[A014]", + "Snowflake has no materialized_view materialization; use materialized " + "maintained, which builds a dynamic table", + ), + ), + MaintainedCompileE2ETestCase( + description="a lag below Snowflake's minimum", + adapter="snowflake", + daily_totals_header="materialized maintained, refresh (lag 30s)", + expected_exit_code=1, + expected_fragments=( + "[A015]", + "refresh lag on Snowflake must be at least 1m; got '30s'", + ), + ), + MaintainedCompileE2ETestCase( + description="a last maintained model with no refresh would never refresh", + adapter="snowflake", + daily_totals_header="materialized maintained", + expected_exit_code=1, + expected_fragments=( + "[A016]", + "maintained model 'daily_totals' has no refresh and no maintained model reads " + "it, so Snowflake would never refresh it; add refresh (lag 1h)", + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_maintained_models_when_compiling_then_adapter_rules_apply( + test_case: MaintainedCompileE2ETestCase, tmp_path: Path +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="shop", + repo_files={ + "sqlbuild_project.toml": _PROJECT_TOML[test_case.adapter], + "models/orders.sql": _ORDERS, + "models/clean_orders.sql": _CLEAN_ORDERS, + "models/daily_totals.sql": ( + f"MODEL (description 'Daily totals.', {test_case.daily_totals_header});\n" + f"{_DAILY_TOTALS_QUERY}" + ), + }, + ) + + compiled: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "compile"), project_dir=project_dir + ) + + output: str = compiled.stdout + compiled.stderr + assert compiled.returncode == test_case.expected_exit_code, output + assert all(map(output.__contains__, test_case.expected_fragments)), output From f1e16016ac094e10645f23f27100fdf156ce9ef3 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 18 Nov 2026 22:20:15 +0000 Subject: [PATCH 02/17] feat(config): add maintained-model pause settings Work in progress. [maintained] heartbeat_grace (fixed duration, default 30m) and allow_ignore_grace (default false), read with load_maintained_config and validated on every project load, and the [targets.] pause_maintained flag on project and local targets. Validated: sqlbuild-config tests (45, new cases fail without the loader), native stub check, target config unit tests. Known gap: fensu FPRSL105 until the build reads load_maintained_config. --- .../src/effective/constants.rs | 7 ++ .../src/project/_helpers/documents.rs | 14 ++- .../src/project/_helpers/sections.rs | 58 +++++++-- .../src/project/_helpers/targets.rs | 9 +- .../sqlbuild-config/src/project/constants.rs | 8 +- .../project/main/load_maintained_config.rs | 9 ++ .../sqlbuild-config/src/project/main/mod.rs | 1 + crates/sqlbuild-config/src/project/models.rs | 9 ++ .../src/project/tests/test_load_config.rs | 118 +++++++++++++++++- .../src/project/tests/test_types.rs | 14 ++- .../_helpers/contract_records/constants.rs | 2 + src/sqlbuild/_native.pyi | 4 + 12 files changed, 234 insertions(+), 19 deletions(-) create mode 100644 crates/sqlbuild-config/src/project/main/load_maintained_config.rs diff --git a/crates/sqlbuild-config/src/effective/constants.rs b/crates/sqlbuild-config/src/effective/constants.rs index 683223e6dd..5508b1565a 100644 --- a/crates/sqlbuild-config/src/effective/constants.rs +++ b/crates/sqlbuild-config/src/effective/constants.rs @@ -238,4 +238,11 @@ pub(crate) const TARGET_FIELDS: &[TargetField] = &[ decided_by: &["plan_from"], }, }, + TargetField { + group: None, + name: "pause_maintained", + rule: MergeRule::Overlay { + decided_by: &["pause_maintained"], + }, + }, ]; diff --git a/crates/sqlbuild-config/src/project/_helpers/documents.rs b/crates/sqlbuild-config/src/project/_helpers/documents.rs index 58d867a89b..bcce7e424e 100644 --- a/crates/sqlbuild-config/src/project/_helpers/documents.rs +++ b/crates/sqlbuild-config/src/project/_helpers/documents.rs @@ -15,8 +15,9 @@ use crate::project::_helpers::help::{ use crate::project::_helpers::keys::Place; use crate::project::_helpers::sections::{ load_changes, load_constants, load_control, load_cost, load_cursors, load_dbt, load_diff, - load_git, load_janitor, load_local_dbt, load_microbatches, load_migrations, load_policy_flag, - load_pools, load_review, load_scenario, load_settings, load_snapshots, setting_overrides, + load_git, load_janitor, load_local_dbt, load_maintained, load_microbatches, load_migrations, + load_policy_flag, load_pools, load_review, load_scenario, load_settings, load_snapshots, + setting_overrides, }; use crate::project::_helpers::sinks::load_sinks; use crate::project::_helpers::targets::{ @@ -30,7 +31,8 @@ use crate::project::constants::{ }; use crate::project::errors::{ConfigFailure, FailureClass, LoadFailure, MessagePiece}; use crate::project::models::{ - Built, ChangesConfig, ConfigFile, ConfigPaths, ControlConfig, PoolLimits, ReviewConfig, + Built, ChangesConfig, ConfigFile, ConfigPaths, ControlConfig, MaintainedConfig, PoolLimits, + ReviewConfig, }; use crate::project::types::{Entries, Fields, Loaded}; use crate::recall::constants::RECALL_SECTION; @@ -297,6 +299,7 @@ fn project_fields(payload: &Entries) -> Loaded { } load_git(get(payload, "git"), PROJECT_FILE)?; load_review(get(payload, "review"), PROJECT_FILE)?; + load_maintained(get(payload, "maintained"), PROJECT_FILE)?; load_control(get(payload, "control"), PROJECT_FILE)?; load_changes(get(payload, "changes"), PROJECT_FILE)?; pool_limits(payload)?; @@ -539,6 +542,11 @@ pub(crate) fn changes_config(paths: &ConfigPaths) -> Result Result { + project_section(paths, "maintained", load_maintained) +} + /// `[review]` of `sqlbuild_project.toml`. pub(crate) fn review_config(paths: &ConfigPaths) -> Result { project_section(paths, "review", load_review) diff --git a/crates/sqlbuild-config/src/project/_helpers/sections.rs b/crates/sqlbuild-config/src/project/_helpers/sections.rs index 7ceafd616b..023949de0e 100644 --- a/crates/sqlbuild-config/src/project/_helpers/sections.rs +++ b/crates/sqlbuild-config/src/project/_helpers/sections.rs @@ -21,21 +21,23 @@ use crate::project::constants::{ use crate::project::constants::{ AUDIT_SEVERITIES, CHECK_DECLARED_COLUMNS_KEY, COLLECTION_RENDERINGS, COLUMN_CONTRACT_MODES, CONCURRENCY_KEY, DBT_DEFER_CLONE_KEY, DBT_KEYS, DBT_REMOVED_KEYS, DBT_REUSE_KEYS, - DEFAULT_IN_USE_WINDOW, DEFAULT_QUERY_ARTIFACT_TTL, FUTURE_CURSOR_ACTIONS, - LEGACY_COLUMN_CONTRACT_MODE_KEY, LEGACY_CONCURRENCY_KEY, LEGACY_SQL_VALIDATION_KEY, - LOCAL_DBT_KEYS, MAX_BATCHES_KEY, MICROBATCH_LIMIT_ACTIONS, OLD_NAME_VIEWS_KEY, - QUERY_CHANGE_TRACKING_KEY, REQUIRE_SQL_ANALYSIS_KEY, REVIEW_DOWNSTREAM_POLICIES, REVIEW_KEYS, - RUN_NAMESPACE_KEY, RUN_NAMESPACE_MAX_LENGTH, SCENARIO_KEYS, SETTINGS_KEYS, - SNAPSHOT_FULL_REFRESH_POLICIES, SNAPSHOT_KEYS, SNAPSHOT_LIMIT_KEYS, - SNAPSHOT_SCHEMA_CHANGE_POLICIES, SQL_ANALYSIS_KEY, UNACCOUNTED_PARTITION_POLICIES, - USD_PER_CREDIT_KEY, ZERO_FIXED_DURATION, ZERO_FLOAT, + DEFAULT_HEARTBEAT_GRACE_SECONDS, DEFAULT_IN_USE_WINDOW, DEFAULT_QUERY_ARTIFACT_TTL, + FUTURE_CURSOR_ACTIONS, LEGACY_COLUMN_CONTRACT_MODE_KEY, LEGACY_CONCURRENCY_KEY, + LEGACY_SQL_VALIDATION_KEY, LOCAL_DBT_KEYS, MAINTAINED_KEYS, MAX_BATCHES_KEY, + MICROBATCH_LIMIT_ACTIONS, OLD_NAME_VIEWS_KEY, QUERY_CHANGE_TRACKING_KEY, + REQUIRE_SQL_ANALYSIS_KEY, REVIEW_DOWNSTREAM_POLICIES, REVIEW_KEYS, RUN_NAMESPACE_KEY, + RUN_NAMESPACE_MAX_LENGTH, SCENARIO_KEYS, SETTINGS_KEYS, SNAPSHOT_FULL_REFRESH_POLICIES, + SNAPSHOT_KEYS, SNAPSHOT_LIMIT_KEYS, SNAPSHOT_SCHEMA_CHANGE_POLICIES, SQL_ANALYSIS_KEY, + UNACCOUNTED_PARTITION_POLICIES, USD_PER_CREDIT_KEY, ZERO_FIXED_DURATION, ZERO_FLOAT, }; use crate::project::constants::{ CHANGE_COUNTS, CHANGE_REACHES, CHANGES_KEYS, CONTROL_KEYS, MAX_POOL_LIMIT, MIN_SYNC_INTERVAL_SECONDS, SYNC_INTERVAL_HELP, SYNC_INTERVAL_OFF, }; use crate::project::errors::ConfigFailure; -use crate::project::models::{Built, ChangesConfig, ConfigFile, ControlConfig, ReviewConfig}; +use crate::project::models::{ + Built, ChangesConfig, ConfigFile, ControlConfig, MaintainedConfig, ReviewConfig, +}; use crate::project::types::{Entries, Fields, Loaded}; use sqlbuild_core::constants::ZERO_DAY_DURATION; @@ -626,6 +628,44 @@ pub(crate) fn sync_interval(text: &str) -> Result { Ok(seconds) } +/// `[maintained]`: how long a silent pausing build keeps its maintained models paused. +pub(crate) fn load_maintained( + payload: Option<&ConfigValue>, + file: ConfigFile, +) -> Loaded { + let mapping = coerce_mapping(payload, place(file, "maintained"))?; + validate_allowed_keys(mapping, &MAINTAINED_KEYS, place(file, "maintained"))?; + let heartbeat_grace_seconds: u64 = match get(mapping, "heartbeat_grace") { + None => DEFAULT_HEARTBEAT_GRACE_SECONDS, + Some(ConfigValue::String(text)) => match fixed_duration_seconds(text, false) { + Ok(Some(seconds)) => seconds.parse().map_err(|_| grace_failure(file))?, + Ok(None) | Err(_) => return Err(grace_failure(file)), + }, + Some(_) => return Err(grace_failure(file)), + }; + let allow_ignore_grace: bool = match get(mapping, "allow_ignore_grace") { + None => false, + Some(ConfigValue::Bool(flag)) => *flag, + Some(_) => { + return Err(ConfigFailure::at( + file, + "maintained.allow_ignore_grace must be true or false", + )); + } + }; + Ok(MaintainedConfig { + heartbeat_grace_seconds, + allow_ignore_grace, + }) +} + +fn grace_failure(file: ConfigFile) -> ConfigFailure { + ConfigFailure::at( + file, + "maintained.heartbeat_grace must be a fixed duration such as 30m or 2h", + ) +} + /// `[control]`: where a signed-in build asks for a sync, and how often a location syncs anyway. pub(crate) fn load_control( payload: Option<&ConfigValue>, diff --git a/crates/sqlbuild-config/src/project/_helpers/targets.rs b/crates/sqlbuild-config/src/project/_helpers/targets.rs index e520bb835e..b8bd8df06b 100644 --- a/crates/sqlbuild-config/src/project/_helpers/targets.rs +++ b/crates/sqlbuild-config/src/project/_helpers/targets.rs @@ -16,8 +16,9 @@ use crate::project::_helpers::sections::{fixed_duration_seconds, item_timeout}; use crate::project::_helpers::values::{int_below, is_int}; use crate::project::constants::{ CLONE_KEYS, CONCURRENCY_KEY, CONFIRMATION_POLICIES, EXECUTION_LIMIT_KEYS, - IGNORE_AUDIT_FAILURES_KEY, MAX_FULL_ROWS_KEY, PLAN_FROM_KEY, PLAN_FROM_SOURCES, POOLS_KEY, - RETENTION_MATERIALIZATIONS, TARGET_KEYS, UNLIMITED_DIFF_FULL_ROWS, WAREHOUSE_GROUPS, + IGNORE_AUDIT_FAILURES_KEY, MAX_FULL_ROWS_KEY, PAUSE_MAINTAINED_KEY, PLAN_FROM_KEY, + PLAN_FROM_SOURCES, POOLS_KEY, RETENTION_MATERIALIZATIONS, TARGET_KEYS, + UNLIMITED_DIFF_FULL_ROWS, WAREHOUSE_GROUPS, }; use crate::project::errors::ConfigFailure; use crate::project::models::{Built, ConfigFile}; @@ -433,6 +434,10 @@ fn local_target(payload: &ConfigValue, target: TargetPlace<'_>) -> Loaded Result { + crate::project::_helpers::documents::maintained_config(paths) +} diff --git a/crates/sqlbuild-config/src/project/main/mod.rs b/crates/sqlbuild-config/src/project/main/mod.rs index 52648c7a86..6d54995e7c 100644 --- a/crates/sqlbuild-config/src/project/main/mod.rs +++ b/crates/sqlbuild-config/src/project/main/mod.rs @@ -4,6 +4,7 @@ pub mod load_changes_config; pub mod load_control_config; pub mod load_git_base_ref; pub mod load_local_config; +pub mod load_maintained_config; pub mod load_pool_limits; pub mod load_project_config; pub mod load_project_setting_keys; diff --git a/crates/sqlbuild-config/src/project/models.rs b/crates/sqlbuild-config/src/project/models.rs index 238a914c04..5fab29b4c0 100644 --- a/crates/sqlbuild-config/src/project/models.rs +++ b/crates/sqlbuild-config/src/project/models.rs @@ -91,6 +91,15 @@ pub struct ChangesConfig { pub reach: Option, } +/// `[maintained]` of `sqlbuild_project.toml`: when a build may take over paused maintained models. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct MaintainedConfig { + /// Seconds a pausing build's heartbeat may be silent before another build resumes its models. + pub heartbeat_grace_seconds: u64, + /// Whether `sqb build --ignore-maintained-grace` may take over within the grace. + pub allow_ignore_grace: bool, +} + /// `[review]` of `sqlbuild_project.toml`: the defaults `sqb review` uses when flags are omitted. #[derive(Clone, Debug, Default, PartialEq, Eq)] pub struct ReviewConfig { diff --git a/crates/sqlbuild-config/src/project/tests/test_load_config.rs b/crates/sqlbuild-config/src/project/tests/test_load_config.rs index 8c8a3fc77a..29edf970c0 100644 --- a/crates/sqlbuild-config/src/project/tests/test_load_config.rs +++ b/crates/sqlbuild-config/src/project/tests/test_load_config.rs @@ -2,14 +2,16 @@ use crate::project::errors::ConfigFailure; use crate::project::main::load_changes_config::load_changes_config; use crate::project::main::load_control_config::load_control_config; use crate::project::main::load_git_base_ref::load_git_base_ref; +use crate::project::main::load_maintained_config::load_maintained_config; use crate::project::main::load_pool_limits::load_pool_limits; use crate::project::main::load_review_config::load_review_config; use crate::project::models::ConfigFile; -use crate::project::models::{ChangesConfig, ControlConfig, ReviewConfig}; +use crate::project::models::{ChangesConfig, ControlConfig, MaintainedConfig, ReviewConfig}; use crate::project::tests::helpers::{failure_at, first_failure, project_directory}; use crate::project::tests::test_types::{ - ChangesConfigTestCase, ControlConfigTestCase, GitBaseRefTestCase, PoolsErrorTestCase, - PoolsTestCase, ProjectFailureTestCase, ReviewConfigErrorTestCase, ReviewConfigTestCase, + ChangesConfigTestCase, ControlConfigTestCase, GitBaseRefTestCase, + MaintainedConfigErrorTestCase, MaintainedConfigTestCase, PoolsErrorTestCase, PoolsTestCase, + ProjectFailureTestCase, ReviewConfigErrorTestCase, ReviewConfigTestCase, }; const PROJECT: &str = "name = 'orders'\nadapter = 'duckdb'\n"; @@ -62,6 +64,29 @@ fn given_project_files_when_loading_then_rejections_match_python_loader() { "Expected 'plan_from' to be one of: live, synced_state", )), }, + ProjectFailureTestCase { + description: "a target may keep its maintained models paused between builds", + project: "name = 'orders'\nadapter = 'snowflake'\n[targets.dev]\npause_maintained = true\n", + local: "[targets.dev]\npause_maintained = false\n", + expected_failure: None, + }, + ProjectFailureTestCase { + description: "pause_maintained is a boolean", + project: "name = 'orders'\nadapter = 'snowflake'\n[targets.dev]\npause_maintained = 'yes'\n", + local: "", + expected_failure: Some(ConfigFailure::plain( + "Expected 'pause_maintained' to be a boolean when provided", + )), + }, + ProjectFailureTestCase { + description: "an invalid maintained section fails the project load", + project: "name = 'orders'\nadapter = 'snowflake'\n[maintained]\nheartbeat_grace = 'soon'\n", + local: "", + expected_failure: Some(ConfigFailure::at( + ConfigFile::Project, + "maintained.heartbeat_grace must be a fixed duration such as 30m or 2h", + )), + }, ProjectFailureTestCase { description: "an unknown key suggests the nearest supported key", project: "name = 'orders'\nadapter = 'duckdb'\n[scopes]\nenforce_placment = true\n", @@ -597,3 +622,90 @@ fn given_invalid_pools_when_loading_then_project_loading_rejects_them() { ); } } + +#[test] +fn given_maintained_section_when_reading_then_returns_its_values_or_defaults() { + let test_cases = [ + MaintainedConfigTestCase { + description: "a longer grace that may be ignored", + project: "name = 'orders'\nadapter = 'snowflake'\n[maintained]\n\ + heartbeat_grace = '45m'\nallow_ignore_grace = true\n", + expected_config: MaintainedConfig { + heartbeat_grace_seconds: 2_700, + allow_ignore_grace: true, + }, + }, + MaintainedConfigTestCase { + description: "a grace shorter than one heartbeat is allowed", + project: "name = 'orders'\nadapter = 'snowflake'\n[maintained]\n\ + heartbeat_grace = '30s'\n", + expected_config: MaintainedConfig { + heartbeat_grace_seconds: 30, + allow_ignore_grace: false, + }, + }, + MaintainedConfigTestCase { + description: "no maintained section", + project: PROJECT, + expected_config: MaintainedConfig { + heartbeat_grace_seconds: 1_800, + allow_ignore_grace: false, + }, + }, + ]; + for test_case in test_cases { + let (_directory, paths) = project_directory(test_case.project, ""); + + let config: MaintainedConfig = load_maintained_config(&paths).expect(test_case.description); + + assert_eq!( + config, test_case.expected_config, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_invalid_maintained_section_when_reading_then_it_is_rejected() { + let test_cases = [ + MaintainedConfigErrorTestCase { + description: "a grace that is not a duration", + project: "name = 'orders'\nadapter = 'snowflake'\n[maintained]\n\ + heartbeat_grace = 'soon'\n", + expected_fragment: "maintained.heartbeat_grace must be a fixed duration such as 30m or 2h", + }, + MaintainedConfigErrorTestCase { + description: "a grace in months", + project: "name = 'orders'\nadapter = 'snowflake'\n[maintained]\n\ + heartbeat_grace = '1mo'\n", + expected_fragment: "maintained.heartbeat_grace must be a fixed duration such as 30m or 2h", + }, + MaintainedConfigErrorTestCase { + description: "allow_ignore_grace written as text", + project: "name = 'orders'\nadapter = 'snowflake'\n[maintained]\n\ + allow_ignore_grace = 'yes'\n", + expected_fragment: "maintained.allow_ignore_grace must be true or false", + }, + MaintainedConfigErrorTestCase { + description: "a misspelt maintained key", + project: "name = 'orders'\nadapter = 'snowflake'\n[maintained]\n\ + heartbeat_grase = '30m'\n", + expected_fragment: "maintained contains unknown key(s): heartbeat_grase", + }, + ]; + for test_case in test_cases { + let (_directory, paths) = project_directory(test_case.project, ""); + + let failure: String = format!( + "{:?}", + load_maintained_config(&paths).expect_err(test_case.description) + ); + + assert!( + failure.contains(test_case.expected_fragment), + "{}: {failure}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-config/src/project/tests/test_types.rs b/crates/sqlbuild-config/src/project/tests/test_types.rs index 413f1af9c0..d0a08468db 100644 --- a/crates/sqlbuild-config/src/project/tests/test_types.rs +++ b/crates/sqlbuild-config/src/project/tests/test_types.rs @@ -1,5 +1,5 @@ use crate::project::errors::ConfigFailure; -use crate::project::models::{ChangesConfig, ControlConfig, ReviewConfig}; +use crate::project::models::{ChangesConfig, ControlConfig, MaintainedConfig, ReviewConfig}; pub(super) struct ProjectFailureTestCase { pub(super) description: &'static str, @@ -58,3 +58,15 @@ pub(crate) struct PoolsErrorTestCase { pub(crate) local: &'static str, pub(crate) expected_fragment: &'static str, } + +pub(crate) struct MaintainedConfigTestCase { + pub(crate) description: &'static str, + pub(crate) project: &'static str, + pub(crate) expected_config: MaintainedConfig, +} + +pub(crate) struct MaintainedConfigErrorTestCase { + pub(crate) description: &'static str, + pub(crate) project: &'static str, + pub(crate) expected_fragment: &'static str, +} diff --git a/crates/sqlbuild-python-base/src/records/_helpers/contract_records/constants.rs b/crates/sqlbuild-python-base/src/records/_helpers/contract_records/constants.rs index 75fe5447c4..46587a7e93 100644 --- a/crates/sqlbuild-python-base/src/records/_helpers/contract_records/constants.rs +++ b/crates/sqlbuild-python-base/src/records/_helpers/contract_records/constants.rs @@ -151,6 +151,7 @@ const R_TARGET_CONFIG: RecordSpec = RecordSpec { FieldSpec::new("catalog_connection", FieldDefault::None), FieldSpec::new("ignore_audit_failures", FieldDefault::None), FieldSpec::new("plan_from", FieldDefault::None), + FieldSpec::new("pause_maintained", FieldDefault::None), ], }; const R_TARGET_SETTINGS: RecordSpec = RecordSpec { @@ -210,6 +211,7 @@ const R_LOCAL_TARGET_CONFIG: RecordSpec = RecordSpec { FieldSpec::new("catalog_connection", FieldDefault::None), FieldSpec::new("ignore_audit_failures", FieldDefault::None), FieldSpec::new("plan_from", FieldDefault::None), + FieldSpec::new("pause_maintained", FieldDefault::None), ], }; const R_SETTINGS_CONFIG: RecordSpec = RecordSpec { diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 949d974f84..dff3211435 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -12015,6 +12015,7 @@ class TargetConfig(NativeRecord): catalog_connection: str | None ignore_audit_failures: bool | None plan_from: str | None + pause_maintained: bool | None def __init__( self, connection: dict[str, object] = ..., @@ -12041,6 +12042,7 @@ class TargetConfig(NativeRecord): catalog_connection: str | None = ..., ignore_audit_failures: bool | None = ..., plan_from: str | None = ..., + pause_maintained: bool | None = ..., ) -> None: ... class LocalTargetConfig(NativeRecord): @@ -12070,6 +12072,7 @@ class LocalTargetConfig(NativeRecord): catalog_connection: str | None ignore_audit_failures: bool | None plan_from: str | None + pause_maintained: bool | None def __init__( self, connection: dict[str, object] = ..., @@ -12097,6 +12100,7 @@ class LocalTargetConfig(NativeRecord): catalog_connection: str | None = ..., ignore_audit_failures: bool | None = ..., plan_from: str | None = ..., + pause_maintained: bool | None = ..., ) -> None: ... class SettingsConfig(NativeRecord): From aec47d9f0552720b68b235af33dcfc1f0b66bf2c Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 18 Nov 2026 22:39:55 +0000 Subject: [PATCH 03/17] feat(plan): plan maintained models as create, alter or keep Work in progress. A maintained model is recreated on a first run, a rename, a full refresh or any query, function, config or schema change, and kept otherwise. Plan text and the UI word a recreate as a full recompute with no replay policy, and an alter as an in-place change of refresh settings. AlterMaintained is defined but not yet chosen: it needs the live lag and warehouse read. Validated: planner strategy (14) and cli_text tests, plan-text vitest (new case fails without the change), 895 planner, manifest and compile pytest cases, native stub check. --- .../main/materialization_type.rs | 1 + .../main/model_plan_action.rs | 7 +++ .../src/planner_strategy/models.rs | 8 ++++ .../tests/test_planner_strategy.rs | 48 +++++++++++++++++++ .../src/cli_text/_helpers/plan_sections.rs | 24 +++++++--- .../sqlbuild-core/src/cli_text/constants.rs | 2 + .../src/cli_text/tests/test_cli_text.rs | 17 +++++++ .../_helpers/compiler_enums/constants.rs | 4 ++ .../_helpers/plans/display_plan.rs | 3 ++ .../src/planner_strategy/_helpers/strategy.rs | 3 ++ .../src/planner_strategy/_helpers/values.rs | 1 + src/sqlbuild/_native.pyi | 4 ++ ui/src/lib/run/plan-text.test.ts | 9 ++++ ui/src/lib/run/plan-text.ts | 4 +- 14 files changed, 128 insertions(+), 7 deletions(-) diff --git a/crates/sqlbuild-analysis/src/planner_strategy/main/materialization_type.rs b/crates/sqlbuild-analysis/src/planner_strategy/main/materialization_type.rs index f45163514c..40708d18a5 100644 --- a/crates/sqlbuild-analysis/src/planner_strategy/main/materialization_type.rs +++ b/crates/sqlbuild-analysis/src/planner_strategy/main/materialization_type.rs @@ -10,6 +10,7 @@ pub fn materialization_type(materialized: Option<&str>) -> MaterializationType { Some("incremental") => MaterializationType::Incremental, Some("snapshot") => MaterializationType::Snapshot, Some("seed") => MaterializationType::Seed, + Some("maintained") => MaterializationType::Maintained, Some(_) => MaterializationType::Custom, } } diff --git a/crates/sqlbuild-analysis/src/planner_strategy/main/model_plan_action.rs b/crates/sqlbuild-analysis/src/planner_strategy/main/model_plan_action.rs index 437d66d599..de5c990fc2 100644 --- a/crates/sqlbuild-analysis/src/planner_strategy/main/model_plan_action.rs +++ b/crates/sqlbuild-analysis/src/planner_strategy/main/model_plan_action.rs @@ -41,6 +41,13 @@ pub fn model_plan_action( refreshed(PlanReason::NormalIncremental, false), )); } + MaterializationType::Maintained => { + let reason: PlanReason = refreshed(PlanReason::NoChange, true); + if reason == PlanReason::NoChange { + return Ok((PlanAction::KeepMaintained, reason)); + } + return Ok((PlanAction::CreateMaintained, reason)); + } MaterializationType::Table | MaterializationType::Incremental | MaterializationType::Seed => {} diff --git a/crates/sqlbuild-analysis/src/planner_strategy/models.rs b/crates/sqlbuild-analysis/src/planner_strategy/models.rs index 959f26c038..a760213e79 100644 --- a/crates/sqlbuild-analysis/src/planner_strategy/models.rs +++ b/crates/sqlbuild-analysis/src/planner_strategy/models.rs @@ -11,6 +11,8 @@ pub enum MaterializationType { Snapshot, Seed, Custom, + /// A stored result the warehouse keeps fresh between builds. + Maintained, } /// What the plan does with a model. @@ -24,6 +26,12 @@ pub enum PlanAction { Snapshot, Skip, Custom, + /// Create or replace the warehouse-maintained object. + CreateMaintained, + /// Change its refresh settings in place. + AlterMaintained, + /// Leave the object as it is; builds may still refresh it. + KeepMaintained, } /// Why the plan does it. diff --git a/crates/sqlbuild-analysis/src/planner_strategy/tests/test_planner_strategy.rs b/crates/sqlbuild-analysis/src/planner_strategy/tests/test_planner_strategy.rs index a65170fcaf..66db810b10 100644 --- a/crates/sqlbuild-analysis/src/planner_strategy/tests/test_planner_strategy.rs +++ b/crates/sqlbuild-analysis/src/planner_strategy/tests/test_planner_strategy.rs @@ -102,6 +102,54 @@ fn given_model_change_when_resolving_action_then_matches_python_strategy() { "S202 incremental model 'orders' has unknown strategy 'upsert'".to_owned(), ), }, + ActionTestCase { + description: "maintained first run creates the dynamic table", + materialization: MaterializationType::Maintained, + strategy: None, + change: change(PlanChangeKind::FirstRun), + full_refresh: false, + expected_action: Ok((PlanAction::CreateMaintained, PlanReason::FirstRun)), + }, + ActionTestCase { + description: "maintained query change recreates it", + materialization: MaterializationType::Maintained, + strategy: None, + change: change(PlanChangeKind::QueryChanged), + full_refresh: false, + expected_action: Ok((PlanAction::CreateMaintained, PlanReason::QueryChanged)), + }, + ActionTestCase { + description: "maintained config change such as refresh_mode recreates it", + materialization: MaterializationType::Maintained, + strategy: None, + change: change(PlanChangeKind::ConfigChanged), + full_refresh: false, + expected_action: Ok((PlanAction::CreateMaintained, PlanReason::ConfigChanged)), + }, + ActionTestCase { + description: "maintained rename creates it under the new name", + materialization: MaterializationType::Maintained, + strategy: None, + change: change(PlanChangeKind::Renamed), + full_refresh: false, + expected_action: Ok((PlanAction::CreateMaintained, PlanReason::Renamed)), + }, + ActionTestCase { + description: "maintained full refresh recreates an unchanged model", + materialization: MaterializationType::Maintained, + strategy: None, + change: change(PlanChangeKind::NoChange), + full_refresh: true, + expected_action: Ok((PlanAction::CreateMaintained, PlanReason::FullRefresh)), + }, + ActionTestCase { + description: "unchanged maintained model is kept", + materialization: MaterializationType::Maintained, + strategy: None, + change: change(PlanChangeKind::NoChange), + full_refresh: false, + expected_action: Ok((PlanAction::KeepMaintained, PlanReason::NoChange)), + }, ]; for test_case in test_cases { let model = ModelStrategy { diff --git a/crates/sqlbuild-core/src/cli_text/_helpers/plan_sections.rs b/crates/sqlbuild-core/src/cli_text/_helpers/plan_sections.rs index 3a2ce7ce86..37ffef5ac5 100644 --- a/crates/sqlbuild-core/src/cli_text/_helpers/plan_sections.rs +++ b/crates/sqlbuild-core/src/cli_text/_helpers/plan_sections.rs @@ -6,12 +6,13 @@ use crate::cli_text::_helpers::bounded_sections::{ }; use crate::cli_text::_helpers::layout::tree_connector; use crate::cli_text::constants::{ - BOUNDED_BACKFILL, CHANGED_IDENTITY, COLUMN_ADDED, COLUMN_REMOVED, COLUMN_TYPE_CHANGED, - CONFIG_CHANGED_REASON, CREATE_VIEW_ACTION, CUSTOM_ACTION, DIFF_HEADER_MARKER, - EFFECTIVE_BATCH_SIZE_TOKEN, ERROR_SEVERITY_VALUE, FIRST_RUN_REASON, FORWARD_BACKFILL, - FULL_BACKFILL, FULL_REFRESH_REASON, INFO_SEVERITY_VALUE, NO_CHANGE_REASON, - NORMAL_INCREMENTAL_REASON, QUERY_CHANGED_REASON, REASON_GROUPS, SCHEMA_CHANGED_REASON, - SKIP_ACTION, SOURCE_FRESHNESS_UNKNOWN_WARNING_TITLE, STALE_INPUT_WARNING_TITLE, + ALTER_MAINTAINED_ACTION, BOUNDED_BACKFILL, CHANGED_IDENTITY, COLUMN_ADDED, COLUMN_REMOVED, + COLUMN_TYPE_CHANGED, CONFIG_CHANGED_REASON, CREATE_MAINTAINED_ACTION, CREATE_VIEW_ACTION, + CUSTOM_ACTION, DIFF_HEADER_MARKER, EFFECTIVE_BATCH_SIZE_TOKEN, ERROR_SEVERITY_VALUE, + FIRST_RUN_REASON, FORWARD_BACKFILL, FULL_BACKFILL, FULL_REFRESH_REASON, INFO_SEVERITY_VALUE, + NO_CHANGE_REASON, NORMAL_INCREMENTAL_REASON, QUERY_CHANGED_REASON, REASON_GROUPS, + SCHEMA_CHANGED_REASON, SKIP_ACTION, SOURCE_FRESHNESS_UNKNOWN_WARNING_TITLE, + STALE_INPUT_WARNING_TITLE, }; use crate::cli_text::models::{ CliStyle, CursorPlanText, DisplayLimit, FunctionPlanText, ModelPlanText, PlanTextReport, @@ -429,6 +430,16 @@ fn action_text(model: &ModelPlanText) -> String { if model.action == CREATE_VIEW_ACTION { return "recreate view".to_owned(); } + if model.action == CREATE_MAINTAINED_ACTION { + return if model.reason == FIRST_RUN_REASON { + "create".to_owned() + } else { + "recreate, full recompute".to_owned() + }; + } + if model.action == ALTER_MAINTAINED_ACTION { + return "change refresh settings in place".to_owned(); + } if model.action == CUSTOM_ACTION { let materialization: &str = model .custom_materialization_name @@ -454,6 +465,7 @@ fn action_text(model: &ModelPlanText) -> String { fn policy_lines(model: &ModelPlanText) -> Vec { if model.backfill_action == FORWARD_BACKFILL + || model.action == CREATE_MAINTAINED_ACTION || !(model.reason == QUERY_CHANGED_REASON || model.reason == SCHEMA_CHANGED_REASON) { return Vec::new(); diff --git a/crates/sqlbuild-core/src/cli_text/constants.rs b/crates/sqlbuild-core/src/cli_text/constants.rs index d1a4378b23..f00f5abc48 100644 --- a/crates/sqlbuild-core/src/cli_text/constants.rs +++ b/crates/sqlbuild-core/src/cli_text/constants.rs @@ -63,6 +63,8 @@ pub(crate) const REASON_GROUPS: [(&str, &str); 7] = [ pub(crate) const SKIP_ACTION: &str = "skip"; pub(crate) const CREATE_VIEW_ACTION: &str = "create_view"; pub(crate) const CUSTOM_ACTION: &str = "custom"; +pub(crate) const CREATE_MAINTAINED_ACTION: &str = "create_maintained"; +pub(crate) const ALTER_MAINTAINED_ACTION: &str = "alter_maintained"; pub(crate) const FULL_REFRESH_REASON: &str = "full_refresh"; pub(crate) const FIRST_RUN_REASON: &str = "first_run"; pub(crate) const QUERY_CHANGED_REASON: &str = "query_changed"; diff --git a/crates/sqlbuild-core/src/cli_text/tests/test_cli_text.rs b/crates/sqlbuild-core/src/cli_text/tests/test_cli_text.rs index 7b1c772206..be0c037c79 100644 --- a/crates/sqlbuild-core/src/cli_text/tests/test_cli_text.rs +++ b/crates/sqlbuild-core/src/cli_text/tests/test_cli_text.rs @@ -226,6 +226,23 @@ fn given_plan_report_when_rendering_then_groups_models_by_reason() { expected_text: "Plan ready 1 selected\nAudit failures won't stop this build\n\nModels (1)\n\u{2514}\u{2500}\u{2500} orders table", expected_completion: "\u{2713} Plan complete 1 selected", }, + PlanTextTestCase { + description: "maintained models recompute on a query change and alter refresh settings in place", + report: plan_report(vec![ + ModelPlanText { + action: "create_maintained".to_owned(), + materialization_label: "maintained".to_owned(), + ..table_model("daily_totals", "query_changed") + }, + ModelPlanText { + action: "alter_maintained".to_owned(), + materialization_label: "maintained".to_owned(), + ..table_model("clean_orders", "config_changed") + }, + ]), + expected_text: "Plan ready 2 selected\n\nQuery changed (1)\n\u{2514}\u{2500}\u{2500} daily_totals recreate, full recompute\n\nConfig changed (1)\n\u{2514}\u{2500}\u{2500} clean_orders change refresh settings in place", + expected_completion: "\u{2713} Plan complete 2 selected", + }, PlanTextTestCase { description: "every active model is a full refresh", report: plan_report(vec![ diff --git a/crates/sqlbuild-python-base/src/records/_helpers/compiler_enums/constants.rs b/crates/sqlbuild-python-base/src/records/_helpers/compiler_enums/constants.rs index 63f2aa3e7b..5b65da0fd2 100644 --- a/crates/sqlbuild-python-base/src/records/_helpers/compiler_enums/constants.rs +++ b/crates/sqlbuild-python-base/src/records/_helpers/compiler_enums/constants.rs @@ -586,6 +586,9 @@ const E_PLAN_ACTION: EnumSpec = EnumSpec { ("LOAD_SEED", "load_seed"), ("SKIP", "skip"), ("CUSTOM", "custom"), + ("CREATE_MAINTAINED", "create_maintained"), + ("ALTER_MAINTAINED", "alter_maintained"), + ("KEEP_MAINTAINED", "keep_maintained"), ], }; const E_PLAN_REASON: EnumSpec = EnumSpec { @@ -683,6 +686,7 @@ const E_MATERIALIZATION_TYPE: EnumSpec = EnumSpec { ("SNAPSHOT", "snapshot"), ("SEED", "seed"), ("CUSTOM", "custom"), + ("MAINTAINED", "maintained"), ], }; const E_SCENARIO_ARTIFACT_KIND: EnumSpec = EnumSpec { diff --git a/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/display_plan.rs b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/display_plan.rs index 77c64a33bc..cabdb8403f 100644 --- a/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/display_plan.rs +++ b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/display_plan.rs @@ -24,6 +24,7 @@ use sqlbuild_python_project::planner_phases::main::object::object; const VIEW: &str = "view"; const INCREMENTAL: &str = "incremental"; const CUSTOM: &str = "custom"; +const MAINTAINED: &str = "maintained"; fn display_action( materialization_type: &Bound<'_, PyAny>, @@ -39,6 +40,8 @@ fn display_action( } } else if materialization_type.eq(CUSTOM)? { "CUSTOM" + } else if materialization_type.eq(MAINTAINED)? { + "CREATE_MAINTAINED" } else { "CREATE_TABLE" }) diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/strategy.rs b/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/strategy.rs index b620572c5d..89cc09cce8 100644 --- a/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/strategy.rs +++ b/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/strategy.rs @@ -30,6 +30,9 @@ pub(crate) fn action_value(action: PlanAction) -> &'static str { PlanAction::Snapshot => "snapshot", PlanAction::Skip => "skip", PlanAction::Custom => "custom", + PlanAction::CreateMaintained => "create_maintained", + PlanAction::AlterMaintained => "alter_maintained", + PlanAction::KeepMaintained => "keep_maintained", } } diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/values.rs b/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/values.rs index d910fd16b3..284859ce87 100644 --- a/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/values.rs +++ b/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/values.rs @@ -55,6 +55,7 @@ pub(crate) fn materialization_value(materialization: MaterializationType) -> &'s MaterializationType::Snapshot => "snapshot", MaterializationType::Seed => "seed", MaterializationType::Custom => "custom", + MaterializationType::Maintained => "maintained", } } diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index dff3211435..5c1495af74 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -13527,6 +13527,9 @@ class PlanAction(StrEnum): LOAD_SEED = "load_seed" SKIP = "skip" CUSTOM = "custom" + CREATE_MAINTAINED = "create_maintained" + ALTER_MAINTAINED = "alter_maintained" + KEEP_MAINTAINED = "keep_maintained" class PlanReason(StrEnum): FIRST_RUN = "first_run" @@ -13594,6 +13597,7 @@ class MaterializationType(StrEnum): SNAPSHOT = "snapshot" SEED = "seed" CUSTOM = "custom" + MAINTAINED = "maintained" @classmethod def is_table_backed(cls, *, materialized: object | None) -> bool: ... diff --git a/ui/src/lib/run/plan-text.test.ts b/ui/src/lib/run/plan-text.test.ts index 86723bb729..2ab282275e 100644 --- a/ui/src/lib/run/plan-text.test.ts +++ b/ui/src/lib/run/plan-text.test.ts @@ -35,6 +35,15 @@ describe('plan text, as sqb plan prints it', () => { expect(actionText(model('a', { reason: 'first_run', backfill: { action: 'full', duration: null } }))).toBe('continue forward'); }); + it('words maintained models as the CLI does, with no replay policy', () => { + const recreated = model('a', { action: 'create_maintained', reason: 'query_changed', backfill: { action: 'full', duration: null } }); + expect(actionText(recreated)).toBe('recreate, full recompute'); + expect(actionText(model('a', { action: 'create_maintained', reason: 'first_run' }))).toBe('create'); + expect(actionText(model('a', { action: 'alter_maintained', reason: 'config_changed' }))).toBe('change refresh settings in place'); + const [section] = planSections(plan([recreated])); + expect(section.entries[0].detail).toEqual([]); + }); + it('puts each change in its own section in reason order, then the routine models, and leaves skipped ones out', () => { const sections = planSections( plan([ diff --git a/ui/src/lib/run/plan-text.ts b/ui/src/lib/run/plan-text.ts index a9180cf2ea..4fbf432b62 100644 --- a/ui/src/lib/run/plan-text.ts +++ b/ui/src/lib/run/plan-text.ts @@ -97,6 +97,8 @@ export function materializationLabel(m: PlanModel): string { export function actionText(m: PlanModel): string { if (m.action === 'create_view') return 'recreate view'; if (m.action === 'custom') return 'run custom materialization'; + if (m.action === 'create_maintained') return m.reason === 'first_run' ? 'create' : 'recreate, full recompute'; + if (m.action === 'alter_maintained') return 'change refresh settings in place'; if (m.backfill.action === 'bounded' && m.backfill.duration) return `rebuild last ${m.backfill.duration}`; if (m.backfill.action === 'full' && m.reason !== 'first_run') return 'full rebuild'; return 'continue forward'; @@ -116,7 +118,7 @@ function cursorLines(m: PlanModel, showRange: boolean, windowStart?: string): st } function policyLine(m: PlanModel): string[] { - if (m.backfill.action === 'forward' || !(m.reason === 'query_changed' || m.reason === 'schema_changed')) return []; + if (m.backfill.action === 'forward' || m.action === 'create_maintained' || !(m.reason === 'query_changed' || m.reason === 'schema_changed')) return []; const value = m.backfill.action === 'bounded' ? `bounded-${m.backfill.duration ?? 'full'}` : m.backfill.action; return [`policy: replay_on_change=${value}`]; } From 524899a16e57de4bbe7082ad35dae1014d97d513 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 18 Nov 2026 22:43:32 +0000 Subject: [PATCH 04/17] feat(plan): recreate a maintained model when its refresh_mode changes Work in progress. refresh_mode joins the version identity keys, so a change plans as a config change and recreates the model. Absent keys stay out of the identity payload, so other models keep their identity. Lag and warehouse stay out of identity: they will be compared with the live dynamic table and altered in place. Validated: 993 planner unit and integration tests; the refresh_mode case fails without the key. --- .../src/planner_identity/constants.rs | 3 +- .../planner/_helpers/changes/test_detect.py | 29 +++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/crates/sqlbuild-analysis/src/planner_identity/constants.rs b/crates/sqlbuild-analysis/src/planner_identity/constants.rs index 79f8e591ff..fcdf538680 100644 --- a/crates/sqlbuild-analysis/src/planner_identity/constants.rs +++ b/crates/sqlbuild-analysis/src/planner_identity/constants.rs @@ -1,7 +1,7 @@ //! Planner identity constants shared with the Python planner. /// Model config keys whose values participate in version identity, in Python's insertion order after `cursor_inputs`. -pub const VERSION_IDENTITY_CONFIG_KEYS: [&str; 40] = [ +pub const VERSION_IDENTITY_CONFIG_KEYS: [&str; 41] = [ "append_cursor_inclusive", "batch_size", "check_columns", @@ -42,6 +42,7 @@ pub const VERSION_IDENTITY_CONFIG_KEYS: [&str; 40] = [ "partition_by", "primary_key", "table_settings", + "refresh_mode", ]; /// Model config key of cursor inputs, recorded before every other identity key. pub const CURSOR_INPUTS_CONFIG_KEY: &str = "cursor_inputs"; diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_detect.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_detect.py index 8187638751..6edecd18ea 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_detect.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_detect.py @@ -697,6 +697,35 @@ def test_given_model_identity_metadata_when_detecting_changes_then_uses_aligned_ expected_config_changed=True, expected_backfill=BackfillResult(action=BackfillAction.FORWARD_ONLY), ), + DetectFunctionCallerChangeTestCase( + description="a maintained model's refresh_mode change is a config change", + metadata_case=build_function_caller_metadata_case( + config_values={"materialized": "maintained", "refresh_mode": "full"}, + function_local_hashes={}, + previous_function_hashes={}, + previous_config_values={ + "materialized": "maintained", + "refresh_mode": "incremental", + }, + ), + query_change_tracking=True, + expected_changed_functions=(), + expected_config_changed=True, + expected_backfill=BackfillResult(action=BackfillAction.FORWARD_ONLY), + ), + DetectFunctionCallerChangeTestCase( + description="a maintained model's lag change is altered in place, not a config change", + metadata_case=build_function_caller_metadata_case( + config_values={"materialized": "maintained", "refresh": {"lag": "5m"}}, + function_local_hashes={}, + previous_function_hashes={}, + previous_config_values={"materialized": "maintained", "refresh": {"lag": "1h"}}, + ), + query_change_tracking=True, + expected_changed_functions=(), + expected_config_changed=False, + expected_backfill=BackfillResult(action=BackfillAction.FORWARD_ONLY), + ), ], ids=lambda case: case.description, ) From 29cbedb193dbb9cc62913989566db34ddfef82ef Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 18 Nov 2026 23:52:13 +0000 Subject: [PATCH 05/17] feat(plan): alter maintained models from their live refresh settings Work in progress. A kept maintained model now reads its listed target lag and warehouse through SHOW DYNAMIC TABLES: a drifted lag or declared warehouse plans an in-place alter, and a name that holds no dynamic table plans a create. Every maintained plan entry carries its rendered target_lag. A kept maintained model whose upstream the plan replaces is marked reinitialised_by that upstream, and sqb plan and the UI say it recomputes in full. The read is an optional adapter method; adapters without dynamic tables list nothing, and a preview plan leaves the action as planned. Validated: maintained_refresh crate tests, cli_text tests, the Snowflake SHOW DYNAMIC TABLES read, a DuckDB-backed replan integration test (keep, alter, create, reinitialise), the strict adapter contract, the plan text UI tests, and the planner, plan, compile, adapter and run suites. --- crates/sqlbuild-analysis/src/lib.rs | 1 + .../src/maintained_refresh/_helpers/mod.rs | 1 + .../maintained_refresh/_helpers/target_lag.rs | 24 ++ .../src/maintained_refresh/constants.rs | 11 + .../main/maintained_settings_drift.rs | 15 ++ .../src/maintained_refresh/main/mod.rs | 3 + .../main/reinitialised_readers.rs | 32 +++ .../main/render_target_lag.rs | 19 ++ .../src/maintained_refresh/mod.rs | 9 + .../src/maintained_refresh/models.rs | 34 +++ .../src/maintained_refresh/tests.rs | 6 + .../src/maintained_refresh/tests/helpers.rs | 26 +++ .../tests/test_maintained_refresh.rs | 209 ++++++++++++++++++ .../maintained_refresh/tests/test_types.rs | 28 +++ .../src/cli_text/_helpers/plan_sections.rs | 3 + crates/sqlbuild-core/src/cli_text/models.rs | 2 + .../src/cli_text/tests/test_cli_text.rs | 14 ++ .../_helpers/planner_records/constants.rs | 2 + .../src/live_reads/models.rs | 4 + .../src/plan_output/_helpers/plan_json.rs | 4 + .../src/plan_output/_helpers/plan_text.rs | 1 + .../_helpers/plans/execution_plan.rs | 2 + .../_helpers/outputs/maintained.rs | 162 ++++++++++++++ .../planner_phases/_helpers/outputs/mod.rs | 1 + .../src/planner_phases/main/outputs/mod.rs | 1 + .../main/outputs/plan_maintained_refresh.rs | 15 ++ .../src/planner_warehouse/models.rs | 5 + src/sqlbuild/_native.pyi | 4 + .../contract/classes/strict_adapter.py | 8 + .../snowflake/classes/snowflake_adapter.py | 25 +++ .../compiler/planner/main/_test_types.py | 12 + .../sqlbuild/compiler/planner/main/helpers.py | 43 ++++ .../main/test_maintained_refresh_plan.py | 109 +++++++++ .../strict_adapter/test_strict_adapter.py | 1 + .../adapters/snowflake/_test_types.py | 7 + .../test_maintained_refresh_settings.py | 50 +++++ ui/src/lib/api/types/operate.ts | 2 + ui/src/lib/run/plan-text.test.ts | 6 + ui/src/lib/run/plan-text.ts | 2 +- 39 files changed, 902 insertions(+), 1 deletion(-) create mode 100644 crates/sqlbuild-analysis/src/maintained_refresh/_helpers/mod.rs create mode 100644 crates/sqlbuild-analysis/src/maintained_refresh/_helpers/target_lag.rs create mode 100644 crates/sqlbuild-analysis/src/maintained_refresh/constants.rs create mode 100644 crates/sqlbuild-analysis/src/maintained_refresh/main/maintained_settings_drift.rs create mode 100644 crates/sqlbuild-analysis/src/maintained_refresh/main/mod.rs create mode 100644 crates/sqlbuild-analysis/src/maintained_refresh/main/reinitialised_readers.rs create mode 100644 crates/sqlbuild-analysis/src/maintained_refresh/main/render_target_lag.rs create mode 100644 crates/sqlbuild-analysis/src/maintained_refresh/mod.rs create mode 100644 crates/sqlbuild-analysis/src/maintained_refresh/models.rs create mode 100644 crates/sqlbuild-analysis/src/maintained_refresh/tests.rs create mode 100644 crates/sqlbuild-analysis/src/maintained_refresh/tests/helpers.rs create mode 100644 crates/sqlbuild-analysis/src/maintained_refresh/tests/test_maintained_refresh.rs create mode 100644 crates/sqlbuild-analysis/src/maintained_refresh/tests/test_types.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/maintained.rs create mode 100644 crates/sqlbuild-python-planner/src/planner_phases/main/outputs/plan_maintained_refresh.rs create mode 100644 tests/integration/src/sqlbuild/compiler/planner/main/test_maintained_refresh_plan.py create mode 100644 tests/unit/src/sqlbuild/adapters/snowflake/test_maintained_refresh_settings.py diff --git a/crates/sqlbuild-analysis/src/lib.rs b/crates/sqlbuild-analysis/src/lib.rs index 5bb2c63eae..303bf144da 100644 --- a/crates/sqlbuild-analysis/src/lib.rs +++ b/crates/sqlbuild-analysis/src/lib.rs @@ -22,6 +22,7 @@ pub mod janitor_planning; pub mod lineage; pub mod lineage_command; pub mod lineage_graph; +pub mod maintained_refresh; pub mod manifest; pub mod node_set_dependencies; pub mod nullability_rules; diff --git a/crates/sqlbuild-analysis/src/maintained_refresh/_helpers/mod.rs b/crates/sqlbuild-analysis/src/maintained_refresh/_helpers/mod.rs new file mode 100644 index 0000000000..f72a321364 --- /dev/null +++ b/crates/sqlbuild-analysis/src/maintained_refresh/_helpers/mod.rs @@ -0,0 +1 @@ +pub(crate) mod target_lag; diff --git a/crates/sqlbuild-analysis/src/maintained_refresh/_helpers/target_lag.rs b/crates/sqlbuild-analysis/src/maintained_refresh/_helpers/target_lag.rs new file mode 100644 index 0000000000..6123f16214 --- /dev/null +++ b/crates/sqlbuild-analysis/src/maintained_refresh/_helpers/target_lag.rs @@ -0,0 +1,24 @@ +//! A dynamic table's listed `target_lag`, such as `DOWNSTREAM` or `2 hours`. + +use crate::maintained_refresh::constants::{DOWNSTREAM_TARGET_LAG, TARGET_LAG_UNITS}; +use crate::maintained_refresh::models::RefreshLag; + +/// The lag `text` lists, or `None` when it is not a lag Snowflake lists. +pub(crate) fn parse_target_lag(text: &str) -> Option { + let text: &str = text.trim(); + if text.eq_ignore_ascii_case(DOWNSTREAM_TARGET_LAG) { + return Some(RefreshLag::Downstream); + } + let (amount, unit) = text.split_once(' ')?; + let amount: u64 = match amount.parse::() { + Ok(amount) => amount, + Err(_) => return None, + }; + let unit: String = unit.trim().to_ascii_lowercase(); + let unit: &str = unit.strip_suffix('s').unwrap_or(&unit); + TARGET_LAG_UNITS + .iter() + .find(|(name, _)| *name == unit) + .and_then(|(_, seconds)| amount.checked_mul(*seconds)) + .map(RefreshLag::Seconds) +} diff --git a/crates/sqlbuild-analysis/src/maintained_refresh/constants.rs b/crates/sqlbuild-analysis/src/maintained_refresh/constants.rs new file mode 100644 index 0000000000..97d74e91a2 --- /dev/null +++ b/crates/sqlbuild-analysis/src/maintained_refresh/constants.rs @@ -0,0 +1,11 @@ +//! Snowflake target lag spellings. + +/// The `target_lag` of a dynamic table refreshed for its readers. +pub const DOWNSTREAM_TARGET_LAG: &str = "DOWNSTREAM"; +/// Target lag units, largest first, with their seconds. +pub(crate) const TARGET_LAG_UNITS: [(&str, u64); 4] = [ + ("day", 86_400), + ("hour", 3_600), + ("minute", 60), + ("second", 1), +]; diff --git a/crates/sqlbuild-analysis/src/maintained_refresh/main/maintained_settings_drift.rs b/crates/sqlbuild-analysis/src/maintained_refresh/main/maintained_settings_drift.rs new file mode 100644 index 0000000000..6c2f11b86c --- /dev/null +++ b/crates/sqlbuild-analysis/src/maintained_refresh/main/maintained_settings_drift.rs @@ -0,0 +1,15 @@ +//! Whether a maintained model's listed refresh settings differ from its declared ones. + +use crate::maintained_refresh::_helpers::target_lag::parse_target_lag; +use crate::maintained_refresh::models::{LiveDynamicTable, MaintainedSettings}; + +/// True when the lag differs or cannot be read, or a declared warehouse differs. +pub fn maintained_settings_drift(desired: &MaintainedSettings, live: &LiveDynamicTable) -> bool { + if parse_target_lag(&live.target_lag) != Some(desired.lag) { + return true; + } + desired + .warehouse + .as_deref() + .is_some_and(|warehouse| !warehouse.eq_ignore_ascii_case(live.warehouse.trim())) +} diff --git a/crates/sqlbuild-analysis/src/maintained_refresh/main/mod.rs b/crates/sqlbuild-analysis/src/maintained_refresh/main/mod.rs new file mode 100644 index 0000000000..a48b621304 --- /dev/null +++ b/crates/sqlbuild-analysis/src/maintained_refresh/main/mod.rs @@ -0,0 +1,3 @@ +pub mod maintained_settings_drift; +pub mod reinitialised_readers; +pub mod render_target_lag; diff --git a/crates/sqlbuild-analysis/src/maintained_refresh/main/reinitialised_readers.rs b/crates/sqlbuild-analysis/src/maintained_refresh/main/reinitialised_readers.rs new file mode 100644 index 0000000000..82f825c8f9 --- /dev/null +++ b/crates/sqlbuild-analysis/src/maintained_refresh/main/reinitialised_readers.rs @@ -0,0 +1,32 @@ +//! Maintained models the plan keeps whose upstream it replaces, so they recompute in full. + +use std::collections::BTreeSet; + +use crate::maintained_refresh::models::MaintainedRead; + +/// Each kept maintained reader by name, with the first replaced upstream it reads by name. +pub fn reinitialised_readers(reads: &[MaintainedRead]) -> Vec<(String, String)> { + let replaced: BTreeSet<&str> = reads + .iter() + .filter(|read| read.replaced) + .map(|read| read.name.as_str()) + .collect(); + let mut readers: Vec<(String, String)> = Vec::new(); + for read in reads + .iter() + .filter(|read| read.maintained && !read.replaced) + { + let mut upstreams: Vec<&str> = read + .reads + .iter() + .map(String::as_str) + .filter(|upstream| replaced.contains(upstream)) + .collect(); + upstreams.sort_unstable(); + if let Some(upstream) = upstreams.first() { + readers.push((read.name.clone(), (*upstream).to_owned())); + } + } + readers.sort(); + readers +} diff --git a/crates/sqlbuild-analysis/src/maintained_refresh/main/render_target_lag.rs b/crates/sqlbuild-analysis/src/maintained_refresh/main/render_target_lag.rs new file mode 100644 index 0000000000..ca61aa4d47 --- /dev/null +++ b/crates/sqlbuild-analysis/src/maintained_refresh/main/render_target_lag.rs @@ -0,0 +1,19 @@ +//! The `TARGET_LAG` value of a refresh lag, in its largest exact unit. + +use crate::maintained_refresh::constants::{DOWNSTREAM_TARGET_LAG, TARGET_LAG_UNITS}; +use crate::maintained_refresh::models::RefreshLag; + +/// `DOWNSTREAM`, or a quoted lag such as `'2 hours'`. +pub fn render_target_lag(lag: RefreshLag) -> String { + let RefreshLag::Seconds(seconds) = lag else { + return DOWNSTREAM_TARGET_LAG.to_owned(); + }; + let (unit, size) = TARGET_LAG_UNITS + .iter() + .find(|(_, size)| seconds % size == 0) + .copied() + .unwrap_or(("second", 1)); + let amount: u64 = seconds / size; + let plural: &str = if amount == 1 { "" } else { "s" }; + format!("'{amount} {unit}{plural}'") +} diff --git a/crates/sqlbuild-analysis/src/maintained_refresh/mod.rs b/crates/sqlbuild-analysis/src/maintained_refresh/mod.rs new file mode 100644 index 0000000000..31a6ce51c4 --- /dev/null +++ b/crates/sqlbuild-analysis/src/maintained_refresh/mod.rs @@ -0,0 +1,9 @@ +//! Maintained refresh: dynamic table lag text, refresh setting drift, and readers a rebuild reinitialises. + +pub(crate) mod _helpers; +pub mod constants; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-analysis/src/maintained_refresh/models.rs b/crates/sqlbuild-analysis/src/maintained_refresh/models.rs new file mode 100644 index 0000000000..057d87a6e7 --- /dev/null +++ b/crates/sqlbuild-analysis/src/maintained_refresh/models.rs @@ -0,0 +1,34 @@ +//! Declared and listed refresh settings of maintained models. + +/// How stale a maintained model may get: refreshed for its readers, or within a fixed lag. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum RefreshLag { + /// Refreshed when the maintained models reading it need it. + Downstream, + /// Refreshed within this many seconds. + Seconds(u64), +} + +/// The refresh settings a maintained model declares. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct MaintainedSettings { + pub lag: RefreshLag, + /// The declared refresh warehouse; unset keeps whichever the object has. + pub warehouse: Option, +} + +/// A maintained model's refresh settings as the warehouse lists them. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct LiveDynamicTable { + pub target_lag: String, + pub warehouse: String, +} + +/// One planned model: whether it is maintained, whether this plan replaces it, and what it reads. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct MaintainedRead { + pub name: String, + pub maintained: bool, + pub replaced: bool, + pub reads: Vec, +} diff --git a/crates/sqlbuild-analysis/src/maintained_refresh/tests.rs b/crates/sqlbuild-analysis/src/maintained_refresh/tests.rs new file mode 100644 index 0000000000..0b70bcf139 --- /dev/null +++ b/crates/sqlbuild-analysis/src/maintained_refresh/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_maintained_refresh.rs"] +mod maintained_refresh; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-analysis/src/maintained_refresh/tests/helpers.rs b/crates/sqlbuild-analysis/src/maintained_refresh/tests/helpers.rs new file mode 100644 index 0000000000..9fa27f7945 --- /dev/null +++ b/crates/sqlbuild-analysis/src/maintained_refresh/tests/helpers.rs @@ -0,0 +1,26 @@ +use crate::maintained_refresh::models::{ + LiveDynamicTable, MaintainedRead, MaintainedSettings, RefreshLag, +}; + +pub(super) fn settings(lag: RefreshLag, warehouse: Option<&str>) -> MaintainedSettings { + MaintainedSettings { + lag, + warehouse: warehouse.map(str::to_owned), + } +} + +pub(super) fn live(target_lag: &str, warehouse: &str) -> LiveDynamicTable { + LiveDynamicTable { + target_lag: target_lag.to_owned(), + warehouse: warehouse.to_owned(), + } +} + +pub(super) fn read(name: &str, maintained: bool, replaced: bool, reads: &[&str]) -> MaintainedRead { + MaintainedRead { + name: name.to_owned(), + maintained, + replaced, + reads: reads.iter().map(|read| (*read).to_owned()).collect(), + } +} diff --git a/crates/sqlbuild-analysis/src/maintained_refresh/tests/test_maintained_refresh.rs b/crates/sqlbuild-analysis/src/maintained_refresh/tests/test_maintained_refresh.rs new file mode 100644 index 0000000000..145cdbf5bd --- /dev/null +++ b/crates/sqlbuild-analysis/src/maintained_refresh/tests/test_maintained_refresh.rs @@ -0,0 +1,209 @@ +use crate::maintained_refresh::_helpers::target_lag::parse_target_lag; +use crate::maintained_refresh::main::maintained_settings_drift::maintained_settings_drift; +use crate::maintained_refresh::main::reinitialised_readers::reinitialised_readers; +use crate::maintained_refresh::main::render_target_lag::render_target_lag; +use crate::maintained_refresh::models::RefreshLag; +use crate::maintained_refresh::tests::helpers::{live, read, settings}; +use crate::maintained_refresh::tests::test_types::{ + DriftTestCase, ReinitialiseTestCase, RenderLagTestCase, TargetLagTestCase, +}; + +#[test] +fn given_snowflake_target_lag_text_when_parsing_then_lag_is_read_in_seconds() { + let test_cases = [ + TargetLagTestCase { + description: "lag set by readers", + text: "DOWNSTREAM", + expected_lag: Some(RefreshLag::Downstream), + }, + TargetLagTestCase { + description: "one minute, singular", + text: "1 minute", + expected_lag: Some(RefreshLag::Seconds(60)), + }, + TargetLagTestCase { + description: "hours, plural and padded", + text: " 2 hours ", + expected_lag: Some(RefreshLag::Seconds(7_200)), + }, + TargetLagTestCase { + description: "seconds", + text: "90 seconds", + expected_lag: Some(RefreshLag::Seconds(90)), + }, + TargetLagTestCase { + description: "days", + text: "1 day", + expected_lag: Some(RefreshLag::Seconds(86_400)), + }, + TargetLagTestCase { + description: "an unknown unit", + text: "1 fortnight", + expected_lag: None, + }, + ]; + for test_case in test_cases { + assert_eq!( + parse_target_lag(test_case.text), + test_case.expected_lag, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_refresh_lag_when_rendering_then_largest_exact_unit_is_used() { + let test_cases = [ + RenderLagTestCase { + description: "lag set by readers", + lag: RefreshLag::Downstream, + expected_sql: "DOWNSTREAM", + }, + RenderLagTestCase { + description: "a whole hour", + lag: RefreshLag::Seconds(3_600), + expected_sql: "'1 hour'", + }, + RenderLagTestCase { + description: "ninety minutes stays in minutes", + lag: RefreshLag::Seconds(5_400), + expected_sql: "'90 minutes'", + }, + RenderLagTestCase { + description: "two days", + lag: RefreshLag::Seconds(172_800), + expected_sql: "'2 days'", + }, + RenderLagTestCase { + description: "seconds that are not whole minutes", + lag: RefreshLag::Seconds(61), + expected_sql: "'61 seconds'", + }, + ]; + for test_case in test_cases { + assert_eq!( + render_target_lag(test_case.lag), + test_case.expected_sql, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_live_dynamic_table_when_comparing_settings_then_only_real_drift_alters() { + let test_cases = [ + DriftTestCase { + description: "the same lag written in another unit", + desired: settings(RefreshLag::Seconds(3_600), None), + live: live("60 minutes", "TRANSFORM_WH"), + expected_drift: false, + }, + DriftTestCase { + description: "a shorter lag", + desired: settings(RefreshLag::Seconds(300), None), + live: live("1 hour", "TRANSFORM_WH"), + expected_drift: true, + }, + DriftTestCase { + description: "a lag now set by readers", + desired: settings(RefreshLag::Downstream, None), + live: live("1 hour", "TRANSFORM_WH"), + expected_drift: true, + }, + DriftTestCase { + description: "an unset refresh warehouse keeps the existing one", + desired: settings(RefreshLag::Downstream, None), + live: live("DOWNSTREAM", "ANY_WH"), + expected_drift: false, + }, + DriftTestCase { + description: "a warehouse matched without case", + desired: settings(RefreshLag::Downstream, Some("transform_wh")), + live: live("DOWNSTREAM", "TRANSFORM_WH"), + expected_drift: false, + }, + DriftTestCase { + description: "a different warehouse", + desired: settings(RefreshLag::Downstream, Some("reporting_wh")), + live: live("DOWNSTREAM", "TRANSFORM_WH"), + expected_drift: true, + }, + DriftTestCase { + description: "an unreadable live lag is set again", + desired: settings(RefreshLag::Seconds(60), None), + live: live("whenever", "TRANSFORM_WH"), + expected_drift: true, + }, + ]; + for test_case in test_cases { + assert_eq!( + maintained_settings_drift(&test_case.desired, &test_case.live), + test_case.expected_drift, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_planned_rebuilds_when_checking_maintained_readers_then_reinitialised_ones_are_named() { + let test_cases = [ + ReinitialiseTestCase { + description: "a maintained model reading a rebuilt table", + reads: vec![ + read("orders", false, true, &[]), + read("daily_totals", true, false, &["orders"]), + ], + expected_readers: vec![("daily_totals", "orders")], + }, + ReinitialiseTestCase { + description: "an upstream kept in place, such as a merged incremental", + reads: vec![ + read("orders", false, false, &[]), + read("daily_totals", true, false, &["orders"]), + ], + expected_readers: vec![], + }, + ReinitialiseTestCase { + description: "a recreated maintained model reinitialises its maintained readers", + reads: vec![ + read("clean_orders", true, true, &["orders"]), + read("daily_totals", true, false, &["clean_orders"]), + ], + expected_readers: vec![("daily_totals", "clean_orders")], + }, + ReinitialiseTestCase { + description: "a recreated model itself is not reported, and a table reader is ignored", + reads: vec![ + read("orders", false, true, &[]), + read("daily_totals", true, true, &["orders"]), + read("report", false, false, &["orders"]), + ], + expected_readers: vec![], + }, + ReinitialiseTestCase { + description: "the first replaced upstream by name is named", + reads: vec![ + read("orders", false, true, &[]), + read("customers", false, true, &[]), + read("daily_totals", true, false, &["orders", "customers"]), + ], + expected_readers: vec![("daily_totals", "customers")], + }, + ]; + for test_case in test_cases { + let readers: Vec<(String, String)> = reinitialised_readers(&test_case.reads); + let named: Vec<(&str, &str)> = readers + .iter() + .map(|(reader, upstream)| (reader.as_str(), upstream.as_str())) + .collect(); + + assert_eq!( + named, test_case.expected_readers, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/maintained_refresh/tests/test_types.rs b/crates/sqlbuild-analysis/src/maintained_refresh/tests/test_types.rs new file mode 100644 index 0000000000..70aedbf086 --- /dev/null +++ b/crates/sqlbuild-analysis/src/maintained_refresh/tests/test_types.rs @@ -0,0 +1,28 @@ +use crate::maintained_refresh::models::{ + LiveDynamicTable, MaintainedRead, MaintainedSettings, RefreshLag, +}; + +pub(super) struct TargetLagTestCase { + pub(super) description: &'static str, + pub(super) text: &'static str, + pub(super) expected_lag: Option, +} + +pub(super) struct RenderLagTestCase { + pub(super) description: &'static str, + pub(super) lag: RefreshLag, + pub(super) expected_sql: &'static str, +} + +pub(super) struct DriftTestCase { + pub(super) description: &'static str, + pub(super) desired: MaintainedSettings, + pub(super) live: LiveDynamicTable, + pub(super) expected_drift: bool, +} + +pub(super) struct ReinitialiseTestCase { + pub(super) description: &'static str, + pub(super) reads: Vec, + pub(super) expected_readers: Vec<(&'static str, &'static str)>, +} diff --git a/crates/sqlbuild-core/src/cli_text/_helpers/plan_sections.rs b/crates/sqlbuild-core/src/cli_text/_helpers/plan_sections.rs index 37ffef5ac5..3d68442d25 100644 --- a/crates/sqlbuild-core/src/cli_text/_helpers/plan_sections.rs +++ b/crates/sqlbuild-core/src/cli_text/_helpers/plan_sections.rs @@ -630,6 +630,9 @@ pub(crate) fn routine_model_lines( true, )]; lines.extend(cursor_lines(model, true)); + if let Some(upstream) = &model.reinitialised_by { + lines.push(format!(" recomputes in full: {upstream} is rebuilt")); + } lines }) } diff --git a/crates/sqlbuild-core/src/cli_text/models.rs b/crates/sqlbuild-core/src/cli_text/models.rs index 34817c0b7b..881e5b4ace 100644 --- a/crates/sqlbuild-core/src/cli_text/models.rs +++ b/crates/sqlbuild-core/src/cli_text/models.rs @@ -322,6 +322,8 @@ pub struct ModelPlanText { pub config_diff: Option>, /// The rendered query diff, when the query changed and the previous query is known. pub query_diff: Option>, + /// The rebuilt upstream that makes this kept maintained model recompute in full. + pub reinitialised_by: Option, } /// One planned function. diff --git a/crates/sqlbuild-core/src/cli_text/tests/test_cli_text.rs b/crates/sqlbuild-core/src/cli_text/tests/test_cli_text.rs index be0c037c79..f5ce8beea5 100644 --- a/crates/sqlbuild-core/src/cli_text/tests/test_cli_text.rs +++ b/crates/sqlbuild-core/src/cli_text/tests/test_cli_text.rs @@ -243,6 +243,20 @@ fn given_plan_report_when_rendering_then_groups_models_by_reason() { expected_text: "Plan ready 2 selected\n\nQuery changed (1)\n\u{2514}\u{2500}\u{2500} daily_totals recreate, full recompute\n\nConfig changed (1)\n\u{2514}\u{2500}\u{2500} clean_orders change refresh settings in place", expected_completion: "\u{2713} Plan complete 2 selected", }, + PlanTextTestCase { + description: "a kept maintained model whose upstream is rebuilt says it recomputes in full", + report: plan_report(vec![ + table_model("orders", "query_changed"), + ModelPlanText { + action: "keep_maintained".to_owned(), + materialization_label: "maintained".to_owned(), + reinitialised_by: Some("orders".to_owned()), + ..table_model("daily_totals", "no_change") + }, + ]), + expected_text: "Plan ready 2 selected\n\nQuery changed (1)\n\u{2514}\u{2500}\u{2500} orders full rebuild\n \u{2514}\u{2500}\u{2500} policy replay_on_change=full\n\nModels (1)\n\u{2514}\u{2500}\u{2500} daily_totals maintained\n \u{2514}\u{2500}\u{2500} recomputes in full orders is rebuilt", + expected_completion: "\u{2713} Plan complete 2 selected", + }, PlanTextTestCase { description: "every active model is a full refresh", report: plan_report(vec![ diff --git a/crates/sqlbuild-python-base/src/records/_helpers/planner_records/constants.rs b/crates/sqlbuild-python-base/src/records/_helpers/planner_records/constants.rs index bd568fa583..5826828d83 100644 --- a/crates/sqlbuild-python-base/src/records/_helpers/planner_records/constants.rs +++ b/crates/sqlbuild-python-base/src/records/_helpers/planner_records/constants.rs @@ -666,6 +666,8 @@ const R_MODEL_PLAN_ENTRY: RecordSpec = RecordSpec { FieldSpec::new("max_duration", FieldDefault::None), FieldSpec::new("max_duration_seconds", FieldDefault::None), FieldSpec::new("max_duration_source", FieldDefault::None), + FieldSpec::new("reinitialised_by", FieldDefault::None), + FieldSpec::new("target_lag", FieldDefault::None), ], }; const R_CLONE_SOURCE_PLAN_ENTRY: RecordSpec = RecordSpec { diff --git a/crates/sqlbuild-python-planner/src/live_reads/models.rs b/crates/sqlbuild-python-planner/src/live_reads/models.rs index 6fb7db6992..df284dd646 100644 --- a/crates/sqlbuild-python-planner/src/live_reads/models.rs +++ b/crates/sqlbuild-python-planner/src/live_reads/models.rs @@ -438,6 +438,10 @@ impl WarehouseReads for LiveReads { "column_rename_unavailable_reason", vec![("destination", destination)], ), + AdapterRead::MaintainedRefresh([database, schema, name]) => ( + "maintained_refresh_settings", + vec![("database", database), ("schema", schema), ("name", name)], + ), }; self.known_call(py, method, arguments) } diff --git a/crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_json.rs b/crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_json.rs index 1f1979f407..a0cd5faf9e 100644 --- a/crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_json.rs +++ b/crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_json.rs @@ -230,6 +230,10 @@ pub(crate) fn model_json<'py>(entry: &Bound<'py, PyAny>) -> PyResult = native_field(entry, "reinitialised_by")?; + if !reinitialised_by.is_none() { + model.set_item("reinitialised_by", reinitialised_by)?; + } let qualified: Bound<'py, PyAny> = native_field(&native_field(entry, "destination")?, "qualified_name")?; if !qualified.is_none() { diff --git a/crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_text.rs b/crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_text.rs index 10a1bfb6ac..e1d71c4f68 100644 --- a/crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_text.rs +++ b/crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_text.rs @@ -250,6 +250,7 @@ fn model_text(entry: &Bound<'_, PyAny>) -> PyResult { rename_hints, config_diff: coloured(model_config_diff(entry, &reason)?), query_diff: coloured(model_query_diff(entry, &reason)?), + reinitialised_by: optional_text(&native_field(entry, "reinitialised_by")?)?, name, action, reason, diff --git a/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/execution_plan.rs b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/execution_plan.rs index ff6617a080..f33159b252 100644 --- a/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/execution_plan.rs +++ b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/execution_plan.rs @@ -27,6 +27,7 @@ use crate::planner_phases::main::outputs::base_plan_output::base_plan_output; use crate::planner_phases::main::outputs::build_planner_identity_context::build_planner_identity_context; use crate::planner_phases::main::outputs::check_selected_scope_buildability::check_selected_scope_buildability; use crate::planner_phases::main::outputs::detect_stale_warning_changes::detect_stale_warning_changes; +use crate::planner_phases::main::outputs::plan_maintained_refresh::plan_maintained_refresh; use crate::planner_phases::main::outputs::unselected_upstream_models::unselected_upstream_models; use crate::planner_phases::main::rebuilds::build_planner_entry_results::build_planner_entry_results; use crate::planner_phases::main::rebuilds::defer_to_migration_origins::defer_to_migration_origins; @@ -196,6 +197,7 @@ pub(crate) fn planned_output<'py>( (&inspection_scope, &reconciliation), &source_freshness, )?; + let entries = plan_maintained_refresh(runtime, &inspection_scope, entries)?; record_step(BUILDING_ENTRIES, entries_started.elapsed().as_secs_f64()); let planned_sql_tests = timed(TEST_FIXTURES, || test_planning.result(py))?.into_bound(py); let assembling_started = Instant::now(); diff --git a/crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/maintained.rs b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/maintained.rs new file mode 100644 index 0000000000..9297abf940 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/maintained.rs @@ -0,0 +1,162 @@ +//! Maintained models whose live refresh settings drifted, and kept ones a replaced upstream reinitialises. + +use crate::planner_phases::main::inputs::runtime_reads::runtime_reads; +use crate::planner_phases::models::ModelEntryResults; +use crate::planner_strategy::main::values::text::text; +use crate::planner_warehouse::models::{AdapterRead, PlannerReads, Read}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::PyString; +use sqlbuild_analysis::maintained_refresh::main::maintained_settings_drift::maintained_settings_drift; +use sqlbuild_analysis::maintained_refresh::main::reinitialised_readers::reinitialised_readers; +use sqlbuild_analysis::maintained_refresh::main::render_target_lag::render_target_lag; +use sqlbuild_analysis::maintained_refresh::models::{ + LiveDynamicTable, MaintainedRead, MaintainedSettings, RefreshLag, +}; +use sqlbuild_model_config::model_validation::main::fixed_duration_seconds::fixed_duration_seconds; +use sqlbuild_python_base::records::main::construct::enum_member::enum_member; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::main::fields::replaced::replaced; +use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_front::model_config::main::config::config_text_object::config_text_object; +use sqlbuild_python_front::model_config::main::config::config_value::config_value; +use sqlbuild_python_front::model_config::main::config::model_values::model_values; +use sqlbuild_python_project::graph::main::nodes::native_key::native_key; +use sqlbuild_python_project::native_host::models::ModelLookup; +use sqlbuild_python_project::planner_scope::main::scope_data::scope_data; +use std::collections::HashMap; + +const MAINTAINED: &str = "maintained"; +const KEEP_MAINTAINED: &str = "keep_maintained"; +/// Actions that recreate the relation, which reinitialises every dynamic table reading it. +const REPLACING_ACTIONS: [&str; 3] = ["create_table", "create_view", "create_maintained"]; + +/// Alter drifted maintained models, set their `target_lag`, and mark kept readers of a replaced upstream. +pub(crate) fn plan_maintained_refresh<'py>( + runtime: &Bound<'py, PyAny>, + execution_scope: &Bound<'py, PyAny>, + results: ModelEntryResults<'py>, +) -> PyResult> { + let mut maintained: Vec = Vec::with_capacity(results.entries.len()); + for entry in &results.entries { + maintained.push(text(&native_field(entry, "materialization_type")?)? == MAINTAINED); + } + if !maintained.contains(&true) { + return Ok(results); + } + let py: Python<'py> = runtime.py(); + let scope = scope_data(execution_scope)?; + let mut upstream: HashMap<&(String, String), Vec> = HashMap::new(); + for (key, deps) in &scope.upstream { + upstream.insert(key, deps.iter().map(|(_, name)| name.clone()).collect()); + } + let models = ModelLookup::Native(scope.clone()); + let reads = runtime_reads(runtime)?; + let mut entries: Vec> = Vec::with_capacity(results.entries.len()); + let mut rows: Vec = Vec::with_capacity(results.entries.len()); + for (entry, is_maintained) in results.entries.into_iter().zip(maintained) { + let entry = if is_maintained { + with_refresh_settings(entry, &models, &reads)? + } else { + entry + }; + let key = native_key(&native_field(&entry, "key")?)?; + let action: String = text(&native_field(&entry, "action")?)?; + rows.push(MaintainedRead { + name: key.1.clone(), + maintained: is_maintained, + replaced: REPLACING_ACTIONS.contains(&action.as_str()), + reads: upstream.remove(&key).unwrap_or_default(), + }); + entries.push(entry); + } + let readers: HashMap = reinitialised_readers(&rows).into_iter().collect(); + for (entry, row) in entries.iter_mut().zip(&rows) { + if let Some(upstream) = readers.get(&row.name) { + *entry = replaced( + entry, + &[("reinitialised_by", PyString::new(py, upstream).into_any())], + )?; + } + } + Ok(ModelEntryResults { + entries, + warnings: results.warnings, + }) +} + +/// The entry with its `target_lag`; a kept one alters on drift, or creates when no dynamic table holds its name. +fn with_refresh_settings<'py>( + entry: Bound<'py, PyAny>, + models: &ModelLookup, + reads: &Bound<'py, PlannerReads>, +) -> PyResult> { + let py: Python<'py> = entry.py(); + let model = models.get(py, &native_field(&entry, "name")?.extract::()?)?; + let Some(desired) = declared_settings(&model)? else { + return Ok(entry); + }; + let mut changes: Vec<(&str, Bound<'py, PyAny>)> = vec![( + "target_lag", + PyString::new(py, &render_target_lag(desired.lag)).into_any(), + )]; + if text(&native_field(&entry, "action")?)? == KEEP_MAINTAINED { + if let Some((action, reason)) = live_action(&entry, &desired, reads)? { + changes.push(("action", enum_member(py, RecordId::PlanAction, action)?)); + changes.push(("reason", enum_member(py, RecordId::PlanReason, reason)?)); + } + } + replaced(&entry, &changes) +} + +/// The action and reason replacing a kept model's, or `None` when its listed settings match or are unknown. +fn live_action( + entry: &Bound<'_, PyAny>, + desired: &MaintainedSettings, + reads: &Bound<'_, PlannerReads>, +) -> PyResult> { + let destination = native_field(entry, "destination")?; + let read = reads.get().reads().adapter_read( + entry.py(), + AdapterRead::MaintainedRefresh([ + native_field(&destination, "database")?, + native_field(&destination, "schema")?, + native_field(&destination, "name")?, + ]), + )?; + let listed = match read { + Read::Unknown(_) => return Ok(None), + Read::Known(listed) => listed, + }; + if listed.is_none() { + return Ok(Some(("CREATE_MAINTAINED", "FIRST_RUN"))); + } + let (target_lag, warehouse): (String, String) = listed.extract()?; + let live = LiveDynamicTable { + target_lag, + warehouse, + }; + Ok(maintained_settings_drift(desired, &live).then_some(("ALTER_MAINTAINED", "CONFIG_CHANGED"))) +} + +/// The model's declared lag and refresh warehouse, or `None` when its lag does not read. +fn declared_settings(model: &Bound<'_, PyAny>) -> PyResult> { + let py = model.py(); + let values = model_values(model)?; + let refresh = config_value(py, &values, "refresh")?; + let lag: RefreshLag = if refresh.is_none() { + RefreshLag::Downstream + } else { + let text: String = refresh.get_item("lag")?.extract()?; + let Some(seconds) = fixed_duration_seconds(&text) else { + return Ok(None); + }; + match u64::try_from(seconds) { + Ok(seconds) => RefreshLag::Seconds(seconds), + Err(_) => return Ok(None), + } + }; + let warehouse: Option = config_text_object(py, &values, "refresh_warehouse")? + .map(|warehouse| warehouse.extract()) + .transpose()?; + Ok(Some(MaintainedSettings { lag, warehouse })) +} diff --git a/crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/mod.rs b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/mod.rs index fcf2752f38..1295d1283b 100644 --- a/crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/mod.rs +++ b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/mod.rs @@ -1,3 +1,4 @@ pub(crate) mod buildability; pub(crate) mod identities; +pub(crate) mod maintained; pub(crate) mod output_assembly; diff --git a/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/mod.rs b/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/mod.rs index 69b6f2dbe9..28947d1f49 100644 --- a/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/mod.rs +++ b/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/mod.rs @@ -7,5 +7,6 @@ pub mod build_planner_identity_context; pub mod check_selected_scope_buildability; pub mod detect_stale_warning_changes; pub mod detect_stale_warning_changes_of_function; +pub mod plan_maintained_refresh; pub mod planner_identity_work_info_function; pub mod unselected_upstream_models; diff --git a/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/plan_maintained_refresh.rs b/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/plan_maintained_refresh.rs new file mode 100644 index 0000000000..2b789a1dc5 --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/plan_maintained_refresh.rs @@ -0,0 +1,15 @@ +//! Alter drifted maintained models, set their target lag, and mark kept readers of a replaced upstream. + +use crate::planner_phases::_helpers::outputs::maintained; +use crate::planner_phases::models::ModelEntryResults; +use pyo3::prelude::{Bound, PyAny, PyResult}; + +/// Alter drifted maintained models, set their target lag, and mark kept readers of a replaced upstream. +#[inline] +pub fn plan_maintained_refresh<'py>( + runtime: &Bound<'py, PyAny>, + execution_scope: &Bound<'py, PyAny>, + entries: ModelEntryResults<'py>, +) -> PyResult> { + maintained::plan_maintained_refresh(runtime, execution_scope, entries) +} diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/models.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/models.rs index da5c4f3d27..44098a75b8 100644 --- a/crates/sqlbuild-python-planner/src/planner_warehouse/models.rs +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/models.rs @@ -383,6 +383,8 @@ pub enum AdapterRead<'py> { FreshnessQuery(Bound<'py, PyAny>), /// Why a destination cannot rename columns, `None` when it can. ColumnRenameUnavailable(Bound<'py, PyAny>), + /// One maintained model's `[database, schema, name]` listed target lag and refresh warehouse. + MaintainedRefresh([Bound<'py, PyAny>; 3]), } /// A read of what the warehouse state tables recorded. @@ -754,6 +756,9 @@ impl WarehouseReads for SyncedReads { AdapterRead::ColumnRenameUnavailable(_) => { return Ok(unknown(self, "column rename support")); } + AdapterRead::MaintainedRefresh(_) => { + return Ok(unknown(self, "maintained refresh settings")); + } }; let unavailable = host_entry(py, AdapterHost::TableFreshnessMetadata)?.getattr("unavailable")?; diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 5c1495af74..b96441627a 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -14119,6 +14119,8 @@ class ModelPlanEntry(NativeRecord): max_duration: str | None max_duration_seconds: int | None max_duration_source: str | None + reinitialised_by: str | None + target_lag: str | None def __init__( self, key: CompiledObjectKey, @@ -14208,6 +14210,8 @@ class ModelPlanEntry(NativeRecord): max_duration: str | None = ..., max_duration_seconds: int | None = ..., max_duration_source: str | None = ..., + reinitialised_by: str | None = ..., + target_lag: str | None = ..., ) -> None: ... class CloneSourcePlanEntry(NativeRecord): diff --git a/src/sqlbuild/adapter/contract/classes/strict_adapter.py b/src/sqlbuild/adapter/contract/classes/strict_adapter.py index 19023aa933..2d102a0760 100644 --- a/src/sqlbuild/adapter/contract/classes/strict_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/strict_adapter.py @@ -321,6 +321,14 @@ def render_drop_schema( help="sqb review needs a warehouse that can drop a schema it created", ) + def maintained_refresh_settings( + self, *, connection: Any, database: str | None, schema: str | None, name: str + ) -> tuple[str, str] | None: + """Optional: a maintained model's listed target lag and refresh warehouse, or None.""" + + del connection, database, schema, name + return None + @abstractmethod def render_create_table_as(self, *, destination: str, sql: str) -> tuple[str, ...]: """Render SQL statements that create or replace a table from a query.""" diff --git a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py index ed578f251e..99c3c5f565 100644 --- a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py +++ b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py @@ -3665,6 +3665,31 @@ def column_rename_unavailable_reason(self, *, connection: Any, destination: str) del connection, destination return None + def maintained_refresh_settings( + self, *, connection: Any, database: str | None, schema: str | None, name: str + ) -> tuple[str, str] | None: + """The dynamic table's target lag and warehouse from SHOW DYNAMIC TABLES, or None.""" + + stored_name: str = self._information_schema_identifier(name) + if schema is None: + return None + rows: list[dict[str, object]] = self._fetch_schema_show_rows( + connection=connection, + query=( + f"SHOW DYNAMIC TABLES LIKE '{show_like_pattern(stored_name)}' IN SCHEMA " + + self._show_schema_scope(connection=connection, database=database, schema=schema) + ), + show_database=self._show_database( + connection=connection, database=database, schema=schema + ), + schema=schema, + ) + row: dict[str, object] + for row in rows: + if str(row.get("name")) == stored_name: + return (str(row.get("target_lag")), str(row.get("warehouse"))) + return None + def widen_target_nullability( self, *, diff --git a/tests/integration/src/sqlbuild/compiler/planner/main/_test_types.py b/tests/integration/src/sqlbuild/compiler/planner/main/_test_types.py index 7c19d1c40c..bb4901a9ae 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/main/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/planner/main/_test_types.py @@ -205,3 +205,15 @@ class SyncedScanTestCase: last_ddl_after_scan_start: timedelta expected_live_describe: bool location_clock_ahead: timedelta = timedelta() + + +@dataclass(frozen=True) +class MaintainedRefreshPlanTestCase: + description: str + listed_settings: tuple[str, str] | None + upstream_query: str + expected_upstream_action: PlanAction + expected_action: PlanAction + expected_reason: PlanReason + expected_reinitialised_by: str | None + expected_read_names: tuple[str, ...] diff --git a/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py b/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py index 9c63c10938..3c1661bbf8 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py @@ -827,3 +827,46 @@ def planned_entries(result: CompilePipelineResult) -> list[tuple[str, str]]: """Each planned model's name and reason, in name order.""" return sorted((entry.name, str(entry.reason)) for entry in result.plan_output.model_entries) + + +CLEAN_ORDERS_QUERY: str = "SELECT 1 AS order_id" +_MAINTAINED: dict[str, object] = { + "materialized": "maintained", + "refresh": {"lag": "1h"}, + "refresh_warehouse": "transform_wh", +} + + +class ListingDuckDbAdapter(DuckDbAdapter): + """DuckDB that lists one dynamic table's refresh settings, as Snowflake's SHOW would.""" + + def __init__(self, listed: tuple[str, str] | None) -> None: + super().__init__() + self.listed: tuple[str, str] | None = listed + self.read_names: list[str] = [] + + def maintained_refresh_settings( + self, *, connection: Any, database: str | None, schema: str | None, name: str + ) -> tuple[str, str] | None: + del connection, database, schema + self.read_names.append(name) + return self.listed + + +def maintained_project_case(upstream_query: str) -> BuildExecutionPlanTestCase: + """Clean orders and the daily totals maintained from it, both refreshed within an hour.""" + + return BuildExecutionPlanTestCase( + description="clean orders and the daily totals maintained from it", + setup_sql=(), + model_locations={"clean_orders": "staging", "daily_totals": "staging"}, + model_configs={"clean_orders": _MAINTAINED, "daily_totals": _MAINTAINED}, + model_queries={ + "clean_orders": upstream_query, + "daily_totals": "SELECT count(*) AS orders FROM staging.clean_orders", + }, + model_deps={"daily_totals": ("clean_orders",)}, + full_refresh=False, + expected_action={}, + expected_reason={}, + ) diff --git a/tests/integration/src/sqlbuild/compiler/planner/main/test_maintained_refresh_plan.py b/tests/integration/src/sqlbuild/compiler/planner/main/test_maintained_refresh_plan.py new file mode 100644 index 0000000000..913b2b126d --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/planner/main/test_maintained_refresh_plan.py @@ -0,0 +1,109 @@ +"""A replan of a maintained model reads its live refresh settings and marks rebuilt upstreams.""" + +from __future__ import annotations + +from typing import Any + +import pytest + +from sqlbuild._native import ModelPlanEntry, PlanAction, PlanOutput, PlanReason +from sqlbuild.executor.run._helpers.reuse.fingerprinting import try_write_fingerprint +from tests.integration.src.sqlbuild.compiler.planner.main._test_types import ( + MaintainedRefreshPlanTestCase, +) +from tests.integration.src.sqlbuild.compiler.planner.main.helpers import ( + CLEAN_ORDERS_QUERY, + ListingDuckDbAdapter, + build_execution_plan_from_kwargs, + build_project_from_test_case, + maintained_project_case, +) + + +@pytest.mark.parametrize( + "test_case", + [ + MaintainedRefreshPlanTestCase( + description="listed settings that match keep both models", + listed_settings=("60 minutes", "TRANSFORM_WH"), + upstream_query=CLEAN_ORDERS_QUERY, + expected_upstream_action=PlanAction.KEEP_MAINTAINED, + expected_action=PlanAction.KEEP_MAINTAINED, + expected_reason=PlanReason.NO_CHANGE, + expected_reinitialised_by=None, + expected_read_names=("clean_orders", "daily_totals"), + ), + MaintainedRefreshPlanTestCase( + description="a lag changed outside the project is altered in place", + listed_settings=("2 hours", "TRANSFORM_WH"), + upstream_query=CLEAN_ORDERS_QUERY, + expected_upstream_action=PlanAction.ALTER_MAINTAINED, + expected_action=PlanAction.ALTER_MAINTAINED, + expected_reason=PlanReason.CONFIG_CHANGED, + expected_reinitialised_by=None, + expected_read_names=("clean_orders", "daily_totals"), + ), + MaintainedRefreshPlanTestCase( + description="a name that holds no dynamic table is created", + listed_settings=None, + upstream_query=CLEAN_ORDERS_QUERY, + expected_upstream_action=PlanAction.CREATE_MAINTAINED, + expected_action=PlanAction.CREATE_MAINTAINED, + expected_reason=PlanReason.FIRST_RUN, + expected_reinitialised_by=None, + expected_read_names=("clean_orders", "daily_totals"), + ), + MaintainedRefreshPlanTestCase( + description="a recreated upstream marks the kept reader as recomputing in full", + listed_settings=("1 hour", "TRANSFORM_WH"), + upstream_query="SELECT 2 AS order_id", + expected_upstream_action=PlanAction.CREATE_MAINTAINED, + expected_action=PlanAction.KEEP_MAINTAINED, + expected_reason=PlanReason.NO_CHANGE, + expected_reinitialised_by="clean_orders", + expected_read_names=("daily_totals",), + ), + ], + ids=lambda case: case.description, +) +def test_given_built_maintained_models_when_replanning_then_live_settings_and_rebuilds_decide( + test_case: MaintainedRefreshPlanTestCase, +) -> None: + adapter: ListingDuckDbAdapter = ListingDuckDbAdapter(test_case.listed_settings) + connection: Any = adapter.connect({"database": ":memory:"}) + connection.execute("CREATE SCHEMA staging") + first: PlanOutput = build_execution_plan_from_kwargs( + project=build_project_from_test_case(maintained_project_case(CLEAN_ORDERS_QUERY)), + adapter=adapter, + connection=connection, + ) + connection.execute("CREATE TABLE staging.clean_orders AS SELECT 1 AS order_id") + connection.execute("CREATE TABLE staging.daily_totals AS SELECT 1 AS orders") + warnings: list[str] = [] + for entry in first.model_entries: + warnings.extend( + try_write_fingerprint( + entry=entry, + adapter=adapter, + connection=connection, + run_id="first_build", + query_change_tracking=True, + ) + ) + + plan: PlanOutput = build_execution_plan_from_kwargs( + project=build_project_from_test_case(maintained_project_case(test_case.upstream_query)), + adapter=adapter, + connection=connection, + ) + + entries: dict[str, ModelPlanEntry] = {entry.name: entry for entry in plan.model_entries} + assert warnings == [] + assert entries["clean_orders"].action == test_case.expected_upstream_action + assert entries["daily_totals"].action == test_case.expected_action + assert entries["daily_totals"].reason == test_case.expected_reason + assert entries["daily_totals"].reinitialised_by == test_case.expected_reinitialised_by + assert entries["clean_orders"].reinitialised_by is None + assert entries["daily_totals"].target_lag == "'1 hour'" + assert tuple(sorted(adapter.read_names)) == test_case.expected_read_names + adapter.close(connection) diff --git a/tests/unit/src/sqlbuild/adapter/contract/classes/strict_adapter/test_strict_adapter.py b/tests/unit/src/sqlbuild/adapter/contract/classes/strict_adapter/test_strict_adapter.py index faad1a7145..66ba9eeb66 100644 --- a/tests/unit/src/sqlbuild/adapter/contract/classes/strict_adapter/test_strict_adapter.py +++ b/tests/unit/src/sqlbuild/adapter/contract/classes/strict_adapter/test_strict_adapter.py @@ -89,6 +89,7 @@ def test_given_first_class_adapters_when_checking_contract_then_no_method_comes_ { "create_sql_test_output", "drop_sql_test_output", + "maintained_refresh_settings", "render_create_temporary_function", "render_drop_schema", } diff --git a/tests/unit/src/sqlbuild/adapters/snowflake/_test_types.py b/tests/unit/src/sqlbuild/adapters/snowflake/_test_types.py index 0fb684cdf3..adfe4381a6 100644 --- a/tests/unit/src/sqlbuild/adapters/snowflake/_test_types.py +++ b/tests/unit/src/sqlbuild/adapters/snowflake/_test_types.py @@ -453,3 +453,10 @@ class SnowflakeKeyedDeleteInsertFailureTestCase: class SnowflakeWarehouseUserTestCase: description: str expected_expression: str + + +@dataclass(frozen=True) +class SnowflakeMaintainedRefreshTestCase: + description: str + rows: tuple[tuple[object, ...], ...] + expected_settings: tuple[str, str] | None diff --git a/tests/unit/src/sqlbuild/adapters/snowflake/test_maintained_refresh_settings.py b/tests/unit/src/sqlbuild/adapters/snowflake/test_maintained_refresh_settings.py new file mode 100644 index 0000000000..450e68f0ed --- /dev/null +++ b/tests/unit/src/sqlbuild/adapters/snowflake/test_maintained_refresh_settings.py @@ -0,0 +1,50 @@ +import pytest + +from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter +from tests.unit.src.sqlbuild.adapters.helpers import MetadataConnection, execute_through_cursor +from tests.unit.src.sqlbuild.adapters.snowflake._test_types import ( + SnowflakeMaintainedRefreshTestCase, +) + +_SHOW_DYNAMIC_TABLES_COLUMNS: tuple[str, ...] = ("name", "target_lag", "warehouse") + + +@pytest.mark.parametrize( + "test_case", + [ + SnowflakeMaintainedRefreshTestCase( + description="a dynamic table lists its lag and refresh warehouse", + rows=(("DAILY_TOTALS", "1 hour", "TRANSFORM_WH"),), + expected_settings=("1 hour", "TRANSFORM_WH"), + ), + SnowflakeMaintainedRefreshTestCase( + description="a neighbour matched by the LIKE wildcard is not the model", + rows=(("DAILY_TOTALS_ARCHIVE", "DOWNSTREAM", "TRANSFORM_WH"),), + expected_settings=None, + ), + SnowflakeMaintainedRefreshTestCase( + description="a relation that is no dynamic table lists nothing", + rows=(), + expected_settings=None, + ), + ], + ids=lambda case: case.description, +) +def test_given_dynamic_table_listing_when_reading_refresh_settings_then_reads_one_show( + test_case: SnowflakeMaintainedRefreshTestCase, monkeypatch: pytest.MonkeyPatch +) -> None: + adapter: SnowflakeAdapter = SnowflakeAdapter() + connection: MetadataConnection = MetadataConnection( + test_case.rows, _SHOW_DYNAMIC_TABLES_COLUMNS + ) + monkeypatch.setattr(adapter, "execute", execute_through_cursor) + + settings: tuple[str, str] | None = adapter.maintained_refresh_settings( + connection=connection, database="ANALYTICS", schema="MARTS", name="daily_totals" + ) + + assert settings == test_case.expected_settings + assert len(connection.executed) == 1 + assert connection.executed[0].startswith( + 'SHOW DYNAMIC TABLES LIKE \'DAILY_TOTALS\' IN SCHEMA "ANALYTICS"."MARTS"' + ) diff --git a/ui/src/lib/api/types/operate.ts b/ui/src/lib/api/types/operate.ts index e6ae455140..c73e4ece68 100644 --- a/ui/src/lib/api/types/operate.ts +++ b/ui/src/lib/api/types/operate.ts @@ -656,6 +656,8 @@ export type PlanModel = { backfill: { action: string; duration: string | null }; qualified_name?: string; changed_functions?: string[]; + /** The rebuilt upstream that makes this kept maintained model recompute in full. */ + reinitialised_by?: string; cursor_type?: string; cursor?: { requested_bounds: { start: string | null; end: string | null }; resolved_bounds: { start: string | null; end: string | null } | null } | null; /** Unified diff lines of the query since the model's last build, as `sqb plan` prints them. */ diff --git a/ui/src/lib/run/plan-text.test.ts b/ui/src/lib/run/plan-text.test.ts index 2ab282275e..d0ebe9db34 100644 --- a/ui/src/lib/run/plan-text.test.ts +++ b/ui/src/lib/run/plan-text.test.ts @@ -44,6 +44,12 @@ describe('plan text, as sqb plan prints it', () => { expect(section.entries[0].detail).toEqual([]); }); + it('says a kept maintained model recomputes in full when an upstream it reads is rebuilt', () => { + const kept = model('daily_totals', { materialization_type: 'maintained', action: 'keep_maintained', reinitialised_by: 'orders' }); + const [routine] = planSections(plan([kept])); + expect(routine.entries).toEqual([{ name: 'daily_totals', value: 'maintained', detail: ['recomputes in full: orders is rebuilt'] }]); + }); + it('puts each change in its own section in reason order, then the routine models, and leaves skipped ones out', () => { const sections = planSections( plan([ diff --git a/ui/src/lib/run/plan-text.ts b/ui/src/lib/run/plan-text.ts index 4fbf432b62..f8304e9750 100644 --- a/ui/src/lib/run/plan-text.ts +++ b/ui/src/lib/run/plan-text.ts @@ -189,7 +189,7 @@ export function planSections(plan: PlanDocument): PlanSection[] { const why = changedRuns(plan); add('Models', active.filter((m) => ROUTINE.has(m.reason)).map((m) => { const run = why.get(m.name); - return { name: m.name, value: materializationLabel(m), detail: [...cursorLines(m, true, starts[m.name]), ...(run ? [run.downstream_of ? `reads ${keyName(run.downstream_of)}` : run.reason] : [])] }; + return { name: m.name, value: materializationLabel(m), detail: [...cursorLines(m, true, starts[m.name]), ...(m.reinitialised_by ? [`recomputes in full: ${m.reinitialised_by} is rebuilt`] : []), ...(run ? [run.downstream_of ? `reads ${keyName(run.downstream_of)}` : run.reason] : [])] }; }), false, 'routine_models'); } add('Functions', plan.functions.filter((f) => !f.reason || f.reason === 'no_change').map(functionEntry), false, 'functions'); From 3e9c1bab5984d3d94f90cb6d6404639293292e1a Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 19 Nov 2026 00:07:25 +0000 Subject: [PATCH 06/17] feat(build): create, alter and keep maintained models on Snowflake Work in progress. A build now creates a maintained model as a dynamic table with CREATE OR REPLACE, an explicit REFRESH_MODE (incremental by default, so a query that cannot refresh incrementally fails at create) and INITIALIZE = ON_CREATE, so the table is readable at once. It runs on refresh_warehouse, or else the session's warehouse; with neither, the build fails with A717 before any statement. A drifted lag or declared warehouse is set in place with ALTER, and a kept model runs no refresh statement. A table or view the model was before is dropped and replaced; a relation the warehouse lists as a dynamic table is never dropped. Plan entries carry the model's target_lag, refresh_mode and refresh_warehouse. The create, alter, suspend, resume and refresh SQL are optional adapter methods; adapters without dynamic tables raise A716. The view and maintained executors share one lifecycle for hooks, final audits and the fingerprint. Validated: Snowflake SQL rendering, the strict adapter contract, the maintained executor on DuckDB (create, declared warehouse, replace a former view, alter, keep, no warehouse, never drop a dynamic table), the view executor unchanged, and the run, scenario, executor and adapter suites. --- .../_helpers/planner_records/constants.rs | 2 + .../_helpers/outputs/maintained.rs | 23 ++- src/sqlbuild/_native.pyi | 4 + .../contract/classes/strict_adapter.py | 38 +++++ src/sqlbuild/adapter/contract/types.py | 8 + .../snowflake/classes/snowflake_adapter.py | 47 ++++++ .../executor/build/_helpers/scheduler.py | 4 + src/sqlbuild/executor/build/constants.py | 7 + .../_helpers/materializations/maintained.py | 124 +++++++++++++++ .../run/_helpers/materializations/view.py | 38 +++-- src/sqlbuild/executor/run/main/_execute.py | 3 + .../main/test_maintained_refresh_plan.py | 2 + .../executor/run/maintained/__init__.py | 0 .../executor/run/maintained/_test_types.py | 26 ++++ .../executor/run/maintained/helpers.py | 108 +++++++++++++ .../maintained/test_maintained_execution.py | 146 ++++++++++++++++++ .../strict_adapter/test_strict_adapter.py | 14 ++ .../adapters/snowflake/_test_types.py | 26 +++- .../test_maintained_refresh_settings.py | 123 +++++++++++++++ 19 files changed, 727 insertions(+), 16 deletions(-) create mode 100644 src/sqlbuild/executor/run/_helpers/materializations/maintained.py create mode 100644 tests/integration/src/sqlbuild/executor/run/maintained/__init__.py create mode 100644 tests/integration/src/sqlbuild/executor/run/maintained/_test_types.py create mode 100644 tests/integration/src/sqlbuild/executor/run/maintained/helpers.py create mode 100644 tests/integration/src/sqlbuild/executor/run/maintained/test_maintained_execution.py diff --git a/crates/sqlbuild-python-base/src/records/_helpers/planner_records/constants.rs b/crates/sqlbuild-python-base/src/records/_helpers/planner_records/constants.rs index 5826828d83..55ba072f4c 100644 --- a/crates/sqlbuild-python-base/src/records/_helpers/planner_records/constants.rs +++ b/crates/sqlbuild-python-base/src/records/_helpers/planner_records/constants.rs @@ -668,6 +668,8 @@ const R_MODEL_PLAN_ENTRY: RecordSpec = RecordSpec { FieldSpec::new("max_duration_source", FieldDefault::None), FieldSpec::new("reinitialised_by", FieldDefault::None), FieldSpec::new("target_lag", FieldDefault::None), + FieldSpec::new("refresh_mode", FieldDefault::None), + FieldSpec::new("refresh_warehouse", FieldDefault::None), ], }; const R_CLONE_SOURCE_PLAN_ENTRY: RecordSpec = RecordSpec { diff --git a/crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/maintained.rs b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/maintained.rs index 9297abf940..988ccf7267 100644 --- a/crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/maintained.rs +++ b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/maintained.rs @@ -4,6 +4,7 @@ use crate::planner_phases::main::inputs::runtime_reads::runtime_reads; use crate::planner_phases::models::ModelEntryResults; use crate::planner_strategy::main::values::text::text; use crate::planner_warehouse::models::{AdapterRead, PlannerReads, Read}; +use pyo3::IntoPyObject; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::PyString; use sqlbuild_analysis::maintained_refresh::main::maintained_settings_drift::maintained_settings_drift; @@ -27,6 +28,8 @@ use std::collections::HashMap; const MAINTAINED: &str = "maintained"; const KEEP_MAINTAINED: &str = "keep_maintained"; +/// The refresh mode of a maintained model that declares none, so Snowflake never picks one quietly. +const DEFAULT_REFRESH_MODE: &str = "incremental"; /// Actions that recreate the relation, which reinitialises every dynamic table reading it. const REPLACING_ACTIONS: [&str; 3] = ["create_table", "create_view", "create_maintained"]; @@ -95,10 +98,22 @@ fn with_refresh_settings<'py>( let Some(desired) = declared_settings(&model)? else { return Ok(entry); }; - let mut changes: Vec<(&str, Bound<'py, PyAny>)> = vec![( - "target_lag", - PyString::new(py, &render_target_lag(desired.lag)).into_any(), - )]; + let values = model_values(&model)?; + let mode: String = match config_text_object(py, &values, "refresh_mode")? { + Some(mode) => mode.extract()?, + None => DEFAULT_REFRESH_MODE.to_owned(), + }; + let mut changes: Vec<(&str, Bound<'py, PyAny>)> = vec![ + ( + "target_lag", + PyString::new(py, &render_target_lag(desired.lag)).into_any(), + ), + ("refresh_mode", PyString::new(py, &mode).into_any()), + ( + "refresh_warehouse", + desired.warehouse.as_deref().into_pyobject(py)?.into_any(), + ), + ]; if text(&native_field(&entry, "action")?)? == KEEP_MAINTAINED { if let Some((action, reason)) = live_action(&entry, &desired, reads)? { changes.push(("action", enum_member(py, RecordId::PlanAction, action)?)); diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index b96441627a..5b3f4a149a 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -14121,6 +14121,8 @@ class ModelPlanEntry(NativeRecord): max_duration_source: str | None reinitialised_by: str | None target_lag: str | None + refresh_mode: str | None + refresh_warehouse: str | None def __init__( self, key: CompiledObjectKey, @@ -14212,6 +14214,8 @@ class ModelPlanEntry(NativeRecord): max_duration_source: str | None = ..., reinitialised_by: str | None = ..., target_lag: str | None = ..., + refresh_mode: str | None = ..., + refresh_warehouse: str | None = ..., ) -> None: ... class CloneSourcePlanEntry(NativeRecord): diff --git a/src/sqlbuild/adapter/contract/classes/strict_adapter.py b/src/sqlbuild/adapter/contract/classes/strict_adapter.py index 2d102a0760..6b9f40d251 100644 --- a/src/sqlbuild/adapter/contract/classes/strict_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/strict_adapter.py @@ -33,6 +33,7 @@ from sqlbuild.adapter.contract.types import ( FrameworkType, LoaderLogicalType, + MaintainedOperation, PromotionStrategy, StatementSizeLimit, TablePromotionMode, @@ -321,6 +322,43 @@ def render_drop_schema( help="sqb review needs a warehouse that can drop a schema it created", ) + def session_warehouse(self, *, connection: Any) -> str | None: + """Optional: the warehouse the session runs on, or None when the adapter has none.""" + + del connection + return None + + def render_create_maintained( + self, *, destination: str, sql: str, target_lag: str, refresh_mode: str, warehouse: str + ) -> tuple[str, ...]: + """Optional: SQL that creates or replaces a maintained model; unsupported by default.""" + + del sql, target_lag, refresh_mode, warehouse + raise self._maintained_unsupported(destination) + + def render_alter_maintained( + self, *, destination: str, target_lag: str, warehouse: str | None + ) -> tuple[str, ...]: + """Optional: SQL that changes a maintained model's refresh settings in place.""" + + del target_lag, warehouse + raise self._maintained_unsupported(destination) + + def render_maintained_operation( + self, *, destination: str, operation: MaintainedOperation + ) -> tuple[str, ...]: + """Optional: SQL that suspends, resumes or refreshes a maintained model.""" + + del operation + raise self._maintained_unsupported(destination) + + def _maintained_unsupported(self, destination: str) -> AdapterUserError: + return AdapterUserError( + f"the {self.adapter_name} adapter cannot build maintained model '{destination}'", + code="A716", + help="materialized maintained needs an adapter whose warehouse refreshes it", + ) + def maintained_refresh_settings( self, *, connection: Any, database: str | None, schema: str | None, name: str ) -> tuple[str, str] | None: diff --git a/src/sqlbuild/adapter/contract/types.py b/src/sqlbuild/adapter/contract/types.py index 0ef3963263..843ae80ca9 100644 --- a/src/sqlbuild/adapter/contract/types.py +++ b/src/sqlbuild/adapter/contract/types.py @@ -82,6 +82,14 @@ class RetentionChangePhase(StrEnum): FINALIZE = "finalize" +class MaintainedOperation(StrEnum): + """What a build does to a maintained model's scheduled refreshes.""" + + SUSPEND = "suspend" + RESUME = "resume" + REFRESH = "refresh" + + class RelationType(StrEnum): TABLE = "table" VIEW = "view" diff --git a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py index 99c3c5f565..bce960cbe6 100644 --- a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py +++ b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py @@ -6,6 +6,7 @@ import json import logging import os +import re import time from collections.abc import Callable from dataclasses import replace @@ -98,6 +99,7 @@ HistoricalSnapshotCloseStyle, HistoricalSnapshotInsertStyle, LoaderLogicalType, + MaintainedOperation, MigrationTransfer, PromotionStrategy, RelationReadStatus, @@ -204,6 +206,7 @@ "table_name, table_schema, table_type, is_transient, created, last_altered, retention_time" ) _BULK_COLUMN_RELATION_CHUNK_SIZE: int = 2_000 +_PLAIN_WAREHOUSE_NAME: re.Pattern[str] = re.compile(r"[A-Za-z_][A-Za-z0-9_$]*") _RELATION_CHANGE_COLUMNS: str = ( "table_name, table_schema, table_type, is_transient, created, last_altered, last_ddl, " "retention_time" @@ -3665,6 +3668,50 @@ def column_rename_unavailable_reason(self, *, connection: Any, destination: str) del connection, destination return None + def session_warehouse(self, *, connection: Any) -> str | None: + """The session's warehouse, which refreshes a maintained model that names none.""" + + rows: tuple[tuple[object, ...], ...] = self.query( + connection=connection, sql="SELECT CURRENT_WAREHOUSE()", limit=None + ).rows + return None if not rows or rows[0][0] is None else str(rows[0][0]) + + def render_create_maintained( + self, *, destination: str, sql: str, target_lag: str, refresh_mode: str, warehouse: str + ) -> tuple[str, ...]: + """A dynamic table with an explicit mode and ON_CREATE, so it is readable at once.""" + + return ( + f"CREATE OR REPLACE DYNAMIC TABLE {destination} TARGET_LAG = {target_lag} " + f"WAREHOUSE = {self._warehouse_name(warehouse)} " + f"REFRESH_MODE = {refresh_mode.upper()} INITIALIZE = ON_CREATE AS {sql}", + ) + + def render_alter_maintained( + self, *, destination: str, target_lag: str, warehouse: str | None + ) -> tuple[str, ...]: + """The lag, and a declared warehouse, set in place.""" + + statements: list[str] = [f"ALTER DYNAMIC TABLE {destination} SET TARGET_LAG = {target_lag}"] + if warehouse is not None: + statements.append( + f"ALTER DYNAMIC TABLE {destination} " + f"SET WAREHOUSE = {self._warehouse_name(warehouse)}" + ) + return tuple(statements) + + def render_maintained_operation( + self, *, destination: str, operation: MaintainedOperation + ) -> tuple[str, ...]: + return (f"ALTER DYNAMIC TABLE {destination} {operation.value.upper()}",) + + def _warehouse_name(self, warehouse: str) -> str: + """A plain warehouse name as written, resolved without case; any other quoted exactly.""" + + if _PLAIN_WAREHOUSE_NAME.fullmatch(warehouse): + return warehouse + return '"' + warehouse.replace('"', '""') + '"' + def maintained_refresh_settings( self, *, connection: Any, database: str | None, schema: str | None, name: str ) -> tuple[str, str] | None: diff --git a/src/sqlbuild/executor/build/_helpers/scheduler.py b/src/sqlbuild/executor/build/_helpers/scheduler.py index 727e48f49e..527dd72fff 100644 --- a/src/sqlbuild/executor/build/_helpers/scheduler.py +++ b/src/sqlbuild/executor/build/_helpers/scheduler.py @@ -25,6 +25,7 @@ BUILD_CUSTOM_MATERIALIZATION_MISSING_CODE, BUILD_SOURCE_FRESHNESS_BLOCKED_CODE, INCREMENTAL_ACTIONS, + MAINTAINED_ACTIONS, ) from sqlbuild.executor.build.models import ( FunctionExecutionResult, @@ -36,6 +37,7 @@ from sqlbuild.executor.run.main._execute import ( execute_custom_entry, execute_incremental_entry, + execute_maintained_entry, execute_microbatch_entry, execute_snapshot_entry, execute_table_entry, @@ -163,6 +165,8 @@ def _dispatch_model_entry( context=context, declared_columns=entry.declared_columns, ) + if entry.action in MAINTAINED_ACTIONS: + return execute_maintained_entry(context=context) if entry.action == PlanAction.CREATE_VIEW: return execute_view_entry(context=context) if entry.action == PlanAction.SNAPSHOT: diff --git a/src/sqlbuild/executor/build/constants.py b/src/sqlbuild/executor/build/constants.py index f7109ec129..f4d7636187 100644 --- a/src/sqlbuild/executor/build/constants.py +++ b/src/sqlbuild/executor/build/constants.py @@ -11,6 +11,13 @@ PlanAction.INCREMENTAL_MERGE, } ) +MAINTAINED_ACTIONS: frozenset[PlanAction] = frozenset( + { + PlanAction.CREATE_MAINTAINED, + PlanAction.ALTER_MAINTAINED, + PlanAction.KEEP_MAINTAINED, + } +) BUILD_WORKER_FAILED_CODE: str = "B001" BUILD_UNKNOWN_RESOURCE_FAILED_CODE: str = "B002" diff --git a/src/sqlbuild/executor/run/_helpers/materializations/maintained.py b/src/sqlbuild/executor/run/_helpers/materializations/maintained.py new file mode 100644 index 0000000000..4fabbd0b29 --- /dev/null +++ b/src/sqlbuild/executor/run/_helpers/materializations/maintained.py @@ -0,0 +1,124 @@ +"""Maintained model lifecycle: the warehouse refreshes the model; a build creates or alters it.""" + +from __future__ import annotations + +from collections.abc import Callable +from typing import Any + +from sqlbuild._native import CompiledRelationLocation, ModelPlanEntry, PlanAction +from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter +from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder +from sqlbuild.adapter.contract.exceptions import AdapterUserError +from sqlbuild.adapter.contract.types import RelationType +from sqlbuild.adapter.relations.main.cached_relation_type import cached_relation_type +from sqlbuild.executor.run._helpers.materializations.view import execute_in_place_entry +from sqlbuild.executor.run.models import ModelExecutionResult, ModelMaterializationContext + +_NO_WAREHOUSE_CODE: str = "A717" + + +def execute_maintained_entry(*, context: ModelMaterializationContext) -> ModelExecutionResult: + """Create, alter or keep one maintained model, then run its audits and hooks.""" + + def materialize(*, target_qualified: str, statement_recorder: StatementRecorder) -> None: + entry: ModelPlanEntry = context.entry + if entry.action == PlanAction.CREATE_MAINTAINED: + _create(context=context, destination=target_qualified, recorder=statement_recorder) + elif entry.action == PlanAction.ALTER_MAINTAINED: + _run( + adapter=context.adapter, + connection=context.connection, + statements=context.adapter.render_alter_maintained( + destination=target_qualified, + target_lag=str(entry.target_lag), + warehouse=entry.refresh_warehouse, + ), + recorder=statement_recorder, + ) + + return execute_in_place_entry( + context=context, + materialize=materialize, + action_name=str(context.entry.action), + built="the maintained model's refresh", + ) + + +def _create( + *, context: ModelMaterializationContext, destination: str, recorder: StatementRecorder +) -> None: + """Create or replace; a table or view the model was before is dropped, never a dynamic table.""" + + entry: ModelPlanEntry = context.entry + adapter: BaseAdapter = context.adapter + connection: Any = context.connection + warehouse: str | None = entry.refresh_warehouse or adapter.session_warehouse( + connection=connection + ) + if warehouse is None: + raise AdapterUserError( + f"maintained model '{entry.name}' has no warehouse to refresh it", + code=_NO_WAREHOUSE_CODE, + help="set refresh_warehouse on the model, or a warehouse on the target's connection", + ) + statements: tuple[str, ...] = adapter.render_create_maintained( + destination=destination, + sql=entry.resolved_sql, + target_lag=str(entry.target_lag), + refresh_mode=str(entry.refresh_mode), + warehouse=warehouse, + ) + try: + _run(adapter=adapter, connection=connection, statements=statements, recorder=recorder) + except Exception as error: + former: RelationType | None = _former_relation( + adapter=adapter, connection=connection, entry=entry + ) + if former is None: + raise error + drop: Callable[..., None] = ( + adapter.drop_view if former == RelationType.VIEW else adapter.drop + ) + drop( + connection=connection, + destination=destination, + if_exists=True, + statement_recorder=recorder, + ) + _run(adapter=adapter, connection=connection, statements=statements, recorder=recorder) + + +def _former_relation( + *, adapter: BaseAdapter, connection: Any, entry: ModelPlanEntry +) -> RelationType | None: + """The type of a relation at the destination the warehouse does not list as maintained.""" + + location: CompiledRelationLocation = entry.destination + existing: RelationType | None = cached_relation_type( + adapter=adapter, + connection=connection, + database=location.database, + schema=location.schema, + name=location.name, + ) + if existing is None: + return None + listed: tuple[str, str] | None = adapter.maintained_refresh_settings( + connection=connection, + database=location.database, + schema=location.schema, + name=location.name, + ) + return existing if listed is None else None + + +def _run( + *, + adapter: BaseAdapter, + connection: Any, + statements: tuple[str, ...], + recorder: StatementRecorder, +) -> None: + recorder.record_many(statements) + for statement in statements: + adapter.execute(connection=connection, sql=statement) diff --git a/src/sqlbuild/executor/run/_helpers/materializations/view.py b/src/sqlbuild/executor/run/_helpers/materializations/view.py index 1ab396dd2c..b266b0b004 100644 --- a/src/sqlbuild/executor/run/_helpers/materializations/view.py +++ b/src/sqlbuild/executor/run/_helpers/materializations/view.py @@ -2,7 +2,7 @@ from __future__ import annotations -from collections.abc import Mapping +from collections.abc import Callable, Mapping from typing import Any from sqlbuild._native import ( @@ -45,6 +45,29 @@ def execute_view_entry( ) -> ModelExecutionResult: """Execute one view model through its full materialization lifecycle.""" + def materialize(*, target_qualified: str, statement_recorder: StatementRecorder) -> None: + _create_view_replacing_table( + adapter=context.adapter, + connection=context.connection, + entry=context.entry, + target_qualified=target_qualified, + statement_recorder=statement_recorder, + ) + + return execute_in_place_entry( + context=context, materialize=materialize, action_name="create_view", built="view creation" + ) + + +def execute_in_place_entry( + *, + context: ModelMaterializationContext, + materialize: Callable[..., None], + action_name: str, + built: str, +) -> ModelExecutionResult: + """Hooks, schema, `materialize` on the destination, final audits and the fingerprint.""" + entry: ModelPlanEntry = context.entry adapter: BaseAdapter = context.adapter connection: Any = context.connection @@ -112,14 +135,8 @@ def execute_view_entry( schema=target_schema, statement_recorder=statement_recorder, ) - with diagnostics_context(sqlbuild_phase="materialize", sqlbuild_action_name="create_view"): - _create_view_replacing_table( - adapter=adapter, - connection=connection, - entry=entry, - target_qualified=target_qualified, - statement_recorder=statement_recorder, - ) + with diagnostics_context(sqlbuild_phase="materialize", sqlbuild_action_name=action_name): + materialize(target_qualified=target_qualified, statement_recorder=statement_recorder) except Exception as exc: return build_failed_result( entry=entry, @@ -154,8 +171,7 @@ def execute_view_entry( entry=entry, phase=ExecutionPhase.AUDIT, error=( - f"final audit for '{entry.name}' failed after view creation " - "with severity level: error" + f"final audit for '{entry.name}' failed after {built} with severity level: error" ), promoted_relation=target_qualified, warnings=warnings, diff --git a/src/sqlbuild/executor/run/main/_execute.py b/src/sqlbuild/executor/run/main/_execute.py index 7089ac50e8..1d4cf77f2d 100644 --- a/src/sqlbuild/executor/run/main/_execute.py +++ b/src/sqlbuild/executor/run/main/_execute.py @@ -44,6 +44,9 @@ from sqlbuild.executor.run._helpers.materializations.incremental import ( execute_incremental_entry as execute_incremental_entry, ) +from sqlbuild.executor.run._helpers.materializations.maintained import ( + execute_maintained_entry as execute_maintained_entry, +) from sqlbuild.executor.run._helpers.materializations.microbatch import ( execute_microbatch_entry as execute_microbatch_entry, ) diff --git a/tests/integration/src/sqlbuild/compiler/planner/main/test_maintained_refresh_plan.py b/tests/integration/src/sqlbuild/compiler/planner/main/test_maintained_refresh_plan.py index 913b2b126d..c037cd3f2a 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/main/test_maintained_refresh_plan.py +++ b/tests/integration/src/sqlbuild/compiler/planner/main/test_maintained_refresh_plan.py @@ -105,5 +105,7 @@ def test_given_built_maintained_models_when_replanning_then_live_settings_and_re assert entries["daily_totals"].reinitialised_by == test_case.expected_reinitialised_by assert entries["clean_orders"].reinitialised_by is None assert entries["daily_totals"].target_lag == "'1 hour'" + assert entries["daily_totals"].refresh_mode == "incremental" + assert entries["daily_totals"].refresh_warehouse == "transform_wh" assert tuple(sorted(adapter.read_names)) == test_case.expected_read_names adapter.close(connection) diff --git a/tests/integration/src/sqlbuild/executor/run/maintained/__init__.py b/tests/integration/src/sqlbuild/executor/run/maintained/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/integration/src/sqlbuild/executor/run/maintained/_test_types.py b/tests/integration/src/sqlbuild/executor/run/maintained/_test_types.py new file mode 100644 index 0000000000..617c6d926a --- /dev/null +++ b/tests/integration/src/sqlbuild/executor/run/maintained/_test_types.py @@ -0,0 +1,26 @@ +from __future__ import annotations + +from dataclasses import dataclass + +from sqlbuild._native import PlanAction + + +@dataclass(frozen=True) +class MaintainedSuccessTestCase: + description: str + action: PlanAction + setup_sql: tuple[str, ...] + refresh_warehouse: str | None + expected_created: tuple[tuple[str, str, str], ...] + expected_altered: tuple[tuple[str, str | None], ...] + expected_rows: tuple[tuple[int, ...], ...] + + +@dataclass(frozen=True) +class MaintainedFailureTestCase: + description: str + setup_sql: tuple[str, ...] + session_warehouse: str | None + listed_settings: tuple[str, str] | None + expected_error_code: str + expected_listed_types: tuple[tuple[str], ...] diff --git a/tests/integration/src/sqlbuild/executor/run/maintained/helpers.py b/tests/integration/src/sqlbuild/executor/run/maintained/helpers.py new file mode 100644 index 0000000000..d9e297440d --- /dev/null +++ b/tests/integration/src/sqlbuild/executor/run/maintained/helpers.py @@ -0,0 +1,108 @@ +"""A DuckDB adapter that builds maintained models as tables, for executor lifecycle tests.""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +from sqlbuild._native import ( + CompiledObjectKey, + CompiledRelationLocation, + CompiledResourceType, + MaterializationType, + ModelPlanEntry, + PlanAction, + PlanReason, +) +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from sqlbuild.executor.run.models import ModelMaterializationContext + +DAILY_TOTALS_SQL: str = "SELECT 3 AS orders" + + +class MaintainedDuckDbAdapter(DuckDbAdapter): + """Builds a maintained model as a table that, like a dynamic table, replaces nothing else.""" + + def __init__( + self, *, session_warehouse: str | None, listed_settings: tuple[str, str] | None = None + ) -> None: + super().__init__() + self.warehouse: str | None = session_warehouse + self.listed_settings: tuple[str, str] | None = listed_settings + self.created: list[tuple[str, str, str]] = [] + self.altered: list[tuple[str, str | None]] = [] + + def session_warehouse(self, *, connection: Any) -> str | None: + del connection + return self.warehouse + + def maintained_refresh_settings( + self, *, connection: Any, database: str | None, schema: str | None, name: str + ) -> tuple[str, str] | None: + del connection, database, schema, name + return self.listed_settings + + def render_create_maintained( + self, *, destination: str, sql: str, target_lag: str, refresh_mode: str, warehouse: str + ) -> tuple[str, ...]: + self.created.append((target_lag, refresh_mode, warehouse)) + return (f"CREATE TABLE {destination} AS {sql}",) + + def render_alter_maintained( + self, *, destination: str, target_lag: str, warehouse: str | None + ) -> tuple[str, ...]: + self.altered.append((target_lag, warehouse)) + lag: str = target_lag.replace("'", "''") + return (f"COMMENT ON TABLE {destination} IS 'refreshed within {lag}'",) + + +def maintained_context( + *, + adapter: MaintainedDuckDbAdapter, + connection: Any, + action: PlanAction, + refresh_warehouse: str | None, +) -> ModelMaterializationContext: + """Daily totals, maintained within an hour, planned with `action`.""" + + entry: ModelPlanEntry = ModelPlanEntry( + key=CompiledObjectKey(resource_type=CompiledResourceType.MODEL, name="daily_totals"), + name="daily_totals", + relative_path=Path("models/daily_totals.sql"), + materialization_type=MaterializationType.MAINTAINED, + action=action, + reason=PlanReason.FIRST_RUN, + destination=CompiledRelationLocation( + database=None, + schema="staging", + name="daily_totals", + qualified_name="staging.daily_totals", + ), + fingerprint_query_sql=DAILY_TOTALS_SQL, + resolved_sql=DAILY_TOTALS_SQL, + logical_ddl="", + target_lag="'1 hour'", + refresh_mode="incremental", + refresh_warehouse=refresh_warehouse, + ) + return ModelMaterializationContext( + entry=entry, + adapter=adapter, + connection=connection, + model_locations={}, + seed_locations={}, + source_map={}, + python_source_read_map={}, + model_audits=(), + run_id="maintained_run", + query_change_tracking=True, + ) + + +def listed_types(*, connection: Any) -> list[tuple[Any, ...]]: + """The relation types daily_totals is listed as; empty when nothing holds the name.""" + + return connection.execute( + "SELECT table_type FROM information_schema.tables " + "WHERE table_schema = 'staging' AND table_name = 'daily_totals'" + ).fetchall() diff --git a/tests/integration/src/sqlbuild/executor/run/maintained/test_maintained_execution.py b/tests/integration/src/sqlbuild/executor/run/maintained/test_maintained_execution.py new file mode 100644 index 0000000000..52dcb36ae5 --- /dev/null +++ b/tests/integration/src/sqlbuild/executor/run/maintained/test_maintained_execution.py @@ -0,0 +1,146 @@ +"""Integration tests for the maintained model lifecycle: create, alter and keep.""" + +from __future__ import annotations + +from typing import Any + +import pytest + +from sqlbuild._native import PlanAction +from sqlbuild.executor.run.main._execute import execute_maintained_entry +from sqlbuild.executor.run.models import ModelExecutionResult +from sqlbuild.executor.scheduling.types import ExecutionStatus +from tests.integration.src.sqlbuild.executor.run.maintained._test_types import ( + MaintainedFailureTestCase, + MaintainedSuccessTestCase, +) +from tests.integration.src.sqlbuild.executor.run.maintained.helpers import ( + MaintainedDuckDbAdapter, + listed_types, + maintained_context, +) + +_BUILT: tuple[str, ...] = ("CREATE TABLE staging.daily_totals AS SELECT 1 AS orders",) + + +@pytest.mark.parametrize( + "test_case", + [ + MaintainedSuccessTestCase( + description="a first run creates on the session warehouse", + action=PlanAction.CREATE_MAINTAINED, + setup_sql=(), + refresh_warehouse=None, + expected_created=(("'1 hour'", "incremental", "SESSION_WH"),), + expected_altered=(), + expected_rows=((3,),), + ), + MaintainedSuccessTestCase( + description="a declared refresh warehouse is used over the session's", + action=PlanAction.CREATE_MAINTAINED, + setup_sql=(), + refresh_warehouse="transform_wh", + expected_created=(("'1 hour'", "incremental", "transform_wh"),), + expected_altered=(), + expected_rows=((3,),), + ), + MaintainedSuccessTestCase( + description="a view the model was before is dropped and replaced", + action=PlanAction.CREATE_MAINTAINED, + setup_sql=("CREATE VIEW staging.daily_totals AS SELECT 1 AS orders",), + refresh_warehouse=None, + expected_created=(("'1 hour'", "incremental", "SESSION_WH"),), + expected_altered=(), + expected_rows=((3,),), + ), + MaintainedSuccessTestCase( + description="drifted settings alter in place and keep the data", + action=PlanAction.ALTER_MAINTAINED, + setup_sql=_BUILT, + refresh_warehouse="transform_wh", + expected_created=(), + expected_altered=(("'1 hour'", "transform_wh"),), + expected_rows=((1,),), + ), + MaintainedSuccessTestCase( + description="a kept model runs no refresh statement", + action=PlanAction.KEEP_MAINTAINED, + setup_sql=_BUILT, + refresh_warehouse=None, + expected_created=(), + expected_altered=(), + expected_rows=((1,),), + ), + ], + ids=lambda case: case.description, +) +def test_given_maintained_entry_when_executing_then_settings_reach_the_warehouse( + test_case: MaintainedSuccessTestCase, connection: Any +) -> None: + adapter: MaintainedDuckDbAdapter = MaintainedDuckDbAdapter(session_warehouse="SESSION_WH") + for sql in test_case.setup_sql: + connection.execute(sql) + + result: ModelExecutionResult = execute_maintained_entry( + context=maintained_context( + adapter=adapter, + connection=connection, + action=test_case.action, + refresh_warehouse=test_case.refresh_warehouse, + ) + ) + + assert result.status == ExecutionStatus.SUCCESS, result.error_message + assert tuple(adapter.created) == test_case.expected_created + assert tuple(adapter.altered) == test_case.expected_altered + assert connection.execute("SELECT orders FROM staging.daily_totals").fetchall() == list( + test_case.expected_rows + ) + assert connection.execute( + "SELECT node_name FROM staging._sqlbuild_fingerprints" + ).fetchall() == [("daily_totals",)] + + +@pytest.mark.parametrize( + "test_case", + [ + MaintainedFailureTestCase( + description="no declared or session warehouse fails before any statement", + setup_sql=(), + session_warehouse=None, + listed_settings=None, + expected_error_code="A717", + expected_listed_types=(), + ), + MaintainedFailureTestCase( + description="a failed create never drops a relation listed as a dynamic table", + setup_sql=("CREATE VIEW staging.daily_totals AS SELECT 1 AS orders",), + session_warehouse="SESSION_WH", + listed_settings=("1 hour", "SESSION_WH"), + expected_error_code="R002", + expected_listed_types=(("VIEW",),), + ), + ], + ids=lambda case: case.description, +) +def test_given_unbuildable_maintained_entry_when_executing_then_fails_keeping_the_relation( + test_case: MaintainedFailureTestCase, connection: Any +) -> None: + adapter: MaintainedDuckDbAdapter = MaintainedDuckDbAdapter( + session_warehouse=test_case.session_warehouse, listed_settings=test_case.listed_settings + ) + for sql in test_case.setup_sql: + connection.execute(sql) + + result: ModelExecutionResult = execute_maintained_entry( + context=maintained_context( + adapter=adapter, + connection=connection, + action=PlanAction.CREATE_MAINTAINED, + refresh_warehouse=None, + ) + ) + + assert result.status == ExecutionStatus.FAILED + assert result.error_code == test_case.expected_error_code + assert listed_types(connection=connection) == list(test_case.expected_listed_types) diff --git a/tests/unit/src/sqlbuild/adapter/contract/classes/strict_adapter/test_strict_adapter.py b/tests/unit/src/sqlbuild/adapter/contract/classes/strict_adapter/test_strict_adapter.py index 66ba9eeb66..c9826ee25e 100644 --- a/tests/unit/src/sqlbuild/adapter/contract/classes/strict_adapter/test_strict_adapter.py +++ b/tests/unit/src/sqlbuild/adapter/contract/classes/strict_adapter/test_strict_adapter.py @@ -90,8 +90,12 @@ def test_given_first_class_adapters_when_checking_contract_then_no_method_comes_ "create_sql_test_output", "drop_sql_test_output", "maintained_refresh_settings", + "render_alter_maintained", + "render_create_maintained", "render_create_temporary_function", "render_drop_schema", + "render_maintained_operation", + "session_warehouse", } ), expected_output_relation=None, @@ -125,6 +129,16 @@ def test_given_strict_adapter_when_reading_optional_methods_then_defaults_keep_i assert relation is test_case.expected_output_relation assert temporary is test_case.expected_temporary_function assert drop_error.value.code == "A714" + assert adapter.session_warehouse(connection=None) is None + with pytest.raises(AdapterUserError) as maintained_error: + adapter.render_create_maintained( + destination="main.daily_totals", + sql="SELECT 1", + target_lag="DOWNSTREAM", + refresh_mode="incremental", + warehouse="transform_wh", + ) + assert maintained_error.value.code == "A716" @pytest.mark.parametrize( diff --git a/tests/unit/src/sqlbuild/adapters/snowflake/_test_types.py b/tests/unit/src/sqlbuild/adapters/snowflake/_test_types.py index adfe4381a6..e3303fe8b1 100644 --- a/tests/unit/src/sqlbuild/adapters/snowflake/_test_types.py +++ b/tests/unit/src/sqlbuild/adapters/snowflake/_test_types.py @@ -4,7 +4,7 @@ from sqlbuild._native import InferredNullability from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.models import FunctionInfo, RelationUsageReport, SchemaDiffResult -from sqlbuild.adapter.contract.types import TableFreshnessStatus +from sqlbuild.adapter.contract.types import MaintainedOperation, TableFreshnessStatus from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter from sqlbuild.cost.types import CostStatus from tests.unit.src.sqlbuild.adapters.snowflake.inspection.helpers import FakeRelation @@ -460,3 +460,27 @@ class SnowflakeMaintainedRefreshTestCase: description: str rows: tuple[tuple[object, ...], ...] expected_settings: tuple[str, str] | None + + +@dataclass(frozen=True) +class SnowflakeMaintainedCreateTestCase: + description: str + target_lag: str + refresh_mode: str + warehouse: str + expected_sql: str + + +@dataclass(frozen=True) +class SnowflakeMaintainedAlterTestCase: + description: str + target_lag: str + warehouse: str | None + expected_sql: tuple[str, ...] + + +@dataclass(frozen=True) +class SnowflakeMaintainedOperationTestCase: + description: str + operation: MaintainedOperation + expected_sql: str diff --git a/tests/unit/src/sqlbuild/adapters/snowflake/test_maintained_refresh_settings.py b/tests/unit/src/sqlbuild/adapters/snowflake/test_maintained_refresh_settings.py index 450e68f0ed..f9f82636bc 100644 --- a/tests/unit/src/sqlbuild/adapters/snowflake/test_maintained_refresh_settings.py +++ b/tests/unit/src/sqlbuild/adapters/snowflake/test_maintained_refresh_settings.py @@ -1,8 +1,12 @@ import pytest +from sqlbuild.adapter.contract.types import MaintainedOperation from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter from tests.unit.src.sqlbuild.adapters.helpers import MetadataConnection, execute_through_cursor from tests.unit.src.sqlbuild.adapters.snowflake._test_types import ( + SnowflakeMaintainedAlterTestCase, + SnowflakeMaintainedCreateTestCase, + SnowflakeMaintainedOperationTestCase, SnowflakeMaintainedRefreshTestCase, ) @@ -48,3 +52,122 @@ def test_given_dynamic_table_listing_when_reading_refresh_settings_then_reads_on assert connection.executed[0].startswith( 'SHOW DYNAMIC TABLES LIKE \'DAILY_TOTALS\' IN SCHEMA "ANALYTICS"."MARTS"' ) + + +@pytest.mark.parametrize( + "test_case", + [ + SnowflakeMaintainedCreateTestCase( + description="a lag, the default incremental mode and an unquoted warehouse", + target_lag="'1 hour'", + refresh_mode="incremental", + warehouse="transform_wh", + expected_sql=( + "CREATE OR REPLACE DYNAMIC TABLE ANALYTICS.MARTS.DAILY_TOTALS " + "TARGET_LAG = '1 hour' WAREHOUSE = transform_wh REFRESH_MODE = INCREMENTAL " + "INITIALIZE = ON_CREATE AS SELECT 1 AS orders" + ), + ), + SnowflakeMaintainedCreateTestCase( + description="refreshed for its readers, in adaptive mode", + target_lag="DOWNSTREAM", + refresh_mode="adaptive", + warehouse="TRANSFORM_WH", + expected_sql=( + "CREATE OR REPLACE DYNAMIC TABLE ANALYTICS.MARTS.DAILY_TOTALS " + "TARGET_LAG = DOWNSTREAM WAREHOUSE = TRANSFORM_WH REFRESH_MODE = ADAPTIVE " + "INITIALIZE = ON_CREATE AS SELECT 1 AS orders" + ), + ), + SnowflakeMaintainedCreateTestCase( + description="a warehouse name that needs quoting", + target_lag="'5 minutes'", + refresh_mode="full", + warehouse="transform-wh", + expected_sql=( + "CREATE OR REPLACE DYNAMIC TABLE ANALYTICS.MARTS.DAILY_TOTALS " + "TARGET_LAG = '5 minutes' WAREHOUSE = \"transform-wh\" REFRESH_MODE = FULL " + "INITIALIZE = ON_CREATE AS SELECT 1 AS orders" + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_maintained_settings_when_rendering_create_then_mode_and_initialize_are_explicit( + test_case: SnowflakeMaintainedCreateTestCase, +) -> None: + statements: tuple[str, ...] = SnowflakeAdapter().render_create_maintained( + destination="ANALYTICS.MARTS.DAILY_TOTALS", + sql="SELECT 1 AS orders", + target_lag=test_case.target_lag, + refresh_mode=test_case.refresh_mode, + warehouse=test_case.warehouse, + ) + + assert statements == (test_case.expected_sql,) + + +@pytest.mark.parametrize( + "test_case", + [ + SnowflakeMaintainedAlterTestCase( + description="an unset refresh warehouse alters only the lag", + target_lag="'2 hours'", + warehouse=None, + expected_sql=( + "ALTER DYNAMIC TABLE ANALYTICS.MARTS.DAILY_TOTALS SET TARGET_LAG = '2 hours'", + ), + ), + SnowflakeMaintainedAlterTestCase( + description="a declared refresh warehouse is set too", + target_lag="DOWNSTREAM", + warehouse="reporting_wh", + expected_sql=( + "ALTER DYNAMIC TABLE ANALYTICS.MARTS.DAILY_TOTALS SET TARGET_LAG = DOWNSTREAM", + "ALTER DYNAMIC TABLE ANALYTICS.MARTS.DAILY_TOTALS SET WAREHOUSE = reporting_wh", + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_drifted_settings_when_rendering_alter_then_settings_change_in_place( + test_case: SnowflakeMaintainedAlterTestCase, +) -> None: + statements: tuple[str, ...] = SnowflakeAdapter().render_alter_maintained( + destination="ANALYTICS.MARTS.DAILY_TOTALS", + target_lag=test_case.target_lag, + warehouse=test_case.warehouse, + ) + + assert statements == test_case.expected_sql + + +@pytest.mark.parametrize( + "test_case", + [ + SnowflakeMaintainedOperationTestCase( + description="suspend", + operation=MaintainedOperation.SUSPEND, + expected_sql="ALTER DYNAMIC TABLE ANALYTICS.MARTS.DAILY_TOTALS SUSPEND", + ), + SnowflakeMaintainedOperationTestCase( + description="resume", + operation=MaintainedOperation.RESUME, + expected_sql="ALTER DYNAMIC TABLE ANALYTICS.MARTS.DAILY_TOTALS RESUME", + ), + SnowflakeMaintainedOperationTestCase( + description="refresh", + operation=MaintainedOperation.REFRESH, + expected_sql="ALTER DYNAMIC TABLE ANALYTICS.MARTS.DAILY_TOTALS REFRESH", + ), + ], + ids=lambda case: case.description, +) +def test_given_operation_when_rendering_then_one_alter_dynamic_table_statement( + test_case: SnowflakeMaintainedOperationTestCase, +) -> None: + statements: tuple[str, ...] = SnowflakeAdapter().render_maintained_operation( + destination="ANALYTICS.MARTS.DAILY_TOTALS", operation=test_case.operation + ) + + assert statements == (test_case.expected_sql,) From 4cedaf087736540e3b336763586aa1b4050cb5c0 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 19 Nov 2026 00:17:39 +0000 Subject: [PATCH 07/17] feat(build): track changes on tables a maintained model reads Work in progress. A staged build swaps a freshly created table into place; on Snowflake, a swapped-in table built without change tracking makes the next refresh of every dynamic table reading it fail with "Change tracking is not enabled". The planner now marks every model a maintained model reads, selected or not, with change_tracking, and the table, type-enforcement and microbatch full-refresh paths create those tables with CHANGE_TRACKING = TRUE. A live Snowflake run confirmed that a stage built this way refreshes the reader after the swap. Each reader's materialization is read once per plan, only for models downstream of planned entries. Type enforcement now takes the table settings as one mapping. Not covered: a snapshot target that is rebuilt in full. Validated: Snowflake CTAS rendering, the planner integration test (tracked, untracked, and a reader outside the selection), the table executor on DuckDB (staged, type enforcement, untracked control, and a fails-without-fix check), and the executor, planner and Snowflake adapter suites. --- .../_helpers/planner_records/constants.rs | 1 + .../_helpers/outputs/maintained.rs | 70 ++++++++++++--- src/sqlbuild/_native.pyi | 2 + .../snowflake/classes/snowflake_adapter.py | 10 ++- .../run/_helpers/execution/table_config.py | 3 +- .../_helpers/materializations/incremental.py | 2 +- .../_helpers/materializations/microbatch.py | 3 +- .../_helpers/validation/type_enforcement.py | 10 +-- src/sqlbuild/executor/run/main/_execute.py | 7 +- .../src/sqlbuild/adapters/sqlite/helpers.py | 2 +- .../compiler/planner/main/_test_types.py | 7 ++ .../sqlbuild/compiler/planner/main/helpers.py | 24 ++++++ .../main/test_maintained_refresh_plan.py | 33 +++++++ .../executor/run/maintained/_test_types.py | 10 +++ .../executor/run/maintained/helpers.py | 27 ++++++ .../run/maintained/test_change_tracking.py | 85 +++++++++++++++++++ .../adapters/snowflake/_test_types.py | 7 ++ .../test_maintained_refresh_settings.py | 41 +++++++++ 18 files changed, 316 insertions(+), 28 deletions(-) create mode 100644 tests/integration/src/sqlbuild/executor/run/maintained/test_change_tracking.py diff --git a/crates/sqlbuild-python-base/src/records/_helpers/planner_records/constants.rs b/crates/sqlbuild-python-base/src/records/_helpers/planner_records/constants.rs index 55ba072f4c..7a79955e8d 100644 --- a/crates/sqlbuild-python-base/src/records/_helpers/planner_records/constants.rs +++ b/crates/sqlbuild-python-base/src/records/_helpers/planner_records/constants.rs @@ -670,6 +670,7 @@ const R_MODEL_PLAN_ENTRY: RecordSpec = RecordSpec { FieldSpec::new("target_lag", FieldDefault::None), FieldSpec::new("refresh_mode", FieldDefault::None), FieldSpec::new("refresh_warehouse", FieldDefault::None), + FieldSpec::new("change_tracking", FieldDefault::False), ], }; const R_CLONE_SOURCE_PLAN_ENTRY: RecordSpec = RecordSpec { diff --git a/crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/maintained.rs b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/maintained.rs index 988ccf7267..e9e3159c0b 100644 --- a/crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/maintained.rs +++ b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/maintained.rs @@ -2,17 +2,20 @@ use crate::planner_phases::main::inputs::runtime_reads::runtime_reads; use crate::planner_phases::models::ModelEntryResults; +use crate::planner_strategy::main::strategy::materialization_of::materialization_of; use crate::planner_strategy::main::values::text::text; use crate::planner_warehouse::models::{AdapterRead, PlannerReads, Read}; use pyo3::IntoPyObject; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; -use pyo3::types::PyString; +use pyo3::types::{PyBool, PyString}; +use sqlbuild_analysis::assembly::project::types::ObjectKey; use sqlbuild_analysis::maintained_refresh::main::maintained_settings_drift::maintained_settings_drift; use sqlbuild_analysis::maintained_refresh::main::reinitialised_readers::reinitialised_readers; use sqlbuild_analysis::maintained_refresh::main::render_target_lag::render_target_lag; use sqlbuild_analysis::maintained_refresh::models::{ LiveDynamicTable, MaintainedRead, MaintainedSettings, RefreshLag, }; +use sqlbuild_analysis::planner_scope::constants::MODEL; use sqlbuild_model_config::model_validation::main::fixed_duration_seconds::fixed_duration_seconds; use sqlbuild_python_base::records::main::construct::enum_member::enum_member; use sqlbuild_python_base::records::main::fields::field::field as native_field; @@ -33,36 +36,49 @@ const DEFAULT_REFRESH_MODE: &str = "incremental"; /// Actions that recreate the relation, which reinitialises every dynamic table reading it. const REPLACING_ACTIONS: [&str; 3] = ["create_table", "create_view", "create_maintained"]; -/// Alter drifted maintained models, set their `target_lag`, and mark kept readers of a replaced upstream. +/// Plan maintained refresh settings and reinitialised readers, and track changes on what they read. pub(crate) fn plan_maintained_refresh<'py>( runtime: &Bound<'py, PyAny>, execution_scope: &Bound<'py, PyAny>, results: ModelEntryResults<'py>, ) -> PyResult> { - let mut maintained: Vec = Vec::with_capacity(results.entries.len()); - for entry in &results.entries { - maintained.push(text(&native_field(entry, "materialization_type")?)? == MAINTAINED); - } - if !maintained.contains(&true) { - return Ok(results); - } let py: Python<'py> = runtime.py(); let scope = scope_data(execution_scope)?; - let mut upstream: HashMap<&(String, String), Vec> = HashMap::new(); + let mut upstream: HashMap<&ObjectKey, Vec> = HashMap::new(); for (key, deps) in &scope.upstream { upstream.insert(key, deps.iter().map(|(_, name)| name.clone()).collect()); } + let downstream: HashMap<&ObjectKey, &Vec> = scope + .downstream + .iter() + .map(|(key, readers)| (key, readers)) + .collect(); let models = ModelLookup::Native(scope.clone()); let reads = runtime_reads(runtime)?; + let mut maintained_models: HashMap = HashMap::new(); let mut entries: Vec> = Vec::with_capacity(results.entries.len()); let mut rows: Vec = Vec::with_capacity(results.entries.len()); - for (entry, is_maintained) in results.entries.into_iter().zip(maintained) { + for entry in results.entries { + let key: ObjectKey = native_key(&native_field(&entry, "key")?)?; + let is_maintained: bool = + text(&native_field(&entry, "materialization_type")?)? == MAINTAINED; let entry = if is_maintained { with_refresh_settings(entry, &models, &reads)? + } else if read_by_maintained( + py, + downstream.get(&key).copied(), + (&models, &mut maintained_models), + )? { + replaced( + &entry, + &[( + "change_tracking", + PyBool::new(py, true).to_owned().into_any(), + )], + )? } else { entry }; - let key = native_key(&native_field(&entry, "key")?)?; let action: String = text(&native_field(&entry, "action")?)?; rows.push(MaintainedRead { name: key.1.clone(), @@ -87,6 +103,36 @@ pub(crate) fn plan_maintained_refresh<'py>( }) } +/// Whether any model reading the entry is maintained, checking each reader once per plan. +fn read_by_maintained( + py: Python<'_>, + readers: Option<&Vec>, + (models, maintained_models): (&ModelLookup, &mut HashMap), +) -> PyResult { + let Some(readers) = readers else { + return Ok(false); + }; + for (kind, name) in readers { + if kind != MODEL { + continue; + } + let maintained: bool = match maintained_models.get(name) { + Some(maintained) => *maintained, + None => { + let model = models.get(py, name)?; + let maintained: bool = + !model.is_none() && text(&materialization_of(&model)?)? == MAINTAINED; + maintained_models.insert(name.clone(), maintained); + maintained + } + }; + if maintained { + return Ok(true); + } + } + Ok(false) +} + /// The entry with its `target_lag`; a kept one alters on drift, or creates when no dynamic table holds its name. fn with_refresh_settings<'py>( entry: Bound<'py, PyAny>, diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 5b3f4a149a..15f6ae2876 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -14123,6 +14123,7 @@ class ModelPlanEntry(NativeRecord): target_lag: str | None refresh_mode: str | None refresh_warehouse: str | None + change_tracking: bool def __init__( self, key: CompiledObjectKey, @@ -14216,6 +14217,7 @@ class ModelPlanEntry(NativeRecord): target_lag: str | None = ..., refresh_mode: str | None = ..., refresh_warehouse: str | None = ..., + change_tracking: bool = ..., ) -> None: ... class CloneSourcePlanEntry(NativeRecord): diff --git a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py index bce960cbe6..ef894eddb8 100644 --- a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py +++ b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py @@ -1161,10 +1161,11 @@ def render_create_table_as(self, *, destination: str, sql: str) -> tuple[str, .. ) def _render_create_table_as_type( - self, *, destination: str, sql: str, table_type: str + self, *, destination: str, sql: str, table_type: str, change_tracking: bool = False ) -> tuple[str, ...]: table_kind: str = "TABLE" if table_type == TableType.PERMANENT else "TRANSIENT TABLE" - return (f"CREATE OR REPLACE {table_kind} {destination} AS {sql}",) + tracking: str = " CHANGE_TRACKING = TRUE" if change_tracking else "" + return (f"CREATE OR REPLACE {table_kind} {destination}{tracking} AS {sql}",) def render_create_view_as(self, *, destination: str, sql: str) -> tuple[str, ...]: return (f"CREATE OR REPLACE VIEW {destination} AS {sql}",) @@ -1332,7 +1333,10 @@ def create_table_as( ) -> None: table_type: str = str((config or {}).get("table_type", "transient")) statements: tuple[str, ...] = self._render_create_table_as_type( - destination=destination, sql=sql, table_type=table_type + destination=destination, + sql=sql, + table_type=table_type, + change_tracking=bool((config or {}).get("change_tracking", False)), ) statement_recorder.record_many(statements) stmt: str diff --git a/src/sqlbuild/executor/run/_helpers/execution/table_config.py b/src/sqlbuild/executor/run/_helpers/execution/table_config.py index 7f71b3f645..44cdf5e270 100644 --- a/src/sqlbuild/executor/run/_helpers/execution/table_config.py +++ b/src/sqlbuild/executor/run/_helpers/execution/table_config.py @@ -17,10 +17,11 @@ def model_output_config(entry: ModelPlanEntry) -> dict[str, object]: def model_table_config(entry: ModelPlanEntry) -> dict[str, object]: - """The model's table type, physical layout and expected output columns.""" + """The model's table type, physical layout, change tracking and expected output columns.""" return { "table_type": entry.table_type, + "change_tracking": entry.change_tracking, "physical_storage": entry.physical_storage, **model_output_config(entry), } diff --git a/src/sqlbuild/executor/run/_helpers/materializations/incremental.py b/src/sqlbuild/executor/run/_helpers/materializations/incremental.py index 01b9fa6f56..700add7c56 100644 --- a/src/sqlbuild/executor/run/_helpers/materializations/incremental.py +++ b/src/sqlbuild/executor/run/_helpers/materializations/incremental.py @@ -689,7 +689,7 @@ def _prepare_incremental_delta_types( staging_schema=inputs.target_schema, staging_table=inputs.delta_table, declared_columns=tuple(desired_columns), - table_type=TableType.TRANSIENT, + table_config={"table_type": TableType.TRANSIENT, "physical_storage": {}}, statement_recorder=statement_recorder, ) diff --git a/src/sqlbuild/executor/run/_helpers/materializations/microbatch.py b/src/sqlbuild/executor/run/_helpers/materializations/microbatch.py index d6f3a5e9ce..c0784aedea 100644 --- a/src/sqlbuild/executor/run/_helpers/materializations/microbatch.py +++ b/src/sqlbuild/executor/run/_helpers/materializations/microbatch.py @@ -2957,7 +2957,7 @@ def _enforce_microbatch_types( staging_schema=targets.target_schema, staging_table=targets.delta_table, declared_columns=declared_columns, - table_type=TableType.TRANSIENT, + table_config={"table_type": TableType.TRANSIENT, "physical_storage": {}}, statement_recorder=state.statement_recorder, ) except Exception as exc: @@ -3057,6 +3057,7 @@ def _apply_microbatch_dml( config={ "table_type": context.entry.table_type, "physical_storage": context.entry.physical_storage, + "change_tracking": context.entry.change_tracking, }, statement_recorder=state.statement_recorder, ) diff --git a/src/sqlbuild/executor/run/_helpers/validation/type_enforcement.py b/src/sqlbuild/executor/run/_helpers/validation/type_enforcement.py index 09649f37a5..9da1bd78c0 100644 --- a/src/sqlbuild/executor/run/_helpers/validation/type_enforcement.py +++ b/src/sqlbuild/executor/run/_helpers/validation/type_enforcement.py @@ -2,6 +2,7 @@ from __future__ import annotations +from collections.abc import Mapping from typing import Any from sqlbuild._native import fit_auxiliary_relation_name @@ -26,9 +27,8 @@ def enforce_types_staged( staging_schema: str | None, staging_table: str, declared_columns: tuple[ColumnInfo, ...], - table_type: str, + table_config: Mapping[str, object], statement_recorder: StatementRecorder, - physical_storage: dict[str, object] | None = None, ) -> None: """Inspect staging columns and rebuild with casts for declared types.""" @@ -76,11 +76,7 @@ def enforce_types_staged( connection=connection, destination=enforced_qualified, sql=f"SELECT {projection_sql} FROM {staging_qualified}", - config={ - "table_type": table_type, - "column_types": declared_map, - "physical_storage": physical_storage or {}, - }, + config={**table_config, "column_types": declared_map}, statement_recorder=statement_recorder, ) adapter.drop_table( diff --git a/src/sqlbuild/executor/run/main/_execute.py b/src/sqlbuild/executor/run/main/_execute.py index 1d4cf77f2d..9a8359f13d 100644 --- a/src/sqlbuild/executor/run/main/_execute.py +++ b/src/sqlbuild/executor/run/main/_execute.py @@ -246,8 +246,11 @@ def _staged_lifecycle( staging_schema=targets.target_schema, staging_table=targets.staging_table, declared_columns=declared_columns, - table_type=entry.table_type, - physical_storage=entry.physical_storage, + table_config={ + "table_type": entry.table_type, + "physical_storage": entry.physical_storage, + "change_tracking": entry.change_tracking, + }, statement_recorder=statement_recorder, ) except Exception as exc: diff --git a/tests/integration/src/sqlbuild/adapters/sqlite/helpers.py b/tests/integration/src/sqlbuild/adapters/sqlite/helpers.py index 0584f027d5..2188e2a42c 100644 --- a/tests/integration/src/sqlbuild/adapters/sqlite/helpers.py +++ b/tests/integration/src/sqlbuild/adapters/sqlite/helpers.py @@ -54,7 +54,7 @@ def stage_contract_relation( ColumnInfo("payment_id", "INTEGER"), ColumnInfo("amount", "DECIMAL(10,2)"), ), - table_type="table", + table_config={"table_type": "table", "physical_storage": {}}, statement_recorder=recorder(), ) diff --git a/tests/integration/src/sqlbuild/compiler/planner/main/_test_types.py b/tests/integration/src/sqlbuild/compiler/planner/main/_test_types.py index bb4901a9ae..b4a9864101 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/main/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/planner/main/_test_types.py @@ -217,3 +217,10 @@ class MaintainedRefreshPlanTestCase: expected_reason: PlanReason expected_reinitialised_by: str | None expected_read_names: tuple[str, ...] + + +@dataclass(frozen=True) +class ChangeTrackingPlanTestCase: + description: str + select: tuple[str, ...] + expected_change_tracking: dict[str, bool] diff --git a/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py b/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py index 3c1661bbf8..42b9eeb2fb 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py @@ -870,3 +870,27 @@ def maintained_project_case(upstream_query: str) -> BuildExecutionPlanTestCase: expected_action={}, expected_reason={}, ) + + +def change_tracking_project_case() -> BuildExecutionPlanTestCase: + """Orders and customers tables; daily totals is maintained from orders alone.""" + + return BuildExecutionPlanTestCase( + description="tables that a maintained model reads, and one it does not", + setup_sql=(), + model_locations={"orders": "staging", "customers": "staging", "daily_totals": "staging"}, + model_configs={ + "orders": {"materialized": "table"}, + "customers": {"materialized": "table"}, + "daily_totals": {"materialized": "maintained", "refresh": {"lag": "1h"}}, + }, + model_queries={ + "orders": "SELECT 1 AS order_id", + "customers": "SELECT 1 AS customer_id", + "daily_totals": "SELECT count(*) AS orders FROM staging.orders", + }, + model_deps={"daily_totals": ("orders",)}, + full_refresh=False, + expected_action={}, + expected_reason={}, + ) diff --git a/tests/integration/src/sqlbuild/compiler/planner/main/test_maintained_refresh_plan.py b/tests/integration/src/sqlbuild/compiler/planner/main/test_maintained_refresh_plan.py index c037cd3f2a..331eef06fc 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/main/test_maintained_refresh_plan.py +++ b/tests/integration/src/sqlbuild/compiler/planner/main/test_maintained_refresh_plan.py @@ -7,8 +7,10 @@ import pytest from sqlbuild._native import ModelPlanEntry, PlanAction, PlanOutput, PlanReason +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.executor.run._helpers.reuse.fingerprinting import try_write_fingerprint from tests.integration.src.sqlbuild.compiler.planner.main._test_types import ( + ChangeTrackingPlanTestCase, MaintainedRefreshPlanTestCase, ) from tests.integration.src.sqlbuild.compiler.planner.main.helpers import ( @@ -16,6 +18,7 @@ ListingDuckDbAdapter, build_execution_plan_from_kwargs, build_project_from_test_case, + change_tracking_project_case, maintained_project_case, ) @@ -109,3 +112,33 @@ def test_given_built_maintained_models_when_replanning_then_live_settings_and_re assert entries["daily_totals"].refresh_warehouse == "transform_wh" assert tuple(sorted(adapter.read_names)) == test_case.expected_read_names adapter.close(connection) + + +@pytest.mark.parametrize( + "test_case", + [ + ChangeTrackingPlanTestCase( + description="a table a maintained model reads is built with change tracking", + select=(), + expected_change_tracking={"orders": True, "customers": False, "daily_totals": False}, + ), + ChangeTrackingPlanTestCase( + description="the reader need not be selected for its upstream to be tracked", + select=("orders", "customers"), + expected_change_tracking={"orders": True, "customers": False}, + ), + ], + ids=lambda case: case.description, +) +def test_given_table_read_by_maintained_model_when_planning_then_it_is_built_with_change_tracking( + test_case: ChangeTrackingPlanTestCase, adapter: DuckDbAdapter, connection: Any +) -> None: + plan: PlanOutput = build_execution_plan_from_kwargs( + project=build_project_from_test_case(change_tracking_project_case()), + adapter=adapter, + connection=connection, + select=test_case.select, + ) + + tracked: dict[str, bool] = {entry.name: entry.change_tracking for entry in plan.model_entries} + assert tracked == test_case.expected_change_tracking diff --git a/tests/integration/src/sqlbuild/executor/run/maintained/_test_types.py b/tests/integration/src/sqlbuild/executor/run/maintained/_test_types.py index 617c6d926a..9f2d57e13a 100644 --- a/tests/integration/src/sqlbuild/executor/run/maintained/_test_types.py +++ b/tests/integration/src/sqlbuild/executor/run/maintained/_test_types.py @@ -3,6 +3,7 @@ from dataclasses import dataclass from sqlbuild._native import PlanAction +from sqlbuild.adapter.contract.types import TablePromotionMode @dataclass(frozen=True) @@ -24,3 +25,12 @@ class MaintainedFailureTestCase: listed_settings: tuple[str, str] | None expected_error_code: str expected_listed_types: tuple[tuple[str], ...] + + +@dataclass(frozen=True) +class ChangeTrackingTableTestCase: + description: str + change_tracking: bool + promotion_mode: TablePromotionMode + declared_columns: tuple[tuple[str, str], ...] + expected_tracking: tuple[bool, ...] diff --git a/tests/integration/src/sqlbuild/executor/run/maintained/helpers.py b/tests/integration/src/sqlbuild/executor/run/maintained/helpers.py index d9e297440d..517c152a49 100644 --- a/tests/integration/src/sqlbuild/executor/run/maintained/helpers.py +++ b/tests/integration/src/sqlbuild/executor/run/maintained/helpers.py @@ -14,6 +14,7 @@ PlanAction, PlanReason, ) +from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.executor.run.models import ModelMaterializationContext @@ -106,3 +107,29 @@ def listed_types(*, connection: Any) -> list[tuple[Any, ...]]: "SELECT table_type FROM information_schema.tables " "WHERE table_schema = 'staging' AND table_name = 'daily_totals'" ).fetchall() + + +class TrackingDuckDbAdapter(DuckDbAdapter): + """Records whether each table it creates was asked to track changes.""" + + def __init__(self) -> None: + super().__init__() + self.tracking: list[bool] = [] + + def create_table_as( + self, + *, + connection: Any, + destination: str, + sql: str, + config: dict[str, Any] | None = None, + statement_recorder: StatementRecorder, + ) -> None: + self.tracking.append(bool((config or {}).get("change_tracking"))) + super().create_table_as( + connection=connection, + destination=destination, + sql=sql, + config=config, + statement_recorder=statement_recorder, + ) diff --git a/tests/integration/src/sqlbuild/executor/run/maintained/test_change_tracking.py b/tests/integration/src/sqlbuild/executor/run/maintained/test_change_tracking.py new file mode 100644 index 0000000000..167ba289a4 --- /dev/null +++ b/tests/integration/src/sqlbuild/executor/run/maintained/test_change_tracking.py @@ -0,0 +1,85 @@ +"""A table a maintained model reads is created with change tracking on every build path.""" + +from __future__ import annotations + +from dataclasses import replace +from typing import Any + +import pytest + +from sqlbuild._native import ModelPlanEntry +from sqlbuild.adapter.contract.types import TablePromotionMode +from sqlbuild.executor.run.main._execute import execute_table_entry +from sqlbuild.executor.run.models import ModelExecutionResult, ModelMaterializationContext +from sqlbuild.executor.scheduling.types import ExecutionStatus +from tests.integration.src.sqlbuild.executor.run.helpers import ( + build_declared_columns, + build_table_plan_entry, +) +from tests.integration.src.sqlbuild.executor.run.maintained._test_types import ( + ChangeTrackingTableTestCase, +) +from tests.integration.src.sqlbuild.executor.run.maintained.helpers import TrackingDuckDbAdapter + + +@pytest.mark.parametrize( + "test_case", + [ + ChangeTrackingTableTestCase( + description="a staged build tracks changes on the stage it swaps in", + change_tracking=True, + promotion_mode=TablePromotionMode.STAGED, + declared_columns=(), + expected_tracking=(True,), + ), + ChangeTrackingTableTestCase( + description="type enforcement rebuilds the stage with change tracking too", + change_tracking=True, + promotion_mode=TablePromotionMode.STAGED, + declared_columns=(("order_id", "BIGINT"),), + expected_tracking=(True, True), + ), + ChangeTrackingTableTestCase( + description="a table no maintained model reads is created as before", + change_tracking=False, + promotion_mode=TablePromotionMode.STAGED, + declared_columns=(("order_id", "BIGINT"),), + expected_tracking=(False, False), + ), + ], + ids=lambda case: case.description, +) +def test_given_table_read_by_maintained_model_when_building_then_every_create_tracks_changes( + test_case: ChangeTrackingTableTestCase, connection: Any +) -> None: + adapter: TrackingDuckDbAdapter = TrackingDuckDbAdapter() + entry: ModelPlanEntry = replace( + build_table_plan_entry( + name="orders", + sql="SELECT 1::INTEGER AS order_id", + target_schema="staging", + target_name="orders", + type_enforcement=bool(test_case.declared_columns), + ), + change_tracking=test_case.change_tracking, + ) + + result: ModelExecutionResult = execute_table_entry( + context=ModelMaterializationContext( + entry=entry, + adapter=adapter, + connection=connection, + model_locations={}, + seed_locations={}, + source_map={}, + python_source_read_map={}, + model_audits=(), + run_id="tracking_run", + query_change_tracking=False, + ), + declared_columns=build_declared_columns(test_case.declared_columns), + promotion_mode=test_case.promotion_mode, + ) + + assert result.status == ExecutionStatus.SUCCESS, result.error_message + assert tuple(adapter.tracking) == test_case.expected_tracking diff --git a/tests/unit/src/sqlbuild/adapters/snowflake/_test_types.py b/tests/unit/src/sqlbuild/adapters/snowflake/_test_types.py index e3303fe8b1..03d54e874b 100644 --- a/tests/unit/src/sqlbuild/adapters/snowflake/_test_types.py +++ b/tests/unit/src/sqlbuild/adapters/snowflake/_test_types.py @@ -484,3 +484,10 @@ class SnowflakeMaintainedOperationTestCase: description: str operation: MaintainedOperation expected_sql: str + + +@dataclass(frozen=True) +class SnowflakeChangeTrackingTestCase: + description: str + config: dict[str, object] + expected_sql: str diff --git a/tests/unit/src/sqlbuild/adapters/snowflake/test_maintained_refresh_settings.py b/tests/unit/src/sqlbuild/adapters/snowflake/test_maintained_refresh_settings.py index f9f82636bc..9d8721ef6e 100644 --- a/tests/unit/src/sqlbuild/adapters/snowflake/test_maintained_refresh_settings.py +++ b/tests/unit/src/sqlbuild/adapters/snowflake/test_maintained_refresh_settings.py @@ -1,9 +1,11 @@ import pytest +from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.types import MaintainedOperation from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter from tests.unit.src.sqlbuild.adapters.helpers import MetadataConnection, execute_through_cursor from tests.unit.src.sqlbuild.adapters.snowflake._test_types import ( + SnowflakeChangeTrackingTestCase, SnowflakeMaintainedAlterTestCase, SnowflakeMaintainedCreateTestCase, SnowflakeMaintainedOperationTestCase, @@ -171,3 +173,42 @@ def test_given_operation_when_rendering_then_one_alter_dynamic_table_statement( ) assert statements == (test_case.expected_sql,) + + +@pytest.mark.parametrize( + "test_case", + [ + SnowflakeChangeTrackingTestCase( + description="a table a maintained model reads tracks changes from creation", + config={"table_type": "transient", "change_tracking": True}, + expected_sql=( + "CREATE OR REPLACE TRANSIENT TABLE ANALYTICS.STAGING.ORDERS " + "CHANGE_TRACKING = TRUE AS SELECT 1 AS order_id" + ), + ), + SnowflakeChangeTrackingTestCase( + description="a permanent table read by none keeps the plain form", + config={"table_type": "permanent", "change_tracking": False}, + expected_sql=( + "CREATE OR REPLACE TABLE ANALYTICS.STAGING.ORDERS AS SELECT 1 AS order_id" + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_table_config_when_creating_table_then_change_tracking_follows_it( + test_case: SnowflakeChangeTrackingTestCase, monkeypatch: pytest.MonkeyPatch +) -> None: + adapter: SnowflakeAdapter = SnowflakeAdapter() + connection: MetadataConnection = MetadataConnection(()) + monkeypatch.setattr(adapter, "execute", execute_through_cursor) + + adapter.create_table_as( + connection=connection, + destination="ANALYTICS.STAGING.ORDERS", + sql="SELECT 1 AS order_id", + config=test_case.config, + statement_recorder=StatementRecorder(), + ) + + assert [sql.removesuffix(" ()") for sql in connection.executed] == [test_case.expected_sql] From c1b5a9e1d198a8261a3e4ae3418c8b4c7ef99fcc Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 19 Nov 2026 01:09:54 +0000 Subject: [PATCH 08/17] feat(build): pause maintained models during a build and record the hold Work in progress. A build now pauses the maintained models it touches before its first write and resumes them in a finally step: the models it creates, and those reading anything it writes, upstream first. Only models the build found active are paused and resumed; one a person had suspended stays suspended, even when the build recreates it. Each pause and resume is appended to _sqlbuild_maintained_pauses in the target schema with a deterministic event ID, and a heartbeat is written every 60 s on its own connection, timed by the warehouse clock. A build that finds a hold whose heartbeat has been silent past [maintained] heartbeat_grace takes it over and resumes it; a live hold stops the build before any change (A718) and names the run holding it. sqb build --ignore-maintained-grace takes over at once, and only when [maintained] allow_ignore_grace = true (otherwise A719). The flag is not offered to ad-hoc server runs. A kept or altered maintained model is refreshed by hand when it sets refresh_on_build, when a model built later reads it, or on a pause_maintained target, where models stay suspended and no hold is recorded. Plan entries now list each model's maintained readers, which also decide change tracking, in place of the change_tracking flag. Not covered: the location's warehouse sync does not resume expired holds; the next build does. Validated: lease integration tests on DuckDB with a recording adapter (first build, unchanged and rebuilt upstreams, a person's suspend kept across a recreate, refresh for a later reader and refresh_on_build, pause_maintained, expiry takeover, the flag, a live-hold refusal and the heartbeat), the planner and Snowflake adapter tests, and the executor, planner, adapter, CLI and janitor suites. --- .../src/janitor_planning/constants.rs | 2 +- .../sqlbuild-cli/src/argv/command_table.json | 11 + crates/sqlbuild-cli/src/argv/constants.rs | 3 +- crates/sqlbuild-cli/src/dispatch/constants.rs | 3 +- crates/sqlbuild-core/src/constants.rs | 5 +- .../tests/api/test_warehouse_metadata.rs | 7 +- .../_helpers/planner_records/constants.rs | 3 +- .../src/records/constants.rs | 1 + .../main/project/configured_maintained.rs | 50 ++ .../src/discovery/main/project/mod.rs | 1 + .../_helpers/outputs/maintained.rs | 43 +- .../_helpers/build_command/execution.rs | 123 +++-- .../_helpers/exported_constants/constants.rs | 9 +- src/sqlbuild/_native.pyi | 9 +- .../contract/classes/strict_adapter.py | 8 + .../snowflake/classes/snowflake_adapter.py | 20 +- .../executor/build/classes/build_scheduler.py | 1 + src/sqlbuild/executor/build/main/_execute.py | 22 +- src/sqlbuild/executor/build/models.py | 6 + .../executor/maintained_pauses/__init__.py | 1 + .../maintained_pauses/_helpers/__init__.py | 1 + .../maintained_pauses/_helpers/scope.py | 92 ++++ .../maintained_pauses/_helpers/sql.py | 156 +++++++ .../maintained_pauses/classes/__init__.py | 1 + .../classes/maintained_heartbeat.py | 62 +++ .../classes/maintained_hold.py | 353 ++++++++++++++ .../executor/maintained_pauses/constants.py | 49 ++ .../maintained_pauses/main/__init__.py | 1 + .../main/hold_maintained_models.py | 71 +++ .../executor/maintained_pauses/models.py | 28 ++ .../executor/maintained_pauses/types.py | 15 + .../run/_helpers/execution/table_config.py | 2 +- .../_helpers/materializations/maintained.py | 20 +- .../_helpers/materializations/microbatch.py | 2 +- src/sqlbuild/executor/run/main/_execute.py | 2 +- src/sqlbuild/executor/run/models.py | 2 + .../compiler/planner/main/_test_types.py | 3 +- .../sqlbuild/compiler/planner/main/helpers.py | 6 +- .../main/test_maintained_refresh_plan.py | 22 +- .../executor/maintained_pauses/__init__.py | 1 + .../executor/maintained_pauses/_test_types.py | 52 +++ .../executor/maintained_pauses/conftest.py | 31 ++ .../executor/maintained_pauses/helpers.py | 170 +++++++ .../test_maintained_pause_lease.py | 440 ++++++++++++++++++ .../executor/run/maintained/_test_types.py | 2 +- .../run/maintained/test_change_tracking.py | 8 +- .../adapters/snowflake/_test_types.py | 7 + .../test_maintained_refresh_settings.py | 33 ++ 48 files changed, 1843 insertions(+), 117 deletions(-) create mode 100644 crates/sqlbuild-python-front/src/discovery/main/project/configured_maintained.rs create mode 100644 src/sqlbuild/executor/maintained_pauses/__init__.py create mode 100644 src/sqlbuild/executor/maintained_pauses/_helpers/__init__.py create mode 100644 src/sqlbuild/executor/maintained_pauses/_helpers/scope.py create mode 100644 src/sqlbuild/executor/maintained_pauses/_helpers/sql.py create mode 100644 src/sqlbuild/executor/maintained_pauses/classes/__init__.py create mode 100644 src/sqlbuild/executor/maintained_pauses/classes/maintained_heartbeat.py create mode 100644 src/sqlbuild/executor/maintained_pauses/classes/maintained_hold.py create mode 100644 src/sqlbuild/executor/maintained_pauses/constants.py create mode 100644 src/sqlbuild/executor/maintained_pauses/main/__init__.py create mode 100644 src/sqlbuild/executor/maintained_pauses/main/hold_maintained_models.py create mode 100644 src/sqlbuild/executor/maintained_pauses/models.py create mode 100644 src/sqlbuild/executor/maintained_pauses/types.py create mode 100644 tests/integration/src/sqlbuild/executor/maintained_pauses/__init__.py create mode 100644 tests/integration/src/sqlbuild/executor/maintained_pauses/_test_types.py create mode 100644 tests/integration/src/sqlbuild/executor/maintained_pauses/conftest.py create mode 100644 tests/integration/src/sqlbuild/executor/maintained_pauses/helpers.py create mode 100644 tests/integration/src/sqlbuild/executor/maintained_pauses/test_maintained_pause_lease.py diff --git a/crates/sqlbuild-analysis/src/janitor_planning/constants.rs b/crates/sqlbuild-analysis/src/janitor_planning/constants.rs index 30781736f8..0984a15101 100644 --- a/crates/sqlbuild-analysis/src/janitor_planning/constants.rs +++ b/crates/sqlbuild-analysis/src/janitor_planning/constants.rs @@ -3,7 +3,7 @@ use sqlbuild_core::constants::STATE_TABLE_NAMES; /// SQLBuild state tables the janitor never acts on. -pub const BUILT_IN_EXCLUDE_PATTERNS: [&str; 13] = STATE_TABLE_NAMES; +pub const BUILT_IN_EXCLUDE_PATTERNS: [&str; 14] = STATE_TABLE_NAMES; /// Prefix of every strict janitor archive name. pub const ARCHIVE_NAME_PREFIX: &str = "_SQB_ARCHIVE__"; /// Separator between archive name segments. diff --git a/crates/sqlbuild-cli/src/argv/command_table.json b/crates/sqlbuild-cli/src/argv/command_table.json index 0a0e5235bd..80148a54d7 100644 --- a/crates/sqlbuild-cli/src/argv/command_table.json +++ b/crates/sqlbuild-cli/src/argv/command_table.json @@ -2630,6 +2630,17 @@ "default": false, "group": 1 }, + { + "kind": "store_true", + "options": [ + "--ignore-maintained-grace" + ], + "dest": "ignore_maintained_grace", + "nargs": 0, + "const": true, + "default": false, + "group": 1 + }, { "kind": "store", "options": [ diff --git a/crates/sqlbuild-cli/src/argv/constants.rs b/crates/sqlbuild-cli/src/argv/constants.rs index 991c78bae2..14b4b94ec7 100644 --- a/crates/sqlbuild-cli/src/argv/constants.rs +++ b/crates/sqlbuild-cli/src/argv/constants.rs @@ -77,7 +77,7 @@ pub(crate) const INT_CONVERTER: &str = "int"; pub(crate) const PATH_CONVERTER: &str = "Path"; /// Every `CliNamespace` attribute with a default, in declaration order. -pub(crate) const CLI_NAMESPACE_DEFAULTS: [(&str, NamespaceDefault); 192] = [ +pub(crate) const CLI_NAMESPACE_DEFAULTS: [(&str, NamespaceDefault); 193] = [ ("command", NamespaceDefault::None), ("project_dir", NamespaceDefault::None), ("dbt_project_dir", NamespaceDefault::None), @@ -157,6 +157,7 @@ pub(crate) const CLI_NAMESPACE_DEFAULTS: [(&str, NamespaceDefault); 192] = [ "allow_snapshot_schema_change", NamespaceDefault::Bool(false), ), + ("ignore_maintained_grace", NamespaceDefault::Bool(false)), ("concurrency", NamespaceDefault::None), ("verbose", NamespaceDefault::Bool(false)), ("debug", NamespaceDefault::Bool(false)), diff --git a/crates/sqlbuild-cli/src/dispatch/constants.rs b/crates/sqlbuild-cli/src/dispatch/constants.rs index 2910f50607..6b048c7dbb 100644 --- a/crates/sqlbuild-cli/src/dispatch/constants.rs +++ b/crates/sqlbuild-cli/src/dispatch/constants.rs @@ -145,7 +145,7 @@ pub(crate) const PLAN_ROUTE: Route = routed( REQUEST_KEYWORD_SHAPE, ); -const BUILD_FIELDS: [Field; 40] = [ +const BUILD_FIELDS: [Field; 41] = [ PROJECT_DIR, NO_SQL_VALIDATION, same("no_cache"), @@ -169,6 +169,7 @@ const BUILD_FIELDS: [Field; 40] = [ same("allow_retention_decrease"), same("allow_missing_migration_origin"), same("allow_snapshot_schema_change"), + same("ignore_maintained_grace"), same("concurrency"), SELECT, field("selector_files", Source::SelectorFiles), diff --git a/crates/sqlbuild-core/src/constants.rs b/crates/sqlbuild-core/src/constants.rs index 95966a7de9..82fb646f95 100644 --- a/crates/sqlbuild-core/src/constants.rs +++ b/crates/sqlbuild-core/src/constants.rs @@ -44,8 +44,10 @@ pub const OBSERVATIONS_TABLE_NAME: &str = "_sqlbuild_observations"; pub const RESOURCE_COSTS_TABLE_NAME: &str = "_sqlbuild_resource_costs"; /// State table of each build's start and end, which shows runs the control plane did not start. pub const RUNS_TABLE_NAME: &str = "_sqlbuild_runs"; +/// State table of the maintained models a build paused, and its heartbeats while it holds them. +pub const MAINTAINED_PAUSES_TABLE_NAME: &str = "_sqlbuild_maintained_pauses"; /// Every state table SQLBuild writes in a target's state schema. -pub const STATE_TABLE_NAMES: [&str; 13] = [ +pub const STATE_TABLE_NAMES: [&str; 14] = [ FINGERPRINT_TABLE_NAME, MICROBATCH_TABLE_NAME, SOURCE_FRESHNESS_TABLE_NAME, @@ -59,6 +61,7 @@ pub const STATE_TABLE_NAMES: [&str; 13] = [ OBSERVATIONS_TABLE_NAME, RESOURCE_COSTS_TABLE_NAME, RUNS_TABLE_NAME, + MAINTAINED_PAUSES_TABLE_NAME, ]; /// A zero-length cursor duration. pub const ZERO_DAY_DURATION: &str = "0d"; diff --git a/crates/sqlbuild-orchestration/tests/api/test_warehouse_metadata.rs b/crates/sqlbuild-orchestration/tests/api/test_warehouse_metadata.rs index 6d0b5e4ff7..8df1af7c22 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_warehouse_metadata.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_warehouse_metadata.rs @@ -3,7 +3,7 @@ use chrono::{DateTime, Duration, Utc}; use reqwest::StatusCode; use serde_json::json; -use sqlbuild_core::constants::{RUNS_TABLE_NAME, STATE_TABLE_NAMES}; +use sqlbuild_core::constants::{MAINTAINED_PAUSES_TABLE_NAME, RUNS_TABLE_NAME, STATE_TABLE_NAMES}; use uuid::Uuid; use crate::api::tests::changes_helpers::{connect, expect_topics}; @@ -36,10 +36,11 @@ use crate::warehouse_metadata::models::{MetadataBatch, WarehouseHistoryKind}; #[test] fn given_state_tables_when_classified_then_each_is_mirrored_by_exactly_one_kind() { - // The runs table has no mirror until the control store decides where observed runs live. + // The runs table has no mirror until the control store decides where observed runs live; + // maintained pauses are a lease that builds read from the warehouse, never a mirror. let mut written: Vec<&str> = STATE_TABLE_NAMES .into_iter() - .filter(|table| *table != RUNS_TABLE_NAME) + .filter(|table| ![RUNS_TABLE_NAME, MAINTAINED_PAUSES_TABLE_NAME].contains(table)) .collect(); written.sort_unstable(); let test_cases = [StateTableClassificationTestCase { diff --git a/crates/sqlbuild-python-base/src/records/_helpers/planner_records/constants.rs b/crates/sqlbuild-python-base/src/records/_helpers/planner_records/constants.rs index 7a79955e8d..469a8f2610 100644 --- a/crates/sqlbuild-python-base/src/records/_helpers/planner_records/constants.rs +++ b/crates/sqlbuild-python-base/src/records/_helpers/planner_records/constants.rs @@ -670,7 +670,8 @@ const R_MODEL_PLAN_ENTRY: RecordSpec = RecordSpec { FieldSpec::new("target_lag", FieldDefault::None), FieldSpec::new("refresh_mode", FieldDefault::None), FieldSpec::new("refresh_warehouse", FieldDefault::None), - FieldSpec::new("change_tracking", FieldDefault::False), + FieldSpec::new("refresh_on_build", FieldDefault::False), + FieldSpec::new("maintained_readers", FieldDefault::EmptyTuple), ], }; const R_CLONE_SOURCE_PLAN_ENTRY: RecordSpec = RecordSpec { diff --git a/crates/sqlbuild-python-base/src/records/constants.rs b/crates/sqlbuild-python-base/src/records/constants.rs index ca6de97cfa..6825d659f6 100644 --- a/crates/sqlbuild-python-base/src/records/constants.rs +++ b/crates/sqlbuild-python-base/src/records/constants.rs @@ -255,6 +255,7 @@ const BUILD_COMMAND_REQUEST: RecordSpec = RecordSpec { FieldSpec::new("allow_retention_decrease", FieldDefault::False), FieldSpec::new("allow_missing_migration_origin", FieldDefault::False), FieldSpec::new("allow_snapshot_schema_change", FieldDefault::False), + FieldSpec::new("ignore_maintained_grace", FieldDefault::False), FieldSpec::new("concurrency", FieldDefault::None), FieldSpec::new("select", FieldDefault::EmptyTuple), FieldSpec::new("exclude", FieldDefault::EmptyTuple), diff --git a/crates/sqlbuild-python-front/src/discovery/main/project/configured_maintained.rs b/crates/sqlbuild-python-front/src/discovery/main/project/configured_maintained.rs new file mode 100644 index 0000000000..daf7d1ec7f --- /dev/null +++ b/crates/sqlbuild-python-front/src/discovery/main/project/configured_maintained.rs @@ -0,0 +1,50 @@ +//! A build's maintained-pause runtime settings: the target's `pause_maintained` and `[maintained]`. + +use crate::contracts::main::resolve_target_config::resolve_target_config; +use crate::contracts::main::resolve_target_name::resolve_target_name; +use crate::discovery::_helpers::project::project_config::configured; +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_config::project::main::load_maintained_config::load_maintained_config; +use sqlbuild_config::project::models::MaintainedConfig; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// `BuildRuntimeParams` fields for the request's target and flag, and the project's `[maintained]`. +pub fn configured_maintained<'py>( + discovered: &Bound<'py, PyAny>, + request: &Bound<'py, PyAny>, + project_dir: &Bound<'py, PyAny>, +) -> PyResult)>> { + let py = discovered.py(); + let selected_target: Bound<'py, PyAny> = native_field(request, "selected_target")?; + let project_config: Bound<'py, PyAny> = native_field(discovered, "project_config")?; + let local_config: Bound<'py, PyAny> = native_field(discovered, "local_config")?; + let paused: Option = + match resolve_target_name(py, &project_config, &local_config, &selected_target)? { + Some(name) => native_field( + &resolve_target_config(py, &project_config, &local_config, name.as_any())?, + "pause_maintained", + )? + .extract()?, + None => None, + }; + let config: MaintainedConfig = configured(project_dir, load_maintained_config)?; + Ok(vec![ + ( + "pause_maintained", + paused.unwrap_or(false).into_bound_py_any(py)?, + ), + ( + "maintained_heartbeat_grace_seconds", + config.heartbeat_grace_seconds.into_bound_py_any(py)?, + ), + ( + "allow_ignore_maintained_grace", + config.allow_ignore_grace.into_bound_py_any(py)?, + ), + ( + "ignore_maintained_grace", + native_field(request, "ignore_maintained_grace")?, + ), + ]) +} diff --git a/crates/sqlbuild-python-front/src/discovery/main/project/mod.rs b/crates/sqlbuild-python-front/src/discovery/main/project/mod.rs index 00673e75cd..1f0aeb28e9 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/project/mod.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/project/mod.rs @@ -1,6 +1,7 @@ pub mod column_name; pub mod configured_changes; pub mod configured_git_base_ref; +pub mod configured_maintained; pub mod configured_review; pub mod discover_configured_providers; pub mod discover_connection_configuration; diff --git a/crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/maintained.rs b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/maintained.rs index e9e3159c0b..d724ff0f87 100644 --- a/crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/maintained.rs +++ b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/maintained.rs @@ -1,4 +1,4 @@ -//! Maintained models whose live refresh settings drifted, and kept ones a replaced upstream reinitialises. +//! Maintained models whose live refresh settings drifted, kept ones a replaced upstream reinitialises, and who reads what. use crate::planner_phases::main::inputs::runtime_reads::runtime_reads; use crate::planner_phases::models::ModelEntryResults; @@ -7,7 +7,7 @@ use crate::planner_strategy::main::values::text::text; use crate::planner_warehouse::models::{AdapterRead, PlannerReads, Read}; use pyo3::IntoPyObject; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; -use pyo3::types::{PyBool, PyString}; +use pyo3::types::{PyBool, PyString, PyTuple}; use sqlbuild_analysis::assembly::project::types::ObjectKey; use sqlbuild_analysis::maintained_refresh::main::maintained_settings_drift::maintained_settings_drift; use sqlbuild_analysis::maintained_refresh::main::reinitialised_readers::reinitialised_readers; @@ -36,7 +36,7 @@ const DEFAULT_REFRESH_MODE: &str = "incremental"; /// Actions that recreate the relation, which reinitialises every dynamic table reading it. const REPLACING_ACTIONS: [&str; 3] = ["create_table", "create_view", "create_maintained"]; -/// Plan maintained refresh settings and reinitialised readers, and track changes on what they read. +/// Plan maintained refresh settings and reinitialised readers, and name the maintained readers of each entry. pub(crate) fn plan_maintained_refresh<'py>( runtime: &Bound<'py, PyAny>, execution_scope: &Bound<'py, PyAny>, @@ -64,20 +64,21 @@ pub(crate) fn plan_maintained_refresh<'py>( text(&native_field(&entry, "materialization_type")?)? == MAINTAINED; let entry = if is_maintained { with_refresh_settings(entry, &models, &reads)? - } else if read_by_maintained( + } else { + entry + }; + let readers: Vec = maintained_readers( py, downstream.get(&key).copied(), (&models, &mut maintained_models), - )? { + )?; + let entry = if readers.is_empty() { + entry + } else { replaced( &entry, - &[( - "change_tracking", - PyBool::new(py, true).to_owned().into_any(), - )], + &[("maintained_readers", PyTuple::new(py, readers)?.into_any())], )? - } else { - entry }; let action: String = text(&native_field(&entry, "action")?)?; rows.push(MaintainedRead { @@ -103,15 +104,16 @@ pub(crate) fn plan_maintained_refresh<'py>( }) } -/// Whether any model reading the entry is maintained, checking each reader once per plan. -fn read_by_maintained( +/// The maintained models reading the entry, selected or not, checking each reader once per plan. +fn maintained_readers( py: Python<'_>, readers: Option<&Vec>, (models, maintained_models): (&ModelLookup, &mut HashMap), -) -> PyResult { +) -> PyResult> { let Some(readers) = readers else { - return Ok(false); + return Ok(Vec::new()); }; + let mut found: Vec = Vec::new(); for (kind, name) in readers { if kind != MODEL { continue; @@ -127,10 +129,11 @@ fn read_by_maintained( } }; if maintained { - return Ok(true); + found.push(name.clone()); } } - Ok(false) + found.sort(); + Ok(found) } /// The entry with its `target_lag`; a kept one alters on drift, or creates when no dynamic table holds its name. @@ -159,6 +162,12 @@ fn with_refresh_settings<'py>( "refresh_warehouse", desired.warehouse.as_deref().into_pyobject(py)?.into_any(), ), + ( + "refresh_on_build", + PyBool::new(py, config_value(py, &values, "refresh_on_build")?.is_truthy()?) + .to_owned() + .into_any(), + ), ]; if text(&native_field(&entry, "action")?)? == KEEP_MAINTAINED { if let Some((action, reason)) = live_action(&entry, &desired, reads)? { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs index 7186236739..eeaa517957 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs @@ -49,6 +49,7 @@ use sqlbuild_python_compile::compile_driver::models::TargetSettingsValues; use sqlbuild_python_front::cost_context::models::CostContext; use sqlbuild_python_front::cursor::main::overrides::cursor_integer::cursor_integer; use sqlbuild_python_front::cursor::main::overrides::cursor_timestamp::cursor_timestamp; +use sqlbuild_python_front::discovery::main::project::configured_maintained::configured_maintained; use sqlbuild_python_front::native_host::main::recorder_callbacks::recorder_callbacks; use sqlbuild_python_output::build_progress::main::batch_complete_callback::batch_complete_callback; use sqlbuild_python_output::build_progress::main::check_callback::check_callback; @@ -336,77 +337,71 @@ pub(crate) fn execute_build_plan<'py>( let lifecycle: &DirectPythonLifecycle = preparation.lifecycle.get(); let [start_ts, end_ts, start_int, end_int] = &preparation.cursors; let request = |name: &str| native_field(&context.request, name); - let target_config: TargetSettingsValues = project_target_settings( - &context.get("discovered_inputs")?, - &project, - (&connection_config, &adapter), - )?; + let discovered: Bound<'py, PyAny> = context.get("discovered_inputs")?; + let project_dir: Bound<'py, PyAny> = context.get("effective_project_dir")?; + let target_config: TargetSettingsValues = + project_target_settings(&discovered, &project, (&connection_config, &adapter))?; let (default_database, default_schema) = target_config.node_locations(&adapter)?; + let maintained = configured_maintained(&discovered, &context.request, &project_dir)?; let runtime: Bound<'py, PyAny> = call_with( &host_entry(py, ExecutorHost::BuildRuntimeParams)?, &[ - ("run_id", native_field(&project, "run_id")?), - ( - "runtime_dir", - joined(&context.get("effective_project_dir")?, &["target"])?, - ), - ( - "snapshots", - context - .get("discovered_inputs")? - .getattr("project_config")? - .getattr("snapshots")?, - ), - ( - "allow_snapshot_schema_change", - request("allow_snapshot_schema_change")?, - ), - ("run_tests", request("run_tests")?), - ("run_audits", request("run_audits")?), - ("fail_fast", request("fail_fast")?), - ( - "max_concurrency", - preparation.concurrency.into_pyobject(py)?.into_any(), - ), - ("loader_is_reload", request("reload_sources")?), - ("target_config", target_config.record(py)?), - ("default_database", default_database), - ("default_schema", default_schema), - ("start_cursor_ts", start_ts.clone()), - ("end_cursor_ts", end_ts.clone()), - ("start_cursor_int", start_int.clone()), - ("end_cursor_int", end_int.clone()), - ("use_color", boolean(py, context.use_color)), - ("providers", providers.clone()), - ( - "test_timeouts", - timeout_record( - py, - &resolve_run_timeout( - (&project, &context.get("discovered_inputs")?), - request("test_timeout")?.extract()?, - TimeoutKind::Test, + &maintained[..], + &[ + ("run_id", native_field(&project, "run_id")?), + ("runtime_dir", joined(&project_dir, &["target"])?), + ( + "snapshots", + discovered.getattr("project_config")?.getattr("snapshots")?, + ), + ( + "allow_snapshot_schema_change", + request("allow_snapshot_schema_change")?, + ), + ("run_tests", request("run_tests")?), + ("run_audits", request("run_audits")?), + ("fail_fast", request("fail_fast")?), + ( + "max_concurrency", + preparation.concurrency.into_pyobject(py)?.into_any(), + ), + ("loader_is_reload", request("reload_sources")?), + ("target_config", target_config.record(py)?), + ("default_database", default_database), + ("default_schema", default_schema), + ("start_cursor_ts", start_ts.clone()), + ("end_cursor_ts", end_ts.clone()), + ("start_cursor_int", start_int.clone()), + ("end_cursor_int", end_int.clone()), + ("use_color", boolean(py, context.use_color)), + ("providers", providers.clone()), + ( + "test_timeouts", + timeout_record( + py, + &resolve_run_timeout( + (&project, &discovered), + request("test_timeout")?.extract()?, + TimeoutKind::Test, + )?, )?, - )?, - ), - ( - "project_relations", - build_project_relation_targets( - py, - Some(keywords( + ), + ( + "project_relations", + build_project_relation_targets( py, - &[("adapter", adapter.clone()), ("plan_output", plan.clone())], - )?), - ) - .and_then(|value| pyo3::IntoPyObjectExt::into_bound_py_any(value, py))?, - ), - ], - )?; - let policy: Option = changed_policy( - &context.get("effective_project_dir")?, - &context.request, - &project, + Some(keywords( + py, + &[("adapter", adapter.clone()), ("plan_output", plan.clone())], + )?), + ) + .and_then(|value| pyo3::IntoPyObjectExt::into_bound_py_any(value, py))?, + ), + ], + ] + .concat(), )?; + let policy: Option = changed_policy(&project_dir, &context.request, &project)?; let prepared: Prepared<'py> = prepare_consumption( (&plan, &project), (&adapter, &connection_config), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/constants.rs index f63a0244ed..a0c36d1e5c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/constants.rs @@ -19,10 +19,10 @@ use sqlbuild_core::command_output::constants::{ }; use sqlbuild_core::constants::{ AUDIT_RESULTS_TABLE_NAME, COLUMN_MIGRATION_TABLE_NAME, CONSUMPTION_TABLE_NAME, - FINGERPRINT_TABLE_NAME, JANITOR_EVENTS_TABLE_NAME, MICROBATCH_TABLE_NAME, MIGRATION_TABLE_NAME, - NODE_RESULTS_TABLE_NAME, OBSERVATIONS_TABLE_NAME, OLD_NAME_VIEW_TABLE_NAME, - PROJECT_CONFIG_FILENAME, PYTHON_INIT_MODULE_STEM, PYTHON_NODE_ROOT, RUNS_TABLE_NAME, - SETTINGS_SECTION, SOURCE_FRESHNESS_TABLE_NAME, STAGED_PROMOTION_MODE, + FINGERPRINT_TABLE_NAME, JANITOR_EVENTS_TABLE_NAME, MAINTAINED_PAUSES_TABLE_NAME, + MICROBATCH_TABLE_NAME, MIGRATION_TABLE_NAME, NODE_RESULTS_TABLE_NAME, OBSERVATIONS_TABLE_NAME, + OLD_NAME_VIEW_TABLE_NAME, PROJECT_CONFIG_FILENAME, PYTHON_INIT_MODULE_STEM, PYTHON_NODE_ROOT, + RUNS_TABLE_NAME, SETTINGS_SECTION, SOURCE_FRESHNESS_TABLE_NAME, STAGED_PROMOTION_MODE, TABLE_PROMOTION_MODE_SETTING_KEY, ZERO_DAY_DURATION, }; use sqlbuild_core::diagnostic_routing::constants::{ @@ -95,6 +95,7 @@ pub(crate) const EXPORTED_TEXTS: &[(&str, &str)] = &[ ("JANITOR_EVENTS_TABLE_NAME", JANITOR_EVENTS_TABLE_NAME), ("OBSERVATIONS_TABLE_NAME", OBSERVATIONS_TABLE_NAME), ("RUNS_TABLE_NAME", RUNS_TABLE_NAME), + ("MAINTAINED_PAUSES_TABLE_NAME", MAINTAINED_PAUSES_TABLE_NAME), ("COLUMN_NODE_TYPE", COLUMN_NODE_TYPE), ("COLUMN_NODE_NAME", COLUMN_NODE_NAME), ("COLUMN_TARGET_DATABASE", COLUMN_TARGET_DATABASE), diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 15f6ae2876..5b45ec48d0 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -192,6 +192,7 @@ MICROBATCH_TABLE_NAME: str NODE_RESULTS_TABLE_NAME: str OBSERVATIONS_TABLE_NAME: str RUNS_TABLE_NAME: str +MAINTAINED_PAUSES_TABLE_NAME: str AUDIT_RESULTS_TABLE_NAME: str JANITOR_EVENTS_TABLE_NAME: str COLUMN_NODE_TYPE: str @@ -6592,6 +6593,7 @@ class BuildCommandRequest(NativeRecord): allow_retention_decrease: bool allow_missing_migration_origin: bool allow_snapshot_schema_change: bool + ignore_maintained_grace: bool concurrency: int | None select: tuple[str, ...] exclude: tuple[str, ...] @@ -6636,6 +6638,7 @@ class BuildCommandRequest(NativeRecord): allow_retention_decrease: bool = False, allow_missing_migration_origin: bool = False, allow_snapshot_schema_change: bool = False, + ignore_maintained_grace: bool = False, concurrency: int | None = None, select: tuple[str, ...] = (), exclude: tuple[str, ...] = (), @@ -14123,7 +14126,8 @@ class ModelPlanEntry(NativeRecord): target_lag: str | None refresh_mode: str | None refresh_warehouse: str | None - change_tracking: bool + refresh_on_build: bool + maintained_readers: tuple[str, ...] def __init__( self, key: CompiledObjectKey, @@ -14217,7 +14221,8 @@ class ModelPlanEntry(NativeRecord): target_lag: str | None = ..., refresh_mode: str | None = ..., refresh_warehouse: str | None = ..., - change_tracking: bool = ..., + refresh_on_build: bool = ..., + maintained_readers: tuple[str, ...] = ..., ) -> None: ... class CloneSourcePlanEntry(NativeRecord): diff --git a/src/sqlbuild/adapter/contract/classes/strict_adapter.py b/src/sqlbuild/adapter/contract/classes/strict_adapter.py index 6b9f40d251..e0695c23d0 100644 --- a/src/sqlbuild/adapter/contract/classes/strict_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/strict_adapter.py @@ -367,6 +367,14 @@ def maintained_refresh_settings( del connection, database, schema, name return None + def maintained_scheduling_state( + self, *, connection: Any, database: str | None, schema: str | None, name: str + ) -> str | None: + """Optional: a maintained model's `ACTIVE` or `SUSPENDED` state, or None when unlisted.""" + + del connection, database, schema, name + return None + @abstractmethod def render_create_table_as(self, *, destination: str, sql: str) -> tuple[str, ...]: """Render SQL statements that create or replace a table from a query.""" diff --git a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py index ef894eddb8..02060f65cc 100644 --- a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py +++ b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py @@ -3721,6 +3721,24 @@ def maintained_refresh_settings( ) -> tuple[str, str] | None: """The dynamic table's target lag and warehouse from SHOW DYNAMIC TABLES, or None.""" + row: dict[str, object] | None = self._listed_dynamic_table( + connection=connection, database=database, schema=schema, name=name + ) + return None if row is None else (str(row.get("target_lag")), str(row.get("warehouse"))) + + def maintained_scheduling_state( + self, *, connection: Any, database: str | None, schema: str | None, name: str + ) -> str | None: + """The dynamic table's scheduling state from SHOW DYNAMIC TABLES, or None.""" + + row: dict[str, object] | None = self._listed_dynamic_table( + connection=connection, database=database, schema=schema, name=name + ) + return None if row is None else str(row.get("scheduling_state")) + + def _listed_dynamic_table( + self, *, connection: Any, database: str | None, schema: str | None, name: str + ) -> dict[str, object] | None: stored_name: str = self._information_schema_identifier(name) if schema is None: return None @@ -3738,7 +3756,7 @@ def maintained_refresh_settings( row: dict[str, object] for row in rows: if str(row.get("name")) == stored_name: - return (str(row.get("target_lag")), str(row.get("warehouse"))) + return row return None def widen_target_nullability( diff --git a/src/sqlbuild/executor/build/classes/build_scheduler.py b/src/sqlbuild/executor/build/classes/build_scheduler.py index 5790b24ac8..b8c628ecfd 100644 --- a/src/sqlbuild/executor/build/classes/build_scheduler.py +++ b/src/sqlbuild/executor/build/classes/build_scheduler.py @@ -1256,6 +1256,7 @@ def _execute_model_node( watermark_resolver=self._watermark_resolver, bound_views=bound_views, fingerprint_writes=self._fingerprint_writes, + maintained_hold=self._runtime.maintained_hold, ), promotion_mode=self._promotion_mode, snapshots=self._snapshots, diff --git a/src/sqlbuild/executor/build/main/_execute.py b/src/sqlbuild/executor/build/main/_execute.py index 87f8f4bdc1..bc0433b760 100644 --- a/src/sqlbuild/executor/build/main/_execute.py +++ b/src/sqlbuild/executor/build/main/_execute.py @@ -2,6 +2,7 @@ from __future__ import annotations +from contextlib import AbstractContextManager, nullcontext from dataclasses import replace from typing import Any @@ -35,6 +36,10 @@ ) from sqlbuild.executor.build.types import BuildStatus from sqlbuild.executor.load.models import LoadExecutionResult +from sqlbuild.executor.maintained_pauses.classes.maintained_hold import MaintainedHold +from sqlbuild.executor.maintained_pauses.main.hold_maintained_models import ( + hold_maintained_models, +) from sqlbuild.executor.migrations.main._apply import apply_model_migrations from sqlbuild.executor.migrations.main._apply_columns import apply_column_migrations from sqlbuild.executor.run.models import ModelExecutionResult @@ -54,16 +59,27 @@ def execute_build_plan( initial_state: BuildInitialState | None = None, schema_prepared: bool = False, ) -> BuildExecutionResult: - """Execute a full build plan using the DAG scheduler.""" + """Execute a build plan with the DAG scheduler, holding the maintained models it touches.""" - with relation_metadata_cache_scope(adapter=adapter): + hold: MaintainedHold | None = hold_maintained_models( + plan=plan, adapter=adapter, runtime=runtime, connection_config=connection_config + ) + held: AbstractContextManager[object] = ( + nullcontext() + if hold is None + else hold.holding( + connection=scheduler_connection, + on_progress=None if callbacks is None else callbacks.on_progress, + ) + ) + with relation_metadata_cache_scope(adapter=adapter), held: return _execute_build_plan( plan=plan, adapter=adapter, connection_config=connection_config, connections=connections, scheduler_connection=scheduler_connection, - runtime=runtime, + runtime=replace(runtime, maintained_hold=hold), callbacks=callbacks, customizations=customizations, initial_state=initial_state, diff --git a/src/sqlbuild/executor/build/models.py b/src/sqlbuild/executor/build/models.py index 2883e40c8f..005ad584c8 100644 --- a/src/sqlbuild/executor/build/models.py +++ b/src/sqlbuild/executor/build/models.py @@ -28,6 +28,7 @@ from sqlbuild.executor.build.types import BeforeNodeWriteCallback, BuildStatus from sqlbuild.executor.custom.models import MaterializationResult from sqlbuild.executor.load.models import LoadExecutionResult +from sqlbuild.executor.maintained_pauses.classes.maintained_hold import MaintainedHold from sqlbuild.executor.python_nodes.types import PythonIdentityRecorder from sqlbuild.executor.run.models import MicrobatchBatchProgress, ModelExecutionResult from sqlbuild.executor.scheduling.types import ExecutionStatus @@ -101,6 +102,11 @@ class BuildRuntimeParams: microbatch_lease_check: Callable[[], None] | None = None project_relations: Mapping[SqlResourceRef, str] | None = None test_timeouts: RunItemTimeouts | None = None + pause_maintained: bool = False + maintained_heartbeat_grace_seconds: int = 1800 + allow_ignore_maintained_grace: bool = False + ignore_maintained_grace: bool = False + maintained_hold: MaintainedHold | None = None @dataclass(frozen=True) diff --git a/src/sqlbuild/executor/maintained_pauses/__init__.py b/src/sqlbuild/executor/maintained_pauses/__init__.py new file mode 100644 index 0000000000..9aae34706c --- /dev/null +++ b/src/sqlbuild/executor/maintained_pauses/__init__.py @@ -0,0 +1 @@ +"""Maintained models a build holds paused, and the append-only lease that records it.""" diff --git a/src/sqlbuild/executor/maintained_pauses/_helpers/__init__.py b/src/sqlbuild/executor/maintained_pauses/_helpers/__init__.py new file mode 100644 index 0000000000..928816ab73 --- /dev/null +++ b/src/sqlbuild/executor/maintained_pauses/_helpers/__init__.py @@ -0,0 +1 @@ +"""Maintained pause helpers.""" diff --git a/src/sqlbuild/executor/maintained_pauses/_helpers/scope.py b/src/sqlbuild/executor/maintained_pauses/_helpers/scope.py new file mode 100644 index 0000000000..eff45d123d --- /dev/null +++ b/src/sqlbuild/executor/maintained_pauses/_helpers/scope.py @@ -0,0 +1,92 @@ +"""Which maintained models a build holds, upstream first, and which ones it refreshes by hand.""" + +from __future__ import annotations + +from sqlbuild._native import ( + CompiledObjectKey, + CompiledResourceType, + MaterializationType, + ModelPlanEntry, + PlanAction, + PlanOutput, +) + +_UNWRITTEN: frozenset[PlanAction] = frozenset( + {PlanAction.SKIP, PlanAction.KEEP_MAINTAINED, PlanAction.ALTER_MAINTAINED} +) + + +def held_names(*, plan: PlanOutput, pause_maintained: bool) -> tuple[str, ...]: + """Maintained models built, or reading what is written (every planned one when paused).""" + + names: set[str] = set() + entry: ModelPlanEntry + for entry in plan.model_entries: + maintained: bool = entry.materialization_type == MaterializationType.MAINTAINED + if maintained and (pause_maintained or entry.action not in _UNWRITTEN): + names.add(entry.name) + if entry.action not in _UNWRITTEN: + names.update(entry.maintained_readers) + return _upstream_first(names=frozenset(names), plan=plan) + + +def refreshed_for_readers(*, plan: PlanOutput) -> frozenset[str]: + """Maintained models that a model built later in this build reads, directly or not.""" + + entries: dict[str, ModelPlanEntry] = {entry.name: entry for entry in plan.model_entries} + readers: dict[str, tuple[str, ...]] = _model_edges(edges=plan.downstream_deps) + found: set[str] = set() + name: str + for name, entry in entries.items(): + if entry.materialization_type != MaterializationType.MAINTAINED: + continue + reached: set[str] = _reachable(start=name, edges=readers) + if any(_built_table(entry=entries.get(reader)) for reader in reached): + found.add(name) + return frozenset(found) + + +def _built_table(*, entry: ModelPlanEntry | None) -> bool: + return ( + entry is not None + and entry.materialization_type != MaterializationType.MAINTAINED + and entry.action != PlanAction.SKIP + ) + + +def _upstream_first(*, names: frozenset[str], plan: PlanOutput) -> tuple[str, ...]: + """A model upstream of another has strictly fewer held ancestors, so it sorts first.""" + + upstream: dict[str, tuple[str, ...]] = _model_edges(edges=plan.upstream_deps) + held_ancestors: dict[str, int] = { + name: len(_reachable(start=name, edges=upstream) & names) for name in names + } + return tuple(sorted(names, key=lambda name: (held_ancestors[name], name))) + + +def _model_edges( + *, edges: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]] +) -> dict[str, tuple[str, ...]]: + models: dict[str, tuple[str, ...]] = {} + key: CompiledObjectKey + others: tuple[CompiledObjectKey, ...] + for key, others in edges.items(): + if key.resource_type == CompiledResourceType.MODEL: + models[key.name] = _model_names(keys=others) + return models + + +def _model_names(*, keys: tuple[CompiledObjectKey, ...]) -> tuple[str, ...]: + return tuple(key.name for key in keys if key.resource_type == CompiledResourceType.MODEL) + + +def _reachable(*, start: str, edges: dict[str, tuple[str, ...]]) -> set[str]: + seen: set[str] = set() + pending: list[str] = list(edges.get(start, ())) + while pending: + name: str = pending.pop() + if name in seen: + continue + seen.add(name) + pending.extend(edges.get(name, ())) + return seen diff --git a/src/sqlbuild/executor/maintained_pauses/_helpers/sql.py b/src/sqlbuild/executor/maintained_pauses/_helpers/sql.py new file mode 100644 index 0000000000..35a7711180 --- /dev/null +++ b/src/sqlbuild/executor/maintained_pauses/_helpers/sql.py @@ -0,0 +1,156 @@ +"""Portable SQL for the append-only maintained pause table, timed by the warehouse clock.""" + +from __future__ import annotations + +import hashlib +import json +from datetime import datetime +from typing import Any + +from sqlbuild._native import ( + MAINTAINED_PAUSES_TABLE_NAME, + CompiledRelationLocation, + render_state_sql_literal, + render_state_table_create_sql, +) +from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter +from sqlbuild.errors.contracts.exceptions import ExecutorInputError +from sqlbuild.executor.maintained_pauses.constants import ( + COLUMN_EVENT_ID, + COLUMN_EVENT_TYPE, + COLUMN_RECORDED_AT, + COLUMN_RECORDED_BY, + COLUMN_RELATION_DATABASE, + COLUMN_RELATION_NAME, + COLUMN_RELATION_SCHEMA, + COLUMN_RUN_ID, + COLUMN_SCHEMA_VERSION, + MAINTAINED_PAUSE_COLUMN_TYPES, + MAINTAINED_PAUSE_COLUMNS, + MAINTAINED_PAUSE_REQUIRED_COLUMNS, + MAINTAINED_PAUSE_SCHEMA_VERSION, +) +from sqlbuild.executor.maintained_pauses.models import MaintainedPauseEvent, OpenHold +from sqlbuild.executor.maintained_pauses.types import MaintainedPauseEventType, StateLocation + + +def pause_table_name(*, adapter: BaseAdapter, location: StateLocation) -> str: + qualified: str | None = adapter.render_qualified_name( + database=location[0], schema=location[1], name=MAINTAINED_PAUSES_TABLE_NAME + ) + if qualified is None: + raise ExecutorInputError("the maintained pause table needs a schema") + return qualified + + +def setup_statements(*, adapter: BaseAdapter, location: StateLocation) -> tuple[str, ...]: + """Create the schema and the pause table when they are missing.""" + + return ( + *adapter.render_create_schema(database=location[0], schema=location[1]), + render_state_table_create_sql( + qualified_name=pause_table_name(adapter=adapter, location=location), + columns=MAINTAINED_PAUSE_COLUMNS, + column_types=MAINTAINED_PAUSE_COLUMN_TYPES, + required_columns=MAINTAINED_PAUSE_REQUIRED_COLUMNS, + render_framework_type=adapter.render_framework_type, + ), + ) + + +def insert_sql( + *, adapter: BaseAdapter, location: StateLocation, events: tuple[MaintainedPauseEvent, ...] +) -> str: + """Append every event in one statement, so a hand-over closes and opens holds together.""" + + now: str = adapter.render_current_utc_timestamp() + rows: str = " UNION ALL ".join( + f"SELECT {_row_literals(event=event)}, {now}" for event in events + ) + return ( + f"INSERT INTO {pause_table_name(adapter=adapter, location=location)} " + f"({', '.join(MAINTAINED_PAUSE_COLUMNS)}) {rows}" + ) + + +def open_holds_sql(*, adapter: BaseAdapter, location: StateLocation) -> str: + """Paused models nobody resumed, with when their run last wrote and the warehouse's now.""" + + table: str = pause_table_name(adapter=adapter, location=location) + now: str = adapter.render_current_utc_timestamp() + same_relation: str = " AND ".join( + f"COALESCE(r.{column}, '') = COALESCE(p.{column}, '')" + for column in (COLUMN_RELATION_DATABASE, COLUMN_RELATION_SCHEMA, COLUMN_RELATION_NAME) + ) + return ( + f"SELECT p.{COLUMN_RUN_ID}, p.{COLUMN_RELATION_DATABASE}, p.{COLUMN_RELATION_SCHEMA}, " + f"p.{COLUMN_RELATION_NAME}, CAST(h.last_seen AS TIMESTAMP), CAST({now} AS TIMESTAMP) " + f"FROM {table} p JOIN (SELECT {COLUMN_RUN_ID}, MAX({COLUMN_RECORDED_AT}) AS last_seen " + f"FROM {table} WHERE {COLUMN_RECORDED_BY} = {COLUMN_RUN_ID} GROUP BY {COLUMN_RUN_ID}) h " + f"ON h.{COLUMN_RUN_ID} = p.{COLUMN_RUN_ID} " + f"WHERE p.{COLUMN_EVENT_TYPE} = '{MaintainedPauseEventType.PAUSED.value}' " + f"AND NOT EXISTS (SELECT 1 FROM {table} r " + f"WHERE r.{COLUMN_EVENT_TYPE} = '{MaintainedPauseEventType.RESUMED.value}' " + f"AND r.{COLUMN_RUN_ID} = p.{COLUMN_RUN_ID} AND {same_relation})" + ) + + +def open_holds(*, rows: tuple[tuple[Any, ...], ...]) -> tuple[OpenHold, ...]: + """The open holds of `open_holds_sql` rows.""" + + return tuple( + OpenHold( + run_id=str(row[0]), + relation=CompiledRelationLocation( + database=_optional(row[1]), + schema=_optional(row[2]), + name=str(row[3]), + qualified_name=None, + ), + idle_seconds=_seconds_between(start=row[4], end=row[5]), + ) + for row in rows + ) + + +def _seconds_between(*, start: datetime, end: datetime) -> float: + return (end.replace(tzinfo=None) - start.replace(tzinfo=None)).total_seconds() + + +def _optional(value: object) -> str | None: + return None if value is None else str(value) + + +def _row_literals(*, event: MaintainedPauseEvent) -> str: + relation: CompiledRelationLocation | None = event.relation + values: dict[str, object | None] = { + COLUMN_EVENT_ID: _event_id(event=event), + COLUMN_SCHEMA_VERSION: MAINTAINED_PAUSE_SCHEMA_VERSION, + COLUMN_EVENT_TYPE: event.event_type.value, + COLUMN_RUN_ID: event.run_id, + COLUMN_RELATION_DATABASE: None if relation is None else relation.database, + COLUMN_RELATION_SCHEMA: None if relation is None else relation.schema, + COLUMN_RELATION_NAME: None if relation is None else relation.name, + COLUMN_RECORDED_BY: event.recorded_by, + } + return ", ".join( + render_state_sql_literal( + value=values[column], declared_type=MAINTAINED_PAUSE_COLUMN_TYPES[column] + ) + for column in MAINTAINED_PAUSE_COLUMNS + if column != COLUMN_RECORDED_AT + ) + + +def _event_id(*, event: MaintainedPauseEvent) -> str: + """Content-addressed: one pause and one resume per run and model, one ID per heartbeat.""" + + relation: CompiledRelationLocation | None = event.relation + identity: tuple[object, ...] = ( + event.event_type.value, + event.run_id, + None if relation is None else (relation.database, relation.schema, relation.name), + event.sequence, + ) + encoded: str = json.dumps(identity, separators=(",", ":"), ensure_ascii=True) + return hashlib.sha256(encoded.encode("utf-8")).hexdigest() diff --git a/src/sqlbuild/executor/maintained_pauses/classes/__init__.py b/src/sqlbuild/executor/maintained_pauses/classes/__init__.py new file mode 100644 index 0000000000..4dbf76c0aa --- /dev/null +++ b/src/sqlbuild/executor/maintained_pauses/classes/__init__.py @@ -0,0 +1 @@ +"""Maintained pause classes.""" diff --git a/src/sqlbuild/executor/maintained_pauses/classes/maintained_heartbeat.py b/src/sqlbuild/executor/maintained_pauses/classes/maintained_heartbeat.py new file mode 100644 index 0000000000..d26cdee712 --- /dev/null +++ b/src/sqlbuild/executor/maintained_pauses/classes/maintained_heartbeat.py @@ -0,0 +1,62 @@ +"""A thread that tells other builds, through the warehouse, that a build still holds its models.""" + +from __future__ import annotations + +import threading +from collections.abc import Callable +from typing import Any + +from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter + + +class MaintainedHeartbeat: + """Write a heartbeat every interval on its own connection, which a busy build never blocks.""" + + def __init__( + self, + *, + adapter: BaseAdapter, + connection_config: dict[str, object], + write: Callable[..., None], + interval_seconds: float, + ) -> None: + self._adapter: BaseAdapter = adapter + self._connection_config: dict[str, object] = connection_config + self._write: Callable[..., None] = write + self._interval_seconds: float = interval_seconds + self._stopped: threading.Event = threading.Event() + self._thread: threading.Thread = threading.Thread( + target=self._beat, name="sqlbuild-maintained-heartbeat", daemon=True + ) + self.failure: Exception | None = None + + def start(self) -> None: + self._thread.start() + + def stop(self) -> None: + self._stopped.set() + self._thread.join() + + def _beat(self) -> None: + """Reconnect after a failed beat; the grace, not one beat, decides when a hold expires.""" + + connection: Any = None + sequence: int = 0 + while not self._stopped.wait(self._interval_seconds): + sequence += 1 + try: + if connection is None: + connection = self._adapter.connect(dict(self._connection_config)) + self._write(connection=connection, sequence=sequence) + except Exception as error: + self.failure = error + connection = self._closed(connection=connection) + _ = self._closed(connection=connection) + + def _closed(self, *, connection: Any) -> None: + if connection is not None: + try: + self._adapter.close(connection) + except Exception as error: + self.failure = error + return None diff --git a/src/sqlbuild/executor/maintained_pauses/classes/maintained_hold.py b/src/sqlbuild/executor/maintained_pauses/classes/maintained_hold.py new file mode 100644 index 0000000000..64b9fb2117 --- /dev/null +++ b/src/sqlbuild/executor/maintained_pauses/classes/maintained_hold.py @@ -0,0 +1,353 @@ +"""One build's hold on the maintained models it touches: pause, record, beat, then resume.""" + +from __future__ import annotations + +import threading +from collections.abc import Callable, Iterator +from contextlib import contextmanager +from typing import Any + +from sqlbuild._native import CompiledRelationLocation +from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter +from sqlbuild.adapter.contract.exceptions import AdapterUserError +from sqlbuild.adapter.contract.types import MaintainedOperation +from sqlbuild.executor.maintained_pauses._helpers.sql import ( + insert_sql, + open_holds, + open_holds_sql, + setup_statements, +) +from sqlbuild.executor.maintained_pauses.classes.maintained_heartbeat import ( + MaintainedHeartbeat, +) +from sqlbuild.executor.maintained_pauses.constants import ( + ACTIVE_STATE, + HELD_ELSEWHERE_CODE, + SUSPENDED_STATE, +) +from sqlbuild.executor.maintained_pauses.models import MaintainedPauseEvent, OpenHold +from sqlbuild.executor.maintained_pauses.types import MaintainedPauseEventType, StateLocation + +type _RelationKey = tuple[str | None, str | None, str] + + +class MaintainedHold: + """Pause the held models a build found active, record it, and resume only those at the end.""" + + def __init__( + self, + *, + adapter: BaseAdapter, + held: dict[str, CompiledRelationLocation], + refreshed: frozenset[str], + state_location: StateLocation | None, + settings: tuple[str, bool, float, bool], + heartbeat: Callable[[Callable[..., None]], MaintainedHeartbeat], + ) -> None: + self._adapter: BaseAdapter = adapter + self._held: dict[str, CompiledRelationLocation] = held + self._refreshed: frozenset[str] = refreshed + self._state_location: StateLocation | None = state_location + self._run_id: str + self._pause_maintained: bool + self._grace_seconds: float + self._ignore_grace: bool + self._run_id, self._pause_maintained, self._grace_seconds, self._ignore_grace = settings + self._new_heartbeat: Callable[[Callable[..., None]], MaintainedHeartbeat] = heartbeat + self._heartbeat: MaintainedHeartbeat | None = None + self._paused: list[str] = [] + self._left_paused: frozenset[str] = frozenset() + self._lock: threading.Lock = threading.Lock() + self._on_progress: Callable[[str], None] | None = None + + @contextmanager + def holding( + self, *, connection: Any, on_progress: Callable[[str], None] | None + ) -> Iterator[None]: + """Hold for the build; after a failure, resume without hiding its error.""" + + try: + self.acquire(connection=connection, on_progress=on_progress) + yield + except BaseException: + try: + self.release(connection=connection) + except Exception as error: + self._warn(f"could not resume the maintained models this build paused: {error}") + raise + self.release(connection=connection) + + def refreshes(self, *, name: str) -> bool: + """Whether a kept or altered model is refreshed by hand once its inputs are built.""" + + return self._pause_maintained or name in self._refreshed + + def acquire(self, *, connection: Any, on_progress: Callable[[str], None] | None) -> None: + """Take over expired holds, refuse live ones, then pause the held models found active.""" + + self._on_progress = on_progress + if self._pause_maintained: + self._suspend_active(connection=connection, names=tuple(self._held)) + return + location: StateLocation | None = self._state_location + if location is None: + return + for statement in setup_statements(adapter=self._adapter, location=location): + self._adapter.execute(connection=connection, sql=statement) + taken: tuple[OpenHold, ...] = self._take_over(connection=connection, location=location) + taken_keys: set[_RelationKey] = {_key(hold.relation) for hold in taken} + states: dict[str, str | None] = { + name: self._state(connection=connection, name=name) for name in self._held + } + self._paused = [ + name + for name in self._held + if states[name] == ACTIVE_STATE or _key(self._held[name]) in taken_keys + ] + self._left_paused = frozenset( + name + for name in self._held + if states[name] == SUSPENDED_STATE and name not in self._paused + ) + for name in sorted(self._left_paused): + self._warn(f"maintained model {name} was paused outside SQLBuild; it stays paused") + self._record( + connection=connection, + events=( + *( + self._event(event_type=MaintainedPauseEventType.RESUMED, hold=hold) + for hold in taken + ), + *( + self._own(event_type=MaintainedPauseEventType.PAUSED, name=name) + for name in self._paused + ), + ), + ) + self._suspend_active(connection=connection, names=tuple(self._paused), states=states) + self._heartbeat = self._new_heartbeat(self._write_heartbeat) + self._heartbeat.start() + + def after_create(self, *, connection: Any, name: str) -> None: + """Suspend a held model that CREATE OR REPLACE made active; resume it only if ours.""" + + if name not in self._held: + return + with self._lock: + newly_paused: bool = not ( + self._pause_maintained or name in self._paused or name in self._left_paused + ) + if newly_paused: + self._paused.append(name) + if newly_paused: + self._record( + connection=connection, + events=(self._own(event_type=MaintainedPauseEventType.PAUSED, name=name),), + ) + self._operate(connection=connection, name=name, operation=MaintainedOperation.SUSPEND) + + def release(self, *, connection: Any) -> None: + """Stop the heartbeat and resume what this build paused, upstream first.""" + + if self._heartbeat is not None: + self._heartbeat.stop() + self._report_heartbeat_failure(failure=self._heartbeat.failure) + order: list[str] = list(self._held) + resumed: list[str] = [] + failures: list[Exception] = [] + for name in sorted(self._paused, key=order.index): + try: + self._operate( + connection=connection, name=name, operation=MaintainedOperation.RESUME + ) + resumed.append(name) + except Exception as error: + failures.append(error) + self._record( + connection=connection, + events=tuple( + self._own(event_type=MaintainedPauseEventType.RESUMED, name=name) + for name in resumed + ), + ) + if failures: + raise failures[0] + + def _take_over(self, *, connection: Any, location: StateLocation) -> tuple[OpenHold, ...]: + """Resume expired holds on other models now; return the held models' holds to take over.""" + + rows: tuple[tuple[Any, ...], ...] = tuple( + self._adapter.query( + connection=connection, + sql=open_holds_sql(adapter=self._adapter, location=location), + limit=None, + ).rows + ) + held_keys: set[_RelationKey] = {_key(location) for location in self._held.values()} + holds: tuple[OpenHold, ...] = tuple( + hold for hold in open_holds(rows=rows) if hold.run_id != self._run_id + ) + live: tuple[OpenHold, ...] = tuple( + hold + for hold in holds + if hold.idle_seconds < self._grace_seconds and _key(hold.relation) in held_keys + ) + if live and not self._ignore_grace: + raise _held_elsewhere(holds=live) + expired: tuple[OpenHold, ...] = tuple( + hold for hold in holds if hold.idle_seconds >= self._grace_seconds + ) + elsewhere: tuple[OpenHold, ...] = tuple( + hold for hold in expired if _key(hold.relation) not in held_keys + ) + for hold in elsewhere: + self._resume_found(connection=connection, relation=hold.relation) + self._record( + connection=connection, + events=tuple( + self._event(event_type=MaintainedPauseEventType.RESUMED, hold=hold) + for hold in elsewhere + ), + ) + taken: tuple[OpenHold, ...] = (*live, *(h for h in expired if h not in elsewhere)) + self._report(holds=elsewhere, verb="Resumed") + self._report(holds=taken, verb="Took over") + return taken + + def _resume_found(self, *, connection: Any, relation: CompiledRelationLocation) -> None: + state: str | None = self._adapter.maintained_scheduling_state( + connection=connection, + database=relation.database, + schema=relation.schema, + name=relation.name, + ) + if state == SUSPENDED_STATE: + self._run( + connection=connection, + destination=self._destination(relation), + operation=MaintainedOperation.RESUME, + ) + + def _suspend_active( + self, + *, + connection: Any, + names: tuple[str, ...], + states: dict[str, str | None] | None = None, + ) -> None: + for name in names: + state: str | None = ( + states[name] + if states is not None + else self._state(connection=connection, name=name) + ) + if state == ACTIVE_STATE: + self._operate( + connection=connection, name=name, operation=MaintainedOperation.SUSPEND + ) + + def _state(self, *, connection: Any, name: str) -> str | None: + location: CompiledRelationLocation = self._held[name] + return self._adapter.maintained_scheduling_state( + connection=connection, + database=location.database, + schema=location.schema, + name=location.name, + ) + + def _operate(self, *, connection: Any, name: str, operation: MaintainedOperation) -> None: + self._run( + connection=connection, + destination=self._destination(self._held[name]), + operation=operation, + ) + + def _run(self, *, connection: Any, destination: str, operation: MaintainedOperation) -> None: + for statement in self._adapter.render_maintained_operation( + destination=destination, operation=operation + ): + self._adapter.execute(connection=connection, sql=statement) + + def _destination(self, location: CompiledRelationLocation) -> str: + return location.qualified_name or str( + self._adapter.render_qualified_name( + database=location.database, schema=location.schema, name=location.name + ) + ) + + def _record(self, *, connection: Any, events: tuple[MaintainedPauseEvent, ...]) -> None: + location: StateLocation | None = self._state_location + if not events or location is None: + return + self._adapter.execute( + connection=connection, + sql=insert_sql(adapter=self._adapter, location=location, events=events), + ) + + def _write_heartbeat(self, *, connection: Any, sequence: int) -> None: + self._record( + connection=connection, + events=( + MaintainedPauseEvent( + event_type=MaintainedPauseEventType.HEARTBEAT, + run_id=self._run_id, + recorded_by=self._run_id, + relation=None, + sequence=sequence, + ), + ), + ) + + def _own(self, *, event_type: MaintainedPauseEventType, name: str) -> MaintainedPauseEvent: + return MaintainedPauseEvent( + event_type=event_type, + run_id=self._run_id, + recorded_by=self._run_id, + relation=self._held[name], + ) + + def _event( + self, *, event_type: MaintainedPauseEventType, hold: OpenHold + ) -> MaintainedPauseEvent: + return MaintainedPauseEvent( + event_type=event_type, + run_id=hold.run_id, + recorded_by=self._run_id, + relation=hold.relation, + ) + + def _report(self, *, holds: tuple[OpenHold, ...], verb: str) -> None: + if self._on_progress is None: + return + for hold in holds: + self._on_progress( + f"{verb} maintained model {hold.relation.name}, paused by run {hold.run_id} " + f"whose heartbeat has been silent for {int(hold.idle_seconds)}s" + ) + + def _report_heartbeat_failure(self, *, failure: Exception | None) -> None: + if failure is not None: + self._warn(f"a maintained pause heartbeat failed: {failure}") + + def _warn(self, message: str) -> None: + if self._on_progress is not None: + self._on_progress(f"Warning: {message}") + + +def _key(location: CompiledRelationLocation) -> _RelationKey: + return (location.database, location.schema, location.name) + + +def _held_elsewhere(*, holds: tuple[OpenHold, ...]) -> AdapterUserError: + first: OpenHold = holds[0] + names: str = ", ".join(sorted({hold.relation.name for hold in holds})) + return AdapterUserError( + f"run {first.run_id} holds maintained models this build needs ({names}); " + f"its last heartbeat was {int(first.idle_seconds)}s ago", + code=HELD_ELSEWHERE_CODE, + help=( + "wait for that build to finish, or for its hold to pass [maintained] " + "heartbeat_grace; with allow_ignore_grace = true, " + "sqb build --ignore-maintained-grace takes over at once" + ), + ) diff --git a/src/sqlbuild/executor/maintained_pauses/constants.py b/src/sqlbuild/executor/maintained_pauses/constants.py new file mode 100644 index 0000000000..22fd73f5fa --- /dev/null +++ b/src/sqlbuild/executor/maintained_pauses/constants.py @@ -0,0 +1,49 @@ +"""Maintained pause table columns, timings and error codes.""" + +from sqlbuild._native import StateSqlValueType + +MAINTAINED_PAUSE_SCHEMA_VERSION: int = 1 +HEARTBEAT_SECONDS: float = 60.0 +ACTIVE_STATE: str = "ACTIVE" +SUSPENDED_STATE: str = "SUSPENDED" +HELD_ELSEWHERE_CODE: str = "A718" +IGNORE_GRACE_REFUSED_CODE: str = "A719" + +COLUMN_EVENT_ID: str = "event_id" +COLUMN_SCHEMA_VERSION: str = "schema_version" +COLUMN_EVENT_TYPE: str = "event_type" +COLUMN_RUN_ID: str = "run_id" +COLUMN_RELATION_DATABASE: str = "relation_database" +COLUMN_RELATION_SCHEMA: str = "relation_schema" +COLUMN_RELATION_NAME: str = "relation_name" +COLUMN_RECORDED_BY: str = "recorded_by" +COLUMN_RECORDED_AT: str = "recorded_at" + +MAINTAINED_PAUSE_COLUMNS: tuple[str, ...] = ( + COLUMN_EVENT_ID, + COLUMN_SCHEMA_VERSION, + COLUMN_EVENT_TYPE, + COLUMN_RUN_ID, + COLUMN_RELATION_DATABASE, + COLUMN_RELATION_SCHEMA, + COLUMN_RELATION_NAME, + COLUMN_RECORDED_BY, + COLUMN_RECORDED_AT, +) + +MAINTAINED_PAUSE_REQUIRED_COLUMNS: frozenset[str] = frozenset( + { + COLUMN_EVENT_ID, + COLUMN_SCHEMA_VERSION, + COLUMN_EVENT_TYPE, + COLUMN_RUN_ID, + COLUMN_RECORDED_BY, + COLUMN_RECORDED_AT, + } +) + +MAINTAINED_PAUSE_COLUMN_TYPES: dict[str, StateSqlValueType] = { + **{column: StateSqlValueType.STRING for column in MAINTAINED_PAUSE_COLUMNS}, + COLUMN_SCHEMA_VERSION: StateSqlValueType.INTEGER, + COLUMN_RECORDED_AT: StateSqlValueType.TEXT_TIMESTAMP, +} diff --git a/src/sqlbuild/executor/maintained_pauses/main/__init__.py b/src/sqlbuild/executor/maintained_pauses/main/__init__.py new file mode 100644 index 0000000000..36ef4cff68 --- /dev/null +++ b/src/sqlbuild/executor/maintained_pauses/main/__init__.py @@ -0,0 +1 @@ +"""Maintained pause entrypoints.""" diff --git a/src/sqlbuild/executor/maintained_pauses/main/hold_maintained_models.py b/src/sqlbuild/executor/maintained_pauses/main/hold_maintained_models.py new file mode 100644 index 0000000000..dc8045be4d --- /dev/null +++ b/src/sqlbuild/executor/maintained_pauses/main/hold_maintained_models.py @@ -0,0 +1,71 @@ +"""The hold a build keeps on the maintained models it touches, or None when it touches none.""" + +from __future__ import annotations + +from sqlbuild._native import CompiledRelationLocation, PlanOutput, TargetSettings +from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter +from sqlbuild.adapter.contract.exceptions import AdapterUserError +from sqlbuild.executor.build.models import BuildRuntimeParams +from sqlbuild.executor.maintained_pauses._helpers.scope import held_names, refreshed_for_readers +from sqlbuild.executor.maintained_pauses.classes.maintained_heartbeat import ( + MaintainedHeartbeat, +) +from sqlbuild.executor.maintained_pauses.classes.maintained_hold import MaintainedHold +from sqlbuild.executor.maintained_pauses.constants import ( + HEARTBEAT_SECONDS, + IGNORE_GRACE_REFUSED_CODE, +) +from sqlbuild.executor.maintained_pauses.types import StateLocation + + +def hold_maintained_models( + *, + plan: PlanOutput, + adapter: BaseAdapter, + runtime: BuildRuntimeParams, + connection_config: dict[str, object], + heartbeat_seconds: float = HEARTBEAT_SECONDS, +) -> MaintainedHold | None: + """The build's hold, unacquired; `--ignore-maintained-grace` needs `allow_ignore_grace`.""" + + if runtime.ignore_maintained_grace and not runtime.allow_ignore_maintained_grace: + raise AdapterUserError( + "--ignore-maintained-grace needs [maintained] allow_ignore_grace = true", + code=IGNORE_GRACE_REFUSED_CODE, + help="set allow_ignore_grace = true under [maintained] in sqlbuild_project.toml", + ) + names: tuple[str, ...] = held_names(plan=plan, pause_maintained=runtime.pause_maintained) + refreshed: frozenset[str] = refreshed_for_readers(plan=plan) + if not names and not refreshed: + return None + held: dict[str, CompiledRelationLocation] = {name: plan.model_locations[name] for name in names} + return MaintainedHold( + adapter=adapter, + held=held, + refreshed=refreshed, + state_location=_state_location(runtime=runtime, held=held) if held else None, + settings=( + runtime.run_id, + runtime.pause_maintained, + float(runtime.maintained_heartbeat_grace_seconds), + runtime.ignore_maintained_grace, + ), + heartbeat=lambda write: MaintainedHeartbeat( + adapter=adapter, + connection_config=connection_config, + write=write, + interval_seconds=heartbeat_seconds, + ), + ) + + +def _state_location( + *, runtime: BuildRuntimeParams, held: dict[str, CompiledRelationLocation] +) -> StateLocation: + """The target's schema, next to the runs table; the first held model's for a preserve schema.""" + + target: TargetSettings | None = runtime.target_config + if target is not None and target.schema is not None and not target.schema_preserved: + return (target.database, target.schema) + first: CompiledRelationLocation = next(iter(held.values())) + return (first.database, str(first.schema)) diff --git a/src/sqlbuild/executor/maintained_pauses/models.py b/src/sqlbuild/executor/maintained_pauses/models.py new file mode 100644 index 0000000000..fc51dd9b4e --- /dev/null +++ b/src/sqlbuild/executor/maintained_pauses/models.py @@ -0,0 +1,28 @@ +"""Records of the maintained pause lease.""" + +from __future__ import annotations + +from dataclasses import dataclass + +from sqlbuild._native import CompiledRelationLocation +from sqlbuild.executor.maintained_pauses.types import MaintainedPauseEventType + + +@dataclass(frozen=True) +class MaintainedPauseEvent: + """One row to append: a pause, a resume, or a heartbeat of the run holding models.""" + + event_type: MaintainedPauseEventType + run_id: str + recorded_by: str + relation: CompiledRelationLocation | None + sequence: int = 0 + + +@dataclass(frozen=True) +class OpenHold: + """A model a run paused and nobody resumed, with seconds since that run last wrote.""" + + run_id: str + relation: CompiledRelationLocation + idle_seconds: float diff --git a/src/sqlbuild/executor/maintained_pauses/types.py b/src/sqlbuild/executor/maintained_pauses/types.py new file mode 100644 index 0000000000..c199bdd358 --- /dev/null +++ b/src/sqlbuild/executor/maintained_pauses/types.py @@ -0,0 +1,15 @@ +"""Maintained pause event types.""" + +from __future__ import annotations + +from enum import StrEnum + +type StateLocation = tuple[str | None, str] + + +class MaintainedPauseEventType(StrEnum): + """What one row of the pause table records.""" + + PAUSED = "paused" + RESUMED = "resumed" + HEARTBEAT = "heartbeat" diff --git a/src/sqlbuild/executor/run/_helpers/execution/table_config.py b/src/sqlbuild/executor/run/_helpers/execution/table_config.py index 44cdf5e270..ceeee3e044 100644 --- a/src/sqlbuild/executor/run/_helpers/execution/table_config.py +++ b/src/sqlbuild/executor/run/_helpers/execution/table_config.py @@ -21,7 +21,7 @@ def model_table_config(entry: ModelPlanEntry) -> dict[str, object]: return { "table_type": entry.table_type, - "change_tracking": entry.change_tracking, + "change_tracking": bool(entry.maintained_readers), "physical_storage": entry.physical_storage, **model_output_config(entry), } diff --git a/src/sqlbuild/executor/run/_helpers/materializations/maintained.py b/src/sqlbuild/executor/run/_helpers/materializations/maintained.py index 4fabbd0b29..a1340b7578 100644 --- a/src/sqlbuild/executor/run/_helpers/materializations/maintained.py +++ b/src/sqlbuild/executor/run/_helpers/materializations/maintained.py @@ -9,8 +9,9 @@ from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.exceptions import AdapterUserError -from sqlbuild.adapter.contract.types import RelationType +from sqlbuild.adapter.contract.types import MaintainedOperation, RelationType from sqlbuild.adapter.relations.main.cached_relation_type import cached_relation_type +from sqlbuild.executor.maintained_pauses.classes.maintained_hold import MaintainedHold from sqlbuild.executor.run._helpers.materializations.view import execute_in_place_entry from sqlbuild.executor.run.models import ModelExecutionResult, ModelMaterializationContext @@ -18,13 +19,17 @@ def execute_maintained_entry(*, context: ModelMaterializationContext) -> ModelExecutionResult: - """Create, alter or keep one maintained model, then run its audits and hooks.""" + """Create, alter or keep one maintained model, refresh a kept one when asked, then audit it.""" def materialize(*, target_qualified: str, statement_recorder: StatementRecorder) -> None: entry: ModelPlanEntry = context.entry + hold: MaintainedHold | None = context.maintained_hold if entry.action == PlanAction.CREATE_MAINTAINED: _create(context=context, destination=target_qualified, recorder=statement_recorder) - elif entry.action == PlanAction.ALTER_MAINTAINED: + if hold is not None: + hold.after_create(connection=context.connection, name=entry.name) + return + if entry.action == PlanAction.ALTER_MAINTAINED: _run( adapter=context.adapter, connection=context.connection, @@ -35,6 +40,15 @@ def materialize(*, target_qualified: str, statement_recorder: StatementRecorder) ), recorder=statement_recorder, ) + if entry.refresh_on_build or (hold is not None and hold.refreshes(name=entry.name)): + _run( + adapter=context.adapter, + connection=context.connection, + statements=context.adapter.render_maintained_operation( + destination=target_qualified, operation=MaintainedOperation.REFRESH + ), + recorder=statement_recorder, + ) return execute_in_place_entry( context=context, diff --git a/src/sqlbuild/executor/run/_helpers/materializations/microbatch.py b/src/sqlbuild/executor/run/_helpers/materializations/microbatch.py index c0784aedea..d3389b7b36 100644 --- a/src/sqlbuild/executor/run/_helpers/materializations/microbatch.py +++ b/src/sqlbuild/executor/run/_helpers/materializations/microbatch.py @@ -3057,7 +3057,7 @@ def _apply_microbatch_dml( config={ "table_type": context.entry.table_type, "physical_storage": context.entry.physical_storage, - "change_tracking": context.entry.change_tracking, + "change_tracking": bool(context.entry.maintained_readers), }, statement_recorder=state.statement_recorder, ) diff --git a/src/sqlbuild/executor/run/main/_execute.py b/src/sqlbuild/executor/run/main/_execute.py index 9a8359f13d..1d8584e21b 100644 --- a/src/sqlbuild/executor/run/main/_execute.py +++ b/src/sqlbuild/executor/run/main/_execute.py @@ -249,7 +249,7 @@ def _staged_lifecycle( table_config={ "table_type": entry.table_type, "physical_storage": entry.physical_storage, - "change_tracking": entry.change_tracking, + "change_tracking": bool(entry.maintained_readers), }, statement_recorder=statement_recorder, ) diff --git a/src/sqlbuild/executor/run/models.py b/src/sqlbuild/executor/run/models.py index 674a5fba62..84bec90f9f 100644 --- a/src/sqlbuild/executor/run/models.py +++ b/src/sqlbuild/executor/run/models.py @@ -40,6 +40,7 @@ from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.auditing.models import AuditExecutionResult from sqlbuild.executor.custom.models import MaterializationResult +from sqlbuild.executor.maintained_pauses.classes.maintained_hold import MaintainedHold from sqlbuild.executor.python_nodes.classes.runtime_relation_guard import RuntimeRelationGuard from sqlbuild.executor.python_nodes.types import PythonIdentityRecorder from sqlbuild.executor.run.classes.fingerprint_writes import FingerprintWrites @@ -333,6 +334,7 @@ class ModelMaterializationContext: enforce_explicit_references: bool = False bound_views: BoundViewGuard = field(default_factory=BoundViewGuard) fingerprint_writes: FingerprintWrites | None = None + maintained_hold: MaintainedHold | None = None @dataclass(frozen=True, kw_only=True, init=False) diff --git a/tests/integration/src/sqlbuild/compiler/planner/main/_test_types.py b/tests/integration/src/sqlbuild/compiler/planner/main/_test_types.py index b4a9864101..905f2491f3 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/main/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/planner/main/_test_types.py @@ -223,4 +223,5 @@ class MaintainedRefreshPlanTestCase: class ChangeTrackingPlanTestCase: description: str select: tuple[str, ...] - expected_change_tracking: dict[str, bool] + expected_maintained_readers: dict[str, tuple[str, ...]] + expected_refresh_on_build: dict[str, bool] diff --git a/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py b/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py index 42b9eeb2fb..4a1985c027 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py @@ -882,7 +882,11 @@ def change_tracking_project_case() -> BuildExecutionPlanTestCase: model_configs={ "orders": {"materialized": "table"}, "customers": {"materialized": "table"}, - "daily_totals": {"materialized": "maintained", "refresh": {"lag": "1h"}}, + "daily_totals": { + "materialized": "maintained", + "refresh": {"lag": "1h"}, + "refresh_on_build": True, + }, }, model_queries={ "orders": "SELECT 1 AS order_id", diff --git a/tests/integration/src/sqlbuild/compiler/planner/main/test_maintained_refresh_plan.py b/tests/integration/src/sqlbuild/compiler/planner/main/test_maintained_refresh_plan.py index 331eef06fc..0f4394c07a 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/main/test_maintained_refresh_plan.py +++ b/tests/integration/src/sqlbuild/compiler/planner/main/test_maintained_refresh_plan.py @@ -120,17 +120,23 @@ def test_given_built_maintained_models_when_replanning_then_live_settings_and_re ChangeTrackingPlanTestCase( description="a table a maintained model reads is built with change tracking", select=(), - expected_change_tracking={"orders": True, "customers": False, "daily_totals": False}, + expected_maintained_readers={ + "orders": ("daily_totals",), + "customers": (), + "daily_totals": (), + }, + expected_refresh_on_build={"orders": False, "customers": False, "daily_totals": True}, ), ChangeTrackingPlanTestCase( description="the reader need not be selected for its upstream to be tracked", select=("orders", "customers"), - expected_change_tracking={"orders": True, "customers": False}, + expected_maintained_readers={"orders": ("daily_totals",), "customers": ()}, + expected_refresh_on_build={"orders": False, "customers": False}, ), ], ids=lambda case: case.description, ) -def test_given_table_read_by_maintained_model_when_planning_then_it_is_built_with_change_tracking( +def test_given_table_read_by_maintained_model_when_planning_then_it_names_its_maintained_readers( test_case: ChangeTrackingPlanTestCase, adapter: DuckDbAdapter, connection: Any ) -> None: plan: PlanOutput = build_execution_plan_from_kwargs( @@ -140,5 +146,11 @@ def test_given_table_read_by_maintained_model_when_planning_then_it_is_built_wit select=test_case.select, ) - tracked: dict[str, bool] = {entry.name: entry.change_tracking for entry in plan.model_entries} - assert tracked == test_case.expected_change_tracking + readers: dict[str, tuple[str, ...]] = { + entry.name: entry.maintained_readers for entry in plan.model_entries + } + assert readers == test_case.expected_maintained_readers + assert { + entry.name: entry.refresh_on_build for entry in plan.model_entries + } == test_case.expected_refresh_on_build + assert plan.model_locations["daily_totals"].name == "daily_totals" diff --git a/tests/integration/src/sqlbuild/executor/maintained_pauses/__init__.py b/tests/integration/src/sqlbuild/executor/maintained_pauses/__init__.py new file mode 100644 index 0000000000..11f5a11f26 --- /dev/null +++ b/tests/integration/src/sqlbuild/executor/maintained_pauses/__init__.py @@ -0,0 +1 @@ +"""Maintained model pause lease integration tests.""" diff --git a/tests/integration/src/sqlbuild/executor/maintained_pauses/_test_types.py b/tests/integration/src/sqlbuild/executor/maintained_pauses/_test_types.py new file mode 100644 index 0000000000..acc075ecf3 --- /dev/null +++ b/tests/integration/src/sqlbuild/executor/maintained_pauses/_test_types.py @@ -0,0 +1,52 @@ +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class PauseLeaseTestCase: + description: str + second_select: tuple[str, ...] + second_orders_query: str + second_totals_query: str + report_locations: dict[str, str] + refresh_on_build: bool + states_before_second: dict[str, str] + expected_operations: tuple[tuple[str, str], ...] + expected_events: tuple[tuple[str, str, str, str], ...] + + +@dataclass(frozen=True) +class HoldTakeoverTestCase: + description: str + grace_seconds: int + ignore_grace: bool + allow_ignore_grace: bool + expected_operations: tuple[tuple[str, str], ...] + expected_events: tuple[tuple[str, str, str, str], ...] + + +@dataclass(frozen=True) +class HoldRefusalTestCase: + description: str + grace_seconds: int + ignore_grace: bool + allow_ignore_grace: bool + expected_code: str + expected_message_fragment: str + + +@dataclass(frozen=True) +class FirstBuildTestCase: + description: str + pause_maintained: bool + expected_operations: tuple[tuple[str, str], ...] + expected_states: dict[str, str] + + +@dataclass(frozen=True) +class HeartbeatTestCase: + description: str + heartbeat_seconds: float + wait_seconds: float + expected_min_beats: int diff --git a/tests/integration/src/sqlbuild/executor/maintained_pauses/conftest.py b/tests/integration/src/sqlbuild/executor/maintained_pauses/conftest.py new file mode 100644 index 0000000000..1634ea5edb --- /dev/null +++ b/tests/integration/src/sqlbuild/executor/maintained_pauses/conftest.py @@ -0,0 +1,31 @@ +"""A file-backed DuckDB with a staging schema, so a heartbeat can open its own connection.""" + +from __future__ import annotations + +from collections.abc import Iterator +from pathlib import Path +from typing import Any + +import pytest + +from tests.integration.src.sqlbuild.executor.maintained_pauses.helpers import ( + PausingDuckDbAdapter, +) + + +@pytest.fixture +def database_path(tmp_path: Path) -> Path: + return tmp_path / "lease.duckdb" + + +@pytest.fixture +def adapter() -> PausingDuckDbAdapter: + return PausingDuckDbAdapter() + + +@pytest.fixture +def connection(adapter: PausingDuckDbAdapter, database_path: Path) -> Iterator[Any]: + opened: Any = adapter.connect({"database": str(database_path)}) + opened.execute("CREATE SCHEMA staging") + yield opened + adapter.close(opened) diff --git a/tests/integration/src/sqlbuild/executor/maintained_pauses/helpers.py b/tests/integration/src/sqlbuild/executor/maintained_pauses/helpers.py new file mode 100644 index 0000000000..ad776a6d5a --- /dev/null +++ b/tests/integration/src/sqlbuild/executor/maintained_pauses/helpers.py @@ -0,0 +1,170 @@ +"""A DuckDB adapter that tracks dynamic-table scheduling, and the plans the lease tests build.""" + +from __future__ import annotations + +from dataclasses import replace +from pathlib import Path +from typing import Any + +from sqlbuild._native import PlanOutput +from sqlbuild.adapter.contract.types import MaintainedOperation, TablePromotionMode +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from sqlbuild.executor.build.main._execute import execute_build_plan +from sqlbuild.executor.build.models import BuildExecutionResult, BuildRuntimeParams +from tests.integration.src.sqlbuild.compiler.planner.main._test_types import ( + BuildExecutionPlanTestCase, +) +from tests.integration.src.sqlbuild.compiler.planner.main.helpers import ( + build_execution_plan_from_kwargs, + build_project_from_test_case, +) + +type LeaseProject = BuildExecutionPlanTestCase + +_MAINTAINED: dict[str, object] = { + "materialized": "maintained", + "refresh": {"lag": "1h"}, + "refresh_warehouse": "transform_wh", +} +_CREATED_STATE: str = "ACTIVE" +TOTALS_QUERY: str = "SELECT count(*) AS orders FROM staging.orders" +_STATES: dict[MaintainedOperation, str] = { + MaintainedOperation.SUSPEND: "SUSPENDED", + MaintainedOperation.RESUME: "ACTIVE", +} + + +class PausingDuckDbAdapter(DuckDbAdapter): + """Builds maintained models as tables and keeps their scheduling state as Snowflake would.""" + + def __init__(self) -> None: + super().__init__() + self.states: dict[str, str] = {} + self.listed: dict[str, tuple[str, str]] = {} + self.operations: list[tuple[str, str]] = [] + + def session_warehouse(self, *, connection: Any) -> str | None: + del connection + return "SESSION_WH" + + def maintained_refresh_settings( + self, *, connection: Any, database: str | None, schema: str | None, name: str + ) -> tuple[str, str] | None: + del connection, database, schema + return self.listed.get(name) + + def maintained_scheduling_state( + self, *, connection: Any, database: str | None, schema: str | None, name: str + ) -> str | None: + del connection, database, schema + return self.states.get(name) + + def render_create_maintained( + self, *, destination: str, sql: str, target_lag: str, refresh_mode: str, warehouse: str + ) -> tuple[str, ...]: + del target_lag, refresh_mode, warehouse + name: str = destination.rsplit(".", 1)[-1] + self.states[name] = _CREATED_STATE + self.listed[name] = ("1 hour", "TRANSFORM_WH") + self.operations.append(("create", name)) + return (f"CREATE OR REPLACE TABLE {destination} AS {sql}",) + + def render_maintained_operation( + self, *, destination: str, operation: MaintainedOperation + ) -> tuple[str, ...]: + name: str = destination.rsplit(".", 1)[-1] + self.states[name] = _STATES.get(operation, self.states[name]) + self.operations.append((operation.value, name)) + return ("SELECT 1",) + + +def lease_project_case( + *, + orders_query: str, + report_locations: dict[str, str], + refresh_on_build: bool, + totals_query: str = TOTALS_QUERY, +) -> BuildExecutionPlanTestCase: + """Orders, daily totals maintained from them, and any report table located to read those.""" + + maintained: dict[str, object] = {**_MAINTAINED, "refresh_on_build": refresh_on_build} + return BuildExecutionPlanTestCase( + description="orders and the daily totals maintained from them", + setup_sql=(), + model_locations={"orders": "staging", "daily_totals": "staging", **report_locations}, + model_configs={ + "orders": {"materialized": "table"}, + "daily_totals": maintained, + "order_report": {"materialized": "table"}, + }, + model_queries={ + "orders": orders_query, + "daily_totals": totals_query, + "order_report": "SELECT orders FROM staging.daily_totals", + }, + model_deps={"daily_totals": ("orders",), "order_report": ("daily_totals",)}, + full_refresh=False, + expected_action={}, + expected_reason={}, + ) + + +def plan_lease_project( + *, + adapter: PausingDuckDbAdapter, + connection: Any, + case: BuildExecutionPlanTestCase, + select: tuple[str, ...] = (), +) -> PlanOutput: + """Plan the selected models against the warehouse as it stands.""" + + return build_execution_plan_from_kwargs( + project=build_project_from_test_case(case), + adapter=adapter, + connection=connection, + select=select, + ) + + +def build_lease_project( + *, + adapter: PausingDuckDbAdapter, + connection: Any, + case: BuildExecutionPlanTestCase, + runtime: BuildRuntimeParams, + database_path: Path, + select: tuple[str, ...] = (), +) -> BuildExecutionResult: + """Plan the selected models, then build them with `runtime`.""" + + return execute_build_plan( + plan=plan_lease_project(adapter=adapter, connection=connection, case=case, select=select), + adapter=adapter, + connection_config={"database": str(database_path)}, + connections=(connection,), + scheduler_connection=connection, + runtime=replace( + runtime, + promotion_mode=TablePromotionMode(adapter.default_table_promotion_mode()), + ), + ) + + +def pause_events(*, connection: Any) -> list[tuple[str, str, str, str]]: + """Each recorded pause event's type, holding run, recording run and model, sorted.""" + + rows: list[tuple[Any, ...]] = connection.execute( + "SELECT event_type, run_id, recorded_by, relation_name " + "FROM staging._sqlbuild_maintained_pauses WHERE event_type <> 'heartbeat' " + "ORDER BY 1, 2, 3, 4" + ).fetchall() + return [(str(row[0]), str(row[1]), str(row[2]), str(row[3])) for row in rows] + + +def pause_table_names(*, connection: Any) -> list[tuple[Any, ...]]: + """The pause state table, when a build created one in staging.""" + + return connection.execute( + "SELECT table_name FROM information_schema.tables " + "WHERE table_schema = 'staging' AND table_name = '_sqlbuild_maintained_pauses'" + ).fetchall() diff --git a/tests/integration/src/sqlbuild/executor/maintained_pauses/test_maintained_pause_lease.py b/tests/integration/src/sqlbuild/executor/maintained_pauses/test_maintained_pause_lease.py new file mode 100644 index 0000000000..c1763d9f17 --- /dev/null +++ b/tests/integration/src/sqlbuild/executor/maintained_pauses/test_maintained_pause_lease.py @@ -0,0 +1,440 @@ +"""A build pauses the maintained models it touches, records its hold, and resumes them at the end.""" + +from __future__ import annotations + +import time +from pathlib import Path +from typing import Any + +import pytest + +from sqlbuild.adapter.contract.exceptions import AdapterUserError +from sqlbuild.executor.build.models import BuildExecutionResult, BuildRuntimeParams +from sqlbuild.executor.build.types import BuildStatus +from sqlbuild.executor.maintained_pauses.classes.maintained_hold import MaintainedHold +from sqlbuild.executor.maintained_pauses.main.hold_maintained_models import ( + hold_maintained_models, +) +from tests.integration.src.sqlbuild.executor.maintained_pauses._test_types import ( + FirstBuildTestCase, + HeartbeatTestCase, + HoldRefusalTestCase, + HoldTakeoverTestCase, + PauseLeaseTestCase, +) +from tests.integration.src.sqlbuild.executor.maintained_pauses.helpers import ( + TOTALS_QUERY, + LeaseProject, + PausingDuckDbAdapter, + build_lease_project, + lease_project_case, + pause_events, + pause_table_names, + plan_lease_project, +) + +_ORDERS: str = "SELECT 1 AS order_id" +_MORE_ORDERS: str = "SELECT 1 AS order_id UNION ALL SELECT 2" +_REPORT: dict[str, str] = {"order_report": "staging"} +_FIRST_HOLD: tuple[tuple[str, str, str, str], ...] = ( + ("paused", "first_run", "first_run", "daily_totals"), + ("resumed", "first_run", "first_run", "daily_totals"), +) +_SECOND_HOLD: tuple[tuple[str, str, str, str], ...] = ( + ("paused", "second_run", "second_run", "daily_totals"), + ("resumed", "second_run", "second_run", "daily_totals"), +) +_TAKEN_OVER: tuple[tuple[str, str, str, str], ...] = ( + ("paused", "dead_run", "dead_run", "daily_totals"), + ("paused", "first_run", "first_run", "daily_totals"), + ("paused", "second_run", "second_run", "daily_totals"), + ("resumed", "dead_run", "second_run", "daily_totals"), + ("resumed", "first_run", "first_run", "daily_totals"), + ("resumed", "second_run", "second_run", "daily_totals"), +) + + +@pytest.mark.parametrize( + "test_case", + [ + FirstBuildTestCase( + description="a created model is paused for the build and resumed at its end", + pause_maintained=False, + expected_operations=( + ("create", "daily_totals"), + ("suspend", "daily_totals"), + ("resume", "daily_totals"), + ), + expected_states={"daily_totals": "ACTIVE"}, + ), + ], + ids=lambda case: case.description, +) +def test_given_first_build_when_creating_maintained_model_then_pauses_it_until_the_end( + test_case: FirstBuildTestCase, + adapter: PausingDuckDbAdapter, + connection: Any, + database_path: Path, +) -> None: + result: BuildExecutionResult = build_lease_project( + adapter=adapter, + connection=connection, + case=lease_project_case(orders_query=_ORDERS, report_locations={}, refresh_on_build=False), + runtime=BuildRuntimeParams(run_id="first_run", pause_maintained=test_case.pause_maintained), + database_path=database_path, + ) + + assert result.status == BuildStatus.SUCCESS + assert tuple(adapter.operations) == test_case.expected_operations + assert adapter.states == test_case.expected_states + assert pause_events(connection=connection) == sorted(_FIRST_HOLD) + + +@pytest.mark.parametrize( + "test_case", + [ + PauseLeaseTestCase( + description="a build of the maintained model alone writes nothing it reads", + second_select=("daily_totals",), + second_orders_query=_ORDERS, + second_totals_query=TOTALS_QUERY, + report_locations={}, + refresh_on_build=False, + states_before_second={}, + expected_operations=(), + expected_events=_FIRST_HOLD, + ), + PauseLeaseTestCase( + description="a rebuilt upstream pauses the model for the build", + second_select=(), + second_orders_query=_MORE_ORDERS, + second_totals_query=TOTALS_QUERY, + report_locations={}, + refresh_on_build=False, + states_before_second={}, + expected_operations=(("suspend", "daily_totals"), ("resume", "daily_totals")), + expected_events=(*_FIRST_HOLD, *_SECOND_HOLD), + ), + PauseLeaseTestCase( + description="a model a person suspended is neither paused nor resumed", + second_select=(), + second_orders_query=_MORE_ORDERS, + second_totals_query=TOTALS_QUERY, + report_locations={}, + refresh_on_build=False, + states_before_second={"daily_totals": "SUSPENDED"}, + expected_operations=(), + expected_events=_FIRST_HOLD, + ), + PauseLeaseTestCase( + description="a model a person suspended stays suspended when it is recreated", + second_select=(), + second_orders_query=_ORDERS, + second_totals_query="SELECT count(*) + 0 AS orders FROM staging.orders", + report_locations={}, + refresh_on_build=False, + states_before_second={"daily_totals": "SUSPENDED"}, + expected_operations=(("create", "daily_totals"), ("suspend", "daily_totals")), + expected_events=_FIRST_HOLD, + ), + PauseLeaseTestCase( + description="a table built from the model refreshes it first", + second_select=(), + second_orders_query=_MORE_ORDERS, + second_totals_query=TOTALS_QUERY, + report_locations=_REPORT, + refresh_on_build=False, + states_before_second={}, + expected_operations=( + ("suspend", "daily_totals"), + ("refresh", "daily_totals"), + ("resume", "daily_totals"), + ), + expected_events=(*_FIRST_HOLD, *_SECOND_HOLD), + ), + PauseLeaseTestCase( + description="refresh_on_build refreshes a kept model", + second_select=(), + second_orders_query=_MORE_ORDERS, + second_totals_query=TOTALS_QUERY, + report_locations={}, + refresh_on_build=True, + states_before_second={}, + expected_operations=( + ("suspend", "daily_totals"), + ("refresh", "daily_totals"), + ("resume", "daily_totals"), + ), + expected_events=(*_FIRST_HOLD, *_SECOND_HOLD), + ), + ], + ids=lambda case: case.description, +) +def test_given_built_maintained_model_when_building_again_then_pauses_only_what_it_touches( + test_case: PauseLeaseTestCase, + adapter: PausingDuckDbAdapter, + connection: Any, + database_path: Path, +) -> None: + _ = build_lease_project( + adapter=adapter, + connection=connection, + case=lease_project_case( + orders_query=_ORDERS, + report_locations=test_case.report_locations, + refresh_on_build=test_case.refresh_on_build, + ), + runtime=BuildRuntimeParams(run_id="first_run"), + database_path=database_path, + ) + adapter.operations.clear() + adapter.states.update(test_case.states_before_second) + + result: BuildExecutionResult = build_lease_project( + adapter=adapter, + connection=connection, + case=lease_project_case( + orders_query=test_case.second_orders_query, + report_locations=test_case.report_locations, + refresh_on_build=test_case.refresh_on_build, + totals_query=test_case.second_totals_query, + ), + runtime=BuildRuntimeParams(run_id="second_run"), + database_path=database_path, + select=test_case.second_select, + ) + + assert result.status == BuildStatus.SUCCESS + assert tuple(adapter.operations) == test_case.expected_operations + assert pause_events(connection=connection) == sorted(test_case.expected_events) + + +@pytest.mark.parametrize( + "test_case", + [ + FirstBuildTestCase( + description="a pause_maintained target leaves models suspended and records nothing", + pause_maintained=True, + expected_operations=(("refresh", "daily_totals"),), + expected_states={"daily_totals": "SUSPENDED"}, + ), + ], + ids=lambda case: case.description, +) +def test_given_pause_maintained_target_when_building_then_models_stay_suspended_and_refresh( + test_case: FirstBuildTestCase, + adapter: PausingDuckDbAdapter, + connection: Any, + database_path: Path, +) -> None: + paused: BuildRuntimeParams = BuildRuntimeParams( + run_id="first_run", pause_maintained=test_case.pause_maintained + ) + _ = build_lease_project( + adapter=adapter, + connection=connection, + case=lease_project_case(orders_query=_ORDERS, report_locations={}, refresh_on_build=False), + runtime=paused, + database_path=database_path, + ) + first: list[tuple[str, str]] = list(adapter.operations) + adapter.operations.clear() + + result: BuildExecutionResult = build_lease_project( + adapter=adapter, + connection=connection, + case=lease_project_case( + orders_query=_MORE_ORDERS, report_locations={}, refresh_on_build=False + ), + runtime=paused, + database_path=database_path, + ) + + assert result.status == BuildStatus.SUCCESS + assert first == [("create", "daily_totals"), ("suspend", "daily_totals")] + assert tuple(adapter.operations) == test_case.expected_operations + assert adapter.states == test_case.expected_states + assert pause_table_names(connection=connection) == [] + + +@pytest.mark.parametrize( + "test_case", + [ + HoldTakeoverTestCase( + description="a hold silent past the grace is taken over", + grace_seconds=0, + ignore_grace=False, + allow_ignore_grace=False, + expected_operations=(("resume", "daily_totals"),), + expected_events=_TAKEN_OVER, + ), + HoldTakeoverTestCase( + description="--ignore-maintained-grace takes over a live hold when the project allows", + grace_seconds=1800, + ignore_grace=True, + allow_ignore_grace=True, + expected_operations=(("resume", "daily_totals"),), + expected_events=_TAKEN_OVER, + ), + ], + ids=lambda case: case.description, +) +def test_given_hold_of_a_stopped_build_when_building_then_takes_it_over_and_resumes_at_the_end( + test_case: HoldTakeoverTestCase, + adapter: PausingDuckDbAdapter, + connection: Any, + database_path: Path, +) -> None: + _ = build_lease_project( + adapter=adapter, + connection=connection, + case=lease_project_case(orders_query=_ORDERS, report_locations={}, refresh_on_build=False), + runtime=BuildRuntimeParams(run_id="first_run"), + database_path=database_path, + ) + changed: LeaseProject = lease_project_case( + orders_query=_MORE_ORDERS, report_locations={}, refresh_on_build=False + ) + dead: MaintainedHold | None = hold_maintained_models( + plan=plan_lease_project(adapter=adapter, connection=connection, case=changed), + adapter=adapter, + runtime=BuildRuntimeParams(run_id="dead_run"), + connection_config={"database": str(database_path)}, + ) + assert dead is not None + dead.acquire(connection=connection, on_progress=None) + adapter.operations.clear() + + result: BuildExecutionResult = build_lease_project( + adapter=adapter, + connection=connection, + case=changed, + runtime=BuildRuntimeParams( + run_id="second_run", + maintained_heartbeat_grace_seconds=test_case.grace_seconds, + ignore_maintained_grace=test_case.ignore_grace, + allow_ignore_maintained_grace=test_case.allow_ignore_grace, + ), + database_path=database_path, + ) + + assert result.status == BuildStatus.SUCCESS + assert tuple(adapter.operations) == test_case.expected_operations + assert adapter.states == {"daily_totals": "ACTIVE"} + assert pause_events(connection=connection) == sorted(test_case.expected_events) + + +@pytest.mark.parametrize( + "test_case", + [ + HoldRefusalTestCase( + description="a live hold stops the build and names the run holding it", + grace_seconds=1800, + ignore_grace=False, + allow_ignore_grace=False, + expected_code="A718", + expected_message_fragment="dead_run", + ), + HoldRefusalTestCase( + description="--ignore-maintained-grace needs allow_ignore_grace", + grace_seconds=1800, + ignore_grace=True, + allow_ignore_grace=False, + expected_code="A719", + expected_message_fragment="allow_ignore_grace", + ), + ], + ids=lambda case: case.description, +) +def test_given_live_hold_when_building_then_stops_before_any_change( + test_case: HoldRefusalTestCase, + adapter: PausingDuckDbAdapter, + connection: Any, + database_path: Path, +) -> None: + _ = build_lease_project( + adapter=adapter, + connection=connection, + case=lease_project_case(orders_query=_ORDERS, report_locations={}, refresh_on_build=False), + runtime=BuildRuntimeParams(run_id="first_run"), + database_path=database_path, + ) + changed: LeaseProject = lease_project_case( + orders_query=_MORE_ORDERS, report_locations={}, refresh_on_build=False + ) + dead: MaintainedHold | None = hold_maintained_models( + plan=plan_lease_project(adapter=adapter, connection=connection, case=changed), + adapter=adapter, + runtime=BuildRuntimeParams(run_id="dead_run"), + connection_config={"database": str(database_path)}, + ) + assert dead is not None + dead.acquire(connection=connection, on_progress=None) + adapter.operations.clear() + + with pytest.raises(AdapterUserError) as raised: + _ = build_lease_project( + adapter=adapter, + connection=connection, + case=changed, + runtime=BuildRuntimeParams( + run_id="second_run", + maintained_heartbeat_grace_seconds=test_case.grace_seconds, + ignore_maintained_grace=test_case.ignore_grace, + allow_ignore_maintained_grace=test_case.allow_ignore_grace, + ), + database_path=database_path, + ) + + assert raised.value.code == test_case.expected_code + assert test_case.expected_message_fragment in str(raised.value) + assert adapter.operations == [] + assert connection.execute("SELECT count(*) FROM staging.orders").fetchall() == [(1,)] + + +@pytest.mark.parametrize( + "test_case", + [ + HeartbeatTestCase( + description="a hold beats on its own connection while the build runs", + heartbeat_seconds=0.05, + wait_seconds=0.4, + expected_min_beats=2, + ), + ], + ids=lambda case: case.description, +) +def test_given_held_models_when_build_runs_then_heartbeat_writes_on_its_own_connection( + test_case: HeartbeatTestCase, + adapter: PausingDuckDbAdapter, + connection: Any, + database_path: Path, +) -> None: + _ = build_lease_project( + adapter=adapter, + connection=connection, + case=lease_project_case(orders_query=_ORDERS, report_locations={}, refresh_on_build=False), + runtime=BuildRuntimeParams(run_id="first_run"), + database_path=database_path, + ) + changed: LeaseProject = lease_project_case( + orders_query=_MORE_ORDERS, report_locations={}, refresh_on_build=False + ) + hold: MaintainedHold | None = hold_maintained_models( + plan=plan_lease_project(adapter=adapter, connection=connection, case=changed), + adapter=adapter, + runtime=BuildRuntimeParams(run_id="beating_run"), + connection_config={"database": str(database_path)}, + heartbeat_seconds=test_case.heartbeat_seconds, + ) + assert hold is not None + + hold.acquire(connection=connection, on_progress=None) + time.sleep(test_case.wait_seconds) + hold.release(connection=connection) + + beats: list[tuple[Any, ...]] = connection.execute( + "SELECT count(*) FROM staging._sqlbuild_maintained_pauses " + "WHERE event_type = 'heartbeat' AND run_id = 'beating_run'" + ).fetchall() + assert beats[0][0] >= test_case.expected_min_beats diff --git a/tests/integration/src/sqlbuild/executor/run/maintained/_test_types.py b/tests/integration/src/sqlbuild/executor/run/maintained/_test_types.py index 9f2d57e13a..66a0b19f55 100644 --- a/tests/integration/src/sqlbuild/executor/run/maintained/_test_types.py +++ b/tests/integration/src/sqlbuild/executor/run/maintained/_test_types.py @@ -30,7 +30,7 @@ class MaintainedFailureTestCase: @dataclass(frozen=True) class ChangeTrackingTableTestCase: description: str - change_tracking: bool + maintained_readers: tuple[str, ...] promotion_mode: TablePromotionMode declared_columns: tuple[tuple[str, str], ...] expected_tracking: tuple[bool, ...] diff --git a/tests/integration/src/sqlbuild/executor/run/maintained/test_change_tracking.py b/tests/integration/src/sqlbuild/executor/run/maintained/test_change_tracking.py index 167ba289a4..4321614ce8 100644 --- a/tests/integration/src/sqlbuild/executor/run/maintained/test_change_tracking.py +++ b/tests/integration/src/sqlbuild/executor/run/maintained/test_change_tracking.py @@ -27,21 +27,21 @@ [ ChangeTrackingTableTestCase( description="a staged build tracks changes on the stage it swaps in", - change_tracking=True, + maintained_readers=("daily_totals",), promotion_mode=TablePromotionMode.STAGED, declared_columns=(), expected_tracking=(True,), ), ChangeTrackingTableTestCase( description="type enforcement rebuilds the stage with change tracking too", - change_tracking=True, + maintained_readers=("daily_totals",), promotion_mode=TablePromotionMode.STAGED, declared_columns=(("order_id", "BIGINT"),), expected_tracking=(True, True), ), ChangeTrackingTableTestCase( description="a table no maintained model reads is created as before", - change_tracking=False, + maintained_readers=(), promotion_mode=TablePromotionMode.STAGED, declared_columns=(("order_id", "BIGINT"),), expected_tracking=(False, False), @@ -61,7 +61,7 @@ def test_given_table_read_by_maintained_model_when_building_then_every_create_tr target_name="orders", type_enforcement=bool(test_case.declared_columns), ), - change_tracking=test_case.change_tracking, + maintained_readers=test_case.maintained_readers, ) result: ModelExecutionResult = execute_table_entry( diff --git a/tests/unit/src/sqlbuild/adapters/snowflake/_test_types.py b/tests/unit/src/sqlbuild/adapters/snowflake/_test_types.py index 03d54e874b..e768378233 100644 --- a/tests/unit/src/sqlbuild/adapters/snowflake/_test_types.py +++ b/tests/unit/src/sqlbuild/adapters/snowflake/_test_types.py @@ -491,3 +491,10 @@ class SnowflakeChangeTrackingTestCase: description: str config: dict[str, object] expected_sql: str + + +@dataclass(frozen=True) +class SnowflakeMaintainedStateTestCase: + description: str + rows: tuple[tuple[str, str], ...] + expected_state: str | None diff --git a/tests/unit/src/sqlbuild/adapters/snowflake/test_maintained_refresh_settings.py b/tests/unit/src/sqlbuild/adapters/snowflake/test_maintained_refresh_settings.py index 9d8721ef6e..6af8a6f118 100644 --- a/tests/unit/src/sqlbuild/adapters/snowflake/test_maintained_refresh_settings.py +++ b/tests/unit/src/sqlbuild/adapters/snowflake/test_maintained_refresh_settings.py @@ -10,9 +10,11 @@ SnowflakeMaintainedCreateTestCase, SnowflakeMaintainedOperationTestCase, SnowflakeMaintainedRefreshTestCase, + SnowflakeMaintainedStateTestCase, ) _SHOW_DYNAMIC_TABLES_COLUMNS: tuple[str, ...] = ("name", "target_lag", "warehouse") +_SCHEDULING_COLUMNS: tuple[str, ...] = ("name", "scheduling_state") @pytest.mark.parametrize( @@ -56,6 +58,37 @@ def test_given_dynamic_table_listing_when_reading_refresh_settings_then_reads_on ) +@pytest.mark.parametrize( + "test_case", + [ + SnowflakeMaintainedStateTestCase( + description="a suspended dynamic table reads as suspended", + rows=(("DAILY_TOTALS", "SUSPENDED"),), + expected_state="SUSPENDED", + ), + SnowflakeMaintainedStateTestCase( + description="a neighbour matched by the LIKE wildcard is not the model", + rows=(("DAILY_TOTALS_ARCHIVE", "ACTIVE"),), + expected_state=None, + ), + ], + ids=lambda case: case.description, +) +def test_given_dynamic_table_listing_when_reading_scheduling_state_then_reads_its_row( + test_case: SnowflakeMaintainedStateTestCase, monkeypatch: pytest.MonkeyPatch +) -> None: + adapter: SnowflakeAdapter = SnowflakeAdapter() + connection: MetadataConnection = MetadataConnection(test_case.rows, _SCHEDULING_COLUMNS) + monkeypatch.setattr(adapter, "execute", execute_through_cursor) + + state: str | None = adapter.maintained_scheduling_state( + connection=connection, database="ANALYTICS", schema="MARTS", name="daily_totals" + ) + + assert state == test_case.expected_state + assert connection.executed[0].startswith("SHOW DYNAMIC TABLES LIKE 'DAILY_TOTALS'") + + @pytest.mark.parametrize( "test_case", [ From ee400d2551cfcde4372e5d3b63fd3fbdda960ea0 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 19 Nov 2026 01:19:31 +0000 Subject: [PATCH 09/17] feat(plan): report kept maintained models that are paused Work in progress. sqb plan now says "paused: scheduled refreshes are suspended" under a kept maintained model that the warehouse lists as suspended, and plan JSON marks it with "paused": true; the plan page shows the same line. The plan never resumes anything. The live read that already decides whether a kept model's settings drifted now also returns its scheduling state: the optional adapter method maintained_listing returns the lag, refresh warehouse and scheduling state from one SHOW DYNAMIC TABLES, replacing the two methods it merges. The pause lease reads the same listing. Validated: the planner integration test (a suspended kept model is reported, active ones are not), plan text in Rust and in the UI, the Snowflake listing unit tests, and the executor, planner and adapter suites. --- .../src/cli_text/_helpers/plan_sections.rs | 3 ++ crates/sqlbuild-core/src/cli_text/models.rs | 2 + .../src/cli_text/tests/test_cli_text.rs | 11 ++++ .../_helpers/planner_records/constants.rs | 1 + .../src/live_reads/models.rs | 2 +- .../src/plan_output/_helpers/plan_json.rs | 3 ++ .../src/plan_output/_helpers/plan_text.rs | 1 + .../_helpers/outputs/maintained.rs | 32 +++++++---- .../src/planner_warehouse/models.rs | 2 +- src/sqlbuild/_native.pyi | 2 + .../contract/classes/strict_adapter.py | 15 ++---- src/sqlbuild/adapter/contract/types.py | 10 +++- .../snowflake/classes/snowflake_adapter.py | 23 ++++---- .../classes/maintained_hold.py | 32 ++++++----- .../_helpers/materializations/maintained.py | 4 +- .../compiler/planner/main/_test_types.py | 4 +- .../sqlbuild/compiler/planner/main/helpers.py | 17 +++--- .../main/test_maintained_refresh_plan.py | 19 +++++-- .../executor/maintained_pauses/helpers.py | 24 ++++----- .../executor/run/maintained/_test_types.py | 4 +- .../executor/run/maintained/helpers.py | 9 ++-- .../maintained/test_maintained_execution.py | 3 +- .../strict_adapter/test_strict_adapter.py | 2 +- .../adapters/snowflake/_test_types.py | 21 ++++---- ..._settings.py => test_maintained_models.py} | 54 +++++-------------- ui/src/lib/api/types/operate.ts | 2 + ui/src/lib/run/plan-text.test.ts | 6 +++ ui/src/lib/run/plan-text.ts | 2 +- 28 files changed, 175 insertions(+), 135 deletions(-) rename tests/unit/src/sqlbuild/adapters/snowflake/{test_maintained_refresh_settings.py => test_maintained_models.py} (80%) diff --git a/crates/sqlbuild-core/src/cli_text/_helpers/plan_sections.rs b/crates/sqlbuild-core/src/cli_text/_helpers/plan_sections.rs index 3d68442d25..03b6cb37ea 100644 --- a/crates/sqlbuild-core/src/cli_text/_helpers/plan_sections.rs +++ b/crates/sqlbuild-core/src/cli_text/_helpers/plan_sections.rs @@ -633,6 +633,9 @@ pub(crate) fn routine_model_lines( if let Some(upstream) = &model.reinitialised_by { lines.push(format!(" recomputes in full: {upstream} is rebuilt")); } + if model.paused { + lines.push(" paused: scheduled refreshes are suspended".to_owned()); + } lines }) } diff --git a/crates/sqlbuild-core/src/cli_text/models.rs b/crates/sqlbuild-core/src/cli_text/models.rs index 881e5b4ace..0274627d1e 100644 --- a/crates/sqlbuild-core/src/cli_text/models.rs +++ b/crates/sqlbuild-core/src/cli_text/models.rs @@ -324,6 +324,8 @@ pub struct ModelPlanText { pub query_diff: Option>, /// The rebuilt upstream that makes this kept maintained model recompute in full. pub reinitialised_by: Option, + /// Whether the warehouse lists this kept maintained model as suspended. + pub paused: bool, } /// One planned function. diff --git a/crates/sqlbuild-core/src/cli_text/tests/test_cli_text.rs b/crates/sqlbuild-core/src/cli_text/tests/test_cli_text.rs index f5ce8beea5..51f8d9a1e0 100644 --- a/crates/sqlbuild-core/src/cli_text/tests/test_cli_text.rs +++ b/crates/sqlbuild-core/src/cli_text/tests/test_cli_text.rs @@ -257,6 +257,17 @@ fn given_plan_report_when_rendering_then_groups_models_by_reason() { expected_text: "Plan ready 2 selected\n\nQuery changed (1)\n\u{2514}\u{2500}\u{2500} orders full rebuild\n \u{2514}\u{2500}\u{2500} policy replay_on_change=full\n\nModels (1)\n\u{2514}\u{2500}\u{2500} daily_totals maintained\n \u{2514}\u{2500}\u{2500} recomputes in full orders is rebuilt", expected_completion: "\u{2713} Plan complete 2 selected", }, + PlanTextTestCase { + description: "a kept maintained model that is suspended is reported as paused", + report: plan_report(vec![ModelPlanText { + action: "keep_maintained".to_owned(), + materialization_label: "maintained".to_owned(), + paused: true, + ..table_model("daily_totals", "no_change") + }]), + expected_text: "Plan ready 1 selected\n\nModels (1)\n\u{2514}\u{2500}\u{2500} daily_totals maintained\n \u{2514}\u{2500}\u{2500} paused scheduled refreshes are suspended", + expected_completion: "\u{2713} Plan complete 1 selected", + }, PlanTextTestCase { description: "every active model is a full refresh", report: plan_report(vec![ diff --git a/crates/sqlbuild-python-base/src/records/_helpers/planner_records/constants.rs b/crates/sqlbuild-python-base/src/records/_helpers/planner_records/constants.rs index 469a8f2610..a5bb652b75 100644 --- a/crates/sqlbuild-python-base/src/records/_helpers/planner_records/constants.rs +++ b/crates/sqlbuild-python-base/src/records/_helpers/planner_records/constants.rs @@ -672,6 +672,7 @@ const R_MODEL_PLAN_ENTRY: RecordSpec = RecordSpec { FieldSpec::new("refresh_warehouse", FieldDefault::None), FieldSpec::new("refresh_on_build", FieldDefault::False), FieldSpec::new("maintained_readers", FieldDefault::EmptyTuple), + FieldSpec::new("maintained_paused", FieldDefault::False), ], }; const R_CLONE_SOURCE_PLAN_ENTRY: RecordSpec = RecordSpec { diff --git a/crates/sqlbuild-python-planner/src/live_reads/models.rs b/crates/sqlbuild-python-planner/src/live_reads/models.rs index df284dd646..3841b31896 100644 --- a/crates/sqlbuild-python-planner/src/live_reads/models.rs +++ b/crates/sqlbuild-python-planner/src/live_reads/models.rs @@ -439,7 +439,7 @@ impl WarehouseReads for LiveReads { vec![("destination", destination)], ), AdapterRead::MaintainedRefresh([database, schema, name]) => ( - "maintained_refresh_settings", + "maintained_listing", vec![("database", database), ("schema", schema), ("name", name)], ), }; diff --git a/crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_json.rs b/crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_json.rs index a0cd5faf9e..d92a09036f 100644 --- a/crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_json.rs +++ b/crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_json.rs @@ -234,6 +234,9 @@ pub(crate) fn model_json<'py>(entry: &Bound<'py, PyAny>) -> PyResult = native_field(&native_field(entry, "destination")?, "qualified_name")?; if !qualified.is_none() { diff --git a/crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_text.rs b/crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_text.rs index e1d71c4f68..e67e1c9a06 100644 --- a/crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_text.rs +++ b/crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_text.rs @@ -251,6 +251,7 @@ fn model_text(entry: &Bound<'_, PyAny>) -> PyResult { config_diff: coloured(model_config_diff(entry, &reason)?), query_diff: coloured(model_query_diff(entry, &reason)?), reinitialised_by: optional_text(&native_field(entry, "reinitialised_by")?)?, + paused: native_field(entry, "maintained_paused")?.is_truthy()?, name, action, reason, diff --git a/crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/maintained.rs b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/maintained.rs index d724ff0f87..0f2d167ff6 100644 --- a/crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/maintained.rs +++ b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/maintained.rs @@ -31,6 +31,8 @@ use std::collections::HashMap; const MAINTAINED: &str = "maintained"; const KEEP_MAINTAINED: &str = "keep_maintained"; +/// The scheduling state Snowflake lists for a dynamic table whose refreshes are paused. +const SUSPENDED: &str = "SUSPENDED"; /// The refresh mode of a maintained model that declares none, so Snowflake never picks one quietly. const DEFAULT_REFRESH_MODE: &str = "incremental"; /// Actions that recreate the relation, which reinitialises every dynamic table reading it. @@ -164,26 +166,34 @@ fn with_refresh_settings<'py>( ), ( "refresh_on_build", - PyBool::new(py, config_value(py, &values, "refresh_on_build")?.is_truthy()?) - .to_owned() - .into_any(), + PyBool::new( + py, + config_value(py, &values, "refresh_on_build")?.is_truthy()?, + ) + .to_owned() + .into_any(), ), ]; if text(&native_field(&entry, "action")?)? == KEEP_MAINTAINED { - if let Some((action, reason)) = live_action(&entry, &desired, reads)? { + let (action, paused) = live_action(&entry, &desired, reads)?; + if let Some((action, reason)) = action { changes.push(("action", enum_member(py, RecordId::PlanAction, action)?)); changes.push(("reason", enum_member(py, RecordId::PlanReason, reason)?)); } + changes.push(( + "maintained_paused", + PyBool::new(py, paused).to_owned().into_any(), + )); } replaced(&entry, &changes) } -/// The action and reason replacing a kept model's, or `None` when its listed settings match or are unknown. +/// The action and reason replacing a kept model's, `None` when its listing matches or is unknown, and whether it is suspended. fn live_action( entry: &Bound<'_, PyAny>, desired: &MaintainedSettings, reads: &Bound<'_, PlannerReads>, -) -> PyResult> { +) -> PyResult<(Option<(&'static str, &'static str)>, bool)> { let destination = native_field(entry, "destination")?; let read = reads.get().reads().adapter_read( entry.py(), @@ -194,18 +204,20 @@ fn live_action( ]), )?; let listed = match read { - Read::Unknown(_) => return Ok(None), + Read::Unknown(_) => return Ok((None, false)), Read::Known(listed) => listed, }; if listed.is_none() { - return Ok(Some(("CREATE_MAINTAINED", "FIRST_RUN"))); + return Ok((Some(("CREATE_MAINTAINED", "FIRST_RUN")), false)); } - let (target_lag, warehouse): (String, String) = listed.extract()?; + let (target_lag, warehouse, state): (String, String, String) = listed.extract()?; let live = LiveDynamicTable { target_lag, warehouse, }; - Ok(maintained_settings_drift(desired, &live).then_some(("ALTER_MAINTAINED", "CONFIG_CHANGED"))) + let action = + maintained_settings_drift(desired, &live).then_some(("ALTER_MAINTAINED", "CONFIG_CHANGED")); + Ok((action, state == SUSPENDED)) } /// The model's declared lag and refresh warehouse, or `None` when its lag does not read. diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/models.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/models.rs index 44098a75b8..b09c27b27d 100644 --- a/crates/sqlbuild-python-planner/src/planner_warehouse/models.rs +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/models.rs @@ -757,7 +757,7 @@ impl WarehouseReads for SyncedReads { return Ok(unknown(self, "column rename support")); } AdapterRead::MaintainedRefresh(_) => { - return Ok(unknown(self, "maintained refresh settings")); + return Ok(unknown(self, "maintained model listings")); } }; let unavailable = diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 5b45ec48d0..eced41585a 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -14128,6 +14128,7 @@ class ModelPlanEntry(NativeRecord): refresh_warehouse: str | None refresh_on_build: bool maintained_readers: tuple[str, ...] + maintained_paused: bool def __init__( self, key: CompiledObjectKey, @@ -14223,6 +14224,7 @@ class ModelPlanEntry(NativeRecord): refresh_warehouse: str | None = ..., refresh_on_build: bool = ..., maintained_readers: tuple[str, ...] = ..., + maintained_paused: bool = ..., ) -> None: ... class CloneSourcePlanEntry(NativeRecord): diff --git a/src/sqlbuild/adapter/contract/classes/strict_adapter.py b/src/sqlbuild/adapter/contract/classes/strict_adapter.py index e0695c23d0..06cc06a89e 100644 --- a/src/sqlbuild/adapter/contract/classes/strict_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/strict_adapter.py @@ -33,6 +33,7 @@ from sqlbuild.adapter.contract.types import ( FrameworkType, LoaderLogicalType, + MaintainedListing, MaintainedOperation, PromotionStrategy, StatementSizeLimit, @@ -359,18 +360,10 @@ def _maintained_unsupported(self, destination: str) -> AdapterUserError: help="materialized maintained needs an adapter whose warehouse refreshes it", ) - def maintained_refresh_settings( + def maintained_listing( self, *, connection: Any, database: str | None, schema: str | None, name: str - ) -> tuple[str, str] | None: - """Optional: a maintained model's listed target lag and refresh warehouse, or None.""" - - del connection, database, schema, name - return None - - def maintained_scheduling_state( - self, *, connection: Any, database: str | None, schema: str | None, name: str - ) -> str | None: - """Optional: a maintained model's `ACTIVE` or `SUSPENDED` state, or None when unlisted.""" + ) -> MaintainedListing | None: + """Optional: how the warehouse lists a maintained model, or None when it lists none.""" del connection, database, schema, name return None diff --git a/src/sqlbuild/adapter/contract/types.py b/src/sqlbuild/adapter/contract/types.py index 843ae80ca9..cf88c3cb4e 100644 --- a/src/sqlbuild/adapter/contract/types.py +++ b/src/sqlbuild/adapter/contract/types.py @@ -4,7 +4,7 @@ from collections.abc import Callable from enum import StrEnum -from typing import Literal, Protocol +from typing import Literal, NamedTuple, Protocol from sqlbuild._native import InferredNullability @@ -172,3 +172,11 @@ class HistoricalSnapshotInsertStyle(StrEnum): WITH_INSERT = "with_insert" INSERT_WITH = "insert_with" TSQL = "tsql" + + +class MaintainedListing(NamedTuple): + """A maintained model as the warehouse lists it: lag, refresh warehouse and schedule state.""" + + target_lag: str + warehouse: str + scheduling_state: str diff --git a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py index 02060f65cc..00f720a399 100644 --- a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py +++ b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py @@ -99,6 +99,7 @@ HistoricalSnapshotCloseStyle, HistoricalSnapshotInsertStyle, LoaderLogicalType, + MaintainedListing, MaintainedOperation, MigrationTransfer, PromotionStrategy, @@ -3716,25 +3717,21 @@ def _warehouse_name(self, warehouse: str) -> str: return warehouse return '"' + warehouse.replace('"', '""') + '"' - def maintained_refresh_settings( + def maintained_listing( self, *, connection: Any, database: str | None, schema: str | None, name: str - ) -> tuple[str, str] | None: - """The dynamic table's target lag and warehouse from SHOW DYNAMIC TABLES, or None.""" + ) -> MaintainedListing | None: + """The dynamic table's lag, warehouse and scheduling state from SHOW DYNAMIC TABLES.""" row: dict[str, object] | None = self._listed_dynamic_table( connection=connection, database=database, schema=schema, name=name ) - return None if row is None else (str(row.get("target_lag")), str(row.get("warehouse"))) - - def maintained_scheduling_state( - self, *, connection: Any, database: str | None, schema: str | None, name: str - ) -> str | None: - """The dynamic table's scheduling state from SHOW DYNAMIC TABLES, or None.""" - - row: dict[str, object] | None = self._listed_dynamic_table( - connection=connection, database=database, schema=schema, name=name + if row is None: + return None + return MaintainedListing( + target_lag=str(row.get("target_lag")), + warehouse=str(row.get("warehouse")), + scheduling_state=str(row.get("scheduling_state")), ) - return None if row is None else str(row.get("scheduling_state")) def _listed_dynamic_table( self, *, connection: Any, database: str | None, schema: str | None, name: str diff --git a/src/sqlbuild/executor/maintained_pauses/classes/maintained_hold.py b/src/sqlbuild/executor/maintained_pauses/classes/maintained_hold.py index 64b9fb2117..0cbedc10f9 100644 --- a/src/sqlbuild/executor/maintained_pauses/classes/maintained_hold.py +++ b/src/sqlbuild/executor/maintained_pauses/classes/maintained_hold.py @@ -10,7 +10,7 @@ from sqlbuild._native import CompiledRelationLocation from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.exceptions import AdapterUserError -from sqlbuild.adapter.contract.types import MaintainedOperation +from sqlbuild.adapter.contract.types import MaintainedListing, MaintainedOperation from sqlbuild.executor.maintained_pauses._helpers.sql import ( insert_sql, open_holds, @@ -215,13 +215,9 @@ def _take_over(self, *, connection: Any, location: StateLocation) -> tuple[OpenH return taken def _resume_found(self, *, connection: Any, relation: CompiledRelationLocation) -> None: - state: str | None = self._adapter.maintained_scheduling_state( - connection=connection, - database=relation.database, - schema=relation.schema, - name=relation.name, - ) - if state == SUSPENDED_STATE: + if _scheduling_state(adapter=self._adapter, connection=connection, relation=relation) == ( + SUSPENDED_STATE + ): self._run( connection=connection, destination=self._destination(relation), @@ -247,12 +243,8 @@ def _suspend_active( ) def _state(self, *, connection: Any, name: str) -> str | None: - location: CompiledRelationLocation = self._held[name] - return self._adapter.maintained_scheduling_state( - connection=connection, - database=location.database, - schema=location.schema, - name=location.name, + return _scheduling_state( + adapter=self._adapter, connection=connection, relation=self._held[name] ) def _operate(self, *, connection: Any, name: str, operation: MaintainedOperation) -> None: @@ -334,6 +326,18 @@ def _warn(self, message: str) -> None: self._on_progress(f"Warning: {message}") +def _scheduling_state( + *, adapter: BaseAdapter, connection: Any, relation: CompiledRelationLocation +) -> str | None: + listing: MaintainedListing | None = adapter.maintained_listing( + connection=connection, + database=relation.database, + schema=relation.schema, + name=relation.name, + ) + return None if listing is None else listing.scheduling_state + + def _key(location: CompiledRelationLocation) -> _RelationKey: return (location.database, location.schema, location.name) diff --git a/src/sqlbuild/executor/run/_helpers/materializations/maintained.py b/src/sqlbuild/executor/run/_helpers/materializations/maintained.py index a1340b7578..7f67470325 100644 --- a/src/sqlbuild/executor/run/_helpers/materializations/maintained.py +++ b/src/sqlbuild/executor/run/_helpers/materializations/maintained.py @@ -9,7 +9,7 @@ from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.exceptions import AdapterUserError -from sqlbuild.adapter.contract.types import MaintainedOperation, RelationType +from sqlbuild.adapter.contract.types import MaintainedListing, MaintainedOperation, RelationType from sqlbuild.adapter.relations.main.cached_relation_type import cached_relation_type from sqlbuild.executor.maintained_pauses.classes.maintained_hold import MaintainedHold from sqlbuild.executor.run._helpers.materializations.view import execute_in_place_entry @@ -117,7 +117,7 @@ def _former_relation( ) if existing is None: return None - listed: tuple[str, str] | None = adapter.maintained_refresh_settings( + listed: MaintainedListing | None = adapter.maintained_listing( connection=connection, database=location.database, schema=location.schema, diff --git a/tests/integration/src/sqlbuild/compiler/planner/main/_test_types.py b/tests/integration/src/sqlbuild/compiler/planner/main/_test_types.py index 905f2491f3..ac3e1abfbe 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/main/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/planner/main/_test_types.py @@ -7,6 +7,7 @@ PlanReason, WarningSeverity, ) +from sqlbuild.adapter.contract.types import MaintainedListing from sqlbuild.compiler.compile.types import FunctionLanguage @@ -210,13 +211,14 @@ class SyncedScanTestCase: @dataclass(frozen=True) class MaintainedRefreshPlanTestCase: description: str - listed_settings: tuple[str, str] | None + listed_settings: MaintainedListing | None upstream_query: str expected_upstream_action: PlanAction expected_action: PlanAction expected_reason: PlanReason expected_reinitialised_by: str | None expected_read_names: tuple[str, ...] + expected_paused: bool = False @dataclass(frozen=True) diff --git a/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py b/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py index 4a1985c027..178ee9fae0 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py @@ -56,7 +56,12 @@ run_compile_pipeline, write_fingerprint, ) -from sqlbuild.adapter.contract.models import ColumnInfo, RelationInfo, RelationReadProbe +from sqlbuild.adapter.contract.models import ( + ColumnInfo, + RelationInfo, + RelationReadProbe, +) +from sqlbuild.adapter.contract.types import MaintainedListing from sqlbuild.adapter.relations.classes.inspection_catalog import InspectionCatalog from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.types import ( @@ -838,16 +843,16 @@ def planned_entries(result: CompilePipelineResult) -> list[tuple[str, str]]: class ListingDuckDbAdapter(DuckDbAdapter): - """DuckDB that lists one dynamic table's refresh settings, as Snowflake's SHOW would.""" + """DuckDB that lists every dynamic table the same way, as Snowflake's SHOW would.""" - def __init__(self, listed: tuple[str, str] | None) -> None: + def __init__(self, listed: MaintainedListing | None) -> None: super().__init__() - self.listed: tuple[str, str] | None = listed + self.listed: MaintainedListing | None = listed self.read_names: list[str] = [] - def maintained_refresh_settings( + def maintained_listing( self, *, connection: Any, database: str | None, schema: str | None, name: str - ) -> tuple[str, str] | None: + ) -> MaintainedListing | None: del connection, database, schema self.read_names.append(name) return self.listed diff --git a/tests/integration/src/sqlbuild/compiler/planner/main/test_maintained_refresh_plan.py b/tests/integration/src/sqlbuild/compiler/planner/main/test_maintained_refresh_plan.py index 0f4394c07a..1703f38efb 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/main/test_maintained_refresh_plan.py +++ b/tests/integration/src/sqlbuild/compiler/planner/main/test_maintained_refresh_plan.py @@ -7,6 +7,7 @@ import pytest from sqlbuild._native import ModelPlanEntry, PlanAction, PlanOutput, PlanReason +from sqlbuild.adapter.contract.types import MaintainedListing from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.executor.run._helpers.reuse.fingerprinting import try_write_fingerprint from tests.integration.src.sqlbuild.compiler.planner.main._test_types import ( @@ -28,7 +29,7 @@ [ MaintainedRefreshPlanTestCase( description="listed settings that match keep both models", - listed_settings=("60 minutes", "TRANSFORM_WH"), + listed_settings=MaintainedListing("60 minutes", "TRANSFORM_WH", "ACTIVE"), upstream_query=CLEAN_ORDERS_QUERY, expected_upstream_action=PlanAction.KEEP_MAINTAINED, expected_action=PlanAction.KEEP_MAINTAINED, @@ -36,9 +37,20 @@ expected_reinitialised_by=None, expected_read_names=("clean_orders", "daily_totals"), ), + MaintainedRefreshPlanTestCase( + description="a kept model listed as suspended is reported as paused", + listed_settings=MaintainedListing("60 minutes", "TRANSFORM_WH", "SUSPENDED"), + upstream_query=CLEAN_ORDERS_QUERY, + expected_upstream_action=PlanAction.KEEP_MAINTAINED, + expected_action=PlanAction.KEEP_MAINTAINED, + expected_reason=PlanReason.NO_CHANGE, + expected_reinitialised_by=None, + expected_read_names=("clean_orders", "daily_totals"), + expected_paused=True, + ), MaintainedRefreshPlanTestCase( description="a lag changed outside the project is altered in place", - listed_settings=("2 hours", "TRANSFORM_WH"), + listed_settings=MaintainedListing("2 hours", "TRANSFORM_WH", "ACTIVE"), upstream_query=CLEAN_ORDERS_QUERY, expected_upstream_action=PlanAction.ALTER_MAINTAINED, expected_action=PlanAction.ALTER_MAINTAINED, @@ -58,7 +70,7 @@ ), MaintainedRefreshPlanTestCase( description="a recreated upstream marks the kept reader as recomputing in full", - listed_settings=("1 hour", "TRANSFORM_WH"), + listed_settings=MaintainedListing("1 hour", "TRANSFORM_WH", "ACTIVE"), upstream_query="SELECT 2 AS order_id", expected_upstream_action=PlanAction.CREATE_MAINTAINED, expected_action=PlanAction.KEEP_MAINTAINED, @@ -106,6 +118,7 @@ def test_given_built_maintained_models_when_replanning_then_live_settings_and_re assert entries["daily_totals"].action == test_case.expected_action assert entries["daily_totals"].reason == test_case.expected_reason assert entries["daily_totals"].reinitialised_by == test_case.expected_reinitialised_by + assert entries["daily_totals"].maintained_paused == test_case.expected_paused assert entries["clean_orders"].reinitialised_by is None assert entries["daily_totals"].target_lag == "'1 hour'" assert entries["daily_totals"].refresh_mode == "incremental" diff --git a/tests/integration/src/sqlbuild/executor/maintained_pauses/helpers.py b/tests/integration/src/sqlbuild/executor/maintained_pauses/helpers.py index ad776a6d5a..62e5187c79 100644 --- a/tests/integration/src/sqlbuild/executor/maintained_pauses/helpers.py +++ b/tests/integration/src/sqlbuild/executor/maintained_pauses/helpers.py @@ -7,7 +7,11 @@ from typing import Any from sqlbuild._native import PlanOutput -from sqlbuild.adapter.contract.types import MaintainedOperation, TablePromotionMode +from sqlbuild.adapter.contract.types import ( + MaintainedListing, + MaintainedOperation, + TablePromotionMode, +) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.executor.build.main._execute import execute_build_plan from sqlbuild.executor.build.models import BuildExecutionResult, BuildRuntimeParams @@ -40,24 +44,21 @@ class PausingDuckDbAdapter(DuckDbAdapter): def __init__(self) -> None: super().__init__() self.states: dict[str, str] = {} - self.listed: dict[str, tuple[str, str]] = {} self.operations: list[tuple[str, str]] = [] def session_warehouse(self, *, connection: Any) -> str | None: del connection return "SESSION_WH" - def maintained_refresh_settings( - self, *, connection: Any, database: str | None, schema: str | None, name: str - ) -> tuple[str, str] | None: - del connection, database, schema - return self.listed.get(name) - - def maintained_scheduling_state( + def maintained_listing( self, *, connection: Any, database: str | None, schema: str | None, name: str - ) -> str | None: + ) -> MaintainedListing | None: del connection, database, schema - return self.states.get(name) + listings: dict[str, MaintainedListing] = { + listed: MaintainedListing("1 hour", "TRANSFORM_WH", state) + for listed, state in self.states.items() + } + return listings.get(name) def render_create_maintained( self, *, destination: str, sql: str, target_lag: str, refresh_mode: str, warehouse: str @@ -65,7 +66,6 @@ def render_create_maintained( del target_lag, refresh_mode, warehouse name: str = destination.rsplit(".", 1)[-1] self.states[name] = _CREATED_STATE - self.listed[name] = ("1 hour", "TRANSFORM_WH") self.operations.append(("create", name)) return (f"CREATE OR REPLACE TABLE {destination} AS {sql}",) diff --git a/tests/integration/src/sqlbuild/executor/run/maintained/_test_types.py b/tests/integration/src/sqlbuild/executor/run/maintained/_test_types.py index 66a0b19f55..efe049ac4f 100644 --- a/tests/integration/src/sqlbuild/executor/run/maintained/_test_types.py +++ b/tests/integration/src/sqlbuild/executor/run/maintained/_test_types.py @@ -3,7 +3,7 @@ from dataclasses import dataclass from sqlbuild._native import PlanAction -from sqlbuild.adapter.contract.types import TablePromotionMode +from sqlbuild.adapter.contract.types import MaintainedListing, TablePromotionMode @dataclass(frozen=True) @@ -22,7 +22,7 @@ class MaintainedFailureTestCase: description: str setup_sql: tuple[str, ...] session_warehouse: str | None - listed_settings: tuple[str, str] | None + listed_settings: MaintainedListing | None expected_error_code: str expected_listed_types: tuple[tuple[str], ...] diff --git a/tests/integration/src/sqlbuild/executor/run/maintained/helpers.py b/tests/integration/src/sqlbuild/executor/run/maintained/helpers.py index 517c152a49..7c216b915b 100644 --- a/tests/integration/src/sqlbuild/executor/run/maintained/helpers.py +++ b/tests/integration/src/sqlbuild/executor/run/maintained/helpers.py @@ -15,6 +15,7 @@ PlanReason, ) from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder +from sqlbuild.adapter.contract.types import MaintainedListing from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.executor.run.models import ModelMaterializationContext @@ -25,11 +26,11 @@ class MaintainedDuckDbAdapter(DuckDbAdapter): """Builds a maintained model as a table that, like a dynamic table, replaces nothing else.""" def __init__( - self, *, session_warehouse: str | None, listed_settings: tuple[str, str] | None = None + self, *, session_warehouse: str | None, listed_settings: MaintainedListing | None = None ) -> None: super().__init__() self.warehouse: str | None = session_warehouse - self.listed_settings: tuple[str, str] | None = listed_settings + self.listed_settings: MaintainedListing | None = listed_settings self.created: list[tuple[str, str, str]] = [] self.altered: list[tuple[str, str | None]] = [] @@ -37,9 +38,9 @@ def session_warehouse(self, *, connection: Any) -> str | None: del connection return self.warehouse - def maintained_refresh_settings( + def maintained_listing( self, *, connection: Any, database: str | None, schema: str | None, name: str - ) -> tuple[str, str] | None: + ) -> MaintainedListing | None: del connection, database, schema, name return self.listed_settings diff --git a/tests/integration/src/sqlbuild/executor/run/maintained/test_maintained_execution.py b/tests/integration/src/sqlbuild/executor/run/maintained/test_maintained_execution.py index 52dcb36ae5..86b17d3c3c 100644 --- a/tests/integration/src/sqlbuild/executor/run/maintained/test_maintained_execution.py +++ b/tests/integration/src/sqlbuild/executor/run/maintained/test_maintained_execution.py @@ -7,6 +7,7 @@ import pytest from sqlbuild._native import PlanAction +from sqlbuild.adapter.contract.types import MaintainedListing from sqlbuild.executor.run.main._execute import execute_maintained_entry from sqlbuild.executor.run.models import ModelExecutionResult from sqlbuild.executor.scheduling.types import ExecutionStatus @@ -116,7 +117,7 @@ def test_given_maintained_entry_when_executing_then_settings_reach_the_warehouse description="a failed create never drops a relation listed as a dynamic table", setup_sql=("CREATE VIEW staging.daily_totals AS SELECT 1 AS orders",), session_warehouse="SESSION_WH", - listed_settings=("1 hour", "SESSION_WH"), + listed_settings=MaintainedListing("1 hour", "SESSION_WH", "ACTIVE"), expected_error_code="R002", expected_listed_types=(("VIEW",),), ), diff --git a/tests/unit/src/sqlbuild/adapter/contract/classes/strict_adapter/test_strict_adapter.py b/tests/unit/src/sqlbuild/adapter/contract/classes/strict_adapter/test_strict_adapter.py index c9826ee25e..301ac4504f 100644 --- a/tests/unit/src/sqlbuild/adapter/contract/classes/strict_adapter/test_strict_adapter.py +++ b/tests/unit/src/sqlbuild/adapter/contract/classes/strict_adapter/test_strict_adapter.py @@ -89,7 +89,7 @@ def test_given_first_class_adapters_when_checking_contract_then_no_method_comes_ { "create_sql_test_output", "drop_sql_test_output", - "maintained_refresh_settings", + "maintained_listing", "render_alter_maintained", "render_create_maintained", "render_create_temporary_function", diff --git a/tests/unit/src/sqlbuild/adapters/snowflake/_test_types.py b/tests/unit/src/sqlbuild/adapters/snowflake/_test_types.py index e768378233..e73f7229f2 100644 --- a/tests/unit/src/sqlbuild/adapters/snowflake/_test_types.py +++ b/tests/unit/src/sqlbuild/adapters/snowflake/_test_types.py @@ -3,8 +3,16 @@ from sqlbuild._native import InferredNullability from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.adapter.contract.models import FunctionInfo, RelationUsageReport, SchemaDiffResult -from sqlbuild.adapter.contract.types import MaintainedOperation, TableFreshnessStatus +from sqlbuild.adapter.contract.models import ( + FunctionInfo, + RelationUsageReport, + SchemaDiffResult, +) +from sqlbuild.adapter.contract.types import ( + MaintainedListing, + MaintainedOperation, + TableFreshnessStatus, +) from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter from sqlbuild.cost.types import CostStatus from tests.unit.src.sqlbuild.adapters.snowflake.inspection.helpers import FakeRelation @@ -459,7 +467,7 @@ class SnowflakeWarehouseUserTestCase: class SnowflakeMaintainedRefreshTestCase: description: str rows: tuple[tuple[object, ...], ...] - expected_settings: tuple[str, str] | None + expected_settings: MaintainedListing | None @dataclass(frozen=True) @@ -491,10 +499,3 @@ class SnowflakeChangeTrackingTestCase: description: str config: dict[str, object] expected_sql: str - - -@dataclass(frozen=True) -class SnowflakeMaintainedStateTestCase: - description: str - rows: tuple[tuple[str, str], ...] - expected_state: str | None diff --git a/tests/unit/src/sqlbuild/adapters/snowflake/test_maintained_refresh_settings.py b/tests/unit/src/sqlbuild/adapters/snowflake/test_maintained_models.py similarity index 80% rename from tests/unit/src/sqlbuild/adapters/snowflake/test_maintained_refresh_settings.py rename to tests/unit/src/sqlbuild/adapters/snowflake/test_maintained_models.py index 6af8a6f118..a050065022 100644 --- a/tests/unit/src/sqlbuild/adapters/snowflake/test_maintained_refresh_settings.py +++ b/tests/unit/src/sqlbuild/adapters/snowflake/test_maintained_models.py @@ -1,7 +1,7 @@ import pytest from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder -from sqlbuild.adapter.contract.types import MaintainedOperation +from sqlbuild.adapter.contract.types import MaintainedListing, MaintainedOperation from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter from tests.unit.src.sqlbuild.adapters.helpers import MetadataConnection, execute_through_cursor from tests.unit.src.sqlbuild.adapters.snowflake._test_types import ( @@ -10,24 +10,27 @@ SnowflakeMaintainedCreateTestCase, SnowflakeMaintainedOperationTestCase, SnowflakeMaintainedRefreshTestCase, - SnowflakeMaintainedStateTestCase, ) -_SHOW_DYNAMIC_TABLES_COLUMNS: tuple[str, ...] = ("name", "target_lag", "warehouse") -_SCHEDULING_COLUMNS: tuple[str, ...] = ("name", "scheduling_state") +_SHOW_DYNAMIC_TABLES_COLUMNS: tuple[str, ...] = ( + "name", + "target_lag", + "warehouse", + "scheduling_state", +) @pytest.mark.parametrize( "test_case", [ SnowflakeMaintainedRefreshTestCase( - description="a dynamic table lists its lag and refresh warehouse", - rows=(("DAILY_TOTALS", "1 hour", "TRANSFORM_WH"),), - expected_settings=("1 hour", "TRANSFORM_WH"), + description="a dynamic table lists its lag, refresh warehouse and schedule state", + rows=(("DAILY_TOTALS", "1 hour", "TRANSFORM_WH", "SUSPENDED"),), + expected_settings=MaintainedListing("1 hour", "TRANSFORM_WH", "SUSPENDED"), ), SnowflakeMaintainedRefreshTestCase( description="a neighbour matched by the LIKE wildcard is not the model", - rows=(("DAILY_TOTALS_ARCHIVE", "DOWNSTREAM", "TRANSFORM_WH"),), + rows=(("DAILY_TOTALS_ARCHIVE", "DOWNSTREAM", "TRANSFORM_WH", "ACTIVE"),), expected_settings=None, ), SnowflakeMaintainedRefreshTestCase( @@ -38,7 +41,7 @@ ], ids=lambda case: case.description, ) -def test_given_dynamic_table_listing_when_reading_refresh_settings_then_reads_one_show( +def test_given_dynamic_table_listing_when_reading_it_then_reads_one_show( test_case: SnowflakeMaintainedRefreshTestCase, monkeypatch: pytest.MonkeyPatch ) -> None: adapter: SnowflakeAdapter = SnowflakeAdapter() @@ -47,7 +50,7 @@ def test_given_dynamic_table_listing_when_reading_refresh_settings_then_reads_on ) monkeypatch.setattr(adapter, "execute", execute_through_cursor) - settings: tuple[str, str] | None = adapter.maintained_refresh_settings( + settings: MaintainedListing | None = adapter.maintained_listing( connection=connection, database="ANALYTICS", schema="MARTS", name="daily_totals" ) @@ -58,37 +61,6 @@ def test_given_dynamic_table_listing_when_reading_refresh_settings_then_reads_on ) -@pytest.mark.parametrize( - "test_case", - [ - SnowflakeMaintainedStateTestCase( - description="a suspended dynamic table reads as suspended", - rows=(("DAILY_TOTALS", "SUSPENDED"),), - expected_state="SUSPENDED", - ), - SnowflakeMaintainedStateTestCase( - description="a neighbour matched by the LIKE wildcard is not the model", - rows=(("DAILY_TOTALS_ARCHIVE", "ACTIVE"),), - expected_state=None, - ), - ], - ids=lambda case: case.description, -) -def test_given_dynamic_table_listing_when_reading_scheduling_state_then_reads_its_row( - test_case: SnowflakeMaintainedStateTestCase, monkeypatch: pytest.MonkeyPatch -) -> None: - adapter: SnowflakeAdapter = SnowflakeAdapter() - connection: MetadataConnection = MetadataConnection(test_case.rows, _SCHEDULING_COLUMNS) - monkeypatch.setattr(adapter, "execute", execute_through_cursor) - - state: str | None = adapter.maintained_scheduling_state( - connection=connection, database="ANALYTICS", schema="MARTS", name="daily_totals" - ) - - assert state == test_case.expected_state - assert connection.executed[0].startswith("SHOW DYNAMIC TABLES LIKE 'DAILY_TOTALS'") - - @pytest.mark.parametrize( "test_case", [ diff --git a/ui/src/lib/api/types/operate.ts b/ui/src/lib/api/types/operate.ts index c73e4ece68..38eeff8328 100644 --- a/ui/src/lib/api/types/operate.ts +++ b/ui/src/lib/api/types/operate.ts @@ -658,6 +658,8 @@ export type PlanModel = { changed_functions?: string[]; /** The rebuilt upstream that makes this kept maintained model recompute in full. */ reinitialised_by?: string; + /** Whether the warehouse lists this kept maintained model as suspended. */ + paused?: boolean; cursor_type?: string; cursor?: { requested_bounds: { start: string | null; end: string | null }; resolved_bounds: { start: string | null; end: string | null } | null } | null; /** Unified diff lines of the query since the model's last build, as `sqb plan` prints them. */ diff --git a/ui/src/lib/run/plan-text.test.ts b/ui/src/lib/run/plan-text.test.ts index d0ebe9db34..e80b045adc 100644 --- a/ui/src/lib/run/plan-text.test.ts +++ b/ui/src/lib/run/plan-text.test.ts @@ -50,6 +50,12 @@ describe('plan text, as sqb plan prints it', () => { expect(routine.entries).toEqual([{ name: 'daily_totals', value: 'maintained', detail: ['recomputes in full: orders is rebuilt'] }]); }); + it('says a kept maintained model is paused when the warehouse lists it as suspended', () => { + const kept = model('daily_totals', { materialization_type: 'maintained', action: 'keep_maintained', paused: true }); + const [routine] = planSections(plan([kept])); + expect(routine.entries).toEqual([{ name: 'daily_totals', value: 'maintained', detail: ['paused: scheduled refreshes are suspended'] }]); + }); + it('puts each change in its own section in reason order, then the routine models, and leaves skipped ones out', () => { const sections = planSections( plan([ diff --git a/ui/src/lib/run/plan-text.ts b/ui/src/lib/run/plan-text.ts index f8304e9750..97825f856f 100644 --- a/ui/src/lib/run/plan-text.ts +++ b/ui/src/lib/run/plan-text.ts @@ -189,7 +189,7 @@ export function planSections(plan: PlanDocument): PlanSection[] { const why = changedRuns(plan); add('Models', active.filter((m) => ROUTINE.has(m.reason)).map((m) => { const run = why.get(m.name); - return { name: m.name, value: materializationLabel(m), detail: [...cursorLines(m, true, starts[m.name]), ...(m.reinitialised_by ? [`recomputes in full: ${m.reinitialised_by} is rebuilt`] : []), ...(run ? [run.downstream_of ? `reads ${keyName(run.downstream_of)}` : run.reason] : [])] }; + return { name: m.name, value: materializationLabel(m), detail: [...cursorLines(m, true, starts[m.name]), ...(m.reinitialised_by ? [`recomputes in full: ${m.reinitialised_by} is rebuilt`] : []), ...(m.paused ? ['paused: scheduled refreshes are suspended'] : []), ...(run ? [run.downstream_of ? `reads ${keyName(run.downstream_of)}` : run.reason] : [])] }; }), false, 'routine_models'); } add('Functions', plan.functions.filter((f) => !f.reason || f.reason === 'no_change').map(functionEntry), false, 'functions'); From f412a97b00b883bd7dacca85529e94e4cd7bb005 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 19 Nov 2026 01:28:08 +0000 Subject: [PATCH 10/17] docs: document maintained models, their pause during builds and Snowflake grants --- .../sqlbuild/references/docs/cli/build.md | 1 + .../sqlbuild/references/docs/cli/dag.md | 2 +- .../docs/concepts/adapters/snowflake.md | 23 ++++++ .../docs/concepts/models/configuration.md | 2 +- .../docs/concepts/models/materializations.md | 73 +++++++++++++++++++ .../docs/concepts/project-configuration.md | 17 +++++ website/src/content/docs/docs/cli/build.mdx | 1 + website/src/content/docs/docs/cli/dag.mdx | 2 +- .../docs/docs/concepts/adapters/snowflake.mdx | 22 ++++++ .../docs/concepts/models/configuration.mdx | 2 +- .../docs/concepts/models/materializations.mdx | 72 ++++++++++++++++++ .../docs/concepts/project-configuration.mdx | 16 ++++ 12 files changed, 229 insertions(+), 4 deletions(-) diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/build.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/build.md index 129b506831..f006832b56 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/build.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/build.md @@ -66,6 +66,7 @@ own option. `--version` works only directly after `sqb`, as in `sqb --version`: | `--defer-sources-to` | Read managed source data from another target | | `--plan-from` | Read warehouse state `live`, or from the target's `synced_state` checked against the warehouse first; overrides the target's `plan_from`. See [Planning from synced state](../concepts/project-configuration.md#planning-from-synced-state) | | `--fail-fast` | Stop on first failure and skip remaining nodes | +| `--ignore-maintained-grace` | Take over [maintained models](../concepts/models/materializations.md#during-a-build) another build holds, without waiting for `[maintained] heartbeat_grace`. Needs `allow_ignore_grace = true`, else fails with `A719` | | `--concurrency` | Number of worker connections (default: 1) | | `--test-timeout ` | Time limit for each unit test the build runs, such as `30s`, or `disabled`; see [Time limits](../concepts/testing-reference.md#time-limits) | | `--verbose`, `-v` | Show lifecycle SQL inline after each model | diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/dag.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/dag.md index dd9ae1bcfd..c87285ddeb 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/dag.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/dag.md @@ -105,7 +105,7 @@ Each node represents a source, seed, model, or function: | `description` | Model or source description, if declared | | `tags` | Model tags | | `columns` | Column metadata (name, type, nullable, description) | -| `materialization_type` | For models: `view`, `table`, `incremental`, or custom name | +| `materialization_type` | For models: `view`, `table`, `incremental`, `snapshot`, `maintained`, or custom name | | `language` | For functions: `sql` or `python` | | `return_kind` | For functions: `scalar` or `table` | | `arguments` | For functions: argument name and type pairs | diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/snowflake.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/snowflake.md index dcedd89b3c..e4e39c0407 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/snowflake.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/snowflake.md @@ -14,6 +14,7 @@ Online: https://sqlbuild.com/docs/concepts/adapters/snowflake/ - Case-sensitive column names - Metadata reads - State tables +- Maintained models - Shared connections across targets - Cost estimates - Functions in unit tests @@ -165,6 +166,28 @@ SQLBuild records its state in `_sqlbuild_*` tables. On Snowflake it creates them with 90 days of time travel, or 1 day on editions that allow only that. It never alters an existing state table. Models are not affected: their table type follows the model's `table_type` setting. +## Maintained models + +A [maintained model](../models/materializations.md#maintained) is a Snowflake dynamic +table. `materialized materialized_view` fails with `A014`; use `materialized maintained`. + +The build role needs these grants: + +| Grant | Why | +|-------|-----| +| `CREATE DYNAMIC TABLE` on the model's schema | Create and recreate the model | +| Ownership or `OPERATE` on the dynamic table | Suspend, resume, refresh and alter it during builds | +| `USAGE` on the `refresh_warehouse` | Run the scheduled refreshes | + +Dynamic tables refresh from changes, which needs change tracking on every table they read: + +- A table model that a maintained model reads is built with change tracking on. +- For a source table, Snowflake turns change tracking on when the build role owns the table. + Otherwise its owner runs `ALTER TABLE SET CHANGE_TRACKING = TRUE`. + +Builds record the models they pause in the `_sqlbuild_maintained_pauses` state table, so another +build can tell whether they are still held. + ## Shared connections across targets Use multiple targets to reuse one Snowflake connection while selecting different authoritative diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/configuration.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/configuration.md index 94cda6a657..00a9c2512f 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/configuration.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/configuration.md @@ -36,7 +36,7 @@ which accepts any keys. | Field | Description | |-------|-------------| -| `materialized` | `view`, `table`, `incremental`, `snapshot`, or a custom materialization name | +| `materialized` | `view`, `table`, `incremental`, `snapshot`, `maintained`, or a custom materialization name; see [Materializations](materializations.md) | | `tags` | Tags used by selectors | | `critical` | `true` marks the model as critical, so [`sqb review`](../../cli/review.md) builds and compares it whenever something upstream of it changes (default `false`) | | `description` | Required human-readable model description; see [Descriptions](#descriptions) | diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/materializations.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/materializations.md index 3baa7ed8c0..3ba8897054 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/materializations.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/materializations.md @@ -12,6 +12,7 @@ Online: https://sqlbuild.com/docs/concepts/models/materializations/ - Table - Incremental - Snapshot +- Maintained - Custom - When a build fails @@ -83,6 +84,77 @@ FROM __source("customers") See [Snapshots](../snapshots.md) for timestamp and check strategies, historical inputs, and full-refresh policies. +## Maintained + +The warehouse keeps a maintained model up to date on a schedule you set, between builds. On Snowflake it is a dynamic table; a build creates it, changes it when its definition changes, and otherwise leaves the refreshing to Snowflake. + +```sql +MODEL ( + description "One row per customer per order date", + materialized maintained, + refresh (lag 1h), +); + +SELECT customer_id, CAST(ordered_at AS DATE) AS order_date, SUM(amount) AS revenue +FROM __ref("stg_orders") +GROUP BY customer_id, CAST(ordered_at AS DATE) +``` + +| Field | Default | Description | +|-------|---------|-------------| +| `refresh (lag )` | none | How far behind its inputs the model may fall: a fixed duration such as `5m`, `1h` or `1d`, at least `1m` on Snowflake. Without it, the model refreshes when a maintained model that reads it refreshes | +| `refresh_mode` | `incremental` | `incremental`, `full`, `auto` or `adaptive`. Snowflake rejects `incremental` for a query it can't refresh incrementally; use `full` for that query | +| `refresh_warehouse` | the connection's warehouse | Warehouse the scheduled refreshes run on | +| `refresh_on_build` | `false` | Refresh the model in every build that selects it, not only on its schedule | + +Only Snowflake supports maintained models. See [Snowflake](../adapters/snowflake.md#maintained-models) for the grants and change tracking they need. + +### What the plan shows + +| Change | Plan | What the build does | +|--------|------|---------------------| +| First build, or the dynamic table is missing | `create` | Creates it and fills it | +| Query or `refresh_mode` changed | `recreate, full recompute` | Replaces it; every row is computed again | +| `refresh` lag or `refresh_warehouse` differs from the warehouse | `change refresh settings in place` | Alters the schedule; rows stay | +| Nothing | `maintained` | Leaves it alone | + +A model the plan keeps can carry one more line: + +- `recomputes in full: stg_orders is rebuilt`: the build replaces a model it reads, so its next refresh computes every row again. +- `paused: scheduled refreshes are suspended`: Snowflake lists it as suspended, so it is not refreshing. + +### During a build + +A build pauses the maintained models it changes, and those that read a model it rebuilds, so they never refresh from half-built inputs. It resumes them when it ends, whether it succeeds or fails. + +- Dynamic tables outside the project that read a paused model pause too, until the build resumes it. One that someone had suspended before stays suspended. +- A model that was already suspended when the build started stays suspended, and the build warns about it. +- If someone suspends a model during the build, the build's resume undoes that suspend. +- A kept model is refreshed during the build when a model built later in the same build reads it, so that model reads current rows. `refresh_on_build` refreshes it in every build. +- A target with `pause_maintained = true` leaves its maintained models suspended between builds and refreshes them only in builds; see [Targets](../project-configuration.md#targets). + +Two builds can't pause the same models at once. A build that needs a model another build holds fails with `A718`, naming that run. A build that stops without resuming, for example because its process was killed, leaves its models suspended. The next build that pauses maintained models on the target resumes them once that build has been silent for `[maintained] heartbeat_grace` (see [Maintained](../project-configuration.md#maintained)). Until then, resume them in Snowflake by hand, or run `sqb build --ignore-maintained-grace` where the project allows it. + +### Error codes + +| Code | When | Fix | +|------|------|-----| +| `A013` | `materialized maintained` on an adapter other than Snowflake | Use `table` or `incremental` on that adapter | +| `A014` | `materialized materialized_view` on Snowflake | Use `materialized maintained` | +| `A015` | A `refresh` lag under `1m` on Snowflake | Set a lag of at least `1m` | +| `A016` | A model with no `refresh` that no maintained model reads, so it would never refresh | Add `refresh (lag 1h)` or another lag | +| `A716` | The adapter can't build a maintained model | Use an adapter that supports them | +| `A717` | No warehouse to refresh with | Set `refresh_warehouse`, or a `warehouse` on the connection | +| `A718` | Another build holds the models this build needs | Wait for it to finish, or for its hold to pass `heartbeat_grace` | +| `A719` | `--ignore-maintained-grace` without `allow_ignore_grace = true` | Set `[maintained] allow_ignore_grace = true`, or wait for the grace to pass | + +### Limitations + +- Writing out the default `refresh_mode incremental` changes the model's definition, so the next build recreates it once. +- A custom materialization named `maintained` is not used; the built-in one takes the name. +- Unit tests and scenarios treat a maintained model as a table. +- A snapshot rebuilt in full is created without change tracking. Before a maintained model reads it again, run `ALTER TABLE SET CHANGE_TRACKING = TRUE`. + ## Custom A project-local Python materialization can manage specialized persistence with adapter access, schema findings, query-change state, declared columns, and `ctx.run_audits`. @@ -120,4 +192,5 @@ The custom function owns staging, runtime type or contract enforcement, audit ti | Table (`immediate`) | The new table. Audits run after it is replaced. | | Incremental | The old table if a delta audit fails. Rows already written stay if a final audit or a later microbatch fails. | | Microbatch full rebuild | A partly rebuilt table; the old one was dropped first. | +| Maintained | The created or altered dynamic table, resumed. Audits run after it changes. | | Custom | Whatever the custom materialization does. | diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md index 6b7b5893af..82acb4b67b 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md @@ -20,6 +20,7 @@ Online: https://sqlbuild.com/docs/concepts/project-configuration/ - Project variables - Janitor - Migrations +- Maintained - Scenario - Git - Review @@ -189,6 +190,7 @@ A target is a named build context - the database and schema you build into, plus | `concurrency` | Worker limit for commands run against this target, replacing `[settings] concurrency` (see below) | | `ignore_audit_failures` | Whether builds on this target let audit failures through, replacing `[settings] ignore_audit_failures`; see [Ignoring audit failures](audits.md#ignoring-audit-failures) | | `plan_from` | Where `sqb plan` and `sqb build` read warehouse state: `live` (default) or `synced_state`; see [Planning from synced state](#planning-from-synced-state) | +| `pause_maintained` | `true` keeps this target's [maintained models](models/materializations.md#maintained) suspended between builds; each build refreshes the ones it selects. Default `false` | ```toml [connections.warehouse] @@ -695,6 +697,21 @@ old_name_views = "30d" |-------|---------|-------------| | `old_name_views` | `"30d"` | How long a renamed model's old name keeps working through a [compatibility view](models/migrations.md#old-names) before janitor drops it. `false` keeps the old relation in place instead. A model's `old_name_view` header overrides it. | +## Maintained + +How builds pause [maintained models](models/materializations.md#maintained) while they run: + +```toml +[maintained] +heartbeat_grace = "30m" +allow_ignore_grace = false +``` + +| Field | Default | Description | +|-------|---------|-------------| +| `heartbeat_grace` | `"30m"` | How long a build that paused maintained models may go silent before another build resumes them or takes them over. A fixed duration such as `"30m"` or `"2h"` | +| `allow_ignore_grace` | `false` | Allow `sqb build --ignore-maintained-grace`, which takes over models another build holds without waiting for the grace. Without it the flag fails with `A719` | + ## Scenario Configuration for scenario run isolation, snapshot capture safety limits, and local type overrides: diff --git a/website/src/content/docs/docs/cli/build.mdx b/website/src/content/docs/docs/cli/build.mdx index 195c93647a..31fed956bc 100644 --- a/website/src/content/docs/docs/cli/build.mdx +++ b/website/src/content/docs/docs/cli/build.mdx @@ -49,6 +49,7 @@ own option. `--version` works only directly after `sqb`, as in `sqb --version`: | `--defer-sources-to` | Read managed source data from another target | | `--plan-from` | Read warehouse state `live`, or from the target's `synced_state` checked against the warehouse first; overrides the target's `plan_from`. See [Planning from synced state](/docs/concepts/project-configuration#planning-from-synced-state) | | `--fail-fast` | Stop on first failure and skip remaining nodes | +| `--ignore-maintained-grace` | Take over [maintained models](/docs/concepts/models/materializations#during-a-build) another build holds, without waiting for `[maintained] heartbeat_grace`. Needs `allow_ignore_grace = true`, else fails with `A719` | | `--concurrency` | Number of worker connections (default: 1) | | `--test-timeout ` | Time limit for each unit test the build runs, such as `30s`, or `disabled`; see [Time limits](/docs/concepts/testing-reference#time-limits) | | `--verbose`, `-v` | Show lifecycle SQL inline after each model | diff --git a/website/src/content/docs/docs/cli/dag.mdx b/website/src/content/docs/docs/cli/dag.mdx index dbefae2d51..cd8a98262e 100644 --- a/website/src/content/docs/docs/cli/dag.mdx +++ b/website/src/content/docs/docs/cli/dag.mdx @@ -96,7 +96,7 @@ Each node represents a source, seed, model, or function: | `description` | Model or source description, if declared | | `tags` | Model tags | | `columns` | Column metadata (name, type, nullable, description) | -| `materialization_type` | For models: `view`, `table`, `incremental`, or custom name | +| `materialization_type` | For models: `view`, `table`, `incremental`, `snapshot`, `maintained`, or custom name | | `language` | For functions: `sql` or `python` | | `return_kind` | For functions: `scalar` or `table` | | `arguments` | For functions: argument name and type pairs | diff --git a/website/src/content/docs/docs/concepts/adapters/snowflake.mdx b/website/src/content/docs/docs/concepts/adapters/snowflake.mdx index c0c1319e5b..946b866cc9 100644 --- a/website/src/content/docs/docs/concepts/adapters/snowflake.mdx +++ b/website/src/content/docs/docs/concepts/adapters/snowflake.mdx @@ -150,6 +150,28 @@ SQLBuild records its state in `_sqlbuild_*` tables. On Snowflake it creates them with 90 days of time travel, or 1 day on editions that allow only that. It never alters an existing state table. Models are not affected: their table type follows the model's `table_type` setting. +## Maintained models + +A [maintained model](/docs/concepts/models/materializations#maintained) is a Snowflake dynamic +table. `materialized materialized_view` fails with `A014`; use `materialized maintained`. + +The build role needs these grants: + +| Grant | Why | +|-------|-----| +| `CREATE DYNAMIC TABLE` on the model's schema | Create and recreate the model | +| Ownership or `OPERATE` on the dynamic table | Suspend, resume, refresh and alter it during builds | +| `USAGE` on the `refresh_warehouse` | Run the scheduled refreshes | + +Dynamic tables refresh from changes, which needs change tracking on every table they read: + +- A table model that a maintained model reads is built with change tracking on. +- For a source table, Snowflake turns change tracking on when the build role owns the table. + Otherwise its owner runs `ALTER TABLE SET CHANGE_TRACKING = TRUE`. + +Builds record the models they pause in the `_sqlbuild_maintained_pauses` state table, so another +build can tell whether they are still held. + ## Shared connections across targets Use multiple targets to reuse one Snowflake connection while selecting different authoritative diff --git a/website/src/content/docs/docs/concepts/models/configuration.mdx b/website/src/content/docs/docs/concepts/models/configuration.mdx index ce135807da..c72e3487cf 100644 --- a/website/src/content/docs/docs/concepts/models/configuration.mdx +++ b/website/src/content/docs/docs/concepts/models/configuration.mdx @@ -20,7 +20,7 @@ which accepts any keys. | Field | Description | |-------|-------------| -| `materialized` | `view`, `table`, `incremental`, `snapshot`, or a custom materialization name | +| `materialized` | `view`, `table`, `incremental`, `snapshot`, `maintained`, or a custom materialization name; see [Materializations](/docs/concepts/models/materializations) | | `tags` | Tags used by selectors | | `critical` | `true` marks the model as critical, so [`sqb review`](/docs/cli/review) builds and compares it whenever something upstream of it changes (default `false`) | | `description` | Required human-readable model description; see [Descriptions](#descriptions) | diff --git a/website/src/content/docs/docs/concepts/models/materializations.mdx b/website/src/content/docs/docs/concepts/models/materializations.mdx index 14e18f9791..74626c8422 100644 --- a/website/src/content/docs/docs/concepts/models/materializations.mdx +++ b/website/src/content/docs/docs/concepts/models/materializations.mdx @@ -71,6 +71,77 @@ FROM __source("customers") See [Snapshots](/docs/concepts/snapshots) for timestamp and check strategies, historical inputs, and full-refresh policies. +## Maintained + +The warehouse keeps a maintained model up to date on a schedule you set, between builds. On Snowflake it is a dynamic table; a build creates it, changes it when its definition changes, and otherwise leaves the refreshing to Snowflake. + +```sql +MODEL ( + description "One row per customer per order date", + materialized maintained, + refresh (lag 1h), +); + +SELECT customer_id, CAST(ordered_at AS DATE) AS order_date, SUM(amount) AS revenue +FROM __ref("stg_orders") +GROUP BY customer_id, CAST(ordered_at AS DATE) +``` + +| Field | Default | Description | +|-------|---------|-------------| +| `refresh (lag )` | none | How far behind its inputs the model may fall: a fixed duration such as `5m`, `1h` or `1d`, at least `1m` on Snowflake. Without it, the model refreshes when a maintained model that reads it refreshes | +| `refresh_mode` | `incremental` | `incremental`, `full`, `auto` or `adaptive`. Snowflake rejects `incremental` for a query it can't refresh incrementally; use `full` for that query | +| `refresh_warehouse` | the connection's warehouse | Warehouse the scheduled refreshes run on | +| `refresh_on_build` | `false` | Refresh the model in every build that selects it, not only on its schedule | + +Only Snowflake supports maintained models. See [Snowflake](/docs/concepts/adapters/snowflake#maintained-models) for the grants and change tracking they need. + +### What the plan shows + +| Change | Plan | What the build does | +|--------|------|---------------------| +| First build, or the dynamic table is missing | `create` | Creates it and fills it | +| Query or `refresh_mode` changed | `recreate, full recompute` | Replaces it; every row is computed again | +| `refresh` lag or `refresh_warehouse` differs from the warehouse | `change refresh settings in place` | Alters the schedule; rows stay | +| Nothing | `maintained` | Leaves it alone | + +A model the plan keeps can carry one more line: + +- `recomputes in full: stg_orders is rebuilt`: the build replaces a model it reads, so its next refresh computes every row again. +- `paused: scheduled refreshes are suspended`: Snowflake lists it as suspended, so it is not refreshing. + +### During a build + +A build pauses the maintained models it changes, and those that read a model it rebuilds, so they never refresh from half-built inputs. It resumes them when it ends, whether it succeeds or fails. + +- Dynamic tables outside the project that read a paused model pause too, until the build resumes it. One that someone had suspended before stays suspended. +- A model that was already suspended when the build started stays suspended, and the build warns about it. +- If someone suspends a model during the build, the build's resume undoes that suspend. +- A kept model is refreshed during the build when a model built later in the same build reads it, so that model reads current rows. `refresh_on_build` refreshes it in every build. +- A target with `pause_maintained = true` leaves its maintained models suspended between builds and refreshes them only in builds; see [Targets](/docs/concepts/project-configuration#targets). + +Two builds can't pause the same models at once. A build that needs a model another build holds fails with `A718`, naming that run. A build that stops without resuming, for example because its process was killed, leaves its models suspended. The next build that pauses maintained models on the target resumes them once that build has been silent for `[maintained] heartbeat_grace` (see [Maintained](/docs/concepts/project-configuration#maintained)). Until then, resume them in Snowflake by hand, or run `sqb build --ignore-maintained-grace` where the project allows it. + +### Error codes + +| Code | When | Fix | +|------|------|-----| +| `A013` | `materialized maintained` on an adapter other than Snowflake | Use `table` or `incremental` on that adapter | +| `A014` | `materialized materialized_view` on Snowflake | Use `materialized maintained` | +| `A015` | A `refresh` lag under `1m` on Snowflake | Set a lag of at least `1m` | +| `A016` | A model with no `refresh` that no maintained model reads, so it would never refresh | Add `refresh (lag 1h)` or another lag | +| `A716` | The adapter can't build a maintained model | Use an adapter that supports them | +| `A717` | No warehouse to refresh with | Set `refresh_warehouse`, or a `warehouse` on the connection | +| `A718` | Another build holds the models this build needs | Wait for it to finish, or for its hold to pass `heartbeat_grace` | +| `A719` | `--ignore-maintained-grace` without `allow_ignore_grace = true` | Set `[maintained] allow_ignore_grace = true`, or wait for the grace to pass | + +### Limitations + +- Writing out the default `refresh_mode incremental` changes the model's definition, so the next build recreates it once. +- A custom materialization named `maintained` is not used; the built-in one takes the name. +- Unit tests and scenarios treat a maintained model as a table. +- A snapshot rebuilt in full is created without change tracking. Before a maintained model reads it again, run `ALTER TABLE SET CHANGE_TRACKING = TRUE`. + ## Custom A project-local Python materialization can manage specialized persistence with adapter access, schema findings, query-change state, declared columns, and `ctx.run_audits`. @@ -108,4 +179,5 @@ The custom function owns staging, runtime type or contract enforcement, audit ti | Table (`immediate`) | The new table. Audits run after it is replaced. | | Incremental | The old table if a delta audit fails. Rows already written stay if a final audit or a later microbatch fails. | | Microbatch full rebuild | A partly rebuilt table; the old one was dropped first. | +| Maintained | The created or altered dynamic table, resumed. Audits run after it changes. | | Custom | Whatever the custom materialization does. | diff --git a/website/src/content/docs/docs/concepts/project-configuration.mdx b/website/src/content/docs/docs/concepts/project-configuration.mdx index 30f05b29f4..fe4b1cb8d1 100644 --- a/website/src/content/docs/docs/concepts/project-configuration.mdx +++ b/website/src/content/docs/docs/concepts/project-configuration.mdx @@ -163,6 +163,7 @@ A target is a named build context - the database and schema you build into, plus | `concurrency` | Worker limit for commands run against this target, replacing `[settings] concurrency` (see below) | | `ignore_audit_failures` | Whether builds on this target let audit failures through, replacing `[settings] ignore_audit_failures`; see [Ignoring audit failures](/docs/concepts/audits#ignoring-audit-failures) | | `plan_from` | Where `sqb plan` and `sqb build` read warehouse state: `live` (default) or `synced_state`; see [Planning from synced state](#planning-from-synced-state) | +| `pause_maintained` | `true` keeps this target's [maintained models](/docs/concepts/models/materializations#maintained) suspended between builds; each build refreshes the ones it selects. Default `false` | ```toml [connections.warehouse] @@ -669,6 +670,21 @@ old_name_views = "30d" |-------|---------|-------------| | `old_name_views` | `"30d"` | How long a renamed model's old name keeps working through a [compatibility view](/docs/concepts/models/migrations#old-names) before janitor drops it. `false` keeps the old relation in place instead. A model's `old_name_view` header overrides it. | +## Maintained + +How builds pause [maintained models](/docs/concepts/models/materializations#maintained) while they run: + +```toml +[maintained] +heartbeat_grace = "30m" +allow_ignore_grace = false +``` + +| Field | Default | Description | +|-------|---------|-------------| +| `heartbeat_grace` | `"30m"` | How long a build that paused maintained models may go silent before another build resumes them or takes them over. A fixed duration such as `"30m"` or `"2h"` | +| `allow_ignore_grace` | `false` | Allow `sqb build --ignore-maintained-grace`, which takes over models another build holds without waiting for the grace. Without it the flag fails with `A719` | + ## Scenario Configuration for scenario run isolation, snapshot capture safety limits, and local type overrides: From 6989d68fc7e57582e8c8dbed531094724f758f6a Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 19 Nov 2026 01:30:00 +0000 Subject: [PATCH 11/17] refactor(snowflake): share the exact-name SHOW lookup between object counts and dynamic tables --- .../snowflake/classes/snowflake_adapter.py | 44 +++++++++---------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py index 00f720a399..19dc7d7195 100644 --- a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py +++ b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py @@ -3736,13 +3736,26 @@ def maintained_listing( def _listed_dynamic_table( self, *, connection: Any, database: str | None, schema: str | None, name: str ) -> dict[str, object] | None: - stored_name: str = self._information_schema_identifier(name) if schema is None: return None + return self._show_named_row( + connection=connection, + listing="DYNAMIC TABLES", + database=database, + schema=schema, + name=name, + ) + + def _show_named_row( + self, *, connection: Any, listing: str, database: str | None, schema: str, name: str + ) -> dict[str, object] | None: + """The SHOW row for exactly this name in the schema, if it lists one.""" + + stored_name: str = self._information_schema_identifier(name) rows: list[dict[str, object]] = self._fetch_schema_show_rows( connection=connection, query=( - f"SHOW DYNAMIC TABLES LIKE '{show_like_pattern(stored_name)}' IN SCHEMA " + f"SHOW {listing} LIKE '{show_like_pattern(stored_name)}' IN SCHEMA " + self._show_schema_scope(connection=connection, database=database, schema=schema) ), show_database=self._show_database( @@ -3888,27 +3901,14 @@ def _show_object_row_count( ) -> tuple[Any, Any] | None: """Rows and bytes of one table; a view or other object lists neither.""" - stored_name: str = self._information_schema_identifier(name) - rows: list[dict[str, object]] = self._fetch_schema_show_rows( - connection=connection, - query=( - f"SHOW OBJECTS LIKE '{show_like_pattern(stored_name)}' IN SCHEMA " - + self._show_schema_scope(connection=connection, database=database, schema=schema) - ), - show_database=self._show_database( - connection=connection, database=database, schema=schema - ), - schema=schema, + row: dict[str, object] | None = self._show_named_row( + connection=connection, listing="OBJECTS", database=database, schema=schema, name=name ) - row: dict[str, object] - for row in rows: - if str(row.get("name")) == stored_name: - return ( - (row.get("rows"), row.get("bytes")) - if str(row.get("kind")) == SHOW_OBJECTS_TABLE_KIND - else (None, None) - ) - return None + if row is None: + return None + if str(row.get("kind")) != SHOW_OBJECTS_TABLE_KIND: + return (None, None) + return (row.get("rows"), row.get("bytes")) def inspect_row_diff_coverage( self, From 9a2847b0f0c09837262aa1de9253258c23631538 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 19 Nov 2026 01:55:20 +0000 Subject: [PATCH 12/17] fix(build): harden maintained models after review - Hold warnings and takeovers reach the build's output. - A build whose hold another build took over leaves those models paused. - [maintained] heartbeat_grace is at least 3m, three heartbeats. - Tables a maintained model reads through views are tracked and paused. - Switching a table or view to maintained restores it if the create fails. --- .../src/project/_helpers/sections.rs | 17 ++- .../sqlbuild-config/src/project/constants.rs | 2 + .../src/project/tests/test_load_config.rs | 15 ++- .../_helpers/outputs/maintained.rs | 98 ++++++++++----- .../_helpers/build_command/execution.rs | 2 +- .../docs/concepts/adapters/snowflake.md | 2 +- .../docs/concepts/models/materializations.md | 3 +- .../docs/concepts/project-configuration.md | 2 +- src/sqlbuild/executor/build/main/_execute.py | 6 +- src/sqlbuild/executor/build/models.py | 2 +- .../classes/maintained_hold.py | 24 ++++ .../_helpers/materializations/maintained.py | 62 ++++++++-- src/sqlbuild/executor/run/constants.py | 1 + .../sqlbuild/compiler/planner/main/helpers.py | 26 +++- .../main/test_maintained_refresh_plan.py | 22 +++- .../executor/maintained_pauses/_test_types.py | 15 +++ .../executor/maintained_pauses/helpers.py | 10 +- .../test_maintained_pause_lease.py | 117 +++++++++++++++++- .../executor/run/maintained/_test_types.py | 1 + .../executor/run/maintained/helpers.py | 15 ++- .../maintained/test_maintained_execution.py | 24 +++- .../docs/docs/concepts/adapters/snowflake.mdx | 2 +- .../docs/concepts/models/materializations.mdx | 3 +- .../docs/concepts/project-configuration.mdx | 2 +- 24 files changed, 397 insertions(+), 76 deletions(-) diff --git a/crates/sqlbuild-config/src/project/_helpers/sections.rs b/crates/sqlbuild-config/src/project/_helpers/sections.rs index 023949de0e..c944e56106 100644 --- a/crates/sqlbuild-config/src/project/_helpers/sections.rs +++ b/crates/sqlbuild-config/src/project/_helpers/sections.rs @@ -24,11 +24,12 @@ use crate::project::constants::{ DEFAULT_HEARTBEAT_GRACE_SECONDS, DEFAULT_IN_USE_WINDOW, DEFAULT_QUERY_ARTIFACT_TTL, FUTURE_CURSOR_ACTIONS, LEGACY_COLUMN_CONTRACT_MODE_KEY, LEGACY_CONCURRENCY_KEY, LEGACY_SQL_VALIDATION_KEY, LOCAL_DBT_KEYS, MAINTAINED_KEYS, MAX_BATCHES_KEY, - MICROBATCH_LIMIT_ACTIONS, OLD_NAME_VIEWS_KEY, QUERY_CHANGE_TRACKING_KEY, - REQUIRE_SQL_ANALYSIS_KEY, REVIEW_DOWNSTREAM_POLICIES, REVIEW_KEYS, RUN_NAMESPACE_KEY, - RUN_NAMESPACE_MAX_LENGTH, SCENARIO_KEYS, SETTINGS_KEYS, SNAPSHOT_FULL_REFRESH_POLICIES, - SNAPSHOT_KEYS, SNAPSHOT_LIMIT_KEYS, SNAPSHOT_SCHEMA_CHANGE_POLICIES, SQL_ANALYSIS_KEY, - UNACCOUNTED_PARTITION_POLICIES, USD_PER_CREDIT_KEY, ZERO_FIXED_DURATION, ZERO_FLOAT, + MICROBATCH_LIMIT_ACTIONS, MIN_HEARTBEAT_GRACE_SECONDS, OLD_NAME_VIEWS_KEY, + QUERY_CHANGE_TRACKING_KEY, REQUIRE_SQL_ANALYSIS_KEY, REVIEW_DOWNSTREAM_POLICIES, REVIEW_KEYS, + RUN_NAMESPACE_KEY, RUN_NAMESPACE_MAX_LENGTH, SCENARIO_KEYS, SETTINGS_KEYS, + SNAPSHOT_FULL_REFRESH_POLICIES, SNAPSHOT_KEYS, SNAPSHOT_LIMIT_KEYS, + SNAPSHOT_SCHEMA_CHANGE_POLICIES, SQL_ANALYSIS_KEY, UNACCOUNTED_PARTITION_POLICIES, + USD_PER_CREDIT_KEY, ZERO_FIXED_DURATION, ZERO_FLOAT, }; use crate::project::constants::{ CHANGE_COUNTS, CHANGE_REACHES, CHANGES_KEYS, CONTROL_KEYS, MAX_POOL_LIMIT, @@ -643,6 +644,12 @@ pub(crate) fn load_maintained( }, Some(_) => return Err(grace_failure(file)), }; + if heartbeat_grace_seconds < MIN_HEARTBEAT_GRACE_SECONDS { + return Err(ConfigFailure::at( + file, + "maintained.heartbeat_grace must be at least 3m, three of a build's one-minute heartbeats", + )); + } let allow_ignore_grace: bool = match get(mapping, "allow_ignore_grace") { None => false, Some(ConfigValue::Bool(flag)) => *flag, diff --git a/crates/sqlbuild-config/src/project/constants.rs b/crates/sqlbuild-config/src/project/constants.rs index 85c47e5b56..7059fcc081 100644 --- a/crates/sqlbuild-config/src/project/constants.rs +++ b/crates/sqlbuild-config/src/project/constants.rs @@ -242,6 +242,8 @@ pub(crate) const SYNC_INTERVAL_HELP: &str = pub(crate) const MAINTAINED_KEYS: [&str; 2] = ["allow_ignore_grace", "heartbeat_grace"]; /// How long a paused maintained model's heartbeat may be silent before another build takes over. pub(crate) const DEFAULT_HEARTBEAT_GRACE_SECONDS: u64 = 1_800; +/// Three of a build's one-minute heartbeats, so a healthy build never looks silent. +pub(crate) const MIN_HEARTBEAT_GRACE_SECONDS: u64 = 180; pub(crate) const REVIEW_KEYS: [&str; 4] = ["against", "critical_tags", "downstream", "schema_prefix"]; pub(crate) const REVIEW_DOWNSTREAM_POLICIES: [&str; 3] = ["critical", "all", "none"]; diff --git a/crates/sqlbuild-config/src/project/tests/test_load_config.rs b/crates/sqlbuild-config/src/project/tests/test_load_config.rs index 29edf970c0..7e86023a6f 100644 --- a/crates/sqlbuild-config/src/project/tests/test_load_config.rs +++ b/crates/sqlbuild-config/src/project/tests/test_load_config.rs @@ -87,6 +87,15 @@ fn given_project_files_when_loading_then_rejections_match_python_loader() { "maintained.heartbeat_grace must be a fixed duration such as 30m or 2h", )), }, + ProjectFailureTestCase { + description: "a grace shorter than three heartbeats would expire a healthy build", + project: "name = 'orders'\nadapter = 'snowflake'\n[maintained]\nheartbeat_grace = '2m'\n", + local: "", + expected_failure: Some(ConfigFailure::at( + ConfigFile::Project, + "maintained.heartbeat_grace must be at least 3m, three of a build's one-minute heartbeats", + )), + }, ProjectFailureTestCase { description: "an unknown key suggests the nearest supported key", project: "name = 'orders'\nadapter = 'duckdb'\n[scopes]\nenforce_placment = true\n", @@ -636,11 +645,11 @@ fn given_maintained_section_when_reading_then_returns_its_values_or_defaults() { }, }, MaintainedConfigTestCase { - description: "a grace shorter than one heartbeat is allowed", + description: "the shortest grace, three heartbeats", project: "name = 'orders'\nadapter = 'snowflake'\n[maintained]\n\ - heartbeat_grace = '30s'\n", + heartbeat_grace = '3m'\n", expected_config: MaintainedConfig { - heartbeat_grace_seconds: 30, + heartbeat_grace_seconds: 180, allow_ignore_grace: false, }, }, diff --git a/crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/maintained.rs b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/maintained.rs index 0f2d167ff6..4f0650b1be 100644 --- a/crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/maintained.rs +++ b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/maintained.rs @@ -27,9 +27,11 @@ use sqlbuild_python_front::model_config::main::config::model_values::model_value use sqlbuild_python_project::graph::main::nodes::native_key::native_key; use sqlbuild_python_project::native_host::models::ModelLookup; use sqlbuild_python_project::planner_scope::main::scope_data::scope_data; -use std::collections::HashMap; +use std::collections::{BTreeSet, HashMap}; const MAINTAINED: &str = "maintained"; +/// Views pass reads through, so a maintained model reading a view reads the view's inputs. +const VIEW: &str = "view"; const KEEP_MAINTAINED: &str = "keep_maintained"; /// The scheduling state Snowflake lists for a dynamic table whose refreshes are paused. const SUSPENDED: &str = "SUSPENDED"; @@ -57,7 +59,11 @@ pub(crate) fn plan_maintained_refresh<'py>( .collect(); let models = ModelLookup::Native(scope.clone()); let reads = runtime_reads(runtime)?; - let mut maintained_models: HashMap = HashMap::new(); + let mut known = KnownReaders { + models: &models, + materializations: HashMap::new(), + through_view: HashMap::new(), + }; let mut entries: Vec> = Vec::with_capacity(results.entries.len()); let mut rows: Vec = Vec::with_capacity(results.entries.len()); for entry in results.entries { @@ -69,11 +75,7 @@ pub(crate) fn plan_maintained_refresh<'py>( } else { entry }; - let readers: Vec = maintained_readers( - py, - downstream.get(&key).copied(), - (&models, &mut maintained_models), - )?; + let readers: Vec = known.maintained_readers(py, &key, &downstream)?; let entry = if readers.is_empty() { entry } else { @@ -106,36 +108,64 @@ pub(crate) fn plan_maintained_refresh<'py>( }) } -/// The maintained models reading the entry, selected or not, checking each reader once per plan. -fn maintained_readers( - py: Python<'_>, - readers: Option<&Vec>, - (models, maintained_models): (&ModelLookup, &mut HashMap), -) -> PyResult> { - let Some(readers) = readers else { - return Ok(Vec::new()); - }; - let mut found: Vec = Vec::new(); - for (kind, name) in readers { - if kind != MODEL { - continue; - } - let maintained: bool = match maintained_models.get(name) { - Some(maintained) => *maintained, - None => { - let model = models.get(py, name)?; - let maintained: bool = - !model.is_none() && text(&materialization_of(&model)?)? == MAINTAINED; - maintained_models.insert(name.clone(), maintained); - maintained +/// Per-plan lookups: each model's materialization, and the maintained readers behind each view. +struct KnownReaders<'a> { + models: &'a ModelLookup, + materializations: HashMap, + through_view: HashMap>, +} + +impl KnownReaders<'_> { + /// The maintained models reading the entry directly or through views, selected or not. + fn maintained_readers( + &mut self, + py: Python<'_>, + key: &ObjectKey, + downstream: &HashMap<&ObjectKey, &Vec>, + ) -> PyResult> { + let mut found: BTreeSet = BTreeSet::new(); + for reader in downstream + .get(key) + .into_iter() + .flat_map(|readers| readers.iter()) + { + let (kind, name) = reader; + if kind != MODEL { + continue; } - }; - if maintained { - found.push(name.clone()); + match self.materialization(py, name)?.as_str() { + MAINTAINED => { + found.insert(name.clone()); + } + VIEW => { + if !self.through_view.contains_key(name) { + let through: Vec = + self.maintained_readers(py, reader, downstream)?; + self.through_view.insert(name.clone(), through); + } + found.extend(self.through_view[name].iter().cloned()); + } + _ => {} + } + } + Ok(found.into_iter().collect()) + } + + /// A model's materialization, looked up once per plan. + fn materialization(&mut self, py: Python<'_>, name: &str) -> PyResult { + if let Some(materialization) = self.materializations.get(name) { + return Ok(materialization.clone()); } + let model = self.models.get(py, name)?; + let materialization: String = if model.is_none() { + String::new() + } else { + text(&materialization_of(&model)?)? + }; + self.materializations + .insert(name.to_owned(), materialization.clone()); + Ok(materialization) } - found.sort(); - Ok(found) } /// The entry with its `target_lag`; a kept one alters on drift, or creates when no dynamic table holds its name. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs index eeaa517957..5a0a7a7829 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs @@ -418,7 +418,7 @@ pub(crate) fn execute_build_plan<'py>( let build_callbacks: Bound<'py, PyAny> = call_with( &host_entry(py, ExecutorHost::BuildCallbacks)?, &[ - ("on_migration_progress", line_writer(&context.stream)?), + ("on_lifecycle_progress", line_writer(&context.stream)?), ( "on_node_start", keyword_relay(native_field(callbacks, "on_node_start")?, &NODE_START)?, diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/snowflake.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/snowflake.md index e4e39c0407..3074e3168e 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/snowflake.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/snowflake.md @@ -181,7 +181,7 @@ The build role needs these grants: Dynamic tables refresh from changes, which needs change tracking on every table they read: -- A table model that a maintained model reads is built with change tracking on. +- A table model that a maintained model reads, directly or through views, is built with change tracking on. - For a source table, Snowflake turns change tracking on when the build role owns the table. Otherwise its owner runs `ALTER TABLE SET CHANGE_TRACKING = TRUE`. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/materializations.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/materializations.md index 3ba8897054..7e009f8747 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/materializations.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/materializations.md @@ -125,7 +125,7 @@ A model the plan keeps can carry one more line: ### During a build -A build pauses the maintained models it changes, and those that read a model it rebuilds, so they never refresh from half-built inputs. It resumes them when it ends, whether it succeeds or fails. +A build pauses the maintained models it changes, and those that read a model it rebuilds, directly or through views, so they never refresh from half-built inputs. It resumes them when it ends, whether it succeeds or fails. - Dynamic tables outside the project that read a paused model pause too, until the build resumes it. One that someone had suspended before stays suspended. - A model that was already suspended when the build started stays suspended, and the build warns about it. @@ -153,6 +153,7 @@ Two builds can't pause the same models at once. A build that needs a model anoth - Writing out the default `refresh_mode incremental` changes the model's definition, so the next build recreates it once. - A custom materialization named `maintained` is not used; the built-in one takes the name. - Unit tests and scenarios treat a maintained model as a table. +- On a target with `schema = "preserve"`, builds see each other's pauses only when their maintained models share a schema. - A snapshot rebuilt in full is created without change tracking. Before a maintained model reads it again, run `ALTER TABLE SET CHANGE_TRACKING = TRUE`. ## Custom diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md index 82acb4b67b..6dfb1072e7 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md @@ -709,7 +709,7 @@ allow_ignore_grace = false | Field | Default | Description | |-------|---------|-------------| -| `heartbeat_grace` | `"30m"` | How long a build that paused maintained models may go silent before another build resumes them or takes them over. A fixed duration such as `"30m"` or `"2h"` | +| `heartbeat_grace` | `"30m"` | How long a build that paused maintained models may go silent before another build resumes them or takes them over. A fixed duration of at least `"3m"`, such as `"30m"` or `"2h"`; a build writes a heartbeat every minute | | `allow_ignore_grace` | `false` | Allow `sqb build --ignore-maintained-grace`, which takes over models another build holds without waiting for the grace. Without it the flag fails with `A719` | ## Scenario diff --git a/src/sqlbuild/executor/build/main/_execute.py b/src/sqlbuild/executor/build/main/_execute.py index bc0433b760..1d48b1c2f0 100644 --- a/src/sqlbuild/executor/build/main/_execute.py +++ b/src/sqlbuild/executor/build/main/_execute.py @@ -69,7 +69,7 @@ def execute_build_plan( if hold is None else hold.holding( connection=scheduler_connection, - on_progress=None if callbacks is None else callbacks.on_progress, + on_progress=None if callbacks is None else callbacks.on_lifecycle_progress, ) ) with relation_metadata_cache_scope(adapter=adapter), held: @@ -125,14 +125,14 @@ def _execute_build_plan( adapter=adapter, connection=scheduler_connection, run_id=runtime.run_id, - on_progress=resolved_callbacks.on_migration_progress, + on_progress=resolved_callbacks.on_lifecycle_progress, ) _ = apply_column_migrations( plan=plan, adapter=adapter, connection=scheduler_connection, run_id=runtime.run_id, - on_progress=resolved_callbacks.on_migration_progress, + on_progress=resolved_callbacks.on_lifecycle_progress, ) _ = apply_retention_phase( plan=plan, diff --git a/src/sqlbuild/executor/build/models.py b/src/sqlbuild/executor/build/models.py index 005ad584c8..d0c19dfe0a 100644 --- a/src/sqlbuild/executor/build/models.py +++ b/src/sqlbuild/executor/build/models.py @@ -125,7 +125,7 @@ class BuildCallbacks: python_identity_recorder: PythonIdentityRecorder | None = None on_scheduler_state: Callable[[SchedulerState], None] | None = None on_statement_complete: Callable[[StatementExecutionTelemetry], None] | None = None - on_migration_progress: Callable[[str], None] | None = None + on_lifecycle_progress: Callable[[str], None] | None = None @dataclass(frozen=True) diff --git a/src/sqlbuild/executor/maintained_pauses/classes/maintained_hold.py b/src/sqlbuild/executor/maintained_pauses/classes/maintained_hold.py index 0cbedc10f9..c1acf55645 100644 --- a/src/sqlbuild/executor/maintained_pauses/classes/maintained_hold.py +++ b/src/sqlbuild/executor/maintained_pauses/classes/maintained_hold.py @@ -153,9 +153,12 @@ def release(self, *, connection: Any) -> None: self._heartbeat.stop() self._report_heartbeat_failure(failure=self._heartbeat.failure) order: list[str] = list(self._held) + still_held: frozenset[_RelationKey] | None = self._still_held(connection=connection) resumed: list[str] = [] failures: list[Exception] = [] for name in sorted(self._paused, key=order.index): + if still_held is not None and _key(self._held[name]) not in still_held: + continue try: self._operate( connection=connection, name=name, operation=MaintainedOperation.RESUME @@ -214,6 +217,27 @@ def _take_over(self, *, connection: Any, location: StateLocation) -> tuple[OpenH self._report(holds=taken, verb="Took over") return taken + def _still_held(self, *, connection: Any) -> frozenset[_RelationKey] | None: + """This run's holds nobody took over, or None when the pause table can't say.""" + + location: StateLocation | None = self._state_location + if location is None or self._pause_maintained: + return None + try: + rows: tuple[tuple[Any, ...], ...] = tuple( + self._adapter.query( + connection=connection, + sql=open_holds_sql(adapter=self._adapter, location=location), + limit=None, + ).rows + ) + except Exception as error: + self._warn(f"could not read which maintained models this build still holds: {error}") + return None + return frozenset( + _key(hold.relation) for hold in open_holds(rows=rows) if hold.run_id == self._run_id + ) + def _resume_found(self, *, connection: Any, relation: CompiledRelationLocation) -> None: if _scheduling_state(adapter=self._adapter, connection=connection, relation=relation) == ( SUSPENDED_STATE diff --git a/src/sqlbuild/executor/run/_helpers/materializations/maintained.py b/src/sqlbuild/executor/run/_helpers/materializations/maintained.py index 7f67470325..d1f22a0797 100644 --- a/src/sqlbuild/executor/run/_helpers/materializations/maintained.py +++ b/src/sqlbuild/executor/run/_helpers/materializations/maintained.py @@ -5,14 +5,23 @@ from collections.abc import Callable from typing import Any -from sqlbuild._native import CompiledRelationLocation, ModelPlanEntry, PlanAction +from sqlbuild._native import ( + CompiledRelationLocation, + ModelPlanEntry, + PlanAction, + fit_auxiliary_relation_name, +) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.exceptions import AdapterUserError from sqlbuild.adapter.contract.types import MaintainedListing, MaintainedOperation, RelationType from sqlbuild.adapter.relations.main.cached_relation_type import cached_relation_type +from sqlbuild.adapter.relations.main.resolve_qualified_name_parts import ( + resolve_qualified_name_parts, +) from sqlbuild.executor.maintained_pauses.classes.maintained_hold import MaintainedHold from sqlbuild.executor.run._helpers.materializations.view import execute_in_place_entry +from sqlbuild.executor.run.constants import REPLACED_RELATION_SUFFIX from sqlbuild.executor.run.models import ModelExecutionResult, ModelMaterializationContext _NO_WAREHOUSE_CODE: str = "A717" @@ -61,7 +70,7 @@ def materialize(*, target_qualified: str, statement_recorder: StatementRecorder) def _create( *, context: ModelMaterializationContext, destination: str, recorder: StatementRecorder ) -> None: - """Create or replace; a table or view the model was before is dropped, never a dynamic table.""" + """Create or replace; a table or view the model was before is swapped out, never a DT.""" entry: ModelPlanEntry = context.entry adapter: BaseAdapter = context.adapter @@ -90,16 +99,55 @@ def _create( ) if former is None: raise error - drop: Callable[..., None] = ( - adapter.drop_view if former == RelationType.VIEW else adapter.drop + _replace_former( + context=context, + destination=destination, + statements=statements, + former=former, + recorder=recorder, ) - drop( + + +def _replace_former( + *, + context: ModelMaterializationContext, + destination: str, + statements: tuple[str, ...], + former: RelationType, + recorder: StatementRecorder, +) -> None: + """Move the former table or view aside, create, then drop it; a failed create restores it.""" + + adapter: BaseAdapter = context.adapter + connection: Any = context.connection + location: CompiledRelationLocation = context.entry.destination + aside: str = resolve_qualified_name_parts( + adapter=adapter, + database=location.database, + schema=location.schema, + name=fit_auxiliary_relation_name( + base_name=location.name, + suffix=REPLACED_RELATION_SUFFIX, + identifier_limit=adapter.maximum_identifier_length(), + ), + ) + view: bool = former == RelationType.VIEW + rename: Callable[..., None] = adapter.rename_view if view else adapter.rename + drop: Callable[..., None] = adapter.drop_view if view else adapter.drop + rename( + connection=connection, origin=destination, destination=aside, statement_recorder=recorder + ) + try: + _run(adapter=adapter, connection=connection, statements=statements, recorder=recorder) + except Exception: + rename( connection=connection, + origin=aside, destination=destination, - if_exists=True, statement_recorder=recorder, ) - _run(adapter=adapter, connection=connection, statements=statements, recorder=recorder) + raise + drop(connection=connection, destination=aside, if_exists=True, statement_recorder=recorder) def _former_relation( diff --git a/src/sqlbuild/executor/run/constants.py b/src/sqlbuild/executor/run/constants.py index d5486ad907..397bb1587e 100644 --- a/src/sqlbuild/executor/run/constants.py +++ b/src/sqlbuild/executor/run/constants.py @@ -21,6 +21,7 @@ SNAPSHOT_ALL_CHECK_COLUMNS: tuple[str, ...] = ("*",) STAGING_RELATION_SUFFIX: str = "__staging" +REPLACED_RELATION_SUFFIX: str = "__replaced" TYPE_ENFORCEMENT_RELATION_SUFFIX: str = "__enforced" DELTA_RELATION_SUFFIX: str = "__delta" SNAPSHOT_DELTA_RELATION_SUFFIX: str = "__snapshot_delta" diff --git a/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py b/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py index 178ee9fae0..b13f754f11 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py @@ -878,15 +878,23 @@ def maintained_project_case(upstream_query: str) -> BuildExecutionPlanTestCase: def change_tracking_project_case() -> BuildExecutionPlanTestCase: - """Orders and customers tables; daily totals is maintained from orders alone.""" + """Daily totals is maintained from orders, and from payments through a view; not customers.""" return BuildExecutionPlanTestCase( - description="tables that a maintained model reads, and one it does not", + description="tables a maintained model reads, directly or via a view, and one it does not", setup_sql=(), - model_locations={"orders": "staging", "customers": "staging", "daily_totals": "staging"}, + model_locations={ + "orders": "staging", + "customers": "staging", + "payments": "staging", + "settled_payments": "staging", + "daily_totals": "staging", + }, model_configs={ "orders": {"materialized": "table"}, "customers": {"materialized": "table"}, + "payments": {"materialized": "table"}, + "settled_payments": {"materialized": "view"}, "daily_totals": { "materialized": "maintained", "refresh": {"lag": "1h"}, @@ -896,9 +904,17 @@ def change_tracking_project_case() -> BuildExecutionPlanTestCase: model_queries={ "orders": "SELECT 1 AS order_id", "customers": "SELECT 1 AS customer_id", - "daily_totals": "SELECT count(*) AS orders FROM staging.orders", + "payments": "SELECT 1 AS order_id", + "settled_payments": "SELECT order_id FROM staging.payments", + "daily_totals": ( + "SELECT count(*) AS orders FROM staging.orders " + "JOIN staging.settled_payments USING (order_id)" + ), + }, + model_deps={ + "daily_totals": ("orders", "settled_payments"), + "settled_payments": ("payments",), }, - model_deps={"daily_totals": ("orders",)}, full_refresh=False, expected_action={}, expected_reason={}, diff --git a/tests/integration/src/sqlbuild/compiler/planner/main/test_maintained_refresh_plan.py b/tests/integration/src/sqlbuild/compiler/planner/main/test_maintained_refresh_plan.py index 1703f38efb..351c65275e 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/main/test_maintained_refresh_plan.py +++ b/tests/integration/src/sqlbuild/compiler/planner/main/test_maintained_refresh_plan.py @@ -131,20 +131,32 @@ def test_given_built_maintained_models_when_replanning_then_live_settings_and_re "test_case", [ ChangeTrackingPlanTestCase( - description="a table a maintained model reads is built with change tracking", + description="a table a maintained model reads, directly or through a view, is tracked", select=(), expected_maintained_readers={ "orders": ("daily_totals",), "customers": (), + "payments": ("daily_totals",), + "settled_payments": ("daily_totals",), "daily_totals": (), }, - expected_refresh_on_build={"orders": False, "customers": False, "daily_totals": True}, + expected_refresh_on_build={ + "orders": False, + "customers": False, + "payments": False, + "settled_payments": False, + "daily_totals": True, + }, ), ChangeTrackingPlanTestCase( description="the reader need not be selected for its upstream to be tracked", - select=("orders", "customers"), - expected_maintained_readers={"orders": ("daily_totals",), "customers": ()}, - expected_refresh_on_build={"orders": False, "customers": False}, + select=("orders", "customers", "payments"), + expected_maintained_readers={ + "orders": ("daily_totals",), + "customers": (), + "payments": ("daily_totals",), + }, + expected_refresh_on_build={"orders": False, "customers": False, "payments": False}, ), ], ids=lambda case: case.description, diff --git a/tests/integration/src/sqlbuild/executor/maintained_pauses/_test_types.py b/tests/integration/src/sqlbuild/executor/maintained_pauses/_test_types.py index acc075ecf3..69de1e88eb 100644 --- a/tests/integration/src/sqlbuild/executor/maintained_pauses/_test_types.py +++ b/tests/integration/src/sqlbuild/executor/maintained_pauses/_test_types.py @@ -44,6 +44,21 @@ class FirstBuildTestCase: expected_states: dict[str, str] +@dataclass(frozen=True) +class TakenOverReleaseTestCase: + description: str + expected_release_operations: tuple[tuple[str, str], ...] + expected_states: dict[str, str] + expected_events: tuple[tuple[str, str, str, str], ...] + + +@dataclass(frozen=True) +class LifecycleLineTestCase: + description: str + states_before_second: dict[str, str] + expected_line: str + + @dataclass(frozen=True) class HeartbeatTestCase: description: str diff --git a/tests/integration/src/sqlbuild/executor/maintained_pauses/helpers.py b/tests/integration/src/sqlbuild/executor/maintained_pauses/helpers.py index 62e5187c79..f956db03e5 100644 --- a/tests/integration/src/sqlbuild/executor/maintained_pauses/helpers.py +++ b/tests/integration/src/sqlbuild/executor/maintained_pauses/helpers.py @@ -14,7 +14,11 @@ ) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.executor.build.main._execute import execute_build_plan -from sqlbuild.executor.build.models import BuildExecutionResult, BuildRuntimeParams +from sqlbuild.executor.build.models import ( + BuildCallbacks, + BuildExecutionResult, + BuildRuntimeParams, +) from tests.integration.src.sqlbuild.compiler.planner.main._test_types import ( BuildExecutionPlanTestCase, ) @@ -134,8 +138,9 @@ def build_lease_project( runtime: BuildRuntimeParams, database_path: Path, select: tuple[str, ...] = (), + callbacks: BuildCallbacks | None = None, ) -> BuildExecutionResult: - """Plan the selected models, then build them with `runtime`.""" + """Plan the selected models, then build them with `runtime` and `callbacks`.""" return execute_build_plan( plan=plan_lease_project(adapter=adapter, connection=connection, case=case, select=select), @@ -147,6 +152,7 @@ def build_lease_project( runtime, promotion_mode=TablePromotionMode(adapter.default_table_promotion_mode()), ), + callbacks=callbacks, ) diff --git a/tests/integration/src/sqlbuild/executor/maintained_pauses/test_maintained_pause_lease.py b/tests/integration/src/sqlbuild/executor/maintained_pauses/test_maintained_pause_lease.py index c1763d9f17..523c09225e 100644 --- a/tests/integration/src/sqlbuild/executor/maintained_pauses/test_maintained_pause_lease.py +++ b/tests/integration/src/sqlbuild/executor/maintained_pauses/test_maintained_pause_lease.py @@ -8,8 +8,13 @@ import pytest +from sqlbuild._native import PlanOutput from sqlbuild.adapter.contract.exceptions import AdapterUserError -from sqlbuild.executor.build.models import BuildExecutionResult, BuildRuntimeParams +from sqlbuild.executor.build.models import ( + BuildCallbacks, + BuildExecutionResult, + BuildRuntimeParams, +) from sqlbuild.executor.build.types import BuildStatus from sqlbuild.executor.maintained_pauses.classes.maintained_hold import MaintainedHold from sqlbuild.executor.maintained_pauses.main.hold_maintained_models import ( @@ -20,7 +25,9 @@ HeartbeatTestCase, HoldRefusalTestCase, HoldTakeoverTestCase, + LifecycleLineTestCase, PauseLeaseTestCase, + TakenOverReleaseTestCase, ) from tests.integration.src.sqlbuild.executor.maintained_pauses.helpers import ( TOTALS_QUERY, @@ -438,3 +445,111 @@ def test_given_held_models_when_build_runs_then_heartbeat_writes_on_its_own_conn "WHERE event_type = 'heartbeat' AND run_id = 'beating_run'" ).fetchall() assert beats[0][0] >= test_case.expected_min_beats + + +@pytest.mark.parametrize( + "test_case", + [ + LifecycleLineTestCase( + description="a model suspended outside SQLBuild is reported on the build's output", + states_before_second={"daily_totals": "SUSPENDED"}, + expected_line=( + "Warning: maintained model daily_totals was paused outside SQLBuild; " + "it stays paused" + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_build_output_when_hold_reports_then_line_reaches_the_lifecycle_writer( + test_case: LifecycleLineTestCase, + adapter: PausingDuckDbAdapter, + connection: Any, + database_path: Path, +) -> None: + _ = build_lease_project( + adapter=adapter, + connection=connection, + case=lease_project_case(orders_query=_ORDERS, report_locations={}, refresh_on_build=False), + runtime=BuildRuntimeParams(run_id="first_run"), + database_path=database_path, + ) + adapter.states.update(test_case.states_before_second) + lines: list[str] = [] + + result: BuildExecutionResult = build_lease_project( + adapter=adapter, + connection=connection, + case=lease_project_case( + orders_query=_MORE_ORDERS, report_locations={}, refresh_on_build=False + ), + runtime=BuildRuntimeParams(run_id="second_run"), + database_path=database_path, + callbacks=BuildCallbacks(on_lifecycle_progress=lines.append), + ) + + assert result.status == BuildStatus.SUCCESS + assert test_case.expected_line in lines + + +@pytest.mark.parametrize( + "test_case", + [ + TakenOverReleaseTestCase( + description="a build whose hold was taken over leaves the model to the new holder", + expected_release_operations=(), + expected_states={"daily_totals": "SUSPENDED"}, + expected_events=( + ("paused", "first_run", "first_run", "daily_totals"), + ("paused", "second_run", "second_run", "daily_totals"), + ("paused", "slow_run", "slow_run", "daily_totals"), + ("resumed", "first_run", "first_run", "daily_totals"), + ("resumed", "slow_run", "second_run", "daily_totals"), + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_hold_taken_over_when_first_build_releases_then_it_resumes_nothing( + test_case: TakenOverReleaseTestCase, + adapter: PausingDuckDbAdapter, + connection: Any, + database_path: Path, +) -> None: + _ = build_lease_project( + adapter=adapter, + connection=connection, + case=lease_project_case(orders_query=_ORDERS, report_locations={}, refresh_on_build=False), + runtime=BuildRuntimeParams(run_id="first_run"), + database_path=database_path, + ) + changed: LeaseProject = lease_project_case( + orders_query=_MORE_ORDERS, report_locations={}, refresh_on_build=False + ) + plan: PlanOutput = plan_lease_project(adapter=adapter, connection=connection, case=changed) + holds: list[MaintainedHold] = [] + for runtime in ( + BuildRuntimeParams(run_id="slow_run"), + BuildRuntimeParams( + run_id="second_run", + ignore_maintained_grace=True, + allow_ignore_maintained_grace=True, + ), + ): + hold: MaintainedHold | None = hold_maintained_models( + plan=plan, + adapter=adapter, + runtime=runtime, + connection_config={"database": str(database_path)}, + ) + assert hold is not None + hold.acquire(connection=connection, on_progress=None) + holds.append(hold) + adapter.operations.clear() + + holds[0].release(connection=connection) + + assert tuple(adapter.operations) == test_case.expected_release_operations + assert adapter.states == test_case.expected_states + assert pause_events(connection=connection) == sorted(test_case.expected_events) + holds[1].release(connection=connection) diff --git a/tests/integration/src/sqlbuild/executor/run/maintained/_test_types.py b/tests/integration/src/sqlbuild/executor/run/maintained/_test_types.py index efe049ac4f..b199a9d5d6 100644 --- a/tests/integration/src/sqlbuild/executor/run/maintained/_test_types.py +++ b/tests/integration/src/sqlbuild/executor/run/maintained/_test_types.py @@ -23,6 +23,7 @@ class MaintainedFailureTestCase: setup_sql: tuple[str, ...] session_warehouse: str | None listed_settings: MaintainedListing | None + failing_query: str | None expected_error_code: str expected_listed_types: tuple[tuple[str], ...] diff --git a/tests/integration/src/sqlbuild/executor/run/maintained/helpers.py b/tests/integration/src/sqlbuild/executor/run/maintained/helpers.py index 7c216b915b..a45b76e0ee 100644 --- a/tests/integration/src/sqlbuild/executor/run/maintained/helpers.py +++ b/tests/integration/src/sqlbuild/executor/run/maintained/helpers.py @@ -26,11 +26,16 @@ class MaintainedDuckDbAdapter(DuckDbAdapter): """Builds a maintained model as a table that, like a dynamic table, replaces nothing else.""" def __init__( - self, *, session_warehouse: str | None, listed_settings: MaintainedListing | None = None + self, + *, + session_warehouse: str | None, + listed_settings: MaintainedListing | None = None, + failing_query: str | None = None, ) -> None: super().__init__() self.warehouse: str | None = session_warehouse self.listed_settings: MaintainedListing | None = listed_settings + self.failing_query: str | None = failing_query self.created: list[tuple[str, str, str]] = [] self.altered: list[tuple[str, str | None]] = [] @@ -48,7 +53,13 @@ def render_create_maintained( self, *, destination: str, sql: str, target_lag: str, refresh_mode: str, warehouse: str ) -> tuple[str, ...]: self.created.append((target_lag, refresh_mode, warehouse)) - return (f"CREATE TABLE {destination} AS {sql}",) + return (f"CREATE TABLE {destination} AS {self.failing_query or sql}",) + + def render_rename(self, *, origin: str, destination: str) -> tuple[str, ...]: + return (f"ALTER TABLE {origin} RENAME TO {destination.split('.')[-1]}",) + + def render_rename_view(self, *, origin: str, destination: str) -> tuple[str, ...]: + return (f"ALTER VIEW {origin} RENAME TO {destination.split('.')[-1]}",) def render_alter_maintained( self, *, destination: str, target_lag: str, warehouse: str | None diff --git a/tests/integration/src/sqlbuild/executor/run/maintained/test_maintained_execution.py b/tests/integration/src/sqlbuild/executor/run/maintained/test_maintained_execution.py index 86b17d3c3c..ecfcdbb579 100644 --- a/tests/integration/src/sqlbuild/executor/run/maintained/test_maintained_execution.py +++ b/tests/integration/src/sqlbuild/executor/run/maintained/test_maintained_execution.py @@ -110,6 +110,7 @@ def test_given_maintained_entry_when_executing_then_settings_reach_the_warehouse setup_sql=(), session_warehouse=None, listed_settings=None, + failing_query=None, expected_error_code="A717", expected_listed_types=(), ), @@ -118,6 +119,25 @@ def test_given_maintained_entry_when_executing_then_settings_reach_the_warehouse setup_sql=("CREATE VIEW staging.daily_totals AS SELECT 1 AS orders",), session_warehouse="SESSION_WH", listed_settings=MaintainedListing("1 hour", "SESSION_WH", "ACTIVE"), + failing_query=None, + expected_error_code="R002", + expected_listed_types=(("VIEW",),), + ), + MaintainedFailureTestCase( + description="a table the model was before is restored when the create keeps failing", + setup_sql=("CREATE TABLE staging.daily_totals AS SELECT 1 AS orders",), + session_warehouse="SESSION_WH", + listed_settings=None, + failing_query="SELECT missing_column FROM staging.no_such_table", + expected_error_code="R002", + expected_listed_types=(("BASE TABLE",),), + ), + MaintainedFailureTestCase( + description="a view the model was before is restored when the create keeps failing", + setup_sql=("CREATE VIEW staging.daily_totals AS SELECT 1 AS orders",), + session_warehouse="SESSION_WH", + listed_settings=None, + failing_query="SELECT missing_column FROM staging.no_such_table", expected_error_code="R002", expected_listed_types=(("VIEW",),), ), @@ -128,7 +148,9 @@ def test_given_unbuildable_maintained_entry_when_executing_then_fails_keeping_th test_case: MaintainedFailureTestCase, connection: Any ) -> None: adapter: MaintainedDuckDbAdapter = MaintainedDuckDbAdapter( - session_warehouse=test_case.session_warehouse, listed_settings=test_case.listed_settings + session_warehouse=test_case.session_warehouse, + listed_settings=test_case.listed_settings, + failing_query=test_case.failing_query, ) for sql in test_case.setup_sql: connection.execute(sql) diff --git a/website/src/content/docs/docs/concepts/adapters/snowflake.mdx b/website/src/content/docs/docs/concepts/adapters/snowflake.mdx index 946b866cc9..8dddc56ef3 100644 --- a/website/src/content/docs/docs/concepts/adapters/snowflake.mdx +++ b/website/src/content/docs/docs/concepts/adapters/snowflake.mdx @@ -165,7 +165,7 @@ The build role needs these grants: Dynamic tables refresh from changes, which needs change tracking on every table they read: -- A table model that a maintained model reads is built with change tracking on. +- A table model that a maintained model reads, directly or through views, is built with change tracking on. - For a source table, Snowflake turns change tracking on when the build role owns the table. Otherwise its owner runs `ALTER TABLE SET CHANGE_TRACKING = TRUE`. diff --git a/website/src/content/docs/docs/concepts/models/materializations.mdx b/website/src/content/docs/docs/concepts/models/materializations.mdx index 74626c8422..8640445d44 100644 --- a/website/src/content/docs/docs/concepts/models/materializations.mdx +++ b/website/src/content/docs/docs/concepts/models/materializations.mdx @@ -112,7 +112,7 @@ A model the plan keeps can carry one more line: ### During a build -A build pauses the maintained models it changes, and those that read a model it rebuilds, so they never refresh from half-built inputs. It resumes them when it ends, whether it succeeds or fails. +A build pauses the maintained models it changes, and those that read a model it rebuilds, directly or through views, so they never refresh from half-built inputs. It resumes them when it ends, whether it succeeds or fails. - Dynamic tables outside the project that read a paused model pause too, until the build resumes it. One that someone had suspended before stays suspended. - A model that was already suspended when the build started stays suspended, and the build warns about it. @@ -140,6 +140,7 @@ Two builds can't pause the same models at once. A build that needs a model anoth - Writing out the default `refresh_mode incremental` changes the model's definition, so the next build recreates it once. - A custom materialization named `maintained` is not used; the built-in one takes the name. - Unit tests and scenarios treat a maintained model as a table. +- On a target with `schema = "preserve"`, builds see each other's pauses only when their maintained models share a schema. - A snapshot rebuilt in full is created without change tracking. Before a maintained model reads it again, run `ALTER TABLE SET CHANGE_TRACKING = TRUE`. ## Custom diff --git a/website/src/content/docs/docs/concepts/project-configuration.mdx b/website/src/content/docs/docs/concepts/project-configuration.mdx index fe4b1cb8d1..ae56abf4ce 100644 --- a/website/src/content/docs/docs/concepts/project-configuration.mdx +++ b/website/src/content/docs/docs/concepts/project-configuration.mdx @@ -682,7 +682,7 @@ allow_ignore_grace = false | Field | Default | Description | |-------|---------|-------------| -| `heartbeat_grace` | `"30m"` | How long a build that paused maintained models may go silent before another build resumes them or takes them over. A fixed duration such as `"30m"` or `"2h"` | +| `heartbeat_grace` | `"30m"` | How long a build that paused maintained models may go silent before another build resumes them or takes them over. A fixed duration of at least `"3m"`, such as `"30m"` or `"2h"`; a build writes a heartbeat every minute | | `allow_ignore_grace` | `false` | Allow `sqb build --ignore-maintained-grace`, which takes over models another build holds without waiting for the grace. Without it the flag fails with `A719` | ## Scenario From 402ff4d7f2702d0327ff2e0a768008d878a9810e Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 19 Nov 2026 01:58:33 +0000 Subject: [PATCH 13/17] docs: note that a maintained model's lag can't undercut its maintained inputs --- .../references/docs/concepts/models/materializations.md | 2 +- .../src/content/docs/docs/concepts/models/materializations.mdx | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/materializations.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/materializations.md index 7e009f8747..02a8ed69d1 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/materializations.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/materializations.md @@ -102,7 +102,7 @@ GROUP BY customer_id, CAST(ordered_at AS DATE) | Field | Default | Description | |-------|---------|-------------| -| `refresh (lag )` | none | How far behind its inputs the model may fall: a fixed duration such as `5m`, `1h` or `1d`, at least `1m` on Snowflake. Without it, the model refreshes when a maintained model that reads it refreshes | +| `refresh (lag )` | none | How far behind its inputs the model may fall: a fixed duration such as `5m`, `1h` or `1d`, at least `1m` on Snowflake and no shorter than the lag of a maintained model it reads. Without it, the model refreshes when a maintained model that reads it refreshes | | `refresh_mode` | `incremental` | `incremental`, `full`, `auto` or `adaptive`. Snowflake rejects `incremental` for a query it can't refresh incrementally; use `full` for that query | | `refresh_warehouse` | the connection's warehouse | Warehouse the scheduled refreshes run on | | `refresh_on_build` | `false` | Refresh the model in every build that selects it, not only on its schedule | diff --git a/website/src/content/docs/docs/concepts/models/materializations.mdx b/website/src/content/docs/docs/concepts/models/materializations.mdx index 8640445d44..cabfec288e 100644 --- a/website/src/content/docs/docs/concepts/models/materializations.mdx +++ b/website/src/content/docs/docs/concepts/models/materializations.mdx @@ -89,7 +89,7 @@ GROUP BY customer_id, CAST(ordered_at AS DATE) | Field | Default | Description | |-------|---------|-------------| -| `refresh (lag )` | none | How far behind its inputs the model may fall: a fixed duration such as `5m`, `1h` or `1d`, at least `1m` on Snowflake. Without it, the model refreshes when a maintained model that reads it refreshes | +| `refresh (lag )` | none | How far behind its inputs the model may fall: a fixed duration such as `5m`, `1h` or `1d`, at least `1m` on Snowflake and no shorter than the lag of a maintained model it reads. Without it, the model refreshes when a maintained model that reads it refreshes | | `refresh_mode` | `incremental` | `incremental`, `full`, `auto` or `adaptive`. Snowflake rejects `incremental` for a query it can't refresh incrementally; use `full` for that query | | `refresh_warehouse` | the connection's warehouse | Warehouse the scheduled refreshes run on | | `refresh_on_build` | `false` | Refresh the model in every build that selects it, not only on its schedule | From 933e07de102116d815b517817c57dde86c361c29 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 19 Nov 2026 02:09:09 +0000 Subject: [PATCH 14/17] chore(fensu): allow the graph walk mirrored in layering tooling and the maintained hold --- fensu.toml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/fensu.toml b/fensu.toml index 9fec3e7dee..2a1becc261 100644 --- a/fensu.toml +++ b/fensu.toml @@ -1351,6 +1351,10 @@ rule = "FFR601" path = "src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py" reason = "The SQL Server adapter intentionally keeps overrideable backend behavior on one class." +[[dupes.allowlist]] +paths = ["scripts/native_layering/_helpers/layering.py", "src/sqlbuild/executor/maintained_pauses/_helpers/scope.py"] +reason = "A ten-line depth-first walk in two packages that must not import each other: the dependency-layering tool walks Cargo.lock packages, the build's maintained-model hold walks plan edges between models." + [[dupes.allowlist]] paths = ["crates/sqlbuild-python-planner/src/planner_changes/_helpers/detect.rs", "crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/python_selection.rs"] reason = "One reads a column's name and type for schema-change detection, the other the kind and name of each SQL resource a Python node reads; only the two-field record loop is alike." From f6c4ce0d6690ced1c29e6ce61b71332aa7f54fd3 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 19 Nov 2026 03:01:19 +0000 Subject: [PATCH 15/17] fix(build): settle concurrent maintained pauses and keep both errors of a failed restore - Only a heartbeat_grace no longer than the 60s heartbeat interval is rejected. - After recording its pauses a build reads them back; the earliest live hold on a model wins, and a later build closes its hold and stops with A718 before suspending anything. - When moving a former table or view back fails, the error reports both the create failure and the restore failure. --- .../src/project/_helpers/sections.rs | 19 +++-- .../sqlbuild-config/src/project/constants.rs | 4 +- .../src/project/tests/test_load_config.rs | 12 +-- .../docs/concepts/models/materializations.md | 2 +- .../docs/concepts/project-configuration.md | 2 +- .../maintained_pauses/_helpers/sql.py | 4 +- .../classes/maintained_hold.py | 33 ++++++++ .../executor/maintained_pauses/models.py | 4 +- .../_helpers/materializations/maintained.py | 26 +++++-- .../executor/maintained_pauses/_test_types.py | 9 +++ .../test_maintained_pause_lease.py | 77 +++++++++++++++++++ .../executor/run/maintained/_test_types.py | 11 +++ .../executor/run/maintained/helpers.py | 5 +- .../maintained/test_maintained_execution.py | 50 ++++++++++++ .../docs/concepts/models/materializations.mdx | 2 +- .../docs/concepts/project-configuration.mdx | 2 +- 16 files changed, 229 insertions(+), 33 deletions(-) diff --git a/crates/sqlbuild-config/src/project/_helpers/sections.rs b/crates/sqlbuild-config/src/project/_helpers/sections.rs index c944e56106..be75473f5f 100644 --- a/crates/sqlbuild-config/src/project/_helpers/sections.rs +++ b/crates/sqlbuild-config/src/project/_helpers/sections.rs @@ -22,14 +22,13 @@ use crate::project::constants::{ AUDIT_SEVERITIES, CHECK_DECLARED_COLUMNS_KEY, COLLECTION_RENDERINGS, COLUMN_CONTRACT_MODES, CONCURRENCY_KEY, DBT_DEFER_CLONE_KEY, DBT_KEYS, DBT_REMOVED_KEYS, DBT_REUSE_KEYS, DEFAULT_HEARTBEAT_GRACE_SECONDS, DEFAULT_IN_USE_WINDOW, DEFAULT_QUERY_ARTIFACT_TTL, - FUTURE_CURSOR_ACTIONS, LEGACY_COLUMN_CONTRACT_MODE_KEY, LEGACY_CONCURRENCY_KEY, - LEGACY_SQL_VALIDATION_KEY, LOCAL_DBT_KEYS, MAINTAINED_KEYS, MAX_BATCHES_KEY, - MICROBATCH_LIMIT_ACTIONS, MIN_HEARTBEAT_GRACE_SECONDS, OLD_NAME_VIEWS_KEY, - QUERY_CHANGE_TRACKING_KEY, REQUIRE_SQL_ANALYSIS_KEY, REVIEW_DOWNSTREAM_POLICIES, REVIEW_KEYS, - RUN_NAMESPACE_KEY, RUN_NAMESPACE_MAX_LENGTH, SCENARIO_KEYS, SETTINGS_KEYS, - SNAPSHOT_FULL_REFRESH_POLICIES, SNAPSHOT_KEYS, SNAPSHOT_LIMIT_KEYS, - SNAPSHOT_SCHEMA_CHANGE_POLICIES, SQL_ANALYSIS_KEY, UNACCOUNTED_PARTITION_POLICIES, - USD_PER_CREDIT_KEY, ZERO_FIXED_DURATION, ZERO_FLOAT, + FUTURE_CURSOR_ACTIONS, HEARTBEAT_INTERVAL_SECONDS, LEGACY_COLUMN_CONTRACT_MODE_KEY, + LEGACY_CONCURRENCY_KEY, LEGACY_SQL_VALIDATION_KEY, LOCAL_DBT_KEYS, MAINTAINED_KEYS, + MAX_BATCHES_KEY, MICROBATCH_LIMIT_ACTIONS, OLD_NAME_VIEWS_KEY, QUERY_CHANGE_TRACKING_KEY, + REQUIRE_SQL_ANALYSIS_KEY, REVIEW_DOWNSTREAM_POLICIES, REVIEW_KEYS, RUN_NAMESPACE_KEY, + RUN_NAMESPACE_MAX_LENGTH, SCENARIO_KEYS, SETTINGS_KEYS, SNAPSHOT_FULL_REFRESH_POLICIES, + SNAPSHOT_KEYS, SNAPSHOT_LIMIT_KEYS, SNAPSHOT_SCHEMA_CHANGE_POLICIES, SQL_ANALYSIS_KEY, + UNACCOUNTED_PARTITION_POLICIES, USD_PER_CREDIT_KEY, ZERO_FIXED_DURATION, ZERO_FLOAT, }; use crate::project::constants::{ CHANGE_COUNTS, CHANGE_REACHES, CHANGES_KEYS, CONTROL_KEYS, MAX_POOL_LIMIT, @@ -644,10 +643,10 @@ pub(crate) fn load_maintained( }, Some(_) => return Err(grace_failure(file)), }; - if heartbeat_grace_seconds < MIN_HEARTBEAT_GRACE_SECONDS { + if heartbeat_grace_seconds <= HEARTBEAT_INTERVAL_SECONDS { return Err(ConfigFailure::at( file, - "maintained.heartbeat_grace must be at least 3m, three of a build's one-minute heartbeats", + "maintained.heartbeat_grace must be longer than 1m: a build writes a heartbeat every minute, so a shorter grace expires builds that are still running", )); } let allow_ignore_grace: bool = match get(mapping, "allow_ignore_grace") { diff --git a/crates/sqlbuild-config/src/project/constants.rs b/crates/sqlbuild-config/src/project/constants.rs index 7059fcc081..1668872450 100644 --- a/crates/sqlbuild-config/src/project/constants.rs +++ b/crates/sqlbuild-config/src/project/constants.rs @@ -242,8 +242,8 @@ pub(crate) const SYNC_INTERVAL_HELP: &str = pub(crate) const MAINTAINED_KEYS: [&str; 2] = ["allow_ignore_grace", "heartbeat_grace"]; /// How long a paused maintained model's heartbeat may be silent before another build takes over. pub(crate) const DEFAULT_HEARTBEAT_GRACE_SECONDS: u64 = 1_800; -/// Three of a build's one-minute heartbeats, so a healthy build never looks silent. -pub(crate) const MIN_HEARTBEAT_GRACE_SECONDS: u64 = 180; +/// How often a build that holds maintained models writes a heartbeat; a grace must exceed it. +pub(crate) const HEARTBEAT_INTERVAL_SECONDS: u64 = 60; pub(crate) const REVIEW_KEYS: [&str; 4] = ["against", "critical_tags", "downstream", "schema_prefix"]; pub(crate) const REVIEW_DOWNSTREAM_POLICIES: [&str; 3] = ["critical", "all", "none"]; diff --git a/crates/sqlbuild-config/src/project/tests/test_load_config.rs b/crates/sqlbuild-config/src/project/tests/test_load_config.rs index 7e86023a6f..aa14ab21bd 100644 --- a/crates/sqlbuild-config/src/project/tests/test_load_config.rs +++ b/crates/sqlbuild-config/src/project/tests/test_load_config.rs @@ -88,12 +88,12 @@ fn given_project_files_when_loading_then_rejections_match_python_loader() { )), }, ProjectFailureTestCase { - description: "a grace shorter than three heartbeats would expire a healthy build", - project: "name = 'orders'\nadapter = 'snowflake'\n[maintained]\nheartbeat_grace = '2m'\n", + description: "a grace no longer than the heartbeat interval would expire every build", + project: "name = 'orders'\nadapter = 'snowflake'\n[maintained]\nheartbeat_grace = '1m'\n", local: "", expected_failure: Some(ConfigFailure::at( ConfigFile::Project, - "maintained.heartbeat_grace must be at least 3m, three of a build's one-minute heartbeats", + "maintained.heartbeat_grace must be longer than 1m: a build writes a heartbeat every minute, so a shorter grace expires builds that are still running", )), }, ProjectFailureTestCase { @@ -645,11 +645,11 @@ fn given_maintained_section_when_reading_then_returns_its_values_or_defaults() { }, }, MaintainedConfigTestCase { - description: "the shortest grace, three heartbeats", + description: "any grace longer than the heartbeat interval", project: "name = 'orders'\nadapter = 'snowflake'\n[maintained]\n\ - heartbeat_grace = '3m'\n", + heartbeat_grace = '61s'\n", expected_config: MaintainedConfig { - heartbeat_grace_seconds: 180, + heartbeat_grace_seconds: 61, allow_ignore_grace: false, }, }, diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/materializations.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/materializations.md index 02a8ed69d1..a6c8c4d203 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/materializations.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/materializations.md @@ -133,7 +133,7 @@ A build pauses the maintained models it changes, and those that read a model it - A kept model is refreshed during the build when a model built later in the same build reads it, so that model reads current rows. `refresh_on_build` refreshes it in every build. - A target with `pause_maintained = true` leaves its maintained models suspended between builds and refreshes them only in builds; see [Targets](../project-configuration.md#targets). -Two builds can't pause the same models at once. A build that needs a model another build holds fails with `A718`, naming that run. A build that stops without resuming, for example because its process was killed, leaves its models suspended. The next build that pauses maintained models on the target resumes them once that build has been silent for `[maintained] heartbeat_grace` (see [Maintained](../project-configuration.md#maintained)). Until then, resume them in Snowflake by hand, or run `sqb build --ignore-maintained-grace` where the project allows it. +Two builds can't pause the same models at once. A build that needs a model another build holds fails with `A718`, naming that run. When two builds pause a model at the same moment, the one that recorded its pause first keeps it, and the other fails with `A718` without suspending or resuming anything. A build that stops without resuming, for example because its process was killed, leaves its models suspended. The next build that pauses maintained models on the target resumes them once that build has been silent for `[maintained] heartbeat_grace` (see [Maintained](../project-configuration.md#maintained)). Until then, resume them in Snowflake by hand, or run `sqb build --ignore-maintained-grace` where the project allows it. ### Error codes diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md index 6dfb1072e7..38ca3a22f6 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md @@ -709,7 +709,7 @@ allow_ignore_grace = false | Field | Default | Description | |-------|---------|-------------| -| `heartbeat_grace` | `"30m"` | How long a build that paused maintained models may go silent before another build resumes them or takes them over. A fixed duration of at least `"3m"`, such as `"30m"` or `"2h"`; a build writes a heartbeat every minute | +| `heartbeat_grace` | `"30m"` | How long a build that paused maintained models may go silent before another build resumes them or takes them over. A fixed duration longer than `"1m"`, such as `"30m"` or `"2h"`; a build writes a heartbeat every minute | | `allow_ignore_grace` | `false` | Allow `sqb build --ignore-maintained-grace`, which takes over models another build holds without waiting for the grace. Without it the flag fails with `A719` | ## Scenario diff --git a/src/sqlbuild/executor/maintained_pauses/_helpers/sql.py b/src/sqlbuild/executor/maintained_pauses/_helpers/sql.py index 35a7711180..1a83fb5548 100644 --- a/src/sqlbuild/executor/maintained_pauses/_helpers/sql.py +++ b/src/sqlbuild/executor/maintained_pauses/_helpers/sql.py @@ -84,7 +84,8 @@ def open_holds_sql(*, adapter: BaseAdapter, location: StateLocation) -> str: ) return ( f"SELECT p.{COLUMN_RUN_ID}, p.{COLUMN_RELATION_DATABASE}, p.{COLUMN_RELATION_SCHEMA}, " - f"p.{COLUMN_RELATION_NAME}, CAST(h.last_seen AS TIMESTAMP), CAST({now} AS TIMESTAMP) " + f"p.{COLUMN_RELATION_NAME}, CAST(h.last_seen AS TIMESTAMP), CAST({now} AS TIMESTAMP), " + f"CAST(p.{COLUMN_RECORDED_AT} AS TIMESTAMP) " f"FROM {table} p JOIN (SELECT {COLUMN_RUN_ID}, MAX({COLUMN_RECORDED_AT}) AS last_seen " f"FROM {table} WHERE {COLUMN_RECORDED_BY} = {COLUMN_RUN_ID} GROUP BY {COLUMN_RUN_ID}) h " f"ON h.{COLUMN_RUN_ID} = p.{COLUMN_RUN_ID} " @@ -108,6 +109,7 @@ def open_holds(*, rows: tuple[tuple[Any, ...], ...]) -> tuple[OpenHold, ...]: qualified_name=None, ), idle_seconds=_seconds_between(start=row[4], end=row[5]), + paused_at=row[6].replace(tzinfo=None), ) for row in rows ) diff --git a/src/sqlbuild/executor/maintained_pauses/classes/maintained_hold.py b/src/sqlbuild/executor/maintained_pauses/classes/maintained_hold.py index c1acf55645..9a5c14654b 100644 --- a/src/sqlbuild/executor/maintained_pauses/classes/maintained_hold.py +++ b/src/sqlbuild/executor/maintained_pauses/classes/maintained_hold.py @@ -124,6 +124,7 @@ def acquire(self, *, connection: Any, on_progress: Callable[[str], None] | None) ), ), ) + self._yield_to_earlier(connection=connection, location=location) self._suspend_active(connection=connection, names=tuple(self._paused), states=states) self._heartbeat = self._new_heartbeat(self._write_heartbeat) self._heartbeat.start() @@ -217,6 +218,38 @@ def _take_over(self, *, connection: Any, location: StateLocation) -> tuple[OpenH self._report(holds=taken, verb="Took over") return taken + def _yield_to_earlier(self, *, connection: Any, location: StateLocation) -> None: + """Read the pauses back; if another live hold on a model came first, close ours and stop.""" + + mine: set[_RelationKey] = {_key(self._held[name]) for name in self._paused} + if not mine: + return + rows: tuple[tuple[Any, ...], ...] = tuple( + self._adapter.query( + connection=connection, + sql=open_holds_sql(adapter=self._adapter, location=location), + limit=None, + ).rows + ) + first: dict[_RelationKey, OpenHold] = {} + for hold in sorted(open_holds(rows=rows), key=lambda hold: (hold.paused_at, hold.run_id)): + if hold.idle_seconds < self._grace_seconds and _key(hold.relation) in mine: + first.setdefault(_key(hold.relation), hold) + winners: tuple[OpenHold, ...] = tuple( + hold for hold in first.values() if hold.run_id != self._run_id + ) + if not winners: + return + self._record( + connection=connection, + events=tuple( + self._own(event_type=MaintainedPauseEventType.RESUMED, name=name) + for name in self._paused + ), + ) + self._paused = [] + raise _held_elsewhere(holds=winners) + def _still_held(self, *, connection: Any) -> frozenset[_RelationKey] | None: """This run's holds nobody took over, or None when the pause table can't say.""" diff --git a/src/sqlbuild/executor/maintained_pauses/models.py b/src/sqlbuild/executor/maintained_pauses/models.py index fc51dd9b4e..f76cca16b9 100644 --- a/src/sqlbuild/executor/maintained_pauses/models.py +++ b/src/sqlbuild/executor/maintained_pauses/models.py @@ -3,6 +3,7 @@ from __future__ import annotations from dataclasses import dataclass +from datetime import datetime from sqlbuild._native import CompiledRelationLocation from sqlbuild.executor.maintained_pauses.types import MaintainedPauseEventType @@ -21,8 +22,9 @@ class MaintainedPauseEvent: @dataclass(frozen=True) class OpenHold: - """A model a run paused and nobody resumed, with seconds since that run last wrote.""" + """A model a run paused and nobody resumed, when it paused, and how long the run is silent.""" run_id: str relation: CompiledRelationLocation idle_seconds: float + paused_at: datetime diff --git a/src/sqlbuild/executor/run/_helpers/materializations/maintained.py b/src/sqlbuild/executor/run/_helpers/materializations/maintained.py index d1f22a0797..55821b6928 100644 --- a/src/sqlbuild/executor/run/_helpers/materializations/maintained.py +++ b/src/sqlbuild/executor/run/_helpers/materializations/maintained.py @@ -9,6 +9,8 @@ CompiledRelationLocation, ModelPlanEntry, PlanAction, + error_code, + error_message, fit_auxiliary_relation_name, ) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter @@ -21,7 +23,7 @@ ) from sqlbuild.executor.maintained_pauses.classes.maintained_hold import MaintainedHold from sqlbuild.executor.run._helpers.materializations.view import execute_in_place_entry -from sqlbuild.executor.run.constants import REPLACED_RELATION_SUFFIX +from sqlbuild.executor.run.constants import REPLACED_RELATION_SUFFIX, RUN_STAGING_FAILED_CODE from sqlbuild.executor.run.models import ModelExecutionResult, ModelMaterializationContext _NO_WAREHOUSE_CODE: str = "A717" @@ -139,13 +141,21 @@ def _replace_former( ) try: _run(adapter=adapter, connection=connection, statements=statements, recorder=recorder) - except Exception: - rename( - connection=connection, - origin=aside, - destination=destination, - statement_recorder=recorder, - ) + except Exception as create_error: + try: + rename( + connection=connection, + origin=aside, + destination=destination, + statement_recorder=recorder, + ) + except Exception as restore_error: + raise AdapterUserError( + f"{error_message(create_error)}; restoring the former {former.value} " + f"also failed, so it is left at {aside}: {error_message(restore_error)}", + code=error_code(error=create_error, fallback_code=RUN_STAGING_FAILED_CODE), + help=f"rename {aside} back to {destination}, then build the model again", + ) from create_error raise drop(connection=connection, destination=aside, if_exists=True, statement_recorder=recorder) diff --git a/tests/integration/src/sqlbuild/executor/maintained_pauses/_test_types.py b/tests/integration/src/sqlbuild/executor/maintained_pauses/_test_types.py index 69de1e88eb..01cd40ee39 100644 --- a/tests/integration/src/sqlbuild/executor/maintained_pauses/_test_types.py +++ b/tests/integration/src/sqlbuild/executor/maintained_pauses/_test_types.py @@ -52,6 +52,15 @@ class TakenOverReleaseTestCase: expected_events: tuple[tuple[str, str, str, str], ...] +@dataclass(frozen=True) +class ConcurrentHoldTestCase: + description: str + expected_code: str + expected_winner: str + expected_states: dict[str, str] + expected_events: tuple[tuple[str, str, str, str], ...] + + @dataclass(frozen=True) class LifecycleLineTestCase: description: str diff --git a/tests/integration/src/sqlbuild/executor/maintained_pauses/test_maintained_pause_lease.py b/tests/integration/src/sqlbuild/executor/maintained_pauses/test_maintained_pause_lease.py index 523c09225e..d24a464446 100644 --- a/tests/integration/src/sqlbuild/executor/maintained_pauses/test_maintained_pause_lease.py +++ b/tests/integration/src/sqlbuild/executor/maintained_pauses/test_maintained_pause_lease.py @@ -3,6 +3,7 @@ from __future__ import annotations import time +from collections.abc import Callable from pathlib import Path from typing import Any @@ -27,6 +28,7 @@ HoldTakeoverTestCase, LifecycleLineTestCase, PauseLeaseTestCase, + ConcurrentHoldTestCase, TakenOverReleaseTestCase, ) from tests.integration.src.sqlbuild.executor.maintained_pauses.helpers import ( @@ -553,3 +555,78 @@ def test_given_hold_taken_over_when_first_build_releases_then_it_resumes_nothing assert adapter.states == test_case.expected_states assert pause_events(connection=connection) == sorted(test_case.expected_events) holds[1].release(connection=connection) + + +@pytest.mark.parametrize( + "test_case", + [ + ConcurrentHoldTestCase( + description="of two builds pausing a model at once, the later one backs out", + expected_code="A718", + expected_winner="early_run", + expected_states={"daily_totals": "SUSPENDED"}, + expected_events=( + ("paused", "early_run", "early_run", "daily_totals"), + ("paused", "first_run", "first_run", "daily_totals"), + ("paused", "late_run", "late_run", "daily_totals"), + ("resumed", "first_run", "first_run", "daily_totals"), + ("resumed", "late_run", "late_run", "daily_totals"), + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_two_builds_pausing_together_when_both_record_a_pause_then_only_the_earliest_holds( + test_case: ConcurrentHoldTestCase, + adapter: PausingDuckDbAdapter, + connection: Any, + database_path: Path, +) -> None: + _ = build_lease_project( + adapter=adapter, + connection=connection, + case=lease_project_case(orders_query=_ORDERS, report_locations={}, refresh_on_build=False), + runtime=BuildRuntimeParams(run_id="first_run"), + database_path=database_path, + ) + plan: PlanOutput = plan_lease_project( + adapter=adapter, + connection=connection, + case=lease_project_case( + orders_query=_MORE_ORDERS, report_locations={}, refresh_on_build=False + ), + ) + early: MaintainedHold | None = hold_maintained_models( + plan=plan, + adapter=adapter, + runtime=BuildRuntimeParams(run_id="early_run"), + connection_config={"database": str(database_path)}, + ) + late: MaintainedHold | None = hold_maintained_models( + plan=plan, + adapter=adapter, + runtime=BuildRuntimeParams(run_id="late_run"), + connection_config={"database": str(database_path)}, + ) + assert early is not None + assert late is not None + read_state: Callable[..., str | None] = late._state + + def state_then_rival_acquires(*, connection: Any, name: str) -> str | None: + state: str | None = read_state(connection=connection, name=name) + early.acquire(connection=connection, on_progress=None) + adapter.operations.clear() + return state + + late._state = state_then_rival_acquires # ty: ignore[invalid-assignment] + + with pytest.raises(AdapterUserError) as raised: + with late.holding(connection=connection, on_progress=None): + pass + + assert raised.value.code == test_case.expected_code + assert test_case.expected_winner in str(raised.value) + assert adapter.operations == [] + assert adapter.states == test_case.expected_states + assert pause_events(connection=connection) == sorted(test_case.expected_events) + early.release(connection=connection) diff --git a/tests/integration/src/sqlbuild/executor/run/maintained/_test_types.py b/tests/integration/src/sqlbuild/executor/run/maintained/_test_types.py index b199a9d5d6..3958be16a0 100644 --- a/tests/integration/src/sqlbuild/executor/run/maintained/_test_types.py +++ b/tests/integration/src/sqlbuild/executor/run/maintained/_test_types.py @@ -28,6 +28,17 @@ class MaintainedFailureTestCase: expected_listed_types: tuple[tuple[str], ...] +@dataclass(frozen=True) +class FailedRestoreTestCase: + description: str + setup_sql: tuple[str, ...] + failing_query: str + failing_renames: dict[str, str] + expected_error_code: str + expected_fragments: tuple[str, ...] + expected_relations: tuple[tuple[str, str], ...] + + @dataclass(frozen=True) class ChangeTrackingTableTestCase: description: str diff --git a/tests/integration/src/sqlbuild/executor/run/maintained/helpers.py b/tests/integration/src/sqlbuild/executor/run/maintained/helpers.py index a45b76e0ee..1f2a9d098b 100644 --- a/tests/integration/src/sqlbuild/executor/run/maintained/helpers.py +++ b/tests/integration/src/sqlbuild/executor/run/maintained/helpers.py @@ -31,11 +31,13 @@ def __init__( session_warehouse: str | None, listed_settings: MaintainedListing | None = None, failing_query: str | None = None, + failing_renames: dict[str, str] | None = None, ) -> None: super().__init__() self.warehouse: str | None = session_warehouse self.listed_settings: MaintainedListing | None = listed_settings self.failing_query: str | None = failing_query + self.failing_renames: dict[str, str] = dict(failing_renames or {}) self.created: list[tuple[str, str, str]] = [] self.altered: list[tuple[str, str | None]] = [] @@ -56,7 +58,8 @@ def render_create_maintained( return (f"CREATE TABLE {destination} AS {self.failing_query or sql}",) def render_rename(self, *, origin: str, destination: str) -> tuple[str, ...]: - return (f"ALTER TABLE {origin} RENAME TO {destination.split('.')[-1]}",) + renamed: str = f"ALTER TABLE {origin} RENAME TO {destination.split('.')[-1]}" + return (self.failing_renames.get(origin, renamed),) def render_rename_view(self, *, origin: str, destination: str) -> tuple[str, ...]: return (f"ALTER VIEW {origin} RENAME TO {destination.split('.')[-1]}",) diff --git a/tests/integration/src/sqlbuild/executor/run/maintained/test_maintained_execution.py b/tests/integration/src/sqlbuild/executor/run/maintained/test_maintained_execution.py index ecfcdbb579..1df742d105 100644 --- a/tests/integration/src/sqlbuild/executor/run/maintained/test_maintained_execution.py +++ b/tests/integration/src/sqlbuild/executor/run/maintained/test_maintained_execution.py @@ -12,6 +12,7 @@ from sqlbuild.executor.run.models import ModelExecutionResult from sqlbuild.executor.scheduling.types import ExecutionStatus from tests.integration.src.sqlbuild.executor.run.maintained._test_types import ( + FailedRestoreTestCase, MaintainedFailureTestCase, MaintainedSuccessTestCase, ) @@ -167,3 +168,52 @@ def test_given_unbuildable_maintained_entry_when_executing_then_fails_keeping_th assert result.status == ExecutionStatus.FAILED assert result.error_code == test_case.expected_error_code assert listed_types(connection=connection) == list(test_case.expected_listed_types) + + +@pytest.mark.parametrize( + "test_case", + [ + FailedRestoreTestCase( + description="a failed create and a failed restore are both reported", + setup_sql=("CREATE TABLE staging.daily_totals AS SELECT 1 AS orders",), + failing_query="SELECT missing_column FROM staging.no_such_table", + failing_renames={ + "staging.daily_totals__replaced": "SELECT restore_column FROM staging.no_restore" + }, + expected_error_code="R002", + expected_fragments=("no_such_table", "no_restore", "daily_totals__replaced"), + expected_relations=(("daily_totals__replaced", "BASE TABLE"),), + ), + ], + ids=lambda case: case.description, +) +def test_given_failed_create_and_restore_when_executing_then_reports_both_errors( + test_case: FailedRestoreTestCase, connection: Any +) -> None: + adapter: MaintainedDuckDbAdapter = MaintainedDuckDbAdapter( + session_warehouse="SESSION_WH", + failing_query=test_case.failing_query, + failing_renames=test_case.failing_renames, + ) + for sql in test_case.setup_sql: + connection.execute(sql) + + result: ModelExecutionResult = execute_maintained_entry( + context=maintained_context( + adapter=adapter, + connection=connection, + action=PlanAction.CREATE_MAINTAINED, + refresh_warehouse=None, + ) + ) + + assert result.status == ExecutionStatus.FAILED + assert result.error_code == test_case.expected_error_code + message: str = str(result.error_message) + assert [fragment in message for fragment in test_case.expected_fragments] == [True] * len( + test_case.expected_fragments + ) + assert connection.execute( + "SELECT table_name, table_type FROM information_schema.tables " + "WHERE table_schema = 'staging' AND table_name LIKE 'daily_totals%' ORDER BY 1" + ).fetchall() == list(test_case.expected_relations) diff --git a/website/src/content/docs/docs/concepts/models/materializations.mdx b/website/src/content/docs/docs/concepts/models/materializations.mdx index cabfec288e..9fa53d5d07 100644 --- a/website/src/content/docs/docs/concepts/models/materializations.mdx +++ b/website/src/content/docs/docs/concepts/models/materializations.mdx @@ -120,7 +120,7 @@ A build pauses the maintained models it changes, and those that read a model it - A kept model is refreshed during the build when a model built later in the same build reads it, so that model reads current rows. `refresh_on_build` refreshes it in every build. - A target with `pause_maintained = true` leaves its maintained models suspended between builds and refreshes them only in builds; see [Targets](/docs/concepts/project-configuration#targets). -Two builds can't pause the same models at once. A build that needs a model another build holds fails with `A718`, naming that run. A build that stops without resuming, for example because its process was killed, leaves its models suspended. The next build that pauses maintained models on the target resumes them once that build has been silent for `[maintained] heartbeat_grace` (see [Maintained](/docs/concepts/project-configuration#maintained)). Until then, resume them in Snowflake by hand, or run `sqb build --ignore-maintained-grace` where the project allows it. +Two builds can't pause the same models at once. A build that needs a model another build holds fails with `A718`, naming that run. When two builds pause a model at the same moment, the one that recorded its pause first keeps it, and the other fails with `A718` without suspending or resuming anything. A build that stops without resuming, for example because its process was killed, leaves its models suspended. The next build that pauses maintained models on the target resumes them once that build has been silent for `[maintained] heartbeat_grace` (see [Maintained](/docs/concepts/project-configuration#maintained)). Until then, resume them in Snowflake by hand, or run `sqb build --ignore-maintained-grace` where the project allows it. ### Error codes diff --git a/website/src/content/docs/docs/concepts/project-configuration.mdx b/website/src/content/docs/docs/concepts/project-configuration.mdx index ae56abf4ce..2d11732db2 100644 --- a/website/src/content/docs/docs/concepts/project-configuration.mdx +++ b/website/src/content/docs/docs/concepts/project-configuration.mdx @@ -682,7 +682,7 @@ allow_ignore_grace = false | Field | Default | Description | |-------|---------|-------------| -| `heartbeat_grace` | `"30m"` | How long a build that paused maintained models may go silent before another build resumes them or takes them over. A fixed duration of at least `"3m"`, such as `"30m"` or `"2h"`; a build writes a heartbeat every minute | +| `heartbeat_grace` | `"30m"` | How long a build that paused maintained models may go silent before another build resumes them or takes them over. A fixed duration longer than `"1m"`, such as `"30m"` or `"2h"`; a build writes a heartbeat every minute | | `allow_ignore_grace` | `false` | Allow `sqb build --ignore-maintained-grace`, which takes over models another build holds without waiting for the grace. Without it the flag fails with `A719` | ## Scenario From d43d1029683e822b58d669ff131acb01e3511275 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 19 Nov 2026 03:06:34 +0000 Subject: [PATCH 16/17] fix(build): write the first maintained heartbeat when the hold is taken A grace just over the 60s interval could otherwise expire a running build, because its first beat landed one interval plus a connect after its pauses. --- .../maintained_pauses/classes/maintained_heartbeat.py | 6 ++++-- .../maintained_pauses/test_maintained_pause_lease.py | 8 +++++++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/src/sqlbuild/executor/maintained_pauses/classes/maintained_heartbeat.py b/src/sqlbuild/executor/maintained_pauses/classes/maintained_heartbeat.py index d26cdee712..a689c16004 100644 --- a/src/sqlbuild/executor/maintained_pauses/classes/maintained_heartbeat.py +++ b/src/sqlbuild/executor/maintained_pauses/classes/maintained_heartbeat.py @@ -38,11 +38,12 @@ def stop(self) -> None: self._thread.join() def _beat(self) -> None: - """Reconnect after a failed beat; the grace, not one beat, decides when a hold expires.""" + """Beat at once, then every interval, so no gap exceeds one interval plus a write.""" connection: Any = None sequence: int = 0 - while not self._stopped.wait(self._interval_seconds): + stopped: bool = False + while not stopped: sequence += 1 try: if connection is None: @@ -51,6 +52,7 @@ def _beat(self) -> None: except Exception as error: self.failure = error connection = self._closed(connection=connection) + stopped = self._stopped.wait(self._interval_seconds) _ = self._closed(connection=connection) def _closed(self, *, connection: Any) -> None: diff --git a/tests/integration/src/sqlbuild/executor/maintained_pauses/test_maintained_pause_lease.py b/tests/integration/src/sqlbuild/executor/maintained_pauses/test_maintained_pause_lease.py index d24a464446..0bb8cbd9e6 100644 --- a/tests/integration/src/sqlbuild/executor/maintained_pauses/test_maintained_pause_lease.py +++ b/tests/integration/src/sqlbuild/executor/maintained_pauses/test_maintained_pause_lease.py @@ -22,13 +22,13 @@ hold_maintained_models, ) from tests.integration.src.sqlbuild.executor.maintained_pauses._test_types import ( + ConcurrentHoldTestCase, FirstBuildTestCase, HeartbeatTestCase, HoldRefusalTestCase, HoldTakeoverTestCase, LifecycleLineTestCase, PauseLeaseTestCase, - ConcurrentHoldTestCase, TakenOverReleaseTestCase, ) from tests.integration.src.sqlbuild.executor.maintained_pauses.helpers import ( @@ -410,6 +410,12 @@ def test_given_live_hold_when_building_then_stops_before_any_change( wait_seconds=0.4, expected_min_beats=2, ), + HeartbeatTestCase( + description="a hold beats as soon as it is taken, not one interval later", + heartbeat_seconds=30.0, + wait_seconds=0.3, + expected_min_beats=1, + ), ], ids=lambda case: case.description, ) From fe720b4e7020533df55eb873f6b3e3eca1f2c682 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 19 Nov 2026 05:01:16 +0000 Subject: [PATCH 17/17] test(goldens): list pause_maintained among allowed target keys The target-unknown-key failure golden lists every allowed target key; maintained models add pause_maintained. --- tests/goldens/compiler/failure/target-unknown-key.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/goldens/compiler/failure/target-unknown-key.json b/tests/goldens/compiler/failure/target-unknown-key.json index 18a234ac3f..9d222e2426 100644 --- a/tests/goldens/compiler/failure/target-unknown-key.json +++ b/tests/goldens/compiler/failure/target-unknown-key.json @@ -8,7 +8,7 @@ "phase": "compile", "severity": "error", "code": "D001", - "message": "/sqlbuild_project.toml targets.ci contains unknown key(s): warehouse_size. Allowed keys: catalog_connection, clone, compile_cache, concurrency, connection, database, default_table_type, defer_clone_from, defer_sources_to, diff, execution_limits, ignore_audit_failures, loader_schema, missing_migration_origin, owns_time_travel_retention_namespace, plan_from, pools, recall, schema, table_type_downgrade, time_travel_retention, time_travel_retention_decrease, vars, warehouses", + "message": "/sqlbuild_project.toml targets.ci contains unknown key(s): warehouse_size. Allowed keys: catalog_connection, clone, compile_cache, concurrency, connection, database, default_table_type, defer_clone_from, defer_sources_to, diff, execution_limits, ignore_audit_failures, loader_schema, missing_migration_origin, owns_time_travel_retention_namespace, pause_maintained, plan_from, pools, recall, schema, table_type_downgrade, time_travel_retention, time_travel_retention_decrease, vars, warehouses", "help": "did you mean 'warehouses'?" } ]