Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
17 commits
Select commit Hold shift + click to select a range
7c39a06
feat: accept maintained models in MODEL headers with offline adapter …
kvlonge Nov 18, 2026
f1e1601
feat(config): add maintained-model pause settings
kvlonge Nov 18, 2026
aec47d9
feat(plan): plan maintained models as create, alter or keep
kvlonge Nov 18, 2026
524899a
feat(plan): recreate a maintained model when its refresh_mode changes
kvlonge Nov 18, 2026
29cbedb
feat(plan): alter maintained models from their live refresh settings
kvlonge Nov 18, 2026
3e9c1ba
feat(build): create, alter and keep maintained models on Snowflake
kvlonge Nov 19, 2026
4cedaf0
feat(build): track changes on tables a maintained model reads
kvlonge Nov 19, 2026
c1b5a9e
feat(build): pause maintained models during a build and record the hold
kvlonge Nov 19, 2026
ee400d2
feat(plan): report kept maintained models that are paused
kvlonge Nov 19, 2026
f412a97
docs: document maintained models, their pause during builds and Snowf…
kvlonge Nov 19, 2026
6989d68
refactor(snowflake): share the exact-name SHOW lookup between object …
kvlonge Nov 19, 2026
9a2847b
fix(build): harden maintained models after review
kvlonge Nov 19, 2026
402ff4d
docs: note that a maintained model's lag can't undercut its maintaine…
kvlonge Nov 19, 2026
933e07d
chore(fensu): allow the graph walk mirrored in layering tooling and t…
kvlonge Nov 19, 2026
f6c4ce0
fix(build): settle concurrent maintained pauses and keep both errors …
kvlonge Nov 19, 2026
d43d102
fix(build): write the first maintained heartbeat when the hold is taken
kvlonge Nov 19, 2026
fe720b4
test(goldens): list pause_maintained among allowed target keys
kvlonge Nov 19, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
//! Error codes of the project-wide maintained model checks.

/// A maintained model that nothing would ever refresh.
pub const MAINTAINED_NEVER_REFRESHED_CODE: &str = "A016";
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
pub mod never_refreshed_maintained;
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
//! Maintained models with no refresh of their own that no maintained model reads.

use std::collections::HashSet;

use crate::assembly::maintained_checks::models::MaintainedModel;

/// The first by name; without a reader or its own refresh, the warehouse never refreshes it.
pub fn never_refreshed_maintained(models: &[MaintainedModel]) -> Option<&str> {
let maintained: HashSet<&str> = models.iter().map(|model| model.name.as_str()).collect();
let read: HashSet<&str> = models
.iter()
.flat_map(|model| model.reads.iter().map(String::as_str))
.filter(|name| maintained.contains(name))
.collect();
models
.iter()
.filter(|model| !model.has_refresh && !read.contains(model.name.as_str()))
.map(|model| model.name.as_str())
.min()
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
pub mod constants;
pub mod main;
pub mod models;

#[cfg(test)]
mod tests;
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
//! What the project-wide maintained model checks read.

/// One maintained model: whether it sets its own refresh, and the models it reads.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct MaintainedModel {
pub name: String,
pub has_refresh: bool,
pub reads: Vec<String>,
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
#[path = "tests/helpers.rs"]
mod helpers;
#[path = "tests/test_maintained_checks.rs"]
mod maintained_checks;
#[path = "tests/test_types.rs"]
mod test_types;
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
use crate::assembly::maintained_checks::models::MaintainedModel;

/// A maintained model named `name` reading `reads`.
pub(crate) fn maintained(name: &str, has_refresh: bool, reads: &[&str]) -> MaintainedModel {
MaintainedModel {
name: name.to_owned(),
has_refresh,
reads: reads.iter().map(|read| (*read).to_owned()).collect(),
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
use crate::assembly::maintained_checks::main::never_refreshed_maintained::never_refreshed_maintained;
use crate::assembly::maintained_checks::tests::helpers::maintained;
use crate::assembly::maintained_checks::tests::test_types::NeverRefreshedTestCase;

#[test]
fn given_maintained_chains_when_checking_then_unreached_models_without_refresh_are_named() {
let test_cases = [
NeverRefreshedTestCase {
description: "an upstream model refreshed for its reader",
models: vec![
maintained("clean_orders", false, &["orders"]),
maintained("daily_totals", true, &["clean_orders"]),
],
expected_name: None,
},
NeverRefreshedTestCase {
description: "the last model of a chain without refresh",
models: vec![
maintained("clean_orders", false, &["orders"]),
maintained("daily_totals", false, &["clean_orders"]),
],
expected_name: Some("daily_totals"),
},
NeverRefreshedTestCase {
description: "a model read only by a table, which never asks for a refresh",
models: vec![maintained("clean_orders", false, &["orders"])],
expected_name: Some("clean_orders"),
},
NeverRefreshedTestCase {
description: "two unread models report the first by name",
models: vec![
maintained("b_totals", false, &["orders"]),
maintained("a_totals", false, &["orders"]),
],
expected_name: Some("a_totals"),
},
NeverRefreshedTestCase {
description: "every model sets its own refresh",
models: vec![maintained("daily_totals", true, &[])],
expected_name: None,
},
];

for test_case in test_cases {
assert_eq!(
never_refreshed_maintained(&test_case.models),
test_case.expected_name,
"{}",
test_case.description
);
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
use crate::assembly::maintained_checks::models::MaintainedModel;

pub(crate) struct NeverRefreshedTestCase {
pub(crate) description: &'static str,
pub(crate) models: Vec<MaintainedModel>,
pub(crate) expected_name: Option<&'static str>,
}
1 change: 1 addition & 0 deletions crates/sqlbuild-analysis/src/assembly/mod.rs
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
//! The native model analysis session: catalog, dataflow scheduling, projection and shape publication.

pub mod analysis_session;
pub mod maintained_checks;
pub mod project;
pub mod target_checks;
2 changes: 1 addition & 1 deletion crates/sqlbuild-analysis/src/janitor_planning/constants.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
use sqlbuild_core::constants::STATE_TABLE_NAMES;

/// SQLBuild state tables the janitor never acts on.
pub const BUILT_IN_EXCLUDE_PATTERNS: [&str; 13] = STATE_TABLE_NAMES;
pub const BUILT_IN_EXCLUDE_PATTERNS: [&str; 14] = STATE_TABLE_NAMES;
/// Prefix of every strict janitor archive name.
pub const ARCHIVE_NAME_PREFIX: &str = "_SQB_ARCHIVE__";
/// Separator between archive name segments.
Expand Down
1 change: 1 addition & 0 deletions crates/sqlbuild-analysis/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ pub mod janitor_planning;
pub mod lineage;
pub mod lineage_command;
pub mod lineage_graph;
pub mod maintained_refresh;
pub mod manifest;
pub mod node_set_dependencies;
pub mod nullability_rules;
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
pub(crate) mod target_lag;
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
//! A dynamic table's listed `target_lag`, such as `DOWNSTREAM` or `2 hours`.

use crate::maintained_refresh::constants::{DOWNSTREAM_TARGET_LAG, TARGET_LAG_UNITS};
use crate::maintained_refresh::models::RefreshLag;

/// The lag `text` lists, or `None` when it is not a lag Snowflake lists.
pub(crate) fn parse_target_lag(text: &str) -> Option<RefreshLag> {
let text: &str = text.trim();
if text.eq_ignore_ascii_case(DOWNSTREAM_TARGET_LAG) {
return Some(RefreshLag::Downstream);
}
let (amount, unit) = text.split_once(' ')?;
let amount: u64 = match amount.parse::<u64>() {
Ok(amount) => amount,
Err(_) => return None,
};
let unit: String = unit.trim().to_ascii_lowercase();
let unit: &str = unit.strip_suffix('s').unwrap_or(&unit);
TARGET_LAG_UNITS
.iter()
.find(|(name, _)| *name == unit)
.and_then(|(_, seconds)| amount.checked_mul(*seconds))
.map(RefreshLag::Seconds)
}
11 changes: 11 additions & 0 deletions crates/sqlbuild-analysis/src/maintained_refresh/constants.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
//! Snowflake target lag spellings.

/// The `target_lag` of a dynamic table refreshed for its readers.
pub const DOWNSTREAM_TARGET_LAG: &str = "DOWNSTREAM";
/// Target lag units, largest first, with their seconds.
pub(crate) const TARGET_LAG_UNITS: [(&str, u64); 4] = [
("day", 86_400),
("hour", 3_600),
("minute", 60),
("second", 1),
];
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
//! Whether a maintained model's listed refresh settings differ from its declared ones.

use crate::maintained_refresh::_helpers::target_lag::parse_target_lag;
use crate::maintained_refresh::models::{LiveDynamicTable, MaintainedSettings};

/// True when the lag differs or cannot be read, or a declared warehouse differs.
pub fn maintained_settings_drift(desired: &MaintainedSettings, live: &LiveDynamicTable) -> bool {
if parse_target_lag(&live.target_lag) != Some(desired.lag) {
return true;
}
desired
.warehouse
.as_deref()
.is_some_and(|warehouse| !warehouse.eq_ignore_ascii_case(live.warehouse.trim()))
}
3 changes: 3 additions & 0 deletions crates/sqlbuild-analysis/src/maintained_refresh/main/mod.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
pub mod maintained_settings_drift;
pub mod reinitialised_readers;
pub mod render_target_lag;
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
//! Maintained models the plan keeps whose upstream it replaces, so they recompute in full.

use std::collections::BTreeSet;

use crate::maintained_refresh::models::MaintainedRead;

/// Each kept maintained reader by name, with the first replaced upstream it reads by name.
pub fn reinitialised_readers(reads: &[MaintainedRead]) -> Vec<(String, String)> {
let replaced: BTreeSet<&str> = reads
.iter()
.filter(|read| read.replaced)
.map(|read| read.name.as_str())
.collect();
let mut readers: Vec<(String, String)> = Vec::new();
for read in reads
.iter()
.filter(|read| read.maintained && !read.replaced)
{
let mut upstreams: Vec<&str> = read
.reads
.iter()
.map(String::as_str)
.filter(|upstream| replaced.contains(upstream))
.collect();
upstreams.sort_unstable();
if let Some(upstream) = upstreams.first() {
readers.push((read.name.clone(), (*upstream).to_owned()));
}
}
readers.sort();
readers
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
//! The `TARGET_LAG` value of a refresh lag, in its largest exact unit.

use crate::maintained_refresh::constants::{DOWNSTREAM_TARGET_LAG, TARGET_LAG_UNITS};
use crate::maintained_refresh::models::RefreshLag;

/// `DOWNSTREAM`, or a quoted lag such as `'2 hours'`.
pub fn render_target_lag(lag: RefreshLag) -> String {
let RefreshLag::Seconds(seconds) = lag else {
return DOWNSTREAM_TARGET_LAG.to_owned();
};
let (unit, size) = TARGET_LAG_UNITS
.iter()
.find(|(_, size)| seconds % size == 0)
.copied()
.unwrap_or(("second", 1));
let amount: u64 = seconds / size;
let plural: &str = if amount == 1 { "" } else { "s" };
format!("'{amount} {unit}{plural}'")
}
9 changes: 9 additions & 0 deletions crates/sqlbuild-analysis/src/maintained_refresh/mod.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
//! Maintained refresh: dynamic table lag text, refresh setting drift, and readers a rebuild reinitialises.

pub(crate) mod _helpers;
pub mod constants;
pub mod main;
pub mod models;

#[cfg(test)]
mod tests;
34 changes: 34 additions & 0 deletions crates/sqlbuild-analysis/src/maintained_refresh/models.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
//! Declared and listed refresh settings of maintained models.

/// How stale a maintained model may get: refreshed for its readers, or within a fixed lag.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum RefreshLag {
/// Refreshed when the maintained models reading it need it.
Downstream,
/// Refreshed within this many seconds.
Seconds(u64),
}

/// The refresh settings a maintained model declares.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct MaintainedSettings {
pub lag: RefreshLag,
/// The declared refresh warehouse; unset keeps whichever the object has.
pub warehouse: Option<String>,
}

/// A maintained model's refresh settings as the warehouse lists them.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct LiveDynamicTable {
pub target_lag: String,
pub warehouse: String,
}

/// One planned model: whether it is maintained, whether this plan replaces it, and what it reads.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct MaintainedRead {
pub name: String,
pub maintained: bool,
pub replaced: bool,
pub reads: Vec<String>,
}
6 changes: 6 additions & 0 deletions crates/sqlbuild-analysis/src/maintained_refresh/tests.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
#[path = "tests/helpers.rs"]
mod helpers;
#[path = "tests/test_maintained_refresh.rs"]
mod maintained_refresh;
#[path = "tests/test_types.rs"]
mod test_types;
26 changes: 26 additions & 0 deletions crates/sqlbuild-analysis/src/maintained_refresh/tests/helpers.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
use crate::maintained_refresh::models::{
LiveDynamicTable, MaintainedRead, MaintainedSettings, RefreshLag,
};

pub(super) fn settings(lag: RefreshLag, warehouse: Option<&str>) -> MaintainedSettings {
MaintainedSettings {
lag,
warehouse: warehouse.map(str::to_owned),
}
}

pub(super) fn live(target_lag: &str, warehouse: &str) -> LiveDynamicTable {
LiveDynamicTable {
target_lag: target_lag.to_owned(),
warehouse: warehouse.to_owned(),
}
}

pub(super) fn read(name: &str, maintained: bool, replaced: bool, reads: &[&str]) -> MaintainedRead {
MaintainedRead {
name: name.to_owned(),
maintained,
replaced,
reads: reads.iter().map(|read| (*read).to_owned()).collect(),
}
}
Loading
Loading