Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
//! Error codes of the project-wide dynamic model checks.

/// A dynamic model that nothing would ever refresh.
pub const DYNAMIC_NEVER_REFRESHED_CODE: &str = "A016";
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
pub mod never_refreshed_dynamic;
Original file line number Diff line number Diff line change
@@ -1,16 +1,16 @@
//! Maintained models with no refresh of their own that no maintained model reads.
//! Dynamic models with no refresh of their own that no dynamic model reads.

use std::collections::HashSet;

use crate::assembly::maintained_checks::models::MaintainedModel;
use crate::assembly::dynamic_checks::models::DynamicModel;

/// The first by name; without a reader or its own refresh, the warehouse never refreshes it.
pub fn never_refreshed_maintained(models: &[MaintainedModel]) -> Option<&str> {
let maintained: HashSet<&str> = models.iter().map(|model| model.name.as_str()).collect();
pub fn never_refreshed_dynamic(models: &[DynamicModel]) -> Option<&str> {
let dynamic: HashSet<&str> = models.iter().map(|model| model.name.as_str()).collect();
let read: HashSet<&str> = models
.iter()
.flat_map(|model| model.reads.iter().map(String::as_str))
.filter(|name| maintained.contains(name))
.filter(|name| dynamic.contains(name))
.collect();
models
.iter()
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
//! What the project-wide dynamic model checks read.

/// One dynamic model: whether it sets its own refresh, and the models it reads.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct DynamicModel {
pub name: String,
pub has_refresh: bool,
pub reads: Vec<String>,
}
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
#[path = "tests/test_dynamic_checks.rs"]
mod dynamic_checks;
#[path = "tests/helpers.rs"]
mod helpers;
#[path = "tests/test_maintained_checks.rs"]
mod maintained_checks;
#[path = "tests/test_types.rs"]
mod test_types;
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
use crate::assembly::dynamic_checks::models::DynamicModel;

/// A dynamic model named `name` reading `reads`.
pub(crate) fn dynamic(name: &str, has_refresh: bool, reads: &[&str]) -> DynamicModel {
DynamicModel {
name: name.to_owned(),
has_refresh,
reads: reads.iter().map(|read| (*read).to_owned()).collect(),
}
}
Original file line number Diff line number Diff line change
@@ -1,49 +1,49 @@
use crate::assembly::maintained_checks::main::never_refreshed_maintained::never_refreshed_maintained;
use crate::assembly::maintained_checks::tests::helpers::maintained;
use crate::assembly::maintained_checks::tests::test_types::NeverRefreshedTestCase;
use crate::assembly::dynamic_checks::main::never_refreshed_dynamic::never_refreshed_dynamic;
use crate::assembly::dynamic_checks::tests::helpers::dynamic;
use crate::assembly::dynamic_checks::tests::test_types::NeverRefreshedTestCase;

#[test]
fn given_maintained_chains_when_checking_then_unreached_models_without_refresh_are_named() {
fn given_dynamic_chains_when_checking_then_unreached_models_without_refresh_are_named() {
let test_cases = [
NeverRefreshedTestCase {
description: "an upstream model refreshed for its reader",
models: vec![
maintained("clean_orders", false, &["orders"]),
maintained("daily_totals", true, &["clean_orders"]),
dynamic("clean_orders", false, &["orders"]),
dynamic("daily_totals", true, &["clean_orders"]),
],
expected_name: None,
},
NeverRefreshedTestCase {
description: "the last model of a chain without refresh",
models: vec![
maintained("clean_orders", false, &["orders"]),
maintained("daily_totals", false, &["clean_orders"]),
dynamic("clean_orders", false, &["orders"]),
dynamic("daily_totals", false, &["clean_orders"]),
],
expected_name: Some("daily_totals"),
},
NeverRefreshedTestCase {
description: "a model read only by a table, which never asks for a refresh",
models: vec![maintained("clean_orders", false, &["orders"])],
models: vec![dynamic("clean_orders", false, &["orders"])],
expected_name: Some("clean_orders"),
},
NeverRefreshedTestCase {
description: "two unread models report the first by name",
models: vec![
maintained("b_totals", false, &["orders"]),
maintained("a_totals", false, &["orders"]),
dynamic("b_totals", false, &["orders"]),
dynamic("a_totals", false, &["orders"]),
],
expected_name: Some("a_totals"),
},
NeverRefreshedTestCase {
description: "every model sets its own refresh",
models: vec![maintained("daily_totals", true, &[])],
models: vec![dynamic("daily_totals", true, &[])],
expected_name: None,
},
];

for test_case in test_cases {
assert_eq!(
never_refreshed_maintained(&test_case.models),
never_refreshed_dynamic(&test_case.models),
test_case.expected_name,
"{}",
test_case.description
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
use crate::assembly::maintained_checks::models::MaintainedModel;
use crate::assembly::dynamic_checks::models::DynamicModel;

pub(crate) struct NeverRefreshedTestCase {
pub(crate) description: &'static str,
pub(crate) models: Vec<MaintainedModel>,
pub(crate) models: Vec<DynamicModel>,
pub(crate) expected_name: Option<&'static str>,
}

This file was deleted.

This file was deleted.

This file was deleted.

This file was deleted.

2 changes: 1 addition & 1 deletion crates/sqlbuild-analysis/src/assembly/mod.rs
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
//! The native model analysis session: catalog, dataflow scheduling, projection and shape publication.

pub mod analysis_session;
pub mod maintained_checks;
pub mod dynamic_checks;
pub mod project;
pub mod target_checks;
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
//! A dynamic table's listed `target_lag`, such as `DOWNSTREAM` or `2 hours`.

use crate::maintained_refresh::constants::{DOWNSTREAM_TARGET_LAG, TARGET_LAG_UNITS};
use crate::maintained_refresh::models::RefreshLag;
use crate::dynamic_refresh::constants::{DOWNSTREAM_TARGET_LAG, TARGET_LAG_UNITS};
use crate::dynamic_refresh::models::RefreshLag;

/// The lag `text` lists, or `None` when it is not a lag Snowflake lists.
pub(crate) fn parse_target_lag(text: &str) -> Option<RefreshLag> {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
//! Whether a dynamic model's listed refresh settings differ from its declared ones.

use crate::dynamic_refresh::_helpers::target_lag::parse_target_lag;
use crate::dynamic_refresh::models::{DynamicSettings, LiveDynamicTable};

/// True when the lag differs or cannot be read, or a declared warehouse differs.
pub fn dynamic_settings_drift(desired: &DynamicSettings, live: &LiveDynamicTable) -> bool {
if parse_target_lag(&live.target_lag) != Some(desired.lag) {
return true;
}
desired
.warehouse
.as_deref()
.is_some_and(|warehouse| !warehouse.eq_ignore_ascii_case(live.warehouse.trim()))
}
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
pub mod maintained_settings_drift;
pub mod dynamic_settings_drift;
pub mod reinitialised_readers;
pub mod render_target_lag;
Original file line number Diff line number Diff line change
@@ -1,21 +1,18 @@
//! Maintained models the plan keeps whose upstream it replaces, so they recompute in full.
//! Dynamic models the plan keeps whose upstream it replaces, so they recompute in full.

use std::collections::BTreeSet;

use crate::maintained_refresh::models::MaintainedRead;
use crate::dynamic_refresh::models::DynamicRead;

/// Each kept maintained reader by name, with the first replaced upstream it reads by name.
pub fn reinitialised_readers(reads: &[MaintainedRead]) -> Vec<(String, String)> {
/// Each kept dynamic reader by name, with the first replaced upstream it reads by name.
pub fn reinitialised_readers(reads: &[DynamicRead]) -> Vec<(String, String)> {
let replaced: BTreeSet<&str> = reads
.iter()
.filter(|read| read.replaced)
.map(|read| read.name.as_str())
.collect();
let mut readers: Vec<(String, String)> = Vec::new();
for read in reads
.iter()
.filter(|read| read.maintained && !read.replaced)
{
for read in reads.iter().filter(|read| read.dynamic && !read.replaced) {
let mut upstreams: Vec<&str> = read
.reads
.iter()
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
//! The `TARGET_LAG` value of a refresh lag, in its largest exact unit.

use crate::maintained_refresh::constants::{DOWNSTREAM_TARGET_LAG, TARGET_LAG_UNITS};
use crate::maintained_refresh::models::RefreshLag;
use crate::dynamic_refresh::constants::{DOWNSTREAM_TARGET_LAG, TARGET_LAG_UNITS};
use crate::dynamic_refresh::models::RefreshLag;

/// `DOWNSTREAM`, or a quoted lag such as `'2 hours'`.
pub fn render_target_lag(lag: RefreshLag) -> String {
Expand Down
9 changes: 9 additions & 0 deletions crates/sqlbuild-analysis/src/dynamic_refresh/mod.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
//! Dynamic refresh: dynamic table lag text, refresh setting drift, and readers a rebuild reinitialises.

pub(crate) mod _helpers;
pub mod constants;
pub mod main;
pub mod models;

#[cfg(test)]
mod tests;
Original file line number Diff line number Diff line change
@@ -1,34 +1,34 @@
//! Declared and listed refresh settings of maintained models.
//! Declared and listed refresh settings of dynamic models.

/// How stale a maintained model may get: refreshed for its readers, or within a fixed lag.
/// How stale a dynamic model may get: refreshed for its readers, or within a fixed lag.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum RefreshLag {
/// Refreshed when the maintained models reading it need it.
/// Refreshed when the dynamic models reading it need it.
Downstream,
/// Refreshed within this many seconds.
Seconds(u64),
}

/// The refresh settings a maintained model declares.
/// The refresh settings a dynamic model declares.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct MaintainedSettings {
pub struct DynamicSettings {
pub lag: RefreshLag,
/// The declared refresh warehouse; unset keeps whichever the object has.
pub warehouse: Option<String>,
}

/// A maintained model's refresh settings as the warehouse lists them.
/// A dynamic model's refresh settings as the warehouse lists them.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct LiveDynamicTable {
pub target_lag: String,
pub warehouse: String,
}

/// One planned model: whether it is maintained, whether this plan replaces it, and what it reads.
/// One planned model: whether it is dynamic, whether this plan replaces it, and what it reads.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct MaintainedRead {
pub struct DynamicRead {
pub name: String,
pub maintained: bool,
pub dynamic: bool,
pub replaced: bool,
pub reads: Vec<String>,
}
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
#[path = "tests/test_dynamic_refresh.rs"]
mod dynamic_refresh;
#[path = "tests/helpers.rs"]
mod helpers;
#[path = "tests/test_maintained_refresh.rs"]
mod maintained_refresh;
#[path = "tests/test_types.rs"]
mod test_types;
Original file line number Diff line number Diff line change
@@ -1,9 +1,7 @@
use crate::maintained_refresh::models::{
LiveDynamicTable, MaintainedRead, MaintainedSettings, RefreshLag,
};
use crate::dynamic_refresh::models::{DynamicRead, DynamicSettings, LiveDynamicTable, RefreshLag};

pub(super) fn settings(lag: RefreshLag, warehouse: Option<&str>) -> MaintainedSettings {
MaintainedSettings {
pub(super) fn settings(lag: RefreshLag, warehouse: Option<&str>) -> DynamicSettings {
DynamicSettings {
lag,
warehouse: warehouse.map(str::to_owned),
}
Expand All @@ -16,10 +14,10 @@ pub(super) fn live(target_lag: &str, warehouse: &str) -> LiveDynamicTable {
}
}

pub(super) fn read(name: &str, maintained: bool, replaced: bool, reads: &[&str]) -> MaintainedRead {
MaintainedRead {
pub(super) fn read(name: &str, dynamic: bool, replaced: bool, reads: &[&str]) -> DynamicRead {
DynamicRead {
name: name.to_owned(),
maintained,
dynamic,
replaced,
reads: reads.iter().map(|read| (*read).to_owned()).collect(),
}
Expand Down
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
use crate::maintained_refresh::_helpers::target_lag::parse_target_lag;
use crate::maintained_refresh::main::maintained_settings_drift::maintained_settings_drift;
use crate::maintained_refresh::main::reinitialised_readers::reinitialised_readers;
use crate::maintained_refresh::main::render_target_lag::render_target_lag;
use crate::maintained_refresh::models::RefreshLag;
use crate::maintained_refresh::tests::helpers::{live, read, settings};
use crate::maintained_refresh::tests::test_types::{
use crate::dynamic_refresh::_helpers::target_lag::parse_target_lag;
use crate::dynamic_refresh::main::dynamic_settings_drift::dynamic_settings_drift;
use crate::dynamic_refresh::main::reinitialised_readers::reinitialised_readers;
use crate::dynamic_refresh::main::render_target_lag::render_target_lag;
use crate::dynamic_refresh::models::RefreshLag;
use crate::dynamic_refresh::tests::helpers::{live, read, settings};
use crate::dynamic_refresh::tests::test_types::{
DriftTestCase, ReinitialiseTestCase, RenderLagTestCase, TargetLagTestCase,
};

Expand Down Expand Up @@ -139,7 +139,7 @@ fn given_live_dynamic_table_when_comparing_settings_then_only_real_drift_alters(
];
for test_case in test_cases {
assert_eq!(
maintained_settings_drift(&test_case.desired, &test_case.live),
dynamic_settings_drift(&test_case.desired, &test_case.live),
test_case.expected_drift,
"{}",
test_case.description
Expand All @@ -148,10 +148,10 @@ fn given_live_dynamic_table_when_comparing_settings_then_only_real_drift_alters(
}

#[test]
fn given_planned_rebuilds_when_checking_maintained_readers_then_reinitialised_ones_are_named() {
fn given_planned_rebuilds_when_checking_dynamic_readers_then_reinitialised_ones_are_named() {
let test_cases = [
ReinitialiseTestCase {
description: "a maintained model reading a rebuilt table",
description: "a dynamic model reading a rebuilt table",
reads: vec![
read("orders", false, true, &[]),
read("daily_totals", true, false, &["orders"]),
Expand All @@ -167,7 +167,7 @@ fn given_planned_rebuilds_when_checking_maintained_readers_then_reinitialised_on
expected_readers: vec![],
},
ReinitialiseTestCase {
description: "a recreated maintained model reinitialises its maintained readers",
description: "a recreated dynamic model reinitialises its dynamic readers",
reads: vec![
read("clean_orders", true, true, &["orders"]),
read("daily_totals", true, false, &["clean_orders"]),
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,4 @@
use crate::maintained_refresh::models::{
LiveDynamicTable, MaintainedRead, MaintainedSettings, RefreshLag,
};
use crate::dynamic_refresh::models::{DynamicRead, DynamicSettings, LiveDynamicTable, RefreshLag};

pub(super) struct TargetLagTestCase {
pub(super) description: &'static str,
Expand All @@ -16,13 +14,13 @@ pub(super) struct RenderLagTestCase {

pub(super) struct DriftTestCase {
pub(super) description: &'static str,
pub(super) desired: MaintainedSettings,
pub(super) desired: DynamicSettings,
pub(super) live: LiveDynamicTable,
pub(super) expected_drift: bool,
}

pub(super) struct ReinitialiseTestCase {
pub(super) description: &'static str,
pub(super) reads: Vec<MaintainedRead>,
pub(super) reads: Vec<DynamicRead>,
pub(super) expected_readers: Vec<(&'static str, &'static str)>,
}
Loading
Loading