From e6b04c200079a150ff39a71d33dcfbfc03b286c1 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 19 Nov 2026 07:34:55 +0000 Subject: [PATCH 1/2] fix(build): close the maintained-model hold gaps and two rebuild residuals - A model created during a build now runs the same earliest-hold check as acquire once its pause is recorded, so two builds creating the same model no longer both hold it. Our create made the model active, so it is suspended either way; the build holding it resumes it at its end, and a build that backed out resumes it itself at its end when no other build holds it, so a model never stays suspended after both back out. - When the losing build cannot close its own pause, the A718 error now names that failure too instead of being replaced by it. - Pauses are ordered by the warehouse clock when each insert starts, but an insert commits later. A build now also backs out for a rival that paused within the time its own insert took, since that rival may have read the table before our pause committed, and when a takeover closed its pause before it read the table back. Two builds that start at the same moment can both stop with A718; neither builds alongside the other. - A snapshot's first build and full rebuild turn change tracking on when a maintained model reads it, as table builds already do (new optional render_track_changes adapter method; Snowflake runs ALTER TABLE ... SET CHANGE_TRACKING = TRUE). - The planner's change check drops refresh_mode when it is the default, on both the recorded and the current side, so writing out or removing the default no longer recreates the model. Stored-status hashes are unchanged. --- .../main/version_identity_payload.rs | 5 +- .../src/fingerprint_identity/constants.rs | 6 + .../src/fingerprint_identity/main/mod.rs | 1 + .../main/without_default_refresh_mode.rs | 16 + .../src/fingerprint_identity/mod.rs | 3 + .../src/fingerprint_identity/tests.rs | 4 + .../tests/test_fingerprint_identity.rs | 37 +++ .../fingerprint_identity/tests/test_types.rs | 5 + .../_helpers/outputs/maintained.rs | 3 +- .../docs/concepts/adapters/snowflake.md | 3 +- .../docs/concepts/models/maintained.md | 5 +- .../contract/classes/strict_adapter.py | 6 + .../snowflake/classes/snowflake_adapter.py | 3 + .../classes/maintained_hold.py | 142 +++++++-- .../run/_helpers/materializations/snapshot.py | 2 + .../compiler/planner/main/_test_types.py | 1 + .../sqlbuild/compiler/planner/main/helpers.py | 9 +- .../main/test_maintained_refresh_plan.py | 15 +- .../executor/maintained_pauses/_test_types.py | 32 ++ .../executor/maintained_pauses/helpers.py | 8 + .../test_maintained_pause_lease.py | 286 ++++++++++++++++++ .../executor/run/maintained/_test_types.py | 7 + .../executor/run/maintained/helpers.py | 12 + .../run/maintained/test_change_tracking.py | 65 +++- .../strict_adapter/test_strict_adapter.py | 1 + .../planner/_helpers/changes/test_detect.py | 26 ++ .../docs/docs/concepts/adapters/snowflake.mdx | 3 +- .../docs/docs/concepts/models/maintained.mdx | 5 +- 28 files changed, 663 insertions(+), 48 deletions(-) create mode 100644 crates/sqlbuild-core/src/fingerprint_identity/main/without_default_refresh_mode.rs create mode 100644 crates/sqlbuild-core/src/fingerprint_identity/tests.rs create mode 100644 crates/sqlbuild-core/src/fingerprint_identity/tests/test_fingerprint_identity.rs create mode 100644 crates/sqlbuild-core/src/fingerprint_identity/tests/test_types.rs diff --git a/crates/sqlbuild-analysis/src/planner_changes/main/version_identity_payload.rs b/crates/sqlbuild-analysis/src/planner_changes/main/version_identity_payload.rs index 62976bbb3b..155921eda4 100644 --- a/crates/sqlbuild-analysis/src/planner_changes/main/version_identity_payload.rs +++ b/crates/sqlbuild-analysis/src/planner_changes/main/version_identity_payload.rs @@ -3,8 +3,9 @@ use crate::planner_changes::models::IdentityPayload; use sqlbuild_core::fingerprint_identity::main::decoded_object::decoded_object; use sqlbuild_core::fingerprint_identity::main::normalize_version_identity_metadata::normalize_version_identity_metadata; +use sqlbuild_core::fingerprint_identity::main::without_default_refresh_mode::without_default_refresh_mode; -/// The metadata without audit-gate, migration and declared-column entries; with `without_functions`, also without function hashes. +/// The metadata without audit-gate, migration, declared-column and default refresh-mode entries; with `without_functions`, also without function hashes. pub fn version_identity_payload( metadata_json: Option<&str>, without_functions: bool, @@ -17,7 +18,7 @@ pub fn version_identity_payload( Ok(None) => IdentityPayload::Absent, Ok(Some(mut object)) => { normalize_version_identity_metadata(&mut object, without_functions); - IdentityPayload::Object(object) + IdentityPayload::Object(without_default_refresh_mode(object)) } } } diff --git a/crates/sqlbuild-core/src/fingerprint_identity/constants.rs b/crates/sqlbuild-core/src/fingerprint_identity/constants.rs index c32ef975c0..3da98bc0a3 100644 --- a/crates/sqlbuild-core/src/fingerprint_identity/constants.rs +++ b/crates/sqlbuild-core/src/fingerprint_identity/constants.rs @@ -4,6 +4,12 @@ pub const AUDIT_GATE_METADATA_KEY: &str = "audit_gate"; pub const MIGRATION_FINGERPRINT_METADATA_KEY: &str = "migration_fingerprint"; pub const DECLARED_COLUMNS_METADATA_KEY: &str = "declared_columns_hash"; pub const LOCAL_FUNCTION_HASHES_METADATA_KEY: &str = "local_function_hashes"; +/// Metadata key of the model config entries in version identity. +pub const CONFIG_METADATA_KEY: &str = "config"; +/// Config key of a maintained model's refresh mode. +pub const REFRESH_MODE_CONFIG_KEY: &str = "refresh_mode"; +/// The refresh mode of a maintained model that declares none; writing it out changes nothing. +pub const DEFAULT_REFRESH_MODE: &str = "incremental"; /// The fingerprint dialect when a project names none. pub(crate) const GENERIC_FINGERPRINT_DIALECT: &str = "generic"; /// Python's `NATIVE_DIALECT_ALIASES`. diff --git a/crates/sqlbuild-core/src/fingerprint_identity/main/mod.rs b/crates/sqlbuild-core/src/fingerprint_identity/main/mod.rs index 86131ac87e..e5bc381e33 100644 --- a/crates/sqlbuild-core/src/fingerprint_identity/main/mod.rs +++ b/crates/sqlbuild-core/src/fingerprint_identity/main/mod.rs @@ -6,3 +6,4 @@ pub mod normalize_version_identity_metadata; pub mod python_equal; pub mod recorded_model_local_hash; pub mod stable_identity_hash; +pub mod without_default_refresh_mode; diff --git a/crates/sqlbuild-core/src/fingerprint_identity/main/without_default_refresh_mode.rs b/crates/sqlbuild-core/src/fingerprint_identity/main/without_default_refresh_mode.rs new file mode 100644 index 0000000000..43d32c10a6 --- /dev/null +++ b/crates/sqlbuild-core/src/fingerprint_identity/main/without_default_refresh_mode.rs @@ -0,0 +1,16 @@ +//! Drop a written-out default refresh mode, which plans the same as none. + +use serde_json::{Map, Value}; + +use crate::fingerprint_identity::constants::{ + CONFIG_METADATA_KEY, DEFAULT_REFRESH_MODE, REFRESH_MODE_CONFIG_KEY, +}; + +pub fn without_default_refresh_mode(mut object: Map) -> Map { + if let Some(Value::Object(config)) = object.get_mut(CONFIG_METADATA_KEY) + && config.get(REFRESH_MODE_CONFIG_KEY) == Some(&Value::from(DEFAULT_REFRESH_MODE)) + { + config.shift_remove(REFRESH_MODE_CONFIG_KEY); + } + object +} diff --git a/crates/sqlbuild-core/src/fingerprint_identity/mod.rs b/crates/sqlbuild-core/src/fingerprint_identity/mod.rs index 919648bfd1..0cc81d3427 100644 --- a/crates/sqlbuild-core/src/fingerprint_identity/mod.rs +++ b/crates/sqlbuild-core/src/fingerprint_identity/mod.rs @@ -2,3 +2,6 @@ pub mod constants; pub mod main; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-core/src/fingerprint_identity/tests.rs b/crates/sqlbuild-core/src/fingerprint_identity/tests.rs new file mode 100644 index 0000000000..9f95f97207 --- /dev/null +++ b/crates/sqlbuild-core/src/fingerprint_identity/tests.rs @@ -0,0 +1,4 @@ +#[path = "tests/test_fingerprint_identity.rs"] +mod fingerprint_identity; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-core/src/fingerprint_identity/tests/test_fingerprint_identity.rs b/crates/sqlbuild-core/src/fingerprint_identity/tests/test_fingerprint_identity.rs new file mode 100644 index 0000000000..1f89bfcfcd --- /dev/null +++ b/crates/sqlbuild-core/src/fingerprint_identity/tests/test_fingerprint_identity.rs @@ -0,0 +1,37 @@ +use crate::fingerprint_identity::main::canonical_text::canonical_text; +use crate::fingerprint_identity::main::decoded_object::decoded_object; +use crate::fingerprint_identity::main::model_local_identity_hash::model_local_identity_hash; +use crate::fingerprint_identity::main::recorded_model_local_hash::recorded_model_local_hash; +use crate::fingerprint_identity::tests::test_types::RecordedLocalHashTestCase; + +#[test] +fn given_recorded_metadata_when_hashing_then_it_matches_the_config_as_written() { + let test_cases = [ + RecordedLocalHashTestCase { + description: "a written-out default refresh mode stays in the hash", + metadata_json: r#"{"config":{"materialized":"maintained","refresh_mode":"incremental"},"execution_signature":{},"local_function_hashes":{},"model_name":"daily_totals"}"#, + expected_hashed_json: r#"{"config":{"materialized":"maintained","refresh_mode":"incremental"},"execution_signature":{},"local_function_hashes":{},"model_name":"daily_totals"}"#, + }, + RecordedLocalHashTestCase { + description: "a model that declares no refresh mode", + metadata_json: r#"{"config":{"materialized":"maintained"},"execution_signature":{},"local_function_hashes":{},"model_name":"daily_totals"}"#, + expected_hashed_json: r#"{"config":{"materialized":"maintained"},"execution_signature":{},"local_function_hashes":{},"model_name":"daily_totals"}"#, + }, + ]; + + for test_case in test_cases { + assert_eq!( + recorded_model_local_hash("query_hash", test_case.metadata_json), + Some(model_local_identity_hash( + "query_hash", + &canonical_text( + decoded_object(test_case.expected_hashed_json) + .expect("metadata is JSON") + .expect("metadata is an object") + ) + )), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-core/src/fingerprint_identity/tests/test_types.rs b/crates/sqlbuild-core/src/fingerprint_identity/tests/test_types.rs new file mode 100644 index 0000000000..669e385090 --- /dev/null +++ b/crates/sqlbuild-core/src/fingerprint_identity/tests/test_types.rs @@ -0,0 +1,5 @@ +pub(super) struct RecordedLocalHashTestCase { + pub(super) description: &'static str, + pub(super) metadata_json: &'static str, + pub(super) expected_hashed_json: &'static str, +} diff --git a/crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/maintained.rs b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/maintained.rs index 4f0650b1be..cafd4de572 100644 --- a/crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/maintained.rs +++ b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/maintained.rs @@ -16,6 +16,7 @@ use sqlbuild_analysis::maintained_refresh::models::{ LiveDynamicTable, MaintainedRead, MaintainedSettings, RefreshLag, }; use sqlbuild_analysis::planner_scope::constants::MODEL; +use sqlbuild_core::fingerprint_identity::constants::DEFAULT_REFRESH_MODE; use sqlbuild_model_config::model_validation::main::fixed_duration_seconds::fixed_duration_seconds; use sqlbuild_python_base::records::main::construct::enum_member::enum_member; use sqlbuild_python_base::records::main::fields::field::field as native_field; @@ -35,8 +36,6 @@ const VIEW: &str = "view"; const KEEP_MAINTAINED: &str = "keep_maintained"; /// The scheduling state Snowflake lists for a dynamic table whose refreshes are paused. const SUSPENDED: &str = "SUSPENDED"; -/// The refresh mode of a maintained model that declares none, so Snowflake never picks one quietly. -const DEFAULT_REFRESH_MODE: &str = "incremental"; /// Actions that recreate the relation, which reinitialises every dynamic table reading it. const REPLACING_ACTIONS: [&str; 3] = ["create_table", "create_view", "create_maintained"]; diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/snowflake.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/snowflake.md index fa4c878d51..085bf739f4 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/snowflake.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/snowflake.md @@ -181,7 +181,8 @@ The build role needs these grants: Dynamic tables refresh from changes, which needs change tracking on every table they read: -- A table model that a maintained model reads, directly or through views, is built with change tracking on. +- A table or snapshot model that a maintained model reads, directly or through views, is built with + change tracking on. - For a source table, Snowflake turns change tracking on when the build role owns the table. Otherwise its owner runs `ALTER TABLE SET CHANGE_TRACKING = TRUE`. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/maintained.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/maintained.md index 777c4de3c5..cc52805737 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/maintained.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/maintained.md @@ -64,7 +64,8 @@ whether it succeeds or fails. ## When a build dies - **One build at a time.** A build that needs a maintained model another build holds fails with - `A718`, naming that run. Wait for it to finish. + `A718`, naming that run. Wait for it to finish. Two builds that start at the same moment can + both fail this way; run one of them again. - **A stopped build leaves its models suspended.** A build killed before it resumes them, for example by `SIGKILL`, can't resume them itself. - **The next build resumes them.** The next build that pauses maintained models on the target @@ -76,7 +77,5 @@ whether it succeeds or fails. ## Limitations -- Writing out the default `refresh_mode incremental` recreates the model once. -- A snapshot rebuilt in full loses change tracking; run `ALTER TABLE SET CHANGE_TRACKING = TRUE` before a maintained model reads it again. - Unit tests and scenarios treat a maintained model as a table. - On a target with `schema = "preserve"`, builds see each other's pauses only when their maintained models share a schema. diff --git a/src/sqlbuild/adapter/contract/classes/strict_adapter.py b/src/sqlbuild/adapter/contract/classes/strict_adapter.py index 06cc06a89e..5af764ab19 100644 --- a/src/sqlbuild/adapter/contract/classes/strict_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/strict_adapter.py @@ -353,6 +353,12 @@ def render_maintained_operation( del operation raise self._maintained_unsupported(destination) + def render_track_changes(self, *, destination: str) -> tuple[str, ...]: + """Optional: SQL that turns on change tracking for a table a maintained model reads.""" + + del destination + return () + def _maintained_unsupported(self, destination: str) -> AdapterUserError: return AdapterUserError( f"the {self.adapter_name} adapter cannot build maintained model '{destination}'", diff --git a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py index 19dc7d7195..62c48475c8 100644 --- a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py +++ b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py @@ -3710,6 +3710,9 @@ def render_maintained_operation( ) -> tuple[str, ...]: return (f"ALTER DYNAMIC TABLE {destination} {operation.value.upper()}",) + def render_track_changes(self, *, destination: str) -> tuple[str, ...]: + return (f"ALTER TABLE {destination} SET CHANGE_TRACKING = TRUE",) + def _warehouse_name(self, warehouse: str) -> str: """A plain warehouse name as written, resolved without case; any other quoted exactly.""" diff --git a/src/sqlbuild/executor/maintained_pauses/classes/maintained_hold.py b/src/sqlbuild/executor/maintained_pauses/classes/maintained_hold.py index 9a5c14654b..2c1e1db4c4 100644 --- a/src/sqlbuild/executor/maintained_pauses/classes/maintained_hold.py +++ b/src/sqlbuild/executor/maintained_pauses/classes/maintained_hold.py @@ -3,8 +3,10 @@ from __future__ import annotations import threading +import time from collections.abc import Callable, Iterator from contextlib import contextmanager +from datetime import datetime, timedelta from typing import Any from sqlbuild._native import CompiledRelationLocation @@ -57,6 +59,7 @@ def __init__( self._heartbeat: MaintainedHeartbeat | None = None self._paused: list[str] = [] self._left_paused: frozenset[str] = frozenset() + self._suspended_unheld: list[str] = [] self._lock: threading.Lock = threading.Lock() self._on_progress: Callable[[str], None] | None = None @@ -111,7 +114,7 @@ def acquire(self, *, connection: Any, on_progress: Callable[[str], None] | None) ) for name in sorted(self._left_paused): self._warn(f"maintained model {name} was paused outside SQLBuild; it stays paused") - self._record( + window: timedelta = self._timed_record( connection=connection, events=( *( @@ -124,13 +127,15 @@ def acquire(self, *, connection: Any, on_progress: Callable[[str], None] | None) ), ), ) - self._yield_to_earlier(connection=connection, location=location) + self._yield_to_rivals( + connection=connection, location=location, names=tuple(self._paused), window=window + ) self._suspend_active(connection=connection, names=tuple(self._paused), states=states) self._heartbeat = self._new_heartbeat(self._write_heartbeat) self._heartbeat.start() def after_create(self, *, connection: Any, name: str) -> None: - """Suspend a held model that CREATE OR REPLACE made active; resume it only if ours.""" + """Suspend a model CREATE OR REPLACE made active; check its new hold as acquire does.""" if name not in self._held: return @@ -140,12 +145,23 @@ def after_create(self, *, connection: Any, name: str) -> None: ) if newly_paused: self._paused.append(name) + window: timedelta = timedelta() if newly_paused: - self._record( + window = self._timed_record( connection=connection, events=(self._own(event_type=MaintainedPauseEventType.PAUSED, name=name),), ) self._operate(connection=connection, name=name, operation=MaintainedOperation.SUSPEND) + location: StateLocation | None = self._state_location + if newly_paused and location is not None: + try: + self._yield_to_rivals( + connection=connection, location=location, names=(name,), window=window + ) + except AdapterUserError: + with self._lock: + self._suspended_unheld.append(name) + raise def release(self, *, connection: Any) -> None: """Stop the heartbeat and resume what this build paused, upstream first.""" @@ -154,9 +170,14 @@ def release(self, *, connection: Any) -> None: self._heartbeat.stop() self._report_heartbeat_failure(failure=self._heartbeat.failure) order: list[str] = list(self._held) - still_held: frozenset[_RelationKey] | None = self._still_held(connection=connection) + holds: tuple[OpenHold, ...] | None = self._open_holds(connection=connection) + still_held: frozenset[_RelationKey] | None = ( + None + if holds is None + else frozenset(_key(hold.relation) for hold in holds if hold.run_id == self._run_id) + ) resumed: list[str] = [] - failures: list[Exception] = [] + failures: list[Exception] = self._resume_unheld(connection=connection, holds=holds) for name in sorted(self._paused, key=order.index): if still_held is not None and _key(self._held[name]) not in still_held: continue @@ -218,10 +239,17 @@ def _take_over(self, *, connection: Any, location: StateLocation) -> tuple[OpenH self._report(holds=taken, verb="Took over") return taken - def _yield_to_earlier(self, *, connection: Any, location: StateLocation) -> None: - """Read the pauses back; if another live hold on a model came first, close ours and stop.""" + def _yield_to_rivals( + self, + *, + connection: Any, + location: StateLocation, + names: tuple[str, ...], + window: timedelta, + ) -> None: + """Read the pauses back; if another live hold may have come first, close ours and stop.""" - mine: set[_RelationKey] = {_key(self._held[name]) for name in self._paused} + mine: set[_RelationKey] = {_key(self._held[name]) for name in names} if not mine: return rows: tuple[tuple[Any, ...], ...] = tuple( @@ -231,27 +259,65 @@ def _yield_to_earlier(self, *, connection: Any, location: StateLocation) -> None limit=None, ).rows ) - first: dict[_RelationKey, OpenHold] = {} - for hold in sorted(open_holds(rows=rows), key=lambda hold: (hold.paused_at, hold.run_id)): - if hold.idle_seconds < self._grace_seconds and _key(hold.relation) in mine: - first.setdefault(_key(hold.relation), hold) - winners: tuple[OpenHold, ...] = tuple( - hold for hold in first.values() if hold.run_id != self._run_id + holds: tuple[OpenHold, ...] = open_holds(rows=rows) + paused_at: dict[_RelationKey, datetime] = { + _key(hold.relation): hold.paused_at + for hold in holds + if hold.run_id == self._run_id and _key(hold.relation) in mine + } + rivals: list[OpenHold] = sorted( + ( + hold + for hold in holds + if hold.run_id != self._run_id + and hold.idle_seconds < self._grace_seconds + and _key(hold.relation) in mine + and ( + _key(hold.relation) not in paused_at + or hold.paused_at <= paused_at[_key(hold.relation)] + window + ) + ), + key=lambda hold: (hold.paused_at, hold.run_id), ) - if not winners: + if not rivals: return - self._record( - connection=connection, - events=tuple( - self._own(event_type=MaintainedPauseEventType.RESUMED, name=name) - for name in self._paused - ), + with self._lock: + self._paused = [name for name in self._paused if name not in names] + try: + self._record( + connection=connection, + events=tuple( + self._own(event_type=MaintainedPauseEventType.RESUMED, name=name) + for name in names + ), + ) + except Exception as error: + raise _held_elsewhere(holds=tuple(rivals), close_failure=error) from error + raise _held_elsewhere(holds=tuple(rivals)) + + def _resume_unheld( + self, *, connection: Any, holds: tuple[OpenHold, ...] | None + ) -> list[Exception]: + """Resume the models this build suspended after creating them, then gave up, if unheld.""" + + held_elsewhere: frozenset[_RelationKey] = frozenset( + _key(hold.relation) for hold in holds or () if hold.run_id != self._run_id ) - self._paused = [] - raise _held_elsewhere(holds=winners) + failures: list[Exception] = [] + for name in self._suspended_unheld: + if holds is None or _key(self._held[name]) in held_elsewhere: + continue + try: + if self._state(connection=connection, name=name) == SUSPENDED_STATE: + self._operate( + connection=connection, name=name, operation=MaintainedOperation.RESUME + ) + except Exception as error: + failures.append(error) + return failures - def _still_held(self, *, connection: Any) -> frozenset[_RelationKey] | None: - """This run's holds nobody took over, or None when the pause table can't say.""" + def _open_holds(self, *, connection: Any) -> tuple[OpenHold, ...] | None: + """Every open hold, or None when the pause table can't say.""" location: StateLocation | None = self._state_location if location is None or self._pause_maintained: @@ -267,9 +333,7 @@ def _still_held(self, *, connection: Any) -> frozenset[_RelationKey] | None: except Exception as error: self._warn(f"could not read which maintained models this build still holds: {error}") return None - return frozenset( - _key(hold.relation) for hold in open_holds(rows=rows) if hold.run_id == self._run_id - ) + return open_holds(rows=rows) def _resume_found(self, *, connection: Any, relation: CompiledRelationLocation) -> None: if _scheduling_state(adapter=self._adapter, connection=connection, relation=relation) == ( @@ -324,6 +388,15 @@ def _destination(self, location: CompiledRelationLocation) -> str: ) ) + def _timed_record( + self, *, connection: Any, events: tuple[MaintainedPauseEvent, ...] + ) -> timedelta: + """Record, and return how long the insert took: an upper bound on its commit delay.""" + + started: float = time.monotonic() + self._record(connection=connection, events=events) + return timedelta(seconds=time.monotonic() - started) + def _record(self, *, connection: Any, events: tuple[MaintainedPauseEvent, ...]) -> None: location: StateLocation | None = self._state_location if not events or location is None: @@ -399,12 +472,19 @@ def _key(location: CompiledRelationLocation) -> _RelationKey: return (location.database, location.schema, location.name) -def _held_elsewhere(*, holds: tuple[OpenHold, ...]) -> AdapterUserError: +def _held_elsewhere( + *, holds: tuple[OpenHold, ...], close_failure: Exception | None = None +) -> AdapterUserError: first: OpenHold = holds[0] names: str = ", ".join(sorted({hold.relation.name for hold in holds})) + closing: str = ( + "" + if close_failure is None + else f"; closing this build's own pause also failed: {close_failure}" + ) return AdapterUserError( f"run {first.run_id} holds maintained models this build needs ({names}); " - f"its last heartbeat was {int(first.idle_seconds)}s ago", + f"its last heartbeat was {int(first.idle_seconds)}s ago{closing}", code=HELD_ELSEWHERE_CODE, help=( "wait for that build to finish, or for its hold to pass [maintained] " diff --git a/src/sqlbuild/executor/run/_helpers/materializations/snapshot.py b/src/sqlbuild/executor/run/_helpers/materializations/snapshot.py index cf3b7bf4e7..96f46bf335 100644 --- a/src/sqlbuild/executor/run/_helpers/materializations/snapshot.py +++ b/src/sqlbuild/executor/run/_helpers/materializations/snapshot.py @@ -800,6 +800,8 @@ def _create_initial_snapshot_target( valid_to_column=valid_to_column, initial_valid_from=entry.initial_valid_from, ) + if entry.maintained_readers: + statements = (*statements, *adapter.render_track_changes(destination=target_qualified)) statement_recorder.record_many(statements) statement: str for statement in statements: diff --git a/tests/integration/src/sqlbuild/compiler/planner/main/_test_types.py b/tests/integration/src/sqlbuild/compiler/planner/main/_test_types.py index ac3e1abfbe..2651952329 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/main/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/planner/main/_test_types.py @@ -219,6 +219,7 @@ class MaintainedRefreshPlanTestCase: expected_reinitialised_by: str | None expected_read_names: tuple[str, ...] expected_paused: bool = False + totals_config: dict[str, object] | None = None @dataclass(frozen=True) diff --git a/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py b/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py index b13f754f11..ba0c256eab 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py @@ -858,14 +858,19 @@ def maintained_listing( return self.listed -def maintained_project_case(upstream_query: str) -> BuildExecutionPlanTestCase: +def maintained_project_case( + upstream_query: str, totals_config: dict[str, object] | None = None +) -> BuildExecutionPlanTestCase: """Clean orders and the daily totals maintained from it, both refreshed within an hour.""" return BuildExecutionPlanTestCase( description="clean orders and the daily totals maintained from it", setup_sql=(), model_locations={"clean_orders": "staging", "daily_totals": "staging"}, - model_configs={"clean_orders": _MAINTAINED, "daily_totals": _MAINTAINED}, + model_configs={ + "clean_orders": _MAINTAINED, + "daily_totals": {**_MAINTAINED, **(totals_config or {})}, + }, model_queries={ "clean_orders": upstream_query, "daily_totals": "SELECT count(*) AS orders FROM staging.clean_orders", diff --git a/tests/integration/src/sqlbuild/compiler/planner/main/test_maintained_refresh_plan.py b/tests/integration/src/sqlbuild/compiler/planner/main/test_maintained_refresh_plan.py index 351c65275e..ac00d92fd7 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/main/test_maintained_refresh_plan.py +++ b/tests/integration/src/sqlbuild/compiler/planner/main/test_maintained_refresh_plan.py @@ -37,6 +37,17 @@ expected_reinitialised_by=None, expected_read_names=("clean_orders", "daily_totals"), ), + MaintainedRefreshPlanTestCase( + description="writing out the default refresh_mode keeps the model", + listed_settings=MaintainedListing("60 minutes", "TRANSFORM_WH", "ACTIVE"), + upstream_query=CLEAN_ORDERS_QUERY, + expected_upstream_action=PlanAction.KEEP_MAINTAINED, + expected_action=PlanAction.KEEP_MAINTAINED, + expected_reason=PlanReason.NO_CHANGE, + expected_reinitialised_by=None, + expected_read_names=("clean_orders", "daily_totals"), + totals_config={"refresh_mode": "incremental"}, + ), MaintainedRefreshPlanTestCase( description="a kept model listed as suspended is reported as paused", listed_settings=MaintainedListing("60 minutes", "TRANSFORM_WH", "SUSPENDED"), @@ -107,7 +118,9 @@ def test_given_built_maintained_models_when_replanning_then_live_settings_and_re ) plan: PlanOutput = build_execution_plan_from_kwargs( - project=build_project_from_test_case(maintained_project_case(test_case.upstream_query)), + project=build_project_from_test_case( + maintained_project_case(test_case.upstream_query, test_case.totals_config) + ), adapter=adapter, connection=connection, ) diff --git a/tests/integration/src/sqlbuild/executor/maintained_pauses/_test_types.py b/tests/integration/src/sqlbuild/executor/maintained_pauses/_test_types.py index 01cd40ee39..19e1bb611e 100644 --- a/tests/integration/src/sqlbuild/executor/maintained_pauses/_test_types.py +++ b/tests/integration/src/sqlbuild/executor/maintained_pauses/_test_types.py @@ -2,6 +2,8 @@ from dataclasses import dataclass +from sqlbuild.executor.maintained_pauses.types import MaintainedPauseEventType + @dataclass(frozen=True) class PauseLeaseTestCase: @@ -74,3 +76,33 @@ class HeartbeatTestCase: heartbeat_seconds: float wait_seconds: float expected_min_beats: int + + +@dataclass(frozen=True) +class CreatedModelHoldTestCase: + description: str + refused_events: tuple[MaintainedPauseEventType, ...] + rival_closes: tuple[str, ...] + expected_release_operations: tuple[tuple[str, str], ...] + expected_code: str + expected_message_fragments: tuple[str, ...] + expected_states: dict[str, str] + expected_events: tuple[tuple[str, str, str, str], ...] + + +@dataclass(frozen=True) +class LateCommitTestCase: + description: str + expected_code: str + expected_rival: str + expected_states: dict[str, str] + expected_events: tuple[tuple[str, str, str, str], ...] + + +@dataclass(frozen=True) +class TakenOverPauseTestCase: + description: str + expected_code: str + expected_rival: str + expected_states: dict[str, str] + expected_events: tuple[tuple[str, str, str, str], ...] diff --git a/tests/integration/src/sqlbuild/executor/maintained_pauses/helpers.py b/tests/integration/src/sqlbuild/executor/maintained_pauses/helpers.py index f956db03e5..ff8ef4bf33 100644 --- a/tests/integration/src/sqlbuild/executor/maintained_pauses/helpers.py +++ b/tests/integration/src/sqlbuild/executor/maintained_pauses/helpers.py @@ -19,6 +19,7 @@ BuildExecutionResult, BuildRuntimeParams, ) +from sqlbuild.executor.maintained_pauses.models import MaintainedPauseEvent from tests.integration.src.sqlbuild.compiler.planner.main._test_types import ( BuildExecutionPlanTestCase, ) @@ -174,3 +175,10 @@ def pause_table_names(*, connection: Any) -> list[tuple[Any, ...]]: "SELECT table_name FROM information_schema.tables " "WHERE table_schema = 'staging' AND table_name = '_sqlbuild_maintained_pauses'" ).fetchall() + + +def refuse_insert(*, connection: Any, events: tuple[MaintainedPauseEvent, ...]) -> None: + """Stand in for a pause-table insert the warehouse refuses.""" + + del connection, events + raise RuntimeError("pause table is read-only") diff --git a/tests/integration/src/sqlbuild/executor/maintained_pauses/test_maintained_pause_lease.py b/tests/integration/src/sqlbuild/executor/maintained_pauses/test_maintained_pause_lease.py index 0bb8cbd9e6..aa95449702 100644 --- a/tests/integration/src/sqlbuild/executor/maintained_pauses/test_maintained_pause_lease.py +++ b/tests/integration/src/sqlbuild/executor/maintained_pauses/test_maintained_pause_lease.py @@ -17,18 +17,24 @@ BuildRuntimeParams, ) from sqlbuild.executor.build.types import BuildStatus +from sqlbuild.executor.maintained_pauses._helpers.sql import insert_sql from sqlbuild.executor.maintained_pauses.classes.maintained_hold import MaintainedHold from sqlbuild.executor.maintained_pauses.main.hold_maintained_models import ( hold_maintained_models, ) +from sqlbuild.executor.maintained_pauses.models import MaintainedPauseEvent +from sqlbuild.executor.maintained_pauses.types import MaintainedPauseEventType, StateLocation from tests.integration.src.sqlbuild.executor.maintained_pauses._test_types import ( ConcurrentHoldTestCase, + CreatedModelHoldTestCase, FirstBuildTestCase, HeartbeatTestCase, HoldRefusalTestCase, HoldTakeoverTestCase, + LateCommitTestCase, LifecycleLineTestCase, PauseLeaseTestCase, + TakenOverPauseTestCase, TakenOverReleaseTestCase, ) from tests.integration.src.sqlbuild.executor.maintained_pauses.helpers import ( @@ -40,6 +46,7 @@ pause_events, pause_table_names, plan_lease_project, + refuse_insert, ) _ORDERS: str = "SELECT 1 AS order_id" @@ -636,3 +643,282 @@ def state_then_rival_acquires(*, connection: Any, name: str) -> str | None: assert adapter.states == test_case.expected_states assert pause_events(connection=connection) == sorted(test_case.expected_events) early.release(connection=connection) + + +@pytest.mark.parametrize( + "test_case", + [ + CreatedModelHoldTestCase( + description="of two builds creating a model, the later one backs out", + refused_events=(), + rival_closes=(), + expected_release_operations=(), + expected_code="A718", + expected_message_fragments=("early_run",), + expected_states={"daily_totals": "SUSPENDED"}, + expected_events=( + ("paused", "early_run", "early_run", "daily_totals"), + ("paused", "late_run", "late_run", "daily_totals"), + ("resumed", "late_run", "late_run", "daily_totals"), + ), + ), + CreatedModelHoldTestCase( + description="a failed close of the later pause is reported with the hold", + refused_events=(MaintainedPauseEventType.RESUMED,), + rival_closes=(), + expected_release_operations=(), + expected_code="A718", + expected_message_fragments=("early_run", "pause table is read-only"), + expected_states={"daily_totals": "SUSPENDED"}, + expected_events=( + ("paused", "early_run", "early_run", "daily_totals"), + ("paused", "late_run", "late_run", "daily_totals"), + ), + ), + CreatedModelHoldTestCase( + description="when the earlier build backs out too, the later one resumes the model", + refused_events=(), + rival_closes=("daily_totals",), + expected_release_operations=(("resume", "daily_totals"),), + expected_code="A718", + expected_message_fragments=("early_run",), + expected_states={"daily_totals": "ACTIVE"}, + expected_events=( + ("paused", "early_run", "early_run", "daily_totals"), + ("paused", "late_run", "late_run", "daily_totals"), + ("resumed", "early_run", "early_run", "daily_totals"), + ("resumed", "late_run", "late_run", "daily_totals"), + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_two_builds_creating_a_model_when_both_pause_it_then_only_the_earliest_holds( + test_case: CreatedModelHoldTestCase, + adapter: PausingDuckDbAdapter, + connection: Any, + database_path: Path, +) -> None: + plan: PlanOutput = plan_lease_project( + adapter=adapter, + connection=connection, + case=lease_project_case(orders_query=_ORDERS, report_locations={}, refresh_on_build=False), + ) + holds: dict[str, MaintainedHold] = {} + for run_id in ("early_run", "late_run"): + hold: MaintainedHold | None = hold_maintained_models( + plan=plan, + adapter=adapter, + runtime=BuildRuntimeParams(run_id=run_id), + connection_config={"database": str(database_path)}, + ) + assert hold is not None + hold.acquire(connection=connection, on_progress=None) + holds[run_id] = hold + late: MaintainedHold = holds["late_run"] + recorders: dict[tuple[MaintainedPauseEventType, ...], Callable[..., None]] = { + (): late._record, + **{(event_type,): late._record for event_type in MaintainedPauseEventType}, + **{(event_type,): refuse_insert for event_type in test_case.refused_events}, + } + + def record_or_refuse(*, connection: Any, events: tuple[MaintainedPauseEvent, ...]) -> None: + recorders[tuple(event.event_type for event in events)](connection=connection, events=events) + + late._record = record_or_refuse # ty: ignore[invalid-assignment] + adapter.states["daily_totals"] = "ACTIVE" + holds["early_run"].after_create(connection=connection, name="daily_totals") + adapter.states["daily_totals"] = "ACTIVE" + + with pytest.raises(AdapterUserError) as raised: + late.after_create(connection=connection, name="daily_totals") + early: MaintainedHold = holds["early_run"] + early._record( + connection=connection, + events=tuple( + early._own(event_type=MaintainedPauseEventType.RESUMED, name=name) + for name in test_case.rival_closes + ), + ) + adapter.operations.clear() + late.release(connection=connection) + + assert raised.value.code == test_case.expected_code + for fragment in test_case.expected_message_fragments: + assert fragment in str(raised.value) + assert tuple(adapter.operations) == test_case.expected_release_operations + assert adapter.states == test_case.expected_states + assert pause_events(connection=connection) == sorted(test_case.expected_events) + holds["early_run"].release(connection=connection) + + +@pytest.mark.parametrize( + "test_case", + [ + LateCommitTestCase( + description="a build whose pause commits after a rival read the table backs out", + expected_code="A718", + expected_rival="late_run", + expected_states={"daily_totals": "SUSPENDED"}, + expected_events=( + ("paused", "early_run", "early_run", "daily_totals"), + ("paused", "first_run", "first_run", "daily_totals"), + ("paused", "late_run", "late_run", "daily_totals"), + ("resumed", "early_run", "early_run", "daily_totals"), + ("resumed", "first_run", "first_run", "daily_totals"), + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_pause_committing_late_when_rival_paused_meanwhile_then_the_late_one_backs_out( + test_case: LateCommitTestCase, + adapter: PausingDuckDbAdapter, + connection: Any, + database_path: Path, +) -> None: + _ = build_lease_project( + adapter=adapter, + connection=connection, + case=lease_project_case(orders_query=_ORDERS, report_locations={}, refresh_on_build=False), + runtime=BuildRuntimeParams(run_id="first_run"), + database_path=database_path, + ) + plan: PlanOutput = plan_lease_project( + adapter=adapter, + connection=connection, + case=lease_project_case( + orders_query=_MORE_ORDERS, report_locations={}, refresh_on_build=False + ), + ) + holds: dict[str, MaintainedHold] = {} + for run_id in ("early_run", "late_run"): + hold: MaintainedHold | None = hold_maintained_models( + plan=plan, + adapter=adapter, + runtime=BuildRuntimeParams(run_id=run_id), + connection_config={"database": str(database_path)}, + ) + assert hold is not None + holds[run_id] = hold + early: MaintainedHold = holds["early_run"] + location: StateLocation | None = early._state_location + assert location is not None + now: str = adapter.render_current_utc_timestamp() + + def commit_after_rival(*, connection: Any, events: tuple[MaintainedPauseEvent, ...]) -> None: + started: Any = connection.execute(f"SELECT {now}").fetchone()[0] + holds["late_run"].acquire(connection=connection, on_progress=None) + adapter.execute( + connection=connection, + sql=insert_sql(adapter=adapter, location=location, events=events).replace( + now, f"TIMESTAMP '{started.isoformat()}'" + ), + ) + + recorders: dict[tuple[MaintainedPauseEventType, ...], Callable[..., None]] = { + (): early._record, + (MaintainedPauseEventType.PAUSED,): commit_after_rival, + (MaintainedPauseEventType.RESUMED,): early._record, + } + + def record_committing_late( + *, connection: Any, events: tuple[MaintainedPauseEvent, ...] + ) -> None: + recorders[tuple(event.event_type for event in events)](connection=connection, events=events) + + early._record = record_committing_late # ty: ignore[invalid-assignment] + + with pytest.raises(AdapterUserError) as raised: + early.acquire(connection=connection, on_progress=None) + + assert raised.value.code == test_case.expected_code + assert test_case.expected_rival in str(raised.value) + assert adapter.states == test_case.expected_states + assert pause_events(connection=connection) == sorted(test_case.expected_events) + holds["late_run"].release(connection=connection) + + +@pytest.mark.parametrize( + "test_case", + [ + TakenOverPauseTestCase( + description="a build whose pause is taken over before it reads back backs out", + expected_code="A718", + expected_rival="taker_run", + expected_states={"daily_totals": "SUSPENDED"}, + expected_events=( + ("paused", "early_run", "early_run", "daily_totals"), + ("paused", "first_run", "first_run", "daily_totals"), + ("paused", "taker_run", "taker_run", "daily_totals"), + ("resumed", "early_run", "early_run", "daily_totals"), + ("resumed", "early_run", "taker_run", "daily_totals"), + ("resumed", "first_run", "first_run", "daily_totals"), + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_pause_taken_over_before_read_back_when_acquiring_then_the_build_backs_out( + test_case: TakenOverPauseTestCase, + adapter: PausingDuckDbAdapter, + connection: Any, + database_path: Path, +) -> None: + _ = build_lease_project( + adapter=adapter, + connection=connection, + case=lease_project_case(orders_query=_ORDERS, report_locations={}, refresh_on_build=False), + runtime=BuildRuntimeParams(run_id="first_run"), + database_path=database_path, + ) + plan: PlanOutput = plan_lease_project( + adapter=adapter, + connection=connection, + case=lease_project_case( + orders_query=_MORE_ORDERS, report_locations={}, refresh_on_build=False + ), + ) + holds: dict[str, MaintainedHold] = {} + for runtime in ( + BuildRuntimeParams(run_id="early_run"), + BuildRuntimeParams( + run_id="taker_run", ignore_maintained_grace=True, allow_ignore_maintained_grace=True + ), + ): + hold: MaintainedHold | None = hold_maintained_models( + plan=plan, + adapter=adapter, + runtime=runtime, + connection_config={"database": str(database_path)}, + ) + assert hold is not None + holds[runtime.run_id] = hold + early: MaintainedHold = holds["early_run"] + record: Callable[..., None] = early._record + + def record_then_rival_takes_over( + *, connection: Any, events: tuple[MaintainedPauseEvent, ...] + ) -> None: + record(connection=connection, events=events) + holds["taker_run"].acquire(connection=connection, on_progress=None) + + recorders: dict[tuple[MaintainedPauseEventType, ...], Callable[..., None]] = { + (): record, + (MaintainedPauseEventType.PAUSED,): record_then_rival_takes_over, + (MaintainedPauseEventType.RESUMED,): record, + } + + def record_or_take_over(*, connection: Any, events: tuple[MaintainedPauseEvent, ...]) -> None: + recorders[tuple(event.event_type for event in events)](connection=connection, events=events) + + early._record = record_or_take_over # ty: ignore[invalid-assignment] + + with pytest.raises(AdapterUserError) as raised: + early.acquire(connection=connection, on_progress=None) + + assert raised.value.code == test_case.expected_code + assert test_case.expected_rival in str(raised.value) + assert adapter.states == test_case.expected_states + assert pause_events(connection=connection) == sorted(test_case.expected_events) + holds["taker_run"].release(connection=connection) diff --git a/tests/integration/src/sqlbuild/executor/run/maintained/_test_types.py b/tests/integration/src/sqlbuild/executor/run/maintained/_test_types.py index 3958be16a0..9debeb3066 100644 --- a/tests/integration/src/sqlbuild/executor/run/maintained/_test_types.py +++ b/tests/integration/src/sqlbuild/executor/run/maintained/_test_types.py @@ -46,3 +46,10 @@ class ChangeTrackingTableTestCase: promotion_mode: TablePromotionMode declared_columns: tuple[tuple[str, str], ...] expected_tracking: tuple[bool, ...] + + +@dataclass(frozen=True) +class ChangeTrackingSnapshotTestCase: + description: str + maintained_readers: tuple[str, ...] + expected_tracked: tuple[str, ...] diff --git a/tests/integration/src/sqlbuild/executor/run/maintained/helpers.py b/tests/integration/src/sqlbuild/executor/run/maintained/helpers.py index 1f2a9d098b..3848f0de8c 100644 --- a/tests/integration/src/sqlbuild/executor/run/maintained/helpers.py +++ b/tests/integration/src/sqlbuild/executor/run/maintained/helpers.py @@ -148,3 +148,15 @@ def create_table_as( config=config, statement_recorder=statement_recorder, ) + + +class TrackedSnapshotDuckDbAdapter(DuckDbAdapter): + """Records each table it is asked to track changes on after creating it.""" + + def __init__(self) -> None: + super().__init__() + self.tracked: list[str] = [] + + def render_track_changes(self, *, destination: str) -> tuple[str, ...]: + self.tracked.append(destination) + return () diff --git a/tests/integration/src/sqlbuild/executor/run/maintained/test_change_tracking.py b/tests/integration/src/sqlbuild/executor/run/maintained/test_change_tracking.py index 4321614ce8..d9d9fd5fe0 100644 --- a/tests/integration/src/sqlbuild/executor/run/maintained/test_change_tracking.py +++ b/tests/integration/src/sqlbuild/executor/run/maintained/test_change_tracking.py @@ -7,8 +7,9 @@ import pytest -from sqlbuild._native import ModelPlanEntry +from sqlbuild._native import ModelPlanEntry, PlanReason from sqlbuild.adapter.contract.types import TablePromotionMode +from sqlbuild.executor.run._helpers.materializations.snapshot import execute_snapshot_entry from sqlbuild.executor.run.main._execute import execute_table_entry from sqlbuild.executor.run.models import ModelExecutionResult, ModelMaterializationContext from sqlbuild.executor.scheduling.types import ExecutionStatus @@ -17,9 +18,16 @@ build_table_plan_entry, ) from tests.integration.src.sqlbuild.executor.run.maintained._test_types import ( + ChangeTrackingSnapshotTestCase, ChangeTrackingTableTestCase, ) -from tests.integration.src.sqlbuild.executor.run.maintained.helpers import TrackingDuckDbAdapter +from tests.integration.src.sqlbuild.executor.run.maintained.helpers import ( + TrackedSnapshotDuckDbAdapter, + TrackingDuckDbAdapter, +) +from tests.unit.src.sqlbuild.executor.run._helpers.helpers import ( + build_snapshot_execution_plan_entry, +) @pytest.mark.parametrize( @@ -83,3 +91,56 @@ def test_given_table_read_by_maintained_model_when_building_then_every_create_tr assert result.status == ExecutionStatus.SUCCESS, result.error_message assert tuple(adapter.tracking) == test_case.expected_tracking + + +@pytest.mark.parametrize( + "test_case", + [ + ChangeTrackingSnapshotTestCase( + description="a snapshot's first build and its full rebuild both track changes", + maintained_readers=("daily_totals",), + expected_tracked=( + "main.customer_snapshot", + "main.__sqb_rebuild__customer_snapshot", + ), + ), + ChangeTrackingSnapshotTestCase( + description="a snapshot no maintained model reads is built as before", + maintained_readers=(), + expected_tracked=(), + ), + ], + ids=lambda case: case.description, +) +def test_given_snapshot_read_by_maintained_model_when_rebuilding_then_it_tracks_changes( + test_case: ChangeTrackingSnapshotTestCase, +) -> None: + adapter: TrackedSnapshotDuckDbAdapter = TrackedSnapshotDuckDbAdapter() + connection: Any = adapter.connect({"database": ":memory:"}) + entry: ModelPlanEntry = replace( + build_snapshot_execution_plan_entry(), maintained_readers=test_case.maintained_readers + ) + + results: list[ModelExecutionResult] = [ + execute_snapshot_entry( + context=ModelMaterializationContext( + entry=replace(entry, reason=reason), + adapter=adapter, + connection=connection, + model_locations={}, + seed_locations={}, + source_map={}, + python_source_read_map={}, + model_audits=(), + run_id="tracking_run", + query_change_tracking=False, + ) + ) + for reason in (PlanReason.FIRST_RUN, PlanReason.FULL_REFRESH) + ] + + assert [result.status for result in results] == [ExecutionStatus.SUCCESS] * 2, [ + result.error_message for result in results + ] + assert tuple(adapter.tracked) == test_case.expected_tracked + adapter.close(connection) diff --git a/tests/unit/src/sqlbuild/adapter/contract/classes/strict_adapter/test_strict_adapter.py b/tests/unit/src/sqlbuild/adapter/contract/classes/strict_adapter/test_strict_adapter.py index 301ac4504f..c8a16a45e9 100644 --- a/tests/unit/src/sqlbuild/adapter/contract/classes/strict_adapter/test_strict_adapter.py +++ b/tests/unit/src/sqlbuild/adapter/contract/classes/strict_adapter/test_strict_adapter.py @@ -95,6 +95,7 @@ def test_given_first_class_adapters_when_checking_contract_then_no_method_comes_ "render_create_temporary_function", "render_drop_schema", "render_maintained_operation", + "render_track_changes", "session_warehouse", } ), diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_detect.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_detect.py index 6edecd18ea..ef5d2d2836 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_detect.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_detect.py @@ -590,6 +590,32 @@ def test_given_dropped_relation_with_recorded_build_when_planning_then_plans_fir '"post_hooks":[{"statement":"INSERT INTO audit_log SELECT \'new\'","type":"sql"}]', ), ), + DetectModelMetadataTestCase( + description="writing out the default refresh_mode is no change", + config_values={"materialized": "maintained", "refresh_mode": "incremental"}, + schema_columns=(), + deps=(), + function_local_hashes={}, + previous_metadata_json=build_version_identity_metadata_json( + model_name="orders", + config_values={"materialized": "maintained"}, + execution_signature={}, + ), + expected_change_kind=ChangeKind.NO_CHANGE, + ), + DetectModelMetadataTestCase( + description="removing a written-out default refresh_mode is no change", + config_values={"materialized": "maintained"}, + schema_columns=(), + deps=(), + function_local_hashes={}, + previous_metadata_json=build_version_identity_metadata_json( + model_name="orders", + config_values={"materialized": "maintained", "refresh_mode": "incremental"}, + execution_signature={}, + ), + expected_change_kind=ChangeKind.NO_CHANGE, + ), DetectModelMetadataTestCase( description="ignores runtime audit gate proof when version identity matches", config_values={"materialized": "table"}, diff --git a/website/src/content/docs/docs/concepts/adapters/snowflake.mdx b/website/src/content/docs/docs/concepts/adapters/snowflake.mdx index f229f05121..87400a7ee8 100644 --- a/website/src/content/docs/docs/concepts/adapters/snowflake.mdx +++ b/website/src/content/docs/docs/concepts/adapters/snowflake.mdx @@ -165,7 +165,8 @@ The build role needs these grants: Dynamic tables refresh from changes, which needs change tracking on every table they read: -- A table model that a maintained model reads, directly or through views, is built with change tracking on. +- A table or snapshot model that a maintained model reads, directly or through views, is built with + change tracking on. - For a source table, Snowflake turns change tracking on when the build role owns the table. Otherwise its owner runs `ALTER TABLE SET CHANGE_TRACKING = TRUE`. diff --git a/website/src/content/docs/docs/concepts/models/maintained.mdx b/website/src/content/docs/docs/concepts/models/maintained.mdx index e825a6db5e..3ab5dedc62 100644 --- a/website/src/content/docs/docs/concepts/models/maintained.mdx +++ b/website/src/content/docs/docs/concepts/models/maintained.mdx @@ -61,7 +61,8 @@ whether it succeeds or fails. ## When a build dies - **One build at a time.** A build that needs a maintained model another build holds fails with - `A718`, naming that run. Wait for it to finish. + `A718`, naming that run. Wait for it to finish. Two builds that start at the same moment can + both fail this way; run one of them again. - **A stopped build leaves its models suspended.** A build killed before it resumes them, for example by `SIGKILL`, can't resume them itself. - **The next build resumes them.** The next build that pauses maintained models on the target @@ -73,7 +74,5 @@ whether it succeeds or fails. ## Limitations -- Writing out the default `refresh_mode incremental` recreates the model once. -- A snapshot rebuilt in full loses change tracking; run `ALTER TABLE SET CHANGE_TRACKING = TRUE` before a maintained model reads it again. - Unit tests and scenarios treat a maintained model as a table. - On a target with `schema = "preserve"`, builds see each other's pauses only when their maintained models share a schema. From 648484b3fe319710e13cff43c07cdafde2c36f65 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 19 Nov 2026 08:42:49 +0000 Subject: [PATCH 2/2] feat(build)!: rename maintained models to dynamic models The model kind is now called dynamic everywhere: `materialized dynamic`, the `[dynamic]` project section (still `heartbeat_grace` and `allow_ignore_grace`), the `pause_dynamic` target setting, the `--ignore-dynamic-grace` build flag, the `_sqlbuild_dynamic_pauses` state table, plan actions and wording, error text, the Dynamic models docs page and the tests. Error codes are unchanged. BREAKING CHANGE: `materialized maintained`, `[maintained]`, `pause_maintained` and `--ignore-maintained-grace` are no longer accepted; use `materialized dynamic`, `[dynamic]`, `pause_dynamic` and `--ignore-dynamic-grace`. The materialization is part of a model's version identity, so each existing model is recreated once, with a full recompute, by its first build after the rename. Builds record pauses in `_sqlbuild_dynamic_pauses` and no longer read `_sqlbuild_maintained_pauses`: let builds on the previous version finish before upgrading, resume by hand any model a killed older build left suspended, then drop the old table. --- .../src/assembly/dynamic_checks/constants.rs | 4 + .../src/assembly/dynamic_checks/main/mod.rs | 1 + .../main/never_refreshed_dynamic.rs} | 10 +- .../mod.rs | 0 .../src/assembly/dynamic_checks/models.rs | 9 ++ .../tests.rs | 4 +- .../assembly/dynamic_checks/tests/helpers.rs | 10 ++ .../tests/test_dynamic_checks.rs} | 26 ++-- .../tests/test_types.rs | 4 +- .../assembly/maintained_checks/constants.rs | 4 - .../assembly/maintained_checks/main/mod.rs | 1 - .../src/assembly/maintained_checks/models.rs | 9 -- .../maintained_checks/tests/helpers.rs | 10 -- crates/sqlbuild-analysis/src/assembly/mod.rs | 2 +- .../_helpers/mod.rs | 0 .../_helpers/target_lag.rs | 4 +- .../constants.rs | 0 .../main/dynamic_settings_drift.rs | 15 ++ .../main/mod.rs | 2 +- .../main/reinitialised_readers.rs | 13 +- .../main/render_target_lag.rs | 4 +- .../src/dynamic_refresh/mod.rs | 9 ++ .../models.rs | 18 +-- .../tests.rs | 4 +- .../tests/helpers.rs | 14 +- .../tests/test_dynamic_refresh.rs} | 22 +-- .../tests/test_types.rs | 8 +- crates/sqlbuild-analysis/src/lib.rs | 2 +- .../main/maintained_settings_drift.rs | 15 -- .../src/maintained_refresh/mod.rs | 9 -- .../main/materialization_type.rs | 2 +- .../main/model_plan_action.rs | 6 +- .../src/planner_strategy/models.rs | 10 +- .../tests/test_planner_strategy.rs | 36 ++--- .../sqlbuild-cli/src/argv/command_table.json | 4 +- crates/sqlbuild-cli/src/argv/constants.rs | 2 +- crates/sqlbuild-cli/src/dispatch/constants.rs | 2 +- .../src/effective/constants.rs | 4 +- .../src/project/_helpers/documents.rs | 12 +- .../src/project/_helpers/sections.rs | 35 ++--- .../src/project/_helpers/targets.rs | 6 +- .../sqlbuild-config/src/project/constants.rs | 12 +- .../src/project/main/load_dynamic_config.rs | 9 ++ .../project/main/load_maintained_config.rs | 9 -- .../sqlbuild-config/src/project/main/mod.rs | 2 +- crates/sqlbuild-config/src/project/models.rs | 6 +- .../src/project/tests/test_load_config.rs | 82 +++++------ .../src/project/tests/test_types.rs | 8 +- .../src/cli_text/_helpers/plan_sections.rs | 10 +- .../sqlbuild-core/src/cli_text/constants.rs | 4 +- crates/sqlbuild-core/src/cli_text/models.rs | 4 +- .../src/cli_text/tests/test_cli_text.rs | 26 ++-- crates/sqlbuild-core/src/constants.rs | 6 +- .../src/fingerprint_identity/constants.rs | 4 +- .../tests/test_fingerprint_identity.rs | 8 +- .../_helpers/materialization.rs | 42 +++--- .../src/model_validation/constants.rs | 26 ++-- .../main/validate_model_config.rs | 4 +- .../tests/test_validate_model_config.rs | 46 +++---- .../tests/api/test_warehouse_metadata.rs | 6 +- .../_helpers/compiler_enums/constants.rs | 8 +- .../_helpers/contract_records/constants.rs | 4 +- .../_helpers/planner_records/constants.rs | 4 +- .../src/records/constants.rs | 2 +- .../setup/{maintained.rs => dynamic.rs} | 27 ++-- .../src/compile_driver/_helpers/setup/mod.rs | 2 +- .../compile_driver/_helpers/setup/project.rs | 4 +- ...ed_maintained.rs => configured_dynamic.rs} | 24 ++-- .../src/discovery/main/project/mod.rs | 2 +- .../src/live_reads/models.rs | 4 +- .../src/plan_output/_helpers/plan_json.rs | 2 +- .../src/plan_output/_helpers/plan_text.rs | 2 +- .../_helpers/plans/display_plan.rs | 6 +- .../_helpers/plans/execution_plan.rs | 4 +- .../outputs/{maintained.rs => dynamic.rs} | 66 +++++---- .../planner_phases/_helpers/outputs/mod.rs | 2 +- .../src/planner_phases/main/outputs/mod.rs | 2 +- .../main/outputs/plan_dynamic_refresh.rs | 15 ++ .../main/outputs/plan_maintained_refresh.rs | 15 -- .../src/planner_strategy/_helpers/strategy.rs | 6 +- .../src/planner_strategy/_helpers/values.rs | 2 +- .../src/planner_warehouse/models.rs | 8 +- .../_helpers/build_command/execution.rs | 6 +- .../_helpers/exported_constants/constants.rs | 4 +- fensu.toml | 4 +- .../sqlbuild/references/docs/CONTENTS.md | 2 +- .../sqlbuild/references/docs/cli/build.md | 2 +- .../sqlbuild/references/docs/cli/dag.md | 2 +- .../docs/concepts/adapters/snowflake.md | 12 +- .../docs/concepts/models/configuration.md | 2 +- .../models/{maintained.md => dynamic.md} | 34 ++--- .../docs/concepts/models/materializations.md | 10 +- .../docs/concepts/project-configuration.md | 14 +- src/sqlbuild/_native.pyi | 30 ++-- .../contract/classes/strict_adapter.py | 38 +++--- src/sqlbuild/adapter/contract/types.py | 8 +- .../snowflake/classes/snowflake_adapter.py | 20 +-- .../executor/build/_helpers/scheduler.py | 8 +- .../executor/build/classes/build_scheduler.py | 2 +- src/sqlbuild/executor/build/constants.py | 8 +- src/sqlbuild/executor/build/main/_execute.py | 14 +- src/sqlbuild/executor/build/models.py | 12 +- .../executor/dynamic_pauses/__init__.py | 1 + .../dynamic_pauses/_helpers/__init__.py | 1 + .../_helpers/scope.py | 20 +-- .../_helpers/sql.py | 46 +++---- .../dynamic_pauses/classes/__init__.py | 1 + .../classes/dynamic_heartbeat.py} | 4 +- .../classes/dynamic_hold.py} | 108 +++++++-------- .../constants.py | 12 +- .../executor/dynamic_pauses/main/__init__.py | 1 + .../main/hold_dynamic_models.py} | 38 +++--- .../models.py | 8 +- .../types.py | 4 +- .../executor/maintained_pauses/__init__.py | 1 - .../maintained_pauses/_helpers/__init__.py | 1 - .../maintained_pauses/classes/__init__.py | 1 - .../maintained_pauses/main/__init__.py | 1 - .../run/_helpers/execution/table_config.py | 2 +- .../{maintained.py => dynamic.py} | 32 ++--- .../_helpers/materializations/microbatch.py | 2 +- .../run/_helpers/materializations/snapshot.py | 2 +- src/sqlbuild/executor/run/main/_execute.py | 8 +- src/sqlbuild/executor/run/models.py | 4 +- .../cli/commands/main/compile/_test_types.py | 4 +- ...ained_models.py => test_dynamic_models.py} | 40 +++--- .../compiler/failure/target-unknown-key.json | 2 +- .../compiler/planner/main/_test_types.py | 8 +- .../sqlbuild/compiler/planner/main/helpers.py | 30 ++-- ...h_plan.py => test_dynamic_refresh_plan.py} | 76 +++++------ .../executor/dynamic_pauses/__init__.py | 1 + .../_test_types.py | 6 +- .../conftest.py | 2 +- .../helpers.py | 46 +++---- .../test_dynamic_pause_lease.py} | 129 +++++++++--------- .../executor/maintained_pauses/__init__.py | 1 - .../run/{maintained => dynamic}/__init__.py | 0 .../{maintained => dynamic}/_test_types.py | 12 +- .../run/{maintained => dynamic}/helpers.py | 30 ++-- .../test_change_tracking.py | 36 ++--- .../test_dynamic_execution.py} | 78 +++++------ .../strict_adapter/test_strict_adapter.py | 14 +- .../adapters/snowflake/_test_types.py | 16 +-- ...ained_models.py => test_dynamic_models.py} | 60 ++++---- .../planner/_helpers/changes/test_detect.py | 20 +-- ui/src/lib/api/types/operate.ts | 4 +- ui/src/lib/run/plan-text.test.ts | 20 +-- ui/src/lib/run/plan-text.ts | 6 +- website/src/content/docs/docs/cli/build.mdx | 2 +- website/src/content/docs/docs/cli/dag.mdx | 2 +- .../docs/docs/concepts/adapters/snowflake.mdx | 10 +- .../docs/concepts/models/configuration.mdx | 2 +- .../models/{maintained.mdx => dynamic.mdx} | 32 ++--- .../docs/concepts/models/materializations.mdx | 8 +- .../docs/concepts/project-configuration.mdx | 12 +- website/src/sidebar.json | 2 +- 156 files changed, 1048 insertions(+), 1070 deletions(-) create mode 100644 crates/sqlbuild-analysis/src/assembly/dynamic_checks/constants.rs create mode 100644 crates/sqlbuild-analysis/src/assembly/dynamic_checks/main/mod.rs rename crates/sqlbuild-analysis/src/assembly/{maintained_checks/main/never_refreshed_maintained.rs => dynamic_checks/main/never_refreshed_dynamic.rs} (53%) rename crates/sqlbuild-analysis/src/assembly/{maintained_checks => dynamic_checks}/mod.rs (100%) create mode 100644 crates/sqlbuild-analysis/src/assembly/dynamic_checks/models.rs rename crates/sqlbuild-analysis/src/assembly/{maintained_checks => dynamic_checks}/tests.rs (57%) create mode 100644 crates/sqlbuild-analysis/src/assembly/dynamic_checks/tests/helpers.rs rename crates/sqlbuild-analysis/src/assembly/{maintained_checks/tests/test_maintained_checks.rs => dynamic_checks/tests/test_dynamic_checks.rs} (54%) rename crates/sqlbuild-analysis/src/assembly/{maintained_checks => dynamic_checks}/tests/test_types.rs (56%) delete mode 100644 crates/sqlbuild-analysis/src/assembly/maintained_checks/constants.rs delete mode 100644 crates/sqlbuild-analysis/src/assembly/maintained_checks/main/mod.rs delete mode 100644 crates/sqlbuild-analysis/src/assembly/maintained_checks/models.rs delete mode 100644 crates/sqlbuild-analysis/src/assembly/maintained_checks/tests/helpers.rs rename crates/sqlbuild-analysis/src/{maintained_refresh => dynamic_refresh}/_helpers/mod.rs (100%) rename crates/sqlbuild-analysis/src/{maintained_refresh => dynamic_refresh}/_helpers/target_lag.rs (85%) rename crates/sqlbuild-analysis/src/{maintained_refresh => dynamic_refresh}/constants.rs (100%) create mode 100644 crates/sqlbuild-analysis/src/dynamic_refresh/main/dynamic_settings_drift.rs rename crates/sqlbuild-analysis/src/{maintained_refresh => dynamic_refresh}/main/mod.rs (62%) rename crates/sqlbuild-analysis/src/{maintained_refresh => dynamic_refresh}/main/reinitialised_readers.rs (60%) rename crates/sqlbuild-analysis/src/{maintained_refresh => dynamic_refresh}/main/render_target_lag.rs (81%) create mode 100644 crates/sqlbuild-analysis/src/dynamic_refresh/mod.rs rename crates/sqlbuild-analysis/src/{maintained_refresh => dynamic_refresh}/models.rs (51%) rename crates/sqlbuild-analysis/src/{maintained_refresh => dynamic_refresh}/tests.rs (56%) rename crates/sqlbuild-analysis/src/{maintained_refresh => dynamic_refresh}/tests/helpers.rs (59%) rename crates/sqlbuild-analysis/src/{maintained_refresh/tests/test_maintained_refresh.rs => dynamic_refresh/tests/test_dynamic_refresh.rs} (88%) rename crates/sqlbuild-analysis/src/{maintained_refresh => dynamic_refresh}/tests/test_types.rs (75%) delete mode 100644 crates/sqlbuild-analysis/src/maintained_refresh/main/maintained_settings_drift.rs delete mode 100644 crates/sqlbuild-analysis/src/maintained_refresh/mod.rs create mode 100644 crates/sqlbuild-config/src/project/main/load_dynamic_config.rs delete mode 100644 crates/sqlbuild-config/src/project/main/load_maintained_config.rs rename crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/{maintained.rs => dynamic.rs} (71%) rename crates/sqlbuild-python-front/src/discovery/main/project/{configured_maintained.rs => configured_dynamic.rs} (69%) rename crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/{maintained.rs => dynamic.rs} (80%) create mode 100644 crates/sqlbuild-python-planner/src/planner_phases/main/outputs/plan_dynamic_refresh.rs delete mode 100644 crates/sqlbuild-python-planner/src/planner_phases/main/outputs/plan_maintained_refresh.rs rename src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/{maintained.md => dynamic.md} (67%) create mode 100644 src/sqlbuild/executor/dynamic_pauses/__init__.py create mode 100644 src/sqlbuild/executor/dynamic_pauses/_helpers/__init__.py rename src/sqlbuild/executor/{maintained_pauses => dynamic_pauses}/_helpers/scope.py (76%) rename src/sqlbuild/executor/{maintained_pauses => dynamic_pauses}/_helpers/sql.py (78%) create mode 100644 src/sqlbuild/executor/dynamic_pauses/classes/__init__.py rename src/sqlbuild/executor/{maintained_pauses/classes/maintained_heartbeat.py => dynamic_pauses/classes/dynamic_heartbeat.py} (95%) rename src/sqlbuild/executor/{maintained_pauses/classes/maintained_hold.py => dynamic_pauses/classes/dynamic_hold.py} (81%) rename src/sqlbuild/executor/{maintained_pauses => dynamic_pauses}/constants.py (75%) create mode 100644 src/sqlbuild/executor/dynamic_pauses/main/__init__.py rename src/sqlbuild/executor/{maintained_pauses/main/hold_maintained_models.py => dynamic_pauses/main/hold_dynamic_models.py} (58%) rename src/sqlbuild/executor/{maintained_pauses => dynamic_pauses}/models.py (76%) rename src/sqlbuild/executor/{maintained_pauses => dynamic_pauses}/types.py (75%) delete mode 100644 src/sqlbuild/executor/maintained_pauses/__init__.py delete mode 100644 src/sqlbuild/executor/maintained_pauses/_helpers/__init__.py delete mode 100644 src/sqlbuild/executor/maintained_pauses/classes/__init__.py delete mode 100644 src/sqlbuild/executor/maintained_pauses/main/__init__.py rename src/sqlbuild/executor/run/_helpers/materializations/{maintained.py => dynamic.py} (84%) rename tests/e2e/src/sqlbuild/cli/commands/main/compile/{test_maintained_models.py => test_dynamic_models.py} (72%) rename tests/integration/src/sqlbuild/compiler/planner/main/{test_maintained_refresh_plan.py => test_dynamic_refresh_plan.py} (71%) create mode 100644 tests/integration/src/sqlbuild/executor/dynamic_pauses/__init__.py rename tests/integration/src/sqlbuild/executor/{maintained_pauses => dynamic_pauses}/_test_types.py (94%) rename tests/integration/src/sqlbuild/executor/{maintained_pauses => dynamic_pauses}/conftest.py (89%) rename tests/integration/src/sqlbuild/executor/{maintained_pauses => dynamic_pauses}/helpers.py (80%) rename tests/integration/src/sqlbuild/executor/{maintained_pauses/test_maintained_pause_lease.py => dynamic_pauses/test_dynamic_pause_lease.py} (87%) delete mode 100644 tests/integration/src/sqlbuild/executor/maintained_pauses/__init__.py rename tests/integration/src/sqlbuild/executor/run/{maintained => dynamic}/__init__.py (100%) rename tests/integration/src/sqlbuild/executor/run/{maintained => dynamic}/_test_types.py (81%) rename tests/integration/src/sqlbuild/executor/run/{maintained => dynamic}/helpers.py (86%) rename tests/integration/src/sqlbuild/executor/run/{maintained => dynamic}/test_change_tracking.py (82%) rename tests/integration/src/sqlbuild/executor/run/{maintained/test_maintained_execution.py => dynamic/test_dynamic_execution.py} (76%) rename tests/unit/src/sqlbuild/adapters/snowflake/{test_maintained_models.py => test_dynamic_models.py} (81%) rename website/src/content/docs/docs/concepts/models/{maintained.mdx => dynamic.mdx} (67%) diff --git a/crates/sqlbuild-analysis/src/assembly/dynamic_checks/constants.rs b/crates/sqlbuild-analysis/src/assembly/dynamic_checks/constants.rs new file mode 100644 index 0000000000..f3d94893ac --- /dev/null +++ b/crates/sqlbuild-analysis/src/assembly/dynamic_checks/constants.rs @@ -0,0 +1,4 @@ +//! Error codes of the project-wide dynamic model checks. + +/// A dynamic model that nothing would ever refresh. +pub const DYNAMIC_NEVER_REFRESHED_CODE: &str = "A016"; diff --git a/crates/sqlbuild-analysis/src/assembly/dynamic_checks/main/mod.rs b/crates/sqlbuild-analysis/src/assembly/dynamic_checks/main/mod.rs new file mode 100644 index 0000000000..cb3c63689b --- /dev/null +++ b/crates/sqlbuild-analysis/src/assembly/dynamic_checks/main/mod.rs @@ -0,0 +1 @@ +pub mod never_refreshed_dynamic; diff --git a/crates/sqlbuild-analysis/src/assembly/maintained_checks/main/never_refreshed_maintained.rs b/crates/sqlbuild-analysis/src/assembly/dynamic_checks/main/never_refreshed_dynamic.rs similarity index 53% rename from crates/sqlbuild-analysis/src/assembly/maintained_checks/main/never_refreshed_maintained.rs rename to crates/sqlbuild-analysis/src/assembly/dynamic_checks/main/never_refreshed_dynamic.rs index 078332bc87..74ecb201d2 100644 --- a/crates/sqlbuild-analysis/src/assembly/maintained_checks/main/never_refreshed_maintained.rs +++ b/crates/sqlbuild-analysis/src/assembly/dynamic_checks/main/never_refreshed_dynamic.rs @@ -1,16 +1,16 @@ -//! Maintained models with no refresh of their own that no maintained model reads. +//! Dynamic models with no refresh of their own that no dynamic model reads. use std::collections::HashSet; -use crate::assembly::maintained_checks::models::MaintainedModel; +use crate::assembly::dynamic_checks::models::DynamicModel; /// The first by name; without a reader or its own refresh, the warehouse never refreshes it. -pub fn never_refreshed_maintained(models: &[MaintainedModel]) -> Option<&str> { - let maintained: HashSet<&str> = models.iter().map(|model| model.name.as_str()).collect(); +pub fn never_refreshed_dynamic(models: &[DynamicModel]) -> Option<&str> { + let dynamic: HashSet<&str> = models.iter().map(|model| model.name.as_str()).collect(); let read: HashSet<&str> = models .iter() .flat_map(|model| model.reads.iter().map(String::as_str)) - .filter(|name| maintained.contains(name)) + .filter(|name| dynamic.contains(name)) .collect(); models .iter() diff --git a/crates/sqlbuild-analysis/src/assembly/maintained_checks/mod.rs b/crates/sqlbuild-analysis/src/assembly/dynamic_checks/mod.rs similarity index 100% rename from crates/sqlbuild-analysis/src/assembly/maintained_checks/mod.rs rename to crates/sqlbuild-analysis/src/assembly/dynamic_checks/mod.rs diff --git a/crates/sqlbuild-analysis/src/assembly/dynamic_checks/models.rs b/crates/sqlbuild-analysis/src/assembly/dynamic_checks/models.rs new file mode 100644 index 0000000000..9d23af7451 --- /dev/null +++ b/crates/sqlbuild-analysis/src/assembly/dynamic_checks/models.rs @@ -0,0 +1,9 @@ +//! What the project-wide dynamic model checks read. + +/// One dynamic model: whether it sets its own refresh, and the models it reads. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct DynamicModel { + pub name: String, + pub has_refresh: bool, + pub reads: Vec, +} diff --git a/crates/sqlbuild-analysis/src/assembly/maintained_checks/tests.rs b/crates/sqlbuild-analysis/src/assembly/dynamic_checks/tests.rs similarity index 57% rename from crates/sqlbuild-analysis/src/assembly/maintained_checks/tests.rs rename to crates/sqlbuild-analysis/src/assembly/dynamic_checks/tests.rs index e40ce2efc9..66cecbeac6 100644 --- a/crates/sqlbuild-analysis/src/assembly/maintained_checks/tests.rs +++ b/crates/sqlbuild-analysis/src/assembly/dynamic_checks/tests.rs @@ -1,6 +1,6 @@ +#[path = "tests/test_dynamic_checks.rs"] +mod dynamic_checks; #[path = "tests/helpers.rs"] mod helpers; -#[path = "tests/test_maintained_checks.rs"] -mod maintained_checks; #[path = "tests/test_types.rs"] mod test_types; diff --git a/crates/sqlbuild-analysis/src/assembly/dynamic_checks/tests/helpers.rs b/crates/sqlbuild-analysis/src/assembly/dynamic_checks/tests/helpers.rs new file mode 100644 index 0000000000..b1d8b22730 --- /dev/null +++ b/crates/sqlbuild-analysis/src/assembly/dynamic_checks/tests/helpers.rs @@ -0,0 +1,10 @@ +use crate::assembly::dynamic_checks::models::DynamicModel; + +/// A dynamic model named `name` reading `reads`. +pub(crate) fn dynamic(name: &str, has_refresh: bool, reads: &[&str]) -> DynamicModel { + DynamicModel { + name: name.to_owned(), + has_refresh, + reads: reads.iter().map(|read| (*read).to_owned()).collect(), + } +} diff --git a/crates/sqlbuild-analysis/src/assembly/maintained_checks/tests/test_maintained_checks.rs b/crates/sqlbuild-analysis/src/assembly/dynamic_checks/tests/test_dynamic_checks.rs similarity index 54% rename from crates/sqlbuild-analysis/src/assembly/maintained_checks/tests/test_maintained_checks.rs rename to crates/sqlbuild-analysis/src/assembly/dynamic_checks/tests/test_dynamic_checks.rs index f363cfca59..577186ee4e 100644 --- a/crates/sqlbuild-analysis/src/assembly/maintained_checks/tests/test_maintained_checks.rs +++ b/crates/sqlbuild-analysis/src/assembly/dynamic_checks/tests/test_dynamic_checks.rs @@ -1,49 +1,49 @@ -use crate::assembly::maintained_checks::main::never_refreshed_maintained::never_refreshed_maintained; -use crate::assembly::maintained_checks::tests::helpers::maintained; -use crate::assembly::maintained_checks::tests::test_types::NeverRefreshedTestCase; +use crate::assembly::dynamic_checks::main::never_refreshed_dynamic::never_refreshed_dynamic; +use crate::assembly::dynamic_checks::tests::helpers::dynamic; +use crate::assembly::dynamic_checks::tests::test_types::NeverRefreshedTestCase; #[test] -fn given_maintained_chains_when_checking_then_unreached_models_without_refresh_are_named() { +fn given_dynamic_chains_when_checking_then_unreached_models_without_refresh_are_named() { let test_cases = [ NeverRefreshedTestCase { description: "an upstream model refreshed for its reader", models: vec![ - maintained("clean_orders", false, &["orders"]), - maintained("daily_totals", true, &["clean_orders"]), + dynamic("clean_orders", false, &["orders"]), + dynamic("daily_totals", true, &["clean_orders"]), ], expected_name: None, }, NeverRefreshedTestCase { description: "the last model of a chain without refresh", models: vec![ - maintained("clean_orders", false, &["orders"]), - maintained("daily_totals", false, &["clean_orders"]), + dynamic("clean_orders", false, &["orders"]), + dynamic("daily_totals", false, &["clean_orders"]), ], expected_name: Some("daily_totals"), }, NeverRefreshedTestCase { description: "a model read only by a table, which never asks for a refresh", - models: vec![maintained("clean_orders", false, &["orders"])], + models: vec![dynamic("clean_orders", false, &["orders"])], expected_name: Some("clean_orders"), }, NeverRefreshedTestCase { description: "two unread models report the first by name", models: vec![ - maintained("b_totals", false, &["orders"]), - maintained("a_totals", false, &["orders"]), + dynamic("b_totals", false, &["orders"]), + dynamic("a_totals", false, &["orders"]), ], expected_name: Some("a_totals"), }, NeverRefreshedTestCase { description: "every model sets its own refresh", - models: vec![maintained("daily_totals", true, &[])], + models: vec![dynamic("daily_totals", true, &[])], expected_name: None, }, ]; for test_case in test_cases { assert_eq!( - never_refreshed_maintained(&test_case.models), + never_refreshed_dynamic(&test_case.models), test_case.expected_name, "{}", test_case.description diff --git a/crates/sqlbuild-analysis/src/assembly/maintained_checks/tests/test_types.rs b/crates/sqlbuild-analysis/src/assembly/dynamic_checks/tests/test_types.rs similarity index 56% rename from crates/sqlbuild-analysis/src/assembly/maintained_checks/tests/test_types.rs rename to crates/sqlbuild-analysis/src/assembly/dynamic_checks/tests/test_types.rs index 66f9fbe2f1..0bd0f37594 100644 --- a/crates/sqlbuild-analysis/src/assembly/maintained_checks/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/assembly/dynamic_checks/tests/test_types.rs @@ -1,7 +1,7 @@ -use crate::assembly::maintained_checks::models::MaintainedModel; +use crate::assembly::dynamic_checks::models::DynamicModel; pub(crate) struct NeverRefreshedTestCase { pub(crate) description: &'static str, - pub(crate) models: Vec, + pub(crate) models: Vec, pub(crate) expected_name: Option<&'static str>, } diff --git a/crates/sqlbuild-analysis/src/assembly/maintained_checks/constants.rs b/crates/sqlbuild-analysis/src/assembly/maintained_checks/constants.rs deleted file mode 100644 index 385fec1fd5..0000000000 --- a/crates/sqlbuild-analysis/src/assembly/maintained_checks/constants.rs +++ /dev/null @@ -1,4 +0,0 @@ -//! Error codes of the project-wide maintained model checks. - -/// A maintained model that nothing would ever refresh. -pub const MAINTAINED_NEVER_REFRESHED_CODE: &str = "A016"; diff --git a/crates/sqlbuild-analysis/src/assembly/maintained_checks/main/mod.rs b/crates/sqlbuild-analysis/src/assembly/maintained_checks/main/mod.rs deleted file mode 100644 index d43fa0186a..0000000000 --- a/crates/sqlbuild-analysis/src/assembly/maintained_checks/main/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub mod never_refreshed_maintained; diff --git a/crates/sqlbuild-analysis/src/assembly/maintained_checks/models.rs b/crates/sqlbuild-analysis/src/assembly/maintained_checks/models.rs deleted file mode 100644 index 4724ca32f3..0000000000 --- a/crates/sqlbuild-analysis/src/assembly/maintained_checks/models.rs +++ /dev/null @@ -1,9 +0,0 @@ -//! What the project-wide maintained model checks read. - -/// One maintained model: whether it sets its own refresh, and the models it reads. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct MaintainedModel { - pub name: String, - pub has_refresh: bool, - pub reads: Vec, -} diff --git a/crates/sqlbuild-analysis/src/assembly/maintained_checks/tests/helpers.rs b/crates/sqlbuild-analysis/src/assembly/maintained_checks/tests/helpers.rs deleted file mode 100644 index 2ad27cec20..0000000000 --- a/crates/sqlbuild-analysis/src/assembly/maintained_checks/tests/helpers.rs +++ /dev/null @@ -1,10 +0,0 @@ -use crate::assembly::maintained_checks::models::MaintainedModel; - -/// A maintained model named `name` reading `reads`. -pub(crate) fn maintained(name: &str, has_refresh: bool, reads: &[&str]) -> MaintainedModel { - MaintainedModel { - name: name.to_owned(), - has_refresh, - reads: reads.iter().map(|read| (*read).to_owned()).collect(), - } -} diff --git a/crates/sqlbuild-analysis/src/assembly/mod.rs b/crates/sqlbuild-analysis/src/assembly/mod.rs index 4e62fc77e8..4405d0dfa9 100644 --- a/crates/sqlbuild-analysis/src/assembly/mod.rs +++ b/crates/sqlbuild-analysis/src/assembly/mod.rs @@ -1,6 +1,6 @@ //! The native model analysis session: catalog, dataflow scheduling, projection and shape publication. pub mod analysis_session; -pub mod maintained_checks; +pub mod dynamic_checks; pub mod project; pub mod target_checks; diff --git a/crates/sqlbuild-analysis/src/maintained_refresh/_helpers/mod.rs b/crates/sqlbuild-analysis/src/dynamic_refresh/_helpers/mod.rs similarity index 100% rename from crates/sqlbuild-analysis/src/maintained_refresh/_helpers/mod.rs rename to crates/sqlbuild-analysis/src/dynamic_refresh/_helpers/mod.rs diff --git a/crates/sqlbuild-analysis/src/maintained_refresh/_helpers/target_lag.rs b/crates/sqlbuild-analysis/src/dynamic_refresh/_helpers/target_lag.rs similarity index 85% rename from crates/sqlbuild-analysis/src/maintained_refresh/_helpers/target_lag.rs rename to crates/sqlbuild-analysis/src/dynamic_refresh/_helpers/target_lag.rs index 6123f16214..764a87e693 100644 --- a/crates/sqlbuild-analysis/src/maintained_refresh/_helpers/target_lag.rs +++ b/crates/sqlbuild-analysis/src/dynamic_refresh/_helpers/target_lag.rs @@ -1,7 +1,7 @@ //! A dynamic table's listed `target_lag`, such as `DOWNSTREAM` or `2 hours`. -use crate::maintained_refresh::constants::{DOWNSTREAM_TARGET_LAG, TARGET_LAG_UNITS}; -use crate::maintained_refresh::models::RefreshLag; +use crate::dynamic_refresh::constants::{DOWNSTREAM_TARGET_LAG, TARGET_LAG_UNITS}; +use crate::dynamic_refresh::models::RefreshLag; /// The lag `text` lists, or `None` when it is not a lag Snowflake lists. pub(crate) fn parse_target_lag(text: &str) -> Option { diff --git a/crates/sqlbuild-analysis/src/maintained_refresh/constants.rs b/crates/sqlbuild-analysis/src/dynamic_refresh/constants.rs similarity index 100% rename from crates/sqlbuild-analysis/src/maintained_refresh/constants.rs rename to crates/sqlbuild-analysis/src/dynamic_refresh/constants.rs diff --git a/crates/sqlbuild-analysis/src/dynamic_refresh/main/dynamic_settings_drift.rs b/crates/sqlbuild-analysis/src/dynamic_refresh/main/dynamic_settings_drift.rs new file mode 100644 index 0000000000..db0191481c --- /dev/null +++ b/crates/sqlbuild-analysis/src/dynamic_refresh/main/dynamic_settings_drift.rs @@ -0,0 +1,15 @@ +//! Whether a dynamic model's listed refresh settings differ from its declared ones. + +use crate::dynamic_refresh::_helpers::target_lag::parse_target_lag; +use crate::dynamic_refresh::models::{DynamicSettings, LiveDynamicTable}; + +/// True when the lag differs or cannot be read, or a declared warehouse differs. +pub fn dynamic_settings_drift(desired: &DynamicSettings, live: &LiveDynamicTable) -> bool { + if parse_target_lag(&live.target_lag) != Some(desired.lag) { + return true; + } + desired + .warehouse + .as_deref() + .is_some_and(|warehouse| !warehouse.eq_ignore_ascii_case(live.warehouse.trim())) +} diff --git a/crates/sqlbuild-analysis/src/maintained_refresh/main/mod.rs b/crates/sqlbuild-analysis/src/dynamic_refresh/main/mod.rs similarity index 62% rename from crates/sqlbuild-analysis/src/maintained_refresh/main/mod.rs rename to crates/sqlbuild-analysis/src/dynamic_refresh/main/mod.rs index a48b621304..d9a08bc028 100644 --- a/crates/sqlbuild-analysis/src/maintained_refresh/main/mod.rs +++ b/crates/sqlbuild-analysis/src/dynamic_refresh/main/mod.rs @@ -1,3 +1,3 @@ -pub mod maintained_settings_drift; +pub mod dynamic_settings_drift; pub mod reinitialised_readers; pub mod render_target_lag; diff --git a/crates/sqlbuild-analysis/src/maintained_refresh/main/reinitialised_readers.rs b/crates/sqlbuild-analysis/src/dynamic_refresh/main/reinitialised_readers.rs similarity index 60% rename from crates/sqlbuild-analysis/src/maintained_refresh/main/reinitialised_readers.rs rename to crates/sqlbuild-analysis/src/dynamic_refresh/main/reinitialised_readers.rs index 82f825c8f9..da5b210da9 100644 --- a/crates/sqlbuild-analysis/src/maintained_refresh/main/reinitialised_readers.rs +++ b/crates/sqlbuild-analysis/src/dynamic_refresh/main/reinitialised_readers.rs @@ -1,21 +1,18 @@ -//! Maintained models the plan keeps whose upstream it replaces, so they recompute in full. +//! Dynamic models the plan keeps whose upstream it replaces, so they recompute in full. use std::collections::BTreeSet; -use crate::maintained_refresh::models::MaintainedRead; +use crate::dynamic_refresh::models::DynamicRead; -/// Each kept maintained reader by name, with the first replaced upstream it reads by name. -pub fn reinitialised_readers(reads: &[MaintainedRead]) -> Vec<(String, String)> { +/// Each kept dynamic reader by name, with the first replaced upstream it reads by name. +pub fn reinitialised_readers(reads: &[DynamicRead]) -> Vec<(String, String)> { let replaced: BTreeSet<&str> = reads .iter() .filter(|read| read.replaced) .map(|read| read.name.as_str()) .collect(); let mut readers: Vec<(String, String)> = Vec::new(); - for read in reads - .iter() - .filter(|read| read.maintained && !read.replaced) - { + for read in reads.iter().filter(|read| read.dynamic && !read.replaced) { let mut upstreams: Vec<&str> = read .reads .iter() diff --git a/crates/sqlbuild-analysis/src/maintained_refresh/main/render_target_lag.rs b/crates/sqlbuild-analysis/src/dynamic_refresh/main/render_target_lag.rs similarity index 81% rename from crates/sqlbuild-analysis/src/maintained_refresh/main/render_target_lag.rs rename to crates/sqlbuild-analysis/src/dynamic_refresh/main/render_target_lag.rs index ca61aa4d47..c76758be3e 100644 --- a/crates/sqlbuild-analysis/src/maintained_refresh/main/render_target_lag.rs +++ b/crates/sqlbuild-analysis/src/dynamic_refresh/main/render_target_lag.rs @@ -1,7 +1,7 @@ //! The `TARGET_LAG` value of a refresh lag, in its largest exact unit. -use crate::maintained_refresh::constants::{DOWNSTREAM_TARGET_LAG, TARGET_LAG_UNITS}; -use crate::maintained_refresh::models::RefreshLag; +use crate::dynamic_refresh::constants::{DOWNSTREAM_TARGET_LAG, TARGET_LAG_UNITS}; +use crate::dynamic_refresh::models::RefreshLag; /// `DOWNSTREAM`, or a quoted lag such as `'2 hours'`. pub fn render_target_lag(lag: RefreshLag) -> String { diff --git a/crates/sqlbuild-analysis/src/dynamic_refresh/mod.rs b/crates/sqlbuild-analysis/src/dynamic_refresh/mod.rs new file mode 100644 index 0000000000..2c9f901902 --- /dev/null +++ b/crates/sqlbuild-analysis/src/dynamic_refresh/mod.rs @@ -0,0 +1,9 @@ +//! Dynamic refresh: dynamic table lag text, refresh setting drift, and readers a rebuild reinitialises. + +pub(crate) mod _helpers; +pub mod constants; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-analysis/src/maintained_refresh/models.rs b/crates/sqlbuild-analysis/src/dynamic_refresh/models.rs similarity index 51% rename from crates/sqlbuild-analysis/src/maintained_refresh/models.rs rename to crates/sqlbuild-analysis/src/dynamic_refresh/models.rs index 057d87a6e7..68b0d154a9 100644 --- a/crates/sqlbuild-analysis/src/maintained_refresh/models.rs +++ b/crates/sqlbuild-analysis/src/dynamic_refresh/models.rs @@ -1,34 +1,34 @@ -//! Declared and listed refresh settings of maintained models. +//! Declared and listed refresh settings of dynamic models. -/// How stale a maintained model may get: refreshed for its readers, or within a fixed lag. +/// How stale a dynamic model may get: refreshed for its readers, or within a fixed lag. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum RefreshLag { - /// Refreshed when the maintained models reading it need it. + /// Refreshed when the dynamic models reading it need it. Downstream, /// Refreshed within this many seconds. Seconds(u64), } -/// The refresh settings a maintained model declares. +/// The refresh settings a dynamic model declares. #[derive(Clone, Debug, PartialEq, Eq)] -pub struct MaintainedSettings { +pub struct DynamicSettings { pub lag: RefreshLag, /// The declared refresh warehouse; unset keeps whichever the object has. pub warehouse: Option, } -/// A maintained model's refresh settings as the warehouse lists them. +/// A dynamic model's refresh settings as the warehouse lists them. #[derive(Clone, Debug, PartialEq, Eq)] pub struct LiveDynamicTable { pub target_lag: String, pub warehouse: String, } -/// One planned model: whether it is maintained, whether this plan replaces it, and what it reads. +/// One planned model: whether it is dynamic, whether this plan replaces it, and what it reads. #[derive(Clone, Debug, PartialEq, Eq)] -pub struct MaintainedRead { +pub struct DynamicRead { pub name: String, - pub maintained: bool, + pub dynamic: bool, pub replaced: bool, pub reads: Vec, } diff --git a/crates/sqlbuild-analysis/src/maintained_refresh/tests.rs b/crates/sqlbuild-analysis/src/dynamic_refresh/tests.rs similarity index 56% rename from crates/sqlbuild-analysis/src/maintained_refresh/tests.rs rename to crates/sqlbuild-analysis/src/dynamic_refresh/tests.rs index 0b70bcf139..141538094c 100644 --- a/crates/sqlbuild-analysis/src/maintained_refresh/tests.rs +++ b/crates/sqlbuild-analysis/src/dynamic_refresh/tests.rs @@ -1,6 +1,6 @@ +#[path = "tests/test_dynamic_refresh.rs"] +mod dynamic_refresh; #[path = "tests/helpers.rs"] mod helpers; -#[path = "tests/test_maintained_refresh.rs"] -mod maintained_refresh; #[path = "tests/test_types.rs"] mod test_types; diff --git a/crates/sqlbuild-analysis/src/maintained_refresh/tests/helpers.rs b/crates/sqlbuild-analysis/src/dynamic_refresh/tests/helpers.rs similarity index 59% rename from crates/sqlbuild-analysis/src/maintained_refresh/tests/helpers.rs rename to crates/sqlbuild-analysis/src/dynamic_refresh/tests/helpers.rs index 9fa27f7945..3f10885306 100644 --- a/crates/sqlbuild-analysis/src/maintained_refresh/tests/helpers.rs +++ b/crates/sqlbuild-analysis/src/dynamic_refresh/tests/helpers.rs @@ -1,9 +1,7 @@ -use crate::maintained_refresh::models::{ - LiveDynamicTable, MaintainedRead, MaintainedSettings, RefreshLag, -}; +use crate::dynamic_refresh::models::{DynamicRead, DynamicSettings, LiveDynamicTable, RefreshLag}; -pub(super) fn settings(lag: RefreshLag, warehouse: Option<&str>) -> MaintainedSettings { - MaintainedSettings { +pub(super) fn settings(lag: RefreshLag, warehouse: Option<&str>) -> DynamicSettings { + DynamicSettings { lag, warehouse: warehouse.map(str::to_owned), } @@ -16,10 +14,10 @@ pub(super) fn live(target_lag: &str, warehouse: &str) -> LiveDynamicTable { } } -pub(super) fn read(name: &str, maintained: bool, replaced: bool, reads: &[&str]) -> MaintainedRead { - MaintainedRead { +pub(super) fn read(name: &str, dynamic: bool, replaced: bool, reads: &[&str]) -> DynamicRead { + DynamicRead { name: name.to_owned(), - maintained, + dynamic, replaced, reads: reads.iter().map(|read| (*read).to_owned()).collect(), } diff --git a/crates/sqlbuild-analysis/src/maintained_refresh/tests/test_maintained_refresh.rs b/crates/sqlbuild-analysis/src/dynamic_refresh/tests/test_dynamic_refresh.rs similarity index 88% rename from crates/sqlbuild-analysis/src/maintained_refresh/tests/test_maintained_refresh.rs rename to crates/sqlbuild-analysis/src/dynamic_refresh/tests/test_dynamic_refresh.rs index 145cdbf5bd..6938568355 100644 --- a/crates/sqlbuild-analysis/src/maintained_refresh/tests/test_maintained_refresh.rs +++ b/crates/sqlbuild-analysis/src/dynamic_refresh/tests/test_dynamic_refresh.rs @@ -1,10 +1,10 @@ -use crate::maintained_refresh::_helpers::target_lag::parse_target_lag; -use crate::maintained_refresh::main::maintained_settings_drift::maintained_settings_drift; -use crate::maintained_refresh::main::reinitialised_readers::reinitialised_readers; -use crate::maintained_refresh::main::render_target_lag::render_target_lag; -use crate::maintained_refresh::models::RefreshLag; -use crate::maintained_refresh::tests::helpers::{live, read, settings}; -use crate::maintained_refresh::tests::test_types::{ +use crate::dynamic_refresh::_helpers::target_lag::parse_target_lag; +use crate::dynamic_refresh::main::dynamic_settings_drift::dynamic_settings_drift; +use crate::dynamic_refresh::main::reinitialised_readers::reinitialised_readers; +use crate::dynamic_refresh::main::render_target_lag::render_target_lag; +use crate::dynamic_refresh::models::RefreshLag; +use crate::dynamic_refresh::tests::helpers::{live, read, settings}; +use crate::dynamic_refresh::tests::test_types::{ DriftTestCase, ReinitialiseTestCase, RenderLagTestCase, TargetLagTestCase, }; @@ -139,7 +139,7 @@ fn given_live_dynamic_table_when_comparing_settings_then_only_real_drift_alters( ]; for test_case in test_cases { assert_eq!( - maintained_settings_drift(&test_case.desired, &test_case.live), + dynamic_settings_drift(&test_case.desired, &test_case.live), test_case.expected_drift, "{}", test_case.description @@ -148,10 +148,10 @@ fn given_live_dynamic_table_when_comparing_settings_then_only_real_drift_alters( } #[test] -fn given_planned_rebuilds_when_checking_maintained_readers_then_reinitialised_ones_are_named() { +fn given_planned_rebuilds_when_checking_dynamic_readers_then_reinitialised_ones_are_named() { let test_cases = [ ReinitialiseTestCase { - description: "a maintained model reading a rebuilt table", + description: "a dynamic model reading a rebuilt table", reads: vec![ read("orders", false, true, &[]), read("daily_totals", true, false, &["orders"]), @@ -167,7 +167,7 @@ fn given_planned_rebuilds_when_checking_maintained_readers_then_reinitialised_on expected_readers: vec![], }, ReinitialiseTestCase { - description: "a recreated maintained model reinitialises its maintained readers", + description: "a recreated dynamic model reinitialises its dynamic readers", reads: vec![ read("clean_orders", true, true, &["orders"]), read("daily_totals", true, false, &["clean_orders"]), diff --git a/crates/sqlbuild-analysis/src/maintained_refresh/tests/test_types.rs b/crates/sqlbuild-analysis/src/dynamic_refresh/tests/test_types.rs similarity index 75% rename from crates/sqlbuild-analysis/src/maintained_refresh/tests/test_types.rs rename to crates/sqlbuild-analysis/src/dynamic_refresh/tests/test_types.rs index 70aedbf086..bf88ee50a8 100644 --- a/crates/sqlbuild-analysis/src/maintained_refresh/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/dynamic_refresh/tests/test_types.rs @@ -1,6 +1,4 @@ -use crate::maintained_refresh::models::{ - LiveDynamicTable, MaintainedRead, MaintainedSettings, RefreshLag, -}; +use crate::dynamic_refresh::models::{DynamicRead, DynamicSettings, LiveDynamicTable, RefreshLag}; pub(super) struct TargetLagTestCase { pub(super) description: &'static str, @@ -16,13 +14,13 @@ pub(super) struct RenderLagTestCase { pub(super) struct DriftTestCase { pub(super) description: &'static str, - pub(super) desired: MaintainedSettings, + pub(super) desired: DynamicSettings, pub(super) live: LiveDynamicTable, pub(super) expected_drift: bool, } pub(super) struct ReinitialiseTestCase { pub(super) description: &'static str, - pub(super) reads: Vec, + pub(super) reads: Vec, pub(super) expected_readers: Vec<(&'static str, &'static str)>, } diff --git a/crates/sqlbuild-analysis/src/lib.rs b/crates/sqlbuild-analysis/src/lib.rs index 303bf144da..0f0ca871f0 100644 --- a/crates/sqlbuild-analysis/src/lib.rs +++ b/crates/sqlbuild-analysis/src/lib.rs @@ -14,6 +14,7 @@ pub mod contracts; pub mod dag_artifact; pub mod definition_changes; pub mod diff_selection; +pub mod dynamic_refresh; pub mod explicit_references; pub mod fingerprint_storage; pub mod graph; @@ -22,7 +23,6 @@ pub mod janitor_planning; pub mod lineage; pub mod lineage_command; pub mod lineage_graph; -pub mod maintained_refresh; pub mod manifest; pub mod node_set_dependencies; pub mod nullability_rules; diff --git a/crates/sqlbuild-analysis/src/maintained_refresh/main/maintained_settings_drift.rs b/crates/sqlbuild-analysis/src/maintained_refresh/main/maintained_settings_drift.rs deleted file mode 100644 index 6c2f11b86c..0000000000 --- a/crates/sqlbuild-analysis/src/maintained_refresh/main/maintained_settings_drift.rs +++ /dev/null @@ -1,15 +0,0 @@ -//! Whether a maintained model's listed refresh settings differ from its declared ones. - -use crate::maintained_refresh::_helpers::target_lag::parse_target_lag; -use crate::maintained_refresh::models::{LiveDynamicTable, MaintainedSettings}; - -/// True when the lag differs or cannot be read, or a declared warehouse differs. -pub fn maintained_settings_drift(desired: &MaintainedSettings, live: &LiveDynamicTable) -> bool { - if parse_target_lag(&live.target_lag) != Some(desired.lag) { - return true; - } - desired - .warehouse - .as_deref() - .is_some_and(|warehouse| !warehouse.eq_ignore_ascii_case(live.warehouse.trim())) -} diff --git a/crates/sqlbuild-analysis/src/maintained_refresh/mod.rs b/crates/sqlbuild-analysis/src/maintained_refresh/mod.rs deleted file mode 100644 index 31a6ce51c4..0000000000 --- a/crates/sqlbuild-analysis/src/maintained_refresh/mod.rs +++ /dev/null @@ -1,9 +0,0 @@ -//! Maintained refresh: dynamic table lag text, refresh setting drift, and readers a rebuild reinitialises. - -pub(crate) mod _helpers; -pub mod constants; -pub mod main; -pub mod models; - -#[cfg(test)] -mod tests; diff --git a/crates/sqlbuild-analysis/src/planner_strategy/main/materialization_type.rs b/crates/sqlbuild-analysis/src/planner_strategy/main/materialization_type.rs index 40708d18a5..80ac505416 100644 --- a/crates/sqlbuild-analysis/src/planner_strategy/main/materialization_type.rs +++ b/crates/sqlbuild-analysis/src/planner_strategy/main/materialization_type.rs @@ -10,7 +10,7 @@ pub fn materialization_type(materialized: Option<&str>) -> MaterializationType { Some("incremental") => MaterializationType::Incremental, Some("snapshot") => MaterializationType::Snapshot, Some("seed") => MaterializationType::Seed, - Some("maintained") => MaterializationType::Maintained, + Some("dynamic") => MaterializationType::Dynamic, Some(_) => MaterializationType::Custom, } } diff --git a/crates/sqlbuild-analysis/src/planner_strategy/main/model_plan_action.rs b/crates/sqlbuild-analysis/src/planner_strategy/main/model_plan_action.rs index de5c990fc2..8612eca45d 100644 --- a/crates/sqlbuild-analysis/src/planner_strategy/main/model_plan_action.rs +++ b/crates/sqlbuild-analysis/src/planner_strategy/main/model_plan_action.rs @@ -41,12 +41,12 @@ pub fn model_plan_action( refreshed(PlanReason::NormalIncremental, false), )); } - MaterializationType::Maintained => { + MaterializationType::Dynamic => { let reason: PlanReason = refreshed(PlanReason::NoChange, true); if reason == PlanReason::NoChange { - return Ok((PlanAction::KeepMaintained, reason)); + return Ok((PlanAction::KeepDynamic, reason)); } - return Ok((PlanAction::CreateMaintained, reason)); + return Ok((PlanAction::CreateDynamic, reason)); } MaterializationType::Table | MaterializationType::Incremental diff --git a/crates/sqlbuild-analysis/src/planner_strategy/models.rs b/crates/sqlbuild-analysis/src/planner_strategy/models.rs index a760213e79..d72cc734cb 100644 --- a/crates/sqlbuild-analysis/src/planner_strategy/models.rs +++ b/crates/sqlbuild-analysis/src/planner_strategy/models.rs @@ -12,7 +12,7 @@ pub enum MaterializationType { Seed, Custom, /// A stored result the warehouse keeps fresh between builds. - Maintained, + Dynamic, } /// What the plan does with a model. @@ -26,12 +26,12 @@ pub enum PlanAction { Snapshot, Skip, Custom, - /// Create or replace the warehouse-maintained object. - CreateMaintained, + /// Create or replace the warehouse-dynamic object. + CreateDynamic, /// Change its refresh settings in place. - AlterMaintained, + AlterDynamic, /// Leave the object as it is; builds may still refresh it. - KeepMaintained, + KeepDynamic, } /// Why the plan does it. diff --git a/crates/sqlbuild-analysis/src/planner_strategy/tests/test_planner_strategy.rs b/crates/sqlbuild-analysis/src/planner_strategy/tests/test_planner_strategy.rs index 66db810b10..d980633888 100644 --- a/crates/sqlbuild-analysis/src/planner_strategy/tests/test_planner_strategy.rs +++ b/crates/sqlbuild-analysis/src/planner_strategy/tests/test_planner_strategy.rs @@ -103,52 +103,52 @@ fn given_model_change_when_resolving_action_then_matches_python_strategy() { ), }, ActionTestCase { - description: "maintained first run creates the dynamic table", - materialization: MaterializationType::Maintained, + description: "dynamic first run creates the dynamic table", + materialization: MaterializationType::Dynamic, strategy: None, change: change(PlanChangeKind::FirstRun), full_refresh: false, - expected_action: Ok((PlanAction::CreateMaintained, PlanReason::FirstRun)), + expected_action: Ok((PlanAction::CreateDynamic, PlanReason::FirstRun)), }, ActionTestCase { - description: "maintained query change recreates it", - materialization: MaterializationType::Maintained, + description: "dynamic query change recreates it", + materialization: MaterializationType::Dynamic, strategy: None, change: change(PlanChangeKind::QueryChanged), full_refresh: false, - expected_action: Ok((PlanAction::CreateMaintained, PlanReason::QueryChanged)), + expected_action: Ok((PlanAction::CreateDynamic, PlanReason::QueryChanged)), }, ActionTestCase { - description: "maintained config change such as refresh_mode recreates it", - materialization: MaterializationType::Maintained, + description: "dynamic config change such as refresh_mode recreates it", + materialization: MaterializationType::Dynamic, strategy: None, change: change(PlanChangeKind::ConfigChanged), full_refresh: false, - expected_action: Ok((PlanAction::CreateMaintained, PlanReason::ConfigChanged)), + expected_action: Ok((PlanAction::CreateDynamic, PlanReason::ConfigChanged)), }, ActionTestCase { - description: "maintained rename creates it under the new name", - materialization: MaterializationType::Maintained, + description: "dynamic rename creates it under the new name", + materialization: MaterializationType::Dynamic, strategy: None, change: change(PlanChangeKind::Renamed), full_refresh: false, - expected_action: Ok((PlanAction::CreateMaintained, PlanReason::Renamed)), + expected_action: Ok((PlanAction::CreateDynamic, PlanReason::Renamed)), }, ActionTestCase { - description: "maintained full refresh recreates an unchanged model", - materialization: MaterializationType::Maintained, + description: "dynamic full refresh recreates an unchanged model", + materialization: MaterializationType::Dynamic, strategy: None, change: change(PlanChangeKind::NoChange), full_refresh: true, - expected_action: Ok((PlanAction::CreateMaintained, PlanReason::FullRefresh)), + expected_action: Ok((PlanAction::CreateDynamic, PlanReason::FullRefresh)), }, ActionTestCase { - description: "unchanged maintained model is kept", - materialization: MaterializationType::Maintained, + description: "unchanged dynamic model is kept", + materialization: MaterializationType::Dynamic, strategy: None, change: change(PlanChangeKind::NoChange), full_refresh: false, - expected_action: Ok((PlanAction::KeepMaintained, PlanReason::NoChange)), + expected_action: Ok((PlanAction::KeepDynamic, PlanReason::NoChange)), }, ]; for test_case in test_cases { diff --git a/crates/sqlbuild-cli/src/argv/command_table.json b/crates/sqlbuild-cli/src/argv/command_table.json index 80148a54d7..6f19fa1d5d 100644 --- a/crates/sqlbuild-cli/src/argv/command_table.json +++ b/crates/sqlbuild-cli/src/argv/command_table.json @@ -2633,9 +2633,9 @@ { "kind": "store_true", "options": [ - "--ignore-maintained-grace" + "--ignore-dynamic-grace" ], - "dest": "ignore_maintained_grace", + "dest": "ignore_dynamic_grace", "nargs": 0, "const": true, "default": false, diff --git a/crates/sqlbuild-cli/src/argv/constants.rs b/crates/sqlbuild-cli/src/argv/constants.rs index 14b4b94ec7..c99a0c09a5 100644 --- a/crates/sqlbuild-cli/src/argv/constants.rs +++ b/crates/sqlbuild-cli/src/argv/constants.rs @@ -157,7 +157,7 @@ pub(crate) const CLI_NAMESPACE_DEFAULTS: [(&str, NamespaceDefault); 193] = [ "allow_snapshot_schema_change", NamespaceDefault::Bool(false), ), - ("ignore_maintained_grace", NamespaceDefault::Bool(false)), + ("ignore_dynamic_grace", NamespaceDefault::Bool(false)), ("concurrency", NamespaceDefault::None), ("verbose", NamespaceDefault::Bool(false)), ("debug", NamespaceDefault::Bool(false)), diff --git a/crates/sqlbuild-cli/src/dispatch/constants.rs b/crates/sqlbuild-cli/src/dispatch/constants.rs index 6b048c7dbb..953ee7474c 100644 --- a/crates/sqlbuild-cli/src/dispatch/constants.rs +++ b/crates/sqlbuild-cli/src/dispatch/constants.rs @@ -169,7 +169,7 @@ const BUILD_FIELDS: [Field; 41] = [ same("allow_retention_decrease"), same("allow_missing_migration_origin"), same("allow_snapshot_schema_change"), - same("ignore_maintained_grace"), + same("ignore_dynamic_grace"), same("concurrency"), SELECT, field("selector_files", Source::SelectorFiles), diff --git a/crates/sqlbuild-config/src/effective/constants.rs b/crates/sqlbuild-config/src/effective/constants.rs index 5508b1565a..18b48d3613 100644 --- a/crates/sqlbuild-config/src/effective/constants.rs +++ b/crates/sqlbuild-config/src/effective/constants.rs @@ -240,9 +240,9 @@ pub(crate) const TARGET_FIELDS: &[TargetField] = &[ }, TargetField { group: None, - name: "pause_maintained", + name: "pause_dynamic", rule: MergeRule::Overlay { - decided_by: &["pause_maintained"], + decided_by: &["pause_dynamic"], }, }, ]; diff --git a/crates/sqlbuild-config/src/project/_helpers/documents.rs b/crates/sqlbuild-config/src/project/_helpers/documents.rs index bcce7e424e..9b85455043 100644 --- a/crates/sqlbuild-config/src/project/_helpers/documents.rs +++ b/crates/sqlbuild-config/src/project/_helpers/documents.rs @@ -15,7 +15,7 @@ use crate::project::_helpers::help::{ use crate::project::_helpers::keys::Place; use crate::project::_helpers::sections::{ load_changes, load_constants, load_control, load_cost, load_cursors, load_dbt, load_diff, - load_git, load_janitor, load_local_dbt, load_maintained, load_microbatches, load_migrations, + load_dynamic, load_git, load_janitor, load_local_dbt, load_microbatches, load_migrations, load_policy_flag, load_pools, load_review, load_scenario, load_settings, load_snapshots, setting_overrides, }; @@ -31,7 +31,7 @@ use crate::project::constants::{ }; use crate::project::errors::{ConfigFailure, FailureClass, LoadFailure, MessagePiece}; use crate::project::models::{ - Built, ChangesConfig, ConfigFile, ConfigPaths, ControlConfig, MaintainedConfig, PoolLimits, + Built, ChangesConfig, ConfigFile, ConfigPaths, ControlConfig, DynamicConfig, PoolLimits, ReviewConfig, }; use crate::project::types::{Entries, Fields, Loaded}; @@ -299,7 +299,7 @@ fn project_fields(payload: &Entries) -> Loaded { } load_git(get(payload, "git"), PROJECT_FILE)?; load_review(get(payload, "review"), PROJECT_FILE)?; - load_maintained(get(payload, "maintained"), PROJECT_FILE)?; + load_dynamic(get(payload, "dynamic"), PROJECT_FILE)?; load_control(get(payload, "control"), PROJECT_FILE)?; load_changes(get(payload, "changes"), PROJECT_FILE)?; pool_limits(payload)?; @@ -542,9 +542,9 @@ pub(crate) fn changes_config(paths: &ConfigPaths) -> Result Result { - project_section(paths, "maintained", load_maintained) +/// `[dynamic]` of `sqlbuild_project.toml`. +pub(crate) fn dynamic_config(paths: &ConfigPaths) -> Result { + project_section(paths, "dynamic", load_dynamic) } /// `[review]` of `sqlbuild_project.toml`. diff --git a/crates/sqlbuild-config/src/project/_helpers/sections.rs b/crates/sqlbuild-config/src/project/_helpers/sections.rs index be75473f5f..c8c1f739b2 100644 --- a/crates/sqlbuild-config/src/project/_helpers/sections.rs +++ b/crates/sqlbuild-config/src/project/_helpers/sections.rs @@ -22,13 +22,14 @@ use crate::project::constants::{ AUDIT_SEVERITIES, CHECK_DECLARED_COLUMNS_KEY, COLLECTION_RENDERINGS, COLUMN_CONTRACT_MODES, CONCURRENCY_KEY, DBT_DEFER_CLONE_KEY, DBT_KEYS, DBT_REMOVED_KEYS, DBT_REUSE_KEYS, DEFAULT_HEARTBEAT_GRACE_SECONDS, DEFAULT_IN_USE_WINDOW, DEFAULT_QUERY_ARTIFACT_TTL, - FUTURE_CURSOR_ACTIONS, HEARTBEAT_INTERVAL_SECONDS, LEGACY_COLUMN_CONTRACT_MODE_KEY, - LEGACY_CONCURRENCY_KEY, LEGACY_SQL_VALIDATION_KEY, LOCAL_DBT_KEYS, MAINTAINED_KEYS, - MAX_BATCHES_KEY, MICROBATCH_LIMIT_ACTIONS, OLD_NAME_VIEWS_KEY, QUERY_CHANGE_TRACKING_KEY, - REQUIRE_SQL_ANALYSIS_KEY, REVIEW_DOWNSTREAM_POLICIES, REVIEW_KEYS, RUN_NAMESPACE_KEY, - RUN_NAMESPACE_MAX_LENGTH, SCENARIO_KEYS, SETTINGS_KEYS, SNAPSHOT_FULL_REFRESH_POLICIES, - SNAPSHOT_KEYS, SNAPSHOT_LIMIT_KEYS, SNAPSHOT_SCHEMA_CHANGE_POLICIES, SQL_ANALYSIS_KEY, - UNACCOUNTED_PARTITION_POLICIES, USD_PER_CREDIT_KEY, ZERO_FIXED_DURATION, ZERO_FLOAT, + DYNAMIC_KEYS, FUTURE_CURSOR_ACTIONS, HEARTBEAT_INTERVAL_SECONDS, + LEGACY_COLUMN_CONTRACT_MODE_KEY, LEGACY_CONCURRENCY_KEY, LEGACY_SQL_VALIDATION_KEY, + LOCAL_DBT_KEYS, MAX_BATCHES_KEY, MICROBATCH_LIMIT_ACTIONS, OLD_NAME_VIEWS_KEY, + QUERY_CHANGE_TRACKING_KEY, REQUIRE_SQL_ANALYSIS_KEY, REVIEW_DOWNSTREAM_POLICIES, REVIEW_KEYS, + RUN_NAMESPACE_KEY, RUN_NAMESPACE_MAX_LENGTH, SCENARIO_KEYS, SETTINGS_KEYS, + SNAPSHOT_FULL_REFRESH_POLICIES, SNAPSHOT_KEYS, SNAPSHOT_LIMIT_KEYS, + SNAPSHOT_SCHEMA_CHANGE_POLICIES, SQL_ANALYSIS_KEY, UNACCOUNTED_PARTITION_POLICIES, + USD_PER_CREDIT_KEY, ZERO_FIXED_DURATION, ZERO_FLOAT, }; use crate::project::constants::{ CHANGE_COUNTS, CHANGE_REACHES, CHANGES_KEYS, CONTROL_KEYS, MAX_POOL_LIMIT, @@ -36,7 +37,7 @@ use crate::project::constants::{ }; use crate::project::errors::ConfigFailure; use crate::project::models::{ - Built, ChangesConfig, ConfigFile, ControlConfig, MaintainedConfig, ReviewConfig, + Built, ChangesConfig, ConfigFile, ControlConfig, DynamicConfig, ReviewConfig, }; use crate::project::types::{Entries, Fields, Loaded}; use sqlbuild_core::constants::ZERO_DAY_DURATION; @@ -628,13 +629,13 @@ pub(crate) fn sync_interval(text: &str) -> Result { Ok(seconds) } -/// `[maintained]`: how long a silent pausing build keeps its maintained models paused. -pub(crate) fn load_maintained( +/// `[dynamic]`: how long a silent pausing build keeps its dynamic models paused. +pub(crate) fn load_dynamic( payload: Option<&ConfigValue>, file: ConfigFile, -) -> Loaded { - let mapping = coerce_mapping(payload, place(file, "maintained"))?; - validate_allowed_keys(mapping, &MAINTAINED_KEYS, place(file, "maintained"))?; +) -> Loaded { + let mapping = coerce_mapping(payload, place(file, "dynamic"))?; + validate_allowed_keys(mapping, &DYNAMIC_KEYS, place(file, "dynamic"))?; let heartbeat_grace_seconds: u64 = match get(mapping, "heartbeat_grace") { None => DEFAULT_HEARTBEAT_GRACE_SECONDS, Some(ConfigValue::String(text)) => match fixed_duration_seconds(text, false) { @@ -646,7 +647,7 @@ pub(crate) fn load_maintained( if heartbeat_grace_seconds <= HEARTBEAT_INTERVAL_SECONDS { return Err(ConfigFailure::at( file, - "maintained.heartbeat_grace must be longer than 1m: a build writes a heartbeat every minute, so a shorter grace expires builds that are still running", + "dynamic.heartbeat_grace must be longer than 1m: a build writes a heartbeat every minute, so a shorter grace expires builds that are still running", )); } let allow_ignore_grace: bool = match get(mapping, "allow_ignore_grace") { @@ -655,11 +656,11 @@ pub(crate) fn load_maintained( Some(_) => { return Err(ConfigFailure::at( file, - "maintained.allow_ignore_grace must be true or false", + "dynamic.allow_ignore_grace must be true or false", )); } }; - Ok(MaintainedConfig { + Ok(DynamicConfig { heartbeat_grace_seconds, allow_ignore_grace, }) @@ -668,7 +669,7 @@ pub(crate) fn load_maintained( fn grace_failure(file: ConfigFile) -> ConfigFailure { ConfigFailure::at( file, - "maintained.heartbeat_grace must be a fixed duration such as 30m or 2h", + "dynamic.heartbeat_grace must be a fixed duration such as 30m or 2h", ) } diff --git a/crates/sqlbuild-config/src/project/_helpers/targets.rs b/crates/sqlbuild-config/src/project/_helpers/targets.rs index b8bd8df06b..73b6fd74f9 100644 --- a/crates/sqlbuild-config/src/project/_helpers/targets.rs +++ b/crates/sqlbuild-config/src/project/_helpers/targets.rs @@ -16,7 +16,7 @@ use crate::project::_helpers::sections::{fixed_duration_seconds, item_timeout}; use crate::project::_helpers::values::{int_below, is_int}; use crate::project::constants::{ CLONE_KEYS, CONCURRENCY_KEY, CONFIRMATION_POLICIES, EXECUTION_LIMIT_KEYS, - IGNORE_AUDIT_FAILURES_KEY, MAX_FULL_ROWS_KEY, PAUSE_MAINTAINED_KEY, PLAN_FROM_KEY, + IGNORE_AUDIT_FAILURES_KEY, MAX_FULL_ROWS_KEY, PAUSE_DYNAMIC_KEY, PLAN_FROM_KEY, PLAN_FROM_SOURCES, POOLS_KEY, RETENTION_MATERIALIZATIONS, TARGET_KEYS, UNLIMITED_DIFF_FULL_ROWS, WAREHOUSE_GROUPS, }; @@ -435,8 +435,8 @@ fn local_target(payload: &ConfigValue, target: TargetPlace<'_>) -> Loaded Result { + crate::project::_helpers::documents::dynamic_config(paths) +} diff --git a/crates/sqlbuild-config/src/project/main/load_maintained_config.rs b/crates/sqlbuild-config/src/project/main/load_maintained_config.rs deleted file mode 100644 index b5740f203d..0000000000 --- a/crates/sqlbuild-config/src/project/main/load_maintained_config.rs +++ /dev/null @@ -1,9 +0,0 @@ -//! Read `[maintained]` from `sqlbuild_project.toml`. - -use crate::project::errors::LoadFailure; -use crate::project::models::{ConfigPaths, MaintainedConfig}; - -/// The maintained-model pause settings, with their defaults when `[maintained]` sets none. -pub fn load_maintained_config(paths: &ConfigPaths) -> Result { - crate::project::_helpers::documents::maintained_config(paths) -} diff --git a/crates/sqlbuild-config/src/project/main/mod.rs b/crates/sqlbuild-config/src/project/main/mod.rs index 6d54995e7c..abf7b235dc 100644 --- a/crates/sqlbuild-config/src/project/main/mod.rs +++ b/crates/sqlbuild-config/src/project/main/mod.rs @@ -2,9 +2,9 @@ pub mod deprecated_setting_warnings; pub mod explicit_references_help; pub mod load_changes_config; pub mod load_control_config; +pub mod load_dynamic_config; pub mod load_git_base_ref; pub mod load_local_config; -pub mod load_maintained_config; pub mod load_pool_limits; pub mod load_project_config; pub mod load_project_setting_keys; diff --git a/crates/sqlbuild-config/src/project/models.rs b/crates/sqlbuild-config/src/project/models.rs index 5fab29b4c0..6a6f921209 100644 --- a/crates/sqlbuild-config/src/project/models.rs +++ b/crates/sqlbuild-config/src/project/models.rs @@ -91,12 +91,12 @@ pub struct ChangesConfig { pub reach: Option, } -/// `[maintained]` of `sqlbuild_project.toml`: when a build may take over paused maintained models. +/// `[dynamic]` of `sqlbuild_project.toml`: when a build may take over paused dynamic models. #[derive(Clone, Debug, PartialEq, Eq)] -pub struct MaintainedConfig { +pub struct DynamicConfig { /// Seconds a pausing build's heartbeat may be silent before another build resumes its models. pub heartbeat_grace_seconds: u64, - /// Whether `sqb build --ignore-maintained-grace` may take over within the grace. + /// Whether `sqb build --ignore-dynamic-grace` may take over within the grace. pub allow_ignore_grace: bool, } diff --git a/crates/sqlbuild-config/src/project/tests/test_load_config.rs b/crates/sqlbuild-config/src/project/tests/test_load_config.rs index aa14ab21bd..84dbfb4f22 100644 --- a/crates/sqlbuild-config/src/project/tests/test_load_config.rs +++ b/crates/sqlbuild-config/src/project/tests/test_load_config.rs @@ -1,16 +1,16 @@ use crate::project::errors::ConfigFailure; use crate::project::main::load_changes_config::load_changes_config; use crate::project::main::load_control_config::load_control_config; +use crate::project::main::load_dynamic_config::load_dynamic_config; use crate::project::main::load_git_base_ref::load_git_base_ref; -use crate::project::main::load_maintained_config::load_maintained_config; use crate::project::main::load_pool_limits::load_pool_limits; use crate::project::main::load_review_config::load_review_config; use crate::project::models::ConfigFile; -use crate::project::models::{ChangesConfig, ControlConfig, MaintainedConfig, ReviewConfig}; +use crate::project::models::{ChangesConfig, ControlConfig, DynamicConfig, ReviewConfig}; use crate::project::tests::helpers::{failure_at, first_failure, project_directory}; use crate::project::tests::test_types::{ - ChangesConfigTestCase, ControlConfigTestCase, GitBaseRefTestCase, - MaintainedConfigErrorTestCase, MaintainedConfigTestCase, PoolsErrorTestCase, PoolsTestCase, + ChangesConfigTestCase, ControlConfigTestCase, DynamicConfigErrorTestCase, + DynamicConfigTestCase, GitBaseRefTestCase, PoolsErrorTestCase, PoolsTestCase, ProjectFailureTestCase, ReviewConfigErrorTestCase, ReviewConfigTestCase, }; @@ -65,35 +65,35 @@ fn given_project_files_when_loading_then_rejections_match_python_loader() { )), }, ProjectFailureTestCase { - description: "a target may keep its maintained models paused between builds", - project: "name = 'orders'\nadapter = 'snowflake'\n[targets.dev]\npause_maintained = true\n", - local: "[targets.dev]\npause_maintained = false\n", + description: "a target may keep its dynamic models paused between builds", + project: "name = 'orders'\nadapter = 'snowflake'\n[targets.dev]\npause_dynamic = true\n", + local: "[targets.dev]\npause_dynamic = false\n", expected_failure: None, }, ProjectFailureTestCase { - description: "pause_maintained is a boolean", - project: "name = 'orders'\nadapter = 'snowflake'\n[targets.dev]\npause_maintained = 'yes'\n", + description: "pause_dynamic is a boolean", + project: "name = 'orders'\nadapter = 'snowflake'\n[targets.dev]\npause_dynamic = 'yes'\n", local: "", expected_failure: Some(ConfigFailure::plain( - "Expected 'pause_maintained' to be a boolean when provided", + "Expected 'pause_dynamic' to be a boolean when provided", )), }, ProjectFailureTestCase { - description: "an invalid maintained section fails the project load", - project: "name = 'orders'\nadapter = 'snowflake'\n[maintained]\nheartbeat_grace = 'soon'\n", + description: "an invalid dynamic section fails the project load", + project: "name = 'orders'\nadapter = 'snowflake'\n[dynamic]\nheartbeat_grace = 'soon'\n", local: "", expected_failure: Some(ConfigFailure::at( ConfigFile::Project, - "maintained.heartbeat_grace must be a fixed duration such as 30m or 2h", + "dynamic.heartbeat_grace must be a fixed duration such as 30m or 2h", )), }, ProjectFailureTestCase { description: "a grace no longer than the heartbeat interval would expire every build", - project: "name = 'orders'\nadapter = 'snowflake'\n[maintained]\nheartbeat_grace = '1m'\n", + project: "name = 'orders'\nadapter = 'snowflake'\n[dynamic]\nheartbeat_grace = '1m'\n", local: "", expected_failure: Some(ConfigFailure::at( ConfigFile::Project, - "maintained.heartbeat_grace must be longer than 1m: a build writes a heartbeat every minute, so a shorter grace expires builds that are still running", + "dynamic.heartbeat_grace must be longer than 1m: a build writes a heartbeat every minute, so a shorter grace expires builds that are still running", )), }, ProjectFailureTestCase { @@ -633,30 +633,30 @@ fn given_invalid_pools_when_loading_then_project_loading_rejects_them() { } #[test] -fn given_maintained_section_when_reading_then_returns_its_values_or_defaults() { +fn given_dynamic_section_when_reading_then_returns_its_values_or_defaults() { let test_cases = [ - MaintainedConfigTestCase { + DynamicConfigTestCase { description: "a longer grace that may be ignored", - project: "name = 'orders'\nadapter = 'snowflake'\n[maintained]\n\ + project: "name = 'orders'\nadapter = 'snowflake'\n[dynamic]\n\ heartbeat_grace = '45m'\nallow_ignore_grace = true\n", - expected_config: MaintainedConfig { + expected_config: DynamicConfig { heartbeat_grace_seconds: 2_700, allow_ignore_grace: true, }, }, - MaintainedConfigTestCase { + DynamicConfigTestCase { description: "any grace longer than the heartbeat interval", - project: "name = 'orders'\nadapter = 'snowflake'\n[maintained]\n\ + project: "name = 'orders'\nadapter = 'snowflake'\n[dynamic]\n\ heartbeat_grace = '61s'\n", - expected_config: MaintainedConfig { + expected_config: DynamicConfig { heartbeat_grace_seconds: 61, allow_ignore_grace: false, }, }, - MaintainedConfigTestCase { - description: "no maintained section", + DynamicConfigTestCase { + description: "no dynamic section", project: PROJECT, - expected_config: MaintainedConfig { + expected_config: DynamicConfig { heartbeat_grace_seconds: 1_800, allow_ignore_grace: false, }, @@ -665,7 +665,7 @@ fn given_maintained_section_when_reading_then_returns_its_values_or_defaults() { for test_case in test_cases { let (_directory, paths) = project_directory(test_case.project, ""); - let config: MaintainedConfig = load_maintained_config(&paths).expect(test_case.description); + let config: DynamicConfig = load_dynamic_config(&paths).expect(test_case.description); assert_eq!( config, test_case.expected_config, @@ -676,31 +676,31 @@ fn given_maintained_section_when_reading_then_returns_its_values_or_defaults() { } #[test] -fn given_invalid_maintained_section_when_reading_then_it_is_rejected() { +fn given_invalid_dynamic_section_when_reading_then_it_is_rejected() { let test_cases = [ - MaintainedConfigErrorTestCase { + DynamicConfigErrorTestCase { description: "a grace that is not a duration", - project: "name = 'orders'\nadapter = 'snowflake'\n[maintained]\n\ + project: "name = 'orders'\nadapter = 'snowflake'\n[dynamic]\n\ heartbeat_grace = 'soon'\n", - expected_fragment: "maintained.heartbeat_grace must be a fixed duration such as 30m or 2h", + expected_fragment: "dynamic.heartbeat_grace must be a fixed duration such as 30m or 2h", }, - MaintainedConfigErrorTestCase { + DynamicConfigErrorTestCase { description: "a grace in months", - project: "name = 'orders'\nadapter = 'snowflake'\n[maintained]\n\ + project: "name = 'orders'\nadapter = 'snowflake'\n[dynamic]\n\ heartbeat_grace = '1mo'\n", - expected_fragment: "maintained.heartbeat_grace must be a fixed duration such as 30m or 2h", + expected_fragment: "dynamic.heartbeat_grace must be a fixed duration such as 30m or 2h", }, - MaintainedConfigErrorTestCase { + DynamicConfigErrorTestCase { description: "allow_ignore_grace written as text", - project: "name = 'orders'\nadapter = 'snowflake'\n[maintained]\n\ + project: "name = 'orders'\nadapter = 'snowflake'\n[dynamic]\n\ allow_ignore_grace = 'yes'\n", - expected_fragment: "maintained.allow_ignore_grace must be true or false", + expected_fragment: "dynamic.allow_ignore_grace must be true or false", }, - MaintainedConfigErrorTestCase { - description: "a misspelt maintained key", - project: "name = 'orders'\nadapter = 'snowflake'\n[maintained]\n\ + DynamicConfigErrorTestCase { + description: "a misspelt dynamic key", + project: "name = 'orders'\nadapter = 'snowflake'\n[dynamic]\n\ heartbeat_grase = '30m'\n", - expected_fragment: "maintained contains unknown key(s): heartbeat_grase", + expected_fragment: "dynamic contains unknown key(s): heartbeat_grase", }, ]; for test_case in test_cases { @@ -708,7 +708,7 @@ fn given_invalid_maintained_section_when_reading_then_it_is_rejected() { let failure: String = format!( "{:?}", - load_maintained_config(&paths).expect_err(test_case.description) + load_dynamic_config(&paths).expect_err(test_case.description) ); assert!( diff --git a/crates/sqlbuild-config/src/project/tests/test_types.rs b/crates/sqlbuild-config/src/project/tests/test_types.rs index d0a08468db..2189e764b3 100644 --- a/crates/sqlbuild-config/src/project/tests/test_types.rs +++ b/crates/sqlbuild-config/src/project/tests/test_types.rs @@ -1,5 +1,5 @@ use crate::project::errors::ConfigFailure; -use crate::project::models::{ChangesConfig, ControlConfig, MaintainedConfig, ReviewConfig}; +use crate::project::models::{ChangesConfig, ControlConfig, DynamicConfig, ReviewConfig}; pub(super) struct ProjectFailureTestCase { pub(super) description: &'static str, @@ -59,13 +59,13 @@ pub(crate) struct PoolsErrorTestCase { pub(crate) expected_fragment: &'static str, } -pub(crate) struct MaintainedConfigTestCase { +pub(crate) struct DynamicConfigTestCase { pub(crate) description: &'static str, pub(crate) project: &'static str, - pub(crate) expected_config: MaintainedConfig, + pub(crate) expected_config: DynamicConfig, } -pub(crate) struct MaintainedConfigErrorTestCase { +pub(crate) struct DynamicConfigErrorTestCase { pub(crate) description: &'static str, pub(crate) project: &'static str, pub(crate) expected_fragment: &'static str, diff --git a/crates/sqlbuild-core/src/cli_text/_helpers/plan_sections.rs b/crates/sqlbuild-core/src/cli_text/_helpers/plan_sections.rs index 03b6cb37ea..11cccfe5ed 100644 --- a/crates/sqlbuild-core/src/cli_text/_helpers/plan_sections.rs +++ b/crates/sqlbuild-core/src/cli_text/_helpers/plan_sections.rs @@ -6,8 +6,8 @@ use crate::cli_text::_helpers::bounded_sections::{ }; use crate::cli_text::_helpers::layout::tree_connector; use crate::cli_text::constants::{ - ALTER_MAINTAINED_ACTION, BOUNDED_BACKFILL, CHANGED_IDENTITY, COLUMN_ADDED, COLUMN_REMOVED, - COLUMN_TYPE_CHANGED, CONFIG_CHANGED_REASON, CREATE_MAINTAINED_ACTION, CREATE_VIEW_ACTION, + ALTER_DYNAMIC_ACTION, BOUNDED_BACKFILL, CHANGED_IDENTITY, COLUMN_ADDED, COLUMN_REMOVED, + COLUMN_TYPE_CHANGED, CONFIG_CHANGED_REASON, CREATE_DYNAMIC_ACTION, CREATE_VIEW_ACTION, CUSTOM_ACTION, DIFF_HEADER_MARKER, EFFECTIVE_BATCH_SIZE_TOKEN, ERROR_SEVERITY_VALUE, FIRST_RUN_REASON, FORWARD_BACKFILL, FULL_BACKFILL, FULL_REFRESH_REASON, INFO_SEVERITY_VALUE, NO_CHANGE_REASON, NORMAL_INCREMENTAL_REASON, QUERY_CHANGED_REASON, REASON_GROUPS, @@ -430,14 +430,14 @@ fn action_text(model: &ModelPlanText) -> String { if model.action == CREATE_VIEW_ACTION { return "recreate view".to_owned(); } - if model.action == CREATE_MAINTAINED_ACTION { + if model.action == CREATE_DYNAMIC_ACTION { return if model.reason == FIRST_RUN_REASON { "create".to_owned() } else { "recreate, full recompute".to_owned() }; } - if model.action == ALTER_MAINTAINED_ACTION { + if model.action == ALTER_DYNAMIC_ACTION { return "change refresh settings in place".to_owned(); } if model.action == CUSTOM_ACTION { @@ -465,7 +465,7 @@ fn action_text(model: &ModelPlanText) -> String { fn policy_lines(model: &ModelPlanText) -> Vec { if model.backfill_action == FORWARD_BACKFILL - || model.action == CREATE_MAINTAINED_ACTION + || model.action == CREATE_DYNAMIC_ACTION || !(model.reason == QUERY_CHANGED_REASON || model.reason == SCHEMA_CHANGED_REASON) { return Vec::new(); diff --git a/crates/sqlbuild-core/src/cli_text/constants.rs b/crates/sqlbuild-core/src/cli_text/constants.rs index f00f5abc48..bd63958abd 100644 --- a/crates/sqlbuild-core/src/cli_text/constants.rs +++ b/crates/sqlbuild-core/src/cli_text/constants.rs @@ -63,8 +63,8 @@ pub(crate) const REASON_GROUPS: [(&str, &str); 7] = [ pub(crate) const SKIP_ACTION: &str = "skip"; pub(crate) const CREATE_VIEW_ACTION: &str = "create_view"; pub(crate) const CUSTOM_ACTION: &str = "custom"; -pub(crate) const CREATE_MAINTAINED_ACTION: &str = "create_maintained"; -pub(crate) const ALTER_MAINTAINED_ACTION: &str = "alter_maintained"; +pub(crate) const CREATE_DYNAMIC_ACTION: &str = "create_dynamic"; +pub(crate) const ALTER_DYNAMIC_ACTION: &str = "alter_dynamic"; pub(crate) const FULL_REFRESH_REASON: &str = "full_refresh"; pub(crate) const FIRST_RUN_REASON: &str = "first_run"; pub(crate) const QUERY_CHANGED_REASON: &str = "query_changed"; diff --git a/crates/sqlbuild-core/src/cli_text/models.rs b/crates/sqlbuild-core/src/cli_text/models.rs index 0274627d1e..189916b4e5 100644 --- a/crates/sqlbuild-core/src/cli_text/models.rs +++ b/crates/sqlbuild-core/src/cli_text/models.rs @@ -322,9 +322,9 @@ pub struct ModelPlanText { pub config_diff: Option>, /// The rendered query diff, when the query changed and the previous query is known. pub query_diff: Option>, - /// The rebuilt upstream that makes this kept maintained model recompute in full. + /// The rebuilt upstream that makes this kept dynamic model recompute in full. pub reinitialised_by: Option, - /// Whether the warehouse lists this kept maintained model as suspended. + /// Whether the warehouse lists this kept dynamic model as suspended. pub paused: bool, } diff --git a/crates/sqlbuild-core/src/cli_text/tests/test_cli_text.rs b/crates/sqlbuild-core/src/cli_text/tests/test_cli_text.rs index 51f8d9a1e0..511bd83618 100644 --- a/crates/sqlbuild-core/src/cli_text/tests/test_cli_text.rs +++ b/crates/sqlbuild-core/src/cli_text/tests/test_cli_text.rs @@ -227,16 +227,16 @@ fn given_plan_report_when_rendering_then_groups_models_by_reason() { expected_completion: "\u{2713} Plan complete 1 selected", }, PlanTextTestCase { - description: "maintained models recompute on a query change and alter refresh settings in place", + description: "dynamic models recompute on a query change and alter refresh settings in place", report: plan_report(vec![ ModelPlanText { - action: "create_maintained".to_owned(), - materialization_label: "maintained".to_owned(), + action: "create_dynamic".to_owned(), + materialization_label: "dynamic".to_owned(), ..table_model("daily_totals", "query_changed") }, ModelPlanText { - action: "alter_maintained".to_owned(), - materialization_label: "maintained".to_owned(), + action: "alter_dynamic".to_owned(), + materialization_label: "dynamic".to_owned(), ..table_model("clean_orders", "config_changed") }, ]), @@ -244,28 +244,28 @@ fn given_plan_report_when_rendering_then_groups_models_by_reason() { expected_completion: "\u{2713} Plan complete 2 selected", }, PlanTextTestCase { - description: "a kept maintained model whose upstream is rebuilt says it recomputes in full", + description: "a kept dynamic model whose upstream is rebuilt says it recomputes in full", report: plan_report(vec![ table_model("orders", "query_changed"), ModelPlanText { - action: "keep_maintained".to_owned(), - materialization_label: "maintained".to_owned(), + action: "keep_dynamic".to_owned(), + materialization_label: "dynamic".to_owned(), reinitialised_by: Some("orders".to_owned()), ..table_model("daily_totals", "no_change") }, ]), - expected_text: "Plan ready 2 selected\n\nQuery changed (1)\n\u{2514}\u{2500}\u{2500} orders full rebuild\n \u{2514}\u{2500}\u{2500} policy replay_on_change=full\n\nModels (1)\n\u{2514}\u{2500}\u{2500} daily_totals maintained\n \u{2514}\u{2500}\u{2500} recomputes in full orders is rebuilt", + expected_text: "Plan ready 2 selected\n\nQuery changed (1)\n\u{2514}\u{2500}\u{2500} orders full rebuild\n \u{2514}\u{2500}\u{2500} policy replay_on_change=full\n\nModels (1)\n\u{2514}\u{2500}\u{2500} daily_totals dynamic\n \u{2514}\u{2500}\u{2500} recomputes in full orders is rebuilt", expected_completion: "\u{2713} Plan complete 2 selected", }, PlanTextTestCase { - description: "a kept maintained model that is suspended is reported as paused", + description: "a kept dynamic model that is suspended is reported as paused", report: plan_report(vec![ModelPlanText { - action: "keep_maintained".to_owned(), - materialization_label: "maintained".to_owned(), + action: "keep_dynamic".to_owned(), + materialization_label: "dynamic".to_owned(), paused: true, ..table_model("daily_totals", "no_change") }]), - expected_text: "Plan ready 1 selected\n\nModels (1)\n\u{2514}\u{2500}\u{2500} daily_totals maintained\n \u{2514}\u{2500}\u{2500} paused scheduled refreshes are suspended", + expected_text: "Plan ready 1 selected\n\nModels (1)\n\u{2514}\u{2500}\u{2500} daily_totals dynamic\n \u{2514}\u{2500}\u{2500} paused scheduled refreshes are suspended", expected_completion: "\u{2713} Plan complete 1 selected", }, PlanTextTestCase { diff --git a/crates/sqlbuild-core/src/constants.rs b/crates/sqlbuild-core/src/constants.rs index 82fb646f95..a6629f8912 100644 --- a/crates/sqlbuild-core/src/constants.rs +++ b/crates/sqlbuild-core/src/constants.rs @@ -44,8 +44,8 @@ pub const OBSERVATIONS_TABLE_NAME: &str = "_sqlbuild_observations"; pub const RESOURCE_COSTS_TABLE_NAME: &str = "_sqlbuild_resource_costs"; /// State table of each build's start and end, which shows runs the control plane did not start. pub const RUNS_TABLE_NAME: &str = "_sqlbuild_runs"; -/// State table of the maintained models a build paused, and its heartbeats while it holds them. -pub const MAINTAINED_PAUSES_TABLE_NAME: &str = "_sqlbuild_maintained_pauses"; +/// State table of the dynamic models a build paused, and its heartbeats while it holds them. +pub const DYNAMIC_PAUSES_TABLE_NAME: &str = "_sqlbuild_dynamic_pauses"; /// Every state table SQLBuild writes in a target's state schema. pub const STATE_TABLE_NAMES: [&str; 14] = [ FINGERPRINT_TABLE_NAME, @@ -61,7 +61,7 @@ pub const STATE_TABLE_NAMES: [&str; 14] = [ OBSERVATIONS_TABLE_NAME, RESOURCE_COSTS_TABLE_NAME, RUNS_TABLE_NAME, - MAINTAINED_PAUSES_TABLE_NAME, + DYNAMIC_PAUSES_TABLE_NAME, ]; /// A zero-length cursor duration. pub const ZERO_DAY_DURATION: &str = "0d"; diff --git a/crates/sqlbuild-core/src/fingerprint_identity/constants.rs b/crates/sqlbuild-core/src/fingerprint_identity/constants.rs index 3da98bc0a3..ac688e2b22 100644 --- a/crates/sqlbuild-core/src/fingerprint_identity/constants.rs +++ b/crates/sqlbuild-core/src/fingerprint_identity/constants.rs @@ -6,9 +6,9 @@ pub const DECLARED_COLUMNS_METADATA_KEY: &str = "declared_columns_hash"; pub const LOCAL_FUNCTION_HASHES_METADATA_KEY: &str = "local_function_hashes"; /// Metadata key of the model config entries in version identity. pub const CONFIG_METADATA_KEY: &str = "config"; -/// Config key of a maintained model's refresh mode. +/// Config key of a dynamic model's refresh mode. pub const REFRESH_MODE_CONFIG_KEY: &str = "refresh_mode"; -/// The refresh mode of a maintained model that declares none; writing it out changes nothing. +/// The refresh mode of a dynamic model that declares none; writing it out changes nothing. pub const DEFAULT_REFRESH_MODE: &str = "incremental"; /// The fingerprint dialect when a project names none. pub(crate) const GENERIC_FINGERPRINT_DIALECT: &str = "generic"; diff --git a/crates/sqlbuild-core/src/fingerprint_identity/tests/test_fingerprint_identity.rs b/crates/sqlbuild-core/src/fingerprint_identity/tests/test_fingerprint_identity.rs index 1f89bfcfcd..dded25744f 100644 --- a/crates/sqlbuild-core/src/fingerprint_identity/tests/test_fingerprint_identity.rs +++ b/crates/sqlbuild-core/src/fingerprint_identity/tests/test_fingerprint_identity.rs @@ -9,13 +9,13 @@ fn given_recorded_metadata_when_hashing_then_it_matches_the_config_as_written() let test_cases = [ RecordedLocalHashTestCase { description: "a written-out default refresh mode stays in the hash", - metadata_json: r#"{"config":{"materialized":"maintained","refresh_mode":"incremental"},"execution_signature":{},"local_function_hashes":{},"model_name":"daily_totals"}"#, - expected_hashed_json: r#"{"config":{"materialized":"maintained","refresh_mode":"incremental"},"execution_signature":{},"local_function_hashes":{},"model_name":"daily_totals"}"#, + metadata_json: r#"{"config":{"materialized":"dynamic","refresh_mode":"incremental"},"execution_signature":{},"local_function_hashes":{},"model_name":"daily_totals"}"#, + expected_hashed_json: r#"{"config":{"materialized":"dynamic","refresh_mode":"incremental"},"execution_signature":{},"local_function_hashes":{},"model_name":"daily_totals"}"#, }, RecordedLocalHashTestCase { description: "a model that declares no refresh mode", - metadata_json: r#"{"config":{"materialized":"maintained"},"execution_signature":{},"local_function_hashes":{},"model_name":"daily_totals"}"#, - expected_hashed_json: r#"{"config":{"materialized":"maintained"},"execution_signature":{},"local_function_hashes":{},"model_name":"daily_totals"}"#, + metadata_json: r#"{"config":{"materialized":"dynamic"},"execution_signature":{},"local_function_hashes":{},"model_name":"daily_totals"}"#, + expected_hashed_json: r#"{"config":{"materialized":"dynamic"},"execution_signature":{},"local_function_hashes":{},"model_name":"daily_totals"}"#, }, ]; diff --git a/crates/sqlbuild-model-config/src/model_validation/_helpers/materialization.rs b/crates/sqlbuild-model-config/src/model_validation/_helpers/materialization.rs index e3633aac1a..6e61dd7b8e 100644 --- a/crates/sqlbuild-model-config/src/model_validation/_helpers/materialization.rs +++ b/crates/sqlbuild-model-config/src/model_validation/_helpers/materialization.rs @@ -22,17 +22,17 @@ use crate::model_validation::constants::{ CHAINED_HELP, CLAIMED_HELP, MATERIALIZATION_HELP, MIGRATABLE_MATERIALIZATIONS, ORIGIN_HELP, QUALIFIED_NAME_SEPARATOR, }; +use crate::model_validation::constants::{ + DYNAMIC_ADAPTER_CODE, DYNAMIC_ADAPTERS, DYNAMIC_MATERIALIZATION, DYNAMIC_ONLY_KEYS, + DYNAMIC_REFRESH_FORMS, DYNAMIC_REFRESH_MODES, DYNAMIC_REFRESH_UNSUPPORTED_CODE, + MATERIALIZED_VIEW_ADAPTER_CODE, MATERIALIZED_VIEW_MATERIALIZATION, SNAPSHOT_DISALLOWED_KEYS, + SNOWFLAKE_ADAPTER, SNOWFLAKE_MIN_LAG_SECONDS, +}; use crate::model_validation::constants::{ ENGINE_FAMILY_SUFFIX, ENGINE_KEY, ORDER_BY_KEY, PARTITION_BY_KEY, PHYSICAL_STORAGE_KEYS, PHYSICAL_STORAGE_MATERIALIZATIONS, PRIMARY_KEY_KEY, TABLE_SETTINGS_KEY, UNSUPPORTED_ENGINE_PREFIXES, }; -use crate::model_validation::constants::{ - MAINTAINED_ADAPTER_CODE, MAINTAINED_ADAPTERS, MAINTAINED_MATERIALIZATION, MAINTAINED_ONLY_KEYS, - MAINTAINED_REFRESH_FORMS, MAINTAINED_REFRESH_MODES, MAINTAINED_REFRESH_UNSUPPORTED_CODE, - MATERIALIZED_VIEW_ADAPTER_CODE, MATERIALIZED_VIEW_MATERIALIZATION, SNAPSHOT_DISALLOWED_KEYS, - SNOWFLAKE_ADAPTER, SNOWFLAKE_MIN_LAG_SECONDS, -}; use crate::model_validation::models::{ ModelValidationFacts, ProjectValidationFacts, ValidationStop, }; @@ -622,8 +622,8 @@ fn setting_name(name: &str) -> bool { .all(|character| character.is_ascii_alphanumeric() || character == '_') } -/// Check the refresh fields: only maintained models on an adapter that builds them take them. -pub(crate) fn check_maintained( +/// Check the refresh fields: only dynamic models on an adapter that builds them take them. +pub(crate) fn check_dynamic( config: &ConfigView<'_, N>, project: &ProjectValidationFacts, ) -> Check { @@ -638,30 +638,30 @@ pub(crate) fn check_maintained( return Err(coded( config, MATERIALIZED_VIEW_ADAPTER_CODE, - "Snowflake has no materialized_view materialization; use materialized maintained, \ + "Snowflake has no materialized_view materialization; use materialized dynamic, \ which builds a dynamic table", )); } - if materialized.as_deref() != Some(MAINTAINED_MATERIALIZATION) { - return match config.first_present(&MAINTAINED_ONLY_KEYS) { - Some(key) => Err(config.error(format!("{key} is only valid for maintained models"))), + if materialized.as_deref() != Some(DYNAMIC_MATERIALIZATION) { + return match config.first_present(&DYNAMIC_ONLY_KEYS) { + Some(key) => Err(config.error(format!("{key} is only valid for dynamic models"))), None => Ok(()), }; } - if let Some(adapter) = adapter.filter(|adapter| !one_of(adapter, &MAINTAINED_ADAPTERS)) { + if let Some(adapter) = adapter.filter(|adapter| !one_of(adapter, &DYNAMIC_ADAPTERS)) { return Err(coded( config, - MAINTAINED_ADAPTER_CODE, + DYNAMIC_ADAPTER_CODE, format!( - "materialized maintained is not supported on the {adapter} adapter; adapters \ + "materialized dynamic is not supported on the {adapter} adapter; adapters \ that support it: {}", - MAINTAINED_ADAPTERS.join(", ") + DYNAMIC_ADAPTERS.join(", ") ), )); } if let Some(key) = config.first_present(&SNAPSHOT_DISALLOWED_KEYS) { return Err(config.error(format!( - "{key} is not valid for maintained models: the warehouse refreshes them from the \ + "{key} is not valid for dynamic models: the warehouse refreshes them from the \ whole query" ))); } @@ -670,11 +670,11 @@ pub(crate) fn check_maintained( let text: Option = text_if_string(mode); if !text .as_deref() - .is_some_and(|text| one_of(text, &MAINTAINED_REFRESH_MODES)) + .is_some_and(|text| one_of(text, &DYNAMIC_REFRESH_MODES)) { return Err(config.error(format!( "{REFRESH_MODE_KEY} must be one of: {}; got '{}'", - sorted_values(&MAINTAINED_REFRESH_MODES), + sorted_values(&DYNAMIC_REFRESH_MODES), text.unwrap_or_default() ))); } @@ -711,7 +711,7 @@ fn check_refresh(config: &ConfigView<'_, N>, adapter: Option<&s let [(form, value)] = block.as_slice() else { return Err(unknown_form(config, &forms)); }; - if !MAINTAINED_REFRESH_FORMS + if !DYNAMIC_REFRESH_FORMS .iter() .any(|known| form.is_text(known)) { @@ -727,7 +727,7 @@ fn check_refresh(config: &ConfigView<'_, N>, adapter: Option<&s if adapter == Some(SNOWFLAKE_ADAPTER) && lag.fixed_seconds < SNOWFLAKE_MIN_LAG_SECONDS { return Err(coded( config, - MAINTAINED_REFRESH_UNSUPPORTED_CODE, + DYNAMIC_REFRESH_UNSUPPORTED_CODE, format!("{REFRESH_KEY} {LAG_FORM} on Snowflake must be at least 1m; got '{text}'"), )); } diff --git a/crates/sqlbuild-model-config/src/model_validation/constants.rs b/crates/sqlbuild-model-config/src/model_validation/constants.rs index bb0dad808a..c0749a7410 100644 --- a/crates/sqlbuild-model-config/src/model_validation/constants.rs +++ b/crates/sqlbuild-model-config/src/model_validation/constants.rs @@ -2,7 +2,7 @@ /// Built-in materializations that never name a custom materialization. pub const BUILTIN_MATERIALIZATIONS: [&str; 5] = - ["view", "table", "incremental", "snapshot", "maintained"]; + ["view", "table", "incremental", "snapshot", "dynamic"]; /// Materializations that own a physical managed table. pub const TABLE_BACKED_MATERIALIZATIONS: [&str; 3] = ["table", "incremental", "snapshot"]; /// Materializations that keep history across runs. @@ -16,32 +16,32 @@ pub const INCREMENTAL_MATERIALIZATION: &str = "incremental"; /// The snapshot materialization. pub const SNAPSHOT_MATERIALIZATION: &str = "snapshot"; /// The materialization whose stored result the warehouse keeps fresh between builds. -pub const MAINTAINED_MATERIALIZATION: &str = "maintained"; -/// Keys only a maintained model accepts. -pub const MAINTAINED_ONLY_KEYS: [&str; 4] = [ +pub const DYNAMIC_MATERIALIZATION: &str = "dynamic"; +/// Keys only a dynamic model accepts. +pub const DYNAMIC_ONLY_KEYS: [&str; 4] = [ "refresh", "refresh_mode", "refresh_warehouse", "refresh_on_build", ]; /// The `refresh ( … )` forms any adapter accepts; each adapter accepts only its own. -pub const MAINTAINED_REFRESH_FORMS: [&str; 1] = ["lag"]; +pub const DYNAMIC_REFRESH_FORMS: [&str; 1] = ["lag"]; /// `refresh_mode` values. -pub const MAINTAINED_REFRESH_MODES: [&str; 4] = ["incremental", "full", "auto", "adaptive"]; -/// Adapters that build maintained models, sorted. -pub const MAINTAINED_ADAPTERS: [&str; 1] = ["snowflake"]; +pub const DYNAMIC_REFRESH_MODES: [&str; 4] = ["incremental", "full", "auto", "adaptive"]; +/// Adapters that build dynamic models, sorted. +pub const DYNAMIC_ADAPTERS: [&str; 1] = ["snowflake"]; /// The Snowflake adapter name. pub const SNOWFLAKE_ADAPTER: &str = "snowflake"; -/// The materialization name other warehouses use, which Snowflake builds as `maintained`. +/// The materialization name other warehouses use, which Snowflake builds as `dynamic`. pub const MATERIALIZED_VIEW_MATERIALIZATION: &str = "materialized_view"; /// Snowflake's smallest dynamic table target lag, in seconds. pub const SNOWFLAKE_MIN_LAG_SECONDS: u128 = 60; -/// The code of a maintained model on an adapter that cannot build one. -pub const MAINTAINED_ADAPTER_CODE: &str = "A013"; -/// The code of `materialized_view` on Snowflake, which builds `maintained` instead. +/// The code of a dynamic model on an adapter that cannot build one. +pub const DYNAMIC_ADAPTER_CODE: &str = "A013"; +/// The code of `materialized_view` on Snowflake, which builds `dynamic` instead. pub const MATERIALIZED_VIEW_ADAPTER_CODE: &str = "A014"; /// The code of a refresh form or value the adapter does not accept. -pub const MAINTAINED_REFRESH_UNSUPPORTED_CODE: &str = "A015"; +pub const DYNAMIC_REFRESH_UNSUPPORTED_CODE: &str = "A015"; /// `IncrementalStrategy` values. pub const INCREMENTAL_STRATEGIES: [&str; 3] = ["append", "delete_insert", "merge"]; diff --git a/crates/sqlbuild-model-config/src/model_validation/main/validate_model_config.rs b/crates/sqlbuild-model-config/src/model_validation/main/validate_model_config.rs index 23eb12f947..85fad21212 100644 --- a/crates/sqlbuild-model-config/src/model_validation/main/validate_model_config.rs +++ b/crates/sqlbuild-model-config/src/model_validation/main/validate_model_config.rs @@ -3,7 +3,7 @@ use crate::model_validation::_helpers::config::ConfigView; use crate::model_validation::_helpers::incremental::check_incremental; use crate::model_validation::_helpers::materialization::{ - check_contract, check_custom_materialization, check_execution_limit, check_maintained, + check_contract, check_custom_materialization, check_dynamic, check_execution_limit, check_migration, check_non_incremental, check_physical_storage, check_placeholders, check_project_capability, check_review_marker, check_storage_policies, }; @@ -29,7 +29,7 @@ pub fn validate_model_config( check_review_marker(&config)?; check_non_incremental(&config)?; check_snapshot(&config, facts)?; - check_maintained(&config, project)?; + check_dynamic(&config, project)?; check_custom_materialization(&config, project)?; check_storage_policies(&config, facts)?; check_physical_storage(&config)?; diff --git a/crates/sqlbuild-model-config/src/model_validation/tests/test_validate_model_config.rs b/crates/sqlbuild-model-config/src/model_validation/tests/test_validate_model_config.rs index 6375dca5f1..6362687338 100644 --- a/crates/sqlbuild-model-config/src/model_validation/tests/test_validate_model_config.rs +++ b/crates/sqlbuild-model-config/src/model_validation/tests/test_validate_model_config.rs @@ -555,16 +555,16 @@ fn given_effective_configs_when_validating_natively_then_python_outcomes_result( } #[test] -fn given_maintained_model_headers_when_validating_then_refresh_fields_are_checked() { - let maintained = |extra: Vec<(&'static str, Value)>| { - let mut config = vec![("materialized", Value::Str("maintained"))]; +fn given_dynamic_model_headers_when_validating_then_refresh_fields_are_checked() { + let dynamic = |extra: Vec<(&'static str, Value)>| { + let mut config = vec![("materialized", Value::Str("dynamic"))]; config.extend(extra); config }; let test_cases = [ ValidateTestCase { - description: "a maintained model with a lag, a mode and a warehouse", - config: maintained(vec![ + description: "a dynamic model with a lag, a mode and a warehouse", + config: dynamic(vec![ ("refresh", map(vec![("lag", Value::Str("1h"))])), ("refresh_mode", Value::Str("adaptive")), ("refresh_warehouse", Value::Str("transform_wh")), @@ -575,8 +575,8 @@ fn given_maintained_model_headers_when_validating_then_refresh_fields_are_checke expected_outcome: "accepted", }, ValidateTestCase { - description: "a maintained model refreshed by its readers", - config: maintained(vec![]), + description: "a dynamic model refreshed by its readers", + config: dynamic(vec![]), references: &["ref:orders"], query_sql: "select 1", expected_outcome: "accepted", @@ -589,25 +589,25 @@ fn given_maintained_model_headers_when_validating_then_refresh_fields_are_checke ], references: &[], query_sql: "select 1", - expected_outcome: "model 'orders_daily': refresh is only valid for maintained models", + expected_outcome: "model 'orders_daily': refresh is only valid for dynamic models", }, ValidateTestCase { description: "refresh_on_build on a model with no materialization", config: vec![("refresh_on_build", Value::Bool(true))], references: &[], query_sql: "select 1", - expected_outcome: "model 'orders_daily': refresh_on_build is only valid for maintained models", + expected_outcome: "model 'orders_daily': refresh_on_build is only valid for dynamic models", }, ValidateTestCase { description: "a refresh block with a form no adapter has", - config: maintained(vec![("refresh", map(vec![("every", Value::Str("1h"))]))]), + config: dynamic(vec![("refresh", map(vec![("every", Value::Str("1h"))]))]), references: &[], query_sql: "select 1", expected_outcome: "model 'orders_daily': refresh takes exactly one form, such as refresh (lag 1h); got: every", }, ValidateTestCase { description: "a refresh block with two forms", - config: maintained(vec![( + config: dynamic(vec![( "refresh", map(vec![ ("lag", Value::Str("1h")), @@ -620,63 +620,63 @@ fn given_maintained_model_headers_when_validating_then_refresh_fields_are_checke }, ValidateTestCase { description: "refresh written as a plain value", - config: maintained(vec![("refresh", Value::Str("1h"))]), + config: dynamic(vec![("refresh", Value::Str("1h"))]), references: &[], query_sql: "select 1", expected_outcome: "model 'orders_daily': refresh must be a block such as refresh (lag 1h)", }, ValidateTestCase { description: "a lag in months", - config: maintained(vec![("refresh", map(vec![("lag", Value::Str("1mo"))]))]), + config: dynamic(vec![("refresh", map(vec![("lag", Value::Str("1mo"))]))]), references: &[], query_sql: "select 1", expected_outcome: "model 'orders_daily': refresh lag must be a fixed duration such as 5m, 1h or 1d; got '1mo'", }, ValidateTestCase { description: "a lag that is not a duration", - config: maintained(vec![("refresh", map(vec![("lag", Value::Str("hourly"))]))]), + config: dynamic(vec![("refresh", map(vec![("lag", Value::Str("hourly"))]))]), references: &[], query_sql: "select 1", expected_outcome: "model 'orders_daily': refresh lag must be a fixed duration such as 5m, 1h or 1d; got 'hourly'", }, ValidateTestCase { description: "an unknown refresh mode", - config: maintained(vec![("refresh_mode", Value::Str("fast"))]), + config: dynamic(vec![("refresh_mode", Value::Str("fast"))]), references: &[], query_sql: "select 1", expected_outcome: "model 'orders_daily': refresh_mode must be one of: adaptive, auto, full, incremental; got 'fast'", }, ValidateTestCase { description: "refresh_on_build that is not a boolean", - config: maintained(vec![("refresh_on_build", Value::Str("yes"))]), + config: dynamic(vec![("refresh_on_build", Value::Str("yes"))]), references: &[], query_sql: "select 1", expected_outcome: "model 'orders_daily': refresh_on_build must be a boolean", }, ValidateTestCase { description: "a blank refresh warehouse", - config: maintained(vec![("refresh_warehouse", Value::Str(" "))]), + config: dynamic(vec![("refresh_warehouse", Value::Str(" "))]), references: &[], query_sql: "select 1", expected_outcome: "model 'orders_daily': refresh_warehouse must be a non-empty warehouse name", }, ValidateTestCase { - description: "an incremental-only key on a maintained model", - config: maintained(vec![("cursor", Value::Str("updated_at"))]), + description: "an incremental-only key on a dynamic model", + config: dynamic(vec![("cursor", Value::Str("updated_at"))]), references: &[], query_sql: "select 1", - expected_outcome: "model 'orders_daily': cursor is not valid for maintained models: the warehouse refreshes them from the whole query", + expected_outcome: "model 'orders_daily': cursor is not valid for dynamic models: the warehouse refreshes them from the whole query", }, ValidateTestCase { description: "the smallest lag Snowflake accepts", - config: maintained(vec![("refresh", map(vec![("lag", Value::Str("1m"))]))]), + config: dynamic(vec![("refresh", map(vec![("lag", Value::Str("1m"))]))]), references: &[], query_sql: "select 1", expected_outcome: "accepted", }, ValidateTestCase { description: "a lag one second under Snowflake's minimum", - config: maintained(vec![("refresh", map(vec![("lag", Value::Str("59s"))]))]), + config: dynamic(vec![("refresh", map(vec![("lag", Value::Str("59s"))]))]), references: &[], query_sql: "select 1", expected_outcome: "model 'orders_daily': refresh lag on Snowflake must be at least 1m; got '59s'", @@ -686,7 +686,7 @@ fn given_maintained_model_headers_when_validating_then_refresh_fields_are_checke config: vec![("materialized", Value::Str("materialized_view"))], references: &[], query_sql: "select 1", - expected_outcome: "model 'orders_daily': Snowflake has no materialized_view materialization; use materialized maintained, which builds a dynamic table", + expected_outcome: "model 'orders_daily': Snowflake has no materialized_view materialization; use materialized dynamic, which builds a dynamic table", }, ]; diff --git a/crates/sqlbuild-orchestration/tests/api/test_warehouse_metadata.rs b/crates/sqlbuild-orchestration/tests/api/test_warehouse_metadata.rs index 8df1af7c22..901f154c74 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_warehouse_metadata.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_warehouse_metadata.rs @@ -3,7 +3,7 @@ use chrono::{DateTime, Duration, Utc}; use reqwest::StatusCode; use serde_json::json; -use sqlbuild_core::constants::{MAINTAINED_PAUSES_TABLE_NAME, RUNS_TABLE_NAME, STATE_TABLE_NAMES}; +use sqlbuild_core::constants::{DYNAMIC_PAUSES_TABLE_NAME, RUNS_TABLE_NAME, STATE_TABLE_NAMES}; use uuid::Uuid; use crate::api::tests::changes_helpers::{connect, expect_topics}; @@ -37,10 +37,10 @@ use crate::warehouse_metadata::models::{MetadataBatch, WarehouseHistoryKind}; #[test] fn given_state_tables_when_classified_then_each_is_mirrored_by_exactly_one_kind() { // The runs table has no mirror until the control store decides where observed runs live; - // maintained pauses are a lease that builds read from the warehouse, never a mirror. + // dynamic pauses are a lease that builds read from the warehouse, never a mirror. let mut written: Vec<&str> = STATE_TABLE_NAMES .into_iter() - .filter(|table| ![RUNS_TABLE_NAME, MAINTAINED_PAUSES_TABLE_NAME].contains(table)) + .filter(|table| ![RUNS_TABLE_NAME, DYNAMIC_PAUSES_TABLE_NAME].contains(table)) .collect(); written.sort_unstable(); let test_cases = [StateTableClassificationTestCase { diff --git a/crates/sqlbuild-python-base/src/records/_helpers/compiler_enums/constants.rs b/crates/sqlbuild-python-base/src/records/_helpers/compiler_enums/constants.rs index 5b65da0fd2..8621d90ffb 100644 --- a/crates/sqlbuild-python-base/src/records/_helpers/compiler_enums/constants.rs +++ b/crates/sqlbuild-python-base/src/records/_helpers/compiler_enums/constants.rs @@ -586,9 +586,9 @@ const E_PLAN_ACTION: EnumSpec = EnumSpec { ("LOAD_SEED", "load_seed"), ("SKIP", "skip"), ("CUSTOM", "custom"), - ("CREATE_MAINTAINED", "create_maintained"), - ("ALTER_MAINTAINED", "alter_maintained"), - ("KEEP_MAINTAINED", "keep_maintained"), + ("CREATE_DYNAMIC", "create_dynamic"), + ("ALTER_DYNAMIC", "alter_dynamic"), + ("KEEP_DYNAMIC", "keep_dynamic"), ], }; const E_PLAN_REASON: EnumSpec = EnumSpec { @@ -686,7 +686,7 @@ const E_MATERIALIZATION_TYPE: EnumSpec = EnumSpec { ("SNAPSHOT", "snapshot"), ("SEED", "seed"), ("CUSTOM", "custom"), - ("MAINTAINED", "maintained"), + ("DYNAMIC", "dynamic"), ], }; const E_SCENARIO_ARTIFACT_KIND: EnumSpec = EnumSpec { diff --git a/crates/sqlbuild-python-base/src/records/_helpers/contract_records/constants.rs b/crates/sqlbuild-python-base/src/records/_helpers/contract_records/constants.rs index 46587a7e93..58bc8be8fa 100644 --- a/crates/sqlbuild-python-base/src/records/_helpers/contract_records/constants.rs +++ b/crates/sqlbuild-python-base/src/records/_helpers/contract_records/constants.rs @@ -151,7 +151,7 @@ const R_TARGET_CONFIG: RecordSpec = RecordSpec { FieldSpec::new("catalog_connection", FieldDefault::None), FieldSpec::new("ignore_audit_failures", FieldDefault::None), FieldSpec::new("plan_from", FieldDefault::None), - FieldSpec::new("pause_maintained", FieldDefault::None), + FieldSpec::new("pause_dynamic", FieldDefault::None), ], }; const R_TARGET_SETTINGS: RecordSpec = RecordSpec { @@ -211,7 +211,7 @@ const R_LOCAL_TARGET_CONFIG: RecordSpec = RecordSpec { FieldSpec::new("catalog_connection", FieldDefault::None), FieldSpec::new("ignore_audit_failures", FieldDefault::None), FieldSpec::new("plan_from", FieldDefault::None), - FieldSpec::new("pause_maintained", FieldDefault::None), + FieldSpec::new("pause_dynamic", FieldDefault::None), ], }; const R_SETTINGS_CONFIG: RecordSpec = RecordSpec { diff --git a/crates/sqlbuild-python-base/src/records/_helpers/planner_records/constants.rs b/crates/sqlbuild-python-base/src/records/_helpers/planner_records/constants.rs index a5bb652b75..be7daa312e 100644 --- a/crates/sqlbuild-python-base/src/records/_helpers/planner_records/constants.rs +++ b/crates/sqlbuild-python-base/src/records/_helpers/planner_records/constants.rs @@ -671,8 +671,8 @@ const R_MODEL_PLAN_ENTRY: RecordSpec = RecordSpec { FieldSpec::new("refresh_mode", FieldDefault::None), FieldSpec::new("refresh_warehouse", FieldDefault::None), FieldSpec::new("refresh_on_build", FieldDefault::False), - FieldSpec::new("maintained_readers", FieldDefault::EmptyTuple), - FieldSpec::new("maintained_paused", FieldDefault::False), + FieldSpec::new("dynamic_readers", FieldDefault::EmptyTuple), + FieldSpec::new("dynamic_paused", FieldDefault::False), ], }; const R_CLONE_SOURCE_PLAN_ENTRY: RecordSpec = RecordSpec { diff --git a/crates/sqlbuild-python-base/src/records/constants.rs b/crates/sqlbuild-python-base/src/records/constants.rs index 6825d659f6..d2b0e21e1b 100644 --- a/crates/sqlbuild-python-base/src/records/constants.rs +++ b/crates/sqlbuild-python-base/src/records/constants.rs @@ -255,7 +255,7 @@ const BUILD_COMMAND_REQUEST: RecordSpec = RecordSpec { FieldSpec::new("allow_retention_decrease", FieldDefault::False), FieldSpec::new("allow_missing_migration_origin", FieldDefault::False), FieldSpec::new("allow_snapshot_schema_change", FieldDefault::False), - FieldSpec::new("ignore_maintained_grace", FieldDefault::False), + FieldSpec::new("ignore_dynamic_grace", FieldDefault::False), FieldSpec::new("concurrency", FieldDefault::None), FieldSpec::new("select", FieldDefault::EmptyTuple), FieldSpec::new("exclude", FieldDefault::EmptyTuple), diff --git a/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/maintained.rs b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/dynamic.rs similarity index 71% rename from crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/maintained.rs rename to crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/dynamic.rs index cb045e0beb..76f6016522 100644 --- a/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/maintained.rs +++ b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/dynamic.rs @@ -1,10 +1,10 @@ -//! Project-wide checks of maintained models on the effective adapter. +//! Project-wide checks of dynamic models on the effective adapter. use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; -use sqlbuild_analysis::assembly::maintained_checks::constants::MAINTAINED_NEVER_REFRESHED_CODE; -use sqlbuild_analysis::assembly::maintained_checks::main::never_refreshed_maintained::never_refreshed_maintained; -use sqlbuild_analysis::assembly::maintained_checks::models::MaintainedModel; +use sqlbuild_analysis::assembly::dynamic_checks::constants::DYNAMIC_NEVER_REFRESHED_CODE; +use sqlbuild_analysis::assembly::dynamic_checks::main::never_refreshed_dynamic::never_refreshed_dynamic; +use sqlbuild_analysis::assembly::dynamic_checks::models::DynamicModel; use sqlbuild_python_base::boundary::main::optional_text::optional_text; use sqlbuild_python_base::records::main::construct::enum_member::enum_member; use sqlbuild_python_base::records::main::fields::field::field as native_field; @@ -17,11 +17,11 @@ use sqlbuild_python_front::model_config::types::ConfigValues; use crate::compile_driver::_helpers::setup::targets::planner_error; const SNOWFLAKE_ADAPTER: &str = "snowflake"; -const MAINTAINED_MATERIALIZATION: &str = "maintained"; +const DYNAMIC_MATERIALIZATION: &str = "dynamic"; const REFRESH_KEY: &str = "refresh"; -/// Raise when a Snowflake maintained model has no refresh and no maintained model reads it. -pub(crate) fn maintained_refresh( +/// Raise when a Snowflake dynamic model has no refresh and no dynamic model reads it. +pub(crate) fn dynamic_refresh( adapter_name: &Bound<'_, PyAny>, project: &Bound<'_, PyAny>, ) -> PyResult<()> { @@ -30,12 +30,11 @@ pub(crate) fn maintained_refresh( } let py: Python<'_> = project.py(); let model_type: Bound<'_, PyAny> = enum_member(py, RecordId::CompiledResourceType, "MODEL")?; - let mut maintained: Vec = Vec::new(); + let mut dynamic: Vec = Vec::new(); for model in native_field(project, "models")?.try_iter()? { let model: Bound<'_, PyAny> = model?; let values: ConfigValues<'_> = model_values(&model)?; - if config_text(py, &values, "materialized")?.as_deref() != Some(MAINTAINED_MATERIALIZATION) - { + if config_text(py, &values, "materialized")?.as_deref() != Some(DYNAMIC_MATERIALIZATION) { continue; } let mut reads: Vec = Vec::new(); @@ -45,22 +44,22 @@ pub(crate) fn maintained_refresh( reads.push(native_field(&key, "name")?.extract()?); } } - maintained.push(MaintainedModel { + dynamic.push(DynamicModel { name: native_field(&model, "name")?.extract()?, has_refresh: !config_value(py, &values, REFRESH_KEY)?.is_none(), reads, }); } - let Some(name) = never_refreshed_maintained(&maintained) else { + let Some(name) = never_refreshed_dynamic(&dynamic) else { return Ok(()); }; Err(planner_error( py, format!( - "maintained model '{name}' has no refresh and no maintained model reads it, so \ + "dynamic model '{name}' has no refresh and no dynamic model reads it, so \ Snowflake would never refresh it; add refresh (lag 1h)" ), - MAINTAINED_NEVER_REFRESHED_CODE, + DYNAMIC_NEVER_REFRESHED_CODE, Some(format!( "set how stale '{name}' may get in its MODEL header:\n refresh (lag 1h)," )), diff --git a/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/mod.rs b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/mod.rs index 961c650c8f..7c158b10a8 100644 --- a/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/mod.rs +++ b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/mod.rs @@ -1,5 +1,5 @@ +pub(crate) mod dynamic; pub(crate) mod inputs; -pub(crate) mod maintained; pub(crate) mod model_free_locations; pub(crate) mod node_results; pub(crate) mod pipeline; diff --git a/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/project.rs b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/project.rs index 16444bd260..4f640436b0 100644 --- a/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/project.rs +++ b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/project.rs @@ -14,7 +14,7 @@ use pyo3::types::{PyBool, PyCFunction, PyDict, PyTuple}; use sqlbuild_host::compile_context::main::record_compile_timing::record_compile_timing; use sqlbuild_python_project::graph::main::python_nodes::resolve_node_sets::resolve_node_sets; -use crate::compile_driver::_helpers::setup::maintained::maintained_refresh; +use crate::compile_driver::_helpers::setup::dynamic::dynamic_refresh; use crate::compile_driver::_helpers::setup::target_defaults::TargetDefaults; use crate::compile_driver::_helpers::setup::target_defaults::apply_target_defaults; use crate::compile_driver::_helpers::setup::targets::managed_loader_target_isolation; @@ -243,7 +243,7 @@ fn targeted_project<'py>( }; let project: Bound<'py, PyAny> = apply_target_defaults(project, &defaults)?; project_targets(&adapter_name, &project)?; - maintained_refresh(&adapter_name, &project)?; + dynamic_refresh(&adapter_name, &project)?; let promotion_key: Bound<'py, PyAny> = pyo3::types::PyString::new(py, TABLE_PROMOTION_MODE_SETTING_KEY).into_any(); let settings_file: Bound<'py, PyAny> = diff --git a/crates/sqlbuild-python-front/src/discovery/main/project/configured_maintained.rs b/crates/sqlbuild-python-front/src/discovery/main/project/configured_dynamic.rs similarity index 69% rename from crates/sqlbuild-python-front/src/discovery/main/project/configured_maintained.rs rename to crates/sqlbuild-python-front/src/discovery/main/project/configured_dynamic.rs index daf7d1ec7f..71b2570941 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/project/configured_maintained.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/project/configured_dynamic.rs @@ -1,16 +1,16 @@ -//! A build's maintained-pause runtime settings: the target's `pause_maintained` and `[maintained]`. +//! A build's dynamic-pause runtime settings: the target's `pause_dynamic` and `[dynamic]`. use crate::contracts::main::resolve_target_config::resolve_target_config; use crate::contracts::main::resolve_target_name::resolve_target_name; use crate::discovery::_helpers::project::project_config::configured; use pyo3::IntoPyObjectExt; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; -use sqlbuild_config::project::main::load_maintained_config::load_maintained_config; -use sqlbuild_config::project::models::MaintainedConfig; +use sqlbuild_config::project::main::load_dynamic_config::load_dynamic_config; +use sqlbuild_config::project::models::DynamicConfig; use sqlbuild_python_base::records::main::fields::field::field as native_field; -/// `BuildRuntimeParams` fields for the request's target and flag, and the project's `[maintained]`. -pub fn configured_maintained<'py>( +/// `BuildRuntimeParams` fields for the request's target and flag, and the project's `[dynamic]`. +pub fn configured_dynamic<'py>( discovered: &Bound<'py, PyAny>, request: &Bound<'py, PyAny>, project_dir: &Bound<'py, PyAny>, @@ -23,28 +23,28 @@ pub fn configured_maintained<'py>( match resolve_target_name(py, &project_config, &local_config, &selected_target)? { Some(name) => native_field( &resolve_target_config(py, &project_config, &local_config, name.as_any())?, - "pause_maintained", + "pause_dynamic", )? .extract()?, None => None, }; - let config: MaintainedConfig = configured(project_dir, load_maintained_config)?; + let config: DynamicConfig = configured(project_dir, load_dynamic_config)?; Ok(vec![ ( - "pause_maintained", + "pause_dynamic", paused.unwrap_or(false).into_bound_py_any(py)?, ), ( - "maintained_heartbeat_grace_seconds", + "dynamic_heartbeat_grace_seconds", config.heartbeat_grace_seconds.into_bound_py_any(py)?, ), ( - "allow_ignore_maintained_grace", + "allow_ignore_dynamic_grace", config.allow_ignore_grace.into_bound_py_any(py)?, ), ( - "ignore_maintained_grace", - native_field(request, "ignore_maintained_grace")?, + "ignore_dynamic_grace", + native_field(request, "ignore_dynamic_grace")?, ), ]) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/project/mod.rs b/crates/sqlbuild-python-front/src/discovery/main/project/mod.rs index 1f0aeb28e9..85eb080380 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/project/mod.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/project/mod.rs @@ -1,7 +1,7 @@ pub mod column_name; pub mod configured_changes; +pub mod configured_dynamic; pub mod configured_git_base_ref; -pub mod configured_maintained; pub mod configured_review; pub mod discover_configured_providers; pub mod discover_connection_configuration; diff --git a/crates/sqlbuild-python-planner/src/live_reads/models.rs b/crates/sqlbuild-python-planner/src/live_reads/models.rs index 3841b31896..06110c27ea 100644 --- a/crates/sqlbuild-python-planner/src/live_reads/models.rs +++ b/crates/sqlbuild-python-planner/src/live_reads/models.rs @@ -438,8 +438,8 @@ impl WarehouseReads for LiveReads { "column_rename_unavailable_reason", vec![("destination", destination)], ), - AdapterRead::MaintainedRefresh([database, schema, name]) => ( - "maintained_listing", + AdapterRead::DynamicRefresh([database, schema, name]) => ( + "dynamic_listing", vec![("database", database), ("schema", schema), ("name", name)], ), }; diff --git a/crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_json.rs b/crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_json.rs index d92a09036f..186f30f55c 100644 --- a/crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_json.rs +++ b/crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_json.rs @@ -234,7 +234,7 @@ pub(crate) fn model_json<'py>(entry: &Bound<'py, PyAny>) -> PyResult = diff --git a/crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_text.rs b/crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_text.rs index e67e1c9a06..9dcef5903c 100644 --- a/crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_text.rs +++ b/crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_text.rs @@ -251,7 +251,7 @@ fn model_text(entry: &Bound<'_, PyAny>) -> PyResult { config_diff: coloured(model_config_diff(entry, &reason)?), query_diff: coloured(model_query_diff(entry, &reason)?), reinitialised_by: optional_text(&native_field(entry, "reinitialised_by")?)?, - paused: native_field(entry, "maintained_paused")?.is_truthy()?, + paused: native_field(entry, "dynamic_paused")?.is_truthy()?, name, action, reason, diff --git a/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/display_plan.rs b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/display_plan.rs index cabdb8403f..c26c222219 100644 --- a/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/display_plan.rs +++ b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/display_plan.rs @@ -24,7 +24,7 @@ use sqlbuild_python_project::planner_phases::main::object::object; const VIEW: &str = "view"; const INCREMENTAL: &str = "incremental"; const CUSTOM: &str = "custom"; -const MAINTAINED: &str = "maintained"; +const DYNAMIC: &str = "dynamic"; fn display_action( materialization_type: &Bound<'_, PyAny>, @@ -40,8 +40,8 @@ fn display_action( } } else if materialization_type.eq(CUSTOM)? { "CUSTOM" - } else if materialization_type.eq(MAINTAINED)? { - "CREATE_MAINTAINED" + } else if materialization_type.eq(DYNAMIC)? { + "CREATE_DYNAMIC" } else { "CREATE_TABLE" }) diff --git a/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/execution_plan.rs b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/execution_plan.rs index f33159b252..a36dd6b897 100644 --- a/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/execution_plan.rs +++ b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/execution_plan.rs @@ -27,7 +27,7 @@ use crate::planner_phases::main::outputs::base_plan_output::base_plan_output; use crate::planner_phases::main::outputs::build_planner_identity_context::build_planner_identity_context; use crate::planner_phases::main::outputs::check_selected_scope_buildability::check_selected_scope_buildability; use crate::planner_phases::main::outputs::detect_stale_warning_changes::detect_stale_warning_changes; -use crate::planner_phases::main::outputs::plan_maintained_refresh::plan_maintained_refresh; +use crate::planner_phases::main::outputs::plan_dynamic_refresh::plan_dynamic_refresh; use crate::planner_phases::main::outputs::unselected_upstream_models::unselected_upstream_models; use crate::planner_phases::main::rebuilds::build_planner_entry_results::build_planner_entry_results; use crate::planner_phases::main::rebuilds::defer_to_migration_origins::defer_to_migration_origins; @@ -197,7 +197,7 @@ pub(crate) fn planned_output<'py>( (&inspection_scope, &reconciliation), &source_freshness, )?; - let entries = plan_maintained_refresh(runtime, &inspection_scope, entries)?; + let entries = plan_dynamic_refresh(runtime, &inspection_scope, entries)?; record_step(BUILDING_ENTRIES, entries_started.elapsed().as_secs_f64()); let planned_sql_tests = timed(TEST_FIXTURES, || test_planning.result(py))?.into_bound(py); let assembling_started = Instant::now(); diff --git a/crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/maintained.rs b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/dynamic.rs similarity index 80% rename from crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/maintained.rs rename to crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/dynamic.rs index cafd4de572..d073a6e2e2 100644 --- a/crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/maintained.rs +++ b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/dynamic.rs @@ -1,4 +1,4 @@ -//! Maintained models whose live refresh settings drifted, kept ones a replaced upstream reinitialises, and who reads what. +//! Dynamic models whose live refresh settings drifted, kept ones a replaced upstream reinitialises, and who reads what. use crate::planner_phases::main::inputs::runtime_reads::runtime_reads; use crate::planner_phases::models::ModelEntryResults; @@ -9,11 +9,11 @@ use pyo3::IntoPyObject; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyBool, PyString, PyTuple}; use sqlbuild_analysis::assembly::project::types::ObjectKey; -use sqlbuild_analysis::maintained_refresh::main::maintained_settings_drift::maintained_settings_drift; -use sqlbuild_analysis::maintained_refresh::main::reinitialised_readers::reinitialised_readers; -use sqlbuild_analysis::maintained_refresh::main::render_target_lag::render_target_lag; -use sqlbuild_analysis::maintained_refresh::models::{ - LiveDynamicTable, MaintainedRead, MaintainedSettings, RefreshLag, +use sqlbuild_analysis::dynamic_refresh::main::dynamic_settings_drift::dynamic_settings_drift; +use sqlbuild_analysis::dynamic_refresh::main::reinitialised_readers::reinitialised_readers; +use sqlbuild_analysis::dynamic_refresh::main::render_target_lag::render_target_lag; +use sqlbuild_analysis::dynamic_refresh::models::{ + DynamicRead, DynamicSettings, LiveDynamicTable, RefreshLag, }; use sqlbuild_analysis::planner_scope::constants::MODEL; use sqlbuild_core::fingerprint_identity::constants::DEFAULT_REFRESH_MODE; @@ -30,17 +30,17 @@ use sqlbuild_python_project::native_host::models::ModelLookup; use sqlbuild_python_project::planner_scope::main::scope_data::scope_data; use std::collections::{BTreeSet, HashMap}; -const MAINTAINED: &str = "maintained"; -/// Views pass reads through, so a maintained model reading a view reads the view's inputs. +const DYNAMIC: &str = "dynamic"; +/// Views pass reads through, so a dynamic model reading a view reads the view's inputs. const VIEW: &str = "view"; -const KEEP_MAINTAINED: &str = "keep_maintained"; +const KEEP_DYNAMIC: &str = "keep_dynamic"; /// The scheduling state Snowflake lists for a dynamic table whose refreshes are paused. const SUSPENDED: &str = "SUSPENDED"; /// Actions that recreate the relation, which reinitialises every dynamic table reading it. -const REPLACING_ACTIONS: [&str; 3] = ["create_table", "create_view", "create_maintained"]; +const REPLACING_ACTIONS: [&str; 3] = ["create_table", "create_view", "create_dynamic"]; -/// Plan maintained refresh settings and reinitialised readers, and name the maintained readers of each entry. -pub(crate) fn plan_maintained_refresh<'py>( +/// Plan dynamic refresh settings and reinitialised readers, and name the dynamic readers of each entry. +pub(crate) fn plan_dynamic_refresh<'py>( runtime: &Bound<'py, PyAny>, execution_scope: &Bound<'py, PyAny>, results: ModelEntryResults<'py>, @@ -64,29 +64,28 @@ pub(crate) fn plan_maintained_refresh<'py>( through_view: HashMap::new(), }; let mut entries: Vec> = Vec::with_capacity(results.entries.len()); - let mut rows: Vec = Vec::with_capacity(results.entries.len()); + let mut rows: Vec = Vec::with_capacity(results.entries.len()); for entry in results.entries { let key: ObjectKey = native_key(&native_field(&entry, "key")?)?; - let is_maintained: bool = - text(&native_field(&entry, "materialization_type")?)? == MAINTAINED; - let entry = if is_maintained { + let is_dynamic: bool = text(&native_field(&entry, "materialization_type")?)? == DYNAMIC; + let entry = if is_dynamic { with_refresh_settings(entry, &models, &reads)? } else { entry }; - let readers: Vec = known.maintained_readers(py, &key, &downstream)?; + let readers: Vec = known.dynamic_readers(py, &key, &downstream)?; let entry = if readers.is_empty() { entry } else { replaced( &entry, - &[("maintained_readers", PyTuple::new(py, readers)?.into_any())], + &[("dynamic_readers", PyTuple::new(py, readers)?.into_any())], )? }; let action: String = text(&native_field(&entry, "action")?)?; - rows.push(MaintainedRead { + rows.push(DynamicRead { name: key.1.clone(), - maintained: is_maintained, + dynamic: is_dynamic, replaced: REPLACING_ACTIONS.contains(&action.as_str()), reads: upstream.remove(&key).unwrap_or_default(), }); @@ -107,7 +106,7 @@ pub(crate) fn plan_maintained_refresh<'py>( }) } -/// Per-plan lookups: each model's materialization, and the maintained readers behind each view. +/// Per-plan lookups: each model's materialization, and the dynamic readers behind each view. struct KnownReaders<'a> { models: &'a ModelLookup, materializations: HashMap, @@ -115,8 +114,8 @@ struct KnownReaders<'a> { } impl KnownReaders<'_> { - /// The maintained models reading the entry directly or through views, selected or not. - fn maintained_readers( + /// The dynamic models reading the entry directly or through views, selected or not. + fn dynamic_readers( &mut self, py: Python<'_>, key: &ObjectKey, @@ -133,13 +132,12 @@ impl KnownReaders<'_> { continue; } match self.materialization(py, name)?.as_str() { - MAINTAINED => { + DYNAMIC => { found.insert(name.clone()); } VIEW => { if !self.through_view.contains_key(name) { - let through: Vec = - self.maintained_readers(py, reader, downstream)?; + let through: Vec = self.dynamic_readers(py, reader, downstream)?; self.through_view.insert(name.clone(), through); } found.extend(self.through_view[name].iter().cloned()); @@ -203,14 +201,14 @@ fn with_refresh_settings<'py>( .into_any(), ), ]; - if text(&native_field(&entry, "action")?)? == KEEP_MAINTAINED { + if text(&native_field(&entry, "action")?)? == KEEP_DYNAMIC { let (action, paused) = live_action(&entry, &desired, reads)?; if let Some((action, reason)) = action { changes.push(("action", enum_member(py, RecordId::PlanAction, action)?)); changes.push(("reason", enum_member(py, RecordId::PlanReason, reason)?)); } changes.push(( - "maintained_paused", + "dynamic_paused", PyBool::new(py, paused).to_owned().into_any(), )); } @@ -220,13 +218,13 @@ fn with_refresh_settings<'py>( /// The action and reason replacing a kept model's, `None` when its listing matches or is unknown, and whether it is suspended. fn live_action( entry: &Bound<'_, PyAny>, - desired: &MaintainedSettings, + desired: &DynamicSettings, reads: &Bound<'_, PlannerReads>, ) -> PyResult<(Option<(&'static str, &'static str)>, bool)> { let destination = native_field(entry, "destination")?; let read = reads.get().reads().adapter_read( entry.py(), - AdapterRead::MaintainedRefresh([ + AdapterRead::DynamicRefresh([ native_field(&destination, "database")?, native_field(&destination, "schema")?, native_field(&destination, "name")?, @@ -237,7 +235,7 @@ fn live_action( Read::Known(listed) => listed, }; if listed.is_none() { - return Ok((Some(("CREATE_MAINTAINED", "FIRST_RUN")), false)); + return Ok((Some(("CREATE_DYNAMIC", "FIRST_RUN")), false)); } let (target_lag, warehouse, state): (String, String, String) = listed.extract()?; let live = LiveDynamicTable { @@ -245,12 +243,12 @@ fn live_action( warehouse, }; let action = - maintained_settings_drift(desired, &live).then_some(("ALTER_MAINTAINED", "CONFIG_CHANGED")); + dynamic_settings_drift(desired, &live).then_some(("ALTER_DYNAMIC", "CONFIG_CHANGED")); Ok((action, state == SUSPENDED)) } /// The model's declared lag and refresh warehouse, or `None` when its lag does not read. -fn declared_settings(model: &Bound<'_, PyAny>) -> PyResult> { +fn declared_settings(model: &Bound<'_, PyAny>) -> PyResult> { let py = model.py(); let values = model_values(model)?; let refresh = config_value(py, &values, "refresh")?; @@ -269,5 +267,5 @@ fn declared_settings(model: &Bound<'_, PyAny>) -> PyResult = config_text_object(py, &values, "refresh_warehouse")? .map(|warehouse| warehouse.extract()) .transpose()?; - Ok(Some(MaintainedSettings { lag, warehouse })) + Ok(Some(DynamicSettings { lag, warehouse })) } diff --git a/crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/mod.rs b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/mod.rs index 1295d1283b..b1dc8c7e1a 100644 --- a/crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/mod.rs +++ b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/outputs/mod.rs @@ -1,4 +1,4 @@ pub(crate) mod buildability; +pub(crate) mod dynamic; pub(crate) mod identities; -pub(crate) mod maintained; pub(crate) mod output_assembly; diff --git a/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/mod.rs b/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/mod.rs index 28947d1f49..64d9efc5fb 100644 --- a/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/mod.rs +++ b/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/mod.rs @@ -7,6 +7,6 @@ pub mod build_planner_identity_context; pub mod check_selected_scope_buildability; pub mod detect_stale_warning_changes; pub mod detect_stale_warning_changes_of_function; -pub mod plan_maintained_refresh; +pub mod plan_dynamic_refresh; pub mod planner_identity_work_info_function; pub mod unselected_upstream_models; diff --git a/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/plan_dynamic_refresh.rs b/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/plan_dynamic_refresh.rs new file mode 100644 index 0000000000..b0cc4aef7b --- /dev/null +++ b/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/plan_dynamic_refresh.rs @@ -0,0 +1,15 @@ +//! Alter drifted dynamic models, set their target lag, and mark kept readers of a replaced upstream. + +use crate::planner_phases::_helpers::outputs::dynamic; +use crate::planner_phases::models::ModelEntryResults; +use pyo3::prelude::{Bound, PyAny, PyResult}; + +/// Alter drifted dynamic models, set their target lag, and mark kept readers of a replaced upstream. +#[inline] +pub fn plan_dynamic_refresh<'py>( + runtime: &Bound<'py, PyAny>, + execution_scope: &Bound<'py, PyAny>, + entries: ModelEntryResults<'py>, +) -> PyResult> { + dynamic::plan_dynamic_refresh(runtime, execution_scope, entries) +} diff --git a/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/plan_maintained_refresh.rs b/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/plan_maintained_refresh.rs deleted file mode 100644 index 2b789a1dc5..0000000000 --- a/crates/sqlbuild-python-planner/src/planner_phases/main/outputs/plan_maintained_refresh.rs +++ /dev/null @@ -1,15 +0,0 @@ -//! Alter drifted maintained models, set their target lag, and mark kept readers of a replaced upstream. - -use crate::planner_phases::_helpers::outputs::maintained; -use crate::planner_phases::models::ModelEntryResults; -use pyo3::prelude::{Bound, PyAny, PyResult}; - -/// Alter drifted maintained models, set their target lag, and mark kept readers of a replaced upstream. -#[inline] -pub fn plan_maintained_refresh<'py>( - runtime: &Bound<'py, PyAny>, - execution_scope: &Bound<'py, PyAny>, - entries: ModelEntryResults<'py>, -) -> PyResult> { - maintained::plan_maintained_refresh(runtime, execution_scope, entries) -} diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/strategy.rs b/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/strategy.rs index 89cc09cce8..271f327da0 100644 --- a/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/strategy.rs +++ b/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/strategy.rs @@ -30,9 +30,9 @@ pub(crate) fn action_value(action: PlanAction) -> &'static str { PlanAction::Snapshot => "snapshot", PlanAction::Skip => "skip", PlanAction::Custom => "custom", - PlanAction::CreateMaintained => "create_maintained", - PlanAction::AlterMaintained => "alter_maintained", - PlanAction::KeepMaintained => "keep_maintained", + PlanAction::CreateDynamic => "create_dynamic", + PlanAction::AlterDynamic => "alter_dynamic", + PlanAction::KeepDynamic => "keep_dynamic", } } diff --git a/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/values.rs b/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/values.rs index 284859ce87..5a67e42a1c 100644 --- a/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/values.rs +++ b/crates/sqlbuild-python-planner/src/planner_strategy/_helpers/values.rs @@ -55,7 +55,7 @@ pub(crate) fn materialization_value(materialization: MaterializationType) -> &'s MaterializationType::Snapshot => "snapshot", MaterializationType::Seed => "seed", MaterializationType::Custom => "custom", - MaterializationType::Maintained => "maintained", + MaterializationType::Dynamic => "dynamic", } } diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/models.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/models.rs index b09c27b27d..bf6b4d06d9 100644 --- a/crates/sqlbuild-python-planner/src/planner_warehouse/models.rs +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/models.rs @@ -383,8 +383,8 @@ pub enum AdapterRead<'py> { FreshnessQuery(Bound<'py, PyAny>), /// Why a destination cannot rename columns, `None` when it can. ColumnRenameUnavailable(Bound<'py, PyAny>), - /// One maintained model's `[database, schema, name]` listed target lag and refresh warehouse. - MaintainedRefresh([Bound<'py, PyAny>; 3]), + /// One dynamic model's `[database, schema, name]` listed target lag and refresh warehouse. + DynamicRefresh([Bound<'py, PyAny>; 3]), } /// A read of what the warehouse state tables recorded. @@ -756,8 +756,8 @@ impl WarehouseReads for SyncedReads { AdapterRead::ColumnRenameUnavailable(_) => { return Ok(unknown(self, "column rename support")); } - AdapterRead::MaintainedRefresh(_) => { - return Ok(unknown(self, "maintained model listings")); + AdapterRead::DynamicRefresh(_) => { + return Ok(unknown(self, "dynamic model listings")); } }; let unavailable = diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs index 5a0a7a7829..cba13215c6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs @@ -49,7 +49,7 @@ use sqlbuild_python_compile::compile_driver::models::TargetSettingsValues; use sqlbuild_python_front::cost_context::models::CostContext; use sqlbuild_python_front::cursor::main::overrides::cursor_integer::cursor_integer; use sqlbuild_python_front::cursor::main::overrides::cursor_timestamp::cursor_timestamp; -use sqlbuild_python_front::discovery::main::project::configured_maintained::configured_maintained; +use sqlbuild_python_front::discovery::main::project::configured_dynamic::configured_dynamic; use sqlbuild_python_front::native_host::main::recorder_callbacks::recorder_callbacks; use sqlbuild_python_output::build_progress::main::batch_complete_callback::batch_complete_callback; use sqlbuild_python_output::build_progress::main::check_callback::check_callback; @@ -342,11 +342,11 @@ pub(crate) fn execute_build_plan<'py>( let target_config: TargetSettingsValues = project_target_settings(&discovered, &project, (&connection_config, &adapter))?; let (default_database, default_schema) = target_config.node_locations(&adapter)?; - let maintained = configured_maintained(&discovered, &context.request, &project_dir)?; + let dynamic = configured_dynamic(&discovered, &context.request, &project_dir)?; let runtime: Bound<'py, PyAny> = call_with( &host_entry(py, ExecutorHost::BuildRuntimeParams)?, &[ - &maintained[..], + &dynamic[..], &[ ("run_id", native_field(&project, "run_id")?), ("runtime_dir", joined(&project_dir, &["target"])?), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/constants.rs index a0c36d1e5c..510cd06cc0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/constants.rs @@ -19,7 +19,7 @@ use sqlbuild_core::command_output::constants::{ }; use sqlbuild_core::constants::{ AUDIT_RESULTS_TABLE_NAME, COLUMN_MIGRATION_TABLE_NAME, CONSUMPTION_TABLE_NAME, - FINGERPRINT_TABLE_NAME, JANITOR_EVENTS_TABLE_NAME, MAINTAINED_PAUSES_TABLE_NAME, + DYNAMIC_PAUSES_TABLE_NAME, FINGERPRINT_TABLE_NAME, JANITOR_EVENTS_TABLE_NAME, MICROBATCH_TABLE_NAME, MIGRATION_TABLE_NAME, NODE_RESULTS_TABLE_NAME, OBSERVATIONS_TABLE_NAME, OLD_NAME_VIEW_TABLE_NAME, PROJECT_CONFIG_FILENAME, PYTHON_INIT_MODULE_STEM, PYTHON_NODE_ROOT, RUNS_TABLE_NAME, SETTINGS_SECTION, SOURCE_FRESHNESS_TABLE_NAME, STAGED_PROMOTION_MODE, @@ -95,7 +95,7 @@ pub(crate) const EXPORTED_TEXTS: &[(&str, &str)] = &[ ("JANITOR_EVENTS_TABLE_NAME", JANITOR_EVENTS_TABLE_NAME), ("OBSERVATIONS_TABLE_NAME", OBSERVATIONS_TABLE_NAME), ("RUNS_TABLE_NAME", RUNS_TABLE_NAME), - ("MAINTAINED_PAUSES_TABLE_NAME", MAINTAINED_PAUSES_TABLE_NAME), + ("DYNAMIC_PAUSES_TABLE_NAME", DYNAMIC_PAUSES_TABLE_NAME), ("COLUMN_NODE_TYPE", COLUMN_NODE_TYPE), ("COLUMN_NODE_NAME", COLUMN_NODE_NAME), ("COLUMN_TARGET_DATABASE", COLUMN_TARGET_DATABASE), diff --git a/fensu.toml b/fensu.toml index 2a1becc261..73dcbfa978 100644 --- a/fensu.toml +++ b/fensu.toml @@ -1352,8 +1352,8 @@ path = "src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py" reason = "The SQL Server adapter intentionally keeps overrideable backend behavior on one class." [[dupes.allowlist]] -paths = ["scripts/native_layering/_helpers/layering.py", "src/sqlbuild/executor/maintained_pauses/_helpers/scope.py"] -reason = "A ten-line depth-first walk in two packages that must not import each other: the dependency-layering tool walks Cargo.lock packages, the build's maintained-model hold walks plan edges between models." +paths = ["scripts/native_layering/_helpers/layering.py", "src/sqlbuild/executor/dynamic_pauses/_helpers/scope.py"] +reason = "A ten-line depth-first walk in two packages that must not import each other: the dependency-layering tool walks Cargo.lock packages, the build's dynamic-model hold walks plan edges between models." [[dupes.allowlist]] paths = ["crates/sqlbuild-python-planner/src/planner_changes/_helpers/detect.rs", "crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/python_selection.rs"] diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/CONTENTS.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/CONTENTS.md index 484e052615..47720be1f4 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/CONTENTS.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/CONTENTS.md @@ -24,7 +24,7 @@ Bundled copies of every page on the SQLBuild documentation site, matching the in - [Seeds](concepts/seeds.md) (`concepts/seeds`) - Load static CSV data into your pipeline as tables. - [Incremental Models](concepts/incremental.md) (`concepts/incremental`) - Cursor-based incremental strategies, microbatch execution, and backfill policies. - [Snapshots (SCD Type 2)](concepts/snapshots.md) (`concepts/snapshots`) - Preserve row history over time using SCD Type 2 semantics with timestamp or check-based change detection. -- [Maintained Models](concepts/models/maintained.md) (`concepts/models/maintained`) - Let the warehouse keep a model up to date on a schedule, paused safely while builds run. +- [Dynamic models](concepts/models/dynamic.md) (`concepts/models/dynamic`) - Let the warehouse keep a model up to date on a schedule, paused safely while builds run. - [Functions](concepts/functions.md) (`concepts/functions`) - User-defined functions and table functions managed as part of your project. - [Macros](concepts/macros.md) (`concepts/macros`) - Reuse SQL with typed SQL macros, or generate it with Python macros when it needs logic. - [SQL Macros](concepts/macros/sql-macros.md) (`concepts/macros/sql-macros`) - Write typed, logic-free SQL fragments that the compiler checks and sees inside. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/build.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/build.md index cc013924c9..ab495b4a1d 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/build.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/build.md @@ -66,7 +66,7 @@ own option. `--version` works only directly after `sqb`, as in `sqb --version`: | `--defer-sources-to` | Read managed source data from another target | | `--plan-from` | Read warehouse state `live`, or from the target's `synced_state` checked against the warehouse first; overrides the target's `plan_from`. See [Planning from synced state](../concepts/project-configuration.md#planning-from-synced-state) | | `--fail-fast` | Stop on first failure and skip remaining nodes | -| `--ignore-maintained-grace` | Take over [maintained models](../concepts/models/maintained.md#when-a-build-dies) another build holds, without waiting for `[maintained] heartbeat_grace`. Needs `allow_ignore_grace = true`, else fails with `A719` | +| `--ignore-dynamic-grace` | Take over [dynamic models](../concepts/models/dynamic.md#when-a-build-dies) another build holds, without waiting for `[dynamic] heartbeat_grace`. Needs `allow_ignore_grace = true`, else fails with `A719` | | `--concurrency` | Number of worker connections (default: 1) | | `--test-timeout ` | Time limit for each unit test the build runs, such as `30s`, or `disabled`; see [Time limits](../concepts/testing-reference.md#time-limits) | | `--verbose`, `-v` | Show lifecycle SQL inline after each model | diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/dag.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/dag.md index c87285ddeb..3b1f83954c 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/dag.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/dag.md @@ -105,7 +105,7 @@ Each node represents a source, seed, model, or function: | `description` | Model or source description, if declared | | `tags` | Model tags | | `columns` | Column metadata (name, type, nullable, description) | -| `materialization_type` | For models: `view`, `table`, `incremental`, `snapshot`, `maintained`, or custom name | +| `materialization_type` | For models: `view`, `table`, `incremental`, `snapshot`, `dynamic`, or custom name | | `language` | For functions: `sql` or `python` | | `return_kind` | For functions: `scalar` or `table` | | `arguments` | For functions: argument name and type pairs | diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/snowflake.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/snowflake.md index 085bf739f4..85c44a8187 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/snowflake.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/snowflake.md @@ -14,7 +14,7 @@ Online: https://sqlbuild.com/docs/concepts/adapters/snowflake/ - Case-sensitive column names - Metadata reads - State tables -- Maintained models +- Dynamic models - Shared connections across targets - Cost estimates - Functions in unit tests @@ -166,10 +166,10 @@ SQLBuild records its state in `_sqlbuild_*` tables. On Snowflake it creates them with 90 days of time travel, or 1 day on editions that allow only that. It never alters an existing state table. Models are not affected: their table type follows the model's `table_type` setting. -## Maintained models +## Dynamic models -A [maintained model](../models/maintained.md) is a Snowflake dynamic -table. `materialized materialized_view` fails with `A014`; use `materialized maintained`. +A [dynamic model](../models/dynamic.md) is a Snowflake dynamic +table. `materialized materialized_view` fails with `A014`; use `materialized dynamic`. The build role needs these grants: @@ -181,12 +181,12 @@ The build role needs these grants: Dynamic tables refresh from changes, which needs change tracking on every table they read: -- A table or snapshot model that a maintained model reads, directly or through views, is built with +- A table or snapshot model that a dynamic model reads, directly or through views, is built with change tracking on. - For a source table, Snowflake turns change tracking on when the build role owns the table. Otherwise its owner runs `ALTER TABLE SET CHANGE_TRACKING = TRUE`. -Builds record the models they pause in the `_sqlbuild_maintained_pauses` state table, so another +Builds record the models they pause in the `_sqlbuild_dynamic_pauses` state table, so another build can tell whether they are still held. ## Shared connections across targets diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/configuration.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/configuration.md index 00a9c2512f..754c10adf1 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/configuration.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/configuration.md @@ -36,7 +36,7 @@ which accepts any keys. | Field | Description | |-------|-------------| -| `materialized` | `view`, `table`, `incremental`, `snapshot`, `maintained`, or a custom materialization name; see [Materializations](materializations.md) | +| `materialized` | `view`, `table`, `incremental`, `snapshot`, `dynamic`, or a custom materialization name; see [Materializations](materializations.md) | | `tags` | Tags used by selectors | | `critical` | `true` marks the model as critical, so [`sqb review`](../../cli/review.md) builds and compares it whenever something upstream of it changes (default `false`) | | `description` | Required human-readable model description; see [Descriptions](#descriptions) | diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/maintained.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/dynamic.md similarity index 67% rename from src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/maintained.md rename to src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/dynamic.md index cc52805737..53cc1d2d88 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/maintained.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/dynamic.md @@ -1,19 +1,19 @@ -# Maintained Models +# Dynamic models > Let the warehouse keep a model up to date on a schedule, paused safely while builds run. -Online: https://sqlbuild.com/docs/concepts/models/maintained/ +Online: https://sqlbuild.com/docs/concepts/models/dynamic/ -A maintained model is one the warehouse keeps up to date on a schedule you set, between builds. A +A dynamic model is one the warehouse keeps up to date on a schedule you set, between builds. A build creates it, changes it when its definition changes, and otherwise leaves the refreshing to the warehouse; on Snowflake it is a dynamic table. ```sql MODEL ( description "One row per customer per order date", - materialized maintained, + materialized dynamic, refresh (lag 1h), ); @@ -22,14 +22,14 @@ FROM __ref("stg_orders") GROUP BY customer_id, CAST(ordered_at AS DATE) ``` -Only Snowflake supports maintained models. See [Snowflake](../adapters/snowflake.md#maintained-models) +Only Snowflake supports dynamic models. See [Snowflake](../adapters/snowflake.md#dynamic-models) for the grants and change tracking they need. ## Settings | Field | Default | Description | |-------|---------|-------------| -| `refresh (lag )` | none | How far behind its inputs the model may fall: a fixed duration such as `5m`, `1h` or `1d`, at least `1m` on Snowflake and no shorter than the lag of a maintained model it reads. Without it, the model refreshes when a maintained model that reads it refreshes | +| `refresh (lag )` | none | How far behind its inputs the model may fall: a fixed duration such as `5m`, `1h` or `1d`, at least `1m` on Snowflake and no shorter than the lag of a dynamic model it reads. Without it, the model refreshes when a dynamic model that reads it refreshes | | `refresh_mode` | `incremental` | `incremental`, `full`, `auto` or `adaptive`. Snowflake rejects `incremental` for a query it can't refresh incrementally; use `full` for that query | | `refresh_warehouse` | the connection's warehouse | Warehouse the scheduled refreshes run on | | `refresh_on_build` | `false` | Refresh the model in every build that selects it, not only on its schedule | @@ -41,7 +41,7 @@ for the grants and change tracking they need. | First build, or the dynamic table is missing | `create` | Creates it and fills it | | Query or `refresh_mode` changed | `recreate, full recompute` | Replaces it; every row is computed again | | `refresh` lag or `refresh_warehouse` differs from the warehouse | `change refresh settings in place` | Alters the schedule; rows stay | -| Nothing | `maintained` | Leaves it alone | +| Nothing | `dynamic` | Leaves it alone | A model the plan keeps can carry one more line: @@ -50,7 +50,7 @@ A model the plan keeps can carry one more line: ## During a build -A build pauses the maintained models it changes, and those that read a model it rebuilds, directly +A build pauses the dynamic models it changes, and those that read a model it rebuilds, directly or through views, so they never refresh from half-built inputs. It resumes them when it ends, whether it succeeds or fails. @@ -59,23 +59,23 @@ whether it succeeds or fails. - A suspend made by someone else during the build is undone when the build resumes the model. - A kept model refreshes during the build when a model built later in the same build reads it. - `refresh_on_build` refreshes the model in every build that selects it. -- A target with `pause_maintained = true` leaves its maintained models suspended between builds; see [Targets](../project-configuration.md#targets). +- A target with `pause_dynamic = true` leaves its dynamic models suspended between builds; see [Targets](../project-configuration.md#targets). ## When a build dies -- **One build at a time.** A build that needs a maintained model another build holds fails with +- **One build at a time.** A build that needs a dynamic model another build holds fails with `A718`, naming that run. Wait for it to finish. Two builds that start at the same moment can both fail this way; run one of them again. - **A stopped build leaves its models suspended.** A build killed before it resumes them, for example by `SIGKILL`, can't resume them itself. -- **The next build resumes them.** The next build that pauses maintained models on the target - resumes them once the stopped build has been silent for `[maintained] heartbeat_grace`, 30 - minutes by default; see [Maintained](../project-configuration.md#maintained). -- **Taking over at once.** Run `sqb build --ignore-maintained-grace` to take the models over - without waiting. It needs `[maintained] allow_ignore_grace = true`; without it the flag fails +- **The next build resumes them.** The next build that pauses dynamic models on the target + resumes them once the stopped build has been silent for `[dynamic] heartbeat_grace`, 30 + minutes by default; see [Dynamic](../project-configuration.md#dynamic). +- **Taking over at once.** Run `sqb build --ignore-dynamic-grace` to take the models over + without waiting. It needs `[dynamic] allow_ignore_grace = true`; without it the flag fails with `A719`. You can also resume the models in Snowflake by hand. ## Limitations -- Unit tests and scenarios treat a maintained model as a table. -- On a target with `schema = "preserve"`, builds see each other's pauses only when their maintained models share a schema. +- Unit tests and scenarios treat a dynamic model as a table. +- On a target with `schema = "preserve"`, builds see each other's pauses only when their dynamic models share a schema. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/materializations.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/materializations.md index 5ca09cd193..2aa4141724 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/materializations.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/materializations.md @@ -12,7 +12,7 @@ Online: https://sqlbuild.com/docs/concepts/models/materializations/ - Table - Incremental - Snapshot -- Maintained +- Dynamic - Custom - When a build fails @@ -84,19 +84,19 @@ FROM __source("customers") See [Snapshots](../snapshots.md) for timestamp and check strategies, historical inputs, and full-refresh policies. -## Maintained +## Dynamic The warehouse keeps the model up to date on a schedule you set, and builds pause it while they change its inputs: ```sql -MODEL (description "One row per customer per order date", materialized maintained, refresh (lag 1h)); +MODEL (description "One row per customer per order date", materialized dynamic, refresh (lag 1h)); SELECT customer_id, CAST(ordered_at AS DATE) AS order_date, SUM(amount) AS revenue FROM __ref("stg_orders") GROUP BY customer_id, CAST(ordered_at AS DATE) ``` -See [Maintained models](maintained.md) for its settings, what the plan shows and what happens when a build dies. +See [Dynamic models](dynamic.md) for its settings, what the plan shows and what happens when a build dies. ## Custom @@ -135,5 +135,5 @@ The custom function owns staging, runtime type or contract enforcement, audit ti | Table (`immediate`) | The new table. Audits run after it is replaced. | | Incremental | The old table if a delta audit fails. Rows already written stay if a final audit or a later microbatch fails. | | Microbatch full rebuild | A partly rebuilt table; the old one was dropped first. | -| Maintained | The created or altered dynamic table, resumed. Audits run after it changes. | +| Dynamic | The created or altered dynamic table, resumed. Audits run after it changes. | | Custom | Whatever the custom materialization does. | diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md index 9a40443f25..539bf642a3 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md @@ -20,7 +20,7 @@ Online: https://sqlbuild.com/docs/concepts/project-configuration/ - Project variables - Janitor - Migrations -- Maintained +- Dynamic - Scenario - Git - Review @@ -190,7 +190,7 @@ A target is a named build context - the database and schema you build into, plus | `concurrency` | Worker limit for commands run against this target, replacing `[settings] concurrency` (see below) | | `ignore_audit_failures` | Whether builds on this target let audit failures through, replacing `[settings] ignore_audit_failures`; see [Ignoring audit failures](audits.md#ignoring-audit-failures) | | `plan_from` | Where `sqb plan` and `sqb build` read warehouse state: `live` (default) or `synced_state`; see [Planning from synced state](#planning-from-synced-state) | -| `pause_maintained` | `true` keeps this target's [maintained models](models/maintained.md) suspended between builds; each build refreshes the ones it selects. Default `false` | +| `pause_dynamic` | `true` keeps this target's [dynamic models](models/dynamic.md) suspended between builds; each build refreshes the ones it selects. Default `false` | ```toml [connections.warehouse] @@ -697,20 +697,20 @@ old_name_views = "30d" |-------|---------|-------------| | `old_name_views` | `"30d"` | How long a renamed model's old name keeps working through a [compatibility view](models/migrations.md#old-names) before janitor drops it. `false` keeps the old relation in place instead. A model's `old_name_view` header overrides it. | -## Maintained +## Dynamic -How builds pause [maintained models](models/maintained.md) while they run: +How builds pause [dynamic models](models/dynamic.md) while they run: ```toml -[maintained] +[dynamic] heartbeat_grace = "30m" allow_ignore_grace = false ``` | Field | Default | Description | |-------|---------|-------------| -| `heartbeat_grace` | `"30m"` | How long a build that paused maintained models may go silent before another build resumes them or takes them over. A fixed duration longer than `"1m"`, such as `"30m"` or `"2h"`; a build writes a heartbeat every minute | -| `allow_ignore_grace` | `false` | Allow `sqb build --ignore-maintained-grace`, which takes over models another build holds without waiting for the grace. Without it the flag fails with `A719` | +| `heartbeat_grace` | `"30m"` | How long a build that paused dynamic models may go silent before another build resumes them or takes them over. A fixed duration longer than `"1m"`, such as `"30m"` or `"2h"`; a build writes a heartbeat every minute | +| `allow_ignore_grace` | `false` | Allow `sqb build --ignore-dynamic-grace`, which takes over models another build holds without waiting for the grace. Without it the flag fails with `A719` | ## Scenario diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index eced41585a..47b8971951 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -192,7 +192,7 @@ MICROBATCH_TABLE_NAME: str NODE_RESULTS_TABLE_NAME: str OBSERVATIONS_TABLE_NAME: str RUNS_TABLE_NAME: str -MAINTAINED_PAUSES_TABLE_NAME: str +DYNAMIC_PAUSES_TABLE_NAME: str AUDIT_RESULTS_TABLE_NAME: str JANITOR_EVENTS_TABLE_NAME: str COLUMN_NODE_TYPE: str @@ -6593,7 +6593,7 @@ class BuildCommandRequest(NativeRecord): allow_retention_decrease: bool allow_missing_migration_origin: bool allow_snapshot_schema_change: bool - ignore_maintained_grace: bool + ignore_dynamic_grace: bool concurrency: int | None select: tuple[str, ...] exclude: tuple[str, ...] @@ -6638,7 +6638,7 @@ class BuildCommandRequest(NativeRecord): allow_retention_decrease: bool = False, allow_missing_migration_origin: bool = False, allow_snapshot_schema_change: bool = False, - ignore_maintained_grace: bool = False, + ignore_dynamic_grace: bool = False, concurrency: int | None = None, select: tuple[str, ...] = (), exclude: tuple[str, ...] = (), @@ -12018,7 +12018,7 @@ class TargetConfig(NativeRecord): catalog_connection: str | None ignore_audit_failures: bool | None plan_from: str | None - pause_maintained: bool | None + pause_dynamic: bool | None def __init__( self, connection: dict[str, object] = ..., @@ -12045,7 +12045,7 @@ class TargetConfig(NativeRecord): catalog_connection: str | None = ..., ignore_audit_failures: bool | None = ..., plan_from: str | None = ..., - pause_maintained: bool | None = ..., + pause_dynamic: bool | None = ..., ) -> None: ... class LocalTargetConfig(NativeRecord): @@ -12075,7 +12075,7 @@ class LocalTargetConfig(NativeRecord): catalog_connection: str | None ignore_audit_failures: bool | None plan_from: str | None - pause_maintained: bool | None + pause_dynamic: bool | None def __init__( self, connection: dict[str, object] = ..., @@ -12103,7 +12103,7 @@ class LocalTargetConfig(NativeRecord): catalog_connection: str | None = ..., ignore_audit_failures: bool | None = ..., plan_from: str | None = ..., - pause_maintained: bool | None = ..., + pause_dynamic: bool | None = ..., ) -> None: ... class SettingsConfig(NativeRecord): @@ -13530,9 +13530,9 @@ class PlanAction(StrEnum): LOAD_SEED = "load_seed" SKIP = "skip" CUSTOM = "custom" - CREATE_MAINTAINED = "create_maintained" - ALTER_MAINTAINED = "alter_maintained" - KEEP_MAINTAINED = "keep_maintained" + CREATE_DYNAMIC = "create_dynamic" + ALTER_DYNAMIC = "alter_dynamic" + KEEP_DYNAMIC = "keep_dynamic" class PlanReason(StrEnum): FIRST_RUN = "first_run" @@ -13600,7 +13600,7 @@ class MaterializationType(StrEnum): SNAPSHOT = "snapshot" SEED = "seed" CUSTOM = "custom" - MAINTAINED = "maintained" + DYNAMIC = "dynamic" @classmethod def is_table_backed(cls, *, materialized: object | None) -> bool: ... @@ -14127,8 +14127,8 @@ class ModelPlanEntry(NativeRecord): refresh_mode: str | None refresh_warehouse: str | None refresh_on_build: bool - maintained_readers: tuple[str, ...] - maintained_paused: bool + dynamic_readers: tuple[str, ...] + dynamic_paused: bool def __init__( self, key: CompiledObjectKey, @@ -14223,8 +14223,8 @@ class ModelPlanEntry(NativeRecord): refresh_mode: str | None = ..., refresh_warehouse: str | None = ..., refresh_on_build: bool = ..., - maintained_readers: tuple[str, ...] = ..., - maintained_paused: bool = ..., + dynamic_readers: tuple[str, ...] = ..., + dynamic_paused: bool = ..., ) -> None: ... class CloneSourcePlanEntry(NativeRecord): diff --git a/src/sqlbuild/adapter/contract/classes/strict_adapter.py b/src/sqlbuild/adapter/contract/classes/strict_adapter.py index 5af764ab19..89c03cf168 100644 --- a/src/sqlbuild/adapter/contract/classes/strict_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/strict_adapter.py @@ -31,10 +31,10 @@ TemporaryFunction, ) from sqlbuild.adapter.contract.types import ( + DynamicListing, + DynamicOperation, FrameworkType, LoaderLogicalType, - MaintainedListing, - MaintainedOperation, PromotionStrategy, StatementSizeLimit, TablePromotionMode, @@ -329,47 +329,47 @@ def session_warehouse(self, *, connection: Any) -> str | None: del connection return None - def render_create_maintained( + def render_create_dynamic( self, *, destination: str, sql: str, target_lag: str, refresh_mode: str, warehouse: str ) -> tuple[str, ...]: - """Optional: SQL that creates or replaces a maintained model; unsupported by default.""" + """Optional: SQL that creates or replaces a dynamic model; unsupported by default.""" del sql, target_lag, refresh_mode, warehouse - raise self._maintained_unsupported(destination) + raise self._dynamic_unsupported(destination) - def render_alter_maintained( + def render_alter_dynamic( self, *, destination: str, target_lag: str, warehouse: str | None ) -> tuple[str, ...]: - """Optional: SQL that changes a maintained model's refresh settings in place.""" + """Optional: SQL that changes a dynamic model's refresh settings in place.""" del target_lag, warehouse - raise self._maintained_unsupported(destination) + raise self._dynamic_unsupported(destination) - def render_maintained_operation( - self, *, destination: str, operation: MaintainedOperation + def render_dynamic_operation( + self, *, destination: str, operation: DynamicOperation ) -> tuple[str, ...]: - """Optional: SQL that suspends, resumes or refreshes a maintained model.""" + """Optional: SQL that suspends, resumes or refreshes a dynamic model.""" del operation - raise self._maintained_unsupported(destination) + raise self._dynamic_unsupported(destination) def render_track_changes(self, *, destination: str) -> tuple[str, ...]: - """Optional: SQL that turns on change tracking for a table a maintained model reads.""" + """Optional: SQL that turns on change tracking for a table a dynamic model reads.""" del destination return () - def _maintained_unsupported(self, destination: str) -> AdapterUserError: + def _dynamic_unsupported(self, destination: str) -> AdapterUserError: return AdapterUserError( - f"the {self.adapter_name} adapter cannot build maintained model '{destination}'", + f"the {self.adapter_name} adapter cannot build dynamic model '{destination}'", code="A716", - help="materialized maintained needs an adapter whose warehouse refreshes it", + help="materialized dynamic needs an adapter whose warehouse refreshes it", ) - def maintained_listing( + def dynamic_listing( self, *, connection: Any, database: str | None, schema: str | None, name: str - ) -> MaintainedListing | None: - """Optional: how the warehouse lists a maintained model, or None when it lists none.""" + ) -> DynamicListing | None: + """Optional: how the warehouse lists a dynamic model, or None when it lists none.""" del connection, database, schema, name return None diff --git a/src/sqlbuild/adapter/contract/types.py b/src/sqlbuild/adapter/contract/types.py index cf88c3cb4e..80d52e02f0 100644 --- a/src/sqlbuild/adapter/contract/types.py +++ b/src/sqlbuild/adapter/contract/types.py @@ -82,8 +82,8 @@ class RetentionChangePhase(StrEnum): FINALIZE = "finalize" -class MaintainedOperation(StrEnum): - """What a build does to a maintained model's scheduled refreshes.""" +class DynamicOperation(StrEnum): + """What a build does to a dynamic model's scheduled refreshes.""" SUSPEND = "suspend" RESUME = "resume" @@ -174,8 +174,8 @@ class HistoricalSnapshotInsertStyle(StrEnum): TSQL = "tsql" -class MaintainedListing(NamedTuple): - """A maintained model as the warehouse lists it: lag, refresh warehouse and schedule state.""" +class DynamicListing(NamedTuple): + """A dynamic model as the warehouse lists it: lag, refresh warehouse and schedule state.""" target_lag: str warehouse: str diff --git a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py index 62c48475c8..6d62b24af0 100644 --- a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py +++ b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py @@ -95,12 +95,12 @@ from sqlbuild.adapter.contract.types import ( BuiltinAdapter, CursorKind, + DynamicListing, + DynamicOperation, FrameworkType, HistoricalSnapshotCloseStyle, HistoricalSnapshotInsertStyle, LoaderLogicalType, - MaintainedListing, - MaintainedOperation, MigrationTransfer, PromotionStrategy, RelationReadStatus, @@ -3674,14 +3674,14 @@ def column_rename_unavailable_reason(self, *, connection: Any, destination: str) return None def session_warehouse(self, *, connection: Any) -> str | None: - """The session's warehouse, which refreshes a maintained model that names none.""" + """The session's warehouse, which refreshes a dynamic model that names none.""" rows: tuple[tuple[object, ...], ...] = self.query( connection=connection, sql="SELECT CURRENT_WAREHOUSE()", limit=None ).rows return None if not rows or rows[0][0] is None else str(rows[0][0]) - def render_create_maintained( + def render_create_dynamic( self, *, destination: str, sql: str, target_lag: str, refresh_mode: str, warehouse: str ) -> tuple[str, ...]: """A dynamic table with an explicit mode and ON_CREATE, so it is readable at once.""" @@ -3692,7 +3692,7 @@ def render_create_maintained( f"REFRESH_MODE = {refresh_mode.upper()} INITIALIZE = ON_CREATE AS {sql}", ) - def render_alter_maintained( + def render_alter_dynamic( self, *, destination: str, target_lag: str, warehouse: str | None ) -> tuple[str, ...]: """The lag, and a declared warehouse, set in place.""" @@ -3705,8 +3705,8 @@ def render_alter_maintained( ) return tuple(statements) - def render_maintained_operation( - self, *, destination: str, operation: MaintainedOperation + def render_dynamic_operation( + self, *, destination: str, operation: DynamicOperation ) -> tuple[str, ...]: return (f"ALTER DYNAMIC TABLE {destination} {operation.value.upper()}",) @@ -3720,9 +3720,9 @@ def _warehouse_name(self, warehouse: str) -> str: return warehouse return '"' + warehouse.replace('"', '""') + '"' - def maintained_listing( + def dynamic_listing( self, *, connection: Any, database: str | None, schema: str | None, name: str - ) -> MaintainedListing | None: + ) -> DynamicListing | None: """The dynamic table's lag, warehouse and scheduling state from SHOW DYNAMIC TABLES.""" row: dict[str, object] | None = self._listed_dynamic_table( @@ -3730,7 +3730,7 @@ def maintained_listing( ) if row is None: return None - return MaintainedListing( + return DynamicListing( target_lag=str(row.get("target_lag")), warehouse=str(row.get("warehouse")), scheduling_state=str(row.get("scheduling_state")), diff --git a/src/sqlbuild/executor/build/_helpers/scheduler.py b/src/sqlbuild/executor/build/_helpers/scheduler.py index 527dd72fff..3e03ac59d4 100644 --- a/src/sqlbuild/executor/build/_helpers/scheduler.py +++ b/src/sqlbuild/executor/build/_helpers/scheduler.py @@ -24,8 +24,8 @@ from sqlbuild.executor.build.constants import ( BUILD_CUSTOM_MATERIALIZATION_MISSING_CODE, BUILD_SOURCE_FRESHNESS_BLOCKED_CODE, + DYNAMIC_ACTIONS, INCREMENTAL_ACTIONS, - MAINTAINED_ACTIONS, ) from sqlbuild.executor.build.models import ( FunctionExecutionResult, @@ -36,8 +36,8 @@ from sqlbuild.executor.load.models import LoadExecutionResult from sqlbuild.executor.run.main._execute import ( execute_custom_entry, + execute_dynamic_entry, execute_incremental_entry, - execute_maintained_entry, execute_microbatch_entry, execute_snapshot_entry, execute_table_entry, @@ -165,8 +165,8 @@ def _dispatch_model_entry( context=context, declared_columns=entry.declared_columns, ) - if entry.action in MAINTAINED_ACTIONS: - return execute_maintained_entry(context=context) + if entry.action in DYNAMIC_ACTIONS: + return execute_dynamic_entry(context=context) if entry.action == PlanAction.CREATE_VIEW: return execute_view_entry(context=context) if entry.action == PlanAction.SNAPSHOT: diff --git a/src/sqlbuild/executor/build/classes/build_scheduler.py b/src/sqlbuild/executor/build/classes/build_scheduler.py index b8c628ecfd..395f78de8b 100644 --- a/src/sqlbuild/executor/build/classes/build_scheduler.py +++ b/src/sqlbuild/executor/build/classes/build_scheduler.py @@ -1256,7 +1256,7 @@ def _execute_model_node( watermark_resolver=self._watermark_resolver, bound_views=bound_views, fingerprint_writes=self._fingerprint_writes, - maintained_hold=self._runtime.maintained_hold, + dynamic_hold=self._runtime.dynamic_hold, ), promotion_mode=self._promotion_mode, snapshots=self._snapshots, diff --git a/src/sqlbuild/executor/build/constants.py b/src/sqlbuild/executor/build/constants.py index f4d7636187..ef38d54f7a 100644 --- a/src/sqlbuild/executor/build/constants.py +++ b/src/sqlbuild/executor/build/constants.py @@ -11,11 +11,11 @@ PlanAction.INCREMENTAL_MERGE, } ) -MAINTAINED_ACTIONS: frozenset[PlanAction] = frozenset( +DYNAMIC_ACTIONS: frozenset[PlanAction] = frozenset( { - PlanAction.CREATE_MAINTAINED, - PlanAction.ALTER_MAINTAINED, - PlanAction.KEEP_MAINTAINED, + PlanAction.CREATE_DYNAMIC, + PlanAction.ALTER_DYNAMIC, + PlanAction.KEEP_DYNAMIC, } ) diff --git a/src/sqlbuild/executor/build/main/_execute.py b/src/sqlbuild/executor/build/main/_execute.py index 1d48b1c2f0..b0191f6d04 100644 --- a/src/sqlbuild/executor/build/main/_execute.py +++ b/src/sqlbuild/executor/build/main/_execute.py @@ -35,11 +35,11 @@ SeedExecutionResult, ) from sqlbuild.executor.build.types import BuildStatus -from sqlbuild.executor.load.models import LoadExecutionResult -from sqlbuild.executor.maintained_pauses.classes.maintained_hold import MaintainedHold -from sqlbuild.executor.maintained_pauses.main.hold_maintained_models import ( - hold_maintained_models, +from sqlbuild.executor.dynamic_pauses.classes.dynamic_hold import DynamicHold +from sqlbuild.executor.dynamic_pauses.main.hold_dynamic_models import ( + hold_dynamic_models, ) +from sqlbuild.executor.load.models import LoadExecutionResult from sqlbuild.executor.migrations.main._apply import apply_model_migrations from sqlbuild.executor.migrations.main._apply_columns import apply_column_migrations from sqlbuild.executor.run.models import ModelExecutionResult @@ -59,9 +59,9 @@ def execute_build_plan( initial_state: BuildInitialState | None = None, schema_prepared: bool = False, ) -> BuildExecutionResult: - """Execute a build plan with the DAG scheduler, holding the maintained models it touches.""" + """Execute a build plan with the DAG scheduler, holding the dynamic models it touches.""" - hold: MaintainedHold | None = hold_maintained_models( + hold: DynamicHold | None = hold_dynamic_models( plan=plan, adapter=adapter, runtime=runtime, connection_config=connection_config ) held: AbstractContextManager[object] = ( @@ -79,7 +79,7 @@ def execute_build_plan( connection_config=connection_config, connections=connections, scheduler_connection=scheduler_connection, - runtime=replace(runtime, maintained_hold=hold), + runtime=replace(runtime, dynamic_hold=hold), callbacks=callbacks, customizations=customizations, initial_state=initial_state, diff --git a/src/sqlbuild/executor/build/models.py b/src/sqlbuild/executor/build/models.py index d0c19dfe0a..d245ce43a3 100644 --- a/src/sqlbuild/executor/build/models.py +++ b/src/sqlbuild/executor/build/models.py @@ -27,8 +27,8 @@ from sqlbuild.executor.auditing.models import AuditExecutionResult from sqlbuild.executor.build.types import BeforeNodeWriteCallback, BuildStatus from sqlbuild.executor.custom.models import MaterializationResult +from sqlbuild.executor.dynamic_pauses.classes.dynamic_hold import DynamicHold from sqlbuild.executor.load.models import LoadExecutionResult -from sqlbuild.executor.maintained_pauses.classes.maintained_hold import MaintainedHold from sqlbuild.executor.python_nodes.types import PythonIdentityRecorder from sqlbuild.executor.run.models import MicrobatchBatchProgress, ModelExecutionResult from sqlbuild.executor.scheduling.types import ExecutionStatus @@ -102,11 +102,11 @@ class BuildRuntimeParams: microbatch_lease_check: Callable[[], None] | None = None project_relations: Mapping[SqlResourceRef, str] | None = None test_timeouts: RunItemTimeouts | None = None - pause_maintained: bool = False - maintained_heartbeat_grace_seconds: int = 1800 - allow_ignore_maintained_grace: bool = False - ignore_maintained_grace: bool = False - maintained_hold: MaintainedHold | None = None + pause_dynamic: bool = False + dynamic_heartbeat_grace_seconds: int = 1800 + allow_ignore_dynamic_grace: bool = False + ignore_dynamic_grace: bool = False + dynamic_hold: DynamicHold | None = None @dataclass(frozen=True) diff --git a/src/sqlbuild/executor/dynamic_pauses/__init__.py b/src/sqlbuild/executor/dynamic_pauses/__init__.py new file mode 100644 index 0000000000..22d49862b2 --- /dev/null +++ b/src/sqlbuild/executor/dynamic_pauses/__init__.py @@ -0,0 +1 @@ +"""Dynamic models a build holds paused, and the append-only lease that records it.""" diff --git a/src/sqlbuild/executor/dynamic_pauses/_helpers/__init__.py b/src/sqlbuild/executor/dynamic_pauses/_helpers/__init__.py new file mode 100644 index 0000000000..b14a823fbc --- /dev/null +++ b/src/sqlbuild/executor/dynamic_pauses/_helpers/__init__.py @@ -0,0 +1 @@ +"""Dynamic pause helpers.""" diff --git a/src/sqlbuild/executor/maintained_pauses/_helpers/scope.py b/src/sqlbuild/executor/dynamic_pauses/_helpers/scope.py similarity index 76% rename from src/sqlbuild/executor/maintained_pauses/_helpers/scope.py rename to src/sqlbuild/executor/dynamic_pauses/_helpers/scope.py index eff45d123d..6cafffdb03 100644 --- a/src/sqlbuild/executor/maintained_pauses/_helpers/scope.py +++ b/src/sqlbuild/executor/dynamic_pauses/_helpers/scope.py @@ -1,4 +1,4 @@ -"""Which maintained models a build holds, upstream first, and which ones it refreshes by hand.""" +"""Which dynamic models a build holds, upstream first, and which ones it refreshes by hand.""" from __future__ import annotations @@ -12,33 +12,33 @@ ) _UNWRITTEN: frozenset[PlanAction] = frozenset( - {PlanAction.SKIP, PlanAction.KEEP_MAINTAINED, PlanAction.ALTER_MAINTAINED} + {PlanAction.SKIP, PlanAction.KEEP_DYNAMIC, PlanAction.ALTER_DYNAMIC} ) -def held_names(*, plan: PlanOutput, pause_maintained: bool) -> tuple[str, ...]: - """Maintained models built, or reading what is written (every planned one when paused).""" +def held_names(*, plan: PlanOutput, pause_dynamic: bool) -> tuple[str, ...]: + """Dynamic models built, or reading what is written (every planned one when paused).""" names: set[str] = set() entry: ModelPlanEntry for entry in plan.model_entries: - maintained: bool = entry.materialization_type == MaterializationType.MAINTAINED - if maintained and (pause_maintained or entry.action not in _UNWRITTEN): + dynamic: bool = entry.materialization_type == MaterializationType.DYNAMIC + if dynamic and (pause_dynamic or entry.action not in _UNWRITTEN): names.add(entry.name) if entry.action not in _UNWRITTEN: - names.update(entry.maintained_readers) + names.update(entry.dynamic_readers) return _upstream_first(names=frozenset(names), plan=plan) def refreshed_for_readers(*, plan: PlanOutput) -> frozenset[str]: - """Maintained models that a model built later in this build reads, directly or not.""" + """Dynamic models that a model built later in this build reads, directly or not.""" entries: dict[str, ModelPlanEntry] = {entry.name: entry for entry in plan.model_entries} readers: dict[str, tuple[str, ...]] = _model_edges(edges=plan.downstream_deps) found: set[str] = set() name: str for name, entry in entries.items(): - if entry.materialization_type != MaterializationType.MAINTAINED: + if entry.materialization_type != MaterializationType.DYNAMIC: continue reached: set[str] = _reachable(start=name, edges=readers) if any(_built_table(entry=entries.get(reader)) for reader in reached): @@ -49,7 +49,7 @@ def refreshed_for_readers(*, plan: PlanOutput) -> frozenset[str]: def _built_table(*, entry: ModelPlanEntry | None) -> bool: return ( entry is not None - and entry.materialization_type != MaterializationType.MAINTAINED + and entry.materialization_type != MaterializationType.DYNAMIC and entry.action != PlanAction.SKIP ) diff --git a/src/sqlbuild/executor/maintained_pauses/_helpers/sql.py b/src/sqlbuild/executor/dynamic_pauses/_helpers/sql.py similarity index 78% rename from src/sqlbuild/executor/maintained_pauses/_helpers/sql.py rename to src/sqlbuild/executor/dynamic_pauses/_helpers/sql.py index 1a83fb5548..1870e75994 100644 --- a/src/sqlbuild/executor/maintained_pauses/_helpers/sql.py +++ b/src/sqlbuild/executor/dynamic_pauses/_helpers/sql.py @@ -1,4 +1,4 @@ -"""Portable SQL for the append-only maintained pause table, timed by the warehouse clock.""" +"""Portable SQL for the append-only dynamic pause table, timed by the warehouse clock.""" from __future__ import annotations @@ -8,14 +8,14 @@ from typing import Any from sqlbuild._native import ( - MAINTAINED_PAUSES_TABLE_NAME, + DYNAMIC_PAUSES_TABLE_NAME, CompiledRelationLocation, render_state_sql_literal, render_state_table_create_sql, ) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.errors.contracts.exceptions import ExecutorInputError -from sqlbuild.executor.maintained_pauses.constants import ( +from sqlbuild.executor.dynamic_pauses.constants import ( COLUMN_EVENT_ID, COLUMN_EVENT_TYPE, COLUMN_RECORDED_AT, @@ -25,21 +25,21 @@ COLUMN_RELATION_SCHEMA, COLUMN_RUN_ID, COLUMN_SCHEMA_VERSION, - MAINTAINED_PAUSE_COLUMN_TYPES, - MAINTAINED_PAUSE_COLUMNS, - MAINTAINED_PAUSE_REQUIRED_COLUMNS, - MAINTAINED_PAUSE_SCHEMA_VERSION, + DYNAMIC_PAUSE_COLUMN_TYPES, + DYNAMIC_PAUSE_COLUMNS, + DYNAMIC_PAUSE_REQUIRED_COLUMNS, + DYNAMIC_PAUSE_SCHEMA_VERSION, ) -from sqlbuild.executor.maintained_pauses.models import MaintainedPauseEvent, OpenHold -from sqlbuild.executor.maintained_pauses.types import MaintainedPauseEventType, StateLocation +from sqlbuild.executor.dynamic_pauses.models import DynamicPauseEvent, OpenHold +from sqlbuild.executor.dynamic_pauses.types import DynamicPauseEventType, StateLocation def pause_table_name(*, adapter: BaseAdapter, location: StateLocation) -> str: qualified: str | None = adapter.render_qualified_name( - database=location[0], schema=location[1], name=MAINTAINED_PAUSES_TABLE_NAME + database=location[0], schema=location[1], name=DYNAMIC_PAUSES_TABLE_NAME ) if qualified is None: - raise ExecutorInputError("the maintained pause table needs a schema") + raise ExecutorInputError("the dynamic pause table needs a schema") return qualified @@ -50,16 +50,16 @@ def setup_statements(*, adapter: BaseAdapter, location: StateLocation) -> tuple[ *adapter.render_create_schema(database=location[0], schema=location[1]), render_state_table_create_sql( qualified_name=pause_table_name(adapter=adapter, location=location), - columns=MAINTAINED_PAUSE_COLUMNS, - column_types=MAINTAINED_PAUSE_COLUMN_TYPES, - required_columns=MAINTAINED_PAUSE_REQUIRED_COLUMNS, + columns=DYNAMIC_PAUSE_COLUMNS, + column_types=DYNAMIC_PAUSE_COLUMN_TYPES, + required_columns=DYNAMIC_PAUSE_REQUIRED_COLUMNS, render_framework_type=adapter.render_framework_type, ), ) def insert_sql( - *, adapter: BaseAdapter, location: StateLocation, events: tuple[MaintainedPauseEvent, ...] + *, adapter: BaseAdapter, location: StateLocation, events: tuple[DynamicPauseEvent, ...] ) -> str: """Append every event in one statement, so a hand-over closes and opens holds together.""" @@ -69,7 +69,7 @@ def insert_sql( ) return ( f"INSERT INTO {pause_table_name(adapter=adapter, location=location)} " - f"({', '.join(MAINTAINED_PAUSE_COLUMNS)}) {rows}" + f"({', '.join(DYNAMIC_PAUSE_COLUMNS)}) {rows}" ) @@ -89,9 +89,9 @@ def open_holds_sql(*, adapter: BaseAdapter, location: StateLocation) -> str: f"FROM {table} p JOIN (SELECT {COLUMN_RUN_ID}, MAX({COLUMN_RECORDED_AT}) AS last_seen " f"FROM {table} WHERE {COLUMN_RECORDED_BY} = {COLUMN_RUN_ID} GROUP BY {COLUMN_RUN_ID}) h " f"ON h.{COLUMN_RUN_ID} = p.{COLUMN_RUN_ID} " - f"WHERE p.{COLUMN_EVENT_TYPE} = '{MaintainedPauseEventType.PAUSED.value}' " + f"WHERE p.{COLUMN_EVENT_TYPE} = '{DynamicPauseEventType.PAUSED.value}' " f"AND NOT EXISTS (SELECT 1 FROM {table} r " - f"WHERE r.{COLUMN_EVENT_TYPE} = '{MaintainedPauseEventType.RESUMED.value}' " + f"WHERE r.{COLUMN_EVENT_TYPE} = '{DynamicPauseEventType.RESUMED.value}' " f"AND r.{COLUMN_RUN_ID} = p.{COLUMN_RUN_ID} AND {same_relation})" ) @@ -123,11 +123,11 @@ def _optional(value: object) -> str | None: return None if value is None else str(value) -def _row_literals(*, event: MaintainedPauseEvent) -> str: +def _row_literals(*, event: DynamicPauseEvent) -> str: relation: CompiledRelationLocation | None = event.relation values: dict[str, object | None] = { COLUMN_EVENT_ID: _event_id(event=event), - COLUMN_SCHEMA_VERSION: MAINTAINED_PAUSE_SCHEMA_VERSION, + COLUMN_SCHEMA_VERSION: DYNAMIC_PAUSE_SCHEMA_VERSION, COLUMN_EVENT_TYPE: event.event_type.value, COLUMN_RUN_ID: event.run_id, COLUMN_RELATION_DATABASE: None if relation is None else relation.database, @@ -137,14 +137,14 @@ def _row_literals(*, event: MaintainedPauseEvent) -> str: } return ", ".join( render_state_sql_literal( - value=values[column], declared_type=MAINTAINED_PAUSE_COLUMN_TYPES[column] + value=values[column], declared_type=DYNAMIC_PAUSE_COLUMN_TYPES[column] ) - for column in MAINTAINED_PAUSE_COLUMNS + for column in DYNAMIC_PAUSE_COLUMNS if column != COLUMN_RECORDED_AT ) -def _event_id(*, event: MaintainedPauseEvent) -> str: +def _event_id(*, event: DynamicPauseEvent) -> str: """Content-addressed: one pause and one resume per run and model, one ID per heartbeat.""" relation: CompiledRelationLocation | None = event.relation diff --git a/src/sqlbuild/executor/dynamic_pauses/classes/__init__.py b/src/sqlbuild/executor/dynamic_pauses/classes/__init__.py new file mode 100644 index 0000000000..128dde7b94 --- /dev/null +++ b/src/sqlbuild/executor/dynamic_pauses/classes/__init__.py @@ -0,0 +1 @@ +"""Dynamic pause classes.""" diff --git a/src/sqlbuild/executor/maintained_pauses/classes/maintained_heartbeat.py b/src/sqlbuild/executor/dynamic_pauses/classes/dynamic_heartbeat.py similarity index 95% rename from src/sqlbuild/executor/maintained_pauses/classes/maintained_heartbeat.py rename to src/sqlbuild/executor/dynamic_pauses/classes/dynamic_heartbeat.py index a689c16004..1e9da2861e 100644 --- a/src/sqlbuild/executor/maintained_pauses/classes/maintained_heartbeat.py +++ b/src/sqlbuild/executor/dynamic_pauses/classes/dynamic_heartbeat.py @@ -9,7 +9,7 @@ from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -class MaintainedHeartbeat: +class DynamicHeartbeat: """Write a heartbeat every interval on its own connection, which a busy build never blocks.""" def __init__( @@ -26,7 +26,7 @@ def __init__( self._interval_seconds: float = interval_seconds self._stopped: threading.Event = threading.Event() self._thread: threading.Thread = threading.Thread( - target=self._beat, name="sqlbuild-maintained-heartbeat", daemon=True + target=self._beat, name="sqlbuild-dynamic-heartbeat", daemon=True ) self.failure: Exception | None = None diff --git a/src/sqlbuild/executor/maintained_pauses/classes/maintained_hold.py b/src/sqlbuild/executor/dynamic_pauses/classes/dynamic_hold.py similarity index 81% rename from src/sqlbuild/executor/maintained_pauses/classes/maintained_hold.py rename to src/sqlbuild/executor/dynamic_pauses/classes/dynamic_hold.py index 2c1e1db4c4..9db3018d4c 100644 --- a/src/sqlbuild/executor/maintained_pauses/classes/maintained_hold.py +++ b/src/sqlbuild/executor/dynamic_pauses/classes/dynamic_hold.py @@ -1,4 +1,4 @@ -"""One build's hold on the maintained models it touches: pause, record, beat, then resume.""" +"""One build's hold on the dynamic models it touches: pause, record, beat, then resume.""" from __future__ import annotations @@ -12,28 +12,28 @@ from sqlbuild._native import CompiledRelationLocation from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.exceptions import AdapterUserError -from sqlbuild.adapter.contract.types import MaintainedListing, MaintainedOperation -from sqlbuild.executor.maintained_pauses._helpers.sql import ( +from sqlbuild.adapter.contract.types import DynamicListing, DynamicOperation +from sqlbuild.executor.dynamic_pauses._helpers.sql import ( insert_sql, open_holds, open_holds_sql, setup_statements, ) -from sqlbuild.executor.maintained_pauses.classes.maintained_heartbeat import ( - MaintainedHeartbeat, +from sqlbuild.executor.dynamic_pauses.classes.dynamic_heartbeat import ( + DynamicHeartbeat, ) -from sqlbuild.executor.maintained_pauses.constants import ( +from sqlbuild.executor.dynamic_pauses.constants import ( ACTIVE_STATE, HELD_ELSEWHERE_CODE, SUSPENDED_STATE, ) -from sqlbuild.executor.maintained_pauses.models import MaintainedPauseEvent, OpenHold -from sqlbuild.executor.maintained_pauses.types import MaintainedPauseEventType, StateLocation +from sqlbuild.executor.dynamic_pauses.models import DynamicPauseEvent, OpenHold +from sqlbuild.executor.dynamic_pauses.types import DynamicPauseEventType, StateLocation type _RelationKey = tuple[str | None, str | None, str] -class MaintainedHold: +class DynamicHold: """Pause the held models a build found active, record it, and resume only those at the end.""" def __init__( @@ -44,19 +44,19 @@ def __init__( refreshed: frozenset[str], state_location: StateLocation | None, settings: tuple[str, bool, float, bool], - heartbeat: Callable[[Callable[..., None]], MaintainedHeartbeat], + heartbeat: Callable[[Callable[..., None]], DynamicHeartbeat], ) -> None: self._adapter: BaseAdapter = adapter self._held: dict[str, CompiledRelationLocation] = held self._refreshed: frozenset[str] = refreshed self._state_location: StateLocation | None = state_location self._run_id: str - self._pause_maintained: bool + self._pause_dynamic: bool self._grace_seconds: float self._ignore_grace: bool - self._run_id, self._pause_maintained, self._grace_seconds, self._ignore_grace = settings - self._new_heartbeat: Callable[[Callable[..., None]], MaintainedHeartbeat] = heartbeat - self._heartbeat: MaintainedHeartbeat | None = None + self._run_id, self._pause_dynamic, self._grace_seconds, self._ignore_grace = settings + self._new_heartbeat: Callable[[Callable[..., None]], DynamicHeartbeat] = heartbeat + self._heartbeat: DynamicHeartbeat | None = None self._paused: list[str] = [] self._left_paused: frozenset[str] = frozenset() self._suspended_unheld: list[str] = [] @@ -76,20 +76,20 @@ def holding( try: self.release(connection=connection) except Exception as error: - self._warn(f"could not resume the maintained models this build paused: {error}") + self._warn(f"could not resume the dynamic models this build paused: {error}") raise self.release(connection=connection) def refreshes(self, *, name: str) -> bool: """Whether a kept or altered model is refreshed by hand once its inputs are built.""" - return self._pause_maintained or name in self._refreshed + return self._pause_dynamic or name in self._refreshed def acquire(self, *, connection: Any, on_progress: Callable[[str], None] | None) -> None: """Take over expired holds, refuse live ones, then pause the held models found active.""" self._on_progress = on_progress - if self._pause_maintained: + if self._pause_dynamic: self._suspend_active(connection=connection, names=tuple(self._held)) return location: StateLocation | None = self._state_location @@ -113,16 +113,16 @@ def acquire(self, *, connection: Any, on_progress: Callable[[str], None] | None) if states[name] == SUSPENDED_STATE and name not in self._paused ) for name in sorted(self._left_paused): - self._warn(f"maintained model {name} was paused outside SQLBuild; it stays paused") + self._warn(f"dynamic model {name} was paused outside SQLBuild; it stays paused") window: timedelta = self._timed_record( connection=connection, events=( *( - self._event(event_type=MaintainedPauseEventType.RESUMED, hold=hold) + self._event(event_type=DynamicPauseEventType.RESUMED, hold=hold) for hold in taken ), *( - self._own(event_type=MaintainedPauseEventType.PAUSED, name=name) + self._own(event_type=DynamicPauseEventType.PAUSED, name=name) for name in self._paused ), ), @@ -141,7 +141,7 @@ def after_create(self, *, connection: Any, name: str) -> None: return with self._lock: newly_paused: bool = not ( - self._pause_maintained or name in self._paused or name in self._left_paused + self._pause_dynamic or name in self._paused or name in self._left_paused ) if newly_paused: self._paused.append(name) @@ -149,9 +149,9 @@ def after_create(self, *, connection: Any, name: str) -> None: if newly_paused: window = self._timed_record( connection=connection, - events=(self._own(event_type=MaintainedPauseEventType.PAUSED, name=name),), + events=(self._own(event_type=DynamicPauseEventType.PAUSED, name=name),), ) - self._operate(connection=connection, name=name, operation=MaintainedOperation.SUSPEND) + self._operate(connection=connection, name=name, operation=DynamicOperation.SUSPEND) location: StateLocation | None = self._state_location if newly_paused and location is not None: try: @@ -182,17 +182,14 @@ def release(self, *, connection: Any) -> None: if still_held is not None and _key(self._held[name]) not in still_held: continue try: - self._operate( - connection=connection, name=name, operation=MaintainedOperation.RESUME - ) + self._operate(connection=connection, name=name, operation=DynamicOperation.RESUME) resumed.append(name) except Exception as error: failures.append(error) self._record( connection=connection, events=tuple( - self._own(event_type=MaintainedPauseEventType.RESUMED, name=name) - for name in resumed + self._own(event_type=DynamicPauseEventType.RESUMED, name=name) for name in resumed ), ) if failures: @@ -230,7 +227,7 @@ def _take_over(self, *, connection: Any, location: StateLocation) -> tuple[OpenH self._record( connection=connection, events=tuple( - self._event(event_type=MaintainedPauseEventType.RESUMED, hold=hold) + self._event(event_type=DynamicPauseEventType.RESUMED, hold=hold) for hold in elsewhere ), ) @@ -287,8 +284,7 @@ def _yield_to_rivals( self._record( connection=connection, events=tuple( - self._own(event_type=MaintainedPauseEventType.RESUMED, name=name) - for name in names + self._own(event_type=DynamicPauseEventType.RESUMED, name=name) for name in names ), ) except Exception as error: @@ -310,7 +306,7 @@ def _resume_unheld( try: if self._state(connection=connection, name=name) == SUSPENDED_STATE: self._operate( - connection=connection, name=name, operation=MaintainedOperation.RESUME + connection=connection, name=name, operation=DynamicOperation.RESUME ) except Exception as error: failures.append(error) @@ -320,7 +316,7 @@ def _open_holds(self, *, connection: Any) -> tuple[OpenHold, ...] | None: """Every open hold, or None when the pause table can't say.""" location: StateLocation | None = self._state_location - if location is None or self._pause_maintained: + if location is None or self._pause_dynamic: return None try: rows: tuple[tuple[Any, ...], ...] = tuple( @@ -331,7 +327,7 @@ def _open_holds(self, *, connection: Any) -> tuple[OpenHold, ...] | None: ).rows ) except Exception as error: - self._warn(f"could not read which maintained models this build still holds: {error}") + self._warn(f"could not read which dynamic models this build still holds: {error}") return None return open_holds(rows=rows) @@ -342,7 +338,7 @@ def _resume_found(self, *, connection: Any, relation: CompiledRelationLocation) self._run( connection=connection, destination=self._destination(relation), - operation=MaintainedOperation.RESUME, + operation=DynamicOperation.RESUME, ) def _suspend_active( @@ -359,24 +355,22 @@ def _suspend_active( else self._state(connection=connection, name=name) ) if state == ACTIVE_STATE: - self._operate( - connection=connection, name=name, operation=MaintainedOperation.SUSPEND - ) + self._operate(connection=connection, name=name, operation=DynamicOperation.SUSPEND) def _state(self, *, connection: Any, name: str) -> str | None: return _scheduling_state( adapter=self._adapter, connection=connection, relation=self._held[name] ) - def _operate(self, *, connection: Any, name: str, operation: MaintainedOperation) -> None: + def _operate(self, *, connection: Any, name: str, operation: DynamicOperation) -> None: self._run( connection=connection, destination=self._destination(self._held[name]), operation=operation, ) - def _run(self, *, connection: Any, destination: str, operation: MaintainedOperation) -> None: - for statement in self._adapter.render_maintained_operation( + def _run(self, *, connection: Any, destination: str, operation: DynamicOperation) -> None: + for statement in self._adapter.render_dynamic_operation( destination=destination, operation=operation ): self._adapter.execute(connection=connection, sql=statement) @@ -388,16 +382,14 @@ def _destination(self, location: CompiledRelationLocation) -> str: ) ) - def _timed_record( - self, *, connection: Any, events: tuple[MaintainedPauseEvent, ...] - ) -> timedelta: + def _timed_record(self, *, connection: Any, events: tuple[DynamicPauseEvent, ...]) -> timedelta: """Record, and return how long the insert took: an upper bound on its commit delay.""" started: float = time.monotonic() self._record(connection=connection, events=events) return timedelta(seconds=time.monotonic() - started) - def _record(self, *, connection: Any, events: tuple[MaintainedPauseEvent, ...]) -> None: + def _record(self, *, connection: Any, events: tuple[DynamicPauseEvent, ...]) -> None: location: StateLocation | None = self._state_location if not events or location is None: return @@ -410,8 +402,8 @@ def _write_heartbeat(self, *, connection: Any, sequence: int) -> None: self._record( connection=connection, events=( - MaintainedPauseEvent( - event_type=MaintainedPauseEventType.HEARTBEAT, + DynamicPauseEvent( + event_type=DynamicPauseEventType.HEARTBEAT, run_id=self._run_id, recorded_by=self._run_id, relation=None, @@ -420,18 +412,16 @@ def _write_heartbeat(self, *, connection: Any, sequence: int) -> None: ), ) - def _own(self, *, event_type: MaintainedPauseEventType, name: str) -> MaintainedPauseEvent: - return MaintainedPauseEvent( + def _own(self, *, event_type: DynamicPauseEventType, name: str) -> DynamicPauseEvent: + return DynamicPauseEvent( event_type=event_type, run_id=self._run_id, recorded_by=self._run_id, relation=self._held[name], ) - def _event( - self, *, event_type: MaintainedPauseEventType, hold: OpenHold - ) -> MaintainedPauseEvent: - return MaintainedPauseEvent( + def _event(self, *, event_type: DynamicPauseEventType, hold: OpenHold) -> DynamicPauseEvent: + return DynamicPauseEvent( event_type=event_type, run_id=hold.run_id, recorded_by=self._run_id, @@ -443,13 +433,13 @@ def _report(self, *, holds: tuple[OpenHold, ...], verb: str) -> None: return for hold in holds: self._on_progress( - f"{verb} maintained model {hold.relation.name}, paused by run {hold.run_id} " + f"{verb} dynamic model {hold.relation.name}, paused by run {hold.run_id} " f"whose heartbeat has been silent for {int(hold.idle_seconds)}s" ) def _report_heartbeat_failure(self, *, failure: Exception | None) -> None: if failure is not None: - self._warn(f"a maintained pause heartbeat failed: {failure}") + self._warn(f"a dynamic pause heartbeat failed: {failure}") def _warn(self, message: str) -> None: if self._on_progress is not None: @@ -459,7 +449,7 @@ def _warn(self, message: str) -> None: def _scheduling_state( *, adapter: BaseAdapter, connection: Any, relation: CompiledRelationLocation ) -> str | None: - listing: MaintainedListing | None = adapter.maintained_listing( + listing: DynamicListing | None = adapter.dynamic_listing( connection=connection, database=relation.database, schema=relation.schema, @@ -483,12 +473,12 @@ def _held_elsewhere( else f"; closing this build's own pause also failed: {close_failure}" ) return AdapterUserError( - f"run {first.run_id} holds maintained models this build needs ({names}); " + f"run {first.run_id} holds dynamic models this build needs ({names}); " f"its last heartbeat was {int(first.idle_seconds)}s ago{closing}", code=HELD_ELSEWHERE_CODE, help=( - "wait for that build to finish, or for its hold to pass [maintained] " + "wait for that build to finish, or for its hold to pass [dynamic] " "heartbeat_grace; with allow_ignore_grace = true, " - "sqb build --ignore-maintained-grace takes over at once" + "sqb build --ignore-dynamic-grace takes over at once" ), ) diff --git a/src/sqlbuild/executor/maintained_pauses/constants.py b/src/sqlbuild/executor/dynamic_pauses/constants.py similarity index 75% rename from src/sqlbuild/executor/maintained_pauses/constants.py rename to src/sqlbuild/executor/dynamic_pauses/constants.py index 22fd73f5fa..1abaac9786 100644 --- a/src/sqlbuild/executor/maintained_pauses/constants.py +++ b/src/sqlbuild/executor/dynamic_pauses/constants.py @@ -1,8 +1,8 @@ -"""Maintained pause table columns, timings and error codes.""" +"""Dynamic pause table columns, timings and error codes.""" from sqlbuild._native import StateSqlValueType -MAINTAINED_PAUSE_SCHEMA_VERSION: int = 1 +DYNAMIC_PAUSE_SCHEMA_VERSION: int = 1 HEARTBEAT_SECONDS: float = 60.0 ACTIVE_STATE: str = "ACTIVE" SUSPENDED_STATE: str = "SUSPENDED" @@ -19,7 +19,7 @@ COLUMN_RECORDED_BY: str = "recorded_by" COLUMN_RECORDED_AT: str = "recorded_at" -MAINTAINED_PAUSE_COLUMNS: tuple[str, ...] = ( +DYNAMIC_PAUSE_COLUMNS: tuple[str, ...] = ( COLUMN_EVENT_ID, COLUMN_SCHEMA_VERSION, COLUMN_EVENT_TYPE, @@ -31,7 +31,7 @@ COLUMN_RECORDED_AT, ) -MAINTAINED_PAUSE_REQUIRED_COLUMNS: frozenset[str] = frozenset( +DYNAMIC_PAUSE_REQUIRED_COLUMNS: frozenset[str] = frozenset( { COLUMN_EVENT_ID, COLUMN_SCHEMA_VERSION, @@ -42,8 +42,8 @@ } ) -MAINTAINED_PAUSE_COLUMN_TYPES: dict[str, StateSqlValueType] = { - **{column: StateSqlValueType.STRING for column in MAINTAINED_PAUSE_COLUMNS}, +DYNAMIC_PAUSE_COLUMN_TYPES: dict[str, StateSqlValueType] = { + **{column: StateSqlValueType.STRING for column in DYNAMIC_PAUSE_COLUMNS}, COLUMN_SCHEMA_VERSION: StateSqlValueType.INTEGER, COLUMN_RECORDED_AT: StateSqlValueType.TEXT_TIMESTAMP, } diff --git a/src/sqlbuild/executor/dynamic_pauses/main/__init__.py b/src/sqlbuild/executor/dynamic_pauses/main/__init__.py new file mode 100644 index 0000000000..3fa138e58c --- /dev/null +++ b/src/sqlbuild/executor/dynamic_pauses/main/__init__.py @@ -0,0 +1 @@ +"""Dynamic pause entrypoints.""" diff --git a/src/sqlbuild/executor/maintained_pauses/main/hold_maintained_models.py b/src/sqlbuild/executor/dynamic_pauses/main/hold_dynamic_models.py similarity index 58% rename from src/sqlbuild/executor/maintained_pauses/main/hold_maintained_models.py rename to src/sqlbuild/executor/dynamic_pauses/main/hold_dynamic_models.py index dc8045be4d..f6a3112bc0 100644 --- a/src/sqlbuild/executor/maintained_pauses/main/hold_maintained_models.py +++ b/src/sqlbuild/executor/dynamic_pauses/main/hold_dynamic_models.py @@ -1,4 +1,4 @@ -"""The hold a build keeps on the maintained models it touches, or None when it touches none.""" +"""The hold a build keeps on the dynamic models it touches, or None when it touches none.""" from __future__ import annotations @@ -6,51 +6,51 @@ from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.exceptions import AdapterUserError from sqlbuild.executor.build.models import BuildRuntimeParams -from sqlbuild.executor.maintained_pauses._helpers.scope import held_names, refreshed_for_readers -from sqlbuild.executor.maintained_pauses.classes.maintained_heartbeat import ( - MaintainedHeartbeat, +from sqlbuild.executor.dynamic_pauses._helpers.scope import held_names, refreshed_for_readers +from sqlbuild.executor.dynamic_pauses.classes.dynamic_heartbeat import ( + DynamicHeartbeat, ) -from sqlbuild.executor.maintained_pauses.classes.maintained_hold import MaintainedHold -from sqlbuild.executor.maintained_pauses.constants import ( +from sqlbuild.executor.dynamic_pauses.classes.dynamic_hold import DynamicHold +from sqlbuild.executor.dynamic_pauses.constants import ( HEARTBEAT_SECONDS, IGNORE_GRACE_REFUSED_CODE, ) -from sqlbuild.executor.maintained_pauses.types import StateLocation +from sqlbuild.executor.dynamic_pauses.types import StateLocation -def hold_maintained_models( +def hold_dynamic_models( *, plan: PlanOutput, adapter: BaseAdapter, runtime: BuildRuntimeParams, connection_config: dict[str, object], heartbeat_seconds: float = HEARTBEAT_SECONDS, -) -> MaintainedHold | None: - """The build's hold, unacquired; `--ignore-maintained-grace` needs `allow_ignore_grace`.""" +) -> DynamicHold | None: + """The build's hold, unacquired; `--ignore-dynamic-grace` needs `allow_ignore_grace`.""" - if runtime.ignore_maintained_grace and not runtime.allow_ignore_maintained_grace: + if runtime.ignore_dynamic_grace and not runtime.allow_ignore_dynamic_grace: raise AdapterUserError( - "--ignore-maintained-grace needs [maintained] allow_ignore_grace = true", + "--ignore-dynamic-grace needs [dynamic] allow_ignore_grace = true", code=IGNORE_GRACE_REFUSED_CODE, - help="set allow_ignore_grace = true under [maintained] in sqlbuild_project.toml", + help="set allow_ignore_grace = true under [dynamic] in sqlbuild_project.toml", ) - names: tuple[str, ...] = held_names(plan=plan, pause_maintained=runtime.pause_maintained) + names: tuple[str, ...] = held_names(plan=plan, pause_dynamic=runtime.pause_dynamic) refreshed: frozenset[str] = refreshed_for_readers(plan=plan) if not names and not refreshed: return None held: dict[str, CompiledRelationLocation] = {name: plan.model_locations[name] for name in names} - return MaintainedHold( + return DynamicHold( adapter=adapter, held=held, refreshed=refreshed, state_location=_state_location(runtime=runtime, held=held) if held else None, settings=( runtime.run_id, - runtime.pause_maintained, - float(runtime.maintained_heartbeat_grace_seconds), - runtime.ignore_maintained_grace, + runtime.pause_dynamic, + float(runtime.dynamic_heartbeat_grace_seconds), + runtime.ignore_dynamic_grace, ), - heartbeat=lambda write: MaintainedHeartbeat( + heartbeat=lambda write: DynamicHeartbeat( adapter=adapter, connection_config=connection_config, write=write, diff --git a/src/sqlbuild/executor/maintained_pauses/models.py b/src/sqlbuild/executor/dynamic_pauses/models.py similarity index 76% rename from src/sqlbuild/executor/maintained_pauses/models.py rename to src/sqlbuild/executor/dynamic_pauses/models.py index f76cca16b9..62f103abc2 100644 --- a/src/sqlbuild/executor/maintained_pauses/models.py +++ b/src/sqlbuild/executor/dynamic_pauses/models.py @@ -1,4 +1,4 @@ -"""Records of the maintained pause lease.""" +"""Records of the dynamic pause lease.""" from __future__ import annotations @@ -6,14 +6,14 @@ from datetime import datetime from sqlbuild._native import CompiledRelationLocation -from sqlbuild.executor.maintained_pauses.types import MaintainedPauseEventType +from sqlbuild.executor.dynamic_pauses.types import DynamicPauseEventType @dataclass(frozen=True) -class MaintainedPauseEvent: +class DynamicPauseEvent: """One row to append: a pause, a resume, or a heartbeat of the run holding models.""" - event_type: MaintainedPauseEventType + event_type: DynamicPauseEventType run_id: str recorded_by: str relation: CompiledRelationLocation | None diff --git a/src/sqlbuild/executor/maintained_pauses/types.py b/src/sqlbuild/executor/dynamic_pauses/types.py similarity index 75% rename from src/sqlbuild/executor/maintained_pauses/types.py rename to src/sqlbuild/executor/dynamic_pauses/types.py index c199bdd358..11f347b513 100644 --- a/src/sqlbuild/executor/maintained_pauses/types.py +++ b/src/sqlbuild/executor/dynamic_pauses/types.py @@ -1,4 +1,4 @@ -"""Maintained pause event types.""" +"""Dynamic pause event types.""" from __future__ import annotations @@ -7,7 +7,7 @@ type StateLocation = tuple[str | None, str] -class MaintainedPauseEventType(StrEnum): +class DynamicPauseEventType(StrEnum): """What one row of the pause table records.""" PAUSED = "paused" diff --git a/src/sqlbuild/executor/maintained_pauses/__init__.py b/src/sqlbuild/executor/maintained_pauses/__init__.py deleted file mode 100644 index 9aae34706c..0000000000 --- a/src/sqlbuild/executor/maintained_pauses/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Maintained models a build holds paused, and the append-only lease that records it.""" diff --git a/src/sqlbuild/executor/maintained_pauses/_helpers/__init__.py b/src/sqlbuild/executor/maintained_pauses/_helpers/__init__.py deleted file mode 100644 index 928816ab73..0000000000 --- a/src/sqlbuild/executor/maintained_pauses/_helpers/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Maintained pause helpers.""" diff --git a/src/sqlbuild/executor/maintained_pauses/classes/__init__.py b/src/sqlbuild/executor/maintained_pauses/classes/__init__.py deleted file mode 100644 index 4dbf76c0aa..0000000000 --- a/src/sqlbuild/executor/maintained_pauses/classes/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Maintained pause classes.""" diff --git a/src/sqlbuild/executor/maintained_pauses/main/__init__.py b/src/sqlbuild/executor/maintained_pauses/main/__init__.py deleted file mode 100644 index 36ef4cff68..0000000000 --- a/src/sqlbuild/executor/maintained_pauses/main/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Maintained pause entrypoints.""" diff --git a/src/sqlbuild/executor/run/_helpers/execution/table_config.py b/src/sqlbuild/executor/run/_helpers/execution/table_config.py index ceeee3e044..cd4c5ede42 100644 --- a/src/sqlbuild/executor/run/_helpers/execution/table_config.py +++ b/src/sqlbuild/executor/run/_helpers/execution/table_config.py @@ -21,7 +21,7 @@ def model_table_config(entry: ModelPlanEntry) -> dict[str, object]: return { "table_type": entry.table_type, - "change_tracking": bool(entry.maintained_readers), + "change_tracking": bool(entry.dynamic_readers), "physical_storage": entry.physical_storage, **model_output_config(entry), } diff --git a/src/sqlbuild/executor/run/_helpers/materializations/maintained.py b/src/sqlbuild/executor/run/_helpers/materializations/dynamic.py similarity index 84% rename from src/sqlbuild/executor/run/_helpers/materializations/maintained.py rename to src/sqlbuild/executor/run/_helpers/materializations/dynamic.py index 55821b6928..7d104dcbe9 100644 --- a/src/sqlbuild/executor/run/_helpers/materializations/maintained.py +++ b/src/sqlbuild/executor/run/_helpers/materializations/dynamic.py @@ -1,4 +1,4 @@ -"""Maintained model lifecycle: the warehouse refreshes the model; a build creates or alters it.""" +"""Dynamic model lifecycle: the warehouse refreshes the model; a build creates or alters it.""" from __future__ import annotations @@ -16,12 +16,12 @@ from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.exceptions import AdapterUserError -from sqlbuild.adapter.contract.types import MaintainedListing, MaintainedOperation, RelationType +from sqlbuild.adapter.contract.types import DynamicListing, DynamicOperation, RelationType from sqlbuild.adapter.relations.main.cached_relation_type import cached_relation_type from sqlbuild.adapter.relations.main.resolve_qualified_name_parts import ( resolve_qualified_name_parts, ) -from sqlbuild.executor.maintained_pauses.classes.maintained_hold import MaintainedHold +from sqlbuild.executor.dynamic_pauses.classes.dynamic_hold import DynamicHold from sqlbuild.executor.run._helpers.materializations.view import execute_in_place_entry from sqlbuild.executor.run.constants import REPLACED_RELATION_SUFFIX, RUN_STAGING_FAILED_CODE from sqlbuild.executor.run.models import ModelExecutionResult, ModelMaterializationContext @@ -29,22 +29,22 @@ _NO_WAREHOUSE_CODE: str = "A717" -def execute_maintained_entry(*, context: ModelMaterializationContext) -> ModelExecutionResult: - """Create, alter or keep one maintained model, refresh a kept one when asked, then audit it.""" +def execute_dynamic_entry(*, context: ModelMaterializationContext) -> ModelExecutionResult: + """Create, alter or keep one dynamic model, refresh a kept one when asked, then audit it.""" def materialize(*, target_qualified: str, statement_recorder: StatementRecorder) -> None: entry: ModelPlanEntry = context.entry - hold: MaintainedHold | None = context.maintained_hold - if entry.action == PlanAction.CREATE_MAINTAINED: + hold: DynamicHold | None = context.dynamic_hold + if entry.action == PlanAction.CREATE_DYNAMIC: _create(context=context, destination=target_qualified, recorder=statement_recorder) if hold is not None: hold.after_create(connection=context.connection, name=entry.name) return - if entry.action == PlanAction.ALTER_MAINTAINED: + if entry.action == PlanAction.ALTER_DYNAMIC: _run( adapter=context.adapter, connection=context.connection, - statements=context.adapter.render_alter_maintained( + statements=context.adapter.render_alter_dynamic( destination=target_qualified, target_lag=str(entry.target_lag), warehouse=entry.refresh_warehouse, @@ -55,8 +55,8 @@ def materialize(*, target_qualified: str, statement_recorder: StatementRecorder) _run( adapter=context.adapter, connection=context.connection, - statements=context.adapter.render_maintained_operation( - destination=target_qualified, operation=MaintainedOperation.REFRESH + statements=context.adapter.render_dynamic_operation( + destination=target_qualified, operation=DynamicOperation.REFRESH ), recorder=statement_recorder, ) @@ -65,7 +65,7 @@ def materialize(*, target_qualified: str, statement_recorder: StatementRecorder) context=context, materialize=materialize, action_name=str(context.entry.action), - built="the maintained model's refresh", + built="the dynamic model's refresh", ) @@ -82,11 +82,11 @@ def _create( ) if warehouse is None: raise AdapterUserError( - f"maintained model '{entry.name}' has no warehouse to refresh it", + f"dynamic model '{entry.name}' has no warehouse to refresh it", code=_NO_WAREHOUSE_CODE, help="set refresh_warehouse on the model, or a warehouse on the target's connection", ) - statements: tuple[str, ...] = adapter.render_create_maintained( + statements: tuple[str, ...] = adapter.render_create_dynamic( destination=destination, sql=entry.resolved_sql, target_lag=str(entry.target_lag), @@ -163,7 +163,7 @@ def _replace_former( def _former_relation( *, adapter: BaseAdapter, connection: Any, entry: ModelPlanEntry ) -> RelationType | None: - """The type of a relation at the destination the warehouse does not list as maintained.""" + """The type of a relation at the destination the warehouse does not list as dynamic.""" location: CompiledRelationLocation = entry.destination existing: RelationType | None = cached_relation_type( @@ -175,7 +175,7 @@ def _former_relation( ) if existing is None: return None - listed: MaintainedListing | None = adapter.maintained_listing( + listed: DynamicListing | None = adapter.dynamic_listing( connection=connection, database=location.database, schema=location.schema, diff --git a/src/sqlbuild/executor/run/_helpers/materializations/microbatch.py b/src/sqlbuild/executor/run/_helpers/materializations/microbatch.py index d3389b7b36..d9886af16f 100644 --- a/src/sqlbuild/executor/run/_helpers/materializations/microbatch.py +++ b/src/sqlbuild/executor/run/_helpers/materializations/microbatch.py @@ -3057,7 +3057,7 @@ def _apply_microbatch_dml( config={ "table_type": context.entry.table_type, "physical_storage": context.entry.physical_storage, - "change_tracking": bool(context.entry.maintained_readers), + "change_tracking": bool(context.entry.dynamic_readers), }, statement_recorder=state.statement_recorder, ) diff --git a/src/sqlbuild/executor/run/_helpers/materializations/snapshot.py b/src/sqlbuild/executor/run/_helpers/materializations/snapshot.py index 96f46bf335..4243567e3f 100644 --- a/src/sqlbuild/executor/run/_helpers/materializations/snapshot.py +++ b/src/sqlbuild/executor/run/_helpers/materializations/snapshot.py @@ -800,7 +800,7 @@ def _create_initial_snapshot_target( valid_to_column=valid_to_column, initial_valid_from=entry.initial_valid_from, ) - if entry.maintained_readers: + if entry.dynamic_readers: statements = (*statements, *adapter.render_track_changes(destination=target_qualified)) statement_recorder.record_many(statements) statement: str diff --git a/src/sqlbuild/executor/run/main/_execute.py b/src/sqlbuild/executor/run/main/_execute.py index 1d8584e21b..fe13cbb17c 100644 --- a/src/sqlbuild/executor/run/main/_execute.py +++ b/src/sqlbuild/executor/run/main/_execute.py @@ -41,12 +41,12 @@ from sqlbuild.executor.run._helpers.materializations.custom import ( execute_custom_entry as execute_custom_entry, ) +from sqlbuild.executor.run._helpers.materializations.dynamic import ( + execute_dynamic_entry as execute_dynamic_entry, +) from sqlbuild.executor.run._helpers.materializations.incremental import ( execute_incremental_entry as execute_incremental_entry, ) -from sqlbuild.executor.run._helpers.materializations.maintained import ( - execute_maintained_entry as execute_maintained_entry, -) from sqlbuild.executor.run._helpers.materializations.microbatch import ( execute_microbatch_entry as execute_microbatch_entry, ) @@ -249,7 +249,7 @@ def _staged_lifecycle( table_config={ "table_type": entry.table_type, "physical_storage": entry.physical_storage, - "change_tracking": bool(entry.maintained_readers), + "change_tracking": bool(entry.dynamic_readers), }, statement_recorder=statement_recorder, ) diff --git a/src/sqlbuild/executor/run/models.py b/src/sqlbuild/executor/run/models.py index 84bec90f9f..fc47787c66 100644 --- a/src/sqlbuild/executor/run/models.py +++ b/src/sqlbuild/executor/run/models.py @@ -40,7 +40,7 @@ from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.auditing.models import AuditExecutionResult from sqlbuild.executor.custom.models import MaterializationResult -from sqlbuild.executor.maintained_pauses.classes.maintained_hold import MaintainedHold +from sqlbuild.executor.dynamic_pauses.classes.dynamic_hold import DynamicHold from sqlbuild.executor.python_nodes.classes.runtime_relation_guard import RuntimeRelationGuard from sqlbuild.executor.python_nodes.types import PythonIdentityRecorder from sqlbuild.executor.run.classes.fingerprint_writes import FingerprintWrites @@ -334,7 +334,7 @@ class ModelMaterializationContext: enforce_explicit_references: bool = False bound_views: BoundViewGuard = field(default_factory=BoundViewGuard) fingerprint_writes: FingerprintWrites | None = None - maintained_hold: MaintainedHold | None = None + dynamic_hold: DynamicHold | None = None @dataclass(frozen=True, kw_only=True, init=False) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py index 639178b99d..c50594341d 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py @@ -1464,8 +1464,8 @@ class JobGroupCyclesCase: @dataclass(frozen=True) -class MaintainedCompileE2ETestCase: - """A project with maintained models, its exit code and text its compile output contains.""" +class DynamicCompileE2ETestCase: + """A project with dynamic models, its exit code and text its compile output contains.""" description: str adapter: str diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_maintained_models.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dynamic_models.py similarity index 72% rename from tests/e2e/src/sqlbuild/cli/commands/main/compile/test_maintained_models.py rename to tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dynamic_models.py index da2449debf..ca307d8ad0 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_maintained_models.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dynamic_models.py @@ -1,4 +1,4 @@ -"""E2E coverage of maintained models: which adapters and headers compile, and the errors they get.""" +"""E2E coverage of dynamic models: which adapters and headers compile, and the errors they get.""" from __future__ import annotations @@ -8,7 +8,7 @@ import pytest from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import ( - MaintainedCompileE2ETestCase, + DynamicCompileE2ETestCase, ) from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import prepare_inline_project, run_sqb @@ -27,7 +27,7 @@ "MODEL (description 'Orders.', materialized table);\nSELECT 1 AS order_id, 10 AS amount\n" ) _CLEAN_ORDERS: str = ( - "MODEL (description 'Orders with an amount.', materialized maintained);\n" + "MODEL (description 'Orders with an amount.', materialized dynamic);\n" 'SELECT order_id, amount FROM __ref("orders") WHERE amount > 0\n' ) _DAILY_TOTALS_QUERY: str = 'SELECT SUM(amount) AS total FROM __ref("clean_orders")\n' @@ -36,61 +36,61 @@ @pytest.mark.parametrize( "test_case", [ - MaintainedCompileE2ETestCase( + DynamicCompileE2ETestCase( description="a chain refreshed by the lag of its last model compiles on Snowflake", adapter="snowflake", - daily_totals_header="materialized maintained, refresh (lag 1h), refresh_mode full", + daily_totals_header="materialized dynamic, refresh (lag 1h), refresh_mode full", expected_exit_code=0, expected_fragments=("Project compiled 3 models",), ), - MaintainedCompileE2ETestCase( - description="maintained models are refused on DuckDB", + DynamicCompileE2ETestCase( + description="dynamic models are refused on DuckDB", adapter="duckdb", - daily_totals_header="materialized maintained, refresh (lag 1h)", + daily_totals_header="materialized dynamic, refresh (lag 1h)", expected_exit_code=1, expected_fragments=( "[A013]", - "materialized maintained is not supported on the duckdb adapter; " + "materialized dynamic is not supported on the duckdb adapter; " "adapters that support it: snowflake", ), ), - MaintainedCompileE2ETestCase( - description="materialized_view on Snowflake points to maintained", + DynamicCompileE2ETestCase( + description="materialized_view on Snowflake points to dynamic", adapter="snowflake", daily_totals_header="materialized materialized_view", expected_exit_code=1, expected_fragments=( "[A014]", "Snowflake has no materialized_view materialization; use materialized " - "maintained, which builds a dynamic table", + "dynamic, which builds a dynamic table", ), ), - MaintainedCompileE2ETestCase( + DynamicCompileE2ETestCase( description="a lag below Snowflake's minimum", adapter="snowflake", - daily_totals_header="materialized maintained, refresh (lag 30s)", + daily_totals_header="materialized dynamic, refresh (lag 30s)", expected_exit_code=1, expected_fragments=( "[A015]", "refresh lag on Snowflake must be at least 1m; got '30s'", ), ), - MaintainedCompileE2ETestCase( - description="a last maintained model with no refresh would never refresh", + DynamicCompileE2ETestCase( + description="a last dynamic model with no refresh would never refresh", adapter="snowflake", - daily_totals_header="materialized maintained", + daily_totals_header="materialized dynamic", expected_exit_code=1, expected_fragments=( "[A016]", - "maintained model 'daily_totals' has no refresh and no maintained model reads " + "dynamic model 'daily_totals' has no refresh and no dynamic model reads " "it, so Snowflake would never refresh it; add refresh (lag 1h)", ), ), ], ids=lambda case: case.description, ) -def test_given_maintained_models_when_compiling_then_adapter_rules_apply( - test_case: MaintainedCompileE2ETestCase, tmp_path: Path +def test_given_dynamic_models_when_compiling_then_adapter_rules_apply( + test_case: DynamicCompileE2ETestCase, tmp_path: Path ) -> None: project_dir: Path = prepare_inline_project( tmp_path=tmp_path, diff --git a/tests/goldens/compiler/failure/target-unknown-key.json b/tests/goldens/compiler/failure/target-unknown-key.json index 9d222e2426..fc7580ddd4 100644 --- a/tests/goldens/compiler/failure/target-unknown-key.json +++ b/tests/goldens/compiler/failure/target-unknown-key.json @@ -8,7 +8,7 @@ "phase": "compile", "severity": "error", "code": "D001", - "message": "/sqlbuild_project.toml targets.ci contains unknown key(s): warehouse_size. Allowed keys: catalog_connection, clone, compile_cache, concurrency, connection, database, default_table_type, defer_clone_from, defer_sources_to, diff, execution_limits, ignore_audit_failures, loader_schema, missing_migration_origin, owns_time_travel_retention_namespace, pause_maintained, plan_from, pools, recall, schema, table_type_downgrade, time_travel_retention, time_travel_retention_decrease, vars, warehouses", + "message": "/sqlbuild_project.toml targets.ci contains unknown key(s): warehouse_size. Allowed keys: catalog_connection, clone, compile_cache, concurrency, connection, database, default_table_type, defer_clone_from, defer_sources_to, diff, execution_limits, ignore_audit_failures, loader_schema, missing_migration_origin, owns_time_travel_retention_namespace, pause_dynamic, plan_from, pools, recall, schema, table_type_downgrade, time_travel_retention, time_travel_retention_decrease, vars, warehouses", "help": "did you mean 'warehouses'?" } ] diff --git a/tests/integration/src/sqlbuild/compiler/planner/main/_test_types.py b/tests/integration/src/sqlbuild/compiler/planner/main/_test_types.py index 2651952329..2add956198 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/main/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/planner/main/_test_types.py @@ -7,7 +7,7 @@ PlanReason, WarningSeverity, ) -from sqlbuild.adapter.contract.types import MaintainedListing +from sqlbuild.adapter.contract.types import DynamicListing from sqlbuild.compiler.compile.types import FunctionLanguage @@ -209,9 +209,9 @@ class SyncedScanTestCase: @dataclass(frozen=True) -class MaintainedRefreshPlanTestCase: +class DynamicRefreshPlanTestCase: description: str - listed_settings: MaintainedListing | None + listed_settings: DynamicListing | None upstream_query: str expected_upstream_action: PlanAction expected_action: PlanAction @@ -226,5 +226,5 @@ class MaintainedRefreshPlanTestCase: class ChangeTrackingPlanTestCase: description: str select: tuple[str, ...] - expected_maintained_readers: dict[str, tuple[str, ...]] + expected_dynamic_readers: dict[str, tuple[str, ...]] expected_refresh_on_build: dict[str, bool] diff --git a/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py b/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py index ba0c256eab..9a963f36b6 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py @@ -61,7 +61,7 @@ RelationInfo, RelationReadProbe, ) -from sqlbuild.adapter.contract.types import MaintainedListing +from sqlbuild.adapter.contract.types import DynamicListing from sqlbuild.adapter.relations.classes.inspection_catalog import InspectionCatalog from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.types import ( @@ -835,8 +835,8 @@ def planned_entries(result: CompilePipelineResult) -> list[tuple[str, str]]: CLEAN_ORDERS_QUERY: str = "SELECT 1 AS order_id" -_MAINTAINED: dict[str, object] = { - "materialized": "maintained", +_DYNAMIC: dict[str, object] = { + "materialized": "dynamic", "refresh": {"lag": "1h"}, "refresh_warehouse": "transform_wh", } @@ -845,31 +845,31 @@ def planned_entries(result: CompilePipelineResult) -> list[tuple[str, str]]: class ListingDuckDbAdapter(DuckDbAdapter): """DuckDB that lists every dynamic table the same way, as Snowflake's SHOW would.""" - def __init__(self, listed: MaintainedListing | None) -> None: + def __init__(self, listed: DynamicListing | None) -> None: super().__init__() - self.listed: MaintainedListing | None = listed + self.listed: DynamicListing | None = listed self.read_names: list[str] = [] - def maintained_listing( + def dynamic_listing( self, *, connection: Any, database: str | None, schema: str | None, name: str - ) -> MaintainedListing | None: + ) -> DynamicListing | None: del connection, database, schema self.read_names.append(name) return self.listed -def maintained_project_case( +def dynamic_project_case( upstream_query: str, totals_config: dict[str, object] | None = None ) -> BuildExecutionPlanTestCase: - """Clean orders and the daily totals maintained from it, both refreshed within an hour.""" + """Clean orders and dynamic daily totals built from it, both refreshed within an hour.""" return BuildExecutionPlanTestCase( - description="clean orders and the daily totals maintained from it", + description="clean orders and the daily totals built from it as a dynamic model", setup_sql=(), model_locations={"clean_orders": "staging", "daily_totals": "staging"}, model_configs={ - "clean_orders": _MAINTAINED, - "daily_totals": {**_MAINTAINED, **(totals_config or {})}, + "clean_orders": _DYNAMIC, + "daily_totals": {**_DYNAMIC, **(totals_config or {})}, }, model_queries={ "clean_orders": upstream_query, @@ -883,10 +883,10 @@ def maintained_project_case( def change_tracking_project_case() -> BuildExecutionPlanTestCase: - """Daily totals is maintained from orders, and from payments through a view; not customers.""" + """Dynamic daily totals read orders, and payments through a view; not customers.""" return BuildExecutionPlanTestCase( - description="tables a maintained model reads, directly or via a view, and one it does not", + description="tables a dynamic model reads, directly or via a view, and one it does not", setup_sql=(), model_locations={ "orders": "staging", @@ -901,7 +901,7 @@ def change_tracking_project_case() -> BuildExecutionPlanTestCase: "payments": {"materialized": "table"}, "settled_payments": {"materialized": "view"}, "daily_totals": { - "materialized": "maintained", + "materialized": "dynamic", "refresh": {"lag": "1h"}, "refresh_on_build": True, }, diff --git a/tests/integration/src/sqlbuild/compiler/planner/main/test_maintained_refresh_plan.py b/tests/integration/src/sqlbuild/compiler/planner/main/test_dynamic_refresh_plan.py similarity index 71% rename from tests/integration/src/sqlbuild/compiler/planner/main/test_maintained_refresh_plan.py rename to tests/integration/src/sqlbuild/compiler/planner/main/test_dynamic_refresh_plan.py index ac00d92fd7..1ccff4c8e3 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/main/test_maintained_refresh_plan.py +++ b/tests/integration/src/sqlbuild/compiler/planner/main/test_dynamic_refresh_plan.py @@ -1,4 +1,4 @@ -"""A replan of a maintained model reads its live refresh settings and marks rebuilt upstreams.""" +"""A replan of a dynamic model reads its live refresh settings and marks rebuilt upstreams.""" from __future__ import annotations @@ -7,12 +7,12 @@ import pytest from sqlbuild._native import ModelPlanEntry, PlanAction, PlanOutput, PlanReason -from sqlbuild.adapter.contract.types import MaintainedListing +from sqlbuild.adapter.contract.types import DynamicListing from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.executor.run._helpers.reuse.fingerprinting import try_write_fingerprint from tests.integration.src.sqlbuild.compiler.planner.main._test_types import ( ChangeTrackingPlanTestCase, - MaintainedRefreshPlanTestCase, + DynamicRefreshPlanTestCase, ) from tests.integration.src.sqlbuild.compiler.planner.main.helpers import ( CLEAN_ORDERS_QUERY, @@ -20,71 +20,71 @@ build_execution_plan_from_kwargs, build_project_from_test_case, change_tracking_project_case, - maintained_project_case, + dynamic_project_case, ) @pytest.mark.parametrize( "test_case", [ - MaintainedRefreshPlanTestCase( + DynamicRefreshPlanTestCase( description="listed settings that match keep both models", - listed_settings=MaintainedListing("60 minutes", "TRANSFORM_WH", "ACTIVE"), + listed_settings=DynamicListing("60 minutes", "TRANSFORM_WH", "ACTIVE"), upstream_query=CLEAN_ORDERS_QUERY, - expected_upstream_action=PlanAction.KEEP_MAINTAINED, - expected_action=PlanAction.KEEP_MAINTAINED, + expected_upstream_action=PlanAction.KEEP_DYNAMIC, + expected_action=PlanAction.KEEP_DYNAMIC, expected_reason=PlanReason.NO_CHANGE, expected_reinitialised_by=None, expected_read_names=("clean_orders", "daily_totals"), ), - MaintainedRefreshPlanTestCase( + DynamicRefreshPlanTestCase( description="writing out the default refresh_mode keeps the model", - listed_settings=MaintainedListing("60 minutes", "TRANSFORM_WH", "ACTIVE"), + listed_settings=DynamicListing("60 minutes", "TRANSFORM_WH", "ACTIVE"), upstream_query=CLEAN_ORDERS_QUERY, - expected_upstream_action=PlanAction.KEEP_MAINTAINED, - expected_action=PlanAction.KEEP_MAINTAINED, + expected_upstream_action=PlanAction.KEEP_DYNAMIC, + expected_action=PlanAction.KEEP_DYNAMIC, expected_reason=PlanReason.NO_CHANGE, expected_reinitialised_by=None, expected_read_names=("clean_orders", "daily_totals"), totals_config={"refresh_mode": "incremental"}, ), - MaintainedRefreshPlanTestCase( + DynamicRefreshPlanTestCase( description="a kept model listed as suspended is reported as paused", - listed_settings=MaintainedListing("60 minutes", "TRANSFORM_WH", "SUSPENDED"), + listed_settings=DynamicListing("60 minutes", "TRANSFORM_WH", "SUSPENDED"), upstream_query=CLEAN_ORDERS_QUERY, - expected_upstream_action=PlanAction.KEEP_MAINTAINED, - expected_action=PlanAction.KEEP_MAINTAINED, + expected_upstream_action=PlanAction.KEEP_DYNAMIC, + expected_action=PlanAction.KEEP_DYNAMIC, expected_reason=PlanReason.NO_CHANGE, expected_reinitialised_by=None, expected_read_names=("clean_orders", "daily_totals"), expected_paused=True, ), - MaintainedRefreshPlanTestCase( + DynamicRefreshPlanTestCase( description="a lag changed outside the project is altered in place", - listed_settings=MaintainedListing("2 hours", "TRANSFORM_WH", "ACTIVE"), + listed_settings=DynamicListing("2 hours", "TRANSFORM_WH", "ACTIVE"), upstream_query=CLEAN_ORDERS_QUERY, - expected_upstream_action=PlanAction.ALTER_MAINTAINED, - expected_action=PlanAction.ALTER_MAINTAINED, + expected_upstream_action=PlanAction.ALTER_DYNAMIC, + expected_action=PlanAction.ALTER_DYNAMIC, expected_reason=PlanReason.CONFIG_CHANGED, expected_reinitialised_by=None, expected_read_names=("clean_orders", "daily_totals"), ), - MaintainedRefreshPlanTestCase( + DynamicRefreshPlanTestCase( description="a name that holds no dynamic table is created", listed_settings=None, upstream_query=CLEAN_ORDERS_QUERY, - expected_upstream_action=PlanAction.CREATE_MAINTAINED, - expected_action=PlanAction.CREATE_MAINTAINED, + expected_upstream_action=PlanAction.CREATE_DYNAMIC, + expected_action=PlanAction.CREATE_DYNAMIC, expected_reason=PlanReason.FIRST_RUN, expected_reinitialised_by=None, expected_read_names=("clean_orders", "daily_totals"), ), - MaintainedRefreshPlanTestCase( + DynamicRefreshPlanTestCase( description="a recreated upstream marks the kept reader as recomputing in full", - listed_settings=MaintainedListing("1 hour", "TRANSFORM_WH", "ACTIVE"), + listed_settings=DynamicListing("1 hour", "TRANSFORM_WH", "ACTIVE"), upstream_query="SELECT 2 AS order_id", - expected_upstream_action=PlanAction.CREATE_MAINTAINED, - expected_action=PlanAction.KEEP_MAINTAINED, + expected_upstream_action=PlanAction.CREATE_DYNAMIC, + expected_action=PlanAction.KEEP_DYNAMIC, expected_reason=PlanReason.NO_CHANGE, expected_reinitialised_by="clean_orders", expected_read_names=("daily_totals",), @@ -92,14 +92,14 @@ ], ids=lambda case: case.description, ) -def test_given_built_maintained_models_when_replanning_then_live_settings_and_rebuilds_decide( - test_case: MaintainedRefreshPlanTestCase, +def test_given_built_dynamic_models_when_replanning_then_live_settings_and_rebuilds_decide( + test_case: DynamicRefreshPlanTestCase, ) -> None: adapter: ListingDuckDbAdapter = ListingDuckDbAdapter(test_case.listed_settings) connection: Any = adapter.connect({"database": ":memory:"}) connection.execute("CREATE SCHEMA staging") first: PlanOutput = build_execution_plan_from_kwargs( - project=build_project_from_test_case(maintained_project_case(CLEAN_ORDERS_QUERY)), + project=build_project_from_test_case(dynamic_project_case(CLEAN_ORDERS_QUERY)), adapter=adapter, connection=connection, ) @@ -119,7 +119,7 @@ def test_given_built_maintained_models_when_replanning_then_live_settings_and_re plan: PlanOutput = build_execution_plan_from_kwargs( project=build_project_from_test_case( - maintained_project_case(test_case.upstream_query, test_case.totals_config) + dynamic_project_case(test_case.upstream_query, test_case.totals_config) ), adapter=adapter, connection=connection, @@ -131,7 +131,7 @@ def test_given_built_maintained_models_when_replanning_then_live_settings_and_re assert entries["daily_totals"].action == test_case.expected_action assert entries["daily_totals"].reason == test_case.expected_reason assert entries["daily_totals"].reinitialised_by == test_case.expected_reinitialised_by - assert entries["daily_totals"].maintained_paused == test_case.expected_paused + assert entries["daily_totals"].dynamic_paused == test_case.expected_paused assert entries["clean_orders"].reinitialised_by is None assert entries["daily_totals"].target_lag == "'1 hour'" assert entries["daily_totals"].refresh_mode == "incremental" @@ -144,9 +144,9 @@ def test_given_built_maintained_models_when_replanning_then_live_settings_and_re "test_case", [ ChangeTrackingPlanTestCase( - description="a table a maintained model reads, directly or through a view, is tracked", + description="a table a dynamic model reads, directly or through a view, is tracked", select=(), - expected_maintained_readers={ + expected_dynamic_readers={ "orders": ("daily_totals",), "customers": (), "payments": ("daily_totals",), @@ -164,7 +164,7 @@ def test_given_built_maintained_models_when_replanning_then_live_settings_and_re ChangeTrackingPlanTestCase( description="the reader need not be selected for its upstream to be tracked", select=("orders", "customers", "payments"), - expected_maintained_readers={ + expected_dynamic_readers={ "orders": ("daily_totals",), "customers": (), "payments": ("daily_totals",), @@ -174,7 +174,7 @@ def test_given_built_maintained_models_when_replanning_then_live_settings_and_re ], ids=lambda case: case.description, ) -def test_given_table_read_by_maintained_model_when_planning_then_it_names_its_maintained_readers( +def test_given_table_read_by_dynamic_model_when_planning_then_it_names_its_dynamic_readers( test_case: ChangeTrackingPlanTestCase, adapter: DuckDbAdapter, connection: Any ) -> None: plan: PlanOutput = build_execution_plan_from_kwargs( @@ -185,9 +185,9 @@ def test_given_table_read_by_maintained_model_when_planning_then_it_names_its_ma ) readers: dict[str, tuple[str, ...]] = { - entry.name: entry.maintained_readers for entry in plan.model_entries + entry.name: entry.dynamic_readers for entry in plan.model_entries } - assert readers == test_case.expected_maintained_readers + assert readers == test_case.expected_dynamic_readers assert { entry.name: entry.refresh_on_build for entry in plan.model_entries } == test_case.expected_refresh_on_build diff --git a/tests/integration/src/sqlbuild/executor/dynamic_pauses/__init__.py b/tests/integration/src/sqlbuild/executor/dynamic_pauses/__init__.py new file mode 100644 index 0000000000..2097ead2ba --- /dev/null +++ b/tests/integration/src/sqlbuild/executor/dynamic_pauses/__init__.py @@ -0,0 +1 @@ +"""Dynamic model pause lease integration tests.""" diff --git a/tests/integration/src/sqlbuild/executor/maintained_pauses/_test_types.py b/tests/integration/src/sqlbuild/executor/dynamic_pauses/_test_types.py similarity index 94% rename from tests/integration/src/sqlbuild/executor/maintained_pauses/_test_types.py rename to tests/integration/src/sqlbuild/executor/dynamic_pauses/_test_types.py index 19e1bb611e..f6b867c312 100644 --- a/tests/integration/src/sqlbuild/executor/maintained_pauses/_test_types.py +++ b/tests/integration/src/sqlbuild/executor/dynamic_pauses/_test_types.py @@ -2,7 +2,7 @@ from dataclasses import dataclass -from sqlbuild.executor.maintained_pauses.types import MaintainedPauseEventType +from sqlbuild.executor.dynamic_pauses.types import DynamicPauseEventType @dataclass(frozen=True) @@ -41,7 +41,7 @@ class HoldRefusalTestCase: @dataclass(frozen=True) class FirstBuildTestCase: description: str - pause_maintained: bool + pause_dynamic: bool expected_operations: tuple[tuple[str, str], ...] expected_states: dict[str, str] @@ -81,7 +81,7 @@ class HeartbeatTestCase: @dataclass(frozen=True) class CreatedModelHoldTestCase: description: str - refused_events: tuple[MaintainedPauseEventType, ...] + refused_events: tuple[DynamicPauseEventType, ...] rival_closes: tuple[str, ...] expected_release_operations: tuple[tuple[str, str], ...] expected_code: str diff --git a/tests/integration/src/sqlbuild/executor/maintained_pauses/conftest.py b/tests/integration/src/sqlbuild/executor/dynamic_pauses/conftest.py similarity index 89% rename from tests/integration/src/sqlbuild/executor/maintained_pauses/conftest.py rename to tests/integration/src/sqlbuild/executor/dynamic_pauses/conftest.py index 1634ea5edb..a575688845 100644 --- a/tests/integration/src/sqlbuild/executor/maintained_pauses/conftest.py +++ b/tests/integration/src/sqlbuild/executor/dynamic_pauses/conftest.py @@ -8,7 +8,7 @@ import pytest -from tests.integration.src.sqlbuild.executor.maintained_pauses.helpers import ( +from tests.integration.src.sqlbuild.executor.dynamic_pauses.helpers import ( PausingDuckDbAdapter, ) diff --git a/tests/integration/src/sqlbuild/executor/maintained_pauses/helpers.py b/tests/integration/src/sqlbuild/executor/dynamic_pauses/helpers.py similarity index 80% rename from tests/integration/src/sqlbuild/executor/maintained_pauses/helpers.py rename to tests/integration/src/sqlbuild/executor/dynamic_pauses/helpers.py index ff8ef4bf33..f430e65aef 100644 --- a/tests/integration/src/sqlbuild/executor/maintained_pauses/helpers.py +++ b/tests/integration/src/sqlbuild/executor/dynamic_pauses/helpers.py @@ -8,8 +8,8 @@ from sqlbuild._native import PlanOutput from sqlbuild.adapter.contract.types import ( - MaintainedListing, - MaintainedOperation, + DynamicListing, + DynamicOperation, TablePromotionMode, ) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter @@ -19,7 +19,7 @@ BuildExecutionResult, BuildRuntimeParams, ) -from sqlbuild.executor.maintained_pauses.models import MaintainedPauseEvent +from sqlbuild.executor.dynamic_pauses.models import DynamicPauseEvent from tests.integration.src.sqlbuild.compiler.planner.main._test_types import ( BuildExecutionPlanTestCase, ) @@ -30,21 +30,21 @@ type LeaseProject = BuildExecutionPlanTestCase -_MAINTAINED: dict[str, object] = { - "materialized": "maintained", +_DYNAMIC: dict[str, object] = { + "materialized": "dynamic", "refresh": {"lag": "1h"}, "refresh_warehouse": "transform_wh", } _CREATED_STATE: str = "ACTIVE" TOTALS_QUERY: str = "SELECT count(*) AS orders FROM staging.orders" -_STATES: dict[MaintainedOperation, str] = { - MaintainedOperation.SUSPEND: "SUSPENDED", - MaintainedOperation.RESUME: "ACTIVE", +_STATES: dict[DynamicOperation, str] = { + DynamicOperation.SUSPEND: "SUSPENDED", + DynamicOperation.RESUME: "ACTIVE", } class PausingDuckDbAdapter(DuckDbAdapter): - """Builds maintained models as tables and keeps their scheduling state as Snowflake would.""" + """Builds dynamic models as tables and keeps their scheduling state as Snowflake would.""" def __init__(self) -> None: super().__init__() @@ -55,17 +55,17 @@ def session_warehouse(self, *, connection: Any) -> str | None: del connection return "SESSION_WH" - def maintained_listing( + def dynamic_listing( self, *, connection: Any, database: str | None, schema: str | None, name: str - ) -> MaintainedListing | None: + ) -> DynamicListing | None: del connection, database, schema - listings: dict[str, MaintainedListing] = { - listed: MaintainedListing("1 hour", "TRANSFORM_WH", state) + listings: dict[str, DynamicListing] = { + listed: DynamicListing("1 hour", "TRANSFORM_WH", state) for listed, state in self.states.items() } return listings.get(name) - def render_create_maintained( + def render_create_dynamic( self, *, destination: str, sql: str, target_lag: str, refresh_mode: str, warehouse: str ) -> tuple[str, ...]: del target_lag, refresh_mode, warehouse @@ -74,8 +74,8 @@ def render_create_maintained( self.operations.append(("create", name)) return (f"CREATE OR REPLACE TABLE {destination} AS {sql}",) - def render_maintained_operation( - self, *, destination: str, operation: MaintainedOperation + def render_dynamic_operation( + self, *, destination: str, operation: DynamicOperation ) -> tuple[str, ...]: name: str = destination.rsplit(".", 1)[-1] self.states[name] = _STATES.get(operation, self.states[name]) @@ -90,16 +90,16 @@ def lease_project_case( refresh_on_build: bool, totals_query: str = TOTALS_QUERY, ) -> BuildExecutionPlanTestCase: - """Orders, daily totals maintained from them, and any report table located to read those.""" + """Orders, dynamic daily totals built from them, and any report table that reads those.""" - maintained: dict[str, object] = {**_MAINTAINED, "refresh_on_build": refresh_on_build} + dynamic: dict[str, object] = {**_DYNAMIC, "refresh_on_build": refresh_on_build} return BuildExecutionPlanTestCase( - description="orders and the daily totals maintained from them", + description="orders and the daily totals built from them as a dynamic model", setup_sql=(), model_locations={"orders": "staging", "daily_totals": "staging", **report_locations}, model_configs={ "orders": {"materialized": "table"}, - "daily_totals": maintained, + "daily_totals": dynamic, "order_report": {"materialized": "table"}, }, model_queries={ @@ -162,7 +162,7 @@ def pause_events(*, connection: Any) -> list[tuple[str, str, str, str]]: rows: list[tuple[Any, ...]] = connection.execute( "SELECT event_type, run_id, recorded_by, relation_name " - "FROM staging._sqlbuild_maintained_pauses WHERE event_type <> 'heartbeat' " + "FROM staging._sqlbuild_dynamic_pauses WHERE event_type <> 'heartbeat' " "ORDER BY 1, 2, 3, 4" ).fetchall() return [(str(row[0]), str(row[1]), str(row[2]), str(row[3])) for row in rows] @@ -173,11 +173,11 @@ def pause_table_names(*, connection: Any) -> list[tuple[Any, ...]]: return connection.execute( "SELECT table_name FROM information_schema.tables " - "WHERE table_schema = 'staging' AND table_name = '_sqlbuild_maintained_pauses'" + "WHERE table_schema = 'staging' AND table_name = '_sqlbuild_dynamic_pauses'" ).fetchall() -def refuse_insert(*, connection: Any, events: tuple[MaintainedPauseEvent, ...]) -> None: +def refuse_insert(*, connection: Any, events: tuple[DynamicPauseEvent, ...]) -> None: """Stand in for a pause-table insert the warehouse refuses.""" del connection, events diff --git a/tests/integration/src/sqlbuild/executor/maintained_pauses/test_maintained_pause_lease.py b/tests/integration/src/sqlbuild/executor/dynamic_pauses/test_dynamic_pause_lease.py similarity index 87% rename from tests/integration/src/sqlbuild/executor/maintained_pauses/test_maintained_pause_lease.py rename to tests/integration/src/sqlbuild/executor/dynamic_pauses/test_dynamic_pause_lease.py index aa95449702..ebb12aa730 100644 --- a/tests/integration/src/sqlbuild/executor/maintained_pauses/test_maintained_pause_lease.py +++ b/tests/integration/src/sqlbuild/executor/dynamic_pauses/test_dynamic_pause_lease.py @@ -1,4 +1,4 @@ -"""A build pauses the maintained models it touches, records its hold, and resumes them at the end.""" +"""A build pauses the dynamic models it touches, records its hold, and resumes them at the end.""" from __future__ import annotations @@ -17,14 +17,14 @@ BuildRuntimeParams, ) from sqlbuild.executor.build.types import BuildStatus -from sqlbuild.executor.maintained_pauses._helpers.sql import insert_sql -from sqlbuild.executor.maintained_pauses.classes.maintained_hold import MaintainedHold -from sqlbuild.executor.maintained_pauses.main.hold_maintained_models import ( - hold_maintained_models, +from sqlbuild.executor.dynamic_pauses._helpers.sql import insert_sql +from sqlbuild.executor.dynamic_pauses.classes.dynamic_hold import DynamicHold +from sqlbuild.executor.dynamic_pauses.main.hold_dynamic_models import ( + hold_dynamic_models, ) -from sqlbuild.executor.maintained_pauses.models import MaintainedPauseEvent -from sqlbuild.executor.maintained_pauses.types import MaintainedPauseEventType, StateLocation -from tests.integration.src.sqlbuild.executor.maintained_pauses._test_types import ( +from sqlbuild.executor.dynamic_pauses.models import DynamicPauseEvent +from sqlbuild.executor.dynamic_pauses.types import DynamicPauseEventType, StateLocation +from tests.integration.src.sqlbuild.executor.dynamic_pauses._test_types import ( ConcurrentHoldTestCase, CreatedModelHoldTestCase, FirstBuildTestCase, @@ -37,7 +37,7 @@ TakenOverPauseTestCase, TakenOverReleaseTestCase, ) -from tests.integration.src.sqlbuild.executor.maintained_pauses.helpers import ( +from tests.integration.src.sqlbuild.executor.dynamic_pauses.helpers import ( TOTALS_QUERY, LeaseProject, PausingDuckDbAdapter, @@ -75,7 +75,7 @@ [ FirstBuildTestCase( description="a created model is paused for the build and resumed at its end", - pause_maintained=False, + pause_dynamic=False, expected_operations=( ("create", "daily_totals"), ("suspend", "daily_totals"), @@ -86,7 +86,7 @@ ], ids=lambda case: case.description, ) -def test_given_first_build_when_creating_maintained_model_then_pauses_it_until_the_end( +def test_given_first_build_when_creating_dynamic_model_then_pauses_it_until_the_end( test_case: FirstBuildTestCase, adapter: PausingDuckDbAdapter, connection: Any, @@ -96,7 +96,7 @@ def test_given_first_build_when_creating_maintained_model_then_pauses_it_until_t adapter=adapter, connection=connection, case=lease_project_case(orders_query=_ORDERS, report_locations={}, refresh_on_build=False), - runtime=BuildRuntimeParams(run_id="first_run", pause_maintained=test_case.pause_maintained), + runtime=BuildRuntimeParams(run_id="first_run", pause_dynamic=test_case.pause_dynamic), database_path=database_path, ) @@ -110,7 +110,7 @@ def test_given_first_build_when_creating_maintained_model_then_pauses_it_until_t "test_case", [ PauseLeaseTestCase( - description="a build of the maintained model alone writes nothing it reads", + description="a build of the dynamic model alone writes nothing it reads", second_select=("daily_totals",), second_orders_query=_ORDERS, second_totals_query=TOTALS_QUERY, @@ -186,7 +186,7 @@ def test_given_first_build_when_creating_maintained_model_then_pauses_it_until_t ], ids=lambda case: case.description, ) -def test_given_built_maintained_model_when_building_again_then_pauses_only_what_it_touches( +def test_given_built_dynamic_model_when_building_again_then_pauses_only_what_it_touches( test_case: PauseLeaseTestCase, adapter: PausingDuckDbAdapter, connection: Any, @@ -229,22 +229,22 @@ def test_given_built_maintained_model_when_building_again_then_pauses_only_what_ "test_case", [ FirstBuildTestCase( - description="a pause_maintained target leaves models suspended and records nothing", - pause_maintained=True, + description="a pause_dynamic target leaves models suspended and records nothing", + pause_dynamic=True, expected_operations=(("refresh", "daily_totals"),), expected_states={"daily_totals": "SUSPENDED"}, ), ], ids=lambda case: case.description, ) -def test_given_pause_maintained_target_when_building_then_models_stay_suspended_and_refresh( +def test_given_pause_dynamic_target_when_building_then_models_stay_suspended_and_refresh( test_case: FirstBuildTestCase, adapter: PausingDuckDbAdapter, connection: Any, database_path: Path, ) -> None: paused: BuildRuntimeParams = BuildRuntimeParams( - run_id="first_run", pause_maintained=test_case.pause_maintained + run_id="first_run", pause_dynamic=test_case.pause_dynamic ) _ = build_lease_project( adapter=adapter, @@ -285,7 +285,7 @@ def test_given_pause_maintained_target_when_building_then_models_stay_suspended_ expected_events=_TAKEN_OVER, ), HoldTakeoverTestCase( - description="--ignore-maintained-grace takes over a live hold when the project allows", + description="--ignore-dynamic-grace takes over a live hold when the project allows", grace_seconds=1800, ignore_grace=True, allow_ignore_grace=True, @@ -311,7 +311,7 @@ def test_given_hold_of_a_stopped_build_when_building_then_takes_it_over_and_resu changed: LeaseProject = lease_project_case( orders_query=_MORE_ORDERS, report_locations={}, refresh_on_build=False ) - dead: MaintainedHold | None = hold_maintained_models( + dead: DynamicHold | None = hold_dynamic_models( plan=plan_lease_project(adapter=adapter, connection=connection, case=changed), adapter=adapter, runtime=BuildRuntimeParams(run_id="dead_run"), @@ -327,9 +327,9 @@ def test_given_hold_of_a_stopped_build_when_building_then_takes_it_over_and_resu case=changed, runtime=BuildRuntimeParams( run_id="second_run", - maintained_heartbeat_grace_seconds=test_case.grace_seconds, - ignore_maintained_grace=test_case.ignore_grace, - allow_ignore_maintained_grace=test_case.allow_ignore_grace, + dynamic_heartbeat_grace_seconds=test_case.grace_seconds, + ignore_dynamic_grace=test_case.ignore_grace, + allow_ignore_dynamic_grace=test_case.allow_ignore_grace, ), database_path=database_path, ) @@ -352,7 +352,7 @@ def test_given_hold_of_a_stopped_build_when_building_then_takes_it_over_and_resu expected_message_fragment="dead_run", ), HoldRefusalTestCase( - description="--ignore-maintained-grace needs allow_ignore_grace", + description="--ignore-dynamic-grace needs allow_ignore_grace", grace_seconds=1800, ignore_grace=True, allow_ignore_grace=False, @@ -378,7 +378,7 @@ def test_given_live_hold_when_building_then_stops_before_any_change( changed: LeaseProject = lease_project_case( orders_query=_MORE_ORDERS, report_locations={}, refresh_on_build=False ) - dead: MaintainedHold | None = hold_maintained_models( + dead: DynamicHold | None = hold_dynamic_models( plan=plan_lease_project(adapter=adapter, connection=connection, case=changed), adapter=adapter, runtime=BuildRuntimeParams(run_id="dead_run"), @@ -395,9 +395,9 @@ def test_given_live_hold_when_building_then_stops_before_any_change( case=changed, runtime=BuildRuntimeParams( run_id="second_run", - maintained_heartbeat_grace_seconds=test_case.grace_seconds, - ignore_maintained_grace=test_case.ignore_grace, - allow_ignore_maintained_grace=test_case.allow_ignore_grace, + dynamic_heartbeat_grace_seconds=test_case.grace_seconds, + ignore_dynamic_grace=test_case.ignore_grace, + allow_ignore_dynamic_grace=test_case.allow_ignore_grace, ), database_path=database_path, ) @@ -442,7 +442,7 @@ def test_given_held_models_when_build_runs_then_heartbeat_writes_on_its_own_conn changed: LeaseProject = lease_project_case( orders_query=_MORE_ORDERS, report_locations={}, refresh_on_build=False ) - hold: MaintainedHold | None = hold_maintained_models( + hold: DynamicHold | None = hold_dynamic_models( plan=plan_lease_project(adapter=adapter, connection=connection, case=changed), adapter=adapter, runtime=BuildRuntimeParams(run_id="beating_run"), @@ -456,7 +456,7 @@ def test_given_held_models_when_build_runs_then_heartbeat_writes_on_its_own_conn hold.release(connection=connection) beats: list[tuple[Any, ...]] = connection.execute( - "SELECT count(*) FROM staging._sqlbuild_maintained_pauses " + "SELECT count(*) FROM staging._sqlbuild_dynamic_pauses " "WHERE event_type = 'heartbeat' AND run_id = 'beating_run'" ).fetchall() assert beats[0][0] >= test_case.expected_min_beats @@ -469,8 +469,7 @@ def test_given_held_models_when_build_runs_then_heartbeat_writes_on_its_own_conn description="a model suspended outside SQLBuild is reported on the build's output", states_before_second={"daily_totals": "SUSPENDED"}, expected_line=( - "Warning: maintained model daily_totals was paused outside SQLBuild; " - "it stays paused" + "Warning: dynamic model daily_totals was paused outside SQLBuild; it stays paused" ), ), ], @@ -542,16 +541,16 @@ def test_given_hold_taken_over_when_first_build_releases_then_it_resumes_nothing orders_query=_MORE_ORDERS, report_locations={}, refresh_on_build=False ) plan: PlanOutput = plan_lease_project(adapter=adapter, connection=connection, case=changed) - holds: list[MaintainedHold] = [] + holds: list[DynamicHold] = [] for runtime in ( BuildRuntimeParams(run_id="slow_run"), BuildRuntimeParams( run_id="second_run", - ignore_maintained_grace=True, - allow_ignore_maintained_grace=True, + ignore_dynamic_grace=True, + allow_ignore_dynamic_grace=True, ), ): - hold: MaintainedHold | None = hold_maintained_models( + hold: DynamicHold | None = hold_dynamic_models( plan=plan, adapter=adapter, runtime=runtime, @@ -609,13 +608,13 @@ def test_given_two_builds_pausing_together_when_both_record_a_pause_then_only_th orders_query=_MORE_ORDERS, report_locations={}, refresh_on_build=False ), ) - early: MaintainedHold | None = hold_maintained_models( + early: DynamicHold | None = hold_dynamic_models( plan=plan, adapter=adapter, runtime=BuildRuntimeParams(run_id="early_run"), connection_config={"database": str(database_path)}, ) - late: MaintainedHold | None = hold_maintained_models( + late: DynamicHold | None = hold_dynamic_models( plan=plan, adapter=adapter, runtime=BuildRuntimeParams(run_id="late_run"), @@ -664,7 +663,7 @@ def state_then_rival_acquires(*, connection: Any, name: str) -> str | None: ), CreatedModelHoldTestCase( description="a failed close of the later pause is reported with the hold", - refused_events=(MaintainedPauseEventType.RESUMED,), + refused_events=(DynamicPauseEventType.RESUMED,), rival_closes=(), expected_release_operations=(), expected_code="A718", @@ -704,9 +703,9 @@ def test_given_two_builds_creating_a_model_when_both_pause_it_then_only_the_earl connection=connection, case=lease_project_case(orders_query=_ORDERS, report_locations={}, refresh_on_build=False), ) - holds: dict[str, MaintainedHold] = {} + holds: dict[str, DynamicHold] = {} for run_id in ("early_run", "late_run"): - hold: MaintainedHold | None = hold_maintained_models( + hold: DynamicHold | None = hold_dynamic_models( plan=plan, adapter=adapter, runtime=BuildRuntimeParams(run_id=run_id), @@ -715,14 +714,14 @@ def test_given_two_builds_creating_a_model_when_both_pause_it_then_only_the_earl assert hold is not None hold.acquire(connection=connection, on_progress=None) holds[run_id] = hold - late: MaintainedHold = holds["late_run"] - recorders: dict[tuple[MaintainedPauseEventType, ...], Callable[..., None]] = { + late: DynamicHold = holds["late_run"] + recorders: dict[tuple[DynamicPauseEventType, ...], Callable[..., None]] = { (): late._record, - **{(event_type,): late._record for event_type in MaintainedPauseEventType}, + **{(event_type,): late._record for event_type in DynamicPauseEventType}, **{(event_type,): refuse_insert for event_type in test_case.refused_events}, } - def record_or_refuse(*, connection: Any, events: tuple[MaintainedPauseEvent, ...]) -> None: + def record_or_refuse(*, connection: Any, events: tuple[DynamicPauseEvent, ...]) -> None: recorders[tuple(event.event_type for event in events)](connection=connection, events=events) late._record = record_or_refuse # ty: ignore[invalid-assignment] @@ -732,11 +731,11 @@ def record_or_refuse(*, connection: Any, events: tuple[MaintainedPauseEvent, ... with pytest.raises(AdapterUserError) as raised: late.after_create(connection=connection, name="daily_totals") - early: MaintainedHold = holds["early_run"] + early: DynamicHold = holds["early_run"] early._record( connection=connection, events=tuple( - early._own(event_type=MaintainedPauseEventType.RESUMED, name=name) + early._own(event_type=DynamicPauseEventType.RESUMED, name=name) for name in test_case.rival_closes ), ) @@ -791,9 +790,9 @@ def test_given_pause_committing_late_when_rival_paused_meanwhile_then_the_late_o orders_query=_MORE_ORDERS, report_locations={}, refresh_on_build=False ), ) - holds: dict[str, MaintainedHold] = {} + holds: dict[str, DynamicHold] = {} for run_id in ("early_run", "late_run"): - hold: MaintainedHold | None = hold_maintained_models( + hold: DynamicHold | None = hold_dynamic_models( plan=plan, adapter=adapter, runtime=BuildRuntimeParams(run_id=run_id), @@ -801,12 +800,12 @@ def test_given_pause_committing_late_when_rival_paused_meanwhile_then_the_late_o ) assert hold is not None holds[run_id] = hold - early: MaintainedHold = holds["early_run"] + early: DynamicHold = holds["early_run"] location: StateLocation | None = early._state_location assert location is not None now: str = adapter.render_current_utc_timestamp() - def commit_after_rival(*, connection: Any, events: tuple[MaintainedPauseEvent, ...]) -> None: + def commit_after_rival(*, connection: Any, events: tuple[DynamicPauseEvent, ...]) -> None: started: Any = connection.execute(f"SELECT {now}").fetchone()[0] holds["late_run"].acquire(connection=connection, on_progress=None) adapter.execute( @@ -816,15 +815,13 @@ def commit_after_rival(*, connection: Any, events: tuple[MaintainedPauseEvent, . ), ) - recorders: dict[tuple[MaintainedPauseEventType, ...], Callable[..., None]] = { + recorders: dict[tuple[DynamicPauseEventType, ...], Callable[..., None]] = { (): early._record, - (MaintainedPauseEventType.PAUSED,): commit_after_rival, - (MaintainedPauseEventType.RESUMED,): early._record, + (DynamicPauseEventType.PAUSED,): commit_after_rival, + (DynamicPauseEventType.RESUMED,): early._record, } - def record_committing_late( - *, connection: Any, events: tuple[MaintainedPauseEvent, ...] - ) -> None: + def record_committing_late(*, connection: Any, events: tuple[DynamicPauseEvent, ...]) -> None: recorders[tuple(event.event_type for event in events)](connection=connection, events=events) early._record = record_committing_late # ty: ignore[invalid-assignment] @@ -879,14 +876,14 @@ def test_given_pause_taken_over_before_read_back_when_acquiring_then_the_build_b orders_query=_MORE_ORDERS, report_locations={}, refresh_on_build=False ), ) - holds: dict[str, MaintainedHold] = {} + holds: dict[str, DynamicHold] = {} for runtime in ( BuildRuntimeParams(run_id="early_run"), BuildRuntimeParams( - run_id="taker_run", ignore_maintained_grace=True, allow_ignore_maintained_grace=True + run_id="taker_run", ignore_dynamic_grace=True, allow_ignore_dynamic_grace=True ), ): - hold: MaintainedHold | None = hold_maintained_models( + hold: DynamicHold | None = hold_dynamic_models( plan=plan, adapter=adapter, runtime=runtime, @@ -894,22 +891,22 @@ def test_given_pause_taken_over_before_read_back_when_acquiring_then_the_build_b ) assert hold is not None holds[runtime.run_id] = hold - early: MaintainedHold = holds["early_run"] + early: DynamicHold = holds["early_run"] record: Callable[..., None] = early._record def record_then_rival_takes_over( - *, connection: Any, events: tuple[MaintainedPauseEvent, ...] + *, connection: Any, events: tuple[DynamicPauseEvent, ...] ) -> None: record(connection=connection, events=events) holds["taker_run"].acquire(connection=connection, on_progress=None) - recorders: dict[tuple[MaintainedPauseEventType, ...], Callable[..., None]] = { + recorders: dict[tuple[DynamicPauseEventType, ...], Callable[..., None]] = { (): record, - (MaintainedPauseEventType.PAUSED,): record_then_rival_takes_over, - (MaintainedPauseEventType.RESUMED,): record, + (DynamicPauseEventType.PAUSED,): record_then_rival_takes_over, + (DynamicPauseEventType.RESUMED,): record, } - def record_or_take_over(*, connection: Any, events: tuple[MaintainedPauseEvent, ...]) -> None: + def record_or_take_over(*, connection: Any, events: tuple[DynamicPauseEvent, ...]) -> None: recorders[tuple(event.event_type for event in events)](connection=connection, events=events) early._record = record_or_take_over # ty: ignore[invalid-assignment] diff --git a/tests/integration/src/sqlbuild/executor/maintained_pauses/__init__.py b/tests/integration/src/sqlbuild/executor/maintained_pauses/__init__.py deleted file mode 100644 index 11f5a11f26..0000000000 --- a/tests/integration/src/sqlbuild/executor/maintained_pauses/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Maintained model pause lease integration tests.""" diff --git a/tests/integration/src/sqlbuild/executor/run/maintained/__init__.py b/tests/integration/src/sqlbuild/executor/run/dynamic/__init__.py similarity index 100% rename from tests/integration/src/sqlbuild/executor/run/maintained/__init__.py rename to tests/integration/src/sqlbuild/executor/run/dynamic/__init__.py diff --git a/tests/integration/src/sqlbuild/executor/run/maintained/_test_types.py b/tests/integration/src/sqlbuild/executor/run/dynamic/_test_types.py similarity index 81% rename from tests/integration/src/sqlbuild/executor/run/maintained/_test_types.py rename to tests/integration/src/sqlbuild/executor/run/dynamic/_test_types.py index 9debeb3066..5d3a14525c 100644 --- a/tests/integration/src/sqlbuild/executor/run/maintained/_test_types.py +++ b/tests/integration/src/sqlbuild/executor/run/dynamic/_test_types.py @@ -3,11 +3,11 @@ from dataclasses import dataclass from sqlbuild._native import PlanAction -from sqlbuild.adapter.contract.types import MaintainedListing, TablePromotionMode +from sqlbuild.adapter.contract.types import DynamicListing, TablePromotionMode @dataclass(frozen=True) -class MaintainedSuccessTestCase: +class DynamicSuccessTestCase: description: str action: PlanAction setup_sql: tuple[str, ...] @@ -18,11 +18,11 @@ class MaintainedSuccessTestCase: @dataclass(frozen=True) -class MaintainedFailureTestCase: +class DynamicFailureTestCase: description: str setup_sql: tuple[str, ...] session_warehouse: str | None - listed_settings: MaintainedListing | None + listed_settings: DynamicListing | None failing_query: str | None expected_error_code: str expected_listed_types: tuple[tuple[str], ...] @@ -42,7 +42,7 @@ class FailedRestoreTestCase: @dataclass(frozen=True) class ChangeTrackingTableTestCase: description: str - maintained_readers: tuple[str, ...] + dynamic_readers: tuple[str, ...] promotion_mode: TablePromotionMode declared_columns: tuple[tuple[str, str], ...] expected_tracking: tuple[bool, ...] @@ -51,5 +51,5 @@ class ChangeTrackingTableTestCase: @dataclass(frozen=True) class ChangeTrackingSnapshotTestCase: description: str - maintained_readers: tuple[str, ...] + dynamic_readers: tuple[str, ...] expected_tracked: tuple[str, ...] diff --git a/tests/integration/src/sqlbuild/executor/run/maintained/helpers.py b/tests/integration/src/sqlbuild/executor/run/dynamic/helpers.py similarity index 86% rename from tests/integration/src/sqlbuild/executor/run/maintained/helpers.py rename to tests/integration/src/sqlbuild/executor/run/dynamic/helpers.py index 3848f0de8c..146f816ebe 100644 --- a/tests/integration/src/sqlbuild/executor/run/maintained/helpers.py +++ b/tests/integration/src/sqlbuild/executor/run/dynamic/helpers.py @@ -1,4 +1,4 @@ -"""A DuckDB adapter that builds maintained models as tables, for executor lifecycle tests.""" +"""A DuckDB adapter that builds dynamic models as tables, for executor lifecycle tests.""" from __future__ import annotations @@ -15,27 +15,27 @@ PlanReason, ) from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder -from sqlbuild.adapter.contract.types import MaintainedListing +from sqlbuild.adapter.contract.types import DynamicListing from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.executor.run.models import ModelMaterializationContext DAILY_TOTALS_SQL: str = "SELECT 3 AS orders" -class MaintainedDuckDbAdapter(DuckDbAdapter): - """Builds a maintained model as a table that, like a dynamic table, replaces nothing else.""" +class DynamicDuckDbAdapter(DuckDbAdapter): + """Builds a dynamic model as a table that, like a dynamic table, replaces nothing else.""" def __init__( self, *, session_warehouse: str | None, - listed_settings: MaintainedListing | None = None, + listed_settings: DynamicListing | None = None, failing_query: str | None = None, failing_renames: dict[str, str] | None = None, ) -> None: super().__init__() self.warehouse: str | None = session_warehouse - self.listed_settings: MaintainedListing | None = listed_settings + self.listed_settings: DynamicListing | None = listed_settings self.failing_query: str | None = failing_query self.failing_renames: dict[str, str] = dict(failing_renames or {}) self.created: list[tuple[str, str, str]] = [] @@ -45,13 +45,13 @@ def session_warehouse(self, *, connection: Any) -> str | None: del connection return self.warehouse - def maintained_listing( + def dynamic_listing( self, *, connection: Any, database: str | None, schema: str | None, name: str - ) -> MaintainedListing | None: + ) -> DynamicListing | None: del connection, database, schema, name return self.listed_settings - def render_create_maintained( + def render_create_dynamic( self, *, destination: str, sql: str, target_lag: str, refresh_mode: str, warehouse: str ) -> tuple[str, ...]: self.created.append((target_lag, refresh_mode, warehouse)) @@ -64,7 +64,7 @@ def render_rename(self, *, origin: str, destination: str) -> tuple[str, ...]: def render_rename_view(self, *, origin: str, destination: str) -> tuple[str, ...]: return (f"ALTER VIEW {origin} RENAME TO {destination.split('.')[-1]}",) - def render_alter_maintained( + def render_alter_dynamic( self, *, destination: str, target_lag: str, warehouse: str | None ) -> tuple[str, ...]: self.altered.append((target_lag, warehouse)) @@ -72,20 +72,20 @@ def render_alter_maintained( return (f"COMMENT ON TABLE {destination} IS 'refreshed within {lag}'",) -def maintained_context( +def dynamic_context( *, - adapter: MaintainedDuckDbAdapter, + adapter: DynamicDuckDbAdapter, connection: Any, action: PlanAction, refresh_warehouse: str | None, ) -> ModelMaterializationContext: - """Daily totals, maintained within an hour, planned with `action`.""" + """Daily totals, dynamic within an hour, planned with `action`.""" entry: ModelPlanEntry = ModelPlanEntry( key=CompiledObjectKey(resource_type=CompiledResourceType.MODEL, name="daily_totals"), name="daily_totals", relative_path=Path("models/daily_totals.sql"), - materialization_type=MaterializationType.MAINTAINED, + materialization_type=MaterializationType.DYNAMIC, action=action, reason=PlanReason.FIRST_RUN, destination=CompiledRelationLocation( @@ -110,7 +110,7 @@ def maintained_context( source_map={}, python_source_read_map={}, model_audits=(), - run_id="maintained_run", + run_id="dynamic_run", query_change_tracking=True, ) diff --git a/tests/integration/src/sqlbuild/executor/run/maintained/test_change_tracking.py b/tests/integration/src/sqlbuild/executor/run/dynamic/test_change_tracking.py similarity index 82% rename from tests/integration/src/sqlbuild/executor/run/maintained/test_change_tracking.py rename to tests/integration/src/sqlbuild/executor/run/dynamic/test_change_tracking.py index d9d9fd5fe0..516ca26749 100644 --- a/tests/integration/src/sqlbuild/executor/run/maintained/test_change_tracking.py +++ b/tests/integration/src/sqlbuild/executor/run/dynamic/test_change_tracking.py @@ -1,4 +1,4 @@ -"""A table a maintained model reads is created with change tracking on every build path.""" +"""A table a dynamic model reads is created with change tracking on every build path.""" from __future__ import annotations @@ -13,18 +13,18 @@ from sqlbuild.executor.run.main._execute import execute_table_entry from sqlbuild.executor.run.models import ModelExecutionResult, ModelMaterializationContext from sqlbuild.executor.scheduling.types import ExecutionStatus -from tests.integration.src.sqlbuild.executor.run.helpers import ( - build_declared_columns, - build_table_plan_entry, -) -from tests.integration.src.sqlbuild.executor.run.maintained._test_types import ( +from tests.integration.src.sqlbuild.executor.run.dynamic._test_types import ( ChangeTrackingSnapshotTestCase, ChangeTrackingTableTestCase, ) -from tests.integration.src.sqlbuild.executor.run.maintained.helpers import ( +from tests.integration.src.sqlbuild.executor.run.dynamic.helpers import ( TrackedSnapshotDuckDbAdapter, TrackingDuckDbAdapter, ) +from tests.integration.src.sqlbuild.executor.run.helpers import ( + build_declared_columns, + build_table_plan_entry, +) from tests.unit.src.sqlbuild.executor.run._helpers.helpers import ( build_snapshot_execution_plan_entry, ) @@ -35,21 +35,21 @@ [ ChangeTrackingTableTestCase( description="a staged build tracks changes on the stage it swaps in", - maintained_readers=("daily_totals",), + dynamic_readers=("daily_totals",), promotion_mode=TablePromotionMode.STAGED, declared_columns=(), expected_tracking=(True,), ), ChangeTrackingTableTestCase( description="type enforcement rebuilds the stage with change tracking too", - maintained_readers=("daily_totals",), + dynamic_readers=("daily_totals",), promotion_mode=TablePromotionMode.STAGED, declared_columns=(("order_id", "BIGINT"),), expected_tracking=(True, True), ), ChangeTrackingTableTestCase( - description="a table no maintained model reads is created as before", - maintained_readers=(), + description="a table no dynamic model reads is created as before", + dynamic_readers=(), promotion_mode=TablePromotionMode.STAGED, declared_columns=(("order_id", "BIGINT"),), expected_tracking=(False, False), @@ -57,7 +57,7 @@ ], ids=lambda case: case.description, ) -def test_given_table_read_by_maintained_model_when_building_then_every_create_tracks_changes( +def test_given_table_read_by_dynamic_model_when_building_then_every_create_tracks_changes( test_case: ChangeTrackingTableTestCase, connection: Any ) -> None: adapter: TrackingDuckDbAdapter = TrackingDuckDbAdapter() @@ -69,7 +69,7 @@ def test_given_table_read_by_maintained_model_when_building_then_every_create_tr target_name="orders", type_enforcement=bool(test_case.declared_columns), ), - maintained_readers=test_case.maintained_readers, + dynamic_readers=test_case.dynamic_readers, ) result: ModelExecutionResult = execute_table_entry( @@ -98,27 +98,27 @@ def test_given_table_read_by_maintained_model_when_building_then_every_create_tr [ ChangeTrackingSnapshotTestCase( description="a snapshot's first build and its full rebuild both track changes", - maintained_readers=("daily_totals",), + dynamic_readers=("daily_totals",), expected_tracked=( "main.customer_snapshot", "main.__sqb_rebuild__customer_snapshot", ), ), ChangeTrackingSnapshotTestCase( - description="a snapshot no maintained model reads is built as before", - maintained_readers=(), + description="a snapshot no dynamic model reads is built as before", + dynamic_readers=(), expected_tracked=(), ), ], ids=lambda case: case.description, ) -def test_given_snapshot_read_by_maintained_model_when_rebuilding_then_it_tracks_changes( +def test_given_snapshot_read_by_dynamic_model_when_rebuilding_then_it_tracks_changes( test_case: ChangeTrackingSnapshotTestCase, ) -> None: adapter: TrackedSnapshotDuckDbAdapter = TrackedSnapshotDuckDbAdapter() connection: Any = adapter.connect({"database": ":memory:"}) entry: ModelPlanEntry = replace( - build_snapshot_execution_plan_entry(), maintained_readers=test_case.maintained_readers + build_snapshot_execution_plan_entry(), dynamic_readers=test_case.dynamic_readers ) results: list[ModelExecutionResult] = [ diff --git a/tests/integration/src/sqlbuild/executor/run/maintained/test_maintained_execution.py b/tests/integration/src/sqlbuild/executor/run/dynamic/test_dynamic_execution.py similarity index 76% rename from tests/integration/src/sqlbuild/executor/run/maintained/test_maintained_execution.py rename to tests/integration/src/sqlbuild/executor/run/dynamic/test_dynamic_execution.py index 1df742d105..3dd65a043b 100644 --- a/tests/integration/src/sqlbuild/executor/run/maintained/test_maintained_execution.py +++ b/tests/integration/src/sqlbuild/executor/run/dynamic/test_dynamic_execution.py @@ -1,4 +1,4 @@ -"""Integration tests for the maintained model lifecycle: create, alter and keep.""" +"""Integration tests for the dynamic model lifecycle: create, alter and keep.""" from __future__ import annotations @@ -7,19 +7,19 @@ import pytest from sqlbuild._native import PlanAction -from sqlbuild.adapter.contract.types import MaintainedListing -from sqlbuild.executor.run.main._execute import execute_maintained_entry +from sqlbuild.adapter.contract.types import DynamicListing +from sqlbuild.executor.run.main._execute import execute_dynamic_entry from sqlbuild.executor.run.models import ModelExecutionResult from sqlbuild.executor.scheduling.types import ExecutionStatus -from tests.integration.src.sqlbuild.executor.run.maintained._test_types import ( +from tests.integration.src.sqlbuild.executor.run.dynamic._test_types import ( + DynamicFailureTestCase, + DynamicSuccessTestCase, FailedRestoreTestCase, - MaintainedFailureTestCase, - MaintainedSuccessTestCase, ) -from tests.integration.src.sqlbuild.executor.run.maintained.helpers import ( - MaintainedDuckDbAdapter, +from tests.integration.src.sqlbuild.executor.run.dynamic.helpers import ( + DynamicDuckDbAdapter, + dynamic_context, listed_types, - maintained_context, ) _BUILT: tuple[str, ...] = ("CREATE TABLE staging.daily_totals AS SELECT 1 AS orders",) @@ -28,45 +28,45 @@ @pytest.mark.parametrize( "test_case", [ - MaintainedSuccessTestCase( + DynamicSuccessTestCase( description="a first run creates on the session warehouse", - action=PlanAction.CREATE_MAINTAINED, + action=PlanAction.CREATE_DYNAMIC, setup_sql=(), refresh_warehouse=None, expected_created=(("'1 hour'", "incremental", "SESSION_WH"),), expected_altered=(), expected_rows=((3,),), ), - MaintainedSuccessTestCase( + DynamicSuccessTestCase( description="a declared refresh warehouse is used over the session's", - action=PlanAction.CREATE_MAINTAINED, + action=PlanAction.CREATE_DYNAMIC, setup_sql=(), refresh_warehouse="transform_wh", expected_created=(("'1 hour'", "incremental", "transform_wh"),), expected_altered=(), expected_rows=((3,),), ), - MaintainedSuccessTestCase( + DynamicSuccessTestCase( description="a view the model was before is dropped and replaced", - action=PlanAction.CREATE_MAINTAINED, + action=PlanAction.CREATE_DYNAMIC, setup_sql=("CREATE VIEW staging.daily_totals AS SELECT 1 AS orders",), refresh_warehouse=None, expected_created=(("'1 hour'", "incremental", "SESSION_WH"),), expected_altered=(), expected_rows=((3,),), ), - MaintainedSuccessTestCase( + DynamicSuccessTestCase( description="drifted settings alter in place and keep the data", - action=PlanAction.ALTER_MAINTAINED, + action=PlanAction.ALTER_DYNAMIC, setup_sql=_BUILT, refresh_warehouse="transform_wh", expected_created=(), expected_altered=(("'1 hour'", "transform_wh"),), expected_rows=((1,),), ), - MaintainedSuccessTestCase( + DynamicSuccessTestCase( description="a kept model runs no refresh statement", - action=PlanAction.KEEP_MAINTAINED, + action=PlanAction.KEEP_DYNAMIC, setup_sql=_BUILT, refresh_warehouse=None, expected_created=(), @@ -76,15 +76,15 @@ ], ids=lambda case: case.description, ) -def test_given_maintained_entry_when_executing_then_settings_reach_the_warehouse( - test_case: MaintainedSuccessTestCase, connection: Any +def test_given_dynamic_entry_when_executing_then_settings_reach_the_warehouse( + test_case: DynamicSuccessTestCase, connection: Any ) -> None: - adapter: MaintainedDuckDbAdapter = MaintainedDuckDbAdapter(session_warehouse="SESSION_WH") + adapter: DynamicDuckDbAdapter = DynamicDuckDbAdapter(session_warehouse="SESSION_WH") for sql in test_case.setup_sql: connection.execute(sql) - result: ModelExecutionResult = execute_maintained_entry( - context=maintained_context( + result: ModelExecutionResult = execute_dynamic_entry( + context=dynamic_context( adapter=adapter, connection=connection, action=test_case.action, @@ -106,7 +106,7 @@ def test_given_maintained_entry_when_executing_then_settings_reach_the_warehouse @pytest.mark.parametrize( "test_case", [ - MaintainedFailureTestCase( + DynamicFailureTestCase( description="no declared or session warehouse fails before any statement", setup_sql=(), session_warehouse=None, @@ -115,16 +115,16 @@ def test_given_maintained_entry_when_executing_then_settings_reach_the_warehouse expected_error_code="A717", expected_listed_types=(), ), - MaintainedFailureTestCase( + DynamicFailureTestCase( description="a failed create never drops a relation listed as a dynamic table", setup_sql=("CREATE VIEW staging.daily_totals AS SELECT 1 AS orders",), session_warehouse="SESSION_WH", - listed_settings=MaintainedListing("1 hour", "SESSION_WH", "ACTIVE"), + listed_settings=DynamicListing("1 hour", "SESSION_WH", "ACTIVE"), failing_query=None, expected_error_code="R002", expected_listed_types=(("VIEW",),), ), - MaintainedFailureTestCase( + DynamicFailureTestCase( description="a table the model was before is restored when the create keeps failing", setup_sql=("CREATE TABLE staging.daily_totals AS SELECT 1 AS orders",), session_warehouse="SESSION_WH", @@ -133,7 +133,7 @@ def test_given_maintained_entry_when_executing_then_settings_reach_the_warehouse expected_error_code="R002", expected_listed_types=(("BASE TABLE",),), ), - MaintainedFailureTestCase( + DynamicFailureTestCase( description="a view the model was before is restored when the create keeps failing", setup_sql=("CREATE VIEW staging.daily_totals AS SELECT 1 AS orders",), session_warehouse="SESSION_WH", @@ -145,10 +145,10 @@ def test_given_maintained_entry_when_executing_then_settings_reach_the_warehouse ], ids=lambda case: case.description, ) -def test_given_unbuildable_maintained_entry_when_executing_then_fails_keeping_the_relation( - test_case: MaintainedFailureTestCase, connection: Any +def test_given_unbuildable_dynamic_entry_when_executing_then_fails_keeping_the_relation( + test_case: DynamicFailureTestCase, connection: Any ) -> None: - adapter: MaintainedDuckDbAdapter = MaintainedDuckDbAdapter( + adapter: DynamicDuckDbAdapter = DynamicDuckDbAdapter( session_warehouse=test_case.session_warehouse, listed_settings=test_case.listed_settings, failing_query=test_case.failing_query, @@ -156,11 +156,11 @@ def test_given_unbuildable_maintained_entry_when_executing_then_fails_keeping_th for sql in test_case.setup_sql: connection.execute(sql) - result: ModelExecutionResult = execute_maintained_entry( - context=maintained_context( + result: ModelExecutionResult = execute_dynamic_entry( + context=dynamic_context( adapter=adapter, connection=connection, - action=PlanAction.CREATE_MAINTAINED, + action=PlanAction.CREATE_DYNAMIC, refresh_warehouse=None, ) ) @@ -190,7 +190,7 @@ def test_given_unbuildable_maintained_entry_when_executing_then_fails_keeping_th def test_given_failed_create_and_restore_when_executing_then_reports_both_errors( test_case: FailedRestoreTestCase, connection: Any ) -> None: - adapter: MaintainedDuckDbAdapter = MaintainedDuckDbAdapter( + adapter: DynamicDuckDbAdapter = DynamicDuckDbAdapter( session_warehouse="SESSION_WH", failing_query=test_case.failing_query, failing_renames=test_case.failing_renames, @@ -198,11 +198,11 @@ def test_given_failed_create_and_restore_when_executing_then_reports_both_errors for sql in test_case.setup_sql: connection.execute(sql) - result: ModelExecutionResult = execute_maintained_entry( - context=maintained_context( + result: ModelExecutionResult = execute_dynamic_entry( + context=dynamic_context( adapter=adapter, connection=connection, - action=PlanAction.CREATE_MAINTAINED, + action=PlanAction.CREATE_DYNAMIC, refresh_warehouse=None, ) ) diff --git a/tests/unit/src/sqlbuild/adapter/contract/classes/strict_adapter/test_strict_adapter.py b/tests/unit/src/sqlbuild/adapter/contract/classes/strict_adapter/test_strict_adapter.py index c8a16a45e9..6605bc233c 100644 --- a/tests/unit/src/sqlbuild/adapter/contract/classes/strict_adapter/test_strict_adapter.py +++ b/tests/unit/src/sqlbuild/adapter/contract/classes/strict_adapter/test_strict_adapter.py @@ -89,12 +89,12 @@ def test_given_first_class_adapters_when_checking_contract_then_no_method_comes_ { "create_sql_test_output", "drop_sql_test_output", - "maintained_listing", - "render_alter_maintained", - "render_create_maintained", + "dynamic_listing", + "render_alter_dynamic", + "render_create_dynamic", "render_create_temporary_function", "render_drop_schema", - "render_maintained_operation", + "render_dynamic_operation", "render_track_changes", "session_warehouse", } @@ -131,15 +131,15 @@ def test_given_strict_adapter_when_reading_optional_methods_then_defaults_keep_i assert temporary is test_case.expected_temporary_function assert drop_error.value.code == "A714" assert adapter.session_warehouse(connection=None) is None - with pytest.raises(AdapterUserError) as maintained_error: - adapter.render_create_maintained( + with pytest.raises(AdapterUserError) as dynamic_error: + adapter.render_create_dynamic( destination="main.daily_totals", sql="SELECT 1", target_lag="DOWNSTREAM", refresh_mode="incremental", warehouse="transform_wh", ) - assert maintained_error.value.code == "A716" + assert dynamic_error.value.code == "A716" @pytest.mark.parametrize( diff --git a/tests/unit/src/sqlbuild/adapters/snowflake/_test_types.py b/tests/unit/src/sqlbuild/adapters/snowflake/_test_types.py index e73f7229f2..6065bcc473 100644 --- a/tests/unit/src/sqlbuild/adapters/snowflake/_test_types.py +++ b/tests/unit/src/sqlbuild/adapters/snowflake/_test_types.py @@ -9,8 +9,8 @@ SchemaDiffResult, ) from sqlbuild.adapter.contract.types import ( - MaintainedListing, - MaintainedOperation, + DynamicListing, + DynamicOperation, TableFreshnessStatus, ) from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter @@ -464,14 +464,14 @@ class SnowflakeWarehouseUserTestCase: @dataclass(frozen=True) -class SnowflakeMaintainedRefreshTestCase: +class SnowflakeDynamicRefreshTestCase: description: str rows: tuple[tuple[object, ...], ...] - expected_settings: MaintainedListing | None + expected_settings: DynamicListing | None @dataclass(frozen=True) -class SnowflakeMaintainedCreateTestCase: +class SnowflakeDynamicCreateTestCase: description: str target_lag: str refresh_mode: str @@ -480,7 +480,7 @@ class SnowflakeMaintainedCreateTestCase: @dataclass(frozen=True) -class SnowflakeMaintainedAlterTestCase: +class SnowflakeDynamicAlterTestCase: description: str target_lag: str warehouse: str | None @@ -488,9 +488,9 @@ class SnowflakeMaintainedAlterTestCase: @dataclass(frozen=True) -class SnowflakeMaintainedOperationTestCase: +class SnowflakeDynamicOperationTestCase: description: str - operation: MaintainedOperation + operation: DynamicOperation expected_sql: str diff --git a/tests/unit/src/sqlbuild/adapters/snowflake/test_maintained_models.py b/tests/unit/src/sqlbuild/adapters/snowflake/test_dynamic_models.py similarity index 81% rename from tests/unit/src/sqlbuild/adapters/snowflake/test_maintained_models.py rename to tests/unit/src/sqlbuild/adapters/snowflake/test_dynamic_models.py index a050065022..8790a71a29 100644 --- a/tests/unit/src/sqlbuild/adapters/snowflake/test_maintained_models.py +++ b/tests/unit/src/sqlbuild/adapters/snowflake/test_dynamic_models.py @@ -1,15 +1,15 @@ import pytest from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder -from sqlbuild.adapter.contract.types import MaintainedListing, MaintainedOperation +from sqlbuild.adapter.contract.types import DynamicListing, DynamicOperation from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter from tests.unit.src.sqlbuild.adapters.helpers import MetadataConnection, execute_through_cursor from tests.unit.src.sqlbuild.adapters.snowflake._test_types import ( SnowflakeChangeTrackingTestCase, - SnowflakeMaintainedAlterTestCase, - SnowflakeMaintainedCreateTestCase, - SnowflakeMaintainedOperationTestCase, - SnowflakeMaintainedRefreshTestCase, + SnowflakeDynamicAlterTestCase, + SnowflakeDynamicCreateTestCase, + SnowflakeDynamicOperationTestCase, + SnowflakeDynamicRefreshTestCase, ) _SHOW_DYNAMIC_TABLES_COLUMNS: tuple[str, ...] = ( @@ -23,17 +23,17 @@ @pytest.mark.parametrize( "test_case", [ - SnowflakeMaintainedRefreshTestCase( + SnowflakeDynamicRefreshTestCase( description="a dynamic table lists its lag, refresh warehouse and schedule state", rows=(("DAILY_TOTALS", "1 hour", "TRANSFORM_WH", "SUSPENDED"),), - expected_settings=MaintainedListing("1 hour", "TRANSFORM_WH", "SUSPENDED"), + expected_settings=DynamicListing("1 hour", "TRANSFORM_WH", "SUSPENDED"), ), - SnowflakeMaintainedRefreshTestCase( + SnowflakeDynamicRefreshTestCase( description="a neighbour matched by the LIKE wildcard is not the model", rows=(("DAILY_TOTALS_ARCHIVE", "DOWNSTREAM", "TRANSFORM_WH", "ACTIVE"),), expected_settings=None, ), - SnowflakeMaintainedRefreshTestCase( + SnowflakeDynamicRefreshTestCase( description="a relation that is no dynamic table lists nothing", rows=(), expected_settings=None, @@ -42,7 +42,7 @@ ids=lambda case: case.description, ) def test_given_dynamic_table_listing_when_reading_it_then_reads_one_show( - test_case: SnowflakeMaintainedRefreshTestCase, monkeypatch: pytest.MonkeyPatch + test_case: SnowflakeDynamicRefreshTestCase, monkeypatch: pytest.MonkeyPatch ) -> None: adapter: SnowflakeAdapter = SnowflakeAdapter() connection: MetadataConnection = MetadataConnection( @@ -50,7 +50,7 @@ def test_given_dynamic_table_listing_when_reading_it_then_reads_one_show( ) monkeypatch.setattr(adapter, "execute", execute_through_cursor) - settings: MaintainedListing | None = adapter.maintained_listing( + settings: DynamicListing | None = adapter.dynamic_listing( connection=connection, database="ANALYTICS", schema="MARTS", name="daily_totals" ) @@ -64,7 +64,7 @@ def test_given_dynamic_table_listing_when_reading_it_then_reads_one_show( @pytest.mark.parametrize( "test_case", [ - SnowflakeMaintainedCreateTestCase( + SnowflakeDynamicCreateTestCase( description="a lag, the default incremental mode and an unquoted warehouse", target_lag="'1 hour'", refresh_mode="incremental", @@ -75,7 +75,7 @@ def test_given_dynamic_table_listing_when_reading_it_then_reads_one_show( "INITIALIZE = ON_CREATE AS SELECT 1 AS orders" ), ), - SnowflakeMaintainedCreateTestCase( + SnowflakeDynamicCreateTestCase( description="refreshed for its readers, in adaptive mode", target_lag="DOWNSTREAM", refresh_mode="adaptive", @@ -86,7 +86,7 @@ def test_given_dynamic_table_listing_when_reading_it_then_reads_one_show( "INITIALIZE = ON_CREATE AS SELECT 1 AS orders" ), ), - SnowflakeMaintainedCreateTestCase( + SnowflakeDynamicCreateTestCase( description="a warehouse name that needs quoting", target_lag="'5 minutes'", refresh_mode="full", @@ -100,10 +100,10 @@ def test_given_dynamic_table_listing_when_reading_it_then_reads_one_show( ], ids=lambda case: case.description, ) -def test_given_maintained_settings_when_rendering_create_then_mode_and_initialize_are_explicit( - test_case: SnowflakeMaintainedCreateTestCase, +def test_given_dynamic_settings_when_rendering_create_then_mode_and_initialize_are_explicit( + test_case: SnowflakeDynamicCreateTestCase, ) -> None: - statements: tuple[str, ...] = SnowflakeAdapter().render_create_maintained( + statements: tuple[str, ...] = SnowflakeAdapter().render_create_dynamic( destination="ANALYTICS.MARTS.DAILY_TOTALS", sql="SELECT 1 AS orders", target_lag=test_case.target_lag, @@ -117,7 +117,7 @@ def test_given_maintained_settings_when_rendering_create_then_mode_and_initializ @pytest.mark.parametrize( "test_case", [ - SnowflakeMaintainedAlterTestCase( + SnowflakeDynamicAlterTestCase( description="an unset refresh warehouse alters only the lag", target_lag="'2 hours'", warehouse=None, @@ -125,7 +125,7 @@ def test_given_maintained_settings_when_rendering_create_then_mode_and_initializ "ALTER DYNAMIC TABLE ANALYTICS.MARTS.DAILY_TOTALS SET TARGET_LAG = '2 hours'", ), ), - SnowflakeMaintainedAlterTestCase( + SnowflakeDynamicAlterTestCase( description="a declared refresh warehouse is set too", target_lag="DOWNSTREAM", warehouse="reporting_wh", @@ -138,9 +138,9 @@ def test_given_maintained_settings_when_rendering_create_then_mode_and_initializ ids=lambda case: case.description, ) def test_given_drifted_settings_when_rendering_alter_then_settings_change_in_place( - test_case: SnowflakeMaintainedAlterTestCase, + test_case: SnowflakeDynamicAlterTestCase, ) -> None: - statements: tuple[str, ...] = SnowflakeAdapter().render_alter_maintained( + statements: tuple[str, ...] = SnowflakeAdapter().render_alter_dynamic( destination="ANALYTICS.MARTS.DAILY_TOTALS", target_lag=test_case.target_lag, warehouse=test_case.warehouse, @@ -152,28 +152,28 @@ def test_given_drifted_settings_when_rendering_alter_then_settings_change_in_pla @pytest.mark.parametrize( "test_case", [ - SnowflakeMaintainedOperationTestCase( + SnowflakeDynamicOperationTestCase( description="suspend", - operation=MaintainedOperation.SUSPEND, + operation=DynamicOperation.SUSPEND, expected_sql="ALTER DYNAMIC TABLE ANALYTICS.MARTS.DAILY_TOTALS SUSPEND", ), - SnowflakeMaintainedOperationTestCase( + SnowflakeDynamicOperationTestCase( description="resume", - operation=MaintainedOperation.RESUME, + operation=DynamicOperation.RESUME, expected_sql="ALTER DYNAMIC TABLE ANALYTICS.MARTS.DAILY_TOTALS RESUME", ), - SnowflakeMaintainedOperationTestCase( + SnowflakeDynamicOperationTestCase( description="refresh", - operation=MaintainedOperation.REFRESH, + operation=DynamicOperation.REFRESH, expected_sql="ALTER DYNAMIC TABLE ANALYTICS.MARTS.DAILY_TOTALS REFRESH", ), ], ids=lambda case: case.description, ) def test_given_operation_when_rendering_then_one_alter_dynamic_table_statement( - test_case: SnowflakeMaintainedOperationTestCase, + test_case: SnowflakeDynamicOperationTestCase, ) -> None: - statements: tuple[str, ...] = SnowflakeAdapter().render_maintained_operation( + statements: tuple[str, ...] = SnowflakeAdapter().render_dynamic_operation( destination="ANALYTICS.MARTS.DAILY_TOTALS", operation=test_case.operation ) @@ -184,7 +184,7 @@ def test_given_operation_when_rendering_then_one_alter_dynamic_table_statement( "test_case", [ SnowflakeChangeTrackingTestCase( - description="a table a maintained model reads tracks changes from creation", + description="a table a dynamic model reads tracks changes from creation", config={"table_type": "transient", "change_tracking": True}, expected_sql=( "CREATE OR REPLACE TRANSIENT TABLE ANALYTICS.STAGING.ORDERS " diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_detect.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_detect.py index ef5d2d2836..9a3c6b2bee 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_detect.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_detect.py @@ -592,26 +592,26 @@ def test_given_dropped_relation_with_recorded_build_when_planning_then_plans_fir ), DetectModelMetadataTestCase( description="writing out the default refresh_mode is no change", - config_values={"materialized": "maintained", "refresh_mode": "incremental"}, + config_values={"materialized": "dynamic", "refresh_mode": "incremental"}, schema_columns=(), deps=(), function_local_hashes={}, previous_metadata_json=build_version_identity_metadata_json( model_name="orders", - config_values={"materialized": "maintained"}, + config_values={"materialized": "dynamic"}, execution_signature={}, ), expected_change_kind=ChangeKind.NO_CHANGE, ), DetectModelMetadataTestCase( description="removing a written-out default refresh_mode is no change", - config_values={"materialized": "maintained"}, + config_values={"materialized": "dynamic"}, schema_columns=(), deps=(), function_local_hashes={}, previous_metadata_json=build_version_identity_metadata_json( model_name="orders", - config_values={"materialized": "maintained", "refresh_mode": "incremental"}, + config_values={"materialized": "dynamic", "refresh_mode": "incremental"}, execution_signature={}, ), expected_change_kind=ChangeKind.NO_CHANGE, @@ -724,13 +724,13 @@ def test_given_model_identity_metadata_when_detecting_changes_then_uses_aligned_ expected_backfill=BackfillResult(action=BackfillAction.FORWARD_ONLY), ), DetectFunctionCallerChangeTestCase( - description="a maintained model's refresh_mode change is a config change", + description="a dynamic model's refresh_mode change is a config change", metadata_case=build_function_caller_metadata_case( - config_values={"materialized": "maintained", "refresh_mode": "full"}, + config_values={"materialized": "dynamic", "refresh_mode": "full"}, function_local_hashes={}, previous_function_hashes={}, previous_config_values={ - "materialized": "maintained", + "materialized": "dynamic", "refresh_mode": "incremental", }, ), @@ -740,12 +740,12 @@ def test_given_model_identity_metadata_when_detecting_changes_then_uses_aligned_ expected_backfill=BackfillResult(action=BackfillAction.FORWARD_ONLY), ), DetectFunctionCallerChangeTestCase( - description="a maintained model's lag change is altered in place, not a config change", + description="a dynamic model's lag change is altered in place, not a config change", metadata_case=build_function_caller_metadata_case( - config_values={"materialized": "maintained", "refresh": {"lag": "5m"}}, + config_values={"materialized": "dynamic", "refresh": {"lag": "5m"}}, function_local_hashes={}, previous_function_hashes={}, - previous_config_values={"materialized": "maintained", "refresh": {"lag": "1h"}}, + previous_config_values={"materialized": "dynamic", "refresh": {"lag": "1h"}}, ), query_change_tracking=True, expected_changed_functions=(), diff --git a/ui/src/lib/api/types/operate.ts b/ui/src/lib/api/types/operate.ts index 38eeff8328..4d3dc595c0 100644 --- a/ui/src/lib/api/types/operate.ts +++ b/ui/src/lib/api/types/operate.ts @@ -656,9 +656,9 @@ export type PlanModel = { backfill: { action: string; duration: string | null }; qualified_name?: string; changed_functions?: string[]; - /** The rebuilt upstream that makes this kept maintained model recompute in full. */ + /** The rebuilt upstream that makes this kept dynamic model recompute in full. */ reinitialised_by?: string; - /** Whether the warehouse lists this kept maintained model as suspended. */ + /** Whether the warehouse lists this kept dynamic model as suspended. */ paused?: boolean; cursor_type?: string; cursor?: { requested_bounds: { start: string | null; end: string | null }; resolved_bounds: { start: string | null; end: string | null } | null } | null; diff --git a/ui/src/lib/run/plan-text.test.ts b/ui/src/lib/run/plan-text.test.ts index e80b045adc..45e6d4f4cd 100644 --- a/ui/src/lib/run/plan-text.test.ts +++ b/ui/src/lib/run/plan-text.test.ts @@ -35,25 +35,25 @@ describe('plan text, as sqb plan prints it', () => { expect(actionText(model('a', { reason: 'first_run', backfill: { action: 'full', duration: null } }))).toBe('continue forward'); }); - it('words maintained models as the CLI does, with no replay policy', () => { - const recreated = model('a', { action: 'create_maintained', reason: 'query_changed', backfill: { action: 'full', duration: null } }); + it('words dynamic models as the CLI does, with no replay policy', () => { + const recreated = model('a', { action: 'create_dynamic', reason: 'query_changed', backfill: { action: 'full', duration: null } }); expect(actionText(recreated)).toBe('recreate, full recompute'); - expect(actionText(model('a', { action: 'create_maintained', reason: 'first_run' }))).toBe('create'); - expect(actionText(model('a', { action: 'alter_maintained', reason: 'config_changed' }))).toBe('change refresh settings in place'); + expect(actionText(model('a', { action: 'create_dynamic', reason: 'first_run' }))).toBe('create'); + expect(actionText(model('a', { action: 'alter_dynamic', reason: 'config_changed' }))).toBe('change refresh settings in place'); const [section] = planSections(plan([recreated])); expect(section.entries[0].detail).toEqual([]); }); - it('says a kept maintained model recomputes in full when an upstream it reads is rebuilt', () => { - const kept = model('daily_totals', { materialization_type: 'maintained', action: 'keep_maintained', reinitialised_by: 'orders' }); + it('says a kept dynamic model recomputes in full when an upstream it reads is rebuilt', () => { + const kept = model('daily_totals', { materialization_type: 'dynamic', action: 'keep_dynamic', reinitialised_by: 'orders' }); const [routine] = planSections(plan([kept])); - expect(routine.entries).toEqual([{ name: 'daily_totals', value: 'maintained', detail: ['recomputes in full: orders is rebuilt'] }]); + expect(routine.entries).toEqual([{ name: 'daily_totals', value: 'dynamic', detail: ['recomputes in full: orders is rebuilt'] }]); }); - it('says a kept maintained model is paused when the warehouse lists it as suspended', () => { - const kept = model('daily_totals', { materialization_type: 'maintained', action: 'keep_maintained', paused: true }); + it('says a kept dynamic model is paused when the warehouse lists it as suspended', () => { + const kept = model('daily_totals', { materialization_type: 'dynamic', action: 'keep_dynamic', paused: true }); const [routine] = planSections(plan([kept])); - expect(routine.entries).toEqual([{ name: 'daily_totals', value: 'maintained', detail: ['paused: scheduled refreshes are suspended'] }]); + expect(routine.entries).toEqual([{ name: 'daily_totals', value: 'dynamic', detail: ['paused: scheduled refreshes are suspended'] }]); }); it('puts each change in its own section in reason order, then the routine models, and leaves skipped ones out', () => { diff --git a/ui/src/lib/run/plan-text.ts b/ui/src/lib/run/plan-text.ts index 97825f856f..6b20698164 100644 --- a/ui/src/lib/run/plan-text.ts +++ b/ui/src/lib/run/plan-text.ts @@ -97,8 +97,8 @@ export function materializationLabel(m: PlanModel): string { export function actionText(m: PlanModel): string { if (m.action === 'create_view') return 'recreate view'; if (m.action === 'custom') return 'run custom materialization'; - if (m.action === 'create_maintained') return m.reason === 'first_run' ? 'create' : 'recreate, full recompute'; - if (m.action === 'alter_maintained') return 'change refresh settings in place'; + if (m.action === 'create_dynamic') return m.reason === 'first_run' ? 'create' : 'recreate, full recompute'; + if (m.action === 'alter_dynamic') return 'change refresh settings in place'; if (m.backfill.action === 'bounded' && m.backfill.duration) return `rebuild last ${m.backfill.duration}`; if (m.backfill.action === 'full' && m.reason !== 'first_run') return 'full rebuild'; return 'continue forward'; @@ -118,7 +118,7 @@ function cursorLines(m: PlanModel, showRange: boolean, windowStart?: string): st } function policyLine(m: PlanModel): string[] { - if (m.backfill.action === 'forward' || m.action === 'create_maintained' || !(m.reason === 'query_changed' || m.reason === 'schema_changed')) return []; + if (m.backfill.action === 'forward' || m.action === 'create_dynamic' || !(m.reason === 'query_changed' || m.reason === 'schema_changed')) return []; const value = m.backfill.action === 'bounded' ? `bounded-${m.backfill.duration ?? 'full'}` : m.backfill.action; return [`policy: replay_on_change=${value}`]; } diff --git a/website/src/content/docs/docs/cli/build.mdx b/website/src/content/docs/docs/cli/build.mdx index 2a69160633..12e7916aca 100644 --- a/website/src/content/docs/docs/cli/build.mdx +++ b/website/src/content/docs/docs/cli/build.mdx @@ -49,7 +49,7 @@ own option. `--version` works only directly after `sqb`, as in `sqb --version`: | `--defer-sources-to` | Read managed source data from another target | | `--plan-from` | Read warehouse state `live`, or from the target's `synced_state` checked against the warehouse first; overrides the target's `plan_from`. See [Planning from synced state](/docs/concepts/project-configuration#planning-from-synced-state) | | `--fail-fast` | Stop on first failure and skip remaining nodes | -| `--ignore-maintained-grace` | Take over [maintained models](/docs/concepts/models/maintained#when-a-build-dies) another build holds, without waiting for `[maintained] heartbeat_grace`. Needs `allow_ignore_grace = true`, else fails with `A719` | +| `--ignore-dynamic-grace` | Take over [dynamic models](/docs/concepts/models/dynamic#when-a-build-dies) another build holds, without waiting for `[dynamic] heartbeat_grace`. Needs `allow_ignore_grace = true`, else fails with `A719` | | `--concurrency` | Number of worker connections (default: 1) | | `--test-timeout ` | Time limit for each unit test the build runs, such as `30s`, or `disabled`; see [Time limits](/docs/concepts/testing-reference#time-limits) | | `--verbose`, `-v` | Show lifecycle SQL inline after each model | diff --git a/website/src/content/docs/docs/cli/dag.mdx b/website/src/content/docs/docs/cli/dag.mdx index cd8a98262e..f945007864 100644 --- a/website/src/content/docs/docs/cli/dag.mdx +++ b/website/src/content/docs/docs/cli/dag.mdx @@ -96,7 +96,7 @@ Each node represents a source, seed, model, or function: | `description` | Model or source description, if declared | | `tags` | Model tags | | `columns` | Column metadata (name, type, nullable, description) | -| `materialization_type` | For models: `view`, `table`, `incremental`, `snapshot`, `maintained`, or custom name | +| `materialization_type` | For models: `view`, `table`, `incremental`, `snapshot`, `dynamic`, or custom name | | `language` | For functions: `sql` or `python` | | `return_kind` | For functions: `scalar` or `table` | | `arguments` | For functions: argument name and type pairs | diff --git a/website/src/content/docs/docs/concepts/adapters/snowflake.mdx b/website/src/content/docs/docs/concepts/adapters/snowflake.mdx index 87400a7ee8..29194563a5 100644 --- a/website/src/content/docs/docs/concepts/adapters/snowflake.mdx +++ b/website/src/content/docs/docs/concepts/adapters/snowflake.mdx @@ -150,10 +150,10 @@ SQLBuild records its state in `_sqlbuild_*` tables. On Snowflake it creates them with 90 days of time travel, or 1 day on editions that allow only that. It never alters an existing state table. Models are not affected: their table type follows the model's `table_type` setting. -## Maintained models +## Dynamic models -A [maintained model](/docs/concepts/models/maintained) is a Snowflake dynamic -table. `materialized materialized_view` fails with `A014`; use `materialized maintained`. +A [dynamic model](/docs/concepts/models/dynamic) is a Snowflake dynamic +table. `materialized materialized_view` fails with `A014`; use `materialized dynamic`. The build role needs these grants: @@ -165,12 +165,12 @@ The build role needs these grants: Dynamic tables refresh from changes, which needs change tracking on every table they read: -- A table or snapshot model that a maintained model reads, directly or through views, is built with +- A table or snapshot model that a dynamic model reads, directly or through views, is built with change tracking on. - For a source table, Snowflake turns change tracking on when the build role owns the table. Otherwise its owner runs `ALTER TABLE SET CHANGE_TRACKING = TRUE`. -Builds record the models they pause in the `_sqlbuild_maintained_pauses` state table, so another +Builds record the models they pause in the `_sqlbuild_dynamic_pauses` state table, so another build can tell whether they are still held. ## Shared connections across targets diff --git a/website/src/content/docs/docs/concepts/models/configuration.mdx b/website/src/content/docs/docs/concepts/models/configuration.mdx index c72e3487cf..8755d418db 100644 --- a/website/src/content/docs/docs/concepts/models/configuration.mdx +++ b/website/src/content/docs/docs/concepts/models/configuration.mdx @@ -20,7 +20,7 @@ which accepts any keys. | Field | Description | |-------|-------------| -| `materialized` | `view`, `table`, `incremental`, `snapshot`, `maintained`, or a custom materialization name; see [Materializations](/docs/concepts/models/materializations) | +| `materialized` | `view`, `table`, `incremental`, `snapshot`, `dynamic`, or a custom materialization name; see [Materializations](/docs/concepts/models/materializations) | | `tags` | Tags used by selectors | | `critical` | `true` marks the model as critical, so [`sqb review`](/docs/cli/review) builds and compares it whenever something upstream of it changes (default `false`) | | `description` | Required human-readable model description; see [Descriptions](#descriptions) | diff --git a/website/src/content/docs/docs/concepts/models/maintained.mdx b/website/src/content/docs/docs/concepts/models/dynamic.mdx similarity index 67% rename from website/src/content/docs/docs/concepts/models/maintained.mdx rename to website/src/content/docs/docs/concepts/models/dynamic.mdx index 3ab5dedc62..9cb53ce8ff 100644 --- a/website/src/content/docs/docs/concepts/models/maintained.mdx +++ b/website/src/content/docs/docs/concepts/models/dynamic.mdx @@ -1,16 +1,16 @@ --- -title: "Maintained Models" +title: "Dynamic models" description: "Let the warehouse keep a model up to date on a schedule, paused safely while builds run." --- -A maintained model is one the warehouse keeps up to date on a schedule you set, between builds. A +A dynamic model is one the warehouse keeps up to date on a schedule you set, between builds. A build creates it, changes it when its definition changes, and otherwise leaves the refreshing to the warehouse; on Snowflake it is a dynamic table. ```sql MODEL ( description "One row per customer per order date", - materialized maintained, + materialized dynamic, refresh (lag 1h), ); @@ -19,14 +19,14 @@ FROM __ref("stg_orders") GROUP BY customer_id, CAST(ordered_at AS DATE) ``` -Only Snowflake supports maintained models. See [Snowflake](/docs/concepts/adapters/snowflake#maintained-models) +Only Snowflake supports dynamic models. See [Snowflake](/docs/concepts/adapters/snowflake#dynamic-models) for the grants and change tracking they need. ## Settings | Field | Default | Description | |-------|---------|-------------| -| `refresh (lag )` | none | How far behind its inputs the model may fall: a fixed duration such as `5m`, `1h` or `1d`, at least `1m` on Snowflake and no shorter than the lag of a maintained model it reads. Without it, the model refreshes when a maintained model that reads it refreshes | +| `refresh (lag )` | none | How far behind its inputs the model may fall: a fixed duration such as `5m`, `1h` or `1d`, at least `1m` on Snowflake and no shorter than the lag of a dynamic model it reads. Without it, the model refreshes when a dynamic model that reads it refreshes | | `refresh_mode` | `incremental` | `incremental`, `full`, `auto` or `adaptive`. Snowflake rejects `incremental` for a query it can't refresh incrementally; use `full` for that query | | `refresh_warehouse` | the connection's warehouse | Warehouse the scheduled refreshes run on | | `refresh_on_build` | `false` | Refresh the model in every build that selects it, not only on its schedule | @@ -38,7 +38,7 @@ for the grants and change tracking they need. | First build, or the dynamic table is missing | `create` | Creates it and fills it | | Query or `refresh_mode` changed | `recreate, full recompute` | Replaces it; every row is computed again | | `refresh` lag or `refresh_warehouse` differs from the warehouse | `change refresh settings in place` | Alters the schedule; rows stay | -| Nothing | `maintained` | Leaves it alone | +| Nothing | `dynamic` | Leaves it alone | A model the plan keeps can carry one more line: @@ -47,7 +47,7 @@ A model the plan keeps can carry one more line: ## During a build -A build pauses the maintained models it changes, and those that read a model it rebuilds, directly +A build pauses the dynamic models it changes, and those that read a model it rebuilds, directly or through views, so they never refresh from half-built inputs. It resumes them when it ends, whether it succeeds or fails. @@ -56,23 +56,23 @@ whether it succeeds or fails. - A suspend made by someone else during the build is undone when the build resumes the model. - A kept model refreshes during the build when a model built later in the same build reads it. - `refresh_on_build` refreshes the model in every build that selects it. -- A target with `pause_maintained = true` leaves its maintained models suspended between builds; see [Targets](/docs/concepts/project-configuration#targets). +- A target with `pause_dynamic = true` leaves its dynamic models suspended between builds; see [Targets](/docs/concepts/project-configuration#targets). ## When a build dies -- **One build at a time.** A build that needs a maintained model another build holds fails with +- **One build at a time.** A build that needs a dynamic model another build holds fails with `A718`, naming that run. Wait for it to finish. Two builds that start at the same moment can both fail this way; run one of them again. - **A stopped build leaves its models suspended.** A build killed before it resumes them, for example by `SIGKILL`, can't resume them itself. -- **The next build resumes them.** The next build that pauses maintained models on the target - resumes them once the stopped build has been silent for `[maintained] heartbeat_grace`, 30 - minutes by default; see [Maintained](/docs/concepts/project-configuration#maintained). -- **Taking over at once.** Run `sqb build --ignore-maintained-grace` to take the models over - without waiting. It needs `[maintained] allow_ignore_grace = true`; without it the flag fails +- **The next build resumes them.** The next build that pauses dynamic models on the target + resumes them once the stopped build has been silent for `[dynamic] heartbeat_grace`, 30 + minutes by default; see [Dynamic](/docs/concepts/project-configuration#dynamic). +- **Taking over at once.** Run `sqb build --ignore-dynamic-grace` to take the models over + without waiting. It needs `[dynamic] allow_ignore_grace = true`; without it the flag fails with `A719`. You can also resume the models in Snowflake by hand. ## Limitations -- Unit tests and scenarios treat a maintained model as a table. -- On a target with `schema = "preserve"`, builds see each other's pauses only when their maintained models share a schema. +- Unit tests and scenarios treat a dynamic model as a table. +- On a target with `schema = "preserve"`, builds see each other's pauses only when their dynamic models share a schema. diff --git a/website/src/content/docs/docs/concepts/models/materializations.mdx b/website/src/content/docs/docs/concepts/models/materializations.mdx index 827b8616ed..f18aef09bb 100644 --- a/website/src/content/docs/docs/concepts/models/materializations.mdx +++ b/website/src/content/docs/docs/concepts/models/materializations.mdx @@ -71,19 +71,19 @@ FROM __source("customers") See [Snapshots](/docs/concepts/snapshots) for timestamp and check strategies, historical inputs, and full-refresh policies. -## Maintained +## Dynamic The warehouse keeps the model up to date on a schedule you set, and builds pause it while they change its inputs: ```sql -MODEL (description "One row per customer per order date", materialized maintained, refresh (lag 1h)); +MODEL (description "One row per customer per order date", materialized dynamic, refresh (lag 1h)); SELECT customer_id, CAST(ordered_at AS DATE) AS order_date, SUM(amount) AS revenue FROM __ref("stg_orders") GROUP BY customer_id, CAST(ordered_at AS DATE) ``` -See [Maintained models](/docs/concepts/models/maintained) for its settings, what the plan shows and what happens when a build dies. +See [Dynamic models](/docs/concepts/models/dynamic) for its settings, what the plan shows and what happens when a build dies. ## Custom @@ -122,5 +122,5 @@ The custom function owns staging, runtime type or contract enforcement, audit ti | Table (`immediate`) | The new table. Audits run after it is replaced. | | Incremental | The old table if a delta audit fails. Rows already written stay if a final audit or a later microbatch fails. | | Microbatch full rebuild | A partly rebuilt table; the old one was dropped first. | -| Maintained | The created or altered dynamic table, resumed. Audits run after it changes. | +| Dynamic | The created or altered dynamic table, resumed. Audits run after it changes. | | Custom | Whatever the custom materialization does. | diff --git a/website/src/content/docs/docs/concepts/project-configuration.mdx b/website/src/content/docs/docs/concepts/project-configuration.mdx index 52fb28a2c0..827f300ec7 100644 --- a/website/src/content/docs/docs/concepts/project-configuration.mdx +++ b/website/src/content/docs/docs/concepts/project-configuration.mdx @@ -163,7 +163,7 @@ A target is a named build context - the database and schema you build into, plus | `concurrency` | Worker limit for commands run against this target, replacing `[settings] concurrency` (see below) | | `ignore_audit_failures` | Whether builds on this target let audit failures through, replacing `[settings] ignore_audit_failures`; see [Ignoring audit failures](/docs/concepts/audits#ignoring-audit-failures) | | `plan_from` | Where `sqb plan` and `sqb build` read warehouse state: `live` (default) or `synced_state`; see [Planning from synced state](#planning-from-synced-state) | -| `pause_maintained` | `true` keeps this target's [maintained models](/docs/concepts/models/maintained) suspended between builds; each build refreshes the ones it selects. Default `false` | +| `pause_dynamic` | `true` keeps this target's [dynamic models](/docs/concepts/models/dynamic) suspended between builds; each build refreshes the ones it selects. Default `false` | ```toml [connections.warehouse] @@ -670,20 +670,20 @@ old_name_views = "30d" |-------|---------|-------------| | `old_name_views` | `"30d"` | How long a renamed model's old name keeps working through a [compatibility view](/docs/concepts/models/migrations#old-names) before janitor drops it. `false` keeps the old relation in place instead. A model's `old_name_view` header overrides it. | -## Maintained +## Dynamic -How builds pause [maintained models](/docs/concepts/models/maintained) while they run: +How builds pause [dynamic models](/docs/concepts/models/dynamic) while they run: ```toml -[maintained] +[dynamic] heartbeat_grace = "30m" allow_ignore_grace = false ``` | Field | Default | Description | |-------|---------|-------------| -| `heartbeat_grace` | `"30m"` | How long a build that paused maintained models may go silent before another build resumes them or takes them over. A fixed duration longer than `"1m"`, such as `"30m"` or `"2h"`; a build writes a heartbeat every minute | -| `allow_ignore_grace` | `false` | Allow `sqb build --ignore-maintained-grace`, which takes over models another build holds without waiting for the grace. Without it the flag fails with `A719` | +| `heartbeat_grace` | `"30m"` | How long a build that paused dynamic models may go silent before another build resumes them or takes them over. A fixed duration longer than `"1m"`, such as `"30m"` or `"2h"`; a build writes a heartbeat every minute | +| `allow_ignore_grace` | `false` | Allow `sqb build --ignore-dynamic-grace`, which takes over models another build holds without waiting for the grace. Without it the flag fails with `A719` | ## Scenario diff --git a/website/src/sidebar.json b/website/src/sidebar.json index 0be8c87a80..a2d01907f1 100644 --- a/website/src/sidebar.json +++ b/website/src/sidebar.json @@ -31,7 +31,7 @@ "docs/concepts/seeds", "docs/concepts/incremental", "docs/concepts/snapshots", - "docs/concepts/models/maintained", + "docs/concepts/models/dynamic", "docs/concepts/functions", { "label": "Macros",