diff --git a/benchmarks/duckshop/edge_cases/orchestration/dynamic_hold_without_dynamic_adapter.json b/benchmarks/duckshop/edge_cases/orchestration/dynamic_hold_without_dynamic_adapter.json new file mode 100644 index 0000000000..67241b1cb7 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/dynamic_hold_without_dynamic_adapter.json @@ -0,0 +1,13 @@ +{ + "model": "MODEL (description 'Orders.', materialized table); SELECT 17 AS amount\n", + "dynamic": "[dynamic]\nheartbeat_grace = \"10m\"\n", + "holds": [ + {"run_id": "stopped-build", "model": "daily_totals", "idle_seconds": 900}, + {"run_id": "running-build", "model": "weekly_totals", "idle_seconds": 0} + ], + "lifecycle": [ + "a DuckDB warehouse holds a pause table with one hold silent past the project's grace and one live hold", + "a served location's sync completes", + "DuckDB cannot build dynamic models, so the sync opens no resume session and both holds stay open" + ] +} diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/location/dynamic_resume.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/location/dynamic_resume.rs new file mode 100644 index 0000000000..3890a69b17 --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/location/dynamic_resume.rs @@ -0,0 +1,122 @@ +//! A completed synchronization resumes the dynamic models whose pausing builds stopped heartbeating. + +use std::time::Duration; + +use sqlbuild_core::constants::DYNAMIC_RESUME_SECONDS; +use sqlbuild_orchestration::warehouse_metadata::constants::DYNAMIC_RESUME_REQUEST; +use sqlbuild_orchestration::warehouse_metadata::models::{ + DynamicResumeLocation, DynamicResumeReply, +}; +use tokio::io::{AsyncBufReadExt, AsyncReadExt, AsyncWriteExt}; + +use crate::argv::models::Stream; +use crate::native_commands::_helpers::command::terminal::write_output; +use crate::native_commands::_helpers::location::execution::location_warning; +use crate::native_commands::constants::METADATA_REPLY_BYTES; +use crate::native_commands::models::{LocationCompiler, LocationMetadataSync}; + +/// Ask the reader to resume expired holds, sending each location's quiet token; a failure only warns. +pub(crate) async fn resume_expired_holds(mut state: LocationMetadataSync) -> LocationMetadataSync { + if state.halted { + return state; + } + let Some(reader) = state.reader.take() else { + return state; + }; + let locations: Vec = state + .locations + .iter() + .chain( + state + .dynamic_quiet_tokens + .keys() + .filter(|location| !state.locations.contains(location)), + ) + .map(|location| DynamicResumeLocation { + location: location.clone(), + quiet_token: state.dynamic_quiet_tokens.get(location).cloned(), + }) + .collect(); + let request = serde_json::json!({ DYNAMIC_RESUME_REQUEST: locations }).to_string(); + let reason = match tokio::time::timeout( + Duration::from_secs(DYNAMIC_RESUME_SECONDS), + exchange(reader, request), + ) + .await + { + Ok((reader, Ok(DynamicResumeReply::Resumed(resumed)))) => { + state.reader = Some(reader); + state.dynamic_quiet_tokens = resumed + .quiet_tokens + .into_iter() + .filter_map(|resume| Some((resume.location, resume.quiet_token?))) + .collect(); + for failure in &resumed.failures { + drop(write_output(&[( + Stream::Stderr, + &format!("Resuming an expired dynamic hold failed: {failure}\n"), + )])); + } + for hold in resumed.holds { + drop(write_output(&[( + Stream::Stderr, + &format!( + "Resumed dynamic model {}: run {} stopped heartbeating.\n", + hold.relation, hold.run_id + ), + )])); + } + return state; + } + Ok((reader, Ok(DynamicResumeReply::Failed(reason)))) => { + state.reader = Some(reader); + reason + } + Ok((reader, Err(reason))) => { + state.reader = Some(reader); + drop(write_output(&[( + Stream::Stderr, + &format!("Resuming expired dynamic holds failed: {reason}\n"), + )])); + reason + } + Err(_) => { + let reason = format!("no reply in {DYNAMIC_RESUME_SECONDS} s"); + drop(write_output(&[( + Stream::Stderr, + &format!("Resuming expired dynamic holds failed: {reason}\n"), + )])); + reason + } + }; + location_warning("dynamic resume failed", reason); + state +} + +/// One request line and its bounded reply line, handing the reader back. +async fn exchange( + mut reader: LocationCompiler, + request: String, +) -> (LocationCompiler, Result) { + let reply: Result = async { + let fail = |error: std::io::Error| error.to_string(); + reader + .stdin + .write_all(format!("{request}\n").as_bytes()) + .await + .map_err(fail)?; + reader.stdin.flush().await.map_err(fail)?; + let mut line = String::new(); + let read = (&mut reader.stdout) + .take(METADATA_REPLY_BYTES + 1) + .read_line(&mut line) + .await + .map_err(fail)?; + if read == 0 { + return Err("location runtime exited before replying".into()); + } + serde_json::from_str(&line).map_err(|_| "invalid location reply".to_owned()) + } + .await; + (reader, reply) +} diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/location/execution.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/location/execution.rs index 5252ad6ac4..f46ed7d471 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/location/execution.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/location/execution.rs @@ -31,6 +31,7 @@ use crate::native_commands::_helpers::command::terminal::{ finish_lost_stdout, terminal_columns, write_output, }; use crate::native_commands::_helpers::command::terminal::{live_screen, stderr_columns}; +use crate::native_commands::_helpers::location::dynamic_resume::resume_expired_holds; use crate::native_commands::_helpers::location::sync_report::{ metadata_health, read_timeout_seconds, rows_text, scan_line, }; @@ -512,6 +513,7 @@ pub(crate) fn metadata_sync( state_history_versions: None, progress: SyncProgress::new(), reads_wake_compute: false, + dynamic_quiet_tokens: std::collections::BTreeMap::new(), } } @@ -1438,14 +1440,20 @@ pub(crate) async fn read_metadata_page( Ok(Some((state, read))) } -/// A completed synchronization closes its reader, so no warehouse session outlives the sync. +/// A completed synchronization resumes expired dynamic holds, then closes its reader, so no warehouse session outlives the sync. pub(crate) async fn finish_metadata_page( state: LocationMetadataSync, + stopped: impl Future>, ) -> Result, ControlError> { - if state.idle { - return Ok(Some(close_metadata_reader(state).await?)); + if !state.idle { + return Ok(Some(state)); } - Ok(Some(state)) + tokio::pin!(stopped); + let state: LocationMetadataSync = tokio::select! { + result = &mut stopped => { result?; return Ok(None); }, + state = resume_expired_holds(state) => state, + }; + Ok(Some(close_metadata_reader(state).await?)) } /// End of input lets the reader close its warehouse session; one that ignores it is killed. @@ -1636,13 +1644,16 @@ pub(crate) async fn settle_publication( stopped: impl Future>, ) -> Result, ControlError> { let (mut metadata, deferred) = if connect { - let sync: LocationMetadataSync = metadata_sync( + let mut sync: LocationMetadataSync = metadata_sync( (locations, generation), ( options.metadata_reconcile_seconds, options.safety_sync_seconds, ), ); + sync.dynamic_quiet_tokens = metadata + .map(|previous| previous.dynamic_quiet_tokens) + .unwrap_or_default(); (Some(sync), None) } else { (metadata, Some((locations, generation))) diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/location/mod.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/location/mod.rs index ad8a09edea..d4763cff42 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/location/mod.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/location/mod.rs @@ -1,4 +1,5 @@ pub(crate) mod changed_build; +pub(crate) mod dynamic_resume; pub(crate) mod execution; pub(crate) mod held_copy; pub(crate) mod plan_document; diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/location/sync_outcomes.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/location/sync_outcomes.rs index e488f2b107..72fb8c532a 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/location/sync_outcomes.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/location/sync_outcomes.rs @@ -141,5 +141,5 @@ pub(crate) async fn deliver_metadata_page( result = &mut stopped => { result?; return Ok(None); }, () = metadata_health(&state, (client, poll), (started, None)) => {}, } - finish_metadata_page(acknowledge_metadata_page(state)).await + finish_metadata_page(acknowledge_metadata_page(state), stopped).await } diff --git a/crates/sqlbuild-cli/src/native_commands/main/_location_metadata.rs b/crates/sqlbuild-cli/src/native_commands/main/_location_metadata.rs index 5c4216cbe4..eaab7273b5 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/_location_metadata.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/_location_metadata.rs @@ -99,7 +99,7 @@ pub(crate) async fn sync_location_metadata( result = &mut stopped => { result?; return Ok(None); }, () = metadata_health(&state, (client, poll), (started, None)) => {}, } - return finish_metadata_page(next_scan(state)).await; + return finish_metadata_page(next_scan(state), stopped.as_mut()).await; } state = freeze_history_page(state, page, poll)?; } diff --git a/crates/sqlbuild-cli/src/native_commands/models.rs b/crates/sqlbuild-cli/src/native_commands/models.rs index 46655c1562..09ee9d6a56 100644 --- a/crates/sqlbuild-cli/src/native_commands/models.rs +++ b/crates/sqlbuild-cli/src/native_commands/models.rs @@ -287,6 +287,8 @@ pub(crate) struct LocationMetadataSync { pub(crate) applied: bool, /// The generation was published after the last plan of what changed, which waits for this sync. pub(crate) replan: bool, + /// Per state location, the pause table's change token after a resume pass left no hold open. + pub(crate) dynamic_quiet_tokens: std::collections::BTreeMap, } /// Rows delivered and time taken by the current synchronization and its current table. diff --git a/crates/sqlbuild-cli/src/native_commands/tests/helpers.rs b/crates/sqlbuild-cli/src/native_commands/tests/helpers.rs index 775f6eb238..b6b61c438a 100644 --- a/crates/sqlbuild-cli/src/native_commands/tests/helpers.rs +++ b/crates/sqlbuild-cli/src/native_commands/tests/helpers.rs @@ -85,6 +85,20 @@ pub(super) fn one_location() -> Vec { }] } +/// A location in `directory` served by an unreachable server. +pub(super) fn location_options(directory: &Path) -> LocationServe { + LocationServe { + project_dir: directory.to_path_buf(), + server_url: "http://127.0.0.1:1".into(), + location_id: Uuid::nil(), + target: "development".into(), + image_id: None, + metadata_reconcile_seconds: 3600, + safety_sync_seconds: 1800, + dev_events: false, + } +} + /// A location in `directory` whose `sqb debug` reports `wakes_compute` and whose every history read fails, published to an unreachable server: its options, script, work channel and the sync the publication starts. pub(super) fn published_location( directory: &Path, @@ -93,6 +107,22 @@ pub(super) fn published_location( (LocationServe, std::path::PathBuf), (LocationWorkClient, LocationWorkPoll), LocationMetadataSync, +) { + let (location, work, mut state) = republished_location(directory, wakes_compute, None); + state.begin_sync(chrono::Utc::now()); + state.idle = false; + (location, work, state) +} + +/// A location in `directory` published again over the sync `previous` left, and the new sync. +pub(super) fn republished_location( + directory: &Path, + wakes_compute: bool, + previous: Option, +) -> ( + (LocationServe, std::path::PathBuf), + (LocationWorkClient, LocationWorkPoll), + LocationMetadataSync, ) { let script = directory.join(format!("location-{wakes_compute}")); std::fs::write( @@ -107,16 +137,7 @@ pub(super) fn published_location( let mut permissions = std::fs::metadata(&script).expect("metadata").permissions(); std::os::unix::fs::PermissionsExt::set_mode(&mut permissions, 0o755); std::fs::set_permissions(&script, permissions).expect("executable"); - let options = LocationServe { - project_dir: directory.to_path_buf(), - server_url: "http://127.0.0.1:1".into(), - location_id: Uuid::nil(), - target: "development".into(), - image_id: None, - metadata_reconcile_seconds: 3600, - safety_sync_seconds: 1800, - dev_events: false, - }; + let options = location_options(directory); let client = connect_location_work((&options.server_url, "token"), options.location_id) .expect("work client"); let poll = LocationWorkPoll { @@ -130,14 +151,12 @@ pub(super) fn published_location( .block_on(settle_publication( (&options, &script, ("token", Uuid::nil())), ((one_location(), 1), true, ConnectionCheckTrigger::Publish), - (location_watch(directory), None), + (location_watch(directory), previous), std::future::pending(), )) .expect("no control error") .expect("not stopped"); - let mut state = metadata.expect("a connected publication starts a sync"); - state.begin_sync(chrono::Utc::now()); - state.idle = false; + let state = metadata.expect("a connected publication starts a sync"); ((options, script), (client, poll), state) } diff --git a/crates/sqlbuild-cli/src/native_commands/tests/test_location_metadata.rs b/crates/sqlbuild-cli/src/native_commands/tests/test_location_metadata.rs index a5c34766e6..b77835d974 100644 --- a/crates/sqlbuild-cli/src/native_commands/tests/test_location_metadata.rs +++ b/crates/sqlbuild-cli/src/native_commands/tests/test_location_metadata.rs @@ -1,5 +1,8 @@ use crate::native_commands::_helpers::command::paths::location_watch; use crate::native_commands::_helpers::location::execution::metadata_sync; +use crate::native_commands::_helpers::location::execution::{ + finish_metadata_page, open_location_runtime, +}; use crate::native_commands::_helpers::location::execution::{ history_request, read_metadata_page, safety_sync_setting, }; @@ -11,10 +14,13 @@ use crate::native_commands::constants::{METADATA_FAILURE_LIMIT, METADATA_SAFETY_ use crate::native_commands::main::_location_metadata::sync_location_metadata; use crate::native_commands::models::{LocationMetadataPage, LocationServe, ReadFailure}; use crate::native_commands::tests::helpers::{ - acknowledge_complete_pages, one_location, published_location, + acknowledge_complete_pages, location_options, one_location, published_location, + republished_location, }; use crate::native_commands::tests::test_types::BilledDeferralTestCase; +use crate::native_commands::tests::test_types::DynamicResumeTestCase; use crate::native_commands::tests::test_types::FastClockScanTestCase; +use crate::native_commands::tests::test_types::RepublishTokenTestCase; use crate::native_commands::tests::test_types::{ FailedReadHealthTestCase, NewestVersionsTestCase, ReadFailureTestCase, }; @@ -1268,3 +1274,182 @@ fn given_a_fast_location_clock_when_scanning_then_a_ddl_after_the_page_read_is_s ); } } + +#[test] +fn given_a_completed_sync_when_finishing_then_its_reader_resumes_expired_holds_once_before_closing() +{ + const UNTOKENED: &str = r#"{"dynamic_resume":[{"location":{"database":"analytics","schema":"state"},"quiet_token":null}]}"#; + const TOKENED: &str = r#"{"dynamic_resume":[{"location":{"database":"analytics","schema":"state"},"quiet_token":"7/12:00"}]}"#; + let test_cases = [ + DynamicResumeTestCase { + description: "a completed sync sends the locations it read, then closes the reader", + idle: true, + halted: false, + quiet_token: None, + reply: r#"{"resumed":{"holds":[{"run_id":"run-1","relation":"analytics.marts.orders"}],"quiet_tokens":[{"location":{"database":"analytics","schema":"state"},"quiet_token":null}],"failures":[]}}"#, + expected_requests: &[UNTOKENED], + expected_reader_open: false, + expected_quiet_token: None, + }, + DynamicResumeTestCase { + description: "a pass that leaves no hold open keeps its token for the next sync", + idle: true, + halted: false, + quiet_token: None, + reply: r#"{"resumed":{"holds":[],"quiet_tokens":[{"location":{"database":"analytics","schema":"state"},"quiet_token":"7/12:00"}],"failures":[]}}"#, + expected_requests: &[UNTOKENED], + expected_reader_open: false, + expected_quiet_token: Some("7/12:00"), + }, + DynamicResumeTestCase { + description: "a kept token is sent, and a pass that finds a hold drops it", + idle: true, + halted: false, + quiet_token: Some("7/12:00"), + reply: r#"{"resumed":{"holds":[],"quiet_tokens":[{"location":{"database":"analytics","schema":"state"},"quiet_token":null}],"failures":[]}}"#, + expected_requests: &[TOKENED], + expected_reader_open: false, + expected_quiet_token: None, + }, + DynamicResumeTestCase { + description: "a hold that failed to resume is reported, and the pass's tokens still apply", + idle: true, + halted: false, + quiet_token: Some("7/12:00"), + reply: r#"{"resumed":{"holds":[],"quiet_tokens":[{"location":{"database":"analytics","schema":"state"},"quiet_token":null}],"failures":["analytics.marts.orders: insufficient privileges"]}}"#, + expected_requests: &[TOKENED], + expected_reader_open: false, + expected_quiet_token: None, + }, + DynamicResumeTestCase { + description: "a failed resume only warns, keeps the token, and the reader still closes", + idle: true, + halted: false, + quiet_token: Some("7/12:00"), + reply: r#"{"failed":"permission denied"}"#, + expected_requests: &[TOKENED], + expected_reader_open: false, + expected_quiet_token: Some("7/12:00"), + }, + DynamicResumeTestCase { + description: "an unreadable reply only warns, and the reader still closes", + idle: true, + halted: false, + quiet_token: None, + reply: "not json", + expected_requests: &[UNTOKENED], + expected_reader_open: false, + expected_quiet_token: None, + }, + DynamicResumeTestCase { + description: "a sync stopped by failed reads writes nothing", + idle: true, + halted: true, + quiet_token: None, + reply: r#"{"resumed":{"holds":[],"quiet_tokens":[{"location":{"database":"analytics","schema":"state"},"quiet_token":"7/12:00"}],"failures":[]}}"#, + expected_requests: &[], + expected_reader_open: false, + expected_quiet_token: None, + }, + DynamicResumeTestCase { + description: "a sync with scans left neither resumes nor closes its reader", + idle: false, + halted: false, + quiet_token: None, + reply: r#"{"resumed":{"holds":[],"quiet_tokens":[{"location":{"database":"analytics","schema":"state"},"quiet_token":"7/12:00"}],"failures":[]}}"#, + expected_requests: &[], + expected_reader_open: true, + expected_quiet_token: None, + }, + ]; + for test_case in test_cases { + let directory = tempfile::tempdir().expect("temporary directory"); + let requests = directory.path().join("requests"); + let script = directory.path().join("location"); + std::fs::write( + &script, + format!( + "#!/bin/sh\nwhile read line; do echo \"$line\" >> '{}'; echo '{}'; done\n", + requests.display(), + test_case.reply + ), + ) + .expect("location script"); + let mut permissions = std::fs::metadata(&script).expect("metadata").permissions(); + std::os::unix::fs::PermissionsExt::set_mode(&mut permissions, 0o755); + std::fs::set_permissions(&script, permissions).expect("executable"); + let options = location_options(directory.path()); + let state = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("runtime") + .block_on(async { + let mut state = metadata_sync((one_location(), 1), (3600, 1800)); + state.idle = test_case.idle; + state.halted = test_case.halted; + state.dynamic_quiet_tokens.extend( + test_case + .quiet_token + .map(|token| (one_location().remove(0), token.to_owned())), + ); + state.reader = + Some(open_location_runtime(&options, &script, Vec::new()).expect("runtime")); + finish_metadata_page(state, std::future::pending()).await + }) + .expect("no control error") + .expect("not stopped"); + let received = std::fs::read_to_string(&requests).unwrap_or_default(); + assert_eq!( + ( + received.lines().collect::>(), + state.reader.is_some(), + state + .dynamic_quiet_tokens + .get(&one_location().remove(0)) + .map(String::as_str), + ), + ( + test_case.expected_requests.to_vec(), + test_case.expected_reader_open, + test_case.expected_quiet_token, + ), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_a_quiet_pause_table_when_the_location_publishes_again_then_the_new_sync_keeps_its_token() { + let test_cases = [ + RepublishTokenTestCase { + description: "a publication after a quiet sync keeps the pause table's token", + previous_token: Some("7/12:00"), + expected_token: Some("7/12:00"), + }, + RepublishTokenTestCase { + description: "the first publication has no token yet", + previous_token: None, + expected_token: None, + }, + ]; + for test_case in test_cases { + let directory = tempfile::tempdir().expect("temporary directory"); + let mut previous = metadata_sync((one_location(), 1), (3600, 1800)); + previous.dynamic_quiet_tokens.extend( + test_case + .previous_token + .map(|token| (one_location().remove(0), token.to_owned())), + ); + let (_, _, state) = republished_location(directory.path(), false, Some(previous)); + assert_eq!( + state + .dynamic_quiet_tokens + .get(&one_location().remove(0)) + .map(String::as_str), + test_case.expected_token, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-cli/src/native_commands/tests/test_types.rs b/crates/sqlbuild-cli/src/native_commands/tests/test_types.rs index 1d42d8c8d6..4c06297646 100644 --- a/crates/sqlbuild-cli/src/native_commands/tests/test_types.rs +++ b/crates/sqlbuild-cli/src/native_commands/tests/test_types.rs @@ -353,3 +353,26 @@ pub(super) struct FastClockScanTestCase { /// Whether a table altered a second after the page read is still described live. pub(super) expected_described_live: bool, } + +/// The token a quiet sync left before the location published again, and the token kept. +pub(super) struct RepublishTokenTestCase { + pub(super) description: &'static str, + pub(super) previous_token: Option<&'static str>, + pub(super) expected_token: Option<&'static str>, +} + +pub(super) struct DynamicResumeTestCase { + pub(super) description: &'static str, + /// Whether the synchronization completed, and whether failed reads stopped it. + pub(super) idle: bool, + pub(super) halted: bool, + /// The pause table token the location's last quiet pass left, before this sync finishes. + pub(super) quiet_token: Option<&'static str>, + /// The runtime's reply line to any request. + pub(super) reply: &'static str, + /// The request lines the runtime received, and whether the sync still holds its reader. + pub(super) expected_requests: &'static [&'static str], + pub(super) expected_reader_open: bool, + /// The token the next sync sends for the location. + pub(super) expected_quiet_token: Option<&'static str>, +} diff --git a/crates/sqlbuild-core/src/constants.rs b/crates/sqlbuild-core/src/constants.rs index a6629f8912..5ea4219c11 100644 --- a/crates/sqlbuild-core/src/constants.rs +++ b/crates/sqlbuild-core/src/constants.rs @@ -46,6 +46,8 @@ pub const RESOURCE_COSTS_TABLE_NAME: &str = "_sqlbuild_resource_costs"; pub const RUNS_TABLE_NAME: &str = "_sqlbuild_runs"; /// State table of the dynamic models a build paused, and its heartbeats while it holds them. pub const DYNAMIC_PAUSES_TABLE_NAME: &str = "_sqlbuild_dynamic_pauses"; +/// How long a location may spend resuming dynamic holds before it is killed; older claims are dead. +pub const DYNAMIC_RESUME_SECONDS: u64 = 120; /// Every state table SQLBuild writes in a target's state schema. pub const STATE_TABLE_NAMES: [&str; 14] = [ FINGERPRINT_TABLE_NAME, diff --git a/crates/sqlbuild-host/src/entries/_helpers/executor.rs b/crates/sqlbuild-host/src/entries/_helpers/executor.rs index 5589a7946b..453758fbe4 100644 --- a/crates/sqlbuild-host/src/entries/_helpers/executor.rs +++ b/crates/sqlbuild-host/src/entries/_helpers/executor.rs @@ -859,4 +859,9 @@ pub(crate) const ENTRIES: &[HostEntry] = &[ kind: HostKind::Function, edge: HostEdge::Executor, }, + HostEntry { + name: "resume_expired_holds", + kind: HostKind::Function, + edge: HostEdge::Executor, + }, ]; diff --git a/crates/sqlbuild-host/src/entries/models.rs b/crates/sqlbuild-host/src/entries/models.rs index 9fedeafd28..2b8e9e968e 100644 --- a/crates/sqlbuild-host/src/entries/models.rs +++ b/crates/sqlbuild-host/src/entries/models.rs @@ -378,6 +378,7 @@ pub enum ExecutorHost { CreateReviewSchema, DropReviewSchema, LeftoverReviewSchemas, + ResumeExpiredDynamicHolds, } impl ExecutorHost { diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/constants.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/constants.rs index a495a62b01..67c79dac2d 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/constants.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/constants.rs @@ -59,3 +59,7 @@ pub(crate) const SYNCED_CATALOG_SQL: &str = "SELECT record_json, observed_at, sc /// The development locations of one target, by project path. pub(crate) const DEVELOPMENT_LOCATIONS_SQL: &str = "SELECT project_path, location_id FROM sqb_development_locations WHERE target=$1"; +/// The runtime request key a completed sync sends with the state locations it read and their quiet tokens: `{"dynamic_resume": [...]}`. +pub const DYNAMIC_RESUME_REQUEST: &str = "dynamic_resume"; +/// Prefix of the `recorded_by` a location sync writes on the holds it resumes, before the location ID. +pub const DYNAMIC_RESUME_RECORDER: &str = "location-sync:"; diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/dynamic_resume/mod.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/dynamic_resume/mod.rs new file mode 100644 index 0000000000..02714195aa --- /dev/null +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/dynamic_resume/mod.rs @@ -0,0 +1 @@ +pub mod resume_locations; diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/dynamic_resume/resume_locations.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/dynamic_resume/resume_locations.rs new file mode 100644 index 0000000000..39b59d6b0e --- /dev/null +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/dynamic_resume/resume_locations.rs @@ -0,0 +1,25 @@ +//! The state locations a sync resumes expired dynamic-model holds in. + +use crate::warehouse_metadata::models::{DynamicResumeLocation, WarehouseStateLocation}; + +/// None unless the adapter builds dynamic models; else the requested ones and the pause table's. +pub fn resume_locations( + requested: Vec, + (dynamic, pause_location): (bool, Option), +) -> Vec { + if !dynamic { + return Vec::new(); + } + let mut locations: Vec = requested; + if let Some(location) = pause_location + && !locations + .iter() + .any(|requested| requested.location == location) + { + locations.push(DynamicResumeLocation { + location, + quiet_token: None, + }); + } + locations +} diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/mod.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/mod.rs index 91a0c97ff7..6b107228d2 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/mod.rs @@ -1,3 +1,4 @@ +pub mod dynamic_resume; pub mod ingest; pub mod location_reads; pub(crate) mod node_reads; diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs index 78d596d4b2..ca888807dc 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs @@ -725,6 +725,41 @@ pub struct WarehouseHistoryFailure { pub page_size: bool, } +/// A dynamic model a completed sync resumed: the run whose hold expired, and its dotted relation. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ResumedHold { + pub run_id: String, + pub relation: String, +} + +/// One state location a completed sync resumes holds in, and its pause table's token when the last pass left no hold open. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct DynamicResumeLocation { + pub location: WarehouseStateLocation, + /// A pause table whose change token still equals this is not read, so a quiet sync wakes no compute. + pub quiet_token: Option, +} + +/// The holds a pass closed, each location it resumed in with its new quiet token, and its failures. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct DynamicResumed { + pub holds: Vec, + pub quiet_tokens: Vec, + /// Each hold or location the pass could not resume, with the warehouse's error. + pub failures: Vec, +} + +/// A location runtime's reply to a dynamic resume: what it closed, or its redacted error. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum DynamicResumeReply { + Resumed(DynamicResumed), + Failed(String), +} + /// Typed import columns are selected by code, never from location-supplied field names. #[derive(Debug)] pub(crate) struct StateEventImport { diff --git a/crates/sqlbuild-orchestration/tests/warehouse_metadata/test_contracts.rs b/crates/sqlbuild-orchestration/tests/warehouse_metadata/test_contracts.rs index 801bd93736..825282689f 100644 --- a/crates/sqlbuild-orchestration/tests/warehouse_metadata/test_contracts.rs +++ b/crates/sqlbuild-orchestration/tests/warehouse_metadata/test_contracts.rs @@ -4,5 +4,7 @@ mod helpers; #[path = "test_preview_state.rs"] mod preview_state; +#[path = "test_resume_locations.rs"] +mod resume_locations; #[path = "test_types.rs"] mod test_types; diff --git a/crates/sqlbuild-orchestration/tests/warehouse_metadata/test_resume_locations.rs b/crates/sqlbuild-orchestration/tests/warehouse_metadata/test_resume_locations.rs new file mode 100644 index 0000000000..2f5599534a --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/warehouse_metadata/test_resume_locations.rs @@ -0,0 +1,68 @@ +//! A sync resumes expired holds only where a dynamic-model adapter keeps its pause table. + +use crate::warehouse_metadata::main::dynamic_resume::resume_locations::resume_locations; +use crate::warehouse_metadata::models::{DynamicResumeLocation, WarehouseStateLocation}; +use crate::warehouse_metadata::tests::test_types::ResumeLocationsTestCase; + +#[test] +fn given_sync_locations_when_choosing_where_to_resume_then_the_pause_table_is_included_and_other_adapters_get_none() + { + let test_cases = [ + ResumeLocationsTestCase { + description: "every model in custom schemas still resumes in the target's default schema", + requested: &[("orders_mart", None), ("finance_mart", Some("token-1"))], + dynamic: true, + pause_schema: Some("analytics"), + expected_locations: &[ + ("orders_mart", None), + ("finance_mart", Some("token-1")), + ("analytics", None), + ], + }, + ResumeLocationsTestCase { + description: "a default schema the models already write to is asked once, with its token", + requested: &[("analytics", Some("token-2"))], + dynamic: true, + pause_schema: Some("analytics"), + expected_locations: &[("analytics", Some("token-2"))], + }, + ResumeLocationsTestCase { + description: "a preserved target schema adds no location", + requested: &[("orders_mart", None)], + dynamic: true, + pause_schema: None, + expected_locations: &[("orders_mart", None)], + }, + ResumeLocationsTestCase { + description: "an adapter that cannot build dynamic models resumes nowhere", + requested: &[("analytics", None)], + dynamic: false, + pause_schema: Some("analytics"), + expected_locations: &[], + }, + ]; + let location = |schema: &str| WarehouseStateLocation { + database: Some("warehouse".into()), + schema: schema.into(), + }; + let resume = |locations: &[(&str, Option<&str>)]| -> Vec { + locations + .iter() + .map(|(schema, token)| DynamicResumeLocation { + location: location(schema), + quiet_token: token.map(str::to_owned), + }) + .collect() + }; + for test_case in test_cases { + assert_eq!( + resume_locations( + resume(test_case.requested), + (test_case.dynamic, test_case.pause_schema.map(location)), + ), + resume(test_case.expected_locations), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-orchestration/tests/warehouse_metadata/test_types.rs b/crates/sqlbuild-orchestration/tests/warehouse_metadata/test_types.rs index 5973d893e3..2484635b3a 100644 --- a/crates/sqlbuild-orchestration/tests/warehouse_metadata/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/warehouse_metadata/test_types.rs @@ -5,6 +5,16 @@ use serde_json::Value; /// A state's cursor `MAX` of orders, newest shop fingerprint runs, consumption row count and version. pub(crate) type StateAnswers = (Option, Vec, usize, Option); +/// The locations a sync asks to resume in, and the adapter and pause table it resumes for. +pub(crate) struct ResumeLocationsTestCase { + pub(crate) description: &'static str, + pub(crate) requested: &'static [(&'static str, Option<&'static str>)], + pub(crate) dynamic: bool, + pub(crate) pause_schema: Option<&'static str>, + /// Each location's schema and quiet token, in order. + pub(crate) expected_locations: &'static [(&'static str, Option<&'static str>)], +} + /// A served copy and a stored fixture of the same facts, and what both states answer. pub(crate) struct PreviewStateTestCase { pub(crate) description: &'static str, diff --git a/crates/sqlbuild-python-front/src/discovery/main/project/configured_dynamic_grace.rs b/crates/sqlbuild-python-front/src/discovery/main/project/configured_dynamic_grace.rs new file mode 100644 index 0000000000..cfe6e1dfd9 --- /dev/null +++ b/crates/sqlbuild-python-front/src/discovery/main/project/configured_dynamic_grace.rs @@ -0,0 +1,10 @@ +//! How long a dynamic-model hold may go without a heartbeat: the project's `[dynamic]` grace. + +use crate::discovery::_helpers::project::project_config::configured; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use sqlbuild_config::project::main::load_dynamic_config::load_dynamic_config; + +/// `[dynamic] heartbeat_grace` in seconds for the project in `project_dir`. +pub fn configured_dynamic_grace(project_dir: &Bound<'_, PyAny>) -> PyResult { + Ok(configured(project_dir, load_dynamic_config)?.heartbeat_grace_seconds) +} diff --git a/crates/sqlbuild-python-front/src/discovery/main/project/mod.rs b/crates/sqlbuild-python-front/src/discovery/main/project/mod.rs index 85eb080380..21a917f902 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/project/mod.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/project/mod.rs @@ -1,6 +1,7 @@ pub mod column_name; pub mod configured_changes; pub mod configured_dynamic; +pub mod configured_dynamic_grace; pub mod configured_git_base_ref; pub mod configured_review; pub mod discover_configured_providers; diff --git a/crates/sqlbuild-python-front/src/native_host/main/mod.rs b/crates/sqlbuild-python-front/src/native_host/main/mod.rs index 393249f52a..7154dbd7ad 100644 --- a/crates/sqlbuild-python-front/src/native_host/main/mod.rs +++ b/crates/sqlbuild-python-front/src/native_host/main/mod.rs @@ -11,5 +11,6 @@ pub mod register_audit_results_state; pub mod register_consumption_state; pub mod register_observations_state; pub mod reports_cost_capability; +pub mod resume_expired_dynamic_holds; pub mod seconds_since; pub mod state_table_name; diff --git a/crates/sqlbuild-python-front/src/native_host/main/resume_expired_dynamic_holds.rs b/crates/sqlbuild-python-front/src/native_host/main/resume_expired_dynamic_holds.rs new file mode 100644 index 0000000000..ae5fd1653e --- /dev/null +++ b/crates/sqlbuild-python-front/src/native_host/main/resume_expired_dynamic_holds.rs @@ -0,0 +1,89 @@ +//! Resume the dynamic models whose holds expired, in each state location a sync read. + +use crate::discovery::main::project::configured_dynamic_grace::configured_dynamic_grace; +use crate::native_host::types::{ResumeConnection, ResumeLocation, ResumedHolds}; +use pyo3::IntoPyObjectExt; +use pyo3::exceptions::PyException; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_host::entries::models::ExecutorHost; +use sqlbuild_python_base::boundary::main::call_with::call_with; +use sqlbuild_python_base::native_host::main::host_entry::host_entry; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// Resume holds silent past `[dynamic] heartbeat_grace`, each location on its own, over one session. +pub fn resume_expired_dynamic_holds<'py>( + (adapter, connection_config): ResumeConnection<'_, 'py>, + project_dir: &Bound<'py, PyAny>, + locations: &[ResumeLocation], + recorded_by: &str, +) -> PyResult { + let connection: Bound<'py, PyAny> = + native_field(adapter, "connect")?.call1((connection_config,))?; + let resumed: PyResult = + resume_each((adapter, &connection), project_dir, locations, recorded_by); + let closed: PyResult> = native_field(adapter, "close")?.call1((&connection,)); + let resumed: ResumedHolds = resumed?; + closed?; + Ok(resumed) +} + +fn resume_each<'py>( + (adapter, connection): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + project_dir: &Bound<'py, PyAny>, + locations: &[ResumeLocation], + recorded_by: &str, +) -> PyResult { + let py = adapter.py(); + let grace_seconds: u64 = configured_dynamic_grace(project_dir)?; + let resume: Bound<'py, PyAny> = host_entry(py, ExecutorHost::ResumeExpiredDynamicHolds)?; + let mut resumed: Vec<(String, String)> = Vec::new(); + let mut quiet_tokens: Vec = Vec::new(); + let mut failures: Vec = Vec::new(); + for (location, quiet_token) in locations { + let outcome: PyResult> = call_with( + &resume, + &[ + ("adapter", adapter.clone()), + ("connection", connection.clone()), + ("location", location.clone().into_bound_py_any(py)?), + ( + "grace_seconds", + (grace_seconds as f64).into_bound_py_any(py)?, + ), + ("recorded_by", recorded_by.into_bound_py_any(py)?), + ("quiet_token", quiet_token.clone().into_bound_py_any(py)?), + ], + ); + let outcome: Bound<'py, PyAny> = match outcome { + Ok(outcome) => outcome, + Err(error) if error.is_instance_of::(py) => { + let (database, schema) = location; + let dotted: String = database + .iter() + .chain([schema]) + .cloned() + .collect::>() + .join("."); + failures.push(format!("{dotted}: {error}")); + quiet_tokens.push((location.clone(), None)); + continue; + } + Err(error) => return Err(error), + }; + for hold in native_field(&outcome, "resumed")?.try_iter()? { + let hold: Bound<'py, PyAny> = hold?; + let relation: Bound<'py, PyAny> = native_field(&hold, "relation")?; + let mut parts: Vec = Vec::new(); + for part in ["database", "schema", "name"] { + parts.extend(native_field(&relation, part)?.extract::>()?); + } + resumed.push((native_field(&hold, "run_id")?.extract()?, parts.join("."))); + } + failures.extend(native_field(&outcome, "failures")?.extract::>()?); + quiet_tokens.push(( + location.clone(), + native_field(&outcome, "quiet_token")?.extract()?, + )); + } + Ok((resumed, quiet_tokens, failures)) +} diff --git a/crates/sqlbuild-python-front/src/native_host/types.rs b/crates/sqlbuild-python-front/src/native_host/types.rs index 0d0b83f623..ba9e8ffd4b 100644 --- a/crates/sqlbuild-python-front/src/native_host/types.rs +++ b/crates/sqlbuild-python-front/src/native_host/types.rs @@ -3,6 +3,7 @@ use crate::native_host::models::Location; use pyo3::prelude::PyResult; use pyo3::prelude::Python; +use pyo3::prelude::{Bound, PyAny}; use sqlbuild_state::consumption::models::ConsumptionRow; /// Reading recorded consumption: from the warehouse table, or from a synced state. @@ -17,3 +18,12 @@ pub trait ConsumptionReads { execution_ids: &[&str], ) -> PyResult>; } + +/// One state location's database and schema, and the pause table token its last quiet pass left. +pub type ResumeLocation = ((Option, String), Option); + +/// Each resumed hold's run ID and dotted relation, each location's new quiet token, and failures. +pub type ResumedHolds = (Vec<(String, String)>, Vec, Vec); + +/// A project's adapter and the connection configuration of its selected target. +pub type ResumeConnection<'a, 'py> = (&'a Bound<'py, PyAny>, &'a Bound<'py, PyAny>); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_events/event.rs b/crates/sqlbuild-python-output/src/dbt_events/_helpers/event.rs similarity index 100% rename from crates/sqlbuild-python/src/bindings/_helpers/dbt_events/event.rs rename to crates/sqlbuild-python-output/src/dbt_events/_helpers/event.rs diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_events/getters.rs b/crates/sqlbuild-python-output/src/dbt_events/_helpers/getters.rs similarity index 63% rename from crates/sqlbuild-python/src/bindings/_helpers/dbt_events/getters.rs rename to crates/sqlbuild-python-output/src/dbt_events/_helpers/getters.rs index 52ae91c091..d81e52d057 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_events/getters.rs +++ b/crates/sqlbuild-python-output/src/dbt_events/_helpers/getters.rs @@ -3,17 +3,10 @@ use pyo3::IntoPyObjectExt; use pyo3::prelude::{Bound, PyAny, PyResult, Python}; -use crate::bindings::_helpers::dbt_events::stream::DbtEventStreamUpdate; -use sqlbuild_python_base::records::main::registry::class_of::class_of; -use sqlbuild_python_base::records::models::GetterEntry; +use crate::dbt_events::models::DbtEventStreamUpdate; -/// The getter readers of this module's native pyclasses. -pub(crate) const GETTERS: &[GetterEntry] = &[GetterEntry { - class: class_of::, - read: read_dbt_event_stream_update, -}]; - -fn read_dbt_event_stream_update<'py>( +/// The getter values of a `DbtEventStreamUpdate`. +pub(crate) fn read_dbt_event_stream_update<'py>( value: &Bound<'py, PyAny>, name: &str, ) -> Option>> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_events/mod.rs b/crates/sqlbuild-python-output/src/dbt_events/_helpers/mod.rs similarity index 57% rename from crates/sqlbuild-python/src/bindings/_helpers/dbt_events/mod.rs rename to crates/sqlbuild-python-output/src/dbt_events/_helpers/mod.rs index 8124b7b4a9..922dfc296c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_events/mod.rs +++ b/crates/sqlbuild-python-output/src/dbt_events/_helpers/mod.rs @@ -1,5 +1,3 @@ pub(crate) mod event; pub(crate) mod getters; -pub(crate) mod registration; pub(crate) mod results; -pub(crate) mod stream; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_events/results.rs b/crates/sqlbuild-python-output/src/dbt_events/_helpers/results.rs similarity index 100% rename from crates/sqlbuild-python/src/bindings/_helpers/dbt_events/results.rs rename to crates/sqlbuild-python-output/src/dbt_events/_helpers/results.rs diff --git a/crates/sqlbuild-python-output/src/dbt_events/constants.rs b/crates/sqlbuild-python-output/src/dbt_events/constants.rs new file mode 100644 index 0000000000..edd6dcb130 --- /dev/null +++ b/crates/sqlbuild-python-output/src/dbt_events/constants.rs @@ -0,0 +1,21 @@ +//! Public constants of the dbt event stream. + +use crate::dbt_events::_helpers::getters::read_dbt_event_stream_update; +use crate::dbt_events::models::DbtEventStreamUpdate; +use sqlbuild_python_base::records::main::registry::class_of::class_of; +use sqlbuild_python_base::records::models::GetterEntry; + +/// The getter readers of this module's native pyclasses. +pub const GETTERS: &[GetterEntry] = &[GetterEntry { + class: class_of::, + read: read_dbt_event_stream_update, +}]; + +/// The keywords `DbtEventStream` accepts: required, then optional. +pub(crate) const STREAM_OPTIONS: (&[&str], &[&str]) = + (&["display_total", "detail_by_unique_id", "use_color"], &[]); +/// The keywords `render_dbt_node_result` accepts: required, then optional. +pub(crate) const RENDER_OPTIONS: (&[&str], &[&str]) = ( + &["result", "use_color"], + &["display_index", "display_total", "detail"], +); diff --git a/crates/sqlbuild-python-output/src/dbt_events/main/mod.rs b/crates/sqlbuild-python-output/src/dbt_events/main/mod.rs new file mode 100644 index 0000000000..6b008861f6 --- /dev/null +++ b/crates/sqlbuild-python-output/src/dbt_events/main/mod.rs @@ -0,0 +1,2 @@ +pub mod register_dbt_events; +pub mod render_dbt_node_result; diff --git a/crates/sqlbuild-python-output/src/dbt_events/main/register_dbt_events.rs b/crates/sqlbuild-python-output/src/dbt_events/main/register_dbt_events.rs new file mode 100644 index 0000000000..90a8ab7a74 --- /dev/null +++ b/crates/sqlbuild-python-output/src/dbt_events/main/register_dbt_events.rs @@ -0,0 +1,15 @@ +//! Register the native dbt event stream. + +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +use crate::dbt_events::main::render_dbt_node_result::render_dbt_node_result; +use crate::dbt_events::models::{DbtEventStream, DbtEventStreamUpdate}; + +/// Register the dbt event stream classes and the node row renderer. +pub fn register_dbt_events(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::()?; + module.add_class::()?; + module.add_function(wrap_pyfunction!(render_dbt_node_result, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python-output/src/dbt_events/main/render_dbt_node_result.rs b/crates/sqlbuild-python-output/src/dbt_events/main/render_dbt_node_result.rs new file mode 100644 index 0000000000..04cd36a418 --- /dev/null +++ b/crates/sqlbuild-python-output/src/dbt_events/main/render_dbt_node_result.rs @@ -0,0 +1,39 @@ +//! One dbt node row and its messages, rendered from a Python `DbtNodeExecutionResult`. + +use pyo3::prelude::{Bound, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::PyDict; +use sqlbuild_core::cli_text::models::CliStyle; +use sqlbuild_core::dbt_events::main::render_dbt_node_row::render_dbt_node_row; +use sqlbuild_core::dbt_events::models::DbtRowPosition; +use sqlbuild_python_base::boundary::models::KeywordArguments; + +use crate::dbt_events::_helpers::results::native_result; +use crate::dbt_events::constants::RENDER_OPTIONS; + +/// One dbt node row and its messages. +#[pyfunction] +#[pyo3(signature = (**options))] +pub fn render_dbt_node_result( + py: Python<'_>, + options: Option>, +) -> PyResult { + let arguments: KeywordArguments<'_> = + KeywordArguments::read(py, "render_dbt_node_result", options, RENDER_OPTIONS)?; + let position: DbtRowPosition = DbtRowPosition { + display_index: arguments.value("display_index")?.extract()?, + display_total: arguments.value("display_total")?.extract()?, + }; + let detail: String = arguments + .value("detail")? + .extract::>()? + .unwrap_or_default(); + Ok(render_dbt_node_row( + &native_result(&arguments.value("result")?)?, + position, + &detail, + CliStyle { + use_color: arguments.flag("use_color")?, + }, + )) +} diff --git a/crates/sqlbuild-python-output/src/dbt_events/mod.rs b/crates/sqlbuild-python-output/src/dbt_events/mod.rs new file mode 100644 index 0000000000..396d4420fc --- /dev/null +++ b/crates/sqlbuild-python-output/src/dbt_events/mod.rs @@ -0,0 +1,6 @@ +//! The dbt event stream: dbt's JSON log lines read into node progress, rows and Python results. + +pub(crate) mod _helpers; +pub mod constants; +pub mod main; +pub mod models; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_events/stream.rs b/crates/sqlbuild-python-output/src/dbt_events/models.rs similarity index 66% rename from crates/sqlbuild-python/src/bindings/_helpers/dbt_events/stream.rs rename to crates/sqlbuild-python-output/src/dbt_events/models.rs index 25ecb5a60e..1a5d7733b2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_events/stream.rs +++ b/crates/sqlbuild-python-output/src/dbt_events/models.rs @@ -1,33 +1,26 @@ -//! The dbt event stream: one output line in, the progress and row it adds out. +//! Public records of the dbt event stream: one output line in, the progress and row it adds out. use std::sync::Mutex; use pyo3::exceptions::PyRuntimeError; -use pyo3::prelude::{ - Bound, Py, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::PyDict; -use pyo3::{pyclass, pyfunction, pymethods, wrap_pyfunction}; +use pyo3::{pyclass, pymethods}; use sqlbuild_core::cli_text::models::CliStyle; use sqlbuild_core::dbt_events::main::consume_dbt_event::consume_dbt_event; use sqlbuild_core::dbt_events::main::render_dbt_node_row::render_dbt_node_row; use sqlbuild_core::dbt_events::models::{DbtRowPosition, DbtStreamState, DbtStreamUpdate}; -use crate::bindings::_helpers::dbt_events::event::dbt_event; -use crate::bindings::_helpers::dbt_events::results::{native_result, python_result}; +use crate::dbt_events::_helpers::event::dbt_event; +use crate::dbt_events::_helpers::results::python_result; +use crate::dbt_events::constants::STREAM_OPTIONS; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::records::main::fields::mapping_get_or::mapping_get_or; -const STREAM_OPTIONS: (&[&str], &[&str]) = - (&["display_total", "detail_by_unique_id", "use_color"], &[]); -const RENDER_OPTIONS: (&[&str], &[&str]) = ( - &["result", "use_color"], - &["display_index", "display_total", "detail"], -); - /// What one dbt output line adds: a progress message, a node row, or both. +#[derive(Debug)] #[pyclass(module = "sqlbuild._native", frozen, get_all)] -pub(crate) struct DbtEventStreamUpdate { +pub struct DbtEventStreamUpdate { /// The node-started progress message. pub(crate) status_message: Option, /// The started or finished node. @@ -39,8 +32,9 @@ pub(crate) struct DbtEventStreamUpdate { } /// Read dbt's JSON log lines into node progress, rows and results. +#[derive(Debug)] #[pyclass(module = "sqlbuild._native", frozen)] -pub(crate) struct DbtEventStream { +pub struct DbtEventStream { state: Mutex, display_total: Option, detail_by_unique_id: Py, @@ -88,7 +82,7 @@ impl DbtEventStream { impl DbtEventStream { #[new] #[pyo3(signature = (**options))] - pub(crate) fn new(py: Python<'_>, options: Option>) -> PyResult { + pub fn new(py: Python<'_>, options: Option>) -> PyResult { let arguments: KeywordArguments<'_> = KeywordArguments::read(py, "DbtEventStream", options, STREAM_OPTIONS)?; Ok(Self { @@ -117,37 +111,3 @@ impl DbtEventStream { update.map(|update| self.update(py, update)).transpose() } } - -/// One dbt node row and its messages. -#[pyfunction] -#[pyo3(signature = (**options))] -pub(crate) fn render_dbt_node_result( - py: Python<'_>, - options: Option>, -) -> PyResult { - let arguments: KeywordArguments<'_> = - KeywordArguments::read(py, "render_dbt_node_result", options, RENDER_OPTIONS)?; - let position: DbtRowPosition = DbtRowPosition { - display_index: arguments.value("display_index")?.extract()?, - display_total: arguments.value("display_total")?.extract()?, - }; - let detail: String = arguments - .value("detail")? - .extract::>()? - .unwrap_or_default(); - Ok(render_dbt_node_row( - &native_result(&arguments.value("result")?)?, - position, - &detail, - CliStyle { - use_color: arguments.flag("use_color")?, - }, - )) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_class::()?; - module.add_class::()?; - module.add_function(wrap_pyfunction!(render_dbt_node_result, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python-output/src/lib.rs b/crates/sqlbuild-python-output/src/lib.rs index a3704154f9..a4c4ad3c81 100644 --- a/crates/sqlbuild-python-output/src/lib.rs +++ b/crates/sqlbuild-python-output/src/lib.rs @@ -1,10 +1,11 @@ -//! The output layer of the `sqlbuild._native` bindings: build progress, result output, output capture, cost telemetry, compute logs and the Recall session. +//! The output layer of the `sqlbuild._native` bindings: build progress, result output, output capture, cost telemetry, compute logs, the dbt event stream and the Recall session. #![forbid(unsafe_code)] pub mod build_progress; pub mod compute_logs; pub mod cost_telemetry; +pub mod dbt_events; pub mod event_exporting; pub mod output_capture; pub mod recall; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs index 9962d9ccc8..c3c15b7c99 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs @@ -6,10 +6,10 @@ use crate::bindings::_helpers::cli::{argv, entry, namespace, observability, term use crate::bindings::_helpers::{ audit_command, build_command, build_progress, change_selection, check, clone, command_runtime, compute_logs, contract_command, cost_command, cost_report, cost_telemetry, dbt_command, - dbt_events, dbt_interop, debug_command, diff_report, event_exporting, execution_history, - format_command, freshness, init_command, janitor, load, output_capture, plan_command, - presentation_text, query_command, recall, result_output, review_command, scenario_command, - seed_command, sql_test_command, workspace_commands, + dbt_interop, debug_command, diff_report, event_exporting, execution_history, format_command, + freshness, init_command, janitor, load, output_capture, plan_command, presentation_text, + query_command, recall, result_output, review_command, scenario_command, seed_command, + sql_test_command, workspace_commands, }; use sqlbuild_python_base::build_progress::main::progress::register_progress::register_progress; use sqlbuild_python_base::cli::main::register_launch_state::register_launch_state; @@ -44,7 +44,7 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { cost_report::registration::register(module)?; cost_telemetry::registration::register(module)?; debug_command::registration::register(module)?; - dbt_events::registration::register(module)?; + sqlbuild_python_output::dbt_events::main::register_dbt_events::register_dbt_events(module)?; dbt_interop::registration::register(module)?; diff_report::registration::register(module)?; review_command::registration::register(module)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_events/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_events/registration.rs deleted file mode 100644 index b6e7d9653f..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_events/registration.rs +++ /dev/null @@ -1,9 +0,0 @@ -//! Register the native dbt event stream. - -use pyo3::prelude::{Bound, PyModule, PyResult}; - -use crate::bindings::_helpers::dbt_events::stream; - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - stream::register(module) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/event_stream.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/event_stream.rs index f130400958..99a33e6053 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/event_stream.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/event_stream.rs @@ -22,7 +22,6 @@ use sqlbuild_core::dbt_process::main::spawn_dbt_json_process::spawn_dbt_json_pro use sqlbuild_core::dbt_process::main::stop_dbt_process::stop_dbt_process; use sqlbuild_core::dbt_process::models::DbtJsonProcess; -use crate::bindings::_helpers::dbt_events::stream::DbtEventStream; use crate::bindings::_helpers::dbt_interop::runner::{ dbt_command_lifecycle, launch_failure, launch_os_error, report_exit, }; @@ -32,6 +31,7 @@ use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::build_progress::main::transient_status::status_reporter::status_reporter; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::types::CompilerDetach; +use sqlbuild_python_output::dbt_events::models::DbtEventStream; const STREAM_OPTIONS: (&[&str], &[&str]) = ( &["argv", "cwd", "stream", "use_color", "target_path"], diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/runtime_execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/runtime_execution.rs index 85e0f56b1d..23c9480c3a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/runtime_execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/runtime_execution.rs @@ -1,6 +1,5 @@ //! Execute dbt through the subprocess boundary and finalize results in artifact order. -use crate::bindings::_helpers::dbt_events::stream::render_dbt_node_result; use crate::bindings::_helpers::dbt_interop::cli::parse_dbt_ls_json_lines; use crate::bindings::_helpers::dbt_interop::event_stream::execute_dbt_json_event_stream; use crate::bindings::_helpers::dbt_interop::execution::dbt_execution_ls_argv; @@ -16,6 +15,7 @@ use sqlbuild_python_base::presentation_text::models::PythonCliStyle; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_output::dbt_events::main::render_dbt_node_result::render_dbt_node_result; use std::collections::HashSet; use std::time::Instant; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/constants.rs index 510cd06cc0..25989526f3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/exported_constants/constants.rs @@ -19,10 +19,11 @@ use sqlbuild_core::command_output::constants::{ }; use sqlbuild_core::constants::{ AUDIT_RESULTS_TABLE_NAME, COLUMN_MIGRATION_TABLE_NAME, CONSUMPTION_TABLE_NAME, - DYNAMIC_PAUSES_TABLE_NAME, FINGERPRINT_TABLE_NAME, JANITOR_EVENTS_TABLE_NAME, - MICROBATCH_TABLE_NAME, MIGRATION_TABLE_NAME, NODE_RESULTS_TABLE_NAME, OBSERVATIONS_TABLE_NAME, - OLD_NAME_VIEW_TABLE_NAME, PROJECT_CONFIG_FILENAME, PYTHON_INIT_MODULE_STEM, PYTHON_NODE_ROOT, - RUNS_TABLE_NAME, SETTINGS_SECTION, SOURCE_FRESHNESS_TABLE_NAME, STAGED_PROMOTION_MODE, + DYNAMIC_PAUSES_TABLE_NAME, DYNAMIC_RESUME_SECONDS, FINGERPRINT_TABLE_NAME, + JANITOR_EVENTS_TABLE_NAME, MICROBATCH_TABLE_NAME, MIGRATION_TABLE_NAME, + NODE_RESULTS_TABLE_NAME, OBSERVATIONS_TABLE_NAME, OLD_NAME_VIEW_TABLE_NAME, + PROJECT_CONFIG_FILENAME, PYTHON_INIT_MODULE_STEM, PYTHON_NODE_ROOT, RUNS_TABLE_NAME, + SETTINGS_SECTION, SOURCE_FRESHNESS_TABLE_NAME, STAGED_PROMOTION_MODE, TABLE_PROMOTION_MODE_SETTING_KEY, ZERO_DAY_DURATION, }; use sqlbuild_core::diagnostic_routing::constants::{ @@ -131,6 +132,7 @@ pub(crate) const EXPORTED_INTEGERS: &[(&str, i64)] = &[ ("CURRENT_RUN_STORAGE_SCHEMA_VERSION", SCHEMA_VERSION), ("COMPUTE_LOG_FORMAT_VERSION", COMPUTE_LOG_FORMAT_VERSION), ("DEFAULT_RETENTION_COUNT", DEFAULT_RETENTION_COUNT as i64), + ("DYNAMIC_RESUME_SECONDS", DYNAMIC_RESUME_SECONDS as i64), ("MAX_READ_BYTES", MAX_READ_BYTES as i64), ( "CURRENT_COMMAND_OUTPUT_SCHEMA_VERSION", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/mod.rs index 914ed0ee35..41528fa598 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/mod.rs @@ -3,6 +3,7 @@ mod constants; mod load_rules; mod runtime; mod runtime_plan; +mod runtime_resume; mod snapshot; mod snapshot_connection; mod snapshot_lineage; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/runtime.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/runtime.rs index 3fdb5dbfcd..1f5ffa5be3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/runtime.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/runtime.rs @@ -10,8 +10,9 @@ use sqlbuild_core::redaction::main::redact_error_text::redact_error_text; use sqlbuild_orchestration::dev_events::main::emit_dev_event::emit_dev_event; use sqlbuild_orchestration::dev_events::main::generation_failed::generation_failed; use sqlbuild_orchestration::sensors::models::{CustomSensorOutcome, SensorEvaluationRequest}; +use sqlbuild_orchestration::warehouse_metadata::constants::DYNAMIC_RESUME_REQUEST; use sqlbuild_orchestration::warehouse_metadata::models::{ - WarehouseHistoryFailure, WarehouseHistoryKind, WarehouseHistoryRead, + DynamicResumeReply, WarehouseHistoryFailure, WarehouseHistoryKind, WarehouseHistoryRead, }; use crate::bindings::_helpers::location_command::command::location_arguments; @@ -19,6 +20,7 @@ use crate::bindings::_helpers::location_command::command::publish_location; use crate::bindings::_helpers::location_command::constants::DUCKDB_LOCK_CONFLICT; use crate::bindings::_helpers::location_command::constants::PLAN_REQUEST; use crate::bindings::_helpers::location_command::runtime_plan::runtime_plan; +use crate::bindings::_helpers::location_command::runtime_resume::runtime_resume; use crate::bindings::_helpers::location_command::types::HeldProject; use crate::bindings::_helpers::native_host::orchestration_sensors::evaluate_sensor; use crate::bindings::_helpers::native_host::warehouse_metadata::close_history_session; @@ -76,6 +78,18 @@ pub(super) fn run_runtime(py: Python<'_>, options: Bound<'_, PyDict>) -> PyResul })) } else if let Some(plan) = request.get(PLAN_REQUEST) { serde_json::to_string(&runtime_plan(py, held.as_ref(), plan)?) + } else if let Some(locations) = request.get(DYNAMIC_RESUME_REQUEST) { + serde_json::to_string(&match runtime_resume(py, &options, locations) { + Ok(resumed) => DynamicResumeReply::Resumed(resumed), + Err(error) => { + let reason = failure_reason(&error); + write_stderr( + py, + &format!("Resuming expired dynamic holds failed: {reason}\n"), + )?; + DynamicResumeReply::Failed(reason) + } + }) } else if let Some(kind) = WarehouseHistoryKind::ALL .iter() .find(|kind| request.get(kind.as_str()).is_some()) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/runtime_resume.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/runtime_resume.rs new file mode 100644 index 0000000000..bf3c65f4e1 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/runtime_resume.rs @@ -0,0 +1,117 @@ +//! A completed sync's only warehouse write: resume the dynamic models whose holds expired. + +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::PyDict; +use serde_json::Value; +use sqlbuild_model_config::model_validation::constants::DYNAMIC_ADAPTERS; +use sqlbuild_orchestration::warehouse_metadata::constants::DYNAMIC_RESUME_RECORDER; +use sqlbuild_orchestration::warehouse_metadata::main::dynamic_resume::resume_locations::resume_locations; +use sqlbuild_orchestration::warehouse_metadata::models::{ + DynamicResumeLocation, DynamicResumed, ResumedHold, WarehouseStateLocation, +}; +use sqlbuild_python_base::records::main::fields::field::field; +use sqlbuild_python_compile::compile_driver::main::setup::selected_target_settings::selected_target_settings; +use sqlbuild_python_front::compile_attachment::main::build_effective_runtime_config::build_effective_runtime_config; +use sqlbuild_python_front::native_host::main::resume_expired_dynamic_holds::resume_expired_dynamic_holds; +use sqlbuild_python_front::native_host::types::ResumeLocation; + +use crate::bindings::_helpers::command_runtime::connection::project_connection_config; +use crate::bindings::_helpers::location_command::command::location_arguments; +use crate::bindings::_helpers::query_command::project::{ + ProjectAdapter, ProjectInputs, project_adapter, +}; + +/// Resume expired holds in `locations` and the target's pause table; no session without dynamic models. +pub(super) fn runtime_resume( + py: Python<'_>, + options: &Bound<'_, PyDict>, + locations: &Value, +) -> PyResult { + let requested: Vec = serde_json::from_value(locations.clone()) + .map_err(|_| PyValueError::new_err("runtime received an invalid dynamic resume"))?; + let arguments = location_arguments(py, Some(options.clone()))?; + let ProjectAdapter { + project_dir, + discovered, + adapter, + } = project_adapter( + py, + Some(arguments.value("project_dir")?), + ProjectInputs::Configuration, + )?; + let adapter_name: String = field(&adapter, "adapter_name")?.str()?.extract()?; + let dynamic: bool = DYNAMIC_ADAPTERS.contains(&adapter_name.as_str()); + let target: Bound<'_, PyAny> = arguments.value("selected_target")?; + let config: Bound<'_, PyAny> = project_connection_config( + &discovered, + (&project_dir, &target), + (&py.None().into_bound(py), true), + false, + )?; + let pause_location: Option = match dynamic { + true => pause_location((&discovered, &target), &adapter_name, (&config, &adapter))?, + false => None, + }; + let locations: Vec = resume_locations(requested, (dynamic, pause_location)) + .into_iter() + .map(|resume| { + ( + (resume.location.database, resume.location.schema), + resume.quiet_token, + ) + }) + .collect(); + if locations.is_empty() { + return Ok(DynamicResumed { + holds: Vec::new(), + quiet_tokens: Vec::new(), + failures: Vec::new(), + }); + } + let recorded_by = format!( + "{DYNAMIC_RESUME_RECORDER}{}", + arguments.value("location_id")?.str()? + ); + let (holds, quiet_tokens, failures) = + resume_expired_dynamic_holds((&adapter, &config), &project_dir, &locations, &recorded_by)?; + Ok(DynamicResumed { + holds: holds + .into_iter() + .map(|(run_id, relation)| ResumedHold { run_id, relation }) + .collect(), + quiet_tokens: quiet_tokens + .into_iter() + .map(|((database, schema), quiet_token)| DynamicResumeLocation { + location: WarehouseStateLocation { database, schema }, + quiet_token, + }) + .collect(), + failures, + }) +} + +/// Where builds keep the pause table: the target's schema, unless it is `preserve` or unset. +fn pause_location<'py>( + (discovered, target): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + adapter_name: &str, + (config, adapter): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), +) -> PyResult> { + let py: Python<'py> = adapter.py(); + let vars: Bound<'py, PyAny> = build_effective_runtime_config( + discovered, + Some(target.clone()).filter(|target| !target.is_none()), + None, + )? + .get_item(1)?; + let settings: Bound<'py, PyAny> = + selected_target_settings((discovered, target), adapter_name, (config, &vars, adapter))? + .record(py)?; + if field(&settings, "schema_preserved")?.extract::()? { + return Ok(None); + } + let database: Option = field(&settings, "database")?.extract()?; + Ok(field(&settings, "schema")? + .extract::>()? + .map(|schema| WarehouseStateLocation { database, schema })) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs index 01bd265121..eb9513039f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs @@ -27,7 +27,6 @@ pub(crate) mod cursor; pub(crate) mod dag_artifact; pub(crate) mod dagster_invocation; pub(crate) mod dbt_command; -pub(crate) mod dbt_events; pub(crate) mod dbt_interop; pub(crate) mod debug_command; pub(crate) mod diagnostics; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/constants.rs index d8e63e1987..0bdf54fcda 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/constants.rs @@ -1,5 +1,6 @@ //! Native behaviour of the hosted shapes: computed attributes, methods, ordering and validation. +use crate::bindings::_helpers::dbt_interop; use crate::bindings::_helpers::record_behaviours::auditing::{ finish_measurement_contract, finish_threshold_bound, finish_thresholds, }; @@ -68,7 +69,6 @@ use crate::bindings::_helpers::record_behaviours::scopes::{ use crate::bindings::_helpers::record_behaviours::values::{ finish_fingerprint_set, logical_type_display_name, sql_value_kind, }; -use crate::bindings::_helpers::{dbt_events, dbt_interop}; use sqlbuild_python_base::records::models::{GetterEntry, RecordBehaviours}; use sqlbuild_python_base::records::types::ClassMethod; use sqlbuild_python_base::records::types::ClassProperty; @@ -360,7 +360,7 @@ pub(crate) const GETTER_TABLES: &[&[GetterEntry]] = &[ sqlbuild_python_base::compile_context::constants::GETTERS, sqlbuild_python_project::compiled_views::constants::GETTERS, sqlbuild_python_output::compute_logs::constants::GETTERS, - dbt_events::getters::GETTERS, + sqlbuild_python_output::dbt_events::constants::GETTERS, dbt_interop::getters::GETTERS, sqlbuild_python_base::diagnostics::constants::GETTERS, sqlbuild_python_front::discovery::constants::GETTERS, diff --git a/fensu.toml b/fensu.toml index c282764ae4..3983108bc4 100644 --- a/fensu.toml +++ b/fensu.toml @@ -467,6 +467,11 @@ modules = ["sqlbuild.executor.load.models"] symbols = ["LoadCallbacks", "LoadExecutionResult", "LoadRuntimeParams"] reason = "Native code resolves these through the sqlbuild._native_host name table." +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.dynamic_pauses.main.resume_expired_holds"] +symbols = ["resume_expired_holds"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + [[targets.python.dead_code.roots]] modules = ["sqlbuild.executor.old_name_views._helpers.janitor"] symbols = ["is_compatibility_view"] @@ -1657,12 +1662,12 @@ select = [ [[targets.rust.threshold_overrides]] paths = ["crates/sqlbuild-python/Cargo.toml"] -thresholds = { max_crate_code_lines = 51200 } -reason = "The Python bindings crate is above the code-line limit; this caps it near its current size until it is split into crates by area. The location and plan bindings of the large-project server work and #1560 grew it to 51,122; moving them out would need new cross-crate imports or a policy exception. Lower the cap as code moves out." +thresholds = { max_crate_code_lines = 51000 } +reason = "The Python bindings crate is above the code-line limit; this caps it near its current size until it is split into crates by area. The location and plan bindings of the large-project server work and #1560 grew it to 51,122; moving them out would need new cross-crate imports or a policy exception. The dbt event stream moved to sqlbuild-python-output, and the location sync's dynamic-model resume came in (50,945). Lower the cap as code moves out." [[targets.rust.threshold_overrides]] paths = ["crates/sqlbuild-python/src/bindings/_helpers"] -thresholds = { max_helpers_container_buckets = 93 } +thresholds = { max_helpers_container_buckets = 92 } reason = "The bindings helpers container is above the bucket limit; this caps it at its current bucket count until cohesive groups move into their own modules or crates. Lower the cap as buckets move out." [targets.rust.rule_options.XSB076] @@ -2759,7 +2764,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/dbt_command/init.rs=7", "crates/sqlbuild-python/src/bindings/_helpers/dbt_command/init_project.rs=26", "crates/sqlbuild-python/src/bindings/_helpers/dbt_command/support.rs=6", - "crates/sqlbuild-python/src/bindings/_helpers/dbt_events/stream.rs=0", + "crates/sqlbuild-python-output/src/dbt_events/models.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/cli.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/event_stream.rs=14", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/execution.rs=1", @@ -3066,7 +3071,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/dagster_invocation/arguments.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/dbt_command/command.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/dbt_command/init_project.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/dbt_events/stream.rs=2", + "crates/sqlbuild-python-output/src/dbt_events/models.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/event_stream.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/graph.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/manifest.rs=4", @@ -3139,7 +3144,6 @@ module_layers = [ "dag_artifact=registry", "dagster_invocation=commands", "dbt_command=commands", - "dbt_events=commands", "dbt_interop=commands", "debug_command=commands", "diagnostics=registry", @@ -3565,7 +3569,7 @@ paths = ["crates/sqlbuild-python-base/src/boundary/_helpers/python_calls.rs", "c reason = "Each passes its own keyword set to its own Python callable; only building the keyword dict is alike." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python-base/src/compile_context/_helpers/input_reads.rs", "crates/sqlbuild-python-front/src/discovery/main/project/register_project_files.rs", "crates/sqlbuild-python-front/src/discovery/main/snapshots/register_snapshot.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_events/stream.rs"] +paths = ["crates/sqlbuild-python-base/src/compile_context/_helpers/input_reads.rs", "crates/sqlbuild-python-front/src/discovery/main/project/register_project_files.rs", "crates/sqlbuild-python-front/src/discovery/main/snapshots/register_snapshot.rs", "crates/sqlbuild-python-output/src/dbt_events/main/register_dbt_events.rs"] reason = "Each registers its own module's Python functions; only the add-function calls are alike." [[dupes.allowlist]] @@ -3977,7 +3981,7 @@ paths = ["crates/sqlbuild-python-compile/src/rules/_helpers/lint/lint_sql_bodies reason = "Each reads its own record's fields from its own Python object; only the field-read shape is alike." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python-output/src/build_progress/models.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_events/stream.rs"] +paths = ["crates/sqlbuild-python-output/src/build_progress/models.rs", "crates/sqlbuild-python-output/src/dbt_events/models.rs"] reason = "Each Python constructor reads its own keyword set into its own fields; only the keyword parsing is alike." [[dupes.allowlist]] diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md index 255a91828e..a4b23cd799 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md @@ -572,8 +572,19 @@ Details: interval, add one more sync after it, not one each. - The interval restarts after every completed sync, so a location that syncs often for other reasons never syncs extra. -- A sync reads the warehouse and never writes to it. On Snowflake, it skips state tables that - haven't changed since the last sync, so a sync with nothing new doesn't resume the warehouse. +- A sync reads the warehouse and writes only to the dynamic pause table: on Snowflake, after it + completes, it resumes the [dynamic models](../concepts/models/dynamic.md#when-a-build-dies) whose + pausing build has been silent past `[dynamic] heartbeat_grace`, and records that in the pause + table as `location-sync:`. Before resuming, it records a `resuming` claim on each + model and reads the holds again, so a build that paused the model meanwhile keeps it paused; a + build that finds an open claim waits for it to close. A claim older than 2 minutes, the time a + sync gets to resume, is treated as abandoned. It checks the pause table in the target's schema and in every + schema the location's models write to. A model that fails to resume is logged as + `Resuming an expired dynamic hold failed: : `, keeps its hold, and is retried on + the next sync; the others still resume. Other adapters can't build dynamic models, so their + syncs don't open a session for this. +- On Snowflake, a sync skips state tables that haven't changed since the last sync, the pause + table included, so a sync with nothing new doesn't resume the warehouse. - When a read fails, the Connections page shows the warehouse error. A read that ran out of memory or time retries a smaller page. Any other error, such as a missing permission or table, retries after 30 seconds, then waits twice as long after each failure in a row. After 5 failures in a row diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/dynamic.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/dynamic.md index 4d3c304492..7a869a10e6 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/dynamic.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/dynamic.md @@ -68,14 +68,23 @@ whether it succeeds or fails. both fail this way; run one of them again. - **A stopped build leaves its models suspended.** A build killed before it resumes them, for example by `SIGKILL`, can't resume them itself. -- **The next build resumes them.** The next build that pauses dynamic models on the target - resumes them once the stopped build has been silent for `[dynamic] heartbeat_grace`, 30 - minutes by default; see [Dynamic](../project-configuration.md#dynamic). +- **The location's sync resumes them.** Once the stopped build has been silent for + `[dynamic] heartbeat_grace`, 30 minutes by default, the next completed sync of a served location + resumes its models and closes its hold, and the location's log says which. A sync never leaves + a model running while another build holds it, and it leaves models suspended outside SQLBuild + alone. A build that takes a model over while a sync is resuming it waits for the sync, at most + 2 minutes, then pauses the model again if the sync resumed it. A model it fails to resume, for example for a missing privilege, keeps its hold and is retried + on the next sync. See + [Dynamic](../project-configuration.md#dynamic) and + [location syncs](../../cli/server.md#sync-from-the-warehouse-now). +- **Or the next build does.** Without a served location, the next build that pauses dynamic + models on the target resumes them after the same grace. - **Taking over at once.** Run `sqb build --ignore-dynamic-grace` to take the models over without waiting. It needs `[dynamic] allow_ignore_grace = true`; without it the flag fails with `A719`. You can also resume the models in Snowflake by hand. ## Limitations +- A resume that takes longer than its 2-minute claim can still start a model a build has just paused: the last `ALTER` of a sync killed at the limit, or a build's resume of another model on a queued warehouse. - Unit tests and scenarios treat a dynamic model as a table. - On a target with `schema = "preserve"`, builds see each other's pauses only when their dynamic models share a schema. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md index 2de6859ef1..6c8254d7b8 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md @@ -712,7 +712,7 @@ allow_ignore_grace = false | Field | Default | Description | |-------|---------|-------------| -| `heartbeat_grace` | `"30m"` | How long a build that paused dynamic models may go silent before another build resumes them or takes them over. A fixed duration longer than `"1m"`, such as `"30m"` or `"2h"`; a build writes a heartbeat every minute | +| `heartbeat_grace` | `"30m"` | How long a build that paused dynamic models may go silent before a location's sync or another build resumes them, or another build takes them over. A fixed duration longer than `"1m"`, such as `"30m"` or `"2h"`; a build writes a heartbeat every minute | | `allow_ignore_grace` | `false` | Allow `sqb build --ignore-dynamic-grace`, which takes over models another build holds without waiting for the grace. Without it the flag fails with `A719` | ## Scenario diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index d3384c5a85..d93d3101e5 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -227,6 +227,7 @@ CURRENT_EVENT_LOG_SCHEMA_VERSION: int CURRENT_RUN_STORAGE_SCHEMA_VERSION: int COMPUTE_LOG_FORMAT_VERSION: int DEFAULT_RETENTION_COUNT: int +DYNAMIC_RESUME_SECONDS: int MAX_FUNCTION_CALL_DEPTH: int MAX_READ_BYTES: int CURRENT_COMMAND_OUTPUT_SCHEMA_VERSION: int diff --git a/src/sqlbuild/_native_host.py b/src/sqlbuild/_native_host.py index a35b02aae4..d9a2c5cb1b 100644 --- a/src/sqlbuild/_native_host.py +++ b/src/sqlbuild/_native_host.py @@ -390,6 +390,10 @@ "sqlbuild.executor.review.main.leftover_review_schemas", "leftover_review_schemas", ), + "resume_expired_holds": ( + "sqlbuild.executor.dynamic_pauses.main.resume_expired_holds", + "resume_expired_holds", + ), "FullDiffSizeGuardError": ("sqlbuild.executor.diff.exceptions", "FullDiffSizeGuardError"), "execute_query_diff": ("sqlbuild.executor.diff.main._execute_query", "execute_query_diff"), "execute_diff": ("sqlbuild.executor.diff.main.execute", "execute_diff"), diff --git a/src/sqlbuild/_native_host.pyi b/src/sqlbuild/_native_host.pyi index 080ee8da4c..0f6b520fdd 100644 --- a/src/sqlbuild/_native_host.pyi +++ b/src/sqlbuild/_native_host.pyi @@ -265,6 +265,9 @@ from sqlbuild.executor.diff.models import DiffExecutionOptions as DiffExecutionO from sqlbuild.executor.diff.models import DiffExecutionResult as DiffExecutionResult from sqlbuild.executor.diff.models import FullDiffSizeLimits as FullDiffSizeLimits from sqlbuild.executor.diff.models import RowDiffSamplingOverride as RowDiffSamplingOverride +from sqlbuild.executor.dynamic_pauses.main.resume_expired_holds import ( + resume_expired_holds as resume_expired_holds, +) from sqlbuild.executor.janitor.main.execute import execute_janitor_plan as execute_janitor_plan from sqlbuild.executor.janitor.models import JanitorArchiveCandidate as JanitorArchiveCandidate from sqlbuild.executor.janitor.models import JanitorArchivedRelation as JanitorArchivedRelation diff --git a/src/sqlbuild/executor/dynamic_pauses/_helpers/resume.py b/src/sqlbuild/executor/dynamic_pauses/_helpers/resume.py new file mode 100644 index 0000000000..5d6e98857c --- /dev/null +++ b/src/sqlbuild/executor/dynamic_pauses/_helpers/resume.py @@ -0,0 +1,229 @@ +"""Resume dynamic models through the adapter under a claim, and close the holds that paused them.""" + +from __future__ import annotations + +import contextlib +import uuid +from typing import Any + +from sqlbuild._native import DYNAMIC_PAUSES_TABLE_NAME, CompiledRelationLocation +from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter +from sqlbuild.adapter.contract.types import DynamicListing, DynamicOperation +from sqlbuild.adapter.relations.types import RelationChangeTokenReader +from sqlbuild.executor.dynamic_pauses._helpers.sql import insert_sql, read_open_holds +from sqlbuild.executor.dynamic_pauses.constants import CLAIM_RUN_MARKER, SUSPENDED_STATE +from sqlbuild.executor.dynamic_pauses.models import ( + DynamicPauseEvent, + HoldsResumed, + OpenHold, + ResumeOutcome, +) +from sqlbuild.executor.dynamic_pauses.types import DynamicPauseEventType, StateLocation + + +def scheduling_state( + *, adapter: BaseAdapter, connection: Any, relation: CompiledRelationLocation +) -> str | None: + """A dynamic model's scheduling state, or None when the warehouse does not list it.""" + + listing: DynamicListing | None = adapter.dynamic_listing( + connection=connection, + database=relation.database, + schema=relation.schema, + name=relation.name, + ) + return None if listing is None else listing.scheduling_state + + +def run_dynamic_operation( + *, + adapter: BaseAdapter, + connection: Any, + relation: CompiledRelationLocation, + operation: DynamicOperation, +) -> None: + """Suspend or resume one dynamic model.""" + + destination: str = relation.qualified_name or str( + adapter.render_qualified_name( + database=relation.database, schema=relation.schema, name=relation.name + ) + ) + for statement in adapter.render_dynamic_operation(destination=destination, operation=operation): + adapter.execute(connection=connection, sql=statement) + + +def _resume_suspended( + *, adapter: BaseAdapter, connection: Any, relation: CompiledRelationLocation +) -> None: + """Resume if suspended; known gap: an ALTER sent just before a kill may land after its claim.""" + + if scheduling_state(adapter=adapter, connection=connection, relation=relation) == ( + SUSPENDED_STATE + ): + run_dynamic_operation( + adapter=adapter, + connection=connection, + relation=relation, + operation=DynamicOperation.RESUME, + ) + + +def resume_expired( + *, + adapter: BaseAdapter, + connection: Any, + location: StateLocation, + holds_after: tuple[float, str], + quiet_token: str | None, +) -> ResumeOutcome: + """Resume holds silent past the grace seconds unless the pause table still has `quiet_token`.""" + + grace_seconds: float + recorded_by: str + grace_seconds, recorded_by = holds_after + if not adapter.relation_exists( + connection=connection, + database=location[0], + schema=location[1], + name=DYNAMIC_PAUSES_TABLE_NAME, + ): + return ResumeOutcome(resumed=(), failures=(), quiet_token=None) + token: str | None = ( + adapter.read_relation_change_token( + connection=connection, + database=location[0], + schema=location[1], + name=DYNAMIC_PAUSES_TABLE_NAME, + ) + if isinstance(adapter, RelationChangeTokenReader) + else None + ) + if token is not None and token == quiet_token: + return ResumeOutcome(resumed=(), failures=(), quiet_token=token) + holds: tuple[OpenHold, ...] = read_open_holds( + adapter=adapter, connection=connection, location=location + ) + live: set[tuple[str | None, str | None, str]] = { + (hold.relation.database, hold.relation.schema, hold.relation.name) + for hold in holds + if hold.idle_seconds < grace_seconds + } + expired: tuple[OpenHold, ...] = tuple( + hold + for hold in holds + if hold.idle_seconds >= grace_seconds + and (hold.relation.database, hold.relation.schema, hold.relation.name) not in live + ) + outcome: HoldsResumed = claim_and_resume( + adapter=adapter, + connection=connection, + location=location, + holds=expired, + holds_after=holds_after, + ) + return ResumeOutcome( + resumed=outcome.resumed, + failures=tuple(f"{_dotted(hold.relation)}: {error}" for hold, error in outcome.failures), + quiet_token=None if holds else token, + ) + + +def claim_and_resume( + *, + adapter: BaseAdapter, + connection: Any, + location: StateLocation, + holds: tuple[OpenHold, ...], + holds_after: tuple[float, str], +) -> HoldsResumed: + """Claim the expired holds' models, then resume only those no run holds live after the claim.""" + + grace_seconds: float + recorded_by: str + grace_seconds, recorded_by = holds_after + if not holds: + return HoldsResumed(resumed=(), failures=()) + claim_run: str = f"{recorded_by}{CLAIM_RUN_MARKER}{uuid.uuid4().hex}" + claimed: tuple[CompiledRelationLocation, ...] = tuple( + {_relation_key(hold.relation): hold.relation for hold in holds}.values() + ) + claims: tuple[DynamicPauseEvent, ...] = tuple( + DynamicPauseEvent( + event_type=DynamicPauseEventType.RESUMING, + run_id=claim_run, + recorded_by=recorded_by, + relation=relation, + ) + for relation in claimed + ) + _append(adapter=adapter, connection=connection, location=location, events=claims) + closes: tuple[DynamicPauseEvent, ...] = tuple( + DynamicPauseEvent( + event_type=DynamicPauseEventType.RESUMED, + run_id=claim_run, + recorded_by=recorded_by, + relation=relation, + ) + for relation in claimed + ) + try: + live: frozenset[tuple[str | None, str | None, str]] = frozenset( + _relation_key(hold.relation) + for hold in read_open_holds(adapter=adapter, connection=connection, location=location) + if hold.idle_seconds < grace_seconds + ) + except Exception: + with contextlib.suppress(Exception): + _append(adapter=adapter, connection=connection, location=location, events=closes) + raise + resumed: list[OpenHold] = [] + failures: list[tuple[OpenHold, Exception]] = [] + for hold in holds: + if _relation_key(hold.relation) in live: + continue + try: + _resume_suspended(adapter=adapter, connection=connection, relation=hold.relation) + except Exception as error: + failures.append((hold, error)) + continue + resumed.append(hold) + _append( + adapter=adapter, + connection=connection, + location=location, + events=( + *( + DynamicPauseEvent( + event_type=DynamicPauseEventType.RESUMED, + run_id=hold.run_id, + recorded_by=recorded_by, + relation=hold.relation, + ) + for hold in resumed + ), + *closes, + ), + ) + return HoldsResumed(resumed=tuple(resumed), failures=tuple(failures)) + + +def _append( + *, + adapter: BaseAdapter, + connection: Any, + location: StateLocation, + events: tuple[DynamicPauseEvent, ...], +) -> None: + adapter.execute( + connection=connection, + sql=insert_sql(adapter=adapter, location=location, events=events), + ) + + +def _relation_key(relation: CompiledRelationLocation) -> tuple[str | None, str | None, str]: + return (relation.database, relation.schema, relation.name) + + +def _dotted(relation: CompiledRelationLocation) -> str: + return ".".join(part for part in (relation.database, relation.schema, relation.name) if part) diff --git a/src/sqlbuild/executor/dynamic_pauses/_helpers/sql.py b/src/sqlbuild/executor/dynamic_pauses/_helpers/sql.py index 1870e75994..791cbe4f2a 100644 --- a/src/sqlbuild/executor/dynamic_pauses/_helpers/sql.py +++ b/src/sqlbuild/executor/dynamic_pauses/_helpers/sql.py @@ -30,7 +30,7 @@ DYNAMIC_PAUSE_REQUIRED_COLUMNS, DYNAMIC_PAUSE_SCHEMA_VERSION, ) -from sqlbuild.executor.dynamic_pauses.models import DynamicPauseEvent, OpenHold +from sqlbuild.executor.dynamic_pauses.models import DynamicPauseEvent, OpenClaim, OpenHold from sqlbuild.executor.dynamic_pauses.types import DynamicPauseEventType, StateLocation @@ -78,19 +78,89 @@ def open_holds_sql(*, adapter: BaseAdapter, location: StateLocation) -> str: table: str = pause_table_name(adapter=adapter, location=location) now: str = adapter.render_current_utc_timestamp() + return _open_rows_sql( + table=table, + event_type=DynamicPauseEventType.PAUSED, + timings=( + f"CAST(h.last_seen AS TIMESTAMP), CAST({now} AS TIMESTAMP), " + f"CAST(p.{COLUMN_RECORDED_AT} AS TIMESTAMP)" + ), + join=( + f"JOIN (SELECT {COLUMN_RUN_ID}, MAX({COLUMN_RECORDED_AT}) AS last_seen " + f"FROM {table} WHERE {COLUMN_RECORDED_BY} = {COLUMN_RUN_ID} GROUP BY {COLUMN_RUN_ID}) " + f"h ON h.{COLUMN_RUN_ID} = p.{COLUMN_RUN_ID} " + ), + ) + + +def open_claims_sql(*, adapter: BaseAdapter, location: StateLocation) -> str: + """Sync claims no `RESUMED` row of their run closed, with their age by the warehouse clock.""" + + now: str = adapter.render_current_utc_timestamp() + return _open_rows_sql( + table=pause_table_name(adapter=adapter, location=location), + event_type=DynamicPauseEventType.RESUMING, + timings=f"CAST(p.{COLUMN_RECORDED_AT} AS TIMESTAMP), CAST({now} AS TIMESTAMP)", + join="", + ) + + +def open_claims(*, rows: tuple[tuple[Any, ...], ...]) -> tuple[OpenClaim, ...]: + """The open claims of `open_claims_sql` rows.""" + + return tuple( + OpenClaim( + run_id=str(row[0]), + relation=_row_relation(row=row), + age_seconds=_seconds_between(start=row[4], end=row[5]), + ) + for row in rows + ) + + +def read_open_claims( + *, adapter: BaseAdapter, connection: Any, location: StateLocation +) -> tuple[OpenClaim, ...]: + """The pause table's open resume claims as the warehouse sees them now.""" + + return open_claims( + rows=tuple( + adapter.query( + connection=connection, + sql=open_claims_sql(adapter=adapter, location=location), + limit=None, + ).rows + ) + ) + + +def _open_rows_sql( + *, table: str, event_type: DynamicPauseEventType, timings: str, join: str +) -> str: + """Rows of `event_type` no `RESUMED` row of their run closed: run, relation, then `timings`.""" + + return ( + f"SELECT p.{COLUMN_RUN_ID}, p.{COLUMN_RELATION_DATABASE}, p.{COLUMN_RELATION_SCHEMA}, " + f"p.{COLUMN_RELATION_NAME}, {timings} FROM {table} p {join}" + f"WHERE p.{COLUMN_EVENT_TYPE} = '{event_type.value}' AND {_not_resumed(table=table)}" + ) + + +def _row_relation(*, row: tuple[Any, ...]) -> CompiledRelationLocation: + return CompiledRelationLocation( + database=_optional(row[1]), schema=_optional(row[2]), name=str(row[3]), qualified_name=None + ) + + +def _not_resumed(*, table: str) -> str: + """No `RESUMED` row of row `p`'s run closes it for `p`'s model.""" + same_relation: str = " AND ".join( f"COALESCE(r.{column}, '') = COALESCE(p.{column}, '')" for column in (COLUMN_RELATION_DATABASE, COLUMN_RELATION_SCHEMA, COLUMN_RELATION_NAME) ) return ( - f"SELECT p.{COLUMN_RUN_ID}, p.{COLUMN_RELATION_DATABASE}, p.{COLUMN_RELATION_SCHEMA}, " - f"p.{COLUMN_RELATION_NAME}, CAST(h.last_seen AS TIMESTAMP), CAST({now} AS TIMESTAMP), " - f"CAST(p.{COLUMN_RECORDED_AT} AS TIMESTAMP) " - f"FROM {table} p JOIN (SELECT {COLUMN_RUN_ID}, MAX({COLUMN_RECORDED_AT}) AS last_seen " - f"FROM {table} WHERE {COLUMN_RECORDED_BY} = {COLUMN_RUN_ID} GROUP BY {COLUMN_RUN_ID}) h " - f"ON h.{COLUMN_RUN_ID} = p.{COLUMN_RUN_ID} " - f"WHERE p.{COLUMN_EVENT_TYPE} = '{DynamicPauseEventType.PAUSED.value}' " - f"AND NOT EXISTS (SELECT 1 FROM {table} r " + f"NOT EXISTS (SELECT 1 FROM {table} r " f"WHERE r.{COLUMN_EVENT_TYPE} = '{DynamicPauseEventType.RESUMED.value}' " f"AND r.{COLUMN_RUN_ID} = p.{COLUMN_RUN_ID} AND {same_relation})" ) @@ -102,12 +172,7 @@ def open_holds(*, rows: tuple[tuple[Any, ...], ...]) -> tuple[OpenHold, ...]: return tuple( OpenHold( run_id=str(row[0]), - relation=CompiledRelationLocation( - database=_optional(row[1]), - schema=_optional(row[2]), - name=str(row[3]), - qualified_name=None, - ), + relation=_row_relation(row=row), idle_seconds=_seconds_between(start=row[4], end=row[5]), paused_at=row[6].replace(tzinfo=None), ) @@ -115,6 +180,22 @@ def open_holds(*, rows: tuple[tuple[Any, ...], ...]) -> tuple[OpenHold, ...]: ) +def read_open_holds( + *, adapter: BaseAdapter, connection: Any, location: StateLocation +) -> tuple[OpenHold, ...]: + """The pause table's open holds as the warehouse sees them now.""" + + return open_holds( + rows=tuple( + adapter.query( + connection=connection, + sql=open_holds_sql(adapter=adapter, location=location), + limit=None, + ).rows + ) + ) + + def _seconds_between(*, start: datetime, end: datetime) -> float: return (end.replace(tzinfo=None) - start.replace(tzinfo=None)).total_seconds() @@ -145,7 +226,7 @@ def _row_literals(*, event: DynamicPauseEvent) -> str: def _event_id(*, event: DynamicPauseEvent) -> str: - """Content-addressed: one pause and one resume per run and model, one ID per heartbeat.""" + """Content-addressed: one per type, run and model, one per heartbeat; each claim is a run.""" relation: CompiledRelationLocation | None = event.relation identity: tuple[object, ...] = ( diff --git a/src/sqlbuild/executor/dynamic_pauses/classes/dynamic_hold.py b/src/sqlbuild/executor/dynamic_pauses/classes/dynamic_hold.py index 9db3018d4c..339d51f301 100644 --- a/src/sqlbuild/executor/dynamic_pauses/classes/dynamic_hold.py +++ b/src/sqlbuild/executor/dynamic_pauses/classes/dynamic_hold.py @@ -12,11 +12,16 @@ from sqlbuild._native import CompiledRelationLocation from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.exceptions import AdapterUserError -from sqlbuild.adapter.contract.types import DynamicListing, DynamicOperation +from sqlbuild.adapter.contract.types import DynamicOperation +from sqlbuild.executor.dynamic_pauses._helpers.resume import ( + claim_and_resume, + run_dynamic_operation, + scheduling_state, +) from sqlbuild.executor.dynamic_pauses._helpers.sql import ( insert_sql, - open_holds, - open_holds_sql, + read_open_claims, + read_open_holds, setup_statements, ) from sqlbuild.executor.dynamic_pauses.classes.dynamic_heartbeat import ( @@ -27,7 +32,12 @@ HELD_ELSEWHERE_CODE, SUSPENDED_STATE, ) -from sqlbuild.executor.dynamic_pauses.models import DynamicPauseEvent, OpenHold +from sqlbuild.executor.dynamic_pauses.models import ( + DynamicPauseEvent, + HoldsResumed, + OpenClaim, + OpenHold, +) from sqlbuild.executor.dynamic_pauses.types import DynamicPauseEventType, StateLocation type _RelationKey = tuple[str | None, str | None, str] @@ -45,8 +55,10 @@ def __init__( state_location: StateLocation | None, settings: tuple[str, bool, float, bool], heartbeat: Callable[[Callable[..., None]], DynamicHeartbeat], + claim_wait: tuple[float, float], ) -> None: self._adapter: BaseAdapter = adapter + self._claim_wait: tuple[float, float] = claim_wait self._held: dict[str, CompiledRelationLocation] = held self._refreshed: frozenset[str] = refreshed self._state_location: StateLocation | None = state_location @@ -130,9 +142,19 @@ def acquire(self, *, connection: Any, on_progress: Callable[[str], None] | None) self._yield_to_rivals( connection=connection, location=location, names=tuple(self._paused), window=window ) - self._suspend_active(connection=connection, names=tuple(self._paused), states=states) self._heartbeat = self._new_heartbeat(self._write_heartbeat) self._heartbeat.start() + claimed: frozenset[_RelationKey] = self._await_claims( + connection=connection, location=location, names=tuple(self._paused) + ) + states.update( + { + name: self._state(connection=connection, name=name) + for name in self._paused + if _key(self._held[name]) in taken_keys | claimed + } + ) + self._suspend_active(connection=connection, names=tuple(self._paused), states=states) def after_create(self, *, connection: Any, name: str) -> None: """Suspend a model CREATE OR REPLACE made active; check its new hold as acquire does.""" @@ -198,16 +220,13 @@ def release(self, *, connection: Any) -> None: def _take_over(self, *, connection: Any, location: StateLocation) -> tuple[OpenHold, ...]: """Resume expired holds on other models now; return the held models' holds to take over.""" - rows: tuple[tuple[Any, ...], ...] = tuple( - self._adapter.query( - connection=connection, - sql=open_holds_sql(adapter=self._adapter, location=location), - limit=None, - ).rows - ) held_keys: set[_RelationKey] = {_key(location) for location in self._held.values()} holds: tuple[OpenHold, ...] = tuple( - hold for hold in open_holds(rows=rows) if hold.run_id != self._run_id + hold + for hold in read_open_holds( + adapter=self._adapter, connection=connection, location=location + ) + if hold.run_id != self._run_id ) live: tuple[OpenHold, ...] = tuple( hold @@ -222,17 +241,17 @@ def _take_over(self, *, connection: Any, location: StateLocation) -> tuple[OpenH elsewhere: tuple[OpenHold, ...] = tuple( hold for hold in expired if _key(hold.relation) not in held_keys ) - for hold in elsewhere: - self._resume_found(connection=connection, relation=hold.relation) - self._record( + resumed: HoldsResumed = claim_and_resume( + adapter=self._adapter, connection=connection, - events=tuple( - self._event(event_type=DynamicPauseEventType.RESUMED, hold=hold) - for hold in elsewhere - ), + location=location, + holds=elsewhere, + holds_after=(self._grace_seconds, self._run_id), ) + self._report(holds=resumed.resumed, verb="Resumed") + if resumed.failures: + raise resumed.failures[0][1] taken: tuple[OpenHold, ...] = (*live, *(h for h in expired if h not in elsewhere)) - self._report(holds=elsewhere, verb="Resumed") self._report(holds=taken, verb="Took over") return taken @@ -249,14 +268,9 @@ def _yield_to_rivals( mine: set[_RelationKey] = {_key(self._held[name]) for name in names} if not mine: return - rows: tuple[tuple[Any, ...], ...] = tuple( - self._adapter.query( - connection=connection, - sql=open_holds_sql(adapter=self._adapter, location=location), - limit=None, - ).rows + holds: tuple[OpenHold, ...] = read_open_holds( + adapter=self._adapter, connection=connection, location=location ) - holds: tuple[OpenHold, ...] = open_holds(rows=rows) paused_at: dict[_RelationKey, datetime] = { _key(hold.relation): hold.paused_at for hold in holds @@ -291,6 +305,34 @@ def _yield_to_rivals( raise _held_elsewhere(holds=tuple(rivals), close_failure=error) from error raise _held_elsewhere(holds=tuple(rivals)) + def _await_claims( + self, *, connection: Any, location: StateLocation, names: tuple[str, ...] + ) -> frozenset[_RelationKey]: + """Wait while a sync's claim on these models is open and young; return those claimed.""" + + mine: set[_RelationKey] = {_key(self._held[name]) for name in names} + if not mine: + return frozenset() + limit_seconds: float + poll_seconds: float + limit_seconds, poll_seconds = self._claim_wait + deadline: float = time.monotonic() + limit_seconds + claimed: set[_RelationKey] = set() + while True: + claims: tuple[OpenClaim, ...] = tuple( + claim + for claim in read_open_claims( + adapter=self._adapter, connection=connection, location=location + ) + if _key(claim.relation) in mine and claim.age_seconds < limit_seconds + ) + for name in sorted({claim.relation.name for claim in claims} - {k[2] for k in claimed}): + self._progress(f"Waiting for a location sync resuming dynamic model {name}") + claimed.update(_key(claim.relation) for claim in claims) + if not claims or time.monotonic() >= deadline: + return frozenset(claimed) + time.sleep(poll_seconds) + def _resume_unheld( self, *, connection: Any, holds: tuple[OpenHold, ...] | None ) -> list[Exception]: @@ -319,27 +361,13 @@ def _open_holds(self, *, connection: Any) -> tuple[OpenHold, ...] | None: if location is None or self._pause_dynamic: return None try: - rows: tuple[tuple[Any, ...], ...] = tuple( - self._adapter.query( - connection=connection, - sql=open_holds_sql(adapter=self._adapter, location=location), - limit=None, - ).rows + holds: tuple[OpenHold, ...] = read_open_holds( + adapter=self._adapter, connection=connection, location=location ) except Exception as error: self._warn(f"could not read which dynamic models this build still holds: {error}") return None - return open_holds(rows=rows) - - def _resume_found(self, *, connection: Any, relation: CompiledRelationLocation) -> None: - if _scheduling_state(adapter=self._adapter, connection=connection, relation=relation) == ( - SUSPENDED_STATE - ): - self._run( - connection=connection, - destination=self._destination(relation), - operation=DynamicOperation.RESUME, - ) + return holds def _suspend_active( self, @@ -358,30 +386,18 @@ def _suspend_active( self._operate(connection=connection, name=name, operation=DynamicOperation.SUSPEND) def _state(self, *, connection: Any, name: str) -> str | None: - return _scheduling_state( + return scheduling_state( adapter=self._adapter, connection=connection, relation=self._held[name] ) def _operate(self, *, connection: Any, name: str, operation: DynamicOperation) -> None: - self._run( + run_dynamic_operation( + adapter=self._adapter, connection=connection, - destination=self._destination(self._held[name]), + relation=self._held[name], operation=operation, ) - def _run(self, *, connection: Any, destination: str, operation: DynamicOperation) -> None: - for statement in self._adapter.render_dynamic_operation( - destination=destination, operation=operation - ): - self._adapter.execute(connection=connection, sql=statement) - - def _destination(self, location: CompiledRelationLocation) -> str: - return location.qualified_name or str( - self._adapter.render_qualified_name( - database=location.database, schema=location.schema, name=location.name - ) - ) - def _timed_record(self, *, connection: Any, events: tuple[DynamicPauseEvent, ...]) -> timedelta: """Record, and return how long the insert took: an upper bound on its commit delay.""" @@ -441,23 +457,15 @@ def _report_heartbeat_failure(self, *, failure: Exception | None) -> None: if failure is not None: self._warn(f"a dynamic pause heartbeat failed: {failure}") + def _progress(self, message: str) -> None: + if self._on_progress is not None: + self._on_progress(message) + def _warn(self, message: str) -> None: if self._on_progress is not None: self._on_progress(f"Warning: {message}") -def _scheduling_state( - *, adapter: BaseAdapter, connection: Any, relation: CompiledRelationLocation -) -> str | None: - listing: DynamicListing | None = adapter.dynamic_listing( - connection=connection, - database=relation.database, - schema=relation.schema, - name=relation.name, - ) - return None if listing is None else listing.scheduling_state - - def _key(location: CompiledRelationLocation) -> _RelationKey: return (location.database, location.schema, location.name) diff --git a/src/sqlbuild/executor/dynamic_pauses/constants.py b/src/sqlbuild/executor/dynamic_pauses/constants.py index 1abaac9786..5d5f7b4abc 100644 --- a/src/sqlbuild/executor/dynamic_pauses/constants.py +++ b/src/sqlbuild/executor/dynamic_pauses/constants.py @@ -4,6 +4,8 @@ DYNAMIC_PAUSE_SCHEMA_VERSION: int = 1 HEARTBEAT_SECONDS: float = 60.0 +CLAIM_POLL_SECONDS: float = 1.0 +CLAIM_RUN_MARKER: str = "/claim/" ACTIVE_STATE: str = "ACTIVE" SUSPENDED_STATE: str = "SUSPENDED" HELD_ELSEWHERE_CODE: str = "A718" diff --git a/src/sqlbuild/executor/dynamic_pauses/main/hold_dynamic_models.py b/src/sqlbuild/executor/dynamic_pauses/main/hold_dynamic_models.py index f6a3112bc0..a837d9146d 100644 --- a/src/sqlbuild/executor/dynamic_pauses/main/hold_dynamic_models.py +++ b/src/sqlbuild/executor/dynamic_pauses/main/hold_dynamic_models.py @@ -2,7 +2,12 @@ from __future__ import annotations -from sqlbuild._native import CompiledRelationLocation, PlanOutput, TargetSettings +from sqlbuild._native import ( + DYNAMIC_RESUME_SECONDS, + CompiledRelationLocation, + PlanOutput, + TargetSettings, +) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.exceptions import AdapterUserError from sqlbuild.executor.build.models import BuildRuntimeParams @@ -12,6 +17,7 @@ ) from sqlbuild.executor.dynamic_pauses.classes.dynamic_hold import DynamicHold from sqlbuild.executor.dynamic_pauses.constants import ( + CLAIM_POLL_SECONDS, HEARTBEAT_SECONDS, IGNORE_GRACE_REFUSED_CODE, ) @@ -25,8 +31,9 @@ def hold_dynamic_models( runtime: BuildRuntimeParams, connection_config: dict[str, object], heartbeat_seconds: float = HEARTBEAT_SECONDS, + claim_wait: tuple[float, float] = (float(DYNAMIC_RESUME_SECONDS), CLAIM_POLL_SECONDS), ) -> DynamicHold | None: - """The build's hold, unacquired; `--ignore-dynamic-grace` needs `allow_ignore_grace`.""" + """The build's hold, unacquired; `claim_wait` is a sync claim's age limit and poll interval.""" if runtime.ignore_dynamic_grace and not runtime.allow_ignore_dynamic_grace: raise AdapterUserError( @@ -56,6 +63,7 @@ def hold_dynamic_models( write=write, interval_seconds=heartbeat_seconds, ), + claim_wait=claim_wait, ) diff --git a/src/sqlbuild/executor/dynamic_pauses/main/resume_expired_holds.py b/src/sqlbuild/executor/dynamic_pauses/main/resume_expired_holds.py new file mode 100644 index 0000000000..80b8a7c87f --- /dev/null +++ b/src/sqlbuild/executor/dynamic_pauses/main/resume_expired_holds.py @@ -0,0 +1,30 @@ +"""Resume the dynamic models whose holds expired: a location sync's only warehouse write.""" + +from __future__ import annotations + +from typing import Any + +from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter +from sqlbuild.executor.dynamic_pauses._helpers.resume import resume_expired +from sqlbuild.executor.dynamic_pauses.models import ResumeOutcome +from sqlbuild.executor.dynamic_pauses.types import StateLocation + + +def resume_expired_holds( + *, + adapter: BaseAdapter, + connection: Any, + location: StateLocation, + grace_seconds: float, + recorded_by: str, + quiet_token: str | None, +) -> ResumeOutcome: + """Close expired holds on models no run holds live; a still-quiet pause table is not read.""" + + return resume_expired( + adapter=adapter, + connection=connection, + location=location, + holds_after=(grace_seconds, recorded_by), + quiet_token=quiet_token, + ) diff --git a/src/sqlbuild/executor/dynamic_pauses/models.py b/src/sqlbuild/executor/dynamic_pauses/models.py index 62f103abc2..899299142c 100644 --- a/src/sqlbuild/executor/dynamic_pauses/models.py +++ b/src/sqlbuild/executor/dynamic_pauses/models.py @@ -20,6 +20,15 @@ class DynamicPauseEvent: sequence: int = 0 +@dataclass(frozen=True) +class OpenClaim: + """A sync's claim on a model it may be resuming, and its age by the warehouse clock.""" + + run_id: str + relation: CompiledRelationLocation + age_seconds: float + + @dataclass(frozen=True) class OpenHold: """A model a run paused and nobody resumed, when it paused, and how long the run is silent.""" @@ -28,3 +37,20 @@ class OpenHold: relation: CompiledRelationLocation idle_seconds: float paused_at: datetime + + +@dataclass(frozen=True) +class HoldsResumed: + """The holds whose models resumed and were closed, and each hold that failed with its error.""" + + resumed: tuple[OpenHold, ...] + failures: tuple[tuple[OpenHold, Exception], ...] + + +@dataclass(frozen=True) +class ResumeOutcome: + """A pass's resumed holds and failures, and the pause table's token when it left none open.""" + + resumed: tuple[OpenHold, ...] + failures: tuple[str, ...] + quiet_token: str | None diff --git a/src/sqlbuild/executor/dynamic_pauses/types.py b/src/sqlbuild/executor/dynamic_pauses/types.py index 11f347b513..9543ba2b19 100644 --- a/src/sqlbuild/executor/dynamic_pauses/types.py +++ b/src/sqlbuild/executor/dynamic_pauses/types.py @@ -13,3 +13,5 @@ class DynamicPauseEventType(StrEnum): PAUSED = "paused" RESUMED = "resumed" HEARTBEAT = "heartbeat" + RESUMING = "resuming" + """A sync's claim on a model it is about to resume, closed by a `RESUMED` row of its run.""" diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py index 3b0cc10f74..b470e3d1a7 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py @@ -583,6 +583,13 @@ class ChangedBuildPublicationTestCase: expected_published: dict[str, list[str]] +@dataclass(frozen=True) +class NoDynamicAdapterSyncTestCase: + description: str + fixture: str + expected_open_holds: tuple[str, ...] + + @dataclass(frozen=True) class ChangedBuildReplanTestCase: description: str diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py index 20dd587f77..154523ccb2 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_warehouse_metadata.py @@ -22,6 +22,7 @@ import pytest from sqlbuild._native import ( + CompiledRelationLocation, DiscoveredProjectInputs, Fingerprint, SourceFreshnessRecord, @@ -36,6 +37,9 @@ ) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.types import CompiledProjectView +from sqlbuild.executor.dynamic_pauses._helpers.sql import insert_sql, setup_statements +from sqlbuild.executor.dynamic_pauses.models import DynamicPauseEvent +from sqlbuild.executor.dynamic_pauses.types import DynamicPauseEventType, StateLocation from tests.integration.src.sqlbuild.compiler.compiled_project.helpers import ( METADATA, PUBLISHABLE_FILES, @@ -45,6 +49,7 @@ CatalogConnectionTestCase, FingerprintHistoryTestCase, NewestVersionsTestCase, + NoDynamicAdapterSyncTestCase, PageBytesTestCase, ReadFailureTestCase, StateEventsTestCase, @@ -1073,6 +1078,102 @@ def test_given_a_served_location_when_warehouse_history_changes_then_periodic_re record_finished_run(control) +@pytest.mark.parametrize( + "test_case", + [ + NoDynamicAdapterSyncTestCase( + description="an expired hold in a DuckDB pause table stays open after the sync", + fixture=( + "benchmarks/duckshop/edge_cases/orchestration/" + "dynamic_hold_without_dynamic_adapter.json" + ), + expected_open_holds=("running-build", "stopped-build"), + ), + ], + ids=lambda case: case.description, +) +def test_given_a_duckdb_location_when_its_sync_completes_then_it_opens_no_resume_session( + test_case: NoDynamicAdapterSyncTestCase, tmp_path: Path +) -> None: + project: Path = tmp_path / "project" + fixture: dict[str, Any] = json.loads(Path(test_case.fixture).read_text()) + adapter: Any + connection: Any + adapter, connection = open_legacy_warehouse(project, "duckdb", "database") + location: StateLocation = (None, "main") + for statement in setup_statements(adapter=adapter, location=location): + _ = adapter.execute(connection=connection, sql=statement) + for hold in fixture["holds"]: + event: DynamicPauseEvent = DynamicPauseEvent( + event_type=DynamicPauseEventType.PAUSED, + run_id=hold["run_id"], + recorded_by=hold["run_id"], + relation=CompiledRelationLocation( + database=None, schema="main", name=hold["model"], qualified_name=None + ), + ) + _ = adapter.execute( + connection=connection, + sql=insert_sql(adapter=adapter, location=location, events=(event,)), + ) + _ = adapter.execute( + connection=connection, + sql=( + "UPDATE main._sqlbuild_dynamic_pauses SET recorded_at = recorded_at - " + f"INTERVAL {hold['idle_seconds']} SECOND WHERE run_id = '{hold['run_id']}'" + ), + ) + adapter.close(connection) + (project / "orchestration/calendars/unexpected_calendar.py").unlink() + with (project / "sqlbuild_project.toml").open("a") as config: + _ = config.write(fixture["dynamic"]) + write_files(project_dir=project, files={"models/orders.sql": fixture["model"]}) + with snapshot_api(tmp_path / "control") as api: + log: Path = tmp_path / "location.log" + with location_process( + [ + str(Path(sys.executable).with_name("sqb")), + "--project-dir", + str(project), + "location", + "serve", + "--server", + api.url, + "--location-id", + api.location_id, + "--target", + "development", + ], + log, + tmp_path / "location.stdout", + env=os.environ | {"SQLBUILD_LOCATION_TOKEN": api.bearer}, + ) as served: + _ = wait_for_generation(api, "development", 1, served, log) + deadline: float = time.monotonic() + 45 + while "synchronization completed" not in log.read_text(): + assert served.poll() is None, log.read_text() + assert time.monotonic() < deadline, log.read_text() + record_finished_run(tmp_path / "control/.sqlbuild/control.sqlite3") + time.sleep(0.5) + wait_for_idle_sync(served, log) + text: str = log.read_text() + connection = adapter.connect({"database": str(project / "warehouse.db")}) + events: list[tuple[Any, ...]] = adapter.query( + connection=connection, + sql=( + "SELECT run_id, event_type FROM main._sqlbuild_dynamic_pauses " + "ORDER BY run_id, event_type" + ), + limit=10, + ).rows + adapter.close(connection) + assert [tuple(row) for row in events] == [ + (run_id, "paused") for run_id in test_case.expected_open_holds + ], text + assert "Resumed dynamic model" not in text, text + assert "Resuming an expired dynamic hold failed" not in text, text + + @pytest.mark.parametrize( "test_case", [ diff --git a/tests/integration/src/sqlbuild/executor/dynamic_pauses/_test_types.py b/tests/integration/src/sqlbuild/executor/dynamic_pauses/_test_types.py index f6b867c312..22fa9c95ab 100644 --- a/tests/integration/src/sqlbuild/executor/dynamic_pauses/_test_types.py +++ b/tests/integration/src/sqlbuild/executor/dynamic_pauses/_test_types.py @@ -106,3 +106,109 @@ class TakenOverPauseTestCase: expected_rival: str expected_states: dict[str, str] expected_events: tuple[tuple[str, str, str, str], ...] + + +@dataclass(frozen=True) +class SyncResumeTestCase: + description: str + holds: tuple[tuple[str, str, int], ...] + hand_suspended: tuple[str, ...] + pause_table: bool + expected_resumed: tuple[tuple[str, str], ...] + expected_states: dict[str, str] + expected_events: tuple[tuple[str, str, str, str], ...] + + +@dataclass(frozen=True) +class DoubleCloseTestCase: + description: str + closers: tuple[str, ...] + expected_events: tuple[tuple[str, str, str, str], ...] + + +@dataclass(frozen=True) +class SyncDuringTakeoverTestCase: + description: str + expected_operations: tuple[tuple[str, str], ...] + expected_states: dict[str, str] + + +@dataclass(frozen=True) +class ClaimRaceTestCase: + description: str + expected_operations: tuple[tuple[str, str], ...] + expected_states: dict[str, str] + expected_open_holds: tuple[str, ...] + expected_claims: tuple[tuple[str, str, str], ...] + + +@dataclass(frozen=True) +class ClaimWaitTestCase: + description: str + expected_operations: tuple[tuple[str, str], ...] + expected_states: dict[str, str] + expected_claim_reads: int + expected_progress: str + + +@dataclass(frozen=True) +class AbandonedClaimTestCase: + description: str + claim_age_seconds: int + limit_seconds: float + expected_waited: bool + expected_states: dict[str, str] + + +@dataclass(frozen=True) +class LongClaimWaitTestCase: + description: str + grace_seconds: int + waited_seconds: int + expected_open_holds: tuple[str, ...] + expected_states: dict[str, str] + expected_released_states: dict[str, str] + + +@dataclass(frozen=True) +class TwoBuildsTestCase: + description: str + expected_operations: tuple[tuple[str, str], ...] + expected_states: dict[str, str] + expected_open_holds: tuple[str, ...] + expected_claims: tuple[tuple[str, str, str], ...] + + +@dataclass(frozen=True) +class TwoSyncsTestCase: + description: str + expected_operations: tuple[tuple[str, str], ...] + expected_states: dict[str, str] + expected_resumed: tuple[tuple[str, ...], tuple[str, ...]] + expected_claims: tuple[tuple[str, str, str], ...] + + +@dataclass(frozen=True) +class StaleSyncResumeTestCase: + description: str + expected_states: dict[str, str] + + +@dataclass(frozen=True) +class RefusedResumeTestCase: + description: str + refused: frozenset[str] + expected_resumed: tuple[tuple[str, str], ...] + expected_failures: tuple[str, ...] + expected_states: dict[str, str] + expected_events: tuple[tuple[str, str, str, str], ...] + + +@dataclass(frozen=True) +class QuietTokenTestCase: + description: str + holds: tuple[tuple[str, str, int], ...] + tokens: bool + quiet: bool + expected_resumed: tuple[tuple[str, str], ...] + expected_quiet: bool diff --git a/tests/integration/src/sqlbuild/executor/dynamic_pauses/helpers.py b/tests/integration/src/sqlbuild/executor/dynamic_pauses/helpers.py index f430e65aef..f045a46ca6 100644 --- a/tests/integration/src/sqlbuild/executor/dynamic_pauses/helpers.py +++ b/tests/integration/src/sqlbuild/executor/dynamic_pauses/helpers.py @@ -2,11 +2,13 @@ from __future__ import annotations +import time +from collections.abc import Callable from dataclasses import replace from pathlib import Path from typing import Any -from sqlbuild._native import PlanOutput +from sqlbuild._native import CompiledRelationLocation, PlanOutput from sqlbuild.adapter.contract.types import ( DynamicListing, DynamicOperation, @@ -19,7 +21,15 @@ BuildExecutionResult, BuildRuntimeParams, ) +from sqlbuild.executor.dynamic_pauses._helpers.resume import run_dynamic_operation +from sqlbuild.executor.dynamic_pauses._helpers.sql import insert_sql, setup_statements +from sqlbuild.executor.dynamic_pauses.classes.dynamic_heartbeat import DynamicHeartbeat +from sqlbuild.executor.dynamic_pauses.classes.dynamic_hold import DynamicHold +from sqlbuild.executor.dynamic_pauses.constants import CLAIM_RUN_MARKER +from sqlbuild.executor.dynamic_pauses.main.hold_dynamic_models import hold_dynamic_models +from sqlbuild.executor.dynamic_pauses.main.resume_expired_holds import resume_expired_holds from sqlbuild.executor.dynamic_pauses.models import DynamicPauseEvent +from sqlbuild.executor.dynamic_pauses.types import DynamicPauseEventType, StateLocation from tests.integration.src.sqlbuild.compiler.planner.main._test_types import ( BuildExecutionPlanTestCase, ) @@ -83,6 +93,94 @@ def render_dynamic_operation( return ("SELECT 1",) +class SyncDuringTakeoverAdapter(PausingDuckDbAdapter): + """Resumes models as a location sync would, right after `takeover_run` records its pause.""" + + def __init__(self, *, takeover_run: str, resumed: dict[str, str]) -> None: + super().__init__() + self._takeover_run: str = takeover_run + self._resumed: dict[str, str] = resumed + + def execute(self, *, connection: Any, sql: str) -> Any: + cursor: Any = super().execute(connection=connection, sql=sql) + recorded_takeover: bool = "'paused'" in sql and f"'{self._takeover_run}'" in sql + self.states.update({True: self._resumed, False: {}}[recorded_takeover]) + return cursor + + +class ClaimHookAdapter(PausingDuckDbAdapter): + """Runs one queued step right after the next resume-claim write, and after the next read.""" + + def __init__(self) -> None: + super().__init__() + self.after_claim_write: list[Callable[[Any], None]] = [] + self.after_claim_read: list[Callable[[Any], None]] = [] + self.after_holds_read: list[Callable[[Any], None]] = [] + self.claim_reads: int = 0 + + def execute(self, *, connection: Any, sql: str) -> Any: + cursor: Any = super().execute(connection=connection, sql=sql) + wrote_claim: bool = sql.startswith("INSERT") and "'resuming'" in sql + _run_next( + steps={True: self.after_claim_write, False: []}[wrote_claim], connection=connection + ) + return cursor + + def query(self, *, connection: Any, sql: str, limit: int | None) -> Any: + result: Any = super().query(connection=connection, sql=sql, limit=limit) + read_claims: bool = "'resuming'" in sql + read_holds: bool = "'paused'" in sql and not read_claims + self.claim_reads += int(read_claims) + _run_next( + steps={True: self.after_claim_read, False: []}[read_claims], connection=connection + ) + _run_next(steps={True: self.after_holds_read, False: []}[read_holds], connection=connection) + return result + + +def _run_next(*, steps: list[Callable[[Any], None]], connection: Any) -> None: + for step in [steps.pop(0) for _ in steps[:1]]: + step(connection) + + +class RefusingResumeAdapter(PausingDuckDbAdapter): + """Refuses to resume the models in `refused`, as a warehouse refusing the ALTER would.""" + + def __init__(self, *, refused: frozenset[str]) -> None: + super().__init__() + self._refused: frozenset[str] = refused + + def render_dynamic_operation( + self, *, destination: str, operation: DynamicOperation + ) -> tuple[str, ...]: + name: str = destination.rsplit(".", 1)[-1] + refusing: bool = operation == DynamicOperation.RESUME and name in self._refused + {True: _refuse_resume, False: _allow_operation}[refusing](name) + return super().render_dynamic_operation(destination=destination, operation=operation) + + +def _refuse_resume(name: str) -> None: + raise RuntimeError(f"insufficient privileges to resume {name}") + + +def _allow_operation(name: str) -> None: + del name + + +class TokenPausingDuckDbAdapter(PausingDuckDbAdapter): + """Also reads a table's change token, as Snowflake does without a running warehouse.""" + + def read_relation_change_token( + self, *, connection: Any, database: str | None, schema: str, name: str + ) -> str | None: + del database + row: tuple[Any, ...] = connection.execute( + f"SELECT COUNT(*) || '/' || COALESCE(CAST(MAX(recorded_at) AS VARCHAR), '') " + f"FROM {schema}.{name}" + ).fetchone() + return str(row[0]) + + def lease_project_case( *, orders_query: str, @@ -157,17 +255,38 @@ def build_lease_project( ) +def heartbeat_count(*, connection: Any, run_id: str) -> int: + """How many heartbeats the run `run_id` has recorded in the staging pause table.""" + + rows: list[tuple[Any, ...]] = connection.execute( + "SELECT count(*) FROM staging._sqlbuild_dynamic_pauses " + "WHERE event_type = 'heartbeat' AND run_id = ?", + [run_id], + ).fetchall() + return int(rows[0][0]) + + def pause_events(*, connection: Any) -> list[tuple[str, str, str, str]]: """Each recorded pause event's type, holding run, recording run and model, sorted.""" rows: list[tuple[Any, ...]] = connection.execute( "SELECT event_type, run_id, recorded_by, relation_name " - "FROM staging._sqlbuild_dynamic_pauses WHERE event_type <> 'heartbeat' " - "ORDER BY 1, 2, 3, 4" + "FROM staging._sqlbuild_dynamic_pauses WHERE event_type NOT IN ('heartbeat', 'resuming') " + f"AND run_id NOT LIKE '%{CLAIM_RUN_MARKER}%' ORDER BY 1, 2, 3, 4" ).fetchall() return [(str(row[0]), str(row[1]), str(row[2]), str(row[3])) for row in rows] +def recorded_events(*, connection: Any) -> list[tuple[str, str, str, str]]: + """The recorded pause events, or none when no build created the pause table.""" + + read: Callable[..., list[tuple[str, str, str, str]]] = { + True: pause_events, + False: lambda **_: [], + }[bool(pause_table_names(connection=connection))] + return read(connection=connection) + + def pause_table_names(*, connection: Any) -> list[tuple[Any, ...]]: """The pause state table, when a build created one in staging.""" @@ -182,3 +301,251 @@ def refuse_insert(*, connection: Any, events: tuple[DynamicPauseEvent, ...]) -> del connection, events raise RuntimeError("pause table is read-only") + + +def seed_hold( + *, adapter: PausingDuckDbAdapter, connection: Any, run_id: str, name: str, idle_seconds: int +) -> None: + """A run's pause of `name`, with its last write `idle_seconds` ago, and the model suspended.""" + + location: StateLocation = (None, "staging") + for statement in setup_statements(adapter=adapter, location=location): + connection.execute(statement) + relation: CompiledRelationLocation = CompiledRelationLocation( + database=None, schema="staging", name=name, qualified_name=None + ) + connection.execute( + insert_sql( + adapter=adapter, + location=location, + events=( + DynamicPauseEvent( + event_type=DynamicPauseEventType.PAUSED, + run_id=run_id, + recorded_by=run_id, + relation=relation, + ), + ), + ) + ) + connection.execute( + "UPDATE staging._sqlbuild_dynamic_pauses " + f"SET recorded_at = recorded_at - INTERVAL {idle_seconds} SECOND WHERE run_id = '{run_id}'" + ) + adapter.states[name] = "SUSPENDED" + + +def held_by_dead_run( + *, adapter: PausingDuckDbAdapter, connection: Any, database_path: Path +) -> PlanOutput: + """Build once, then let `dead_run` pause `daily_totals` and stop; the plan a rebuild runs.""" + + _ = build_lease_project( + adapter=adapter, + connection=connection, + case=lease_project_case( + orders_query="SELECT 1 AS order_id", report_locations={}, refresh_on_build=False + ), + runtime=BuildRuntimeParams(run_id="first_run"), + database_path=database_path, + ) + plan: PlanOutput = plan_lease_project( + adapter=adapter, + connection=connection, + case=lease_project_case( + orders_query="SELECT 1 AS order_id UNION ALL SELECT 2", + report_locations={}, + refresh_on_build=False, + ), + ) + dead: DynamicHold | None = hold_dynamic_models( + plan=plan, + adapter=adapter, + runtime=BuildRuntimeParams(run_id="dead_run"), + connection_config={"database": str(database_path)}, + ) + assert dead is not None + dead.acquire(connection=connection, on_progress=None) + return plan + + +def expired_hold_plan( + *, adapter: PausingDuckDbAdapter, connection: Any, database_path: Path +) -> PlanOutput: + """Build once, seed `dead_run`'s hold on `daily_totals` an hour silent; the rebuild's plan.""" + + _ = build_lease_project( + adapter=adapter, + connection=connection, + case=lease_project_case( + orders_query="SELECT 1 AS order_id", report_locations={}, refresh_on_build=False + ), + runtime=BuildRuntimeParams(run_id="first_run"), + database_path=database_path, + ) + plan: PlanOutput = plan_lease_project( + adapter=adapter, + connection=connection, + case=lease_project_case( + orders_query="SELECT 1 AS order_id UNION ALL SELECT 2", + report_locations={}, + refresh_on_build=False, + ), + ) + seed_hold( + adapter=adapter, + connection=connection, + run_id="dead_run", + name="daily_totals", + idle_seconds=3600, + ) + return plan + + +def age_beat_and_sync( + *, + adapter: PausingDuckDbAdapter, + connection: Any, + waiting: tuple[str, int], + sync: tuple[str, float, str], +) -> None: + """Age the waiting run as a long wait would, let it beat, run a sync, then close the claim.""" + + run_id, age_seconds = waiting + recorded_by, grace_seconds, claim_run = sync + connection.execute( + "UPDATE staging._sqlbuild_dynamic_pauses " + f"SET recorded_at = recorded_at - INTERVAL {age_seconds} SECOND WHERE run_id = '{run_id}'" + ) + aged: int = heartbeat_count(connection=connection, run_id=run_id) + deadline: float = time.monotonic() + 2 + while heartbeat_count(connection=connection, run_id=run_id) <= aged and ( + time.monotonic() < deadline + ): + time.sleep(0.02) + _ = resume_expired_holds( + adapter=adapter, + connection=connection, + location=(None, "staging"), + grace_seconds=grace_seconds, + recorded_by=recorded_by, + quiet_token=None, + ) + connection.execute( + insert_sql( + adapter=adapter, + location=(None, "staging"), + events=( + DynamicPauseEvent( + event_type=DynamicPauseEventType.RESUMED, + run_id=claim_run, + recorded_by=recorded_by, + relation=_daily_totals(), + ), + ), + ) + ) + + +def other_model_hold(*, adapter: PausingDuckDbAdapter, database_path: Path) -> DynamicHold: + """`other_run`'s unacquired hold on `weekly_totals`, beside `daily_totals`' pause table.""" + + return DynamicHold( + adapter=adapter, + held={ + "weekly_totals": CompiledRelationLocation( + database=None, schema="staging", name="weekly_totals", qualified_name=None + ) + }, + refreshed=frozenset(), + state_location=(None, "staging"), + settings=("other_run", False, 1800.0, False), + heartbeat=lambda write: DynamicHeartbeat( + adapter=adapter, + connection_config={"database": str(database_path)}, + write=write, + interval_seconds=60.0, + ), + claim_wait=(60.0, 0.01), + ) + + +def seed_claim( + *, adapter: PausingDuckDbAdapter, connection: Any, claim_run: str, age_seconds: int +) -> None: + """A sync's open claim on `daily_totals`, written `age_seconds` ago.""" + + connection.execute( + insert_sql( + adapter=adapter, + location=(None, "staging"), + events=( + DynamicPauseEvent( + event_type=DynamicPauseEventType.RESUMING, + run_id=claim_run, + recorded_by="location-sync:warehouse", + relation=_daily_totals(), + ), + ), + ) + ) + connection.execute( + "UPDATE staging._sqlbuild_dynamic_pauses " + f"SET recorded_at = recorded_at - INTERVAL {age_seconds} SECOND " + f"WHERE run_id = '{claim_run}'" + ) + + +def finish_claimed_resume( + *, adapter: PausingDuckDbAdapter, connection: Any, claim_run: str +) -> None: + """The claiming sync's next steps: resume daily_totals; close dead_run's hold and the claim.""" + + run_dynamic_operation( + adapter=adapter, + connection=connection, + relation=_daily_totals(), + operation=DynamicOperation.RESUME, + ) + connection.execute( + insert_sql( + adapter=adapter, + location=(None, "staging"), + events=tuple( + DynamicPauseEvent( + event_type=DynamicPauseEventType.RESUMED, + run_id=run_id, + recorded_by="location-sync:warehouse", + relation=_daily_totals(), + ) + for run_id in ("dead_run", claim_run) + ), + ) + ) + + +def claim_events(*, connection: Any) -> list[tuple[str, str, str]]: + """Each claim row's type, recording sync and model, sorted.""" + + rows: list[tuple[Any, ...]] = connection.execute( + "SELECT event_type, recorded_by, relation_name FROM staging._sqlbuild_dynamic_pauses " + f"WHERE run_id LIKE '%{CLAIM_RUN_MARKER}%' ORDER BY 1, 2, 3" + ).fetchall() + return [(str(row[0]), str(row[1]), str(row[2])) for row in rows] + + +def _daily_totals() -> CompiledRelationLocation: + return CompiledRelationLocation( + database=None, schema="staging", name="daily_totals", qualified_name=None + ) + + +def pause_event_ids(*, connection: Any) -> list[tuple[str, str, str]]: + """Each resume's event ID, holding run and recording run, sorted.""" + + rows: list[tuple[Any, ...]] = connection.execute( + "SELECT event_id, run_id, recorded_by FROM staging._sqlbuild_dynamic_pauses " + f"WHERE event_type = 'resumed' AND run_id NOT LIKE '%{CLAIM_RUN_MARKER}%' " + "ORDER BY 1, 2, 3" + ).fetchall() + return [(str(row[0]), str(row[1]), str(row[2])) for row in rows] diff --git a/tests/integration/src/sqlbuild/executor/dynamic_pauses/test_dynamic_sync_resume.py b/tests/integration/src/sqlbuild/executor/dynamic_pauses/test_dynamic_sync_resume.py new file mode 100644 index 0000000000..a8af2b194d --- /dev/null +++ b/tests/integration/src/sqlbuild/executor/dynamic_pauses/test_dynamic_sync_resume.py @@ -0,0 +1,829 @@ +"""A location sync resumes dynamic models whose pause holds expired, and only those.""" + +from __future__ import annotations + +import time +from pathlib import Path +from typing import Any + +import pytest + +from sqlbuild._native import PlanOutput +from sqlbuild.executor.build.models import BuildExecutionResult, BuildRuntimeParams +from sqlbuild.executor.build.types import BuildStatus +from sqlbuild.executor.dynamic_pauses._helpers.resume import claim_and_resume +from sqlbuild.executor.dynamic_pauses._helpers.sql import ( + read_open_claims, + read_open_holds, + setup_statements, +) +from sqlbuild.executor.dynamic_pauses.classes.dynamic_hold import DynamicHold +from sqlbuild.executor.dynamic_pauses.main.hold_dynamic_models import ( + hold_dynamic_models, +) +from sqlbuild.executor.dynamic_pauses.main.resume_expired_holds import resume_expired_holds +from sqlbuild.executor.dynamic_pauses.models import OpenHold, ResumeOutcome +from tests.integration.src.sqlbuild.executor.dynamic_pauses._test_types import ( + AbandonedClaimTestCase, + ClaimRaceTestCase, + ClaimWaitTestCase, + DoubleCloseTestCase, + LongClaimWaitTestCase, + QuietTokenTestCase, + RefusedResumeTestCase, + StaleSyncResumeTestCase, + SyncDuringTakeoverTestCase, + SyncResumeTestCase, + TwoBuildsTestCase, + TwoSyncsTestCase, +) +from tests.integration.src.sqlbuild.executor.dynamic_pauses.helpers import ( + ClaimHookAdapter, + LeaseProject, + PausingDuckDbAdapter, + RefusingResumeAdapter, + SyncDuringTakeoverAdapter, + TokenPausingDuckDbAdapter, + age_beat_and_sync, + build_lease_project, + claim_events, + expired_hold_plan, + finish_claimed_resume, + heartbeat_count, + held_by_dead_run, + lease_project_case, + other_model_hold, + pause_event_ids, + pause_events, + pause_table_names, + plan_lease_project, + recorded_events, + seed_claim, + seed_hold, +) + +_GRACE_SECONDS: float = 600.0 +_SYNC: str = "location-sync:warehouse" + + +@pytest.mark.parametrize( + "test_case", + [ + SyncResumeTestCase( + description="an expired hold is resumed and closed by the sync", + holds=(("dead_run", "daily_totals", 900),), + hand_suspended=(), + pause_table=True, + expected_resumed=(("dead_run", "daily_totals"),), + expected_states={"daily_totals": "ACTIVE"}, + expected_events=( + ("paused", "dead_run", "dead_run", "daily_totals"), + ("resumed", "dead_run", _SYNC, "daily_totals"), + ), + ), + SyncResumeTestCase( + description="a live hold is left alone", + holds=(("busy_run", "daily_totals", 30),), + hand_suspended=(), + pause_table=True, + expected_resumed=(), + expected_states={"daily_totals": "SUSPENDED"}, + expected_events=(("paused", "busy_run", "busy_run", "daily_totals"),), + ), + SyncResumeTestCase( + description="a model suspended by hand stays suspended", + holds=(("dead_run", "daily_totals", 900),), + hand_suspended=("weekly_totals",), + pause_table=True, + expected_resumed=(("dead_run", "daily_totals"),), + expected_states={"daily_totals": "ACTIVE", "weekly_totals": "SUSPENDED"}, + expected_events=( + ("paused", "dead_run", "dead_run", "daily_totals"), + ("resumed", "dead_run", _SYNC, "daily_totals"), + ), + ), + SyncResumeTestCase( + description="an expired hold on a model another run holds live is left alone", + holds=(("dead_run", "daily_totals", 900), ("busy_run", "daily_totals", 30)), + hand_suspended=(), + pause_table=True, + expected_resumed=(), + expected_states={"daily_totals": "SUSPENDED"}, + expected_events=( + ("paused", "busy_run", "busy_run", "daily_totals"), + ("paused", "dead_run", "dead_run", "daily_totals"), + ), + ), + SyncResumeTestCase( + description="of two models only the one whose hold expired is resumed", + holds=(("dead_run", "daily_totals", 900), ("busy_run", "weekly_totals", 30)), + hand_suspended=(), + pause_table=True, + expected_resumed=(("dead_run", "daily_totals"),), + expected_states={"daily_totals": "ACTIVE", "weekly_totals": "SUSPENDED"}, + expected_events=( + ("paused", "busy_run", "busy_run", "weekly_totals"), + ("paused", "dead_run", "dead_run", "daily_totals"), + ("resumed", "dead_run", _SYNC, "daily_totals"), + ), + ), + SyncResumeTestCase( + description="without a pause table the sync writes nothing", + holds=(), + hand_suspended=("daily_totals",), + pause_table=False, + expected_resumed=(), + expected_states={"daily_totals": "SUSPENDED"}, + expected_events=(), + ), + ], + ids=lambda case: case.description, +) +def test_given_pause_holds_when_a_location_syncs_then_only_expired_holds_are_resumed( + test_case: SyncResumeTestCase, adapter: PausingDuckDbAdapter, connection: Any +) -> None: + for run_id, name, idle_seconds in test_case.holds: + seed_hold( + adapter=adapter, + connection=connection, + run_id=run_id, + name=name, + idle_seconds=idle_seconds, + ) + adapter.states.update({name: "SUSPENDED" for name in test_case.hand_suspended}) + + resumed: tuple[OpenHold, ...] = resume_expired_holds( + adapter=adapter, + connection=connection, + location=(None, "staging"), + grace_seconds=_GRACE_SECONDS, + recorded_by=_SYNC, + quiet_token=None, + ).resumed + + assert tuple((hold.run_id, hold.relation.name) for hold in resumed) == ( + test_case.expected_resumed + ) + assert adapter.states == test_case.expected_states + assert bool(pause_table_names(connection=connection)) == test_case.pause_table + assert recorded_events(connection=connection) == list(test_case.expected_events) + + +@pytest.mark.parametrize( + "test_case", + [ + QuietTokenTestCase( + description="a pause table unchanged since a pass left it quiet is not read", + holds=(("dead_run", "daily_totals", 900),), + tokens=True, + quiet=True, + expected_resumed=(), + expected_quiet=True, + ), + QuietTokenTestCase( + description="a pass that leaves no hold open returns the table's token", + holds=(), + tokens=True, + quiet=False, + expected_resumed=(), + expected_quiet=True, + ), + QuietTokenTestCase( + description="a pass that resumes returns no token, so the next pass reads again", + holds=(("dead_run", "daily_totals", 900),), + tokens=True, + quiet=False, + expected_resumed=(("dead_run", "daily_totals"),), + expected_quiet=False, + ), + QuietTokenTestCase( + description="a live hold keeps the table from going quiet", + holds=(("busy_run", "daily_totals", 30),), + tokens=True, + quiet=False, + expected_resumed=(), + expected_quiet=False, + ), + QuietTokenTestCase( + description="an adapter without change tokens reads every pass", + holds=(("dead_run", "daily_totals", 900),), + tokens=False, + quiet=True, + expected_resumed=(("dead_run", "daily_totals"),), + expected_quiet=False, + ), + ], + ids=lambda case: case.description, +) +def test_given_a_quiet_token_when_a_location_syncs_then_only_a_changed_pause_table_is_read( + test_case: QuietTokenTestCase, connection: Any +) -> None: + adapter: PausingDuckDbAdapter = { + True: TokenPausingDuckDbAdapter, + False: PausingDuckDbAdapter, + }[test_case.tokens]() + for statement in setup_statements(adapter=adapter, location=(None, "staging")): + connection.execute(statement) + for run_id, name, idle_seconds in test_case.holds: + seed_hold( + adapter=adapter, + connection=connection, + run_id=run_id, + name=name, + idle_seconds=idle_seconds, + ) + unchanged: str | None = TokenPausingDuckDbAdapter().read_relation_change_token( + connection=connection, database=None, schema="staging", name="_sqlbuild_dynamic_pauses" + ) + token: str | None = {True: unchanged, False: None}[test_case.quiet] + + outcome: ResumeOutcome = resume_expired_holds( + adapter=adapter, + connection=connection, + location=(None, "staging"), + grace_seconds=_GRACE_SECONDS, + recorded_by=_SYNC, + quiet_token=token, + ) + + current: str | None = TokenPausingDuckDbAdapter().read_relation_change_token( + connection=connection, database=None, schema="staging", name="_sqlbuild_dynamic_pauses" + ) + assert tuple((hold.run_id, hold.relation.name) for hold in outcome.resumed) == ( + test_case.expected_resumed + ) + assert outcome.quiet_token == {True: current, False: None}[test_case.expected_quiet] + + +@pytest.mark.parametrize( + "test_case", + [ + DoubleCloseTestCase( + description="a build and a sync closing one expired hold write the same resume", + closers=("second_run", _SYNC), + expected_events=( + ("paused", "dead_run", "dead_run", "daily_totals"), + ("resumed", "dead_run", _SYNC, "daily_totals"), + ("resumed", "dead_run", "second_run", "daily_totals"), + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_one_expired_hold_when_two_closers_resume_it_then_it_closes_once( + test_case: DoubleCloseTestCase, adapter: PausingDuckDbAdapter, connection: Any +) -> None: + seed_hold( + adapter=adapter, + connection=connection, + run_id="dead_run", + name="daily_totals", + idle_seconds=900, + ) + read: tuple[OpenHold, ...] = read_open_holds( + adapter=adapter, connection=connection, location=(None, "staging") + ) + for closer in test_case.closers: + claim_and_resume( + adapter=adapter, + connection=connection, + location=(None, "staging"), + holds=read, + holds_after=(_GRACE_SECONDS, closer), + ) + + assert pause_events(connection=connection) == list(test_case.expected_events) + assert len({event_id for event_id, _, _ in pause_event_ids(connection=connection)}) == 1 + assert read_open_holds(adapter=adapter, connection=connection, location=(None, "staging")) == () + assert adapter.operations == [("resume", "daily_totals")] + assert adapter.states == {"daily_totals": "ACTIVE"} + + +@pytest.mark.parametrize( + "test_case", + [ + SyncDuringTakeoverTestCase( + description="a model a sync resumes while a build takes its hold over is paused again", + expected_operations=(("suspend", "daily_totals"), ("resume", "daily_totals")), + expected_states={"daily_totals": "ACTIVE"}, + ), + ], + ids=lambda case: case.description, +) +def test_given_a_sync_resuming_mid_takeover_when_the_build_holds_then_it_pauses_the_model( + test_case: SyncDuringTakeoverTestCase, + database_path: Path, +) -> None: + adapter: SyncDuringTakeoverAdapter = SyncDuringTakeoverAdapter( + takeover_run="second_run", resumed={"daily_totals": "ACTIVE"} + ) + connection: Any = adapter.connect({"database": str(database_path)}) + connection.execute("CREATE SCHEMA staging") + case: LeaseProject = lease_project_case( + orders_query="SELECT 1 AS order_id", report_locations={}, refresh_on_build=False + ) + _ = build_lease_project( + adapter=adapter, + connection=connection, + case=case, + runtime=BuildRuntimeParams(run_id="first_run"), + database_path=database_path, + ) + changed: LeaseProject = lease_project_case( + orders_query="SELECT 1 AS order_id UNION ALL SELECT 2", + report_locations={}, + refresh_on_build=False, + ) + dead: DynamicHold | None = hold_dynamic_models( + plan=plan_lease_project(adapter=adapter, connection=connection, case=changed), + adapter=adapter, + runtime=BuildRuntimeParams(run_id="dead_run"), + connection_config={"database": str(database_path)}, + ) + assert dead is not None + dead.acquire(connection=connection, on_progress=None) + adapter.operations.clear() + + result: BuildExecutionResult = build_lease_project( + adapter=adapter, + connection=connection, + case=changed, + runtime=BuildRuntimeParams(run_id="second_run", dynamic_heartbeat_grace_seconds=0), + database_path=database_path, + ) + + adapter.close(connection) + assert result.status == BuildStatus.SUCCESS + assert tuple(adapter.operations) == test_case.expected_operations + assert adapter.states == test_case.expected_states + + +@pytest.mark.parametrize( + "test_case", + [ + RefusedResumeTestCase( + description="a refused resume leaves that hold open and the others still close", + refused=frozenset({"daily_totals"}), + expected_resumed=(("dead_weekly", "weekly_totals"),), + expected_failures=( + "staging.daily_totals: insufficient privileges to resume daily_totals", + ), + expected_states={"daily_totals": "SUSPENDED", "weekly_totals": "ACTIVE"}, + expected_events=( + ("paused", "dead_daily", "dead_daily", "daily_totals"), + ("paused", "dead_weekly", "dead_weekly", "weekly_totals"), + ("resumed", "dead_weekly", _SYNC, "weekly_totals"), + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_one_refused_resume_when_a_location_syncs_then_the_other_holds_still_close( + test_case: RefusedResumeTestCase, connection: Any +) -> None: + adapter: RefusingResumeAdapter = RefusingResumeAdapter(refused=test_case.refused) + for run_id, name in (("dead_daily", "daily_totals"), ("dead_weekly", "weekly_totals")): + seed_hold( + adapter=adapter, connection=connection, run_id=run_id, name=name, idle_seconds=900 + ) + + outcome: ResumeOutcome = resume_expired_holds( + adapter=adapter, + connection=connection, + location=(None, "staging"), + grace_seconds=_GRACE_SECONDS, + recorded_by=_SYNC, + quiet_token=None, + ) + + assert tuple((hold.run_id, hold.relation.name) for hold in outcome.resumed) == ( + test_case.expected_resumed + ) + assert outcome.failures == test_case.expected_failures + assert adapter.states == test_case.expected_states + assert pause_events(connection=connection) == list(test_case.expected_events) + + +@pytest.mark.parametrize( + "test_case", + [ + StaleSyncResumeTestCase( + description="a sync resuming from a stale read after the build's state check", + expected_states={"daily_totals": "SUSPENDED"}, + ), + ], + ids=lambda case: case.description, +) +def test_given_a_sync_resuming_after_the_takeover_state_check_when_the_build_holds_then_the_model_stays_paused( + test_case: StaleSyncResumeTestCase, + adapter: PausingDuckDbAdapter, + connection: Any, + database_path: Path, +) -> None: + case: LeaseProject = lease_project_case( + orders_query="SELECT 1 AS order_id", report_locations={}, refresh_on_build=False + ) + _ = build_lease_project( + adapter=adapter, + connection=connection, + case=case, + runtime=BuildRuntimeParams(run_id="first_run"), + database_path=database_path, + ) + plan: PlanOutput = plan_lease_project( + adapter=adapter, + connection=connection, + case=lease_project_case( + orders_query="SELECT 1 AS order_id UNION ALL SELECT 2", + report_locations={}, + refresh_on_build=False, + ), + ) + dead: DynamicHold | None = hold_dynamic_models( + plan=plan, + adapter=adapter, + runtime=BuildRuntimeParams(run_id="dead_run"), + connection_config={"database": str(database_path)}, + ) + assert dead is not None + dead.acquire(connection=connection, on_progress=None) + stale: tuple[OpenHold, ...] = read_open_holds( + adapter=adapter, connection=connection, location=(None, "staging") + ) + taking_over: DynamicHold | None = hold_dynamic_models( + plan=plan, + adapter=adapter, + runtime=BuildRuntimeParams(run_id="second_run", dynamic_heartbeat_grace_seconds=0), + connection_config={"database": str(database_path)}, + heartbeat_seconds=0.05, + ) + assert taking_over is not None + taking_over.acquire(connection=connection, on_progress=None) + + _ = claim_and_resume( + adapter=adapter, + connection=connection, + location=(None, "staging"), + holds=stale, + holds_after=(_GRACE_SECONDS, _SYNC), + ) + beats_after_pass: int = heartbeat_count(connection=connection, run_id="second_run") + 2 + deadline: float = time.monotonic() + 10 + while heartbeat_count(connection=connection, run_id="second_run") < beats_after_pass: + assert time.monotonic() < deadline, "the taking-over build stopped heartbeating" + time.sleep(0.02) + states: dict[str, str] = dict(adapter.states) + taking_over.release(connection=connection) + + assert states == test_case.expected_states + + +@pytest.mark.parametrize( + "test_case", + [ + ClaimRaceTestCase( + description="a build pausing the model between the claim and the re-read wins", + expected_operations=(), + expected_states={"daily_totals": "SUSPENDED"}, + expected_open_holds=("busy_run", "dead_run"), + expected_claims=( + ("resumed", _SYNC, "daily_totals"), + ("resuming", _SYNC, "daily_totals"), + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_a_build_pausing_after_the_sync_claims_when_the_sync_reads_again_then_it_skips_the_model( + test_case: ClaimRaceTestCase, database_path: Path +) -> None: + adapter: ClaimHookAdapter = ClaimHookAdapter() + connection: Any = adapter.connect({"database": str(database_path)}) + connection.execute("CREATE SCHEMA staging") + seed_hold( + adapter=adapter, + connection=connection, + run_id="dead_run", + name="daily_totals", + idle_seconds=900, + ) + adapter.after_claim_write.append( + lambda claimed: seed_hold( + adapter=adapter, + connection=claimed, + run_id="busy_run", + name="daily_totals", + idle_seconds=0, + ) + ) + + outcome: ResumeOutcome = resume_expired_holds( + adapter=adapter, + connection=connection, + location=(None, "staging"), + grace_seconds=_GRACE_SECONDS, + recorded_by=_SYNC, + quiet_token=None, + ) + + holds: tuple[OpenHold, ...] = read_open_holds( + adapter=adapter, connection=connection, location=(None, "staging") + ) + assert outcome.resumed == () + assert tuple(adapter.operations) == test_case.expected_operations + assert adapter.states == test_case.expected_states + assert tuple(sorted(hold.run_id for hold in holds)) == test_case.expected_open_holds + assert claim_events(connection=connection) == list(test_case.expected_claims) + assert ( + read_open_claims(adapter=adapter, connection=connection, location=(None, "staging")) == () + ) + adapter.close(connection) + + +@pytest.mark.parametrize( + "test_case", + [ + ClaimWaitTestCase( + description="a build finding an open claim waits for it, then pauses the model again", + expected_operations=(("resume", "daily_totals"), ("suspend", "daily_totals")), + expected_states={"daily_totals": "SUSPENDED"}, + expected_claim_reads=2, + expected_progress="Waiting for a location sync resuming dynamic model daily_totals", + ), + ], + ids=lambda case: case.description, +) +def test_given_an_open_resume_claim_when_a_build_takes_the_model_over_then_it_waits_and_pauses_it_again( + test_case: ClaimWaitTestCase, database_path: Path +) -> None: + adapter: ClaimHookAdapter = ClaimHookAdapter() + connection: Any = adapter.connect({"database": str(database_path)}) + connection.execute("CREATE SCHEMA staging") + plan: PlanOutput = held_by_dead_run( + adapter=adapter, connection=connection, database_path=database_path + ) + claim_run: str = f"{_SYNC}/claim/in-flight" + seed_claim(adapter=adapter, connection=connection, claim_run=claim_run, age_seconds=0) + adapter.after_claim_read.append( + lambda read: finish_claimed_resume(adapter=adapter, connection=read, claim_run=claim_run) + ) + building: DynamicHold | None = hold_dynamic_models( + plan=plan, + adapter=adapter, + runtime=BuildRuntimeParams(run_id="second_run", dynamic_heartbeat_grace_seconds=0), + connection_config={"database": str(database_path)}, + claim_wait=(60.0, 0.01), + ) + assert building is not None + adapter.operations.clear() + adapter.claim_reads = 0 + progress: list[str] = [] + + building.acquire(connection=connection, on_progress=progress.append) + + operations: tuple[tuple[str, str], ...] = tuple(adapter.operations) + states: dict[str, str] = dict(adapter.states) + building.release(connection=connection) + adapter.close(connection) + assert operations == test_case.expected_operations + assert states == test_case.expected_states + assert adapter.claim_reads == test_case.expected_claim_reads + assert test_case.expected_progress in progress + + +@pytest.mark.parametrize( + "test_case", + [ + AbandonedClaimTestCase( + description="a claim older than the limit is ignored at once", + claim_age_seconds=300, + limit_seconds=120.0, + expected_waited=False, + expected_states={"daily_totals": "SUSPENDED"}, + ), + AbandonedClaimTestCase( + description="a claim that never closes is waited for only until it reaches the limit", + claim_age_seconds=0, + limit_seconds=0.3, + expected_waited=True, + expected_states={"daily_totals": "SUSPENDED"}, + ), + ], + ids=lambda case: case.description, +) +def test_given_an_abandoned_resume_claim_when_a_build_takes_the_model_over_then_it_waits_no_longer_than_the_limit( + test_case: AbandonedClaimTestCase, database_path: Path +) -> None: + adapter: ClaimHookAdapter = ClaimHookAdapter() + connection: Any = adapter.connect({"database": str(database_path)}) + connection.execute("CREATE SCHEMA staging") + plan: PlanOutput = held_by_dead_run( + adapter=adapter, connection=connection, database_path=database_path + ) + seed_claim( + adapter=adapter, + connection=connection, + claim_run=f"{_SYNC}/claim/crashed", + age_seconds=test_case.claim_age_seconds, + ) + building: DynamicHold | None = hold_dynamic_models( + plan=plan, + adapter=adapter, + runtime=BuildRuntimeParams(run_id="second_run", dynamic_heartbeat_grace_seconds=0), + connection_config={"database": str(database_path)}, + claim_wait=(test_case.limit_seconds, 0.02), + ) + assert building is not None + adapter.claim_reads = 0 + started: float = time.monotonic() + + building.acquire(connection=connection, on_progress=None) + + elapsed: float = time.monotonic() - started + states: dict[str, str] = dict(adapter.states) + building.release(connection=connection) + adapter.close(connection) + assert (adapter.claim_reads > 1) == test_case.expected_waited + assert elapsed < test_case.limit_seconds + 5 + assert states == test_case.expected_states + + +@pytest.mark.parametrize( + "test_case", + [ + TwoSyncsTestCase( + description="two syncs claiming one model resume it once and close both claims", + expected_operations=(("resume", "daily_totals"),), + expected_states={"daily_totals": "ACTIVE"}, + expected_resumed=(("dead_run",), ("dead_run",)), + expected_claims=( + ("resumed", "location-sync:other", "daily_totals"), + ("resumed", _SYNC, "daily_totals"), + ("resuming", "location-sync:other", "daily_totals"), + ("resuming", _SYNC, "daily_totals"), + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_two_syncs_claiming_one_model_when_both_pass_then_it_resumes_once_and_both_claims_close( + test_case: TwoSyncsTestCase, database_path: Path +) -> None: + adapter: ClaimHookAdapter = ClaimHookAdapter() + connection: Any = adapter.connect({"database": str(database_path)}) + connection.execute("CREATE SCHEMA staging") + seed_hold( + adapter=adapter, + connection=connection, + run_id="dead_run", + name="daily_totals", + idle_seconds=900, + ) + other: list[ResumeOutcome] = [] + adapter.after_claim_write.append( + lambda claimed: other.append( + resume_expired_holds( + adapter=adapter, + connection=claimed, + location=(None, "staging"), + grace_seconds=_GRACE_SECONDS, + recorded_by="location-sync:other", + quiet_token=None, + ) + ) + ) + + first: ResumeOutcome = resume_expired_holds( + adapter=adapter, + connection=connection, + location=(None, "staging"), + grace_seconds=_GRACE_SECONDS, + recorded_by=_SYNC, + quiet_token=None, + ) + + resumed: tuple[tuple[str, ...], ...] = ( + tuple(hold.run_id for hold in first.resumed), + tuple(hold.run_id for hold in other[0].resumed), + ) + assert tuple(adapter.operations) == test_case.expected_operations + assert adapter.states == test_case.expected_states + assert resumed == test_case.expected_resumed + assert claim_events(connection=connection) == list(test_case.expected_claims) + assert ( + read_open_claims(adapter=adapter, connection=connection, location=(None, "staging")) == () + ) + assert read_open_holds(adapter=adapter, connection=connection, location=(None, "staging")) == () + adapter.close(connection) + + +@pytest.mark.parametrize( + "test_case", + [ + TwoBuildsTestCase( + description="a build resuming another model from a stale read leaves it to its new holder", + expected_operations=(), + expected_states={"daily_totals": "SUSPENDED"}, + expected_open_holds=("second_run",), + expected_claims=( + ("resumed", "other_run", "daily_totals"), + ("resuming", "other_run", "daily_totals"), + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_two_builds_taking_over_one_expired_hold_when_one_holds_the_model_then_it_stays_paused( + test_case: TwoBuildsTestCase, database_path: Path +) -> None: + adapter: ClaimHookAdapter = ClaimHookAdapter() + connection: Any = adapter.connect({"database": str(database_path)}) + connection.execute("CREATE SCHEMA staging") + plan: PlanOutput = expired_hold_plan( + adapter=adapter, connection=connection, database_path=database_path + ) + holder: DynamicHold | None = hold_dynamic_models( + plan=plan, + adapter=adapter, + runtime=BuildRuntimeParams(run_id="second_run"), + connection_config={"database": str(database_path)}, + ) + assert holder is not None + other: DynamicHold = other_model_hold(adapter=adapter, database_path=database_path) + adapter.after_holds_read.append(lambda read: holder.acquire(connection=read, on_progress=None)) + adapter.operations.clear() + + other.acquire(connection=connection, on_progress=None) + + operations: tuple[tuple[str, str], ...] = tuple(adapter.operations) + states: dict[str, str] = dict(adapter.states) + holds: tuple[OpenHold, ...] = read_open_holds( + adapter=adapter, connection=connection, location=(None, "staging") + ) + claims: list[tuple[str, str, str]] = claim_events(connection=connection) + other.release(connection=connection) + holder.release(connection=connection) + adapter.close(connection) + assert operations == test_case.expected_operations + assert states == test_case.expected_states + assert tuple(sorted(hold.run_id for hold in holds)) == test_case.expected_open_holds + assert claims == list(test_case.expected_claims) + + +@pytest.mark.parametrize( + "test_case", + [ + LongClaimWaitTestCase( + description="a build waiting past its grace on a claim still beats, so it keeps its hold", + grace_seconds=61, + waited_seconds=62, + expected_open_holds=("second_run",), + expected_states={"daily_totals": "SUSPENDED"}, + expected_released_states={"daily_totals": "ACTIVE"}, + ), + ], + ids=lambda case: case.description, +) +def test_given_a_build_waiting_on_a_claim_past_its_grace_when_another_sync_passes_then_its_hold_stays_live( + test_case: LongClaimWaitTestCase, database_path: Path +) -> None: + adapter: ClaimHookAdapter = ClaimHookAdapter() + connection: Any = adapter.connect({"database": str(database_path)}) + connection.execute("CREATE SCHEMA staging") + plan: PlanOutput = expired_hold_plan( + adapter=adapter, connection=connection, database_path=database_path + ) + claim_run: str = f"{_SYNC}/claim/slow" + seed_claim(adapter=adapter, connection=connection, claim_run=claim_run, age_seconds=0) + adapter.after_claim_read.append( + lambda read: age_beat_and_sync( + adapter=adapter, + connection=read, + waiting=("second_run", test_case.waited_seconds), + sync=("location-sync:other", float(test_case.grace_seconds), claim_run), + ) + ) + building: DynamicHold | None = hold_dynamic_models( + plan=plan, + adapter=adapter, + runtime=BuildRuntimeParams( + run_id="second_run", dynamic_heartbeat_grace_seconds=test_case.grace_seconds + ), + connection_config={"database": str(database_path)}, + heartbeat_seconds=0.05, + claim_wait=(60.0, 0.02), + ) + assert building is not None + + building.acquire(connection=connection, on_progress=None) + + holds: tuple[OpenHold, ...] = read_open_holds( + adapter=adapter, connection=connection, location=(None, "staging") + ) + states: dict[str, str] = dict(adapter.states) + building.release(connection=connection) + released: dict[str, str] = dict(adapter.states) + adapter.close(connection) + assert tuple(sorted(hold.run_id for hold in holds)) == test_case.expected_open_holds + assert states == test_case.expected_states + assert released == test_case.expected_released_states diff --git a/website/src/content/docs/docs/cli/server.mdx b/website/src/content/docs/docs/cli/server.mdx index 2cea042e05..9bc28c92ed 100644 --- a/website/src/content/docs/docs/cli/server.mdx +++ b/website/src/content/docs/docs/cli/server.mdx @@ -551,8 +551,19 @@ Details: interval, add one more sync after it, not one each. - The interval restarts after every completed sync, so a location that syncs often for other reasons never syncs extra. -- A sync reads the warehouse and never writes to it. On Snowflake, it skips state tables that - haven't changed since the last sync, so a sync with nothing new doesn't resume the warehouse. +- A sync reads the warehouse and writes only to the dynamic pause table: on Snowflake, after it + completes, it resumes the [dynamic models](/docs/concepts/models/dynamic#when-a-build-dies) whose + pausing build has been silent past `[dynamic] heartbeat_grace`, and records that in the pause + table as `location-sync:`. Before resuming, it records a `resuming` claim on each + model and reads the holds again, so a build that paused the model meanwhile keeps it paused; a + build that finds an open claim waits for it to close. A claim older than 2 minutes, the time a + sync gets to resume, is treated as abandoned. It checks the pause table in the target's schema and in every + schema the location's models write to. A model that fails to resume is logged as + `Resuming an expired dynamic hold failed: : `, keeps its hold, and is retried on + the next sync; the others still resume. Other adapters can't build dynamic models, so their + syncs don't open a session for this. +- On Snowflake, a sync skips state tables that haven't changed since the last sync, the pause + table included, so a sync with nothing new doesn't resume the warehouse. - When a read fails, the Connections page shows the warehouse error. A read that ran out of memory or time retries a smaller page. Any other error, such as a missing permission or table, retries after 30 seconds, then waits twice as long after each failure in a row. After 5 failures in a row diff --git a/website/src/content/docs/docs/concepts/models/dynamic.mdx b/website/src/content/docs/docs/concepts/models/dynamic.mdx index f2434e0b30..1b86eab3fa 100644 --- a/website/src/content/docs/docs/concepts/models/dynamic.mdx +++ b/website/src/content/docs/docs/concepts/models/dynamic.mdx @@ -65,14 +65,23 @@ whether it succeeds or fails. both fail this way; run one of them again. - **A stopped build leaves its models suspended.** A build killed before it resumes them, for example by `SIGKILL`, can't resume them itself. -- **The next build resumes them.** The next build that pauses dynamic models on the target - resumes them once the stopped build has been silent for `[dynamic] heartbeat_grace`, 30 - minutes by default; see [Dynamic](/docs/concepts/project-configuration#dynamic). +- **The location's sync resumes them.** Once the stopped build has been silent for + `[dynamic] heartbeat_grace`, 30 minutes by default, the next completed sync of a served location + resumes its models and closes its hold, and the location's log says which. A sync never leaves + a model running while another build holds it, and it leaves models suspended outside SQLBuild + alone. A build that takes a model over while a sync is resuming it waits for the sync, at most + 2 minutes, then pauses the model again if the sync resumed it. A model it fails to resume, for example for a missing privilege, keeps its hold and is retried + on the next sync. See + [Dynamic](/docs/concepts/project-configuration#dynamic) and + [location syncs](/docs/cli/server#sync-from-the-warehouse-now). +- **Or the next build does.** Without a served location, the next build that pauses dynamic + models on the target resumes them after the same grace. - **Taking over at once.** Run `sqb build --ignore-dynamic-grace` to take the models over without waiting. It needs `[dynamic] allow_ignore_grace = true`; without it the flag fails with `A719`. You can also resume the models in Snowflake by hand. ## Limitations +- A resume that takes longer than its 2-minute claim can still start a model a build has just paused: the last `ALTER` of a sync killed at the limit, or a build's resume of another model on a queued warehouse. - Unit tests and scenarios treat a dynamic model as a table. - On a target with `schema = "preserve"`, builds see each other's pauses only when their dynamic models share a schema. diff --git a/website/src/content/docs/docs/concepts/project-configuration.mdx b/website/src/content/docs/docs/concepts/project-configuration.mdx index b6e4e12f66..eb492eacef 100644 --- a/website/src/content/docs/docs/concepts/project-configuration.mdx +++ b/website/src/content/docs/docs/concepts/project-configuration.mdx @@ -685,7 +685,7 @@ allow_ignore_grace = false | Field | Default | Description | |-------|---------|-------------| -| `heartbeat_grace` | `"30m"` | How long a build that paused dynamic models may go silent before another build resumes them or takes them over. A fixed duration longer than `"1m"`, such as `"30m"` or `"2h"`; a build writes a heartbeat every minute | +| `heartbeat_grace` | `"30m"` | How long a build that paused dynamic models may go silent before a location's sync or another build resumes them, or another build takes them over. A fixed duration longer than `"1m"`, such as `"30m"` or `"2h"`; a build writes a heartbeat every minute | | `allow_ignore_grace` | `false` | Allow `sqb build --ignore-dynamic-grace`, which takes over models another build holds without waiting for the grace. Without it the flag fails with `A719` | ## Scenario