Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
15 commits
Select commit Hold shift + click to select a range
e2f2786
refactor(bindings): move the dbt event stream into sqlbuild-python-ou…
kvlonge Nov 19, 2026
775781c
feat(build): share the expired-hold resume and add the location-sync …
kvlonge Nov 19, 2026
08dd3ed
feat(location): resume expired dynamic holds after each completed sync
kvlonge Nov 19, 2026
f448c92
fix(location): register the sync's resume entry and keep its tests fr…
kvlonge Nov 19, 2026
898f63c
fix(location): open the resume's session on the Python edge and hand …
kvlonge Nov 19, 2026
1ef3ec8
docs: say that a location's sync resumes dynamic models whose build w…
kvlonge Nov 19, 2026
c7da892
chore(fensu): point the registration dupes allowance at the dbt event…
kvlonge Nov 19, 2026
6e7a910
fix(location): resume each expired hold on its own, and document the …
kvlonge Nov 19, 2026
d0b264a
fix(location): stop promptly when shutdown arrives during a sync's dy…
kvlonge Nov 20, 2026
aa491ff
fix(location): keep the pause table's quiet token when the location p…
kvlonge Nov 20, 2026
ea57101
fix(location): resume expired holds in the pause table's schema, on S…
kvlonge Nov 20, 2026
a951e84
test(location): make the sync/takeover race xfail a real tripwire
kvlonge Nov 20, 2026
1b27348
fix(location): claim a dynamic model before a sync resumes it, so a b…
kvlonge Nov 20, 2026
157b75f
fix(location): claim other models before a build resumes their expire…
kvlonge Nov 20, 2026
6fcfcc5
fix(location): keep a build's heartbeat running while it waits on a r…
kvlonge Nov 20, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
{
"model": "MODEL (description 'Orders.', materialized table); SELECT 17 AS amount\n",
"dynamic": "[dynamic]\nheartbeat_grace = \"10m\"\n",
"holds": [
{"run_id": "stopped-build", "model": "daily_totals", "idle_seconds": 900},
{"run_id": "running-build", "model": "weekly_totals", "idle_seconds": 0}
],
"lifecycle": [
"a DuckDB warehouse holds a pause table with one hold silent past the project's grace and one live hold",
"a served location's sync completes",
"DuckDB cannot build dynamic models, so the sync opens no resume session and both holds stay open"
]
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,122 @@
//! A completed synchronization resumes the dynamic models whose pausing builds stopped heartbeating.

use std::time::Duration;

use sqlbuild_core::constants::DYNAMIC_RESUME_SECONDS;
use sqlbuild_orchestration::warehouse_metadata::constants::DYNAMIC_RESUME_REQUEST;
use sqlbuild_orchestration::warehouse_metadata::models::{
DynamicResumeLocation, DynamicResumeReply,
};
use tokio::io::{AsyncBufReadExt, AsyncReadExt, AsyncWriteExt};

use crate::argv::models::Stream;
use crate::native_commands::_helpers::command::terminal::write_output;
use crate::native_commands::_helpers::location::execution::location_warning;
use crate::native_commands::constants::METADATA_REPLY_BYTES;
use crate::native_commands::models::{LocationCompiler, LocationMetadataSync};

/// Ask the reader to resume expired holds, sending each location's quiet token; a failure only warns.
pub(crate) async fn resume_expired_holds(mut state: LocationMetadataSync) -> LocationMetadataSync {
if state.halted {
return state;
}
let Some(reader) = state.reader.take() else {
return state;
};
let locations: Vec<DynamicResumeLocation> = state
.locations
.iter()
.chain(
state
.dynamic_quiet_tokens
.keys()
.filter(|location| !state.locations.contains(location)),
)
.map(|location| DynamicResumeLocation {
location: location.clone(),
quiet_token: state.dynamic_quiet_tokens.get(location).cloned(),
})
.collect();
let request = serde_json::json!({ DYNAMIC_RESUME_REQUEST: locations }).to_string();
let reason = match tokio::time::timeout(
Duration::from_secs(DYNAMIC_RESUME_SECONDS),
exchange(reader, request),
)
.await
{
Ok((reader, Ok(DynamicResumeReply::Resumed(resumed)))) => {
state.reader = Some(reader);
state.dynamic_quiet_tokens = resumed
.quiet_tokens
.into_iter()
.filter_map(|resume| Some((resume.location, resume.quiet_token?)))
.collect();
for failure in &resumed.failures {
drop(write_output(&[(
Stream::Stderr,
&format!("Resuming an expired dynamic hold failed: {failure}\n"),
)]));
}
for hold in resumed.holds {
drop(write_output(&[(
Stream::Stderr,
&format!(
"Resumed dynamic model {}: run {} stopped heartbeating.\n",
hold.relation, hold.run_id
),
)]));
}
return state;
}
Ok((reader, Ok(DynamicResumeReply::Failed(reason)))) => {
state.reader = Some(reader);
reason
}
Ok((reader, Err(reason))) => {
state.reader = Some(reader);
drop(write_output(&[(
Stream::Stderr,
&format!("Resuming expired dynamic holds failed: {reason}\n"),
)]));
reason
}
Err(_) => {
let reason = format!("no reply in {DYNAMIC_RESUME_SECONDS} s");
drop(write_output(&[(
Stream::Stderr,
&format!("Resuming expired dynamic holds failed: {reason}\n"),
)]));
reason
}
};
location_warning("dynamic resume failed", reason);
state
}

/// One request line and its bounded reply line, handing the reader back.
async fn exchange(
mut reader: LocationCompiler,
request: String,
) -> (LocationCompiler, Result<DynamicResumeReply, String>) {
let reply: Result<DynamicResumeReply, String> = async {
let fail = |error: std::io::Error| error.to_string();
reader
.stdin
.write_all(format!("{request}\n").as_bytes())
.await
.map_err(fail)?;
reader.stdin.flush().await.map_err(fail)?;
let mut line = String::new();
let read = (&mut reader.stdout)
.take(METADATA_REPLY_BYTES + 1)
.read_line(&mut line)
.await
.map_err(fail)?;
if read == 0 {
return Err("location runtime exited before replying".into());
}
serde_json::from_str(&line).map_err(|_| "invalid location reply".to_owned())
}
.await;
(reader, reply)
}
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ use crate::native_commands::_helpers::command::terminal::{
finish_lost_stdout, terminal_columns, write_output,
};
use crate::native_commands::_helpers::command::terminal::{live_screen, stderr_columns};
use crate::native_commands::_helpers::location::dynamic_resume::resume_expired_holds;
use crate::native_commands::_helpers::location::sync_report::{
metadata_health, read_timeout_seconds, rows_text, scan_line,
};
Expand Down Expand Up @@ -512,6 +513,7 @@ pub(crate) fn metadata_sync(
state_history_versions: None,
progress: SyncProgress::new(),
reads_wake_compute: false,
dynamic_quiet_tokens: std::collections::BTreeMap::new(),
}
}

Expand Down Expand Up @@ -1438,14 +1440,20 @@ pub(crate) async fn read_metadata_page(
Ok(Some((state, read)))
}

/// A completed synchronization closes its reader, so no warehouse session outlives the sync.
/// A completed synchronization resumes expired dynamic holds, then closes its reader, so no warehouse session outlives the sync.
pub(crate) async fn finish_metadata_page(
state: LocationMetadataSync,
stopped: impl Future<Output = Result<(), ControlError>>,
) -> Result<Option<LocationMetadataSync>, ControlError> {
if state.idle {
return Ok(Some(close_metadata_reader(state).await?));
if !state.idle {
return Ok(Some(state));
}
Ok(Some(state))
tokio::pin!(stopped);
let state: LocationMetadataSync = tokio::select! {
result = &mut stopped => { result?; return Ok(None); },
state = resume_expired_holds(state) => state,
};
Ok(Some(close_metadata_reader(state).await?))
}

/// End of input lets the reader close its warehouse session; one that ignores it is killed.
Expand Down Expand Up @@ -1636,13 +1644,16 @@ pub(crate) async fn settle_publication(
stopped: impl Future<Output = Result<(), ControlError>>,
) -> Result<Option<Settled>, ControlError> {
let (mut metadata, deferred) = if connect {
let sync: LocationMetadataSync = metadata_sync(
let mut sync: LocationMetadataSync = metadata_sync(
(locations, generation),
(
options.metadata_reconcile_seconds,
options.safety_sync_seconds,
),
);
sync.dynamic_quiet_tokens = metadata
.map(|previous| previous.dynamic_quiet_tokens)
.unwrap_or_default();
(Some(sync), None)
} else {
(metadata, Some((locations, generation)))
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
pub(crate) mod changed_build;
pub(crate) mod dynamic_resume;
pub(crate) mod execution;
pub(crate) mod held_copy;
pub(crate) mod plan_document;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -141,5 +141,5 @@ pub(crate) async fn deliver_metadata_page(
result = &mut stopped => { result?; return Ok(None); },
() = metadata_health(&state, (client, poll), (started, None)) => {},
}
finish_metadata_page(acknowledge_metadata_page(state)).await
finish_metadata_page(acknowledge_metadata_page(state), stopped).await
}
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,7 @@ pub(crate) async fn sync_location_metadata(
result = &mut stopped => { result?; return Ok(None); },
() = metadata_health(&state, (client, poll), (started, None)) => {},
}
return finish_metadata_page(next_scan(state)).await;
return finish_metadata_page(next_scan(state), stopped.as_mut()).await;
}
state = freeze_history_page(state, page, poll)?;
}
Expand Down
2 changes: 2 additions & 0 deletions crates/sqlbuild-cli/src/native_commands/models.rs
Original file line number Diff line number Diff line change
Expand Up @@ -287,6 +287,8 @@ pub(crate) struct LocationMetadataSync {
pub(crate) applied: bool,
/// The generation was published after the last plan of what changed, which waits for this sync.
pub(crate) replan: bool,
/// Per state location, the pause table's change token after a resume pass left no hold open.
pub(crate) dynamic_quiet_tokens: std::collections::BTreeMap<WarehouseStateLocation, String>,
}

/// Rows delivered and time taken by the current synchronization and its current table.
Expand Down
47 changes: 33 additions & 14 deletions crates/sqlbuild-cli/src/native_commands/tests/helpers.rs
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,20 @@ pub(super) fn one_location() -> Vec<WarehouseStateLocation> {
}]
}

/// A location in `directory` served by an unreachable server.
pub(super) fn location_options(directory: &Path) -> LocationServe {
LocationServe {
project_dir: directory.to_path_buf(),
server_url: "http://127.0.0.1:1".into(),
location_id: Uuid::nil(),
target: "development".into(),
image_id: None,
metadata_reconcile_seconds: 3600,
safety_sync_seconds: 1800,
dev_events: false,
}
}

/// A location in `directory` whose `sqb debug` reports `wakes_compute` and whose every history read fails, published to an unreachable server: its options, script, work channel and the sync the publication starts.
pub(super) fn published_location(
directory: &Path,
Expand All @@ -93,6 +107,22 @@ pub(super) fn published_location(
(LocationServe, std::path::PathBuf),
(LocationWorkClient, LocationWorkPoll),
LocationMetadataSync,
) {
let (location, work, mut state) = republished_location(directory, wakes_compute, None);
state.begin_sync(chrono::Utc::now());
state.idle = false;
(location, work, state)
}

/// A location in `directory` published again over the sync `previous` left, and the new sync.
pub(super) fn republished_location(
directory: &Path,
wakes_compute: bool,
previous: Option<LocationMetadataSync>,
) -> (
(LocationServe, std::path::PathBuf),
(LocationWorkClient, LocationWorkPoll),
LocationMetadataSync,
) {
let script = directory.join(format!("location-{wakes_compute}"));
std::fs::write(
Expand All @@ -107,16 +137,7 @@ pub(super) fn published_location(
let mut permissions = std::fs::metadata(&script).expect("metadata").permissions();
std::os::unix::fs::PermissionsExt::set_mode(&mut permissions, 0o755);
std::fs::set_permissions(&script, permissions).expect("executable");
let options = LocationServe {
project_dir: directory.to_path_buf(),
server_url: "http://127.0.0.1:1".into(),
location_id: Uuid::nil(),
target: "development".into(),
image_id: None,
metadata_reconcile_seconds: 3600,
safety_sync_seconds: 1800,
dev_events: false,
};
let options = location_options(directory);
let client = connect_location_work((&options.server_url, "token"), options.location_id)
.expect("work client");
let poll = LocationWorkPoll {
Expand All @@ -130,14 +151,12 @@ pub(super) fn published_location(
.block_on(settle_publication(
(&options, &script, ("token", Uuid::nil())),
((one_location(), 1), true, ConnectionCheckTrigger::Publish),
(location_watch(directory), None),
(location_watch(directory), previous),
std::future::pending(),
))
.expect("no control error")
.expect("not stopped");
let mut state = metadata.expect("a connected publication starts a sync");
state.begin_sync(chrono::Utc::now());
state.idle = false;
let state = metadata.expect("a connected publication starts a sync");
((options, script), (client, poll), state)
}

Expand Down
Loading
Loading