Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
15 commits
Select commit Hold shift + click to select a range
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions clients/typescript/src/schema.d.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2311,6 +2311,8 @@ export interface components {
kind: "evaluate_sensor";
} | {
command: string[];
/** @description Plan again even when the location kept a plan for the same request. */
fresh: boolean;
/** @enum {string} */
kind: "plan";
selection: string[];
Expand Down Expand Up @@ -2812,6 +2814,8 @@ export interface components {
/** @description A person's build command to plan on the location; the server freezes it like an ad-hoc run. */
PlanRequest: {
command: string[];
/** @description Plan again instead of returning the plan the location kept for the same request. */
fresh: boolean;
/** Format: uuid */
request_id: string;
};
Expand Down Expand Up @@ -2846,6 +2850,11 @@ export interface components {
PlanTimings: {
/** @description `synced_state` for a preview over the synced warehouse state; `live` when it read the warehouse. */
planned_from: string;
/**
* Format: date-time
* @description When the plan the location kept, and answered with again, was made; `None` for a new plan.
*/
reused_from?: string | null;
steps: components["schemas"]["PlanTimingStep"][];
/** Format: double */
total_seconds: number;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ pub(crate) mod execution;
pub(crate) mod held_copy;
pub(crate) mod plan_document;
pub(crate) mod plan_request;
pub(crate) mod plan_reuse;
pub(crate) mod sync_outcomes;
pub(crate) mod sync_report;
pub(crate) mod work;
Original file line number Diff line number Diff line change
@@ -0,0 +1,128 @@
//! The location's last finished plans, in memory, returned again for an identical request.

use std::collections::{BTreeMap, VecDeque};
use std::sync::{Arc, Mutex, PoisonError};

use chrono::{DateTime, SecondsFormat, Utc};
use serde_json::{Value, json};
use sqlbuild_orchestration::platform::models::PlanModelDiff;

use crate::argv::models::Stream;
use crate::native_commands::_helpers::command::terminal::write_output;
use crate::native_commands::constants::{REUSED_PLAN_AGE, REUSED_PLAN_BYTES, REUSED_PLANS};
use crate::native_commands::models::{PlanKey, PlanReuse, PlanToKeep, ReusedPlan};
use crate::native_commands::types::LocationPlan;

/// The location's kept plans; one location serves one target.
pub(crate) static PLAN_REUSE: Mutex<Option<PlanReuse>> = Mutex::new(None);

/// The kept plan for `key` under this generation and copy, if younger than the reuse age at `now`.
pub(crate) fn reused_plan(
reuse: &Mutex<Option<PlanReuse>>,
(generation, version): (i64, &str),
key: &PlanKey,
now: DateTime<Utc>,
) -> Option<LocationPlan> {
let kept: Arc<ReusedPlan> = {
let mut reuse = reuse.lock().unwrap_or_else(PoisonError::into_inner);
let current: &mut PlanReuse = reuse
.as_mut()
.filter(|reuse| reuse.generation == generation && reuse.version == version)?;
while current
.plans
.front()
.is_some_and(|plan| now - plan.planned_at >= REUSED_PLAN_AGE)
{
if let Some(expired) = current.plans.pop_front() {
current.bytes -= plan_bytes(&expired);
}
}
Arc::clone(current.plans.iter().find(|plan| plan.key == *key)?)
};
let read = serde_json::from_str::<Value>(&kept.document).and_then(|document| {
serde_json::from_str::<BTreeMap<String, PlanModelDiff>>(&kept.diffs)
.map(|diffs| (document, diffs))
});
let (mut document, diffs) = match read {
Ok(read) => read,
Err(error) => {
drop(write_output(&[(
Stream::Stderr,
&format!("A kept plan could not be read, so the location plans again: {error}\n"),
)]));
return None;
}
};
document["timings"]["reused_from"] =
json!(kept.planned_at.to_rfc3339_opts(SecondsFormat::Secs, true));
Some((document, diffs))
}

/// Keep a checked, successful plan within `(plans, bytes)`, oldest out first; a newer generation or copy replaces all.
pub(crate) fn remember_plan(
(reuse, (most_plans, most_bytes)): (&Mutex<Option<PlanReuse>>, (usize, usize)),
keep: PlanToKeep,
(document, diffs): (&Value, &BTreeMap<String, PlanModelDiff>),
) {
if !document["error"].is_null() {
return;
}
let (Ok(text), Ok(diff_text)) = (
serde_json::to_string(document),
serde_json::to_string(diffs),
) else {
return;
};
let kept = Arc::new(ReusedPlan {
key: keep.key,
document: text,
diffs: diff_text,
planned_at: keep.planned_at,
});
if plan_bytes(&kept) > most_bytes {
return;
}
let mut reuse = reuse.lock().unwrap_or_else(PoisonError::into_inner);
let current: &mut PlanReuse = match reuse.as_mut() {
Some(current)
if current.generation == keep.generation && current.version == keep.version =>
{
current
}
_ => reuse.insert(PlanReuse {
generation: keep.generation,
version: keep.version,
plans: VecDeque::new(),
bytes: 0,
}),
};
let replaced: usize = current
.plans
.iter()
.filter(|plan| plan.key == kept.key)
.map(|plan| plan_bytes(plan))
.sum();
current.plans.retain(|plan| plan.key != kept.key);
current.bytes = current.bytes - replaced + plan_bytes(&kept);
current.plans.push_back(kept);
while current.plans.len() > most_plans || current.bytes > most_bytes {
if let Some(dropped) = current.plans.pop_front() {
current.bytes -= plan_bytes(&dropped);
}
}
}

/// Keep a plan for identical requests once the checks after planning passed.
pub(crate) fn keep_plan(
keep: Option<PlanToKeep>,
plan: (&Value, &BTreeMap<String, PlanModelDiff>),
) {
if let Some(keep) = keep {
remember_plan((&PLAN_REUSE, (REUSED_PLANS, REUSED_PLAN_BYTES)), keep, plan);
}
}

/// The bytes a kept plan's text holds.
fn plan_bytes(plan: &ReusedPlan) -> usize {
plan.document.len() + plan.diffs.len()
}
9 changes: 9 additions & 0 deletions crates/sqlbuild-cli/src/native_commands/constants.rs
Original file line number Diff line number Diff line change
Expand Up @@ -218,6 +218,15 @@ pub(crate) const CHANGED_BUILD_COMMAND: [&str; 2] = ["build", "--changed"];
/// How long an idle location waits between checks for named work; after work it checks at once.
pub(crate) const LOCATION_WORK_IDLE_POLL: std::time::Duration = std::time::Duration::from_secs(2);

/// Finished plans a location keeps for identical requests; a newer generation or synced copy drops them.
pub(crate) const REUSED_PLANS: usize = 16;

/// The most bytes of plan text a location keeps for reuse; the oldest plans go first.
pub(crate) const REUSED_PLAN_BYTES: usize = 256 * 1024 * 1024;

/// How long a kept plan may answer a request: its windows and expiry follow the time it planned at.
pub(crate) const REUSED_PLAN_AGE: chrono::TimeDelta = chrono::TimeDelta::minutes(10);

/// What a location reports when `sqb plan` produced no plan.
pub(crate) const PLAN_NOT_PRODUCED: &str =
"sqb plan did not produce a plan; see the location's log";
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ use crate::native_commands::_helpers::command::terminal::write_output;
use crate::native_commands::_helpers::location::changed_build::{
changed_build_nodes, changed_build_settings,
};
use crate::native_commands::_helpers::location::plan_reuse::keep_plan;
use crate::native_commands::main::_location_plan::plan_location;
use crate::native_commands::models::{LocationRuntimeRequest, LocationServe, LocationWatch};

Expand Down Expand Up @@ -44,9 +45,9 @@ pub(super) async fn report_changed_build(
}
};
report("Planning what changed from the synced warehouse copy...\n");
let Some((document, _)) = plan_location(
let Some(((document, diffs), keep)) = plan_location(
(options, runtime),
(&BTreeSet::new(), &settings),
(&BTreeSet::new(), &settings, false),
(client, session_id, generation),
stopped,
)
Expand All @@ -58,6 +59,7 @@ pub(super) async fn report_changed_build(
report("Project files changed while planning what changed; not reported.\n");
return Ok(Some(false));
}
keep_plan(keep, (&document, &diffs));
if let Some(reason) = document["error"]["message"].as_str() {
report(&format!("Could not plan what changed: {reason}\n"));
let request = ChangedBuildReport {
Expand Down
49 changes: 40 additions & 9 deletions crates/sqlbuild-cli/src/native_commands/main/_location_plan.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
use std::collections::{BTreeMap, BTreeSet};
use std::future::Future;

use chrono::{DateTime, Utc};
use serde_json::{Value, json};
use sqlbuild_orchestration::platform::models::LocationWorkClient;
use sqlbuild_orchestration::store::errors::ControlError;
Expand All @@ -15,21 +16,24 @@ use crate::native_commands::_helpers::location::held_copy::{HELD_COPY, held_copy
use crate::native_commands::_helpers::location::plan_request::{
plan_arguments, run_plan, selected_names,
};
use crate::native_commands::models::{HeldCopy, LocationRuntimeRequest, LocationServe};
use crate::native_commands::_helpers::location::plan_reuse::{PLAN_REUSE, reused_plan};
use crate::native_commands::models::{
HeldCopy, LocationRuntimeRequest, LocationServe, PlanKey, PlanToKeep,
};
use crate::native_commands::types::LocationPlan;

/// The plan document with the diffs it left out, an `{"error": ...}` report, or `None` when stopped first.
/// The plan with what to keep once its checks pass, an `{"error": ...}` report, or `None` when stopped first.
pub(super) async fn plan_location(
(options, runtime): (&LocationServe, &mpsc::Sender<LocationRuntimeRequest>),
(selection, settings): (&BTreeSet<String>, &BTreeMap<String, Value>),
(selection, settings, fresh): (&BTreeSet<String>, &BTreeMap<String, Value>, bool),
(client, session_id, generation): (&LocationWorkClient, Uuid, i64),
stopped: impl Future<Output = Result<(), ControlError>>,
) -> Result<Option<LocationPlan>, ControlError> {
) -> Result<Option<(LocationPlan, Option<PlanToKeep>)>, ControlError> {
let (argv, select) = match plan_arguments(options, settings)
.and_then(|argv| Ok((argv, selected_names(selection)?)))
{
Ok(planned) => planned,
Err(message) => return Ok(Some(without_diffs(plan_failure(&message)))),
Err(message) => return Ok(Some((without_diffs(plan_failure(&message)), None))),
};
let consumption: bool = settings
.get("changed")
Expand All @@ -45,21 +49,48 @@ pub(super) async fn plan_location(
let copy = match client.preview_copy(&options.target, &read).await {
Ok(copy) => copy,
Err(error) => {
return Ok(Some(without_diffs(plan_failure(&format!(
"the synced warehouse copy is unavailable: {error}"
)))));
return Ok(Some((
without_diffs(plan_failure(&format!(
"the synced warehouse copy is unavailable: {error}"
))),
None,
)));
}
};
let planned: HeldCopy = match held {
Some(held) if copy.unchanged => held,
_ => hold_copy(&HELD_COPY, options, generation, (copy, read_consumption))?,
};
let key: PlanKey = PlanKey {
argv: argv.clone(),
select: select.clone(),
consumption: planned.consumption,
};
let planned_at: DateTime<Utc> = Utc::now();
if !fresh
&& let Some(reused) = reused_plan(
&PLAN_REUSE,
(generation, &planned.version),
&key,
planned_at,
)
{
return Ok(Some((reused, None)));
}
let request = json!({"plan": {
"argv": argv,
"select": select,
"copy": planned.path.to_string_lossy(),
"version": planned.version,
"consumption": planned.consumption,
}});
run_plan(runtime, request, stopped).await
let keep = PlanToKeep {
generation,
version: planned.version,
key,
planned_at,
};
Ok(run_plan(runtime, request, stopped)
.await?
.map(|plan| (plan, Some(keep))))
}
Loading
Loading