From 9170e45754ea173ba12f1836ff36823605e289a0 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 18 Nov 2026 10:26:56 +0000 Subject: [PATCH 01/22] feat(jobs): resolve the settings named jobs give their nodes (CHI-1533) A job named in a selection, as job: or a bare job name, gives the nodes it selects its settings; naming a group names its jobs. A node takes a job's settings only when a --select token names the job, selects the node and the job's own selection holds it, so an intersection narrows them and + around a job never widens them. run_tests, ignore_audit_failures, full_refresh and [retry] apply per node, and two named jobs giving one node different values conflict. concurrency becomes a limit on the job's own nodes. Every other setting is build-wide and must be the same in every named job, and a group's jobs must agree on vars and changed. --- .../src/job_settings/constants.rs | 11 ++ .../src/job_settings/errors.rs | 17 ++ .../job_settings/main/build_job_settings.rs | 66 +++++++ .../main/group_setting_mismatch.rs | 36 ++++ .../src/job_settings/main/mod.rs | 5 + .../src/job_settings/main/named_jobs.rs | 50 +++++ .../job_settings/main/node_job_settings.rs | 140 ++++++++++++++ .../src/job_settings/main/setting_scope.rs | 16 ++ .../sqlbuild-analysis/src/job_settings/mod.rs | 9 + .../src/job_settings/models.rs | 50 +++++ .../src/job_settings/tests.rs | 8 + .../src/job_settings/tests/helpers.rs | 131 +++++++++++++ .../tests/test_build_job_settings.rs | 104 ++++++++++ .../tests/test_node_job_settings.rs | 177 ++++++++++++++++++ .../src/job_settings/tests/test_types.rs | 33 ++++ crates/sqlbuild-analysis/src/lib.rs | 1 + 16 files changed, 854 insertions(+) create mode 100644 crates/sqlbuild-analysis/src/job_settings/constants.rs create mode 100644 crates/sqlbuild-analysis/src/job_settings/errors.rs create mode 100644 crates/sqlbuild-analysis/src/job_settings/main/build_job_settings.rs create mode 100644 crates/sqlbuild-analysis/src/job_settings/main/group_setting_mismatch.rs create mode 100644 crates/sqlbuild-analysis/src/job_settings/main/mod.rs create mode 100644 crates/sqlbuild-analysis/src/job_settings/main/named_jobs.rs create mode 100644 crates/sqlbuild-analysis/src/job_settings/main/node_job_settings.rs create mode 100644 crates/sqlbuild-analysis/src/job_settings/main/setting_scope.rs create mode 100644 crates/sqlbuild-analysis/src/job_settings/mod.rs create mode 100644 crates/sqlbuild-analysis/src/job_settings/models.rs create mode 100644 crates/sqlbuild-analysis/src/job_settings/tests.rs create mode 100644 crates/sqlbuild-analysis/src/job_settings/tests/helpers.rs create mode 100644 crates/sqlbuild-analysis/src/job_settings/tests/test_build_job_settings.rs create mode 100644 crates/sqlbuild-analysis/src/job_settings/tests/test_node_job_settings.rs create mode 100644 crates/sqlbuild-analysis/src/job_settings/tests/test_types.rs diff --git a/crates/sqlbuild-analysis/src/job_settings/constants.rs b/crates/sqlbuild-analysis/src/job_settings/constants.rs new file mode 100644 index 0000000000..df410ae828 --- /dev/null +++ b/crates/sqlbuild-analysis/src/job_settings/constants.rs @@ -0,0 +1,11 @@ +//! How far each job setting reaches when a selection names the job. + +/// Settings that apply only to the named job's own nodes; two jobs giving one node different values conflict. +pub const NODE_SETTINGS: [&str; 3] = ["run_tests", "ignore_audit_failures", "full_refresh"]; +/// The setting that becomes a limit on how many of the job's nodes run at once inside the build. +pub const JOB_LIMIT_SETTING: &str = "concurrency"; +/// Build-wide settings every named group member must agree on, checked when the group compiles. +pub const GROUP_AGREED_SETTINGS: [&str; 2] = ["vars", "changed"]; +/// The help every job setting conflict gives. +pub const CONFLICT_HELP: &str = "make the jobs agree, set it in only one of them, override it with \ + the command-line flag, or build with --no-job-settings"; diff --git a/crates/sqlbuild-analysis/src/job_settings/errors.rs b/crates/sqlbuild-analysis/src/job_settings/errors.rs new file mode 100644 index 0000000000..a120b3fa72 --- /dev/null +++ b/crates/sqlbuild-analysis/src/job_settings/errors.rs @@ -0,0 +1,17 @@ +//! Job settings that cannot apply together. + +use crate::graph::errors::SelectorError; + +/// Two named jobs give one node, or the whole build, different values of one setting. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct JobSettingConflict { + pub message: String, + pub help: &'static str, +} + +/// Why the jobs a selection names cannot give their nodes settings. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum NodeJobSettingsError { + Selector(SelectorError), + Conflict(JobSettingConflict), +} diff --git a/crates/sqlbuild-analysis/src/job_settings/main/build_job_settings.rs b/crates/sqlbuild-analysis/src/job_settings/main/build_job_settings.rs new file mode 100644 index 0000000000..ed2cf4fd61 --- /dev/null +++ b/crates/sqlbuild-analysis/src/job_settings/main/build_job_settings.rs @@ -0,0 +1,66 @@ +//! The build-wide settings of the named jobs, which must all agree. + +use std::collections::BTreeMap; + +use serde_json::Value as Json; + +use crate::job_settings::constants::CONFLICT_HELP; +use crate::job_settings::errors::JobSettingConflict; +use crate::job_settings::main::setting_scope::setting_scope; +use crate::job_settings::models::{DeclaredRun, SettingScope}; + +fn settings_of<'a>( + job: &str, + declared: &'a BTreeMap, +) -> Option<&'a BTreeMap> { + match declared.get(job) { + Some(DeclaredRun::Job { settings, .. }) => Some(settings), + _ => None, + } +} + +fn shown(value: Option<&Json>) -> String { + value.map_or_else(|| "not set".to_owned(), Json::to_string) +} + +/// Each build-wide setting the named jobs give the build; a job that leaves one unset differs from one that sets it. +pub fn build_job_settings( + named: &[String], + declared: &BTreeMap, +) -> Result, JobSettingConflict> { + let mut keys: Vec<&String> = named + .iter() + .filter_map(|job| settings_of(job, declared)) + .flat_map(BTreeMap::keys) + .filter(|key| setting_scope(key) == SettingScope::Build) + .collect(); + keys.sort_unstable(); + keys.dedup(); + let mut agreed: BTreeMap = BTreeMap::new(); + for key in keys { + let mut first: Option<(&String, Option<&Json>)> = None; + for job in named { + let value: Option<&Json> = + settings_of(job, declared).and_then(|settings| settings.get(key)); + match first { + None => first = Some((job, value)), + Some((other, seen)) if seen != value => { + return Err(JobSettingConflict { + message: format!( + "jobs {other} and {job} set {key} differently ({} and {}); a build \ + applies {key} to everything it runs", + shown(seen), + shown(value) + ), + help: CONFLICT_HELP, + }); + } + Some(_) => {} + } + } + if let Some((_, Some(value))) = first { + agreed.insert(key.clone(), value.clone()); + } + } + Ok(agreed) +} diff --git a/crates/sqlbuild-analysis/src/job_settings/main/group_setting_mismatch.rs b/crates/sqlbuild-analysis/src/job_settings/main/group_setting_mismatch.rs new file mode 100644 index 0000000000..ea9402d087 --- /dev/null +++ b/crates/sqlbuild-analysis/src/job_settings/main/group_setting_mismatch.rs @@ -0,0 +1,36 @@ +//! A group's members must agree on the build-wide settings that can change any compiled SQL. + +use std::collections::BTreeMap; + +use serde_json::Value as Json; + +use crate::job_settings::constants::GROUP_AGREED_SETTINGS; +use crate::job_settings::models::DeclaredRun; + +/// Why `members` cannot run as one build, naming the setting and two members; `None` when they agree. +#[must_use] +pub fn group_setting_mismatch( + members: &[String], + declared: &BTreeMap, +) -> Option { + let value = |job: &str, key: &str| -> Option { + match declared.get(job) { + Some(DeclaredRun::Job { settings, .. }) => settings.get(key).cloned(), + _ => None, + } + }; + for key in GROUP_AGREED_SETTINGS { + let mut members = members.iter(); + let Some(first) = members.next() else { + continue; + }; + let expected: Option = value(first, key); + if let Some(other) = members.find(|member| value(member, key) != expected) { + return Some(format!( + "jobs {first} and {other} set {key} differently; a group runs its jobs as one \ + build, which has one {key}" + )); + } + } + None +} diff --git a/crates/sqlbuild-analysis/src/job_settings/main/mod.rs b/crates/sqlbuild-analysis/src/job_settings/main/mod.rs new file mode 100644 index 0000000000..f768d90cd8 --- /dev/null +++ b/crates/sqlbuild-analysis/src/job_settings/main/mod.rs @@ -0,0 +1,5 @@ +pub mod build_job_settings; +pub mod group_setting_mismatch; +pub mod named_jobs; +pub mod node_job_settings; +pub mod setting_scope; diff --git a/crates/sqlbuild-analysis/src/job_settings/main/named_jobs.rs b/crates/sqlbuild-analysis/src/job_settings/main/named_jobs.rs new file mode 100644 index 0000000000..701a1c1497 --- /dev/null +++ b/crates/sqlbuild-analysis/src/job_settings/main/named_jobs.rs @@ -0,0 +1,50 @@ +//! The jobs a selection names, as `job:` or a bare job name; naming a group names its jobs. + +use std::collections::BTreeMap; + +use crate::graph::_helpers::selectors::parse; +use crate::graph::constants::{INTERSECTION_SEPARATOR, JOB_KIND, NAME_KIND}; +use crate::graph::models::ParsedSelector; +use crate::job_settings::models::DeclaredRun; + +/// The jobs one whitespace-free selector token names, each once in the order named. +#[must_use] +pub fn token_jobs(token: &str, declared: &BTreeMap) -> Vec { + let mut jobs: Vec = Vec::new(); + for part in token.split(INTERSECTION_SEPARATOR) { + let Ok(ParsedSelector::Kind { kind, value, .. }) = parse(part) else { + continue; + }; + if kind != JOB_KIND && kind != NAME_KIND { + continue; + } + let named: Vec = match declared.get(&value) { + Some(DeclaredRun::Job { .. }) => vec![value], + Some(DeclaredRun::Group { members }) => members.clone(), + None => Vec::new(), + }; + for job in named { + if !jobs.contains(&job) { + jobs.push(job); + } + } + } + jobs +} + +/// The jobs `selectors` name, each once in the order first named. +#[must_use] +pub fn named_jobs(selectors: &[String], declared: &BTreeMap) -> Vec { + let mut jobs: Vec = Vec::new(); + for token in selectors + .iter() + .flat_map(|selector| selector.split_whitespace()) + { + for job in token_jobs(token, declared) { + if !jobs.contains(&job) { + jobs.push(job); + } + } + } + jobs +} diff --git a/crates/sqlbuild-analysis/src/job_settings/main/node_job_settings.rs b/crates/sqlbuild-analysis/src/job_settings/main/node_job_settings.rs new file mode 100644 index 0000000000..9cde08f580 --- /dev/null +++ b/crates/sqlbuild-analysis/src/job_settings/main/node_job_settings.rs @@ -0,0 +1,140 @@ +//! The settings each selected node takes from the jobs it is selected through. + +use std::collections::BTreeMap; + +use serde_json::Value as Json; + +use crate::assembly::project::types::ObjectKey; +use crate::graph::_helpers::selectors::{Keys, tokens}; +use crate::graph::models::ProjectGraph; +use crate::job_settings::constants::{CONFLICT_HELP, JOB_LIMIT_SETTING, NODE_SETTINGS}; +use crate::job_settings::errors::{JobSettingConflict, NodeJobSettingsError}; +use crate::job_settings::main::named_jobs::token_jobs; +use crate::job_settings::models::{DeclaredRun, JobRetry, NodeJobSettings}; + +fn conflict( + key: &ObjectKey, + (first, second): (&str, &str), + setting: &str, + values: (String, String), +) -> NodeJobSettingsError { + NodeJobSettingsError::Conflict(JobSettingConflict { + message: format!( + "{}:{} is selected through jobs {first} and {second}, which set {setting} differently \ + ({} and {})", + key.0, key.1, values.0, values.1 + ), + help: CONFLICT_HELP, + }) +} + +/// A node is selected through a job when a `--select` token names the job, selects the node, and +/// the job's own selection holds it; `+` around a job and intersections never widen what it covers. +pub fn node_job_settings( + graph: &ProjectGraph, + (select, exclude): (&[String], &[String]), + declared: &BTreeMap, +) -> Result { + let mut result = NodeJobSettings::default(); + let excluded: Keys = tokens(graph, exclude).map_err(NodeJobSettingsError::Selector)?; + let resolved = graph.jobs.as_ref(); + for token in select + .iter() + .flat_map(|selector| selector.split_whitespace()) + { + let jobs: Vec = token_jobs(token, declared); + if jobs.is_empty() { + continue; + } + let selected: Keys = + tokens(graph, &[token.to_owned()]).map_err(NodeJobSettingsError::Selector)?; + for job in jobs { + let Some(Ok(own)) = resolved.and_then(|resolved| resolved.get(&job)) else { + continue; + }; + for key in selected + .intersection(own) + .filter(|key| !excluded.contains(*key)) + { + let jobs = result.node_jobs.entry(key.clone()).or_default(); + if !jobs.contains(&job) { + jobs.push(job.clone()); + } + } + } + } + for (key, jobs) in &result.node_jobs { + let mut values: BTreeMap = BTreeMap::new(); + let mut retry: Option<(&String, JobRetry)> = None; + for job in jobs { + let Some(DeclaredRun::Job { + settings, + retry: job_retry, + }) = declared.get(job) + else { + continue; + }; + for setting in NODE_SETTINGS { + let Some(value) = settings.get(setting) else { + continue; + }; + match values.get(setting) { + Some((other, seen)) if seen != value => { + return Err(conflict( + key, + (other, job), + setting, + (seen.to_string(), value.to_string()), + )); + } + Some(_) => {} + None => { + values.insert(setting.to_owned(), (job.clone(), value.clone())); + } + } + } + if let Some(job_retry) = job_retry { + match retry { + Some((other, seen)) if seen != *job_retry => { + let shown = |retry: JobRetry| { + format!( + "{} attempts, {}s apart", + retry.attempts, retry.delay_seconds + ) + }; + return Err(conflict( + key, + (other, job), + "[retry]", + (shown(seen), shown(*job_retry)), + )); + } + Some(_) => {} + None => retry = Some((job, *job_retry)), + } + } + } + if !values.is_empty() { + result.values.insert( + key.clone(), + values + .into_iter() + .map(|(setting, (_, value))| (setting, value)) + .collect(), + ); + } + if let Some((_, retry)) = retry { + result.retries.insert(key.clone(), retry); + } + } + for jobs in result.node_jobs.values() { + for job in jobs { + if let Some(DeclaredRun::Job { settings, .. }) = declared.get(job) + && let Some(limit) = settings.get(JOB_LIMIT_SETTING).and_then(Json::as_i64) + { + result.job_limits.insert(job.clone(), limit); + } + } + } + Ok(result) +} diff --git a/crates/sqlbuild-analysis/src/job_settings/main/setting_scope.rs b/crates/sqlbuild-analysis/src/job_settings/main/setting_scope.rs new file mode 100644 index 0000000000..80bda3ed92 --- /dev/null +++ b/crates/sqlbuild-analysis/src/job_settings/main/setting_scope.rs @@ -0,0 +1,16 @@ +//! How far one job setting reaches. + +use crate::job_settings::constants::{JOB_LIMIT_SETTING, NODE_SETTINGS}; +use crate::job_settings::models::SettingScope; + +/// A node setting, the job limit, or else a build-wide setting. +#[must_use] +pub fn setting_scope(name: &str) -> SettingScope { + if NODE_SETTINGS.contains(&name) { + SettingScope::Node + } else if name == JOB_LIMIT_SETTING { + SettingScope::JobLimit + } else { + SettingScope::Build + } +} diff --git a/crates/sqlbuild-analysis/src/job_settings/mod.rs b/crates/sqlbuild-analysis/src/job_settings/mod.rs new file mode 100644 index 0000000000..288d5de7a9 --- /dev/null +++ b/crates/sqlbuild-analysis/src/job_settings/mod.rs @@ -0,0 +1,9 @@ +//! Jobs as named selections: which jobs a selection names and the settings they give its nodes. + +pub mod constants; +pub mod errors; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-analysis/src/job_settings/models.rs b/crates/sqlbuild-analysis/src/job_settings/models.rs new file mode 100644 index 0000000000..1642d3c606 --- /dev/null +++ b/crates/sqlbuild-analysis/src/job_settings/models.rs @@ -0,0 +1,50 @@ +//! Declared jobs and groups, and the settings a selection gives its nodes. + +use std::collections::BTreeMap; + +use serde_json::Value as Json; + +use crate::assembly::project::types::ObjectKey; + +/// A job's `[retry]`: at most `attempts` runs in all, each rerun `delay_seconds` after the last. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct JobRetry { + pub attempts: u8, + pub delay_seconds: u32, +} + +/// A declared job's `[settings]` and `[retry]`, or a group's member jobs. +#[derive(Clone, Debug, PartialEq)] +pub enum DeclaredRun { + Job { + settings: BTreeMap, + retry: Option, + }, + Group { + members: Vec, + }, +} + +/// How far one job setting reaches. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum SettingScope { + /// Only the job's own nodes. + Node, + /// A limit on how many of the job's own nodes run at once. + JobLimit, + /// The whole build. + Build, +} + +/// What the jobs a selection names give each of its nodes. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct NodeJobSettings { + /// The named jobs each node is selected through, in the order the jobs are named. + pub node_jobs: BTreeMap>, + /// Each node's node-level settings, from whichever named job sets them. + pub values: BTreeMap>, + /// Each node's `[retry]`, from whichever named job sets it. + pub retries: BTreeMap, + /// Each named job's limit on how many of its nodes run at once. + pub job_limits: BTreeMap, +} diff --git a/crates/sqlbuild-analysis/src/job_settings/tests.rs b/crates/sqlbuild-analysis/src/job_settings/tests.rs new file mode 100644 index 0000000000..eeac578fe3 --- /dev/null +++ b/crates/sqlbuild-analysis/src/job_settings/tests.rs @@ -0,0 +1,8 @@ +#[path = "tests/test_build_job_settings.rs"] +mod build_job_settings; +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_node_job_settings.rs"] +mod node_job_settings; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-analysis/src/job_settings/tests/helpers.rs b/crates/sqlbuild-analysis/src/job_settings/tests/helpers.rs new file mode 100644 index 0000000000..0b0a80b4c0 --- /dev/null +++ b/crates/sqlbuild-analysis/src/job_settings/tests/helpers.rs @@ -0,0 +1,131 @@ +use std::collections::BTreeMap; + +use serde_json::{Value as Json, json}; + +use crate::assembly::project::types::ObjectKey; +use crate::graph::main::attach_jobs::attach_jobs; +use crate::graph::main::build_project_graph::build_project_graph; +use crate::graph::models::{GraphResource, JobSelection, ProjectGraph}; +use crate::job_settings::models::{DeclaredRun, JobRetry}; + +fn key(kind: &str, name: &str) -> ObjectKey { + (kind.to_owned(), name.to_owned()) +} + +fn texts(values: &[&str]) -> Vec { + values.iter().map(|value| (*value).to_owned()).collect() +} + +/// Staged orders feed orders and refunds; customers stands alone. +fn graph() -> ProjectGraph { + let model = |name: &str, deps: &[&str]| GraphResource { + key: key("model", name), + deps: deps.iter().map(|dep| key("model", dep)).collect(), + tags: Vec::new(), + folder: None, + }; + build_project_graph(&[ + model("stg_orders", &[]), + model("orders", &["stg_orders"]), + model("refunds", &["stg_orders"]), + model("customers", &[]), + ]) +} + +/// Jobs over the graph with their settings, and a group of two of them. +pub(super) fn declared() -> BTreeMap { + let job = |settings: Json, retry: Option| DeclaredRun::Job { + settings: settings + .as_object() + .map(|object| object.clone().into_iter().collect()) + .unwrap_or_default(), + retry, + }; + BTreeMap::from([ + ( + "orders_job".to_owned(), + job( + json!({"concurrency": 2, "run_tests": false, "full_refresh": true}), + None, + ), + ), + ( + "refunds_job".to_owned(), + job(json!({"concurrency": 4}), None), + ), + ( + "staging_job".to_owned(), + job(json!({"run_tests": true}), None), + ), + ( + "quiet_job".to_owned(), + job(json!({"run_tests": false}), None), + ), + ( + "crm_job".to_owned(), + job(json!({"vars": {"region": "eu"}}), None), + ), + ( + "crm_us_job".to_owned(), + job(json!({"vars": {"region": "us"}}), None), + ), + ( + "retried_job".to_owned(), + job( + json!({}), + Some(JobRetry { + attempts: 3, + delay_seconds: 60, + }), + ), + ), + ( + "retried_fast_job".to_owned(), + job( + json!({}), + Some(JobRetry { + attempts: 2, + delay_seconds: 0, + }), + ), + ), + ( + "daily_group".to_owned(), + DeclaredRun::Group { + members: texts(&["orders_job", "refunds_job"]), + }, + ), + ]) +} + +fn selections() -> BTreeMap { + let job = |select: &[&str]| JobSelection::Job { + select: texts(select), + exclude: Vec::new(), + }; + BTreeMap::from([ + ("orders_job".to_owned(), job(&["+orders"])), + ("refunds_job".to_owned(), job(&["+refunds"])), + ("staging_job".to_owned(), job(&["stg_orders"])), + ("quiet_job".to_owned(), job(&["stg_orders"])), + ("crm_job".to_owned(), job(&["customers"])), + ("crm_us_job".to_owned(), job(&["customers"])), + ("retried_job".to_owned(), job(&["stg_orders"])), + ("retried_fast_job".to_owned(), job(&["stg_orders"])), + ( + "daily_group".to_owned(), + JobSelection::Group { + members: texts(&["orders_job", "refunds_job"]), + }, + ), + ]) +} + +/// The graph with the jobs attached, as a command that names a job resolves it. +pub(super) fn graph_with_jobs() -> ProjectGraph { + attach_jobs(graph(), selections()) +} + +pub(super) fn strings(values: &[&str]) -> Vec { + texts(values) +} diff --git a/crates/sqlbuild-analysis/src/job_settings/tests/test_build_job_settings.rs b/crates/sqlbuild-analysis/src/job_settings/tests/test_build_job_settings.rs new file mode 100644 index 0000000000..b712f8a448 --- /dev/null +++ b/crates/sqlbuild-analysis/src/job_settings/tests/test_build_job_settings.rs @@ -0,0 +1,104 @@ +use std::collections::BTreeMap; + +use serde_json::json; + +use crate::job_settings::main::build_job_settings::build_job_settings; +use crate::job_settings::main::group_setting_mismatch::group_setting_mismatch; +use crate::job_settings::main::named_jobs::named_jobs; +use crate::job_settings::tests::helpers::{declared, strings}; +use crate::job_settings::tests::test_types::{BuildJobSettingsTestCase, GroupMismatchTestCase}; + +#[test] +fn given_named_jobs_when_merging_build_settings_then_they_must_agree() { + let test_cases = [ + BuildJobSettingsTestCase { + description: "a job that leaves vars unset differs from one that sets them", + select: &["crm_job", "job:orders_job"], + expected_named: vec!["crm_job", "orders_job"], + expected: Err( + "jobs crm_job and orders_job set vars differently ({\"region\":\"eu\"} \ + and not set); a build applies vars to everything it runs", + ), + }, + BuildJobSettingsTestCase { + description: "a single job's vars", + select: &["job:crm_job,customers"], + expected_named: vec!["crm_job"], + expected: Ok(BTreeMap::from([("vars", json!({"region": "eu"}))])), + }, + BuildJobSettingsTestCase { + description: "two jobs with different vars", + select: &["job:crm_job job:crm_us_job"], + expected_named: vec!["crm_job", "crm_us_job"], + expected: Err( + "jobs crm_job and crm_us_job set vars differently ({\"region\":\"eu\"} \ + and {\"region\":\"us\"}); a build applies vars to everything it runs", + ), + }, + BuildJobSettingsTestCase { + description: "a group's jobs without build-wide settings", + select: &["job:daily_group"], + expected_named: vec!["orders_job", "refunds_job"], + expected: Ok(BTreeMap::new()), + }, + BuildJobSettingsTestCase { + description: "tags, paths and node names name no job", + select: &["tag:daily models/marts orders"], + expected_named: vec![], + expected: Ok(BTreeMap::new()), + }, + ]; + for test_case in test_cases { + let named: Vec = named_jobs(&strings(test_case.select), &declared()); + assert_eq!(named, test_case.expected_named, "{}", test_case.description); + let merged = build_job_settings(&named, &declared()); + let merged = merged + .map(|settings| settings.into_iter().collect::>()) + .map_err(|conflict| conflict.message); + let expected = test_case + .expected + .map(|settings| { + settings + .into_iter() + .map(|(key, value)| (key.to_owned(), value)) + .collect() + }) + .map_err(str::to_owned); + assert_eq!(merged, expected, "{}", test_case.description); + } +} + +#[test] +fn given_a_group_when_its_jobs_differ_on_vars_or_changed_then_it_cannot_run_as_one_build() { + let test_cases = [ + GroupMismatchTestCase { + description: "different vars", + members: &["crm_job", "crm_us_job"], + expected: Some( + "jobs crm_job and crm_us_job set vars differently; a group runs its \ + jobs as one build, which has one vars", + ), + }, + GroupMismatchTestCase { + description: "vars set by only one job still differ", + members: &["orders_job", "crm_job"], + expected: Some( + "jobs orders_job and crm_job set vars differently; a group runs its \ + jobs as one build, which has one vars", + ), + }, + GroupMismatchTestCase { + description: "node settings may differ", + members: &["orders_job", "refunds_job", "quiet_job"], + expected: None, + }, + ]; + for test_case in test_cases { + assert_eq!( + group_setting_mismatch(&strings(test_case.members), &declared()).as_deref(), + test_case.expected, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/job_settings/tests/test_node_job_settings.rs b/crates/sqlbuild-analysis/src/job_settings/tests/test_node_job_settings.rs new file mode 100644 index 0000000000..ec8778c618 --- /dev/null +++ b/crates/sqlbuild-analysis/src/job_settings/tests/test_node_job_settings.rs @@ -0,0 +1,177 @@ +use std::collections::BTreeMap; + +use serde_json::json; + +use crate::job_settings::errors::NodeJobSettingsError; +use crate::job_settings::main::node_job_settings::node_job_settings; +use crate::job_settings::models::NodeJobSettings; +use crate::job_settings::tests::helpers::{declared, graph_with_jobs, strings}; +use crate::job_settings::tests::test_types::{NodeJobConflictTestCase, NodeJobSettingsTestCase}; + +fn settings(select: &[&str], exclude: &[&str]) -> Result { + node_job_settings( + &graph_with_jobs(), + (&strings(select), &strings(exclude)), + &declared(), + ) +} + +#[test] +fn given_named_jobs_when_resolving_node_settings_then_only_their_own_selected_nodes_take_them() { + let orders_values = + || BTreeMap::from([("run_tests", json!(false)), ("full_refresh", json!(true))]); + let test_cases = [ + NodeJobSettingsTestCase { + description: "a named job gives each of its nodes its settings and limit", + select: &["job:orders_job"], + exclude: &[], + expected_node_jobs: BTreeMap::from([ + ("orders", vec!["orders_job"]), + ("stg_orders", vec!["orders_job"]), + ]), + expected_values: BTreeMap::from([ + ("orders", orders_values()), + ("stg_orders", orders_values()), + ]), + expected_job_limits: BTreeMap::from([("orders_job", 2)]), + }, + NodeJobSettingsTestCase { + description: "an intersection gives the job's settings to just the nodes it selects", + select: &["job:orders_job,orders"], + exclude: &[], + expected_node_jobs: BTreeMap::from([("orders", vec!["orders_job"])]), + expected_values: BTreeMap::from([("orders", orders_values())]), + expected_job_limits: BTreeMap::from([("orders_job", 2)]), + }, + NodeJobSettingsTestCase { + description: "nodes added beside the job, or by + around it, take none of its settings", + select: &["job:orders_job customers", "job:orders_job+"], + exclude: &[], + expected_node_jobs: BTreeMap::from([ + ("orders", vec!["orders_job"]), + ("stg_orders", vec!["orders_job"]), + ]), + expected_values: BTreeMap::from([ + ("orders", orders_values()), + ("stg_orders", orders_values()), + ]), + expected_job_limits: BTreeMap::from([("orders_job", 2)]), + }, + NodeJobSettingsTestCase { + description: "an excluded node takes nothing", + select: &["orders_job"], + exclude: &["stg_orders"], + expected_node_jobs: BTreeMap::from([("orders", vec!["orders_job"])]), + expected_values: BTreeMap::from([("orders", orders_values())]), + expected_job_limits: BTreeMap::from([("orders_job", 2)]), + }, + NodeJobSettingsTestCase { + description: "naming a group names its jobs; a shared node keeps both jobs' limits", + select: &["job:daily_group"], + exclude: &[], + expected_node_jobs: BTreeMap::from([ + ("orders", vec!["orders_job"]), + ("refunds", vec!["refunds_job"]), + ("stg_orders", vec!["orders_job", "refunds_job"]), + ]), + expected_values: BTreeMap::from([ + ("orders", orders_values()), + ("stg_orders", orders_values()), + ]), + expected_job_limits: BTreeMap::from([("orders_job", 2), ("refunds_job", 4)]), + }, + NodeJobSettingsTestCase { + description: "jobs that agree, or where only one sets a setting, do not conflict", + select: &["job:quiet_job job:orders_job"], + exclude: &[], + expected_node_jobs: BTreeMap::from([ + ("orders", vec!["orders_job"]), + ("stg_orders", vec!["quiet_job", "orders_job"]), + ]), + expected_values: BTreeMap::from([ + ("orders", orders_values()), + ("stg_orders", orders_values()), + ]), + expected_job_limits: BTreeMap::from([("orders_job", 2)]), + }, + NodeJobSettingsTestCase { + description: "a selection that names no job takes no settings", + select: &["+orders"], + exclude: &[], + expected_node_jobs: BTreeMap::new(), + expected_values: BTreeMap::new(), + expected_job_limits: BTreeMap::new(), + }, + ]; + for test_case in test_cases { + let resolved: NodeJobSettings = + settings(test_case.select, test_case.exclude).expect(test_case.description); + let node_jobs: BTreeMap<&str, Vec<&str>> = resolved + .node_jobs + .iter() + .map(|(key, jobs)| (key.1.as_str(), jobs.iter().map(String::as_str).collect())) + .collect(); + let values: BTreeMap<&str, BTreeMap<&str, serde_json::Value>> = resolved + .values + .iter() + .map(|(key, values)| { + ( + key.1.as_str(), + values + .iter() + .map(|(name, value)| (name.as_str(), value.clone())) + .collect(), + ) + }) + .collect(); + let limits: BTreeMap<&str, i64> = resolved + .job_limits + .iter() + .map(|(job, limit)| (job.as_str(), *limit)) + .collect(); + assert_eq!( + node_jobs, test_case.expected_node_jobs, + "{}", + test_case.description + ); + assert_eq!( + values, test_case.expected_values, + "{}", + test_case.description + ); + assert_eq!( + limits, test_case.expected_job_limits, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_two_named_jobs_setting_one_node_differently_when_resolving_then_it_is_a_conflict() { + let test_cases = [ + NodeJobConflictTestCase { + description: "a node-level setting", + select: &["job:staging_job", "job:orders_job"], + expected_message: "model:stg_orders is selected through jobs staging_job and orders_job, \ + which set run_tests differently (true and false)", + }, + NodeJobConflictTestCase { + description: "a retry policy", + select: &["job:retried_job job:retried_fast_job"], + expected_message: "model:stg_orders is selected through jobs retried_job and \ + retried_fast_job, which set [retry] differently (3 attempts, 60s apart and 2 \ + attempts, 0s apart)", + }, + ]; + for test_case in test_cases { + let Err(NodeJobSettingsError::Conflict(conflict)) = settings(test_case.select, &[]) else { + panic!("{}: expected a conflict", test_case.description); + }; + assert_eq!( + conflict.message, test_case.expected_message, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/job_settings/tests/test_types.rs b/crates/sqlbuild-analysis/src/job_settings/tests/test_types.rs new file mode 100644 index 0000000000..25fd309e19 --- /dev/null +++ b/crates/sqlbuild-analysis/src/job_settings/tests/test_types.rs @@ -0,0 +1,33 @@ +use std::collections::BTreeMap; + +use serde_json::Value as Json; + +pub(super) struct NodeJobSettingsTestCase { + pub(super) description: &'static str, + pub(super) select: &'static [&'static str], + pub(super) exclude: &'static [&'static str], + /// Each node's named jobs, as `name -> jobs`. + pub(super) expected_node_jobs: BTreeMap<&'static str, Vec<&'static str>>, + /// Each node's node-level settings, as `name -> setting -> value`. + pub(super) expected_values: BTreeMap<&'static str, BTreeMap<&'static str, Json>>, + pub(super) expected_job_limits: BTreeMap<&'static str, i64>, +} + +pub(super) struct NodeJobConflictTestCase { + pub(super) description: &'static str, + pub(super) select: &'static [&'static str], + pub(super) expected_message: &'static str, +} + +pub(super) struct BuildJobSettingsTestCase { + pub(super) description: &'static str, + pub(super) select: &'static [&'static str], + pub(super) expected_named: Vec<&'static str>, + pub(super) expected: Result, &'static str>, +} + +pub(super) struct GroupMismatchTestCase { + pub(super) description: &'static str, + pub(super) members: &'static [&'static str], + pub(super) expected: Option<&'static str>, +} diff --git a/crates/sqlbuild-analysis/src/lib.rs b/crates/sqlbuild-analysis/src/lib.rs index 0f0ca871f0..071e868275 100644 --- a/crates/sqlbuild-analysis/src/lib.rs +++ b/crates/sqlbuild-analysis/src/lib.rs @@ -20,6 +20,7 @@ pub mod fingerprint_storage; pub mod graph; pub mod identifier_fitting; pub mod janitor_planning; +pub mod job_settings; pub mod lineage; pub mod lineage_command; pub mod lineage_graph; From 4c536d7357198d1d1f1b1f6ab1598e2706cd4701 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 18 Nov 2026 11:08:16 +0000 Subject: [PATCH 02/22] feat(jobs): apply named jobs' settings to the nodes they select (CHI-1533) sqb build and sqb plan now take the settings of the jobs their selection names. Build-wide settings, such as vars and changed, apply where the command line leaves them unset, as the same flags typed there would; every named job must agree on them. Node-level settings apply to the named job's own nodes: full_refresh rebuilds just those models, run_tests = false drops their SQL tests, and ignore_audit_failures decides their audits. concurrency limits how many of the job's nodes run at once inside the build, and the scheduler starts another ready node while one job is full. --no-job-settings ignores job settings, and a group whose jobs set different vars or changed fails to compile. --- .../src/job_settings/constants.rs | 2 + .../src/argv/_helpers/job_settings.rs | 1 + .../sqlbuild-cli/src/argv/command_table.json | 24 ++ crates/sqlbuild-cli/src/argv/constants.rs | 1 + crates/sqlbuild-cli/src/dispatch/constants.rs | 4 +- .../_helpers/compiler_records/constants.rs | 4 + .../_helpers/planner_records/constants.rs | 1 + .../src/records/constants.rs | 2 + .../_helpers/sources/orchestration_values.rs | 19 ++ .../_helpers/entries/model_entries.rs | 3 +- .../src/planner_entries/models.rs | 3 +- .../_helpers/planning/planning_boundary.rs | 7 +- .../_helpers/inputs/warehouse_state.rs | 7 +- .../_helpers/rebuilds/execution_changes.rs | 3 +- .../src/planner_phases/constants.rs | 4 + .../effectively_full_refreshed_model_names.rs | 6 +- .../src/planner_phases/models.rs | 31 ++- .../_helpers/inspection/snapshot.rs | 9 +- .../compiled_project/_helpers/job_catalog.rs | 64 ++++- .../src/compiled_project/constants.rs | 8 + .../compiled_project/main/job_declarations.rs | 12 + .../src/compiled_project/main/mod.rs | 1 + .../_helpers/build_command/execution.rs | 26 +- .../_helpers/build_command/planning.rs | 2 + .../src/bindings/_helpers/cli/dispatch.rs | 2 + .../src/bindings/_helpers/cli/mod.rs | 1 + .../_helpers/cli/named_job_settings.rs | 96 +++++++ .../compile_driver/job_node_settings.rs | 260 ++++++++++++++++++ .../bindings/_helpers/compile_driver/mod.rs | 1 + .../_helpers/compile_driver/plan_result.rs | 81 ++++-- .../_helpers/compiled_project/job_settings.rs | 78 ++++++ .../bindings/_helpers/compiled_project/mod.rs | 1 + .../_helpers/location_command/constants.rs | 8 - .../_helpers/location_command/worker.rs | 27 +- .../bindings/_helpers/plan_command/plan.rs | 5 + src/sqlbuild/_native.pyi | 14 + .../executor/build/classes/build_scheduler.py | 53 +++- src/sqlbuild/executor/build/models.py | 2 + 38 files changed, 780 insertions(+), 93 deletions(-) create mode 100644 crates/sqlbuild-python-project/src/compiled_project/main/job_declarations.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/cli/named_job_settings.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/compile_driver/job_node_settings.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/compiled_project/job_settings.rs diff --git a/crates/sqlbuild-analysis/src/job_settings/constants.rs b/crates/sqlbuild-analysis/src/job_settings/constants.rs index df410ae828..dd941a907d 100644 --- a/crates/sqlbuild-analysis/src/job_settings/constants.rs +++ b/crates/sqlbuild-analysis/src/job_settings/constants.rs @@ -9,3 +9,5 @@ pub const GROUP_AGREED_SETTINGS: [&str; 2] = ["vars", "changed"]; /// The help every job setting conflict gives. pub const CONFLICT_HELP: &str = "make the jobs agree, set it in only one of them, override it with \ the command-line flag, or build with --no-job-settings"; +/// The error code of job settings that cannot apply together. +pub const JOB_SETTING_CONFLICT_CODE: &str = "S027"; diff --git a/crates/sqlbuild-cli/src/argv/_helpers/job_settings.rs b/crates/sqlbuild-cli/src/argv/_helpers/job_settings.rs index d867bc0438..7fa1c3e300 100644 --- a/crates/sqlbuild-cli/src/argv/_helpers/job_settings.rs +++ b/crates/sqlbuild-cli/src/argv/_helpers/job_settings.rs @@ -30,6 +30,7 @@ pub(crate) fn setting_argument( | "dbt_profiles_dir" | "dbt_target" | "dbt_target_path" + | "job_settings" ) { return Err(format!("job setting {name} is owned by the run request")); } diff --git a/crates/sqlbuild-cli/src/argv/command_table.json b/crates/sqlbuild-cli/src/argv/command_table.json index 6f19fa1d5d..322ddc5b85 100644 --- a/crates/sqlbuild-cli/src/argv/command_table.json +++ b/crates/sqlbuild-cli/src/argv/command_table.json @@ -1888,6 +1888,18 @@ "default": false, "group": 1 }, + { + "kind": "store_false", + "options": [ + "--no-job-settings" + ], + "dest": "job_settings", + "nargs": 0, + "const": false, + "default": true, + "help": "ignore the settings of the jobs the selection names, using only the command line's, the target's and the project's", + "group": 1 + }, { "kind": "store", "options": [ @@ -2548,6 +2560,18 @@ "default": false, "group": 1 }, + { + "kind": "store_false", + "options": [ + "--no-job-settings" + ], + "dest": "job_settings", + "nargs": 0, + "const": false, + "default": true, + "help": "ignore the settings of the jobs the selection names, using only the command line's, the target's and the project's", + "group": 1 + }, { "kind": "store", "options": [ diff --git a/crates/sqlbuild-cli/src/argv/constants.rs b/crates/sqlbuild-cli/src/argv/constants.rs index c99a0c09a5..865fb4cae5 100644 --- a/crates/sqlbuild-cli/src/argv/constants.rs +++ b/crates/sqlbuild-cli/src/argv/constants.rs @@ -135,6 +135,7 @@ pub(crate) const CLI_NAMESPACE_DEFAULTS: [(&str, NamespaceDefault); 193] = [ ("no_color", NamespaceDefault::Bool(false)), ("fail_fast", NamespaceDefault::Bool(false)), ("full_refresh", NamespaceDefault::Bool(false)), + ("job_settings", NamespaceDefault::Bool(true)), ("changed", NamespaceDefault::Bool(false)), ("changed_counts", NamespaceDefault::None), ("reach", NamespaceDefault::None), diff --git a/crates/sqlbuild-cli/src/dispatch/constants.rs b/crates/sqlbuild-cli/src/dispatch/constants.rs index 953ee7474c..b61a1ff058 100644 --- a/crates/sqlbuild-cli/src/dispatch/constants.rs +++ b/crates/sqlbuild-cli/src/dispatch/constants.rs @@ -110,7 +110,7 @@ pub(crate) const DAG_ROUTE: Route = routed( }, ); -const PLAN_FIELDS: [Field; 25] = [ +const PLAN_FIELDS: [Field; 26] = [ PROJECT_DIR, NO_SQL_VALIDATION, same("no_cache"), @@ -121,6 +121,7 @@ const PLAN_FIELDS: [Field; 25] = [ CURSOR_OVERRIDES, JSON_OUTPUT, same("full_refresh"), + same("job_settings"), same("changed"), same("changed_counts"), same("reach"), @@ -158,6 +159,7 @@ const BUILD_FIELDS: [Field; 41] = [ NO_COLOR, same("fail_fast"), same("full_refresh"), + same("job_settings"), same("changed"), same("changed_counts"), same("reach"), diff --git a/crates/sqlbuild-python-base/src/records/_helpers/compiler_records/constants.rs b/crates/sqlbuild-python-base/src/records/_helpers/compiler_records/constants.rs index ca785c24e1..e0e0f48a3e 100644 --- a/crates/sqlbuild-python-base/src/records/_helpers/compiler_records/constants.rs +++ b/crates/sqlbuild-python-base/src/records/_helpers/compiler_records/constants.rs @@ -400,6 +400,8 @@ const R_COMPILE_PIPELINE_OPTIONS: RecordSpec = RecordSpec { FieldSpec::new("selection_diagnostics", FieldDefault::False), FieldSpec::new("plan_sql_tests", FieldDefault::True), FieldSpec::new("ignore_audit_failures", FieldDefault::False), + FieldSpec::new("cli_ignore_audit_failures", FieldDefault::None), + FieldSpec::new("job_settings", FieldDefault::True), FieldSpec::new("record_migration_fingerprints", FieldDefault::True), FieldSpec::new("accepts_unit_test_selectors", FieldDefault::False), FieldSpec::new( @@ -873,6 +875,8 @@ const R_COMPILE_PIPELINE_RESULT: RecordSpec = RecordSpec { FieldSpec::new("custom_materializations", FieldDefault::EmptyDict), FieldSpec::new("python_node_names", FieldDefault::EmptyFrozenset), FieldSpec::new("python_plan_entries", FieldDefault::EmptyTuple), + FieldSpec::new("job_limits", FieldDefault::EmptyDict), + FieldSpec::new("node_jobs", FieldDefault::EmptyDict), FieldSpec::new("compile_seconds", FieldDefault::None), FieldSpec::new("planning_seconds", FieldDefault::None), ], diff --git a/crates/sqlbuild-python-base/src/records/_helpers/planner_records/constants.rs b/crates/sqlbuild-python-base/src/records/_helpers/planner_records/constants.rs index be7daa312e..8c4851223e 100644 --- a/crates/sqlbuild-python-base/src/records/_helpers/planner_records/constants.rs +++ b/crates/sqlbuild-python-base/src/records/_helpers/planner_records/constants.rs @@ -85,6 +85,7 @@ const R_PLANNER_OVERRIDES: RecordSpec = RecordSpec { fields: &[ FieldSpec::new("cursor_overrides", FieldDefault::None), FieldSpec::new("full_refresh", FieldDefault::False), + FieldSpec::new("full_refresh_models", FieldDefault::EmptyFrozenset), FieldSpec::new("reload_sources", FieldDefault::False), FieldSpec::new("forced_stale_model_names", FieldDefault::EmptyTuple), FieldSpec::new("external_blocked_model_names", FieldDefault::EmptyTuple), diff --git a/crates/sqlbuild-python-base/src/records/constants.rs b/crates/sqlbuild-python-base/src/records/constants.rs index d2b0e21e1b..643e4877df 100644 --- a/crates/sqlbuild-python-base/src/records/constants.rs +++ b/crates/sqlbuild-python-base/src/records/constants.rs @@ -244,6 +244,7 @@ const BUILD_COMMAND_REQUEST: RecordSpec = RecordSpec { FieldSpec::new("no_color", FieldDefault::False), FieldSpec::new("fail_fast", FieldDefault::False), FieldSpec::new("full_refresh", FieldDefault::False), + FieldSpec::new("job_settings", FieldDefault::True), FieldSpec::new("changed", FieldDefault::False), FieldSpec::new("changed_counts", FieldDefault::None), FieldSpec::new("reach", FieldDefault::None), @@ -568,6 +569,7 @@ const PLAN_COMMAND_REQUEST: RecordSpec = RecordSpec { FieldSpec::new("cursor_overrides", FieldDefault::None), FieldSpec::new("json_output", FieldDefault::False), FieldSpec::new("full_refresh", FieldDefault::False), + FieldSpec::new("job_settings", FieldDefault::True), FieldSpec::new("changed", FieldDefault::False), FieldSpec::new("changed_counts", FieldDefault::None), FieldSpec::new("reach", FieldDefault::None), diff --git a/crates/sqlbuild-python-front/src/discovery/_helpers/sources/orchestration_values.rs b/crates/sqlbuild-python-front/src/discovery/_helpers/sources/orchestration_values.rs index fbc3692ebc..3c4c731b92 100644 --- a/crates/sqlbuild-python-front/src/discovery/_helpers/sources/orchestration_values.rs +++ b/crates/sqlbuild-python-front/src/discovery/_helpers/sources/orchestration_values.rs @@ -7,6 +7,8 @@ use chrono_tz::Tz; use pyo3::prelude::Python; use serde_json::Value; use sqlbuild_analysis::graph::main::parse_selector::parse_selector; +use sqlbuild_analysis::job_settings::main::group_setting_mismatch::group_setting_mismatch; +use sqlbuild_analysis::job_settings::models::DeclaredRun; use sqlbuild_cli::argv::main::job_settings_arguments::job_settings_arguments; use sqlbuild_discovery::models::{DiscoveryFailure, FailureKind}; use sqlbuild_discovery::orchestration_files::models::{ @@ -58,6 +60,23 @@ pub(crate) fn check_orchestration_values( })?; } } + let declared: BTreeMap = inventory + .jobs + .iter() + .map(|(_, job)| { + let run = DeclaredRun::Job { + settings: job.settings.clone(), + retry: None, + }; + (job.run.name.value.clone(), run) + }) + .collect(); + for (path, group) in &inventory.groups { + let members: Vec = group.jobs.iter().map(|job| job.value.clone()).collect(); + if let Some(message) = group_setting_mismatch(&members, &declared) { + return Err(failure(path, (group.run.name.position, message))); + } + } Ok(()) } diff --git a/crates/sqlbuild-python-planner/src/planner_entries/_helpers/entries/model_entries.rs b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/entries/model_entries.rs index bb20834b7b..3885247bdc 100644 --- a/crates/sqlbuild-python-planner/src/planner_entries/_helpers/entries/model_entries.rs +++ b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/entries/model_entries.rs @@ -478,7 +478,8 @@ pub(crate) fn plan_entries<'py>( &cursor_overrides, (start_override.clone(), end_override.clone()), )?; - let effective_full_refresh = resolve_model_full_refresh(&model, full_refresh)?; + let effective_full_refresh = + resolve_model_full_refresh(&model, full_refresh.applies(name))?; let (mut entry, entry_warnings) = plan_model_from_change(&ModelPlanRequest { model: model.clone(), snapshot: snapshot.clone(), diff --git a/crates/sqlbuild-python-planner/src/planner_entries/models.rs b/crates/sqlbuild-python-planner/src/planner_entries/models.rs index 367bb18a6f..4136195d77 100644 --- a/crates/sqlbuild-python-planner/src/planner_entries/models.rs +++ b/crates/sqlbuild-python-planner/src/planner_entries/models.rs @@ -1,5 +1,6 @@ //! Public records of planner entries. +use crate::planner_phases::models::FullRefreshRequest; use crate::planner_warehouse::models::PlannerReads; use pyo3::prelude::{Bound, Py, PyAny}; use pyo3::pyclass; @@ -31,7 +32,7 @@ pub struct PlanEntriesRequest<'py> { pub(crate) relations: Bound<'py, PyAny>, pub(crate) resolved_models: Bound<'py, PyAny>, pub(crate) cursor_overrides: Bound<'py, PyAny>, - pub(crate) full_refresh: bool, + pub(crate) full_refresh: FullRefreshRequest, pub(crate) build_inputs: Bound<'py, PyAny>, /// The effective worker limit execution will use. pub(crate) concurrency: i64, diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/planning/planning_boundary.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/planning/planning_boundary.rs index 7e12a8e95a..edfd40fd26 100644 --- a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/planning/planning_boundary.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/planning/planning_boundary.rs @@ -7,6 +7,7 @@ use crate::planner_phases::main::inputs::runtime_field::runtime_field; use crate::planner_phases::main::inputs::runtime_reads::runtime_reads; use crate::planner_phases::main::rebuilds::effective_target_config::effective_target_config; use crate::planner_phases::main::rebuilds::effectively_full_refreshed_model_names::effectively_full_refreshed_model_names; +use crate::planner_phases::models::FullRefreshRequest; use crate::planner_warehouse::main::inspection::gather_redirected_cursor_snapshots::gather_redirected_cursor_snapshots; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use sqlbuild_python_base::boundary::main::keywords::keywords; @@ -60,10 +61,8 @@ pub(crate) fn redirected_cursor_snapshots<'py>( ), ], )?; - let full_refresh = effectively_full_refreshed_model_names( - &project, - native_field(overrides, "full_refresh")?.is_truthy()?, - )?; + let full_refresh = + effectively_full_refreshed_model_names(&project, &FullRefreshRequest::of(overrides)?)?; gather_redirected_cursor_snapshots( py, Some(keywords( diff --git a/crates/sqlbuild-python-planner/src/planner_phases/_helpers/inputs/warehouse_state.rs b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/inputs/warehouse_state.rs index 4c5adbe3cd..dbc53ca41c 100644 --- a/crates/sqlbuild-python-planner/src/planner_phases/_helpers/inputs/warehouse_state.rs +++ b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/inputs/warehouse_state.rs @@ -12,6 +12,7 @@ use crate::planner_phases::_helpers::inputs::runtime::runtime_field; use crate::planner_phases::main::inputs::runtime_dialect::runtime_dialect; use crate::planner_phases::main::inputs::runtime_reads::runtime_reads; use crate::planner_phases::main::rebuilds::effectively_full_refreshed_model_names::effectively_full_refreshed_model_names; +use crate::planner_phases::models::FullRefreshRequest; use crate::planner_phases::models::{PlannerRuntime, PlannerWarehouseState}; use crate::planner_warehouse::main::inspection::warehouse_snapshot::warehouse_snapshot; use crate::planner_warehouse::models::MigrationFingerprintCache; @@ -51,7 +52,7 @@ pub(crate) fn gather_planner_warehouse_state<'py>( .into_any(); progress(runtime, "Inspecting warehouse state...")?; let selected_scope = class_field(scopes, |s: &PlannerScopeResolution| &s.selected_scope)?; - let full_refresh: bool = native_field(overrides, "full_refresh")?.extract()?; + let full_refresh: FullRefreshRequest = FullRefreshRequest::of(overrides)?; let config = runtime_field(runtime, |runtime| &runtime.project_config)?; let cursor_scope = native( py, @@ -102,7 +103,7 @@ pub(crate) fn gather_planner_warehouse_state<'py>( reads: reads.clone(), selected_keys: stale_keys, full_refresh: false, - full_refresh_model_names: effectively_full_refreshed_model_names(&project, full_refresh)?, + full_refresh_model_names: effectively_full_refreshed_model_names(&project, &full_refresh)?, on_progress: runtime_field(runtime, |runtime| &runtime.on_progress)?, deferred_locations: native_field(deferral, "deferred_locations")?, cursor_scope, @@ -155,7 +156,7 @@ pub(crate) fn gather_planner_warehouse_state<'py>( &ColumnMigrationInputs { full_refresh_model_names: effectively_full_refreshed_model_names( &project, - full_refresh, + &full_refresh, )?, physical_relations: physical.into_any(), source_columns: native_field(&inspection_relations, "source_warehouse_columns")?, diff --git a/crates/sqlbuild-python-planner/src/planner_phases/_helpers/rebuilds/execution_changes.rs b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/rebuilds/execution_changes.rs index 03d6b014b0..0bcc447d06 100644 --- a/crates/sqlbuild-python-planner/src/planner_phases/_helpers/rebuilds/execution_changes.rs +++ b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/rebuilds/execution_changes.rs @@ -8,6 +8,7 @@ use crate::planner_phases::_helpers::inputs::runtime::runtime_field; use crate::planner_phases::_helpers::inputs::runtime::warehouse_field; use crate::planner_phases::main::inputs::runtime_dialect::runtime_dialect; use crate::planner_phases::models::ChangeReconciliation; +use crate::planner_phases::models::FullRefreshRequest; use crate::planner_phases::models::ModelEntryResults; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::PyFrozenSet; @@ -120,7 +121,7 @@ pub(crate) fn build_planner_entry_results<'py>( relations, resolved_models: native_field(&reconciliation.resolved_actions, "models")?, cursor_overrides: native_field(overrides, "cursor_overrides")?, - full_refresh: native_field(overrides, "full_refresh")?.is_truthy()?, + full_refresh: FullRefreshRequest::of(overrides)?, build_inputs, concurrency, }) diff --git a/crates/sqlbuild-python-planner/src/planner_phases/constants.rs b/crates/sqlbuild-python-planner/src/planner_phases/constants.rs index 3524cae4a7..81ba8b930d 100644 --- a/crates/sqlbuild-python-planner/src/planner_phases/constants.rs +++ b/crates/sqlbuild-python-planner/src/planner_phases/constants.rs @@ -111,3 +111,7 @@ pub const WRITING: &str = "Writing the plan"; /// Time inside the plan that no step covers. pub const OTHER: &str = "Other"; +/// The planner override asking every model to rebuild from scratch. +pub const FULL_REFRESH_KEY: &str = "full_refresh"; +/// The planner override naming the models the named jobs rebuild from scratch. +pub const FULL_REFRESH_MODELS_KEY: &str = "full_refresh_models"; diff --git a/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/effectively_full_refreshed_model_names.rs b/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/effectively_full_refreshed_model_names.rs index 2e93d90235..335058e260 100644 --- a/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/effectively_full_refreshed_model_names.rs +++ b/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/effectively_full_refreshed_model_names.rs @@ -1,7 +1,9 @@ //! Names of models effectively full refreshed for this invocation. use crate::planner_phases::main::rebuilds::resolve_model_full_refresh::resolve_model_full_refresh; +use crate::planner_phases::models::FullRefreshRequest; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_python_project::compiled_views::main::compiled_models::model_name::model_name; use sqlbuild_python_project::compiled_views::main::compiled_models::name_object::name_object; use sqlbuild_python_project::compiled_views::main::project::compiled_models::compiled_models; use sqlbuild_python_project::planner_phases::main::frozenset::frozenset; @@ -9,12 +11,12 @@ use sqlbuild_python_project::planner_phases::main::frozenset::frozenset; /// Names of models effectively full refreshed for this invocation. pub fn effectively_full_refreshed_model_names<'py>( project: &Bound<'py, PyAny>, - cli_full_refresh: bool, + requested: &FullRefreshRequest, ) -> PyResult> { let mut names: Vec<_> = Vec::new(); for model in compiled_models(project)?.try_iter()? { let model = model?; - if resolve_model_full_refresh(&model, cli_full_refresh)? { + if resolve_model_full_refresh(&model, requested.applies(&model_name(&model)?))? { names.push(name_object(&model)?); } } diff --git a/crates/sqlbuild-python-planner/src/planner_phases/models.rs b/crates/sqlbuild-python-planner/src/planner_phases/models.rs index ebe7df27dc..26666a80a2 100644 --- a/crates/sqlbuild-python-planner/src/planner_phases/models.rs +++ b/crates/sqlbuild-python-planner/src/planner_phases/models.rs @@ -4,12 +4,15 @@ use crate::planner_phases::_helpers::timing::timings::STEPS; use crate::planner_phases::_helpers::timing::timings::timings_of; use crate::planner_phases::constants::MILLISECONDS; use crate::planner_phases::constants::PERCENT; +use crate::planner_phases::constants::{FULL_REFRESH_KEY, FULL_REFRESH_MODELS_KEY}; use crate::planner_warehouse::models::PlannerReads; use pyo3::prelude::{Bound, Py, PyAny}; +use pyo3::prelude::{PyAnyMethods, PyResult}; use pyo3::pyclass; use serde_json::{Value, json}; use sqlbuild_python_base::native_host::models::Dialect; -use std::collections::BTreeMap; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use std::collections::{BTreeMap, HashSet}; use std::time::Instant; /// The plan output's fields and metadata entries, each set before the output is built. @@ -198,3 +201,29 @@ impl PlanTimings { }) } } + +/// Which models the invocation asks to rebuild from scratch: every model with `--full-refresh`, or +/// the models of the named jobs that set `full_refresh`. +#[derive(Clone, Debug, Default)] +pub struct FullRefreshRequest { + pub all: bool, + pub models: HashSet, +} + +impl FullRefreshRequest { + /// The request of planner overrides' `full_refresh` and `full_refresh_models`. + pub fn of(overrides: &Bound<'_, PyAny>) -> PyResult { + Ok(Self { + all: native_field(overrides, FULL_REFRESH_KEY)?.is_truthy()?, + models: native_field(overrides, FULL_REFRESH_MODELS_KEY)? + .try_iter()? + .map(|name| name?.extract()) + .collect::>()?, + }) + } + + /// Whether the invocation asks to rebuild `model` from scratch. + pub fn applies(&self, model: &str) -> bool { + self.all || self.models.contains(model) + } +} diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/snapshot.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/snapshot.rs index a580b7064f..ed63ed5361 100644 --- a/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/snapshot.rs +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/snapshot.rs @@ -1,6 +1,7 @@ //! Gather a frozen point-in-time warehouse snapshot for planning. use crate::planner_phases::main::rebuilds::effectively_full_refreshed_model_names::effectively_full_refreshed_model_names; +use crate::planner_phases::models::FullRefreshRequest; use crate::planner_warehouse::_helpers::inspection::cursors::{CursorInputs, cursor_snapshots}; use crate::planner_warehouse::constants::{ FUNCTION_NODE_TYPES, METADATA_NAME_FILTER_LIMIT, MIGRATE_FROM_CONFIG_KEY, @@ -705,7 +706,13 @@ pub(crate) fn warehouse_snapshot<'py>( (&states.fingerprints, &relevant), )?; let full_refresh_names = if request.full_refresh_model_names.is_none() { - effectively_full_refreshed_model_names(project, request.full_refresh)? + effectively_full_refreshed_model_names( + project, + &FullRefreshRequest { + all: request.full_refresh, + ..FullRefreshRequest::default() + }, + )? } else { request.full_refresh_model_names.clone() }; diff --git a/crates/sqlbuild-python-project/src/compiled_project/_helpers/job_catalog.rs b/crates/sqlbuild-python-project/src/compiled_project/_helpers/job_catalog.rs index ce70fdee71..3a1980033a 100644 --- a/crates/sqlbuild-python-project/src/compiled_project/_helpers/job_catalog.rs +++ b/crates/sqlbuild-python-project/src/compiled_project/_helpers/job_catalog.rs @@ -9,6 +9,7 @@ use pyo3::exceptions::PyValueError; use pyo3::prelude::{PyResult, Python}; use sqlbuild_analysis::graph::main::parse_selector::parse_selector; use sqlbuild_analysis::graph::models::{JobSelection, ParsedSelector}; +use sqlbuild_analysis::job_settings::models::{DeclaredRun, JobRetry}; use sqlbuild_cli::native_commands::main::effective_target::effective_target; use sqlbuild_discovery::orchestration_files::models::{OrchestrationInventory, RunFile}; @@ -20,11 +21,11 @@ use sqlbuild_python_front::discovery::main::sources::orchestration_declarations: /// Each job and group's selection, and the names of those a schedule or sensor starts. type Catalog = (BTreeMap, BTreeSet); -/// The running command's project directory, its `--target` and, once loaded, its jobs. +/// The running command's project directory, its `--target` and, once loaded, its jobs and their settings. struct JobScope { project_dir: PathBuf, selected_target: Option, - catalog: Option, + catalog: Option<(Catalog, BTreeMap)>, } thread_local! { @@ -82,29 +83,78 @@ pub(crate) fn ordered_jobs( } pub(crate) fn job_catalog(py: Python<'_>) -> PyResult> { + Ok(loaded_jobs(py)?.map(|(catalog, _)| catalog)) +} + +/// Each job's `[settings]` and `[retry]` and each group's jobs declared for the command's target; none outside a command. +pub(crate) fn job_declarations(py: Python<'_>) -> PyResult>> { + Ok(loaded_jobs(py)?.map(|(_, declared)| declared)) +} + +fn loaded_jobs(py: Python<'_>) -> PyResult)>> { let Some(scope) = JOB_SCOPE.with(|scope| { scope.borrow().as_ref().map(|scope| match &scope.catalog { - Some(catalog) => Ok(catalog.clone()), + Some(loaded) => Ok(loaded.clone()), None => Err((scope.project_dir.clone(), scope.selected_target.clone())), }) }) else { return Ok(None); }; let (project_dir, selected_target): (PathBuf, Option) = match scope { - Ok(catalog) => return Ok(Some(catalog)), + Ok(loaded) => return Ok(Some(loaded)), Err(scope) => scope, }; let target: Option = effective_target(&project_dir, selected_target).map_err(PyValueError::new_err)?; let inventory: OrchestrationInventory = orchestration_declarations(project_dir.clone().into_pyobject(py)?.as_any())?; - let catalog: Catalog = inventory_catalog(&inventory, target.as_deref()); + let loaded = ( + inventory_catalog(&inventory, target.as_deref()), + declared_runs(&inventory, target.as_deref()), + ); JOB_SCOPE.with(|scope| { if let Some(scope) = scope.borrow_mut().as_mut() { - scope.catalog = Some(catalog.clone()); + scope.catalog = Some(loaded.clone()); } }); - Ok(Some(catalog)) + Ok(Some(loaded)) +} + +/// Each job's settings and retry, and each group's jobs, declared for `target`. +pub(crate) fn declared_runs( + inventory: &OrchestrationInventory, + target: Option<&str>, +) -> BTreeMap { + let declared_for = |targets: &[String]| { + targets.is_empty() || target.is_some_and(|target| targets.iter().any(|name| name == target)) + }; + let mut declared: BTreeMap = BTreeMap::new(); + for (_, job) in &inventory.jobs { + if declared_for(&job.run.targets) { + declared.insert( + job.run.name.value.clone(), + DeclaredRun::Job { + settings: job.settings.clone(), + retry: job.run.retry.map(|retry| JobRetry { + attempts: retry.attempts, + delay_seconds: retry.delay_seconds, + }), + }, + ); + } + } + for (_, group) in &inventory.groups { + if declared_for(&group.run.targets) { + let members: Vec = group + .jobs + .iter() + .filter(|job| declared.contains_key(&job.value)) + .map(|job| job.value.clone()) + .collect(); + declared.insert(group.run.name.value.clone(), DeclaredRun::Group { members }); + } + } + declared } /// Each job's selector tokens and each group's members declared for `target`, and the names a schedule or sensor starts there. diff --git a/crates/sqlbuild-python-project/src/compiled_project/constants.rs b/crates/sqlbuild-python-project/src/compiled_project/constants.rs index d72ce96a40..a0c7f959ec 100644 --- a/crates/sqlbuild-python-project/src/compiled_project/constants.rs +++ b/crates/sqlbuild-python-project/src/compiled_project/constants.rs @@ -24,3 +24,11 @@ pub(crate) const SENSOR_RUN_ARGUMENT: &str = "run"; pub const CONTRACT_KEY: &str = "contract"; pub const INCREMENTAL_MODE_KEY: &str = "incremental_mode"; + +/// The build settings a job run's `--changed` reach reads, and the reach every job run uses. +pub const CHANGED_SETTING: &str = "changed"; +pub const REACH_SETTING: &str = "reach"; +/// A job run reaches its whole selection: `reach` only stops builds that are not a job's run. +pub const JOB_RUN_REACH: &str = "all"; +/// What a job run with `changed: True` counts when `[changes] counts` is not set: every change. +pub const JOB_RUN_COUNTS: [&str; 2] = ["definition", "source_data"]; diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/job_declarations.rs b/crates/sqlbuild-python-project/src/compiled_project/main/job_declarations.rs new file mode 100644 index 0000000000..62e96c1415 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_project/main/job_declarations.rs @@ -0,0 +1,12 @@ +//! Each job's `[settings]` and `[retry]` and each group's jobs declared for the command's target. + +use crate::compiled_project::_helpers::job_catalog; +use pyo3::prelude::{PyResult, Python}; +use sqlbuild_analysis::job_settings::models::DeclaredRun; +use std::collections::BTreeMap; + +/// Each job's `[settings]` and `[retry]` and each group's jobs declared for the command's target; none outside a command. +#[inline] +pub fn job_declarations(py: Python<'_>) -> PyResult>> { + job_catalog::job_declarations(py) +} diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/mod.rs b/crates/sqlbuild-python-project/src/compiled_project/main/mod.rs index efde8565a4..8787b16bcd 100644 --- a/crates/sqlbuild-python-project/src/compiled_project/main/mod.rs +++ b/crates/sqlbuild-python-project/src/compiled_project/main/mod.rs @@ -5,6 +5,7 @@ pub mod entry_schema_fact; pub mod family_fact; pub mod graph_with_jobs; pub mod install; +pub mod job_declarations; pub mod job_names; pub mod materialization; pub mod model_facts; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs index cba13215c6..de9042278b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs @@ -255,7 +255,7 @@ pub(crate) fn prepare_build_execution<'py>( }) .and_then(|value| pyo3::Bound::new(py, value)) .map(pyo3::Bound::into_any)?; - let concurrency = context.concurrency()?.limit; + let concurrency = connection_limit(context.concurrency()?.limit, pipeline)?; if context.verbose { write_run_context(context, pipeline)?; } else { @@ -310,6 +310,28 @@ pub(crate) fn prepare_build_execution<'py>( }) } +/// The build's connections: its limit, or no more than its named jobs' limits add up to when each +/// planned node belongs to a limited job, so a small job never opens connections it cannot use. +fn connection_limit(limit: i64, pipeline: &Bound<'_, PyAny>) -> PyResult { + let limits: Bound<'_, PyAny> = native_field(pipeline, "job_limits")?; + if !limits.is_truthy()? { + return Ok(limit); + } + let node_jobs: Bound<'_, PyAny> = native_field(pipeline, "node_jobs")?; + for key in + native_field(&native_field(pipeline, "plan_output")?, "execution_order")?.try_iter()? + { + if !node_jobs.contains(key?)? { + return Ok(limit); + } + } + let mut total: i64 = 0; + for value in limits.call_method0("values")?.try_iter()? { + total = total.saturating_add(value?.extract::()?); + } + Ok(limit.min(total.max(1))) +} + fn verbose_callback<'py>( context: &BuildContext<'py>, callbacks: &Bound<'py, PyAny>, @@ -375,6 +397,8 @@ pub(crate) fn execute_build_plan<'py>( ("end_cursor_int", end_int.clone()), ("use_color", boolean(py, context.use_color)), ("providers", providers.clone()), + ("job_limits", native_field(pipeline, "job_limits")?), + ("node_jobs", native_field(pipeline, "node_jobs")?), ( "test_timeouts", timeout_record( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/planning.rs index 9cbf89ba27..65595cdf52 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/planning.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/planning.rs @@ -96,6 +96,8 @@ fn request_options<'py>(context: &BuildContext<'py>) -> PyResult( ) -> PyResult> { let py: Python<'py> = args.py(); let invocation = Invocation::new(args)?; + apply_named_job_settings(args, &invocation.select.extract::>()?)?; let _secrets = MissingSecretsScope::enter( args.getattr_opt(ALLOW_MISSING_SECRETS_DEST)? .is_some_and(|allowed| allowed.is_truthy().unwrap_or(false)), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/mod.rs index 7a3ebc8d8c..71f400a1b8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/mod.rs @@ -3,6 +3,7 @@ pub(crate) mod dispatch; pub(crate) mod entry; pub(crate) mod failure_cause; pub(crate) mod machine_output; +pub(crate) mod named_job_settings; pub(crate) mod namespace; pub(crate) mod observability; pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/named_job_settings.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/named_job_settings.rs new file mode 100644 index 0000000000..754cadd525 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/named_job_settings.rs @@ -0,0 +1,96 @@ +//! A build or plan that names jobs takes their build-wide settings where its own command line is silent. + +use std::collections::BTreeMap; + +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::PyString; +use serde_json::Value; +use sqlbuild_cli::argv::main::accepted_settings_arguments::accepted_settings_arguments; + +use crate::bindings::_helpers::cli::argv::parse_cli_invocation; +use crate::bindings::_helpers::compiled_project::job_settings::named_job_build_settings; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// The commands whose selections take the settings of the jobs they name. +const JOB_SETTING_COMMANDS: [&str; 2] = ["build", "plan"]; + +/// The namespace `sqb ` parses to. +fn parsed<'py>( + py: Python<'py>, + command: &str, + arguments: &[String], +) -> PyResult> { + let argv: Vec> = std::iter::once(command) + .chain(arguments.iter().map(String::as_str)) + .map(|argument| PyString::new(py, argument)) + .collect(); + let args: Bound<'py, PyAny> = native_field(&parse_cli_invocation(py, argv, false)?, "args")?; + if args.is_none() { + return Err(PyValueError::new_err(format!( + "job settings do not parse as sqb {command} options: {}", + arguments.join(" ") + ))); + } + Ok(args) +} + +/// Set on `args` each build-wide setting of the jobs `selectors` name that its command line left +/// at the default, exactly as the same flags typed on the command line would. +pub(crate) fn apply_named_job_settings( + args: &Bound<'_, PyAny>, + selectors: &[String], +) -> PyResult<()> { + let py: Python<'_> = args.py(); + let Some(command) = args.getattr("command")?.extract::>()? else { + return Ok(()); + }; + if !JOB_SETTING_COMMANDS.contains(&command.as_str()) + || selectors.is_empty() + || !args.getattr("job_settings")?.is_truthy()? + { + return Ok(()); + } + let raw_project_dir: Bound<'_, PyAny> = args.getattr("project_dir")?; + let path = py.import("pathlib")?.getattr("Path")?; + let project_dir: Bound<'_, PyAny> = if raw_project_dir.is_none() { + path.call_method0("cwd")? + } else { + path.call1((raw_project_dir,))?.call_method0("resolve")? + }; + let target: Option = args.getattr("target")?.extract()?; + let settings: BTreeMap = + named_job_build_settings(py, selectors, (&project_dir, target))?; + if settings.is_empty() { + return Ok(()); + } + let defaults: Bound<'_, PyAny> = parsed(py, &command, &[])?; + let mut silent: BTreeMap = BTreeMap::new(); + for (name, value) in settings { + if !args.hasattr(name.as_str())? + || args + .getattr(name.as_str())? + .eq(defaults.getattr(name.as_str())?)? + { + silent.insert(name, value); + } + } + let flags: Vec = + accepted_settings_arguments(&command, &silent).map_err(PyValueError::new_err)?; + if flags.is_empty() { + return Ok(()); + } + let applied: Bound<'_, PyAny> = parsed(py, &command, &flags)?; + for (name, value) in applied + .getattr("__dict__")? + .call_method0("items")? + .try_iter()? + .map(|item| item.and_then(|item| item.extract::<(String, Bound<'_, PyAny>)>())) + .collect::>>()? + { + if !value.eq(defaults.getattr(name.as_str())?)? { + args.setattr(name.as_str(), value)?; + } + } + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/job_node_settings.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/job_node_settings.rs new file mode 100644 index 0000000000..d4a9405235 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/job_node_settings.rs @@ -0,0 +1,260 @@ +//! The settings each planned node takes from the jobs its selection names, applied to the plan. + +use std::collections::{BTreeMap, BTreeSet}; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyBool, PyDict, PyDictMethods, PyFrozenSet, PyTuple}; +use serde_json::Value; +use sqlbuild_analysis::assembly::project::types::ObjectKey; +use sqlbuild_analysis::job_settings::constants::CONFLICT_HELP; +use sqlbuild_analysis::job_settings::errors::{JobSettingConflict, NodeJobSettingsError}; +use sqlbuild_analysis::job_settings::main::named_jobs::named_jobs; +use sqlbuild_analysis::job_settings::main::node_job_settings::node_job_settings; +use sqlbuild_analysis::job_settings::models::{DeclaredRun, NodeJobSettings}; + +use crate::bindings::_helpers::compiled_project::job_settings::job_setting_conflict; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::main::fields::replaced::replaced; +use sqlbuild_python_project::compiled_project::main::graph_with_jobs::graph_with_jobs; +use sqlbuild_python_project::compiled_project::main::job_declarations::job_declarations; +use sqlbuild_python_project::graph::main::nodes::native_key::native_key; +use sqlbuild_python_project::graph::models::KeyCache; +use sqlbuild_python_project::graph::models::NativeProjectGraph; + +const MODEL_KIND: &str = "model"; +const FULL_REFRESH: &str = "full_refresh"; +const RUN_TESTS: &str = "run_tests"; +const IGNORE_AUDIT_FAILURES: &str = "ignore_audit_failures"; + +/// What the planned build takes from its named jobs, or nothing when it names none. +pub(crate) struct PlannedJobSettings { + settings: NodeJobSettings, +} + +fn strings(value: &Bound<'_, PyAny>) -> PyResult> { + value.try_iter()?.map(|item| item?.extract()).collect() +} + +/// The named jobs' settings for each node `options` selects; `None` without named jobs, with +/// `--no-job-settings`, or when the selection fails, which planning then reports itself. +pub(crate) fn planned_job_settings( + project: &Bound<'_, PyAny>, + options: &Bound<'_, PyAny>, +) -> PyResult> { + let py: Python<'_> = project.py(); + if !native_field(options, "job_settings")?.is_truthy()? { + return Ok(None); + } + let select: Vec = strings(&native_field(options, "select")?)?; + let exclude: Vec = strings(&native_field(options, "exclude")?)?; + let Some(declared): Option> = job_declarations(py)? else { + return Ok(None); + }; + if named_jobs(&select, &declared).is_empty() { + return Ok(None); + } + let native: Bound<'_, NativeProjectGraph> = + native_field(project, "lineage_graph")?.cast_into()?; + let native = native.get(); + let all: Vec = select.iter().chain(&exclude).cloned().collect(); + let graph = graph_with_jobs(py, native.graph(), &all, &[])?; + match node_job_settings(&graph, (&select, &exclude), &declared) { + Ok(settings) => Ok(Some(PlannedJobSettings { settings })), + Err(NodeJobSettingsError::Selector(_)) => Ok(None), + Err(NodeJobSettingsError::Conflict(conflict)) => Err(job_setting_conflict(py, &conflict)), + } +} + +impl PlannedJobSettings { + fn value(&self, key: &ObjectKey, setting: &str) -> Option<&Value> { + self.settings + .values + .get(key) + .and_then(|values| values.get(setting)) + } + + /// The models the named jobs rebuild from scratch. + pub(crate) fn full_refresh_models<'py>( + &self, + py: Python<'py>, + ) -> PyResult> { + let names: Vec<&String> = self + .settings + .values + .iter() + .filter(|(key, values)| { + key.0 == MODEL_KIND && values.get(FULL_REFRESH) == Some(&Value::Bool(true)) + }) + .map(|(key, _)| &key.1) + .collect(); + PyFrozenSet::new(py, names) + } + + /// Each named job's limit and each planned node's limited jobs, a SQL test taking its models' + /// jobs; none when `--concurrency` overrides them. + pub(crate) fn job_limits<'py>( + &self, + plan: &Bound<'py, PyAny>, + cli_concurrency: &Bound<'py, PyAny>, + ) -> PyResult<(Bound<'py, PyDict>, Bound<'py, PyDict>)> { + let py: Python<'py> = plan.py(); + let limits: Bound<'py, PyDict> = PyDict::new(py); + let node_jobs: Bound<'py, PyDict> = PyDict::new(py); + if !cli_concurrency.is_none() || self.settings.job_limits.is_empty() { + return Ok((limits, node_jobs)); + } + for (job, limit) in &self.settings.job_limits { + limits.set_item(job, limit)?; + } + let limited = |keys: &[ObjectKey]| -> Vec { + let mut jobs: Vec = Vec::new(); + for job in keys + .iter() + .filter_map(|key| self.settings.node_jobs.get(key)) + .flatten() + .filter(|job| self.settings.job_limits.contains_key(*job)) + { + if !jobs.contains(job) { + jobs.push(job.clone()); + } + } + jobs + }; + let cache = KeyCache::new(py)?; + for key in self.settings.node_jobs.keys() { + let jobs: Vec = limited(std::slice::from_ref(key)); + if !jobs.is_empty() { + node_jobs.set_item(cache.key(key)?, PyTuple::new(py, jobs)?)?; + } + } + for entry in native_field(plan, "test_entries")?.try_iter()? { + let entry = entry?; + let mut keys: Vec = vec![native_key(&native_field(&entry, "key")?)?]; + for dep in native_field(&entry, "scope_deps")?.try_iter()? { + keys.push(native_key(&dep?)?); + } + let jobs: Vec = limited(&keys); + if !jobs.is_empty() { + node_jobs.set_item(native_field(&entry, "key")?, PyTuple::new(py, jobs)?)?; + } + } + Ok((limits, node_jobs)) + } + + /// The plan without the SQL tests whose models' jobs set `run_tests = false`, and with each audit + /// of a named job's node ignoring failures as that job says, unless the command line decided. + pub(crate) fn apply_to_plan<'py>( + &self, + plan: &Bound<'py, PyAny>, + cli_ignore_audit_failures: &Bound<'py, PyAny>, + ) -> PyResult> { + let py: Python<'py> = plan.py(); + let mut skipped: BTreeSet = BTreeSet::new(); + let mut tests: Vec> = Vec::new(); + for entry in native_field(plan, "test_entries")?.try_iter()? { + let entry = entry?; + let key: ObjectKey = native_key(&native_field(&entry, "key")?)?; + if self.skips_test(&key, &native_field(&entry, "scope_deps")?)? { + skipped.insert(key); + } else { + tests.push(entry); + } + } + let mut changes: Vec<(&str, Bound<'py, PyAny>)> = Vec::new(); + if !skipped.is_empty() { + let order: Vec> = native_field(plan, "execution_order")? + .try_iter()? + .filter_map(|key| match key { + Ok(key) => match native_key(&key) { + Ok(native) if skipped.contains(&native) => None, + Ok(_) => Some(Ok(key)), + Err(error) => Some(Err(error)), + }, + Err(error) => Some(Err(error)), + }) + .collect::>()?; + changes.push(("test_entries", PyTuple::new(py, tests)?.into_any())); + changes.push(("execution_order", PyTuple::new(py, order)?.into_any())); + } + if cli_ignore_audit_failures.is_none() { + let mut audits: Vec> = Vec::new(); + let mut changed = false; + for entry in native_field(plan, "audit_entries")?.try_iter()? { + let entry = entry?; + match self.audit_ignores(&entry)? { + Some(ignored) + if native_field(&entry, "ignore_failure")?.is_truthy()? != ignored => + { + changed = true; + audits.push(replaced( + &entry, + &[( + "ignore_failure", + PyBool::new(py, ignored).to_owned().into_any(), + )], + )?); + } + _ => audits.push(entry), + } + } + if changed { + changes.push(("audit_entries", PyTuple::new(py, audits)?.into_any())); + } + } + if changes.is_empty() { + return Ok(plan.clone()); + } + replaced(plan, &changes) + } + + /// Whether a SQL test's models' jobs turn tests off; jobs that disagree conflict. + fn skips_test(&self, test: &ObjectKey, scope_deps: &Bound<'_, PyAny>) -> PyResult { + let mut keys: Vec = vec![test.clone()]; + for dep in scope_deps.try_iter()? { + keys.push(native_key(&dep?)?); + } + let mut seen: Option<(&String, &Value)> = None; + for key in &keys { + let Some(value) = self.value(key, RUN_TESTS) else { + continue; + }; + let job: &String = self + .settings + .node_jobs + .get(key) + .and_then(|jobs| jobs.first()) + .unwrap_or(&key.1); + match seen { + Some((other, earlier)) if earlier != value => { + return Err(job_setting_conflict( + scope_deps.py(), + &JobSettingConflict { + message: format!( + "test {} covers nodes of jobs {other} and {job}, which set run_tests differently ({earlier} and {value})", + test.1 + ), + help: CONFLICT_HELP, + }, + )); + } + Some(_) => {} + None => seen = Some((job, value)), + } + } + Ok(seen.is_some_and(|(_, value)| value == &Value::Bool(false))) + } + + /// The named job's `ignore_audit_failures` for the node an audit is attached to, if it sets one. + fn audit_ignores(&self, entry: &Bound<'_, PyAny>) -> PyResult> { + let kind: Bound<'_, PyAny> = native_field(entry, "attached_target_kind")?; + let name: Bound<'_, PyAny> = native_field(entry, "attached_target_name")?; + if kind.is_none() || name.is_none() { + return Ok(None); + } + let kind: String = kind.getattr("value")?.extract()?; + let key: ObjectKey = (kind, name.extract()?); + Ok(self + .value(&key, IGNORE_AUDIT_FAILURES) + .and_then(Value::as_bool)) + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/mod.rs index 4a1a0e691a..b6d5c2096f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/mod.rs @@ -1,3 +1,4 @@ +pub(crate) mod job_node_settings; pub(crate) mod plan_pipeline; pub(crate) mod plan_result; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs index c9b9c1fcc7..49613a5b8a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs @@ -10,6 +10,9 @@ use sqlbuild_python_compile::compile_driver::main::stages::resolve_deferred_targ use sqlbuild_python_compile::compile_driver::main::setup::build_project_graph_from_compiled_project::build_project_graph_from_compiled_project; use sqlbuild_python_compile::compile_driver::main::setup::build_python_run_plan_outputs::build_python_run_plan_outputs; use sqlbuild_python_compile::compile_driver::main::setup::python_read_source_names_for::python_read_source_names_for; +use crate::bindings::_helpers::compile_driver::job_node_settings::{ + PlannedJobSettings, planned_job_settings, +}; use sqlbuild_analysis::planner_scope::constants::SOURCE; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; @@ -155,20 +158,25 @@ pub(crate) fn pipeline_result<'py>( ("sql_test_selection", sql_test_selection), ], )?; - let overrides: Bound<'py, PyAny> = new_record_with( - py, - RecordId::PlannerOverrides, - &option_fields( - options, - &[ - "cursor_overrides", - "full_refresh", - "reload_sources", - "max_microbatches", - "concurrency", - ], - )?, + let job_settings: Option = planned_job_settings(project, options)?; + let mut override_fields: Vec<(&str, Bound<'py, PyAny>)> = option_fields( + options, + &[ + "cursor_overrides", + "full_refresh", + "reload_sources", + "max_microbatches", + "concurrency", + ], )?; + if let Some(job_settings) = &job_settings { + override_fields.push(( + "full_refresh_models", + job_settings.full_refresh_models(py)?.into_any(), + )); + } + let overrides: Bound<'py, PyAny> = + new_record_with(py, RecordId::PlannerOverrides, &override_fields)?; let mut deferral_fields: Vec<(&str, Bound<'py, PyAny>)> = vec![ ("deferred_locations", deferred_locations), ("deferred_relations", deferred_relations), @@ -218,6 +226,21 @@ pub(crate) fn pipeline_result<'py>( ], )?), )?; + let mut job_fields: Vec<(&str, Bound<'py, PyAny>)> = Vec::new(); + let plan_output: Bound<'py, PyAny> = match &job_settings { + Some(job_settings) => { + let plan_output: Bound<'py, PyAny> = job_settings.apply_to_plan( + &plan_output, + &native_field(options, "cli_ignore_audit_failures")?, + )?; + let (limits, node_jobs) = + job_settings.job_limits(&plan_output, &native_field(options, "concurrency")?)?; + job_fields.push(("job_limits", limits.into_any())); + job_fields.push(("node_jobs", node_jobs.into_any())); + plan_output + } + None => plan_output, + }; let custom_materializations: Bound<'py, PyAny> = host_entry(py, UserCodeHost::LoadCustomMaterializations)? .call1((native_field(discovered, "materialization_files")?,))?; @@ -233,23 +256,21 @@ pub(crate) fn pipeline_result<'py>( ], )?), )?; - new_record_with( - py, - RecordId::CompilePipelineResult, - &[ - ("project", project.clone()), - ("plan_output", native_field(&python_outputs, "plan_output")?), - ("custom_materializations", custom_materializations), - ( - "python_node_names", - native_field(&python_outputs, "selected_python_node_names")?, - ), - ( - "python_plan_entries", - native_field(&python_outputs, "python_plan_entries")?, - ), - ], - ) + let mut result_fields: Vec<(&str, Bound<'py, PyAny>)> = vec![ + ("project", project.clone()), + ("plan_output", native_field(&python_outputs, "plan_output")?), + ("custom_materializations", custom_materializations), + ( + "python_node_names", + native_field(&python_outputs, "selected_python_node_names")?, + ), + ( + "python_plan_entries", + native_field(&python_outputs, "python_plan_entries")?, + ), + ]; + result_fields.extend(job_fields); + new_record_with(py, RecordId::CompilePipelineResult, &result_fields) } /// `(name, options.name)` for each of `names`. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/job_settings.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/job_settings.rs new file mode 100644 index 0000000000..a41a21647d --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/job_settings.rs @@ -0,0 +1,78 @@ +//! The settings named jobs give a command: the build-wide ones as the flags the command line left out. + +use std::collections::BTreeMap; +use std::path::PathBuf; + +use pyo3::PyErr; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods}; +use serde_json::Value; +use sqlbuild_analysis::job_settings::constants::JOB_SETTING_CONFLICT_CODE; +use sqlbuild_analysis::job_settings::errors::JobSettingConflict; +use sqlbuild_analysis::job_settings::main::build_job_settings::build_job_settings; +use sqlbuild_analysis::job_settings::main::named_jobs::named_jobs; + +use sqlbuild_python_base::records::main::construct::error_class::error_class; +use sqlbuild_python_base::records::types::ErrorId; +use sqlbuild_python_front::discovery::main::project::configured_changes::configured_changes; +use sqlbuild_python_project::compiled_project::constants::{ + CHANGED_SETTING, JOB_RUN_COUNTS, JOB_RUN_REACH, REACH_SETTING, +}; +use sqlbuild_python_project::compiled_project::main::job_declarations::job_declarations; +use sqlbuild_python_project::compiled_project::main::with_job_scope::with_job_scope; + +/// A job run's `--changed` settings: its whole selection, and source data counted unless configured. +pub(crate) fn job_run_settings( + project_dir: &Bound<'_, PyAny>, + mut settings: BTreeMap, +) -> PyResult> { + let Some(changed) = settings.get(CHANGED_SETTING).cloned() else { + return Ok(settings); + }; + if changed == Value::Bool(false) { + return Ok(settings); + } + settings.insert(REACH_SETTING.to_owned(), Value::from(JOB_RUN_REACH)); + if changed == Value::Bool(true) && configured_changes(project_dir)?.counts.is_none() { + settings.insert( + CHANGED_SETTING.to_owned(), + Value::from(JOB_RUN_COUNTS.to_vec()), + ); + } + Ok(settings) +} + +/// The build-wide settings the jobs `selectors` name give a build, as job runs take them; a +/// conflict between named jobs is a `CliUserError` naming them. +pub(crate) fn named_job_build_settings( + py: Python<'_>, + selectors: &[String], + (project_dir, selected_target): (&Bound<'_, PyAny>, Option), +) -> PyResult> { + let directory: PathBuf = project_dir.extract()?; + let Some(declared) = with_job_scope((directory, selected_target), || job_declarations(py))? + else { + return Ok(BTreeMap::new()); + }; + let named: Vec = named_jobs(selectors, &declared); + if named.is_empty() { + return Ok(BTreeMap::new()); + } + let agreed: BTreeMap = build_job_settings(&named, &declared) + .map_err(|conflict| job_setting_conflict(py, &conflict))?; + job_run_settings(project_dir, agreed) +} + +/// `CliUserError` for job settings that cannot apply together. +pub(crate) fn job_setting_conflict(py: Python<'_>, conflict: &JobSettingConflict) -> PyErr { + let raised = (|| { + let keywords: Bound<'_, PyDict> = PyDict::new(py); + keywords.set_item("code", JOB_SETTING_CONFLICT_CODE)?; + keywords.set_item("help", conflict.help)?; + error_class(py, ErrorId::CliUser)?.call((conflict.message.as_str(),), Some(&keywords)) + })(); + match raised { + Ok(error) => PyErr::from_value(error), + Err(error) => error, + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/mod.rs index 7444bd0374..878e48e2ae 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/mod.rs @@ -1 +1,2 @@ +pub(crate) mod job_settings; pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/constants.rs index ef9d973459..d132719e93 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/constants.rs @@ -22,14 +22,6 @@ pub(crate) const SOURCE_NODE_KIND: &str = "source"; /// DuckDB's message when another process, such as a running build, holds the database file. pub(crate) const DUCKDB_LOCK_CONFLICT: &str = "Conflicting lock is held"; -/// The build settings a job run's `--changed` reach reads, and the reach every job run uses. -pub(crate) const CHANGED_SETTING: &str = "changed"; -pub(crate) const REACH_SETTING: &str = "reach"; -/// A job run reaches its whole selection: `reach` only stops builds that are not a job's run. -pub(crate) const JOB_RUN_REACH: &str = "all"; -/// What a job run with `changed: True` counts when `[changes] counts` is not set: every change. -pub(crate) const JOB_RUN_COUNTS: [&str; 2] = ["definition", "source_data"]; - /// A SQL unit test's published kind; a frozen run selects it back with `test:`. pub(crate) const SQL_TEST_NODE_KIND: &str = "sql_test"; /// The setting-row field the published providers document leaves out: secret matching reads it, the UI does not. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs index f55b4f5f2f..5397a0b567 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs @@ -29,14 +29,11 @@ use crate::bindings::_helpers::cli::dispatch::warehouse_scope; use crate::bindings::_helpers::command_runtime::warehouses::WarehouseInputs; use crate::bindings::_helpers::command_runtime::warehouses::configured_warehouses; use crate::bindings::_helpers::compile_driver::rules_at_load::RulesAtLoadScope; +use crate::bindings::_helpers::compiled_project::job_settings::job_run_settings; use crate::bindings::_helpers::cost_report::deferred::collect_deferred_cost_after_finish; use crate::bindings::_helpers::freshness::freshness_command::run_freshness_command; use crate::bindings::_helpers::location_command::command::compile_snapshot; -use crate::bindings::_helpers::location_command::constants::CHANGED_SETTING; use crate::bindings::_helpers::location_command::constants::COMMAND_IMPLEMENTATIONS; -use crate::bindings::_helpers::location_command::constants::JOB_RUN_COUNTS; -use crate::bindings::_helpers::location_command::constants::JOB_RUN_REACH; -use crate::bindings::_helpers::location_command::constants::REACH_SETTING; use crate::bindings::_helpers::location_command::constants::SQL_TEST_NODE_KIND; use crate::bindings::_helpers::query_command::project::ProjectAdapter; use crate::bindings::_helpers::seed_command::command::run_seed; @@ -49,7 +46,6 @@ use sqlbuild_python_base::native_host::main::host_entry::host_entry as entry; use sqlbuild_python_base::records::main::fields::field::field; use sqlbuild_python_base::records::main::fields::replaced::replaced; use sqlbuild_python_front::compile_attachment::main::build_effective_connection_config::build_effective_connection_config; -use sqlbuild_python_front::discovery::main::project::configured_changes::configured_changes; use sqlbuild_python_output::recall::main::write_stderr::write_stderr; use sqlbuild_python_output::recall::models::InvocationRecall; use sqlbuild_python_project::compiled_project::constants::SQL_TEST_KEY_PREFIX; @@ -127,27 +123,6 @@ fn upstream_nodes(snapshot: &NativeOrchestrationSnapshot) -> BTreeMap, - mut settings: BTreeMap, -) -> PyResult> { - let Some(changed) = settings.get(CHANGED_SETTING).cloned() else { - return Ok(settings); - }; - if changed == Value::Bool(false) { - return Ok(settings); - } - settings.insert(REACH_SETTING.to_owned(), Value::from(JOB_RUN_REACH)); - if changed == Value::Bool(true) && configured_changes(project_dir)?.counts.is_none() { - settings.insert( - CHANGED_SETTING.to_owned(), - Value::from(JOB_RUN_COUNTS.to_vec()), - ); - } - Ok(settings) -} - /// The frozen command's request and warehouse scope; build carries the verified inputs and compiled project in its request. fn command_request<'py>( py: Python<'py>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs b/crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs index d9fc02c4d7..56da07d274 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs @@ -340,6 +340,11 @@ fn compile_pipeline<'py>( "ignore_audit_failures", resolve_ignore_audit_failures(request, &invocation.discovered, &planned_target)?, )?; + options.set_item( + "cli_ignore_audit_failures", + native_field(request, "ignore_audit_failures")?, + )?; + options.set_item("job_settings", native_field(request, "job_settings")?)?; let pipeline_options: Bound<'py, PyAny> = new_record(py, RecordId::CompilePipelineOptions, (), Some(&options))?; let adapter: Bound<'py, PyAny> = native_field(&invocation.context, "adapter")?; diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 47b8971951..622d547dd5 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -6582,6 +6582,7 @@ class BuildCommandRequest(NativeRecord): no_color: bool fail_fast: bool full_refresh: bool + job_settings: bool changed: bool changed_counts: tuple[str, ...] | None reach: str | None @@ -6627,6 +6628,7 @@ class BuildCommandRequest(NativeRecord): no_color: bool = False, fail_fast: bool = False, full_refresh: bool = False, + job_settings: bool = True, changed: bool = False, changed_counts: tuple[str, ...] | None = None, reach: str | None = None, @@ -6997,6 +6999,7 @@ class PlanCommandRequest(NativeRecord): cursor_overrides: CursorOverrides | None json_output: bool full_refresh: bool + job_settings: bool changed: bool changed_counts: tuple[str, ...] | None reach: str | None @@ -7024,6 +7027,7 @@ class PlanCommandRequest(NativeRecord): cursor_overrides: CursorOverrides | None = None, json_output: bool = False, full_refresh: bool = False, + job_settings: bool = True, changed: bool = False, changed_counts: tuple[str, ...] | None = None, reach: str | None = None, @@ -10258,6 +10262,8 @@ class CompilePipelineOptions(NativeRecord): source_lexical_syntax: SqlLexicalSyntax | None plan_from: str | None ignore_audit_failures: bool + cli_ignore_audit_failures: bool | None + job_settings: bool def __init__( self, selected_target: str | None = ..., @@ -10287,6 +10293,8 @@ class CompilePipelineOptions(NativeRecord): source_lexical_syntax: SqlLexicalSyntax | None = ..., plan_from: str | None = ..., ignore_audit_failures: bool = ..., + cli_ignore_audit_failures: bool | None = ..., + job_settings: bool = ..., ) -> None: ... class PythonPlanEntry(NativeRecord): @@ -10504,6 +10512,7 @@ class PlannerOverrides(NativeRecord): cursor_overrides: CursorOverrides | None full_refresh: bool + full_refresh_models: frozenset[str] reload_sources: bool forced_stale_model_names: tuple[str, ...] external_blocked_model_names: tuple[str, ...] @@ -10513,6 +10522,7 @@ class PlannerOverrides(NativeRecord): self, cursor_overrides: CursorOverrides | None = ..., full_refresh: bool = ..., + full_refresh_models: frozenset[str] = ..., reload_sources: bool = ..., forced_stale_model_names: tuple[str, ...] = ..., external_blocked_model_names: tuple[str, ...] = ..., @@ -15320,6 +15330,8 @@ class CompilePipelineResult(NativeRecord): custom_materializations: dict[str, Callable[..., Any]] python_node_names: frozenset[str] python_plan_entries: tuple[PythonPlanEntry, ...] + job_limits: dict[str, int] + node_jobs: dict[CompiledObjectKey, tuple[str, ...]] compile_seconds: float | None planning_seconds: float | None def __init__( @@ -15329,6 +15341,8 @@ class CompilePipelineResult(NativeRecord): custom_materializations: dict[str, Callable[..., Any]] = ..., python_node_names: frozenset[str] = ..., python_plan_entries: tuple[PythonPlanEntry, ...] = ..., + job_limits: dict[str, int] = ..., + node_jobs: dict[CompiledObjectKey, tuple[str, ...]] = ..., compile_seconds: float | None = ..., planning_seconds: float | None = ..., ) -> None: ... diff --git a/src/sqlbuild/executor/build/classes/build_scheduler.py b/src/sqlbuild/executor/build/classes/build_scheduler.py index 395f78de8b..3ac4001087 100644 --- a/src/sqlbuild/executor/build/classes/build_scheduler.py +++ b/src/sqlbuild/executor/build/classes/build_scheduler.py @@ -242,6 +242,9 @@ def __init__( callbacks.python_identity_recorder ) self._max_concurrency: int = len(connections) + self._job_limits: Mapping[str, int] = runtime.job_limits + self._node_jobs: Mapping[CompiledObjectKey, tuple[str, ...]] = runtime.node_jobs + self._job_running: dict[str, int] = {} self._blocked_keys: set[CompiledObjectKey] = set() self._unbuilt_keys: set[CompiledObjectKey] = set(initial_state.initial_failed_keys) self._completed_keys: set[CompiledObjectKey] = set(initial_state.precompleted_keys) @@ -480,7 +483,10 @@ def _run_concurrent(self) -> None: while self._ready and len(self._in_flight) < self._max_concurrency: if self._stop: break - key: CompiledObjectKey = self._ready.popleft() + next_key: CompiledObjectKey | None = self._take_ready_key() + if next_key is None: + break + key: CompiledObjectKey = next_key if key.resource_type == CompiledResourceType.SOURCE: if key in self._blocked_keys: self._record_skipped(key) @@ -489,7 +495,7 @@ def _run_concurrent(self) -> None: if not self._pre_dispatch(key): self._mark_complete(key) continue - self._in_flight.add(key) + self._start_in_flight(key) pool.submit(copy_context().run, self._worker, key) continue if key in self._blocked_keys: @@ -497,7 +503,9 @@ def _run_concurrent(self) -> None: self._mark_complete(key) continue if self._is_concurrent_microbatch(key) and any( - not self._is_concurrent_microbatch(candidate) for candidate in self._ready + not self._is_concurrent_microbatch(candidate) + and self._has_job_room(candidate) + for candidate in self._ready ): self._ready.append(key) continue @@ -513,7 +521,7 @@ def _run_concurrent(self) -> None: self._ready.appendleft(key) break self._microbatch_coordinators.add(key) - self._in_flight.add(key) + self._start_in_flight(key) pool.submit( copy_context().run, self._concurrent_microbatch_worker, @@ -521,7 +529,7 @@ def _run_concurrent(self) -> None: pool=pool, ) continue - self._in_flight.add(key) + self._start_in_flight(key) pool.submit(copy_context().run, self._worker, key) self._report_scheduler_state() @@ -529,7 +537,7 @@ def _run_concurrent(self) -> None: break completion: NodeCompletion = self._completion_queue.get() - self._in_flight.discard(completion.key) + self._finish_in_flight(completion.key) if completion.error is not None: if escaped is None: escaped = completion.error @@ -541,6 +549,39 @@ def _run_concurrent(self) -> None: if escaped is not None: raise escaped + def _take_ready_key(self) -> CompiledObjectKey | None: + """Remove and return the first ready node whose named jobs all have a free slot.""" + + index: int + key: CompiledObjectKey + for index, key in enumerate(self._ready): + if self._has_job_room(key): + del self._ready[index] + return key + return None + + def _has_job_room(self, key: CompiledObjectKey) -> bool: + """Whether each job limiting `key` has a free slot; a node to skip never waits.""" + + return key in self._blocked_keys or all( + self._job_running.get(job, 0) < self._job_limits[job] + for job in self._node_jobs.get(key, ()) + ) + + def _start_in_flight(self, key: CompiledObjectKey) -> None: + self._in_flight.add(key) + job: str + for job in self._node_jobs.get(key, ()): + self._job_running[job] = self._job_running.get(job, 0) + 1 + + def _finish_in_flight(self, key: CompiledObjectKey) -> None: + if key not in self._in_flight: + return + self._in_flight.discard(key) + job: str + for job in self._node_jobs.get(key, ()): + self._job_running[job] -= 1 + def _report_scheduler_state(self) -> None: callback: Callable[[SchedulerState], None] | None = self._callbacks.on_scheduler_state if callback is None: diff --git a/src/sqlbuild/executor/build/models.py b/src/sqlbuild/executor/build/models.py index d245ce43a3..14459bd99c 100644 --- a/src/sqlbuild/executor/build/models.py +++ b/src/sqlbuild/executor/build/models.py @@ -107,6 +107,8 @@ class BuildRuntimeParams: allow_ignore_dynamic_grace: bool = False ignore_dynamic_grace: bool = False dynamic_hold: DynamicHold | None = None + job_limits: Mapping[str, int] = field(default_factory=dict) + node_jobs: Mapping[CompiledObjectKey, tuple[str, ...]] = field(default_factory=dict) @dataclass(frozen=True) From c65a4686c01bdccb18ab5ae4200389291a6a5ac8 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 18 Nov 2026 15:17:49 +0000 Subject: [PATCH 03/22] feat(jobs): take named jobs' settings below the command line's own flags (CHI-1533) Apply a named job's build-wide settings as the flags the command line leaves out, decided from the parser's record of which options it wrote, so abbreviated and global options count, and parse the command line once more. A job conflict, invalid configuration or an unreadable file found while doing so fails again, and is reported, when the command runs; the planner now also checks that named jobs agree on build-wide settings, so sqb plan reports the conflict too. Skip the work when a selection cannot name a job or the project has no orchestration folder, test the scheduler's per-job dispatch, the integration of settings, conflicts and per-job full refresh through the CLI, and document job settings on the jobs, selectors, concurrency, build, plan and configuration pages. --- .../src/job_settings/_helpers/mod.rs | 4 + .../src/job_settings/_helpers/node_values.rs | 87 +++++++ .../setting_scope.rs => _helpers/scope.rs} | 2 +- .../src/job_settings/_helpers/selection.rs | 50 ++++ .../src/job_settings/_helpers/tokens.rs | 33 +++ .../src/job_settings/constants.rs | 4 +- .../job_settings/main/build_job_settings.rs | 4 +- .../main/group_setting_mismatch.rs | 2 +- .../src/job_settings/main/mod.rs | 1 - .../src/job_settings/main/named_jobs.rs | 29 +-- .../job_settings/main/node_job_settings.rs | 128 ++-------- .../sqlbuild-analysis/src/job_settings/mod.rs | 1 + .../src/job_settings/tests/helpers.rs | 16 +- .../tests/test_build_job_settings.rs | 20 +- .../tests/test_node_job_settings.rs | 17 +- .../src/job_settings/tests/test_types.rs | 4 +- .../src/argv/_helpers/job_settings.rs | 26 +- .../sqlbuild-cli/src/argv/_helpers/parsing.rs | 11 +- .../src/argv/main/given_setting_dests.rs | 22 ++ crates/sqlbuild-cli/src/argv/main/mod.rs | 1 + .../src/argv/main/parse_cli_arguments.rs | 4 + .../main/parse_dbt_execution_arguments.rs | 3 + crates/sqlbuild-cli/src/argv/models.rs | 4 +- crates/sqlbuild-cli/src/argv/tests.rs | 2 + .../argv/tests/test_given_setting_dests.rs | 54 ++++ .../sqlbuild-cli/src/argv/tests/test_types.rs | 7 + .../src/planner_phases/models.rs | 3 +- .../src/compiled_project/constants.rs | 4 +- .../_helpers/build_command/execution.rs | 13 +- .../src/bindings/_helpers/cli/dispatch.rs | 2 - .../src/bindings/_helpers/cli/entry.rs | 21 +- .../_helpers/cli/named_job_settings.rs | 171 +++++++------ .../compile_driver/job_node_settings.rs | 32 ++- .../_helpers/compiled_project/job_settings.rs | 52 ++-- .../sqlbuild/references/docs/cli/build.md | 1 + .../sqlbuild/references/docs/cli/plan.md | 1 + .../references/docs/concepts/concurrency.md | 5 +- .../references/docs/concepts/orchestration.md | 37 +++ .../docs/concepts/project-configuration.md | 8 +- .../references/docs/concepts/selectors.md | 2 + .../sqlbuild/cli/commands/main/_test_types.py | 23 ++ .../cli/commands/main/test_job_settings.py | 231 ++++++++++++++++++ .../executor/build/classes/_test_types.py | 15 ++ .../executor/build/classes/helpers.py | 35 +++ .../classes/test_build_scheduler_identity.py | 6 + .../test_build_scheduler_job_limits.py | 51 ++++ website/src/content/docs/docs/cli/build.mdx | 1 + website/src/content/docs/docs/cli/plan.mdx | 1 + .../docs/docs/concepts/concurrency.mdx | 5 +- .../docs/docs/concepts/orchestration.mdx | 37 +++ .../docs/concepts/project-configuration.mdx | 8 +- .../content/docs/docs/concepts/selectors.mdx | 2 + 52 files changed, 996 insertions(+), 307 deletions(-) create mode 100644 crates/sqlbuild-analysis/src/job_settings/_helpers/mod.rs create mode 100644 crates/sqlbuild-analysis/src/job_settings/_helpers/node_values.rs rename crates/sqlbuild-analysis/src/job_settings/{main/setting_scope.rs => _helpers/scope.rs} (87%) create mode 100644 crates/sqlbuild-analysis/src/job_settings/_helpers/selection.rs create mode 100644 crates/sqlbuild-analysis/src/job_settings/_helpers/tokens.rs create mode 100644 crates/sqlbuild-cli/src/argv/main/given_setting_dests.rs create mode 100644 crates/sqlbuild-cli/src/argv/tests/test_given_setting_dests.rs create mode 100644 tests/integration/src/sqlbuild/cli/commands/main/test_job_settings.py create mode 100644 tests/unit/src/sqlbuild/executor/build/classes/helpers.py create mode 100644 tests/unit/src/sqlbuild/executor/build/classes/test_build_scheduler_job_limits.py diff --git a/crates/sqlbuild-analysis/src/job_settings/_helpers/mod.rs b/crates/sqlbuild-analysis/src/job_settings/_helpers/mod.rs new file mode 100644 index 0000000000..25aecee76d --- /dev/null +++ b/crates/sqlbuild-analysis/src/job_settings/_helpers/mod.rs @@ -0,0 +1,4 @@ +pub(crate) mod node_values; +pub(crate) mod scope; +pub(crate) mod selection; +pub(crate) mod tokens; diff --git a/crates/sqlbuild-analysis/src/job_settings/_helpers/node_values.rs b/crates/sqlbuild-analysis/src/job_settings/_helpers/node_values.rs new file mode 100644 index 0000000000..3d93f3c319 --- /dev/null +++ b/crates/sqlbuild-analysis/src/job_settings/_helpers/node_values.rs @@ -0,0 +1,87 @@ +//! The node-level settings and retry one node takes from the jobs it is selected through. + +use std::collections::BTreeMap; + +use serde_json::Value as Json; + +use crate::assembly::project::types::ObjectKey; +use crate::job_settings::constants::{CONFLICT_HELP, NODE_SETTINGS, RETRY_SETTING}; +use crate::job_settings::errors::JobSettingConflict; +use crate::job_settings::models::{DeclaredRun, JobRetry}; + +/// One node's settings, and its retry, each from whichever of its jobs sets it. +pub(crate) type NodeValues = (BTreeMap, Option); + +fn conflict( + key: &ObjectKey, + (first, second): (&str, &str), + setting: &str, + values: (String, String), +) -> JobSettingConflict { + JobSettingConflict { + message: format!( + "{}:{} is selected through jobs {first} and {second}, which set {setting} differently \ + ({} and {})", + key.0, key.1, values.0, values.1 + ), + help: CONFLICT_HELP, + } +} + +fn shown(retry: JobRetry) -> String { + format!( + "{} attempts, {}s apart", + retry.attempts, retry.delay_seconds + ) +} + +/// The settings `jobs` give `key`; two jobs giving it different values of one setting conflict. +pub(crate) fn node_values( + key: &ObjectKey, + jobs: &[String], + declared: &BTreeMap, +) -> Result { + let mut values: BTreeMap = BTreeMap::new(); + let mut retry: Option<(&String, JobRetry)> = None; + for job in jobs { + let Some(DeclaredRun::Job { + settings, + retry: job_retry, + }) = declared.get(job) + else { + continue; + }; + for setting in NODE_SETTINGS { + let Some(value) = settings.get(setting) else { + continue; + }; + match values.get(setting) { + Some((other, seen)) if *seen != value => { + let shown_values = (seen.to_string(), value.to_string()); + return Err(conflict(key, (other, job), setting, shown_values)); + } + Some(_) => {} + None => { + values.insert(setting.to_owned(), (job, value)); + } + } + } + match (retry, job_retry) { + (Some((other, seen)), Some(own)) if seen != *own => { + return Err(conflict( + key, + (other, job), + RETRY_SETTING, + (shown(seen), shown(*own)), + )); + } + (None, Some(own)) => retry = Some((job, *own)), + _ => {} + } + } + let values: BTreeMap = values + .into_iter() + .map(|(name, (_, value))| (name, value.clone())) + .collect(); + Ok((values, retry.map(|(_, retry)| retry))) +} diff --git a/crates/sqlbuild-analysis/src/job_settings/main/setting_scope.rs b/crates/sqlbuild-analysis/src/job_settings/_helpers/scope.rs similarity index 87% rename from crates/sqlbuild-analysis/src/job_settings/main/setting_scope.rs rename to crates/sqlbuild-analysis/src/job_settings/_helpers/scope.rs index 80bda3ed92..4fd31e197b 100644 --- a/crates/sqlbuild-analysis/src/job_settings/main/setting_scope.rs +++ b/crates/sqlbuild-analysis/src/job_settings/_helpers/scope.rs @@ -5,7 +5,7 @@ use crate::job_settings::models::SettingScope; /// A node setting, the job limit, or else a build-wide setting. #[must_use] -pub fn setting_scope(name: &str) -> SettingScope { +pub(crate) fn setting_scope(name: &str) -> SettingScope { if NODE_SETTINGS.contains(&name) { SettingScope::Node } else if name == JOB_LIMIT_SETTING { diff --git a/crates/sqlbuild-analysis/src/job_settings/_helpers/selection.rs b/crates/sqlbuild-analysis/src/job_settings/_helpers/selection.rs new file mode 100644 index 0000000000..98a819a507 --- /dev/null +++ b/crates/sqlbuild-analysis/src/job_settings/_helpers/selection.rs @@ -0,0 +1,50 @@ +//! The named jobs each selected node is selected through. + +use std::collections::{BTreeMap, BTreeSet}; + +use crate::assembly::project::types::ObjectKey; +use crate::graph::errors::SelectorError; +use crate::graph::main::resolve_selector_tokens::resolve_selector_tokens; +use crate::graph::models::ProjectGraph; +use crate::job_settings::_helpers::tokens::token_jobs; +use crate::job_settings::models::DeclaredRun; + +fn keys(graph: &ProjectGraph, selectors: &[String]) -> Result, SelectorError> { + resolve_selector_tokens(graph, selectors).map(|keys| keys.into_iter().collect()) +} + +/// Each node a `--select` token selects through a job it names whose own selection holds it. +pub(crate) fn node_jobs( + graph: &ProjectGraph, + (select, exclude): (&[String], &[String]), + declared: &BTreeMap, +) -> Result>, SelectorError> { + let mut node_jobs: BTreeMap> = BTreeMap::new(); + let excluded: BTreeSet = keys(graph, exclude)?; + let resolved = graph.jobs.as_ref(); + for token in select + .iter() + .flat_map(|selector| selector.split_whitespace()) + { + let jobs: Vec = token_jobs(token, declared); + if jobs.is_empty() { + continue; + } + let selected: BTreeSet = keys(graph, &[token.to_owned()])?; + for job in jobs { + let Some(Ok(own)) = resolved.and_then(|resolved| resolved.get(&job)) else { + continue; + }; + for key in selected + .intersection(own) + .filter(|key| !excluded.contains(*key)) + { + let jobs: &mut Vec = node_jobs.entry(key.clone()).or_default(); + if !jobs.contains(&job) { + jobs.push(job.clone()); + } + } + } + } + Ok(node_jobs) +} diff --git a/crates/sqlbuild-analysis/src/job_settings/_helpers/tokens.rs b/crates/sqlbuild-analysis/src/job_settings/_helpers/tokens.rs new file mode 100644 index 0000000000..cf3413d092 --- /dev/null +++ b/crates/sqlbuild-analysis/src/job_settings/_helpers/tokens.rs @@ -0,0 +1,33 @@ +//! The jobs one selector token names. + +use std::collections::BTreeMap; + +use crate::graph::constants::{INTERSECTION_SEPARATOR, JOB_KIND, NAME_KIND}; +use crate::graph::main::parse_selector::parse_selector; +use crate::graph::models::ParsedSelector; +use crate::job_settings::models::DeclaredRun; + +/// The jobs one whitespace-free selector token names, each once in the order named. +#[must_use] +pub(crate) fn token_jobs(token: &str, declared: &BTreeMap) -> Vec { + let mut jobs: Vec = Vec::new(); + for part in token.split(INTERSECTION_SEPARATOR) { + let Ok(ParsedSelector::Kind { kind, value, .. }) = parse_selector(part) else { + continue; + }; + if kind != JOB_KIND && kind != NAME_KIND { + continue; + } + let named: Vec = match declared.get(&value) { + Some(DeclaredRun::Job { .. }) => vec![value], + Some(DeclaredRun::Group { members }) => members.clone(), + None => Vec::new(), + }; + for job in named { + if !jobs.contains(&job) { + jobs.push(job); + } + } + } + jobs +} diff --git a/crates/sqlbuild-analysis/src/job_settings/constants.rs b/crates/sqlbuild-analysis/src/job_settings/constants.rs index dd941a907d..cddaa1c18e 100644 --- a/crates/sqlbuild-analysis/src/job_settings/constants.rs +++ b/crates/sqlbuild-analysis/src/job_settings/constants.rs @@ -1,6 +1,6 @@ //! How far each job setting reaches when a selection names the job. -/// Settings that apply only to the named job's own nodes; two jobs giving one node different values conflict. +/// Settings that apply only to the named job's own nodes; two jobs may not disagree on one node. pub const NODE_SETTINGS: [&str; 3] = ["run_tests", "ignore_audit_failures", "full_refresh"]; /// The setting that becomes a limit on how many of the job's nodes run at once inside the build. pub const JOB_LIMIT_SETTING: &str = "concurrency"; @@ -11,3 +11,5 @@ pub const CONFLICT_HELP: &str = "make the jobs agree, set it in only one of them the command-line flag, or build with --no-job-settings"; /// The error code of job settings that cannot apply together. pub const JOB_SETTING_CONFLICT_CODE: &str = "S027"; +/// How a conflict names a job's `[retry]` table. +pub const RETRY_SETTING: &str = "[retry]"; diff --git a/crates/sqlbuild-analysis/src/job_settings/main/build_job_settings.rs b/crates/sqlbuild-analysis/src/job_settings/main/build_job_settings.rs index ed2cf4fd61..a29f8c9953 100644 --- a/crates/sqlbuild-analysis/src/job_settings/main/build_job_settings.rs +++ b/crates/sqlbuild-analysis/src/job_settings/main/build_job_settings.rs @@ -4,9 +4,9 @@ use std::collections::BTreeMap; use serde_json::Value as Json; +use crate::job_settings::_helpers::scope::setting_scope; use crate::job_settings::constants::CONFLICT_HELP; use crate::job_settings::errors::JobSettingConflict; -use crate::job_settings::main::setting_scope::setting_scope; use crate::job_settings::models::{DeclaredRun, SettingScope}; fn settings_of<'a>( @@ -23,7 +23,7 @@ fn shown(value: Option<&Json>) -> String { value.map_or_else(|| "not set".to_owned(), Json::to_string) } -/// Each build-wide setting the named jobs give the build; a job that leaves one unset differs from one that sets it. +/// The named jobs' build-wide settings, which must agree; unset differs from set. pub fn build_job_settings( named: &[String], declared: &BTreeMap, diff --git a/crates/sqlbuild-analysis/src/job_settings/main/group_setting_mismatch.rs b/crates/sqlbuild-analysis/src/job_settings/main/group_setting_mismatch.rs index ea9402d087..0b393968ff 100644 --- a/crates/sqlbuild-analysis/src/job_settings/main/group_setting_mismatch.rs +++ b/crates/sqlbuild-analysis/src/job_settings/main/group_setting_mismatch.rs @@ -7,7 +7,7 @@ use serde_json::Value as Json; use crate::job_settings::constants::GROUP_AGREED_SETTINGS; use crate::job_settings::models::DeclaredRun; -/// Why `members` cannot run as one build, naming the setting and two members; `None` when they agree. +/// Why `members` cannot run as one build, naming the setting and two members. #[must_use] pub fn group_setting_mismatch( members: &[String], diff --git a/crates/sqlbuild-analysis/src/job_settings/main/mod.rs b/crates/sqlbuild-analysis/src/job_settings/main/mod.rs index f768d90cd8..b642609e6f 100644 --- a/crates/sqlbuild-analysis/src/job_settings/main/mod.rs +++ b/crates/sqlbuild-analysis/src/job_settings/main/mod.rs @@ -2,4 +2,3 @@ pub mod build_job_settings; pub mod group_setting_mismatch; pub mod named_jobs; pub mod node_job_settings; -pub mod setting_scope; diff --git a/crates/sqlbuild-analysis/src/job_settings/main/named_jobs.rs b/crates/sqlbuild-analysis/src/job_settings/main/named_jobs.rs index 701a1c1497..d27d2ce421 100644 --- a/crates/sqlbuild-analysis/src/job_settings/main/named_jobs.rs +++ b/crates/sqlbuild-analysis/src/job_settings/main/named_jobs.rs @@ -2,36 +2,9 @@ use std::collections::BTreeMap; -use crate::graph::_helpers::selectors::parse; -use crate::graph::constants::{INTERSECTION_SEPARATOR, JOB_KIND, NAME_KIND}; -use crate::graph::models::ParsedSelector; +use crate::job_settings::_helpers::tokens::token_jobs; use crate::job_settings::models::DeclaredRun; -/// The jobs one whitespace-free selector token names, each once in the order named. -#[must_use] -pub fn token_jobs(token: &str, declared: &BTreeMap) -> Vec { - let mut jobs: Vec = Vec::new(); - for part in token.split(INTERSECTION_SEPARATOR) { - let Ok(ParsedSelector::Kind { kind, value, .. }) = parse(part) else { - continue; - }; - if kind != JOB_KIND && kind != NAME_KIND { - continue; - } - let named: Vec = match declared.get(&value) { - Some(DeclaredRun::Job { .. }) => vec![value], - Some(DeclaredRun::Group { members }) => members.clone(), - None => Vec::new(), - }; - for job in named { - if !jobs.contains(&job) { - jobs.push(job); - } - } - } - jobs -} - /// The jobs `selectors` name, each once in the order first named. #[must_use] pub fn named_jobs(selectors: &[String], declared: &BTreeMap) -> Vec { diff --git a/crates/sqlbuild-analysis/src/job_settings/main/node_job_settings.rs b/crates/sqlbuild-analysis/src/job_settings/main/node_job_settings.rs index 9cde08f580..6b96394673 100644 --- a/crates/sqlbuild-analysis/src/job_settings/main/node_job_settings.rs +++ b/crates/sqlbuild-analysis/src/job_settings/main/node_job_settings.rs @@ -4,130 +4,32 @@ use std::collections::BTreeMap; use serde_json::Value as Json; -use crate::assembly::project::types::ObjectKey; -use crate::graph::_helpers::selectors::{Keys, tokens}; use crate::graph::models::ProjectGraph; -use crate::job_settings::constants::{CONFLICT_HELP, JOB_LIMIT_SETTING, NODE_SETTINGS}; -use crate::job_settings::errors::{JobSettingConflict, NodeJobSettingsError}; -use crate::job_settings::main::named_jobs::token_jobs; -use crate::job_settings::models::{DeclaredRun, JobRetry, NodeJobSettings}; +use crate::job_settings::_helpers::node_values::node_values; +use crate::job_settings::_helpers::selection::node_jobs; +use crate::job_settings::constants::JOB_LIMIT_SETTING; +use crate::job_settings::errors::NodeJobSettingsError; +use crate::job_settings::models::{DeclaredRun, NodeJobSettings}; -fn conflict( - key: &ObjectKey, - (first, second): (&str, &str), - setting: &str, - values: (String, String), -) -> NodeJobSettingsError { - NodeJobSettingsError::Conflict(JobSettingConflict { - message: format!( - "{}:{} is selected through jobs {first} and {second}, which set {setting} differently \ - ({} and {})", - key.0, key.1, values.0, values.1 - ), - help: CONFLICT_HELP, - }) -} - -/// A node is selected through a job when a `--select` token names the job, selects the node, and -/// the job's own selection holds it; `+` around a job and intersections never widen what it covers. +/// Each node's named jobs, the settings they give it, and their limits. pub fn node_job_settings( graph: &ProjectGraph, - (select, exclude): (&[String], &[String]), + selection: (&[String], &[String]), declared: &BTreeMap, ) -> Result { - let mut result = NodeJobSettings::default(); - let excluded: Keys = tokens(graph, exclude).map_err(NodeJobSettingsError::Selector)?; - let resolved = graph.jobs.as_ref(); - for token in select - .iter() - .flat_map(|selector| selector.split_whitespace()) - { - let jobs: Vec = token_jobs(token, declared); - if jobs.is_empty() { - continue; - } - let selected: Keys = - tokens(graph, &[token.to_owned()]).map_err(NodeJobSettingsError::Selector)?; - for job in jobs { - let Some(Ok(own)) = resolved.and_then(|resolved| resolved.get(&job)) else { - continue; - }; - for key in selected - .intersection(own) - .filter(|key| !excluded.contains(*key)) - { - let jobs = result.node_jobs.entry(key.clone()).or_default(); - if !jobs.contains(&job) { - jobs.push(job.clone()); - } - } - } - } + let mut result = NodeJobSettings { + node_jobs: node_jobs(graph, selection, declared).map_err(NodeJobSettingsError::Selector)?, + ..NodeJobSettings::default() + }; for (key, jobs) in &result.node_jobs { - let mut values: BTreeMap = BTreeMap::new(); - let mut retry: Option<(&String, JobRetry)> = None; - for job in jobs { - let Some(DeclaredRun::Job { - settings, - retry: job_retry, - }) = declared.get(job) - else { - continue; - }; - for setting in NODE_SETTINGS { - let Some(value) = settings.get(setting) else { - continue; - }; - match values.get(setting) { - Some((other, seen)) if seen != value => { - return Err(conflict( - key, - (other, job), - setting, - (seen.to_string(), value.to_string()), - )); - } - Some(_) => {} - None => { - values.insert(setting.to_owned(), (job.clone(), value.clone())); - } - } - } - if let Some(job_retry) = job_retry { - match retry { - Some((other, seen)) if seen != *job_retry => { - let shown = |retry: JobRetry| { - format!( - "{} attempts, {}s apart", - retry.attempts, retry.delay_seconds - ) - }; - return Err(conflict( - key, - (other, job), - "[retry]", - (shown(seen), shown(*job_retry)), - )); - } - Some(_) => {} - None => retry = Some((job, *job_retry)), - } - } - } + let (values, retry) = + node_values(key, jobs, declared).map_err(NodeJobSettingsError::Conflict)?; if !values.is_empty() { - result.values.insert( - key.clone(), - values - .into_iter() - .map(|(setting, (_, value))| (setting, value)) - .collect(), - ); + result.values.insert(key.clone(), values); } - if let Some((_, retry)) = retry { + if let Some(retry) = retry { result.retries.insert(key.clone(), retry); } - } - for jobs in result.node_jobs.values() { for job in jobs { if let Some(DeclaredRun::Job { settings, .. }) = declared.get(job) && let Some(limit) = settings.get(JOB_LIMIT_SETTING).and_then(Json::as_i64) diff --git a/crates/sqlbuild-analysis/src/job_settings/mod.rs b/crates/sqlbuild-analysis/src/job_settings/mod.rs index 288d5de7a9..c4d7004a23 100644 --- a/crates/sqlbuild-analysis/src/job_settings/mod.rs +++ b/crates/sqlbuild-analysis/src/job_settings/mod.rs @@ -1,5 +1,6 @@ //! Jobs as named selections: which jobs a selection names and the settings they give its nodes. +pub(crate) mod _helpers; pub mod constants; pub mod errors; pub mod main; diff --git a/crates/sqlbuild-analysis/src/job_settings/tests/helpers.rs b/crates/sqlbuild-analysis/src/job_settings/tests/helpers.rs index 0b0a80b4c0..a5617dfffb 100644 --- a/crates/sqlbuild-analysis/src/job_settings/tests/helpers.rs +++ b/crates/sqlbuild-analysis/src/job_settings/tests/helpers.rs @@ -6,7 +6,9 @@ use crate::assembly::project::types::ObjectKey; use crate::graph::main::attach_jobs::attach_jobs; use crate::graph::main::build_project_graph::build_project_graph; use crate::graph::models::{GraphResource, JobSelection, ProjectGraph}; -use crate::job_settings::models::{DeclaredRun, JobRetry}; +use crate::job_settings::errors::NodeJobSettingsError; +use crate::job_settings::main::node_job_settings::node_job_settings; +use crate::job_settings::models::{DeclaredRun, JobRetry, NodeJobSettings}; fn key(kind: &str, name: &str) -> ObjectKey { (kind.to_owned(), name.to_owned()) @@ -129,3 +131,15 @@ pub(super) fn graph_with_jobs() -> ProjectGraph { pub(super) fn strings(values: &[&str]) -> Vec { texts(values) } + +/// What selecting `select` minus `exclude` gives each node of the graph's jobs. +pub(super) fn resolved_settings( + select: &[&str], + exclude: &[&str], +) -> Result { + node_job_settings( + &graph_with_jobs(), + (&texts(select), &texts(exclude)), + &declared(), + ) +} diff --git a/crates/sqlbuild-analysis/src/job_settings/tests/test_build_job_settings.rs b/crates/sqlbuild-analysis/src/job_settings/tests/test_build_job_settings.rs index b712f8a448..0540a65afc 100644 --- a/crates/sqlbuild-analysis/src/job_settings/tests/test_build_job_settings.rs +++ b/crates/sqlbuild-analysis/src/job_settings/tests/test_build_job_settings.rs @@ -15,7 +15,7 @@ fn given_named_jobs_when_merging_build_settings_then_they_must_agree() { description: "a job that leaves vars unset differs from one that sets them", select: &["crm_job", "job:orders_job"], expected_named: vec!["crm_job", "orders_job"], - expected: Err( + expected_settings: Err( "jobs crm_job and orders_job set vars differently ({\"region\":\"eu\"} \ and not set); a build applies vars to everything it runs", ), @@ -24,13 +24,13 @@ fn given_named_jobs_when_merging_build_settings_then_they_must_agree() { description: "a single job's vars", select: &["job:crm_job,customers"], expected_named: vec!["crm_job"], - expected: Ok(BTreeMap::from([("vars", json!({"region": "eu"}))])), + expected_settings: Ok(BTreeMap::from([("vars", json!({"region": "eu"}))])), }, BuildJobSettingsTestCase { description: "two jobs with different vars", select: &["job:crm_job job:crm_us_job"], expected_named: vec!["crm_job", "crm_us_job"], - expected: Err( + expected_settings: Err( "jobs crm_job and crm_us_job set vars differently ({\"region\":\"eu\"} \ and {\"region\":\"us\"}); a build applies vars to everything it runs", ), @@ -39,13 +39,13 @@ fn given_named_jobs_when_merging_build_settings_then_they_must_agree() { description: "a group's jobs without build-wide settings", select: &["job:daily_group"], expected_named: vec!["orders_job", "refunds_job"], - expected: Ok(BTreeMap::new()), + expected_settings: Ok(BTreeMap::new()), }, BuildJobSettingsTestCase { description: "tags, paths and node names name no job", select: &["tag:daily models/marts orders"], expected_named: vec![], - expected: Ok(BTreeMap::new()), + expected_settings: Ok(BTreeMap::new()), }, ]; for test_case in test_cases { @@ -56,7 +56,7 @@ fn given_named_jobs_when_merging_build_settings_then_they_must_agree() { .map(|settings| settings.into_iter().collect::>()) .map_err(|conflict| conflict.message); let expected = test_case - .expected + .expected_settings .map(|settings| { settings .into_iter() @@ -74,7 +74,7 @@ fn given_a_group_when_its_jobs_differ_on_vars_or_changed_then_it_cannot_run_as_o GroupMismatchTestCase { description: "different vars", members: &["crm_job", "crm_us_job"], - expected: Some( + expected_mismatch: Some( "jobs crm_job and crm_us_job set vars differently; a group runs its \ jobs as one build, which has one vars", ), @@ -82,7 +82,7 @@ fn given_a_group_when_its_jobs_differ_on_vars_or_changed_then_it_cannot_run_as_o GroupMismatchTestCase { description: "vars set by only one job still differ", members: &["orders_job", "crm_job"], - expected: Some( + expected_mismatch: Some( "jobs orders_job and crm_job set vars differently; a group runs its \ jobs as one build, which has one vars", ), @@ -90,13 +90,13 @@ fn given_a_group_when_its_jobs_differ_on_vars_or_changed_then_it_cannot_run_as_o GroupMismatchTestCase { description: "node settings may differ", members: &["orders_job", "refunds_job", "quiet_job"], - expected: None, + expected_mismatch: None, }, ]; for test_case in test_cases { assert_eq!( group_setting_mismatch(&strings(test_case.members), &declared()).as_deref(), - test_case.expected, + test_case.expected_mismatch, "{}", test_case.description ); diff --git a/crates/sqlbuild-analysis/src/job_settings/tests/test_node_job_settings.rs b/crates/sqlbuild-analysis/src/job_settings/tests/test_node_job_settings.rs index ec8778c618..33529aa04f 100644 --- a/crates/sqlbuild-analysis/src/job_settings/tests/test_node_job_settings.rs +++ b/crates/sqlbuild-analysis/src/job_settings/tests/test_node_job_settings.rs @@ -3,19 +3,10 @@ use std::collections::BTreeMap; use serde_json::json; use crate::job_settings::errors::NodeJobSettingsError; -use crate::job_settings::main::node_job_settings::node_job_settings; use crate::job_settings::models::NodeJobSettings; -use crate::job_settings::tests::helpers::{declared, graph_with_jobs, strings}; +use crate::job_settings::tests::helpers::resolved_settings; use crate::job_settings::tests::test_types::{NodeJobConflictTestCase, NodeJobSettingsTestCase}; -fn settings(select: &[&str], exclude: &[&str]) -> Result { - node_job_settings( - &graph_with_jobs(), - (&strings(select), &strings(exclude)), - &declared(), - ) -} - #[test] fn given_named_jobs_when_resolving_node_settings_then_only_their_own_selected_nodes_take_them() { let orders_values = @@ -105,7 +96,7 @@ fn given_named_jobs_when_resolving_node_settings_then_only_their_own_selected_no ]; for test_case in test_cases { let resolved: NodeJobSettings = - settings(test_case.select, test_case.exclude).expect(test_case.description); + resolved_settings(test_case.select, test_case.exclude).expect(test_case.description); let node_jobs: BTreeMap<&str, Vec<&str>> = resolved .node_jobs .iter() @@ -165,7 +156,9 @@ fn given_two_named_jobs_setting_one_node_differently_when_resolving_then_it_is_a }, ]; for test_case in test_cases { - let Err(NodeJobSettingsError::Conflict(conflict)) = settings(test_case.select, &[]) else { + let Err(NodeJobSettingsError::Conflict(conflict)) = + resolved_settings(test_case.select, &[]) + else { panic!("{}: expected a conflict", test_case.description); }; assert_eq!( diff --git a/crates/sqlbuild-analysis/src/job_settings/tests/test_types.rs b/crates/sqlbuild-analysis/src/job_settings/tests/test_types.rs index 25fd309e19..582e8fc74f 100644 --- a/crates/sqlbuild-analysis/src/job_settings/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/job_settings/tests/test_types.rs @@ -23,11 +23,11 @@ pub(super) struct BuildJobSettingsTestCase { pub(super) description: &'static str, pub(super) select: &'static [&'static str], pub(super) expected_named: Vec<&'static str>, - pub(super) expected: Result, &'static str>, + pub(super) expected_settings: Result, &'static str>, } pub(super) struct GroupMismatchTestCase { pub(super) description: &'static str, pub(super) members: &'static [&'static str], - pub(super) expected: Option<&'static str>, + pub(super) expected_mismatch: Option<&'static str>, } diff --git a/crates/sqlbuild-cli/src/argv/_helpers/job_settings.rs b/crates/sqlbuild-cli/src/argv/_helpers/job_settings.rs index 7fa1c3e300..492b59f87a 100644 --- a/crates/sqlbuild-cli/src/argv/_helpers/job_settings.rs +++ b/crates/sqlbuild-cli/src/argv/_helpers/job_settings.rs @@ -1,12 +1,14 @@ -//! Render one authored setting and check emitted options against the build parser. +//! Render job settings as options, check them against the build parser and parse without Python. use std::collections::BTreeMap; use serde_json::Value as Json; +use sqlbuild_core::text::main::python_str_repr::python_str_repr; use crate::argv::_helpers::table::root_parser; use crate::argv::constants::CONCURRENCY_DEST; -use crate::argv::models::{ActionKind, ActionSpec, Default, Nargs, ParserSpec, Value}; +use crate::argv::models::{ActionKind, ActionSpec, Default, Nargs, ParserSpec, TypeFailure, Value}; +use crate::argv::types::ArgvHost; pub(crate) fn setting_argument( parser: &ParserSpec, @@ -149,3 +151,23 @@ pub(crate) fn validate_mutex(parser: &ParserSpec, arguments: &[String]) -> Resul } Ok(()) } + +/// Converts `int`-like options so the parse validates them, and keeps every other value as text. +pub(crate) struct IntegerHost; + +impl ArgvHost for IntegerHost { + fn convert(&mut self, _converter: &str, text: &str) -> Result { + Ok(text.trim().parse::().map_or_else( + |_| Value::Str(text.to_owned()), + |number| Value::Int(number.to_string()), + )) + } + + fn repr(&self, text: &str) -> String { + python_str_repr(text) + } + + fn program_version(&mut self) -> String { + String::new() + } +} diff --git a/crates/sqlbuild-cli/src/argv/_helpers/parsing.rs b/crates/sqlbuild-cli/src/argv/_helpers/parsing.rs index cb7c78461e..8dc4a339bf 100644 --- a/crates/sqlbuild-cli/src/argv/_helpers/parsing.rs +++ b/crates/sqlbuild-cli/src/argv/_helpers/parsing.rs @@ -1,6 +1,6 @@ //! `ArgumentParser.parse_known_args`, reproduced over the command table. -use std::collections::{HashMap, HashSet}; +use std::collections::{BTreeSet, HashMap, HashSet}; use sqlbuild_core::text::main::is_decimal_digit::is_decimal_digit; @@ -25,6 +25,8 @@ pub(crate) struct Run<'a> { /// The `prog` of the deepest sub-command parser entered so far. pub(crate) command_prog: Option, pub(crate) usage_failure: Option, + /// The destination of every option the arguments wrote, at any parser level. + pub(crate) given: BTreeSet, } /// `SystemExit` with this status. @@ -363,6 +365,13 @@ impl<'p, 'r, 'h> Level<'p, 'r, 'h> { let parser: &ParserSpec = self.parser; let action: &ActionSpec = &parser.actions[index]; self.seen.insert(action.id); + if let Some(dest) = action + .dest + .as_ref() + .filter(|_| !action.option_strings.is_empty()) + { + self.run.given.insert(dest.clone()); + } let values: Option = self .run .action_values(action, &args) diff --git a/crates/sqlbuild-cli/src/argv/main/given_setting_dests.rs b/crates/sqlbuild-cli/src/argv/main/given_setting_dests.rs new file mode 100644 index 0000000000..419b6de573 --- /dev/null +++ b/crates/sqlbuild-cli/src/argv/main/given_setting_dests.rs @@ -0,0 +1,22 @@ +//! The settings a command line gives itself, which the settings of a job it names never override. + +use std::collections::BTreeSet; + +use crate::argv::_helpers::job_settings::IntegerHost; +use crate::argv::main::parse_cli_arguments::parse_cli_arguments; +use crate::argv::models::ArgvSettings; + +/// The destination of every option `argv` writes, however abbreviated; `None` if it fails to parse. +#[must_use] +pub fn given_setting_dests(argv: &[String]) -> Option> { + let mut host: IntegerHost = IntegerHost; + let settings: ArgvSettings = ArgvSettings { + columns: 80, + use_color: false, + }; + let outcome = parse_cli_arguments(argv, Vec::new(), settings, &mut host); + match outcome.parsed { + Ok(_) => Some(outcome.given), + Err(_exit_status) => None, + } +} diff --git a/crates/sqlbuild-cli/src/argv/main/mod.rs b/crates/sqlbuild-cli/src/argv/main/mod.rs index 10cdc0c9bf..2db8594625 100644 --- a/crates/sqlbuild-cli/src/argv/main/mod.rs +++ b/crates/sqlbuild-cli/src/argv/main/mod.rs @@ -1,6 +1,7 @@ pub mod accepted_settings_arguments; pub mod cli_namespace_defaults; pub mod command_settings_arguments; +pub mod given_setting_dests; pub mod job_settings_arguments; pub mod parse_cli_arguments; pub mod parse_dbt_execution_arguments; diff --git a/crates/sqlbuild-cli/src/argv/main/parse_cli_arguments.rs b/crates/sqlbuild-cli/src/argv/main/parse_cli_arguments.rs index c0fce019da..e3632f3c38 100644 --- a/crates/sqlbuild-cli/src/argv/main/parse_cli_arguments.rs +++ b/crates/sqlbuild-cli/src/argv/main/parse_cli_arguments.rs @@ -1,5 +1,7 @@ //! Parse `sqb`'s argv as its `argparse` parser and `parse_cli_invocation` did. +use std::collections::BTreeSet; + use crate::argv::_helpers::parsing::Run; use crate::argv::_helpers::table::root_parser; use crate::argv::models::{ArgvOutcome, ArgvSettings, Namespace, ParserSpec, Value}; @@ -19,6 +21,7 @@ pub fn parse_cli_arguments( writes: Vec::new(), command_prog: None, usage_failure: None, + given: BTreeSet::new(), }; let namespace: Namespace = Namespace { entries: defaults, @@ -32,5 +35,6 @@ pub fn parse_cli_arguments( writes: run.writes, parsed, usage_failure: run.usage_failure, + given: run.given, } } diff --git a/crates/sqlbuild-cli/src/argv/main/parse_dbt_execution_arguments.rs b/crates/sqlbuild-cli/src/argv/main/parse_dbt_execution_arguments.rs index 40a24ce62f..cf17f5871b 100644 --- a/crates/sqlbuild-cli/src/argv/main/parse_dbt_execution_arguments.rs +++ b/crates/sqlbuild-cli/src/argv/main/parse_dbt_execution_arguments.rs @@ -1,5 +1,7 @@ //! Parse `sqb dbt plan|run|build` arguments as their `argparse` parser did. +use std::collections::BTreeSet; + use sqlbuild_core::dbt_cli::models::DbtParsedFlags; use crate::argv::_helpers::dbt_execution::{PlainHost, parsed_flags}; @@ -29,6 +31,7 @@ pub fn parse_dbt_execution_arguments( writes: Vec::new(), command_prog: None, usage_failure: None, + given: BTreeSet::new(), }; let known: Result = run.known_args(&parser, head.to_vec(), Namespace::default()); diff --git a/crates/sqlbuild-cli/src/argv/models.rs b/crates/sqlbuild-cli/src/argv/models.rs index 35da9d6276..fa77c26b1a 100644 --- a/crates/sqlbuild-cli/src/argv/models.rs +++ b/crates/sqlbuild-cli/src/argv/models.rs @@ -1,6 +1,6 @@ //! The `sqb` command table, parsed values and the outcome of one argv parse. -use std::collections::HashMap; +use std::collections::{BTreeSet, HashMap}; use sqlbuild_core::dbt_cli::models::DbtParsedFlags; @@ -174,6 +174,8 @@ pub struct ArgvOutcome { pub parsed: Result, /// The usage failure that ended the parse, if one did. pub usage_failure: Option, + /// The destination of every option the arguments wrote. + pub given: BTreeSet, } /// Why a `sqb dbt` execution parse stopped before producing flags. diff --git a/crates/sqlbuild-cli/src/argv/tests.rs b/crates/sqlbuild-cli/src/argv/tests.rs index 0289a8fd3d..ce72058936 100644 --- a/crates/sqlbuild-cli/src/argv/tests.rs +++ b/crates/sqlbuild-cli/src/argv/tests.rs @@ -1,3 +1,5 @@ +#[path = "tests/test_given_setting_dests.rs"] +mod given_setting_dests; #[path = "tests/helpers.rs"] mod helpers; #[path = "tests/test_parse_cli_arguments.rs"] diff --git a/crates/sqlbuild-cli/src/argv/tests/test_given_setting_dests.rs b/crates/sqlbuild-cli/src/argv/tests/test_given_setting_dests.rs new file mode 100644 index 0000000000..04035d176e --- /dev/null +++ b/crates/sqlbuild-cli/src/argv/tests/test_given_setting_dests.rs @@ -0,0 +1,54 @@ +use crate::argv::main::given_setting_dests::given_setting_dests; +use crate::argv::tests::test_types::GivenSettingsTestCase; + +#[test] +fn given_a_command_line_when_reading_its_own_settings_then_every_written_option_counts() { + let test_cases = [ + GivenSettingsTestCase { + description: "long options, with = or a separate value, and global options", + argv: &[ + "--no-color", + "build", + "--vars={\"a\": \"1\"}", + "--concurrency", + "3", + "orders_job", + ], + expected_given: Some(&["concurrency", "no_color", "vars"]), + }, + GivenSettingsTestCase { + description: "an abbreviated option names its full option's setting", + argv: &["build", "--concur", "3", "--select", "job:orders_job"], + expected_given: Some(&["concurrency", "select"]), + }, + GivenSettingsTestCase { + description: "a negative flag gives the setting it shares", + argv: &["plan", "--no-ignore-audit-failures", "--no-job-settings"], + expected_given: Some(&["ignore_audit_failures", "job_settings"]), + }, + GivenSettingsTestCase { + description: "a selection alone gives no other setting", + argv: &["build", "orders_job"], + expected_given: Some(&[]), + }, + GivenSettingsTestCase { + description: "a command line that does not parse", + argv: &["build", "--no-such-option"], + expected_given: None, + }, + ]; + for test_case in test_cases { + let argv: Vec = test_case.argv.iter().map(|arg| (*arg).to_owned()).collect(); + let given: Option> = + given_setting_dests(&argv).map(|dests| dests.into_iter().collect()); + let given: Option> = given + .as_ref() + .map(|dests| dests.iter().map(String::as_str).collect()); + assert_eq!( + given.as_deref(), + test_case.expected_given, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-cli/src/argv/tests/test_types.rs b/crates/sqlbuild-cli/src/argv/tests/test_types.rs index 793e4a87a3..68b9999e77 100644 --- a/crates/sqlbuild-cli/src/argv/tests/test_types.rs +++ b/crates/sqlbuild-cli/src/argv/tests/test_types.rs @@ -81,3 +81,10 @@ pub(super) struct JobSettingsRejectionTestCase { pub(super) settings: BTreeMap, pub(super) expected_error: &'static str, } + +/// A command line and the settings it gives itself. +pub(super) struct GivenSettingsTestCase { + pub(super) description: &'static str, + pub(super) argv: &'static [&'static str], + pub(super) expected_given: Option<&'static [&'static str]>, +} diff --git a/crates/sqlbuild-python-planner/src/planner_phases/models.rs b/crates/sqlbuild-python-planner/src/planner_phases/models.rs index 26666a80a2..01d21ffe63 100644 --- a/crates/sqlbuild-python-planner/src/planner_phases/models.rs +++ b/crates/sqlbuild-python-planner/src/planner_phases/models.rs @@ -202,8 +202,7 @@ impl PlanTimings { } } -/// Which models the invocation asks to rebuild from scratch: every model with `--full-refresh`, or -/// the models of the named jobs that set `full_refresh`. +/// The models to rebuild: all with `--full-refresh`, else those of `full_refresh` jobs. #[derive(Clone, Debug, Default)] pub struct FullRefreshRequest { pub all: bool, diff --git a/crates/sqlbuild-python-project/src/compiled_project/constants.rs b/crates/sqlbuild-python-project/src/compiled_project/constants.rs index a0c7f959ec..f0aec16d9e 100644 --- a/crates/sqlbuild-python-project/src/compiled_project/constants.rs +++ b/crates/sqlbuild-python-project/src/compiled_project/constants.rs @@ -10,10 +10,10 @@ pub const REPORT_ATTEMPTS: usize = 3; pub const STORE_BUSY_STATUS: u16 = 503; /// Selector kinds that can name a job: `job:` and a bare name. -pub(crate) const JOB_SELECTOR_KINDS: [&str; 2] = ["job", "name"]; +pub const JOB_SELECTOR_KINDS: [&str; 2] = ["job", "name"]; /// Separates the parts of an intersection selector. -pub(crate) const SELECTOR_INTERSECTION: char = ','; +pub const SELECTOR_INTERSECTION: char = ','; /// The key prefix of a SQL unit test's result, as the snapshot names it. pub const SQL_TEST_KEY_PREFIX: &str = "sql_test:"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs index de9042278b..a27eeb9fab 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs @@ -310,14 +310,13 @@ pub(crate) fn prepare_build_execution<'py>( }) } -/// The build's connections: its limit, or no more than its named jobs' limits add up to when each -/// planned node belongs to a limited job, so a small job never opens connections it cannot use. +/// The build's limit, capped at the sum of its jobs' limits when every planned node has one. fn connection_limit(limit: i64, pipeline: &Bound<'_, PyAny>) -> PyResult { - let limits: Bound<'_, PyAny> = native_field(pipeline, "job_limits")?; - if !limits.is_truthy()? { + let limits: Bound<'_, PyDict> = native_field(pipeline, "job_limits")?.cast_into()?; + if limits.is_empty() { return Ok(limit); } - let node_jobs: Bound<'_, PyAny> = native_field(pipeline, "node_jobs")?; + let node_jobs: Bound<'_, PyDict> = native_field(pipeline, "node_jobs")?.cast_into()?; for key in native_field(&native_field(pipeline, "plan_output")?, "execution_order")?.try_iter()? { @@ -326,8 +325,8 @@ fn connection_limit(limit: i64, pipeline: &Bound<'_, PyAny>) -> PyResult { } } let mut total: i64 = 0; - for value in limits.call_method0("values")?.try_iter()? { - total = total.saturating_add(value?.extract::()?); + for value in limits.values() { + total = total.saturating_add(value.extract::()?); } Ok(limit.min(total.max(1))) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs index 6fdad57ee9..7627073e23 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs @@ -6,7 +6,6 @@ use std::path::PathBuf; use crate::bindings::_helpers::audit_command::command::run_audit_command; use crate::bindings::_helpers::build_command::command::run_build_command; use crate::bindings::_helpers::check::check_command::run_check_command; -use crate::bindings::_helpers::cli::named_job_settings::apply_named_job_settings; use crate::bindings::_helpers::clone::clone_command::run_clone_command; use crate::bindings::_helpers::command_runtime::warehouse_scope::command_warehouse_scope; use crate::bindings::_helpers::command_runtime::warehouses::validate_cli_warehouse; @@ -499,7 +498,6 @@ fn routed_command<'py>( ) -> PyResult> { let py: Python<'py> = args.py(); let invocation = Invocation::new(args)?; - apply_named_job_settings(args, &invocation.select.extract::>()?)?; let _secrets = MissingSecretsScope::enter( args.getattr_opt(ALLOW_MISSING_SECRETS_DEST)? .is_some_and(|allowed| allowed.is_truthy().unwrap_or(false)), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs index 8636c86891..3ee4eb0383 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs @@ -25,6 +25,7 @@ use crate::bindings::_helpers::cli::dispatch::{command_facts, dispatch_command}; use crate::bindings::_helpers::cli::failure_cause::take_failure_cause; use sqlbuild_python_base::cli::main::offer_launch_record::offer_launch_record; use crate::bindings::_helpers::cli::machine_output::take_machine_output_written; +use crate::bindings::_helpers::cli::named_job_settings::job_settings_argv; use crate::bindings::_helpers::cli::observability::{ CliObservabilityScope, cli_observability_scope, }; @@ -797,6 +798,11 @@ fn external_context(py: Python<'_>) -> PyResult> { } } +/// The attributes a parse set on its namespace. +fn namespace_of<'py>(args: &Bound<'py, PyAny>) -> PyResult> { + Ok(args.getattr("__dict__")?.cast_into::()?) +} + fn parsed_invocation<'py>( argv: &Bound<'py, PyAny>, handlers: Option>, @@ -805,7 +811,7 @@ fn parsed_invocation<'py>( let py: Python<'py> = argv.py(); let use_color: bool = !argv.contains(NO_COLOR_OPTION)? && stdout_supports_color(py)?; let parsed: Bound<'py, PyAny> = parse_cli_invocation(py, argv.extract()?, use_color)?; - let args: Bound<'py, PyAny> = native_field(&parsed, "args")?; + let mut args: Bound<'py, PyAny> = native_field(&parsed, "args")?; if args.is_none() { let exit_code: Bound<'py, PyAny> = native_field(&parsed, "exit_code")?; return if exit_code.is_none() { @@ -814,7 +820,18 @@ fn parsed_invocation<'py>( Ok(exit_code) }; } - let namespace = args.getattr("__dict__")?.cast_into::()?; + let mut namespace: Bound<'py, PyDict> = namespace_of(&args)?; + if let Some(job_argv) = job_settings_argv(argv, &namespace)? { + let job_argv: Vec> = job_argv + .iter() + .map(|argument| PyString::new(py, argument)) + .collect(); + args = native_field(&parse_cli_invocation(py, job_argv, use_color)?, "args")?; + if args.is_none() { + return exit(py, 1); + } + namespace = namespace_of(&args)?; + } let command: Option = namespace .get_item("command")? .map(|value| value.extract::>()) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/named_job_settings.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/named_job_settings.rs index 754cadd525..ec362ee046 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/named_job_settings.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/named_job_settings.rs @@ -1,96 +1,121 @@ -//! A build or plan that names jobs takes their build-wide settings where its own command line is silent. +//! A build or plan naming jobs takes their build-wide settings where its command line is silent. -use std::collections::BTreeMap; +use std::collections::{BTreeMap, BTreeSet}; +use std::path::{Path, PathBuf}; -use pyo3::exceptions::PyValueError; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; -use pyo3::types::PyString; +use pyo3::exceptions::{PyOSError, PyRuntimeError, PyUnicodeError, PyValueError}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods}; use serde_json::Value; use sqlbuild_cli::argv::main::accepted_settings_arguments::accepted_settings_arguments; +use sqlbuild_cli::argv::main::given_setting_dests::given_setting_dests; +use sqlbuild_cli::dispatch::main::selector_file_selectors::selector_file_selectors; +use sqlbuild_discovery::constants::ORCHESTRATION_DIRECTORY; -use crate::bindings::_helpers::cli::argv::parse_cli_invocation; -use crate::bindings::_helpers::compiled_project::job_settings::named_job_build_settings; -use sqlbuild_python_base::records::main::fields::field::field as native_field; +use crate::bindings::_helpers::compiled_project::job_settings::{ + could_name_job, named_job_build_settings, +}; +use sqlbuild_python_base::records::main::construct::error_class::error_class; +use sqlbuild_python_base::records::types::ErrorId; /// The commands whose selections take the settings of the jobs they name. const JOB_SETTING_COMMANDS: [&str; 2] = ["build", "plan"]; -/// The namespace `sqb ` parses to. -fn parsed<'py>( - py: Python<'py>, - command: &str, - arguments: &[String], -) -> PyResult> { - let argv: Vec> = std::iter::once(command) - .chain(arguments.iter().map(String::as_str)) - .map(|argument| PyString::new(py, argument)) - .collect(); - let args: Bound<'py, PyAny> = native_field(&parse_cli_invocation(py, argv, false)?, "args")?; - if args.is_none() { - return Err(PyValueError::new_err(format!( - "job settings do not parse as sqb {command} options: {}", - arguments.join(" ") - ))); +/// One parsed argument, `None` when unset. +fn argument<'py, T: pyo3::FromPyObject<'py>>( + namespace: &Bound<'py, PyDict>, + name: &str, +) -> PyResult> { + Ok(namespace + .get_item(name)? + .map(|value| value.extract::>()) + .transpose()? + .flatten()) +} + +/// The `--select` selectors and those of each `--select-file`. +fn selectors(namespace: &Bound<'_, PyDict>) -> PyResult> { + let mut selectors: Vec = argument(namespace, "select")?.unwrap_or_default(); + let files: Vec = argument(namespace, "select_file")?.unwrap_or_default(); + for path in files { + let text: String = std::fs::read_to_string(&path) + .map_err(|error| PyOSError::new_err(error.to_string()))?; + selectors.extend(selector_file_selectors(&text)); } - Ok(args) + Ok(selectors) } -/// Set on `args` each build-wide setting of the jobs `selectors` name that its command line left -/// at the default, exactly as the same flags typed on the command line would. -pub(crate) fn apply_named_job_settings( - args: &Bound<'_, PyAny>, - selectors: &[String], -) -> PyResult<()> { - let py: Python<'_> = args.py(); - let Some(command) = args.getattr("command")?.extract::>()? else { - return Ok(()); +/// The project directory a command runs in, resolved as dispatch resolves it. +fn project_dir(namespace: &Bound<'_, PyDict>) -> PyResult { + let os_error = |error: std::io::Error| PyOSError::new_err(error.to_string()); + let Some(raw): Option = argument(namespace, "project_dir")? else { + return std::env::current_dir().map_err(os_error); }; - if !JOB_SETTING_COMMANDS.contains(&command.as_str()) - || selectors.is_empty() - || !args.getattr("job_settings")?.is_truthy()? - { - return Ok(()); - } - let raw_project_dir: Bound<'_, PyAny> = args.getattr("project_dir")?; - let path = py.import("pathlib")?.getattr("Path")?; - let project_dir: Bound<'_, PyAny> = if raw_project_dir.is_none() { - path.call_method0("cwd")? - } else { - path.call1((raw_project_dir,))?.call_method0("resolve")? + std::fs::canonicalize(&raw).or_else(|_| std::path::absolute(&raw).map_err(os_error)) +} + +/// `argv` plus the flags it leaves out of its named jobs' build-wide settings; `None` if unchanged. +fn named_job_argv( + py: Python<'_>, + argv: &[String], + namespace: &Bound<'_, PyDict>, +) -> PyResult>> { + let Some(command): Option = argument(namespace, "command")? else { + return Ok(None); }; - let target: Option = args.getattr("target")?.extract()?; + let job_settings: bool = argument(namespace, "job_settings")?.unwrap_or(false); + if !JOB_SETTING_COMMANDS.contains(&command.as_str()) || !job_settings { + return Ok(None); + } + let selectors: Vec = selectors(namespace)?; + if !selectors.iter().any(|selector| could_name_job(selector)) { + return Ok(None); + } + let directory: PathBuf = project_dir(namespace)?; + if !Path::new(&directory).join(ORCHESTRATION_DIRECTORY).is_dir() { + return Ok(None); + } + let target: Option = argument(namespace, "target")?; let settings: BTreeMap = - named_job_build_settings(py, selectors, (&project_dir, target))?; + named_job_build_settings(py, &selectors, (&directory, target))?; if settings.is_empty() { - return Ok(()); - } - let defaults: Bound<'_, PyAny> = parsed(py, &command, &[])?; - let mut silent: BTreeMap = BTreeMap::new(); - for (name, value) in settings { - if !args.hasattr(name.as_str())? - || args - .getattr(name.as_str())? - .eq(defaults.getattr(name.as_str())?)? - { - silent.insert(name, value); - } + return Ok(None); } + let given: BTreeSet = given_setting_dests(argv).ok_or_else(|| { + PyRuntimeError::new_err("cannot tell which settings the command line gives") + })?; + let silent: BTreeMap = settings + .into_iter() + .filter(|(name, _)| !given.contains(name)) + .collect(); let flags: Vec = accepted_settings_arguments(&command, &silent).map_err(PyValueError::new_err)?; if flags.is_empty() { - return Ok(()); + return Ok(None); } - let applied: Bound<'_, PyAny> = parsed(py, &command, &flags)?; - for (name, value) in applied - .getattr("__dict__")? - .call_method0("items")? - .try_iter()? - .map(|item| item.and_then(|item| item.extract::<(String, Bound<'_, PyAny>)>())) - .collect::>>()? - { - if !value.eq(defaults.getattr(name.as_str())?)? { - args.setattr(name.as_str(), value)?; - } + Ok(Some(argv.iter().cloned().chain(flags).collect())) +} + +/// Whether the command meets `error` again, and reports it, when it runs. +fn reported_when_run(py: Python<'_>, error: &PyErr) -> PyResult { + Ok(error.is_instance(py, &error_class(py, ErrorId::CliUser)?) + || error.is_instance_of::(py) + || error.is_instance_of::(py)) +} + +/// The argv to parse again with its named jobs' build-wide flags, or `None` to keep the parse. +pub(crate) fn job_settings_argv( + argv: &Bound<'_, PyAny>, + namespace: &Bound<'_, PyDict>, +) -> PyResult>> { + let py: Python<'_> = argv.py(); + let arguments: Vec = match argv.extract() { + Ok(arguments) => arguments, + Err(undecodable) if undecodable.is_instance_of::(py) => return Ok(None), + Err(error) => return Err(error), + }; + match named_job_argv(py, &arguments, namespace) { + Err(error) if reported_when_run(py, &error)? => Ok(None), + result => result, } - Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/job_node_settings.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/job_node_settings.rs index d4a9405235..5378a611e2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/job_node_settings.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/job_node_settings.rs @@ -6,8 +6,10 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyBool, PyDict, PyDictMethods, PyFrozenSet, PyTuple}; use serde_json::Value; use sqlbuild_analysis::assembly::project::types::ObjectKey; +use sqlbuild_analysis::graph::main::selectors_need_jobs::selectors_need_jobs; use sqlbuild_analysis::job_settings::constants::CONFLICT_HELP; use sqlbuild_analysis::job_settings::errors::{JobSettingConflict, NodeJobSettingsError}; +use sqlbuild_analysis::job_settings::main::build_job_settings::build_job_settings; use sqlbuild_analysis::job_settings::main::named_jobs::named_jobs; use sqlbuild_analysis::job_settings::main::node_job_settings::node_job_settings; use sqlbuild_analysis::job_settings::models::{DeclaredRun, NodeJobSettings}; @@ -35,8 +37,7 @@ fn strings(value: &Bound<'_, PyAny>) -> PyResult> { value.try_iter()?.map(|item| item?.extract()).collect() } -/// The named jobs' settings for each node `options` selects; `None` without named jobs, with -/// `--no-job-settings`, or when the selection fails, which planning then reports itself. +/// The named jobs' settings per selected node; `None` when ignored, unnamed or unselectable. pub(crate) fn planned_job_settings( project: &Bound<'_, PyAny>, options: &Bound<'_, PyAny>, @@ -46,16 +47,22 @@ pub(crate) fn planned_job_settings( return Ok(None); } let select: Vec = strings(&native_field(options, "select")?)?; + let native: Bound<'_, NativeProjectGraph> = + native_field(project, "lineage_graph")?.cast_into()?; + let native = native.get(); + if !selectors_need_jobs(native.graph(), &select, &[]) { + return Ok(None); + } let exclude: Vec = strings(&native_field(options, "exclude")?)?; let Some(declared): Option> = job_declarations(py)? else { return Ok(None); }; - if named_jobs(&select, &declared).is_empty() { + let named: Vec = named_jobs(&select, &declared); + if named.is_empty() { return Ok(None); } - let native: Bound<'_, NativeProjectGraph> = - native_field(project, "lineage_graph")?.cast_into()?; - let native = native.get(); + build_job_settings(&named, &declared) + .map_err(|conflict| job_setting_conflict(py, &conflict))?; let all: Vec = select.iter().chain(&exclude).cloned().collect(); let graph = graph_with_jobs(py, native.graph(), &all, &[])?; match node_job_settings(&graph, (&select, &exclude), &declared) { @@ -90,8 +97,7 @@ impl PlannedJobSettings { PyFrozenSet::new(py, names) } - /// Each named job's limit and each planned node's limited jobs, a SQL test taking its models' - /// jobs; none when `--concurrency` overrides them. + /// Each job's limit and each planned node's limited jobs; none under `--concurrency`. pub(crate) fn job_limits<'py>( &self, plan: &Bound<'py, PyAny>, @@ -141,8 +147,7 @@ impl PlannedJobSettings { Ok((limits, node_jobs)) } - /// The plan without the SQL tests whose models' jobs set `run_tests = false`, and with each audit - /// of a named job's node ignoring failures as that job says, unless the command line decided. + /// The plan with its jobs' `run_tests` and, unless the command line decided, audit failures. pub(crate) fn apply_to_plan<'py>( &self, plan: &Bound<'py, PyAny>, @@ -230,7 +235,8 @@ impl PlannedJobSettings { scope_deps.py(), &JobSettingConflict { message: format!( - "test {} covers nodes of jobs {other} and {job}, which set run_tests differently ({earlier} and {value})", + "test {} covers nodes of jobs {other} and {job}, which set \ + run_tests differently ({earlier} and {value})", test.1 ), help: CONFLICT_HELP, @@ -244,14 +250,14 @@ impl PlannedJobSettings { Ok(seen.is_some_and(|(_, value)| value == &Value::Bool(false))) } - /// The named job's `ignore_audit_failures` for the node an audit is attached to, if it sets one. + /// The `ignore_audit_failures` the audited node's job sets, if any. fn audit_ignores(&self, entry: &Bound<'_, PyAny>) -> PyResult> { let kind: Bound<'_, PyAny> = native_field(entry, "attached_target_kind")?; let name: Bound<'_, PyAny> = native_field(entry, "attached_target_name")?; if kind.is_none() || name.is_none() { return Ok(None); } - let kind: String = kind.getattr("value")?.extract()?; + let kind: String = native_field(&kind, "value")?.extract()?; let key: ObjectKey = (kind, name.extract()?); Ok(self .value(&key, IGNORE_AUDIT_FAILURES) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/job_settings.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/job_settings.rs index a41a21647d..52d0dfe164 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/job_settings.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/job_settings.rs @@ -1,22 +1,25 @@ //! The settings named jobs give a command: the build-wide ones as the flags the command line left out. use std::collections::BTreeMap; -use std::path::PathBuf; +use std::path::Path; +use pyo3::IntoPyObject; use pyo3::PyErr; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; -use pyo3::types::{PyDict, PyDictMethods}; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; use serde_json::Value; +use sqlbuild_analysis::graph::main::parse_selector::parse_selector; +use sqlbuild_analysis::graph::models::ParsedSelector; use sqlbuild_analysis::job_settings::constants::JOB_SETTING_CONFLICT_CODE; use sqlbuild_analysis::job_settings::errors::JobSettingConflict; use sqlbuild_analysis::job_settings::main::build_job_settings::build_job_settings; use sqlbuild_analysis::job_settings::main::named_jobs::named_jobs; -use sqlbuild_python_base::records::main::construct::error_class::error_class; +use sqlbuild_python_base::records::main::construct::new_error::new_error; use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_front::discovery::main::project::configured_changes::configured_changes; use sqlbuild_python_project::compiled_project::constants::{ - CHANGED_SETTING, JOB_RUN_COUNTS, JOB_RUN_REACH, REACH_SETTING, + CHANGED_SETTING, JOB_RUN_COUNTS, JOB_RUN_REACH, JOB_SELECTOR_KINDS, REACH_SETTING, + SELECTOR_INTERSECTION, }; use sqlbuild_python_project::compiled_project::main::job_declarations::job_declarations; use sqlbuild_python_project::compiled_project::main::with_job_scope::with_job_scope; @@ -42,16 +45,14 @@ pub(crate) fn job_run_settings( Ok(settings) } -/// The build-wide settings the jobs `selectors` name give a build, as job runs take them; a -/// conflict between named jobs is a `CliUserError` naming them. +/// The build-wide settings the named jobs give a build, as their runs take them, or `S027`. pub(crate) fn named_job_build_settings( py: Python<'_>, selectors: &[String], - (project_dir, selected_target): (&Bound<'_, PyAny>, Option), + (directory, selected_target): (&Path, Option), ) -> PyResult> { - let directory: PathBuf = project_dir.extract()?; - let Some(declared) = with_job_scope((directory, selected_target), || job_declarations(py))? - else { + let scope = (directory.to_path_buf(), selected_target); + let Some(declared) = with_job_scope(scope, || job_declarations(py))? else { return Ok(BTreeMap::new()); }; let named: Vec = named_jobs(selectors, &declared); @@ -60,19 +61,26 @@ pub(crate) fn named_job_build_settings( } let agreed: BTreeMap = build_job_settings(&named, &declared) .map_err(|conflict| job_setting_conflict(py, &conflict))?; - job_run_settings(project_dir, agreed) + job_run_settings(&directory.into_pyobject(py)?.into_any(), agreed) +} + +/// Whether a selector has an atom that can name a job: `job:`, or a bare name. +pub(crate) fn could_name_job(selector: &str) -> bool { + selector + .split_whitespace() + .flat_map(|token| token.split(SELECTOR_INTERSECTION)) + .any(|atom| match parse_selector(atom) { + Ok(ParsedSelector::Kind { kind, .. }) => JOB_SELECTOR_KINDS.contains(&kind.as_str()), + _ => false, + }) } /// `CliUserError` for job settings that cannot apply together. pub(crate) fn job_setting_conflict(py: Python<'_>, conflict: &JobSettingConflict) -> PyErr { - let raised = (|| { - let keywords: Bound<'_, PyDict> = PyDict::new(py); - keywords.set_item("code", JOB_SETTING_CONFLICT_CODE)?; - keywords.set_item("help", conflict.help)?; - error_class(py, ErrorId::CliUser)?.call((conflict.message.as_str(),), Some(&keywords)) - })(); - match raised { - Ok(error) => PyErr::from_value(error), - Err(error) => error, - } + new_error( + py, + ErrorId::CliUser, + &conflict.message, + (Some(JOB_SETTING_CONFLICT_CODE), Some(conflict.help)), + ) } diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/build.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/build.md index ab495b4a1d..30c069f116 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/build.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/build.md @@ -60,6 +60,7 @@ own option. `--version` works only directly after `sqb`, as in `sqb --version`: | `--no-sql-analysis` | Disable compile-time SQL analysis (`--no-sql-validation` is an alias) | | `--no-cache` | Bypass the reusable compile-analysis cache for this invocation | | `--full-refresh` | Drop and rebuild selected models unless a model sets `full_refresh false`; `full_refresh true` forces a model even without this flag | +| `--no-job-settings` | Ignore the settings of the jobs the selection names; see [Settings when a build names a job](../concepts/orchestration.md#settings-when-a-build-names-a-job) | | `--changed` | Build only the selected models and seeds that changed, and what they reach downstream; `--changed=definition,source_data` picks what counts. See [Building only what changed](#building-only-what-changed) | | `--reach` | With `--changed`: `job` stops at models another scheduled job refreshes, `all` builds everything downstream | | `--defer-to` | Resolve unselected model references against another target | diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/plan.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/plan.md index 66d69eead9..2cce0a9bf2 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/plan.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/plan.md @@ -41,6 +41,7 @@ Positional `SELECTOR` arguments work like `--select`, and both forms can be comb | `--ignore-audit-failures`, `--no-ignore-audit-failures` | Plan the build `sqb build` would run with the same flag; a plan whose audit failures are ignored starts with `Audit failures won't stop this build`. See [Ignoring audit failures](../concepts/audits.md#ignoring-audit-failures) | | `--selection-diagnostics` | Warn when a selected model will build on changed upstream models that are not selected; see [Selection and staleness](../concepts/planning/selection-and-staleness.md) | | `--full-refresh` | Plan a full rebuild for selected models unless a model sets `full_refresh false`; `full_refresh true` forces a model without this flag | +| `--no-job-settings` | Ignore the settings of the jobs the selection names; see [Settings when a build names a job](../concepts/orchestration.md#settings-when-a-build-names-a-job) | | `--changed` | Show what `sqb build --changed` would run, which nodes it holds back and why; takes `--changed=definition,source_data` and `--reach` like [`sqb build`](build.md#building-only-what-changed). In `--json`, the `changed` object lists `runs`, `held_back` and `unchanged` | | `--start-cursor` | Override the start cursor, typed from the selected incremental models' cursor type; fails with `S303` if the selection mixes timestamp and integer cursors. With no cursor model selected, an integer value is read as an integer cursor and anything else as an ISO timestamp | | `--end-cursor` | Override the end cursor, typed the same way as `--start-cursor` | diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/concurrency.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/concurrency.md index 957db31cb5..fe5dc414a9 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/concurrency.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/concurrency.md @@ -46,7 +46,10 @@ concurrency = 2 ``` `--concurrency` on the command line wins over all of them; see -[Target concurrency](project-configuration.md#target-concurrency) for the full order. The job's page shows the value its runs use on the target and where it came from. +[Target concurrency](project-configuration.md#target-concurrency) for the full order. +A job's `concurrency` limits how many of that job's nodes build at once in a build that names the +job, within the build's own limit. When one job is at its limit, a ready node of another job starts +instead; see [Settings when a build names a job](orchestration.md#settings-when-a-build-names-a-job). The job's page shows the value its runs use on the target and where it came from. ## Runs at once on a server: `max_workers` diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/orchestration.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/orchestration.md index 4b6cc3d451..ab90288647 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/orchestration.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/orchestration.md @@ -98,6 +98,39 @@ cpus = 2 A job's `[settings]` cannot set `select` or `exclude`: the job's own selection is the run's selection. +### Settings when a build names a job + +A build or plan whose selection names a job, as `job:nightly_orders_job` or the bare name, takes the +job's `[settings]`. Selecting the same nodes some other way, such as by tag, takes none of them. +Naming a group names each of its jobs. The first value set wins: + +1. A flag on the command line, such as `--vars` or `--concurrency` +2. The named job's setting +3. The target's setting +4. The project's setting + +`--no-job-settings` leaves out every job's settings. + +Some settings apply to the job's own nodes only, so a build that names two jobs can treat their +nodes differently: + +| Setting | Applies to | +| --- | --- | +| `full_refresh` | The job's incremental models, rebuilt from scratch | +| `run_tests` | The SQL unit tests of the job's models | +| `ignore_audit_failures` | The audits of the job's nodes | +| `concurrency` | How many of the job's nodes build at once. The build keeps its own limit too, and a node in two named jobs fits within both | +| `[retry]` | The job's nodes | + +A job's settings reach only the nodes its own selection holds. `--select job:nightly_orders_job,orders` +gives them to `orders` alone, and `+job:nightly_orders_job` builds the job's upstream nodes without +them. + +Every other setting, such as `vars` or `changed`, applies to the whole build, so every named job +must set it the same way. A group whose jobs set different `vars` or `changed` fails to compile. +Two named jobs that give one node different values of the same setting fail the build with `S027`, +naming the node, both jobs and the setting. A setting only one of them sets is no conflict. + Select a pipeline by its folder or a tag rather than listing its nodes. Rule `SQBRORCHESTRATION101` reports a job or sensor `select` that names more than three nodes one by one; see [Orchestration Rules](rules/orchestration-rules.md). @@ -283,6 +316,7 @@ Every problem with an orchestration file is a `D017` error at its file, line and | `job nightly_orders_job resources: cpus must be a positive number` | Set `cpus` above zero | | `job 'nightly_orders_job' uses pool 'warehouse_hevy', which [pools] in sqlbuild_project.toml does not declare` | Fix the name, or declare the pool in [`[pools]`](project-configuration.md#pools) | | `job 'nightly_orders_job' takes 2 slots of pool 'supplier_api', more than its limit of 1 on target 'production'` | Take fewer slots, or raise the limit in `[targets.production.pools]` | +| `jobs orders_job and crm_job set vars differently; a group runs its jobs as one build, which has one vars` | Give the group's jobs the same `vars` and `changed`, or move one job to another group | | `sensor 'orders_sensor' is declared here, but importing the module does not define it` | Assign the sensor to a module-level name or list, not a dictionary or function body | | `group dependencies contain a cycle in job group daily_orders_job_group: orders_marts_job waits for staging_orders_job (model:orders reads model:stg_orders); staging_orders_job waits for orders_marts_job (model:stg_orders reads seed:regions)` | Move each node to one job, and give seeds and nodes that several members read their own job that runs first | @@ -292,6 +326,9 @@ Details: a job selector never imports a calendar or sensor module. - A job or group declared with `targets` is selectable only when the command's target is one of them, as on the server. +- Job settings that cannot apply together fail the build or plan with `S027`, such as `model:orders + is selected through jobs orders_job and finance_job, which set run_tests differently (false and + true)`. Make the jobs agree, set it in one of them, pass the flag, or use `--no-job-settings`. - Selectors are checked for syntax at compile. A selector that names a node the project does not have fails when a location publishes, such as `job nightly_orders_job: unknown selector name 'order_total'`. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md index 539bf642a3..872c80e52c 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md @@ -275,9 +275,11 @@ concurrency = 8 value set in this order: 1. `--concurrency` on the command line, for that invocation only -2. The selected target's `concurrency`, from `[targets.]` in either file -3. `[settings] concurrency` in `sqlbuild_local.toml` -4. `[settings] concurrency` in `sqlbuild_project.toml` (default `1`) +2. For `sqb build`, a named job's `concurrency`, which limits that job's nodes within the build's + own limit; see [Settings when a build names a job](orchestration.md#settings-when-a-build-names-a-job) +3. The selected target's `concurrency`, from `[targets.]` in either file +4. `[settings] concurrency` in `sqlbuild_local.toml` +5. `[settings] concurrency` in `sqlbuild_project.toml` (default `1`) The value must be an integer of at least `1`; anything else fails with `D001`. `sqb build --verbose` shows the effective limit and the setting it came from, for example diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/selectors.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/selectors.md index 9a25d769ef..e5af77df77 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/selectors.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/selectors.md @@ -108,6 +108,8 @@ sqb build --select "job:nightly_orders_job,tag:finance" ``` A job selects its own `select` minus its own `exclude`. A job group selects the union of its jobs. +Naming a job also applies its settings to its nodes; see +[Settings when a build names a job](orchestration.md#settings-when-a-build-names-a-job). A job or job group declared with `targets` is selectable only when the command's target is one of them, as on the server. Job selectors compose with `+`, `,` and `--exclude` like any other selector. A job's own selection diff --git a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py index a07f1a708e..7ab20b846b 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py @@ -1475,3 +1475,26 @@ class SeedRunRecordTestCase: description: str # Each row as (phase, command, select_json, named_jobs_json, var_keys_json, status, counts). expected_rows: list[tuple[str, str, str, str, str, str | None, str | None]] + + +@dataclass(frozen=True) +class JobSettingsBuildTestCase: + description: str + build_args: tuple[str, ...] + expected_amount: int + expected_fragments: tuple[str, ...] + + +@dataclass(frozen=True) +class JobSettingsConflictTestCase: + description: str + command_args: tuple[str, ...] + expected_fragment: str + extra_files: dict[str, str] = field(default_factory=dict) + + +@dataclass(frozen=True) +class JobFullRefreshTestCase: + description: str + build_args: tuple[str, ...] + expected_counts: tuple[tuple[int, int], ...] diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_job_settings.py b/tests/integration/src/sqlbuild/cli/commands/main/test_job_settings.py new file mode 100644 index 0000000000..25b9628e3e --- /dev/null +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_job_settings.py @@ -0,0 +1,231 @@ +"""A build that names jobs takes their settings: build-wide ones once, node-level ones per node.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from sqlbuild._native import run_cli as main +from tests.integration.src.sqlbuild.cli.commands.main._test_types import ( + JobFullRefreshTestCase, + JobSettingsBuildTestCase, + JobSettingsConflictTestCase, +) +from tests.integration.src.sqlbuild.cli.commands.main.helpers import ( + query_duckdb_rows, + write_project_files, +) + +_PROJECT_FILES: dict[str, str] = { + "sqlbuild_project.toml": ( + 'name = "demo"\nadapter = "duckdb"\ndefault_target = "dev"\n\n' + '[connection]\ndatabase = "demo.duckdb"\n\n[targets.dev]\n\n[vars]\namount = "1"\n' + ), + "models/stg_orders.sql": ( + "MODEL (description 'Staged orders.', materialized table);\n" + "SELECT CAST(@@amount AS INTEGER) AS order_id\n" + ), + "models/orders.sql": ( + "MODEL (description 'Orders.', materialized table);\n" + 'SELECT order_id FROM __ref("stg_orders")\n' + ), + "models/customers.sql": ( + "MODEL (description 'Customers.', materialized table);\nSELECT 7 AS customer_id\n" + ), + "models/order_events.sql": ( + "MODEL (description 'Order events.', materialized incremental, " + "incremental_strategy append);\nSELECT 1 AS event_id\n" + ), + "models/refund_events.sql": ( + "MODEL (description 'Refund events.', materialized incremental, " + "incremental_strategy append);\nSELECT 1 AS event_id\n" + ), + "tests/unit/test_orders.sql": ( + "TEST();\n\nWITH\n__ref__stg_orders AS (SELECT 4 AS order_id),\n" + "__expected__orders AS (SELECT 4 AS order_id)\nSELECT 1\n" + ), + "orchestration/jobs/orders_job.toml": ( + 'name = "orders_job"\ndescription = "Orders."\nselect = "+orders"\n\n' + '[settings]\nconcurrency = 1\nrun_tests = false\nvars = { amount = "5" }\n' + ), + "orchestration/jobs/checked_orders_job.toml": ( + 'name = "checked_orders_job"\ndescription = "Orders, tested."\nselect = "orders"\n\n' + '[settings]\nrun_tests = true\nvars = { amount = "5" }\n' + ), + "orchestration/jobs/crm_job.toml": ( + 'name = "crm_job"\ndescription = "Customers."\nselect = "customers"\n\n' + '[settings]\nvars = { amount = "9" }\n' + ), + "orchestration/jobs/events_job.toml": ( + 'name = "events_job"\ndescription = "Events."\nselect = "order_events"\n\n' + "[settings]\nfull_refresh = true\n" + ), +} + + +@pytest.mark.parametrize( + "test_case", + [ + JobSettingsBuildTestCase( + description="a named job's vars, concurrency and run_tests apply", + build_args=("--select", "job:orders_job"), + expected_amount=5, + expected_fragments=("(concurrency: 1)", "TOTAL=2"), + ), + JobSettingsBuildTestCase( + description="a bare job name names the job", + build_args=("--select", "orders_job"), + expected_amount=5, + expected_fragments=("(concurrency: 1)", "TOTAL=2"), + ), + JobSettingsBuildTestCase( + description="command-line flags beat the job's settings", + build_args=( + "--select", + "job:orders_job", + "--vars", + '{"amount": "2"}', + "--concurrency", + "3", + ), + expected_amount=2, + expected_fragments=("(concurrency: 3)", "TOTAL=2"), + ), + JobSettingsBuildTestCase( + description="an abbreviated command-line flag still beats the job's setting", + build_args=("--select", "job:orders_job", "--concur", "2"), + expected_amount=5, + expected_fragments=("(concurrency: 2)", "TOTAL=2"), + ), + JobSettingsBuildTestCase( + description="--no-job-settings builds with the project's settings", + build_args=("--select", "job:orders_job", "--no-job-settings"), + expected_amount=1, + expected_fragments=("TOTAL=3",), + ), + JobSettingsBuildTestCase( + description="selecting the job's nodes without naming it takes none of its settings", + build_args=("--select", "+orders"), + expected_amount=1, + expected_fragments=("TOTAL=3",), + ), + ], + ids=lambda case: case.description, +) +def test_given_a_named_job_when_building_then_its_settings_apply_below_the_command_line( + test_case: JobSettingsBuildTestCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + write_project_files(project_dir=tmp_path, files=_PROJECT_FILES) + + exit_code: int = main( + ["--project-dir", str(tmp_path), "--no-color", "build", *test_case.build_args] + ) + captured: str = "".join(capsys.readouterr()) + + assert exit_code == 0, captured + assert query_duckdb_rows( + db_path=tmp_path / "demo.duckdb", sql="SELECT order_id FROM orders" + ) == ((test_case.expected_amount,),) + assert {fragment: fragment in captured for fragment in test_case.expected_fragments} == ( + dict.fromkeys(test_case.expected_fragments, True) + ), captured + + +@pytest.mark.parametrize( + "test_case", + [ + JobSettingsConflictTestCase( + description="two named jobs with different vars cannot build together", + command_args=("build", "--select", "job:orders_job", "job:crm_job"), + expected_fragment=( + 'error[S027]: jobs orders_job and crm_job set vars differently ({"amount":"5"} ' + 'and {"amount":"9"}); a build applies vars to everything it runs' + ), + ), + JobSettingsConflictTestCase( + description="a plan naming jobs with different vars fails the same way", + command_args=("plan", "--select", "job:orders_job", "job:crm_job"), + expected_fragment=( + 'error[S027]: jobs orders_job and crm_job set vars differently ({"amount":"5"} ' + 'and {"amount":"9"}); a build applies vars to everything it runs' + ), + ), + JobSettingsConflictTestCase( + description="two named jobs giving one node different run_tests conflict", + command_args=("build", "--select", "job:orders_job", "job:checked_orders_job"), + expected_fragment=( + "error[S027]: model:orders is selected through jobs orders_job and " + "checked_orders_job, which set run_tests differently (false and true)" + ), + ), + JobSettingsConflictTestCase( + description="a group whose jobs set different vars does not compile", + command_args=("compile",), + expected_fragment=( + "jobs orders_job and crm_job set vars differently; a group runs its jobs as one " + "build, which has one vars" + ), + extra_files={ + "orchestration/job_groups/daily_job_group.toml": ( + 'name = "daily_job_group"\ndescription = "Daily."\n' + 'jobs = ["orders_job", "crm_job"]\n' + ), + }, + ), + ], + ids=lambda case: case.description, +) +def test_given_conflicting_job_settings_when_running_then_the_command_fails_naming_them( + test_case: JobSettingsConflictTestCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + write_project_files(project_dir=tmp_path, files={**_PROJECT_FILES, **test_case.extra_files}) + + exit_code: int = main(["--project-dir", str(tmp_path), "--no-color", *test_case.command_args]) + captured: str = "".join(capsys.readouterr()) + + assert exit_code == 1, captured + assert test_case.expected_fragment in captured.replace("\n", " ").replace(" ", " "), captured + + +@pytest.mark.parametrize( + "test_case", + [ + JobFullRefreshTestCase( + description="a named job's full_refresh rebuilds only its own incremental models", + build_args=("--select", "job:events_job", "refund_events"), + expected_counts=((1, 3),), + ), + JobFullRefreshTestCase( + description="--full-refresh rebuilds every selected incremental model", + build_args=("--select", "job:events_job", "refund_events", "--full-refresh"), + expected_counts=((1, 1),), + ), + ], + ids=lambda case: case.description, +) +def test_given_a_named_job_with_full_refresh_when_building_then_only_its_models_start_over( + test_case: JobFullRefreshTestCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + write_project_files(project_dir=tmp_path, files=_PROJECT_FILES) + arguments: list[str] = ["--project-dir", str(tmp_path), "--no-color", "build", "--no-tests"] + first: int = main([*arguments, "--select", "order_events", "refund_events"]) + second: int = main([*arguments, "--select", "order_events", "refund_events"]) + + exit_code: int = main([*arguments, *test_case.build_args]) + captured: str = "".join(capsys.readouterr()) + + assert (first, second, exit_code) == (0, 0, 0), captured + assert ( + query_duckdb_rows( + db_path=tmp_path / "demo.duckdb", + sql="SELECT (SELECT count(*) FROM order_events), (SELECT count(*) FROM refund_events)", + ) + == test_case.expected_counts + ) diff --git a/tests/unit/src/sqlbuild/executor/build/classes/_test_types.py b/tests/unit/src/sqlbuild/executor/build/classes/_test_types.py index 357a1fdaae..9c8563cc62 100644 --- a/tests/unit/src/sqlbuild/executor/build/classes/_test_types.py +++ b/tests/unit/src/sqlbuild/executor/build/classes/_test_types.py @@ -37,3 +37,18 @@ class SchedulerIdentityTestCase: description: str expected_invocation_id: str expected_run_id: str + + +@dataclass(frozen=True) +class JobLimitDispatchTestCase: + """Ready nodes, each node's limiting jobs and their limits, and the dispatch order expected.""" + + description: str + ready: tuple[str, ...] + node_jobs: dict[str, tuple[str, ...]] + job_limits: dict[str, int] + blocked: frozenset[str] + # Before each dispatch, the running node that finishes first, or "" for none. + finished_before: tuple[str, ...] + # Each dispatch's node, or None when no ready node has room. + expected_dispatched: tuple[str | None, ...] diff --git a/tests/unit/src/sqlbuild/executor/build/classes/helpers.py b/tests/unit/src/sqlbuild/executor/build/classes/helpers.py new file mode 100644 index 0000000000..5b4f7c81ae --- /dev/null +++ b/tests/unit/src/sqlbuild/executor/build/classes/helpers.py @@ -0,0 +1,35 @@ +"""Drive the build scheduler's job-limited dispatch without running nodes.""" + +from __future__ import annotations + +from collections import deque +from typing import Any + +from sqlbuild._native import CompiledObjectKey, CompiledResourceType +from sqlbuild.executor.build.classes.build_scheduler import BuildScheduler +from tests.unit.src.sqlbuild.executor.build.classes._test_types import JobLimitDispatchTestCase + + +def model_key(name: str) -> CompiledObjectKey: + """The key of model `name`.""" + + return CompiledObjectKey(resource_type=CompiledResourceType.MODEL, name=name) + + +def dispatched_names(test_case: JobLimitDispatchTestCase) -> tuple[str | None, ...]: + """Each dispatch's node after `finished_before` ("" for none) finishes, starting it.""" + + scheduler: Any = object.__new__(BuildScheduler) + scheduler._ready = deque(model_key(name) for name in test_case.ready) + scheduler._in_flight = set() + scheduler._blocked_keys = {model_key(name) for name in test_case.blocked} + scheduler._job_limits = test_case.job_limits + scheduler._node_jobs = {model_key(name): jobs for name, jobs in test_case.node_jobs.items()} + scheduler._job_running = {} + dispatched: list[str | None] = [] + for finished in test_case.finished_before: + scheduler._finish_in_flight(model_key(finished)) + taken: CompiledObjectKey | None = scheduler._take_ready_key() + scheduler._start_in_flight(taken) + dispatched.append(getattr(taken, "name", None)) + return tuple(dispatched) diff --git a/tests/unit/src/sqlbuild/executor/build/classes/test_build_scheduler_identity.py b/tests/unit/src/sqlbuild/executor/build/classes/test_build_scheduler_identity.py index 6baf9f4afa..5386e1bad6 100644 --- a/tests/unit/src/sqlbuild/executor/build/classes/test_build_scheduler_identity.py +++ b/tests/unit/src/sqlbuild/executor/build/classes/test_build_scheduler_identity.py @@ -43,6 +43,9 @@ def test_given_regular_primary_work_when_scheduler_submits_then_identity_is_prop scheduler._ready = deque((key,)) scheduler._in_flight = set() scheduler._blocked_keys = set() + scheduler._node_jobs = {} + scheduler._job_limits = {} + scheduler._job_running = {} scheduler._max_concurrency = 1 scheduler._stop = False scheduler._microbatch_coordinator_lock = threading.Lock() @@ -98,6 +101,9 @@ def test_given_microbatch_primary_work_when_scheduler_submits_then_identity_is_p scheduler._ready = deque((key,)) scheduler._in_flight = set() scheduler._blocked_keys = set() + scheduler._node_jobs = {} + scheduler._job_limits = {} + scheduler._job_running = {} scheduler._max_concurrency = 2 scheduler._stop = False scheduler._microbatch_coordinator_lock = threading.Lock() diff --git a/tests/unit/src/sqlbuild/executor/build/classes/test_build_scheduler_job_limits.py b/tests/unit/src/sqlbuild/executor/build/classes/test_build_scheduler_job_limits.py new file mode 100644 index 0000000000..835d15b3e8 --- /dev/null +++ b/tests/unit/src/sqlbuild/executor/build/classes/test_build_scheduler_job_limits.py @@ -0,0 +1,51 @@ +"""The build scheduler keeps each named job within its concurrency without holding up other jobs.""" + +from __future__ import annotations + +import pytest + +from tests.unit.src.sqlbuild.executor.build.classes._test_types import JobLimitDispatchTestCase +from tests.unit.src.sqlbuild.executor.build.classes.helpers import dispatched_names + + +@pytest.mark.parametrize( + "test_case", + [ + JobLimitDispatchTestCase( + description="a full job lets the next ready node of another job start", + ready=("orders_a", "orders_b", "refunds_a"), + node_jobs={"orders_a": ("orders_job",), "orders_b": ("orders_job",)}, + job_limits={"orders_job": 1}, + blocked=frozenset(), + finished_before=("", "", "", "orders_a"), + expected_dispatched=("orders_a", "refunds_a", None, "orders_b"), + ), + JobLimitDispatchTestCase( + description="a node in two jobs fits within the lower limit", + ready=("shared", "orders_a", "refunds_a"), + node_jobs={ + "shared": ("orders_job", "refunds_job"), + "orders_a": ("orders_job",), + "refunds_a": ("refunds_job",), + }, + job_limits={"orders_job": 2, "refunds_job": 1}, + blocked=frozenset(), + finished_before=("", "", "", "shared"), + expected_dispatched=("shared", "orders_a", None, "refunds_a"), + ), + JobLimitDispatchTestCase( + description="a node to skip never waits for its job", + ready=("orders_a", "orders_b"), + node_jobs={"orders_a": ("orders_job",), "orders_b": ("orders_job",)}, + job_limits={"orders_job": 1}, + blocked=frozenset({"orders_b"}), + finished_before=("", ""), + expected_dispatched=("orders_a", "orders_b"), + ), + ], + ids=lambda case: case.description, +) +def test_given_job_limits_when_dispatching_then_each_job_stays_within_its_limit( + test_case: JobLimitDispatchTestCase, +) -> None: + assert dispatched_names(test_case) == test_case.expected_dispatched, test_case.description diff --git a/website/src/content/docs/docs/cli/build.mdx b/website/src/content/docs/docs/cli/build.mdx index 12e7916aca..5aecdf4af2 100644 --- a/website/src/content/docs/docs/cli/build.mdx +++ b/website/src/content/docs/docs/cli/build.mdx @@ -43,6 +43,7 @@ own option. `--version` works only directly after `sqb`, as in `sqb --version`: | `--no-sql-analysis` | Disable compile-time SQL analysis (`--no-sql-validation` is an alias) | | `--no-cache` | Bypass the reusable compile-analysis cache for this invocation | | `--full-refresh` | Drop and rebuild selected models unless a model sets `full_refresh false`; `full_refresh true` forces a model even without this flag | +| `--no-job-settings` | Ignore the settings of the jobs the selection names; see [Settings when a build names a job](/docs/concepts/orchestration#settings-when-a-build-names-a-job) | | `--changed` | Build only the selected models and seeds that changed, and what they reach downstream; `--changed=definition,source_data` picks what counts. See [Building only what changed](#building-only-what-changed) | | `--reach` | With `--changed`: `job` stops at models another scheduled job refreshes, `all` builds everything downstream | | `--defer-to` | Resolve unselected model references against another target | diff --git a/website/src/content/docs/docs/cli/plan.mdx b/website/src/content/docs/docs/cli/plan.mdx index 55f7910385..4fee526405 100644 --- a/website/src/content/docs/docs/cli/plan.mdx +++ b/website/src/content/docs/docs/cli/plan.mdx @@ -30,6 +30,7 @@ Positional `SELECTOR` arguments work like `--select`, and both forms can be comb | `--ignore-audit-failures`, `--no-ignore-audit-failures` | Plan the build `sqb build` would run with the same flag; a plan whose audit failures are ignored starts with `Audit failures won't stop this build`. See [Ignoring audit failures](/docs/concepts/audits#ignoring-audit-failures) | | `--selection-diagnostics` | Warn when a selected model will build on changed upstream models that are not selected; see [Selection and staleness](/docs/concepts/planning/selection-and-staleness) | | `--full-refresh` | Plan a full rebuild for selected models unless a model sets `full_refresh false`; `full_refresh true` forces a model without this flag | +| `--no-job-settings` | Ignore the settings of the jobs the selection names; see [Settings when a build names a job](/docs/concepts/orchestration#settings-when-a-build-names-a-job) | | `--changed` | Show what `sqb build --changed` would run, which nodes it holds back and why; takes `--changed=definition,source_data` and `--reach` like [`sqb build`](/docs/cli/build#building-only-what-changed). In `--json`, the `changed` object lists `runs`, `held_back` and `unchanged` | | `--start-cursor` | Override the start cursor, typed from the selected incremental models' cursor type; fails with `S303` if the selection mixes timestamp and integer cursors. With no cursor model selected, an integer value is read as an integer cursor and anything else as an ISO timestamp | | `--end-cursor` | Override the end cursor, typed the same way as `--start-cursor` | diff --git a/website/src/content/docs/docs/concepts/concurrency.mdx b/website/src/content/docs/docs/concepts/concurrency.mdx index 14089df20c..70b3ffaab3 100644 --- a/website/src/content/docs/docs/concepts/concurrency.mdx +++ b/website/src/content/docs/docs/concepts/concurrency.mdx @@ -35,7 +35,10 @@ concurrency = 2 ``` `--concurrency` on the command line wins over all of them; see -[Target concurrency](/docs/concepts/project-configuration#target-concurrency) for the full order. The job's page shows the value its runs use on the target and where it came from. +[Target concurrency](/docs/concepts/project-configuration#target-concurrency) for the full order. +A job's `concurrency` limits how many of that job's nodes build at once in a build that names the +job, within the build's own limit. When one job is at its limit, a ready node of another job starts +instead; see [Settings when a build names a job](/docs/concepts/orchestration#settings-when-a-build-names-a-job). The job's page shows the value its runs use on the target and where it came from. ## Runs at once on a server: `max_workers` diff --git a/website/src/content/docs/docs/concepts/orchestration.mdx b/website/src/content/docs/docs/concepts/orchestration.mdx index 27b50e0a87..6a880e8c98 100644 --- a/website/src/content/docs/docs/concepts/orchestration.mdx +++ b/website/src/content/docs/docs/concepts/orchestration.mdx @@ -84,6 +84,39 @@ cpus = 2 A job's `[settings]` cannot set `select` or `exclude`: the job's own selection is the run's selection. +### Settings when a build names a job + +A build or plan whose selection names a job, as `job:nightly_orders_job` or the bare name, takes the +job's `[settings]`. Selecting the same nodes some other way, such as by tag, takes none of them. +Naming a group names each of its jobs. The first value set wins: + +1. A flag on the command line, such as `--vars` or `--concurrency` +2. The named job's setting +3. The target's setting +4. The project's setting + +`--no-job-settings` leaves out every job's settings. + +Some settings apply to the job's own nodes only, so a build that names two jobs can treat their +nodes differently: + +| Setting | Applies to | +| --- | --- | +| `full_refresh` | The job's incremental models, rebuilt from scratch | +| `run_tests` | The SQL unit tests of the job's models | +| `ignore_audit_failures` | The audits of the job's nodes | +| `concurrency` | How many of the job's nodes build at once. The build keeps its own limit too, and a node in two named jobs fits within both | +| `[retry]` | The job's nodes | + +A job's settings reach only the nodes its own selection holds. `--select job:nightly_orders_job,orders` +gives them to `orders` alone, and `+job:nightly_orders_job` builds the job's upstream nodes without +them. + +Every other setting, such as `vars` or `changed`, applies to the whole build, so every named job +must set it the same way. A group whose jobs set different `vars` or `changed` fails to compile. +Two named jobs that give one node different values of the same setting fail the build with `S027`, +naming the node, both jobs and the setting. A setting only one of them sets is no conflict. + Select a pipeline by its folder or a tag rather than listing its nodes. Rule `SQBRORCHESTRATION101` reports a job or sensor `select` that names more than three nodes one by one; see [Orchestration Rules](/docs/concepts/rules/orchestration-rules). @@ -270,6 +303,7 @@ Every problem with an orchestration file is a `D017` error at its file, line and | `job nightly_orders_job resources: cpus must be a positive number` | Set `cpus` above zero | | `job 'nightly_orders_job' uses pool 'warehouse_hevy', which [pools] in sqlbuild_project.toml does not declare` | Fix the name, or declare the pool in [`[pools]`](/docs/concepts/project-configuration#pools) | | `job 'nightly_orders_job' takes 2 slots of pool 'supplier_api', more than its limit of 1 on target 'production'` | Take fewer slots, or raise the limit in `[targets.production.pools]` | +| `jobs orders_job and crm_job set vars differently; a group runs its jobs as one build, which has one vars` | Give the group's jobs the same `vars` and `changed`, or move one job to another group | | `sensor 'orders_sensor' is declared here, but importing the module does not define it` | Assign the sensor to a module-level name or list, not a dictionary or function body | | `group dependencies contain a cycle in job group daily_orders_job_group: orders_marts_job waits for staging_orders_job (model:orders reads model:stg_orders); staging_orders_job waits for orders_marts_job (model:stg_orders reads seed:regions)` | Move each node to one job, and give seeds and nodes that several members read their own job that runs first | @@ -279,6 +313,9 @@ Details: a job selector never imports a calendar or sensor module. - A job or group declared with `targets` is selectable only when the command's target is one of them, as on the server. +- Job settings that cannot apply together fail the build or plan with `S027`, such as `model:orders + is selected through jobs orders_job and finance_job, which set run_tests differently (false and + true)`. Make the jobs agree, set it in one of them, pass the flag, or use `--no-job-settings`. - Selectors are checked for syntax at compile. A selector that names a node the project does not have fails when a location publishes, such as `job nightly_orders_job: unknown selector name 'order_total'`. diff --git a/website/src/content/docs/docs/concepts/project-configuration.mdx b/website/src/content/docs/docs/concepts/project-configuration.mdx index 827f300ec7..f563c3b0bf 100644 --- a/website/src/content/docs/docs/concepts/project-configuration.mdx +++ b/website/src/content/docs/docs/concepts/project-configuration.mdx @@ -248,9 +248,11 @@ concurrency = 8 value set in this order: 1. `--concurrency` on the command line, for that invocation only -2. The selected target's `concurrency`, from `[targets.]` in either file -3. `[settings] concurrency` in `sqlbuild_local.toml` -4. `[settings] concurrency` in `sqlbuild_project.toml` (default `1`) +2. For `sqb build`, a named job's `concurrency`, which limits that job's nodes within the build's + own limit; see [Settings when a build names a job](/docs/concepts/orchestration#settings-when-a-build-names-a-job) +3. The selected target's `concurrency`, from `[targets.]` in either file +4. `[settings] concurrency` in `sqlbuild_local.toml` +5. `[settings] concurrency` in `sqlbuild_project.toml` (default `1`) The value must be an integer of at least `1`; anything else fails with `D001`. `sqb build --verbose` shows the effective limit and the setting it came from, for example diff --git a/website/src/content/docs/docs/concepts/selectors.mdx b/website/src/content/docs/docs/concepts/selectors.mdx index e8ec7d9eaa..6d6f2cff26 100644 --- a/website/src/content/docs/docs/concepts/selectors.mdx +++ b/website/src/content/docs/docs/concepts/selectors.mdx @@ -95,6 +95,8 @@ sqb build --select "job:nightly_orders_job,tag:finance" ``` A job selects its own `select` minus its own `exclude`. A job group selects the union of its jobs. +Naming a job also applies its settings to its nodes; see +[Settings when a build names a job](/docs/concepts/orchestration#settings-when-a-build-names-a-job). A job or job group declared with `targets` is selectable only when the command's target is one of them, as on the server. Job selectors compose with `+`, `,` and `--exclude` like any other selector. A job's own selection From 848561e5b1d58aa0887dd4092c95257e056e74e6 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 18 Nov 2026 16:45:12 +0000 Subject: [PATCH 04/22] fix(jobs): apply the settings of jobs a job: pattern names (CHI-1533) job: accepts glob patterns, which selected the matching jobs' nodes but applied none of their settings. A pattern now names every job, and every group's jobs, it matches, as it already does for selection. --- .../src/job_settings/_helpers/tokens.rs | 19 ++++++++++++------- .../tests/test_build_job_settings.rs | 9 +++++++++ .../references/docs/concepts/orchestration.md | 4 ++-- .../docs/docs/concepts/orchestration.mdx | 4 ++-- 4 files changed, 25 insertions(+), 11 deletions(-) diff --git a/crates/sqlbuild-analysis/src/job_settings/_helpers/tokens.rs b/crates/sqlbuild-analysis/src/job_settings/_helpers/tokens.rs index cf3413d092..5a62fa37b9 100644 --- a/crates/sqlbuild-analysis/src/job_settings/_helpers/tokens.rs +++ b/crates/sqlbuild-analysis/src/job_settings/_helpers/tokens.rs @@ -2,12 +2,14 @@ use std::collections::BTreeMap; -use crate::graph::constants::{INTERSECTION_SEPARATOR, JOB_KIND, NAME_KIND}; +use sqlbuild_core::text::main::glob_match::glob_match; + +use crate::graph::constants::{INTERSECTION_SEPARATOR, JOB_KIND, NAME_KIND, PATTERN_CHARACTERS}; use crate::graph::main::parse_selector::parse_selector; use crate::graph::models::ParsedSelector; use crate::job_settings::models::DeclaredRun; -/// The jobs one whitespace-free selector token names, each once in the order named. +/// The jobs one selector token names, by name, `job:` pattern or group, each once. #[must_use] pub(crate) fn token_jobs(token: &str, declared: &BTreeMap) -> Vec { let mut jobs: Vec = Vec::new(); @@ -18,11 +20,14 @@ pub(crate) fn token_jobs(token: &str, declared: &BTreeMap) if kind != JOB_KIND && kind != NAME_KIND { continue; } - let named: Vec = match declared.get(&value) { - Some(DeclaredRun::Job { .. }) => vec![value], - Some(DeclaredRun::Group { members }) => members.clone(), - None => Vec::new(), - }; + let pattern: bool = kind == JOB_KIND && value.contains(PATTERN_CHARACTERS); + let named = declared + .iter() + .filter(|(name, _)| **name == value || (pattern && glob_match(name, &value))) + .flat_map(|(name, run)| match run { + DeclaredRun::Job { .. } => vec![name.clone()], + DeclaredRun::Group { members } => members.clone(), + }); for job in named { if !jobs.contains(&job) { jobs.push(job); diff --git a/crates/sqlbuild-analysis/src/job_settings/tests/test_build_job_settings.rs b/crates/sqlbuild-analysis/src/job_settings/tests/test_build_job_settings.rs index 0540a65afc..21616b4447 100644 --- a/crates/sqlbuild-analysis/src/job_settings/tests/test_build_job_settings.rs +++ b/crates/sqlbuild-analysis/src/job_settings/tests/test_build_job_settings.rs @@ -35,6 +35,15 @@ fn given_named_jobs_when_merging_build_settings_then_they_must_agree() { and {\"region\":\"us\"}); a build applies vars to everything it runs", ), }, + BuildJobSettingsTestCase { + description: "a job: pattern names every job it matches", + select: &["job:crm_*"], + expected_named: vec!["crm_job", "crm_us_job"], + expected_settings: Err( + "jobs crm_job and crm_us_job set vars differently ({\"region\":\"eu\"} \ + and {\"region\":\"us\"}); a build applies vars to everything it runs", + ), + }, BuildJobSettingsTestCase { description: "a group's jobs without build-wide settings", select: &["job:daily_group"], diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/orchestration.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/orchestration.md index ab90288647..484d6b04ba 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/orchestration.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/orchestration.md @@ -100,8 +100,8 @@ selection. ### Settings when a build names a job -A build or plan whose selection names a job, as `job:nightly_orders_job` or the bare name, takes the -job's `[settings]`. Selecting the same nodes some other way, such as by tag, takes none of them. +A build or plan whose selection names a job, as `job:nightly_orders_job`, the bare name or a +`job:` pattern such as `job:nightly_*`, takes the job's `[settings]`. Selecting the same nodes some other way, such as by tag, takes none of them. Naming a group names each of its jobs. The first value set wins: 1. A flag on the command line, such as `--vars` or `--concurrency` diff --git a/website/src/content/docs/docs/concepts/orchestration.mdx b/website/src/content/docs/docs/concepts/orchestration.mdx index 6a880e8c98..96a96f6fd2 100644 --- a/website/src/content/docs/docs/concepts/orchestration.mdx +++ b/website/src/content/docs/docs/concepts/orchestration.mdx @@ -86,8 +86,8 @@ selection. ### Settings when a build names a job -A build or plan whose selection names a job, as `job:nightly_orders_job` or the bare name, takes the -job's `[settings]`. Selecting the same nodes some other way, such as by tag, takes none of them. +A build or plan whose selection names a job, as `job:nightly_orders_job`, the bare name or a +`job:` pattern such as `job:nightly_*`, takes the job's `[settings]`. Selecting the same nodes some other way, such as by tag, takes none of them. Naming a group names each of its jobs. The first value set wins: 1. A flag on the command line, such as `--vars` or `--concurrency` From 4a77931df2057446d8b591218ec92868699ac849 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 18 Nov 2026 18:19:44 +0000 Subject: [PATCH 05/22] fix(jobs): count microbatch sub-work against job limits in the build scheduler (CHI-1533) A queued microbatch coordinator whose job was full still counted towards coordinator demand, so a running coordinator could get no sub-workers. Only coordinators that could start now count, sub-workers take a slot in their coordinator's jobs, and a lock guards the per-job counts that coordinator threads now change. Dispatch takes the head of the ready queue directly when it has room or the build names no limited job. The tests now run the real dispatch loop with recording workers through completions, a failure and the skips it causes. --- .../executor/build/classes/build_scheduler.py | 115 ++++++++----- .../executor/build/classes/_test_types.py | 32 +++- .../executor/build/classes/helpers.py | 153 ++++++++++++++++-- .../classes/test_build_scheduler_identity.py | 2 + .../test_build_scheduler_job_limits.py | 120 +++++++++++--- .../test_build_scheduler_microbatch.py | 8 + 6 files changed, 345 insertions(+), 85 deletions(-) diff --git a/src/sqlbuild/executor/build/classes/build_scheduler.py b/src/sqlbuild/executor/build/classes/build_scheduler.py index 3ac4001087..c6a967629e 100644 --- a/src/sqlbuild/executor/build/classes/build_scheduler.py +++ b/src/sqlbuild/executor/build/classes/build_scheduler.py @@ -242,9 +242,7 @@ def __init__( callbacks.python_identity_recorder ) self._max_concurrency: int = len(connections) - self._job_limits: Mapping[str, int] = runtime.job_limits - self._node_jobs: Mapping[CompiledObjectKey, tuple[str, ...]] = runtime.node_jobs - self._job_running: dict[str, int] = {} + self._init_job_limits(runtime) self._blocked_keys: set[CompiledObjectKey] = set() self._unbuilt_keys: set[CompiledObjectKey] = set(initial_state.initial_failed_keys) self._completed_keys: set[CompiledObjectKey] = set(initial_state.precompleted_keys) @@ -471,14 +469,7 @@ def _run_concurrent(self) -> None: if self._stop and not self._in_flight: break - with self._microbatch_coordinator_lock: - queued_coordinators: int = sum( - 1 for key in tuple(self._ready) if self._is_concurrent_microbatch(key) - ) - self._microbatch_coordinator_demand = min( - self._max_concurrency, - len(self._microbatch_coordinators) + queued_coordinators, - ) + self._update_microbatch_coordinator_demand() while self._ready and len(self._in_flight) < self._max_concurrency: if self._stop: @@ -549,38 +540,76 @@ def _run_concurrent(self) -> None: if escaped is not None: raise escaped + def _update_microbatch_coordinator_demand(self) -> None: + """Count the running coordinators and the queued ones whose jobs would let them start.""" + + with self._microbatch_coordinator_lock: + queued_coordinators: int = sum( + 1 + for key in tuple(self._ready) + if self._is_concurrent_microbatch(key) and self._has_job_room(key) + ) + self._microbatch_coordinator_demand = min( + self._max_concurrency, + len(self._microbatch_coordinators) + queued_coordinators, + ) + + def _init_job_limits(self, runtime: BuildRuntimeParams) -> None: + """Job limits, each node's jobs, and a lock over how many of each job's nodes run. + + Coordinators reserve microbatch sub-work from worker threads, so the counts take a lock. + """ + + self._job_limits: Mapping[str, int] = runtime.job_limits + self._node_jobs: Mapping[CompiledObjectKey, tuple[str, ...]] = runtime.node_jobs + self._job_running: dict[str, int] = {} + self._job_lock: threading.Lock = threading.Lock() + def _take_ready_key(self) -> CompiledObjectKey | None: """Remove and return the first ready node whose named jobs all have a free slot.""" + if not self._job_limits or self._has_job_room(self._ready[0]): + return self._ready.popleft() index: int key: CompiledObjectKey - for index, key in enumerate(self._ready): - if self._has_job_room(key): - del self._ready[index] - return key + with self._job_lock: + for index, key in enumerate(self._ready): + if key in self._blocked_keys or self._jobs_have_room(self._node_jobs.get(key, ())): + del self._ready[index] + return key return None def _has_job_room(self, key: CompiledObjectKey) -> bool: """Whether each job limiting `key` has a free slot; a node to skip never waits.""" - return key in self._blocked_keys or all( - self._job_running.get(job, 0) < self._job_limits[job] - for job in self._node_jobs.get(key, ()) - ) + if key in self._blocked_keys: + return True + with self._job_lock: + return self._jobs_have_room(self._node_jobs.get(key, ())) + + def _jobs_have_room(self, jobs: tuple[str, ...]) -> bool: + """Whether each of `jobs` runs fewer nodes than its limit; the caller holds `_job_lock`.""" + + return all(self._job_running.get(job, 0) < self._job_limits[job] for job in jobs) + + def _add_job_running(self, jobs: tuple[str, ...], *, change: int) -> None: + """Count `change` more running nodes in each of `jobs`; the caller holds `_job_lock`.""" + + job: str + for job in jobs: + self._job_running[job] = self._job_running.get(job, 0) + change def _start_in_flight(self, key: CompiledObjectKey) -> None: self._in_flight.add(key) - job: str - for job in self._node_jobs.get(key, ()): - self._job_running[job] = self._job_running.get(job, 0) + 1 + with self._job_lock: + self._add_job_running(self._node_jobs.get(key, ()), change=1) def _finish_in_flight(self, key: CompiledObjectKey) -> None: if key not in self._in_flight: return self._in_flight.discard(key) - job: str - for job in self._node_jobs.get(key, ()): - self._job_running[job] -= 1 + with self._job_lock: + self._add_job_running(self._node_jobs.get(key, ()), change=-1) def _report_scheduler_state(self) -> None: callback: Callable[[SchedulerState], None] | None = self._callbacks.on_scheduler_state @@ -656,6 +685,7 @@ def _execute_concurrent_microbatch_node( microbatch_batch_runner=lambda batches, concurrency, execute: ( self._run_microbatch_subwork( pool=pool, + jobs=self._node_jobs.get(key, ()), coordinator_connection=connection, batches=batches, concurrency=concurrency, @@ -671,6 +701,7 @@ def _run_microbatch_subwork( self, *, pool: ThreadPoolExecutor, + jobs: tuple[str, ...], coordinator_connection: Any, batches: tuple[BatchWindow, ...], concurrency: int, @@ -693,12 +724,13 @@ def submit_one() -> tuple[Future[MicrobatchPhaseOutcome], BatchWindow]: pool.submit( copy_context().run, self._run_microbatch_batch_worker, + jobs=jobs, batch=batch, execute=execute, ), ) except BaseException: - self._release_microbatch_subworker() + self._release_microbatch_subworker(jobs) raise return future, batch @@ -707,7 +739,7 @@ def submit_one() -> tuple[Future[MicrobatchPhaseOutcome], BatchWindow]: while ( len(pending_batches) > 1 and len(in_flight) < model_subworker_limit - and self._reserve_microbatch_subworker() + and self._reserve_microbatch_subworker(jobs) ): submitted_future, submitted_batch = submit_one() in_flight[submitted_future] = submitted_batch @@ -719,7 +751,7 @@ def submit_one() -> tuple[Future[MicrobatchPhaseOutcome], BatchWindow]: failed: bool = coordinator_result.failure is not None if failed: for future in in_flight: - self._cancel_microbatch_subworker(future=future) + self._cancel_microbatch_subworker(future=future, jobs=jobs) first_exception: BaseException | None = None while in_flight: completed, _ = wait(tuple(in_flight), return_when=FIRST_COMPLETED) @@ -734,18 +766,18 @@ def submit_one() -> tuple[Future[MicrobatchPhaseOutcome], BatchWindow]: first_exception = exc failed = True for sibling in in_flight: - self._cancel_microbatch_subworker(future=sibling) + self._cancel_microbatch_subworker(future=sibling, jobs=jobs) continue results[batch.index] = outcome if outcome.failure is not None: failed = True for sibling in in_flight: - self._cancel_microbatch_subworker(future=sibling) + self._cancel_microbatch_subworker(future=sibling, jobs=jobs) while ( not failed and pending_batches and len(in_flight) < model_subworker_limit - and self._reserve_microbatch_subworker() + and self._reserve_microbatch_subworker(jobs) ): submitted_future, submitted_batch = submit_one() in_flight[submitted_future] = submitted_batch @@ -761,6 +793,7 @@ def submit_one() -> tuple[Future[MicrobatchPhaseOutcome], BatchWindow]: def _run_microbatch_batch_worker( self, *, + jobs: tuple[str, ...], batch: BatchWindow, execute: MicrobatchBatchExecutor, ) -> MicrobatchPhaseOutcome: @@ -771,9 +804,11 @@ def _run_microbatch_batch_worker( try: self._connection_pool.put(connection) finally: - self._release_microbatch_subworker() + self._release_microbatch_subworker(jobs) + + def _reserve_microbatch_subworker(self, jobs: tuple[str, ...]) -> bool: + """Take a sub-worker slot, and one in each of its coordinator's jobs, if all are free.""" - def _reserve_microbatch_subworker(self) -> bool: with self._microbatch_coordinator_lock: if ( max( @@ -784,16 +819,24 @@ def _reserve_microbatch_subworker(self) -> bool: >= self._max_concurrency ): return False + with self._job_lock: + if not self._jobs_have_room(jobs): + return False + self._add_job_running(jobs, change=1) self._microbatch_subworkers += 1 return True - def _release_microbatch_subworker(self) -> None: + def _release_microbatch_subworker(self, jobs: tuple[str, ...]) -> None: with self._microbatch_coordinator_lock: self._microbatch_subworkers -= 1 + with self._job_lock: + self._add_job_running(jobs, change=-1) - def _cancel_microbatch_subworker(self, *, future: Future[MicrobatchPhaseOutcome]) -> None: + def _cancel_microbatch_subworker( + self, *, future: Future[MicrobatchPhaseOutcome], jobs: tuple[str, ...] + ) -> None: if not future.cancelled() and future.cancel(): - self._release_microbatch_subworker() + self._release_microbatch_subworker(jobs) def _pre_dispatch(self, key: CompiledObjectKey) -> bool: """Run pre-dispatch checks. Returns False if the node should be skipped.""" diff --git a/tests/unit/src/sqlbuild/executor/build/classes/_test_types.py b/tests/unit/src/sqlbuild/executor/build/classes/_test_types.py index 9c8563cc62..111920de02 100644 --- a/tests/unit/src/sqlbuild/executor/build/classes/_test_types.py +++ b/tests/unit/src/sqlbuild/executor/build/classes/_test_types.py @@ -40,15 +40,31 @@ class SchedulerIdentityTestCase: @dataclass(frozen=True) -class JobLimitDispatchTestCase: - """Ready nodes, each node's limiting jobs and their limits, and the dispatch order expected.""" +class JobLimitedBuildTestCase: + """A small build graph run through the concurrent dispatch loop with per-job limits.""" description: str - ready: tuple[str, ...] + # Each node and the nodes it waits for. + upstream: dict[str, tuple[str, ...]] node_jobs: dict[str, tuple[str, ...]] job_limits: dict[str, int] - blocked: frozenset[str] - # Before each dispatch, the running node that finishes first, or "" for none. - finished_before: tuple[str, ...] - # Each dispatch's node, or None when no ready node has room. - expected_dispatched: tuple[str | None, ...] + max_concurrency: int + failing: frozenset[str] + # The most nodes of each named job, or of the whole build under "*", seen running at once. + expected_peaks: dict[str, int] + expected_skipped: tuple[str, ...] + + +@dataclass(frozen=True) +class SubworkerReservationTestCase: + """Running and queued microbatch coordinators, and the sub-worker slots one of them gets.""" + + description: str + job_limits: dict[str, int] + # Nodes of each job already running, the coordinator's included. + job_running: dict[str, int] + coordinator_jobs: tuple[str, ...] + queued_coordinator_jobs: tuple[tuple[str, ...], ...] + max_concurrency: int + expected_demand: int + expected_reservations: tuple[bool, ...] diff --git a/tests/unit/src/sqlbuild/executor/build/classes/helpers.py b/tests/unit/src/sqlbuild/executor/build/classes/helpers.py index 5b4f7c81ae..7815de990b 100644 --- a/tests/unit/src/sqlbuild/executor/build/classes/helpers.py +++ b/tests/unit/src/sqlbuild/executor/build/classes/helpers.py @@ -1,13 +1,30 @@ -"""Drive the build scheduler's job-limited dispatch without running nodes.""" +"""Drive the build scheduler's job-limited dispatch with stub workers that record concurrency.""" from __future__ import annotations +import threading +import time from collections import deque +from queue import Queue from typing import Any from sqlbuild._native import CompiledObjectKey, CompiledResourceType from sqlbuild.executor.build.classes.build_scheduler import BuildScheduler -from tests.unit.src.sqlbuild.executor.build.classes._test_types import JobLimitDispatchTestCase +from sqlbuild.executor.build.models import BuildRuntimeParams, NodeCompletion +from sqlbuild.executor.run.models import ModelExecutionResult +from sqlbuild.executor.scheduling.types import ExecutionStatus +from tests.unit.src.sqlbuild.executor.build.classes._test_types import ( + JobLimitedBuildTestCase, + SubworkerReservationTestCase, +) + +# How long each stub node runs, long enough for the loop to start every node it may. +_NODE_SECONDS: float = 0.05 +_ALL_NODES: str = "*" +_STATUS_WHEN_FAILING: dict[bool, ExecutionStatus] = { + True: ExecutionStatus.FAILED, + False: ExecutionStatus.SUCCESS, +} def model_key(name: str) -> CompiledObjectKey: @@ -16,20 +33,122 @@ def model_key(name: str) -> CompiledObjectKey: return CompiledObjectKey(resource_type=CompiledResourceType.MODEL, name=name) -def dispatched_names(test_case: JobLimitDispatchTestCase) -> tuple[str | None, ...]: - """Each dispatch's node after `finished_before` ("" for none) finishes, starting it.""" - +def _bare_scheduler( + *, job_limits: dict[str, int], node_jobs: dict[str, tuple[str, ...]], max_concurrency: int +) -> Any: scheduler: Any = object.__new__(BuildScheduler) - scheduler._ready = deque(model_key(name) for name in test_case.ready) + scheduler._ready = deque() scheduler._in_flight = set() - scheduler._blocked_keys = {model_key(name) for name in test_case.blocked} - scheduler._job_limits = test_case.job_limits - scheduler._node_jobs = {model_key(name): jobs for name, jobs in test_case.node_jobs.items()} - scheduler._job_running = {} - dispatched: list[str | None] = [] - for finished in test_case.finished_before: - scheduler._finish_in_flight(model_key(finished)) - taken: CompiledObjectKey | None = scheduler._take_ready_key() - scheduler._start_in_flight(taken) - dispatched.append(getattr(taken, "name", None)) - return tuple(dispatched) + scheduler._blocked_keys = set() + scheduler._max_concurrency = max_concurrency + scheduler._stop = False + scheduler._microbatch_coordinator_lock = threading.Lock() + scheduler._microbatch_coordinators = set() + scheduler._microbatch_coordinator_demand = 0 + scheduler._microbatch_subworkers = 0 + scheduler._completion_queue = Queue() + scheduler._init_job_limits( + BuildRuntimeParams( + run_id="run", + job_limits=job_limits, + node_jobs={model_key(name): jobs for name, jobs in node_jobs.items()}, + ) + ) + return scheduler + + +def run_job_limited_build( + test_case: JobLimitedBuildTestCase, +) -> tuple[dict[str, int], tuple[str, ...]]: + """Each job's, and the whole build's, peak running nodes, and the nodes skipped.""" + + scheduler: Any = _bare_scheduler( + job_limits=test_case.job_limits, + node_jobs=test_case.node_jobs, + max_concurrency=test_case.max_concurrency, + ) + lock: threading.Lock = threading.Lock() + running: dict[str, int] = {} + peaks: dict[str, int] = {} + finished: set[str] = set() + unbuilt: set[str] = set() + skipped: list[str] = [] + scheduler._ready.extend( + model_key(name) + for name in filter(lambda node: not test_case.upstream[node], test_case.upstream) + ) + + dependents: dict[str, tuple[str, ...]] = { + name: tuple(filter(lambda node: name in test_case.upstream[node], test_case.upstream)) + for name in test_case.upstream + } + + def release(name: str) -> None: + finished.add(name) + ready: tuple[str, ...] = tuple( + filter(lambda node: set(test_case.upstream[node]) <= finished, dependents[name]) + ) + scheduler._blocked_keys.update( + model_key(node) + for node in filter(lambda node: bool(unbuilt & set(test_case.upstream[node])), ready) + ) + scheduler._ready.extend(model_key(node) for node in ready) + + def count(name: str, change: int) -> None: + with lock: + for group in (*test_case.node_jobs.get(name, ()), _ALL_NODES): + running[group] = running.get(group, 0) + change + peaks[group] = max(peaks.get(group, 0), running[group]) + + def worker(key: CompiledObjectKey) -> None: + count(key.name, 1) + time.sleep(_NODE_SECONDS) + count(key.name, -1) + status: ExecutionStatus = _STATUS_WHEN_FAILING[key.name in test_case.failing] + scheduler._completion_queue.put( + NodeCompletion(key=key, result=ModelExecutionResult(model_name=key.name, status=status)) + ) + + def handle_completion(*, key: CompiledObjectKey, result: ModelExecutionResult) -> None: + unbuilt.update({key.name} & test_case.failing) + release(key.name) + + def mark_complete(key: CompiledObjectKey) -> None: + unbuilt.add(key.name) + release(key.name) + + scheduler._worker = worker + scheduler._handle_completion = handle_completion + scheduler._mark_complete = mark_complete + scheduler._record_skipped = lambda key: skipped.append(key.name) + scheduler._pre_dispatch = lambda _key: True + scheduler._is_concurrent_microbatch = lambda _key: False + scheduler._report_scheduler_state = lambda: None + scheduler._run_concurrent() + return ( + {group: peaks.get(group, 0) for group in test_case.expected_peaks}, + tuple(sorted(skipped)), + ) + + +def reserve_subworkers(test_case: SubworkerReservationTestCase) -> tuple[int, tuple[bool, ...]]: + """The coordinator demand with the queued coordinators, then each reservation's outcome.""" + + queued: dict[str, tuple[str, ...]] = { + f"queued_{index}": jobs for index, jobs in enumerate(test_case.queued_coordinator_jobs) + } + scheduler: Any = _bare_scheduler( + job_limits=test_case.job_limits, + node_jobs={"coordinator": test_case.coordinator_jobs, **queued}, + max_concurrency=test_case.max_concurrency, + ) + scheduler._job_running = dict(test_case.job_running) + scheduler._microbatch_coordinators = {model_key("coordinator")} + scheduler._ready.extend(model_key(name) for name in queued) + scheduler._is_concurrent_microbatch = lambda _key: True + scheduler._update_microbatch_coordinator_demand() + reservations: tuple[bool, ...] = tuple( + scheduler._reserve_microbatch_subworker(test_case.coordinator_jobs) + for _ in test_case.expected_reservations + ) + return scheduler._microbatch_coordinator_demand, reservations diff --git a/tests/unit/src/sqlbuild/executor/build/classes/test_build_scheduler_identity.py b/tests/unit/src/sqlbuild/executor/build/classes/test_build_scheduler_identity.py index 5386e1bad6..8335747e25 100644 --- a/tests/unit/src/sqlbuild/executor/build/classes/test_build_scheduler_identity.py +++ b/tests/unit/src/sqlbuild/executor/build/classes/test_build_scheduler_identity.py @@ -46,6 +46,7 @@ def test_given_regular_primary_work_when_scheduler_submits_then_identity_is_prop scheduler._node_jobs = {} scheduler._job_limits = {} scheduler._job_running = {} + scheduler._job_lock = threading.Lock() scheduler._max_concurrency = 1 scheduler._stop = False scheduler._microbatch_coordinator_lock = threading.Lock() @@ -104,6 +105,7 @@ def test_given_microbatch_primary_work_when_scheduler_submits_then_identity_is_p scheduler._node_jobs = {} scheduler._job_limits = {} scheduler._job_running = {} + scheduler._job_lock = threading.Lock() scheduler._max_concurrency = 2 scheduler._stop = False scheduler._microbatch_coordinator_lock = threading.Lock() diff --git a/tests/unit/src/sqlbuild/executor/build/classes/test_build_scheduler_job_limits.py b/tests/unit/src/sqlbuild/executor/build/classes/test_build_scheduler_job_limits.py index 835d15b3e8..528d0db7c1 100644 --- a/tests/unit/src/sqlbuild/executor/build/classes/test_build_scheduler_job_limits.py +++ b/tests/unit/src/sqlbuild/executor/build/classes/test_build_scheduler_job_limits.py @@ -4,48 +4,120 @@ import pytest -from tests.unit.src.sqlbuild.executor.build.classes._test_types import JobLimitDispatchTestCase -from tests.unit.src.sqlbuild.executor.build.classes.helpers import dispatched_names +from tests.unit.src.sqlbuild.executor.build.classes._test_types import ( + JobLimitedBuildTestCase, + SubworkerReservationTestCase, +) +from tests.unit.src.sqlbuild.executor.build.classes.helpers import ( + reserve_subworkers, + run_job_limited_build, +) @pytest.mark.parametrize( "test_case", [ - JobLimitDispatchTestCase( - description="a full job lets the next ready node of another job start", - ready=("orders_a", "orders_b", "refunds_a"), - node_jobs={"orders_a": ("orders_job",), "orders_b": ("orders_job",)}, + JobLimitedBuildTestCase( + description="a full job lets ready nodes of other work start", + upstream={"orders_a": (), "orders_b": (), "orders_c": (), "refunds": (), "crm": ()}, + node_jobs={ + "orders_a": ("orders_job",), + "orders_b": ("orders_job",), + "orders_c": ("orders_job",), + }, job_limits={"orders_job": 1}, - blocked=frozenset(), - finished_before=("", "", "", "orders_a"), - expected_dispatched=("orders_a", "refunds_a", None, "orders_b"), + max_concurrency=4, + failing=frozenset(), + expected_peaks={"orders_job": 1, "*": 3}, + expected_skipped=(), ), - JobLimitDispatchTestCase( + JobLimitedBuildTestCase( description="a node in two jobs fits within the lower limit", - ready=("shared", "orders_a", "refunds_a"), + upstream={"shared": (), "orders_a": (), "orders_b": (), "refunds_a": ()}, node_jobs={ "shared": ("orders_job", "refunds_job"), "orders_a": ("orders_job",), + "orders_b": ("orders_job",), "refunds_a": ("refunds_job",), }, job_limits={"orders_job": 2, "refunds_job": 1}, - blocked=frozenset(), - finished_before=("", "", "", "shared"), - expected_dispatched=("shared", "orders_a", None, "refunds_a"), + max_concurrency=4, + failing=frozenset(), + expected_peaks={"orders_job": 2, "refunds_job": 1}, + expected_skipped=(), ), - JobLimitDispatchTestCase( - description="a node to skip never waits for its job", - ready=("orders_a", "orders_b"), - node_jobs={"orders_a": ("orders_job",), "orders_b": ("orders_job",)}, + JobLimitedBuildTestCase( + description="a failed node frees its slot and its dependents are skipped", + upstream={ + "orders_a": (), + "orders_b": (), + "orders_mart": ("orders_a",), + "orders_report": ("orders_mart",), + }, + node_jobs={ + "orders_a": ("orders_job",), + "orders_b": ("orders_job",), + "orders_mart": ("orders_job",), + "orders_report": ("orders_job",), + }, job_limits={"orders_job": 1}, - blocked=frozenset({"orders_b"}), - finished_before=("", ""), - expected_dispatched=("orders_a", "orders_b"), + max_concurrency=4, + failing=frozenset({"orders_a"}), + expected_peaks={"orders_job": 1, "*": 1}, + expected_skipped=("orders_mart", "orders_report"), + ), + ], + ids=lambda case: case.description, +) +def test_given_job_limits_when_running_the_dispatch_loop_then_each_job_stays_within_its_limit( + test_case: JobLimitedBuildTestCase, +) -> None: + assert run_job_limited_build(test_case) == ( + test_case.expected_peaks, + test_case.expected_skipped, + ), test_case.description + + +@pytest.mark.parametrize( + "test_case", + [ + SubworkerReservationTestCase( + description="sub-workers count against the coordinator's job", + job_limits={"events_job": 2}, + job_running={"events_job": 1}, + coordinator_jobs=("events_job",), + queued_coordinator_jobs=(), + max_concurrency=8, + expected_demand=1, + expected_reservations=(True, False), + ), + SubworkerReservationTestCase( + description="a queued coordinator whose job is full claims no sub-worker slots", + job_limits={"events_job": 3, "refunds_job": 1}, + job_running={"events_job": 1, "refunds_job": 1}, + coordinator_jobs=("events_job",), + queued_coordinator_jobs=(("refunds_job",), ("refunds_job",)), + max_concurrency=3, + expected_demand=1, + expected_reservations=(True, True, False), + ), + SubworkerReservationTestCase( + description="a queued coordinator that could start keeps its slot", + job_limits={"events_job": 3}, + job_running={"events_job": 1}, + coordinator_jobs=("events_job",), + queued_coordinator_jobs=((),), + max_concurrency=3, + expected_demand=2, + expected_reservations=(True, False), ), ], ids=lambda case: case.description, ) -def test_given_job_limits_when_dispatching_then_each_job_stays_within_its_limit( - test_case: JobLimitDispatchTestCase, +def test_given_microbatch_coordinators_when_reserving_sub_workers_then_job_limits_apply( + test_case: SubworkerReservationTestCase, ) -> None: - assert dispatched_names(test_case) == test_case.expected_dispatched, test_case.description + assert reserve_subworkers(test_case) == ( + test_case.expected_demand, + test_case.expected_reservations, + ), test_case.description diff --git a/tests/unit/src/sqlbuild/executor/build/classes/test_build_scheduler_microbatch.py b/tests/unit/src/sqlbuild/executor/build/classes/test_build_scheduler_microbatch.py index 48f68815cb..32d78f7cb2 100644 --- a/tests/unit/src/sqlbuild/executor/build/classes/test_build_scheduler_microbatch.py +++ b/tests/unit/src/sqlbuild/executor/build/classes/test_build_scheduler_microbatch.py @@ -74,6 +74,9 @@ def test_given_large_microbatch_phase_when_workers_are_blocked_then_future_submi scheduler: Any = object.__new__(BuildScheduler) scheduler._max_concurrency = test_case.global_concurrency scheduler._microbatch_coordinator_lock = threading.Lock() + scheduler._job_lock = threading.Lock() + scheduler._job_limits = {} + scheduler._job_running = {} scheduler._microbatch_coordinators = {"orders"} scheduler._microbatch_coordinator_demand = 1 scheduler._microbatch_subworkers = 0 @@ -113,6 +116,7 @@ def run_phase() -> None: results.append( BuildScheduler._run_microbatch_subwork( scheduler, + jobs=(), pool=pool, coordinator_connection="coordinator", batches=batches, @@ -162,6 +166,9 @@ def test_given_coordinator_failure_when_subworkers_are_running_then_pending_batc scheduler: Any = object.__new__(BuildScheduler) scheduler._max_concurrency = test_case.global_concurrency scheduler._microbatch_coordinator_lock = threading.Lock() + scheduler._job_lock = threading.Lock() + scheduler._job_limits = {} + scheduler._job_running = {} scheduler._microbatch_coordinators = {"orders"} scheduler._microbatch_coordinator_demand = 1 scheduler._microbatch_subworkers = 0 @@ -202,6 +209,7 @@ def execute(batch: BatchWindow, connection: object) -> MicrobatchPhaseOutcome: with _TrackingThreadPoolExecutor(max_workers=test_case.expected_max_subworker_futures) as pool: outcomes: tuple[MicrobatchPhaseOutcome, ...] = BuildScheduler._run_microbatch_subwork( scheduler, + jobs=(), pool=pool, coordinator_connection="coordinator", batches=batches, From 96a2c53cc1e477d4abfa42a0d17a7d7c599e2db6 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 18 Nov 2026 18:19:48 +0000 Subject: [PATCH 06/22] fix(jobs): make job settings fail loudly and report per job (CHI-1533) - An option the command line gives also decides every option it rules out, so a job's load_sources = false no longer clashes with --reload, and job flags go before -- so selectors after it still name jobs. - A group compiles only if its jobs agree on every build-wide setting on each target that declares them together, the same check, with the same message, a build of the group makes. - A selector that cannot resolve a named job's nodes fails, naming the job, instead of silently dropping its settings. - Plans and the run context name the jobs whose own audit failures are ignored and whose models are fully refreshed. - Document that command-line flags replace job settings and that --concurrency turns per-job limits off, and test the concurrency limit, connection count, per-node audits, groups, job: patterns and plan --no-job-settings end to end. --- .../src/job_settings/_helpers/selection.rs | 29 ++- .../src/job_settings/constants.rs | 2 - .../src/job_settings/errors.rs | 10 +- .../src/job_settings/main/declared_runs.rs | 45 ++++ .../main/group_setting_mismatch.rs | 45 ++-- .../src/job_settings/main/mod.rs | 1 + .../src/job_settings/tests/helpers.rs | 5 + .../tests/test_build_job_settings.rs | 38 +-- .../tests/test_node_job_settings.rs | 26 +- .../src/job_settings/tests/test_types.rs | 12 +- .../sqlbuild-cli/src/argv/_helpers/parsing.rs | 18 +- crates/sqlbuild-cli/src/argv/models.rs | 2 +- .../argv/tests/test_given_setting_dests.rs | 5 + .../build_command/main/render_run_context.rs | 12 +- .../sqlbuild-core/src/build_command/models.rs | 2 + .../src/cli_text/_helpers/plan_document.rs | 11 + crates/sqlbuild-core/src/cli_text/models.rs | 2 + .../src/cli_text/tests/helpers.rs | 1 + .../_helpers/sources/orchestration_values.rs | 15 +- .../src/plan_output/_helpers/plan_text.rs | 5 + .../src/plan_output/constants.rs | 4 + .../src/plan_output/main/format_plan_json.rs | 25 +- .../compiled_project/_helpers/job_catalog.rs | 40 +-- .../src/compiled_project/constants.rs | 4 + .../_helpers/build_command/execution.rs | 13 + .../_helpers/cli/named_job_settings.rs | 11 +- .../compile_driver/job_node_settings.rs | 61 ++++- .../_helpers/compiled_project/job_settings.rs | 20 +- .../references/docs/concepts/concurrency.md | 3 +- .../references/docs/concepts/orchestration.md | 11 +- .../sqlbuild/cli/commands/main/_test_types.py | 20 ++ .../src/sqlbuild/cli/commands/main/helpers.py | 26 +- .../cli/commands/main/test_job_settings.py | 238 +++++++++++++++++- .../docs/docs/concepts/concurrency.mdx | 3 +- .../docs/docs/concepts/orchestration.mdx | 11 +- 35 files changed, 621 insertions(+), 155 deletions(-) create mode 100644 crates/sqlbuild-analysis/src/job_settings/main/declared_runs.rs diff --git a/crates/sqlbuild-analysis/src/job_settings/_helpers/selection.rs b/crates/sqlbuild-analysis/src/job_settings/_helpers/selection.rs index 98a819a507..772f72cb4f 100644 --- a/crates/sqlbuild-analysis/src/job_settings/_helpers/selection.rs +++ b/crates/sqlbuild-analysis/src/job_settings/_helpers/selection.rs @@ -7,6 +7,7 @@ use crate::graph::errors::SelectorError; use crate::graph::main::resolve_selector_tokens::resolve_selector_tokens; use crate::graph::models::ProjectGraph; use crate::job_settings::_helpers::tokens::token_jobs; +use crate::job_settings::errors::JobSelectorError; use crate::job_settings::models::DeclaredRun; fn keys(graph: &ProjectGraph, selectors: &[String]) -> Result, SelectorError> { @@ -18,9 +19,18 @@ pub(crate) fn node_jobs( graph: &ProjectGraph, (select, exclude): (&[String], &[String]), declared: &BTreeMap, -) -> Result>, SelectorError> { +) -> Result>, JobSelectorError> { + let failed = |jobs: &[String], selector: &str| { + let (jobs, selector) = (jobs.to_vec(), selector.to_owned()); + move |error: SelectorError| JobSelectorError { + jobs, + selector, + error, + } + }; let mut node_jobs: BTreeMap> = BTreeMap::new(); - let excluded: BTreeSet = keys(graph, exclude)?; + let excluded: BTreeSet = + keys(graph, exclude).map_err(failed(&[], &exclude.join(" ")))?; let resolved = graph.jobs.as_ref(); for token in select .iter() @@ -30,17 +40,22 @@ pub(crate) fn node_jobs( if jobs.is_empty() { continue; } - let selected: BTreeSet = keys(graph, &[token.to_owned()])?; - for job in jobs { - let Some(Ok(own)) = resolved.and_then(|resolved| resolved.get(&job)) else { - continue; + let selected: BTreeSet = + keys(graph, &[token.to_owned()]).map_err(failed(&jobs, token))?; + for job in &jobs { + let own: &BTreeSet = match resolved.and_then(|resolved| resolved.get(job)) { + Some(Ok(own)) => own, + Some(Err(error)) => { + return Err(failed(std::slice::from_ref(job), token)(error.clone())); + } + None => continue, }; for key in selected .intersection(own) .filter(|key| !excluded.contains(*key)) { let jobs: &mut Vec = node_jobs.entry(key.clone()).or_default(); - if !jobs.contains(&job) { + if !jobs.contains(job) { jobs.push(job.clone()); } } diff --git a/crates/sqlbuild-analysis/src/job_settings/constants.rs b/crates/sqlbuild-analysis/src/job_settings/constants.rs index cddaa1c18e..64c7266665 100644 --- a/crates/sqlbuild-analysis/src/job_settings/constants.rs +++ b/crates/sqlbuild-analysis/src/job_settings/constants.rs @@ -4,8 +4,6 @@ pub const NODE_SETTINGS: [&str; 3] = ["run_tests", "ignore_audit_failures", "full_refresh"]; /// The setting that becomes a limit on how many of the job's nodes run at once inside the build. pub const JOB_LIMIT_SETTING: &str = "concurrency"; -/// Build-wide settings every named group member must agree on, checked when the group compiles. -pub const GROUP_AGREED_SETTINGS: [&str; 2] = ["vars", "changed"]; /// The help every job setting conflict gives. pub const CONFLICT_HELP: &str = "make the jobs agree, set it in only one of them, override it with \ the command-line flag, or build with --no-job-settings"; diff --git a/crates/sqlbuild-analysis/src/job_settings/errors.rs b/crates/sqlbuild-analysis/src/job_settings/errors.rs index a120b3fa72..d9b5cb5891 100644 --- a/crates/sqlbuild-analysis/src/job_settings/errors.rs +++ b/crates/sqlbuild-analysis/src/job_settings/errors.rs @@ -9,9 +9,17 @@ pub struct JobSettingConflict { pub help: &'static str, } +/// A selector token that names jobs, and why it failed to select their nodes. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct JobSelectorError { + pub jobs: Vec, + pub selector: String, + pub error: SelectorError, +} + /// Why the jobs a selection names cannot give their nodes settings. #[derive(Clone, Debug, PartialEq, Eq)] pub enum NodeJobSettingsError { - Selector(SelectorError), + Selector(JobSelectorError), Conflict(JobSettingConflict), } diff --git a/crates/sqlbuild-analysis/src/job_settings/main/declared_runs.rs b/crates/sqlbuild-analysis/src/job_settings/main/declared_runs.rs new file mode 100644 index 0000000000..0ddafefb85 --- /dev/null +++ b/crates/sqlbuild-analysis/src/job_settings/main/declared_runs.rs @@ -0,0 +1,45 @@ +//! The jobs and groups a target declares, with each job's settings and retry. + +use std::collections::BTreeMap; + +use sqlbuild_discovery::orchestration_files::models::OrchestrationInventory; + +use crate::job_settings::models::{DeclaredRun, JobRetry}; + +/// Each job's settings and retry, and each group's jobs, declared for `target`. +#[must_use] +pub fn declared_runs( + inventory: &OrchestrationInventory, + target: Option<&str>, +) -> BTreeMap { + let declared_for = |targets: &[String]| { + targets.is_empty() || target.is_some_and(|target| targets.iter().any(|name| name == target)) + }; + let mut declared: BTreeMap = BTreeMap::new(); + for (_, job) in &inventory.jobs { + if declared_for(&job.run.targets) { + declared.insert( + job.run.name.value.clone(), + DeclaredRun::Job { + settings: job.settings.clone(), + retry: job.run.retry.map(|retry| JobRetry { + attempts: retry.attempts, + delay_seconds: retry.delay_seconds, + }), + }, + ); + } + } + for (_, group) in &inventory.groups { + if declared_for(&group.run.targets) { + let members: Vec = group + .jobs + .iter() + .filter(|job| declared.contains_key(&job.value)) + .map(|job| job.value.clone()) + .collect(); + declared.insert(group.run.name.value.clone(), DeclaredRun::Group { members }); + } + } + declared +} diff --git a/crates/sqlbuild-analysis/src/job_settings/main/group_setting_mismatch.rs b/crates/sqlbuild-analysis/src/job_settings/main/group_setting_mismatch.rs index 0b393968ff..a2bbe515a2 100644 --- a/crates/sqlbuild-analysis/src/job_settings/main/group_setting_mismatch.rs +++ b/crates/sqlbuild-analysis/src/job_settings/main/group_setting_mismatch.rs @@ -1,34 +1,41 @@ -//! A group's members must agree on the build-wide settings that can change any compiled SQL. +//! A group's jobs must agree on every build-wide setting on each target that declares them. -use std::collections::BTreeMap; +use std::collections::{BTreeMap, BTreeSet}; -use serde_json::Value as Json; +use sqlbuild_discovery::orchestration_files::models::{GroupFile, OrchestrationInventory}; -use crate::job_settings::constants::GROUP_AGREED_SETTINGS; +use crate::job_settings::main::build_job_settings::build_job_settings; +use crate::job_settings::main::declared_runs::declared_runs; use crate::job_settings::models::DeclaredRun; -/// Why `members` cannot run as one build, naming the setting and two members. +/// Why `group` cannot run as one build on some target, as building it would say; `None` if never. #[must_use] pub fn group_setting_mismatch( - members: &[String], - declared: &BTreeMap, + inventory: &OrchestrationInventory, + group: &GroupFile, ) -> Option { - let value = |job: &str, key: &str| -> Option { - match declared.get(job) { - Some(DeclaredRun::Job { settings, .. }) => settings.get(key).cloned(), - _ => None, + let mut targets: BTreeSet> = BTreeSet::from([None]); + targets.extend(group.run.targets.iter().map(|target| Some(target.as_str()))); + for (_, job) in &inventory.jobs { + if group + .jobs + .iter() + .any(|member| member.value == job.run.name.value) + { + targets.extend(job.run.targets.iter().map(|target| Some(target.as_str()))); } - }; - for key in GROUP_AGREED_SETTINGS { - let mut members = members.iter(); - let Some(first) = members.next() else { + } + for target in targets { + let declared: BTreeMap = declared_runs(inventory, target); + let Some(DeclaredRun::Group { members }) = declared.get(&group.run.name.value) else { continue; }; - let expected: Option = value(first, key); - if let Some(other) = members.find(|member| value(member, key) != expected) { + if let Err(conflict) = build_job_settings(members, &declared) { + let on: String = + target.map_or_else(String::new, |target| format!(" on target {target}")); return Some(format!( - "jobs {first} and {other} set {key} differently; a group runs its jobs as one \ - build, which has one {key}" + "{}{on}; group {} builds its jobs together", + conflict.message, group.run.name.value )); } } diff --git a/crates/sqlbuild-analysis/src/job_settings/main/mod.rs b/crates/sqlbuild-analysis/src/job_settings/main/mod.rs index b642609e6f..0f917562f8 100644 --- a/crates/sqlbuild-analysis/src/job_settings/main/mod.rs +++ b/crates/sqlbuild-analysis/src/job_settings/main/mod.rs @@ -1,4 +1,5 @@ pub mod build_job_settings; +pub mod declared_runs; pub mod group_setting_mismatch; pub mod named_jobs; pub mod node_job_settings; diff --git a/crates/sqlbuild-analysis/src/job_settings/tests/helpers.rs b/crates/sqlbuild-analysis/src/job_settings/tests/helpers.rs index a5617dfffb..a1b0480457 100644 --- a/crates/sqlbuild-analysis/src/job_settings/tests/helpers.rs +++ b/crates/sqlbuild-analysis/src/job_settings/tests/helpers.rs @@ -91,6 +91,10 @@ pub(super) fn declared() -> BTreeMap { }), ), ), + ( + "broken_job".to_owned(), + job(json!({"full_refresh": true}), None), + ), ( "daily_group".to_owned(), DeclaredRun::Group { @@ -114,6 +118,7 @@ fn selections() -> BTreeMap { ("crm_us_job".to_owned(), job(&["customers"])), ("retried_job".to_owned(), job(&["stg_orders"])), ("retried_fast_job".to_owned(), job(&["stg_orders"])), + ("broken_job".to_owned(), job(&["missing_model"])), ( "daily_group".to_owned(), JobSelection::Group { diff --git a/crates/sqlbuild-analysis/src/job_settings/tests/test_build_job_settings.rs b/crates/sqlbuild-analysis/src/job_settings/tests/test_build_job_settings.rs index 21616b4447..93dfd1cd3c 100644 --- a/crates/sqlbuild-analysis/src/job_settings/tests/test_build_job_settings.rs +++ b/crates/sqlbuild-analysis/src/job_settings/tests/test_build_job_settings.rs @@ -3,10 +3,9 @@ use std::collections::BTreeMap; use serde_json::json; use crate::job_settings::main::build_job_settings::build_job_settings; -use crate::job_settings::main::group_setting_mismatch::group_setting_mismatch; use crate::job_settings::main::named_jobs::named_jobs; use crate::job_settings::tests::helpers::{declared, strings}; -use crate::job_settings::tests::test_types::{BuildJobSettingsTestCase, GroupMismatchTestCase}; +use crate::job_settings::tests::test_types::BuildJobSettingsTestCase; #[test] fn given_named_jobs_when_merging_build_settings_then_they_must_agree() { @@ -76,38 +75,3 @@ fn given_named_jobs_when_merging_build_settings_then_they_must_agree() { assert_eq!(merged, expected, "{}", test_case.description); } } - -#[test] -fn given_a_group_when_its_jobs_differ_on_vars_or_changed_then_it_cannot_run_as_one_build() { - let test_cases = [ - GroupMismatchTestCase { - description: "different vars", - members: &["crm_job", "crm_us_job"], - expected_mismatch: Some( - "jobs crm_job and crm_us_job set vars differently; a group runs its \ - jobs as one build, which has one vars", - ), - }, - GroupMismatchTestCase { - description: "vars set by only one job still differ", - members: &["orders_job", "crm_job"], - expected_mismatch: Some( - "jobs orders_job and crm_job set vars differently; a group runs its \ - jobs as one build, which has one vars", - ), - }, - GroupMismatchTestCase { - description: "node settings may differ", - members: &["orders_job", "refunds_job", "quiet_job"], - expected_mismatch: None, - }, - ]; - for test_case in test_cases { - assert_eq!( - group_setting_mismatch(&strings(test_case.members), &declared()).as_deref(), - test_case.expected_mismatch, - "{}", - test_case.description - ); - } -} diff --git a/crates/sqlbuild-analysis/src/job_settings/tests/test_node_job_settings.rs b/crates/sqlbuild-analysis/src/job_settings/tests/test_node_job_settings.rs index 33529aa04f..1851d995ce 100644 --- a/crates/sqlbuild-analysis/src/job_settings/tests/test_node_job_settings.rs +++ b/crates/sqlbuild-analysis/src/job_settings/tests/test_node_job_settings.rs @@ -5,7 +5,9 @@ use serde_json::json; use crate::job_settings::errors::NodeJobSettingsError; use crate::job_settings::models::NodeJobSettings; use crate::job_settings::tests::helpers::resolved_settings; -use crate::job_settings::tests::test_types::{NodeJobConflictTestCase, NodeJobSettingsTestCase}; +use crate::job_settings::tests::test_types::{ + JobSelectorFailureTestCase, NodeJobConflictTestCase, NodeJobSettingsTestCase, +}; #[test] fn given_named_jobs_when_resolving_node_settings_then_only_their_own_selected_nodes_take_them() { @@ -168,3 +170,25 @@ fn given_two_named_jobs_setting_one_node_differently_when_resolving_then_it_is_a ); } } + +#[test] +fn given_a_named_job_whose_selection_fails_when_resolving_then_it_fails_naming_the_job() { + let test_cases = [JobSelectorFailureTestCase { + description: "a job selecting a node that does not exist", + select: &["job:broken_job"], + expected_failure: (vec!["broken_job"], "job:broken_job", "S007"), + }]; + for test_case in test_cases { + let Err(NodeJobSettingsError::Selector(failure)) = resolved_settings(test_case.select, &[]) + else { + panic!("{}: expected a selector failure", test_case.description); + }; + let jobs: Vec<&str> = failure.jobs.iter().map(String::as_str).collect(); + assert_eq!( + (jobs, failure.selector.as_str(), failure.error.code), + test_case.expected_failure, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/job_settings/tests/test_types.rs b/crates/sqlbuild-analysis/src/job_settings/tests/test_types.rs index 582e8fc74f..823ec76cc3 100644 --- a/crates/sqlbuild-analysis/src/job_settings/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/job_settings/tests/test_types.rs @@ -13,6 +13,12 @@ pub(super) struct NodeJobSettingsTestCase { pub(super) expected_job_limits: BTreeMap<&'static str, i64>, } +pub(super) struct JobSelectorFailureTestCase { + pub(super) description: &'static str, + pub(super) select: &'static [&'static str], + pub(super) expected_failure: (Vec<&'static str>, &'static str, &'static str), +} + pub(super) struct NodeJobConflictTestCase { pub(super) description: &'static str, pub(super) select: &'static [&'static str], @@ -25,9 +31,3 @@ pub(super) struct BuildJobSettingsTestCase { pub(super) expected_named: Vec<&'static str>, pub(super) expected_settings: Result, &'static str>, } - -pub(super) struct GroupMismatchTestCase { - pub(super) description: &'static str, - pub(super) members: &'static [&'static str], - pub(super) expected_mismatch: Option<&'static str>, -} diff --git a/crates/sqlbuild-cli/src/argv/_helpers/parsing.rs b/crates/sqlbuild-cli/src/argv/_helpers/parsing.rs index 8dc4a339bf..d672165d0c 100644 --- a/crates/sqlbuild-cli/src/argv/_helpers/parsing.rs +++ b/crates/sqlbuild-cli/src/argv/_helpers/parsing.rs @@ -25,7 +25,7 @@ pub(crate) struct Run<'a> { /// The `prog` of the deepest sub-command parser entered so far. pub(crate) command_prog: Option, pub(crate) usage_failure: Option, - /// The destination of every option the arguments wrote, at any parser level. + /// The destination of every option the arguments wrote or excluded, at any parser level. pub(crate) given: BTreeSet, } @@ -365,12 +365,16 @@ impl<'p, 'r, 'h> Level<'p, 'r, 'h> { let parser: &ParserSpec = self.parser; let action: &ActionSpec = &parser.actions[index]; self.seen.insert(action.id); - if let Some(dest) = action - .dest - .as_ref() - .filter(|_| !action.option_strings.is_empty()) - { - self.run.given.insert(dest.clone()); + if !action.option_strings.is_empty() { + let excluded = parser.actions.iter().filter(|other| { + action.mutex_group.is_some() && other.mutex_group == action.mutex_group + }); + for dest in std::iter::once(action) + .chain(excluded) + .filter_map(|given| given.dest.as_ref()) + { + self.run.given.insert(dest.clone()); + } } let values: Option = self .run diff --git a/crates/sqlbuild-cli/src/argv/models.rs b/crates/sqlbuild-cli/src/argv/models.rs index fa77c26b1a..e3b7a7f63a 100644 --- a/crates/sqlbuild-cli/src/argv/models.rs +++ b/crates/sqlbuild-cli/src/argv/models.rs @@ -174,7 +174,7 @@ pub struct ArgvOutcome { pub parsed: Result, /// The usage failure that ended the parse, if one did. pub usage_failure: Option, - /// The destination of every option the arguments wrote. + /// The destination of every option the arguments wrote, or excluded by writing another. pub given: BTreeSet, } diff --git a/crates/sqlbuild-cli/src/argv/tests/test_given_setting_dests.rs b/crates/sqlbuild-cli/src/argv/tests/test_given_setting_dests.rs index 04035d176e..c42d877671 100644 --- a/crates/sqlbuild-cli/src/argv/tests/test_given_setting_dests.rs +++ b/crates/sqlbuild-cli/src/argv/tests/test_given_setting_dests.rs @@ -26,6 +26,11 @@ fn given_a_command_line_when_reading_its_own_settings_then_every_written_option_ argv: &["plan", "--no-ignore-audit-failures", "--no-job-settings"], expected_given: Some(&["ignore_audit_failures", "job_settings"]), }, + GivenSettingsTestCase { + description: "an option gives every setting of its mutually exclusive group", + argv: &["build", "--reload", "orders_job"], + expected_given: Some(&["load_sources", "reload"]), + }, GivenSettingsTestCase { description: "a selection alone gives no other setting", argv: &["build", "orders_job"], diff --git a/crates/sqlbuild-core/src/build_command/main/render_run_context.rs b/crates/sqlbuild-core/src/build_command/main/render_run_context.rs index a87bd1e6d0..3b17f6b792 100644 --- a/crates/sqlbuild-core/src/build_command/main/render_run_context.rs +++ b/crates/sqlbuild-core/src/build_command/main/render_run_context.rs @@ -6,6 +6,16 @@ use crate::build_command::models::RunContextFacts; use crate::cli_text::models::{CliDocument, CliStyle}; use crate::number_text::main::format_thousands::format_thousands; +/// `true`, the models of the named jobs that rebuild theirs from scratch, or `false`. +fn full_refresh(facts: &RunContextFacts) -> String { + match facts.full_refresh_jobs.as_slice() { + _ if facts.full_refresh => true.to_string(), + [] => false.to_string(), + [job] => format!("models of job {}", sanitize(job)), + jobs => format!("models of jobs {}", sanitize(&jobs.join(", "))), + } +} + fn date_vars(facts: &RunContextFacts) -> String { match &facts.date_vars { (None, None) => NOT_SET.to_owned(), @@ -41,7 +51,7 @@ pub fn render_run_context(facts: &RunContextFacts, style: CliStyle) -> String { ) }, ), - ("full_refresh", facts.full_refresh.to_string()), + ("full_refresh", full_refresh(facts)), ( "selected", format!( diff --git a/crates/sqlbuild-core/src/build_command/models.rs b/crates/sqlbuild-core/src/build_command/models.rs index bf8b0d29e7..a0d9ab15cb 100644 --- a/crates/sqlbuild-core/src/build_command/models.rs +++ b/crates/sqlbuild-core/src/build_command/models.rs @@ -182,6 +182,8 @@ pub struct RunContextFacts { /// The configuration layer the concurrency came from. pub concurrency_source: String, pub full_refresh: bool, + /// The named jobs whose own models the build rebuilds from scratch. + pub full_refresh_jobs: Vec, pub selected_count: usize, pub total_count: usize, /// Valid ISO `start_date` and `end_date` variables. diff --git a/crates/sqlbuild-core/src/cli_text/_helpers/plan_document.rs b/crates/sqlbuild-core/src/cli_text/_helpers/plan_document.rs index 5331766ced..92130d76cc 100644 --- a/crates/sqlbuild-core/src/cli_text/_helpers/plan_document.rs +++ b/crates/sqlbuild-core/src/cli_text/_helpers/plan_document.rs @@ -13,6 +13,12 @@ use crate::cli_text::constants::{ }; use crate::cli_text::models::{CliStyle, ModelPlanText, PlanTextReport}; +/// `job a` or `jobs a, b`. +fn job_list(jobs: &[String]) -> String { + let noun: &str = if jobs.len() == 1 { "job" } else { "jobs" }; + format!("{noun} {}", jobs.join(", ")) +} + fn header_lines(report: &PlanTextReport, full_refresh: bool) -> Vec { let style: CliStyle = CliStyle { use_color: true }; let mut lines: Vec = Vec::new(); @@ -25,6 +31,11 @@ fn header_lines(report: &PlanTextReport, full_refresh: bool) -> Vec { } if report.ignore_audit_failures { lines.push(style.warning(IGNORED_AUDIT_FAILURES_NOTICE)); + } else if !report.ignore_audit_failure_jobs.is_empty() { + lines.push(style.warning(&format!( + "{IGNORED_AUDIT_FAILURES_NOTICE} for the audits of {}", + job_list(&report.ignore_audit_failure_jobs) + ))); } lines } diff --git a/crates/sqlbuild-core/src/cli_text/models.rs b/crates/sqlbuild-core/src/cli_text/models.rs index 189916b4e5..6e9b2f5926 100644 --- a/crates/sqlbuild-core/src/cli_text/models.rs +++ b/crates/sqlbuild-core/src/cli_text/models.rs @@ -430,6 +430,8 @@ pub struct PlanTextReport { pub pre_build_lines: Vec, /// Audit failures hold nothing back in the build this plan previews. pub ignore_audit_failures: bool, + /// The named jobs whose own audit failures hold nothing back. + pub ignore_audit_failure_jobs: Vec, } /// The structural kind of a rendered plan line. diff --git a/crates/sqlbuild-core/src/cli_text/tests/helpers.rs b/crates/sqlbuild-core/src/cli_text/tests/helpers.rs index 85d07a22b4..1d4ce415c2 100644 --- a/crates/sqlbuild-core/src/cli_text/tests/helpers.rs +++ b/crates/sqlbuild-core/src/cli_text/tests/helpers.rs @@ -56,6 +56,7 @@ pub(super) fn plan_report(models: Vec) -> PlanTextReport { pruned_models: Vec::new(), pre_build_lines: Vec::new(), ignore_audit_failures: false, + ignore_audit_failure_jobs: Vec::new(), } } diff --git a/crates/sqlbuild-python-front/src/discovery/_helpers/sources/orchestration_values.rs b/crates/sqlbuild-python-front/src/discovery/_helpers/sources/orchestration_values.rs index 3c4c731b92..2a05962a73 100644 --- a/crates/sqlbuild-python-front/src/discovery/_helpers/sources/orchestration_values.rs +++ b/crates/sqlbuild-python-front/src/discovery/_helpers/sources/orchestration_values.rs @@ -8,7 +8,6 @@ use pyo3::prelude::Python; use serde_json::Value; use sqlbuild_analysis::graph::main::parse_selector::parse_selector; use sqlbuild_analysis::job_settings::main::group_setting_mismatch::group_setting_mismatch; -use sqlbuild_analysis::job_settings::models::DeclaredRun; use sqlbuild_cli::argv::main::job_settings_arguments::job_settings_arguments; use sqlbuild_discovery::models::{DiscoveryFailure, FailureKind}; use sqlbuild_discovery::orchestration_files::models::{ @@ -60,20 +59,8 @@ pub(crate) fn check_orchestration_values( })?; } } - let declared: BTreeMap = inventory - .jobs - .iter() - .map(|(_, job)| { - let run = DeclaredRun::Job { - settings: job.settings.clone(), - retry: None, - }; - (job.run.name.value.clone(), run) - }) - .collect(); for (path, group) in &inventory.groups { - let members: Vec = group.jobs.iter().map(|job| job.value.clone()).collect(); - if let Some(message) = group_setting_mismatch(&members, &declared) { + if let Some(message) = group_setting_mismatch(inventory, group) { return Err(failure(path, (group.run.name.position, message))); } } diff --git a/crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_text.rs b/crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_text.rs index 9dcef5903c..937d5ca627 100644 --- a/crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_text.rs +++ b/crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_text.rs @@ -34,6 +34,7 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::cursor::main::scalars::sentinel_to_token::sentinel_to_token; +use sqlbuild_python_project::compiled_project::constants::JOB_IGNORED_AUDIT_FAILURES_KEY; const SKIP_ACTION: &str = "skip"; const QUERY_CHANGED: &str = "query_changed"; @@ -529,5 +530,9 @@ pub(crate) fn report(arguments: &KeywordArguments<'_>) -> PyResult, options: Option>) -> P )?; let metadata: Bound<'_, PyAny> = native_field(&plan, "metadata")?; if metadata.is_truthy()? { - result.set_item("metadata", metadata)?; + result.set_item("metadata", &metadata)?; } let synced_copy: Bound<'_, PyAny> = arguments.value(SYNCED_COPY_KEY)?; if !synced_copy.is_none() { @@ -119,5 +125,20 @@ pub fn format_plan_json(py: Python<'_>, options: Option>) -> P if native_field(&plan, IGNORE_AUDIT_FAILURES_KEY)?.is_truthy()? { result.set_item(IGNORE_AUDIT_FAILURES_KEY, true)?; } + for (key, label) in [ + ( + JOB_IGNORED_AUDIT_FAILURES_KEY, + IGNORE_AUDIT_FAILURES_JOBS_KEY, + ), + (JOB_FULL_REFRESH_KEY, FULL_REFRESH_JOBS_KEY), + ] { + let jobs: Bound<'_, PyAny> = mapping_get(&metadata, key)?; + if !jobs.is_none() { + result.set_item( + label, + PyList::new(py, jobs.try_iter()?.collect::>>()?)?, + )?; + } + } indented_json(result.as_any()) } diff --git a/crates/sqlbuild-python-project/src/compiled_project/_helpers/job_catalog.rs b/crates/sqlbuild-python-project/src/compiled_project/_helpers/job_catalog.rs index 3a1980033a..31f8a54227 100644 --- a/crates/sqlbuild-python-project/src/compiled_project/_helpers/job_catalog.rs +++ b/crates/sqlbuild-python-project/src/compiled_project/_helpers/job_catalog.rs @@ -9,7 +9,8 @@ use pyo3::exceptions::PyValueError; use pyo3::prelude::{PyResult, Python}; use sqlbuild_analysis::graph::main::parse_selector::parse_selector; use sqlbuild_analysis::graph::models::{JobSelection, ParsedSelector}; -use sqlbuild_analysis::job_settings::models::{DeclaredRun, JobRetry}; +use sqlbuild_analysis::job_settings::main::declared_runs::declared_runs; +use sqlbuild_analysis::job_settings::models::DeclaredRun; use sqlbuild_cli::native_commands::main::effective_target::effective_target; use sqlbuild_discovery::orchestration_files::models::{OrchestrationInventory, RunFile}; @@ -120,43 +121,6 @@ fn loaded_jobs(py: Python<'_>) -> PyResult, -) -> BTreeMap { - let declared_for = |targets: &[String]| { - targets.is_empty() || target.is_some_and(|target| targets.iter().any(|name| name == target)) - }; - let mut declared: BTreeMap = BTreeMap::new(); - for (_, job) in &inventory.jobs { - if declared_for(&job.run.targets) { - declared.insert( - job.run.name.value.clone(), - DeclaredRun::Job { - settings: job.settings.clone(), - retry: job.run.retry.map(|retry| JobRetry { - attempts: retry.attempts, - delay_seconds: retry.delay_seconds, - }), - }, - ); - } - } - for (_, group) in &inventory.groups { - if declared_for(&group.run.targets) { - let members: Vec = group - .jobs - .iter() - .filter(|job| declared.contains_key(&job.value)) - .map(|job| job.value.clone()) - .collect(); - declared.insert(group.run.name.value.clone(), DeclaredRun::Group { members }); - } - } - declared -} - /// Each job's selector tokens and each group's members declared for `target`, and the names a schedule or sensor starts there. fn inventory_catalog(inventory: &OrchestrationInventory, target: Option<&str>) -> Catalog { catalog_where(inventory, |targets: &[String]| { diff --git a/crates/sqlbuild-python-project/src/compiled_project/constants.rs b/crates/sqlbuild-python-project/src/compiled_project/constants.rs index f0aec16d9e..637114e246 100644 --- a/crates/sqlbuild-python-project/src/compiled_project/constants.rs +++ b/crates/sqlbuild-python-project/src/compiled_project/constants.rs @@ -32,3 +32,7 @@ pub const REACH_SETTING: &str = "reach"; pub const JOB_RUN_REACH: &str = "all"; /// What a job run with `changed: True` counts when `[changes] counts` is not set: every change. pub const JOB_RUN_COUNTS: [&str; 2] = ["definition", "source_data"]; +/// Plan metadata naming the jobs whose own audits a build lets fail. +pub const JOB_IGNORED_AUDIT_FAILURES_KEY: &str = "job_ignored_audit_failures"; +/// Plan metadata naming the jobs whose own models a build rebuilds from scratch. +pub const JOB_FULL_REFRESH_KEY: &str = "job_full_refresh"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs index a27eeb9fab..64e6101f3e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs @@ -66,6 +66,7 @@ use sqlbuild_python_planner::consumption::models::Prepared; use sqlbuild_python_planner::freshness::main::append_freshness_records::append_freshness_records; use sqlbuild_python_planner::observations::main::append_build_observations::append_build_observations; use sqlbuild_python_planner::planner_resolution::main::cursor::type_cursor_overrides::type_cursor_overrides; +use sqlbuild_python_project::compiled_project::constants::JOB_FULL_REFRESH_KEY; use sqlbuild_python_project::graph::main::python_nodes::python_node_graph::python_node_graph; use sqlbuild_state::consumption::models::ChangedPolicy; @@ -175,6 +176,7 @@ fn run_context_facts( concurrency: concurrency.limit, concurrency_source: concurrency.source.to_string(), full_refresh: context.request_flag("full_refresh")?, + full_refresh_jobs: full_refresh_jobs(pipeline)?, selected_count, total_count, date_vars: ( @@ -185,6 +187,17 @@ fn run_context_facts( }) } +/// The named jobs whose own models the planned build rebuilds from scratch. +fn full_refresh_jobs(pipeline: &Bound<'_, PyAny>) -> PyResult> { + let metadata: Bound<'_, PyAny> = + native_field(&native_field(pipeline, "plan_output")?, "metadata")?; + let jobs: Bound<'_, PyAny> = mapping_get(&metadata, JOB_FULL_REFRESH_KEY)?; + if jobs.is_none() { + return Ok(Vec::new()); + } + jobs.try_iter()?.map(|job| job?.extract()).collect() +} + /// Write the resolved non-secret placement, mode and selection context. pub(crate) fn write_run_context( context: &BuildContext<'_>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/named_job_settings.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/named_job_settings.rs index ec362ee046..1c0e187a78 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/named_job_settings.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/named_job_settings.rs @@ -18,6 +18,9 @@ use crate::bindings::_helpers::compiled_project::job_settings::{ use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::types::ErrorId; +/// Ends the options; job flags go before it so they are never read as selectors. +const OPTIONS_TERMINATOR: &str = "--"; + /// The commands whose selections take the settings of the jobs they name. const JOB_SETTING_COMMANDS: [&str; 2] = ["build", "plan"]; @@ -93,7 +96,13 @@ fn named_job_argv( if flags.is_empty() { return Ok(None); } - Ok(Some(argv.iter().cloned().chain(flags).collect())) + let terminator: usize = argv + .iter() + .position(|argument| argument == OPTIONS_TERMINATOR) + .unwrap_or(argv.len()); + let mut job_argv: Vec = argv.to_vec(); + job_argv.splice(terminator..terminator, flags); + Ok(Some(job_argv)) } /// Whether the command meets `error` again, and reports it, when it runs. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/job_node_settings.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/job_node_settings.rs index 5378a611e2..7a64d3b42c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/job_node_settings.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/job_node_settings.rs @@ -14,9 +14,14 @@ use sqlbuild_analysis::job_settings::main::named_jobs::named_jobs; use sqlbuild_analysis::job_settings::main::node_job_settings::node_job_settings; use sqlbuild_analysis::job_settings::models::{DeclaredRun, NodeJobSettings}; -use crate::bindings::_helpers::compiled_project::job_settings::job_setting_conflict; +use crate::bindings::_helpers::compiled_project::job_settings::{ + job_selector_error, job_setting_conflict, +}; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::replaced::replaced; +use sqlbuild_python_project::compiled_project::constants::{ + JOB_FULL_REFRESH_KEY, JOB_IGNORED_AUDIT_FAILURES_KEY, +}; use sqlbuild_python_project::compiled_project::main::graph_with_jobs::graph_with_jobs; use sqlbuild_python_project::compiled_project::main::job_declarations::job_declarations; use sqlbuild_python_project::graph::main::nodes::native_key::native_key; @@ -31,6 +36,32 @@ const IGNORE_AUDIT_FAILURES: &str = "ignore_audit_failures"; /// What the planned build takes from its named jobs, or nothing when it names none. pub(crate) struct PlannedJobSettings { settings: NodeJobSettings, + /// The named jobs that set `ignore_audit_failures = true` for nodes they select. + ignoring_jobs: Vec, + /// The named jobs that set `full_refresh = true` for models they select. + refreshing_jobs: Vec, +} + +/// The named jobs of `settings` that set `setting` to true for a selected node of `kind`, if given. +fn jobs_setting( + settings: &NodeJobSettings, + declared: &BTreeMap, + (setting, kind): (&str, Option<&str>), +) -> Vec { + let mut jobs: BTreeSet = BTreeSet::new(); + for (key, node_jobs) in &settings.node_jobs { + if kind.is_some_and(|kind| key.0 != kind) { + continue; + } + for job in node_jobs { + if let Some(DeclaredRun::Job { settings, .. }) = declared.get(job) + && settings.get(setting) == Some(&Value::Bool(true)) + { + jobs.insert(job.clone()); + } + } + } + jobs.into_iter().collect() } fn strings(value: &Bound<'_, PyAny>) -> PyResult> { @@ -66,8 +97,12 @@ pub(crate) fn planned_job_settings( let all: Vec = select.iter().chain(&exclude).cloned().collect(); let graph = graph_with_jobs(py, native.graph(), &all, &[])?; match node_job_settings(&graph, (&select, &exclude), &declared) { - Ok(settings) => Ok(Some(PlannedJobSettings { settings })), - Err(NodeJobSettingsError::Selector(_)) => Ok(None), + Ok(settings) => Ok(Some(PlannedJobSettings { + ignoring_jobs: jobs_setting(&settings, &declared, (IGNORE_AUDIT_FAILURES, None)), + refreshing_jobs: jobs_setting(&settings, &declared, (FULL_REFRESH, Some(MODEL_KIND))), + settings, + })), + Err(NodeJobSettingsError::Selector(failure)) => Err(job_selector_error(py, &failure)), Err(NodeJobSettingsError::Conflict(conflict)) => Err(job_setting_conflict(py, &conflict)), } } @@ -206,6 +241,26 @@ impl PlannedJobSettings { changes.push(("audit_entries", PyTuple::new(py, audits)?.into_any())); } } + let ignoring: &[String] = if cli_ignore_audit_failures.is_none() { + &self.ignoring_jobs + } else { + &[] + }; + if !ignoring.is_empty() || !self.refreshing_jobs.is_empty() { + let metadata: Bound<'py, PyDict> = native_field(plan, "metadata")? + .cast_into::()? + .copy()?; + if !ignoring.is_empty() { + metadata.set_item(JOB_IGNORED_AUDIT_FAILURES_KEY, PyTuple::new(py, ignoring)?)?; + } + if !self.refreshing_jobs.is_empty() { + metadata.set_item( + JOB_FULL_REFRESH_KEY, + PyTuple::new(py, &self.refreshing_jobs)?, + )?; + } + changes.push(("metadata", metadata.into_any())); + } if changes.is_empty() { return Ok(plan.clone()); } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/job_settings.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/job_settings.rs index 52d0dfe164..34d361801e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/job_settings.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/job_settings.rs @@ -10,7 +10,7 @@ use serde_json::Value; use sqlbuild_analysis::graph::main::parse_selector::parse_selector; use sqlbuild_analysis::graph::models::ParsedSelector; use sqlbuild_analysis::job_settings::constants::JOB_SETTING_CONFLICT_CODE; -use sqlbuild_analysis::job_settings::errors::JobSettingConflict; +use sqlbuild_analysis::job_settings::errors::{JobSelectorError, JobSettingConflict}; use sqlbuild_analysis::job_settings::main::build_job_settings::build_job_settings; use sqlbuild_analysis::job_settings::main::named_jobs::named_jobs; @@ -84,3 +84,21 @@ pub(crate) fn job_setting_conflict(py: Python<'_>, conflict: &JobSettingConflict (Some(JOB_SETTING_CONFLICT_CODE), Some(conflict.help)), ) } + +/// A `CliUserError` naming the jobs whose selection a selector failed to resolve, with its code. +pub(crate) fn job_selector_error(py: Python<'_>, failure: &JobSelectorError) -> PyErr { + let jobs: String = if failure.jobs.is_empty() { + "the named jobs".to_owned() + } else { + format!("job {}", failure.jobs.join(", ")) + }; + new_error( + py, + ErrorId::CliUser, + &format!( + "cannot apply the settings of {jobs}: selector '{}' failed: {}", + failure.selector, failure.error.message + ), + (Some(failure.error.code), failure.error.help.as_deref()), + ) +} diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/concurrency.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/concurrency.md index fe5dc414a9..7c3a500318 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/concurrency.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/concurrency.md @@ -49,7 +49,8 @@ concurrency = 2 [Target concurrency](project-configuration.md#target-concurrency) for the full order. A job's `concurrency` limits how many of that job's nodes build at once in a build that names the job, within the build's own limit. When one job is at its limit, a ready node of another job starts -instead; see [Settings when a build names a job](orchestration.md#settings-when-a-build-names-a-job). The job's page shows the value its runs use on the target and where it came from. +instead. Microbatch sub-work counts against the job too. `--concurrency` on the command line turns +the per-job limits off; see [Settings when a build names a job](orchestration.md#settings-when-a-build-names-a-job). The job's page shows the value its runs use on the target and where it came from. ## Runs at once on a server: `max_workers` diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/orchestration.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/orchestration.md index 484d6b04ba..2b5e3f6baf 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/orchestration.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/orchestration.md @@ -109,7 +109,11 @@ Naming a group names each of its jobs. The first value set wins: 3. The target's setting 4. The project's setting -`--no-job-settings` leaves out every job's settings. +A command-line flag replaces the job's setting rather than merging with it: `--vars` replaces the +job's whole `vars`. An option that rules another out counts as setting both, so `--reload` wins +over a job's `load_sources = false`. Any `--concurrency` sets the build's limit and turns off the +per-job limits. The job's settings are added before a `--` on the command line, so selectors after +it still name jobs. `--no-job-settings` leaves out every job's settings. Some settings apply to the job's own nodes only, so a build that names two jobs can treat their nodes differently: @@ -127,7 +131,8 @@ gives them to `orders` alone, and `+job:nightly_orders_job` builds the job's ups them. Every other setting, such as `vars` or `changed`, applies to the whole build, so every named job -must set it the same way. A group whose jobs set different `vars` or `changed` fails to compile. +must set it the same way. A group whose jobs set any of these differently on a target that declares +them together fails to compile, with the error a build of the group would give. Two named jobs that give one node different values of the same setting fail the build with `S027`, naming the node, both jobs and the setting. A setting only one of them sets is no conflict. @@ -316,7 +321,7 @@ Every problem with an orchestration file is a `D017` error at its file, line and | `job nightly_orders_job resources: cpus must be a positive number` | Set `cpus` above zero | | `job 'nightly_orders_job' uses pool 'warehouse_hevy', which [pools] in sqlbuild_project.toml does not declare` | Fix the name, or declare the pool in [`[pools]`](project-configuration.md#pools) | | `job 'nightly_orders_job' takes 2 slots of pool 'supplier_api', more than its limit of 1 on target 'production'` | Take fewer slots, or raise the limit in `[targets.production.pools]` | -| `jobs orders_job and crm_job set vars differently; a group runs its jobs as one build, which has one vars` | Give the group's jobs the same `vars` and `changed`, or move one job to another group | +| `jobs orders_job and crm_job set vars differently ({"amount":"5"} and {"amount":"9"}); a build applies vars to everything it runs; group daily_job_group builds its jobs together` | Give the group's jobs the same build-wide settings, or move one job to another group | | `sensor 'orders_sensor' is declared here, but importing the module does not define it` | Assign the sensor to a module-level name or list, not a dictionary or function body | | `group dependencies contain a cycle in job group daily_orders_job_group: orders_marts_job waits for staging_orders_job (model:orders reads model:stg_orders); staging_orders_job waits for orders_marts_job (model:stg_orders reads seed:regions)` | Move each node to one job, and give seeds and nodes that several members read their own job that runs first | diff --git a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py index 7ab20b846b..f093039160 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py @@ -1498,3 +1498,23 @@ class JobFullRefreshTestCase: description: str build_args: tuple[str, ...] expected_counts: tuple[tuple[int, int], ...] + + +@dataclass(frozen=True) +class JobConcurrencyTestCase: + description: str + build_args: tuple[str, ...] + job_nodes: tuple[str, ...] + build_nodes: tuple[str, ...] + expected_job_peak: int + expected_least_build_peak: int + + +@dataclass(frozen=True) +class JobOutputTestCase: + description: str + command_args: tuple[str, ...] + expected_exit_code: int + expected_present: tuple[str, ...] + expected_absent: tuple[str, ...] + extra_files: dict[str, str] = field(default_factory=dict) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/helpers.py b/tests/integration/src/sqlbuild/cli/commands/main/helpers.py index 0be7c535d1..950501f5e8 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/helpers.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/helpers.py @@ -8,8 +8,10 @@ import sys import tempfile import time +from collections import defaultdict from collections.abc import Callable -from itertools import takewhile +from datetime import datetime, timedelta +from itertools import accumulate, takewhile from pathlib import Path from types import ModuleType from typing import Any, cast @@ -55,6 +57,9 @@ ) +_COMPLETED_EVENT: str = "resource_attempt_completed" + + def artifact_tree(root: Path) -> dict[str, bytes]: """Return each file under root by its POSIX path relative to root, or nothing when missing.""" @@ -1455,3 +1460,22 @@ def prepare_warehouse(*, project_dir: Path, sql: str) -> None: with duckdb.connect(str(project_dir / "demo.duckdb")) as warehouse: warehouse.execute(sql) + + +def peak_concurrency(*, events_path: Path, names: tuple[str, ...]) -> int: + """The most of `names` a build's event output shows running at once.""" + + events: dict[tuple[str, str], list[dict[str, Any]]] = defaultdict(list) + for line in events_path.read_text(encoding="utf-8").splitlines(): + event: dict[str, Any] = json.loads(line) + events[(event["event_type"], str(event.get("resource_name")))].append(event) + ends: list[tuple[datetime, float]] = [ + (datetime.fromisoformat(event["occurred_at"]), event["duration_ms"]) + for name in names + for event in events[(_COMPLETED_EVENT, name)] + ] + changes: list[tuple[datetime, int]] = sorted( + [(end - timedelta(milliseconds=duration), 1) for end, duration in ends] + + [(end, -1) for end, _ in ends] + ) + return max(accumulate(change for _, change in changes), default=0) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_job_settings.py b/tests/integration/src/sqlbuild/cli/commands/main/test_job_settings.py index 25b9628e3e..c670c89b47 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_job_settings.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_job_settings.py @@ -8,11 +8,14 @@ from sqlbuild._native import run_cli as main from tests.integration.src.sqlbuild.cli.commands.main._test_types import ( + JobConcurrencyTestCase, JobFullRefreshTestCase, + JobOutputTestCase, JobSettingsBuildTestCase, JobSettingsConflictTestCase, ) from tests.integration.src.sqlbuild.cli.commands.main.helpers import ( + peak_concurrency, query_duckdb_rows, write_project_files, ) @@ -20,7 +23,8 @@ _PROJECT_FILES: dict[str, str] = { "sqlbuild_project.toml": ( 'name = "demo"\nadapter = "duckdb"\ndefault_target = "dev"\n\n' - '[connection]\ndatabase = "demo.duckdb"\n\n[targets.dev]\n\n[vars]\namount = "1"\n' + '[connection]\ndatabase = "demo.duckdb"\n\n[targets.dev]\n\n[targets.prod]\n\n' + '[settings]\nconcurrency = 4\n\n[vars]\namount = "1"\n' ), "models/stg_orders.sql": ( "MODEL (description 'Staged orders.', materialized table);\n" @@ -61,6 +65,36 @@ 'name = "events_job"\ndescription = "Events."\nselect = "order_events"\n\n' "[settings]\nfull_refresh = true\n" ), + "orchestration/jobs/refund_events_job.toml": ( + 'name = "refund_events_job"\ndescription = "Refund events."\nselect = "refund_events"\n' + ), + "orchestration/job_groups/events_job_group.toml": ( + 'name = "events_job_group"\ndescription = "Events."\n' + 'jobs = ["events_job", "refund_events_job"]\n' + ), + "models/lenient_model.sql": ( + "MODEL (description 'Lenient.', materialized table, " + "columns (id (type INTEGER, audits [not_null])));\nSELECT CAST(NULL AS INTEGER) AS id\n" + ), + "orchestration/jobs/lenient_job.toml": ( + 'name = "lenient_job"\ndescription = "Lenient."\nselect = "lenient_model"\n\n' + "[settings]\nignore_audit_failures = true\n" + ), + "orchestration/jobs/loads_off_job.toml": ( + 'name = "loads_off_job"\ndescription = "No loads."\nselect = "customers"\n\n' + "[settings]\nload_sources = false\n" + ), + **{ + f"models/slow_{name}.sql": ( + f"MODEL (description 'Slow {name}.', materialized table);\n" + "SELECT sum(i * i % 7) AS total FROM range(25000000) AS t(i)\n" + ) + for name in ("a", "b", "c", "d") + }, + "orchestration/jobs/slow_job.toml": ( + 'name = "slow_job"\ndescription = "Slow."\nselect = "slow_a slow_b"\n\n' + "[settings]\nconcurrency = 1\n" + ), } @@ -73,6 +107,18 @@ expected_amount=5, expected_fragments=("(concurrency: 1)", "TOTAL=2"), ), + JobSettingsBuildTestCase( + description="a job: pattern names the jobs it matches", + build_args=("--select", "job:orders_*"), + expected_amount=5, + expected_fragments=("(concurrency: 1)", "TOTAL=2"), + ), + JobSettingsBuildTestCase( + description="job flags go before -- so a selector after it still names the job", + build_args=("--", "orders_job"), + expected_amount=5, + expected_fragments=("(concurrency: 1)", "TOTAL=2"), + ), JobSettingsBuildTestCase( description="a bare job name names the job", build_args=("--select", "orders_job"), @@ -102,13 +148,13 @@ description="--no-job-settings builds with the project's settings", build_args=("--select", "job:orders_job", "--no-job-settings"), expected_amount=1, - expected_fragments=("TOTAL=3",), + expected_fragments=("(concurrency: 4)", "TOTAL=3"), ), JobSettingsBuildTestCase( description="selecting the job's nodes without naming it takes none of its settings", build_args=("--select", "+orders"), expected_amount=1, - expected_fragments=("TOTAL=3",), + expected_fragments=("(concurrency: 4)", "TOTAL=3"), ), ], ids=lambda case: case.description, @@ -153,6 +199,25 @@ def test_given_a_named_job_when_building_then_its_settings_apply_below_the_comma 'and {"amount":"9"}); a build applies vars to everything it runs' ), ), + JobSettingsConflictTestCase( + description="a group whose jobs set any build-wide setting differently does not compile", + command_args=("compile",), + expected_fragment=( + "jobs events_job and fast_fail_job set fail_fast differently (not set and true); " + "a build applies fail_fast to everything it runs; group fail_job_group builds its " + "jobs together" + ), + extra_files={ + "orchestration/jobs/fast_fail_job.toml": ( + 'name = "fast_fail_job"\ndescription = "Fails fast."\nselect = "customers"\n\n' + "[settings]\nfail_fast = true\n" + ), + "orchestration/job_groups/fail_job_group.toml": ( + 'name = "fail_job_group"\ndescription = "Fails."\n' + 'jobs = ["events_job", "fast_fail_job"]\n' + ), + }, + ), JobSettingsConflictTestCase( description="two named jobs giving one node different run_tests conflict", command_args=("build", "--select", "job:orders_job", "job:checked_orders_job"), @@ -165,8 +230,9 @@ def test_given_a_named_job_when_building_then_its_settings_apply_below_the_comma description="a group whose jobs set different vars does not compile", command_args=("compile",), expected_fragment=( - "jobs orders_job and crm_job set vars differently; a group runs its jobs as one " - "build, which has one vars" + 'jobs orders_job and crm_job set vars differently ({"amount":"5"} and ' + '{"amount":"9"}); a build applies vars to everything it runs; group ' + "daily_job_group builds its jobs together" ), extra_files={ "orchestration/job_groups/daily_job_group.toml": ( @@ -229,3 +295,165 @@ def test_given_a_named_job_with_full_refresh_when_building_then_only_its_models_ ) == test_case.expected_counts ) + + +@pytest.mark.parametrize( + "test_case", + [ + JobConcurrencyTestCase( + description="a job's concurrency limits its own nodes while other nodes run beside them", + build_args=("--select", "job:slow_job", "slow_c", "slow_d"), + job_nodes=("slow_a", "slow_b"), + build_nodes=("slow_a", "slow_b", "slow_c", "slow_d"), + expected_job_peak=1, + expected_least_build_peak=2, + ), + JobConcurrencyTestCase( + description="without job settings the job's nodes run together", + build_args=("--select", "job:slow_job", "slow_c", "slow_d", "--no-job-settings"), + job_nodes=("slow_a", "slow_b"), + build_nodes=("slow_a", "slow_b", "slow_c", "slow_d"), + expected_job_peak=2, + expected_least_build_peak=2, + ), + ], + ids=lambda case: case.description, +) +def test_given_a_job_concurrency_when_building_then_its_nodes_never_exceed_it( + test_case: JobConcurrencyTestCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + write_project_files(project_dir=tmp_path, files=_PROJECT_FILES) + events: Path = tmp_path / "events.jsonl" + + exit_code: int = main( + [ + "--project-dir", + str(tmp_path), + "--no-color", + "build", + *test_case.build_args, + "--event-output", + str(events), + ] + ) + captured: str = "".join(capsys.readouterr()) + + assert exit_code == 0, captured + assert ( + peak_concurrency(events_path=events, names=test_case.job_nodes), + peak_concurrency(events_path=events, names=test_case.build_nodes) + >= test_case.expected_least_build_peak, + ) == (test_case.expected_job_peak, True), captured + + +@pytest.mark.parametrize( + "test_case", + [ + JobOutputTestCase( + description="a plan shows the job's full refresh", + command_args=("plan", "--select", "job:events_job"), + expected_exit_code=0, + expected_present=("Plan ready full refresh, 1 selected",), + expected_absent=(), + ), + JobOutputTestCase( + description="plan --no-job-settings leaves out the job's full refresh", + command_args=("plan", "--select", "job:events_job", "--no-job-settings"), + expected_exit_code=0, + expected_present=("Plan ready 1 selected",), + expected_absent=("full refresh",), + ), + JobOutputTestCase( + description="plan JSON names the jobs that full refresh their models", + command_args=("plan", "--json", "--select", "job:events_job"), + expected_exit_code=0, + expected_present=('"full_refresh_jobs": [\n "events_job"\n ]',), + expected_absent=(), + ), + JobOutputTestCase( + description="a plan says whose audit failures will not stop the build", + command_args=("plan", "--select", "job:lenient_job"), + expected_exit_code=0, + expected_present=( + "Audit failures won't stop this build for the audits of job lenient_job", + ), + expected_absent=(), + ), + JobOutputTestCase( + description="a named job's ignore_audit_failures lets its audit failures through", + command_args=("build", "--select", "job:lenient_job"), + expected_exit_code=0, + expected_present=("1 audit failure ignored",), + expected_absent=(), + ), + JobOutputTestCase( + description="the same node built without naming the job fails its audit", + command_args=("build", "--select", "lenient_model"), + expected_exit_code=1, + expected_present=("not_null (id)",), + expected_absent=("ignored",), + ), + JobOutputTestCase( + description="the run context names the jobs whose models are fully refreshed", + command_args=("build", "--verbose", "--select", "job:events_job"), + expected_exit_code=0, + expected_present=("full_refresh models of job events_job",), + expected_absent=(), + ), + JobOutputTestCase( + description="naming a group applies each job's settings to its own models", + command_args=("plan", "--json", "--select", "job:events_job_group"), + expected_exit_code=0, + expected_present=('"full_refresh_jobs": [\n "events_job"\n ]',), + expected_absent=("refund_events_job",), + ), + JobOutputTestCase( + description="a group compiles when its disagreeing jobs never share a target", + command_args=("compile",), + expected_exit_code=0, + expected_present=("Project compiled",), + expected_absent=("D017",), + extra_files={ + "orchestration/jobs/prod_fail_job.toml": ( + 'name = "prod_fail_job"\ndescription = "Prod."\nselect = "customers"\n' + 'targets = ["prod"]\n\n[settings]\nfail_fast = true\n' + ), + "orchestration/jobs/dev_job.toml": ( + 'name = "dev_job"\ndescription = "Dev."\nselect = "customers"\n' + 'targets = ["dev"]\n' + ), + "orchestration/job_groups/split_job_group.toml": ( + 'name = "split_job_group"\ndescription = "Split."\n' + 'jobs = ["prod_fail_job", "dev_job"]\n' + ), + }, + ), + JobOutputTestCase( + description="a command-line option beats a job option it excludes", + command_args=("build", "--select", "job:loads_off_job", "--reload"), + expected_exit_code=0, + expected_present=("Completed successfully",), + expected_absent=("not allowed with",), + ), + ], + ids=lambda case: case.description, +) +def test_given_a_named_job_when_planning_or_building_then_the_output_reflects_its_settings( + test_case: JobOutputTestCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + write_project_files(project_dir=tmp_path, files={**_PROJECT_FILES, **test_case.extra_files}) + + exit_code: int = main(["--project-dir", str(tmp_path), "--no-color", *test_case.command_args]) + captured: str = "".join(capsys.readouterr()) + + assert exit_code == test_case.expected_exit_code, captured + assert {fragment: fragment in captured for fragment in test_case.expected_present} == ( + dict.fromkeys(test_case.expected_present, True) + ), captured + assert {fragment: fragment in captured for fragment in test_case.expected_absent} == ( + dict.fromkeys(test_case.expected_absent, False) + ), captured diff --git a/website/src/content/docs/docs/concepts/concurrency.mdx b/website/src/content/docs/docs/concepts/concurrency.mdx index 70b3ffaab3..b948bf1d1a 100644 --- a/website/src/content/docs/docs/concepts/concurrency.mdx +++ b/website/src/content/docs/docs/concepts/concurrency.mdx @@ -38,7 +38,8 @@ concurrency = 2 [Target concurrency](/docs/concepts/project-configuration#target-concurrency) for the full order. A job's `concurrency` limits how many of that job's nodes build at once in a build that names the job, within the build's own limit. When one job is at its limit, a ready node of another job starts -instead; see [Settings when a build names a job](/docs/concepts/orchestration#settings-when-a-build-names-a-job). The job's page shows the value its runs use on the target and where it came from. +instead. Microbatch sub-work counts against the job too. `--concurrency` on the command line turns +the per-job limits off; see [Settings when a build names a job](/docs/concepts/orchestration#settings-when-a-build-names-a-job). The job's page shows the value its runs use on the target and where it came from. ## Runs at once on a server: `max_workers` diff --git a/website/src/content/docs/docs/concepts/orchestration.mdx b/website/src/content/docs/docs/concepts/orchestration.mdx index 96a96f6fd2..6e1022e642 100644 --- a/website/src/content/docs/docs/concepts/orchestration.mdx +++ b/website/src/content/docs/docs/concepts/orchestration.mdx @@ -95,7 +95,11 @@ Naming a group names each of its jobs. The first value set wins: 3. The target's setting 4. The project's setting -`--no-job-settings` leaves out every job's settings. +A command-line flag replaces the job's setting rather than merging with it: `--vars` replaces the +job's whole `vars`. An option that rules another out counts as setting both, so `--reload` wins +over a job's `load_sources = false`. Any `--concurrency` sets the build's limit and turns off the +per-job limits. The job's settings are added before a `--` on the command line, so selectors after +it still name jobs. `--no-job-settings` leaves out every job's settings. Some settings apply to the job's own nodes only, so a build that names two jobs can treat their nodes differently: @@ -113,7 +117,8 @@ gives them to `orders` alone, and `+job:nightly_orders_job` builds the job's ups them. Every other setting, such as `vars` or `changed`, applies to the whole build, so every named job -must set it the same way. A group whose jobs set different `vars` or `changed` fails to compile. +must set it the same way. A group whose jobs set any of these differently on a target that declares +them together fails to compile, with the error a build of the group would give. Two named jobs that give one node different values of the same setting fail the build with `S027`, naming the node, both jobs and the setting. A setting only one of them sets is no conflict. @@ -303,7 +308,7 @@ Every problem with an orchestration file is a `D017` error at its file, line and | `job nightly_orders_job resources: cpus must be a positive number` | Set `cpus` above zero | | `job 'nightly_orders_job' uses pool 'warehouse_hevy', which [pools] in sqlbuild_project.toml does not declare` | Fix the name, or declare the pool in [`[pools]`](/docs/concepts/project-configuration#pools) | | `job 'nightly_orders_job' takes 2 slots of pool 'supplier_api', more than its limit of 1 on target 'production'` | Take fewer slots, or raise the limit in `[targets.production.pools]` | -| `jobs orders_job and crm_job set vars differently; a group runs its jobs as one build, which has one vars` | Give the group's jobs the same `vars` and `changed`, or move one job to another group | +| `jobs orders_job and crm_job set vars differently ({"amount":"5"} and {"amount":"9"}); a build applies vars to everything it runs; group daily_job_group builds its jobs together` | Give the group's jobs the same build-wide settings, or move one job to another group | | `sensor 'orders_sensor' is declared here, but importing the module does not define it` | Assign the sensor to a module-level name or list, not a dictionary or function body | | `group dependencies contain a cycle in job group daily_orders_job_group: orders_marts_job waits for staging_orders_job (model:orders reads model:stg_orders); staging_orders_job waits for orders_marts_job (model:stg_orders reads seed:regions)` | Move each node to one job, and give seeds and nodes that several members read their own job that runs first | From 25c821f039bfc8982da6ad5db8fc19f3f6d3005c Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 18 Nov 2026 19:04:27 +0000 Subject: [PATCH 07/22] feat(jobs): let named jobs raise the build's limit and run server jobs by name (CHI-1533) When a build names jobs and the command line has no --concurrency, its limit is the larger of the configured limit and the named jobs' concurrency added together; each job's own limit still applies inside it, and an explicit --concurrency still wins and turns the per-job limits off. A server job run now builds --select job: over its frozen nodes and settings instead of turning the job's concurrency into --concurrency, so the build applies the job's concurrency and node-level settings as a command-line build does; only build-wide settings still travel as flags. A named job's nodes are found the way the build selects them, Python nodes included, so jobs that select Python paths now take their own settings instead of failing to resolve. --- .../src/job_settings/_helpers/mod.rs | 1 - .../src/job_settings/_helpers/selection.rs | 19 +--- .../job_settings/main/build_job_settings.rs | 2 +- .../src/job_settings/main/mod.rs | 1 + .../job_settings/main/node_job_settings.rs | 6 +- .../scope.rs => main/setting_scope.rs} | 2 +- .../src/job_settings/models.rs | 6 +- .../src/job_settings/tests/helpers.rs | 17 +++- .../src/argv/main/job_run_arguments.rs | 24 +++++ crates/sqlbuild-cli/src/argv/main/mod.rs | 1 + crates/sqlbuild-cli/src/argv/tests.rs | 2 + .../src/argv/tests/test_job_run_arguments.rs | 43 +++++++++ .../sqlbuild-cli/src/argv/tests/test_types.rs | 7 ++ .../sqlbuild-config/src/effective/models.rs | 3 + .../main/published_concurrency.rs | 2 +- .../compiled_project/_helpers/job_catalog.rs | 44 ++++++++- .../src/compiled_project/main/mod.rs | 1 + .../compiled_project/main/with_frozen_job.rs | 16 ++++ .../_helpers/python_nodes/python_selection.rs | 93 ++++++++++++------- .../src/graph/models.rs | 10 ++ .../_helpers/build_command/execution.rs | 46 +++++++-- .../compile_driver/job_node_settings.rs | 31 ++++++- .../_helpers/compile_driver/plan_result.rs | 3 +- .../_helpers/location_command/worker.rs | 49 +++++++--- .../sqlbuild/references/docs/cli/server.md | 4 + .../references/docs/concepts/concurrency.md | 10 +- .../references/docs/concepts/orchestration.md | 2 +- .../docs/concepts/project-configuration.md | 5 +- .../cli/commands/main/test_job_settings.py | 39 ++++++++ website/src/content/docs/docs/cli/server.mdx | 4 + .../docs/docs/concepts/concurrency.mdx | 10 +- .../docs/docs/concepts/orchestration.mdx | 2 +- .../docs/concepts/project-configuration.mdx | 5 +- 33 files changed, 412 insertions(+), 98 deletions(-) rename crates/sqlbuild-analysis/src/job_settings/{_helpers/scope.rs => main/setting_scope.rs} (87%) create mode 100644 crates/sqlbuild-cli/src/argv/main/job_run_arguments.rs create mode 100644 crates/sqlbuild-cli/src/argv/tests/test_job_run_arguments.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_project/main/with_frozen_job.rs diff --git a/crates/sqlbuild-analysis/src/job_settings/_helpers/mod.rs b/crates/sqlbuild-analysis/src/job_settings/_helpers/mod.rs index 25aecee76d..4804846644 100644 --- a/crates/sqlbuild-analysis/src/job_settings/_helpers/mod.rs +++ b/crates/sqlbuild-analysis/src/job_settings/_helpers/mod.rs @@ -1,4 +1,3 @@ pub(crate) mod node_values; -pub(crate) mod scope; pub(crate) mod selection; pub(crate) mod tokens; diff --git a/crates/sqlbuild-analysis/src/job_settings/_helpers/selection.rs b/crates/sqlbuild-analysis/src/job_settings/_helpers/selection.rs index 772f72cb4f..4ba81f1473 100644 --- a/crates/sqlbuild-analysis/src/job_settings/_helpers/selection.rs +++ b/crates/sqlbuild-analysis/src/job_settings/_helpers/selection.rs @@ -4,21 +4,17 @@ use std::collections::{BTreeMap, BTreeSet}; use crate::assembly::project::types::ObjectKey; use crate::graph::errors::SelectorError; -use crate::graph::main::resolve_selector_tokens::resolve_selector_tokens; use crate::graph::models::ProjectGraph; use crate::job_settings::_helpers::tokens::token_jobs; use crate::job_settings::errors::JobSelectorError; -use crate::job_settings::models::DeclaredRun; +use crate::job_settings::models::{DeclaredRun, TokenNodes}; -fn keys(graph: &ProjectGraph, selectors: &[String]) -> Result, SelectorError> { - resolve_selector_tokens(graph, selectors).map(|keys| keys.into_iter().collect()) -} - -/// Each node a `--select` token selects through a job it names whose own selection holds it. +/// Each node a `--select` token, less `--exclude`, selects through a job it names that holds it. pub(crate) fn node_jobs( graph: &ProjectGraph, (select, exclude): (&[String], &[String]), declared: &BTreeMap, + token_nodes: TokenNodes<'_>, ) -> Result>, JobSelectorError> { let failed = |jobs: &[String], selector: &str| { let (jobs, selector) = (jobs.to_vec(), selector.to_owned()); @@ -29,8 +25,6 @@ pub(crate) fn node_jobs( } }; let mut node_jobs: BTreeMap> = BTreeMap::new(); - let excluded: BTreeSet = - keys(graph, exclude).map_err(failed(&[], &exclude.join(" ")))?; let resolved = graph.jobs.as_ref(); for token in select .iter() @@ -41,7 +35,7 @@ pub(crate) fn node_jobs( continue; } let selected: BTreeSet = - keys(graph, &[token.to_owned()]).map_err(failed(&jobs, token))?; + token_nodes(token, exclude).map_err(failed(&jobs, token))?; for job in &jobs { let own: &BTreeSet = match resolved.and_then(|resolved| resolved.get(job)) { Some(Ok(own)) => own, @@ -50,10 +44,7 @@ pub(crate) fn node_jobs( } None => continue, }; - for key in selected - .intersection(own) - .filter(|key| !excluded.contains(*key)) - { + for key in selected.intersection(own) { let jobs: &mut Vec = node_jobs.entry(key.clone()).or_default(); if !jobs.contains(job) { jobs.push(job.clone()); diff --git a/crates/sqlbuild-analysis/src/job_settings/main/build_job_settings.rs b/crates/sqlbuild-analysis/src/job_settings/main/build_job_settings.rs index a29f8c9953..33c9b3d151 100644 --- a/crates/sqlbuild-analysis/src/job_settings/main/build_job_settings.rs +++ b/crates/sqlbuild-analysis/src/job_settings/main/build_job_settings.rs @@ -4,9 +4,9 @@ use std::collections::BTreeMap; use serde_json::Value as Json; -use crate::job_settings::_helpers::scope::setting_scope; use crate::job_settings::constants::CONFLICT_HELP; use crate::job_settings::errors::JobSettingConflict; +use crate::job_settings::main::setting_scope::setting_scope; use crate::job_settings::models::{DeclaredRun, SettingScope}; fn settings_of<'a>( diff --git a/crates/sqlbuild-analysis/src/job_settings/main/mod.rs b/crates/sqlbuild-analysis/src/job_settings/main/mod.rs index 0f917562f8..fa8c731841 100644 --- a/crates/sqlbuild-analysis/src/job_settings/main/mod.rs +++ b/crates/sqlbuild-analysis/src/job_settings/main/mod.rs @@ -3,3 +3,4 @@ pub mod declared_runs; pub mod group_setting_mismatch; pub mod named_jobs; pub mod node_job_settings; +pub mod setting_scope; diff --git a/crates/sqlbuild-analysis/src/job_settings/main/node_job_settings.rs b/crates/sqlbuild-analysis/src/job_settings/main/node_job_settings.rs index 6b96394673..b22ab52b70 100644 --- a/crates/sqlbuild-analysis/src/job_settings/main/node_job_settings.rs +++ b/crates/sqlbuild-analysis/src/job_settings/main/node_job_settings.rs @@ -9,16 +9,18 @@ use crate::job_settings::_helpers::node_values::node_values; use crate::job_settings::_helpers::selection::node_jobs; use crate::job_settings::constants::JOB_LIMIT_SETTING; use crate::job_settings::errors::NodeJobSettingsError; -use crate::job_settings::models::{DeclaredRun, NodeJobSettings}; +use crate::job_settings::models::{DeclaredRun, NodeJobSettings, TokenNodes}; /// Each node's named jobs, the settings they give it, and their limits. pub fn node_job_settings( graph: &ProjectGraph, selection: (&[String], &[String]), declared: &BTreeMap, + token_nodes: TokenNodes<'_>, ) -> Result { let mut result = NodeJobSettings { - node_jobs: node_jobs(graph, selection, declared).map_err(NodeJobSettingsError::Selector)?, + node_jobs: node_jobs(graph, selection, declared, token_nodes) + .map_err(NodeJobSettingsError::Selector)?, ..NodeJobSettings::default() }; for (key, jobs) in &result.node_jobs { diff --git a/crates/sqlbuild-analysis/src/job_settings/_helpers/scope.rs b/crates/sqlbuild-analysis/src/job_settings/main/setting_scope.rs similarity index 87% rename from crates/sqlbuild-analysis/src/job_settings/_helpers/scope.rs rename to crates/sqlbuild-analysis/src/job_settings/main/setting_scope.rs index 4fd31e197b..80bda3ed92 100644 --- a/crates/sqlbuild-analysis/src/job_settings/_helpers/scope.rs +++ b/crates/sqlbuild-analysis/src/job_settings/main/setting_scope.rs @@ -5,7 +5,7 @@ use crate::job_settings::models::SettingScope; /// A node setting, the job limit, or else a build-wide setting. #[must_use] -pub(crate) fn setting_scope(name: &str) -> SettingScope { +pub fn setting_scope(name: &str) -> SettingScope { if NODE_SETTINGS.contains(&name) { SettingScope::Node } else if name == JOB_LIMIT_SETTING { diff --git a/crates/sqlbuild-analysis/src/job_settings/models.rs b/crates/sqlbuild-analysis/src/job_settings/models.rs index 1642d3c606..da1b01dbbc 100644 --- a/crates/sqlbuild-analysis/src/job_settings/models.rs +++ b/crates/sqlbuild-analysis/src/job_settings/models.rs @@ -1,10 +1,11 @@ //! Declared jobs and groups, and the settings a selection gives its nodes. -use std::collections::BTreeMap; +use std::collections::{BTreeMap, BTreeSet}; use serde_json::Value as Json; use crate::assembly::project::types::ObjectKey; +use crate::graph::errors::SelectorError; /// A job's `[retry]`: at most `attempts` runs in all, each rerun `delay_seconds` after the last. #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -36,6 +37,9 @@ pub enum SettingScope { Build, } +/// The nodes one selector token selects, less those `--exclude` removes, as the build resolves them. +pub type TokenNodes<'r> = &'r dyn Fn(&str, &[String]) -> Result, SelectorError>; + /// What the jobs a selection names give each of its nodes. #[derive(Clone, Debug, Default, PartialEq)] pub struct NodeJobSettings { diff --git a/crates/sqlbuild-analysis/src/job_settings/tests/helpers.rs b/crates/sqlbuild-analysis/src/job_settings/tests/helpers.rs index a1b0480457..c7b65cda04 100644 --- a/crates/sqlbuild-analysis/src/job_settings/tests/helpers.rs +++ b/crates/sqlbuild-analysis/src/job_settings/tests/helpers.rs @@ -1,10 +1,11 @@ -use std::collections::BTreeMap; +use std::collections::{BTreeMap, BTreeSet}; use serde_json::{Value as Json, json}; use crate::assembly::project::types::ObjectKey; use crate::graph::main::attach_jobs::attach_jobs; use crate::graph::main::build_project_graph::build_project_graph; +use crate::graph::main::resolve_selector_tokens::resolve_selector_tokens; use crate::graph::models::{GraphResource, JobSelection, ProjectGraph}; use crate::job_settings::errors::NodeJobSettingsError; use crate::job_settings::main::node_job_settings::node_job_settings; @@ -142,9 +143,21 @@ pub(super) fn resolved_settings( select: &[&str], exclude: &[&str], ) -> Result { + let graph: ProjectGraph = graph_with_jobs(); + let token_nodes = |token: &str, exclude: &[String]| { + let excluded: BTreeSet = resolve_selector_tokens(&graph, exclude)? + .into_iter() + .collect(); + let selected = resolve_selector_tokens(&graph, &[token.to_owned()])?; + Ok(selected + .into_iter() + .filter(|key| !excluded.contains(key)) + .collect()) + }; node_job_settings( - &graph_with_jobs(), + &graph, (&texts(select), &texts(exclude)), &declared(), + &token_nodes, ) } diff --git a/crates/sqlbuild-cli/src/argv/main/job_run_arguments.rs b/crates/sqlbuild-cli/src/argv/main/job_run_arguments.rs new file mode 100644 index 0000000000..6a0ea07bbf --- /dev/null +++ b/crates/sqlbuild-cli/src/argv/main/job_run_arguments.rs @@ -0,0 +1,24 @@ +//! The build arguments a server job run passes, leaving the job's own settings to the build. + +use std::collections::BTreeMap; + +use serde_json::Value as Json; +use sqlbuild_analysis::job_settings::main::setting_scope::setting_scope; +use sqlbuild_analysis::job_settings::models::SettingScope; + +use crate::argv::main::job_settings_arguments::job_settings_arguments; + +/// `--select job:` and the job's build-wide settings as flags; the build applies the rest. +pub fn job_run_arguments( + job: &str, + settings: &BTreeMap, +) -> Result, String> { + let build_wide: BTreeMap = settings + .iter() + .filter(|(name, _)| setting_scope(name) == SettingScope::Build) + .map(|(name, value)| (name.clone(), value.clone())) + .collect(); + let mut arguments: Vec = job_settings_arguments(&build_wide)?; + arguments.extend(["--select".to_owned(), format!("job:{job}")]); + Ok(arguments) +} diff --git a/crates/sqlbuild-cli/src/argv/main/mod.rs b/crates/sqlbuild-cli/src/argv/main/mod.rs index 2db8594625..62fed59be5 100644 --- a/crates/sqlbuild-cli/src/argv/main/mod.rs +++ b/crates/sqlbuild-cli/src/argv/main/mod.rs @@ -2,6 +2,7 @@ pub mod accepted_settings_arguments; pub mod cli_namespace_defaults; pub mod command_settings_arguments; pub mod given_setting_dests; +pub mod job_run_arguments; pub mod job_settings_arguments; pub mod parse_cli_arguments; pub mod parse_dbt_execution_arguments; diff --git a/crates/sqlbuild-cli/src/argv/tests.rs b/crates/sqlbuild-cli/src/argv/tests.rs index ce72058936..c053fa0582 100644 --- a/crates/sqlbuild-cli/src/argv/tests.rs +++ b/crates/sqlbuild-cli/src/argv/tests.rs @@ -2,6 +2,8 @@ mod given_setting_dests; #[path = "tests/helpers.rs"] mod helpers; +#[path = "tests/test_job_run_arguments.rs"] +mod job_run_arguments; #[path = "tests/test_parse_cli_arguments.rs"] mod parse_cli_arguments; #[path = "tests/test_parse_dbt_execution_arguments.rs"] diff --git a/crates/sqlbuild-cli/src/argv/tests/test_job_run_arguments.rs b/crates/sqlbuild-cli/src/argv/tests/test_job_run_arguments.rs new file mode 100644 index 0000000000..6e11364310 --- /dev/null +++ b/crates/sqlbuild-cli/src/argv/tests/test_job_run_arguments.rs @@ -0,0 +1,43 @@ +use std::collections::BTreeMap; + +use serde_json::{Value as Json, json}; + +use crate::argv::main::job_run_arguments::job_run_arguments; +use crate::argv::tests::test_types::JobRunArgumentsTestCase; + +#[test] +fn given_a_server_job_run_when_building_its_arguments_then_it_names_the_job_and_leaves_its_own_settings() + { + let test_cases = [ + JobRunArgumentsTestCase { + description: "concurrency and node-level settings are left to the build", + settings: json!({"concurrency": 8, "run_tests": false, "full_refresh": true}), + expected_arguments: &["--select", "job:orders_job"], + }, + JobRunArgumentsTestCase { + description: "build-wide settings still travel as flags", + settings: json!({"concurrency": 4, "vars": {"region": "eu"}, "fail_fast": true}), + expected_arguments: &[ + "--fail-fast", + "--vars={\"region\":\"eu\"}", + "--select", + "job:orders_job", + ], + }, + ]; + for test_case in test_cases { + let settings: BTreeMap = test_case + .settings + .as_object() + .map(|object| object.clone().into_iter().collect()) + .unwrap_or_default(); + let arguments: Vec = + job_run_arguments("orders_job", &settings).expect(test_case.description); + let arguments: Vec<&str> = arguments.iter().map(String::as_str).collect(); + assert_eq!( + arguments, test_case.expected_arguments, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-cli/src/argv/tests/test_types.rs b/crates/sqlbuild-cli/src/argv/tests/test_types.rs index 68b9999e77..e152b76827 100644 --- a/crates/sqlbuild-cli/src/argv/tests/test_types.rs +++ b/crates/sqlbuild-cli/src/argv/tests/test_types.rs @@ -88,3 +88,10 @@ pub(super) struct GivenSettingsTestCase { pub(super) argv: &'static [&'static str], pub(super) expected_given: Option<&'static [&'static str]>, } + +/// A server job run's frozen settings and the build arguments it passes. +pub(super) struct JobRunArgumentsTestCase { + pub(super) description: &'static str, + pub(super) settings: serde_json::Value, + pub(super) expected_arguments: &'static [&'static str], +} diff --git a/crates/sqlbuild-config/src/effective/models.rs b/crates/sqlbuild-config/src/effective/models.rs index cb01766a63..b1b11cd89c 100644 --- a/crates/sqlbuild-config/src/effective/models.rs +++ b/crates/sqlbuild-config/src/effective/models.rs @@ -112,6 +112,8 @@ pub enum ConcurrencySource { Target(String), LocalSettings, ProjectSettings, + /// The named jobs' `concurrency` limits added together, when larger than the configured limit. + NamedJobs, } impl std::fmt::Display for ConcurrencySource { @@ -121,6 +123,7 @@ impl std::fmt::Display for ConcurrencySource { Self::Target(name) => write!(formatter, "targets.{name}.concurrency"), Self::LocalSettings => formatter.write_str("sqlbuild_local.toml settings.concurrency"), Self::ProjectSettings => formatter.write_str("settings.concurrency"), + Self::NamedJobs => formatter.write_str("named jobs' concurrency"), } } } diff --git a/crates/sqlbuild-python-front/src/command_runtime/main/published_concurrency.rs b/crates/sqlbuild-python-front/src/command_runtime/main/published_concurrency.rs index 391ab27b93..a9a9bff66f 100644 --- a/crates/sqlbuild-python-front/src/command_runtime/main/published_concurrency.rs +++ b/crates/sqlbuild-python-front/src/command_runtime/main/published_concurrency.rs @@ -32,7 +32,7 @@ pub fn published_concurrency( Ok(keys.iter().any(|key| key == CONCURRENCY)) }; let source = match effective.source { - Layer::CommandLine => ConcurrencySource::Job, + Layer::CommandLine | Layer::NamedJobs => ConcurrencySource::Job, Layer::Target(_) => ConcurrencySource::Target, Layer::LocalSettings => ConcurrencySource::Local, Layer::ProjectSettings if project_sets()? => ConcurrencySource::Project, diff --git a/crates/sqlbuild-python-project/src/compiled_project/_helpers/job_catalog.rs b/crates/sqlbuild-python-project/src/compiled_project/_helpers/job_catalog.rs index 31f8a54227..419aa3cb17 100644 --- a/crates/sqlbuild-python-project/src/compiled_project/_helpers/job_catalog.rs +++ b/crates/sqlbuild-python-project/src/compiled_project/_helpers/job_catalog.rs @@ -7,6 +7,7 @@ use std::path::PathBuf; use pyo3::IntoPyObject; use pyo3::exceptions::PyValueError; use pyo3::prelude::{PyResult, Python}; +use serde_json::Value; use sqlbuild_analysis::graph::main::parse_selector::parse_selector; use sqlbuild_analysis::graph::models::{JobSelection, ParsedSelector}; use sqlbuild_analysis::job_settings::main::declared_runs::declared_runs; @@ -39,15 +40,48 @@ pub(crate) fn with_job_scope( (project_dir, selected_target): (PathBuf, Option), run: impl FnOnce() -> T, ) -> T { - let previous = JOB_SCOPE.with(|scope| { - scope.replace(Some(JobScope { + within_job_scope( + JobScope { project_dir, selected_target, catalog: None, - })) - }); + }, + run, + ) +} + +/// Run one command with only `job` declared, as its frozen run pins it: its nodes and settings. +pub(crate) fn with_frozen_job( + (project_dir, selected_target): (PathBuf, Option), + (job, select, settings): (&str, Vec, BTreeMap), + run: impl FnOnce() -> T, +) -> T { + let selection = JobSelection::Job { + select, + exclude: Vec::new(), + }; + let declared = DeclaredRun::Job { + settings, + retry: None, + }; + let catalog: Catalog = ( + BTreeMap::from([(job.to_owned(), selection)]), + BTreeSet::new(), + ); + within_job_scope( + JobScope { + project_dir, + selected_target, + catalog: Some((catalog, BTreeMap::from([(job.to_owned(), declared)]))), + }, + run, + ) +} + +fn within_job_scope(scope: JobScope, run: impl FnOnce() -> T) -> T { + let previous = JOB_SCOPE.with(|current| current.replace(Some(scope))); let result: T = run(); - JOB_SCOPE.with(|scope| *scope.borrow_mut() = previous); + JOB_SCOPE.with(|current| *current.borrow_mut() = previous); result } diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/mod.rs b/crates/sqlbuild-python-project/src/compiled_project/main/mod.rs index 8787b16bcd..2dd8eae3e2 100644 --- a/crates/sqlbuild-python-project/src/compiled_project/main/mod.rs +++ b/crates/sqlbuild-python-project/src/compiled_project/main/mod.rs @@ -17,4 +17,5 @@ pub mod python_column_facts; pub mod reference_row_tuple; pub mod register_project; pub mod triggered_jobs; +pub mod with_frozen_job; pub mod with_job_scope; diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/with_frozen_job.rs b/crates/sqlbuild-python-project/src/compiled_project/main/with_frozen_job.rs new file mode 100644 index 0000000000..126554ac7e --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_project/main/with_frozen_job.rs @@ -0,0 +1,16 @@ +//! Run one command with only one job declared, as its frozen run pins it. + +use crate::compiled_project::_helpers::job_catalog; +use serde_json::Value; +use std::collections::BTreeMap; +use std::path::PathBuf; + +/// Run one command with only `job` declared, as its frozen run pins it: its nodes and settings. +#[inline] +pub fn with_frozen_job( + directory: (PathBuf, Option), + frozen: (&str, Vec, BTreeMap), + run: impl FnOnce() -> T, +) -> T { + job_catalog::with_frozen_job(directory, frozen, run) +} diff --git a/crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/python_selection.rs b/crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/python_selection.rs index 6a23dbe2a3..ad201176f4 100644 --- a/crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/python_selection.rs +++ b/crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/python_selection.rs @@ -20,7 +20,7 @@ use sqlbuild_analysis::python_selection::models::{ }; use crate::compiled_project::main::needed_jobs::needed_jobs; -use crate::graph::models::{CompiledProjectGraph, NativeProjectGraph}; +use crate::graph::models::{CompiledProjectGraph, NativeProjectGraph, PythonAwareSelection}; use sqlbuild_python_base::boundary::main::compiler_guard::compiler_guard; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; @@ -153,6 +153,60 @@ pub(crate) fn resolve_python_node_selection<'py>( python_node_names(graph, (select, exclude)) } +impl<'g> PythonAwareSelection<'g> { + /// The graph of `native` with the jobs `selectors` need attached over `python_graph`. + pub fn new( + (project_graph, native): (&Bound<'_, CompiledProjectGraph>, &'g ProjectGraph), + python_graph: &Bound<'_, PyAny>, + selectors: &[String], + ) -> PyResult { + let py: Python<'_> = project_graph.py(); + let sources: Vec<(String, Option)> = + project_sources(project_graph.get().project(py))?; + let allow_warehouse_round_trips: bool = + allows_round_trips(project_graph.get().project(py))?; + let nodes: PythonSelectionGraph = selection_graph(python_graph)?; + let node_names: Vec<&str> = nodes.nodes.iter().map(|node| node.name.as_str()).collect(); + let graph: Cow<'g, ProjectGraph> = match needed_jobs(py, native, selectors, &node_names)? { + Some(catalog) => Cow::Owned(attach_python_jobs( + SqlSelectionGraph { + graph: native, + sources: &sources, + allow_warehouse_round_trips, + }, + &nodes, + catalog, + )), + None => Cow::Borrowed(native), + }; + Ok(Self { + graph, + sources, + allow_warehouse_round_trips, + nodes, + }) + } + + /// The graph with the needed jobs attached. + pub fn graph(&self) -> &ProjectGraph { + &self.graph + } + + /// The SQL keys and Python node names `select` minus `exclude` resolve to. + pub fn resolve( + &self, + select: &[String], + exclude: &[String], + ) -> Result { + let sql = SqlSelectionGraph { + graph: &self.graph, + sources: &self.sources, + allow_warehouse_round_trips: self.allow_warehouse_round_trips, + }; + run_selection(sql, &self.nodes, select, exclude) + } +} + /// `(sql keys, python node names)` run selectors resolve to across SQL resources and Python nodes. #[pyfunction] #[pyo3(signature = (*, project_graph, python_graph, select, exclude))] @@ -165,37 +219,14 @@ pub(crate) fn resolve_python_sql_run_selection( let py: Python<'_> = project_graph.py(); let project_graph: &Bound<'_, CompiledProjectGraph> = project_graph.cast()?; let native: &Bound<'_, NativeProjectGraph> = project_graph.get().native(py); - let sources: Vec<(String, Option)> = project_sources(project_graph.get().project(py))?; - let allow_warehouse_round_trips: bool = allows_round_trips(project_graph.get().project(py))?; - let nodes: PythonSelectionGraph = selection_graph(python_graph)?; - let node_names: Vec<&str> = nodes.nodes.iter().map(|node| node.name.as_str()).collect(); let selectors: Vec = select.iter().chain(&exclude).cloned().collect(); - let graph: Cow<'_, ProjectGraph> = - match needed_jobs(py, native.get().graph(), &selectors, &node_names)? { - Some(catalog) => Cow::Owned(attach_python_jobs( - SqlSelectionGraph { - graph: native.get().graph(), - sources: &sources, - allow_warehouse_round_trips, - }, - &nodes, - catalog, - )), - None => Cow::Borrowed(native.get().graph()), - }; - let selection: RunSelection = compiler_guard(|| { - Ok(run_selection( - SqlSelectionGraph { - graph: &graph, - sources: &sources, - allow_warehouse_round_trips, - }, - &nodes, - &select, - &exclude, - )) - })? - .map_err(|error| planner_error(py, error))?; + let resolution = PythonAwareSelection::new( + (project_graph, native.get().graph()), + python_graph, + &selectors, + )?; + let selection: RunSelection = compiler_guard(|| Ok(resolution.resolve(&select, &exclude)))? + .map_err(|error| planner_error(py, error))?; Ok((selection.sql_keys, selection.python_node_names)) } diff --git a/crates/sqlbuild-python-project/src/graph/models.rs b/crates/sqlbuild-python-project/src/graph/models.rs index 76de4d0736..ad5bfeb558 100644 --- a/crates/sqlbuild-python-project/src/graph/models.rs +++ b/crates/sqlbuild-python-project/src/graph/models.rs @@ -52,6 +52,7 @@ use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::discovery::main::records::stored_node_sets::stored_node_sets; +use std::borrow::Cow; use std::cell::RefCell; use std::collections::BTreeMap; use std::collections::BTreeSet; @@ -566,3 +567,12 @@ pub struct GraphViewMappings<'a, 'py> { pub tag_index: &'a Bound<'py, PyAny>, pub path_index: &'a Bound<'py, PyAny>, } + +/// A project graph with the jobs some selectors name attached as a build selects them, Python +/// nodes included, and the project facts selectors resolve with. +pub struct PythonAwareSelection<'g> { + pub(crate) graph: Cow<'g, ProjectGraph>, + pub(crate) sources: Vec<(String, Option)>, + pub(crate) allow_warehouse_round_trips: bool, + pub(crate) nodes: PythonSelectionGraph, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs index 64e6101f3e..9f8e5628d6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs @@ -4,7 +4,7 @@ use pyo3::IntoPyObject; use pyo3::exceptions::PyValueError; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods}; -use sqlbuild_config::effective::models::EffectiveConcurrency; +use sqlbuild_config::effective::models::{ConcurrencySource, EffectiveConcurrency}; use sqlbuild_core::build_command::main::render_run_context::render_run_context; use sqlbuild_core::build_command::models::{RunContextFacts, SelectorFileFact}; use sqlbuild_core::cli_text::models::CliStyle; @@ -203,7 +203,7 @@ pub(crate) fn write_run_context( context: &BuildContext<'_>, pipeline: &Bound<'_, PyAny>, ) -> PyResult<()> { - let concurrency: EffectiveConcurrency = context.concurrency()?; + let concurrency: EffectiveConcurrency = build_concurrency(context.concurrency()?, pipeline)?; let facts: RunContextFacts = run_context_facts(context, pipeline, &concurrency)?; emit( &context.stream, @@ -268,7 +268,10 @@ pub(crate) fn prepare_build_execution<'py>( }) .and_then(|value| pyo3::Bound::new(py, value)) .map(pyo3::Bound::into_any)?; - let concurrency = connection_limit(context.concurrency()?.limit, pipeline)?; + let concurrency = connection_limit( + build_concurrency(context.concurrency()?, pipeline)?.limit, + pipeline, + )?; if context.verbose { write_run_context(context, pipeline)?; } else { @@ -323,12 +326,39 @@ pub(crate) fn prepare_build_execution<'py>( }) } -/// The build's limit, capped at the sum of its jobs' limits when every planned node has one. -fn connection_limit(limit: i64, pipeline: &Bound<'_, PyAny>) -> PyResult { +/// The named jobs' `concurrency` limits added together; none without a command-line override. +fn job_limit_total(pipeline: &Bound<'_, PyAny>) -> PyResult> { let limits: Bound<'_, PyDict> = native_field(pipeline, "job_limits")?.cast_into()?; if limits.is_empty() { - return Ok(limit); + return Ok(None); } + let mut total: i64 = 0; + for value in limits.values() { + total = total.saturating_add(value.extract::()?); + } + Ok(Some(total)) +} + +/// The build's limit: the configured one, raised to the named jobs' limits added together. +fn build_concurrency( + configured: EffectiveConcurrency, + pipeline: &Bound<'_, PyAny>, +) -> PyResult { + Ok(match job_limit_total(pipeline)? { + Some(total) if total > configured.limit => EffectiveConcurrency { + limit: total, + source: ConcurrencySource::NamedJobs, + }, + _ => configured, + }) +} + +/// The build's connections: its limit, or the named jobs' limits added together when every planned +/// node belongs to one of them, so a job never opens connections it cannot use. +fn connection_limit(limit: i64, pipeline: &Bound<'_, PyAny>) -> PyResult { + let Some(total) = job_limit_total(pipeline)? else { + return Ok(limit); + }; let node_jobs: Bound<'_, PyDict> = native_field(pipeline, "node_jobs")?.cast_into()?; for key in native_field(&native_field(pipeline, "plan_output")?, "execution_order")?.try_iter()? @@ -337,10 +367,6 @@ fn connection_limit(limit: i64, pipeline: &Bound<'_, PyAny>) -> PyResult { return Ok(limit); } } - let mut total: i64 = 0; - for value in limits.values() { - total = total.saturating_add(value.extract::()?); - } Ok(limit.min(total.max(1))) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/job_node_settings.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/job_node_settings.rs index 7a64d3b42c..7c8661e1da 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/job_node_settings.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/job_node_settings.rs @@ -17,16 +17,20 @@ use sqlbuild_analysis::job_settings::models::{DeclaredRun, NodeJobSettings}; use crate::bindings::_helpers::compiled_project::job_settings::{ job_selector_error, job_setting_conflict, }; +use sqlbuild_python_base::boundary::main::compiler_guard::compiler_guard; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::replaced::replaced; +use sqlbuild_python_compile::compile_driver::main::setup::project_graph::project_graph; use sqlbuild_python_project::compiled_project::constants::{ JOB_FULL_REFRESH_KEY, JOB_IGNORED_AUDIT_FAILURES_KEY, }; -use sqlbuild_python_project::compiled_project::main::graph_with_jobs::graph_with_jobs; use sqlbuild_python_project::compiled_project::main::job_declarations::job_declarations; use sqlbuild_python_project::graph::main::nodes::native_key::native_key; +use sqlbuild_python_project::graph::main::python_nodes::python_node_graph::python_node_graph; use sqlbuild_python_project::graph::models::KeyCache; -use sqlbuild_python_project::graph::models::NativeProjectGraph; +use sqlbuild_python_project::graph::models::{ + CompiledProjectGraph, NativeProjectGraph, PythonAwareSelection, +}; const MODEL_KIND: &str = "model"; const FULL_REFRESH: &str = "full_refresh"; @@ -70,7 +74,7 @@ fn strings(value: &Bound<'_, PyAny>) -> PyResult> { /// The named jobs' settings per selected node; `None` when ignored, unnamed or unselectable. pub(crate) fn planned_job_settings( - project: &Bound<'_, PyAny>, + (project, discovered): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), options: &Bound<'_, PyAny>, ) -> PyResult> { let py: Python<'_> = project.py(); @@ -95,8 +99,25 @@ pub(crate) fn planned_job_settings( build_job_settings(&named, &declared) .map_err(|conflict| job_setting_conflict(py, &conflict))?; let all: Vec = select.iter().chain(&exclude).cloned().collect(); - let graph = graph_with_jobs(py, native.graph(), &all, &[])?; - match node_job_settings(&graph, (&select, &exclude), &declared) { + let compiled: Bound<'_, CompiledProjectGraph> = project_graph(project)?.cast_into()?; + let selection = PythonAwareSelection::new( + (&compiled, native.graph()), + &python_node_graph(discovered)?, + &all, + )?; + let token_nodes = |token: &str, exclude: &[String]| { + let resolved = selection.resolve(&[token.to_owned()], exclude)?; + Ok(resolved.sql_keys.into_iter().collect()) + }; + let resolved = compiler_guard(|| { + Ok(node_job_settings( + selection.graph(), + (&select, &exclude), + &declared, + &token_nodes, + )) + })?; + match resolved { Ok(settings) => Ok(Some(PlannedJobSettings { ignoring_jobs: jobs_setting(&settings, &declared, (IGNORE_AUDIT_FAILURES, None)), refreshing_jobs: jobs_setting(&settings, &declared, (FULL_REFRESH, Some(MODEL_KIND))), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs index 49613a5b8a..303d3b1dac 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs @@ -158,7 +158,8 @@ pub(crate) fn pipeline_result<'py>( ("sql_test_selection", sql_test_selection), ], )?; - let job_settings: Option = planned_job_settings(project, options)?; + let job_settings: Option = + planned_job_settings((project, discovered), options)?; let mut override_fields: Vec<(&str, Bound<'py, PyAny>)> = option_fields( options, &[ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs index 5397a0b567..d829d5e3a5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs @@ -1,6 +1,7 @@ //! Execute only the API's frozen request after compiling and verifying the unmodified baseline. use std::collections::BTreeMap; +use std::path::PathBuf; use std::sync::{Arc, mpsc}; use std::time::Duration; @@ -10,6 +11,7 @@ use pyo3::types::PyString; use serde_json::Value; use sqlbuild_analysis::definition_changes::constants::PYTHON_NODE_KINDS; use sqlbuild_cli::argv::main::command_settings_arguments::command_settings_arguments; +use sqlbuild_cli::argv::main::job_run_arguments::job_run_arguments; use sqlbuild_cli::argv::main::job_settings_arguments::job_settings_arguments; use sqlbuild_cli::dispatch::main::command_dispatch::command_dispatch; use sqlbuild_cli::dispatch::models::Dispatch; @@ -49,6 +51,7 @@ use sqlbuild_python_front::compile_attachment::main::build_effective_connection_ use sqlbuild_python_output::recall::main::write_stderr::write_stderr; use sqlbuild_python_output::recall::models::InvocationRecall; use sqlbuild_python_project::compiled_project::constants::SQL_TEST_KEY_PREFIX; +use sqlbuild_python_project::compiled_project::main::with_frozen_job::with_frozen_job; use sqlbuild_python_project::compiled_views::main::project::sql_tests::sql_tests; use sqlbuild_python_project::project_snapshot::models::NativeOrchestrationSnapshot; use sqlbuild_python_project::project_snapshot::models::NativeOrchestrationWorker; @@ -123,13 +126,16 @@ fn upstream_nodes(snapshot: &NativeOrchestrationSnapshot) -> BTreeMap, BTreeMap); + +/// The frozen command's request, warehouse scope and, for a job's build, the frozen job it names. fn command_request<'py>( py: Python<'py>, discovered: &Bound<'py, PyAny>, arguments: &KeywordArguments<'py>, (worker, compiled): (&Bound<'py, NativeOrchestrationWorker>, &Bound<'py, PyAny>), -) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { +) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>, Option)> { let execution = worker.get().execution(py)?; let frozen = execution.claim.request; let mut settings = frozen.settings; @@ -155,10 +161,14 @@ fn command_request<'py>( if frozen.job_key.is_some() { settings = job_run_settings(&arguments.value("project_dir")?, settings)?; } - let flags: Vec = if command == RunCommand::Build { - job_settings_arguments(&settings) - } else { - command_settings_arguments(command.as_str(), &settings) + let job_build: Option<&String> = frozen + .job_key + .as_ref() + .filter(|_| command == RunCommand::Build); + let flags: Vec = match job_build { + Some(job) => job_run_arguments(job, &settings), + None if command == RunCommand::Build => job_settings_arguments(&settings), + None => command_settings_arguments(command.as_str(), &settings), } .map_err(PyValueError::new_err)?; let mut argv: Vec> = std::iter::once(command.as_str().to_owned()) @@ -173,6 +183,7 @@ fn command_request<'py>( PyString::new(py, "--no-color"), PyString::new(py, "--json"), ]); + let mut selectors: Vec = Vec::new(); for key in &frozen.selection { if execution.skipped_nodes.contains_key(key) { continue; @@ -187,7 +198,12 @@ fn command_request<'py>( } else { name.to_owned() }; - argv.extend([PyString::new(py, "--select"), PyString::new(py, &selector)]); + selectors.push(selector); + } + if job_build.is_none() { + for selector in &selectors { + argv.extend([PyString::new(py, "--select"), PyString::new(py, selector)]); + } } let parsed = parse_cli_invocation(py, argv, false)?; let args = field(&parsed, "args")?; @@ -214,7 +230,8 @@ fn command_request<'py>( ], )?; } - Ok((request, warehouse_scope(&args)?)) + let frozen_job: Option = job_build.map(|job| (job.clone(), selectors, settings)); + Ok((request, warehouse_scope(&args)?, frozen_job)) } /// Run the frozen command from the verified inputs, reporting its progress and outcomes to `worker`. @@ -371,11 +388,21 @@ pub(super) fn execute_worker<'py>( return Err(PyValueError::new_err(reason)); } record_compiled_vars(py, (&owner, &frozen), (arguments, input, &project))?; - let (request, scope) = command_request(py, &discovered, arguments, (&owner, compiled))?; + let (request, scope, frozen_job) = + command_request(py, &discovered, arguments, (&owner, compiled))?; write_stderr(py, "Run inputs verified. Executing frozen selection...\n")?; let _rules = RulesAtLoadScope::enter(); - within(&scope, |_| { - run_cancellable(py, &worker, (&request, &discovered)) + let directory: (PathBuf, Option) = ( + arguments.value("project_dir")?.extract()?, + Some(frozen.target.clone()), + ); + within(&scope, |_| match frozen_job { + Some((job, selectors, settings)) => { + with_frozen_job(directory, (&job, selectors, settings), || { + run_cancellable(py, &worker, (&request, &discovered)) + }) + } + None => run_cancellable(py, &worker, (&request, &discovered)), }) })(); let code = result.as_ref().copied().unwrap_or(1); diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md index 2e46bc706b..7e900811e0 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md @@ -780,6 +780,10 @@ warehouse_heavy = 1 `max_workers`, write claims, `overlap` and pools are compared on [Concurrency](../concepts/concurrency.md). +A job's run builds `--select job:`, as `sqb build --select job:` would on the command +line, so the job's `concurrency`, `full_refresh`, `run_tests` and `ignore_audit_failures` apply to +its nodes the same way. A group runs each of its jobs as its own run. + ## Run ad-hoc commands People with `run.adhoc` can run a `sqb` command on the current publication without declaring a diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/concurrency.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/concurrency.md index 7c3a500318..e7cc29868c 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/concurrency.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/concurrency.md @@ -48,9 +48,13 @@ concurrency = 2 `--concurrency` on the command line wins over all of them; see [Target concurrency](project-configuration.md#target-concurrency) for the full order. A job's `concurrency` limits how many of that job's nodes build at once in a build that names the -job, within the build's own limit. When one job is at its limit, a ready node of another job starts -instead. Microbatch sub-work counts against the job too. `--concurrency` on the command line turns -the per-job limits off; see [Settings when a build names a job](orchestration.md#settings-when-a-build-names-a-job). The job's page shows the value its runs use on the target and where it came from. +job. Without `--concurrency`, the build may run as many nodes at once as the larger of the +configured limit and the named jobs' limits added together; jobs without a `concurrency` add +nothing. With a project limit of 1 and a job limit of 4, the build runs up to 4 nodes at once; with +a project limit of 16, the build allows 16 and the job's nodes still run at most 4 at once. When +one job is at its limit, a ready node of another job starts instead. Microbatch sub-work counts +against the job too. A server runs a job the same way. `--concurrency` on the command line sets +the build's limit and turns the per-job limits off; see [Settings when a build names a job](orchestration.md#settings-when-a-build-names-a-job). The job's page shows the value its runs use on the target and where it came from. ## Runs at once on a server: `max_workers` diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/orchestration.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/orchestration.md index 2b5e3f6baf..56c2fb65b5 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/orchestration.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/orchestration.md @@ -123,7 +123,7 @@ nodes differently: | `full_refresh` | The job's incremental models, rebuilt from scratch | | `run_tests` | The SQL unit tests of the job's models | | `ignore_audit_failures` | The audits of the job's nodes | -| `concurrency` | How many of the job's nodes build at once. The build keeps its own limit too, and a node in two named jobs fits within both | +| `concurrency` | How many of the job's nodes build at once, and a node in two named jobs fits within both. The build's own limit becomes the larger of the configured limit and the named jobs' limits added together | | `[retry]` | The job's nodes | A job's settings reach only the nodes its own selection holds. `--select job:nightly_orders_job,orders` diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md index 872c80e52c..2de6859ef1 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md @@ -275,8 +275,9 @@ concurrency = 8 value set in this order: 1. `--concurrency` on the command line, for that invocation only -2. For `sqb build`, a named job's `concurrency`, which limits that job's nodes within the build's - own limit; see [Settings when a build names a job](orchestration.md#settings-when-a-build-names-a-job) +2. For `sqb build`, the named jobs' `concurrency` added together, when larger than the limit the + layers below give; each job's own limit still applies to its nodes. See + [Settings when a build names a job](orchestration.md#settings-when-a-build-names-a-job) 3. The selected target's `concurrency`, from `[targets.]` in either file 4. `[settings] concurrency` in `sqlbuild_local.toml` 5. `[settings] concurrency` in `sqlbuild_project.toml` (default `1`) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_job_settings.py b/tests/integration/src/sqlbuild/cli/commands/main/test_job_settings.py index c670c89b47..7d21c2e329 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_job_settings.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_job_settings.py @@ -91,6 +91,10 @@ ) for name in ("a", "b", "c", "d") }, + "orchestration/jobs/wide_job.toml": ( + 'name = "wide_job"\ndescription = "Wide."\nselect = "customers"\n\n' + '[settings]\nconcurrency = 6\nvars = { amount = "5" }\n' + ), "orchestration/jobs/slow_job.toml": ( 'name = "slow_job"\ndescription = "Slow."\nselect = "slow_a slow_b"\n\n' "[settings]\nconcurrency = 1\n" @@ -351,6 +355,41 @@ def test_given_a_job_concurrency_when_building_then_its_nodes_never_exceed_it( @pytest.mark.parametrize( "test_case", [ + JobOutputTestCase( + description="a named job's concurrency above the configured limit raises the build's", + command_args=("build", "--select", "job:wide_job"), + expected_exit_code=0, + expected_present=("(concurrency: 6)",), + expected_absent=(), + ), + JobOutputTestCase( + description="the named jobs' limits add up", + command_args=("build", "--select", "job:wide_job", "job:orders_job"), + expected_exit_code=0, + expected_present=("(concurrency: 7)",), + expected_absent=(), + ), + JobOutputTestCase( + description="the run context says the named jobs set the limit", + command_args=("build", "--verbose", "--select", "job:wide_job", "+orders"), + expected_exit_code=0, + expected_present=("6 configured limit (named jobs' concurrency)",), + expected_absent=(), + ), + JobOutputTestCase( + description="a configured limit above the named jobs' stays the build's limit", + command_args=("build", "--verbose", "--select", "job:orders_job", "customers"), + expected_exit_code=0, + expected_present=("4 configured limit (settings.concurrency)",), + expected_absent=(), + ), + JobOutputTestCase( + description="an explicit --concurrency wins over the named jobs", + command_args=("build", "--select", "job:wide_job", "--concurrency", "2"), + expected_exit_code=0, + expected_present=("(concurrency: 2)",), + expected_absent=(), + ), JobOutputTestCase( description="a plan shows the job's full refresh", command_args=("plan", "--select", "job:events_job"), diff --git a/website/src/content/docs/docs/cli/server.mdx b/website/src/content/docs/docs/cli/server.mdx index 12b838eb33..2cc6e0df5c 100644 --- a/website/src/content/docs/docs/cli/server.mdx +++ b/website/src/content/docs/docs/cli/server.mdx @@ -759,6 +759,10 @@ warehouse_heavy = 1 `max_workers`, write claims, `overlap` and pools are compared on [Concurrency](/docs/concepts/concurrency). +A job's run builds `--select job:`, as `sqb build --select job:` would on the command +line, so the job's `concurrency`, `full_refresh`, `run_tests` and `ignore_audit_failures` apply to +its nodes the same way. A group runs each of its jobs as its own run. + ## Run ad-hoc commands People with `run.adhoc` can run a `sqb` command on the current publication without declaring a diff --git a/website/src/content/docs/docs/concepts/concurrency.mdx b/website/src/content/docs/docs/concepts/concurrency.mdx index b948bf1d1a..4eb27e70ef 100644 --- a/website/src/content/docs/docs/concepts/concurrency.mdx +++ b/website/src/content/docs/docs/concepts/concurrency.mdx @@ -37,9 +37,13 @@ concurrency = 2 `--concurrency` on the command line wins over all of them; see [Target concurrency](/docs/concepts/project-configuration#target-concurrency) for the full order. A job's `concurrency` limits how many of that job's nodes build at once in a build that names the -job, within the build's own limit. When one job is at its limit, a ready node of another job starts -instead. Microbatch sub-work counts against the job too. `--concurrency` on the command line turns -the per-job limits off; see [Settings when a build names a job](/docs/concepts/orchestration#settings-when-a-build-names-a-job). The job's page shows the value its runs use on the target and where it came from. +job. Without `--concurrency`, the build may run as many nodes at once as the larger of the +configured limit and the named jobs' limits added together; jobs without a `concurrency` add +nothing. With a project limit of 1 and a job limit of 4, the build runs up to 4 nodes at once; with +a project limit of 16, the build allows 16 and the job's nodes still run at most 4 at once. When +one job is at its limit, a ready node of another job starts instead. Microbatch sub-work counts +against the job too. A server runs a job the same way. `--concurrency` on the command line sets +the build's limit and turns the per-job limits off; see [Settings when a build names a job](/docs/concepts/orchestration#settings-when-a-build-names-a-job). The job's page shows the value its runs use on the target and where it came from. ## Runs at once on a server: `max_workers` diff --git a/website/src/content/docs/docs/concepts/orchestration.mdx b/website/src/content/docs/docs/concepts/orchestration.mdx index 6e1022e642..cf80f99ad7 100644 --- a/website/src/content/docs/docs/concepts/orchestration.mdx +++ b/website/src/content/docs/docs/concepts/orchestration.mdx @@ -109,7 +109,7 @@ nodes differently: | `full_refresh` | The job's incremental models, rebuilt from scratch | | `run_tests` | The SQL unit tests of the job's models | | `ignore_audit_failures` | The audits of the job's nodes | -| `concurrency` | How many of the job's nodes build at once. The build keeps its own limit too, and a node in two named jobs fits within both | +| `concurrency` | How many of the job's nodes build at once, and a node in two named jobs fits within both. The build's own limit becomes the larger of the configured limit and the named jobs' limits added together | | `[retry]` | The job's nodes | A job's settings reach only the nodes its own selection holds. `--select job:nightly_orders_job,orders` diff --git a/website/src/content/docs/docs/concepts/project-configuration.mdx b/website/src/content/docs/docs/concepts/project-configuration.mdx index f563c3b0bf..b6e4e12f66 100644 --- a/website/src/content/docs/docs/concepts/project-configuration.mdx +++ b/website/src/content/docs/docs/concepts/project-configuration.mdx @@ -248,8 +248,9 @@ concurrency = 8 value set in this order: 1. `--concurrency` on the command line, for that invocation only -2. For `sqb build`, a named job's `concurrency`, which limits that job's nodes within the build's - own limit; see [Settings when a build names a job](/docs/concepts/orchestration#settings-when-a-build-names-a-job) +2. For `sqb build`, the named jobs' `concurrency` added together, when larger than the limit the + layers below give; each job's own limit still applies to its nodes. See + [Settings when a build names a job](/docs/concepts/orchestration#settings-when-a-build-names-a-job) 3. The selected target's `concurrency`, from `[targets.]` in either file 4. `[settings] concurrency` in `sqlbuild_local.toml` 5. `[settings] concurrency` in `sqlbuild_project.toml` (default `1`) From 9c948d60f10e5fa0bf366048d469ec2b8d9e06be Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 18 Nov 2026 20:51:17 +0000 Subject: [PATCH 08/22] feat(jobs): run each named job's nodes on the job's warehouse (CHI-1533) warehouse is now a node-level job setting. When a build names jobs, each node runs on the warehouse of the named job that selects it; nodes outside the named jobs stay on the build's warehouse. --warehouse still puts the whole build on one warehouse and leaves the jobs' out, and two named jobs giving one node different warehouses fail with S027. A connection running a node switches with USE WAREHOUSE only when the node needs another warehouse than the one the connection is on, and every switched connection returns to the build's warehouse when the scheduler finishes. Microbatch sub-workers follow their coordinator, source loads and Python nodes follow their own node, and code opening its own connections from connection_config gets the node's warehouse. A SQL test runs on its models' job warehouse when they agree on one. A job warehouse is checked like --warehouse: a valid identifier (C261) on an adapter with a session warehouse (C260), worded for the job. Plan text, plan JSON and the verbose run context name each job's warehouse. Server job runs no longer pass a job's warehouse as --warehouse; the build applies it per node. --- .../src/job_settings/_helpers/node_values.rs | 15 +- .../src/job_settings/constants.rs | 11 +- .../src/job_settings/tests/helpers.rs | 16 ++ .../tests/test_build_job_settings.rs | 6 + .../tests/test_node_job_settings.rs | 76 ++++++++- .../src/job_settings/tests/test_types.rs | 8 + .../src/argv/tests/test_job_run_arguments.rs | 2 +- .../main/check_job_warehouse.rs | 32 ++++ .../src/command_connection/main/mod.rs | 2 + .../main/unsupported_job_warehouse.rs | 23 +++ .../tests/test_command_connection.rs | 43 ++++- .../command_connection/tests/test_types.rs | 6 + .../build_command/main/render_run_context.rs | 9 +- .../sqlbuild-core/src/build_command/models.rs | 2 + .../build_command/tests/test_build_command.rs | 13 ++ .../src/cli_text/_helpers/plan_document.rs | 3 + crates/sqlbuild-core/src/cli_text/models.rs | 2 + .../src/cli_text/tests/helpers.rs | 1 + .../_helpers/compiler_records/constants.rs | 2 + .../src/plan_output/_helpers/plan_text.rs | 17 +- .../src/plan_output/constants.rs | 2 + .../src/plan_output/main/format_plan_json.rs | 8 +- .../src/plan_output/main/job_warehouses.rs | 10 ++ .../src/plan_output/main/mod.rs | 1 + .../compiled_project/_helpers/job_catalog.rs | 10 ++ .../src/compiled_project/constants.rs | 2 + .../main/job_scope_project_dir.rs | 10 ++ .../src/compiled_project/main/mod.rs | 1 + .../_helpers/python_nodes/python_selection.rs | 26 +++ .../_helpers/build_command/execution.rs | 13 ++ .../_helpers/build_command/lifecycle.rs | 4 + .../compile_driver/job_node_settings.rs | 144 ++++++++++++++-- .../_helpers/compile_driver/plan_result.rs | 3 + .../docs/concepts/adapters/snowflake.md | 38 ++++- .../references/docs/concepts/orchestration.md | 3 +- src/sqlbuild/_native.pyi | 4 + .../adapter/contract/classes/base_adapter.py | 10 ++ .../snowflake/classes/snowflake_adapter.py | 16 ++ .../executor/build/classes/build_scheduler.py | 56 +++++- src/sqlbuild/executor/build/models.py | 2 + .../_helpers/ingress_execution.py | 2 + .../python_nodes/_helpers/node_warehouse.py | 27 +++ .../python_nodes/_helpers/python_checks.py | 2 + .../read_side_python_execution_tracker.py | 5 +- src/sqlbuild/executor/python_nodes/models.py | 20 ++- .../scheduling/classes/session_warehouses.py | 120 +++++++++++++ .../sqlbuild/cli/commands/main/_test_types.py | 20 +++ .../src/sqlbuild/cli/commands/main/helpers.py | 85 ++++++++++ .../cli/commands/main/test_job_settings.py | 21 +++ .../cli/commands/main/test_job_warehouses.py | 160 ++++++++++++++++++ .../publication/test_adhoc.py | 8 +- .../executor/build/classes/_test_types.py | 18 ++ .../executor/build/classes/helpers.py | 2 + .../test_build_scheduler_microbatch.py | 2 + .../test_build_scheduler_warehouses.py | 126 ++++++++++++++ .../scheduling/classes/_test_types.py | 14 ++ .../executor/scheduling/classes/helpers.py | 20 +++ .../classes/test_session_warehouses.py | 130 ++++++++++++++ .../docs/docs/concepts/adapters/snowflake.mdx | 38 ++++- .../docs/docs/concepts/orchestration.mdx | 3 +- 60 files changed, 1438 insertions(+), 37 deletions(-) create mode 100644 crates/sqlbuild-config/src/command_connection/main/check_job_warehouse.rs create mode 100644 crates/sqlbuild-config/src/command_connection/main/unsupported_job_warehouse.rs create mode 100644 crates/sqlbuild-python-planner/src/plan_output/main/job_warehouses.rs create mode 100644 crates/sqlbuild-python-project/src/compiled_project/main/job_scope_project_dir.rs create mode 100644 src/sqlbuild/executor/python_nodes/_helpers/node_warehouse.py create mode 100644 src/sqlbuild/executor/scheduling/classes/session_warehouses.py create mode 100644 tests/integration/src/sqlbuild/cli/commands/main/test_job_warehouses.py create mode 100644 tests/unit/src/sqlbuild/executor/build/classes/test_build_scheduler_warehouses.py create mode 100644 tests/unit/src/sqlbuild/executor/scheduling/classes/test_session_warehouses.py diff --git a/crates/sqlbuild-analysis/src/job_settings/_helpers/node_values.rs b/crates/sqlbuild-analysis/src/job_settings/_helpers/node_values.rs index 3d93f3c319..9c566b7153 100644 --- a/crates/sqlbuild-analysis/src/job_settings/_helpers/node_values.rs +++ b/crates/sqlbuild-analysis/src/job_settings/_helpers/node_values.rs @@ -5,7 +5,10 @@ use std::collections::BTreeMap; use serde_json::Value as Json; use crate::assembly::project::types::ObjectKey; -use crate::job_settings::constants::{CONFLICT_HELP, NODE_SETTINGS, RETRY_SETTING}; +use crate::definition_changes::constants::PYTHON_NODE_KINDS; +use crate::job_settings::constants::{ + CONFLICT_HELP, NODE_SETTINGS, PYTHON_NODE_SETTINGS, RETRY_SETTING, +}; use crate::job_settings::errors::JobSettingConflict; use crate::job_settings::models::{DeclaredRun, JobRetry}; @@ -41,6 +44,12 @@ pub(crate) fn node_values( jobs: &[String], declared: &BTreeMap, ) -> Result { + let python: bool = PYTHON_NODE_KINDS.contains(&key.0.as_str()); + let node_settings: &[&str] = if python { + &PYTHON_NODE_SETTINGS + } else { + &NODE_SETTINGS + }; let mut values: BTreeMap = BTreeMap::new(); let mut retry: Option<(&String, JobRetry)> = None; for job in jobs { @@ -51,7 +60,7 @@ pub(crate) fn node_values( else { continue; }; - for setting in NODE_SETTINGS { + for &setting in node_settings { let Some(value) = settings.get(setting) else { continue; }; @@ -66,7 +75,7 @@ pub(crate) fn node_values( } } } - match (retry, job_retry) { + match (retry, job_retry.as_ref().filter(|_| !python)) { (Some((other, seen)), Some(own)) if seen != *own => { return Err(conflict( key, diff --git a/crates/sqlbuild-analysis/src/job_settings/constants.rs b/crates/sqlbuild-analysis/src/job_settings/constants.rs index 64c7266665..427f2b3cb3 100644 --- a/crates/sqlbuild-analysis/src/job_settings/constants.rs +++ b/crates/sqlbuild-analysis/src/job_settings/constants.rs @@ -1,7 +1,16 @@ //! How far each job setting reaches when a selection names the job. /// Settings that apply only to the named job's own nodes; two jobs may not disagree on one node. -pub const NODE_SETTINGS: [&str; 3] = ["run_tests", "ignore_audit_failures", "full_refresh"]; +pub const NODE_SETTINGS: [&str; 4] = [ + "run_tests", + "ignore_audit_failures", + "full_refresh", + WAREHOUSE_SETTING, +]; +/// The warehouse the job's nodes run on. +pub const WAREHOUSE_SETTING: &str = "warehouse"; +/// The node settings a Python node takes; the others, and `[retry]`, act on SQL nodes only. +pub const PYTHON_NODE_SETTINGS: [&str; 1] = [WAREHOUSE_SETTING]; /// The setting that becomes a limit on how many of the job's nodes run at once inside the build. pub const JOB_LIMIT_SETTING: &str = "concurrency"; /// The help every job setting conflict gives. diff --git a/crates/sqlbuild-analysis/src/job_settings/tests/helpers.rs b/crates/sqlbuild-analysis/src/job_settings/tests/helpers.rs index c7b65cda04..20346b46dc 100644 --- a/crates/sqlbuild-analysis/src/job_settings/tests/helpers.rs +++ b/crates/sqlbuild-analysis/src/job_settings/tests/helpers.rs @@ -96,6 +96,20 @@ pub(super) fn declared() -> BTreeMap { "broken_job".to_owned(), job(json!({"full_refresh": true}), None), ), + ( + "orders_wh_job".to_owned(), + job( + json!({"warehouse": "ORDERS_WH", "run_tests": false}), + Some(JobRetry { + attempts: 2, + delay_seconds: 0, + }), + ), + ), + ( + "refunds_wh_job".to_owned(), + job(json!({"warehouse": "REFUNDS_WH"}), None), + ), ( "daily_group".to_owned(), DeclaredRun::Group { @@ -120,6 +134,8 @@ fn selections() -> BTreeMap { ("retried_job".to_owned(), job(&["stg_orders"])), ("retried_fast_job".to_owned(), job(&["stg_orders"])), ("broken_job".to_owned(), job(&["missing_model"])), + ("orders_wh_job".to_owned(), job(&["+orders"])), + ("refunds_wh_job".to_owned(), job(&["+refunds"])), ( "daily_group".to_owned(), JobSelection::Group { diff --git a/crates/sqlbuild-analysis/src/job_settings/tests/test_build_job_settings.rs b/crates/sqlbuild-analysis/src/job_settings/tests/test_build_job_settings.rs index 93dfd1cd3c..bdbdd88db3 100644 --- a/crates/sqlbuild-analysis/src/job_settings/tests/test_build_job_settings.rs +++ b/crates/sqlbuild-analysis/src/job_settings/tests/test_build_job_settings.rs @@ -43,6 +43,12 @@ fn given_named_jobs_when_merging_build_settings_then_they_must_agree() { and {\"region\":\"us\"}); a build applies vars to everything it runs", ), }, + BuildJobSettingsTestCase { + description: "jobs on different warehouses build together: warehouse is per node", + select: &["job:orders_wh_job job:refunds_wh_job"], + expected_named: vec!["orders_wh_job", "refunds_wh_job"], + expected_settings: Ok(BTreeMap::new()), + }, BuildJobSettingsTestCase { description: "a group's jobs without build-wide settings", select: &["job:daily_group"], diff --git a/crates/sqlbuild-analysis/src/job_settings/tests/test_node_job_settings.rs b/crates/sqlbuild-analysis/src/job_settings/tests/test_node_job_settings.rs index 1851d995ce..1728fde83e 100644 --- a/crates/sqlbuild-analysis/src/job_settings/tests/test_node_job_settings.rs +++ b/crates/sqlbuild-analysis/src/job_settings/tests/test_node_job_settings.rs @@ -2,17 +2,25 @@ use std::collections::BTreeMap; use serde_json::json; +use crate::job_settings::_helpers::node_values::node_values; use crate::job_settings::errors::NodeJobSettingsError; use crate::job_settings::models::NodeJobSettings; -use crate::job_settings::tests::helpers::resolved_settings; +use crate::job_settings::tests::helpers::{declared, resolved_settings, strings}; use crate::job_settings::tests::test_types::{ JobSelectorFailureTestCase, NodeJobConflictTestCase, NodeJobSettingsTestCase, + PythonNodeValuesTestCase, }; #[test] fn given_named_jobs_when_resolving_node_settings_then_only_their_own_selected_nodes_take_them() { let orders_values = || BTreeMap::from([("run_tests", json!(false)), ("full_refresh", json!(true))]); + let orders_wh_values = || { + BTreeMap::from([ + ("run_tests", json!(false)), + ("warehouse", json!("ORDERS_WH")), + ]) + }; let test_cases = [ NodeJobSettingsTestCase { description: "a named job gives each of its nodes its settings and limit", @@ -87,6 +95,20 @@ fn given_named_jobs_when_resolving_node_settings_then_only_their_own_selected_no ]), expected_job_limits: BTreeMap::from([("orders_job", 2)]), }, + NodeJobSettingsTestCase { + description: "jobs on different warehouses give each its own nodes its warehouse", + select: &["job:orders_wh_job", "refunds"], + exclude: &[], + expected_node_jobs: BTreeMap::from([ + ("orders", vec!["orders_wh_job"]), + ("stg_orders", vec!["orders_wh_job"]), + ]), + expected_values: BTreeMap::from([ + ("orders", orders_wh_values()), + ("stg_orders", orders_wh_values()), + ]), + expected_job_limits: BTreeMap::new(), + }, NodeJobSettingsTestCase { description: "a selection that names no job takes no settings", select: &["+orders"], @@ -149,6 +171,13 @@ fn given_two_named_jobs_setting_one_node_differently_when_resolving_then_it_is_a expected_message: "model:stg_orders is selected through jobs staging_job and orders_job, \ which set run_tests differently (true and false)", }, + NodeJobConflictTestCase { + description: "a warehouse", + select: &["job:orders_wh_job job:refunds_wh_job"], + expected_message: "model:stg_orders is selected through jobs orders_wh_job and \ + refunds_wh_job, which set warehouse differently (\"ORDERS_WH\" and \ + \"REFUNDS_WH\")", + }, NodeJobConflictTestCase { description: "a retry policy", select: &["job:retried_job job:retried_fast_job"], @@ -192,3 +221,48 @@ fn given_a_named_job_whose_selection_fails_when_resolving_then_it_fails_naming_t ); } } + +#[test] +fn given_a_python_node_in_named_jobs_when_resolving_then_it_takes_only_their_warehouse() { + let test_cases = [ + PythonNodeValuesTestCase { + description: "the warehouse applies; run_tests and [retry] do not", + jobs: &["orders_wh_job"], + expected_result: Ok(BTreeMap::from([("warehouse", json!("ORDERS_WH"))])), + }, + PythonNodeValuesTestCase { + description: "jobs that agree on everything Python nodes take do not conflict", + jobs: &["orders_wh_job", "orders_job"], + expected_result: Ok(BTreeMap::from([("warehouse", json!("ORDERS_WH"))])), + }, + PythonNodeValuesTestCase { + description: "two warehouses conflict", + jobs: &["orders_wh_job", "refunds_wh_job"], + expected_result: Err( + "asset:daily_report is selected through jobs orders_wh_job and refunds_wh_job, \ + which set warehouse differently (\"ORDERS_WH\" and \"REFUNDS_WH\")", + ), + }, + ]; + let key = ("asset".to_owned(), "daily_report".to_owned()); + for test_case in test_cases { + let resolved = node_values(&key, &strings(test_case.jobs), &declared()); + let shown = match resolved { + Ok((values, retry)) => { + assert_eq!(retry, None, "{}", test_case.description); + Ok(values) + } + Err(conflict) => Err(conflict.message), + }; + let expected = test_case + .expected_result + .map(|values| { + values + .into_iter() + .map(|(name, value)| (name.to_owned(), value)) + .collect() + }) + .map_err(str::to_owned); + assert_eq!(shown, expected, "{}", test_case.description); + } +} diff --git a/crates/sqlbuild-analysis/src/job_settings/tests/test_types.rs b/crates/sqlbuild-analysis/src/job_settings/tests/test_types.rs index 823ec76cc3..c6b3f5413a 100644 --- a/crates/sqlbuild-analysis/src/job_settings/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/job_settings/tests/test_types.rs @@ -25,6 +25,14 @@ pub(super) struct NodeJobConflictTestCase { pub(super) expected_message: &'static str, } +pub(super) struct PythonNodeValuesTestCase { + pub(super) description: &'static str, + /// The named jobs the Python node is selected through. + pub(super) jobs: &'static [&'static str], + /// Its settings, or the conflict's message. + pub(super) expected_result: Result, &'static str>, +} + pub(super) struct BuildJobSettingsTestCase { pub(super) description: &'static str, pub(super) select: &'static [&'static str], diff --git a/crates/sqlbuild-cli/src/argv/tests/test_job_run_arguments.rs b/crates/sqlbuild-cli/src/argv/tests/test_job_run_arguments.rs index 6e11364310..0bec2c5a5d 100644 --- a/crates/sqlbuild-cli/src/argv/tests/test_job_run_arguments.rs +++ b/crates/sqlbuild-cli/src/argv/tests/test_job_run_arguments.rs @@ -11,7 +11,7 @@ fn given_a_server_job_run_when_building_its_arguments_then_it_names_the_job_and_ let test_cases = [ JobRunArgumentsTestCase { description: "concurrency and node-level settings are left to the build", - settings: json!({"concurrency": 8, "run_tests": false, "full_refresh": true}), + settings: json!({"concurrency": 8, "run_tests": false, "full_refresh": true, "warehouse": "ORDERS_WH"}), expected_arguments: &["--select", "job:orders_job"], }, JobRunArgumentsTestCase { diff --git a/crates/sqlbuild-config/src/command_connection/main/check_job_warehouse.rs b/crates/sqlbuild-config/src/command_connection/main/check_job_warehouse.rs new file mode 100644 index 0000000000..016b9a2645 --- /dev/null +++ b/crates/sqlbuild-config/src/command_connection/main/check_job_warehouse.rs @@ -0,0 +1,32 @@ +//! Validate the `warehouse` a job's `[settings]` gives its nodes. + +use sqlbuild_core::text::main::python_str_repr::python_str_repr; +use sqlbuild_core::text::main::python_strip::python_strip; + +use crate::command_connection::_helpers::identifiers::is_warehouse_identifier; +use crate::command_connection::constants::WAREHOUSE_ERROR_CODE; +use crate::command_connection::errors::CommandConnectionError; + +/// The warehouse name of `job`'s `warehouse` setting; `value` is `None` when the setting is not text. +pub fn check_job_warehouse( + job: &str, + value: Option<&str>, +) -> Result { + let problem: String = match value { + Some(name) if is_warehouse_identifier(name) => return Ok(name.to_owned()), + Some(name) if python_strip(name).is_empty() => "an empty warehouse name".to_owned(), + Some(name) => format!( + "warehouse {}, which is not a valid Snowflake warehouse identifier", + python_str_repr(name) + ), + None => "a warehouse that is not text".to_owned(), + }; + Err(CommandConnectionError::User { + code: Some(WAREHOUSE_ERROR_CODE), + message: format!("job {job} sets {problem}"), + help: Some( + "set warehouse in the job's [settings] to an unquoted identifier such as \"ADHOC_WH\", or a double-quoted one such as '\"Adhoc WH\"'" + .to_owned(), + ), + }) +} diff --git a/crates/sqlbuild-config/src/command_connection/main/mod.rs b/crates/sqlbuild-config/src/command_connection/main/mod.rs index 0cfa5f9805..6a36e4e96f 100644 --- a/crates/sqlbuild-config/src/command_connection/main/mod.rs +++ b/crates/sqlbuild-config/src/command_connection/main/mod.rs @@ -1,7 +1,9 @@ pub mod authored_warehouse; pub mod check_cli_warehouse; pub mod check_expanded_warehouse; +pub mod check_job_warehouse; pub mod check_preview_target; pub mod missing_connection_error; pub mod snowflake_key_warning; pub mod unsupported_cli_warehouse; +pub mod unsupported_job_warehouse; diff --git a/crates/sqlbuild-config/src/command_connection/main/unsupported_job_warehouse.rs b/crates/sqlbuild-config/src/command_connection/main/unsupported_job_warehouse.rs new file mode 100644 index 0000000000..1490235b07 --- /dev/null +++ b/crates/sqlbuild-config/src/command_connection/main/unsupported_job_warehouse.rs @@ -0,0 +1,23 @@ +//! A job `warehouse` for an adapter without a session warehouse. + +use crate::command_connection::constants::UNSUPPORTED_WAREHOUSE_CODE; +use crate::command_connection::errors::CommandConnectionError; + +/// The error for a job's `warehouse` the adapter cannot select. +#[must_use] +pub fn unsupported_job_warehouse( + job: &str, + warehouse: &str, + adapter_name: &str, +) -> CommandConnectionError { + CommandConnectionError::User { + code: Some(UNSUPPORTED_WAREHOUSE_CODE), + message: format!( + "job {job} sets warehouse {warehouse}, but the '{adapter_name}' adapter has no warehouse to select" + ), + help: Some( + "remove warehouse from the job's [settings], or build with --no-job-settings; it needs an adapter with a session warehouse, such as snowflake or an adapter extending it" + .to_owned(), + ), + } +} diff --git a/crates/sqlbuild-config/src/command_connection/tests/test_command_connection.rs b/crates/sqlbuild-config/src/command_connection/tests/test_command_connection.rs index 4c4c02071a..90d621fe26 100644 --- a/crates/sqlbuild-config/src/command_connection/tests/test_command_connection.rs +++ b/crates/sqlbuild-config/src/command_connection/tests/test_command_connection.rs @@ -2,10 +2,11 @@ use crate::command_connection::errors::CommandConnectionError; use crate::command_connection::main::authored_warehouse::authored_warehouse; use crate::command_connection::main::check_cli_warehouse::check_cli_warehouse; use crate::command_connection::main::check_expanded_warehouse::check_expanded_warehouse; +use crate::command_connection::main::check_job_warehouse::check_job_warehouse; use crate::command_connection::main::snowflake_key_warning::snowflake_key_warning; use crate::command_connection::tests::helpers::message; use crate::command_connection::tests::test_types::{ - CliWarehouseTestCase, ExpandedWarehouseTestCase, SnowflakeKeyTestCase, + CliWarehouseTestCase, ExpandedWarehouseTestCase, JobWarehouseTestCase, SnowflakeKeyTestCase, }; #[test] @@ -135,3 +136,43 @@ fn given_connection_keys_when_warning_then_only_duckdb_snowflake_keys_are_listed ); } } + +#[test] +fn given_a_job_warehouse_when_checking_then_it_names_the_job_and_follows_identifier_rules() { + let test_cases = [ + JobWarehouseTestCase { + description: "unquoted identifier", + value: Some("ORDERS_WH"), + expected_result: Ok("ORDERS_WH"), + }, + JobWarehouseTestCase { + description: "blank value", + value: Some(" "), + expected_result: Err("job orders_job sets an empty warehouse name"), + }, + JobWarehouseTestCase { + description: "not an identifier", + value: Some("bad wh"), + expected_result: Err( + "job orders_job sets warehouse 'bad wh', which is not a valid Snowflake warehouse identifier", + ), + }, + JobWarehouseTestCase { + description: "not text", + value: None, + expected_result: Err("job orders_job sets a warehouse that is not text"), + }, + ]; + + for test_case in test_cases { + let result = check_job_warehouse("orders_job", test_case.value); + let shown: Result<&str, Option> = match &result { + Ok(name) => Ok(name.as_str()), + Err(error) => Err(Some(error.message().to_owned())), + }; + let expected: Result<&str, Option> = test_case + .expected_result + .map_err(|text| Some(text.to_owned())); + assert_eq!(shown, expected, "{}", test_case.description); + } +} diff --git a/crates/sqlbuild-config/src/command_connection/tests/test_types.rs b/crates/sqlbuild-config/src/command_connection/tests/test_types.rs index bc3e6c8362..263ea978f2 100644 --- a/crates/sqlbuild-config/src/command_connection/tests/test_types.rs +++ b/crates/sqlbuild-config/src/command_connection/tests/test_types.rs @@ -6,6 +6,12 @@ pub(super) struct CliWarehouseTestCase { pub(super) expected_message: Option<&'static str>, } +pub(super) struct JobWarehouseTestCase { + pub(super) description: &'static str, + pub(super) value: Option<&'static str>, + pub(super) expected_result: Result<&'static str, &'static str>, +} + pub(super) struct SnowflakeKeyTestCase { pub(super) description: &'static str, pub(super) adapter_name: &'static str, diff --git a/crates/sqlbuild-core/src/build_command/main/render_run_context.rs b/crates/sqlbuild-core/src/build_command/main/render_run_context.rs index 3b17f6b792..c5ed1d6cec 100644 --- a/crates/sqlbuild-core/src/build_command/main/render_run_context.rs +++ b/crates/sqlbuild-core/src/build_command/main/render_run_context.rs @@ -28,10 +28,17 @@ fn date_vars(facts: &RunContextFacts) -> String { /// The resolved placement, mode and selection, then the selector files. #[must_use] pub fn render_run_context(facts: &RunContextFacts, style: CliStyle) -> String { - let warehouse: String = facts.warehouse.as_ref().map_or_else( + let mut warehouse: String = facts.warehouse.as_ref().map_or_else( || NOT_SET.to_owned(), |(warehouse, source)| format!("{} ({source})", sanitize(warehouse)), ); + for (job, job_warehouse) in &facts.job_warehouses { + warehouse.push_str(&format!( + "; nodes of job {} on {}", + sanitize(job), + sanitize(job_warehouse) + )); + } let rows: Vec<(String, String)> = [ ("command", sanitize(&facts.command)), ("run_id", display_value(facts.run_id.as_deref())), diff --git a/crates/sqlbuild-core/src/build_command/models.rs b/crates/sqlbuild-core/src/build_command/models.rs index a0d9ab15cb..1ab51524d8 100644 --- a/crates/sqlbuild-core/src/build_command/models.rs +++ b/crates/sqlbuild-core/src/build_command/models.rs @@ -184,6 +184,8 @@ pub struct RunContextFacts { pub full_refresh: bool, /// The named jobs whose own models the build rebuilds from scratch. pub full_refresh_jobs: Vec, + /// Each named job whose nodes run on its own warehouse, and that warehouse. + pub job_warehouses: Vec<(String, String)>, pub selected_count: usize, pub total_count: usize, /// Valid ISO `start_date` and `end_date` variables. diff --git a/crates/sqlbuild-core/src/build_command/tests/test_build_command.rs b/crates/sqlbuild-core/src/build_command/tests/test_build_command.rs index f9bb0b89ad..eec9cd311d 100644 --- a/crates/sqlbuild-core/src/build_command/tests/test_build_command.rs +++ b/crates/sqlbuild-core/src/build_command/tests/test_build_command.rs @@ -151,6 +151,19 @@ fn given_run_context_when_rendering_then_sanitizes_and_groups_counts() { }, expected_text: "Execution\n command sqb build\n run_id not set\n target dev\n database not set\n schema not set\n warehouse small (connection warehouse)\n concurrency 0 configured limit\n full_refresh true\n selected 0 of 0 build resources\n date vars end_date=2026-01-31\n\nSelection files\n selector_file a.txt (1 selector)\n selector_file b.txt (2 selectors)\n\n", }, + ContextTestCase { + description: "named jobs whose nodes run on their own warehouses", + facts: RunContextFacts { + command: "sqb build".to_owned(), + warehouse: Some(("small".to_owned(), "connection warehouse".to_owned())), + job_warehouses: vec![ + ("orders_job".to_owned(), "ORDERS_WH".to_owned()), + ("refunds_job".to_owned(), "REFUNDS_WH".to_owned()), + ], + ..RunContextFacts::default() + }, + expected_text: "Execution\n command sqb build\n run_id not set\n target not set\n database not set\n schema not set\n warehouse small (connection warehouse); nodes of job orders_job on ORDERS_WH; nodes of job refunds_job on REFUNDS_WH\n concurrency 0 configured limit\n full_refresh false\n selected 0 of 0 build resources\n date vars not set\n\n", + }, ]; for test_case in test_cases { diff --git a/crates/sqlbuild-core/src/cli_text/_helpers/plan_document.rs b/crates/sqlbuild-core/src/cli_text/_helpers/plan_document.rs index 92130d76cc..9a4edd216c 100644 --- a/crates/sqlbuild-core/src/cli_text/_helpers/plan_document.rs +++ b/crates/sqlbuild-core/src/cli_text/_helpers/plan_document.rs @@ -37,6 +37,9 @@ fn header_lines(report: &PlanTextReport, full_refresh: bool) -> Vec { job_list(&report.ignore_audit_failure_jobs) ))); } + for (job, warehouse) in &report.job_warehouses { + lines.push(format!("Nodes of job {job} run on warehouse {warehouse}")); + } lines } diff --git a/crates/sqlbuild-core/src/cli_text/models.rs b/crates/sqlbuild-core/src/cli_text/models.rs index 6e9b2f5926..473141944c 100644 --- a/crates/sqlbuild-core/src/cli_text/models.rs +++ b/crates/sqlbuild-core/src/cli_text/models.rs @@ -432,6 +432,8 @@ pub struct PlanTextReport { pub ignore_audit_failures: bool, /// The named jobs whose own audit failures hold nothing back. pub ignore_audit_failure_jobs: Vec, + /// Each named job whose nodes run on its own warehouse, and that warehouse. + pub job_warehouses: Vec<(String, String)>, } /// The structural kind of a rendered plan line. diff --git a/crates/sqlbuild-core/src/cli_text/tests/helpers.rs b/crates/sqlbuild-core/src/cli_text/tests/helpers.rs index 1d4ce415c2..441a5d4032 100644 --- a/crates/sqlbuild-core/src/cli_text/tests/helpers.rs +++ b/crates/sqlbuild-core/src/cli_text/tests/helpers.rs @@ -57,6 +57,7 @@ pub(super) fn plan_report(models: Vec) -> PlanTextReport { pre_build_lines: Vec::new(), ignore_audit_failures: false, ignore_audit_failure_jobs: Vec::new(), + job_warehouses: Vec::new(), } } diff --git a/crates/sqlbuild-python-base/src/records/_helpers/compiler_records/constants.rs b/crates/sqlbuild-python-base/src/records/_helpers/compiler_records/constants.rs index e0e0f48a3e..7b590fc4c3 100644 --- a/crates/sqlbuild-python-base/src/records/_helpers/compiler_records/constants.rs +++ b/crates/sqlbuild-python-base/src/records/_helpers/compiler_records/constants.rs @@ -877,6 +877,8 @@ const R_COMPILE_PIPELINE_RESULT: RecordSpec = RecordSpec { FieldSpec::new("python_plan_entries", FieldDefault::EmptyTuple), FieldSpec::new("job_limits", FieldDefault::EmptyDict), FieldSpec::new("node_jobs", FieldDefault::EmptyDict), + FieldSpec::new("node_warehouses", FieldDefault::EmptyDict), + FieldSpec::new("python_node_warehouses", FieldDefault::EmptyDict), FieldSpec::new("compile_seconds", FieldDefault::None), FieldSpec::new("planning_seconds", FieldDefault::None), ], diff --git a/crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_text.rs b/crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_text.rs index 937d5ca627..6823ccc17c 100644 --- a/crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_text.rs +++ b/crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_text.rs @@ -34,7 +34,9 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::cursor::main::scalars::sentinel_to_token::sentinel_to_token; -use sqlbuild_python_project::compiled_project::constants::JOB_IGNORED_AUDIT_FAILURES_KEY; +use sqlbuild_python_project::compiled_project::constants::{ + JOB_IGNORED_AUDIT_FAILURES_KEY, JOB_WAREHOUSES_KEY, +}; const SKIP_ACTION: &str = "skip"; const QUERY_CHANGED: &str = "query_changed"; @@ -390,6 +392,18 @@ fn provider_usages( } /// `str(item)` of a metadata tuple or list, else nothing. +/// The plan metadata's named jobs and the warehouses their nodes run on. +pub(crate) fn job_warehouses(metadata: &Bound<'_, PyAny>) -> PyResult> { + let value: Bound<'_, PyAny> = mapping_get(metadata, JOB_WAREHOUSES_KEY)?; + let Ok(warehouses) = value.cast::() else { + return Ok(Vec::new()); + }; + warehouses + .iter() + .map(|(job, warehouse)| Ok((job.extract()?, warehouse.extract()?))) + .collect() +} + fn metadata_names(value: Option>) -> PyResult> { match value { Some(value) if value.is_instance_of::() || value.is_instance_of::() => { @@ -534,5 +548,6 @@ pub(crate) fn report(arguments: &KeywordArguments<'_>) -> PyResult, options: Option>) -> P )?; } } + let warehouses: Bound<'_, PyAny> = mapping_get(&metadata, JOB_WAREHOUSES_KEY)?; + if !warehouses.is_none() { + result.set_item(WAREHOUSE_JOBS_KEY, warehouses)?; + } indented_json(result.as_any()) } diff --git a/crates/sqlbuild-python-planner/src/plan_output/main/job_warehouses.rs b/crates/sqlbuild-python-planner/src/plan_output/main/job_warehouses.rs new file mode 100644 index 0000000000..fa12b993ed --- /dev/null +++ b/crates/sqlbuild-python-planner/src/plan_output/main/job_warehouses.rs @@ -0,0 +1,10 @@ +//! The plan metadata's named jobs and the warehouses their nodes run on. + +use crate::plan_output::_helpers::plan_text; +use pyo3::prelude::{Bound, PyAny, PyResult}; + +/// The plan metadata's named jobs and the warehouses their nodes run on. +#[inline] +pub fn job_warehouses(metadata: &Bound<'_, PyAny>) -> PyResult> { + plan_text::job_warehouses(metadata) +} diff --git a/crates/sqlbuild-python-planner/src/plan_output/main/mod.rs b/crates/sqlbuild-python-planner/src/plan_output/main/mod.rs index 1ffbc27d2f..92aea4c7f1 100644 --- a/crates/sqlbuild-python-planner/src/plan_output/main/mod.rs +++ b/crates/sqlbuild-python-planner/src/plan_output/main/mod.rs @@ -7,6 +7,7 @@ pub mod format_plan_json; pub mod future_cursor_safety; pub mod inclusive_cursor_end; pub mod items; +pub mod job_warehouses; pub mod maximum_start_safety; pub mod optional_facts; pub mod register_display; diff --git a/crates/sqlbuild-python-project/src/compiled_project/_helpers/job_catalog.rs b/crates/sqlbuild-python-project/src/compiled_project/_helpers/job_catalog.rs index 419aa3cb17..55dee79117 100644 --- a/crates/sqlbuild-python-project/src/compiled_project/_helpers/job_catalog.rs +++ b/crates/sqlbuild-python-project/src/compiled_project/_helpers/job_catalog.rs @@ -121,6 +121,16 @@ pub(crate) fn job_catalog(py: Python<'_>) -> PyResult> { Ok(loaded_jobs(py)?.map(|(catalog, _)| catalog)) } +/// The project directory of the running command's job scope, if it has one. +pub(crate) fn job_scope_project_dir() -> Option { + JOB_SCOPE.with(|scope| { + scope + .borrow() + .as_ref() + .map(|scope| scope.project_dir.clone()) + }) +} + /// Each job's `[settings]` and `[retry]` and each group's jobs declared for the command's target; none outside a command. pub(crate) fn job_declarations(py: Python<'_>) -> PyResult>> { Ok(loaded_jobs(py)?.map(|(_, declared)| declared)) diff --git a/crates/sqlbuild-python-project/src/compiled_project/constants.rs b/crates/sqlbuild-python-project/src/compiled_project/constants.rs index 637114e246..71ee41fc3c 100644 --- a/crates/sqlbuild-python-project/src/compiled_project/constants.rs +++ b/crates/sqlbuild-python-project/src/compiled_project/constants.rs @@ -36,3 +36,5 @@ pub const JOB_RUN_COUNTS: [&str; 2] = ["definition", "source_data"]; pub const JOB_IGNORED_AUDIT_FAILURES_KEY: &str = "job_ignored_audit_failures"; /// Plan metadata naming the jobs whose own models a build rebuilds from scratch. pub const JOB_FULL_REFRESH_KEY: &str = "job_full_refresh"; +/// Plan metadata mapping each named job whose warehouse its nodes run on to that warehouse. +pub const JOB_WAREHOUSES_KEY: &str = "job_warehouses"; diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/job_scope_project_dir.rs b/crates/sqlbuild-python-project/src/compiled_project/main/job_scope_project_dir.rs new file mode 100644 index 0000000000..5861a6f253 --- /dev/null +++ b/crates/sqlbuild-python-project/src/compiled_project/main/job_scope_project_dir.rs @@ -0,0 +1,10 @@ +//! The project directory of the running command's job scope. + +use crate::compiled_project::_helpers::job_catalog; +use std::path::PathBuf; + +/// The project directory of the running command's job scope, if it has one. +#[inline] +pub fn job_scope_project_dir() -> Option { + job_catalog::job_scope_project_dir() +} diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/mod.rs b/crates/sqlbuild-python-project/src/compiled_project/main/mod.rs index 2dd8eae3e2..eec55f6817 100644 --- a/crates/sqlbuild-python-project/src/compiled_project/main/mod.rs +++ b/crates/sqlbuild-python-project/src/compiled_project/main/mod.rs @@ -7,6 +7,7 @@ pub mod graph_with_jobs; pub mod install; pub mod job_declarations; pub mod job_names; +pub mod job_scope_project_dir; pub mod materialization; pub mod model_facts; pub mod named_jobs; diff --git a/crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/python_selection.rs b/crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/python_selection.rs index ad201176f4..812c37ffbb 100644 --- a/crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/python_selection.rs +++ b/crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/python_selection.rs @@ -4,6 +4,7 @@ use crate::graph::main::python_nodes::python_node_names::python_node_names; use sqlbuild_python_base::positional_record; use sqlbuild_python_base::records::main::construct::new_error::new_error; use std::borrow::Cow; +use std::collections::{BTreeMap, BTreeSet}; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::pyfunction; @@ -192,6 +193,31 @@ impl<'g> PythonAwareSelection<'g> { &self.graph } + /// The SQL keys and Python node `(kind, name)` keys `select` minus `exclude` resolve to. + pub fn resolve_keys( + &self, + select: &[String], + exclude: &[String], + ) -> Result, SelectorError> { + let RunSelection { + sql_keys, + python_node_names, + } = self.resolve(select, exclude)?; + let kinds: BTreeMap<&str, &str> = self + .nodes + .nodes + .iter() + .map(|node| (node.name.as_str(), node.kind.as_str())) + .collect(); + let mut keys: BTreeSet = sql_keys.into_iter().collect(); + for name in python_node_names { + if let Some(kind) = kinds.get(name.as_str()) { + keys.insert(((*kind).to_owned(), name)); + } + } + Ok(keys) + } + /// The SQL keys and Python node names `select` minus `exclude` resolve to. pub fn resolve( &self, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs index 9f8e5628d6..1c84fe89c8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs @@ -65,6 +65,7 @@ use sqlbuild_python_planner::consumption::models::ConsumptionRecorder; use sqlbuild_python_planner::consumption::models::Prepared; use sqlbuild_python_planner::freshness::main::append_freshness_records::append_freshness_records; use sqlbuild_python_planner::observations::main::append_build_observations::append_build_observations; +use sqlbuild_python_planner::plan_output::main::job_warehouses::job_warehouses; use sqlbuild_python_planner::planner_resolution::main::cursor::type_cursor_overrides::type_cursor_overrides; use sqlbuild_python_project::compiled_project::constants::JOB_FULL_REFRESH_KEY; use sqlbuild_python_project::graph::main::python_nodes::python_node_graph::python_node_graph; @@ -177,6 +178,10 @@ fn run_context_facts( concurrency_source: concurrency.source.to_string(), full_refresh: context.request_flag("full_refresh")?, full_refresh_jobs: full_refresh_jobs(pipeline)?, + job_warehouses: job_warehouses(&native_field( + &native_field(pipeline, "plan_output")?, + "metadata", + )?)?, selected_count, total_count, date_vars: ( @@ -437,6 +442,10 @@ pub(crate) fn execute_build_plan<'py>( ("providers", providers.clone()), ("job_limits", native_field(pipeline, "job_limits")?), ("node_jobs", native_field(pipeline, "node_jobs")?), + ( + "node_warehouses", + native_field(pipeline, "node_warehouses")?, + ), ( "test_timeouts", timeout_record( @@ -738,6 +747,10 @@ fn check_runtime<'py>( .and_then(|value| pyo3::IntoPyObjectExt::into_bound_py_any(value, py))?, ), ("providers", providers.clone()), + ( + "node_warehouses", + native_field(pipeline, "python_node_warehouses")?, + ), ], ) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs index f5e23ede68..af968d135f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs @@ -344,6 +344,8 @@ struct RuntimeParts<'a, 'py> { target_config: TargetSettingsValues, relations: (Bound<'py, PyAny>, Bound<'py, PyAny>), inputs: &'a LifecycleInputs<'py>, + /// Each Python node's job warehouse, by name. + warehouses: Bound<'py, PyAny>, } fn node_runtime<'py>( @@ -375,6 +377,7 @@ fn node_runtime<'py>( ("relation_targets", parts.relations.0.clone()), ("project_relations", parts.relations.1.clone()), ("providers", parts.inputs.providers.clone()), + ("node_warehouses", parts.warehouses.clone()), ], ) } @@ -528,6 +531,7 @@ pub(crate) fn prepare_direct_python_lifecycle<'py>( target_config, relations: (relation_targets, project_relations), inputs, + warehouses: native_field(pipeline, "python_node_warehouses")?, }; let mut ingress_python: Bound<'py, PyAny> = tuple(py, &[])?; let mut ingress_loads: Bound<'py, PyAny> = tuple(py, &[])?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/job_node_settings.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/job_node_settings.rs index 7c8661e1da..d2dd302777 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/job_node_settings.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/job_node_settings.rs @@ -2,18 +2,24 @@ use std::collections::{BTreeMap, BTreeSet}; +use pyo3::IntoPyObject; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyBool, PyDict, PyDictMethods, PyFrozenSet, PyTuple}; use serde_json::Value; use sqlbuild_analysis::assembly::project::types::ObjectKey; +use sqlbuild_analysis::definition_changes::constants::PYTHON_NODE_KINDS; use sqlbuild_analysis::graph::main::selectors_need_jobs::selectors_need_jobs; -use sqlbuild_analysis::job_settings::constants::CONFLICT_HELP; +use sqlbuild_analysis::job_settings::constants::{CONFLICT_HELP, WAREHOUSE_SETTING}; use sqlbuild_analysis::job_settings::errors::{JobSettingConflict, NodeJobSettingsError}; use sqlbuild_analysis::job_settings::main::build_job_settings::build_job_settings; use sqlbuild_analysis::job_settings::main::named_jobs::named_jobs; use sqlbuild_analysis::job_settings::main::node_job_settings::node_job_settings; use sqlbuild_analysis::job_settings::models::{DeclaredRun, NodeJobSettings}; +use sqlbuild_config::command_connection::main::check_job_warehouse::check_job_warehouse; +use sqlbuild_config::command_connection::main::unsupported_job_warehouse::unsupported_job_warehouse; +use crate::bindings::_helpers::command_runtime::runtime_errors::raised; +use crate::bindings::_helpers::command_runtime::warehouses::command_scope; use crate::bindings::_helpers::compiled_project::job_settings::{ job_selector_error, job_setting_conflict, }; @@ -21,10 +27,13 @@ use sqlbuild_python_base::boundary::main::compiler_guard::compiler_guard; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::replaced::replaced; use sqlbuild_python_compile::compile_driver::main::setup::project_graph::project_graph; +use sqlbuild_python_front::compile_driver::main::session_warehouse_support::session_warehouse_support; +use sqlbuild_python_front::contracts::main::resolve_effective_adapter_name::resolve_effective_adapter_name; use sqlbuild_python_project::compiled_project::constants::{ - JOB_FULL_REFRESH_KEY, JOB_IGNORED_AUDIT_FAILURES_KEY, + JOB_FULL_REFRESH_KEY, JOB_IGNORED_AUDIT_FAILURES_KEY, JOB_WAREHOUSES_KEY, }; use sqlbuild_python_project::compiled_project::main::job_declarations::job_declarations; +use sqlbuild_python_project::compiled_project::main::job_scope_project_dir::job_scope_project_dir; use sqlbuild_python_project::graph::main::nodes::native_key::native_key; use sqlbuild_python_project::graph::main::python_nodes::python_node_graph::python_node_graph; use sqlbuild_python_project::graph::models::KeyCache; @@ -44,6 +53,61 @@ pub(crate) struct PlannedJobSettings { ignoring_jobs: Vec, /// The named jobs that set `full_refresh = true` for models they select. refreshing_jobs: Vec, + /// Each named job whose `warehouse` its own nodes run on, and that warehouse; `--warehouse` + /// leaves the jobs' warehouses out before they are resolved. + job_warehouses: BTreeMap, +} + +fn is_python(key: &ObjectKey) -> bool { + PYTHON_NODE_KINDS.contains(&key.0.as_str()) +} + +/// The named jobs whose `warehouse` some selected node runs on, checked as `--warehouse` would be: +/// a valid name (`C261`) on an adapter with a session warehouse (`C260`). +fn job_warehouses( + (project, discovered): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), + settings: &NodeJobSettings, + declared: &BTreeMap, +) -> PyResult> { + let py: Python<'_> = project.py(); + let mut warehouses: BTreeMap = BTreeMap::new(); + for (key, jobs) in &settings.node_jobs { + if !settings + .values + .get(key) + .is_some_and(|values| values.contains_key(WAREHOUSE_SETTING)) + { + continue; + } + for job in jobs { + if warehouses.contains_key(job) { + continue; + } + if let Some(DeclaredRun::Job { settings, .. }) = declared.get(job) + && let Some(value) = settings.get(WAREHOUSE_SETTING) + { + let name: String = + check_job_warehouse(job, value.as_str()).map_err(|error| raised(py, error))?; + warehouses.insert(job.clone(), name); + } + } + } + let Some((job, warehouse)) = warehouses.iter().next() else { + return Ok(warehouses); + }; + let adapter_name: Bound<'_, PyAny> = resolve_effective_adapter_name( + &native_field(discovered, "project_config")?, + &native_field(discovered, "local_config")?, + )?; + let project_dir: Bound<'_, PyAny> = job_scope_project_dir().into_pyobject(py)?.into_any(); + if !session_warehouse_support(py, &adapter_name, &project_dir)? { + let adapter: String = adapter_name.str()?.extract()?; + return Err(raised( + py, + unsupported_job_warehouse(job, warehouse, &adapter), + )); + } + Ok(warehouses) } /// The named jobs of `settings` that set `setting` to true for a selected node of `kind`, if given. @@ -54,7 +118,7 @@ fn jobs_setting( ) -> Vec { let mut jobs: BTreeSet = BTreeSet::new(); for (key, node_jobs) in &settings.node_jobs { - if kind.is_some_and(|kind| key.0 != kind) { + if is_python(key) || kind.is_some_and(|kind| key.0 != kind) { continue; } for job in node_jobs { @@ -89,9 +153,16 @@ pub(crate) fn planned_job_settings( return Ok(None); } let exclude: Vec = strings(&native_field(options, "exclude")?)?; - let Some(declared): Option> = job_declarations(py)? else { + let Some(mut declared): Option> = job_declarations(py)? else { return Ok(None); }; + if command_scope(py)?.cli_warehouse.is_some() { + for run in declared.values_mut() { + if let DeclaredRun::Job { settings, .. } = run { + settings.remove(WAREHOUSE_SETTING); + } + } + } let named: Vec = named_jobs(&select, &declared); if named.is_empty() { return Ok(None); @@ -105,10 +176,8 @@ pub(crate) fn planned_job_settings( &python_node_graph(discovered)?, &all, )?; - let token_nodes = |token: &str, exclude: &[String]| { - let resolved = selection.resolve(&[token.to_owned()], exclude)?; - Ok(resolved.sql_keys.into_iter().collect()) - }; + let token_nodes = + |token: &str, exclude: &[String]| selection.resolve_keys(&[token.to_owned()], exclude); let resolved = compiler_guard(|| { Ok(node_job_settings( selection.graph(), @@ -121,6 +190,7 @@ pub(crate) fn planned_job_settings( Ok(settings) => Ok(Some(PlannedJobSettings { ignoring_jobs: jobs_setting(&settings, &declared, (IGNORE_AUDIT_FAILURES, None)), refreshing_jobs: jobs_setting(&settings, &declared, (FULL_REFRESH, Some(MODEL_KIND))), + job_warehouses: job_warehouses((project, discovered), &settings, &declared)?, settings, })), Err(NodeJobSettingsError::Selector(failure)) => Err(job_selector_error(py, &failure)), @@ -183,7 +253,7 @@ impl PlannedJobSettings { jobs }; let cache = KeyCache::new(py)?; - for key in self.settings.node_jobs.keys() { + for key in self.settings.node_jobs.keys().filter(|key| !is_python(key)) { let jobs: Vec = limited(std::slice::from_ref(key)); if !jobs.is_empty() { node_jobs.set_item(cache.key(key)?, PyTuple::new(py, jobs)?)?; @@ -203,6 +273,50 @@ impl PlannedJobSettings { Ok((limits, node_jobs)) } + /// The warehouse of a node's named job, when one applies to it. + fn warehouse(&self, key: &ObjectKey) -> Option<&str> { + if self.job_warehouses.is_empty() { + return None; + } + self.value(key, WAREHOUSE_SETTING).and_then(Value::as_str) + } + + /// Each planned SQL node's job warehouse, and each Python node's by name; a SQL test runs on + /// the warehouse its models' jobs give when they agree. Empty under `--warehouse`. + pub(crate) fn node_warehouses<'py>( + &self, + plan: &Bound<'py, PyAny>, + ) -> PyResult<(Bound<'py, PyDict>, Bound<'py, PyDict>)> { + let py: Python<'py> = plan.py(); + let nodes: Bound<'py, PyDict> = PyDict::new(py); + let python: Bound<'py, PyDict> = PyDict::new(py); + if self.job_warehouses.is_empty() { + return Ok((nodes, python)); + } + for key in native_field(plan, "execution_order")?.try_iter()? { + let key: Bound<'py, PyAny> = key?; + if let Some(warehouse) = self.warehouse(&native_key(&key)?) { + nodes.set_item(key, warehouse)?; + } + } + for entry in native_field(plan, "test_entries")?.try_iter()? { + let entry: Bound<'py, PyAny> = entry?; + let mut covered: BTreeSet<&str> = BTreeSet::new(); + for dep in native_field(&entry, "scope_deps")?.try_iter()? { + covered.extend(self.warehouse(&native_key(&dep?)?)); + } + if let [warehouse] = covered.into_iter().collect::>().as_slice() { + nodes.set_item(native_field(&entry, "key")?, *warehouse)?; + } + } + for key in self.settings.values.keys().filter(|key| is_python(key)) { + if let Some(warehouse) = self.warehouse(key) { + python.set_item(&key.1, warehouse)?; + } + } + Ok((nodes, python)) + } + /// The plan with its jobs' `run_tests` and, unless the command line decided, audit failures. pub(crate) fn apply_to_plan<'py>( &self, @@ -267,7 +381,10 @@ impl PlannedJobSettings { } else { &[] }; - if !ignoring.is_empty() || !self.refreshing_jobs.is_empty() { + if !ignoring.is_empty() + || !self.refreshing_jobs.is_empty() + || !self.job_warehouses.is_empty() + { let metadata: Bound<'py, PyDict> = native_field(plan, "metadata")? .cast_into::()? .copy()?; @@ -280,6 +397,13 @@ impl PlannedJobSettings { PyTuple::new(py, &self.refreshing_jobs)?, )?; } + if !self.job_warehouses.is_empty() { + let warehouses: Bound<'py, PyDict> = PyDict::new(py); + for (job, warehouse) in &self.job_warehouses { + warehouses.set_item(job, warehouse)?; + } + metadata.set_item(JOB_WAREHOUSES_KEY, warehouses)?; + } changes.push(("metadata", metadata.into_any())); } if changes.is_empty() { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs index 303d3b1dac..ede1ec0720 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs @@ -238,6 +238,9 @@ pub(crate) fn pipeline_result<'py>( job_settings.job_limits(&plan_output, &native_field(options, "concurrency")?)?; job_fields.push(("job_limits", limits.into_any())); job_fields.push(("node_jobs", node_jobs.into_any())); + let (nodes, python) = job_settings.node_warehouses(&plan_output)?; + job_fields.push(("node_warehouses", nodes.into_any())); + job_fields.push(("python_node_warehouses", python.into_any())); plan_output } None => plan_output, diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/snowflake.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/snowflake.md index 85c44a8187..0d528d76cb 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/snowflake.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/snowflake.md @@ -90,9 +90,10 @@ query = "ADHOC_WH" Each command uses the first of these that is set: 1. `--warehouse `, for that run only -2. The target's warehouse for the command's group -3. The connection's `warehouse` -4. The Snowflake user's default warehouse +2. For `sqb build`, the `warehouse` of a named job, for that job's nodes only; see below +3. The target's warehouse for the command's group +4. The connection's `warehouse` +5. The Snowflake user's default warehouse A location's syncs use the target's `sync` warehouse, else its `query` warehouse, else the connection's `warehouse`, so a sync never wakes the target's `build` warehouse just to read @@ -108,10 +109,39 @@ sync = "TINY_WH" Runs that a location executes, such as `sqb build`, use their own command's group. -Every connection a command opens uses the same warehouse. Tests and audits that run inside +Every connection a command opens starts on the same warehouse. Tests and audits that run inside `sqb build` use the `build` warehouse. Under `sqb dbt`, dbt's own nodes use the warehouse in their dbt profile. +### A job's own warehouse + +A job can run its nodes on a warehouse of its own: + +```toml +# orchestration/jobs/nightly_orders_job.toml +[settings] +warehouse = "ORDERS_WH" +``` + +When a build names the job, as `sqb build --select job:nightly_orders_job` or a server run of it, +each of the job's nodes runs on `ORDERS_WH`. Its models' audits, its SQL tests, microbatch batches, +source loads and Python nodes go with them, and loaders and Python code that open their own +connections from the context's `connection_config` get `ORDERS_WH` there too. A SQL test runs on +its models' job warehouse when they agree on one. Everything else in the build, including nodes +selected without naming a job and the build's own bookkeeping, stays on the `build` warehouse. + +The connection running a node switches with `USE WAREHOUSE` only when the node needs a different +warehouse than the connection is on, and switches back before it runs a node outside the job. +Connections return to the `build` warehouse when the build ends. Without a configured warehouse, +SQLBuild reads the session's warehouse once before the first switch, so it can switch back. + +`--warehouse` puts the whole build on one warehouse and leaves the jobs' warehouses out. Two named +jobs that give one node different warehouses fail with `S027`; see +[Settings when a build names a job](../orchestration.md#settings-when-a-build-names-a-job). +A job's warehouse must be a Snowflake identifier like a group warehouse (`C261`), and it needs the +Snowflake adapter or one built on it (`C260`). Cost estimates read each query's warehouse, so a +build's cost names every warehouse it used. + A group warehouse wins over a connection `warehouse` from either file. To change one for yourself, set the group in `sqlbuild_local.toml`; the other group keeps its project value: diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/orchestration.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/orchestration.md index 56c2fb65b5..ab50202c83 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/orchestration.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/orchestration.md @@ -110,7 +110,7 @@ Naming a group names each of its jobs. The first value set wins: 4. The project's setting A command-line flag replaces the job's setting rather than merging with it: `--vars` replaces the -job's whole `vars`. An option that rules another out counts as setting both, so `--reload` wins +job's whole `vars`, and `--warehouse` puts every node on its warehouse. An option that rules another out counts as setting both, so `--reload` wins over a job's `load_sources = false`. Any `--concurrency` sets the build's limit and turns off the per-job limits. The job's settings are added before a `--` on the command line, so selectors after it still name jobs. `--no-job-settings` leaves out every job's settings. @@ -123,6 +123,7 @@ nodes differently: | `full_refresh` | The job's incremental models, rebuilt from scratch | | `run_tests` | The SQL unit tests of the job's models | | `ignore_audit_failures` | The audits of the job's nodes | +| `warehouse` | The Snowflake warehouse the job's nodes run on, Python nodes included; see [A job's own warehouse](adapters/snowflake.md#a-jobs-own-warehouse) | | `concurrency` | How many of the job's nodes build at once, and a node in two named jobs fits within both. The build's own limit becomes the larger of the configured limit and the named jobs' limits added together | | `[retry]` | The job's nodes | diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 622d547dd5..6fbe44c29e 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -15332,6 +15332,8 @@ class CompilePipelineResult(NativeRecord): python_plan_entries: tuple[PythonPlanEntry, ...] job_limits: dict[str, int] node_jobs: dict[CompiledObjectKey, tuple[str, ...]] + node_warehouses: dict[CompiledObjectKey, str] + python_node_warehouses: dict[str, str] compile_seconds: float | None planning_seconds: float | None def __init__( @@ -15343,6 +15345,8 @@ class CompilePipelineResult(NativeRecord): python_plan_entries: tuple[PythonPlanEntry, ...] = ..., job_limits: dict[str, int] = ..., node_jobs: dict[CompiledObjectKey, tuple[str, ...]] = ..., + node_warehouses: dict[CompiledObjectKey, str] = ..., + python_node_warehouses: dict[str, str] = ..., compile_seconds: float | None = ..., planning_seconds: float | None = ..., ) -> None: ... diff --git a/src/sqlbuild/adapter/contract/classes/base_adapter.py b/src/sqlbuild/adapter/contract/classes/base_adapter.py index 143a9e7c76..a910587441 100644 --- a/src/sqlbuild/adapter/contract/classes/base_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/base_adapter.py @@ -2345,6 +2345,16 @@ def render_create_observations_table_sql(self, *, database: str | None, schema: render_framework_type=self.render_framework_type, ) + def use_session_warehouse(self, *, connection: Any, warehouse: str) -> None: + """Run the session's next statements on `warehouse`, where the adapter has one.""" + + raise NotImplementedError(f"{type(self).__name__} has no session warehouse to select") + + def current_session_warehouse(self, *, connection: Any) -> str | None: + """The warehouse the session runs on, as an identifier `use_session_warehouse` accepts.""" + + return None + def render_create_consumption_index_sqls( self, *, diff --git a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py index 6d62b24af0..3d1286108f 100644 --- a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py +++ b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py @@ -200,6 +200,8 @@ _SHOW_OBJECT_RELATION_TYPES: dict[str, str] = {"TABLE": "base table", "VIEW": "view"} _COST_LOGIN_TIMEOUT_SECONDS: int = 5 _COST_NETWORK_TIMEOUT_SECONDS: int = 15 +# A warehouse name CURRENT_WAREHOUSE() returns that USE WAREHOUSE accepts without quotes. +_UNQUOTED_WAREHOUSE: re.Pattern[str] = re.compile(r"[A-Z_][A-Z0-9_$]*") _SHOW_COLUMN_KIND: str = "COLUMN" _SHOW_COLUMN_KINDS: frozenset[str] = frozenset({_SHOW_COLUMN_KIND, "VIRTUAL_COLUMN"}) _UNRESOLVED: object = object() @@ -1323,6 +1325,20 @@ def render_current_utc_timestamp(self) -> str: def render_current_warehouse_user(self) -> str | None: return "CURRENT_USER()" + def use_session_warehouse(self, *, connection: Any, warehouse: str) -> None: + self.execute(connection=connection, sql=f"USE WAREHOUSE {warehouse}") + + def current_session_warehouse(self, *, connection: Any) -> str | None: + rows: tuple[tuple[object, ...], ...] = self.query( + connection=connection, sql="SELECT CURRENT_WAREHOUSE()", limit=None + ).rows + name: object = rows[0][0] if rows else None + if not isinstance(name, str) or not name: + return None + if _UNQUOTED_WAREHOUSE.fullmatch(name): + return name + return '"' + name.replace('"', '""') + '"' + def create_table_as( self, *, diff --git a/src/sqlbuild/executor/build/classes/build_scheduler.py b/src/sqlbuild/executor/build/classes/build_scheduler.py index c6a967629e..3e933076c0 100644 --- a/src/sqlbuild/executor/build/classes/build_scheduler.py +++ b/src/sqlbuild/executor/build/classes/build_scheduler.py @@ -115,6 +115,11 @@ ) from sqlbuild.executor.run.types import MicrobatchBatchExecutor from sqlbuild.executor.scheduling.classes.recovering_connection_pool import RecoveringConnectionPool +from sqlbuild.executor.scheduling.classes.session_warehouses import ( + SessionWarehouses, + configured_warehouse, + with_session_warehouse, +) from sqlbuild.executor.scheduling.main._run_worker import run_worker_with_completion from sqlbuild.executor.scheduling.types import ExecutionStatus from sqlbuild.executor.seed.constants import SEED_ENTRY_MISSING_CODE @@ -243,6 +248,12 @@ def __init__( ) self._max_concurrency: int = len(connections) self._init_job_limits(runtime) + self._node_warehouses: Mapping[CompiledObjectKey, str] = runtime.node_warehouses + self._session_warehouses: SessionWarehouses | None = ( + SessionWarehouses(adapter=adapter, home=configured_warehouse(connection_config)) + if runtime.node_warehouses + else None + ) self._blocked_keys: set[CompiledObjectKey] = set() self._unbuilt_keys: set[CompiledObjectKey] = set(initial_state.initial_failed_keys) self._completed_keys: set[CompiledObjectKey] = set(initial_state.precompleted_keys) @@ -306,6 +317,8 @@ def run( else: self._run_concurrent() finally: + if self._session_warehouses is not None: + self._session_warehouses.restore() self._close_replacement_connections() enforce_execution_deadline() @@ -679,6 +692,11 @@ def _execute_concurrent_microbatch_node( ledger_path=(self._runtime_dir / "executions" / self._run_id / "statements.jsonl"), on_statement_complete=self._on_statement_complete, ): + failure: ModelExecutionResult | None = self._enter_node_warehouse( + key=key, connection=connection + ) + if failure is not None: + return failure return self._execute_model_node( key=key, connection=connection, @@ -686,6 +704,7 @@ def _execute_concurrent_microbatch_node( self._run_microbatch_subwork( pool=pool, jobs=self._node_jobs.get(key, ()), + warehouse=self._node_warehouses.get(key), coordinator_connection=connection, batches=batches, concurrency=concurrency, @@ -702,6 +721,7 @@ def _run_microbatch_subwork( *, pool: ThreadPoolExecutor, jobs: tuple[str, ...], + warehouse: str | None = None, coordinator_connection: Any, batches: tuple[BatchWindow, ...], concurrency: int, @@ -725,6 +745,7 @@ def submit_one() -> tuple[Future[MicrobatchPhaseOutcome], BatchWindow]: copy_context().run, self._run_microbatch_batch_worker, jobs=jobs, + warehouse=warehouse, batch=batch, execute=execute, ), @@ -794,11 +815,14 @@ def _run_microbatch_batch_worker( self, *, jobs: tuple[str, ...], + warehouse: str | None = None, batch: BatchWindow, execute: MicrobatchBatchExecutor, ) -> MicrobatchPhaseOutcome: connection: Any = self._connection_pool.get() try: + if self._session_warehouses is not None: + self._session_warehouses.use(connection=connection, warehouse=warehouse) return execute(batch, connection) finally: try: @@ -1029,6 +1053,11 @@ def _execute_node_with_cost_context( ): """Execute a single node. Runs in a worker thread (or scheduler thread for serial).""" + failure: ModelExecutionResult | None = self._enter_node_warehouse( + key=key, connection=connection + ) + if failure is not None: + return failure if key.resource_type == CompiledResourceType.SEED: return self._execute_seed_node(key=key, connection=connection) if key.resource_type == CompiledResourceType.SOURCE: @@ -1046,6 +1075,29 @@ def _execute_node_with_cost_context( error_message=f"unknown executable resource type '{key.resource_type}'", ) + def _enter_node_warehouse( + self, *, key: CompiledObjectKey, connection: Any + ) -> ModelExecutionResult | None: + """Put `connection` on the node's job warehouse, or the build's; the failure if not.""" + + if self._session_warehouses is None: + return None + warehouse: str | None = self._node_warehouses.get(key) + try: + self._session_warehouses.use(connection=connection, warehouse=warehouse) + except Exception as error: # noqa: BLE001 - the node fails with the reason + return ModelExecutionResult( + model_name=key.name, + status=ExecutionStatus.FAILED, + error_code=BUILD_WORKER_FAILED_CODE, + error_message=( + f"cannot run on warehouse {warehouse}: {error}" + if warehouse is not None + else f"cannot return to the build's warehouse: {error}" + ), + ) + return None + def _execute_source_node( self, *, key: CompiledObjectKey, connection: Any ) -> LoadExecutionResult: @@ -1061,7 +1113,9 @@ def _execute_source_node( loader_functions_by_name=self._loader_functions_by_name, loader_ref_entries=self._loader_ref_entries, adapter=self._adapter, - connection_config=self._connection_config, + connection_config=with_session_warehouse( + self._connection_config, self._node_warehouses.get(key) + ), connection=connection, runtime=self._runtime, callbacks=self._callbacks, diff --git a/src/sqlbuild/executor/build/models.py b/src/sqlbuild/executor/build/models.py index 14459bd99c..49c030dee8 100644 --- a/src/sqlbuild/executor/build/models.py +++ b/src/sqlbuild/executor/build/models.py @@ -109,6 +109,8 @@ class BuildRuntimeParams: dynamic_hold: DynamicHold | None = None job_limits: Mapping[str, int] = field(default_factory=dict) node_jobs: Mapping[CompiledObjectKey, tuple[str, ...]] = field(default_factory=dict) + # The warehouse each node runs on when its named job sets one; others use the build's. + node_warehouses: Mapping[CompiledObjectKey, str] = field(default_factory=dict) @dataclass(frozen=True) diff --git a/src/sqlbuild/executor/python_nodes/_helpers/ingress_execution.py b/src/sqlbuild/executor/python_nodes/_helpers/ingress_execution.py index 383e915f9e..c77732a8c9 100644 --- a/src/sqlbuild/executor/python_nodes/_helpers/ingress_execution.py +++ b/src/sqlbuild/executor/python_nodes/_helpers/ingress_execution.py @@ -31,6 +31,7 @@ try_write_python_node_identity_fingerprint, ) from sqlbuild.executor.python_nodes._helpers.lifecycle_nodes import build_ingress_lifecycle_nodes +from sqlbuild.executor.python_nodes._helpers.node_warehouse import node_runtime from sqlbuild.executor.python_nodes.classes.ingress_results import IngressResultAccumulator from sqlbuild.executor.python_nodes.main._ready import run_ready_python_node from sqlbuild.executor.python_nodes.models import ( @@ -149,6 +150,7 @@ def _execute_ingress_lifecycle_node( results: IngressResultAccumulator, ) -> LifecycleNodeResult: discovered_node: DiscoveredPythonNode = python_graph.nodes_by_name[node.name] + runtime = node_runtime(runtime, node.name) if discovered_node.kind == PythonNodeKind.LOADER: return _execute_ingress_loader( node=discovered_node, diff --git a/src/sqlbuild/executor/python_nodes/_helpers/node_warehouse.py b/src/sqlbuild/executor/python_nodes/_helpers/node_warehouse.py new file mode 100644 index 0000000000..d5b3faeb21 --- /dev/null +++ b/src/sqlbuild/executor/python_nodes/_helpers/node_warehouse.py @@ -0,0 +1,27 @@ +"""The runtime one Python node runs with: on its job's warehouse, or on the build's.""" + +from __future__ import annotations + +from dataclasses import replace + +from sqlbuild.executor.python_nodes.models import PythonNodeRuntime +from sqlbuild.executor.scheduling.classes.session_warehouses import ( + SessionWarehouses, + with_session_warehouse, +) + + +def node_runtime(runtime: PythonNodeRuntime, name: str) -> PythonNodeRuntime: + """Put the shared connection on node `name`'s warehouse; code opening its own connections + from `connection_config` gets that warehouse too.""" + + session_warehouses: SessionWarehouses | None = runtime.session_warehouses + if session_warehouses is None: + return runtime + warehouse: str | None = runtime.node_warehouses.get(name) + session_warehouses.use(connection=runtime.connection, warehouse=warehouse) + if warehouse is None: + return runtime + return replace( + runtime, connection_config=with_session_warehouse(runtime.connection_config, warehouse) + ) diff --git a/src/sqlbuild/executor/python_nodes/_helpers/python_checks.py b/src/sqlbuild/executor/python_nodes/_helpers/python_checks.py index 70715affd9..db051af7b2 100644 --- a/src/sqlbuild/executor/python_nodes/_helpers/python_checks.py +++ b/src/sqlbuild/executor/python_nodes/_helpers/python_checks.py @@ -24,6 +24,7 @@ from sqlbuild.executor.python_nodes._helpers.fingerprinting import ( try_write_python_node_identity_fingerprint, ) +from sqlbuild.executor.python_nodes._helpers.node_warehouse import node_runtime from sqlbuild.executor.python_nodes._helpers.relation_guard import build_python_node_relation_guard from sqlbuild.executor.python_nodes._helpers.results import normalize_python_check_return from sqlbuild.executor.python_nodes.models import ( @@ -134,6 +135,7 @@ def _execute_one_python_check( result_store: Any | None, callbacks: PythonCheckCallbacks, ) -> PythonCheckExecutionResult: + runtime = node_runtime(runtime, check_function.name) adapter: BaseAdapter = runtime.adapter providers: ProviderContainer | None = runtime.providers with ( diff --git a/src/sqlbuild/executor/python_nodes/classes/read_side_python_execution_tracker.py b/src/sqlbuild/executor/python_nodes/classes/read_side_python_execution_tracker.py index b063f8a60c..d62e65cd30 100644 --- a/src/sqlbuild/executor/python_nodes/classes/read_side_python_execution_tracker.py +++ b/src/sqlbuild/executor/python_nodes/classes/read_side_python_execution_tracker.py @@ -13,6 +13,7 @@ try_write_python_node_identity_fingerprint, ) from sqlbuild.executor.python_nodes._helpers.ingress_execution import _to_executable_python_node +from sqlbuild.executor.python_nodes._helpers.node_warehouse import node_runtime from sqlbuild.executor.python_nodes._helpers.read_side_execution import ( _sql_result_failed, _sql_result_name, @@ -151,7 +152,9 @@ def _python_dependencies_satisfied(self, node: DiscoveredPythonNode) -> bool: return True def _execute_python_node(self, node: DiscoveredPythonNode) -> None: - runtime: PythonNodeRuntime = replace(self._runtime, result_store=self._result_store) + runtime: PythonNodeRuntime = node_runtime( + replace(self._runtime, result_store=self._result_store), node.name + ) _ = run_ready_python_node( node=_to_executable_python_node(node), upstream_results=tuple( diff --git a/src/sqlbuild/executor/python_nodes/models.py b/src/sqlbuild/executor/python_nodes/models.py index 661af2edc7..4269aad80e 100644 --- a/src/sqlbuild/executor/python_nodes/models.py +++ b/src/sqlbuild/executor/python_nodes/models.py @@ -3,7 +3,7 @@ from __future__ import annotations import logging -from collections.abc import Callable +from collections.abc import Callable, Mapping from dataclasses import dataclass, field from datetime import datetime from typing import Any @@ -28,6 +28,10 @@ PYTHON_NODE_RELATION_QUALIFIER_SEPARATOR, ) from sqlbuild.executor.python_nodes.types import PythonIdentityRecorder +from sqlbuild.executor.scheduling.classes.session_warehouses import ( + SessionWarehouses, + configured_warehouse, +) from sqlbuild.provider.main.runtime import ProviderContainer, _empty_provider_container from sqlbuild.python_nodes.models import SqlResourceRef from sqlbuild.python_nodes.types import PythonCheckSeverity @@ -147,6 +151,20 @@ class PythonNodeRuntime: providers: ProviderContainer | None = None result_store: Any | None = None persist_node_results: bool = True + # The warehouse each Python node runs on when its named job sets one, by node name. + node_warehouses: Mapping[str, str] = field(default_factory=dict) + # Switches `connection` between node warehouses; set whenever `node_warehouses` is not empty. + session_warehouses: SessionWarehouses | None = None + + def __post_init__(self) -> None: + if self.node_warehouses and self.session_warehouses is None: + object.__setattr__( + self, + "session_warehouses", + SessionWarehouses( + adapter=self.adapter, home=configured_warehouse(self.connection_config) + ), + ) @property def resolved_relation_targets(self) -> dict[SqlResourceRef, str]: diff --git a/src/sqlbuild/executor/scheduling/classes/session_warehouses.py b/src/sqlbuild/executor/scheduling/classes/session_warehouses.py new file mode 100644 index 0000000000..bd76719838 --- /dev/null +++ b/src/sqlbuild/executor/scheduling/classes/session_warehouses.py @@ -0,0 +1,120 @@ +"""The warehouse each build connection runs on, switched only when a node needs another.""" + +from __future__ import annotations + +import logging +import threading +from collections.abc import Mapping +from typing import Any + +from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter +from sqlbuild.errors.contracts.exceptions import ExecutorInputError + +_CONNECTION_WAREHOUSE_KEY: str = "warehouse" +_UNSET_PREFIX: str = " str | None: + """The warehouse a connection opened with `connection_config` starts on, if it names one.""" + + value: object | None = connection_config.get(_CONNECTION_WAREHOUSE_KEY) + if not isinstance(value, str): + return None + stripped: str = value.strip() + if not stripped or stripped.startswith(_UNSET_PREFIX): + return None + return stripped + + +def with_session_warehouse( + connection_config: dict[str, object], warehouse: str | None +) -> dict[str, object]: + """`connection_config` for code that opens its own connections on `warehouse`.""" + + if warehouse is None: + return connection_config + return {**connection_config, _CONNECTION_WAREHOUSE_KEY: warehouse} + + +def _same_warehouse(first: str, second: str) -> bool: + """Unquoted warehouse names match case-insensitively, as Snowflake resolves them.""" + + if first.startswith('"') or second.startswith('"'): + return first == second + return first.upper() == second.upper() + + +class SessionWarehouses: + """Moves a connection to a node's warehouse, and back to the build's, only when it changes.""" + + def __init__(self, *, adapter: BaseAdapter, home: str | None) -> None: + self._adapter: BaseAdapter = adapter + self._home: str | None = home + self._home_known: bool = home is not None + self._lock: threading.Lock = threading.Lock() + # Connections away from the build's warehouse: id -> (connection, warehouse). Holding the + # connection keeps its id from being reused by another while it is listed. + self._away: dict[int, tuple[Any, str]] = {} + + def use(self, *, connection: Any, warehouse: str | None) -> None: + """Put `connection` on `warehouse`, or on the build's warehouse when it is None.""" + + with self._lock: + entry: tuple[Any, str] | None = self._away.get(id(connection)) + current: str | None = None if entry is None or entry[0] is not connection else entry[1] + target: str | None = warehouse + if target is not None and self._home is not None and _same_warehouse(target, self._home): + target = None + if current == target or ( + current is not None and target is not None and _same_warehouse(current, target) + ): + return + if target is None: + self._adapter.use_session_warehouse(connection=connection, warehouse=self._return_to()) + with self._lock: + self._away.pop(id(connection), None) + return + if current is None: + self._learn_home(connection) + if self._home is not None and _same_warehouse(target, self._home): + return + self._adapter.use_session_warehouse(connection=connection, warehouse=target) + with self._lock: + self._away[id(connection)] = (connection, target) + + def restore(self) -> None: + """Return every connection still on a node's warehouse to the build's warehouse.""" + + with self._lock: + away: list[tuple[Any, str]] = list(self._away.values()) + self._away.clear() + if not away or self._home is None: + return + connection: Any + for connection, _ in away: + try: + self._adapter.use_session_warehouse(connection=connection, warehouse=self._home) + except Exception as error: # noqa: BLE001 - its owner closes or replaces it next + _LOGGER.warning( + "could not return a connection to warehouse %s: %s", self._home, error + ) + + def _learn_home(self, connection: Any) -> None: + """Read the warehouse the build's connections start on before one first leaves it.""" + + with self._lock: + if self._home_known: + return + self._home = self._adapter.current_session_warehouse(connection=connection) + self._home_known = True + + def _return_to(self) -> str: + if self._home is None: + raise ExecutorInputError( + "cannot return a connection to the build's warehouse: the target sets no " + "warehouse and the session had none; set the target's warehouse to build nodes " + "outside the named jobs beside them" + ) + return self._home diff --git a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py index f093039160..a0b2efbf62 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py @@ -1518,3 +1518,23 @@ class JobOutputTestCase: expected_present: tuple[str, ...] expected_absent: tuple[str, ...] extra_files: dict[str, str] = field(default_factory=dict) + + +@dataclass(frozen=True) +class JobWarehouseSwitchTestCase: + """A build naming jobs, and the warehouses its connections switch to after connecting.""" + + description: str + argv: tuple[str, ...] + expected_connect_warehouses: frozenset[str] + expected_switches: tuple[str, ...] + + +@dataclass(frozen=True) +class JobWarehouseOutputTestCase: + """A command naming jobs with warehouses, its exit code and what its output says.""" + + description: str + argv: tuple[str, ...] + expected_exit_code: int + expected_fragment: str diff --git a/tests/integration/src/sqlbuild/cli/commands/main/helpers.py b/tests/integration/src/sqlbuild/cli/commands/main/helpers.py index 950501f5e8..f976e87214 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/helpers.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/helpers.py @@ -1479,3 +1479,88 @@ def peak_concurrency(*, events_path: Path, names: tuple[str, ...]) -> int: + [(end, -1) for end, _ in ends] ) return max(accumulate(change for _, change in changes), default=0) + + +_CONNECT_MARKER: str = "-- connect" + + +def _job_warehouse_model(dep: str | None) -> str: + source: str = "" if dep is None else f' FROM __ref("{dep}")' + return f"MODEL (description 'Step.', materialized table);\nSELECT 1 AS id{source}\n" + + +def _job_file(name: str, select: str, warehouse: str) -> str: + return ( + f'name = "{name}"\ndescription = "Runs {select}."\nselect = "{select}"\n\n' + f'[settings]\nwarehouse = "{warehouse}"\n' + ) + + +def write_job_warehouse_project(*, project_dir: Path) -> None: + """A Snowflake chain stg -> orders -> orders_daily -> refunds -> summary and a task, with jobs + on their own warehouses: orders_wh_job, refunds_wh_job, refunds_alt_job (also refunds), + tasks_wh_job and bad_wh_job, whose warehouse is not an identifier.""" + + write_snowflake_warehouse_project( + project_dir=project_dir, + warehouses_section='[targets.dev.warehouses]\nbuild = "BUILD_WH"\n', + local_config="", + ) + write_project_files( + project_dir=project_dir, + files={ + "models/stg.sql": _job_warehouse_model(None), + "models/orders.sql": _job_warehouse_model("stg"), + "models/orders_daily.sql": _job_warehouse_model("orders"), + "models/refunds.sql": _job_warehouse_model("orders_daily"), + "models/summary.sql": _job_warehouse_model("refunds"), + "python/tasks.py": ( + "from sqlbuild.tasks import task\n\n\n@task\ndef refresh_marker(ctx):\n" + ' """Refresh."""\n ctx.execute_sql("SELECT \'task marker\'")\n' + ), + "orchestration/jobs/orders_wh_job.toml": _job_file( + "orders_wh_job", "orders orders_daily", "ORDERS_WH" + ), + "orchestration/jobs/refunds_wh_job.toml": _job_file( + "refunds_wh_job", "refunds", "REFUNDS_WH" + ), + "orchestration/jobs/refunds_alt_job.toml": _job_file( + "refunds_alt_job", "refunds", "ALT_WH" + ), + "orchestration/jobs/tasks_wh_job.toml": _job_file( + "tasks_wh_job", "refresh_marker", "TASKS_WH" + ), + "orchestration/jobs/bad_wh_job.toml": _job_file("bad_wh_job", "stg", "bad wh"), + }, + ) + + +def install_writable_offline_snowflake( + monkeypatch: pytest.MonkeyPatch, +) -> tuple[list[dict[str, object]], list[str]]: + """The offline connector, accepting writes, with a marker before each connection's setup.""" + + connect_calls, executed_sql = install_offline_snowflake_connector(monkeypatch) + monkeypatch.setattr( + sys.modules[__name__], "_refuse_offline_snowflake_write", _accept_offline_snowflake_read + ) + connector: ModuleType = sys.modules["snowflake.connector"] + connect: Callable[..., object] = connector.__dict__["connect"] + + def marked_connect(**kwargs: object) -> object: + executed_sql.append(_CONNECT_MARKER) + return connect(**kwargs) + + monkeypatch.setitem(connector.__dict__, "connect", marked_connect) + return connect_calls, executed_sql + + +def warehouse_switches(executed_sql: list[str]) -> tuple[str, ...]: + """The warehouses statements switch to, leaving out each new connection's own setup.""" + + prefix: str = "USE WAREHOUSE " + return tuple( + sql.removeprefix(prefix) + for previous, sql in zip([_CONNECT_MARKER, *executed_sql], executed_sql, strict=False) + if sql.startswith(prefix) and previous != _CONNECT_MARKER + ) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_job_settings.py b/tests/integration/src/sqlbuild/cli/commands/main/test_job_settings.py index 7d21c2e329..be8cc81840 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_job_settings.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_job_settings.py @@ -91,6 +91,10 @@ ) for name in ("a", "b", "c", "d") }, + "orchestration/jobs/warehouse_job.toml": ( + 'name = "warehouse_job"\ndescription = "On a warehouse."\nselect = "customers"\n\n' + '[settings]\nwarehouse = "ORDERS_WH"\n' + ), "orchestration/jobs/wide_job.toml": ( 'name = "wide_job"\ndescription = "Wide."\nselect = "customers"\n\n' '[settings]\nconcurrency = 6\nvars = { amount = "5" }\n' @@ -390,6 +394,23 @@ def test_given_a_job_concurrency_when_building_then_its_nodes_never_exceed_it( expected_present=("(concurrency: 2)",), expected_absent=(), ), + JobOutputTestCase( + description="a job warehouse on an adapter without one fails, naming the job", + command_args=("build", "--select", "job:warehouse_job"), + expected_exit_code=1, + expected_present=( + "error[C260]: job warehouse_job sets warehouse ORDERS_WH, but the 'duckdb' " + "adapter has no warehouse to select", + ), + expected_absent=(), + ), + JobOutputTestCase( + description="--no-job-settings leaves a job warehouse out on such an adapter", + command_args=("build", "--no-job-settings", "--select", "job:warehouse_job"), + expected_exit_code=0, + expected_present=(), + expected_absent=("C260",), + ), JobOutputTestCase( description="a plan shows the job's full refresh", command_args=("plan", "--select", "job:events_job"), diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_job_warehouses.py b/tests/integration/src/sqlbuild/cli/commands/main/test_job_warehouses.py new file mode 100644 index 0000000000..f94080addb --- /dev/null +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_job_warehouses.py @@ -0,0 +1,160 @@ +"""A build naming jobs runs each job's nodes on the job's warehouse, switching only on change.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from sqlbuild._native import run_cli as main +from tests.integration.src.sqlbuild.cli.commands.main._test_types import ( + JobWarehouseOutputTestCase, + JobWarehouseSwitchTestCase, +) +from tests.integration.src.sqlbuild.cli.commands.main.helpers import ( + install_writable_offline_snowflake, + warehouse_switches, + write_job_warehouse_project, +) + +_BOTH_JOBS: tuple[str, ...] = ( + "--select", + "stg", + "job:orders_wh_job", + "job:refunds_wh_job", + "summary", +) + + +def _run(argv: tuple[str, ...], project_dir: Path) -> int: + try: + return int(main(["--no-color", "--project-dir", str(project_dir), *argv]) or 0) + except SystemExit as exited: + return int(exited.code or 0) + + +@pytest.mark.parametrize( + "test_case", + [ + JobWarehouseSwitchTestCase( + description="two jobs' nodes run on their own warehouses, switching only on change", + argv=("build", *_BOTH_JOBS), + expected_connect_warehouses=frozenset({"BUILD_WH"}), + expected_switches=("ORDERS_WH", "REFUNDS_WH", "BUILD_WH"), + ), + JobWarehouseSwitchTestCase( + description="a job's task runs on the job's warehouse", + argv=("build", "--select", "job:tasks_wh_job"), + expected_connect_warehouses=frozenset({"BUILD_WH"}), + expected_switches=("TASKS_WH",), + ), + JobWarehouseSwitchTestCase( + description="--warehouse wins over every named job, even ones that would conflict", + argv=("build", "--warehouse", "CLI_WH", *_BOTH_JOBS, "job:refunds_alt_job"), + expected_connect_warehouses=frozenset({"CLI_WH"}), + expected_switches=(), + ), + JobWarehouseSwitchTestCase( + description="--no-job-settings keeps every node on the build's warehouse", + argv=("build", "--no-job-settings", *_BOTH_JOBS), + expected_connect_warehouses=frozenset({"BUILD_WH"}), + expected_switches=(), + ), + JobWarehouseSwitchTestCase( + description="nodes selected without naming their job stay on the build's warehouse", + argv=("build", "--select", "+refunds"), + expected_connect_warehouses=frozenset({"BUILD_WH"}), + expected_switches=(), + ), + ], + ids=lambda case: case.description, +) +def test_given_named_jobs_with_warehouses_when_building_then_each_node_runs_on_its_jobs( + test_case: JobWarehouseSwitchTestCase, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + write_job_warehouse_project(project_dir=tmp_path) + connect_calls, executed_sql = install_writable_offline_snowflake(monkeypatch) + + exit_code: int = _run(test_case.argv, tmp_path) + + captured = capsys.readouterr() + assert ( + exit_code, + {call.get("warehouse") for call in connect_calls}, + warehouse_switches(executed_sql), + ) == ( + 0, + test_case.expected_connect_warehouses, + test_case.expected_switches, + ), captured.out + captured.err + + +@pytest.mark.parametrize( + "test_case", + [ + JobWarehouseOutputTestCase( + description="a plan names each job's warehouse", + argv=("plan", *_BOTH_JOBS), + expected_exit_code=0, + expected_fragment="Nodes of job orders_wh_job run on warehouse ORDERS_WH", + ), + JobWarehouseOutputTestCase( + description="plan JSON maps each job to its warehouse", + argv=("plan", "--json", *_BOTH_JOBS), + expected_exit_code=0, + expected_fragment=( + '"job_warehouses": {\n "orders_wh_job": "ORDERS_WH",\n' + ' "refunds_wh_job": "REFUNDS_WH"\n }' + ), + ), + JobWarehouseOutputTestCase( + description="the run context shows the jobs' warehouses beside the build's", + argv=("build", "--verbose", *_BOTH_JOBS), + expected_exit_code=0, + expected_fragment=( + "warehouse BUILD_WH (sqlbuild_project.toml [targets.dev.warehouses] build); " + "nodes of job orders_wh_job on ORDERS_WH; nodes of job refunds_wh_job on " + "REFUNDS_WH" + ), + ), + JobWarehouseOutputTestCase( + description="two jobs giving one node different warehouses conflict", + argv=("build", "--select", "job:refunds_wh_job", "job:refunds_alt_job"), + expected_exit_code=1, + expected_fragment=( + "error[S027]: model:refunds is selected through jobs refunds_wh_job and " + 'refunds_alt_job, which set warehouse differently ("REFUNDS_WH" and "ALT_WH")' + ), + ), + JobWarehouseOutputTestCase( + description="a job warehouse that is not an identifier fails before building", + argv=("plan", "--select", "job:bad_wh_job"), + expected_exit_code=1, + expected_fragment=( + "error[C261]: job bad_wh_job sets warehouse 'bad wh', which is not a valid " + "Snowflake warehouse identifier" + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_named_jobs_with_warehouses_when_planning_or_building_then_output_names_them( + test_case: JobWarehouseOutputTestCase, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + write_job_warehouse_project(project_dir=tmp_path) + install_writable_offline_snowflake(monkeypatch) + + exit_code: int = _run(test_case.argv, tmp_path) + + captured = capsys.readouterr() + output: str = captured.out + captured.err + assert (exit_code, test_case.expected_fragment in output) == ( + test_case.expected_exit_code, + True, + ), output diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_adhoc.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_adhoc.py index f761f6232b..f13415759f 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_adhoc.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_adhoc.py @@ -60,11 +60,13 @@ expected_reason=WAREHOUSE_REFUSAL, expected_statements=False, expected_customers_tables=0, - # DuckDB has no session warehouse, so the CLI refuses the job's warehouse: the - # published job reached the CLI instead of being refused by the worker. + # DuckDB has no session warehouse, so the build refuses the job's warehouse: the + # published job reached the build instead of being refused by the worker. job_settings='\n[settings]\nwarehouse = "ANALYST_XS"\n', expected_scheduled_status="failed", - expected_worker_log="error[C260]: --warehouse ANALYST_XS was given", + expected_worker_log=( + "sets warehouse ANALYST_XS, but the 'duckdb' adapter has no warehouse to select" + ), ), ], ids=lambda case: case.description, diff --git a/tests/unit/src/sqlbuild/executor/build/classes/_test_types.py b/tests/unit/src/sqlbuild/executor/build/classes/_test_types.py index 111920de02..60748f763a 100644 --- a/tests/unit/src/sqlbuild/executor/build/classes/_test_types.py +++ b/tests/unit/src/sqlbuild/executor/build/classes/_test_types.py @@ -68,3 +68,21 @@ class SubworkerReservationTestCase: max_concurrency: int expected_demand: int expected_reservations: tuple[bool, ...] + + +@dataclass(frozen=True) +class SubworkerWarehouseTestCase: + """A microbatch sub-worker taking a pooled connection a previous node may have moved.""" + + description: str + coordinator_warehouse: str | None + earlier_warehouse: str | None + expected_statements: tuple[str, ...] + + +@dataclass(frozen=True) +class NodeWarehouseFailureTestCase: + description: str + node_warehouse: str | None + failing: str + expected_message: str diff --git a/tests/unit/src/sqlbuild/executor/build/classes/helpers.py b/tests/unit/src/sqlbuild/executor/build/classes/helpers.py index 7815de990b..8b86a8a3af 100644 --- a/tests/unit/src/sqlbuild/executor/build/classes/helpers.py +++ b/tests/unit/src/sqlbuild/executor/build/classes/helpers.py @@ -47,6 +47,8 @@ def _bare_scheduler( scheduler._microbatch_coordinator_demand = 0 scheduler._microbatch_subworkers = 0 scheduler._completion_queue = Queue() + scheduler._session_warehouses = None + scheduler._node_warehouses = {} scheduler._init_job_limits( BuildRuntimeParams( run_id="run", diff --git a/tests/unit/src/sqlbuild/executor/build/classes/test_build_scheduler_microbatch.py b/tests/unit/src/sqlbuild/executor/build/classes/test_build_scheduler_microbatch.py index 32d78f7cb2..130c580f92 100644 --- a/tests/unit/src/sqlbuild/executor/build/classes/test_build_scheduler_microbatch.py +++ b/tests/unit/src/sqlbuild/executor/build/classes/test_build_scheduler_microbatch.py @@ -81,6 +81,7 @@ def test_given_large_microbatch_phase_when_workers_are_blocked_then_future_submi scheduler._microbatch_coordinator_demand = 1 scheduler._microbatch_subworkers = 0 scheduler._connection_pool = Queue() + scheduler._session_warehouses = None for index in range(test_case.expected_max_subworker_futures): scheduler._connection_pool.put(f"worker-{index}") @@ -173,6 +174,7 @@ def test_given_coordinator_failure_when_subworkers_are_running_then_pending_batc scheduler._microbatch_coordinator_demand = 1 scheduler._microbatch_subworkers = 0 scheduler._connection_pool = Queue() + scheduler._session_warehouses = None for index in range(test_case.expected_max_subworker_futures): scheduler._connection_pool.put(f"worker-{index}") diff --git a/tests/unit/src/sqlbuild/executor/build/classes/test_build_scheduler_warehouses.py b/tests/unit/src/sqlbuild/executor/build/classes/test_build_scheduler_warehouses.py new file mode 100644 index 0000000000..c617ff3eaa --- /dev/null +++ b/tests/unit/src/sqlbuild/executor/build/classes/test_build_scheduler_warehouses.py @@ -0,0 +1,126 @@ +"""Build connections follow each node's job warehouse, microbatch sub-workers included.""" + +from __future__ import annotations + +import threading +from queue import Queue +from typing import Any, cast + +import pytest + +from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter +from sqlbuild.executor.build.classes.build_scheduler import BuildScheduler +from sqlbuild.executor.run.models import BatchWindow, ModelExecutionResult +from sqlbuild.executor.scheduling.classes.session_warehouses import SessionWarehouses +from sqlbuild.executor.scheduling.types import ExecutionStatus +from tests.unit.src.sqlbuild.executor.build.classes._test_types import ( + NodeWarehouseFailureTestCase, + SubworkerWarehouseTestCase, +) +from tests.unit.src.sqlbuild.executor.build.classes.helpers import model_key +from tests.unit.src.sqlbuild.executor.scheduling.classes.helpers import ( + RecordingWarehouseAdapter, +) + + +def _scheduler(adapter: RecordingWarehouseAdapter) -> Any: + scheduler: Any = object.__new__(BuildScheduler) + scheduler._session_warehouses = SessionWarehouses( + adapter=cast(BaseAdapter, adapter), home="BUILD_WH" + ) + scheduler._node_warehouses = {} + scheduler._microbatch_coordinator_lock = threading.Lock() + scheduler._job_lock = threading.Lock() + scheduler._job_limits = {} + scheduler._job_running = {} + scheduler._microbatch_subworkers = 1 + scheduler._connection_pool = Queue() + return scheduler + + +@pytest.mark.parametrize( + "test_case", + [ + SubworkerWarehouseTestCase( + description="a sub-worker moves its connection to the coordinator's job warehouse", + coordinator_warehouse="ORDERS_WH", + earlier_warehouse=None, + expected_statements=("ORDERS_WH",), + ), + SubworkerWarehouseTestCase( + description="a plain model's sub-worker returns a connection a job left elsewhere", + coordinator_warehouse=None, + earlier_warehouse="REFUNDS_WH", + expected_statements=("REFUNDS_WH", "BUILD_WH"), + ), + SubworkerWarehouseTestCase( + description="a connection already on the warehouse issues nothing more", + coordinator_warehouse="ORDERS_WH", + earlier_warehouse="ORDERS_WH", + expected_statements=("ORDERS_WH",), + ), + ], + ids=lambda case: case.description, +) +def test_given_a_coordinator_warehouse_when_a_subworker_runs_a_batch_then_its_connection_follows( + test_case: SubworkerWarehouseTestCase, +) -> None: + adapter: RecordingWarehouseAdapter = RecordingWarehouseAdapter(session_warehouse=None) + scheduler: Any = _scheduler(adapter) + pooled: object = object() + scheduler._session_warehouses.use(connection=pooled, warehouse=test_case.earlier_warehouse) + scheduler._connection_pool.put(pooled) + executed_on: list[object] = [] + + def execute(_batch: BatchWindow, connection: object) -> Any: + executed_on.append(connection) + return "done" + + outcome: Any = BuildScheduler._run_microbatch_batch_worker( + scheduler, + jobs=(), + warehouse=test_case.coordinator_warehouse, + batch=BatchWindow(start="0", end="1", index=0), + execute=execute, + ) + + assert ( + outcome, + executed_on == [pooled], + tuple(warehouse for _, warehouse in adapter.statements), + scheduler._connection_pool.get_nowait() is pooled, + ) == ("done", True, test_case.expected_statements, True) + + +@pytest.mark.parametrize( + "test_case", + [ + NodeWarehouseFailureTestCase( + description="a job warehouse the session cannot use fails the node with the reason", + node_warehouse="MISSING_WH", + failing="MISSING_WH", + expected_message=( + "cannot run on warehouse MISSING_WH: Object does not exist: warehouse MISSING_WH" + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_an_unusable_job_warehouse_when_a_node_starts_then_it_fails_naming_the_warehouse( + test_case: NodeWarehouseFailureTestCase, +) -> None: + adapter: RecordingWarehouseAdapter = RecordingWarehouseAdapter( + session_warehouse=None, failing=test_case.failing + ) + scheduler: Any = _scheduler(adapter) + scheduler._node_warehouses = {model_key("orders"): test_case.node_warehouse} + + result: ModelExecutionResult | None = BuildScheduler._enter_node_warehouse( + scheduler, key=model_key("orders"), connection=object() + ) + + assert result is not None + assert (result.status, result.error_message) == ( + ExecutionStatus.FAILED, + test_case.expected_message, + ) diff --git a/tests/unit/src/sqlbuild/executor/scheduling/classes/_test_types.py b/tests/unit/src/sqlbuild/executor/scheduling/classes/_test_types.py index 8125446930..77a9a5c7b9 100644 --- a/tests/unit/src/sqlbuild/executor/scheduling/classes/_test_types.py +++ b/tests/unit/src/sqlbuild/executor/scheduling/classes/_test_types.py @@ -10,3 +10,17 @@ class UnreplaceableConnectionTestCase: expected_completions: tuple[tuple[str, str], ...] expected_reconnect_attempts: int expected_maximum_seconds: float + + +@dataclass(frozen=True) +class SessionWarehousesTestCase: + """Nodes taking connections in turn, each on its job's warehouse or the build's (None).""" + + description: str + configured: str | None + session_warehouse: str | None + steps: tuple[tuple[str, str | None], ...] + restore: bool + expected_statements: tuple[tuple[str, str], ...] + expected_session_reads: int + expected_error: str | None = None diff --git a/tests/unit/src/sqlbuild/executor/scheduling/classes/helpers.py b/tests/unit/src/sqlbuild/executor/scheduling/classes/helpers.py index e18022c9d4..a08182fd81 100644 --- a/tests/unit/src/sqlbuild/executor/scheduling/classes/helpers.py +++ b/tests/unit/src/sqlbuild/executor/scheduling/classes/helpers.py @@ -54,3 +54,23 @@ def success_completion(*, key: str, result: str) -> tuple[str, str]: def failure_completion(*, key: str, error: BaseException) -> tuple[str, str]: return key, f"failed: {error}" + + +class RecordingWarehouseAdapter: + """Records each warehouse switch; reports `session_warehouse` as the session's warehouse.""" + + def __init__(self, *, session_warehouse: str | None, failing: str | None = None) -> None: + self.session_warehouse: str | None = session_warehouse + self.failing: str | None = failing + self.statements: list[tuple[Any, str]] = [] + self.session_reads: int = 0 + + def use_session_warehouse(self, *, connection: Any, warehouse: str) -> None: + if warehouse == self.failing: + raise RuntimeError(f"Object does not exist: warehouse {warehouse}") + self.statements.append((connection, warehouse)) + + def current_session_warehouse(self, *, connection: Any) -> str | None: + del connection + self.session_reads += 1 + return self.session_warehouse diff --git a/tests/unit/src/sqlbuild/executor/scheduling/classes/test_session_warehouses.py b/tests/unit/src/sqlbuild/executor/scheduling/classes/test_session_warehouses.py new file mode 100644 index 0000000000..5b5f200ca4 --- /dev/null +++ b/tests/unit/src/sqlbuild/executor/scheduling/classes/test_session_warehouses.py @@ -0,0 +1,130 @@ +"""Build connections move to a node's warehouse and back only when the warehouse changes.""" + +from __future__ import annotations + +from typing import Any, cast + +import pytest + +from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter +from sqlbuild.errors.contracts.exceptions import ExecutorInputError +from sqlbuild.executor.scheduling.classes.session_warehouses import SessionWarehouses +from tests.unit.src.sqlbuild.executor.scheduling.classes._test_types import ( + SessionWarehousesTestCase, +) +from tests.unit.src.sqlbuild.executor.scheduling.classes.helpers import ( + RecordingWarehouseAdapter, +) + + +@pytest.mark.parametrize( + "test_case", + [ + SessionWarehousesTestCase( + description="nodes on the build's warehouse issue nothing, however it is spelled", + configured="BUILD_WH", + session_warehouse=None, + steps=(("a", None), ("a", "BUILD_WH"), ("a", "build_wh")), + restore=True, + expected_statements=(), + expected_session_reads=0, + ), + SessionWarehousesTestCase( + description="a job's nodes switch once, the next job's again, a plain node back", + configured="BUILD_WH", + session_warehouse=None, + steps=(("a", "ORDERS_WH"), ("a", "ORDERS_WH"), ("a", "REFUNDS_WH"), ("a", None)), + restore=True, + expected_statements=(("a", "ORDERS_WH"), ("a", "REFUNDS_WH"), ("a", "BUILD_WH")), + expected_session_reads=0, + ), + SessionWarehousesTestCase( + description="each connection keeps its own warehouse", + configured="BUILD_WH", + session_warehouse=None, + steps=(("a", "ORDERS_WH"), ("b", "ORDERS_WH"), ("b", None), ("a", "ORDERS_WH")), + restore=False, + expected_statements=(("a", "ORDERS_WH"), ("b", "ORDERS_WH"), ("b", "BUILD_WH")), + expected_session_reads=0, + ), + SessionWarehousesTestCase( + description="restoring returns every connection still on a job's warehouse", + configured="BUILD_WH", + session_warehouse=None, + steps=(("a", "ORDERS_WH"), ("b", "REFUNDS_WH")), + restore=True, + expected_statements=( + ("a", "ORDERS_WH"), + ("b", "REFUNDS_WH"), + ("a", "BUILD_WH"), + ("b", "BUILD_WH"), + ), + expected_session_reads=0, + ), + SessionWarehousesTestCase( + description="without a configured warehouse the session's is read once, then used", + configured=None, + session_warehouse="USER_WH", + steps=(("a", "ORDERS_WH"), ("b", "ORDERS_WH"), ("a", None)), + restore=True, + expected_statements=( + ("a", "ORDERS_WH"), + ("b", "ORDERS_WH"), + ("a", "USER_WH"), + ("b", "USER_WH"), + ), + expected_session_reads=1, + ), + SessionWarehousesTestCase( + description="a job on the session's own warehouse issues nothing", + configured=None, + session_warehouse="ORDERS_WH", + steps=(("a", "ORDERS_WH"), ("a", None)), + restore=True, + expected_statements=(), + expected_session_reads=1, + ), + SessionWarehousesTestCase( + description="a session without any warehouse cannot be returned to one", + configured=None, + session_warehouse=None, + steps=(("a", "ORDERS_WH"), ("a", None)), + restore=False, + expected_statements=(("a", "ORDERS_WH"),), + expected_session_reads=1, + expected_error="cannot return a connection to the build's warehouse", + ), + ], + ids=lambda case: case.description, +) +def test_given_nodes_on_job_warehouses_when_taking_connections_then_switches_only_on_change( + test_case: SessionWarehousesTestCase, +) -> None: + adapter: RecordingWarehouseAdapter = RecordingWarehouseAdapter( + session_warehouse=test_case.session_warehouse + ) + warehouses: SessionWarehouses = SessionWarehouses( + adapter=cast(BaseAdapter, adapter), home=test_case.configured + ) + connections: dict[str, Any] = {"a": object(), "b": object()} + error: str | None = None + try: + for name, warehouse in test_case.steps: + warehouses.use(connection=connections[name], warehouse=warehouse) + if test_case.restore: + warehouses.restore() + except ExecutorInputError as raised: + error = str(raised) + names: dict[int, str] = {id(connection): name for name, connection in connections.items()} + + assert ( + tuple((names[id(connection)], warehouse) for connection, warehouse in adapter.statements), + adapter.session_reads, + error is not None + and test_case.expected_error is not None + and test_case.expected_error in error, + ) == ( + test_case.expected_statements, + test_case.expected_session_reads, + test_case.expected_error is not None, + ) diff --git a/website/src/content/docs/docs/concepts/adapters/snowflake.mdx b/website/src/content/docs/docs/concepts/adapters/snowflake.mdx index 29194563a5..bca7e8a64d 100644 --- a/website/src/content/docs/docs/concepts/adapters/snowflake.mdx +++ b/website/src/content/docs/docs/concepts/adapters/snowflake.mdx @@ -74,9 +74,10 @@ query = "ADHOC_WH" Each command uses the first of these that is set: 1. `--warehouse `, for that run only -2. The target's warehouse for the command's group -3. The connection's `warehouse` -4. The Snowflake user's default warehouse +2. For `sqb build`, the `warehouse` of a named job, for that job's nodes only; see below +3. The target's warehouse for the command's group +4. The connection's `warehouse` +5. The Snowflake user's default warehouse A location's syncs use the target's `sync` warehouse, else its `query` warehouse, else the connection's `warehouse`, so a sync never wakes the target's `build` warehouse just to read @@ -92,10 +93,39 @@ sync = "TINY_WH" Runs that a location executes, such as `sqb build`, use their own command's group. -Every connection a command opens uses the same warehouse. Tests and audits that run inside +Every connection a command opens starts on the same warehouse. Tests and audits that run inside `sqb build` use the `build` warehouse. Under `sqb dbt`, dbt's own nodes use the warehouse in their dbt profile. +### A job's own warehouse + +A job can run its nodes on a warehouse of its own: + +```toml +# orchestration/jobs/nightly_orders_job.toml +[settings] +warehouse = "ORDERS_WH" +``` + +When a build names the job, as `sqb build --select job:nightly_orders_job` or a server run of it, +each of the job's nodes runs on `ORDERS_WH`. Its models' audits, its SQL tests, microbatch batches, +source loads and Python nodes go with them, and loaders and Python code that open their own +connections from the context's `connection_config` get `ORDERS_WH` there too. A SQL test runs on +its models' job warehouse when they agree on one. Everything else in the build, including nodes +selected without naming a job and the build's own bookkeeping, stays on the `build` warehouse. + +The connection running a node switches with `USE WAREHOUSE` only when the node needs a different +warehouse than the connection is on, and switches back before it runs a node outside the job. +Connections return to the `build` warehouse when the build ends. Without a configured warehouse, +SQLBuild reads the session's warehouse once before the first switch, so it can switch back. + +`--warehouse` puts the whole build on one warehouse and leaves the jobs' warehouses out. Two named +jobs that give one node different warehouses fail with `S027`; see +[Settings when a build names a job](/docs/concepts/orchestration#settings-when-a-build-names-a-job). +A job's warehouse must be a Snowflake identifier like a group warehouse (`C261`), and it needs the +Snowflake adapter or one built on it (`C260`). Cost estimates read each query's warehouse, so a +build's cost names every warehouse it used. + A group warehouse wins over a connection `warehouse` from either file. To change one for yourself, set the group in `sqlbuild_local.toml`; the other group keeps its project value: diff --git a/website/src/content/docs/docs/concepts/orchestration.mdx b/website/src/content/docs/docs/concepts/orchestration.mdx index cf80f99ad7..70d976b395 100644 --- a/website/src/content/docs/docs/concepts/orchestration.mdx +++ b/website/src/content/docs/docs/concepts/orchestration.mdx @@ -96,7 +96,7 @@ Naming a group names each of its jobs. The first value set wins: 4. The project's setting A command-line flag replaces the job's setting rather than merging with it: `--vars` replaces the -job's whole `vars`. An option that rules another out counts as setting both, so `--reload` wins +job's whole `vars`, and `--warehouse` puts every node on its warehouse. An option that rules another out counts as setting both, so `--reload` wins over a job's `load_sources = false`. Any `--concurrency` sets the build's limit and turns off the per-job limits. The job's settings are added before a `--` on the command line, so selectors after it still name jobs. `--no-job-settings` leaves out every job's settings. @@ -109,6 +109,7 @@ nodes differently: | `full_refresh` | The job's incremental models, rebuilt from scratch | | `run_tests` | The SQL unit tests of the job's models | | `ignore_audit_failures` | The audits of the job's nodes | +| `warehouse` | The Snowflake warehouse the job's nodes run on, Python nodes included; see [A job's own warehouse](/docs/concepts/adapters/snowflake#a-jobs-own-warehouse) | | `concurrency` | How many of the job's nodes build at once, and a node in two named jobs fits within both. The build's own limit becomes the larger of the configured limit and the named jobs' limits added together | | `[retry]` | The job's nodes | From bfb232c9dcd63b4b3dd9b7f01a92c7b24f83ef98 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 19 Nov 2026 02:47:46 +0000 Subject: [PATCH 09/22] refactor(jobs): move named-job settings into the project and compile crates (CHI-1533) The bindings crate was above its code-line cap and its helper layers imported upward, so the job settings now live with what they build on: - The job catalog and its entries move from compiled_project into a job_settings area of the project crate, with the build-wide settings, the conflict and selector errors, the argv a build naming jobs parses again, and a frozen run's flags and selectors. - The per-node settings a plan takes from its named jobs, the build's job-raised concurrency and connection count, and the run context's job facts move into compile_driver of the compile crate; the command-line --warehouse reaches it as a flag. - The Python node runtime keeps each node's warehouse as a Rust map, PythonAwareSelection derives Debug, TokenNodes moves to types.rs, and the warehouse tracker, adapter hook and tests follow the keyword-only, one-line-docstring and branch-free test rules. No behaviour changes. --- .../src/job_settings/_helpers/selection.rs | 3 +- .../job_settings/main/node_job_settings.rs | 3 +- .../sqlbuild-analysis/src/job_settings/mod.rs | 1 + .../src/job_settings/models.rs | 6 +- .../tests/test_node_job_settings.rs | 40 ++++--- .../src/job_settings/tests/test_types.rs | 4 +- .../src/job_settings/types.rs | 9 ++ .../tests/test_command_connection.rs | 28 +++-- .../command_connection/tests/test_types.rs | 3 +- .../_helpers/jobs/job_limits.rs | 18 +++ .../_helpers/jobs}/job_node_settings.rs | 58 +++++----- .../src/compile_driver/_helpers/jobs/mod.rs | 2 + .../src/compile_driver/_helpers/mod.rs | 1 + .../main/jobs/build_concurrency.rs | 20 ++++ .../main/jobs/connection_limit.rs | 29 +++++ .../main/jobs/job_override_fields.rs | 20 ++++ .../main/jobs/job_plan_fields.rs | 34 ++++++ .../main/jobs/job_run_context.rs | 34 ++++++ .../src/compile_driver/main/jobs/mod.rs | 6 + .../main/jobs/planned_job_settings.rs | 15 +++ .../src/compile_driver/main/mod.rs | 1 + .../src/compile_driver/models.rs | 14 +++ .../src/compile_driver/types.rs | 5 + .../src/consumption/main/changed_policy.rs | 2 +- .../src/plan_output/_helpers/plan_text.rs | 2 +- .../src/plan_output/main/job_warehouses.rs | 10 -- .../src/plan_output/main/mod.rs | 1 - .../src/compiled_project/_helpers/mod.rs | 1 - .../src/compiled_project/main/mod.rs | 10 -- .../src/compiled_project/mod.rs | 2 +- .../_helpers/python_nodes/python_selection.rs | 2 +- .../main/python_nodes/resolve_node_sets.rs | 2 +- .../src/graph/models.rs | 4 +- .../src/job_settings/_helpers/argv.rs} | 37 ++----- .../job_settings/_helpers/could_name_job.rs | 17 +++ .../_helpers/job_catalog.rs | 0 .../job_settings/_helpers/job_run_settings.rs | 32 ++++++ .../src/job_settings/_helpers/mod.rs | 5 + .../_helpers/named_job_build_settings.rs | 34 ++++++ .../src/job_settings/constants.rs | 17 +++ .../main/declared_jobs.rs | 2 +- .../src/job_settings/main/frozen_run_flags.rs | 49 +++++++++ .../src/job_settings/main/frozen_selectors.rs | 35 ++++++ .../main/graph_with_jobs.rs | 2 +- .../main/job_declarations.rs | 2 +- .../main/job_names.rs | 2 +- .../main/job_scope_project_dir.rs | 2 +- .../job_settings/main/job_selector_error.rs | 25 +++++ .../job_settings/main/job_setting_conflict.rs | 18 +++ .../job_settings/main/job_settings_argv.rs | 24 ++++ .../src/job_settings/main/mod.rs | 15 +++ .../main/named_jobs.rs | 4 +- .../main/needed_jobs.rs | 2 +- .../main/triggered_jobs.rs | 4 +- .../main/with_frozen_job.rs | 2 +- .../main/with_job_scope.rs | 2 +- .../src/job_settings/mod.rs | 6 + .../src/job_settings/models.rs | 13 +++ crates/sqlbuild-python-project/src/lib.rs | 1 + .../src/native_host/_helpers/runs_state.rs | 2 +- .../planner_selection/_helpers/unit_tests.rs | 2 +- .../main/match_project_selector.rs | 2 +- .../main/resolve_graph_keys.rs | 2 +- .../main/resolve_project_selector_tokens.rs | 2 +- .../_helpers/build_command/execution.rs | 75 ++----------- .../_helpers/build_command/lifecycle.rs | 11 +- .../src/bindings/_helpers/cli/dispatch.rs | 2 +- .../src/bindings/_helpers/cli/entry.rs | 2 +- .../src/bindings/_helpers/cli/mod.rs | 1 - .../bindings/_helpers/compile_driver/mod.rs | 1 - .../_helpers/compile_driver/plan_result.rs | 37 ++----- .../_helpers/compiled_project/job_settings.rs | 104 ------------------ .../bindings/_helpers/compiled_project/mod.rs | 1 - .../_helpers/location_command/constants.rs | 2 - .../_helpers/location_command/worker.rs | 71 +++--------- .../adapter/contract/classes/base_adapter.py | 4 +- .../snowflake/classes/snowflake_adapter.py | 1 - .../executor/build/classes/build_scheduler.py | 22 ++-- src/sqlbuild/executor/build/models.py | 1 - .../_helpers/ingress_execution.py | 2 +- .../python_nodes/_helpers/node_warehouse.py | 10 +- .../python_nodes/_helpers/python_checks.py | 2 +- .../read_side_python_execution_tracker.py | 2 +- src/sqlbuild/executor/python_nodes/models.py | 5 +- .../scheduling/classes/session_warehouses.py | 20 ++-- .../src/sqlbuild/cli/commands/main/helpers.py | 39 ++++--- .../cli/commands/main/test_job_warehouses.py | 18 +-- .../executor/build/classes/_test_types.py | 1 - .../executor/build/classes/helpers.py | 22 +++- .../test_build_scheduler_warehouses.py | 36 ++---- .../scheduling/classes/_test_types.py | 11 +- .../executor/scheduling/classes/helpers.py | 13 ++- .../classes/test_session_warehouses.py | 67 +++++------ 93 files changed, 791 insertions(+), 544 deletions(-) create mode 100644 crates/sqlbuild-analysis/src/job_settings/types.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_driver/_helpers/jobs/job_limits.rs rename crates/{sqlbuild-python/src/bindings/_helpers/compile_driver => sqlbuild-python-compile/src/compile_driver/_helpers/jobs}/job_node_settings.rs (90%) create mode 100644 crates/sqlbuild-python-compile/src/compile_driver/_helpers/jobs/mod.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_driver/main/jobs/build_concurrency.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_driver/main/jobs/connection_limit.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_driver/main/jobs/job_override_fields.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_driver/main/jobs/job_plan_fields.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_driver/main/jobs/job_run_context.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_driver/main/jobs/mod.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_driver/main/jobs/planned_job_settings.rs delete mode 100644 crates/sqlbuild-python-planner/src/plan_output/main/job_warehouses.rs rename crates/{sqlbuild-python/src/bindings/_helpers/cli/named_job_settings.rs => sqlbuild-python-project/src/job_settings/_helpers/argv.rs} (76%) create mode 100644 crates/sqlbuild-python-project/src/job_settings/_helpers/could_name_job.rs rename crates/sqlbuild-python-project/src/{compiled_project => job_settings}/_helpers/job_catalog.rs (100%) create mode 100644 crates/sqlbuild-python-project/src/job_settings/_helpers/job_run_settings.rs create mode 100644 crates/sqlbuild-python-project/src/job_settings/_helpers/mod.rs create mode 100644 crates/sqlbuild-python-project/src/job_settings/_helpers/named_job_build_settings.rs create mode 100644 crates/sqlbuild-python-project/src/job_settings/constants.rs rename crates/sqlbuild-python-project/src/{compiled_project => job_settings}/main/declared_jobs.rs (91%) create mode 100644 crates/sqlbuild-python-project/src/job_settings/main/frozen_run_flags.rs create mode 100644 crates/sqlbuild-python-project/src/job_settings/main/frozen_selectors.rs rename crates/sqlbuild-python-project/src/{compiled_project => job_settings}/main/graph_with_jobs.rs (92%) rename crates/sqlbuild-python-project/src/{compiled_project => job_settings}/main/job_declarations.rs (90%) rename crates/sqlbuild-python-project/src/{compiled_project => job_settings}/main/job_names.rs (85%) rename crates/sqlbuild-python-project/src/{compiled_project => job_settings}/main/job_scope_project_dir.rs (83%) create mode 100644 crates/sqlbuild-python-project/src/job_settings/main/job_selector_error.rs create mode 100644 crates/sqlbuild-python-project/src/job_settings/main/job_setting_conflict.rs create mode 100644 crates/sqlbuild-python-project/src/job_settings/main/job_settings_argv.rs create mode 100644 crates/sqlbuild-python-project/src/job_settings/main/mod.rs rename crates/sqlbuild-python-project/src/{compiled_project => job_settings}/main/named_jobs.rs (76%) rename crates/sqlbuild-python-project/src/{compiled_project => job_settings}/main/needed_jobs.rs (91%) rename crates/sqlbuild-python-project/src/{compiled_project => job_settings}/main/triggered_jobs.rs (91%) rename crates/sqlbuild-python-project/src/{compiled_project => job_settings}/main/with_frozen_job.rs (90%) rename crates/sqlbuild-python-project/src/{compiled_project => job_settings}/main/with_job_scope.rs (89%) create mode 100644 crates/sqlbuild-python-project/src/job_settings/mod.rs create mode 100644 crates/sqlbuild-python-project/src/job_settings/models.rs delete mode 100644 crates/sqlbuild-python/src/bindings/_helpers/compiled_project/job_settings.rs diff --git a/crates/sqlbuild-analysis/src/job_settings/_helpers/selection.rs b/crates/sqlbuild-analysis/src/job_settings/_helpers/selection.rs index 4ba81f1473..bee329397c 100644 --- a/crates/sqlbuild-analysis/src/job_settings/_helpers/selection.rs +++ b/crates/sqlbuild-analysis/src/job_settings/_helpers/selection.rs @@ -7,7 +7,8 @@ use crate::graph::errors::SelectorError; use crate::graph::models::ProjectGraph; use crate::job_settings::_helpers::tokens::token_jobs; use crate::job_settings::errors::JobSelectorError; -use crate::job_settings::models::{DeclaredRun, TokenNodes}; +use crate::job_settings::models::DeclaredRun; +use crate::job_settings::types::TokenNodes; /// Each node a `--select` token, less `--exclude`, selects through a job it names that holds it. pub(crate) fn node_jobs( diff --git a/crates/sqlbuild-analysis/src/job_settings/main/node_job_settings.rs b/crates/sqlbuild-analysis/src/job_settings/main/node_job_settings.rs index b22ab52b70..d876b18167 100644 --- a/crates/sqlbuild-analysis/src/job_settings/main/node_job_settings.rs +++ b/crates/sqlbuild-analysis/src/job_settings/main/node_job_settings.rs @@ -9,7 +9,8 @@ use crate::job_settings::_helpers::node_values::node_values; use crate::job_settings::_helpers::selection::node_jobs; use crate::job_settings::constants::JOB_LIMIT_SETTING; use crate::job_settings::errors::NodeJobSettingsError; -use crate::job_settings::models::{DeclaredRun, NodeJobSettings, TokenNodes}; +use crate::job_settings::models::{DeclaredRun, NodeJobSettings}; +use crate::job_settings::types::TokenNodes; /// Each node's named jobs, the settings they give it, and their limits. pub fn node_job_settings( diff --git a/crates/sqlbuild-analysis/src/job_settings/mod.rs b/crates/sqlbuild-analysis/src/job_settings/mod.rs index c4d7004a23..7e40b4e8fa 100644 --- a/crates/sqlbuild-analysis/src/job_settings/mod.rs +++ b/crates/sqlbuild-analysis/src/job_settings/mod.rs @@ -5,6 +5,7 @@ pub mod constants; pub mod errors; pub mod main; pub mod models; +pub mod types; #[cfg(test)] mod tests; diff --git a/crates/sqlbuild-analysis/src/job_settings/models.rs b/crates/sqlbuild-analysis/src/job_settings/models.rs index da1b01dbbc..1642d3c606 100644 --- a/crates/sqlbuild-analysis/src/job_settings/models.rs +++ b/crates/sqlbuild-analysis/src/job_settings/models.rs @@ -1,11 +1,10 @@ //! Declared jobs and groups, and the settings a selection gives its nodes. -use std::collections::{BTreeMap, BTreeSet}; +use std::collections::BTreeMap; use serde_json::Value as Json; use crate::assembly::project::types::ObjectKey; -use crate::graph::errors::SelectorError; /// A job's `[retry]`: at most `attempts` runs in all, each rerun `delay_seconds` after the last. #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -37,9 +36,6 @@ pub enum SettingScope { Build, } -/// The nodes one selector token selects, less those `--exclude` removes, as the build resolves them. -pub type TokenNodes<'r> = &'r dyn Fn(&str, &[String]) -> Result, SelectorError>; - /// What the jobs a selection names give each of its nodes. #[derive(Clone, Debug, Default, PartialEq)] pub struct NodeJobSettings { diff --git a/crates/sqlbuild-analysis/src/job_settings/tests/test_node_job_settings.rs b/crates/sqlbuild-analysis/src/job_settings/tests/test_node_job_settings.rs index 1728fde83e..efc9f24ad8 100644 --- a/crates/sqlbuild-analysis/src/job_settings/tests/test_node_job_settings.rs +++ b/crates/sqlbuild-analysis/src/job_settings/tests/test_node_job_settings.rs @@ -228,17 +228,20 @@ fn given_a_python_node_in_named_jobs_when_resolving_then_it_takes_only_their_war PythonNodeValuesTestCase { description: "the warehouse applies; run_tests and [retry] do not", jobs: &["orders_wh_job"], - expected_result: Ok(BTreeMap::from([("warehouse", json!("ORDERS_WH"))])), + expected_values: Some(BTreeMap::from([("warehouse", json!("ORDERS_WH"))])), + expected_conflict: None, }, PythonNodeValuesTestCase { description: "jobs that agree on everything Python nodes take do not conflict", jobs: &["orders_wh_job", "orders_job"], - expected_result: Ok(BTreeMap::from([("warehouse", json!("ORDERS_WH"))])), + expected_values: Some(BTreeMap::from([("warehouse", json!("ORDERS_WH"))])), + expected_conflict: None, }, PythonNodeValuesTestCase { description: "two warehouses conflict", jobs: &["orders_wh_job", "refunds_wh_job"], - expected_result: Err( + expected_values: None, + expected_conflict: Some( "asset:daily_report is selected through jobs orders_wh_job and refunds_wh_job, \ which set warehouse differently (\"ORDERS_WH\" and \"REFUNDS_WH\")", ), @@ -247,22 +250,23 @@ fn given_a_python_node_in_named_jobs_when_resolving_then_it_takes_only_their_war let key = ("asset".to_owned(), "daily_report".to_owned()); for test_case in test_cases { let resolved = node_values(&key, &strings(test_case.jobs), &declared()); - let shown = match resolved { - Ok((values, retry)) => { - assert_eq!(retry, None, "{}", test_case.description); - Ok(values) - } - Err(conflict) => Err(conflict.message), - }; - let expected = test_case - .expected_result - .map(|values| { + let values: Option> = + resolved.as_ref().ok().map(|(values, _)| { values - .into_iter() - .map(|(name, value)| (name.to_owned(), value)) + .iter() + .map(|(name, value)| (name.as_str(), value.clone())) .collect() - }) - .map_err(str::to_owned); - assert_eq!(shown, expected, "{}", test_case.description); + }); + let retry = resolved.as_ref().ok().and_then(|(_, retry)| *retry); + let conflict: Option<&str> = resolved + .as_ref() + .err() + .map(|conflict| conflict.message.as_str()); + assert_eq!( + (values, retry, conflict), + (test_case.expected_values, None, test_case.expected_conflict), + "{}", + test_case.description + ); } } diff --git a/crates/sqlbuild-analysis/src/job_settings/tests/test_types.rs b/crates/sqlbuild-analysis/src/job_settings/tests/test_types.rs index c6b3f5413a..23bfbdeadb 100644 --- a/crates/sqlbuild-analysis/src/job_settings/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/job_settings/tests/test_types.rs @@ -29,8 +29,8 @@ pub(super) struct PythonNodeValuesTestCase { pub(super) description: &'static str, /// The named jobs the Python node is selected through. pub(super) jobs: &'static [&'static str], - /// Its settings, or the conflict's message. - pub(super) expected_result: Result, &'static str>, + pub(super) expected_values: Option>, + pub(super) expected_conflict: Option<&'static str>, } pub(super) struct BuildJobSettingsTestCase { diff --git a/crates/sqlbuild-analysis/src/job_settings/types.rs b/crates/sqlbuild-analysis/src/job_settings/types.rs new file mode 100644 index 0000000000..a9a4bf2df9 --- /dev/null +++ b/crates/sqlbuild-analysis/src/job_settings/types.rs @@ -0,0 +1,9 @@ +//! Type aliases of named job settings. + +use std::collections::BTreeSet; + +use crate::assembly::project::types::ObjectKey; +use crate::graph::errors::SelectorError; + +/// The nodes one selector token selects, less those `--exclude` removes, as the build resolves them. +pub type TokenNodes<'r> = &'r dyn Fn(&str, &[String]) -> Result, SelectorError>; diff --git a/crates/sqlbuild-config/src/command_connection/tests/test_command_connection.rs b/crates/sqlbuild-config/src/command_connection/tests/test_command_connection.rs index 90d621fe26..699d552ce4 100644 --- a/crates/sqlbuild-config/src/command_connection/tests/test_command_connection.rs +++ b/crates/sqlbuild-config/src/command_connection/tests/test_command_connection.rs @@ -143,36 +143,40 @@ fn given_a_job_warehouse_when_checking_then_it_names_the_job_and_follows_identif JobWarehouseTestCase { description: "unquoted identifier", value: Some("ORDERS_WH"), - expected_result: Ok("ORDERS_WH"), + expected_name: Some("ORDERS_WH"), + expected_message: None, }, JobWarehouseTestCase { description: "blank value", value: Some(" "), - expected_result: Err("job orders_job sets an empty warehouse name"), + expected_name: None, + expected_message: Some("job orders_job sets an empty warehouse name"), }, JobWarehouseTestCase { description: "not an identifier", value: Some("bad wh"), - expected_result: Err( + expected_name: None, + expected_message: Some( "job orders_job sets warehouse 'bad wh', which is not a valid Snowflake warehouse identifier", ), }, JobWarehouseTestCase { description: "not text", value: None, - expected_result: Err("job orders_job sets a warehouse that is not text"), + expected_name: None, + expected_message: Some("job orders_job sets a warehouse that is not text"), }, ]; for test_case in test_cases { let result = check_job_warehouse("orders_job", test_case.value); - let shown: Result<&str, Option> = match &result { - Ok(name) => Ok(name.as_str()), - Err(error) => Err(Some(error.message().to_owned())), - }; - let expected: Result<&str, Option> = test_case - .expected_result - .map_err(|text| Some(text.to_owned())); - assert_eq!(shown, expected, "{}", test_case.description); + let name: Option<&str> = result.as_deref().ok(); + let message: Option<&str> = result.as_ref().err().map(|error| error.message()); + assert_eq!( + (name, message), + (test_case.expected_name, test_case.expected_message), + "{}", + test_case.description + ); } } diff --git a/crates/sqlbuild-config/src/command_connection/tests/test_types.rs b/crates/sqlbuild-config/src/command_connection/tests/test_types.rs index 263ea978f2..c636c5d2c5 100644 --- a/crates/sqlbuild-config/src/command_connection/tests/test_types.rs +++ b/crates/sqlbuild-config/src/command_connection/tests/test_types.rs @@ -9,7 +9,8 @@ pub(super) struct CliWarehouseTestCase { pub(super) struct JobWarehouseTestCase { pub(super) description: &'static str, pub(super) value: Option<&'static str>, - pub(super) expected_result: Result<&'static str, &'static str>, + pub(super) expected_name: Option<&'static str>, + pub(super) expected_message: Option<&'static str>, } pub(super) struct SnowflakeKeyTestCase { diff --git a/crates/sqlbuild-python-compile/src/compile_driver/_helpers/jobs/job_limits.rs b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/jobs/job_limits.rs new file mode 100644 index 0000000000..c74c98a000 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/jobs/job_limits.rs @@ -0,0 +1,18 @@ +//! The named jobs' limits a planned build carries. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::types::{PyDict, PyDictMethods}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// The named jobs' `concurrency` limits added together; none without a command-line override. +pub(crate) fn job_limit_total(pipeline: &Bound<'_, PyAny>) -> PyResult> { + let limits: Bound<'_, PyDict> = native_field(pipeline, "job_limits")?.cast_into()?; + if limits.is_empty() { + return Ok(None); + } + let mut total: i64 = 0; + for value in limits.values() { + total = total.saturating_add(value.extract::()?); + } + Ok(Some(total)) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/job_node_settings.rs b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/jobs/job_node_settings.rs similarity index 90% rename from crates/sqlbuild-python/src/bindings/_helpers/compile_driver/job_node_settings.rs rename to crates/sqlbuild-python-compile/src/compile_driver/_helpers/jobs/job_node_settings.rs index d2dd302777..a1918fc7a1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/job_node_settings.rs +++ b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/jobs/job_node_settings.rs @@ -3,6 +3,7 @@ use std::collections::{BTreeMap, BTreeSet}; use pyo3::IntoPyObject; +use pyo3::PyErr; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyBool, PyDict, PyDictMethods, PyFrozenSet, PyTuple}; use serde_json::Value; @@ -15,55 +16,52 @@ use sqlbuild_analysis::job_settings::main::build_job_settings::build_job_setting use sqlbuild_analysis::job_settings::main::named_jobs::named_jobs; use sqlbuild_analysis::job_settings::main::node_job_settings::node_job_settings; use sqlbuild_analysis::job_settings::models::{DeclaredRun, NodeJobSettings}; +use sqlbuild_config::command_connection::errors::CommandConnectionError; use sqlbuild_config::command_connection::main::check_job_warehouse::check_job_warehouse; use sqlbuild_config::command_connection::main::unsupported_job_warehouse::unsupported_job_warehouse; +use sqlbuild_python_base::records::main::construct::new_error::new_error; +use sqlbuild_python_base::records::types::ErrorId; -use crate::bindings::_helpers::command_runtime::runtime_errors::raised; -use crate::bindings::_helpers::command_runtime::warehouses::command_scope; -use crate::bindings::_helpers::compiled_project::job_settings::{ - job_selector_error, job_setting_conflict, -}; +use crate::compile_driver::main::setup::project_graph::project_graph; +use crate::compile_driver::models::PlannedJobSettings; use sqlbuild_python_base::boundary::main::compiler_guard::compiler_guard; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::replaced::replaced; -use sqlbuild_python_compile::compile_driver::main::setup::project_graph::project_graph; use sqlbuild_python_front::compile_driver::main::session_warehouse_support::session_warehouse_support; use sqlbuild_python_front::contracts::main::resolve_effective_adapter_name::resolve_effective_adapter_name; use sqlbuild_python_project::compiled_project::constants::{ JOB_FULL_REFRESH_KEY, JOB_IGNORED_AUDIT_FAILURES_KEY, JOB_WAREHOUSES_KEY, }; -use sqlbuild_python_project::compiled_project::main::job_declarations::job_declarations; -use sqlbuild_python_project::compiled_project::main::job_scope_project_dir::job_scope_project_dir; use sqlbuild_python_project::graph::main::nodes::native_key::native_key; use sqlbuild_python_project::graph::main::python_nodes::python_node_graph::python_node_graph; use sqlbuild_python_project::graph::models::KeyCache; use sqlbuild_python_project::graph::models::{ CompiledProjectGraph, NativeProjectGraph, PythonAwareSelection, }; +use sqlbuild_python_project::job_settings::main::job_declarations::job_declarations; +use sqlbuild_python_project::job_settings::main::job_scope_project_dir::job_scope_project_dir; +use sqlbuild_python_project::job_settings::main::job_selector_error::job_selector_error; +use sqlbuild_python_project::job_settings::main::job_setting_conflict::job_setting_conflict; const MODEL_KIND: &str = "model"; const FULL_REFRESH: &str = "full_refresh"; const RUN_TESTS: &str = "run_tests"; const IGNORE_AUDIT_FAILURES: &str = "ignore_audit_failures"; -/// What the planned build takes from its named jobs, or nothing when it names none. -pub(crate) struct PlannedJobSettings { - settings: NodeJobSettings, - /// The named jobs that set `ignore_audit_failures = true` for nodes they select. - ignoring_jobs: Vec, - /// The named jobs that set `full_refresh = true` for models they select. - refreshing_jobs: Vec, - /// Each named job whose `warehouse` its own nodes run on, and that warehouse; `--warehouse` - /// leaves the jobs' warehouses out before they are resolved. - job_warehouses: BTreeMap, +/// The `CliUserError` of a job warehouse that cannot be used. +fn cli_error(py: Python<'_>, error: &CommandConnectionError) -> PyErr { + let (code, help) = match error { + CommandConnectionError::User { code, help, .. } => (*code, help.as_deref()), + CommandConnectionError::ProjectConfig(_) => (None, None), + }; + new_error(py, ErrorId::CliUser, error.message(), (code, help)) } fn is_python(key: &ObjectKey) -> bool { PYTHON_NODE_KINDS.contains(&key.0.as_str()) } -/// The named jobs whose `warehouse` some selected node runs on, checked as `--warehouse` would be: -/// a valid name (`C261`) on an adapter with a session warehouse (`C260`). +/// The jobs whose `warehouse` a node runs on, checked like `--warehouse` (`C261`, `C260`). fn job_warehouses( (project, discovered): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), settings: &NodeJobSettings, @@ -86,8 +84,8 @@ fn job_warehouses( if let Some(DeclaredRun::Job { settings, .. }) = declared.get(job) && let Some(value) = settings.get(WAREHOUSE_SETTING) { - let name: String = - check_job_warehouse(job, value.as_str()).map_err(|error| raised(py, error))?; + let name: String = check_job_warehouse(job, value.as_str()) + .map_err(|error| cli_error(py, &error))?; warehouses.insert(job.clone(), name); } } @@ -102,9 +100,9 @@ fn job_warehouses( let project_dir: Bound<'_, PyAny> = job_scope_project_dir().into_pyobject(py)?.into_any(); if !session_warehouse_support(py, &adapter_name, &project_dir)? { let adapter: String = adapter_name.str()?.extract()?; - return Err(raised( + return Err(cli_error( py, - unsupported_job_warehouse(job, warehouse, &adapter), + &unsupported_job_warehouse(job, warehouse, &adapter), )); } Ok(warehouses) @@ -139,7 +137,7 @@ fn strings(value: &Bound<'_, PyAny>) -> PyResult> { /// The named jobs' settings per selected node; `None` when ignored, unnamed or unselectable. pub(crate) fn planned_job_settings( (project, discovered): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), - options: &Bound<'_, PyAny>, + (options, cli_warehouse): (&Bound<'_, PyAny>, bool), ) -> PyResult> { let py: Python<'_> = project.py(); if !native_field(options, "job_settings")?.is_truthy()? { @@ -156,7 +154,7 @@ pub(crate) fn planned_job_settings( let Some(mut declared): Option> = job_declarations(py)? else { return Ok(None); }; - if command_scope(py)?.cli_warehouse.is_some() { + if cli_warehouse { for run in declared.values_mut() { if let DeclaredRun::Job { settings, .. } = run { settings.remove(WAREHOUSE_SETTING); @@ -207,10 +205,7 @@ impl PlannedJobSettings { } /// The models the named jobs rebuild from scratch. - pub(crate) fn full_refresh_models<'py>( - &self, - py: Python<'py>, - ) -> PyResult> { + pub fn full_refresh_models<'py>(&self, py: Python<'py>) -> PyResult> { let names: Vec<&String> = self .settings .values @@ -281,8 +276,7 @@ impl PlannedJobSettings { self.value(key, WAREHOUSE_SETTING).and_then(Value::as_str) } - /// Each planned SQL node's job warehouse, and each Python node's by name; a SQL test runs on - /// the warehouse its models' jobs give when they agree. Empty under `--warehouse`. + /// Each SQL node's job warehouse, tests on their models' agreed one, and Python nodes' by name. pub(crate) fn node_warehouses<'py>( &self, plan: &Bound<'py, PyAny>, diff --git a/crates/sqlbuild-python-compile/src/compile_driver/_helpers/jobs/mod.rs b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/jobs/mod.rs new file mode 100644 index 0000000000..15e1f2d769 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/jobs/mod.rs @@ -0,0 +1,2 @@ +pub(crate) mod job_limits; +pub(crate) mod job_node_settings; diff --git a/crates/sqlbuild-python-compile/src/compile_driver/_helpers/mod.rs b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/mod.rs index 4b77187d1e..1e8b92dab8 100644 --- a/crates/sqlbuild-python-compile/src/compile_driver/_helpers/mod.rs +++ b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/mod.rs @@ -1,3 +1,4 @@ pub(crate) mod assembly; +pub(crate) mod jobs; pub(crate) mod setup; pub(crate) mod stages; diff --git a/crates/sqlbuild-python-compile/src/compile_driver/main/jobs/build_concurrency.rs b/crates/sqlbuild-python-compile/src/compile_driver/main/jobs/build_concurrency.rs new file mode 100644 index 0000000000..3ad7861e98 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_driver/main/jobs/build_concurrency.rs @@ -0,0 +1,20 @@ +//! The concurrency limit of a build that names jobs. + +use pyo3::prelude::{Bound, PyAny, PyResult}; +use sqlbuild_config::effective::models::{ConcurrencySource, EffectiveConcurrency}; + +use crate::compile_driver::_helpers::jobs::job_limits::job_limit_total; + +/// The build's limit: the configured one, raised to the named jobs' limits added together. +pub fn build_concurrency( + configured: EffectiveConcurrency, + pipeline: &Bound<'_, PyAny>, +) -> PyResult { + Ok(match job_limit_total(pipeline)? { + Some(total) if total > configured.limit => EffectiveConcurrency { + limit: total, + source: ConcurrencySource::NamedJobs, + }, + _ => configured, + }) +} diff --git a/crates/sqlbuild-python-compile/src/compile_driver/main/jobs/connection_limit.rs b/crates/sqlbuild-python-compile/src/compile_driver/main/jobs/connection_limit.rs new file mode 100644 index 0000000000..49a52d2902 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_driver/main/jobs/connection_limit.rs @@ -0,0 +1,29 @@ +//! The connections a build that names limited jobs opens. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::types::{PyDict, PyDictMethods}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +use crate::compile_driver::_helpers::jobs::job_limits::job_limit_total; +use crate::compile_driver::main::jobs::build_concurrency::build_concurrency; +use sqlbuild_config::effective::models::EffectiveConcurrency; + +/// The build's limit, or the named jobs' limits added together when every planned node has one. +pub fn connection_limit( + configured: EffectiveConcurrency, + pipeline: &Bound<'_, PyAny>, +) -> PyResult { + let limit: i64 = build_concurrency(configured, pipeline)?.limit; + let Some(total) = job_limit_total(pipeline)? else { + return Ok(limit); + }; + let node_jobs: Bound<'_, PyDict> = native_field(pipeline, "node_jobs")?.cast_into()?; + for key in + native_field(&native_field(pipeline, "plan_output")?, "execution_order")?.try_iter()? + { + if !node_jobs.contains(key?)? { + return Ok(limit); + } + } + Ok(limit.min(total.max(1))) +} diff --git a/crates/sqlbuild-python-compile/src/compile_driver/main/jobs/job_override_fields.rs b/crates/sqlbuild-python-compile/src/compile_driver/main/jobs/job_override_fields.rs new file mode 100644 index 0000000000..ffd3872875 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_driver/main/jobs/job_override_fields.rs @@ -0,0 +1,20 @@ +//! The planner overrides the named jobs' settings add. + +use pyo3::prelude::{PyResult, Python}; + +use crate::compile_driver::models::PlannedJobSettings; +use crate::compile_driver::types::PipelineFields; + +/// The models the named jobs rebuild from scratch, as the planner's `full_refresh_models`. +pub fn job_override_fields<'py>( + py: Python<'py>, + job_settings: Option<&PlannedJobSettings>, +) -> PyResult> { + let Some(job_settings) = job_settings else { + return Ok(Vec::new()); + }; + Ok(vec![( + "full_refresh_models", + job_settings.full_refresh_models(py)?.into_any(), + )]) +} diff --git a/crates/sqlbuild-python-compile/src/compile_driver/main/jobs/job_plan_fields.rs b/crates/sqlbuild-python-compile/src/compile_driver/main/jobs/job_plan_fields.rs new file mode 100644 index 0000000000..5a46a7c457 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_driver/main/jobs/job_plan_fields.rs @@ -0,0 +1,34 @@ +//! The plan and pipeline fields the named jobs' settings give a planned build. + +use pyo3::prelude::{Bound, PyAny, PyResult}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +use crate::compile_driver::models::PlannedJobSettings; +use crate::compile_driver::types::PipelineFields; + +/// The plan with the jobs' settings applied, and their limits and warehouses as pipeline fields. +pub fn job_plan_fields<'py>( + job_settings: Option<&PlannedJobSettings>, + plan_output: Bound<'py, PyAny>, + options: &Bound<'py, PyAny>, +) -> PyResult<(Bound<'py, PyAny>, PipelineFields<'py>)> { + let Some(job_settings) = job_settings else { + return Ok((plan_output, Vec::new())); + }; + let plan_output: Bound<'py, PyAny> = job_settings.apply_to_plan( + &plan_output, + &native_field(options, "cli_ignore_audit_failures")?, + )?; + let (limits, node_jobs) = + job_settings.job_limits(&plan_output, &native_field(options, "concurrency")?)?; + let (nodes, python) = job_settings.node_warehouses(&plan_output)?; + Ok(( + plan_output, + vec![ + ("job_limits", limits.into_any()), + ("node_jobs", node_jobs.into_any()), + ("node_warehouses", nodes.into_any()), + ("python_node_warehouses", python.into_any()), + ], + )) +} diff --git a/crates/sqlbuild-python-compile/src/compile_driver/main/jobs/job_run_context.rs b/crates/sqlbuild-python-compile/src/compile_driver/main/jobs/job_run_context.rs new file mode 100644 index 0000000000..65856ffa6d --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_driver/main/jobs/job_run_context.rs @@ -0,0 +1,34 @@ +//! What a build's run context says about its named jobs. + +use crate::compile_driver::types::JobRunContext; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::types::{PyDict, PyDictMethods}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; +use sqlbuild_python_project::compiled_project::constants::{ + JOB_FULL_REFRESH_KEY, JOB_WAREHOUSES_KEY, +}; + +/// The jobs whose models the planned build rebuilds, and each job's warehouse its nodes run on. +pub fn job_run_context(pipeline: &Bound<'_, PyAny>) -> PyResult { + let metadata: Bound<'_, PyAny> = + native_field(&native_field(pipeline, "plan_output")?, "metadata")?; + let refreshing: Bound<'_, PyAny> = mapping_get(&metadata, JOB_FULL_REFRESH_KEY)?; + let jobs: Vec = if refreshing.is_none() { + Vec::new() + } else { + refreshing + .try_iter()? + .map(|job| job?.extract()) + .collect::>()? + }; + let warehouses: Bound<'_, PyAny> = mapping_get(&metadata, JOB_WAREHOUSES_KEY)?; + let warehouses: Vec<(String, String)> = match warehouses.cast::() { + Ok(warehouses) => warehouses + .iter() + .map(|(job, warehouse)| Ok((job.extract()?, warehouse.extract()?))) + .collect::>()?, + Err(_) => Vec::new(), + }; + Ok((jobs, warehouses)) +} diff --git a/crates/sqlbuild-python-compile/src/compile_driver/main/jobs/mod.rs b/crates/sqlbuild-python-compile/src/compile_driver/main/jobs/mod.rs new file mode 100644 index 0000000000..97102c5ad6 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_driver/main/jobs/mod.rs @@ -0,0 +1,6 @@ +pub mod build_concurrency; +pub mod connection_limit; +pub mod job_override_fields; +pub mod job_plan_fields; +pub mod job_run_context; +pub mod planned_job_settings; diff --git a/crates/sqlbuild-python-compile/src/compile_driver/main/jobs/planned_job_settings.rs b/crates/sqlbuild-python-compile/src/compile_driver/main/jobs/planned_job_settings.rs new file mode 100644 index 0000000000..28eded5b53 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_driver/main/jobs/planned_job_settings.rs @@ -0,0 +1,15 @@ +//! The settings each planned node takes from the jobs its selection names. + +use pyo3::prelude::{Bound, PyAny, PyResult}; + +use crate::compile_driver::_helpers::jobs::job_node_settings; +use crate::compile_driver::models::PlannedJobSettings; + +/// The named jobs' settings per selected node; `None` when ignored, unnamed or unselectable. +#[inline] +pub fn planned_job_settings( + inputs: (&Bound<'_, PyAny>, &Bound<'_, PyAny>), + options: (&Bound<'_, PyAny>, bool), +) -> PyResult> { + job_node_settings::planned_job_settings(inputs, options) +} diff --git a/crates/sqlbuild-python-compile/src/compile_driver/main/mod.rs b/crates/sqlbuild-python-compile/src/compile_driver/main/mod.rs index 2f69896363..57a7fde978 100644 --- a/crates/sqlbuild-python-compile/src/compile_driver/main/mod.rs +++ b/crates/sqlbuild-python-compile/src/compile_driver/main/mod.rs @@ -1,2 +1,3 @@ +pub mod jobs; pub mod setup; pub mod stages; diff --git a/crates/sqlbuild-python-compile/src/compile_driver/models.rs b/crates/sqlbuild-python-compile/src/compile_driver/models.rs index 5838cd685d..0c90c59ac8 100644 --- a/crates/sqlbuild-python-compile/src/compile_driver/models.rs +++ b/crates/sqlbuild-python-compile/src/compile_driver/models.rs @@ -9,9 +9,11 @@ use pyo3::prelude::PyResult; use pyo3::prelude::Python; use pyo3::types::PyBool; use sqlbuild_analysis::assembly::target_checks::constants::PRESERVE_TARGET_VALUE; +use sqlbuild_analysis::job_settings::models::NodeJobSettings; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::types::RecordId; +use std::collections::BTreeMap; /// Semantic analysis cache outcome counts for one invocation. #[derive(Debug)] @@ -148,3 +150,15 @@ impl TargetSettingsValues { ) } } + +/// What the planned build takes from its named jobs, or nothing when it names none. +#[derive(Debug)] +pub struct PlannedJobSettings { + pub(crate) settings: NodeJobSettings, + /// The named jobs that set `ignore_audit_failures = true` for nodes they select. + pub(crate) ignoring_jobs: Vec, + /// The named jobs that set `full_refresh = true` for models they select. + pub(crate) refreshing_jobs: Vec, + /// Each named job whose `warehouse` its own nodes run on, and that warehouse. + pub(crate) job_warehouses: BTreeMap, +} diff --git a/crates/sqlbuild-python-compile/src/compile_driver/types.rs b/crates/sqlbuild-python-compile/src/compile_driver/types.rs index 8e30752bcd..245e1cab39 100644 --- a/crates/sqlbuild-python-compile/src/compile_driver/types.rs +++ b/crates/sqlbuild-python-compile/src/compile_driver/types.rs @@ -6,3 +6,8 @@ use pyo3::prelude::PyAny; /// One compile's outcome, input reads and artifact writes, before it is stored and printed. pub type PhasesOutcome<'py> = (CommandOutcome<'py>, Bound<'py, PyAny>, Bound<'py, PyAny>); + +/// Pipeline result fields by name. +pub type PipelineFields<'py> = Vec<(&'static str, pyo3::Bound<'py, pyo3::PyAny>)>; +/// The jobs whose models a build rebuilds, and each job with the warehouse its nodes run on. +pub type JobRunContext = (Vec, Vec<(String, String)>); diff --git a/crates/sqlbuild-python-planner/src/consumption/main/changed_policy.rs b/crates/sqlbuild-python-planner/src/consumption/main/changed_policy.rs index ca236b42d0..5181e3b661 100644 --- a/crates/sqlbuild-python-planner/src/consumption/main/changed_policy.rs +++ b/crates/sqlbuild-python-planner/src/consumption/main/changed_policy.rs @@ -10,8 +10,8 @@ use pyo3::exceptions::PyValueError; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyRef, PyResult}; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_front::discovery::main::project::configured_changes::configured_changes; -use sqlbuild_python_project::compiled_project::main::triggered_jobs::triggered_jobs; use sqlbuild_python_project::graph::models::NativeProjectGraph; +use sqlbuild_python_project::job_settings::main::triggered_jobs::triggered_jobs; use sqlbuild_state::consumption::models::{ChangeKind, ChangedPolicy, Reach}; use std::collections::BTreeSet; diff --git a/crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_text.rs b/crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_text.rs index 6823ccc17c..f1c6d51258 100644 --- a/crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_text.rs +++ b/crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_text.rs @@ -391,7 +391,6 @@ fn provider_usages( Ok(usages) } -/// `str(item)` of a metadata tuple or list, else nothing. /// The plan metadata's named jobs and the warehouses their nodes run on. pub(crate) fn job_warehouses(metadata: &Bound<'_, PyAny>) -> PyResult> { let value: Bound<'_, PyAny> = mapping_get(metadata, JOB_WAREHOUSES_KEY)?; @@ -404,6 +403,7 @@ pub(crate) fn job_warehouses(metadata: &Bound<'_, PyAny>) -> PyResult>) -> PyResult> { match value { Some(value) if value.is_instance_of::() || value.is_instance_of::() => { diff --git a/crates/sqlbuild-python-planner/src/plan_output/main/job_warehouses.rs b/crates/sqlbuild-python-planner/src/plan_output/main/job_warehouses.rs deleted file mode 100644 index fa12b993ed..0000000000 --- a/crates/sqlbuild-python-planner/src/plan_output/main/job_warehouses.rs +++ /dev/null @@ -1,10 +0,0 @@ -//! The plan metadata's named jobs and the warehouses their nodes run on. - -use crate::plan_output::_helpers::plan_text; -use pyo3::prelude::{Bound, PyAny, PyResult}; - -/// The plan metadata's named jobs and the warehouses their nodes run on. -#[inline] -pub fn job_warehouses(metadata: &Bound<'_, PyAny>) -> PyResult> { - plan_text::job_warehouses(metadata) -} diff --git a/crates/sqlbuild-python-planner/src/plan_output/main/mod.rs b/crates/sqlbuild-python-planner/src/plan_output/main/mod.rs index 92aea4c7f1..1ffbc27d2f 100644 --- a/crates/sqlbuild-python-planner/src/plan_output/main/mod.rs +++ b/crates/sqlbuild-python-planner/src/plan_output/main/mod.rs @@ -7,7 +7,6 @@ pub mod format_plan_json; pub mod future_cursor_safety; pub mod inclusive_cursor_end; pub mod items; -pub mod job_warehouses; pub mod maximum_start_safety; pub mod optional_facts; pub mod register_display; diff --git a/crates/sqlbuild-python-project/src/compiled_project/_helpers/mod.rs b/crates/sqlbuild-python-project/src/compiled_project/_helpers/mod.rs index bf19c152a0..0bf4bc4a65 100644 --- a/crates/sqlbuild-python-project/src/compiled_project/_helpers/mod.rs +++ b/crates/sqlbuild-python-project/src/compiled_project/_helpers/mod.rs @@ -1,5 +1,4 @@ pub(crate) mod attachment_inputs; -pub(crate) mod job_catalog; pub(crate) mod project; pub(crate) mod schema_facts; pub(crate) mod values; diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/mod.rs b/crates/sqlbuild-python-project/src/compiled_project/main/mod.rs index eec55f6817..7495ef0105 100644 --- a/crates/sqlbuild-python-project/src/compiled_project/main/mod.rs +++ b/crates/sqlbuild-python-project/src/compiled_project/main/mod.rs @@ -1,22 +1,12 @@ pub mod config_values; pub mod declarations; -pub mod declared_jobs; pub mod entry_schema_fact; pub mod family_fact; -pub mod graph_with_jobs; pub mod install; -pub mod job_declarations; -pub mod job_names; -pub mod job_scope_project_dir; pub mod materialization; pub mod model_facts; -pub mod named_jobs; -pub mod needed_jobs; pub mod plain_value; pub mod python_column_fact; pub mod python_column_facts; pub mod reference_row_tuple; pub mod register_project; -pub mod triggered_jobs; -pub mod with_frozen_job; -pub mod with_job_scope; diff --git a/crates/sqlbuild-python-project/src/compiled_project/mod.rs b/crates/sqlbuild-python-project/src/compiled_project/mod.rs index 4af01a464c..15f3d76eeb 100644 --- a/crates/sqlbuild-python-project/src/compiled_project/mod.rs +++ b/crates/sqlbuild-python-project/src/compiled_project/mod.rs @@ -1,4 +1,4 @@ -//! The compiled project: its rows, attachment inputs, schema facts, values and job catalog. +//! The compiled project: its rows, attachment inputs, schema facts, and values. pub(crate) mod _helpers; pub mod constants; diff --git a/crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/python_selection.rs b/crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/python_selection.rs index 812c37ffbb..1a37690a10 100644 --- a/crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/python_selection.rs +++ b/crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/python_selection.rs @@ -20,8 +20,8 @@ use sqlbuild_analysis::python_selection::models::{ SqlSelectionGraph, }; -use crate::compiled_project::main::needed_jobs::needed_jobs; use crate::graph::models::{CompiledProjectGraph, NativeProjectGraph, PythonAwareSelection}; +use crate::job_settings::main::needed_jobs::needed_jobs; use sqlbuild_python_base::boundary::main::compiler_guard::compiler_guard; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; diff --git a/crates/sqlbuild-python-project/src/graph/main/python_nodes/resolve_node_sets.rs b/crates/sqlbuild-python-project/src/graph/main/python_nodes/resolve_node_sets.rs index a494b6d3c8..9f037cc092 100644 --- a/crates/sqlbuild-python-project/src/graph/main/python_nodes/resolve_node_sets.rs +++ b/crates/sqlbuild-python-project/src/graph/main/python_nodes/resolve_node_sets.rs @@ -2,13 +2,13 @@ use std::collections::BTreeMap; -use crate::compiled_project::main::declared_jobs::declared_jobs; use crate::graph::_helpers::python_nodes::python_node_graph::explicit_python_node_graph; use crate::graph::_helpers::python_nodes::python_selection::{ allows_round_trips, project_sources, selection_graph, }; use crate::graph::_helpers::python_nodes::set_dependencies::dependants; use crate::graph::models::NativeProjectGraph; +use crate::job_settings::main::declared_jobs::declared_jobs; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use sqlbuild_analysis::graph::models::JobSelection; use sqlbuild_analysis::node_set_dependencies::main::resolve_node_set_dependencies::resolve_node_set_dependencies; diff --git a/crates/sqlbuild-python-project/src/graph/models.rs b/crates/sqlbuild-python-project/src/graph/models.rs index ad5bfeb558..cb48954aa7 100644 --- a/crates/sqlbuild-python-project/src/graph/models.rs +++ b/crates/sqlbuild-python-project/src/graph/models.rs @@ -568,8 +568,8 @@ pub struct GraphViewMappings<'a, 'py> { pub path_index: &'a Bound<'py, PyAny>, } -/// A project graph with the jobs some selectors name attached as a build selects them, Python -/// nodes included, and the project facts selectors resolve with. +/// A project graph with the named jobs attached, Python nodes included, and the facts to resolve. +#[derive(Debug)] pub struct PythonAwareSelection<'g> { pub(crate) graph: Cow<'g, ProjectGraph>, pub(crate) sources: Vec<(String, Option)>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/named_job_settings.rs b/crates/sqlbuild-python-project/src/job_settings/_helpers/argv.rs similarity index 76% rename from crates/sqlbuild-python/src/bindings/_helpers/cli/named_job_settings.rs rename to crates/sqlbuild-python-project/src/job_settings/_helpers/argv.rs index 1c0e187a78..9789f3333e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/named_job_settings.rs +++ b/crates/sqlbuild-python-project/src/job_settings/_helpers/argv.rs @@ -3,26 +3,20 @@ use std::collections::{BTreeMap, BTreeSet}; use std::path::{Path, PathBuf}; -use pyo3::exceptions::{PyOSError, PyRuntimeError, PyUnicodeError, PyValueError}; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::exceptions::{PyOSError, PyRuntimeError, PyValueError}; +use pyo3::prelude::{Bound, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods}; use serde_json::Value; use sqlbuild_cli::argv::main::accepted_settings_arguments::accepted_settings_arguments; use sqlbuild_cli::argv::main::given_setting_dests::given_setting_dests; use sqlbuild_cli::dispatch::main::selector_file_selectors::selector_file_selectors; use sqlbuild_discovery::constants::ORCHESTRATION_DIRECTORY; - -use crate::bindings::_helpers::compiled_project::job_settings::{ - could_name_job, named_job_build_settings, -}; use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::types::ErrorId; -/// Ends the options; job flags go before it so they are never read as selectors. -const OPTIONS_TERMINATOR: &str = "--"; - -/// The commands whose selections take the settings of the jobs they name. -const JOB_SETTING_COMMANDS: [&str; 2] = ["build", "plan"]; +use crate::job_settings::_helpers::could_name_job::could_name_job; +use crate::job_settings::_helpers::named_job_build_settings::named_job_build_settings; +use crate::job_settings::constants::{JOB_SETTING_COMMANDS, OPTIONS_TERMINATOR}; /// One parsed argument, `None` when unset. fn argument<'py, T: pyo3::FromPyObject<'py>>( @@ -58,7 +52,7 @@ fn project_dir(namespace: &Bound<'_, PyDict>) -> PyResult { } /// `argv` plus the flags it leaves out of its named jobs' build-wide settings; `None` if unchanged. -fn named_job_argv( +pub(crate) fn named_job_argv( py: Python<'_>, argv: &[String], namespace: &Bound<'_, PyDict>, @@ -106,25 +100,8 @@ fn named_job_argv( } /// Whether the command meets `error` again, and reports it, when it runs. -fn reported_when_run(py: Python<'_>, error: &PyErr) -> PyResult { +pub(crate) fn reported_when_run(py: Python<'_>, error: &PyErr) -> PyResult { Ok(error.is_instance(py, &error_class(py, ErrorId::CliUser)?) || error.is_instance_of::(py) || error.is_instance_of::(py)) } - -/// The argv to parse again with its named jobs' build-wide flags, or `None` to keep the parse. -pub(crate) fn job_settings_argv( - argv: &Bound<'_, PyAny>, - namespace: &Bound<'_, PyDict>, -) -> PyResult>> { - let py: Python<'_> = argv.py(); - let arguments: Vec = match argv.extract() { - Ok(arguments) => arguments, - Err(undecodable) if undecodable.is_instance_of::(py) => return Ok(None), - Err(error) => return Err(error), - }; - match named_job_argv(py, &arguments, namespace) { - Err(error) if reported_when_run(py, &error)? => Ok(None), - result => result, - } -} diff --git a/crates/sqlbuild-python-project/src/job_settings/_helpers/could_name_job.rs b/crates/sqlbuild-python-project/src/job_settings/_helpers/could_name_job.rs new file mode 100644 index 0000000000..cfaca441f0 --- /dev/null +++ b/crates/sqlbuild-python-project/src/job_settings/_helpers/could_name_job.rs @@ -0,0 +1,17 @@ +//! Whether a selector can name a job. + +use sqlbuild_analysis::graph::main::parse_selector::parse_selector; +use sqlbuild_analysis::graph::models::ParsedSelector; + +use crate::compiled_project::constants::{JOB_SELECTOR_KINDS, SELECTOR_INTERSECTION}; + +/// Whether a selector has an atom that can name a job: `job:`, or a bare name. +pub(crate) fn could_name_job(selector: &str) -> bool { + selector + .split_whitespace() + .flat_map(|token| token.split(SELECTOR_INTERSECTION)) + .any(|atom| match parse_selector(atom) { + Ok(ParsedSelector::Kind { kind, .. }) => JOB_SELECTOR_KINDS.contains(&kind.as_str()), + _ => false, + }) +} diff --git a/crates/sqlbuild-python-project/src/compiled_project/_helpers/job_catalog.rs b/crates/sqlbuild-python-project/src/job_settings/_helpers/job_catalog.rs similarity index 100% rename from crates/sqlbuild-python-project/src/compiled_project/_helpers/job_catalog.rs rename to crates/sqlbuild-python-project/src/job_settings/_helpers/job_catalog.rs diff --git a/crates/sqlbuild-python-project/src/job_settings/_helpers/job_run_settings.rs b/crates/sqlbuild-python-project/src/job_settings/_helpers/job_run_settings.rs new file mode 100644 index 0000000000..9c39d3454f --- /dev/null +++ b/crates/sqlbuild-python-project/src/job_settings/_helpers/job_run_settings.rs @@ -0,0 +1,32 @@ +//! A job run's `--changed` settings. + +use std::collections::BTreeMap; + +use pyo3::prelude::{Bound, PyAny, PyResult}; +use serde_json::Value; +use sqlbuild_python_front::discovery::main::project::configured_changes::configured_changes; + +use crate::compiled_project::constants::{ + CHANGED_SETTING, JOB_RUN_COUNTS, JOB_RUN_REACH, REACH_SETTING, +}; + +/// A job run's `--changed` settings: its whole selection, and source data counted unless configured. +pub(crate) fn job_run_settings( + project_dir: &Bound<'_, PyAny>, + mut settings: BTreeMap, +) -> PyResult> { + let Some(changed) = settings.get(CHANGED_SETTING).cloned() else { + return Ok(settings); + }; + if changed == Value::Bool(false) { + return Ok(settings); + } + settings.insert(REACH_SETTING.to_owned(), Value::from(JOB_RUN_REACH)); + if changed == Value::Bool(true) && configured_changes(project_dir)?.counts.is_none() { + settings.insert( + CHANGED_SETTING.to_owned(), + Value::from(JOB_RUN_COUNTS.to_vec()), + ); + } + Ok(settings) +} diff --git a/crates/sqlbuild-python-project/src/job_settings/_helpers/mod.rs b/crates/sqlbuild-python-project/src/job_settings/_helpers/mod.rs new file mode 100644 index 0000000000..e799fed501 --- /dev/null +++ b/crates/sqlbuild-python-project/src/job_settings/_helpers/mod.rs @@ -0,0 +1,5 @@ +pub(crate) mod argv; +pub(crate) mod could_name_job; +pub(crate) mod job_catalog; +pub(crate) mod job_run_settings; +pub(crate) mod named_job_build_settings; diff --git a/crates/sqlbuild-python-project/src/job_settings/_helpers/named_job_build_settings.rs b/crates/sqlbuild-python-project/src/job_settings/_helpers/named_job_build_settings.rs new file mode 100644 index 0000000000..a368878aaa --- /dev/null +++ b/crates/sqlbuild-python-project/src/job_settings/_helpers/named_job_build_settings.rs @@ -0,0 +1,34 @@ +//! The build-wide settings the named jobs give a build. + +use std::collections::BTreeMap; +use std::path::Path; + +use pyo3::IntoPyObject; +use pyo3::prelude::{PyResult, Python}; +use serde_json::Value; +use sqlbuild_analysis::job_settings::main::build_job_settings::build_job_settings; +use sqlbuild_analysis::job_settings::main::named_jobs::named_jobs; + +use crate::job_settings::_helpers::job_run_settings::job_run_settings; +use crate::job_settings::main::job_declarations::job_declarations; +use crate::job_settings::main::job_setting_conflict::job_setting_conflict; +use crate::job_settings::main::with_job_scope::with_job_scope; + +/// The build-wide settings the named jobs give a build, as their runs take them, or `S027`. +pub(crate) fn named_job_build_settings( + py: Python<'_>, + selectors: &[String], + (directory, selected_target): (&Path, Option), +) -> PyResult> { + let scope = (directory.to_path_buf(), selected_target); + let Some(declared) = with_job_scope(scope, || job_declarations(py))? else { + return Ok(BTreeMap::new()); + }; + let named: Vec = named_jobs(selectors, &declared); + if named.is_empty() { + return Ok(BTreeMap::new()); + } + let agreed: BTreeMap = build_job_settings(&named, &declared) + .map_err(|conflict| job_setting_conflict(py, &conflict))?; + job_run_settings(&directory.into_pyobject(py)?.into_any(), agreed) +} diff --git a/crates/sqlbuild-python-project/src/job_settings/constants.rs b/crates/sqlbuild-python-project/src/job_settings/constants.rs new file mode 100644 index 0000000000..bf826724b1 --- /dev/null +++ b/crates/sqlbuild-python-project/src/job_settings/constants.rs @@ -0,0 +1,17 @@ +//! Constants of named job settings. + +/// Ends the options; job flags go before it so they are never read as selectors. +pub const OPTIONS_TERMINATOR: &str = "--"; +/// The commands whose selections take the settings of the jobs they name. +pub const JOB_SETTING_COMMANDS: [&str; 2] = ["build", "plan"]; +/// A SQL unit test's published kind; a frozen run selects it back with `test:`. +pub const SQL_TEST_NODE_KIND: &str = "sql_test"; +/// The cursor settings a frozen run's window replaces. +pub const WINDOW_CURSOR_SETTINGS: [&str; 6] = [ + "start_cursor", + "end_cursor", + "start_cursor_ts", + "end_cursor_ts", + "start_cursor_int", + "end_cursor_int", +]; diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/declared_jobs.rs b/crates/sqlbuild-python-project/src/job_settings/main/declared_jobs.rs similarity index 91% rename from crates/sqlbuild-python-project/src/compiled_project/main/declared_jobs.rs rename to crates/sqlbuild-python-project/src/job_settings/main/declared_jobs.rs index 6510eb7a92..cbbe081979 100644 --- a/crates/sqlbuild-python-project/src/compiled_project/main/declared_jobs.rs +++ b/crates/sqlbuild-python-project/src/job_settings/main/declared_jobs.rs @@ -1,6 +1,6 @@ //! Every job and group a project declares, for resolving `job(...)` dependencies. -use crate::compiled_project::_helpers::job_catalog::catalog_where; +use crate::job_settings::_helpers::job_catalog::catalog_where; use pyo3::prelude::{Bound, PyAny, PyResult}; use sqlbuild_analysis::graph::models::JobSelection; use sqlbuild_discovery::orchestration_files::models::OrchestrationInventory; diff --git a/crates/sqlbuild-python-project/src/job_settings/main/frozen_run_flags.rs b/crates/sqlbuild-python-project/src/job_settings/main/frozen_run_flags.rs new file mode 100644 index 0000000000..b44d399673 --- /dev/null +++ b/crates/sqlbuild-python-project/src/job_settings/main/frozen_run_flags.rs @@ -0,0 +1,49 @@ +//! The command-line flags of a frozen run's settings. + +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use sqlbuild_cli::argv::main::command_settings_arguments::command_settings_arguments; +use sqlbuild_cli::argv::main::job_run_arguments::job_run_arguments; +use sqlbuild_cli::argv::main::job_settings_arguments::job_settings_arguments; +use sqlbuild_orchestration::runs::models::{RunCommand, RunRequest, RunWindow}; + +use crate::job_settings::_helpers::job_run_settings::job_run_settings; +use crate::job_settings::constants::WINDOW_CURSOR_SETTINGS; +use crate::job_settings::models::FrozenRunFlags; + +/// A frozen run's settings with its window's cursors, and the flags a worker passes for them. +pub fn frozen_run_flags( + frozen: &RunRequest, + project_dir: &Bound<'_, PyAny>, +) -> PyResult { + let mut settings = frozen.settings.clone(); + if let Some(window) = &frozen.window { + for key in WINDOW_CURSOR_SETTINGS { + settings.remove(key); + } + let (kind, start, end) = match window { + RunWindow::Timestamp { start, end } => ("ts", start.to_rfc3339(), end.to_rfc3339()), + RunWindow::Integer { start, end } => ("int", start.to_string(), end.to_string()), + }; + settings.insert(format!("start_cursor_{kind}"), serde_json::json!(start)); + settings.insert(format!("end_cursor_{kind}"), serde_json::json!(end)); + } + if frozen.job_key.is_some() { + settings = job_run_settings(project_dir, settings)?; + } + let job_build: Option = frozen + .job_key + .clone() + .filter(|_| frozen.command == RunCommand::Build); + let flags: Vec = match &job_build { + Some(job) => job_run_arguments(job, &settings), + None if frozen.command == RunCommand::Build => job_settings_arguments(&settings), + None => command_settings_arguments(frozen.command.as_str(), &settings), + } + .map_err(PyValueError::new_err)?; + Ok(FrozenRunFlags { + settings, + flags, + job_build, + }) +} diff --git a/crates/sqlbuild-python-project/src/job_settings/main/frozen_selectors.rs b/crates/sqlbuild-python-project/src/job_settings/main/frozen_selectors.rs new file mode 100644 index 0000000000..fb994a27e4 --- /dev/null +++ b/crates/sqlbuild-python-project/src/job_settings/main/frozen_selectors.rs @@ -0,0 +1,35 @@ +//! The selectors that select a frozen run's nodes back. + +use std::collections::BTreeMap; + +use pyo3::exceptions::PyValueError; +use pyo3::prelude::PyResult; +use sqlbuild_analysis::definition_changes::constants::PYTHON_NODE_KINDS; +use sqlbuild_orchestration::runs::models::RunRequest; + +use crate::job_settings::constants::SQL_TEST_NODE_KIND; + +/// One selector per frozen node the run has not skipped. +pub fn frozen_selectors( + frozen: &RunRequest, + skipped_nodes: &BTreeMap, +) -> PyResult> { + let mut selectors: Vec = Vec::new(); + for key in frozen + .selection + .iter() + .filter(|key| !skipped_nodes.contains_key(*key)) + { + let (kind, name) = key + .split_once(':') + .ok_or_else(|| PyValueError::new_err("frozen run contains an invalid node key"))?; + selectors.push(if PYTHON_NODE_KINDS.contains(&kind) { + key.clone() + } else if kind == SQL_TEST_NODE_KIND { + format!("test:{name}") + } else { + name.to_owned() + }); + } + Ok(selectors) +} diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/graph_with_jobs.rs b/crates/sqlbuild-python-project/src/job_settings/main/graph_with_jobs.rs similarity index 92% rename from crates/sqlbuild-python-project/src/compiled_project/main/graph_with_jobs.rs rename to crates/sqlbuild-python-project/src/job_settings/main/graph_with_jobs.rs index e8fda5ea27..ea1a300df4 100644 --- a/crates/sqlbuild-python-project/src/compiled_project/main/graph_with_jobs.rs +++ b/crates/sqlbuild-python-project/src/job_settings/main/graph_with_jobs.rs @@ -1,6 +1,6 @@ //! `graph` with the command's jobs attached when `selectors` name a job, otherwise `graph` itself. -use crate::compiled_project::main::needed_jobs::needed_jobs; +use crate::job_settings::main::needed_jobs::needed_jobs; use pyo3::prelude::{PyResult, Python}; use sqlbuild_analysis::graph::main::attach_jobs::attach_jobs; use sqlbuild_analysis::graph::models::ProjectGraph; diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/job_declarations.rs b/crates/sqlbuild-python-project/src/job_settings/main/job_declarations.rs similarity index 90% rename from crates/sqlbuild-python-project/src/compiled_project/main/job_declarations.rs rename to crates/sqlbuild-python-project/src/job_settings/main/job_declarations.rs index 62e96c1415..9e114f56d5 100644 --- a/crates/sqlbuild-python-project/src/compiled_project/main/job_declarations.rs +++ b/crates/sqlbuild-python-project/src/job_settings/main/job_declarations.rs @@ -1,6 +1,6 @@ //! Each job's `[settings]` and `[retry]` and each group's jobs declared for the command's target. -use crate::compiled_project::_helpers::job_catalog; +use crate::job_settings::_helpers::job_catalog; use pyo3::prelude::{PyResult, Python}; use sqlbuild_analysis::job_settings::models::DeclaredRun; use std::collections::BTreeMap; diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/job_names.rs b/crates/sqlbuild-python-project/src/job_settings/main/job_names.rs similarity index 85% rename from crates/sqlbuild-python-project/src/compiled_project/main/job_names.rs rename to crates/sqlbuild-python-project/src/job_settings/main/job_names.rs index bd3875b222..742094bf8d 100644 --- a/crates/sqlbuild-python-project/src/compiled_project/main/job_names.rs +++ b/crates/sqlbuild-python-project/src/job_settings/main/job_names.rs @@ -1,6 +1,6 @@ //! The command's job and job group names, loaded on first use; empty outside a command. -use crate::compiled_project::_helpers::job_catalog::job_catalog; +use crate::job_settings::_helpers::job_catalog::job_catalog; use pyo3::prelude::{PyResult, Python}; /// The command's job and job group names, loaded on first use; empty outside a command. diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/job_scope_project_dir.rs b/crates/sqlbuild-python-project/src/job_settings/main/job_scope_project_dir.rs similarity index 83% rename from crates/sqlbuild-python-project/src/compiled_project/main/job_scope_project_dir.rs rename to crates/sqlbuild-python-project/src/job_settings/main/job_scope_project_dir.rs index 5861a6f253..a06e3ecf74 100644 --- a/crates/sqlbuild-python-project/src/compiled_project/main/job_scope_project_dir.rs +++ b/crates/sqlbuild-python-project/src/job_settings/main/job_scope_project_dir.rs @@ -1,6 +1,6 @@ //! The project directory of the running command's job scope. -use crate::compiled_project::_helpers::job_catalog; +use crate::job_settings::_helpers::job_catalog; use std::path::PathBuf; /// The project directory of the running command's job scope, if it has one. diff --git a/crates/sqlbuild-python-project/src/job_settings/main/job_selector_error.rs b/crates/sqlbuild-python-project/src/job_settings/main/job_selector_error.rs new file mode 100644 index 0000000000..88e5b7870a --- /dev/null +++ b/crates/sqlbuild-python-project/src/job_settings/main/job_selector_error.rs @@ -0,0 +1,25 @@ +//! The error of a selector that cannot resolve a named job's nodes. + +use pyo3::PyErr; +use pyo3::prelude::Python; +use sqlbuild_analysis::job_settings::errors::JobSelectorError; +use sqlbuild_python_base::records::main::construct::new_error::new_error; +use sqlbuild_python_base::records::types::ErrorId; + +/// A `CliUserError` naming the jobs whose selection a selector failed to resolve, with its code. +pub fn job_selector_error(py: Python<'_>, failure: &JobSelectorError) -> PyErr { + let jobs: String = if failure.jobs.is_empty() { + "the named jobs".to_owned() + } else { + format!("job {}", failure.jobs.join(", ")) + }; + new_error( + py, + ErrorId::CliUser, + &format!( + "cannot apply the settings of {jobs}: selector '{}' failed: {}", + failure.selector, failure.error.message + ), + (Some(failure.error.code), failure.error.help.as_deref()), + ) +} diff --git a/crates/sqlbuild-python-project/src/job_settings/main/job_setting_conflict.rs b/crates/sqlbuild-python-project/src/job_settings/main/job_setting_conflict.rs new file mode 100644 index 0000000000..937dd91119 --- /dev/null +++ b/crates/sqlbuild-python-project/src/job_settings/main/job_setting_conflict.rs @@ -0,0 +1,18 @@ +//! The error of job settings that cannot apply together. + +use pyo3::PyErr; +use pyo3::prelude::Python; +use sqlbuild_analysis::job_settings::constants::JOB_SETTING_CONFLICT_CODE; +use sqlbuild_analysis::job_settings::errors::JobSettingConflict; +use sqlbuild_python_base::records::main::construct::new_error::new_error; +use sqlbuild_python_base::records::types::ErrorId; + +/// `CliUserError` for job settings that cannot apply together. +pub fn job_setting_conflict(py: Python<'_>, conflict: &JobSettingConflict) -> PyErr { + new_error( + py, + ErrorId::CliUser, + &conflict.message, + (Some(JOB_SETTING_CONFLICT_CODE), Some(conflict.help)), + ) +} diff --git a/crates/sqlbuild-python-project/src/job_settings/main/job_settings_argv.rs b/crates/sqlbuild-python-project/src/job_settings/main/job_settings_argv.rs new file mode 100644 index 0000000000..4d489a9898 --- /dev/null +++ b/crates/sqlbuild-python-project/src/job_settings/main/job_settings_argv.rs @@ -0,0 +1,24 @@ +//! The argv a build or plan naming jobs parses again with their build-wide flags. + +use pyo3::exceptions::PyUnicodeError; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::PyDict; + +use crate::job_settings::_helpers::argv::{named_job_argv, reported_when_run}; + +/// The argv to parse again with its named jobs' build-wide flags, or `None` to keep the parse. +pub fn job_settings_argv( + argv: &Bound<'_, PyAny>, + namespace: &Bound<'_, PyDict>, +) -> PyResult>> { + let py: Python<'_> = argv.py(); + let arguments: Vec = match argv.extract() { + Ok(arguments) => arguments, + Err(undecodable) if undecodable.is_instance_of::(py) => return Ok(None), + Err(error) => return Err(error), + }; + match named_job_argv(py, &arguments, namespace) { + Err(error) if reported_when_run(py, &error)? => Ok(None), + result => result, + } +} diff --git a/crates/sqlbuild-python-project/src/job_settings/main/mod.rs b/crates/sqlbuild-python-project/src/job_settings/main/mod.rs new file mode 100644 index 0000000000..347338ec62 --- /dev/null +++ b/crates/sqlbuild-python-project/src/job_settings/main/mod.rs @@ -0,0 +1,15 @@ +pub mod declared_jobs; +pub mod frozen_run_flags; +pub mod frozen_selectors; +pub mod graph_with_jobs; +pub mod job_declarations; +pub mod job_names; +pub mod job_scope_project_dir; +pub mod job_selector_error; +pub mod job_setting_conflict; +pub mod job_settings_argv; +pub mod named_jobs; +pub mod needed_jobs; +pub mod triggered_jobs; +pub mod with_frozen_job; +pub mod with_job_scope; diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/named_jobs.rs b/crates/sqlbuild-python-project/src/job_settings/main/named_jobs.rs similarity index 76% rename from crates/sqlbuild-python-project/src/compiled_project/main/named_jobs.rs rename to crates/sqlbuild-python-project/src/job_settings/main/named_jobs.rs index b79ebfbee1..5fc7884c95 100644 --- a/crates/sqlbuild-python-project/src/compiled_project/main/named_jobs.rs +++ b/crates/sqlbuild-python-project/src/job_settings/main/named_jobs.rs @@ -1,7 +1,7 @@ //! The jobs a command's selectors name, in the order named. -use crate::compiled_project::_helpers::job_catalog::job_catalog; -use crate::compiled_project::_helpers::job_catalog::ordered_jobs; +use crate::job_settings::_helpers::job_catalog::job_catalog; +use crate::job_settings::_helpers::job_catalog::ordered_jobs; use pyo3::prelude::{PyResult, Python}; /// The jobs `selectors` name, as `job:` or a bare job name, each once in the order named; none outside a command. diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/needed_jobs.rs b/crates/sqlbuild-python-project/src/job_settings/main/needed_jobs.rs similarity index 91% rename from crates/sqlbuild-python-project/src/compiled_project/main/needed_jobs.rs rename to crates/sqlbuild-python-project/src/job_settings/main/needed_jobs.rs index 949d525f66..25378c194d 100644 --- a/crates/sqlbuild-python-project/src/compiled_project/main/needed_jobs.rs +++ b/crates/sqlbuild-python-project/src/job_settings/main/needed_jobs.rs @@ -1,6 +1,6 @@ //! The command's jobs when `selectors` name one and `graph` has none attached yet. -use crate::compiled_project::_helpers::job_catalog::job_catalog; +use crate::job_settings::_helpers::job_catalog::job_catalog; use pyo3::prelude::{PyResult, Python}; use sqlbuild_analysis::graph::main::selectors_need_jobs::selectors_need_jobs; use sqlbuild_analysis::graph::models::{JobSelection, ProjectGraph}; diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/triggered_jobs.rs b/crates/sqlbuild-python-project/src/job_settings/main/triggered_jobs.rs similarity index 91% rename from crates/sqlbuild-python-project/src/compiled_project/main/triggered_jobs.rs rename to crates/sqlbuild-python-project/src/job_settings/main/triggered_jobs.rs index 741f09e695..075d6e339c 100644 --- a/crates/sqlbuild-python-project/src/compiled_project/main/triggered_jobs.rs +++ b/crates/sqlbuild-python-project/src/job_settings/main/triggered_jobs.rs @@ -1,7 +1,7 @@ //! Each job a schedule or sensor starts with the nodes it refreshes, less those of jobs `selectors` run. -use crate::compiled_project::_helpers::job_catalog::job_catalog; -use crate::compiled_project::_helpers::job_catalog::selected_jobs; +use crate::job_settings::_helpers::job_catalog::job_catalog; +use crate::job_settings::_helpers::job_catalog::selected_jobs; use pyo3::prelude::{PyResult, Python}; use sqlbuild_analysis::assembly::project::types::ObjectKey; use sqlbuild_analysis::graph::main::attach_jobs::attach_jobs; diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/with_frozen_job.rs b/crates/sqlbuild-python-project/src/job_settings/main/with_frozen_job.rs similarity index 90% rename from crates/sqlbuild-python-project/src/compiled_project/main/with_frozen_job.rs rename to crates/sqlbuild-python-project/src/job_settings/main/with_frozen_job.rs index 126554ac7e..9c8d428b6e 100644 --- a/crates/sqlbuild-python-project/src/compiled_project/main/with_frozen_job.rs +++ b/crates/sqlbuild-python-project/src/job_settings/main/with_frozen_job.rs @@ -1,6 +1,6 @@ //! Run one command with only one job declared, as its frozen run pins it. -use crate::compiled_project::_helpers::job_catalog; +use crate::job_settings::_helpers::job_catalog; use serde_json::Value; use std::collections::BTreeMap; use std::path::PathBuf; diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/with_job_scope.rs b/crates/sqlbuild-python-project/src/job_settings/main/with_job_scope.rs similarity index 89% rename from crates/sqlbuild-python-project/src/compiled_project/main/with_job_scope.rs rename to crates/sqlbuild-python-project/src/job_settings/main/with_job_scope.rs index bc8fde79ad..0e33891f3e 100644 --- a/crates/sqlbuild-python-project/src/compiled_project/main/with_job_scope.rs +++ b/crates/sqlbuild-python-project/src/job_settings/main/with_job_scope.rs @@ -1,6 +1,6 @@ //! Run one command with jobs selectable from `project_dir`, restoring the enclosing scope after. -use crate::compiled_project::_helpers::job_catalog; +use crate::job_settings::_helpers::job_catalog; use std::path::PathBuf; /// Run one command with jobs selectable from `project_dir`, restoring the enclosing scope after. diff --git a/crates/sqlbuild-python-project/src/job_settings/mod.rs b/crates/sqlbuild-python-project/src/job_settings/mod.rs new file mode 100644 index 0000000000..6be20307a7 --- /dev/null +++ b/crates/sqlbuild-python-project/src/job_settings/mod.rs @@ -0,0 +1,6 @@ +//! Named jobs: the catalog a command's selectors name jobs from, and the settings jobs give it. + +pub(crate) mod _helpers; +pub mod constants; +pub mod main; +pub mod models; diff --git a/crates/sqlbuild-python-project/src/job_settings/models.rs b/crates/sqlbuild-python-project/src/job_settings/models.rs new file mode 100644 index 0000000000..0a6c3ff976 --- /dev/null +++ b/crates/sqlbuild-python-project/src/job_settings/models.rs @@ -0,0 +1,13 @@ +//! Models of named job settings. + +use std::collections::BTreeMap; + +use serde_json::Value; + +/// A frozen run's settings, the command-line flags they become, and the job a job's build names. +#[derive(Debug)] +pub struct FrozenRunFlags { + pub settings: BTreeMap, + pub flags: Vec, + pub job_build: Option, +} diff --git a/crates/sqlbuild-python-project/src/lib.rs b/crates/sqlbuild-python-project/src/lib.rs index 7af797d10d..fb136cbe39 100644 --- a/crates/sqlbuild-python-project/src/lib.rs +++ b/crates/sqlbuild-python-project/src/lib.rs @@ -11,6 +11,7 @@ pub mod compiled_views; pub mod contracts; pub mod fingerprints; pub mod graph; +pub mod job_settings; pub mod lineage; pub mod native_host; pub mod planner_identity; diff --git a/crates/sqlbuild-python-project/src/native_host/_helpers/runs_state.rs b/crates/sqlbuild-python-project/src/native_host/_helpers/runs_state.rs index effdc06fcc..e5e40d9abd 100644 --- a/crates/sqlbuild-python-project/src/native_host/_helpers/runs_state.rs +++ b/crates/sqlbuild-python-project/src/native_host/_helpers/runs_state.rs @@ -1,6 +1,6 @@ //! Helpers of the runs table recorder: its DDL, the run's identity facts and its row cells. -use crate::compiled_project::main::named_jobs::named_jobs; +use crate::job_settings::main::named_jobs::named_jobs; use crate::native_host::models::RunCommandScope; use crate::native_host::models::RunIdentity; use pyo3::exceptions::{PyKeyError, PyOSError, PyValueError}; diff --git a/crates/sqlbuild-python-project/src/planner_selection/_helpers/unit_tests.rs b/crates/sqlbuild-python-project/src/planner_selection/_helpers/unit_tests.rs index 600f939727..7f205bcb85 100644 --- a/crates/sqlbuild-python-project/src/planner_selection/_helpers/unit_tests.rs +++ b/crates/sqlbuild-python-project/src/planner_selection/_helpers/unit_tests.rs @@ -13,7 +13,7 @@ use sqlbuild_analysis::planner_selection::models::{ SelectorGroups, UnitTestSelectorNames, UnitTestSelectorSplit, }; -use crate::compiled_project::main::job_names::job_names; +use crate::job_settings::main::job_names::job_names; use crate::planner_selection::models::Carrier; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; diff --git a/crates/sqlbuild-python-project/src/planner_selection/main/match_project_selector.rs b/crates/sqlbuild-python-project/src/planner_selection/main/match_project_selector.rs index fd2804e314..b579d93dbb 100644 --- a/crates/sqlbuild-python-project/src/planner_selection/main/match_project_selector.rs +++ b/crates/sqlbuild-python-project/src/planner_selection/main/match_project_selector.rs @@ -1,8 +1,8 @@ //! The keys one parsed selector matches before `+` graph expansion. -use crate::compiled_project::main::graph_with_jobs::graph_with_jobs; use crate::graph::models::GraphViewMappings; use crate::graph::models::NativeProjectGraph; +use crate::job_settings::main::graph_with_jobs::graph_with_jobs; use crate::planner_selection::_helpers::selectors::graph_names; use crate::planner_selection::_helpers::selectors::selected; use crate::planner_selection::_helpers::selectors::view_graph; diff --git a/crates/sqlbuild-python-project/src/planner_selection/main/resolve_graph_keys.rs b/crates/sqlbuild-python-project/src/planner_selection/main/resolve_graph_keys.rs index c52bfcc3c6..a0471e7c14 100644 --- a/crates/sqlbuild-python-project/src/planner_selection/main/resolve_graph_keys.rs +++ b/crates/sqlbuild-python-project/src/planner_selection/main/resolve_graph_keys.rs @@ -1,6 +1,6 @@ //! Resolve graph keys for native planning without building a public frozenset. -use crate::compiled_project::main::graph_with_jobs::graph_with_jobs; +use crate::job_settings::main::graph_with_jobs::graph_with_jobs; use crate::planner_selection::_helpers::selectors::graph_names; use crate::planner_selection::main::py_strings::py_strings; use crate::planner_selection::models::Carrier; diff --git a/crates/sqlbuild-python-project/src/planner_selection/main/resolve_project_selector_tokens.rs b/crates/sqlbuild-python-project/src/planner_selection/main/resolve_project_selector_tokens.rs index b480f0018a..77d5c3e88e 100644 --- a/crates/sqlbuild-python-project/src/planner_selection/main/resolve_project_selector_tokens.rs +++ b/crates/sqlbuild-python-project/src/planner_selection/main/resolve_project_selector_tokens.rs @@ -1,7 +1,7 @@ //! The keys whitespace-separated selector tokens match, without required build resources. -use crate::compiled_project::main::graph_with_jobs::graph_with_jobs; use crate::graph::models::NativeProjectGraph; +use crate::job_settings::main::graph_with_jobs::graph_with_jobs; use crate::planner_selection::_helpers::selectors::VIEW_OPTIONS; use crate::planner_selection::_helpers::selectors::arguments_graph; use crate::planner_selection::_helpers::selectors::graph_names; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs index 1c84fe89c8..d913efaefb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs @@ -4,7 +4,7 @@ use pyo3::IntoPyObject; use pyo3::exceptions::PyValueError; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods}; -use sqlbuild_config::effective::models::{ConcurrencySource, EffectiveConcurrency}; +use sqlbuild_config::effective::models::EffectiveConcurrency; use sqlbuild_core::build_command::main::render_run_context::render_run_context; use sqlbuild_core::build_command::models::{RunContextFacts, SelectorFileFact}; use sqlbuild_core::cli_text::models::CliStyle; @@ -43,6 +43,9 @@ use sqlbuild_python_base::records::main::construct::new_record::new_record; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_compile::compile_driver::main::jobs::build_concurrency::build_concurrency; +use sqlbuild_python_compile::compile_driver::main::jobs::connection_limit::connection_limit; +use sqlbuild_python_compile::compile_driver::main::jobs::job_run_context::job_run_context; use sqlbuild_python_compile::compile_driver::main::setup::build_project_relation_targets::build_project_relation_targets; use sqlbuild_python_compile::compile_driver::main::setup::project_target_settings::project_target_settings; use sqlbuild_python_compile::compile_driver::models::TargetSettingsValues; @@ -65,9 +68,7 @@ use sqlbuild_python_planner::consumption::models::ConsumptionRecorder; use sqlbuild_python_planner::consumption::models::Prepared; use sqlbuild_python_planner::freshness::main::append_freshness_records::append_freshness_records; use sqlbuild_python_planner::observations::main::append_build_observations::append_build_observations; -use sqlbuild_python_planner::plan_output::main::job_warehouses::job_warehouses; use sqlbuild_python_planner::planner_resolution::main::cursor::type_cursor_overrides::type_cursor_overrides; -use sqlbuild_python_project::compiled_project::constants::JOB_FULL_REFRESH_KEY; use sqlbuild_python_project::graph::main::python_nodes::python_node_graph::python_node_graph; use sqlbuild_state::consumption::models::ChangedPolicy; @@ -167,6 +168,7 @@ fn run_context_facts( selector_count: native_field(&file, "selector_count")?.extract()?, }); } + let (full_refresh_jobs, job_warehouses) = job_run_context(pipeline)?; Ok(RunContextFacts { command: EXECUTION_COMMAND.to_owned(), run_id: display_text(&native_field(&project, "run_id")?)?, @@ -177,11 +179,8 @@ fn run_context_facts( concurrency: concurrency.limit, concurrency_source: concurrency.source.to_string(), full_refresh: context.request_flag("full_refresh")?, - full_refresh_jobs: full_refresh_jobs(pipeline)?, - job_warehouses: job_warehouses(&native_field( - &native_field(pipeline, "plan_output")?, - "metadata", - )?)?, + full_refresh_jobs, + job_warehouses, selected_count, total_count, date_vars: ( @@ -192,17 +191,6 @@ fn run_context_facts( }) } -/// The named jobs whose own models the planned build rebuilds from scratch. -fn full_refresh_jobs(pipeline: &Bound<'_, PyAny>) -> PyResult> { - let metadata: Bound<'_, PyAny> = - native_field(&native_field(pipeline, "plan_output")?, "metadata")?; - let jobs: Bound<'_, PyAny> = mapping_get(&metadata, JOB_FULL_REFRESH_KEY)?; - if jobs.is_none() { - return Ok(Vec::new()); - } - jobs.try_iter()?.map(|job| job?.extract()).collect() -} - /// Write the resolved non-secret placement, mode and selection context. pub(crate) fn write_run_context( context: &BuildContext<'_>, @@ -273,10 +261,7 @@ pub(crate) fn prepare_build_execution<'py>( }) .and_then(|value| pyo3::Bound::new(py, value)) .map(pyo3::Bound::into_any)?; - let concurrency = connection_limit( - build_concurrency(context.concurrency()?, pipeline)?.limit, - pipeline, - )?; + let concurrency = connection_limit(context.concurrency()?, pipeline)?; if context.verbose { write_run_context(context, pipeline)?; } else { @@ -331,50 +316,6 @@ pub(crate) fn prepare_build_execution<'py>( }) } -/// The named jobs' `concurrency` limits added together; none without a command-line override. -fn job_limit_total(pipeline: &Bound<'_, PyAny>) -> PyResult> { - let limits: Bound<'_, PyDict> = native_field(pipeline, "job_limits")?.cast_into()?; - if limits.is_empty() { - return Ok(None); - } - let mut total: i64 = 0; - for value in limits.values() { - total = total.saturating_add(value.extract::()?); - } - Ok(Some(total)) -} - -/// The build's limit: the configured one, raised to the named jobs' limits added together. -fn build_concurrency( - configured: EffectiveConcurrency, - pipeline: &Bound<'_, PyAny>, -) -> PyResult { - Ok(match job_limit_total(pipeline)? { - Some(total) if total > configured.limit => EffectiveConcurrency { - limit: total, - source: ConcurrencySource::NamedJobs, - }, - _ => configured, - }) -} - -/// The build's connections: its limit, or the named jobs' limits added together when every planned -/// node belongs to one of them, so a job never opens connections it cannot use. -fn connection_limit(limit: i64, pipeline: &Bound<'_, PyAny>) -> PyResult { - let Some(total) = job_limit_total(pipeline)? else { - return Ok(limit); - }; - let node_jobs: Bound<'_, PyDict> = native_field(pipeline, "node_jobs")?.cast_into()?; - for key in - native_field(&native_field(pipeline, "plan_output")?, "execution_order")?.try_iter()? - { - if !node_jobs.contains(key?)? { - return Ok(limit); - } - } - Ok(limit.min(total.max(1))) -} - fn verbose_callback<'py>( context: &BuildContext<'py>, callbacks: &Bound<'py, PyAny>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs index af968d135f..4719e205d5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs @@ -1,6 +1,6 @@ //! Ingress Python before the build and read-side tasks and assets dispatched as SQL completes. -use std::collections::{HashMap, HashSet}; +use std::collections::{BTreeMap, HashMap, HashSet}; use std::sync::Mutex; use pyo3::exceptions::PyRuntimeError; @@ -345,7 +345,7 @@ struct RuntimeParts<'a, 'py> { relations: (Bound<'py, PyAny>, Bound<'py, PyAny>), inputs: &'a LifecycleInputs<'py>, /// Each Python node's job warehouse, by name. - warehouses: Bound<'py, PyAny>, + warehouses: BTreeMap, } fn node_runtime<'py>( @@ -377,7 +377,10 @@ fn node_runtime<'py>( ("relation_targets", parts.relations.0.clone()), ("project_relations", parts.relations.1.clone()), ("providers", parts.inputs.providers.clone()), - ("node_warehouses", parts.warehouses.clone()), + ( + "node_warehouses", + pyo3::IntoPyObject::into_pyobject(parts.warehouses.clone(), py)?.into_any(), + ), ], ) } @@ -531,7 +534,7 @@ pub(crate) fn prepare_direct_python_lifecycle<'py>( target_config, relations: (relation_targets, project_relations), inputs, - warehouses: native_field(pipeline, "python_node_warehouses")?, + warehouses: native_field(pipeline, "python_node_warehouses")?.extract()?, }; let mut ingress_python: Bound<'py, PyAny> = tuple(py, &[])?; let mut ingress_loads: Bound<'py, PyAny> = tuple(py, &[])?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs index 7627073e23..dbb257ea33 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs @@ -56,7 +56,7 @@ use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_compile::compile_command::main::invocation::run_compile::run_compile; use sqlbuild_python_front::discovery::models::MissingSecretsScope; use sqlbuild_python_output::recall::models::InvocationRecall; -use sqlbuild_python_project::compiled_project::main::with_job_scope::with_job_scope; +use sqlbuild_python_project::job_settings::main::with_job_scope::with_job_scope; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::main::within::within; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs index 3ee4eb0383..633587ddeb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs @@ -25,7 +25,7 @@ use crate::bindings::_helpers::cli::dispatch::{command_facts, dispatch_command}; use crate::bindings::_helpers::cli::failure_cause::take_failure_cause; use sqlbuild_python_base::cli::main::offer_launch_record::offer_launch_record; use crate::bindings::_helpers::cli::machine_output::take_machine_output_written; -use crate::bindings::_helpers::cli::named_job_settings::job_settings_argv; +use sqlbuild_python_project::job_settings::main::job_settings_argv::job_settings_argv; use crate::bindings::_helpers::cli::observability::{ CliObservabilityScope, cli_observability_scope, }; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/mod.rs index 71f400a1b8..7a3ebc8d8c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/mod.rs @@ -3,7 +3,6 @@ pub(crate) mod dispatch; pub(crate) mod entry; pub(crate) mod failure_cause; pub(crate) mod machine_output; -pub(crate) mod named_job_settings; pub(crate) mod namespace; pub(crate) mod observability; pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/mod.rs index b6d5c2096f..4a1a0e691a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/mod.rs @@ -1,4 +1,3 @@ -pub(crate) mod job_node_settings; pub(crate) mod plan_pipeline; pub(crate) mod plan_result; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs index ede1ec0720..4ec8868c8b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs @@ -10,9 +10,11 @@ use sqlbuild_python_compile::compile_driver::main::stages::resolve_deferred_targ use sqlbuild_python_compile::compile_driver::main::setup::build_project_graph_from_compiled_project::build_project_graph_from_compiled_project; use sqlbuild_python_compile::compile_driver::main::setup::build_python_run_plan_outputs::build_python_run_plan_outputs; use sqlbuild_python_compile::compile_driver::main::setup::python_read_source_names_for::python_read_source_names_for; -use crate::bindings::_helpers::compile_driver::job_node_settings::{ - PlannedJobSettings, planned_job_settings, -}; +use crate::bindings::_helpers::command_runtime::warehouses::command_scope; +use sqlbuild_python_compile::compile_driver::main::jobs::job_override_fields::job_override_fields; +use sqlbuild_python_compile::compile_driver::main::jobs::job_plan_fields::job_plan_fields; +use sqlbuild_python_compile::compile_driver::main::jobs::planned_job_settings::planned_job_settings; +use sqlbuild_python_compile::compile_driver::models::PlannedJobSettings; use sqlbuild_analysis::planner_scope::constants::SOURCE; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; @@ -158,8 +160,9 @@ pub(crate) fn pipeline_result<'py>( ("sql_test_selection", sql_test_selection), ], )?; + let cli_warehouse: bool = command_scope(py)?.cli_warehouse.is_some(); let job_settings: Option = - planned_job_settings((project, discovered), options)?; + planned_job_settings((project, discovered), (options, cli_warehouse))?; let mut override_fields: Vec<(&str, Bound<'py, PyAny>)> = option_fields( options, &[ @@ -170,12 +173,7 @@ pub(crate) fn pipeline_result<'py>( "concurrency", ], )?; - if let Some(job_settings) = &job_settings { - override_fields.push(( - "full_refresh_models", - job_settings.full_refresh_models(py)?.into_any(), - )); - } + override_fields.extend(job_override_fields(py, job_settings.as_ref())?); let overrides: Bound<'py, PyAny> = new_record_with(py, RecordId::PlannerOverrides, &override_fields)?; let mut deferral_fields: Vec<(&str, Bound<'py, PyAny>)> = vec![ @@ -227,24 +225,7 @@ pub(crate) fn pipeline_result<'py>( ], )?), )?; - let mut job_fields: Vec<(&str, Bound<'py, PyAny>)> = Vec::new(); - let plan_output: Bound<'py, PyAny> = match &job_settings { - Some(job_settings) => { - let plan_output: Bound<'py, PyAny> = job_settings.apply_to_plan( - &plan_output, - &native_field(options, "cli_ignore_audit_failures")?, - )?; - let (limits, node_jobs) = - job_settings.job_limits(&plan_output, &native_field(options, "concurrency")?)?; - job_fields.push(("job_limits", limits.into_any())); - job_fields.push(("node_jobs", node_jobs.into_any())); - let (nodes, python) = job_settings.node_warehouses(&plan_output)?; - job_fields.push(("node_warehouses", nodes.into_any())); - job_fields.push(("python_node_warehouses", python.into_any())); - plan_output - } - None => plan_output, - }; + let (plan_output, job_fields) = job_plan_fields(job_settings.as_ref(), plan_output, options)?; let custom_materializations: Bound<'py, PyAny> = host_entry(py, UserCodeHost::LoadCustomMaterializations)? .call1((native_field(discovered, "materialization_files")?,))?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/job_settings.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/job_settings.rs deleted file mode 100644 index 34d361801e..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/job_settings.rs +++ /dev/null @@ -1,104 +0,0 @@ -//! The settings named jobs give a command: the build-wide ones as the flags the command line left out. - -use std::collections::BTreeMap; -use std::path::Path; - -use pyo3::IntoPyObject; -use pyo3::PyErr; -use pyo3::prelude::{Bound, PyAny, PyResult, Python}; -use serde_json::Value; -use sqlbuild_analysis::graph::main::parse_selector::parse_selector; -use sqlbuild_analysis::graph::models::ParsedSelector; -use sqlbuild_analysis::job_settings::constants::JOB_SETTING_CONFLICT_CODE; -use sqlbuild_analysis::job_settings::errors::{JobSelectorError, JobSettingConflict}; -use sqlbuild_analysis::job_settings::main::build_job_settings::build_job_settings; -use sqlbuild_analysis::job_settings::main::named_jobs::named_jobs; - -use sqlbuild_python_base::records::main::construct::new_error::new_error; -use sqlbuild_python_base::records::types::ErrorId; -use sqlbuild_python_front::discovery::main::project::configured_changes::configured_changes; -use sqlbuild_python_project::compiled_project::constants::{ - CHANGED_SETTING, JOB_RUN_COUNTS, JOB_RUN_REACH, JOB_SELECTOR_KINDS, REACH_SETTING, - SELECTOR_INTERSECTION, -}; -use sqlbuild_python_project::compiled_project::main::job_declarations::job_declarations; -use sqlbuild_python_project::compiled_project::main::with_job_scope::with_job_scope; - -/// A job run's `--changed` settings: its whole selection, and source data counted unless configured. -pub(crate) fn job_run_settings( - project_dir: &Bound<'_, PyAny>, - mut settings: BTreeMap, -) -> PyResult> { - let Some(changed) = settings.get(CHANGED_SETTING).cloned() else { - return Ok(settings); - }; - if changed == Value::Bool(false) { - return Ok(settings); - } - settings.insert(REACH_SETTING.to_owned(), Value::from(JOB_RUN_REACH)); - if changed == Value::Bool(true) && configured_changes(project_dir)?.counts.is_none() { - settings.insert( - CHANGED_SETTING.to_owned(), - Value::from(JOB_RUN_COUNTS.to_vec()), - ); - } - Ok(settings) -} - -/// The build-wide settings the named jobs give a build, as their runs take them, or `S027`. -pub(crate) fn named_job_build_settings( - py: Python<'_>, - selectors: &[String], - (directory, selected_target): (&Path, Option), -) -> PyResult> { - let scope = (directory.to_path_buf(), selected_target); - let Some(declared) = with_job_scope(scope, || job_declarations(py))? else { - return Ok(BTreeMap::new()); - }; - let named: Vec = named_jobs(selectors, &declared); - if named.is_empty() { - return Ok(BTreeMap::new()); - } - let agreed: BTreeMap = build_job_settings(&named, &declared) - .map_err(|conflict| job_setting_conflict(py, &conflict))?; - job_run_settings(&directory.into_pyobject(py)?.into_any(), agreed) -} - -/// Whether a selector has an atom that can name a job: `job:`, or a bare name. -pub(crate) fn could_name_job(selector: &str) -> bool { - selector - .split_whitespace() - .flat_map(|token| token.split(SELECTOR_INTERSECTION)) - .any(|atom| match parse_selector(atom) { - Ok(ParsedSelector::Kind { kind, .. }) => JOB_SELECTOR_KINDS.contains(&kind.as_str()), - _ => false, - }) -} - -/// `CliUserError` for job settings that cannot apply together. -pub(crate) fn job_setting_conflict(py: Python<'_>, conflict: &JobSettingConflict) -> PyErr { - new_error( - py, - ErrorId::CliUser, - &conflict.message, - (Some(JOB_SETTING_CONFLICT_CODE), Some(conflict.help)), - ) -} - -/// A `CliUserError` naming the jobs whose selection a selector failed to resolve, with its code. -pub(crate) fn job_selector_error(py: Python<'_>, failure: &JobSelectorError) -> PyErr { - let jobs: String = if failure.jobs.is_empty() { - "the named jobs".to_owned() - } else { - format!("job {}", failure.jobs.join(", ")) - }; - new_error( - py, - ErrorId::CliUser, - &format!( - "cannot apply the settings of {jobs}: selector '{}' failed: {}", - failure.selector, failure.error.message - ), - (Some(failure.error.code), failure.error.help.as_deref()), - ) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/mod.rs index 878e48e2ae..7444bd0374 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/mod.rs @@ -1,2 +1 @@ -pub(crate) mod job_settings; pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/constants.rs index d132719e93..10a6c67240 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/constants.rs @@ -22,8 +22,6 @@ pub(crate) const SOURCE_NODE_KIND: &str = "source"; /// DuckDB's message when another process, such as a running build, holds the database file. pub(crate) const DUCKDB_LOCK_CONFLICT: &str = "Conflicting lock is held"; -/// A SQL unit test's published kind; a frozen run selects it back with `test:`. -pub(crate) const SQL_TEST_NODE_KIND: &str = "sql_test"; /// The setting-row field the published providers document leaves out: secret matching reads it, the UI does not. pub(crate) const SNAPSHOT_HIDDEN_SETTING: &str = "alias"; /// The runtime request key of a plan preview: `{"plan": {"argv", "copy"}}`. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs index d829d5e3a5..8005156b99 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs @@ -9,15 +9,11 @@ use pyo3::exceptions::{PyRuntimeError, PyValueError}; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::PyString; use serde_json::Value; -use sqlbuild_analysis::definition_changes::constants::PYTHON_NODE_KINDS; -use sqlbuild_cli::argv::main::command_settings_arguments::command_settings_arguments; -use sqlbuild_cli::argv::main::job_run_arguments::job_run_arguments; -use sqlbuild_cli::argv::main::job_settings_arguments::job_settings_arguments; use sqlbuild_cli::dispatch::main::command_dispatch::command_dispatch; use sqlbuild_cli::dispatch::models::Dispatch; use sqlbuild_host::entries::models::ExecutorHost; use sqlbuild_orchestration::runs::models::{ - RunChange, RunCommand, RunKind, RunOutcome, RunRequest, RunWindow, + RunChange, RunCommand, RunKind, RunOutcome, RunRequest, }; use sqlbuild_orchestration::snapshots::main::reads::node_hashes::node_hashes; use sqlbuild_orchestration::snapshots::models::SnapshotInput; @@ -31,12 +27,10 @@ use crate::bindings::_helpers::cli::dispatch::warehouse_scope; use crate::bindings::_helpers::command_runtime::warehouses::WarehouseInputs; use crate::bindings::_helpers::command_runtime::warehouses::configured_warehouses; use crate::bindings::_helpers::compile_driver::rules_at_load::RulesAtLoadScope; -use crate::bindings::_helpers::compiled_project::job_settings::job_run_settings; use crate::bindings::_helpers::cost_report::deferred::collect_deferred_cost_after_finish; use crate::bindings::_helpers::freshness::freshness_command::run_freshness_command; use crate::bindings::_helpers::location_command::command::compile_snapshot; use crate::bindings::_helpers::location_command::constants::COMMAND_IMPLEMENTATIONS; -use crate::bindings::_helpers::location_command::constants::SQL_TEST_NODE_KIND; use crate::bindings::_helpers::query_command::project::ProjectAdapter; use crate::bindings::_helpers::seed_command::command::run_seed; use crate::bindings::_helpers::sql_test_command::test_command::run_test; @@ -51,8 +45,12 @@ use sqlbuild_python_front::compile_attachment::main::build_effective_connection_ use sqlbuild_python_output::recall::main::write_stderr::write_stderr; use sqlbuild_python_output::recall::models::InvocationRecall; use sqlbuild_python_project::compiled_project::constants::SQL_TEST_KEY_PREFIX; -use sqlbuild_python_project::compiled_project::main::with_frozen_job::with_frozen_job; use sqlbuild_python_project::compiled_views::main::project::sql_tests::sql_tests; +use sqlbuild_python_project::job_settings::constants::SQL_TEST_NODE_KIND; +use sqlbuild_python_project::job_settings::main::frozen_run_flags::frozen_run_flags; +use sqlbuild_python_project::job_settings::main::frozen_selectors::frozen_selectors; +use sqlbuild_python_project::job_settings::main::with_frozen_job::with_frozen_job; +use sqlbuild_python_project::job_settings::models::FrozenRunFlags; use sqlbuild_python_project::project_snapshot::models::NativeOrchestrationSnapshot; use sqlbuild_python_project::project_snapshot::models::NativeOrchestrationWorker; use sqlbuild_python_project::project_snapshot::models::WorkerBuild; @@ -138,39 +136,12 @@ fn command_request<'py>( ) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>, Option)> { let execution = worker.get().execution(py)?; let frozen = execution.claim.request; - let mut settings = frozen.settings; - if let Some(window) = frozen.window { - for key in [ - "start_cursor", - "end_cursor", - "start_cursor_ts", - "end_cursor_ts", - "start_cursor_int", - "end_cursor_int", - ] { - settings.remove(key); - } - let (kind, start, end) = match window { - RunWindow::Timestamp { start, end } => ("ts", start.to_rfc3339(), end.to_rfc3339()), - RunWindow::Integer { start, end } => ("int", start.to_string(), end.to_string()), - }; - settings.insert(format!("start_cursor_{kind}"), serde_json::json!(start)); - settings.insert(format!("end_cursor_{kind}"), serde_json::json!(end)); - } + let FrozenRunFlags { + settings, + flags, + job_build, + } = frozen_run_flags(&frozen, &arguments.value("project_dir")?)?; let command: RunCommand = frozen.command; - if frozen.job_key.is_some() { - settings = job_run_settings(&arguments.value("project_dir")?, settings)?; - } - let job_build: Option<&String> = frozen - .job_key - .as_ref() - .filter(|_| command == RunCommand::Build); - let flags: Vec = match job_build { - Some(job) => job_run_arguments(job, &settings), - None if command == RunCommand::Build => job_settings_arguments(&settings), - None => command_settings_arguments(command.as_str(), &settings), - } - .map_err(PyValueError::new_err)?; let mut argv: Vec> = std::iter::once(command.as_str().to_owned()) .chain(flags) .map(|argument| PyString::new(py, &argument)) @@ -183,23 +154,7 @@ fn command_request<'py>( PyString::new(py, "--no-color"), PyString::new(py, "--json"), ]); - let mut selectors: Vec = Vec::new(); - for key in &frozen.selection { - if execution.skipped_nodes.contains_key(key) { - continue; - } - let (kind, name) = key - .split_once(':') - .ok_or_else(|| PyValueError::new_err("frozen run contains an invalid node key"))?; - let selector: String = if PYTHON_NODE_KINDS.contains(&kind) { - key.clone() - } else if kind == SQL_TEST_NODE_KIND { - format!("test:{name}") - } else { - name.to_owned() - }; - selectors.push(selector); - } + let selectors: Vec = frozen_selectors(&frozen, &execution.skipped_nodes)?; if job_build.is_none() { for selector in &selectors { argv.extend([PyString::new(py, "--select"), PyString::new(py, selector)]); @@ -230,7 +185,7 @@ fn command_request<'py>( ], )?; } - let frozen_job: Option = job_build.map(|job| (job.clone(), selectors, settings)); + let frozen_job: Option = job_build.map(|job| (job, selectors, settings)); Ok((request, warehouse_scope(&args)?, frozen_job)) } diff --git a/src/sqlbuild/adapter/contract/classes/base_adapter.py b/src/sqlbuild/adapter/contract/classes/base_adapter.py index a910587441..a693bc34e2 100644 --- a/src/sqlbuild/adapter/contract/classes/base_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/base_adapter.py @@ -2348,7 +2348,9 @@ def render_create_observations_table_sql(self, *, database: str | None, schema: def use_session_warehouse(self, *, connection: Any, warehouse: str) -> None: """Run the session's next statements on `warehouse`, where the adapter has one.""" - raise NotImplementedError(f"{type(self).__name__} has no session warehouse to select") + raise AdapterUserError( + f"the '{self.adapter_name}' adapter has no warehouse to select", code="C260" + ) def current_session_warehouse(self, *, connection: Any) -> str | None: """The warehouse the session runs on, as an identifier `use_session_warehouse` accepts.""" diff --git a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py index 3d1286108f..f2bbc2c0ee 100644 --- a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py +++ b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py @@ -200,7 +200,6 @@ _SHOW_OBJECT_RELATION_TYPES: dict[str, str] = {"TABLE": "base table", "VIEW": "view"} _COST_LOGIN_TIMEOUT_SECONDS: int = 5 _COST_NETWORK_TIMEOUT_SECONDS: int = 15 -# A warehouse name CURRENT_WAREHOUSE() returns that USE WAREHOUSE accepts without quotes. _UNQUOTED_WAREHOUSE: re.Pattern[str] = re.compile(r"[A-Z_][A-Z0-9_$]*") _SHOW_COLUMN_KIND: str = "COLUMN" _SHOW_COLUMN_KINDS: frozenset[str] = frozenset({_SHOW_COLUMN_KIND, "VIRTUAL_COLUMN"}) diff --git a/src/sqlbuild/executor/build/classes/build_scheduler.py b/src/sqlbuild/executor/build/classes/build_scheduler.py index 3e933076c0..aad6c2b23e 100644 --- a/src/sqlbuild/executor/build/classes/build_scheduler.py +++ b/src/sqlbuild/executor/build/classes/build_scheduler.py @@ -250,7 +250,9 @@ def __init__( self._init_job_limits(runtime) self._node_warehouses: Mapping[CompiledObjectKey, str] = runtime.node_warehouses self._session_warehouses: SessionWarehouses | None = ( - SessionWarehouses(adapter=adapter, home=configured_warehouse(connection_config)) + SessionWarehouses( + adapter=adapter, home=configured_warehouse(connection_config=connection_config) + ) if runtime.node_warehouses else None ) @@ -568,10 +570,7 @@ def _update_microbatch_coordinator_demand(self) -> None: ) def _init_job_limits(self, runtime: BuildRuntimeParams) -> None: - """Job limits, each node's jobs, and a lock over how many of each job's nodes run. - - Coordinators reserve microbatch sub-work from worker threads, so the counts take a lock. - """ + """Job limits, each node's jobs, and the lock coordinator threads take to count them.""" self._job_limits: Mapping[str, int] = runtime.job_limits self._node_jobs: Mapping[CompiledObjectKey, tuple[str, ...]] = runtime.node_jobs @@ -605,7 +604,7 @@ def _jobs_have_room(self, jobs: tuple[str, ...]) -> bool: return all(self._job_running.get(job, 0) < self._job_limits[job] for job in jobs) - def _add_job_running(self, jobs: tuple[str, ...], *, change: int) -> None: + def _add_job_running(self, *, jobs: tuple[str, ...], change: int) -> None: """Count `change` more running nodes in each of `jobs`; the caller holds `_job_lock`.""" job: str @@ -615,14 +614,14 @@ def _add_job_running(self, jobs: tuple[str, ...], *, change: int) -> None: def _start_in_flight(self, key: CompiledObjectKey) -> None: self._in_flight.add(key) with self._job_lock: - self._add_job_running(self._node_jobs.get(key, ()), change=1) + self._add_job_running(jobs=self._node_jobs.get(key, ()), change=1) def _finish_in_flight(self, key: CompiledObjectKey) -> None: if key not in self._in_flight: return self._in_flight.discard(key) with self._job_lock: - self._add_job_running(self._node_jobs.get(key, ()), change=-1) + self._add_job_running(jobs=self._node_jobs.get(key, ()), change=-1) def _report_scheduler_state(self) -> None: callback: Callable[[SchedulerState], None] | None = self._callbacks.on_scheduler_state @@ -846,7 +845,7 @@ def _reserve_microbatch_subworker(self, jobs: tuple[str, ...]) -> bool: with self._job_lock: if not self._jobs_have_room(jobs): return False - self._add_job_running(jobs, change=1) + self._add_job_running(jobs=jobs, change=1) self._microbatch_subworkers += 1 return True @@ -854,7 +853,7 @@ def _release_microbatch_subworker(self, jobs: tuple[str, ...]) -> None: with self._microbatch_coordinator_lock: self._microbatch_subworkers -= 1 with self._job_lock: - self._add_job_running(jobs, change=-1) + self._add_job_running(jobs=jobs, change=-1) def _cancel_microbatch_subworker( self, *, future: Future[MicrobatchPhaseOutcome], jobs: tuple[str, ...] @@ -1114,7 +1113,8 @@ def _execute_source_node( loader_ref_entries=self._loader_ref_entries, adapter=self._adapter, connection_config=with_session_warehouse( - self._connection_config, self._node_warehouses.get(key) + connection_config=self._connection_config, + warehouse=self._node_warehouses.get(key), ), connection=connection, runtime=self._runtime, diff --git a/src/sqlbuild/executor/build/models.py b/src/sqlbuild/executor/build/models.py index 49c030dee8..0f3ede6d6a 100644 --- a/src/sqlbuild/executor/build/models.py +++ b/src/sqlbuild/executor/build/models.py @@ -109,7 +109,6 @@ class BuildRuntimeParams: dynamic_hold: DynamicHold | None = None job_limits: Mapping[str, int] = field(default_factory=dict) node_jobs: Mapping[CompiledObjectKey, tuple[str, ...]] = field(default_factory=dict) - # The warehouse each node runs on when its named job sets one; others use the build's. node_warehouses: Mapping[CompiledObjectKey, str] = field(default_factory=dict) diff --git a/src/sqlbuild/executor/python_nodes/_helpers/ingress_execution.py b/src/sqlbuild/executor/python_nodes/_helpers/ingress_execution.py index c77732a8c9..228c059e00 100644 --- a/src/sqlbuild/executor/python_nodes/_helpers/ingress_execution.py +++ b/src/sqlbuild/executor/python_nodes/_helpers/ingress_execution.py @@ -150,7 +150,7 @@ def _execute_ingress_lifecycle_node( results: IngressResultAccumulator, ) -> LifecycleNodeResult: discovered_node: DiscoveredPythonNode = python_graph.nodes_by_name[node.name] - runtime = node_runtime(runtime, node.name) + runtime = node_runtime(runtime=runtime, name=node.name) if discovered_node.kind == PythonNodeKind.LOADER: return _execute_ingress_loader( node=discovered_node, diff --git a/src/sqlbuild/executor/python_nodes/_helpers/node_warehouse.py b/src/sqlbuild/executor/python_nodes/_helpers/node_warehouse.py index d5b3faeb21..5477b8db3b 100644 --- a/src/sqlbuild/executor/python_nodes/_helpers/node_warehouse.py +++ b/src/sqlbuild/executor/python_nodes/_helpers/node_warehouse.py @@ -11,9 +11,8 @@ ) -def node_runtime(runtime: PythonNodeRuntime, name: str) -> PythonNodeRuntime: - """Put the shared connection on node `name`'s warehouse; code opening its own connections - from `connection_config` gets that warehouse too.""" +def node_runtime(*, runtime: PythonNodeRuntime, name: str) -> PythonNodeRuntime: + """The runtime of node `name`, its connection and `connection_config` on its warehouse.""" session_warehouses: SessionWarehouses | None = runtime.session_warehouses if session_warehouses is None: @@ -23,5 +22,8 @@ def node_runtime(runtime: PythonNodeRuntime, name: str) -> PythonNodeRuntime: if warehouse is None: return runtime return replace( - runtime, connection_config=with_session_warehouse(runtime.connection_config, warehouse) + runtime, + connection_config=with_session_warehouse( + connection_config=runtime.connection_config, warehouse=warehouse + ), ) diff --git a/src/sqlbuild/executor/python_nodes/_helpers/python_checks.py b/src/sqlbuild/executor/python_nodes/_helpers/python_checks.py index db051af7b2..48bdd2cd79 100644 --- a/src/sqlbuild/executor/python_nodes/_helpers/python_checks.py +++ b/src/sqlbuild/executor/python_nodes/_helpers/python_checks.py @@ -135,7 +135,7 @@ def _execute_one_python_check( result_store: Any | None, callbacks: PythonCheckCallbacks, ) -> PythonCheckExecutionResult: - runtime = node_runtime(runtime, check_function.name) + runtime = node_runtime(runtime=runtime, name=check_function.name) adapter: BaseAdapter = runtime.adapter providers: ProviderContainer | None = runtime.providers with ( diff --git a/src/sqlbuild/executor/python_nodes/classes/read_side_python_execution_tracker.py b/src/sqlbuild/executor/python_nodes/classes/read_side_python_execution_tracker.py index d62e65cd30..58501b2a90 100644 --- a/src/sqlbuild/executor/python_nodes/classes/read_side_python_execution_tracker.py +++ b/src/sqlbuild/executor/python_nodes/classes/read_side_python_execution_tracker.py @@ -153,7 +153,7 @@ def _python_dependencies_satisfied(self, node: DiscoveredPythonNode) -> bool: def _execute_python_node(self, node: DiscoveredPythonNode) -> None: runtime: PythonNodeRuntime = node_runtime( - replace(self._runtime, result_store=self._result_store), node.name + runtime=replace(self._runtime, result_store=self._result_store), name=node.name ) _ = run_ready_python_node( node=_to_executable_python_node(node), diff --git a/src/sqlbuild/executor/python_nodes/models.py b/src/sqlbuild/executor/python_nodes/models.py index 4269aad80e..d1fbbbec14 100644 --- a/src/sqlbuild/executor/python_nodes/models.py +++ b/src/sqlbuild/executor/python_nodes/models.py @@ -151,9 +151,7 @@ class PythonNodeRuntime: providers: ProviderContainer | None = None result_store: Any | None = None persist_node_results: bool = True - # The warehouse each Python node runs on when its named job sets one, by node name. node_warehouses: Mapping[str, str] = field(default_factory=dict) - # Switches `connection` between node warehouses; set whenever `node_warehouses` is not empty. session_warehouses: SessionWarehouses | None = None def __post_init__(self) -> None: @@ -162,7 +160,8 @@ def __post_init__(self) -> None: self, "session_warehouses", SessionWarehouses( - adapter=self.adapter, home=configured_warehouse(self.connection_config) + adapter=self.adapter, + home=configured_warehouse(connection_config=self.connection_config), ), ) diff --git a/src/sqlbuild/executor/scheduling/classes/session_warehouses.py b/src/sqlbuild/executor/scheduling/classes/session_warehouses.py index bd76719838..052ab18e0b 100644 --- a/src/sqlbuild/executor/scheduling/classes/session_warehouses.py +++ b/src/sqlbuild/executor/scheduling/classes/session_warehouses.py @@ -16,7 +16,7 @@ _LOGGER: logging.Logger = logging.getLogger(__name__) -def configured_warehouse(connection_config: Mapping[str, object]) -> str | None: +def configured_warehouse(*, connection_config: Mapping[str, object]) -> str | None: """The warehouse a connection opened with `connection_config` starts on, if it names one.""" value: object | None = connection_config.get(_CONNECTION_WAREHOUSE_KEY) @@ -29,7 +29,7 @@ def configured_warehouse(connection_config: Mapping[str, object]) -> str | None: def with_session_warehouse( - connection_config: dict[str, object], warehouse: str | None + *, connection_config: dict[str, object], warehouse: str | None ) -> dict[str, object]: """`connection_config` for code that opens its own connections on `warehouse`.""" @@ -38,7 +38,7 @@ def with_session_warehouse( return {**connection_config, _CONNECTION_WAREHOUSE_KEY: warehouse} -def _same_warehouse(first: str, second: str) -> bool: +def _same_warehouse(*, first: str, second: str) -> bool: """Unquoted warehouse names match case-insensitively, as Snowflake resolves them.""" if first.startswith('"') or second.startswith('"'): @@ -54,8 +54,6 @@ def __init__(self, *, adapter: BaseAdapter, home: str | None) -> None: self._home: str | None = home self._home_known: bool = home is not None self._lock: threading.Lock = threading.Lock() - # Connections away from the build's warehouse: id -> (connection, warehouse). Holding the - # connection keeps its id from being reused by another while it is listed. self._away: dict[int, tuple[Any, str]] = {} def use(self, *, connection: Any, warehouse: str | None) -> None: @@ -65,10 +63,16 @@ def use(self, *, connection: Any, warehouse: str | None) -> None: entry: tuple[Any, str] | None = self._away.get(id(connection)) current: str | None = None if entry is None or entry[0] is not connection else entry[1] target: str | None = warehouse - if target is not None and self._home is not None and _same_warehouse(target, self._home): + if ( + target is not None + and self._home is not None + and _same_warehouse(first=target, second=self._home) + ): target = None if current == target or ( - current is not None and target is not None and _same_warehouse(current, target) + current is not None + and target is not None + and _same_warehouse(first=current, second=target) ): return if target is None: @@ -78,7 +82,7 @@ def use(self, *, connection: Any, warehouse: str | None) -> None: return if current is None: self._learn_home(connection) - if self._home is not None and _same_warehouse(target, self._home): + if self._home is not None and _same_warehouse(first=target, second=self._home): return self._adapter.use_session_warehouse(connection=connection, warehouse=target) with self._lock: diff --git a/tests/integration/src/sqlbuild/cli/commands/main/helpers.py b/tests/integration/src/sqlbuild/cli/commands/main/helpers.py index f976e87214..b61fbbf012 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/helpers.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/helpers.py @@ -2,6 +2,7 @@ import json import os +import re import shutil import signal import subprocess @@ -11,7 +12,7 @@ from collections import defaultdict from collections.abc import Callable from datetime import datetime, timedelta -from itertools import accumulate, takewhile +from itertools import accumulate, chain, takewhile from pathlib import Path from types import ModuleType from typing import Any, cast @@ -1469,10 +1470,12 @@ def peak_concurrency(*, events_path: Path, names: tuple[str, ...]) -> int: for line in events_path.read_text(encoding="utf-8").splitlines(): event: dict[str, Any] = json.loads(line) events[(event["event_type"], str(event.get("resource_name")))].append(event) + completions: list[dict[str, Any]] = list( + chain.from_iterable(events[(_COMPLETED_EVENT, name)] for name in names) + ) ends: list[tuple[datetime, float]] = [ (datetime.fromisoformat(event["occurred_at"]), event["duration_ms"]) - for name in names - for event in events[(_COMPLETED_EVENT, name)] + for event in completions ] changes: list[tuple[datetime, int]] = sorted( [(end - timedelta(milliseconds=duration), 1) for end, duration in ends] @@ -1484,8 +1487,7 @@ def peak_concurrency(*, events_path: Path, names: tuple[str, ...]) -> int: _CONNECT_MARKER: str = "-- connect" -def _job_warehouse_model(dep: str | None) -> str: - source: str = "" if dep is None else f' FROM __ref("{dep}")' +def _job_warehouse_model(source: str) -> str: return f"MODEL (description 'Step.', materialized table);\nSELECT 1 AS id{source}\n" @@ -1509,11 +1511,11 @@ def write_job_warehouse_project(*, project_dir: Path) -> None: write_project_files( project_dir=project_dir, files={ - "models/stg.sql": _job_warehouse_model(None), - "models/orders.sql": _job_warehouse_model("stg"), - "models/orders_daily.sql": _job_warehouse_model("orders"), - "models/refunds.sql": _job_warehouse_model("orders_daily"), - "models/summary.sql": _job_warehouse_model("refunds"), + "models/stg.sql": _job_warehouse_model(""), + "models/orders.sql": _job_warehouse_model(' FROM __ref("stg")'), + "models/orders_daily.sql": _job_warehouse_model(' FROM __ref("orders")'), + "models/refunds.sql": _job_warehouse_model(' FROM __ref("orders_daily")'), + "models/summary.sql": _job_warehouse_model(' FROM __ref("refunds")'), "python/tasks.py": ( "from sqlbuild.tasks import task\n\n\n@task\ndef refresh_marker(ctx):\n" ' """Refresh."""\n ctx.execute_sql("SELECT \'task marker\'")\n' @@ -1558,9 +1560,14 @@ def marked_connect(**kwargs: object) -> object: def warehouse_switches(executed_sql: list[str]) -> tuple[str, ...]: """The warehouses statements switch to, leaving out each new connection's own setup.""" - prefix: str = "USE WAREHOUSE " - return tuple( - sql.removeprefix(prefix) - for previous, sql in zip([_CONNECT_MARKER, *executed_sql], executed_sql, strict=False) - if sql.startswith(prefix) and previous != _CONNECT_MARKER - ) + statements: str = "\n".join(executed_sql) + return tuple(re.findall(rf"(? int: + """The exit code of one CLI invocation in `project_dir`, returned or raised.""" + + try: + return int(main(["--no-color", "--project-dir", str(project_dir), *argv]) or 0) + except SystemExit as exited: + return int(exited.code or 0) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_job_warehouses.py b/tests/integration/src/sqlbuild/cli/commands/main/test_job_warehouses.py index f94080addb..f589046605 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_job_warehouses.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_job_warehouses.py @@ -5,14 +5,15 @@ from pathlib import Path import pytest +from _pytest.capture import CaptureResult -from sqlbuild._native import run_cli as main from tests.integration.src.sqlbuild.cli.commands.main._test_types import ( JobWarehouseOutputTestCase, JobWarehouseSwitchTestCase, ) from tests.integration.src.sqlbuild.cli.commands.main.helpers import ( install_writable_offline_snowflake, + run_cli_exit_code, warehouse_switches, write_job_warehouse_project, ) @@ -26,13 +27,6 @@ ) -def _run(argv: tuple[str, ...], project_dir: Path) -> int: - try: - return int(main(["--no-color", "--project-dir", str(project_dir), *argv]) or 0) - except SystemExit as exited: - return int(exited.code or 0) - - @pytest.mark.parametrize( "test_case", [ @@ -78,9 +72,9 @@ def test_given_named_jobs_with_warehouses_when_building_then_each_node_runs_on_i write_job_warehouse_project(project_dir=tmp_path) connect_calls, executed_sql = install_writable_offline_snowflake(monkeypatch) - exit_code: int = _run(test_case.argv, tmp_path) + exit_code: int = run_cli_exit_code(test_case.argv, tmp_path) - captured = capsys.readouterr() + captured: CaptureResult[str] = capsys.readouterr() assert ( exit_code, {call.get("warehouse") for call in connect_calls}, @@ -150,9 +144,9 @@ def test_given_named_jobs_with_warehouses_when_planning_or_building_then_output_ write_job_warehouse_project(project_dir=tmp_path) install_writable_offline_snowflake(monkeypatch) - exit_code: int = _run(test_case.argv, tmp_path) + exit_code: int = run_cli_exit_code(test_case.argv, tmp_path) - captured = capsys.readouterr() + captured: CaptureResult[str] = capsys.readouterr() output: str = captured.out + captured.err assert (exit_code, test_case.expected_fragment in output) == ( test_case.expected_exit_code, diff --git a/tests/unit/src/sqlbuild/executor/build/classes/_test_types.py b/tests/unit/src/sqlbuild/executor/build/classes/_test_types.py index 60748f763a..51db8b9b24 100644 --- a/tests/unit/src/sqlbuild/executor/build/classes/_test_types.py +++ b/tests/unit/src/sqlbuild/executor/build/classes/_test_types.py @@ -84,5 +84,4 @@ class SubworkerWarehouseTestCase: class NodeWarehouseFailureTestCase: description: str node_warehouse: str | None - failing: str expected_message: str diff --git a/tests/unit/src/sqlbuild/executor/build/classes/helpers.py b/tests/unit/src/sqlbuild/executor/build/classes/helpers.py index 8b86a8a3af..d6f4c63d5a 100644 --- a/tests/unit/src/sqlbuild/executor/build/classes/helpers.py +++ b/tests/unit/src/sqlbuild/executor/build/classes/helpers.py @@ -6,17 +6,20 @@ import time from collections import deque from queue import Queue -from typing import Any +from typing import Any, cast from sqlbuild._native import CompiledObjectKey, CompiledResourceType +from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.executor.build.classes.build_scheduler import BuildScheduler from sqlbuild.executor.build.models import BuildRuntimeParams, NodeCompletion from sqlbuild.executor.run.models import ModelExecutionResult +from sqlbuild.executor.scheduling.classes.session_warehouses import SessionWarehouses from sqlbuild.executor.scheduling.types import ExecutionStatus from tests.unit.src.sqlbuild.executor.build.classes._test_types import ( JobLimitedBuildTestCase, SubworkerReservationTestCase, ) +from tests.unit.src.sqlbuild.executor.scheduling.classes.helpers import RecordingWarehouseAdapter # How long each stub node runs, long enough for the loop to start every node it may. _NODE_SECONDS: float = 0.05 @@ -154,3 +157,20 @@ def reserve_subworkers(test_case: SubworkerReservationTestCase) -> tuple[int, tu for _ in test_case.expected_reservations ) return scheduler._microbatch_coordinator_demand, reservations + + +def warehouse_scheduler(adapter: RecordingWarehouseAdapter) -> Any: + """A scheduler with only the state warehouse switching and sub-worker slots read.""" + + scheduler: Any = object.__new__(BuildScheduler) + scheduler._session_warehouses = SessionWarehouses( + adapter=cast(BaseAdapter, adapter), home="BUILD_WH" + ) + scheduler._node_warehouses = {} + scheduler._microbatch_coordinator_lock = threading.Lock() + scheduler._job_lock = threading.Lock() + scheduler._job_limits = {} + scheduler._job_running = {} + scheduler._microbatch_subworkers = 1 + scheduler._connection_pool = Queue() + return scheduler diff --git a/tests/unit/src/sqlbuild/executor/build/classes/test_build_scheduler_warehouses.py b/tests/unit/src/sqlbuild/executor/build/classes/test_build_scheduler_warehouses.py index c617ff3eaa..5f91b249f2 100644 --- a/tests/unit/src/sqlbuild/executor/build/classes/test_build_scheduler_warehouses.py +++ b/tests/unit/src/sqlbuild/executor/build/classes/test_build_scheduler_warehouses.py @@ -2,42 +2,27 @@ from __future__ import annotations -import threading -from queue import Queue -from typing import Any, cast +from typing import Any import pytest -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.executor.build.classes.build_scheduler import BuildScheduler from sqlbuild.executor.run.models import BatchWindow, ModelExecutionResult -from sqlbuild.executor.scheduling.classes.session_warehouses import SessionWarehouses from sqlbuild.executor.scheduling.types import ExecutionStatus from tests.unit.src.sqlbuild.executor.build.classes._test_types import ( NodeWarehouseFailureTestCase, SubworkerWarehouseTestCase, ) -from tests.unit.src.sqlbuild.executor.build.classes.helpers import model_key +from tests.unit.src.sqlbuild.executor.build.classes.helpers import ( + model_key, + warehouse_scheduler, +) from tests.unit.src.sqlbuild.executor.scheduling.classes.helpers import ( + MissingWarehouseAdapter, RecordingWarehouseAdapter, ) -def _scheduler(adapter: RecordingWarehouseAdapter) -> Any: - scheduler: Any = object.__new__(BuildScheduler) - scheduler._session_warehouses = SessionWarehouses( - adapter=cast(BaseAdapter, adapter), home="BUILD_WH" - ) - scheduler._node_warehouses = {} - scheduler._microbatch_coordinator_lock = threading.Lock() - scheduler._job_lock = threading.Lock() - scheduler._job_limits = {} - scheduler._job_running = {} - scheduler._microbatch_subworkers = 1 - scheduler._connection_pool = Queue() - return scheduler - - @pytest.mark.parametrize( "test_case", [ @@ -66,7 +51,7 @@ def test_given_a_coordinator_warehouse_when_a_subworker_runs_a_batch_then_its_co test_case: SubworkerWarehouseTestCase, ) -> None: adapter: RecordingWarehouseAdapter = RecordingWarehouseAdapter(session_warehouse=None) - scheduler: Any = _scheduler(adapter) + scheduler: Any = warehouse_scheduler(adapter) pooled: object = object() scheduler._session_warehouses.use(connection=pooled, warehouse=test_case.earlier_warehouse) scheduler._connection_pool.put(pooled) @@ -98,7 +83,6 @@ def execute(_batch: BatchWindow, connection: object) -> Any: NodeWarehouseFailureTestCase( description="a job warehouse the session cannot use fails the node with the reason", node_warehouse="MISSING_WH", - failing="MISSING_WH", expected_message=( "cannot run on warehouse MISSING_WH: Object does not exist: warehouse MISSING_WH" ), @@ -109,10 +93,8 @@ def execute(_batch: BatchWindow, connection: object) -> Any: def test_given_an_unusable_job_warehouse_when_a_node_starts_then_it_fails_naming_the_warehouse( test_case: NodeWarehouseFailureTestCase, ) -> None: - adapter: RecordingWarehouseAdapter = RecordingWarehouseAdapter( - session_warehouse=None, failing=test_case.failing - ) - scheduler: Any = _scheduler(adapter) + adapter: MissingWarehouseAdapter = MissingWarehouseAdapter(session_warehouse=None) + scheduler: Any = warehouse_scheduler(adapter) scheduler._node_warehouses = {model_key("orders"): test_case.node_warehouse} result: ModelExecutionResult | None = BuildScheduler._enter_node_warehouse( diff --git a/tests/unit/src/sqlbuild/executor/scheduling/classes/_test_types.py b/tests/unit/src/sqlbuild/executor/scheduling/classes/_test_types.py index 77a9a5c7b9..dd1df2ec22 100644 --- a/tests/unit/src/sqlbuild/executor/scheduling/classes/_test_types.py +++ b/tests/unit/src/sqlbuild/executor/scheduling/classes/_test_types.py @@ -20,7 +20,14 @@ class SessionWarehousesTestCase: configured: str | None session_warehouse: str | None steps: tuple[tuple[str, str | None], ...] - restore: bool expected_statements: tuple[tuple[str, str], ...] expected_session_reads: int - expected_error: str | None = None + + +@dataclass(frozen=True) +class SessionWithoutWarehouseTestCase: + """A connection moved to a job's warehouse when the session had no warehouse to return to.""" + + description: str + job_warehouse: str + expected_error: str diff --git a/tests/unit/src/sqlbuild/executor/scheduling/classes/helpers.py b/tests/unit/src/sqlbuild/executor/scheduling/classes/helpers.py index a08182fd81..0814a48d43 100644 --- a/tests/unit/src/sqlbuild/executor/scheduling/classes/helpers.py +++ b/tests/unit/src/sqlbuild/executor/scheduling/classes/helpers.py @@ -59,18 +59,23 @@ def failure_completion(*, key: str, error: BaseException) -> tuple[str, str]: class RecordingWarehouseAdapter: """Records each warehouse switch; reports `session_warehouse` as the session's warehouse.""" - def __init__(self, *, session_warehouse: str | None, failing: str | None = None) -> None: + def __init__(self, *, session_warehouse: str | None) -> None: self.session_warehouse: str | None = session_warehouse - self.failing: str | None = failing self.statements: list[tuple[Any, str]] = [] self.session_reads: int = 0 def use_session_warehouse(self, *, connection: Any, warehouse: str) -> None: - if warehouse == self.failing: - raise RuntimeError(f"Object does not exist: warehouse {warehouse}") self.statements.append((connection, warehouse)) def current_session_warehouse(self, *, connection: Any) -> str | None: del connection self.session_reads += 1 return self.session_warehouse + + +class MissingWarehouseAdapter(RecordingWarehouseAdapter): + """An adapter whose session cannot use any warehouse it is asked for.""" + + def use_session_warehouse(self, *, connection: Any, warehouse: str) -> None: + del connection + raise RuntimeError(f"Object does not exist: warehouse {warehouse}") diff --git a/tests/unit/src/sqlbuild/executor/scheduling/classes/test_session_warehouses.py b/tests/unit/src/sqlbuild/executor/scheduling/classes/test_session_warehouses.py index 5b5f200ca4..8784802905 100644 --- a/tests/unit/src/sqlbuild/executor/scheduling/classes/test_session_warehouses.py +++ b/tests/unit/src/sqlbuild/executor/scheduling/classes/test_session_warehouses.py @@ -11,6 +11,7 @@ from sqlbuild.executor.scheduling.classes.session_warehouses import SessionWarehouses from tests.unit.src.sqlbuild.executor.scheduling.classes._test_types import ( SessionWarehousesTestCase, + SessionWithoutWarehouseTestCase, ) from tests.unit.src.sqlbuild.executor.scheduling.classes.helpers import ( RecordingWarehouseAdapter, @@ -25,7 +26,6 @@ configured="BUILD_WH", session_warehouse=None, steps=(("a", None), ("a", "BUILD_WH"), ("a", "build_wh")), - restore=True, expected_statements=(), expected_session_reads=0, ), @@ -34,7 +34,6 @@ configured="BUILD_WH", session_warehouse=None, steps=(("a", "ORDERS_WH"), ("a", "ORDERS_WH"), ("a", "REFUNDS_WH"), ("a", None)), - restore=True, expected_statements=(("a", "ORDERS_WH"), ("a", "REFUNDS_WH"), ("a", "BUILD_WH")), expected_session_reads=0, ), @@ -43,8 +42,12 @@ configured="BUILD_WH", session_warehouse=None, steps=(("a", "ORDERS_WH"), ("b", "ORDERS_WH"), ("b", None), ("a", "ORDERS_WH")), - restore=False, - expected_statements=(("a", "ORDERS_WH"), ("b", "ORDERS_WH"), ("b", "BUILD_WH")), + expected_statements=( + ("a", "ORDERS_WH"), + ("b", "ORDERS_WH"), + ("b", "BUILD_WH"), + ("a", "BUILD_WH"), + ), expected_session_reads=0, ), SessionWarehousesTestCase( @@ -52,7 +55,6 @@ configured="BUILD_WH", session_warehouse=None, steps=(("a", "ORDERS_WH"), ("b", "REFUNDS_WH")), - restore=True, expected_statements=( ("a", "ORDERS_WH"), ("b", "REFUNDS_WH"), @@ -66,7 +68,6 @@ configured=None, session_warehouse="USER_WH", steps=(("a", "ORDERS_WH"), ("b", "ORDERS_WH"), ("a", None)), - restore=True, expected_statements=( ("a", "ORDERS_WH"), ("b", "ORDERS_WH"), @@ -80,20 +81,9 @@ configured=None, session_warehouse="ORDERS_WH", steps=(("a", "ORDERS_WH"), ("a", None)), - restore=True, expected_statements=(), expected_session_reads=1, ), - SessionWarehousesTestCase( - description="a session without any warehouse cannot be returned to one", - configured=None, - session_warehouse=None, - steps=(("a", "ORDERS_WH"), ("a", None)), - restore=False, - expected_statements=(("a", "ORDERS_WH"),), - expected_session_reads=1, - expected_error="cannot return a connection to the build's warehouse", - ), ], ids=lambda case: case.description, ) @@ -107,24 +97,35 @@ def test_given_nodes_on_job_warehouses_when_taking_connections_then_switches_onl adapter=cast(BaseAdapter, adapter), home=test_case.configured ) connections: dict[str, Any] = {"a": object(), "b": object()} - error: str | None = None - try: - for name, warehouse in test_case.steps: - warehouses.use(connection=connections[name], warehouse=warehouse) - if test_case.restore: - warehouses.restore() - except ExecutorInputError as raised: - error = str(raised) + for name, warehouse in test_case.steps: + warehouses.use(connection=connections[name], warehouse=warehouse) + warehouses.restore() names: dict[int, str] = {id(connection): name for name, connection in connections.items()} assert ( tuple((names[id(connection)], warehouse) for connection, warehouse in adapter.statements), adapter.session_reads, - error is not None - and test_case.expected_error is not None - and test_case.expected_error in error, - ) == ( - test_case.expected_statements, - test_case.expected_session_reads, - test_case.expected_error is not None, - ) + ) == (test_case.expected_statements, test_case.expected_session_reads) + + +@pytest.mark.parametrize( + "test_case", + [ + SessionWithoutWarehouseTestCase( + description="a session without any warehouse cannot be returned to one", + job_warehouse="ORDERS_WH", + expected_error="cannot return a connection to the build's warehouse", + ), + ], + ids=lambda case: case.description, +) +def test_given_a_session_without_any_warehouse_when_returning_to_it_then_it_fails( + test_case: SessionWithoutWarehouseTestCase, +) -> None: + adapter: RecordingWarehouseAdapter = RecordingWarehouseAdapter(session_warehouse=None) + warehouses: SessionWarehouses = SessionWarehouses(adapter=cast(BaseAdapter, adapter), home=None) + connection: object = object() + warehouses.use(connection=connection, warehouse=test_case.job_warehouse) + + with pytest.raises(ExecutorInputError, match=test_case.expected_error): + warehouses.use(connection=connection, warehouse=None) From 0ab6ff33e71217c312b216e5b159e31c6c878006 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 19 Nov 2026 05:30:30 +0000 Subject: [PATCH 10/22] fix(jobs): apply job settings to everything a job selects and let the command line settle conflicts (CHI-1533) A group's jobs again need to agree only on vars and changed while each member runs as its own build (D017); the full agreement check returns with phase 2. A server job run passes the job's warehouse as --warehouse again, so setup, planning and bookkeeping run on it. A CLI build whose named jobs all set the same warehouse treats it as build-wide too. full_refresh also recreates the functions a job selects, and ignore_audit_failures also covers singular audits that read the job's nodes. A setting given on the command line replaces every named job's before conflicts are checked, and conflict help is worded per scope. A failed warehouse switch fails the node with its own kind's result: SQL tests, seeds, sources and functions through the scheduler, and Python tasks, assets, loaders and checks without aborting their loop; blocked checks never switch. A SQL test covering nodes with and without a job warehouse runs on the build warehouse. --- .../src/job_settings/_helpers/node_values.rs | 8 +- .../src/job_settings/constants.rs | 14 +- .../job_settings/main/build_job_settings.rs | 4 +- .../main/group_setting_mismatch.rs | 22 ++- .../src/job_settings/main/mod.rs | 1 + .../main/without_given_settings.rs | 19 +++ .../src/argv/main/job_run_arguments.rs | 7 +- .../src/argv/tests/test_job_run_arguments.rs | 7 +- .../build_command/main/render_run_context.rs | 6 +- .../_helpers/planner_records/constants.rs | 1 + .../_helpers/jobs/job_node_settings.rs | 107 +++++++++++--- .../main/jobs/job_override_fields.rs | 16 +- .../src/compile_driver/models.rs | 3 + .../src/planner_changes/_helpers/detect.rs | 12 +- .../planner_changes/main/detect_changes.rs | 4 +- .../_helpers/plans/execution_plan.rs | 4 +- .../src/planner_phases/constants.rs | 2 + .../src/planner_phases/models.rs | 21 ++- .../src/job_settings/_helpers/argv.rs | 34 ++--- .../src/job_settings/_helpers/job_catalog.rs | 15 ++ .../_helpers/named_job_build_settings.rs | 52 ++++++- .../src/job_settings/main/given_settings.rs | 11 ++ .../job_settings/main/job_settings_argv.rs | 40 +++-- .../src/job_settings/main/mod.rs | 2 + .../job_settings/main/with_given_settings.rs | 11 ++ .../src/job_settings/mod.rs | 1 + .../src/job_settings/types.rs | 9 ++ .../src/bindings/_helpers/cli/entry.rs | 12 +- .../docs/concepts/adapters/snowflake.md | 17 ++- .../references/docs/concepts/orchestration.md | 28 ++-- src/sqlbuild/_native.pyi | 2 + .../executor/build/classes/build_scheduler.py | 89 +++++++++-- .../python_nodes/_helpers/execution.py | 11 ++ .../_helpers/ingress_execution.py | 71 +++++---- .../python_nodes/_helpers/node_warehouse.py | 28 ++-- .../python_nodes/_helpers/python_checks.py | 13 +- .../read_side_python_execution_tracker.py | 5 +- .../sqlbuild/cli/commands/main/_test_types.py | 14 ++ .../src/sqlbuild/cli/commands/main/helpers.py | 88 ++++++++++- .../cli/commands/main/test_job_settings.py | 139 ++++++++++++++++-- .../cli/commands/main/test_job_warehouses.py | 29 +++- .../publication/test_adhoc.py | 8 +- .../publication/test_worker.py | 46 ++++++ .../executor/build/classes/_test_types.py | 5 + .../executor/build/classes/helpers.py | 25 ++++ .../test_build_scheduler_warehouses.py | 70 +++++++-- .../python_nodes/_helpers/_test_types.py | 21 +++ .../executor/python_nodes/_helpers/helpers.py | 22 +++ .../_helpers/test_node_warehouse_failures.py | 134 +++++++++++++++++ .../executor/scheduling/classes/helpers.py | 4 +- .../docs/docs/concepts/adapters/snowflake.mdx | 17 ++- .../docs/docs/concepts/orchestration.mdx | 28 ++-- 52 files changed, 1138 insertions(+), 221 deletions(-) create mode 100644 crates/sqlbuild-analysis/src/job_settings/main/without_given_settings.rs create mode 100644 crates/sqlbuild-python-project/src/job_settings/main/given_settings.rs create mode 100644 crates/sqlbuild-python-project/src/job_settings/main/with_given_settings.rs create mode 100644 crates/sqlbuild-python-project/src/job_settings/types.rs create mode 100644 tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_node_warehouse_failures.py diff --git a/crates/sqlbuild-analysis/src/job_settings/_helpers/node_values.rs b/crates/sqlbuild-analysis/src/job_settings/_helpers/node_values.rs index 9c566b7153..4b529b34d7 100644 --- a/crates/sqlbuild-analysis/src/job_settings/_helpers/node_values.rs +++ b/crates/sqlbuild-analysis/src/job_settings/_helpers/node_values.rs @@ -7,7 +7,7 @@ use serde_json::Value as Json; use crate::assembly::project::types::ObjectKey; use crate::definition_changes::constants::PYTHON_NODE_KINDS; use crate::job_settings::constants::{ - CONFLICT_HELP, NODE_SETTINGS, PYTHON_NODE_SETTINGS, RETRY_SETTING, + NODE_CONFLICT_HELP, NODE_SETTINGS, PYTHON_NODE_SETTINGS, RETRY_CONFLICT_HELP, RETRY_SETTING, }; use crate::job_settings::errors::JobSettingConflict; use crate::job_settings::models::{DeclaredRun, JobRetry}; @@ -27,7 +27,11 @@ fn conflict( ({} and {})", key.0, key.1, values.0, values.1 ), - help: CONFLICT_HELP, + help: if setting == RETRY_SETTING { + RETRY_CONFLICT_HELP + } else { + NODE_CONFLICT_HELP + }, } } diff --git a/crates/sqlbuild-analysis/src/job_settings/constants.rs b/crates/sqlbuild-analysis/src/job_settings/constants.rs index 427f2b3cb3..522e487092 100644 --- a/crates/sqlbuild-analysis/src/job_settings/constants.rs +++ b/crates/sqlbuild-analysis/src/job_settings/constants.rs @@ -13,9 +13,17 @@ pub const WAREHOUSE_SETTING: &str = "warehouse"; pub const PYTHON_NODE_SETTINGS: [&str; 1] = [WAREHOUSE_SETTING]; /// The setting that becomes a limit on how many of the job's nodes run at once inside the build. pub const JOB_LIMIT_SETTING: &str = "concurrency"; -/// The help every job setting conflict gives. -pub const CONFLICT_HELP: &str = "make the jobs agree, set it in only one of them, override it with \ - the command-line flag, or build with --no-job-settings"; +/// The help of two named jobs giving one node different values of a node-level setting. +pub const NODE_CONFLICT_HELP: &str = "make the jobs agree, set it in only one of them, override it \ + with the command-line flag, or build with --no-job-settings"; +/// The help of named jobs disagreeing on a build-wide setting, where unset differs from set. +pub const BUILD_CONFLICT_HELP: &str = + "make the jobs agree, override it with the command-line flag, or build with --no-job-settings"; +/// The help of two named jobs giving one node different `[retry]` tables, which no flag overrides. +pub const RETRY_CONFLICT_HELP: &str = + "make the jobs agree, set [retry] in only one of them, or build with --no-job-settings"; +/// The build-wide settings a group's jobs must agree on while each member runs as its own build. +pub const GROUP_AGREED_SETTINGS: [&str; 2] = ["vars", "changed"]; /// The error code of job settings that cannot apply together. pub const JOB_SETTING_CONFLICT_CODE: &str = "S027"; /// How a conflict names a job's `[retry]` table. diff --git a/crates/sqlbuild-analysis/src/job_settings/main/build_job_settings.rs b/crates/sqlbuild-analysis/src/job_settings/main/build_job_settings.rs index 33c9b3d151..93a81766bc 100644 --- a/crates/sqlbuild-analysis/src/job_settings/main/build_job_settings.rs +++ b/crates/sqlbuild-analysis/src/job_settings/main/build_job_settings.rs @@ -4,7 +4,7 @@ use std::collections::BTreeMap; use serde_json::Value as Json; -use crate::job_settings::constants::CONFLICT_HELP; +use crate::job_settings::constants::BUILD_CONFLICT_HELP; use crate::job_settings::errors::JobSettingConflict; use crate::job_settings::main::setting_scope::setting_scope; use crate::job_settings::models::{DeclaredRun, SettingScope}; @@ -52,7 +52,7 @@ pub fn build_job_settings( shown(seen), shown(value) ), - help: CONFLICT_HELP, + help: BUILD_CONFLICT_HELP, }); } Some(_) => {} diff --git a/crates/sqlbuild-analysis/src/job_settings/main/group_setting_mismatch.rs b/crates/sqlbuild-analysis/src/job_settings/main/group_setting_mismatch.rs index a2bbe515a2..d5b0eea65c 100644 --- a/crates/sqlbuild-analysis/src/job_settings/main/group_setting_mismatch.rs +++ b/crates/sqlbuild-analysis/src/job_settings/main/group_setting_mismatch.rs @@ -1,13 +1,28 @@ -//! A group's jobs must agree on every build-wide setting on each target that declares them. +//! A group's jobs must agree on `vars` and `changed` on each target that declares them. use std::collections::{BTreeMap, BTreeSet}; use sqlbuild_discovery::orchestration_files::models::{GroupFile, OrchestrationInventory}; +use crate::job_settings::constants::GROUP_AGREED_SETTINGS; use crate::job_settings::main::build_job_settings::build_job_settings; use crate::job_settings::main::declared_runs::declared_runs; use crate::job_settings::models::DeclaredRun; +/// `run` keeping only the settings a group's jobs must agree on. +fn agreed_settings_only(run: DeclaredRun) -> DeclaredRun { + match run { + DeclaredRun::Job { settings, retry } => DeclaredRun::Job { + settings: settings + .into_iter() + .filter(|(name, _)| GROUP_AGREED_SETTINGS.contains(&name.as_str())) + .collect(), + retry, + }, + group => group, + } +} + /// Why `group` cannot run as one build on some target, as building it would say; `None` if never. #[must_use] pub fn group_setting_mismatch( @@ -26,7 +41,10 @@ pub fn group_setting_mismatch( } } for target in targets { - let declared: BTreeMap = declared_runs(inventory, target); + let declared: BTreeMap = declared_runs(inventory, target) + .into_iter() + .map(|(name, run)| (name, agreed_settings_only(run))) + .collect(); let Some(DeclaredRun::Group { members }) = declared.get(&group.run.name.value) else { continue; }; diff --git a/crates/sqlbuild-analysis/src/job_settings/main/mod.rs b/crates/sqlbuild-analysis/src/job_settings/main/mod.rs index fa8c731841..749ae1fb0f 100644 --- a/crates/sqlbuild-analysis/src/job_settings/main/mod.rs +++ b/crates/sqlbuild-analysis/src/job_settings/main/mod.rs @@ -4,3 +4,4 @@ pub mod group_setting_mismatch; pub mod named_jobs; pub mod node_job_settings; pub mod setting_scope; +pub mod without_given_settings; diff --git a/crates/sqlbuild-analysis/src/job_settings/main/without_given_settings.rs b/crates/sqlbuild-analysis/src/job_settings/main/without_given_settings.rs new file mode 100644 index 0000000000..031887e3a6 --- /dev/null +++ b/crates/sqlbuild-analysis/src/job_settings/main/without_given_settings.rs @@ -0,0 +1,19 @@ +//! The declared jobs less the settings the command line gives, which replace theirs. + +use std::collections::{BTreeMap, BTreeSet}; + +use crate::job_settings::models::DeclaredRun; + +/// `declared` with each setting named in `given` removed from every job, so it never conflicts. +#[must_use] +pub fn without_given_settings( + mut declared: BTreeMap, + given: &BTreeSet, +) -> BTreeMap { + for run in declared.values_mut() { + if let DeclaredRun::Job { settings, .. } = run { + settings.retain(|setting, _| !given.contains(setting)); + } + } + declared +} diff --git a/crates/sqlbuild-cli/src/argv/main/job_run_arguments.rs b/crates/sqlbuild-cli/src/argv/main/job_run_arguments.rs index 6a0ea07bbf..e7baec5964 100644 --- a/crates/sqlbuild-cli/src/argv/main/job_run_arguments.rs +++ b/crates/sqlbuild-cli/src/argv/main/job_run_arguments.rs @@ -3,19 +3,22 @@ use std::collections::BTreeMap; use serde_json::Value as Json; +use sqlbuild_analysis::job_settings::constants::WAREHOUSE_SETTING; use sqlbuild_analysis::job_settings::main::setting_scope::setting_scope; use sqlbuild_analysis::job_settings::models::SettingScope; use crate::argv::main::job_settings_arguments::job_settings_arguments; -/// `--select job:` and the job's build-wide settings as flags; the build applies the rest. +/// `--select job:` and its build-wide settings and warehouse, the run's own, as flags. pub fn job_run_arguments( job: &str, settings: &BTreeMap, ) -> Result, String> { let build_wide: BTreeMap = settings .iter() - .filter(|(name, _)| setting_scope(name) == SettingScope::Build) + .filter(|(name, _)| { + setting_scope(name) == SettingScope::Build || name.as_str() == WAREHOUSE_SETTING + }) .map(|(name, value)| (name.clone(), value.clone())) .collect(); let mut arguments: Vec = job_settings_arguments(&build_wide)?; diff --git a/crates/sqlbuild-cli/src/argv/tests/test_job_run_arguments.rs b/crates/sqlbuild-cli/src/argv/tests/test_job_run_arguments.rs index 0bec2c5a5d..e5de2b5f29 100644 --- a/crates/sqlbuild-cli/src/argv/tests/test_job_run_arguments.rs +++ b/crates/sqlbuild-cli/src/argv/tests/test_job_run_arguments.rs @@ -11,9 +11,14 @@ fn given_a_server_job_run_when_building_its_arguments_then_it_names_the_job_and_ let test_cases = [ JobRunArgumentsTestCase { description: "concurrency and node-level settings are left to the build", - settings: json!({"concurrency": 8, "run_tests": false, "full_refresh": true, "warehouse": "ORDERS_WH"}), + settings: json!({"concurrency": 8, "run_tests": false, "full_refresh": true}), expected_arguments: &["--select", "job:orders_job"], }, + JobRunArgumentsTestCase { + description: "the job's warehouse is the whole run's, setup and planning included", + settings: json!({"concurrency": 8, "run_tests": false, "warehouse": "ORDERS_WH"}), + expected_arguments: &["--warehouse=ORDERS_WH", "--select", "job:orders_job"], + }, JobRunArgumentsTestCase { description: "build-wide settings still travel as flags", settings: json!({"concurrency": 4, "vars": {"region": "eu"}, "fail_fast": true}), diff --git a/crates/sqlbuild-core/src/build_command/main/render_run_context.rs b/crates/sqlbuild-core/src/build_command/main/render_run_context.rs index c5ed1d6cec..7be1e19f18 100644 --- a/crates/sqlbuild-core/src/build_command/main/render_run_context.rs +++ b/crates/sqlbuild-core/src/build_command/main/render_run_context.rs @@ -6,13 +6,13 @@ use crate::build_command::models::RunContextFacts; use crate::cli_text::models::{CliDocument, CliStyle}; use crate::number_text::main::format_thousands::format_thousands; -/// `true`, the models of the named jobs that rebuild theirs from scratch, or `false`. +/// `true`, the nodes of the named jobs that rebuild theirs from scratch, or `false`. fn full_refresh(facts: &RunContextFacts) -> String { match facts.full_refresh_jobs.as_slice() { _ if facts.full_refresh => true.to_string(), [] => false.to_string(), - [job] => format!("models of job {}", sanitize(job)), - jobs => format!("models of jobs {}", sanitize(&jobs.join(", "))), + [job] => format!("nodes of job {}", sanitize(job)), + jobs => format!("nodes of jobs {}", sanitize(&jobs.join(", "))), } } diff --git a/crates/sqlbuild-python-base/src/records/_helpers/planner_records/constants.rs b/crates/sqlbuild-python-base/src/records/_helpers/planner_records/constants.rs index 8c4851223e..6c629f4d2b 100644 --- a/crates/sqlbuild-python-base/src/records/_helpers/planner_records/constants.rs +++ b/crates/sqlbuild-python-base/src/records/_helpers/planner_records/constants.rs @@ -86,6 +86,7 @@ const R_PLANNER_OVERRIDES: RecordSpec = RecordSpec { FieldSpec::new("cursor_overrides", FieldDefault::None), FieldSpec::new("full_refresh", FieldDefault::False), FieldSpec::new("full_refresh_models", FieldDefault::EmptyFrozenset), + FieldSpec::new("full_refresh_functions", FieldDefault::EmptyFrozenset), FieldSpec::new("reload_sources", FieldDefault::False), FieldSpec::new("forced_stale_model_names", FieldDefault::EmptyTuple), FieldSpec::new("external_blocked_model_names", FieldDefault::EmptyTuple), diff --git a/crates/sqlbuild-python-compile/src/compile_driver/_helpers/jobs/job_node_settings.rs b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/jobs/job_node_settings.rs index a1918fc7a1..93875cac02 100644 --- a/crates/sqlbuild-python-compile/src/compile_driver/_helpers/jobs/job_node_settings.rs +++ b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/jobs/job_node_settings.rs @@ -10,17 +10,19 @@ use serde_json::Value; use sqlbuild_analysis::assembly::project::types::ObjectKey; use sqlbuild_analysis::definition_changes::constants::PYTHON_NODE_KINDS; use sqlbuild_analysis::graph::main::selectors_need_jobs::selectors_need_jobs; -use sqlbuild_analysis::job_settings::constants::{CONFLICT_HELP, WAREHOUSE_SETTING}; +use sqlbuild_analysis::job_settings::constants::{NODE_CONFLICT_HELP, WAREHOUSE_SETTING}; use sqlbuild_analysis::job_settings::errors::{JobSettingConflict, NodeJobSettingsError}; use sqlbuild_analysis::job_settings::main::build_job_settings::build_job_settings; use sqlbuild_analysis::job_settings::main::named_jobs::named_jobs; use sqlbuild_analysis::job_settings::main::node_job_settings::node_job_settings; +use sqlbuild_analysis::job_settings::main::without_given_settings::without_given_settings; use sqlbuild_analysis::job_settings::models::{DeclaredRun, NodeJobSettings}; use sqlbuild_config::command_connection::errors::CommandConnectionError; use sqlbuild_config::command_connection::main::check_job_warehouse::check_job_warehouse; use sqlbuild_config::command_connection::main::unsupported_job_warehouse::unsupported_job_warehouse; use sqlbuild_python_base::records::main::construct::new_error::new_error; use sqlbuild_python_base::records::types::ErrorId; +use sqlbuild_python_project::job_settings::main::given_settings::given_settings; use crate::compile_driver::main::setup::project_graph::project_graph; use crate::compile_driver::models::PlannedJobSettings; @@ -32,6 +34,7 @@ use sqlbuild_python_front::contracts::main::resolve_effective_adapter_name::reso use sqlbuild_python_project::compiled_project::constants::{ JOB_FULL_REFRESH_KEY, JOB_IGNORED_AUDIT_FAILURES_KEY, JOB_WAREHOUSES_KEY, }; +use sqlbuild_python_project::compiled_views::main::project::audits::audits; use sqlbuild_python_project::graph::main::nodes::native_key::native_key; use sqlbuild_python_project::graph::main::python_nodes::python_node_graph::python_node_graph; use sqlbuild_python_project::graph::models::KeyCache; @@ -42,8 +45,13 @@ use sqlbuild_python_project::job_settings::main::job_declarations::job_declarati use sqlbuild_python_project::job_settings::main::job_scope_project_dir::job_scope_project_dir; use sqlbuild_python_project::job_settings::main::job_selector_error::job_selector_error; use sqlbuild_python_project::job_settings::main::job_setting_conflict::job_setting_conflict; +use sqlbuild_python_project::resource_views::main::audit_reads::attached_target_kind::attached_target_kind; +use sqlbuild_python_project::resource_views::main::audit_reads::attached_target_name::attached_target_name; +use sqlbuild_python_project::resource_views::main::audit_reads::key::key as audit_key; +use sqlbuild_python_project::resource_views::main::audit_reads::scope_deps::scope_deps; const MODEL_KIND: &str = "model"; +const FUNCTION_KINDS: [&str; 2] = ["udf", "table_fn"]; const FULL_REFRESH: &str = "full_refresh"; const RUN_TESTS: &str = "run_tests"; const IGNORE_AUDIT_FAILURES: &str = "ignore_audit_failures"; @@ -112,11 +120,11 @@ fn job_warehouses( fn jobs_setting( settings: &NodeJobSettings, declared: &BTreeMap, - (setting, kind): (&str, Option<&str>), + (setting, kinds): (&str, Option<&[&str]>), ) -> Vec { let mut jobs: BTreeSet = BTreeSet::new(); for (key, node_jobs) in &settings.node_jobs { - if is_python(key) || kind.is_some_and(|kind| key.0 != kind) { + if is_python(key) || kinds.is_some_and(|kinds| !kinds.contains(&key.0.as_str())) { continue; } for job in node_jobs { @@ -130,6 +138,25 @@ fn jobs_setting( jobs.into_iter().collect() } +/// The nodes each audit attached to no node reads, by the audit's key. +fn unattached_audit_reads( + project: &Bound<'_, PyAny>, +) -> PyResult>> { + let mut reads: BTreeMap> = BTreeMap::new(); + for audit in audits(project)?.try_iter()? { + let audit: Bound<'_, PyAny> = audit?; + if !attached_target_kind(&audit)?.is_none() && !attached_target_name(&audit)?.is_none() { + continue; + } + let deps: Vec = scope_deps(&audit)? + .try_iter()? + .map(|dep| native_key(&dep?)) + .collect::>()?; + reads.insert(native_key(&audit_key(&audit)?)?, deps); + } + Ok(reads) +} + fn strings(value: &Bound<'_, PyAny>) -> PyResult> { value.try_iter()?.map(|item| item?.extract()).collect() } @@ -151,16 +178,14 @@ pub(crate) fn planned_job_settings( return Ok(None); } let exclude: Vec = strings(&native_field(options, "exclude")?)?; - let Some(mut declared): Option> = job_declarations(py)? else { + let Some(declared): Option> = job_declarations(py)? else { return Ok(None); }; + let mut given: BTreeSet = given_settings(); if cli_warehouse { - for run in declared.values_mut() { - if let DeclaredRun::Job { settings, .. } = run { - settings.remove(WAREHOUSE_SETTING); - } - } + given.insert(WAREHOUSE_SETTING.to_owned()); } + let declared: BTreeMap = without_given_settings(declared, &given); let named: Vec = named_jobs(&select, &declared); if named.is_empty() { return Ok(None); @@ -187,8 +212,16 @@ pub(crate) fn planned_job_settings( match resolved { Ok(settings) => Ok(Some(PlannedJobSettings { ignoring_jobs: jobs_setting(&settings, &declared, (IGNORE_AUDIT_FAILURES, None)), - refreshing_jobs: jobs_setting(&settings, &declared, (FULL_REFRESH, Some(MODEL_KIND))), + refreshing_jobs: jobs_setting( + &settings, + &declared, + ( + FULL_REFRESH, + Some(&[MODEL_KIND, FUNCTION_KINDS[0], FUNCTION_KINDS[1]]), + ), + ), job_warehouses: job_warehouses((project, discovered), &settings, &declared)?, + unattached_audit_reads: unattached_audit_reads(project)?, settings, })), Err(NodeJobSettingsError::Selector(failure)) => Err(job_selector_error(py, &failure)), @@ -204,20 +237,34 @@ impl PlannedJobSettings { .and_then(|values| values.get(setting)) } - /// The models the named jobs rebuild from scratch. - pub fn full_refresh_models<'py>(&self, py: Python<'py>) -> PyResult> { + /// The names of the nodes of `kinds` the named jobs rebuild from scratch. + fn refreshed<'py>(&self, py: Python<'py>, kinds: &[&str]) -> PyResult> { let names: Vec<&String> = self .settings .values .iter() .filter(|(key, values)| { - key.0 == MODEL_KIND && values.get(FULL_REFRESH) == Some(&Value::Bool(true)) + kinds.contains(&key.0.as_str()) + && values.get(FULL_REFRESH) == Some(&Value::Bool(true)) }) .map(|(key, _)| &key.1) .collect(); PyFrozenSet::new(py, names) } + /// The models the named jobs rebuild from scratch. + pub fn full_refresh_models<'py>(&self, py: Python<'py>) -> PyResult> { + self.refreshed(py, &[MODEL_KIND]) + } + + /// The functions the named jobs recreate. + pub fn full_refresh_functions<'py>( + &self, + py: Python<'py>, + ) -> PyResult> { + self.refreshed(py, &FUNCTION_KINDS) + } + /// Each job's limit and each planned node's limited jobs; none under `--concurrency`. pub(crate) fn job_limits<'py>( &self, @@ -276,7 +323,7 @@ impl PlannedJobSettings { self.value(key, WAREHOUSE_SETTING).and_then(Value::as_str) } - /// Each SQL node's job warehouse, tests on their models' agreed one, and Python nodes' by name. + /// Each SQL node's job warehouse, a test's only if all its nodes share it, and Python nodes'. pub(crate) fn node_warehouses<'py>( &self, plan: &Bound<'py, PyAny>, @@ -295,12 +342,15 @@ impl PlannedJobSettings { } for entry in native_field(plan, "test_entries")?.try_iter()? { let entry: Bound<'py, PyAny> = entry?; - let mut covered: BTreeSet<&str> = BTreeSet::new(); + let mut covered: BTreeSet> = BTreeSet::new(); for dep in native_field(&entry, "scope_deps")?.try_iter()? { - covered.extend(self.warehouse(&native_key(&dep?)?)); + covered.insert(self.warehouse(&native_key(&dep?)?)); } - if let [warehouse] = covered.into_iter().collect::>().as_slice() { - nodes.set_item(native_field(&entry, "key")?, *warehouse)?; + let key: Bound<'py, PyAny> = native_field(&entry, "key")?; + match covered.into_iter().collect::>().as_slice() { + [Some(warehouse)] => nodes.set_item(key, *warehouse)?, + _ if nodes.contains(&key)? => nodes.del_item(key)?, + _ => {} } } for key in self.settings.values.keys().filter(|key| is_python(key)) { @@ -433,7 +483,7 @@ impl PlannedJobSettings { run_tests differently ({earlier} and {value})", test.1 ), - help: CONFLICT_HELP, + help: NODE_CONFLICT_HELP, }, )); } @@ -444,12 +494,27 @@ impl PlannedJobSettings { Ok(seen.is_some_and(|(_, value)| value == &Value::Bool(false))) } - /// The `ignore_audit_failures` the audited node's job sets, if any. + /// The audited node's `ignore_audit_failures`, or for a singular audit, every read node's. fn audit_ignores(&self, entry: &Bound<'_, PyAny>) -> PyResult> { let kind: Bound<'_, PyAny> = native_field(entry, "attached_target_kind")?; let name: Bound<'_, PyAny> = native_field(entry, "attached_target_name")?; if kind.is_none() || name.is_none() { - return Ok(None); + let audit: ObjectKey = native_key(&native_field(entry, "key")?)?; + let mut ignored: Option = None; + for key in self + .unattached_audit_reads + .get(&audit) + .into_iter() + .flatten() + { + if let Some(value) = self + .value(key, IGNORE_AUDIT_FAILURES) + .and_then(Value::as_bool) + { + ignored = Some(ignored.unwrap_or(true) && value); + } + } + return Ok(ignored); } let kind: String = native_field(&kind, "value")?.extract()?; let key: ObjectKey = (kind, name.extract()?); diff --git a/crates/sqlbuild-python-compile/src/compile_driver/main/jobs/job_override_fields.rs b/crates/sqlbuild-python-compile/src/compile_driver/main/jobs/job_override_fields.rs index ffd3872875..a175e313ec 100644 --- a/crates/sqlbuild-python-compile/src/compile_driver/main/jobs/job_override_fields.rs +++ b/crates/sqlbuild-python-compile/src/compile_driver/main/jobs/job_override_fields.rs @@ -5,7 +5,7 @@ use pyo3::prelude::{PyResult, Python}; use crate::compile_driver::models::PlannedJobSettings; use crate::compile_driver::types::PipelineFields; -/// The models the named jobs rebuild from scratch, as the planner's `full_refresh_models`. +/// The models and functions the named jobs rebuild, as the planner's `full_refresh_*` overrides. pub fn job_override_fields<'py>( py: Python<'py>, job_settings: Option<&PlannedJobSettings>, @@ -13,8 +13,14 @@ pub fn job_override_fields<'py>( let Some(job_settings) = job_settings else { return Ok(Vec::new()); }; - Ok(vec![( - "full_refresh_models", - job_settings.full_refresh_models(py)?.into_any(), - )]) + Ok(vec![ + ( + "full_refresh_models", + job_settings.full_refresh_models(py)?.into_any(), + ), + ( + "full_refresh_functions", + job_settings.full_refresh_functions(py)?.into_any(), + ), + ]) } diff --git a/crates/sqlbuild-python-compile/src/compile_driver/models.rs b/crates/sqlbuild-python-compile/src/compile_driver/models.rs index 0c90c59ac8..bdc58049f7 100644 --- a/crates/sqlbuild-python-compile/src/compile_driver/models.rs +++ b/crates/sqlbuild-python-compile/src/compile_driver/models.rs @@ -8,6 +8,7 @@ use pyo3::prelude::PyAny; use pyo3::prelude::PyResult; use pyo3::prelude::Python; use pyo3::types::PyBool; +use sqlbuild_analysis::assembly::project::types::ObjectKey; use sqlbuild_analysis::assembly::target_checks::constants::PRESERVE_TARGET_VALUE; use sqlbuild_analysis::job_settings::models::NodeJobSettings; use sqlbuild_python_base::boundary::models::KeywordArguments; @@ -161,4 +162,6 @@ pub struct PlannedJobSettings { pub(crate) refreshing_jobs: Vec, /// Each named job whose `warehouse` its own nodes run on, and that warehouse. pub(crate) job_warehouses: BTreeMap, + /// The nodes each audit attached to no node reads, by the audit's key. + pub(crate) unattached_audit_reads: BTreeMap>, } diff --git a/crates/sqlbuild-python-planner/src/planner_changes/_helpers/detect.rs b/crates/sqlbuild-python-planner/src/planner_changes/_helpers/detect.rs index bd6322ca35..f16c3da16f 100644 --- a/crates/sqlbuild-python-planner/src/planner_changes/_helpers/detect.rs +++ b/crates/sqlbuild-python-planner/src/planner_changes/_helpers/detect.rs @@ -1,5 +1,7 @@ //! Detect how each selected model and function changed since its recorded build. +use std::collections::HashSet; + use crate::planner_changes::main::native_enum::native_enum; use crate::planner_changes::main::planner_enum::planner_enum; use crate::planner_changes::models::ModelPass; @@ -85,7 +87,7 @@ pub(crate) fn detect_changes_of<'py>( &arguments.value("scope")?, &arguments.value("snapshot")?, ), - arguments.flag("full_refresh")?, + (arguments.flag("full_refresh")?, &HashSet::new()), ( arguments.value("expected_version_hashes")?, arguments.value("expected_metadata_jsons")?, @@ -93,10 +95,10 @@ pub(crate) fn detect_changes_of<'py>( ) } -/// Selected model and function changes against expected version hashes and metadata JSON. +/// Selected model and function changes; functions in `refreshed_functions` are recreated too. pub(crate) fn detect_changes<'py>( (project, scope, snapshot): (&Bound<'py, PyAny>, &Bound<'py, PyAny>, &Bound<'py, PyAny>), - full_refresh: bool, + (full_refresh, refreshed_functions): (bool, &HashSet), expected: (Bound<'py, PyAny>, Bound<'py, PyAny>), ) -> PyResult> { let py: Python<'py> = project.py(); @@ -124,7 +126,9 @@ pub(crate) fn detect_changes<'py>( if selected.selected.contains( &sqlbuild_python_project::graph::main::nodes::native_key::native_key(&key(&function)?)?, ) { - let reason: &str = if full_refresh { + let reason: &str = if full_refresh + || refreshed_functions.contains(&name.extract::()?) + { "full_refresh" } else { let fingerprint: Bound<'py, PyAny> = mapping_get(&recorded, &name)?; diff --git a/crates/sqlbuild-python-planner/src/planner_changes/main/detect_changes.rs b/crates/sqlbuild-python-planner/src/planner_changes/main/detect_changes.rs index fee967c759..e95626a336 100644 --- a/crates/sqlbuild-python-planner/src/planner_changes/main/detect_changes.rs +++ b/crates/sqlbuild-python-planner/src/planner_changes/main/detect_changes.rs @@ -1,5 +1,7 @@ //! Selected model and function changes against expected version hashes and metadata JSON. +use std::collections::HashSet; + use crate::planner_changes::_helpers::detect; use pyo3::prelude::{Bound, PyAny, PyResult}; @@ -7,7 +9,7 @@ use pyo3::prelude::{Bound, PyAny, PyResult}; #[inline] pub fn detect_changes<'py>( (project, scope, snapshot): (&Bound<'py, PyAny>, &Bound<'py, PyAny>, &Bound<'py, PyAny>), - full_refresh: bool, + full_refresh: (bool, &HashSet), expected: (Bound<'py, PyAny>, Bound<'py, PyAny>), ) -> PyResult> { detect::detect_changes((project, scope, snapshot), full_refresh, expected) diff --git a/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/execution_plan.rs b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/execution_plan.rs index a36dd6b897..8e86841d98 100644 --- a/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/execution_plan.rs +++ b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/execution_plan.rs @@ -32,6 +32,7 @@ use crate::planner_phases::main::outputs::unselected_upstream_models::unselected use crate::planner_phases::main::rebuilds::build_planner_entry_results::build_planner_entry_results; use crate::planner_phases::main::rebuilds::defer_to_migration_origins::defer_to_migration_origins; use crate::planner_phases::main::rebuilds::reconcile_execution_changes::reconcile_execution_changes; +use crate::planner_phases::models::FullRefreshRequest; use crate::planner_phases::models::PlannerIdentityContext; use crate::planner_phases::models::{ PlanOutputContext, PlanWarningContext, PlannedModelEntries, StoragePolicyContext, @@ -88,13 +89,14 @@ fn change_results<'py>( let versions = class_field(identities, |i: &PlannerIdentityContext| { &i.version_identities })?; + let full_refresh: FullRefreshRequest = FullRefreshRequest::of(overrides)?; let changes = detect_changes( ( project, &class_field(scopes, |s: &PlannerScopeResolution| &s.inspection_scope)?, snapshot, ), - native_field(overrides, "full_refresh")?.is_truthy()?, + (full_refresh.all, &full_refresh.functions), ( native_field(&versions, "model_version_hashes")?, native_field(&versions, "model_metadata_jsons")?, diff --git a/crates/sqlbuild-python-planner/src/planner_phases/constants.rs b/crates/sqlbuild-python-planner/src/planner_phases/constants.rs index 81ba8b930d..d44d8393a1 100644 --- a/crates/sqlbuild-python-planner/src/planner_phases/constants.rs +++ b/crates/sqlbuild-python-planner/src/planner_phases/constants.rs @@ -115,3 +115,5 @@ pub const OTHER: &str = "Other"; pub const FULL_REFRESH_KEY: &str = "full_refresh"; /// The planner override naming the models the named jobs rebuild from scratch. pub const FULL_REFRESH_MODELS_KEY: &str = "full_refresh_models"; +/// The planner override naming the functions the named jobs recreate. +pub const FULL_REFRESH_FUNCTIONS_KEY: &str = "full_refresh_functions"; diff --git a/crates/sqlbuild-python-planner/src/planner_phases/models.rs b/crates/sqlbuild-python-planner/src/planner_phases/models.rs index 01d21ffe63..7f3191446d 100644 --- a/crates/sqlbuild-python-planner/src/planner_phases/models.rs +++ b/crates/sqlbuild-python-planner/src/planner_phases/models.rs @@ -4,7 +4,9 @@ use crate::planner_phases::_helpers::timing::timings::STEPS; use crate::planner_phases::_helpers::timing::timings::timings_of; use crate::planner_phases::constants::MILLISECONDS; use crate::planner_phases::constants::PERCENT; -use crate::planner_phases::constants::{FULL_REFRESH_KEY, FULL_REFRESH_MODELS_KEY}; +use crate::planner_phases::constants::{ + FULL_REFRESH_FUNCTIONS_KEY, FULL_REFRESH_KEY, FULL_REFRESH_MODELS_KEY, +}; use crate::planner_warehouse::models::PlannerReads; use pyo3::prelude::{Bound, Py, PyAny}; use pyo3::prelude::{PyAnyMethods, PyResult}; @@ -202,22 +204,27 @@ impl PlanTimings { } } -/// The models to rebuild: all with `--full-refresh`, else those of `full_refresh` jobs. +/// The models and functions to rebuild: all with `--full-refresh`, else those of `full_refresh` jobs. #[derive(Clone, Debug, Default)] pub struct FullRefreshRequest { pub all: bool, pub models: HashSet, + pub functions: HashSet, } impl FullRefreshRequest { - /// The request of planner overrides' `full_refresh` and `full_refresh_models`. + /// The request of planner overrides' `full_refresh` and `full_refresh_*` names. pub fn of(overrides: &Bound<'_, PyAny>) -> PyResult { - Ok(Self { - all: native_field(overrides, FULL_REFRESH_KEY)?.is_truthy()?, - models: native_field(overrides, FULL_REFRESH_MODELS_KEY)? + let names = |key: &str| -> PyResult> { + native_field(overrides, key)? .try_iter()? .map(|name| name?.extract()) - .collect::>()?, + .collect() + }; + Ok(Self { + all: native_field(overrides, FULL_REFRESH_KEY)?.is_truthy()?, + models: names(FULL_REFRESH_MODELS_KEY)?, + functions: names(FULL_REFRESH_FUNCTIONS_KEY)?, }) } diff --git a/crates/sqlbuild-python-project/src/job_settings/_helpers/argv.rs b/crates/sqlbuild-python-project/src/job_settings/_helpers/argv.rs index 9789f3333e..370edbf3ab 100644 --- a/crates/sqlbuild-python-project/src/job_settings/_helpers/argv.rs +++ b/crates/sqlbuild-python-project/src/job_settings/_helpers/argv.rs @@ -51,44 +51,38 @@ fn project_dir(namespace: &Bound<'_, PyDict>) -> PyResult { std::fs::canonicalize(&raw).or_else(|_| std::path::absolute(&raw).map_err(os_error)) } -/// `argv` plus the flags it leaves out of its named jobs' build-wide settings; `None` if unchanged. +/// The settings `argv` gives, and `argv` plus its named jobs' other build-wide flags, if any. pub(crate) fn named_job_argv( py: Python<'_>, argv: &[String], namespace: &Bound<'_, PyDict>, -) -> PyResult>> { +) -> PyResult<(BTreeSet, Option>)> { + let none = |given: BTreeSet| Ok((given, None)); let Some(command): Option = argument(namespace, "command")? else { - return Ok(None); + return none(BTreeSet::new()); }; let job_settings: bool = argument(namespace, "job_settings")?.unwrap_or(false); if !JOB_SETTING_COMMANDS.contains(&command.as_str()) || !job_settings { - return Ok(None); + return none(BTreeSet::new()); } + let given: BTreeSet = given_setting_dests(argv).ok_or_else(|| { + PyRuntimeError::new_err("cannot tell which settings the command line gives") + })?; let selectors: Vec = selectors(namespace)?; if !selectors.iter().any(|selector| could_name_job(selector)) { - return Ok(None); + return none(given); } let directory: PathBuf = project_dir(namespace)?; if !Path::new(&directory).join(ORCHESTRATION_DIRECTORY).is_dir() { - return Ok(None); + return none(given); } let target: Option = argument(namespace, "target")?; let settings: BTreeMap = - named_job_build_settings(py, &selectors, (&directory, target))?; - if settings.is_empty() { - return Ok(None); - } - let given: BTreeSet = given_setting_dests(argv).ok_or_else(|| { - PyRuntimeError::new_err("cannot tell which settings the command line gives") - })?; - let silent: BTreeMap = settings - .into_iter() - .filter(|(name, _)| !given.contains(name)) - .collect(); + named_job_build_settings(py, (&selectors, &given), (&directory, target))?; let flags: Vec = - accepted_settings_arguments(&command, &silent).map_err(PyValueError::new_err)?; + accepted_settings_arguments(&command, &settings).map_err(PyValueError::new_err)?; if flags.is_empty() { - return Ok(None); + return none(given); } let terminator: usize = argv .iter() @@ -96,7 +90,7 @@ pub(crate) fn named_job_argv( .unwrap_or(argv.len()); let mut job_argv: Vec = argv.to_vec(); job_argv.splice(terminator..terminator, flags); - Ok(Some(job_argv)) + Ok((given, Some(job_argv))) } /// Whether the command meets `error` again, and reports it, when it runs. diff --git a/crates/sqlbuild-python-project/src/job_settings/_helpers/job_catalog.rs b/crates/sqlbuild-python-project/src/job_settings/_helpers/job_catalog.rs index 55dee79117..c41c30b68b 100644 --- a/crates/sqlbuild-python-project/src/job_settings/_helpers/job_catalog.rs +++ b/crates/sqlbuild-python-project/src/job_settings/_helpers/job_catalog.rs @@ -33,6 +33,21 @@ struct JobScope { thread_local! { /// Per thread, since a command resolves its selectors on the thread that dispatches it. static JOB_SCOPE: RefCell> = const { RefCell::new(None) }; + /// The settings the running command's own command line gives, which replace named jobs'. + static GIVEN_SETTINGS: RefCell> = const { RefCell::new(BTreeSet::new()) }; +} + +/// Run one command whose command line gives `given` settings, restoring the enclosing ones after. +pub(crate) fn with_given_settings(given: BTreeSet, run: impl FnOnce() -> T) -> T { + let previous = GIVEN_SETTINGS.with(|current| current.replace(given)); + let result: T = run(); + GIVEN_SETTINGS.with(|current| *current.borrow_mut() = previous); + result +} + +/// The settings the running command's command line gives. +pub(crate) fn given_settings() -> BTreeSet { + GIVEN_SETTINGS.with(|current| current.borrow().clone()) } /// Run one command with jobs selectable from `project_dir`, restoring the enclosing scope after. diff --git a/crates/sqlbuild-python-project/src/job_settings/_helpers/named_job_build_settings.rs b/crates/sqlbuild-python-project/src/job_settings/_helpers/named_job_build_settings.rs index a368878aaa..7aa8be5ed6 100644 --- a/crates/sqlbuild-python-project/src/job_settings/_helpers/named_job_build_settings.rs +++ b/crates/sqlbuild-python-project/src/job_settings/_helpers/named_job_build_settings.rs @@ -1,34 +1,74 @@ //! The build-wide settings the named jobs give a build. -use std::collections::BTreeMap; +use std::collections::{BTreeMap, BTreeSet}; use std::path::Path; use pyo3::IntoPyObject; -use pyo3::prelude::{PyResult, Python}; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; use serde_json::Value; +use sqlbuild_analysis::job_settings::constants::WAREHOUSE_SETTING; use sqlbuild_analysis::job_settings::main::build_job_settings::build_job_settings; use sqlbuild_analysis::job_settings::main::named_jobs::named_jobs; +use sqlbuild_analysis::job_settings::main::without_given_settings::without_given_settings; +use sqlbuild_analysis::job_settings::models::DeclaredRun; +use sqlbuild_config::command_connection::main::check_job_warehouse::check_job_warehouse; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_front::compile_driver::main::session_warehouse_support::session_warehouse_support; +use sqlbuild_python_front::contracts::main::resolve_effective_adapter_name::resolve_effective_adapter_name; +use sqlbuild_python_front::discovery::main::project::discover_project_configuration::discover_project_configuration; use crate::job_settings::_helpers::job_run_settings::job_run_settings; use crate::job_settings::main::job_declarations::job_declarations; use crate::job_settings::main::job_setting_conflict::job_setting_conflict; use crate::job_settings::main::with_job_scope::with_job_scope; -/// The build-wide settings the named jobs give a build, as their runs take them, or `S027`. +/// The warehouse every named job sets, when they all set the same usable one. +fn shared_warehouse( + py: Python<'_>, + named: &[String], + (declared, directory): (&BTreeMap, &Bound<'_, PyAny>), +) -> PyResult> { + let mut warehouses = named.iter().map(|job| match declared.get(job) { + Some(DeclaredRun::Job { settings, .. }) => settings.get(WAREHOUSE_SETTING), + _ => None, + }); + let Some(Some(warehouse)) = warehouses.next() else { + return Ok(None); + }; + if warehouses.any(|other| other != Some(warehouse)) { + return Ok(None); + } + if check_job_warehouse("", warehouse.as_str()).is_err() { + return Ok(None); + } + let discovered: Bound<'_, PyAny> = discover_project_configuration(py, directory)?; + let adapter_name: Bound<'_, PyAny> = resolve_effective_adapter_name( + &native_field(&discovered, "project_config")?, + &native_field(&discovered, "local_config")?, + )?; + Ok(session_warehouse_support(py, &adapter_name, directory)?.then(|| (*warehouse).clone())) +} + +/// The named jobs' build-wide settings, a shared warehouse included, less `given`; or `S027`. pub(crate) fn named_job_build_settings( py: Python<'_>, - selectors: &[String], + (selectors, given): (&[String], &BTreeSet), (directory, selected_target): (&Path, Option), ) -> PyResult> { let scope = (directory.to_path_buf(), selected_target); let Some(declared) = with_job_scope(scope, || job_declarations(py))? else { return Ok(BTreeMap::new()); }; + let declared: BTreeMap = without_given_settings(declared, given); let named: Vec = named_jobs(selectors, &declared); if named.is_empty() { return Ok(BTreeMap::new()); } - let agreed: BTreeMap = build_job_settings(&named, &declared) + let mut agreed: BTreeMap = build_job_settings(&named, &declared) .map_err(|conflict| job_setting_conflict(py, &conflict))?; - job_run_settings(&directory.into_pyobject(py)?.into_any(), agreed) + let directory: Bound<'_, PyAny> = directory.into_pyobject(py)?.into_any(); + if let Some(warehouse) = shared_warehouse(py, &named, (&declared, &directory))? { + agreed.insert(WAREHOUSE_SETTING.to_owned(), warehouse); + } + job_run_settings(&directory, agreed) } diff --git a/crates/sqlbuild-python-project/src/job_settings/main/given_settings.rs b/crates/sqlbuild-python-project/src/job_settings/main/given_settings.rs new file mode 100644 index 0000000000..08cd78b9cb --- /dev/null +++ b/crates/sqlbuild-python-project/src/job_settings/main/given_settings.rs @@ -0,0 +1,11 @@ +//! The settings the running command's own command line gives. + +use std::collections::BTreeSet; + +use crate::job_settings::_helpers::job_catalog; + +/// The settings the running command's command line gives, which replace named jobs' settings. +#[inline] +pub fn given_settings() -> BTreeSet { + job_catalog::given_settings() +} diff --git a/crates/sqlbuild-python-project/src/job_settings/main/job_settings_argv.rs b/crates/sqlbuild-python-project/src/job_settings/main/job_settings_argv.rs index 4d489a9898..d4ff004767 100644 --- a/crates/sqlbuild-python-project/src/job_settings/main/job_settings_argv.rs +++ b/crates/sqlbuild-python-project/src/job_settings/main/job_settings_argv.rs @@ -1,24 +1,40 @@ //! The argv a build or plan naming jobs parses again with their build-wide flags. +use std::collections::BTreeSet; + use pyo3::exceptions::PyUnicodeError; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; -use pyo3::types::PyDict; +use pyo3::types::{PyDict, PyString}; +use sqlbuild_cli::argv::main::given_setting_dests::given_setting_dests; use crate::job_settings::_helpers::argv::{named_job_argv, reported_when_run}; +use crate::job_settings::types::JobSettingsArgv; -/// The argv to parse again with its named jobs' build-wide flags, or `None` to keep the parse. -pub fn job_settings_argv( - argv: &Bound<'_, PyAny>, - namespace: &Bound<'_, PyDict>, -) -> PyResult>> { - let py: Python<'_> = argv.py(); +/// The settings the command line gives, and the argv to parse again with its jobs' flags, if any. +pub fn job_settings_argv<'py>( + argv: &Bound<'py, PyAny>, + namespace: &Bound<'py, PyDict>, +) -> PyResult> { + let py: Python<'py> = argv.py(); let arguments: Vec = match argv.extract() { Ok(arguments) => arguments, - Err(undecodable) if undecodable.is_instance_of::(py) => return Ok(None), + Err(undecodable) if undecodable.is_instance_of::(py) => { + return Ok((BTreeSet::new(), None)); + } Err(error) => return Err(error), }; - match named_job_argv(py, &arguments, namespace) { - Err(error) if reported_when_run(py, &error)? => Ok(None), - result => result, - } + let (given, job_argv) = match named_job_argv(py, &arguments, namespace) { + Err(error) if reported_when_run(py, &error)? => { + (given_setting_dests(&arguments).unwrap_or_default(), None) + } + result => result?, + }; + let Some(job_argv) = job_argv else { + return Ok((given, None)); + }; + let strings: Vec> = job_argv + .iter() + .map(|argument| PyString::new(py, argument)) + .collect(); + Ok((given, Some(strings))) } diff --git a/crates/sqlbuild-python-project/src/job_settings/main/mod.rs b/crates/sqlbuild-python-project/src/job_settings/main/mod.rs index 347338ec62..d9222cb453 100644 --- a/crates/sqlbuild-python-project/src/job_settings/main/mod.rs +++ b/crates/sqlbuild-python-project/src/job_settings/main/mod.rs @@ -1,6 +1,7 @@ pub mod declared_jobs; pub mod frozen_run_flags; pub mod frozen_selectors; +pub mod given_settings; pub mod graph_with_jobs; pub mod job_declarations; pub mod job_names; @@ -12,4 +13,5 @@ pub mod named_jobs; pub mod needed_jobs; pub mod triggered_jobs; pub mod with_frozen_job; +pub mod with_given_settings; pub mod with_job_scope; diff --git a/crates/sqlbuild-python-project/src/job_settings/main/with_given_settings.rs b/crates/sqlbuild-python-project/src/job_settings/main/with_given_settings.rs new file mode 100644 index 0000000000..fe9957d53f --- /dev/null +++ b/crates/sqlbuild-python-project/src/job_settings/main/with_given_settings.rs @@ -0,0 +1,11 @@ +//! Run one command whose command line gives some settings itself. + +use std::collections::BTreeSet; + +use crate::job_settings::_helpers::job_catalog; + +/// Run one command whose command line gives `given` settings, restoring the enclosing ones after. +#[inline] +pub fn with_given_settings(given: BTreeSet, run: impl FnOnce() -> T) -> T { + job_catalog::with_given_settings(given, run) +} diff --git a/crates/sqlbuild-python-project/src/job_settings/mod.rs b/crates/sqlbuild-python-project/src/job_settings/mod.rs index 6be20307a7..66a086ca40 100644 --- a/crates/sqlbuild-python-project/src/job_settings/mod.rs +++ b/crates/sqlbuild-python-project/src/job_settings/mod.rs @@ -4,3 +4,4 @@ pub(crate) mod _helpers; pub mod constants; pub mod main; pub mod models; +pub mod types; diff --git a/crates/sqlbuild-python-project/src/job_settings/types.rs b/crates/sqlbuild-python-project/src/job_settings/types.rs new file mode 100644 index 0000000000..5873fec138 --- /dev/null +++ b/crates/sqlbuild-python-project/src/job_settings/types.rs @@ -0,0 +1,9 @@ +//! Type aliases of named-job settings. + +use std::collections::BTreeSet; + +use pyo3::prelude::Bound; +use pyo3::types::PyString; + +/// The settings a command line gives, and its argv with its named jobs' flags when they add any. +pub type JobSettingsArgv<'py> = (BTreeSet, Option>>); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs index 633587ddeb..a3e5e4b46e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs @@ -26,6 +26,7 @@ use crate::bindings::_helpers::cli::failure_cause::take_failure_cause; use sqlbuild_python_base::cli::main::offer_launch_record::offer_launch_record; use crate::bindings::_helpers::cli::machine_output::take_machine_output_written; use sqlbuild_python_project::job_settings::main::job_settings_argv::job_settings_argv; +use sqlbuild_python_project::job_settings::main::with_given_settings::with_given_settings; use crate::bindings::_helpers::cli::observability::{ CliObservabilityScope, cli_observability_scope, }; @@ -821,11 +822,8 @@ fn parsed_invocation<'py>( }; } let mut namespace: Bound<'py, PyDict> = namespace_of(&args)?; - if let Some(job_argv) = job_settings_argv(argv, &namespace)? { - let job_argv: Vec> = job_argv - .iter() - .map(|argument| PyString::new(py, argument)) - .collect(); + let (given, job_argv) = job_settings_argv(argv, &namespace)?; + if let Some(job_argv) = job_argv { args = native_field(&parse_cli_invocation(py, job_argv, use_color)?, "args")?; if args.is_none() { return exit(py, 1); @@ -847,7 +845,9 @@ fn parsed_invocation<'py>( execute_parsed_invocation(args, handlers, identity, use_color) }); } - execute_parsed_invocation(args, handlers, identity, use_color) + with_given_settings(given, || { + execute_parsed_invocation(args, handlers, identity, use_color) + }) } /// Runtime stream routing must cover extension discovery, invocation scopes and their teardown. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/snowflake.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/snowflake.md index 0d528d76cb..4ce91a45d2 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/snowflake.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/snowflake.md @@ -123,12 +123,17 @@ A job can run its nodes on a warehouse of its own: warehouse = "ORDERS_WH" ``` -When a build names the job, as `sqb build --select job:nightly_orders_job` or a server run of it, -each of the job's nodes runs on `ORDERS_WH`. Its models' audits, its SQL tests, microbatch batches, -source loads and Python nodes go with them, and loaders and Python code that open their own -connections from the context's `connection_config` get `ORDERS_WH` there too. A SQL test runs on -its models' job warehouse when they agree on one. Everything else in the build, including nodes -selected without naming a job and the build's own bookkeeping, stays on the `build` warehouse. +A server run of the job, and a build whose named jobs all set the same warehouse, such as +`sqb build --select job:nightly_orders_job`, runs entirely on `ORDERS_WH`, as if passed +`--warehouse ORDERS_WH`: setup, planning and bookkeeping included. Nodes the build selects outside +the named jobs, such as upstream models in `--select +job:nightly_orders_job`, run there too. + +When the named jobs set different warehouses, or only some set one, each job's nodes run on its +own warehouse. Its models' audits, its SQL tests, microbatch batches, source loads and Python nodes +go with them, and loaders and Python code that open their own connections from the context's +`connection_config` get the job's warehouse there too. A SQL test runs on a job warehouse only +when every node it covers runs there. Everything else in the build, including nodes selected +without naming a job and the build's own bookkeeping, stays on the `build` warehouse. The connection running a node switches with `USE WAREHOUSE` only when the node needs a different warehouse than the connection is on, and switches back before it runs a node outside the job. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/orchestration.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/orchestration.md index ab50202c83..ec8829bdf7 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/orchestration.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/orchestration.md @@ -109,8 +109,9 @@ Naming a group names each of its jobs. The first value set wins: 3. The target's setting 4. The project's setting -A command-line flag replaces the job's setting rather than merging with it: `--vars` replaces the -job's whole `vars`, and `--warehouse` puts every node on its warehouse. An option that rules another out counts as setting both, so `--reload` wins +A command-line flag replaces every named job's setting rather than merging with it, so it also +settles named jobs that disagree: `--vars` replaces the job's whole `vars`, `--full-refresh` +rebuilds every selected model, and `--warehouse` puts every node on its warehouse. An option that rules another out counts as setting both, so `--reload` wins over a job's `load_sources = false`. Any `--concurrency` sets the build's limit and turns off the per-job limits. The job's settings are added before a `--` on the command line, so selectors after it still name jobs. `--no-job-settings` leaves out every job's settings. @@ -120,9 +121,9 @@ nodes differently: | Setting | Applies to | | --- | --- | -| `full_refresh` | The job's incremental models, rebuilt from scratch | +| `full_refresh` | The job's incremental models, rebuilt from scratch, and its functions, recreated | | `run_tests` | The SQL unit tests of the job's models | -| `ignore_audit_failures` | The audits of the job's nodes | +| `ignore_audit_failures` | The audits of the job's nodes, and singular audits that read them; a singular audit is ignored when every named job that sets it for the nodes the audit reads sets `true` | | `warehouse` | The Snowflake warehouse the job's nodes run on, Python nodes included; see [A job's own warehouse](adapters/snowflake.md#a-jobs-own-warehouse) | | `concurrency` | How many of the job's nodes build at once, and a node in two named jobs fits within both. The build's own limit becomes the larger of the configured limit and the named jobs' limits added together | | `[retry]` | The job's nodes | @@ -131,9 +132,16 @@ A job's settings reach only the nodes its own selection holds. `--select job:nig gives them to `orders` alone, and `+job:nightly_orders_job` builds the job's upstream nodes without them. -Every other setting, such as `vars` or `changed`, applies to the whole build, so every named job -must set it the same way. A group whose jobs set any of these differently on a target that declares -them together fails to compile, with the error a build of the group would give. +Every other setting, such as `vars`, `changed` or `fail_fast`, applies to the whole build, so every +named job must set it the same way; a job that leaves it unset disagrees with one that sets it. +Pass the flag to settle a disagreement. When every named job sets the same `warehouse`, it is the +whole build's warehouse, as if passed as `--warehouse`; see +[A job's own warehouse](adapters/snowflake.md#a-jobs-own-warehouse). + +Each job in a group still runs as its own build, so a group's jobs must agree only on `vars` and +`changed`, which change the compiled SQL. A group whose jobs set either differently on a target +that declares them together fails to compile, with the error a build of the group would give. + Two named jobs that give one node different values of the same setting fail the build with `S027`, naming the node, both jobs and the setting. A setting only one of them sets is no conflict. @@ -322,7 +330,7 @@ Every problem with an orchestration file is a `D017` error at its file, line and | `job nightly_orders_job resources: cpus must be a positive number` | Set `cpus` above zero | | `job 'nightly_orders_job' uses pool 'warehouse_hevy', which [pools] in sqlbuild_project.toml does not declare` | Fix the name, or declare the pool in [`[pools]`](project-configuration.md#pools) | | `job 'nightly_orders_job' takes 2 slots of pool 'supplier_api', more than its limit of 1 on target 'production'` | Take fewer slots, or raise the limit in `[targets.production.pools]` | -| `jobs orders_job and crm_job set vars differently ({"amount":"5"} and {"amount":"9"}); a build applies vars to everything it runs; group daily_job_group builds its jobs together` | Give the group's jobs the same build-wide settings, or move one job to another group | +| `jobs orders_job and crm_job set vars differently ({"amount":"5"} and {"amount":"9"}); a build applies vars to everything it runs; group daily_job_group builds its jobs together` | Give the group's jobs the same `vars` and `changed`, or move one job to another group | | `sensor 'orders_sensor' is declared here, but importing the module does not define it` | Assign the sensor to a module-level name or list, not a dictionary or function body | | `group dependencies contain a cycle in job group daily_orders_job_group: orders_marts_job waits for staging_orders_job (model:orders reads model:stg_orders); staging_orders_job waits for orders_marts_job (model:stg_orders reads seed:regions)` | Move each node to one job, and give seeds and nodes that several members read their own job that runs first | @@ -334,7 +342,9 @@ Details: them, as on the server. - Job settings that cannot apply together fail the build or plan with `S027`, such as `model:orders is selected through jobs orders_job and finance_job, which set run_tests differently (false and - true)`. Make the jobs agree, set it in one of them, pass the flag, or use `--no-job-settings`. + true)`. Make the jobs agree, set it in only one of them, pass the flag, or use + `--no-job-settings`. For a build-wide setting, where unset differs from set, make the jobs agree, + pass the flag, or use `--no-job-settings`. - Selectors are checked for syntax at compile. A selector that names a node the project does not have fails when a location publishes, such as `job nightly_orders_job: unknown selector name 'order_total'`. diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 6fbe44c29e..2bdee14a74 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -10513,6 +10513,7 @@ class PlannerOverrides(NativeRecord): cursor_overrides: CursorOverrides | None full_refresh: bool full_refresh_models: frozenset[str] + full_refresh_functions: frozenset[str] reload_sources: bool forced_stale_model_names: tuple[str, ...] external_blocked_model_names: tuple[str, ...] @@ -10523,6 +10524,7 @@ class PlannerOverrides(NativeRecord): cursor_overrides: CursorOverrides | None = ..., full_refresh: bool = ..., full_refresh_models: frozenset[str] = ..., + full_refresh_functions: frozenset[str] = ..., reload_sources: bool = ..., forced_stale_model_names: tuple[str, ...] = ..., external_blocked_model_names: tuple[str, ...] = ..., diff --git a/src/sqlbuild/executor/build/classes/build_scheduler.py b/src/sqlbuild/executor/build/classes/build_scheduler.py index aad6c2b23e..31928ed4d8 100644 --- a/src/sqlbuild/executor/build/classes/build_scheduler.py +++ b/src/sqlbuild/executor/build/classes/build_scheduler.py @@ -691,9 +691,14 @@ def _execute_concurrent_microbatch_node( ledger_path=(self._runtime_dir / "executions" / self._run_id / "statements.jsonl"), on_statement_complete=self._on_statement_complete, ): - failure: ModelExecutionResult | None = self._enter_node_warehouse( - key=key, connection=connection - ) + failure: ( + ModelExecutionResult + | SeedExecutionResult + | FunctionExecutionResult + | SqlTestExecutionResult + | LoadExecutionResult + | None + ) = self._enter_node_warehouse(key=key, connection=connection) if failure is not None: return failure return self._execute_model_node( @@ -1052,9 +1057,14 @@ def _execute_node_with_cost_context( ): """Execute a single node. Runs in a worker thread (or scheduler thread for serial).""" - failure: ModelExecutionResult | None = self._enter_node_warehouse( - key=key, connection=connection - ) + failure: ( + ModelExecutionResult + | SeedExecutionResult + | FunctionExecutionResult + | SqlTestExecutionResult + | LoadExecutionResult + | None + ) = self._enter_node_warehouse(key=key, connection=connection) if failure is not None: return failure if key.resource_type == CompiledResourceType.SEED: @@ -1076,8 +1086,15 @@ def _execute_node_with_cost_context( def _enter_node_warehouse( self, *, key: CompiledObjectKey, connection: Any - ) -> ModelExecutionResult | None: - """Put `connection` on the node's job warehouse, or the build's; the failure if not.""" + ) -> ( + ModelExecutionResult + | SeedExecutionResult + | FunctionExecutionResult + | SqlTestExecutionResult + | LoadExecutionResult + | None + ): + """Put `connection` on the node's job warehouse, or the build's; its failure if not.""" if self._session_warehouses is None: return None @@ -1085,10 +1102,8 @@ def _enter_node_warehouse( try: self._session_warehouses.use(connection=connection, warehouse=warehouse) except Exception as error: # noqa: BLE001 - the node fails with the reason - return ModelExecutionResult( - model_name=key.name, - status=ExecutionStatus.FAILED, - error_code=BUILD_WORKER_FAILED_CODE, + return self._failed_node_result( + key=key, error_message=( f"cannot run on warehouse {warehouse}: {error}" if warehouse is not None @@ -1097,6 +1112,56 @@ def _enter_node_warehouse( ) return None + def _failed_node_result( + self, *, key: CompiledObjectKey, error_message: str + ) -> ( + ModelExecutionResult + | SeedExecutionResult + | FunctionExecutionResult + | SqlTestExecutionResult + | LoadExecutionResult + ): + """The failed result of the node's own kind, so its kind's failure handling applies.""" + + if key.resource_type == CompiledResourceType.SEED: + return SeedExecutionResult( + seed_name=key.name, + status=ExecutionStatus.FAILED, + error_code=BUILD_WORKER_FAILED_CODE, + error_message=error_message, + ) + if key.resource_type == CompiledResourceType.SOURCE: + source: SourceEntry = self._plan.source_map[key.name] + return LoadExecutionResult( + source_name=source.name, + loader_name=source.loader or "", + status=ExecutionStatus.FAILED, + target=source.table or source.name, + resource_kind=load_resource_kind(source), + error_message=error_message, + ) + if key.resource_type in {CompiledResourceType.UDF, CompiledResourceType.TABLE_FN}: + return FunctionExecutionResult( + function_name=key.name, + status=ExecutionStatus.FAILED, + function_kind=str(key.resource_type), + error_code=BUILD_WORKER_FAILED_CODE, + error_message=error_message, + ) + if key.resource_type == CompiledResourceType.SQL_TEST: + return SqlTestExecutionResult( + test_name=key.name, + outcome=SqlTestOutcome.ERROR, + error_code=BUILD_WORKER_FAILED_CODE, + error_message=error_message, + ) + return ModelExecutionResult( + model_name=key.name, + status=ExecutionStatus.FAILED, + error_code=BUILD_WORKER_FAILED_CODE, + error_message=error_message, + ) + def _execute_source_node( self, *, key: CompiledObjectKey, connection: Any ) -> LoadExecutionResult: diff --git a/src/sqlbuild/executor/python_nodes/_helpers/execution.py b/src/sqlbuild/executor/python_nodes/_helpers/execution.py index 4501569516..aa979c659d 100644 --- a/src/sqlbuild/executor/python_nodes/_helpers/execution.py +++ b/src/sqlbuild/executor/python_nodes/_helpers/execution.py @@ -19,6 +19,7 @@ from sqlbuild.executor.node_results.main._direct_store import build_direct_node_result_store from sqlbuild.executor.node_results.models import NodeResultRecord from sqlbuild.executor.node_results.types import NodeResultStatus +from sqlbuild.executor.python_nodes._helpers.node_warehouse import node_runtime from sqlbuild.executor.python_nodes._helpers.relation_guard import build_python_node_relation_guard from sqlbuild.executor.python_nodes._helpers.results import ( build_python_node_failure_result, @@ -180,6 +181,16 @@ def _execute_ready_node_in_cost_scope( ) _persist_python_node_result(result_store=result_store, result=result, run_id=run_id) return result, None + runtime, warehouse_failure = node_runtime(runtime=runtime, name=node.name) + if warehouse_failure is not None: + result = PythonNodeExecutionResult( + node_name=node.name, + kind=node_kind, + status=PythonNodeStatus.FAILED, + error_message=warehouse_failure, + ) + _persist_python_node_result(result_store=result_store, result=result, run_id=run_id) + return result, None warnings: list[str] = [] context: TaskContext | AssetContext = _build_context( node=node, diff --git a/src/sqlbuild/executor/python_nodes/_helpers/ingress_execution.py b/src/sqlbuild/executor/python_nodes/_helpers/ingress_execution.py index 228c059e00..82e990217e 100644 --- a/src/sqlbuild/executor/python_nodes/_helpers/ingress_execution.py +++ b/src/sqlbuild/executor/python_nodes/_helpers/ingress_execution.py @@ -150,7 +150,6 @@ def _execute_ingress_lifecycle_node( results: IngressResultAccumulator, ) -> LifecycleNodeResult: discovered_node: DiscoveredPythonNode = python_graph.nodes_by_name[node.name] - runtime = node_runtime(runtime=runtime, name=node.name) if discovered_node.kind == PythonNodeKind.LOADER: return _execute_ingress_loader( node=discovered_node, @@ -277,35 +276,47 @@ def _execute_ingress_loader( resource_name=loader.name, phase="load", ): - result: LoadExecutionResult = execute_source_load( - source_entry=source_entry, - loader_function=loader, - adapter=runtime.adapter, - connection_config=runtime.connection_config, - connection=runtime.connection, - runtime=LoadRuntimeParams( - run_id=runtime.run_id, - target=runtime.target, - target_config=runtime.target_config, - default_database=runtime.default_database, - default_schema=runtime.default_schema, - vars=runtime.vars, - is_reload=runtime.is_reload, - start_cursor_ts=runtime.start_cursor_ts, - end_cursor_ts=runtime.end_cursor_ts, - start_cursor_int=runtime.start_cursor_int, - end_cursor_int=runtime.end_cursor_int, - use_color=callbacks.use_color, - providers=runtime.providers, - result_store=runtime.result_store, - project_relations=runtime.project_relations, - ), - statement_recorder=StatementRecorder(), - loader_ref_entries=_loader_ref_entries( - loader_by_name=loader_by_name, - source_by_loader_name=source_by_loader_name, - ), - source_ref_entries=source_map, + runtime, warehouse_failure = node_runtime(runtime=runtime, name=node.name) + result: LoadExecutionResult = ( + LoadExecutionResult( + source_name=source_entry.name, + loader_name=loader.name, + status=ExecutionStatus.FAILED, + target=source_entry.table or source_entry.name, + resource_kind=resource_kind, + error_message=warehouse_failure, + ) + if warehouse_failure is not None + else execute_source_load( + source_entry=source_entry, + loader_function=loader, + adapter=runtime.adapter, + connection_config=runtime.connection_config, + connection=runtime.connection, + runtime=LoadRuntimeParams( + run_id=runtime.run_id, + target=runtime.target, + target_config=runtime.target_config, + default_database=runtime.default_database, + default_schema=runtime.default_schema, + vars=runtime.vars, + is_reload=runtime.is_reload, + start_cursor_ts=runtime.start_cursor_ts, + end_cursor_ts=runtime.end_cursor_ts, + start_cursor_int=runtime.start_cursor_int, + end_cursor_int=runtime.end_cursor_int, + use_color=callbacks.use_color, + providers=runtime.providers, + result_store=runtime.result_store, + project_relations=runtime.project_relations, + ), + statement_recorder=StatementRecorder(), + loader_ref_entries=_loader_ref_entries( + loader_by_name=loader_by_name, + source_by_loader_name=source_by_loader_name, + ), + source_ref_entries=source_map, + ) ) results.record_load_result(name=node.name, result=result) persist_loader_result( diff --git a/src/sqlbuild/executor/python_nodes/_helpers/node_warehouse.py b/src/sqlbuild/executor/python_nodes/_helpers/node_warehouse.py index 5477b8db3b..8d6da355d6 100644 --- a/src/sqlbuild/executor/python_nodes/_helpers/node_warehouse.py +++ b/src/sqlbuild/executor/python_nodes/_helpers/node_warehouse.py @@ -11,19 +11,29 @@ ) -def node_runtime(*, runtime: PythonNodeRuntime, name: str) -> PythonNodeRuntime: - """The runtime of node `name`, its connection and `connection_config` on its warehouse.""" +def node_runtime(*, runtime: PythonNodeRuntime, name: str) -> tuple[PythonNodeRuntime, str | None]: + """The runtime of node `name` on its warehouse, and why its connection cannot move there.""" session_warehouses: SessionWarehouses | None = runtime.session_warehouses if session_warehouses is None: - return runtime + return runtime, None warehouse: str | None = runtime.node_warehouses.get(name) - session_warehouses.use(connection=runtime.connection, warehouse=warehouse) + try: + session_warehouses.use(connection=runtime.connection, warehouse=warehouse) + except Exception as error: # noqa: BLE001 - the node fails with the reason + return runtime, ( + f"cannot run on warehouse {warehouse}: {error}" + if warehouse is not None + else f"cannot return to the build's warehouse: {error}" + ) if warehouse is None: - return runtime - return replace( - runtime, - connection_config=with_session_warehouse( - connection_config=runtime.connection_config, warehouse=warehouse + return runtime, None + return ( + replace( + runtime, + connection_config=with_session_warehouse( + connection_config=runtime.connection_config, warehouse=warehouse + ), ), + None, ) diff --git a/src/sqlbuild/executor/python_nodes/_helpers/python_checks.py b/src/sqlbuild/executor/python_nodes/_helpers/python_checks.py index 48bdd2cd79..472ed512aa 100644 --- a/src/sqlbuild/executor/python_nodes/_helpers/python_checks.py +++ b/src/sqlbuild/executor/python_nodes/_helpers/python_checks.py @@ -135,7 +135,18 @@ def _execute_one_python_check( result_store: Any | None, callbacks: PythonCheckCallbacks, ) -> PythonCheckExecutionResult: - runtime = node_runtime(runtime=runtime, name=check_function.name) + if blocked is None: + runtime, warehouse_failure = node_runtime(runtime=runtime, name=check_function.name) + blocked = ( + None + if warehouse_failure is None + else PythonCheckExecutionResult( + node_name=check_function.name, + passed=False, + severity=PythonCheckSeverity.ERROR, + error_message=warehouse_failure, + ) + ) adapter: BaseAdapter = runtime.adapter providers: ProviderContainer | None = runtime.providers with ( diff --git a/src/sqlbuild/executor/python_nodes/classes/read_side_python_execution_tracker.py b/src/sqlbuild/executor/python_nodes/classes/read_side_python_execution_tracker.py index 58501b2a90..b063f8a60c 100644 --- a/src/sqlbuild/executor/python_nodes/classes/read_side_python_execution_tracker.py +++ b/src/sqlbuild/executor/python_nodes/classes/read_side_python_execution_tracker.py @@ -13,7 +13,6 @@ try_write_python_node_identity_fingerprint, ) from sqlbuild.executor.python_nodes._helpers.ingress_execution import _to_executable_python_node -from sqlbuild.executor.python_nodes._helpers.node_warehouse import node_runtime from sqlbuild.executor.python_nodes._helpers.read_side_execution import ( _sql_result_failed, _sql_result_name, @@ -152,9 +151,7 @@ def _python_dependencies_satisfied(self, node: DiscoveredPythonNode) -> bool: return True def _execute_python_node(self, node: DiscoveredPythonNode) -> None: - runtime: PythonNodeRuntime = node_runtime( - runtime=replace(self._runtime, result_store=self._result_store), name=node.name - ) + runtime: PythonNodeRuntime = replace(self._runtime, result_store=self._result_store) _ = run_ready_python_node( node=_to_executable_python_node(node), upstream_results=tuple( diff --git a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py index a0b2efbf62..f0bb2e2b51 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py @@ -1528,6 +1528,7 @@ class JobWarehouseSwitchTestCase: argv: tuple[str, ...] expected_connect_warehouses: frozenset[str] expected_switches: tuple[str, ...] + extra_files: dict[str, str] = field(default_factory=dict) @dataclass(frozen=True) @@ -1538,3 +1539,16 @@ class JobWarehouseOutputTestCase: argv: tuple[str, ...] expected_exit_code: int expected_fragment: str + + +@dataclass(frozen=True) +class JobFlagParityTestCase: + """A single named job's run, and the global flags its node settings used to be.""" + + description: str + job_args: tuple[str, ...] + flag_plan_args: tuple[str, ...] + flag_build_args: tuple[str, ...] + expected_function_reasons: tuple[tuple[str, str], ...] + expected_full_refresh_counts: tuple[str, ...] + expected_exit_code: int diff --git a/tests/integration/src/sqlbuild/cli/commands/main/helpers.py b/tests/integration/src/sqlbuild/cli/commands/main/helpers.py index b61fbbf012..6ecfeec8de 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/helpers.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/helpers.py @@ -1500,8 +1500,9 @@ def _job_file(name: str, select: str, warehouse: str) -> str: def write_job_warehouse_project(*, project_dir: Path) -> None: """A Snowflake chain stg -> orders -> orders_daily -> refunds -> summary and a task, with jobs - on their own warehouses: orders_wh_job, refunds_wh_job, refunds_alt_job (also refunds), - tasks_wh_job and bad_wh_job, whose warehouse is not an identifier.""" + on their own warehouses: orders_wh_job, orders_copy_job (orders_daily on the same warehouse), + refunds_wh_job, refunds_alt_job (also refunds), tasks_wh_job and bad_wh_job, whose warehouse + is not an identifier.""" write_snowflake_warehouse_project( project_dir=project_dir, @@ -1526,6 +1527,9 @@ def write_job_warehouse_project(*, project_dir: Path) -> None: "orchestration/jobs/refunds_wh_job.toml": _job_file( "refunds_wh_job", "refunds", "REFUNDS_WH" ), + "orchestration/jobs/orders_copy_job.toml": _job_file( + "orders_copy_job", "orders_daily", "ORDERS_WH" + ), "orchestration/jobs/refunds_alt_job.toml": _job_file( "refunds_alt_job", "refunds", "ALT_WH" ), @@ -1571,3 +1575,83 @@ def run_cli_exit_code(argv: tuple[str, ...], project_dir: Path) -> int: return int(main(["--no-color", "--project-dir", str(project_dir), *argv]) or 0) except SystemExit as exited: return int(exited.code or 0) + + +_PARITY_PROJECT_FILES: dict[str, str] = { + "sqlbuild_project.toml": ( + 'name = "demo"\nadapter = "duckdb"\ndefault_target = "dev"\n\n' + '[connection]\ndatabase = "demo.duckdb"\n\n[targets.dev]\n' + ), + "functions/sql/udf__band.sql": ( + "FUNCTION (description 'Band.', arguments (amount INT), returns VARCHAR);\n" + "CASE WHEN amount >= 20 THEN 'large' ELSE 'small' END\n" + ), + "models/parity_lookup.sql": ( + "MODEL (description 'Lookup.', materialized table, " + "columns (id (type INTEGER, audits [not_null])));\nSELECT CAST(NULL AS INTEGER) AS id\n" + ), + "models/parity_events.sql": ( + "MODEL (description 'Events.', materialized incremental, incremental_strategy append);\n" + 'SELECT 1 AS event_id, __udf("udf__band")(5) AS band\n' + ), + "tests/unit/test_parity_events.sql": ( + "TEST();\n\nWITH\n__ref__parity_lookup AS (SELECT 1 AS id),\n" + "__expected__parity_events AS (SELECT 99 AS event_id, 'x' AS band)\nSELECT 1\n" + ), + "audits/singular/events_have_lookups.sql": ( + 'AUDIT (name "events_have_lookups", severity error);\n' + 'SELECT e.event_id FROM __ref("parity_events") e ' + 'LEFT JOIN __ref("parity_lookup") l ON e.event_id = l.id WHERE l.id IS NULL\n' + ), + "orchestration/jobs/parity_job.toml": ( + 'name = "parity_job"\ndescription = "Parity."\n' + 'select = "udf__band parity_lookup parity_events"\n\n' + "[settings]\nfull_refresh = true\nrun_tests = false\nignore_audit_failures = true\n" + ), +} +PARITY_NODES: tuple[str, ...] = ("udf__band", "parity_lookup", "parity_events") + + +def job_flag_parity_outcome( + *, + project_dir: Path, + args: tuple[tuple[str, ...], tuple[str, ...]], + capsys: pytest.CaptureFixture[str], +) -> tuple[object, ...]: + """Build a parity project once without its job, then plan and build it with `args`: the + function reasons, full-refresh count, exit code, summary and ignored audits they report.""" + + plan_args, build_args = args + + write_project_files(project_dir=project_dir, files=_PARITY_PROJECT_FILES) + base: list[str] = ["--no-color", "--project-dir", str(project_dir)] + main( + [ + *base, + "build", + "--no-job-settings", + "--no-tests", + "--ignore-audit-failures", + "--select", + *PARITY_NODES, + ] + ) + capsys.readouterr() + main([*base, "plan", *plan_args]) + plan: str = capsys.readouterr().out + exit_code: int = main([*base, "build", *build_args]) + build: str = capsys.readouterr().out + return ( + tuple(re.findall(r"^ (\S+) +sql udf\n +reason: (.*)$", plan, re.M)), + tuple(re.findall(r"^Full refresh \((\d+)\)$", plan, re.M)), + exit_code, + tuple(re.findall(r"Completed [^\n]*?TOTAL=\d+", build)), + tuple(re.findall(r"audit .+ returned \d+ rows? \(ignored\)", build)), + ) + + +def unit_test_sql(*models: str) -> str: + """A unit test mocking orders_daily that expects one row from each of `models`.""" + + expected: str = ",\n".join(f"__expected__{model} AS (SELECT 1 AS id)" for model in models) + return f"TEST();\n\nWITH\n__ref__orders_daily AS (SELECT 1 AS id),\n{expected}\nSELECT 1\n" diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_job_settings.py b/tests/integration/src/sqlbuild/cli/commands/main/test_job_settings.py index be8cc81840..13e91e3249 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_job_settings.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_job_settings.py @@ -9,12 +9,15 @@ from sqlbuild._native import run_cli as main from tests.integration.src.sqlbuild.cli.commands.main._test_types import ( JobConcurrencyTestCase, + JobFlagParityTestCase, JobFullRefreshTestCase, JobOutputTestCase, JobSettingsBuildTestCase, JobSettingsConflictTestCase, ) from tests.integration.src.sqlbuild.cli.commands.main.helpers import ( + PARITY_NODES, + job_flag_parity_outcome, peak_concurrency, query_duckdb_rows, write_project_files, @@ -105,6 +108,22 @@ ), } +_FAIL_FAST_FILES: dict[str, str] = { + "orchestration/jobs/fast_fail_job.toml": ( + 'name = "fast_fail_job"\ndescription = "Fails fast."\nselect = "customers"\n\n' + "[settings]\nfail_fast = true\n" + ), + "orchestration/job_groups/fail_job_group.toml": ( + 'name = "fail_job_group"\ndescription = "Fails."\njobs = ["events_job", "fast_fail_job"]\n' + ), +} +_STEADY_EVENTS_FILES: dict[str, str] = { + "orchestration/jobs/steady_events_job.toml": ( + 'name = "steady_events_job"\ndescription = "Events, kept."\nselect = "order_events"\n\n' + "[settings]\nfull_refresh = false\n" + ), +} + @pytest.mark.parametrize( "test_case", @@ -208,23 +227,23 @@ def test_given_a_named_job_when_building_then_its_settings_apply_below_the_comma ), ), JobSettingsConflictTestCase( - description="a group whose jobs set any build-wide setting differently does not compile", - command_args=("compile",), + description="two named jobs setting a build-wide setting differently cannot build", + command_args=("build", "--select", "job:events_job", "job:fast_fail_job"), expected_fragment=( "jobs events_job and fast_fail_job set fail_fast differently (not set and true); " - "a build applies fail_fast to everything it runs; group fail_job_group builds its " - "jobs together" + "a build applies fail_fast to everything it runs = help: make the jobs agree, " + "override it with the command-line flag, or build with --no-job-settings" ), - extra_files={ - "orchestration/jobs/fast_fail_job.toml": ( - 'name = "fast_fail_job"\ndescription = "Fails fast."\nselect = "customers"\n\n' - "[settings]\nfail_fast = true\n" - ), - "orchestration/job_groups/fail_job_group.toml": ( - 'name = "fail_job_group"\ndescription = "Fails."\n' - 'jobs = ["events_job", "fast_fail_job"]\n' - ), - }, + extra_files=_FAIL_FAST_FILES, + ), + JobSettingsConflictTestCase( + description="two named jobs giving one node different full_refresh conflict", + command_args=("build", "--select", "job:events_job", "job:steady_events_job"), + expected_fragment=( + "error[S027]: model:order_events is selected through jobs events_job and " + "steady_events_job, which set full_refresh differently (true and false)" + ), + extra_files=_STEADY_EVENTS_FILES, ), JobSettingsConflictTestCase( description="two named jobs giving one node different run_tests conflict", @@ -459,7 +478,7 @@ def test_given_a_job_concurrency_when_building_then_its_nodes_never_exceed_it( description="the run context names the jobs whose models are fully refreshed", command_args=("build", "--verbose", "--select", "job:events_job"), expected_exit_code=0, - expected_present=("full_refresh models of job events_job",), + expected_present=("full_refresh nodes of job events_job",), expected_absent=(), ), JobOutputTestCase( @@ -490,6 +509,42 @@ def test_given_a_job_concurrency_when_building_then_its_nodes_never_exceed_it( ), }, ), + JobOutputTestCase( + description="a group whose jobs differ only in fail_fast compiles, each job a build", + command_args=("compile",), + expected_exit_code=0, + expected_present=("Project compiled",), + expected_absent=("D017",), + extra_files=_FAIL_FAST_FILES, + ), + JobOutputTestCase( + description="--fail-fast settles the named jobs' fail_fast disagreement", + command_args=( + "build", + "--fail-fast", + "--select", + "job:events_job", + "job:fast_fail_job", + ), + expected_exit_code=0, + expected_present=("Completed successfully",), + expected_absent=("S027",), + extra_files=_FAIL_FAST_FILES, + ), + JobOutputTestCase( + description="--full-refresh settles two jobs' full_refresh disagreement on one node", + command_args=( + "build", + "--full-refresh", + "--select", + "job:events_job", + "job:steady_events_job", + ), + expected_exit_code=0, + expected_present=("Completed successfully",), + expected_absent=("S027",), + extra_files=_STEADY_EVENTS_FILES, + ), JobOutputTestCase( description="a command-line option beats a job option it excludes", command_args=("build", "--select", "job:loads_off_job", "--reload"), @@ -517,3 +572,57 @@ def test_given_a_named_job_when_planning_or_building_then_the_output_reflects_it assert {fragment: fragment in captured for fragment in test_case.expected_absent} == ( dict.fromkeys(test_case.expected_absent, False) ), captured + + +@pytest.mark.parametrize( + "test_case", + [ + JobFlagParityTestCase( + description="full_refresh, run_tests and ignore_audit_failures act as the flags did", + job_args=("--select", "job:parity_job"), + flag_plan_args=( + "--no-job-settings", + "--full-refresh", + "--ignore-audit-failures", + "--select", + *PARITY_NODES, + ), + flag_build_args=( + "--no-job-settings", + "--full-refresh", + "--no-tests", + "--ignore-audit-failures", + "--select", + *PARITY_NODES, + ), + expected_function_reasons=(("udf__band", "full refresh"),), + expected_full_refresh_counts=("2",), + expected_exit_code=0, + ), + ], + ids=lambda case: case.description, +) +def test_given_a_single_named_job_when_building_then_it_matches_its_settings_as_global_flags( + test_case: JobFlagParityTestCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + job_outcome: tuple[object, ...] = job_flag_parity_outcome( + project_dir=tmp_path / "job", + args=(test_case.job_args, test_case.job_args), + capsys=capsys, + ) + flag_outcome: tuple[object, ...] = job_flag_parity_outcome( + project_dir=tmp_path / "flags", + args=(test_case.flag_plan_args, test_case.flag_build_args), + capsys=capsys, + ) + + assert (job_outcome == flag_outcome, job_outcome[:3]) == ( + True, + ( + test_case.expected_function_reasons, + test_case.expected_full_refresh_counts, + test_case.expected_exit_code, + ), + ) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_job_warehouses.py b/tests/integration/src/sqlbuild/cli/commands/main/test_job_warehouses.py index f589046605..a550d2ea81 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_job_warehouses.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_job_warehouses.py @@ -14,8 +14,10 @@ from tests.integration.src.sqlbuild.cli.commands.main.helpers import ( install_writable_offline_snowflake, run_cli_exit_code, + unit_test_sql, warehouse_switches, write_job_warehouse_project, + write_project_files, ) _BOTH_JOBS: tuple[str, ...] = ( @@ -37,10 +39,16 @@ expected_switches=("ORDERS_WH", "REFUNDS_WH", "BUILD_WH"), ), JobWarehouseSwitchTestCase( - description="a job's task runs on the job's warehouse", + description="a single named job's warehouse is the whole build's, its task included", argv=("build", "--select", "job:tasks_wh_job"), - expected_connect_warehouses=frozenset({"BUILD_WH"}), - expected_switches=("TASKS_WH",), + expected_connect_warehouses=frozenset({"TASKS_WH"}), + expected_switches=(), + ), + JobWarehouseSwitchTestCase( + description="a warehouse every named job shares runs upstream nodes outside them too", + argv=("build", "--select", "stg", "job:orders_wh_job", "job:orders_copy_job"), + expected_connect_warehouses=frozenset({"ORDERS_WH"}), + expected_switches=(), ), JobWarehouseSwitchTestCase( description="--warehouse wins over every named job, even ones that would conflict", @@ -48,6 +56,20 @@ expected_connect_warehouses=frozenset({"CLI_WH"}), expected_switches=(), ), + JobWarehouseSwitchTestCase( + description="a SQL test covering nodes with and without a job warehouse stays put", + argv=("build", *_BOTH_JOBS), + expected_connect_warehouses=frozenset({"BUILD_WH"}), + expected_switches=("ORDERS_WH", "REFUNDS_WH", "BUILD_WH"), + extra_files={"tests/unit/test_refunds.sql": unit_test_sql("refunds", "summary")}, + ), + JobWarehouseSwitchTestCase( + description="a SQL test covering only one job's nodes runs on its warehouse", + argv=("build", *_BOTH_JOBS), + expected_connect_warehouses=frozenset({"BUILD_WH"}), + expected_switches=("REFUNDS_WH", "ORDERS_WH", "REFUNDS_WH", "BUILD_WH"), + extra_files={"tests/unit/test_refunds.sql": unit_test_sql("refunds")}, + ), JobWarehouseSwitchTestCase( description="--no-job-settings keeps every node on the build's warehouse", argv=("build", "--no-job-settings", *_BOTH_JOBS), @@ -70,6 +92,7 @@ def test_given_named_jobs_with_warehouses_when_building_then_each_node_runs_on_i capsys: pytest.CaptureFixture[str], ) -> None: write_job_warehouse_project(project_dir=tmp_path) + write_project_files(project_dir=tmp_path, files=test_case.extra_files) connect_calls, executed_sql = install_writable_offline_snowflake(monkeypatch) exit_code: int = run_cli_exit_code(test_case.argv, tmp_path) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_adhoc.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_adhoc.py index f13415759f..f761f6232b 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_adhoc.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_adhoc.py @@ -60,13 +60,11 @@ expected_reason=WAREHOUSE_REFUSAL, expected_statements=False, expected_customers_tables=0, - # DuckDB has no session warehouse, so the build refuses the job's warehouse: the - # published job reached the build instead of being refused by the worker. + # DuckDB has no session warehouse, so the CLI refuses the job's warehouse: the + # published job reached the CLI instead of being refused by the worker. job_settings='\n[settings]\nwarehouse = "ANALYST_XS"\n', expected_scheduled_status="failed", - expected_worker_log=( - "sets warehouse ANALYST_XS, but the 'duckdb' adapter has no warehouse to select" - ), + expected_worker_log="error[C260]: --warehouse ANALYST_XS was given", ), ], ids=lambda case: case.description, diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py index a2f2f71aec..4cf8ae8513 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py @@ -435,6 +435,52 @@ def test_given_a_scheduled_group_when_workers_execute_then_dependencies_and_fail ("udf:udf__order_band", "success", None, None, None), ], ), + WorkerExecutionTestCase( + "a job's full_refresh, run_tests and ignore_audit_failures act as the global flags did", + "SELECT 7 AS amount", + "SELECT count(*) FROM parity_events", + "success", + [(1,)], + files={ + "functions/sql/udf__band.sql": ( + "FUNCTION (description 'Band.', arguments (amount INT), returns VARCHAR);\n" + "CASE WHEN amount >= 20 THEN 'large' ELSE 'small' END\n" + ), + "models/parity_lookup.sql": ( + "MODEL (description 'Lookup.', materialized table, " + "columns (id (type INTEGER, audits [not_null])));\n" + "SELECT CAST(NULL AS INTEGER) AS id\n" + ), + "models/parity_events.sql": ( + "MODEL (description 'Events.', materialized incremental, " + "incremental_strategy append);\n" + 'SELECT 1 AS event_id, __udf("udf__band")(5) AS band\n' + ), + "tests/unit/test_parity_events.sql": ( + "TEST();\n\nWITH\n__ref__parity_lookup AS (SELECT 1 AS id),\n" + "__expected__parity_events AS (SELECT 99 AS event_id, 'x' AS band)\nSELECT 1\n" + ), + "audits/singular/events_have_lookups.sql": ( + 'AUDIT (name "events_have_lookups", severity error);\n' + 'SELECT e.event_id FROM __ref("parity_events") e ' + 'LEFT JOIN __ref("parity_lookup") l ON e.event_id = l.id WHERE l.id IS NULL\n' + ), + "orchestration/jobs/annual_orders_job.toml": ( + 'name = "annual_orders_job"\n' + 'description = "Build orders."\n' + 'select = "orders udf__band parity_lookup parity_events"\n' + 'schedule = { cron = "0 0 * * *", timezone = "UTC", starts_at = "2000-01-01T00:00:00Z" }\n' + "\n[settings]\nfull_refresh = true\nrun_tests = false\n" + "ignore_audit_failures = true\n" + ), + }, + expected_run_nodes=[ + ("model:orders", "success", None, None, None), + ("model:parity_events", "success", None, None, None), + ("model:parity_lookup", "success", None, None, None), + ("udf:udf__band", "success", None, None, None), + ], + ), ], ids=lambda case: case.description, ) diff --git a/tests/unit/src/sqlbuild/executor/build/classes/_test_types.py b/tests/unit/src/sqlbuild/executor/build/classes/_test_types.py index 51db8b9b24..5ca1560d89 100644 --- a/tests/unit/src/sqlbuild/executor/build/classes/_test_types.py +++ b/tests/unit/src/sqlbuild/executor/build/classes/_test_types.py @@ -2,6 +2,9 @@ from dataclasses import dataclass +from sqlbuild._native import CompiledResourceType +from sqlbuild.executor.run.models import ModelExecutionResult + @dataclass(frozen=True) class MicrobatchBoundedFutureTestCase: @@ -85,3 +88,5 @@ class NodeWarehouseFailureTestCase: description: str node_warehouse: str | None expected_message: str + resource_type: CompiledResourceType = CompiledResourceType.MODEL + expected_result_type: type = ModelExecutionResult diff --git a/tests/unit/src/sqlbuild/executor/build/classes/helpers.py b/tests/unit/src/sqlbuild/executor/build/classes/helpers.py index d6f4c63d5a..f9c8b5bd57 100644 --- a/tests/unit/src/sqlbuild/executor/build/classes/helpers.py +++ b/tests/unit/src/sqlbuild/executor/build/classes/helpers.py @@ -6,6 +6,7 @@ import time from collections import deque from queue import Queue +from types import SimpleNamespace from typing import Any, cast from sqlbuild._native import CompiledObjectKey, CompiledResourceType @@ -15,6 +16,7 @@ from sqlbuild.executor.run.models import ModelExecutionResult from sqlbuild.executor.scheduling.classes.session_warehouses import SessionWarehouses from sqlbuild.executor.scheduling.types import ExecutionStatus +from sqlbuild.executor.testing.types import SqlTestOutcome from tests.unit.src.sqlbuild.executor.build.classes._test_types import ( JobLimitedBuildTestCase, SubworkerReservationTestCase, @@ -159,6 +161,29 @@ def reserve_subworkers(test_case: SubworkerReservationTestCase) -> tuple[int, tu return scheduler._microbatch_coordinator_demand, reservations +def node_key(*, resource_type: CompiledResourceType, name: str) -> CompiledObjectKey: + """The key of node `name` of `resource_type`.""" + + return CompiledObjectKey(resource_type=resource_type, name=name) + + +def failed_node_result(result: Any) -> bool: + """Whether a node result of any kind reports failure.""" + + return ( + getattr(result, "status", None) == ExecutionStatus.FAILED + or getattr(result, "outcome", None) == SqlTestOutcome.ERROR + ) + + +def source_plan(name: str) -> SimpleNamespace: + """A plan with only source `name`'s entry, as a failed load result reads it.""" + + return SimpleNamespace( + source_map={name: SimpleNamespace(name=name, loader=None, table=None, meta={})} + ) + + def warehouse_scheduler(adapter: RecordingWarehouseAdapter) -> Any: """A scheduler with only the state warehouse switching and sub-worker slots read.""" diff --git a/tests/unit/src/sqlbuild/executor/build/classes/test_build_scheduler_warehouses.py b/tests/unit/src/sqlbuild/executor/build/classes/test_build_scheduler_warehouses.py index 5f91b249f2..a9182e80ad 100644 --- a/tests/unit/src/sqlbuild/executor/build/classes/test_build_scheduler_warehouses.py +++ b/tests/unit/src/sqlbuild/executor/build/classes/test_build_scheduler_warehouses.py @@ -6,15 +6,20 @@ import pytest +from sqlbuild._native import CompiledObjectKey, CompiledResourceType from sqlbuild.executor.build.classes.build_scheduler import BuildScheduler -from sqlbuild.executor.run.models import BatchWindow, ModelExecutionResult -from sqlbuild.executor.scheduling.types import ExecutionStatus +from sqlbuild.executor.build.models import FunctionExecutionResult, SeedExecutionResult +from sqlbuild.executor.load.models import LoadExecutionResult +from sqlbuild.executor.run.models import BatchWindow +from sqlbuild.executor.testing.models import SqlTestExecutionResult from tests.unit.src.sqlbuild.executor.build.classes._test_types import ( NodeWarehouseFailureTestCase, SubworkerWarehouseTestCase, ) from tests.unit.src.sqlbuild.executor.build.classes.helpers import ( - model_key, + failed_node_result, + node_key, + source_plan, warehouse_scheduler, ) from tests.unit.src.sqlbuild.executor.scheduling.classes.helpers import ( @@ -87,6 +92,51 @@ def execute(_batch: BatchWindow, connection: object) -> Any: "cannot run on warehouse MISSING_WH: Object does not exist: warehouse MISSING_WH" ), ), + NodeWarehouseFailureTestCase( + description="a seed fails with its own kind's result", + node_warehouse="MISSING_WH", + expected_message=( + "cannot run on warehouse MISSING_WH: Object does not exist: warehouse MISSING_WH" + ), + resource_type=CompiledResourceType.SEED, + expected_result_type=SeedExecutionResult, + ), + NodeWarehouseFailureTestCase( + description="a source fails with its own kind's result", + node_warehouse="MISSING_WH", + expected_message=( + "cannot run on warehouse MISSING_WH: Object does not exist: warehouse MISSING_WH" + ), + resource_type=CompiledResourceType.SOURCE, + expected_result_type=LoadExecutionResult, + ), + NodeWarehouseFailureTestCase( + description="a udf fails with its own kind's result", + node_warehouse="MISSING_WH", + expected_message=( + "cannot run on warehouse MISSING_WH: Object does not exist: warehouse MISSING_WH" + ), + resource_type=CompiledResourceType.UDF, + expected_result_type=FunctionExecutionResult, + ), + NodeWarehouseFailureTestCase( + description="a table_fn fails with its own kind's result", + node_warehouse="MISSING_WH", + expected_message=( + "cannot run on warehouse MISSING_WH: Object does not exist: warehouse MISSING_WH" + ), + resource_type=CompiledResourceType.TABLE_FN, + expected_result_type=FunctionExecutionResult, + ), + NodeWarehouseFailureTestCase( + description="a sql_test fails with its own kind's result", + node_warehouse="MISSING_WH", + expected_message=( + "cannot run on warehouse MISSING_WH: Object does not exist: warehouse MISSING_WH" + ), + resource_type=CompiledResourceType.SQL_TEST, + expected_result_type=SqlTestExecutionResult, + ), ], ids=lambda case: case.description, ) @@ -95,14 +145,14 @@ def test_given_an_unusable_job_warehouse_when_a_node_starts_then_it_fails_naming ) -> None: adapter: MissingWarehouseAdapter = MissingWarehouseAdapter(session_warehouse=None) scheduler: Any = warehouse_scheduler(adapter) - scheduler._node_warehouses = {model_key("orders"): test_case.node_warehouse} + key: CompiledObjectKey = node_key(resource_type=test_case.resource_type, name="orders") + scheduler._node_warehouses = {key: test_case.node_warehouse} + scheduler._plan = source_plan("orders") - result: ModelExecutionResult | None = BuildScheduler._enter_node_warehouse( - scheduler, key=model_key("orders"), connection=object() - ) + result: Any = BuildScheduler._enter_node_warehouse(scheduler, key=key, connection=object()) - assert result is not None - assert (result.status, result.error_message) == ( - ExecutionStatus.FAILED, + assert (type(result), failed_node_result(result), result.error_message) == ( + test_case.expected_result_type, + True, test_case.expected_message, ) diff --git a/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/_test_types.py b/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/_test_types.py index bfd9e54a11..a1bc5d099c 100644 --- a/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/_test_types.py @@ -245,3 +245,24 @@ class PythonIdentityFingerprintWriteTestCase: expected_sql_count: int expected_fragments: tuple[str, ...] unexpected_fragments: tuple[str, ...] = () + + +@dataclass(frozen=True) +class IngressWarehouseFailureTestCase: + """An ingress node whose job warehouse the session cannot use, and what the run records.""" + + description: str + missing_warehouse_node: str + expected_python: tuple[tuple[str, PythonNodeStatus, str | None], ...] + expected_loads: tuple[tuple[str, str, str | None], ...] + expected_call_order: tuple[str, ...] + + +@dataclass(frozen=True) +class CheckWarehouseFailureTestCase: + """A check on a job warehouse the session cannot use, after an upstream of some status.""" + + description: str + upstream_status: PythonNodeStatus + expected_error_message: str | None + expected_switch_attempts: int diff --git a/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/helpers.py b/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/helpers.py index 6a7767909d..cfe319463c 100644 --- a/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/helpers.py @@ -46,6 +46,7 @@ PythonNodeRuntime, ) from sqlbuild.executor.python_nodes.types import ExecutablePythonNode +from sqlbuild.executor.scheduling.classes.session_warehouses import SessionWarehouses from sqlbuild.executor.scheduling.main._unlock_downstream import unlock_downstream_python_nodes from sqlbuild.executor.scheduling.models import LifecycleExecutionNode from sqlbuild.observability import LifecycleEvent @@ -671,3 +672,24 @@ def _raise_missing_result( (False, False): _read_default_result, (False, True): _raise_missing_result, } + + +MISSING_WAREHOUSE_FAILURE: str = ( + "cannot run on warehouse MISSING_WH: Object does not exist: warehouse MISSING_WH" +) + + +def missing_warehouse_runtime(*, adapter: Any, node: str) -> PythonNodeRuntime: + """A runtime putting `node` on MISSING_WH through `adapter`'s session warehouses.""" + + return PythonNodeRuntime( + adapter=PythonNodeContextTestAdapter(), + connection_config={}, + connection=object(), + run_id="warehouse-run", + target="dev", + vars={}, + is_reload=False, + node_warehouses={node: "MISSING_WH"}, + session_warehouses=SessionWarehouses(adapter=cast(BaseAdapter, adapter), home="BUILD_WH"), + ) diff --git a/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_node_warehouse_failures.py b/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_node_warehouse_failures.py new file mode 100644 index 0000000000..68a05f2839 --- /dev/null +++ b/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_node_warehouse_failures.py @@ -0,0 +1,134 @@ +"""A Python node whose job warehouse the session cannot use fails alone, with its own result.""" + +from __future__ import annotations + +from typing import Any + +import pytest + +from sqlbuild._native import DiscoveredCheckFunction, PythonNodeGraph +from sqlbuild.compiler.python_nodes.types import PythonNodeKind, PythonNodeStatus +from sqlbuild.executor.python_nodes._helpers.ingress_execution import ( + execute_ingress_python_loader_nodes, +) +from sqlbuild.executor.python_nodes._helpers.python_checks import execute_python_check_nodes +from sqlbuild.executor.python_nodes.models import ( + IngressCallbacks, + PythonCheckCallbacks, + PythonCheckExecutionResult, + PythonIngressLoaderExecutorResult, + PythonNodeExecutionResult, + PythonNodeRunState, +) +from tests.unit.src.sqlbuild.executor.python_nodes._helpers._test_types import ( + CheckWarehouseFailureTestCase, + IngressWarehouseFailureTestCase, +) +from tests.unit.src.sqlbuild.executor.python_nodes._helpers.helpers import ( + MISSING_WAREHOUSE_FAILURE, + build_ingress_task_loader_graph, + build_python_check_graph, + ingress_calls, + ingress_loader_function, + ingress_source_map, + missing_warehouse_runtime, + python_check_function_for_case, + reset_ingress_calls, +) +from tests.unit.src.sqlbuild.executor.scheduling.classes.helpers import MissingWarehouseAdapter + + +@pytest.mark.parametrize( + "test_case", + [ + IngressWarehouseFailureTestCase( + description="a loader that cannot switch fails as a load, after its task ran", + missing_warehouse_node="load_ingress_orders", + expected_python=(("prepare_ingress_orders", PythonNodeStatus.SUCCESS, None),), + expected_loads=(("raw_orders", "failed", MISSING_WAREHOUSE_FAILURE),), + expected_call_order=("prepare_ingress_orders",), + ), + IngressWarehouseFailureTestCase( + description="a task that cannot switch fails, and its loader never runs", + missing_warehouse_node="prepare_ingress_orders", + expected_python=( + ("prepare_ingress_orders", PythonNodeStatus.FAILED, MISSING_WAREHOUSE_FAILURE), + ), + expected_loads=(("raw_orders", "skipped", None),), + expected_call_order=(), + ), + ], + ids=lambda case: case.description, +) +def test_given_an_unusable_job_warehouse_when_running_ingress_then_only_that_node_fails( + test_case: IngressWarehouseFailureTestCase, +) -> None: + reset_ingress_calls() + graph: PythonNodeGraph = build_ingress_task_loader_graph() + adapter: MissingWarehouseAdapter = MissingWarehouseAdapter(session_warehouse=None) + + result: PythonIngressLoaderExecutorResult = execute_ingress_python_loader_nodes( + python_graph=graph, + selected_python_names=frozenset({"prepare_ingress_orders", "load_ingress_orders"}), + loader_functions=(ingress_loader_function(),), + source_map=ingress_source_map(), + runtime=missing_warehouse_runtime(adapter=adapter, node=test_case.missing_warehouse_node), + callbacks=IngressCallbacks(), + ) + + assert ( + tuple((node.node_name, node.status, node.error_message) for node in result.python_results), + tuple( + (load.source_name, load.status.value, load.error_message) + for load in result.load_results + ), + ingress_calls(), + ) == (test_case.expected_python, test_case.expected_loads, test_case.expected_call_order) + + +@pytest.mark.parametrize( + "test_case", + [ + CheckWarehouseFailureTestCase( + description="a ready check that cannot switch fails with the reason", + upstream_status=PythonNodeStatus.SUCCESS, + expected_error_message=MISSING_WAREHOUSE_FAILURE, + expected_switch_attempts=1, + ), + CheckWarehouseFailureTestCase( + description="a blocked check keeps its block and never switches", + upstream_status=PythonNodeStatus.FAILED, + expected_error_message="Upstream Python node failed: upstream_task", + expected_switch_attempts=0, + ), + ], + ids=lambda case: case.description, +) +def test_given_an_unusable_job_warehouse_when_running_a_check_then_it_fails_alone( + test_case: CheckWarehouseFailureTestCase, +) -> None: + check_function: DiscoveredCheckFunction = python_check_function_for_case("passes") + adapter: MissingWarehouseAdapter = MissingWarehouseAdapter(session_warehouse=None) + + results: tuple[PythonCheckExecutionResult, ...] = execute_python_check_nodes( + check_functions=(check_function,), + python_graph=build_python_check_graph(check_function=check_function), + upstream_python_results=( + PythonNodeExecutionResult( + node_name="upstream_task", + kind=PythonNodeKind.TASK, + status=test_case.upstream_status, + ), + ), + upstream_load_results=(), + run_state=PythonNodeRunState(), + runtime=missing_warehouse_runtime(adapter=adapter, node=check_function.name), + callbacks=PythonCheckCallbacks(), + ) + + checked: Any = results[0] + assert ( + checked.failed, + checked.error_message, + len(adapter.statements), + ) == (True, test_case.expected_error_message, test_case.expected_switch_attempts) diff --git a/tests/unit/src/sqlbuild/executor/scheduling/classes/helpers.py b/tests/unit/src/sqlbuild/executor/scheduling/classes/helpers.py index 0814a48d43..176fb5ed73 100644 --- a/tests/unit/src/sqlbuild/executor/scheduling/classes/helpers.py +++ b/tests/unit/src/sqlbuild/executor/scheduling/classes/helpers.py @@ -74,8 +74,8 @@ def current_session_warehouse(self, *, connection: Any) -> str | None: class MissingWarehouseAdapter(RecordingWarehouseAdapter): - """An adapter whose session cannot use any warehouse it is asked for.""" + """An adapter whose session cannot use any warehouse it is asked for; records each attempt.""" def use_session_warehouse(self, *, connection: Any, warehouse: str) -> None: - del connection + self.statements.append((connection, warehouse)) raise RuntimeError(f"Object does not exist: warehouse {warehouse}") diff --git a/website/src/content/docs/docs/concepts/adapters/snowflake.mdx b/website/src/content/docs/docs/concepts/adapters/snowflake.mdx index bca7e8a64d..5f0b448ef5 100644 --- a/website/src/content/docs/docs/concepts/adapters/snowflake.mdx +++ b/website/src/content/docs/docs/concepts/adapters/snowflake.mdx @@ -107,12 +107,17 @@ A job can run its nodes on a warehouse of its own: warehouse = "ORDERS_WH" ``` -When a build names the job, as `sqb build --select job:nightly_orders_job` or a server run of it, -each of the job's nodes runs on `ORDERS_WH`. Its models' audits, its SQL tests, microbatch batches, -source loads and Python nodes go with them, and loaders and Python code that open their own -connections from the context's `connection_config` get `ORDERS_WH` there too. A SQL test runs on -its models' job warehouse when they agree on one. Everything else in the build, including nodes -selected without naming a job and the build's own bookkeeping, stays on the `build` warehouse. +A server run of the job, and a build whose named jobs all set the same warehouse, such as +`sqb build --select job:nightly_orders_job`, runs entirely on `ORDERS_WH`, as if passed +`--warehouse ORDERS_WH`: setup, planning and bookkeeping included. Nodes the build selects outside +the named jobs, such as upstream models in `--select +job:nightly_orders_job`, run there too. + +When the named jobs set different warehouses, or only some set one, each job's nodes run on its +own warehouse. Its models' audits, its SQL tests, microbatch batches, source loads and Python nodes +go with them, and loaders and Python code that open their own connections from the context's +`connection_config` get the job's warehouse there too. A SQL test runs on a job warehouse only +when every node it covers runs there. Everything else in the build, including nodes selected +without naming a job and the build's own bookkeeping, stays on the `build` warehouse. The connection running a node switches with `USE WAREHOUSE` only when the node needs a different warehouse than the connection is on, and switches back before it runs a node outside the job. diff --git a/website/src/content/docs/docs/concepts/orchestration.mdx b/website/src/content/docs/docs/concepts/orchestration.mdx index 70d976b395..53351bd387 100644 --- a/website/src/content/docs/docs/concepts/orchestration.mdx +++ b/website/src/content/docs/docs/concepts/orchestration.mdx @@ -95,8 +95,9 @@ Naming a group names each of its jobs. The first value set wins: 3. The target's setting 4. The project's setting -A command-line flag replaces the job's setting rather than merging with it: `--vars` replaces the -job's whole `vars`, and `--warehouse` puts every node on its warehouse. An option that rules another out counts as setting both, so `--reload` wins +A command-line flag replaces every named job's setting rather than merging with it, so it also +settles named jobs that disagree: `--vars` replaces the job's whole `vars`, `--full-refresh` +rebuilds every selected model, and `--warehouse` puts every node on its warehouse. An option that rules another out counts as setting both, so `--reload` wins over a job's `load_sources = false`. Any `--concurrency` sets the build's limit and turns off the per-job limits. The job's settings are added before a `--` on the command line, so selectors after it still name jobs. `--no-job-settings` leaves out every job's settings. @@ -106,9 +107,9 @@ nodes differently: | Setting | Applies to | | --- | --- | -| `full_refresh` | The job's incremental models, rebuilt from scratch | +| `full_refresh` | The job's incremental models, rebuilt from scratch, and its functions, recreated | | `run_tests` | The SQL unit tests of the job's models | -| `ignore_audit_failures` | The audits of the job's nodes | +| `ignore_audit_failures` | The audits of the job's nodes, and singular audits that read them; a singular audit is ignored when every named job that sets it for the nodes the audit reads sets `true` | | `warehouse` | The Snowflake warehouse the job's nodes run on, Python nodes included; see [A job's own warehouse](/docs/concepts/adapters/snowflake#a-jobs-own-warehouse) | | `concurrency` | How many of the job's nodes build at once, and a node in two named jobs fits within both. The build's own limit becomes the larger of the configured limit and the named jobs' limits added together | | `[retry]` | The job's nodes | @@ -117,9 +118,16 @@ A job's settings reach only the nodes its own selection holds. `--select job:nig gives them to `orders` alone, and `+job:nightly_orders_job` builds the job's upstream nodes without them. -Every other setting, such as `vars` or `changed`, applies to the whole build, so every named job -must set it the same way. A group whose jobs set any of these differently on a target that declares -them together fails to compile, with the error a build of the group would give. +Every other setting, such as `vars`, `changed` or `fail_fast`, applies to the whole build, so every +named job must set it the same way; a job that leaves it unset disagrees with one that sets it. +Pass the flag to settle a disagreement. When every named job sets the same `warehouse`, it is the +whole build's warehouse, as if passed as `--warehouse`; see +[A job's own warehouse](/docs/concepts/adapters/snowflake#a-jobs-own-warehouse). + +Each job in a group still runs as its own build, so a group's jobs must agree only on `vars` and +`changed`, which change the compiled SQL. A group whose jobs set either differently on a target +that declares them together fails to compile, with the error a build of the group would give. + Two named jobs that give one node different values of the same setting fail the build with `S027`, naming the node, both jobs and the setting. A setting only one of them sets is no conflict. @@ -309,7 +317,7 @@ Every problem with an orchestration file is a `D017` error at its file, line and | `job nightly_orders_job resources: cpus must be a positive number` | Set `cpus` above zero | | `job 'nightly_orders_job' uses pool 'warehouse_hevy', which [pools] in sqlbuild_project.toml does not declare` | Fix the name, or declare the pool in [`[pools]`](/docs/concepts/project-configuration#pools) | | `job 'nightly_orders_job' takes 2 slots of pool 'supplier_api', more than its limit of 1 on target 'production'` | Take fewer slots, or raise the limit in `[targets.production.pools]` | -| `jobs orders_job and crm_job set vars differently ({"amount":"5"} and {"amount":"9"}); a build applies vars to everything it runs; group daily_job_group builds its jobs together` | Give the group's jobs the same build-wide settings, or move one job to another group | +| `jobs orders_job and crm_job set vars differently ({"amount":"5"} and {"amount":"9"}); a build applies vars to everything it runs; group daily_job_group builds its jobs together` | Give the group's jobs the same `vars` and `changed`, or move one job to another group | | `sensor 'orders_sensor' is declared here, but importing the module does not define it` | Assign the sensor to a module-level name or list, not a dictionary or function body | | `group dependencies contain a cycle in job group daily_orders_job_group: orders_marts_job waits for staging_orders_job (model:orders reads model:stg_orders); staging_orders_job waits for orders_marts_job (model:stg_orders reads seed:regions)` | Move each node to one job, and give seeds and nodes that several members read their own job that runs first | @@ -321,7 +329,9 @@ Details: them, as on the server. - Job settings that cannot apply together fail the build or plan with `S027`, such as `model:orders is selected through jobs orders_job and finance_job, which set run_tests differently (false and - true)`. Make the jobs agree, set it in one of them, pass the flag, or use `--no-job-settings`. + true)`. Make the jobs agree, set it in only one of them, pass the flag, or use + `--no-job-settings`. For a build-wide setting, where unset differs from set, make the jobs agree, + pass the flag, or use `--no-job-settings`. - Selectors are checked for syntax at compile. A selector that names a node the project does not have fails when a location publishes, such as `job nightly_orders_job: unknown selector name 'order_total'`. From 03670fa3a92f8289b08d2dd402d5c5919e80f188 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 19 Nov 2026 06:24:38 +0000 Subject: [PATCH 11/22] refactor(jobs): one job setting table and one resolver for named jobs' settings (CHI-1533) JOB_SETTINGS lists every sqb build option once with its command-line options, scope (node, build-wide, per-job limit or owned by the run request), whether Python nodes take it, whether one shared value becomes the whole run's, and when a group's jobs must agree on it. The scope decides how named jobs' values merge and which conflict help applies. The node and Python-node setting lists, the per-job limit, the group agreement list, the run-request-owned names and the warehouse exceptions in job run arguments and CLI builds now all read the table. resolve_job_settings is the one place named jobs' settings meet the command line: it removes the settings the command line gives, checks build-wide agreement, takes whole-run values every named job shares, and resolves per-node settings when the planner asks. The CLI argv path and the planner both call it. Table-driven tests check that the table matches the build parser's options exactly, that each scope merges, conflicts and yields to the command line as specified for every setting, that every build-wide setting renders as its own option, and that every node setting has a planner consumer. --- .../{main => _helpers}/build_job_settings.rs | 2 +- .../src/job_settings/_helpers/mod.rs | 4 + .../{main => _helpers}/named_jobs.rs | 5 +- .../{main => _helpers}/node_job_settings.rs | 13 +- .../src/job_settings/_helpers/node_values.rs | 16 +- .../without_given_settings.rs | 2 +- .../src/job_settings/constants.rs | 438 +++++++++++++++++- .../main/group_setting_mismatch.rs | 11 +- .../src/job_settings/main/job_setting_spec.rs | 10 + .../src/job_settings/main/mod.rs | 6 +- .../job_settings/main/resolve_job_settings.rs | 67 +++ .../src/job_settings/main/setting_scope.rs | 12 +- .../src/job_settings/models.rs | 49 +- .../src/job_settings/tests.rs | 2 + .../src/job_settings/tests/helpers.rs | 59 ++- .../tests/test_build_job_settings.rs | 4 +- .../tests/test_job_setting_table.rs | 112 +++++ .../src/job_settings/tests/test_types.rs | 21 + .../src/job_settings/types.rs | 7 + .../src/argv/_helpers/job_settings.rs | 22 +- .../src/argv/main/job_run_arguments.rs | 6 +- crates/sqlbuild-cli/src/argv/tests.rs | 2 + crates/sqlbuild-cli/src/argv/tests/helpers.rs | 102 ++++ .../src/argv/tests/test_job_setting_table.rs | 47 ++ .../sqlbuild-cli/src/argv/tests/test_types.rs | 15 + .../_helpers/jobs/job_node_settings.rs | 39 +- .../src/compile_driver/mod.rs | 2 + .../src/compile_driver/tests.rs | 4 + .../tests/test_node_setting_consumers.rs | 34 ++ .../src/compile_driver/tests/test_types.rs | 7 + .../_helpers/named_job_build_settings.rs | 60 ++- 31 files changed, 1050 insertions(+), 130 deletions(-) rename crates/sqlbuild-analysis/src/job_settings/{main => _helpers}/build_job_settings.rs (98%) rename crates/sqlbuild-analysis/src/job_settings/{main => _helpers}/named_jobs.rs (84%) rename crates/sqlbuild-analysis/src/job_settings/{main => _helpers}/node_job_settings.rs (78%) rename crates/sqlbuild-analysis/src/job_settings/{main => _helpers}/without_given_settings.rs (94%) create mode 100644 crates/sqlbuild-analysis/src/job_settings/main/job_setting_spec.rs create mode 100644 crates/sqlbuild-analysis/src/job_settings/main/resolve_job_settings.rs create mode 100644 crates/sqlbuild-analysis/src/job_settings/tests/test_job_setting_table.rs create mode 100644 crates/sqlbuild-cli/src/argv/tests/test_job_setting_table.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_driver/tests.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_driver/tests/test_node_setting_consumers.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_driver/tests/test_types.rs diff --git a/crates/sqlbuild-analysis/src/job_settings/main/build_job_settings.rs b/crates/sqlbuild-analysis/src/job_settings/_helpers/build_job_settings.rs similarity index 98% rename from crates/sqlbuild-analysis/src/job_settings/main/build_job_settings.rs rename to crates/sqlbuild-analysis/src/job_settings/_helpers/build_job_settings.rs index 93a81766bc..02f24f036a 100644 --- a/crates/sqlbuild-analysis/src/job_settings/main/build_job_settings.rs +++ b/crates/sqlbuild-analysis/src/job_settings/_helpers/build_job_settings.rs @@ -24,7 +24,7 @@ fn shown(value: Option<&Json>) -> String { } /// The named jobs' build-wide settings, which must agree; unset differs from set. -pub fn build_job_settings( +pub(crate) fn build_job_settings( named: &[String], declared: &BTreeMap, ) -> Result, JobSettingConflict> { diff --git a/crates/sqlbuild-analysis/src/job_settings/_helpers/mod.rs b/crates/sqlbuild-analysis/src/job_settings/_helpers/mod.rs index 4804846644..69e5ae86ec 100644 --- a/crates/sqlbuild-analysis/src/job_settings/_helpers/mod.rs +++ b/crates/sqlbuild-analysis/src/job_settings/_helpers/mod.rs @@ -1,3 +1,7 @@ +pub(crate) mod build_job_settings; +pub(crate) mod named_jobs; +pub(crate) mod node_job_settings; pub(crate) mod node_values; pub(crate) mod selection; pub(crate) mod tokens; +pub(crate) mod without_given_settings; diff --git a/crates/sqlbuild-analysis/src/job_settings/main/named_jobs.rs b/crates/sqlbuild-analysis/src/job_settings/_helpers/named_jobs.rs similarity index 84% rename from crates/sqlbuild-analysis/src/job_settings/main/named_jobs.rs rename to crates/sqlbuild-analysis/src/job_settings/_helpers/named_jobs.rs index d27d2ce421..634c1e316c 100644 --- a/crates/sqlbuild-analysis/src/job_settings/main/named_jobs.rs +++ b/crates/sqlbuild-analysis/src/job_settings/_helpers/named_jobs.rs @@ -7,7 +7,10 @@ use crate::job_settings::models::DeclaredRun; /// The jobs `selectors` name, each once in the order first named. #[must_use] -pub fn named_jobs(selectors: &[String], declared: &BTreeMap) -> Vec { +pub(crate) fn named_jobs( + selectors: &[String], + declared: &BTreeMap, +) -> Vec { let mut jobs: Vec = Vec::new(); for token in selectors .iter() diff --git a/crates/sqlbuild-analysis/src/job_settings/main/node_job_settings.rs b/crates/sqlbuild-analysis/src/job_settings/_helpers/node_job_settings.rs similarity index 78% rename from crates/sqlbuild-analysis/src/job_settings/main/node_job_settings.rs rename to crates/sqlbuild-analysis/src/job_settings/_helpers/node_job_settings.rs index d876b18167..17900baec4 100644 --- a/crates/sqlbuild-analysis/src/job_settings/main/node_job_settings.rs +++ b/crates/sqlbuild-analysis/src/job_settings/_helpers/node_job_settings.rs @@ -2,18 +2,16 @@ use std::collections::BTreeMap; -use serde_json::Value as Json; - use crate::graph::models::ProjectGraph; use crate::job_settings::_helpers::node_values::node_values; use crate::job_settings::_helpers::selection::node_jobs; -use crate::job_settings::constants::JOB_LIMIT_SETTING; use crate::job_settings::errors::NodeJobSettingsError; -use crate::job_settings::models::{DeclaredRun, NodeJobSettings}; +use crate::job_settings::main::setting_scope::setting_scope; +use crate::job_settings::models::{DeclaredRun, NodeJobSettings, SettingScope}; use crate::job_settings::types::TokenNodes; /// Each node's named jobs, the settings they give it, and their limits. -pub fn node_job_settings( +pub(crate) fn node_job_settings( graph: &ProjectGraph, selection: (&[String], &[String]), declared: &BTreeMap, @@ -35,7 +33,10 @@ pub fn node_job_settings( } for job in jobs { if let Some(DeclaredRun::Job { settings, .. }) = declared.get(job) - && let Some(limit) = settings.get(JOB_LIMIT_SETTING).and_then(Json::as_i64) + && let Some(limit) = settings + .iter() + .find(|(name, _)| setting_scope(name) == SettingScope::JobLimit) + .and_then(|(_, value)| value.as_i64()) { result.job_limits.insert(job.clone(), limit); } diff --git a/crates/sqlbuild-analysis/src/job_settings/_helpers/node_values.rs b/crates/sqlbuild-analysis/src/job_settings/_helpers/node_values.rs index 4b529b34d7..4c0d8543b6 100644 --- a/crates/sqlbuild-analysis/src/job_settings/_helpers/node_values.rs +++ b/crates/sqlbuild-analysis/src/job_settings/_helpers/node_values.rs @@ -7,10 +7,10 @@ use serde_json::Value as Json; use crate::assembly::project::types::ObjectKey; use crate::definition_changes::constants::PYTHON_NODE_KINDS; use crate::job_settings::constants::{ - NODE_CONFLICT_HELP, NODE_SETTINGS, PYTHON_NODE_SETTINGS, RETRY_CONFLICT_HELP, RETRY_SETTING, + JOB_SETTINGS, NODE_CONFLICT_HELP, RETRY_CONFLICT_HELP, RETRY_SETTING, }; use crate::job_settings::errors::JobSettingConflict; -use crate::job_settings::models::{DeclaredRun, JobRetry}; +use crate::job_settings::models::{DeclaredRun, JobRetry, SettingScope}; /// One node's settings, and its retry, each from whichever of its jobs sets it. pub(crate) type NodeValues = (BTreeMap, Option); @@ -49,11 +49,11 @@ pub(crate) fn node_values( declared: &BTreeMap, ) -> Result { let python: bool = PYTHON_NODE_KINDS.contains(&key.0.as_str()); - let node_settings: &[&str] = if python { - &PYTHON_NODE_SETTINGS - } else { - &NODE_SETTINGS - }; + let node_settings: Vec<&str> = JOB_SETTINGS + .iter() + .filter(|spec| spec.scope == SettingScope::Node && (spec.python_nodes || !python)) + .map(|spec| spec.name) + .collect(); let mut values: BTreeMap = BTreeMap::new(); let mut retry: Option<(&String, JobRetry)> = None; for job in jobs { @@ -64,7 +64,7 @@ pub(crate) fn node_values( else { continue; }; - for &setting in node_settings { + for &setting in &node_settings { let Some(value) = settings.get(setting) else { continue; }; diff --git a/crates/sqlbuild-analysis/src/job_settings/main/without_given_settings.rs b/crates/sqlbuild-analysis/src/job_settings/_helpers/without_given_settings.rs similarity index 94% rename from crates/sqlbuild-analysis/src/job_settings/main/without_given_settings.rs rename to crates/sqlbuild-analysis/src/job_settings/_helpers/without_given_settings.rs index 031887e3a6..7889a73ffd 100644 --- a/crates/sqlbuild-analysis/src/job_settings/main/without_given_settings.rs +++ b/crates/sqlbuild-analysis/src/job_settings/_helpers/without_given_settings.rs @@ -6,7 +6,7 @@ use crate::job_settings::models::DeclaredRun; /// `declared` with each setting named in `given` removed from every job, so it never conflicts. #[must_use] -pub fn without_given_settings( +pub(crate) fn without_given_settings( mut declared: BTreeMap, given: &BTreeSet, ) -> BTreeMap { diff --git a/crates/sqlbuild-analysis/src/job_settings/constants.rs b/crates/sqlbuild-analysis/src/job_settings/constants.rs index 522e487092..03eb5af982 100644 --- a/crates/sqlbuild-analysis/src/job_settings/constants.rs +++ b/crates/sqlbuild-analysis/src/job_settings/constants.rs @@ -1,18 +1,11 @@ -//! How far each job setting reaches when a selection names the job. +//! Every build option as a job setting, and the wording of job setting conflicts. + +use crate::job_settings::models::GroupAgreement::{EachBuild, OneBuild}; +use crate::job_settings::models::JobSettingSpec; +use crate::job_settings::models::SettingScope::{Build, JobLimit, Node, RunRequest}; -/// Settings that apply only to the named job's own nodes; two jobs may not disagree on one node. -pub const NODE_SETTINGS: [&str; 4] = [ - "run_tests", - "ignore_audit_failures", - "full_refresh", - WAREHOUSE_SETTING, -]; /// The warehouse the job's nodes run on. pub const WAREHOUSE_SETTING: &str = "warehouse"; -/// The node settings a Python node takes; the others, and `[retry]`, act on SQL nodes only. -pub const PYTHON_NODE_SETTINGS: [&str; 1] = [WAREHOUSE_SETTING]; -/// The setting that becomes a limit on how many of the job's nodes run at once inside the build. -pub const JOB_LIMIT_SETTING: &str = "concurrency"; /// The help of two named jobs giving one node different values of a node-level setting. pub const NODE_CONFLICT_HELP: &str = "make the jobs agree, set it in only one of them, override it \ with the command-line flag, or build with --no-job-settings"; @@ -22,9 +15,426 @@ pub const BUILD_CONFLICT_HELP: &str = /// The help of two named jobs giving one node different `[retry]` tables, which no flag overrides. pub const RETRY_CONFLICT_HELP: &str = "make the jobs agree, set [retry] in only one of them, or build with --no-job-settings"; -/// The build-wide settings a group's jobs must agree on while each member runs as its own build. -pub const GROUP_AGREED_SETTINGS: [&str; 2] = ["vars", "changed"]; /// The error code of job settings that cannot apply together. pub const JOB_SETTING_CONFLICT_CODE: &str = "S027"; /// How a conflict names a job's `[retry]` table. pub const RETRY_SETTING: &str = "[retry]"; +/// Every option of `sqb build`, in the parser's order, as a job setting. +pub const JOB_SETTINGS: [JobSettingSpec; 52] = [ + JobSettingSpec { + name: "help", + options: &["-h", "--help"], + scope: RunRequest, + python_nodes: false, + whole_run: false, + group: OneBuild, + }, + JobSettingSpec { + name: "no_sql_validation", + options: &["--no-sql-analysis", "--no-sql-validation"], + scope: Build, + python_nodes: false, + whole_run: false, + group: OneBuild, + }, + JobSettingSpec { + name: "no_cache", + options: &["--no-cache"], + scope: Build, + python_nodes: false, + whole_run: false, + group: OneBuild, + }, + JobSettingSpec { + name: "defer_to", + options: &["--defer-to"], + scope: Build, + python_nodes: false, + whole_run: false, + group: OneBuild, + }, + JobSettingSpec { + name: "defer_clone_from", + options: &["--defer-clone-from"], + scope: Build, + python_nodes: false, + whole_run: false, + group: OneBuild, + }, + JobSettingSpec { + name: "defer_sources_to", + options: &["--defer-sources-to"], + scope: Build, + python_nodes: false, + whole_run: false, + group: OneBuild, + }, + JobSettingSpec { + name: "plan_from", + options: &["--plan-from"], + scope: Build, + python_nodes: false, + whole_run: false, + group: OneBuild, + }, + JobSettingSpec { + name: "target", + options: &["--target"], + scope: RunRequest, + python_nodes: false, + whole_run: false, + group: OneBuild, + }, + JobSettingSpec { + name: "json", + options: &["--json"], + scope: RunRequest, + python_nodes: false, + whole_run: false, + group: OneBuild, + }, + JobSettingSpec { + name: "selection_diagnostics", + options: &["--selection-diagnostics"], + scope: Build, + python_nodes: false, + whole_run: false, + group: OneBuild, + }, + JobSettingSpec { + name: "include_python", + options: &["--no-python"], + scope: Build, + python_nodes: false, + whole_run: false, + group: OneBuild, + }, + JobSettingSpec { + name: "run_tests", + options: &["--no-tests"], + scope: Node, + python_nodes: false, + whole_run: false, + group: OneBuild, + }, + JobSettingSpec { + name: "run_audits", + options: &["--no-audits"], + scope: Build, + python_nodes: false, + whole_run: false, + group: OneBuild, + }, + JobSettingSpec { + name: "ignore_audit_failures", + options: &["--ignore-audit-failures", "--no-ignore-audit-failures"], + scope: Node, + python_nodes: false, + whole_run: false, + group: OneBuild, + }, + JobSettingSpec { + name: "manifest", + options: &["--manifest"], + scope: RunRequest, + python_nodes: false, + whole_run: false, + group: OneBuild, + }, + JobSettingSpec { + name: "json_output", + options: &["--json-output"], + scope: RunRequest, + python_nodes: false, + whole_run: false, + group: OneBuild, + }, + JobSettingSpec { + name: "event_output", + options: &["--event-output"], + scope: RunRequest, + python_nodes: false, + whole_run: false, + group: OneBuild, + }, + JobSettingSpec { + name: "start_cursor", + options: &["--start-cursor"], + scope: Build, + python_nodes: false, + whole_run: false, + group: OneBuild, + }, + JobSettingSpec { + name: "end_cursor", + options: &["--end-cursor"], + scope: Build, + python_nodes: false, + whole_run: false, + group: OneBuild, + }, + JobSettingSpec { + name: "start_cursor_ts", + options: &["--start-cursor-ts"], + scope: Build, + python_nodes: false, + whole_run: false, + group: OneBuild, + }, + JobSettingSpec { + name: "end_cursor_ts", + options: &["--end-cursor-ts"], + scope: Build, + python_nodes: false, + whole_run: false, + group: OneBuild, + }, + JobSettingSpec { + name: "start_cursor_int", + options: &["--start-cursor-int"], + scope: Build, + python_nodes: false, + whole_run: false, + group: OneBuild, + }, + JobSettingSpec { + name: "end_cursor_int", + options: &["--end-cursor-int"], + scope: Build, + python_nodes: false, + whole_run: false, + group: OneBuild, + }, + JobSettingSpec { + name: "max_microbatches", + options: &["--max-microbatches"], + scope: Build, + python_nodes: false, + whole_run: false, + group: OneBuild, + }, + JobSettingSpec { + name: "load_sources", + options: &["--load", "--no-load"], + scope: Build, + python_nodes: false, + whole_run: false, + group: OneBuild, + }, + JobSettingSpec { + name: "reload", + options: &["--reload"], + scope: Build, + python_nodes: false, + whole_run: false, + group: OneBuild, + }, + JobSettingSpec { + name: "fail_fast", + options: &["--fail-fast"], + scope: Build, + python_nodes: false, + whole_run: false, + group: OneBuild, + }, + JobSettingSpec { + name: "full_refresh", + options: &["--full-refresh"], + scope: Node, + python_nodes: false, + whole_run: false, + group: OneBuild, + }, + JobSettingSpec { + name: "job_settings", + options: &["--no-job-settings"], + scope: RunRequest, + python_nodes: false, + whole_run: false, + group: OneBuild, + }, + JobSettingSpec { + name: "changed", + options: &["--changed"], + scope: Build, + python_nodes: false, + whole_run: false, + group: EachBuild, + }, + JobSettingSpec { + name: "reach", + options: &["--reach"], + scope: Build, + python_nodes: false, + whole_run: false, + group: OneBuild, + }, + JobSettingSpec { + name: "allow_snapshot_full_refresh", + options: &["--allow-snapshot-full-refresh"], + scope: Build, + python_nodes: false, + whole_run: false, + group: OneBuild, + }, + JobSettingSpec { + name: "allow_table_type_downgrade", + options: &["--allow-table-type-downgrade"], + scope: Build, + python_nodes: false, + whole_run: false, + group: OneBuild, + }, + JobSettingSpec { + name: "allow_retention_decrease", + options: &["--allow-retention-decrease"], + scope: Build, + python_nodes: false, + whole_run: false, + group: OneBuild, + }, + JobSettingSpec { + name: "allow_missing_migration_origin", + options: &["--allow-missing-migration-origin"], + scope: Build, + python_nodes: false, + whole_run: false, + group: OneBuild, + }, + JobSettingSpec { + name: "allow_snapshot_schema_change", + options: &["--allow-snapshot-schema-change"], + scope: Build, + python_nodes: false, + whole_run: false, + group: OneBuild, + }, + JobSettingSpec { + name: "concurrency", + options: &["--concurrency"], + scope: JobLimit, + python_nodes: false, + whole_run: false, + group: OneBuild, + }, + JobSettingSpec { + name: "test_timeout", + options: &["--test-timeout"], + scope: Build, + python_nodes: false, + whole_run: false, + group: OneBuild, + }, + JobSettingSpec { + name: "verbose", + options: &["--verbose", "-v"], + scope: Build, + python_nodes: false, + whole_run: false, + group: OneBuild, + }, + JobSettingSpec { + name: "select", + options: &["--select", "-s"], + scope: RunRequest, + python_nodes: false, + whole_run: false, + group: OneBuild, + }, + JobSettingSpec { + name: "select_file", + options: &["--select-file"], + scope: RunRequest, + python_nodes: false, + whole_run: false, + group: OneBuild, + }, + JobSettingSpec { + name: "exclude", + options: &["--exclude"], + scope: RunRequest, + python_nodes: false, + whole_run: false, + group: OneBuild, + }, + JobSettingSpec { + name: "positional_select", + options: &[], + scope: RunRequest, + python_nodes: false, + whole_run: false, + group: OneBuild, + }, + JobSettingSpec { + name: "vars", + options: &["--vars"], + scope: Build, + python_nodes: false, + whole_run: false, + group: EachBuild, + }, + JobSettingSpec { + name: "dbt_project_dir", + options: &["--dbt-project-dir"], + scope: RunRequest, + python_nodes: false, + whole_run: false, + group: OneBuild, + }, + JobSettingSpec { + name: "dbt_profiles_dir", + options: &["--dbt-profiles-dir"], + scope: RunRequest, + python_nodes: false, + whole_run: false, + group: OneBuild, + }, + JobSettingSpec { + name: "dbt_target", + options: &["--dbt-target"], + scope: RunRequest, + python_nodes: false, + whole_run: false, + group: OneBuild, + }, + JobSettingSpec { + name: "dbt_target_path", + options: &["--dbt-target-path"], + scope: RunRequest, + python_nodes: false, + whole_run: false, + group: OneBuild, + }, + JobSettingSpec { + name: WAREHOUSE_SETTING, + options: &["--warehouse"], + scope: Node, + python_nodes: true, + whole_run: true, + group: OneBuild, + }, + JobSettingSpec { + name: "project_dir", + options: &["--project-dir", "--sqb-project-dir"], + scope: RunRequest, + python_nodes: false, + whole_run: false, + group: OneBuild, + }, + JobSettingSpec { + name: "no_color", + options: &["--no-color"], + scope: Build, + python_nodes: false, + whole_run: false, + group: OneBuild, + }, + JobSettingSpec { + name: "debug", + options: &["--debug"], + scope: Build, + python_nodes: false, + whole_run: false, + group: OneBuild, + }, +]; diff --git a/crates/sqlbuild-analysis/src/job_settings/main/group_setting_mismatch.rs b/crates/sqlbuild-analysis/src/job_settings/main/group_setting_mismatch.rs index d5b0eea65c..33fdd84323 100644 --- a/crates/sqlbuild-analysis/src/job_settings/main/group_setting_mismatch.rs +++ b/crates/sqlbuild-analysis/src/job_settings/main/group_setting_mismatch.rs @@ -4,10 +4,10 @@ use std::collections::{BTreeMap, BTreeSet}; use sqlbuild_discovery::orchestration_files::models::{GroupFile, OrchestrationInventory}; -use crate::job_settings::constants::GROUP_AGREED_SETTINGS; -use crate::job_settings::main::build_job_settings::build_job_settings; +use crate::job_settings::_helpers::build_job_settings::build_job_settings; use crate::job_settings::main::declared_runs::declared_runs; -use crate::job_settings::models::DeclaredRun; +use crate::job_settings::main::job_setting_spec::job_setting_spec; +use crate::job_settings::models::{DeclaredRun, GroupAgreement}; /// `run` keeping only the settings a group's jobs must agree on. fn agreed_settings_only(run: DeclaredRun) -> DeclaredRun { @@ -15,7 +15,10 @@ fn agreed_settings_only(run: DeclaredRun) -> DeclaredRun { DeclaredRun::Job { settings, retry } => DeclaredRun::Job { settings: settings .into_iter() - .filter(|(name, _)| GROUP_AGREED_SETTINGS.contains(&name.as_str())) + .filter(|(name, _)| { + job_setting_spec(name) + .is_some_and(|spec| spec.group == GroupAgreement::EachBuild) + }) .collect(), retry, }, diff --git a/crates/sqlbuild-analysis/src/job_settings/main/job_setting_spec.rs b/crates/sqlbuild-analysis/src/job_settings/main/job_setting_spec.rs new file mode 100644 index 0000000000..081b254f7f --- /dev/null +++ b/crates/sqlbuild-analysis/src/job_settings/main/job_setting_spec.rs @@ -0,0 +1,10 @@ +//! The table row of one job setting. + +use crate::job_settings::constants::JOB_SETTINGS; +use crate::job_settings::models::JobSettingSpec; + +/// The row of setting `name`; `None` for a name no build option has. +#[must_use] +pub fn job_setting_spec(name: &str) -> Option<&'static JobSettingSpec> { + JOB_SETTINGS.iter().find(|spec| spec.name == name) +} diff --git a/crates/sqlbuild-analysis/src/job_settings/main/mod.rs b/crates/sqlbuild-analysis/src/job_settings/main/mod.rs index 749ae1fb0f..87edd17d2b 100644 --- a/crates/sqlbuild-analysis/src/job_settings/main/mod.rs +++ b/crates/sqlbuild-analysis/src/job_settings/main/mod.rs @@ -1,7 +1,5 @@ -pub mod build_job_settings; pub mod declared_runs; pub mod group_setting_mismatch; -pub mod named_jobs; -pub mod node_job_settings; +pub mod job_setting_spec; +pub mod resolve_job_settings; pub mod setting_scope; -pub mod without_given_settings; diff --git a/crates/sqlbuild-analysis/src/job_settings/main/resolve_job_settings.rs b/crates/sqlbuild-analysis/src/job_settings/main/resolve_job_settings.rs new file mode 100644 index 0000000000..b57c24ab3a --- /dev/null +++ b/crates/sqlbuild-analysis/src/job_settings/main/resolve_job_settings.rs @@ -0,0 +1,67 @@ +//! The one place named jobs' settings meet the command line's: command line > named job. + +use std::collections::{BTreeMap, BTreeSet}; + +use serde_json::Value as Json; + +use crate::job_settings::_helpers::build_job_settings::build_job_settings; +use crate::job_settings::_helpers::named_jobs::named_jobs; +use crate::job_settings::_helpers::node_job_settings::node_job_settings; +use crate::job_settings::_helpers::without_given_settings::without_given_settings; +use crate::job_settings::constants::JOB_SETTINGS; +use crate::job_settings::errors::NodeJobSettingsError; +use crate::job_settings::models::{DeclaredRun, ResolvedJobSettings}; +use crate::job_settings::types::NodeSelection; + +/// The value of whole-run setting `name` every named job sets, if they all set the same one. +fn shared_value( + name: &str, + named: &[String], + declared: &BTreeMap, +) -> Option { + let mut values = named.iter().map(|job| match declared.get(job) { + Some(DeclaredRun::Job { settings, .. }) => settings.get(name), + _ => None, + }); + let first: &Json = values.next()??; + values + .all(|value| value == Some(first)) + .then(|| first.clone()) +} + +/// The named jobs' settings less those `given`, then checked for conflicts; per node with `nodes`. +pub fn resolve_job_settings( + declared: BTreeMap, + (selectors, given): (&[String], &BTreeSet), + nodes: Option>, +) -> Result { + let declared: BTreeMap = without_given_settings(declared, given); + let named: Vec = named_jobs(selectors, &declared); + if named.is_empty() { + return Ok(ResolvedJobSettings { + declared, + ..ResolvedJobSettings::default() + }); + } + let build: BTreeMap = + build_job_settings(&named, &declared).map_err(NodeJobSettingsError::Conflict)?; + let mut whole_run: BTreeMap = BTreeMap::new(); + for spec in JOB_SETTINGS.iter().filter(|spec| spec.whole_run) { + if let Some(value) = shared_value(spec.name, &named, &declared) { + whole_run.insert(spec.name.to_owned(), value); + } + } + let nodes = match nodes { + Some((graph, selection, token_nodes)) => { + Some(node_job_settings(graph, selection, &declared, token_nodes)?) + } + None => None, + }; + Ok(ResolvedJobSettings { + declared, + named, + build, + whole_run, + nodes, + }) +} diff --git a/crates/sqlbuild-analysis/src/job_settings/main/setting_scope.rs b/crates/sqlbuild-analysis/src/job_settings/main/setting_scope.rs index 80bda3ed92..2b6bfbd5d8 100644 --- a/crates/sqlbuild-analysis/src/job_settings/main/setting_scope.rs +++ b/crates/sqlbuild-analysis/src/job_settings/main/setting_scope.rs @@ -1,16 +1,10 @@ //! How far one job setting reaches. -use crate::job_settings::constants::{JOB_LIMIT_SETTING, NODE_SETTINGS}; +use crate::job_settings::main::job_setting_spec::job_setting_spec; use crate::job_settings::models::SettingScope; -/// A node setting, the job limit, or else a build-wide setting. +/// The setting's scope from the table; build-wide for a name the parser then rejects. #[must_use] pub fn setting_scope(name: &str) -> SettingScope { - if NODE_SETTINGS.contains(&name) { - SettingScope::Node - } else if name == JOB_LIMIT_SETTING { - SettingScope::JobLimit - } else { - SettingScope::Build - } + job_setting_spec(name).map_or(SettingScope::Build, |spec| spec.scope) } diff --git a/crates/sqlbuild-analysis/src/job_settings/models.rs b/crates/sqlbuild-analysis/src/job_settings/models.rs index 1642d3c606..bc984d601f 100644 --- a/crates/sqlbuild-analysis/src/job_settings/models.rs +++ b/crates/sqlbuild-analysis/src/job_settings/models.rs @@ -25,15 +25,41 @@ pub enum DeclaredRun { }, } -/// How far one job setting reaches. +/// How far one job setting reaches, which also decides how named jobs' values merge. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum SettingScope { - /// Only the job's own nodes. + /// Only the job's own nodes; two jobs giving one node different values conflict, unset not. Node, - /// A limit on how many of the job's own nodes run at once. + /// A limit on how many of the job's own nodes run at once; each job keeps its own. JobLimit, - /// The whole build. + /// The whole build; every named job must set it the same way, unset differing from set. Build, + /// Not a job setting: the run request owns it. + RunRequest, +} + +/// When a group's jobs must agree on a build-wide setting. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum GroupAgreement { + /// Already while each member runs as its own build, as it changes the compiled SQL. + EachBuild, + /// Once a group runs as one build. + OneBuild, +} + +/// One build option as a job setting: how far it reaches and how named jobs merge it. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct JobSettingSpec { + /// The setting's name, the build parser's destination of its options. + pub name: &'static str, + /// The command-line options that give it, which replace every named job's value. + pub options: &'static [&'static str], + pub scope: SettingScope, + /// Whether a Python node takes it; the other node settings act on SQL nodes only. + pub python_nodes: bool, + /// Whether one value every named job shares becomes the whole build's, setup included. + pub whole_run: bool, + pub group: GroupAgreement, } /// What the jobs a selection names give each of its nodes. @@ -48,3 +74,18 @@ pub struct NodeJobSettings { /// Each named job's limit on how many of its nodes run at once. pub job_limits: BTreeMap, } + +/// What the named jobs give a build once the command line's own settings replace theirs. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct ResolvedJobSettings { + /// The declared jobs less the settings the command line gives. + pub declared: BTreeMap, + /// The jobs the selectors name. + pub named: Vec, + /// The build-wide settings the named jobs agree on. + pub build: BTreeMap, + /// The whole-run node settings every named job sets to one value. + pub whole_run: BTreeMap, + /// Each selected node's settings and the jobs' limits, when the nodes were resolved. + pub nodes: Option, +} diff --git a/crates/sqlbuild-analysis/src/job_settings/tests.rs b/crates/sqlbuild-analysis/src/job_settings/tests.rs index eeac578fe3..1fc45957d0 100644 --- a/crates/sqlbuild-analysis/src/job_settings/tests.rs +++ b/crates/sqlbuild-analysis/src/job_settings/tests.rs @@ -2,6 +2,8 @@ mod build_job_settings; #[path = "tests/helpers.rs"] mod helpers; +#[path = "tests/test_job_setting_table.rs"] +mod job_setting_table; #[path = "tests/test_node_job_settings.rs"] mod node_job_settings; #[path = "tests/test_types.rs"] diff --git a/crates/sqlbuild-analysis/src/job_settings/tests/helpers.rs b/crates/sqlbuild-analysis/src/job_settings/tests/helpers.rs index 20346b46dc..62b1f67668 100644 --- a/crates/sqlbuild-analysis/src/job_settings/tests/helpers.rs +++ b/crates/sqlbuild-analysis/src/job_settings/tests/helpers.rs @@ -7,8 +7,10 @@ use crate::graph::main::attach_jobs::attach_jobs; use crate::graph::main::build_project_graph::build_project_graph; use crate::graph::main::resolve_selector_tokens::resolve_selector_tokens; use crate::graph::models::{GraphResource, JobSelection, ProjectGraph}; +use crate::job_settings::_helpers::node_job_settings::node_job_settings; +use crate::job_settings::_helpers::node_values::node_values; use crate::job_settings::errors::NodeJobSettingsError; -use crate::job_settings::main::node_job_settings::node_job_settings; +use crate::job_settings::main::resolve_job_settings::resolve_job_settings; use crate::job_settings::models::{DeclaredRun, JobRetry, NodeJobSettings}; fn key(kind: &str, name: &str) -> ObjectKey { @@ -177,3 +179,58 @@ pub(super) fn resolved_settings( &token_nodes, ) } + +/// Jobs `a_job`, `b_job` and `c_job` setting `name` to each of `values`, unset where `None`. +pub(super) fn jobs_setting(name: &str, values: [Option; 3]) -> BTreeMap { + ["a_job", "b_job", "c_job"] + .into_iter() + .zip(values) + .map(|(job, value)| { + let settings: BTreeMap = value + .map(|value| (name.to_owned(), value)) + .into_iter() + .collect(); + ( + job.to_owned(), + DeclaredRun::Job { + settings, + retry: None, + }, + ) + }) + .collect() +} + +/// The `job:` selectors of `jobs`. +pub(super) fn job_selectors(jobs: &[&str]) -> Vec { + jobs.iter().map(|job| format!("job:{job}")).collect() +} + +/// Whether naming `named`, `given` on the command line, conflicts on `name`, and the build takes it. +pub(super) fn resolves_build_wide(name: &str, named: &[&str], given: &[&str]) -> (bool, bool) { + let declared = jobs_setting(name, [Some(json!("first")), Some(json!("second")), None]); + let given: BTreeSet = given.iter().map(|setting| (*setting).to_owned()).collect(); + resolve_job_settings(declared, (&job_selectors(named), &given), None) + .map_or((true, false), |resolved| { + (false, resolved.build.contains_key(name)) + }) +} + +/// Whether `a_job` and `b_job` giving one model `first` and `second` for `name` conflict. +pub(super) fn conflicts_on_a_node(name: &str) -> bool { + let declared = jobs_setting(name, [Some(json!("first")), Some(json!("second")), None]); + let model: ObjectKey = key("model", "orders"); + node_values(&model, &["a_job".to_owned(), "b_job".to_owned()], &declared).is_err() +} + +/// The whole-run value of `name` when `a_job` and `b_job` set it to `values`. +pub(super) fn whole_run_value(name: &str, values: [Option; 3]) -> Option { + let declared = jobs_setting(name, values); + resolve_job_settings( + declared, + (&job_selectors(&["a_job", "b_job"]), &BTreeSet::new()), + None, + ) + .ok() + .and_then(|resolved| resolved.whole_run.get(name).cloned()) +} diff --git a/crates/sqlbuild-analysis/src/job_settings/tests/test_build_job_settings.rs b/crates/sqlbuild-analysis/src/job_settings/tests/test_build_job_settings.rs index bdbdd88db3..e34ce90a5d 100644 --- a/crates/sqlbuild-analysis/src/job_settings/tests/test_build_job_settings.rs +++ b/crates/sqlbuild-analysis/src/job_settings/tests/test_build_job_settings.rs @@ -2,8 +2,8 @@ use std::collections::BTreeMap; use serde_json::json; -use crate::job_settings::main::build_job_settings::build_job_settings; -use crate::job_settings::main::named_jobs::named_jobs; +use crate::job_settings::_helpers::build_job_settings::build_job_settings; +use crate::job_settings::_helpers::named_jobs::named_jobs; use crate::job_settings::tests::helpers::{declared, strings}; use crate::job_settings::tests::test_types::BuildJobSettingsTestCase; diff --git a/crates/sqlbuild-analysis/src/job_settings/tests/test_job_setting_table.rs b/crates/sqlbuild-analysis/src/job_settings/tests/test_job_setting_table.rs new file mode 100644 index 0000000000..3961a85d64 --- /dev/null +++ b/crates/sqlbuild-analysis/src/job_settings/tests/test_job_setting_table.rs @@ -0,0 +1,112 @@ +use std::collections::BTreeSet; + +use serde_json::json; + +use crate::job_settings::constants::JOB_SETTINGS; +use crate::job_settings::models::SettingScope; +use crate::job_settings::tests::helpers::{ + conflicts_on_a_node, resolves_build_wide, whole_run_value, +}; +use crate::job_settings::tests::test_types::{JobSettingScopeTestCase, WholeRunTestCase}; + +#[test] +fn given_every_job_setting_when_named_jobs_merge_it_then_its_scope_decides_and_the_command_line_wins() + { + let test_cases = [ + JobSettingScopeTestCase { + description: "a build-wide setting must agree, unset included, and applies to all", + scope: SettingScope::Build, + expected_build_conflict: true, + expected_unset_conflict: true, + expected_build_value: true, + expected_node_conflict: false, + }, + JobSettingScopeTestCase { + description: "a node setting stays per node and conflicts on a shared node", + scope: SettingScope::Node, + expected_build_conflict: false, + expected_unset_conflict: false, + expected_build_value: false, + expected_node_conflict: true, + }, + JobSettingScopeTestCase { + description: "a job limit stays with each job", + scope: SettingScope::JobLimit, + expected_build_conflict: false, + expected_unset_conflict: false, + expected_build_value: false, + expected_node_conflict: false, + }, + ]; + let job_scopes: BTreeSet = JOB_SETTINGS + .iter() + .filter(|spec| spec.scope != SettingScope::RunRequest) + .map(|spec| format!("{:?}", spec.scope)) + .collect(); + let case_scopes: BTreeSet = test_cases + .iter() + .map(|case| format!("{:?}", case.scope)) + .collect(); + assert_eq!( + job_scopes, case_scopes, + "every job setting scope has a case" + ); + for test_case in test_cases { + for spec in JOB_SETTINGS + .iter() + .filter(|spec| spec.scope == test_case.scope) + { + assert_eq!( + ( + resolves_build_wide(spec.name, &["a_job", "b_job"], &[]).0, + resolves_build_wide(spec.name, &["a_job", "c_job"], &[]).0, + resolves_build_wide(spec.name, &["a_job"], &[]).1, + conflicts_on_a_node(spec.name), + resolves_build_wide(spec.name, &["a_job", "b_job"], &[spec.name]), + ), + ( + test_case.expected_build_conflict, + test_case.expected_unset_conflict, + test_case.expected_build_value, + test_case.expected_node_conflict, + (false, false), + ), + "{}: {}", + test_case.description, + spec.name + ); + } + } +} + +#[test] +fn given_a_whole_run_setting_when_named_jobs_share_its_value_then_it_is_the_build_s() { + let test_cases = [ + WholeRunTestCase { + description: "every named job sets one value", + values: [Some(json!("ONE")), Some(json!("ONE")), None], + expected_value: Some(json!("ONE")), + }, + WholeRunTestCase { + description: "the named jobs set different values", + values: [Some(json!("ONE")), Some(json!("TWO")), None], + expected_value: None, + }, + WholeRunTestCase { + description: "one named job leaves it unset", + values: [Some(json!("ONE")), None, None], + expected_value: None, + }, + ]; + for test_case in test_cases { + for spec in JOB_SETTINGS.iter().filter(|spec| spec.whole_run) { + assert_eq!( + whole_run_value(spec.name, test_case.values.clone()), + test_case.expected_value, + "{}: {}", + test_case.description, + spec.name + ); + } + } +} diff --git a/crates/sqlbuild-analysis/src/job_settings/tests/test_types.rs b/crates/sqlbuild-analysis/src/job_settings/tests/test_types.rs index 23bfbdeadb..28bfe5befb 100644 --- a/crates/sqlbuild-analysis/src/job_settings/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/job_settings/tests/test_types.rs @@ -39,3 +39,24 @@ pub(super) struct BuildJobSettingsTestCase { pub(super) expected_named: Vec<&'static str>, pub(super) expected_settings: Result, &'static str>, } + +/// How a setting of one scope merges across two named jobs and against the command line. +pub(super) struct JobSettingScopeTestCase { + pub(super) description: &'static str, + pub(super) scope: crate::job_settings::models::SettingScope, + /// Whether two named jobs giving different values fail as a build-wide conflict. + pub(super) expected_build_conflict: bool, + /// Whether a named job leaving it unset fails against one that sets it. + pub(super) expected_unset_conflict: bool, + /// Whether a single named job's value becomes the build's. + pub(super) expected_build_value: bool, + /// Whether two jobs giving one SQL node different values conflict on that node. + pub(super) expected_node_conflict: bool, +} + +/// A whole-run setting's values in two named jobs, and the build's value. +pub(super) struct WholeRunTestCase { + pub(super) description: &'static str, + pub(super) values: [Option; 3], + pub(super) expected_value: Option, +} diff --git a/crates/sqlbuild-analysis/src/job_settings/types.rs b/crates/sqlbuild-analysis/src/job_settings/types.rs index a9a4bf2df9..2365b5d9ae 100644 --- a/crates/sqlbuild-analysis/src/job_settings/types.rs +++ b/crates/sqlbuild-analysis/src/job_settings/types.rs @@ -7,3 +7,10 @@ use crate::graph::errors::SelectorError; /// The nodes one selector token selects, less those `--exclude` removes, as the build resolves them. pub type TokenNodes<'r> = &'r dyn Fn(&str, &[String]) -> Result, SelectorError>; + +/// The graph, the `--select` and `--exclude` selectors and their token resolution of a build's nodes. +pub type NodeSelection<'r> = ( + &'r crate::graph::models::ProjectGraph, + (&'r [String], &'r [String]), + TokenNodes<'r>, +); diff --git a/crates/sqlbuild-cli/src/argv/_helpers/job_settings.rs b/crates/sqlbuild-cli/src/argv/_helpers/job_settings.rs index 492b59f87a..f05e843f7c 100644 --- a/crates/sqlbuild-cli/src/argv/_helpers/job_settings.rs +++ b/crates/sqlbuild-cli/src/argv/_helpers/job_settings.rs @@ -1,5 +1,7 @@ //! Render job settings as options, check them against the build parser and parse without Python. +use sqlbuild_analysis::job_settings::main::setting_scope::setting_scope; +use sqlbuild_analysis::job_settings::models::SettingScope; use std::collections::BTreeMap; use serde_json::Value as Json; @@ -15,25 +17,7 @@ pub(crate) fn setting_argument( name: &str, value: &Json, ) -> Result, String> { - if matches!( - name, - "help" - | "target" - | "project_dir" - | "select" - | "exclude" - | "select_file" - | "positional_select" - | "json" - | "json_output" - | "event_output" - | "manifest" - | "dbt_project_dir" - | "dbt_profiles_dir" - | "dbt_target" - | "dbt_target_path" - | "job_settings" - ) { + if setting_scope(name) == SettingScope::RunRequest { return Err(format!("job setting {name} is owned by the run request")); } let actions: Vec<&ActionSpec> = parser diff --git a/crates/sqlbuild-cli/src/argv/main/job_run_arguments.rs b/crates/sqlbuild-cli/src/argv/main/job_run_arguments.rs index e7baec5964..15e6520191 100644 --- a/crates/sqlbuild-cli/src/argv/main/job_run_arguments.rs +++ b/crates/sqlbuild-cli/src/argv/main/job_run_arguments.rs @@ -3,8 +3,7 @@ use std::collections::BTreeMap; use serde_json::Value as Json; -use sqlbuild_analysis::job_settings::constants::WAREHOUSE_SETTING; -use sqlbuild_analysis::job_settings::main::setting_scope::setting_scope; +use sqlbuild_analysis::job_settings::main::job_setting_spec::job_setting_spec; use sqlbuild_analysis::job_settings::models::SettingScope; use crate::argv::main::job_settings_arguments::job_settings_arguments; @@ -17,7 +16,8 @@ pub fn job_run_arguments( let build_wide: BTreeMap = settings .iter() .filter(|(name, _)| { - setting_scope(name) == SettingScope::Build || name.as_str() == WAREHOUSE_SETTING + job_setting_spec(name) + .is_some_and(|spec| spec.scope == SettingScope::Build || spec.whole_run) }) .map(|(name, value)| (name.clone(), value.clone())) .collect(); diff --git a/crates/sqlbuild-cli/src/argv/tests.rs b/crates/sqlbuild-cli/src/argv/tests.rs index c053fa0582..c8b4187dfd 100644 --- a/crates/sqlbuild-cli/src/argv/tests.rs +++ b/crates/sqlbuild-cli/src/argv/tests.rs @@ -4,6 +4,8 @@ mod given_setting_dests; mod helpers; #[path = "tests/test_job_run_arguments.rs"] mod job_run_arguments; +#[path = "tests/test_job_setting_table.rs"] +mod job_setting_table; #[path = "tests/test_parse_cli_arguments.rs"] mod parse_cli_arguments; #[path = "tests/test_parse_dbt_execution_arguments.rs"] diff --git a/crates/sqlbuild-cli/src/argv/tests/helpers.rs b/crates/sqlbuild-cli/src/argv/tests/helpers.rs index b735659aeb..54043f3e11 100644 --- a/crates/sqlbuild-cli/src/argv/tests/helpers.rs +++ b/crates/sqlbuild-cli/src/argv/tests/helpers.rs @@ -1,4 +1,13 @@ +use std::collections::{BTreeMap, BTreeSet}; + +use serde_json::{Value as Json, json}; +use sqlbuild_analysis::job_settings::constants::JOB_SETTINGS; +use sqlbuild_analysis::job_settings::models::SettingScope; + +use crate::argv::_helpers::table::command_parser; +use crate::argv::main::command_settings_arguments::command_settings_arguments; use crate::argv::main::parse_cli_arguments::parse_cli_arguments; +use crate::argv::models::{ActionKind, ActionSpec, Nargs}; use crate::argv::models::{ArgvOutcome, ArgvSettings, Value}; use crate::argv::tests::test_types::FakeHost; @@ -26,3 +35,96 @@ pub(super) fn strings(items: &[&str]) -> Value { pub(super) fn owned(items: &[&str]) -> Vec { items.iter().map(|item| (*item).to_owned()).collect() } + +/// Each destination of `command`'s parser with all its option strings, in the parser's order. +pub(super) fn parser_options(command: &str) -> Vec<(String, Vec)> { + let actions: Vec<&ActionSpec> = command_parser(command) + .map(|parser| parser.actions.iter().collect()) + .unwrap_or_default(); + let mut seen: BTreeSet<&str> = BTreeSet::new(); + let dests: Vec<&str> = actions + .iter() + .filter_map(|action| action.dest.as_deref()) + .filter(|dest| seen.insert(dest)) + .collect(); + dests + .into_iter() + .map(|dest| { + let strings: Vec = actions + .iter() + .filter(|action| action.dest.as_deref() == Some(dest)) + .flat_map(|action| action.option_strings.iter().cloned()) + .collect(); + (dest.to_owned(), strings) + }) + .collect() +} + +/// Each job setting with its listed options, in the table's order. +pub(super) fn table_options() -> Vec<(String, Vec)> { + JOB_SETTINGS + .iter() + .map(|spec| { + let strings = spec + .options + .iter() + .map(|option| (*option).to_owned()) + .collect(); + (spec.name.to_owned(), strings) + }) + .collect() +} + +/// A value the option of `action` accepts, as a job would write it. +fn sample(action: &ActionSpec) -> Json { + let converter: Option<&str> = action.converter.as_deref(); + let candidates: [(bool, Json); 6] = [ + ( + action.kind == ActionKind::StoreTrue || action.nargs == Nargs::Attached, + json!(true), + ), + (action.kind == ActionKind::StoreFalse, json!(false)), + ( + action.choices.is_some(), + json!(action.choices.as_ref().and_then(|choices| choices.first())), + ), + ( + matches!(converter, Some("int" | "_positive_integer")), + json!(3), + ), + (converter == Some("parse_cli_vars"), json!({"region": "eu"})), + (true, json!("2026-01-01T00:00:00+00:00")), + ]; + candidates + .into_iter() + .find_map(|(applies, value)| applies.then_some(value)) + .unwrap_or_default() +} + +/// The job settings of `scope` and `whole_run` that do not render as one of their listed options. +pub(super) fn unrendered_settings(scope: SettingScope, whole_run: bool) -> Vec<&'static str> { + JOB_SETTINGS + .iter() + .filter(|spec| spec.scope == scope && spec.whole_run == whole_run) + .filter(|spec| { + !rendered_option(spec.name) + .is_some_and(|option| spec.options.contains(&option.as_str())) + }) + .map(|spec| spec.name) + .collect() +} + +/// The option a `build` job setting `name` renders as, from a sample value. +fn rendered_option(name: &str) -> Option { + let parser = command_parser("build").ok()?; + let action = parser + .actions + .iter() + .find(|action| action.dest.as_deref() == Some(name))?; + let settings = BTreeMap::from([(name.to_owned(), sample(action))]); + let rendered: Vec = command_settings_arguments("build", &settings).ok()?; + rendered + .first() + .and_then(|argument| argument.split('=').next()) + .map(str::to_owned) +} diff --git a/crates/sqlbuild-cli/src/argv/tests/test_job_setting_table.rs b/crates/sqlbuild-cli/src/argv/tests/test_job_setting_table.rs new file mode 100644 index 0000000000..1d09a92bfe --- /dev/null +++ b/crates/sqlbuild-cli/src/argv/tests/test_job_setting_table.rs @@ -0,0 +1,47 @@ +use sqlbuild_analysis::job_settings::models::SettingScope; + +use crate::argv::tests::helpers::{parser_options, table_options, unrendered_settings}; +use crate::argv::tests::test_types::{JobSettingTableTestCase, RenderedSettingTestCase}; + +#[test] +fn given_the_build_parser_when_compared_with_the_job_setting_table_then_every_option_has_one_row() { + let test_cases = [JobSettingTableTestCase { + description: "add a JOB_SETTINGS row for each new build option", + command: "build", + expected_rows: table_options(), + }]; + for test_case in test_cases { + assert_eq!( + parser_options(test_case.command), + test_case.expected_rows, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_a_job_setting_the_build_takes_as_a_flag_when_rendered_then_it_is_one_of_its_options() { + let test_cases = [ + RenderedSettingTestCase { + description: "a build-wide setting travels as its own option", + scope: SettingScope::Build, + whole_run: false, + expected_unrendered: Vec::new(), + }, + RenderedSettingTestCase { + description: "a whole-run node setting travels as its own option", + scope: SettingScope::Node, + whole_run: true, + expected_unrendered: Vec::new(), + }, + ]; + for test_case in test_cases { + assert_eq!( + unrendered_settings(test_case.scope, test_case.whole_run), + test_case.expected_unrendered, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-cli/src/argv/tests/test_types.rs b/crates/sqlbuild-cli/src/argv/tests/test_types.rs index e152b76827..18b69f7111 100644 --- a/crates/sqlbuild-cli/src/argv/tests/test_types.rs +++ b/crates/sqlbuild-cli/src/argv/tests/test_types.rs @@ -95,3 +95,18 @@ pub(super) struct JobRunArgumentsTestCase { pub(super) settings: serde_json::Value, pub(super) expected_arguments: &'static [&'static str], } + +/// A command's parser and the job setting table rows it must match. +pub(super) struct JobSettingTableTestCase { + pub(super) description: &'static str, + pub(super) command: &'static str, + pub(super) expected_rows: Vec<(String, Vec)>, +} + +/// The job settings of some scopes, each of which must render as one of its listed options. +pub(super) struct RenderedSettingTestCase { + pub(super) description: &'static str, + pub(super) scope: sqlbuild_analysis::job_settings::models::SettingScope, + pub(super) whole_run: bool, + pub(super) expected_unrendered: Vec<&'static str>, +} diff --git a/crates/sqlbuild-python-compile/src/compile_driver/_helpers/jobs/job_node_settings.rs b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/jobs/job_node_settings.rs index 93875cac02..e2e24e74d2 100644 --- a/crates/sqlbuild-python-compile/src/compile_driver/_helpers/jobs/job_node_settings.rs +++ b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/jobs/job_node_settings.rs @@ -12,11 +12,8 @@ use sqlbuild_analysis::definition_changes::constants::PYTHON_NODE_KINDS; use sqlbuild_analysis::graph::main::selectors_need_jobs::selectors_need_jobs; use sqlbuild_analysis::job_settings::constants::{NODE_CONFLICT_HELP, WAREHOUSE_SETTING}; use sqlbuild_analysis::job_settings::errors::{JobSettingConflict, NodeJobSettingsError}; -use sqlbuild_analysis::job_settings::main::build_job_settings::build_job_settings; -use sqlbuild_analysis::job_settings::main::named_jobs::named_jobs; -use sqlbuild_analysis::job_settings::main::node_job_settings::node_job_settings; -use sqlbuild_analysis::job_settings::main::without_given_settings::without_given_settings; -use sqlbuild_analysis::job_settings::models::{DeclaredRun, NodeJobSettings}; +use sqlbuild_analysis::job_settings::main::resolve_job_settings::resolve_job_settings; +use sqlbuild_analysis::job_settings::models::{DeclaredRun, NodeJobSettings, ResolvedJobSettings}; use sqlbuild_config::command_connection::errors::CommandConnectionError; use sqlbuild_config::command_connection::main::check_job_warehouse::check_job_warehouse; use sqlbuild_config::command_connection::main::unsupported_job_warehouse::unsupported_job_warehouse; @@ -52,9 +49,12 @@ use sqlbuild_python_project::resource_views::main::audit_reads::scope_deps::scop const MODEL_KIND: &str = "model"; const FUNCTION_KINDS: [&str; 2] = ["udf", "table_fn"]; -const FULL_REFRESH: &str = "full_refresh"; -const RUN_TESTS: &str = "run_tests"; -const IGNORE_AUDIT_FAILURES: &str = "ignore_audit_failures"; +/// The node setting the planner's `full_refresh_models` and `full_refresh_functions` apply. +pub(crate) const FULL_REFRESH: &str = "full_refresh"; +/// The node setting that drops a job's SQL tests from the plan. +pub(crate) const RUN_TESTS: &str = "run_tests"; +/// The node setting each audit entry's `ignore_failure` takes. +pub(crate) const IGNORE_AUDIT_FAILURES: &str = "ignore_audit_failures"; /// The `CliUserError` of a job warehouse that cannot be used. fn cli_error(py: Python<'_>, error: &CommandConnectionError) -> PyErr { @@ -185,13 +185,6 @@ pub(crate) fn planned_job_settings( if cli_warehouse { given.insert(WAREHOUSE_SETTING.to_owned()); } - let declared: BTreeMap = without_given_settings(declared, &given); - let named: Vec = named_jobs(&select, &declared); - if named.is_empty() { - return Ok(None); - } - build_job_settings(&named, &declared) - .map_err(|conflict| job_setting_conflict(py, &conflict))?; let all: Vec = select.iter().chain(&exclude).cloned().collect(); let compiled: Bound<'_, CompiledProjectGraph> = project_graph(project)?.cast_into()?; let selection = PythonAwareSelection::new( @@ -202,15 +195,18 @@ pub(crate) fn planned_job_settings( let token_nodes = |token: &str, exclude: &[String]| selection.resolve_keys(&[token.to_owned()], exclude); let resolved = compiler_guard(|| { - Ok(node_job_settings( - selection.graph(), - (&select, &exclude), - &declared, - &token_nodes, + Ok(resolve_job_settings( + declared, + (&select, &given), + Some((selection.graph(), (&select, &exclude), &token_nodes)), )) })?; match resolved { - Ok(settings) => Ok(Some(PlannedJobSettings { + Ok(ResolvedJobSettings { + nodes: Some(settings), + declared, + .. + }) => Ok(Some(PlannedJobSettings { ignoring_jobs: jobs_setting(&settings, &declared, (IGNORE_AUDIT_FAILURES, None)), refreshing_jobs: jobs_setting( &settings, @@ -224,6 +220,7 @@ pub(crate) fn planned_job_settings( unattached_audit_reads: unattached_audit_reads(project)?, settings, })), + Ok(ResolvedJobSettings { nodes: None, .. }) => Ok(None), Err(NodeJobSettingsError::Selector(failure)) => Err(job_selector_error(py, &failure)), Err(NodeJobSettingsError::Conflict(conflict)) => Err(job_setting_conflict(py, &conflict)), } diff --git a/crates/sqlbuild-python-compile/src/compile_driver/mod.rs b/crates/sqlbuild-python-compile/src/compile_driver/mod.rs index cb9dd32021..88823c59ec 100644 --- a/crates/sqlbuild-python-compile/src/compile_driver/mod.rs +++ b/crates/sqlbuild-python-compile/src/compile_driver/mod.rs @@ -3,4 +3,6 @@ pub(crate) mod _helpers; pub mod main; pub mod models; +#[cfg(test)] +mod tests; pub mod types; diff --git a/crates/sqlbuild-python-compile/src/compile_driver/tests.rs b/crates/sqlbuild-python-compile/src/compile_driver/tests.rs new file mode 100644 index 0000000000..5bdc9477a2 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_driver/tests.rs @@ -0,0 +1,4 @@ +#[path = "tests/test_node_setting_consumers.rs"] +mod node_setting_consumers; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-python-compile/src/compile_driver/tests/test_node_setting_consumers.rs b/crates/sqlbuild-python-compile/src/compile_driver/tests/test_node_setting_consumers.rs new file mode 100644 index 0000000000..f712edb5ee --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_driver/tests/test_node_setting_consumers.rs @@ -0,0 +1,34 @@ +use std::collections::BTreeSet; + +use sqlbuild_analysis::job_settings::constants::{JOB_SETTINGS, WAREHOUSE_SETTING}; +use sqlbuild_analysis::job_settings::models::SettingScope; + +use crate::compile_driver::_helpers::jobs::job_node_settings::{ + FULL_REFRESH, IGNORE_AUDIT_FAILURES, RUN_TESTS, +}; +use crate::compile_driver::tests::test_types::NodeSettingConsumersTestCase; + +#[test] +fn given_every_node_setting_in_the_table_when_planning_then_a_consumer_applies_it() { + let test_cases = [NodeSettingConsumersTestCase { + description: "apply each node setting to the plan in job_node_settings.rs, then list it here", + expected_node_settings: BTreeSet::from([ + FULL_REFRESH, + RUN_TESTS, + IGNORE_AUDIT_FAILURES, + WAREHOUSE_SETTING, + ]), + }]; + for test_case in test_cases { + let node_settings: BTreeSet<&str> = JOB_SETTINGS + .iter() + .filter(|spec| spec.scope == SettingScope::Node) + .map(|spec| spec.name) + .collect(); + assert_eq!( + node_settings, test_case.expected_node_settings, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-python-compile/src/compile_driver/tests/test_types.rs b/crates/sqlbuild-python-compile/src/compile_driver/tests/test_types.rs new file mode 100644 index 0000000000..17742f098a --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_driver/tests/test_types.rs @@ -0,0 +1,7 @@ +use std::collections::BTreeSet; + +/// The node settings the planner applies, which the table's node settings must equal. +pub(super) struct NodeSettingConsumersTestCase { + pub(super) description: &'static str, + pub(super) expected_node_settings: BTreeSet<&'static str>, +} diff --git a/crates/sqlbuild-python-project/src/job_settings/_helpers/named_job_build_settings.rs b/crates/sqlbuild-python-project/src/job_settings/_helpers/named_job_build_settings.rs index 7aa8be5ed6..fb701a05ae 100644 --- a/crates/sqlbuild-python-project/src/job_settings/_helpers/named_job_build_settings.rs +++ b/crates/sqlbuild-python-project/src/job_settings/_helpers/named_job_build_settings.rs @@ -7,10 +7,9 @@ use pyo3::IntoPyObject; use pyo3::prelude::{Bound, PyAny, PyResult, Python}; use serde_json::Value; use sqlbuild_analysis::job_settings::constants::WAREHOUSE_SETTING; -use sqlbuild_analysis::job_settings::main::build_job_settings::build_job_settings; -use sqlbuild_analysis::job_settings::main::named_jobs::named_jobs; -use sqlbuild_analysis::job_settings::main::without_given_settings::without_given_settings; -use sqlbuild_analysis::job_settings::models::DeclaredRun; +use sqlbuild_analysis::job_settings::errors::NodeJobSettingsError; +use sqlbuild_analysis::job_settings::main::resolve_job_settings::resolve_job_settings; +use sqlbuild_analysis::job_settings::models::ResolvedJobSettings; use sqlbuild_config::command_connection::main::check_job_warehouse::check_job_warehouse; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_front::compile_driver::main::session_warehouse_support::session_warehouse_support; @@ -19,37 +18,28 @@ use sqlbuild_python_front::discovery::main::project::discover_project_configurat use crate::job_settings::_helpers::job_run_settings::job_run_settings; use crate::job_settings::main::job_declarations::job_declarations; +use crate::job_settings::main::job_selector_error::job_selector_error; use crate::job_settings::main::job_setting_conflict::job_setting_conflict; use crate::job_settings::main::with_job_scope::with_job_scope; -/// The warehouse every named job sets, when they all set the same usable one. -fn shared_warehouse( +/// Whether whole-run `name` = `value` suits a whole build: a valid warehouse the adapter can use. +fn applies_to_whole_run( py: Python<'_>, - named: &[String], - (declared, directory): (&BTreeMap, &Bound<'_, PyAny>), -) -> PyResult> { - let mut warehouses = named.iter().map(|job| match declared.get(job) { - Some(DeclaredRun::Job { settings, .. }) => settings.get(WAREHOUSE_SETTING), - _ => None, - }); - let Some(Some(warehouse)) = warehouses.next() else { - return Ok(None); - }; - if warehouses.any(|other| other != Some(warehouse)) { - return Ok(None); - } - if check_job_warehouse("", warehouse.as_str()).is_err() { - return Ok(None); + (name, value): (&str, &Value), + directory: &Bound<'_, PyAny>, +) -> PyResult { + if name != WAREHOUSE_SETTING || check_job_warehouse("", value.as_str()).is_err() { + return Ok(false); } let discovered: Bound<'_, PyAny> = discover_project_configuration(py, directory)?; let adapter_name: Bound<'_, PyAny> = resolve_effective_adapter_name( &native_field(&discovered, "project_config")?, &native_field(&discovered, "local_config")?, )?; - Ok(session_warehouse_support(py, &adapter_name, directory)?.then(|| (*warehouse).clone())) + session_warehouse_support(py, &adapter_name, directory) } -/// The named jobs' build-wide settings, a shared warehouse included, less `given`; or `S027`. +/// The named jobs' build-wide and shared whole-run settings, less `given`; or `S027`. pub(crate) fn named_job_build_settings( py: Python<'_>, (selectors, given): (&[String], &BTreeSet), @@ -59,16 +49,22 @@ pub(crate) fn named_job_build_settings( let Some(declared) = with_job_scope(scope, || job_declarations(py))? else { return Ok(BTreeMap::new()); }; - let declared: BTreeMap = without_given_settings(declared, given); - let named: Vec = named_jobs(selectors, &declared); - if named.is_empty() { - return Ok(BTreeMap::new()); - } - let mut agreed: BTreeMap = build_job_settings(&named, &declared) - .map_err(|conflict| job_setting_conflict(py, &conflict))?; + let resolved: ResolvedJobSettings = + match resolve_job_settings(declared, (selectors, given), None) { + Ok(resolved) => resolved, + Err(NodeJobSettingsError::Conflict(conflict)) => { + return Err(job_setting_conflict(py, &conflict)); + } + Err(NodeJobSettingsError::Selector(failure)) => { + return Err(job_selector_error(py, &failure)); + } + }; let directory: Bound<'_, PyAny> = directory.into_pyobject(py)?.into_any(); - if let Some(warehouse) = shared_warehouse(py, &named, (&declared, &directory))? { - agreed.insert(WAREHOUSE_SETTING.to_owned(), warehouse); + let mut agreed: BTreeMap = resolved.build; + for (name, value) in resolved.whole_run { + if applies_to_whole_run(py, (&name, &value), &directory)? { + agreed.insert(name, value); + } } job_run_settings(&directory, agreed) } From e24d55db4b5cb17b41c6188ad45e28d7d1514465 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 19 Nov 2026 06:59:40 +0000 Subject: [PATCH 12/22] fix(jobs): singular audits weigh nodes outside the jobs, and conflict help offers only flags that settle it (CHI-1533) A singular audit reading nodes of an ignoring job and nodes outside the named jobs now takes the build's ignore_audit_failures for the latter and is ignored only when every node it reads ignores failures. Each node setting's row in the job setting table names the plan consumer that applies it, and the planner reads node settings through those consumers, so a node row without one never reaches the plan and fails a test. A test checks that node settings other than the whole-run warehouse never become job run flags. Conflict help is generated from the table: it offers a command-line flag only for the value the flag can give, such as --no-tests to turn tests off for all, and otherwise points to agreeing jobs or --no-job-settings. --- .../_helpers/build_job_settings.rs | 4 +- .../src/job_settings/_helpers/node_values.rs | 11 +- .../src/job_settings/constants.rs | 112 ++++++++++++++++-- .../src/job_settings/errors.rs | 2 +- .../src/job_settings/main/conflict_help.rs | 29 +++++ .../src/job_settings/main/consumed_setting.rs | 13 ++ .../src/job_settings/main/mod.rs | 2 + .../src/job_settings/models.rs | 28 +++++ .../src/job_settings/tests/helpers.rs | 31 ++++- .../tests/test_job_setting_table.rs | 81 ++++++++++++- .../src/job_settings/tests/test_types.rs | 15 +++ crates/sqlbuild-cli/src/argv/tests/helpers.rs | 56 ++++++++- .../src/argv/tests/test_job_setting_table.rs | 32 ++++- .../sqlbuild-cli/src/argv/tests/test_types.rs | 7 ++ .../_helpers/jobs/job_node_settings.rs | 88 ++++++++------ .../main/jobs/job_plan_fields.rs | 7 +- .../src/compile_driver/tests.rs | 2 + .../tests/test_node_setting_consumers.rs | 33 +++--- .../tests/test_singular_audit_ignores.rs | 52 ++++++++ .../src/compile_driver/tests/test_types.rs | 13 +- .../job_settings/main/job_setting_conflict.rs | 5 +- .../references/docs/concepts/orchestration.md | 8 +- .../cli/commands/main/test_job_settings.py | 32 ++++- .../docs/docs/concepts/orchestration.mdx | 8 +- 24 files changed, 583 insertions(+), 88 deletions(-) create mode 100644 crates/sqlbuild-analysis/src/job_settings/main/conflict_help.rs create mode 100644 crates/sqlbuild-analysis/src/job_settings/main/consumed_setting.rs create mode 100644 crates/sqlbuild-python-compile/src/compile_driver/tests/test_singular_audit_ignores.rs diff --git a/crates/sqlbuild-analysis/src/job_settings/_helpers/build_job_settings.rs b/crates/sqlbuild-analysis/src/job_settings/_helpers/build_job_settings.rs index 02f24f036a..b97d16cc20 100644 --- a/crates/sqlbuild-analysis/src/job_settings/_helpers/build_job_settings.rs +++ b/crates/sqlbuild-analysis/src/job_settings/_helpers/build_job_settings.rs @@ -4,8 +4,8 @@ use std::collections::BTreeMap; use serde_json::Value as Json; -use crate::job_settings::constants::BUILD_CONFLICT_HELP; use crate::job_settings::errors::JobSettingConflict; +use crate::job_settings::main::conflict_help::conflict_help; use crate::job_settings::main::setting_scope::setting_scope; use crate::job_settings::models::{DeclaredRun, SettingScope}; @@ -52,7 +52,7 @@ pub(crate) fn build_job_settings( shown(seen), shown(value) ), - help: BUILD_CONFLICT_HELP, + help: conflict_help(key), }); } Some(_) => {} diff --git a/crates/sqlbuild-analysis/src/job_settings/_helpers/node_values.rs b/crates/sqlbuild-analysis/src/job_settings/_helpers/node_values.rs index 4c0d8543b6..db2f2befd8 100644 --- a/crates/sqlbuild-analysis/src/job_settings/_helpers/node_values.rs +++ b/crates/sqlbuild-analysis/src/job_settings/_helpers/node_values.rs @@ -6,10 +6,9 @@ use serde_json::Value as Json; use crate::assembly::project::types::ObjectKey; use crate::definition_changes::constants::PYTHON_NODE_KINDS; -use crate::job_settings::constants::{ - JOB_SETTINGS, NODE_CONFLICT_HELP, RETRY_CONFLICT_HELP, RETRY_SETTING, -}; +use crate::job_settings::constants::{JOB_SETTINGS, RETRY_SETTING}; use crate::job_settings::errors::JobSettingConflict; +use crate::job_settings::main::conflict_help::conflict_help; use crate::job_settings::models::{DeclaredRun, JobRetry, SettingScope}; /// One node's settings, and its retry, each from whichever of its jobs sets it. @@ -27,11 +26,7 @@ fn conflict( ({} and {})", key.0, key.1, values.0, values.1 ), - help: if setting == RETRY_SETTING { - RETRY_CONFLICT_HELP - } else { - NODE_CONFLICT_HELP - }, + help: conflict_help(setting), } } diff --git a/crates/sqlbuild-analysis/src/job_settings/constants.rs b/crates/sqlbuild-analysis/src/job_settings/constants.rs index 03eb5af982..8a108a07fa 100644 --- a/crates/sqlbuild-analysis/src/job_settings/constants.rs +++ b/crates/sqlbuild-analysis/src/job_settings/constants.rs @@ -1,17 +1,11 @@ //! Every build option as a job setting, and the wording of job setting conflicts. use crate::job_settings::models::GroupAgreement::{EachBuild, OneBuild}; -use crate::job_settings::models::JobSettingSpec; use crate::job_settings::models::SettingScope::{Build, JobLimit, Node, RunRequest}; +use crate::job_settings::models::{CommandLineValues, JobSettingSpec, NodeConsumer}; /// The warehouse the job's nodes run on. pub const WAREHOUSE_SETTING: &str = "warehouse"; -/// The help of two named jobs giving one node different values of a node-level setting. -pub const NODE_CONFLICT_HELP: &str = "make the jobs agree, set it in only one of them, override it \ - with the command-line flag, or build with --no-job-settings"; -/// The help of named jobs disagreeing on a build-wide setting, where unset differs from set. -pub const BUILD_CONFLICT_HELP: &str = - "make the jobs agree, override it with the command-line flag, or build with --no-job-settings"; /// The help of two named jobs giving one node different `[retry]` tables, which no flag overrides. pub const RETRY_CONFLICT_HELP: &str = "make the jobs agree, set [retry] in only one of them, or build with --no-job-settings"; @@ -27,6 +21,8 @@ pub const JOB_SETTINGS: [JobSettingSpec; 52] = [ scope: RunRequest, python_nodes: false, whole_run: false, + command_line: CommandLineValues::Any, + consumer: None, group: OneBuild, }, JobSettingSpec { @@ -35,6 +31,8 @@ pub const JOB_SETTINGS: [JobSettingSpec; 52] = [ scope: Build, python_nodes: false, whole_run: false, + command_line: CommandLineValues::OnlyTrue, + consumer: None, group: OneBuild, }, JobSettingSpec { @@ -43,6 +41,8 @@ pub const JOB_SETTINGS: [JobSettingSpec; 52] = [ scope: Build, python_nodes: false, whole_run: false, + command_line: CommandLineValues::OnlyTrue, + consumer: None, group: OneBuild, }, JobSettingSpec { @@ -51,6 +51,8 @@ pub const JOB_SETTINGS: [JobSettingSpec; 52] = [ scope: Build, python_nodes: false, whole_run: false, + command_line: CommandLineValues::Any, + consumer: None, group: OneBuild, }, JobSettingSpec { @@ -59,6 +61,8 @@ pub const JOB_SETTINGS: [JobSettingSpec; 52] = [ scope: Build, python_nodes: false, whole_run: false, + command_line: CommandLineValues::Any, + consumer: None, group: OneBuild, }, JobSettingSpec { @@ -67,6 +71,8 @@ pub const JOB_SETTINGS: [JobSettingSpec; 52] = [ scope: Build, python_nodes: false, whole_run: false, + command_line: CommandLineValues::Any, + consumer: None, group: OneBuild, }, JobSettingSpec { @@ -75,6 +81,8 @@ pub const JOB_SETTINGS: [JobSettingSpec; 52] = [ scope: Build, python_nodes: false, whole_run: false, + command_line: CommandLineValues::Any, + consumer: None, group: OneBuild, }, JobSettingSpec { @@ -83,6 +91,8 @@ pub const JOB_SETTINGS: [JobSettingSpec; 52] = [ scope: RunRequest, python_nodes: false, whole_run: false, + command_line: CommandLineValues::Any, + consumer: None, group: OneBuild, }, JobSettingSpec { @@ -91,6 +101,8 @@ pub const JOB_SETTINGS: [JobSettingSpec; 52] = [ scope: RunRequest, python_nodes: false, whole_run: false, + command_line: CommandLineValues::OnlyTrue, + consumer: None, group: OneBuild, }, JobSettingSpec { @@ -99,6 +111,8 @@ pub const JOB_SETTINGS: [JobSettingSpec; 52] = [ scope: Build, python_nodes: false, whole_run: false, + command_line: CommandLineValues::OnlyTrue, + consumer: None, group: OneBuild, }, JobSettingSpec { @@ -107,6 +121,8 @@ pub const JOB_SETTINGS: [JobSettingSpec; 52] = [ scope: Build, python_nodes: false, whole_run: false, + command_line: CommandLineValues::OnlyFalse, + consumer: None, group: OneBuild, }, JobSettingSpec { @@ -115,6 +131,8 @@ pub const JOB_SETTINGS: [JobSettingSpec; 52] = [ scope: Node, python_nodes: false, whole_run: false, + command_line: CommandLineValues::OnlyFalse, + consumer: Some(NodeConsumer::TestEntries), group: OneBuild, }, JobSettingSpec { @@ -123,6 +141,8 @@ pub const JOB_SETTINGS: [JobSettingSpec; 52] = [ scope: Build, python_nodes: false, whole_run: false, + command_line: CommandLineValues::OnlyFalse, + consumer: None, group: OneBuild, }, JobSettingSpec { @@ -131,6 +151,8 @@ pub const JOB_SETTINGS: [JobSettingSpec; 52] = [ scope: Node, python_nodes: false, whole_run: false, + command_line: CommandLineValues::Any, + consumer: Some(NodeConsumer::AuditEntries), group: OneBuild, }, JobSettingSpec { @@ -139,6 +161,8 @@ pub const JOB_SETTINGS: [JobSettingSpec; 52] = [ scope: RunRequest, python_nodes: false, whole_run: false, + command_line: CommandLineValues::OnlyTrue, + consumer: None, group: OneBuild, }, JobSettingSpec { @@ -147,6 +171,8 @@ pub const JOB_SETTINGS: [JobSettingSpec; 52] = [ scope: RunRequest, python_nodes: false, whole_run: false, + command_line: CommandLineValues::Any, + consumer: None, group: OneBuild, }, JobSettingSpec { @@ -155,6 +181,8 @@ pub const JOB_SETTINGS: [JobSettingSpec; 52] = [ scope: RunRequest, python_nodes: false, whole_run: false, + command_line: CommandLineValues::Any, + consumer: None, group: OneBuild, }, JobSettingSpec { @@ -163,6 +191,8 @@ pub const JOB_SETTINGS: [JobSettingSpec; 52] = [ scope: Build, python_nodes: false, whole_run: false, + command_line: CommandLineValues::Any, + consumer: None, group: OneBuild, }, JobSettingSpec { @@ -171,6 +201,8 @@ pub const JOB_SETTINGS: [JobSettingSpec; 52] = [ scope: Build, python_nodes: false, whole_run: false, + command_line: CommandLineValues::Any, + consumer: None, group: OneBuild, }, JobSettingSpec { @@ -179,6 +211,8 @@ pub const JOB_SETTINGS: [JobSettingSpec; 52] = [ scope: Build, python_nodes: false, whole_run: false, + command_line: CommandLineValues::Any, + consumer: None, group: OneBuild, }, JobSettingSpec { @@ -187,6 +221,8 @@ pub const JOB_SETTINGS: [JobSettingSpec; 52] = [ scope: Build, python_nodes: false, whole_run: false, + command_line: CommandLineValues::Any, + consumer: None, group: OneBuild, }, JobSettingSpec { @@ -195,6 +231,8 @@ pub const JOB_SETTINGS: [JobSettingSpec; 52] = [ scope: Build, python_nodes: false, whole_run: false, + command_line: CommandLineValues::Any, + consumer: None, group: OneBuild, }, JobSettingSpec { @@ -203,6 +241,8 @@ pub const JOB_SETTINGS: [JobSettingSpec; 52] = [ scope: Build, python_nodes: false, whole_run: false, + command_line: CommandLineValues::Any, + consumer: None, group: OneBuild, }, JobSettingSpec { @@ -211,6 +251,8 @@ pub const JOB_SETTINGS: [JobSettingSpec; 52] = [ scope: Build, python_nodes: false, whole_run: false, + command_line: CommandLineValues::Any, + consumer: None, group: OneBuild, }, JobSettingSpec { @@ -219,6 +261,8 @@ pub const JOB_SETTINGS: [JobSettingSpec; 52] = [ scope: Build, python_nodes: false, whole_run: false, + command_line: CommandLineValues::Any, + consumer: None, group: OneBuild, }, JobSettingSpec { @@ -227,6 +271,8 @@ pub const JOB_SETTINGS: [JobSettingSpec; 52] = [ scope: Build, python_nodes: false, whole_run: false, + command_line: CommandLineValues::OnlyTrue, + consumer: None, group: OneBuild, }, JobSettingSpec { @@ -235,6 +281,8 @@ pub const JOB_SETTINGS: [JobSettingSpec; 52] = [ scope: Build, python_nodes: false, whole_run: false, + command_line: CommandLineValues::OnlyTrue, + consumer: None, group: OneBuild, }, JobSettingSpec { @@ -243,6 +291,8 @@ pub const JOB_SETTINGS: [JobSettingSpec; 52] = [ scope: Node, python_nodes: false, whole_run: false, + command_line: CommandLineValues::OnlyTrue, + consumer: Some(NodeConsumer::RefreshOverrides), group: OneBuild, }, JobSettingSpec { @@ -251,6 +301,8 @@ pub const JOB_SETTINGS: [JobSettingSpec; 52] = [ scope: RunRequest, python_nodes: false, whole_run: false, + command_line: CommandLineValues::OnlyFalse, + consumer: None, group: OneBuild, }, JobSettingSpec { @@ -259,6 +311,8 @@ pub const JOB_SETTINGS: [JobSettingSpec; 52] = [ scope: Build, python_nodes: false, whole_run: false, + command_line: CommandLineValues::Any, + consumer: None, group: EachBuild, }, JobSettingSpec { @@ -267,6 +321,8 @@ pub const JOB_SETTINGS: [JobSettingSpec; 52] = [ scope: Build, python_nodes: false, whole_run: false, + command_line: CommandLineValues::Any, + consumer: None, group: OneBuild, }, JobSettingSpec { @@ -275,6 +331,8 @@ pub const JOB_SETTINGS: [JobSettingSpec; 52] = [ scope: Build, python_nodes: false, whole_run: false, + command_line: CommandLineValues::OnlyTrue, + consumer: None, group: OneBuild, }, JobSettingSpec { @@ -283,6 +341,8 @@ pub const JOB_SETTINGS: [JobSettingSpec; 52] = [ scope: Build, python_nodes: false, whole_run: false, + command_line: CommandLineValues::OnlyTrue, + consumer: None, group: OneBuild, }, JobSettingSpec { @@ -291,6 +351,8 @@ pub const JOB_SETTINGS: [JobSettingSpec; 52] = [ scope: Build, python_nodes: false, whole_run: false, + command_line: CommandLineValues::OnlyTrue, + consumer: None, group: OneBuild, }, JobSettingSpec { @@ -299,6 +361,8 @@ pub const JOB_SETTINGS: [JobSettingSpec; 52] = [ scope: Build, python_nodes: false, whole_run: false, + command_line: CommandLineValues::OnlyTrue, + consumer: None, group: OneBuild, }, JobSettingSpec { @@ -307,6 +371,8 @@ pub const JOB_SETTINGS: [JobSettingSpec; 52] = [ scope: Build, python_nodes: false, whole_run: false, + command_line: CommandLineValues::OnlyTrue, + consumer: None, group: OneBuild, }, JobSettingSpec { @@ -315,6 +381,8 @@ pub const JOB_SETTINGS: [JobSettingSpec; 52] = [ scope: JobLimit, python_nodes: false, whole_run: false, + command_line: CommandLineValues::Any, + consumer: None, group: OneBuild, }, JobSettingSpec { @@ -323,6 +391,8 @@ pub const JOB_SETTINGS: [JobSettingSpec; 52] = [ scope: Build, python_nodes: false, whole_run: false, + command_line: CommandLineValues::Any, + consumer: None, group: OneBuild, }, JobSettingSpec { @@ -331,6 +401,8 @@ pub const JOB_SETTINGS: [JobSettingSpec; 52] = [ scope: Build, python_nodes: false, whole_run: false, + command_line: CommandLineValues::OnlyTrue, + consumer: None, group: OneBuild, }, JobSettingSpec { @@ -339,6 +411,8 @@ pub const JOB_SETTINGS: [JobSettingSpec; 52] = [ scope: RunRequest, python_nodes: false, whole_run: false, + command_line: CommandLineValues::Any, + consumer: None, group: OneBuild, }, JobSettingSpec { @@ -347,6 +421,8 @@ pub const JOB_SETTINGS: [JobSettingSpec; 52] = [ scope: RunRequest, python_nodes: false, whole_run: false, + command_line: CommandLineValues::Any, + consumer: None, group: OneBuild, }, JobSettingSpec { @@ -355,6 +431,8 @@ pub const JOB_SETTINGS: [JobSettingSpec; 52] = [ scope: RunRequest, python_nodes: false, whole_run: false, + command_line: CommandLineValues::Any, + consumer: None, group: OneBuild, }, JobSettingSpec { @@ -363,6 +441,8 @@ pub const JOB_SETTINGS: [JobSettingSpec; 52] = [ scope: RunRequest, python_nodes: false, whole_run: false, + command_line: CommandLineValues::Any, + consumer: None, group: OneBuild, }, JobSettingSpec { @@ -371,6 +451,8 @@ pub const JOB_SETTINGS: [JobSettingSpec; 52] = [ scope: Build, python_nodes: false, whole_run: false, + command_line: CommandLineValues::Any, + consumer: None, group: EachBuild, }, JobSettingSpec { @@ -379,6 +461,8 @@ pub const JOB_SETTINGS: [JobSettingSpec; 52] = [ scope: RunRequest, python_nodes: false, whole_run: false, + command_line: CommandLineValues::Any, + consumer: None, group: OneBuild, }, JobSettingSpec { @@ -387,6 +471,8 @@ pub const JOB_SETTINGS: [JobSettingSpec; 52] = [ scope: RunRequest, python_nodes: false, whole_run: false, + command_line: CommandLineValues::Any, + consumer: None, group: OneBuild, }, JobSettingSpec { @@ -395,6 +481,8 @@ pub const JOB_SETTINGS: [JobSettingSpec; 52] = [ scope: RunRequest, python_nodes: false, whole_run: false, + command_line: CommandLineValues::Any, + consumer: None, group: OneBuild, }, JobSettingSpec { @@ -403,6 +491,8 @@ pub const JOB_SETTINGS: [JobSettingSpec; 52] = [ scope: RunRequest, python_nodes: false, whole_run: false, + command_line: CommandLineValues::Any, + consumer: None, group: OneBuild, }, JobSettingSpec { @@ -411,6 +501,8 @@ pub const JOB_SETTINGS: [JobSettingSpec; 52] = [ scope: Node, python_nodes: true, whole_run: true, + command_line: CommandLineValues::Any, + consumer: Some(NodeConsumer::NodeWarehouses), group: OneBuild, }, JobSettingSpec { @@ -419,6 +511,8 @@ pub const JOB_SETTINGS: [JobSettingSpec; 52] = [ scope: RunRequest, python_nodes: false, whole_run: false, + command_line: CommandLineValues::Any, + consumer: None, group: OneBuild, }, JobSettingSpec { @@ -427,6 +521,8 @@ pub const JOB_SETTINGS: [JobSettingSpec; 52] = [ scope: Build, python_nodes: false, whole_run: false, + command_line: CommandLineValues::OnlyTrue, + consumer: None, group: OneBuild, }, JobSettingSpec { @@ -435,6 +531,8 @@ pub const JOB_SETTINGS: [JobSettingSpec; 52] = [ scope: Build, python_nodes: false, whole_run: false, + command_line: CommandLineValues::OnlyTrue, + consumer: None, group: OneBuild, }, ]; diff --git a/crates/sqlbuild-analysis/src/job_settings/errors.rs b/crates/sqlbuild-analysis/src/job_settings/errors.rs index d9b5cb5891..6c131cbc82 100644 --- a/crates/sqlbuild-analysis/src/job_settings/errors.rs +++ b/crates/sqlbuild-analysis/src/job_settings/errors.rs @@ -6,7 +6,7 @@ use crate::graph::errors::SelectorError; #[derive(Clone, Debug, PartialEq, Eq)] pub struct JobSettingConflict { pub message: String, - pub help: &'static str, + pub help: String, } /// A selector token that names jobs, and why it failed to select their nodes. diff --git a/crates/sqlbuild-analysis/src/job_settings/main/conflict_help.rs b/crates/sqlbuild-analysis/src/job_settings/main/conflict_help.rs new file mode 100644 index 0000000000..4f94ec4eb7 --- /dev/null +++ b/crates/sqlbuild-analysis/src/job_settings/main/conflict_help.rs @@ -0,0 +1,29 @@ +//! The help a job setting conflict gives, from the setting's row in the table. + +use crate::job_settings::constants::{RETRY_CONFLICT_HELP, RETRY_SETTING}; +use crate::job_settings::main::job_setting_spec::job_setting_spec; +use crate::job_settings::models::{CommandLineValues, SettingScope}; + +/// How to settle named jobs disagreeing on `setting`; a flag only for the value it can give. +#[must_use] +pub fn conflict_help(setting: &str) -> String { + let Some(spec) = job_setting_spec(setting) else { + return if setting == RETRY_SETTING { + RETRY_CONFLICT_HELP.to_owned() + } else { + "make the jobs agree, or build with --no-job-settings".to_owned() + }; + }; + let only_one: &str = if spec.scope == SettingScope::Node { + "set it in only one of them, " + } else { + "" + }; + let options: String = spec.options.join(" or "); + let flag: String = match spec.command_line { + CommandLineValues::Any => format!("override it with {options}, "), + CommandLineValues::OnlyTrue => format!("pass {options} to make it true for all, "), + CommandLineValues::OnlyFalse => format!("pass {options} to make it false for all, "), + }; + format!("make the jobs agree, {only_one}{flag}or build with --no-job-settings") +} diff --git a/crates/sqlbuild-analysis/src/job_settings/main/consumed_setting.rs b/crates/sqlbuild-analysis/src/job_settings/main/consumed_setting.rs new file mode 100644 index 0000000000..8665eae167 --- /dev/null +++ b/crates/sqlbuild-analysis/src/job_settings/main/consumed_setting.rs @@ -0,0 +1,13 @@ +//! The node setting one plan consumer applies. + +use crate::job_settings::constants::JOB_SETTINGS; +use crate::job_settings::models::NodeConsumer; + +/// The node setting `consumer` applies, from the table; `None` if no row names it. +#[must_use] +pub fn consumed_setting(consumer: NodeConsumer) -> Option<&'static str> { + JOB_SETTINGS + .iter() + .find(|spec| spec.consumer == Some(consumer)) + .map(|spec| spec.name) +} diff --git a/crates/sqlbuild-analysis/src/job_settings/main/mod.rs b/crates/sqlbuild-analysis/src/job_settings/main/mod.rs index 87edd17d2b..f98a28f2a8 100644 --- a/crates/sqlbuild-analysis/src/job_settings/main/mod.rs +++ b/crates/sqlbuild-analysis/src/job_settings/main/mod.rs @@ -1,3 +1,5 @@ +pub mod conflict_help; +pub mod consumed_setting; pub mod declared_runs; pub mod group_setting_mismatch; pub mod job_setting_spec; diff --git a/crates/sqlbuild-analysis/src/job_settings/models.rs b/crates/sqlbuild-analysis/src/job_settings/models.rs index bc984d601f..859f11e232 100644 --- a/crates/sqlbuild-analysis/src/job_settings/models.rs +++ b/crates/sqlbuild-analysis/src/job_settings/models.rs @@ -59,9 +59,37 @@ pub struct JobSettingSpec { pub python_nodes: bool, /// Whether one value every named job shares becomes the whole build's, setup included. pub whole_run: bool, + /// The values its command-line options can give it. + pub command_line: CommandLineValues, + /// For a node setting, what applies it to the plan. + pub consumer: Option, pub group: GroupAgreement, } +/// The values a setting's command-line options can give it. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum CommandLineValues { + /// Any value it takes, or both `true` and `false`. + Any, + /// Only `true`, as a flag such as `--full-refresh`. + OnlyTrue, + /// Only `false`, as a flag such as `--no-tests`. + OnlyFalse, +} + +/// What applies a node setting to the planned build. +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub enum NodeConsumer { + /// The planner's `full_refresh_models` and `full_refresh_functions` overrides. + RefreshOverrides, + /// The plan's SQL test entries, dropping the tests a job turns off. + TestEntries, + /// Each audit entry's `ignore_failure`. + AuditEntries, + /// The warehouse each node runs on. + NodeWarehouses, +} + /// What the jobs a selection names give each of its nodes. #[derive(Clone, Debug, Default, PartialEq)] pub struct NodeJobSettings { diff --git a/crates/sqlbuild-analysis/src/job_settings/tests/helpers.rs b/crates/sqlbuild-analysis/src/job_settings/tests/helpers.rs index 62b1f67668..bb76ab991c 100644 --- a/crates/sqlbuild-analysis/src/job_settings/tests/helpers.rs +++ b/crates/sqlbuild-analysis/src/job_settings/tests/helpers.rs @@ -9,9 +9,12 @@ use crate::graph::main::resolve_selector_tokens::resolve_selector_tokens; use crate::graph::models::{GraphResource, JobSelection, ProjectGraph}; use crate::job_settings::_helpers::node_job_settings::node_job_settings; use crate::job_settings::_helpers::node_values::node_values; +use crate::job_settings::constants::JOB_SETTINGS; use crate::job_settings::errors::NodeJobSettingsError; use crate::job_settings::main::resolve_job_settings::resolve_job_settings; -use crate::job_settings::models::{DeclaredRun, JobRetry, NodeJobSettings}; +use crate::job_settings::models::{ + DeclaredRun, JobRetry, JobSettingSpec, NodeConsumer, NodeJobSettings, SettingScope, +}; fn key(kind: &str, name: &str) -> ObjectKey { (kind.to_owned(), name.to_owned()) @@ -234,3 +237,29 @@ pub(super) fn whole_run_value(name: &str, values: [Option; 3]) -> Option (Vec<&'static str>, Vec<&'static str>, usize) { + let node = |spec: &&JobSettingSpec| spec.scope == SettingScope::Node; + let unconsumed: Vec<&str> = JOB_SETTINGS + .iter() + .filter(node) + .filter(|spec| spec.consumer.is_none()) + .map(|spec| spec.name) + .collect(); + let consumed_others: Vec<&str> = JOB_SETTINGS + .iter() + .filter(|spec| spec.scope != SettingScope::Node && spec.consumer.is_some()) + .map(|spec| spec.name) + .collect(); + let consumers: Vec = JOB_SETTINGS + .iter() + .filter_map(|spec| spec.consumer) + .collect(); + let distinct: BTreeSet = consumers.iter().copied().collect(); + ( + unconsumed, + consumed_others, + consumers.len() - distinct.len(), + ) +} diff --git a/crates/sqlbuild-analysis/src/job_settings/tests/test_job_setting_table.rs b/crates/sqlbuild-analysis/src/job_settings/tests/test_job_setting_table.rs index 3961a85d64..e3c339ad9c 100644 --- a/crates/sqlbuild-analysis/src/job_settings/tests/test_job_setting_table.rs +++ b/crates/sqlbuild-analysis/src/job_settings/tests/test_job_setting_table.rs @@ -3,11 +3,14 @@ use std::collections::BTreeSet; use serde_json::json; use crate::job_settings::constants::JOB_SETTINGS; +use crate::job_settings::main::conflict_help::conflict_help; use crate::job_settings::models::SettingScope; use crate::job_settings::tests::helpers::{ - conflicts_on_a_node, resolves_build_wide, whole_run_value, + conflicts_on_a_node, consumer_faults, resolves_build_wide, whole_run_value, +}; +use crate::job_settings::tests::test_types::{ + ConflictHelpTestCase, JobSettingScopeTestCase, NodeConsumerTestCase, WholeRunTestCase, }; -use crate::job_settings::tests::test_types::{JobSettingScopeTestCase, WholeRunTestCase}; #[test] fn given_every_job_setting_when_named_jobs_merge_it_then_its_scope_decides_and_the_command_line_wins() @@ -110,3 +113,77 @@ fn given_a_whole_run_setting_when_named_jobs_share_its_value_then_it_is_the_buil } } } + +#[test] +fn given_two_jobs_disagreeing_on_a_setting_when_reporting_then_the_help_offers_only_flags_that_settle_it() + { + let test_cases = [ + ConflictHelpTestCase { + description: "--no-tests can only turn tests off", + setting: "run_tests", + expected_help: "make the jobs agree, set it in only one of them, pass --no-tests to make \ + it false for all, or build with --no-job-settings", + }, + ConflictHelpTestCase { + description: "--full-refresh can only turn full refresh on", + setting: "full_refresh", + expected_help: "make the jobs agree, set it in only one of them, pass --full-refresh to \ + make it true for all, or build with --no-job-settings", + }, + ConflictHelpTestCase { + description: "a build-wide flag cannot be left unset by one job", + setting: "fail_fast", + expected_help: "make the jobs agree, pass --fail-fast to make it true for all, or build \ + with --no-job-settings", + }, + ConflictHelpTestCase { + description: "a flag pair gives either value", + setting: "ignore_audit_failures", + expected_help: "make the jobs agree, set it in only one of them, override it with \ + --ignore-audit-failures or --no-ignore-audit-failures, or build with \ + --no-job-settings", + }, + ConflictHelpTestCase { + description: "an option taking a value gives any", + setting: "vars", + expected_help: "make the jobs agree, override it with --vars, or build with \ + --no-job-settings", + }, + ConflictHelpTestCase { + description: "no flag gives a retry table", + setting: "[retry]", + expected_help: "make the jobs agree, set [retry] in only one of them, or build with \ + --no-job-settings", + }, + ]; + for test_case in test_cases { + assert_eq!( + conflict_help(test_case.setting), + test_case.expected_help, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_the_job_setting_table_when_checked_then_each_node_setting_names_its_own_consumer() { + let test_cases = [NodeConsumerTestCase { + description: "a node setting without a consumer would never reach the plan", + expected_unconsumed_node_settings: Vec::new(), + expected_consumed_other_settings: Vec::new(), + expected_shared_consumers: 0, + }]; + for test_case in test_cases { + assert_eq!( + consumer_faults(), + ( + test_case.expected_unconsumed_node_settings, + test_case.expected_consumed_other_settings, + test_case.expected_shared_consumers, + ), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/job_settings/tests/test_types.rs b/crates/sqlbuild-analysis/src/job_settings/tests/test_types.rs index 28bfe5befb..be39b98642 100644 --- a/crates/sqlbuild-analysis/src/job_settings/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/job_settings/tests/test_types.rs @@ -60,3 +60,18 @@ pub(super) struct WholeRunTestCase { pub(super) values: [Option; 3], pub(super) expected_value: Option, } + +/// A setting two named jobs disagree on, and the help the conflict gives. +pub(super) struct ConflictHelpTestCase { + pub(super) description: &'static str, + pub(super) setting: &'static str, + pub(super) expected_help: &'static str, +} + +/// The table's node settings, which must each name one distinct consumer, and the others none. +pub(super) struct NodeConsumerTestCase { + pub(super) description: &'static str, + pub(super) expected_unconsumed_node_settings: Vec<&'static str>, + pub(super) expected_consumed_other_settings: Vec<&'static str>, + pub(super) expected_shared_consumers: usize, +} diff --git a/crates/sqlbuild-cli/src/argv/tests/helpers.rs b/crates/sqlbuild-cli/src/argv/tests/helpers.rs index 54043f3e11..8e4abc656f 100644 --- a/crates/sqlbuild-cli/src/argv/tests/helpers.rs +++ b/crates/sqlbuild-cli/src/argv/tests/helpers.rs @@ -2,10 +2,11 @@ use std::collections::{BTreeMap, BTreeSet}; use serde_json::{Value as Json, json}; use sqlbuild_analysis::job_settings::constants::JOB_SETTINGS; -use sqlbuild_analysis::job_settings::models::SettingScope; +use sqlbuild_analysis::job_settings::models::{CommandLineValues, SettingScope}; use crate::argv::_helpers::table::command_parser; use crate::argv::main::command_settings_arguments::command_settings_arguments; +use crate::argv::main::job_run_arguments::job_run_arguments; use crate::argv::main::parse_cli_arguments::parse_cli_arguments; use crate::argv::models::{ActionKind, ActionSpec, Nargs}; use crate::argv::models::{ArgvOutcome, ArgvSettings, Value}; @@ -128,3 +129,56 @@ fn rendered_option(name: &str) -> Option { .and_then(|argument| argument.split('=').next()) .map(str::to_owned) } + +/// The job settings whose `command_line` disagrees with what the build parser's options give. +pub(super) fn command_line_mismatches() -> Vec<&'static str> { + let actions: Vec<&ActionSpec> = command_parser("build") + .map(|parser| parser.actions.iter().collect()) + .unwrap_or_default(); + JOB_SETTINGS + .iter() + .filter(|spec| { + let kinds: Vec = actions + .iter() + .filter(|action| action.dest.as_deref() == Some(spec.name)) + .map(|action| action.kind) + .collect(); + let only = + |kind: ActionKind| !kinds.is_empty() && kinds.iter().all(|seen| *seen == kind); + let expected: CommandLineValues = [ + (only(ActionKind::StoreTrue), CommandLineValues::OnlyTrue), + (only(ActionKind::StoreFalse), CommandLineValues::OnlyFalse), + ] + .into_iter() + .find_map(|(applies, values)| applies.then_some(values)) + .unwrap_or(CommandLineValues::Any); + spec.command_line != expected + }) + .map(|spec| spec.name) + .collect() +} + +/// The node settings that are not whole-run but still reach a job run's build arguments. +pub(super) fn job_run_leaks() -> Vec<&'static str> { + let parser = command_parser("build").ok(); + JOB_SETTINGS + .iter() + .filter(|spec| spec.scope == SettingScope::Node && !spec.whole_run) + .filter(|spec| { + let value: Json = parser + .and_then(|parser| { + parser + .actions + .iter() + .find(|action| action.dest.as_deref() == Some(spec.name)) + }) + .map(sample) + .unwrap_or_default(); + let settings = BTreeMap::from([(spec.name.to_owned(), value)]); + job_run_arguments("orders_job", &settings).map_or(true, |arguments| { + arguments != ["--select", "job:orders_job"] + }) + }) + .map(|spec| spec.name) + .collect() +} diff --git a/crates/sqlbuild-cli/src/argv/tests/test_job_setting_table.rs b/crates/sqlbuild-cli/src/argv/tests/test_job_setting_table.rs index 1d09a92bfe..14e78ac48f 100644 --- a/crates/sqlbuild-cli/src/argv/tests/test_job_setting_table.rs +++ b/crates/sqlbuild-cli/src/argv/tests/test_job_setting_table.rs @@ -1,7 +1,11 @@ use sqlbuild_analysis::job_settings::models::SettingScope; -use crate::argv::tests::helpers::{parser_options, table_options, unrendered_settings}; -use crate::argv::tests::test_types::{JobSettingTableTestCase, RenderedSettingTestCase}; +use crate::argv::tests::helpers::{ + command_line_mismatches, job_run_leaks, parser_options, table_options, unrendered_settings, +}; +use crate::argv::tests::test_types::{ + JobSettingFaultsTestCase, JobSettingTableTestCase, RenderedSettingTestCase, +}; #[test] fn given_the_build_parser_when_compared_with_the_job_setting_table_then_every_option_has_one_row() { @@ -45,3 +49,27 @@ fn given_a_job_setting_the_build_takes_as_a_flag_when_rendered_then_it_is_one_of ); } } + +#[test] +fn given_the_job_setting_table_when_checked_against_the_parser_and_job_runs_then_nothing_leaks() { + let test_cases = [ + JobSettingFaultsTestCase { + description: "each row's command_line is what its options can give", + faults: command_line_mismatches, + expected_faults: Vec::new(), + }, + JobSettingFaultsTestCase { + description: "a node setting that is not whole-run never travels as a job run flag", + faults: job_run_leaks, + expected_faults: Vec::new(), + }, + ]; + for test_case in test_cases { + assert_eq!( + (test_case.faults)(), + test_case.expected_faults, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-cli/src/argv/tests/test_types.rs b/crates/sqlbuild-cli/src/argv/tests/test_types.rs index 18b69f7111..5ec630d3a1 100644 --- a/crates/sqlbuild-cli/src/argv/tests/test_types.rs +++ b/crates/sqlbuild-cli/src/argv/tests/test_types.rs @@ -110,3 +110,10 @@ pub(super) struct RenderedSettingTestCase { pub(super) whole_run: bool, pub(super) expected_unrendered: Vec<&'static str>, } + +/// Settings whose table row disagrees with the parser, or leaks into a job run's arguments. +pub(super) struct JobSettingFaultsTestCase { + pub(super) description: &'static str, + pub(super) faults: fn() -> Vec<&'static str>, + pub(super) expected_faults: Vec<&'static str>, +} diff --git a/crates/sqlbuild-python-compile/src/compile_driver/_helpers/jobs/job_node_settings.rs b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/jobs/job_node_settings.rs index e2e24e74d2..972fb78498 100644 --- a/crates/sqlbuild-python-compile/src/compile_driver/_helpers/jobs/job_node_settings.rs +++ b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/jobs/job_node_settings.rs @@ -10,10 +10,14 @@ use serde_json::Value; use sqlbuild_analysis::assembly::project::types::ObjectKey; use sqlbuild_analysis::definition_changes::constants::PYTHON_NODE_KINDS; use sqlbuild_analysis::graph::main::selectors_need_jobs::selectors_need_jobs; -use sqlbuild_analysis::job_settings::constants::{NODE_CONFLICT_HELP, WAREHOUSE_SETTING}; +use sqlbuild_analysis::job_settings::constants::WAREHOUSE_SETTING; use sqlbuild_analysis::job_settings::errors::{JobSettingConflict, NodeJobSettingsError}; +use sqlbuild_analysis::job_settings::main::conflict_help::conflict_help; +use sqlbuild_analysis::job_settings::main::consumed_setting::consumed_setting; use sqlbuild_analysis::job_settings::main::resolve_job_settings::resolve_job_settings; -use sqlbuild_analysis::job_settings::models::{DeclaredRun, NodeJobSettings, ResolvedJobSettings}; +use sqlbuild_analysis::job_settings::models::{ + DeclaredRun, NodeConsumer, NodeJobSettings, ResolvedJobSettings, +}; use sqlbuild_config::command_connection::errors::CommandConnectionError; use sqlbuild_config::command_connection::main::check_job_warehouse::check_job_warehouse; use sqlbuild_config::command_connection::main::unsupported_job_warehouse::unsupported_job_warehouse; @@ -49,12 +53,10 @@ use sqlbuild_python_project::resource_views::main::audit_reads::scope_deps::scop const MODEL_KIND: &str = "model"; const FUNCTION_KINDS: [&str; 2] = ["udf", "table_fn"]; -/// The node setting the planner's `full_refresh_models` and `full_refresh_functions` apply. -pub(crate) const FULL_REFRESH: &str = "full_refresh"; -/// The node setting that drops a job's SQL tests from the plan. -pub(crate) const RUN_TESTS: &str = "run_tests"; -/// The node setting each audit entry's `ignore_failure` takes. -pub(crate) const IGNORE_AUDIT_FAILURES: &str = "ignore_audit_failures"; +/// The node setting `consumer` applies, as the job setting table names it. +pub(crate) fn node_setting(consumer: NodeConsumer) -> &'static str { + consumed_setting(consumer).unwrap_or_default() +} /// The `CliUserError` of a job warehouse that cannot be used. fn cli_error(py: Python<'_>, error: &CommandConnectionError) -> PyErr { @@ -81,7 +83,7 @@ fn job_warehouses( if !settings .values .get(key) - .is_some_and(|values| values.contains_key(WAREHOUSE_SETTING)) + .is_some_and(|values| values.contains_key(node_setting(NodeConsumer::NodeWarehouses))) { continue; } @@ -90,7 +92,7 @@ fn job_warehouses( continue; } if let Some(DeclaredRun::Job { settings, .. }) = declared.get(job) - && let Some(value) = settings.get(WAREHOUSE_SETTING) + && let Some(value) = settings.get(node_setting(NodeConsumer::NodeWarehouses)) { let name: String = check_job_warehouse(job, value.as_str()) .map_err(|error| cli_error(py, &error))?; @@ -138,6 +140,18 @@ fn jobs_setting( jobs.into_iter().collect() } +/// Whether every read node ignores, by job value or else `build_ignores`; `None` without job values. +pub(crate) fn singular_audit_ignores( + job_values: impl Iterator>, + build_ignores: bool, +) -> Option { + let values: Vec> = job_values.collect(); + values + .iter() + .any(Option::is_some) + .then(|| values.iter().all(|value| value.unwrap_or(build_ignores))) +} + /// The nodes each audit attached to no node reads, by the audit's key. fn unattached_audit_reads( project: &Bound<'_, PyAny>, @@ -207,12 +221,16 @@ pub(crate) fn planned_job_settings( declared, .. }) => Ok(Some(PlannedJobSettings { - ignoring_jobs: jobs_setting(&settings, &declared, (IGNORE_AUDIT_FAILURES, None)), + ignoring_jobs: jobs_setting( + &settings, + &declared, + (node_setting(NodeConsumer::AuditEntries), None), + ), refreshing_jobs: jobs_setting( &settings, &declared, ( - FULL_REFRESH, + node_setting(NodeConsumer::RefreshOverrides), Some(&[MODEL_KIND, FUNCTION_KINDS[0], FUNCTION_KINDS[1]]), ), ), @@ -242,7 +260,8 @@ impl PlannedJobSettings { .iter() .filter(|(key, values)| { kinds.contains(&key.0.as_str()) - && values.get(FULL_REFRESH) == Some(&Value::Bool(true)) + && values.get(node_setting(NodeConsumer::RefreshOverrides)) + == Some(&Value::Bool(true)) }) .map(|(key, _)| &key.1) .collect(); @@ -317,7 +336,8 @@ impl PlannedJobSettings { if self.job_warehouses.is_empty() { return None; } - self.value(key, WAREHOUSE_SETTING).and_then(Value::as_str) + self.value(key, node_setting(NodeConsumer::NodeWarehouses)) + .and_then(Value::as_str) } /// Each SQL node's job warehouse, a test's only if all its nodes share it, and Python nodes'. @@ -362,7 +382,7 @@ impl PlannedJobSettings { pub(crate) fn apply_to_plan<'py>( &self, plan: &Bound<'py, PyAny>, - cli_ignore_audit_failures: &Bound<'py, PyAny>, + (cli_ignore_audit_failures, build_ignores): (&Bound<'py, PyAny>, bool), ) -> PyResult> { let py: Python<'py> = plan.py(); let mut skipped: BTreeSet = BTreeSet::new(); @@ -397,7 +417,7 @@ impl PlannedJobSettings { let mut changed = false; for entry in native_field(plan, "audit_entries")?.try_iter()? { let entry = entry?; - match self.audit_ignores(&entry)? { + match self.audit_ignores(&entry, build_ignores)? { Some(ignored) if native_field(&entry, "ignore_failure")?.is_truthy()? != ignored => { @@ -461,7 +481,7 @@ impl PlannedJobSettings { } let mut seen: Option<(&String, &Value)> = None; for key in &keys { - let Some(value) = self.value(key, RUN_TESTS) else { + let Some(value) = self.value(key, node_setting(NodeConsumer::TestEntries)) else { continue; }; let job: &String = self @@ -480,7 +500,7 @@ impl PlannedJobSettings { run_tests differently ({earlier} and {value})", test.1 ), - help: NODE_CONFLICT_HELP, + help: conflict_help(node_setting(NodeConsumer::TestEntries)), }, )); } @@ -491,32 +511,32 @@ impl PlannedJobSettings { Ok(seen.is_some_and(|(_, value)| value == &Value::Bool(false))) } - /// The audited node's `ignore_audit_failures`, or for a singular audit, every read node's. - fn audit_ignores(&self, entry: &Bound<'_, PyAny>) -> PyResult> { + /// The audited node's `ignore_audit_failures`, or for a singular audit, its read nodes' together. + fn audit_ignores( + &self, + entry: &Bound<'_, PyAny>, + build_ignores: bool, + ) -> PyResult> { let kind: Bound<'_, PyAny> = native_field(entry, "attached_target_kind")?; let name: Bound<'_, PyAny> = native_field(entry, "attached_target_name")?; if kind.is_none() || name.is_none() { let audit: ObjectKey = native_key(&native_field(entry, "key")?)?; - let mut ignored: Option = None; - for key in self + let reads: &[ObjectKey] = self .unattached_audit_reads .get(&audit) - .into_iter() - .flatten() - { - if let Some(value) = self - .value(key, IGNORE_AUDIT_FAILURES) - .and_then(Value::as_bool) - { - ignored = Some(ignored.unwrap_or(true) && value); - } - } - return Ok(ignored); + .map_or(&[], Vec::as_slice); + return Ok(singular_audit_ignores( + reads.iter().map(|key| { + self.value(key, node_setting(NodeConsumer::AuditEntries)) + .and_then(Value::as_bool) + }), + build_ignores, + )); } let kind: String = native_field(&kind, "value")?.extract()?; let key: ObjectKey = (kind, name.extract()?); Ok(self - .value(&key, IGNORE_AUDIT_FAILURES) + .value(&key, node_setting(NodeConsumer::AuditEntries)) .and_then(Value::as_bool)) } } diff --git a/crates/sqlbuild-python-compile/src/compile_driver/main/jobs/job_plan_fields.rs b/crates/sqlbuild-python-compile/src/compile_driver/main/jobs/job_plan_fields.rs index 5a46a7c457..4b4bc8805a 100644 --- a/crates/sqlbuild-python-compile/src/compile_driver/main/jobs/job_plan_fields.rs +++ b/crates/sqlbuild-python-compile/src/compile_driver/main/jobs/job_plan_fields.rs @@ -1,6 +1,6 @@ //! The plan and pipeline fields the named jobs' settings give a planned build. -use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use sqlbuild_python_base::records::main::fields::field::field as native_field; use crate::compile_driver::models::PlannedJobSettings; @@ -17,7 +17,10 @@ pub fn job_plan_fields<'py>( }; let plan_output: Bound<'py, PyAny> = job_settings.apply_to_plan( &plan_output, - &native_field(options, "cli_ignore_audit_failures")?, + ( + &native_field(options, "cli_ignore_audit_failures")?, + native_field(options, "ignore_audit_failures")?.is_truthy()?, + ), )?; let (limits, node_jobs) = job_settings.job_limits(&plan_output, &native_field(options, "concurrency")?)?; diff --git a/crates/sqlbuild-python-compile/src/compile_driver/tests.rs b/crates/sqlbuild-python-compile/src/compile_driver/tests.rs index 5bdc9477a2..bb7b53d2e8 100644 --- a/crates/sqlbuild-python-compile/src/compile_driver/tests.rs +++ b/crates/sqlbuild-python-compile/src/compile_driver/tests.rs @@ -1,4 +1,6 @@ #[path = "tests/test_node_setting_consumers.rs"] mod node_setting_consumers; +#[path = "tests/test_singular_audit_ignores.rs"] +mod singular_audit_ignores; #[path = "tests/test_types.rs"] mod test_types; diff --git a/crates/sqlbuild-python-compile/src/compile_driver/tests/test_node_setting_consumers.rs b/crates/sqlbuild-python-compile/src/compile_driver/tests/test_node_setting_consumers.rs index f712edb5ee..8abaa6af4b 100644 --- a/crates/sqlbuild-python-compile/src/compile_driver/tests/test_node_setting_consumers.rs +++ b/crates/sqlbuild-python-compile/src/compile_driver/tests/test_node_setting_consumers.rs @@ -1,32 +1,31 @@ use std::collections::BTreeSet; -use sqlbuild_analysis::job_settings::constants::{JOB_SETTINGS, WAREHOUSE_SETTING}; -use sqlbuild_analysis::job_settings::models::SettingScope; +use sqlbuild_analysis::job_settings::constants::JOB_SETTINGS; +use sqlbuild_analysis::job_settings::models::{NodeConsumer, SettingScope}; -use crate::compile_driver::_helpers::jobs::job_node_settings::{ - FULL_REFRESH, IGNORE_AUDIT_FAILURES, RUN_TESTS, -}; +use crate::compile_driver::_helpers::jobs::job_node_settings::node_setting; use crate::compile_driver::tests::test_types::NodeSettingConsumersTestCase; #[test] fn given_every_node_setting_in_the_table_when_planning_then_a_consumer_applies_it() { let test_cases = [NodeSettingConsumersTestCase { - description: "apply each node setting to the plan in job_node_settings.rs, then list it here", - expected_node_settings: BTreeSet::from([ - FULL_REFRESH, - RUN_TESTS, - IGNORE_AUDIT_FAILURES, - WAREHOUSE_SETTING, - ]), - }]; - for test_case in test_cases { - let node_settings: BTreeSet<&str> = JOB_SETTINGS + description: "each node setting is read through the consumer its table row names", + consumers: vec![ + NodeConsumer::RefreshOverrides, + NodeConsumer::TestEntries, + NodeConsumer::AuditEntries, + NodeConsumer::NodeWarehouses, + ], + expected_node_settings: JOB_SETTINGS .iter() .filter(|spec| spec.scope == SettingScope::Node) .map(|spec| spec.name) - .collect(); + .collect(), + }]; + for test_case in test_cases { + let read: BTreeSet<&str> = test_case.consumers.into_iter().map(node_setting).collect(); assert_eq!( - node_settings, test_case.expected_node_settings, + read, test_case.expected_node_settings, "{}", test_case.description ); diff --git a/crates/sqlbuild-python-compile/src/compile_driver/tests/test_singular_audit_ignores.rs b/crates/sqlbuild-python-compile/src/compile_driver/tests/test_singular_audit_ignores.rs new file mode 100644 index 0000000000..c9b3773600 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_driver/tests/test_singular_audit_ignores.rs @@ -0,0 +1,52 @@ +use crate::compile_driver::_helpers::jobs::job_node_settings::singular_audit_ignores; +use crate::compile_driver::tests::test_types::SingularAuditIgnoresTestCase; + +#[test] +fn given_a_singular_audit_when_its_nodes_have_job_and_build_values_then_all_must_ignore() { + let test_cases = [ + SingularAuditIgnoresTestCase { + description: "a job node ignores, a node outside the jobs takes the build's false", + job_values: vec![Some(true), None], + build_ignores: false, + expected_ignores: Some(false), + }, + SingularAuditIgnoresTestCase { + description: "a job node ignores, a node outside the jobs takes the build's true", + job_values: vec![Some(true), None], + build_ignores: true, + expected_ignores: Some(true), + }, + SingularAuditIgnoresTestCase { + description: "a job node that refuses wins over the build's true", + job_values: vec![Some(false), None], + build_ignores: true, + expected_ignores: Some(false), + }, + SingularAuditIgnoresTestCase { + description: "two job nodes that both ignore", + job_values: vec![Some(true), Some(true)], + build_ignores: false, + expected_ignores: Some(true), + }, + SingularAuditIgnoresTestCase { + description: "two job nodes that disagree", + job_values: vec![Some(true), Some(false)], + build_ignores: true, + expected_ignores: Some(false), + }, + SingularAuditIgnoresTestCase { + description: "no read node has a job value, so the build decides", + job_values: vec![None, None], + build_ignores: true, + expected_ignores: None, + }, + ]; + for test_case in test_cases { + assert_eq!( + singular_audit_ignores(test_case.job_values.into_iter(), test_case.build_ignores), + test_case.expected_ignores, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-python-compile/src/compile_driver/tests/test_types.rs b/crates/sqlbuild-python-compile/src/compile_driver/tests/test_types.rs index 17742f098a..4bb86a239e 100644 --- a/crates/sqlbuild-python-compile/src/compile_driver/tests/test_types.rs +++ b/crates/sqlbuild-python-compile/src/compile_driver/tests/test_types.rs @@ -1,7 +1,18 @@ use std::collections::BTreeSet; -/// The node settings the planner applies, which the table's node settings must equal. +use sqlbuild_analysis::job_settings::models::NodeConsumer; + +/// The consumers the planner reads node settings through, and the settings they must name. pub(super) struct NodeSettingConsumersTestCase { pub(super) description: &'static str, + pub(super) consumers: Vec, pub(super) expected_node_settings: BTreeSet<&'static str>, } + +/// A singular audit's read nodes' job values, the build's value, and whether it ignores failures. +pub(super) struct SingularAuditIgnoresTestCase { + pub(super) description: &'static str, + pub(super) job_values: Vec>, + pub(super) build_ignores: bool, + pub(super) expected_ignores: Option, +} diff --git a/crates/sqlbuild-python-project/src/job_settings/main/job_setting_conflict.rs b/crates/sqlbuild-python-project/src/job_settings/main/job_setting_conflict.rs index 937dd91119..23fa8512e6 100644 --- a/crates/sqlbuild-python-project/src/job_settings/main/job_setting_conflict.rs +++ b/crates/sqlbuild-python-project/src/job_settings/main/job_setting_conflict.rs @@ -13,6 +13,9 @@ pub fn job_setting_conflict(py: Python<'_>, conflict: &JobSettingConflict) -> Py py, ErrorId::CliUser, &conflict.message, - (Some(JOB_SETTING_CONFLICT_CODE), Some(conflict.help)), + ( + Some(JOB_SETTING_CONFLICT_CODE), + Some(conflict.help.as_str()), + ), ) } diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/orchestration.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/orchestration.md index ec8829bdf7..81c9a21f2a 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/orchestration.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/orchestration.md @@ -123,7 +123,7 @@ nodes differently: | --- | --- | | `full_refresh` | The job's incremental models, rebuilt from scratch, and its functions, recreated | | `run_tests` | The SQL unit tests of the job's models | -| `ignore_audit_failures` | The audits of the job's nodes, and singular audits that read them; a singular audit is ignored when every named job that sets it for the nodes the audit reads sets `true` | +| `ignore_audit_failures` | The audits of the job's nodes, and singular audits that read them; a singular audit is ignored only when every node it reads ignores failures, by its job's setting or else the build's | | `warehouse` | The Snowflake warehouse the job's nodes run on, Python nodes included; see [A job's own warehouse](adapters/snowflake.md#a-jobs-own-warehouse) | | `concurrency` | How many of the job's nodes build at once, and a node in two named jobs fits within both. The build's own limit becomes the larger of the configured limit and the named jobs' limits added together | | `[retry]` | The job's nodes | @@ -342,9 +342,9 @@ Details: them, as on the server. - Job settings that cannot apply together fail the build or plan with `S027`, such as `model:orders is selected through jobs orders_job and finance_job, which set run_tests differently (false and - true)`. Make the jobs agree, set it in only one of them, pass the flag, or use - `--no-job-settings`. For a build-wide setting, where unset differs from set, make the jobs agree, - pass the flag, or use `--no-job-settings`. + true)`. The help names the way out: make the jobs agree, set a node setting in only one of + them, or use `--no-job-settings`. It offers a flag only for the value the flag gives, such as + `--no-tests` to turn tests off for all; no flag turns them back on. - Selectors are checked for syntax at compile. A selector that names a node the project does not have fails when a location publishes, such as `job nightly_orders_job: unknown selector name 'order_total'`. diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_job_settings.py b/tests/integration/src/sqlbuild/cli/commands/main/test_job_settings.py index 13e91e3249..3cd8eaf94e 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_job_settings.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_job_settings.py @@ -117,6 +117,20 @@ 'name = "fail_job_group"\ndescription = "Fails."\njobs = ["events_job", "fast_fail_job"]\n' ), } +_LENIENT_PAIR_FILES: dict[str, str] = { + "models/lenient_peer.sql": ( + "MODEL (description 'Lenient peer.', materialized table);\nSELECT 1 AS id\n" + ), + "audits/singular/lenient_pair.sql": ( + 'AUDIT (name "lenient_pair", severity error);\n' + 'SELECT p.id FROM __ref("lenient_peer") p ' + 'LEFT JOIN __ref("lenient_model") m ON p.id = m.id WHERE m.id IS NULL\n' + ), + "orchestration/jobs/lenient_peer_job.toml": ( + 'name = "lenient_peer_job"\ndescription = "Lenient peer."\nselect = "lenient_peer"\n\n' + "[settings]\nignore_audit_failures = true\n" + ), +} _STEADY_EVENTS_FILES: dict[str, str] = { "orchestration/jobs/steady_events_job.toml": ( 'name = "steady_events_job"\ndescription = "Events, kept."\nselect = "order_events"\n\n' @@ -232,7 +246,7 @@ def test_given_a_named_job_when_building_then_its_settings_apply_below_the_comma expected_fragment=( "jobs events_job and fast_fail_job set fail_fast differently (not set and true); " "a build applies fail_fast to everything it runs = help: make the jobs agree, " - "override it with the command-line flag, or build with --no-job-settings" + "pass --fail-fast to make it true for all, or build with --no-job-settings" ), extra_files=_FAIL_FAST_FILES, ), @@ -545,6 +559,22 @@ def test_given_a_job_concurrency_when_building_then_its_nodes_never_exceed_it( expected_absent=("S027",), extra_files=_STEADY_EVENTS_FILES, ), + JobOutputTestCase( + description="a singular audit over a lenient job node and a node outside it fails", + command_args=("build", "--select", "job:lenient_job", "lenient_peer"), + expected_exit_code=1, + expected_present=("lenient_pair",), + expected_absent=("lenient_pair returned 1 row (ignored)",), + extra_files=_LENIENT_PAIR_FILES, + ), + JobOutputTestCase( + description="a singular audit whose nodes' jobs all ignore failures is ignored", + command_args=("build", "--select", "job:lenient_job", "job:lenient_peer_job"), + expected_exit_code=0, + expected_present=("audit lenient_pair returned 1 row (ignored)",), + expected_absent=(), + extra_files=_LENIENT_PAIR_FILES, + ), JobOutputTestCase( description="a command-line option beats a job option it excludes", command_args=("build", "--select", "job:loads_off_job", "--reload"), diff --git a/website/src/content/docs/docs/concepts/orchestration.mdx b/website/src/content/docs/docs/concepts/orchestration.mdx index 53351bd387..65ba928b69 100644 --- a/website/src/content/docs/docs/concepts/orchestration.mdx +++ b/website/src/content/docs/docs/concepts/orchestration.mdx @@ -109,7 +109,7 @@ nodes differently: | --- | --- | | `full_refresh` | The job's incremental models, rebuilt from scratch, and its functions, recreated | | `run_tests` | The SQL unit tests of the job's models | -| `ignore_audit_failures` | The audits of the job's nodes, and singular audits that read them; a singular audit is ignored when every named job that sets it for the nodes the audit reads sets `true` | +| `ignore_audit_failures` | The audits of the job's nodes, and singular audits that read them; a singular audit is ignored only when every node it reads ignores failures, by its job's setting or else the build's | | `warehouse` | The Snowflake warehouse the job's nodes run on, Python nodes included; see [A job's own warehouse](/docs/concepts/adapters/snowflake#a-jobs-own-warehouse) | | `concurrency` | How many of the job's nodes build at once, and a node in two named jobs fits within both. The build's own limit becomes the larger of the configured limit and the named jobs' limits added together | | `[retry]` | The job's nodes | @@ -329,9 +329,9 @@ Details: them, as on the server. - Job settings that cannot apply together fail the build or plan with `S027`, such as `model:orders is selected through jobs orders_job and finance_job, which set run_tests differently (false and - true)`. Make the jobs agree, set it in only one of them, pass the flag, or use - `--no-job-settings`. For a build-wide setting, where unset differs from set, make the jobs agree, - pass the flag, or use `--no-job-settings`. + true)`. The help names the way out: make the jobs agree, set a node setting in only one of + them, or use `--no-job-settings`. It offers a flag only for the value the flag gives, such as + `--no-tests` to turn tests off for all; no flag turns them back on. - Selectors are checked for syntax at compile. A selector that names a node the project does not have fails when a location publishes, such as `job nightly_orders_job: unknown selector name 'order_total'`. From 3adfc86c8e7c093d775d31fca3ba70194eacaf75 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 19 Nov 2026 11:13:49 +0000 Subject: [PATCH 13/22] fix(jobs): take --ignore-dynamic-grace into the job setting table (CHI-1533) --ignore-dynamic-grace, added with dynamic models, is a build-wide job setting: named jobs must agree on it, the flag can only turn it on, and it applies to the whole build's dynamic-model holds. The build parser's namespace defaults and build and plan request fields count both the dynamic-model options and job_settings. --- .../sqlbuild-analysis/src/job_settings/constants.rs | 12 +++++++++++- crates/sqlbuild-cli/src/argv/constants.rs | 2 +- crates/sqlbuild-cli/src/dispatch/constants.rs | 2 +- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/crates/sqlbuild-analysis/src/job_settings/constants.rs b/crates/sqlbuild-analysis/src/job_settings/constants.rs index 8a108a07fa..566e04e0f7 100644 --- a/crates/sqlbuild-analysis/src/job_settings/constants.rs +++ b/crates/sqlbuild-analysis/src/job_settings/constants.rs @@ -14,7 +14,7 @@ pub const JOB_SETTING_CONFLICT_CODE: &str = "S027"; /// How a conflict names a job's `[retry]` table. pub const RETRY_SETTING: &str = "[retry]"; /// Every option of `sqb build`, in the parser's order, as a job setting. -pub const JOB_SETTINGS: [JobSettingSpec; 52] = [ +pub const JOB_SETTINGS: [JobSettingSpec; 53] = [ JobSettingSpec { name: "help", options: &["-h", "--help"], @@ -375,6 +375,16 @@ pub const JOB_SETTINGS: [JobSettingSpec; 52] = [ consumer: None, group: OneBuild, }, + JobSettingSpec { + name: "ignore_dynamic_grace", + options: &["--ignore-dynamic-grace"], + scope: Build, + python_nodes: false, + whole_run: false, + command_line: CommandLineValues::OnlyTrue, + consumer: None, + group: OneBuild, + }, JobSettingSpec { name: "concurrency", options: &["--concurrency"], diff --git a/crates/sqlbuild-cli/src/argv/constants.rs b/crates/sqlbuild-cli/src/argv/constants.rs index 865fb4cae5..a7d78b7319 100644 --- a/crates/sqlbuild-cli/src/argv/constants.rs +++ b/crates/sqlbuild-cli/src/argv/constants.rs @@ -77,7 +77,7 @@ pub(crate) const INT_CONVERTER: &str = "int"; pub(crate) const PATH_CONVERTER: &str = "Path"; /// Every `CliNamespace` attribute with a default, in declaration order. -pub(crate) const CLI_NAMESPACE_DEFAULTS: [(&str, NamespaceDefault); 193] = [ +pub(crate) const CLI_NAMESPACE_DEFAULTS: [(&str, NamespaceDefault); 194] = [ ("command", NamespaceDefault::None), ("project_dir", NamespaceDefault::None), ("dbt_project_dir", NamespaceDefault::None), diff --git a/crates/sqlbuild-cli/src/dispatch/constants.rs b/crates/sqlbuild-cli/src/dispatch/constants.rs index b61a1ff058..97bd005245 100644 --- a/crates/sqlbuild-cli/src/dispatch/constants.rs +++ b/crates/sqlbuild-cli/src/dispatch/constants.rs @@ -146,7 +146,7 @@ pub(crate) const PLAN_ROUTE: Route = routed( REQUEST_KEYWORD_SHAPE, ); -const BUILD_FIELDS: [Field; 41] = [ +const BUILD_FIELDS: [Field; 42] = [ PROJECT_DIR, NO_SQL_VALIDATION, same("no_cache"), From 301ff8857005c4e6719c6cb21cfeef6e5e7e8615 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 19 Nov 2026 11:13:50 +0000 Subject: [PATCH 14/22] feat(jobs)!: remove the deprecated [on_failure] retry names (CHI-1533) Job and job group files no longer accept the [on_failure] table or the retries and retry_delay_seconds keys, and no longer print deprecation warnings for them. [retry] with attempts and delay_seconds is the only retry table. BREAKING CHANGE: a job or job group file that still uses [on_failure], retries or retry_delay_seconds now fails to compile with an unknown-key error. Rename [on_failure] to [retry], replace retries = N with attempts = N + 1, and rename retry_delay_seconds to delay_seconds. --- .../src/orchestration_files/_helpers/raw.rs | 5 +- .../orchestration_files/_helpers/reading.rs | 124 ++++-------------- .../src/orchestration_files/constants.rs | 18 +-- .../src/orchestration_files/models.rs | 10 -- .../tests/test_read_files.rs | 82 ++---------- .../orchestration_files/tests/test_types.rs | 2 - .../_helpers/project/project_validation.rs | 31 ----- .../test_orchestration_snapshot.py | 38 +++--- 8 files changed, 60 insertions(+), 250 deletions(-) diff --git a/crates/sqlbuild-discovery/src/orchestration_files/_helpers/raw.rs b/crates/sqlbuild-discovery/src/orchestration_files/_helpers/raw.rs index f3bbf3c737..bdf744b190 100644 --- a/crates/sqlbuild-discovery/src/orchestration_files/_helpers/raw.rs +++ b/crates/sqlbuild-discovery/src/orchestration_files/_helpers/raw.rs @@ -18,14 +18,12 @@ pub(crate) struct RawSchedule { pub(crate) default_status: InitialStatus, } -/// `[retry]`, or `[on_failure]`; each key may still use its deprecated name. +/// `[retry]`. #[derive(Deserialize)] #[serde(deny_unknown_fields)] pub(crate) struct RawRetry { pub(crate) attempts: Option>, - pub(crate) retries: Option>, pub(crate) delay_seconds: Option>, - pub(crate) retry_delay_seconds: Option>, } #[derive(Deserialize)] @@ -41,7 +39,6 @@ pub(crate) struct RawRun { pub(crate) pools: BTreeMap>, pub(crate) schedule: Option>, pub(crate) retry: Option>, - pub(crate) on_failure: Option>, } /// The job-only fields; the shared ones are read as `RawRun` from the same text. diff --git a/crates/sqlbuild-discovery/src/orchestration_files/_helpers/reading.rs b/crates/sqlbuild-discovery/src/orchestration_files/_helpers/reading.rs index d0924df8e3..c85dd7521c 100644 --- a/crates/sqlbuild-discovery/src/orchestration_files/_helpers/reading.rs +++ b/crates/sqlbuild-discovery/src/orchestration_files/_helpers/reading.rs @@ -8,12 +8,12 @@ use toml::de::DeTable; use crate::orchestration_files::_helpers::raw::{RawRetry, RawRun, RawSchedule}; use crate::orchestration_files::constants::{ - ATTEMPTS_RANGE, CONCURRENCY_KEY, MAX_POOL_AMOUNT, RETRIES_RANGE, RETRY_ATTEMPTS_KEY, - RETRY_DELAY_KEY, RETRY_DELAY_RANGE, RETRY_TABLE, SUGGESTION_CUTOFF, + ATTEMPTS_RANGE, CONCURRENCY_KEY, MAX_POOL_AMOUNT, RETRY_ATTEMPTS_KEY, RETRY_DELAY_KEY, + RETRY_DELAY_RANGE, RETRY_TABLE, SUGGESTION_CUTOFF, }; use crate::orchestration_files::errors::FileError; use crate::orchestration_files::models::{ - Deprecation, FilePosition, Positioned, RetryPolicy, RunFile, ScheduleFile, ScheduleSource, + FilePosition, Positioned, RetryPolicy, RunFile, ScheduleFile, ScheduleSource, }; /// The 1-based line and character column of byte `offset` in `text`. @@ -122,39 +122,14 @@ fn schedule(text: &str, raw: Spanned) -> Result>, Option>); - -/// A retry number with whether it was set, and the deprecation its name records. -type RetryNumber = (Option<(T, bool)>, Option); - -/// One retry number within its name's range, and the deprecation a deprecated name records. +/// One `[retry]` number within `range`, if set. fn retry_number>( text: &str, - (table, keys, values): (&str, (&str, &str), RetryValues), - ranges: ((i64, i64), (i64, i64)), -) -> Result, FileError> { - let mut deprecation: Option = None; - let (key, value, range) = match values { - (Some(_), Some(deprecated)) => { - return Err(error_at( - text, - deprecated.span().start, - format!( - "{table} sets both `{}` and its deprecated name `{}`; keep only `{}`", - keys.0, keys.1, keys.0 - ), - )); - } - (Some(current), None) => (keys.0, current, ranges.0), - (None, Some(deprecated)) => { - deprecation = Some(Deprecation { - message: format!("`{}` is deprecated; rename it to `{}`", keys.1, keys.0), - position: position(text, deprecated.span().start), - }); - (keys.1, deprecated, ranges.1) - } - (None, None) => return Ok((None, None)), + (key, value): (&str, Option>), + range: (i64, i64), +) -> Result, FileError> { + let Some(value) = value else { + return Ok(None); }; let start: usize = value.span().start; let number: i64 = value.into_inner(); @@ -163,7 +138,7 @@ fn retry_number>( text, start, format!( - "{table} {key} must be a whole number from {} to {}", + "{RETRY_TABLE} {key} must be a whole number from {} to {}", range.0, range.1 ), ) @@ -171,82 +146,40 @@ fn retry_number>( if !(range.0..=range.1).contains(&number) { return Err(invalid()); } - T::try_from(number) - .map(|number| (Some((number, key == keys.1)), deprecation)) - .map_err(|_| invalid()) + T::try_from(number).map(Some).map_err(|_| invalid()) } -/// `[retry]`, or the deprecated `[on_failure]`, checked, with every deprecated name it uses. -fn retry( - text: &str, - (current, deprecated): (Option>, Option>), -) -> Result<(Option, Vec), FileError> { - let mut deprecations: Vec = Vec::new(); - let (table, raw) = match (current, deprecated) { - (Some(_), Some(deprecated)) => { - return Err(error_at( - text, - deprecated.span().start, - format!( - "`[{}]` and its deprecated name `[{}]` are both set; keep only `[{}]`", - RETRY_TABLE.0, RETRY_TABLE.1, RETRY_TABLE.0 - ), - )); - } - (Some(current), None) => (RETRY_TABLE.0, current), - (None, Some(deprecated)) => { - deprecations.push(Deprecation { - message: format!( - "`[{}]` is deprecated; rename it to `[{}]`", - RETRY_TABLE.1, RETRY_TABLE.0 - ), - position: position(text, deprecated.span().start), - }); - (RETRY_TABLE.1, deprecated) - } - (None, None) => return Ok((None, deprecations)), +/// `[retry]`, checked. +fn retry(text: &str, raw: Option>) -> Result, FileError> { + let Some(raw) = raw else { + return Ok(None); }; let start: usize = raw.span().start; let raw: RawRetry = raw.into_inner(); - let ranges = |range: (u8, u8)| (i64::from(range.0), i64::from(range.1)); - let (attempts, attempts_deprecation) = retry_number::( + let attempts: Option = retry_number( text, - (table, RETRY_ATTEMPTS_KEY, (raw.attempts, raw.retries)), - (ranges(ATTEMPTS_RANGE), ranges(RETRIES_RANGE)), + (RETRY_ATTEMPTS_KEY, raw.attempts), + (i64::from(ATTEMPTS_RANGE.0), i64::from(ATTEMPTS_RANGE.1)), )?; - let attempts: Option = - attempts.map(|(number, reruns)| if reruns { number + 1 } else { number }); - let (delay_seconds, delay_deprecation) = retry_number::( + let delay_seconds: Option = retry_number( text, + (RETRY_DELAY_KEY, raw.delay_seconds), ( - table, - RETRY_DELAY_KEY, - (raw.delay_seconds, raw.retry_delay_seconds), + i64::from(RETRY_DELAY_RANGE.0), + i64::from(RETRY_DELAY_RANGE.1), ), - { - let delay = ( - i64::from(RETRY_DELAY_RANGE.0), - i64::from(RETRY_DELAY_RANGE.1), - ); - (delay, delay) - }, )?; - let delay_seconds: Option = delay_seconds.map(|(seconds, _)| seconds); - deprecations.extend(attempts_deprecation.into_iter().chain(delay_deprecation)); let attempts: u8 = attempts.ok_or_else(|| { error_at( text, start, - format!("{table} needs `{}`", RETRY_ATTEMPTS_KEY.0), + format!("{RETRY_TABLE} needs `{RETRY_ATTEMPTS_KEY}`"), ) })?; - Ok(( - Some(RetryPolicy { - attempts, - delay_seconds: delay_seconds.unwrap_or(RETRY_DELAY_RANGE.0), - }), - deprecations, - )) + Ok(Some(RetryPolicy { + attempts, + delay_seconds: delay_seconds.unwrap_or(RETRY_DELAY_RANGE.0), + })) } /// The fields jobs and groups share, checked. @@ -278,7 +211,7 @@ pub(crate) fn run_file(text: &str, raw: RawRun) -> Result { .schedule .map(|value| schedule(text, value)) .transpose()?; - let (retry, deprecations) = retry(text, (raw.retry, raw.on_failure))?; + let retry: Option = retry(text, raw.retry)?; Ok(RunFile { name, description, @@ -288,6 +221,5 @@ pub(crate) fn run_file(text: &str, raw: RawRun) -> Result { pools, schedule: scheduled, retry, - deprecations, }) } diff --git a/crates/sqlbuild-discovery/src/orchestration_files/constants.rs b/crates/sqlbuild-discovery/src/orchestration_files/constants.rs index f9af4272d7..84f2d7ca81 100644 --- a/crates/sqlbuild-discovery/src/orchestration_files/constants.rs +++ b/crates/sqlbuild-discovery/src/orchestration_files/constants.rs @@ -1,7 +1,7 @@ //! The keys job and job group files accept, and the bounds on their run controls. /// Top-level keys of a job file. -pub const JOB_KEYS: [&str; 13] = [ +pub const JOB_KEYS: [&str; 12] = [ "name", "description", "select", @@ -12,12 +12,11 @@ pub const JOB_KEYS: [&str; 13] = [ "pools", "schedule", "retry", - "on_failure", "resources", "settings", ]; /// Top-level keys of a job group file. -pub const GROUP_KEYS: [&str; 11] = [ +pub const GROUP_KEYS: [&str; 10] = [ "name", "description", "jobs", @@ -27,7 +26,6 @@ pub const GROUP_KEYS: [&str; 11] = [ "pools", "schedule", "retry", - "on_failure", "dependencies", ]; /// The group table that sets how members wait for and react to the jobs they depend on. @@ -40,16 +38,12 @@ pub(crate) const DEPENDENCY_FAILURE_KEY: &str = "on_failure"; pub(crate) const DEPENDENCY_FAILURE_VALUES: [&str; 3] = ["partial", "skip", "run"]; /// The fewest and most runs `retry.attempts` allows, the first run included. pub const ATTEMPTS_RANGE: (u8, u8) = (2, 11); -/// The fewest and most reruns the deprecated `on_failure.retries` allows; `attempts` is one more. -pub const RETRIES_RANGE: (u8, u8) = (1, 10); /// The shortest and longest `retry.delay_seconds`, a day. pub const RETRY_DELAY_RANGE: (u32, u32) = (0, 86_400); -/// The retry table, and the deprecated name it replaces. -pub(crate) const RETRY_TABLE: (&str, &str) = ("retry", "on_failure"); -/// The retry count key, and the deprecated name it replaces. -pub(crate) const RETRY_ATTEMPTS_KEY: (&str, &str) = ("attempts", "retries"); -/// The retry delay key, and the deprecated name it replaces. -pub(crate) const RETRY_DELAY_KEY: (&str, &str) = ("delay_seconds", "retry_delay_seconds"); +/// The retry table, and the keys of its run count and delay. +pub(crate) const RETRY_TABLE: &str = "retry"; +pub(crate) const RETRY_ATTEMPTS_KEY: &str = "attempts"; +pub(crate) const RETRY_DELAY_KEY: &str = "delay_seconds"; /// The job setting that moved under `[settings]`. pub(crate) const CONCURRENCY_KEY: &str = "concurrency"; /// The most slots a job or job group may take of one pool. diff --git a/crates/sqlbuild-discovery/src/orchestration_files/models.rs b/crates/sqlbuild-discovery/src/orchestration_files/models.rs index e27348a7a8..afec0707b2 100644 --- a/crates/sqlbuild-discovery/src/orchestration_files/models.rs +++ b/crates/sqlbuild-discovery/src/orchestration_files/models.rs @@ -34,13 +34,6 @@ pub struct RetryPolicy { pub delay_seconds: u32, } -/// A deprecated name a file still uses, where it is written and what replaces it. -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct Deprecation { - pub message: String, - pub position: FilePosition, -} - /// When a group member starts: once the jobs it reads from end (`job`). #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] pub enum WaitFor { @@ -116,10 +109,7 @@ pub struct RunFile { /// Each pool the run takes, with how many of its slots. pub pools: BTreeMap, pub schedule: Option, - /// `[retry]`, or the deprecated `[on_failure]` it replaces. pub retry: Option, - /// Deprecated names the file uses, in file order. - pub deprecations: Vec, } /// One `orchestration/jobs/**.toml` file. diff --git a/crates/sqlbuild-discovery/src/orchestration_files/tests/test_read_files.rs b/crates/sqlbuild-discovery/src/orchestration_files/tests/test_read_files.rs index ecbbd4c247..5886c184c6 100644 --- a/crates/sqlbuild-discovery/src/orchestration_files/tests/test_read_files.rs +++ b/crates/sqlbuild-discovery/src/orchestration_files/tests/test_read_files.rs @@ -23,8 +23,8 @@ overlap = "skip" pools = { warehouse = 2, supplier_api = 1 } schedule = { calendar = "trading_days_calendar", timezone = "Europe/London", default_status = "paused" } -[on_failure] -retries = 2 +[retry] +attempts = 3 [resources] memory = "2g" @@ -50,16 +50,6 @@ fn given_job_files_when_reading_then_types_every_field_and_keeps_positions() { attempts: 3, delay_seconds: 0, }), - expected_deprecations: vec![ - ( - "`[on_failure]` is deprecated; rename it to `[retry]`", - at(11, 1), - ), - ( - "`retries` is deprecated; rename it to `attempts`", - at(12, 11), - ), - ], expected_schedule: Some(ScheduleFile { source: ScheduleSource::Calendar(written("trading_days_calendar", 9, 25)), timezone: written("Europe/London", 9, 61), @@ -81,7 +71,6 @@ fn given_job_files_when_reading_then_types_every_field_and_keeps_positions() { expected_overlap: Overlap::Queue, expected_pools: &[], expected_retry: None, - expected_deprecations: Vec::new(), expected_schedule: None, expected_cpus: None, expected_settings: BTreeMap::new(), @@ -98,33 +87,6 @@ fn given_job_files_when_reading_then_types_every_field_and_keeps_positions() { attempts: 3, delay_seconds: 900, }), - expected_deprecations: Vec::new(), - expected_schedule: None, - expected_cpus: None, - expected_settings: BTreeMap::new(), - }, - JobReadTestCase { - description: "a retry table may still use the deprecated key names", - text: "name = \"a_job\"\ndescription = \"d\"\nselect = \"orders\"\nretry = { retries = 1, retry_delay_seconds = 60 }\n", - expected_name: written("a_job", 1, 8), - expected_select: written("orders", 3, 10), - expected_exclude: None, - expected_overlap: Overlap::Queue, - expected_pools: &[], - expected_retry: Some(RetryPolicy { - attempts: 2, - delay_seconds: 60, - }), - expected_deprecations: vec![ - ( - "`retries` is deprecated; rename it to `attempts`", - at(4, 21), - ), - ( - "`retry_delay_seconds` is deprecated; rename it to `delay_seconds`", - at(4, 46), - ), - ], expected_schedule: None, expected_cpus: None, expected_settings: BTreeMap::new(), @@ -168,16 +130,6 @@ fn given_job_files_when_reading_then_types_every_field_and_keeps_positions() { "{}", test_case.description ); - assert_eq!( - job.run - .deprecations - .iter() - .map(|deprecation| (deprecation.message.as_str(), deprecation.position)) - .collect::>(), - test_case.expected_deprecations, - "{}", - test_case.description - ); assert_eq!( job.run.schedule, test_case.expected_schedule, "{}", @@ -243,18 +195,11 @@ fn given_invalid_job_files_when_reading_then_reports_message_position_and_help() expected_position: at(4, 12), expected_help: None, }, - FileErrorTestCase { - description: "retries out of range", - text: "name = \"a_job\"\ndescription = \"d\"\nselect = \"orders\"\non_failure = { retries = 0 }\n", - expected_message: "on_failure retries must be a whole number from 1 to 10", - expected_position: at(4, 26), - expected_help: None, - }, FileErrorTestCase { description: "retry delay beyond a day", - text: "name = \"a_job\"\ndescription = \"d\"\nselect = \"orders\"\non_failure = { retries = 1, retry_delay_seconds = 86401 }\n", - expected_message: "on_failure retry_delay_seconds must be a whole number from 0 to 86400", - expected_position: at(4, 51), + text: "name = \"a_job\"\ndescription = \"d\"\nselect = \"orders\"\nretry = { attempts = 2, delay_seconds = 86401 }\n", + expected_message: "retry delay_seconds must be a whole number from 0 to 86400", + expected_position: at(4, 41), expected_help: None, }, FileErrorTestCase { @@ -279,23 +224,16 @@ fn given_invalid_job_files_when_reading_then_reports_message_position_and_help() expected_help: None, }, FileErrorTestCase { - description: "a key and its deprecated name together", - text: "name = \"a_job\"\ndescription = \"d\"\nselect = \"orders\"\nretry = { attempts = 1, retries = 2 }\n", - expected_message: "retry sets both `attempts` and its deprecated name `retries`; keep only `attempts`", - expected_position: at(4, 35), - expected_help: None, - }, - FileErrorTestCase { - description: "the retry table and its deprecated name together", - text: "name = \"a_job\"\ndescription = \"d\"\nselect = \"orders\"\nretry = { attempts = 1 }\non_failure = { retries = 2 }\n", - expected_message: "`[retry]` and its deprecated name `[on_failure]` are both set; keep only `[retry]`", - expected_position: at(5, 14), + description: "the removed retry key names fail", + text: "name = \"a_job\"\ndescription = \"d\"\nselect = \"orders\"\nretry = { retries = 2 }\n", + expected_message: "unknown field `retries`, expected `attempts` or `delay_seconds`", + expected_position: at(4, 11), expected_help: None, }, FileErrorTestCase { description: "unknown retry key", text: "name = \"a_job\"\ndescription = \"d\"\nselect = \"orders\"\nretry = { attempts = 1, delay = 5 }\n", - expected_message: "unknown field `delay`, expected one of `attempts`, `retries`, `delay_seconds`, `retry_delay_seconds`", + expected_message: "unknown field `delay`, expected `attempts` or `delay_seconds`", expected_position: at(4, 25), expected_help: None, }, diff --git a/crates/sqlbuild-discovery/src/orchestration_files/tests/test_types.rs b/crates/sqlbuild-discovery/src/orchestration_files/tests/test_types.rs index 428ed58fce..7c99a04bec 100644 --- a/crates/sqlbuild-discovery/src/orchestration_files/tests/test_types.rs +++ b/crates/sqlbuild-discovery/src/orchestration_files/tests/test_types.rs @@ -13,8 +13,6 @@ pub(super) struct JobReadTestCase { pub(super) expected_overlap: Overlap, pub(super) expected_pools: &'static [(&'static str, u32)], pub(super) expected_retry: Option, - /// Each deprecation message with its line and column. - pub(super) expected_deprecations: Vec<(&'static str, FilePosition)>, pub(super) expected_schedule: Option, pub(super) expected_cpus: Option, pub(super) expected_settings: BTreeMap, diff --git a/crates/sqlbuild-python-front/src/discovery/_helpers/project/project_validation.rs b/crates/sqlbuild-python-front/src/discovery/_helpers/project/project_validation.rs index 16cbfbccca..83fd905355 100644 --- a/crates/sqlbuild-python-front/src/discovery/_helpers/project/project_validation.rs +++ b/crates/sqlbuild-python-front/src/discovery/_helpers/project/project_validation.rs @@ -6,10 +6,8 @@ use crate::discovery::main::sources::orchestration_declarations::orchestration_d use crate::discovery::models::NodeRow; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::pyfunction; -use sqlbuild_discovery::orchestration_files::models::{FilePosition, OrchestrationInventory}; use sqlbuild_discovery::project_validation::main::validate_project_inputs::validate_project_inputs; use sqlbuild_discovery::project_validation::models::ProjectFacts; -use sqlbuild_python_base::native_host::main::write_console::write_console; use sqlbuild_python_base::records::main::fields::field::field as native_field; pub(crate) const CSV_MODULE: &str = "csv"; @@ -39,36 +37,7 @@ pub(crate) fn validate_discovered_project_inputs( let project_dir: Bound<'_, PyAny> = native_field(&inputs, "project_dir")?; if !project_dir.is_none() { let inventory = orchestration_declarations(&project_dir)?; - warn_deprecated_orchestration(py, &inventory)?; facts.orchestration = inventory.names(); } validate_project_inputs(&facts).map_err(|failure| validation_error(py, failure)) } - -/// Write a warning line to stderr for each deprecated name a job or job group file still uses. -fn warn_deprecated_orchestration( - py: Python<'_>, - inventory: &OrchestrationInventory, -) -> PyResult<()> { - let runs = inventory - .jobs - .iter() - .map(|(path, job)| (path, &job.run)) - .chain( - inventory - .groups - .iter() - .map(|(path, group)| (path, &group.run)), - ); - for (path, run) in runs { - for deprecation in &run.deprecations { - let FilePosition { line, column } = deprecation.position; - write_console( - py, - "stderr", - &format!("Warning: {path}:{line}:{column}: {}\n", deprecation.message), - )?; - } - } - Ok(()) -} diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py index d38ad1f21e..d83cd55041 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py @@ -901,7 +901,7 @@ def snapshot(schedule: str) -> dict[str, Any]: 'overlap = "skip"\n' "pools = { order_reporting = 2 }\n" f"schedule = {schedule}\n" - "\n[on_failure]\nretries = 2\nretry_delay_seconds = 600\n" + "\n[retry]\nattempts = 3\ndelay_seconds = 600\n" '\n[settings]\nrun_tests = false\nvars = { batch = 3 }\nwarehouse = "reporting"\n' "concurrency = 4\n" ), @@ -1067,23 +1067,10 @@ def test_given_concurrency_layers_when_publishing_then_each_job_and_group_carrie expected_on_failure={"retries": 2, "retry_delay_seconds": 900}, expected_warnings=(), ), - SnapshotRetryTestCase( - description="deprecated on_failure names", - tables="[on_failure]\nretries = 2\nretry_delay_seconds = 600\n", - expected_on_failure={"retries": 2, "retry_delay_seconds": 600}, - expected_warnings=( - "Warning: orchestration/jobs/daily_build_job.toml:6:1: " - "`[on_failure]` is deprecated; rename it to `[retry]`", - "Warning: orchestration/jobs/daily_build_job.toml:7:11: " - "`retries` is deprecated; rename it to `attempts`", - "Warning: orchestration/jobs/daily_build_job.toml:8:23: " - "`retry_delay_seconds` is deprecated; rename it to `delay_seconds`", - ), - ), ], ids=lambda case: case.description, ) -def test_given_a_job_retry_table_when_compiling_then_both_names_publish_one_policy_and_old_ones_warn( +def test_given_a_job_retry_table_when_compiling_then_it_publishes_one_policy_without_warnings( tmp_path: Path, capfd: pytest.CaptureFixture[str], test_case: SnapshotRetryTestCase ) -> None: text: str = "".join(f"{key} = {value}\n" for key, value in _DAILY_BUILD.items()) @@ -1212,19 +1199,24 @@ def test_given_a_job_retry_table_when_compiling_then_both_names_publish_one_poli expected_message="unknown variant `coalesce`, expected `queue` or `skip`", ), SnapshotDeclarationRejectionTestCase( - description="no retries", - tables="[on_failure]\nretries = 0\n", - expected_message="retries must be a whole number from 1 to 10", + description="a single attempt", + tables="[retry]\nattempts = 1\n", + expected_message="attempts must be a whole number from 2 to 11", ), SnapshotDeclarationRejectionTestCase( - description="too many retries", - tables="[on_failure]\nretries = 11\n", - expected_message="retries must be a whole number from 1 to 10", + description="too many attempts", + tables="[retry]\nattempts = 12\n", + expected_message="attempts must be a whole number from 2 to 11", ), SnapshotDeclarationRejectionTestCase( description="retry delay over a day", - tables="[on_failure]\nretries = 1\nretry_delay_seconds = 86401\n", - expected_message="retry_delay_seconds must be a whole number from 0 to 86400", + tables="[retry]\nattempts = 2\ndelay_seconds = 86401\n", + expected_message="delay_seconds must be a whole number from 0 to 86400", + ), + SnapshotDeclarationRejectionTestCase( + description="the removed on_failure table", + tables="[on_failure]\nretries = 2\n", + expected_message="unknown job key 'on_failure'", ), SnapshotDeclarationRejectionTestCase( description="concurrency belongs under settings", From 3263600e14a87fe38f73a4d38f8103b2348dded9 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 19 Nov 2026 12:10:13 +0000 Subject: [PATCH 15/22] fix(jobs): refresh dynamic models on the target's warehouse, check ignore_dynamic_grace at compile, and name [retry] for old [on_failure] tables (CHI-1533) A dynamic model that sets no refresh_warehouse is now created with the target's build warehouse, not the warehouse its connection happened to be on. A job's warehouse, and a --warehouse override, no longer become a dynamic table's permanent refresh warehouse. When --warehouse overrides the build, the resolved connection records the target's own warehouse for this, and Snowflake leaves the key out of the connection. A job that sets ignore_dynamic_grace = true in a project without [dynamic] allow_ignore_grace = true now fails to compile at the setting, instead of failing every run with A719. An old [on_failure] table in a job or group file now fails with a hint that names [retry], attempts = retries + 1 and delay_seconds = retry_delay_seconds. --- .../src/command_connection/constants.rs | 2 + .../orchestration_files/_helpers/reading.rs | 6 ++- .../src/orchestration_files/constants.rs | 6 +++ .../tests/test_read_files.rs | 10 +++++ .../_helpers/sources/orchestration_values.rs | 17 ++++++++- .../sources/orchestration_declarations.rs | 7 +++- .../_helpers/command_runtime/warehouses.rs | 8 +++- .../docs/concepts/models/dynamic.md | 2 +- .../snowflake/classes/snowflake_adapter.py | 2 +- .../executor/build/classes/build_scheduler.py | 30 ++++++++++++++- .../run/_helpers/materializations/dynamic.py | 10 +++-- src/sqlbuild/executor/run/models.py | 1 + .../scheduling/classes/session_warehouses.py | 9 +++++ src/sqlbuild/executor/scheduling/constants.py | 3 ++ .../sqlbuild/cli/commands/main/_test_types.py | 9 +++++ .../src/sqlbuild/cli/commands/main/helpers.py | 28 ++++++++++++++ .../cli/commands/main/test_job_settings.py | 27 ++++++++++++++ .../cli/commands/main/test_job_warehouses.py | 37 +++++++++++++++++++ .../docs/docs/concepts/models/dynamic.mdx | 2 +- 19 files changed, 202 insertions(+), 14 deletions(-) create mode 100644 src/sqlbuild/executor/scheduling/constants.py diff --git a/crates/sqlbuild-config/src/command_connection/constants.rs b/crates/sqlbuild-config/src/command_connection/constants.rs index 2d6591fd76..bdf1b6aae9 100644 --- a/crates/sqlbuild-config/src/command_connection/constants.rs +++ b/crates/sqlbuild-config/src/command_connection/constants.rs @@ -4,6 +4,8 @@ pub const CLI_WAREHOUSE_SOURCE: &str = "--warehouse"; pub const CONNECTION_WAREHOUSE_SOURCE: &str = "connection warehouse"; pub const CONNECTION_DEFAULT_WAREHOUSE_SOURCE: &str = "connection default"; pub const CONNECTION_WAREHOUSE_KEY: &str = "warehouse"; +/// The target's own build warehouse, recorded beside a `--warehouse` override; never connected with. +pub const CONFIGURED_WAREHOUSE_KEY: &str = "configured_warehouse"; pub const CONNECTION_SOURCE_KEY: &str = "source"; pub const DBT_PROFILE_CONNECTION_SOURCE: &str = "dbt_profile"; pub const DUCKDB_ADAPTER: &str = "duckdb"; diff --git a/crates/sqlbuild-discovery/src/orchestration_files/_helpers/reading.rs b/crates/sqlbuild-discovery/src/orchestration_files/_helpers/reading.rs index c85dd7521c..dab5e5a4a2 100644 --- a/crates/sqlbuild-discovery/src/orchestration_files/_helpers/reading.rs +++ b/crates/sqlbuild-discovery/src/orchestration_files/_helpers/reading.rs @@ -8,8 +8,8 @@ use toml::de::DeTable; use crate::orchestration_files::_helpers::raw::{RawRetry, RawRun, RawSchedule}; use crate::orchestration_files::constants::{ - ATTEMPTS_RANGE, CONCURRENCY_KEY, MAX_POOL_AMOUNT, RETRY_ATTEMPTS_KEY, RETRY_DELAY_KEY, - RETRY_DELAY_RANGE, RETRY_TABLE, SUGGESTION_CUTOFF, + ATTEMPTS_RANGE, CONCURRENCY_KEY, MAX_POOL_AMOUNT, REMOVED_RETRY_TABLE, RETRY_ATTEMPTS_KEY, + RETRY_DELAY_KEY, RETRY_DELAY_RANGE, RETRY_TABLE, SUGGESTION_CUTOFF, }; use crate::orchestration_files::errors::FileError; use crate::orchestration_files::models::{ @@ -63,6 +63,8 @@ pub(crate) fn check_keys(text: &str, allowed: &[&str], kind: &str) -> Result<(), } let help: String = if name == CONCURRENCY_KEY { format!("move it under [settings] as `{CONCURRENCY_KEY} = ...`") + } else if name == REMOVED_RETRY_TABLE.0 { + REMOVED_RETRY_TABLE.1.to_owned() } else if let Some(nearest) = close_matches(name, allowed, 1, SUGGESTION_CUTOFF).first() { format!("did you mean '{nearest}'?") } else { diff --git a/crates/sqlbuild-discovery/src/orchestration_files/constants.rs b/crates/sqlbuild-discovery/src/orchestration_files/constants.rs index 84f2d7ca81..9fbb9d80f8 100644 --- a/crates/sqlbuild-discovery/src/orchestration_files/constants.rs +++ b/crates/sqlbuild-discovery/src/orchestration_files/constants.rs @@ -44,6 +44,12 @@ pub const RETRY_DELAY_RANGE: (u32, u32) = (0, 86_400); pub(crate) const RETRY_TABLE: &str = "retry"; pub(crate) const RETRY_ATTEMPTS_KEY: &str = "attempts"; pub(crate) const RETRY_DELAY_KEY: &str = "delay_seconds"; +/// The removed retry table, and how to write it as `[retry]`. +pub(crate) const REMOVED_RETRY_TABLE: (&str, &str) = ( + "on_failure", + "`[on_failure]` is now `[retry]`; `attempts = retries + 1`, \ + `delay_seconds = retry_delay_seconds`", +); /// The job setting that moved under `[settings]`. pub(crate) const CONCURRENCY_KEY: &str = "concurrency"; /// The most slots a job or job group may take of one pool. diff --git a/crates/sqlbuild-discovery/src/orchestration_files/tests/test_read_files.rs b/crates/sqlbuild-discovery/src/orchestration_files/tests/test_read_files.rs index 5886c184c6..b5bdcb4b33 100644 --- a/crates/sqlbuild-discovery/src/orchestration_files/tests/test_read_files.rs +++ b/crates/sqlbuild-discovery/src/orchestration_files/tests/test_read_files.rs @@ -167,6 +167,16 @@ fn given_invalid_job_files_when_reading_then_reports_message_position_and_help() expected_position: at(4, 1), expected_help: Some("move it under [settings] as `concurrency = ...`"), }, + FileErrorTestCase { + description: "the removed on_failure table names its replacement", + text: "name = \"a_job\"\ndescription = \"d\"\nselect = \"orders\"\n\n[on_failure]\nretries = 2\n", + expected_message: "unknown job key 'on_failure'", + expected_position: at(5, 2), + expected_help: Some( + "`[on_failure]` is now `[retry]`; `attempts = retries + 1`, \ + `delay_seconds = retry_delay_seconds`", + ), + }, FileErrorTestCase { description: "a pool amount of zero", text: "name = \"a_job\"\ndescription = \"d\"\nselect = \"orders\"\npools = { supplier_api = 0 }\n", diff --git a/crates/sqlbuild-python-front/src/discovery/_helpers/sources/orchestration_values.rs b/crates/sqlbuild-python-front/src/discovery/_helpers/sources/orchestration_values.rs index 2a05962a73..5dc7a777cb 100644 --- a/crates/sqlbuild-python-front/src/discovery/_helpers/sources/orchestration_values.rs +++ b/crates/sqlbuild-python-front/src/discovery/_helpers/sources/orchestration_values.rs @@ -18,13 +18,16 @@ use sqlbuild_orchestration::store::errors::ControlError; use sqlbuild_python_base::orchestration_reads::main::run_resources::run_resources; +/// The job setting that takes over held dynamic models, which `[dynamic]` must allow. +const IGNORE_DYNAMIC_GRACE: &str = "ignore_dynamic_grace"; + /// A `D017` at `position` of `path`, worded as discovery words its own. type ValueError = (FilePosition, String); /// The first invalid schedule, setting, resource limit or selector, as `D017` at its file, line and column. pub(crate) fn check_orchestration_values( py: Python<'_>, - inventory: &OrchestrationInventory, + (inventory, allow_ignore_grace): (&OrchestrationInventory, bool), ) -> Result<(), DiscoveryFailure> { let runs = inventory .jobs @@ -49,6 +52,18 @@ pub(crate) fn check_orchestration_values( job_settings_arguments(&BTreeMap::from([(key.clone(), value.clone())])) .map_err(|message| failure(path, (job.setting_positions[key], message)))?; } + if !allow_ignore_grace && job.settings.get(IGNORE_DYNAMIC_GRACE) == Some(&Value::Bool(true)) + { + return Err(failure( + path, + ( + job.setting_positions[IGNORE_DYNAMIC_GRACE], + "ignore_dynamic_grace needs [dynamic] allow_ignore_grace = true in \ + sqlbuild_project.toml" + .to_owned(), + ), + )); + } if let Some(first) = job.setting_positions.values().min() { job_settings_arguments(&settings) .map_err(|message| failure(path, (*first, message)))?; diff --git a/crates/sqlbuild-python-front/src/discovery/main/sources/orchestration_declarations.rs b/crates/sqlbuild-python-front/src/discovery/main/sources/orchestration_declarations.rs index f508e7dd0d..76a4f4d99f 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/sources/orchestration_declarations.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/sources/orchestration_declarations.rs @@ -1,10 +1,11 @@ //! Every job, job group, calendar and sensor of `project_dir`, or `OrchestrationDeclarationError` (D017). use crate::discovery::_helpers::failures::raised::failure_error; -use crate::discovery::_helpers::project::project_config::project_pool_limits; +use crate::discovery::_helpers::project::project_config::{configured, project_pool_limits}; use crate::discovery::_helpers::sources::orchestration_files::orchestration_error; use crate::discovery::_helpers::sources::orchestration_values::check_orchestration_values; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use sqlbuild_config::project::main::load_dynamic_config::load_dynamic_config; use sqlbuild_discovery::orchestration_files::main::check_pools::check_pools; use sqlbuild_discovery::orchestration_files::main::orchestration_inventory::orchestration_inventory; use sqlbuild_discovery::orchestration_files::models::OrchestrationInventory; @@ -25,6 +26,8 @@ pub fn orchestration_declarations( let limits = project_pool_limits(py, project_dir)?; check_pools(&inventory, &limits) .map_err(|failure| orchestration_error(&tree, project_dir, failure))?; - check_orchestration_values(py, &inventory).map_err(|failure| failure_error(py, failure))?; + let allow_ignore_grace: bool = configured(project_dir, load_dynamic_config)?.allow_ignore_grace; + check_orchestration_values(py, (&inventory, allow_ignore_grace)) + .map_err(|failure| failure_error(py, failure))?; Ok(inventory) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouses.rs b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouses.rs index 52ccd14ab5..8a2596b594 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouses.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouses.rs @@ -6,8 +6,8 @@ use pyo3::prelude::{ use pyo3::types::{PyDict, PyDictMethods, PyMapping, PyString}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_config::command_connection::constants::{ - CLI_WAREHOUSE_SOURCE, CONNECTION_DEFAULT_WAREHOUSE_SOURCE, CONNECTION_WAREHOUSE_KEY, - CONNECTION_WAREHOUSE_SOURCE, + CLI_WAREHOUSE_SOURCE, CONFIGURED_WAREHOUSE_KEY, CONNECTION_DEFAULT_WAREHOUSE_SOURCE, + CONNECTION_WAREHOUSE_KEY, CONNECTION_WAREHOUSE_SOURCE, }; use sqlbuild_config::command_connection::main::authored_warehouse::authored_warehouse; use sqlbuild_config::command_connection::main::check_cli_warehouse::check_cli_warehouse; @@ -384,6 +384,10 @@ pub(crate) fn apply_warehouse<'py>( let applied: Bound<'py, PyDict> = PyDict::new(py); applied.update(&config.cast::()?.clone())?; applied.set_item(CONNECTION_WAREHOUSE_KEY, warehouse)?; + if scope.cli_warehouse.is_some() { + let own = resolve_warehouse(inputs, (&scope.group, None), (config, &scope.cli_vars))?; + applied.set_item(CONFIGURED_WAREHOUSE_KEY, own.0)?; + } Ok(applied.into_any()) } diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/dynamic.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/dynamic.md index 53cc1d2d88..4d3c304492 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/dynamic.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/dynamic.md @@ -31,7 +31,7 @@ for the grants and change tracking they need. |-------|---------|-------------| | `refresh (lag )` | none | How far behind its inputs the model may fall: a fixed duration such as `5m`, `1h` or `1d`, at least `1m` on Snowflake and no shorter than the lag of a dynamic model it reads. Without it, the model refreshes when a dynamic model that reads it refreshes | | `refresh_mode` | `incremental` | `incremental`, `full`, `auto` or `adaptive`. Snowflake rejects `incremental` for a query it can't refresh incrementally; use `full` for that query | -| `refresh_warehouse` | the connection's warehouse | Warehouse the scheduled refreshes run on | +| `refresh_warehouse` | the target's build warehouse | Warehouse the scheduled refreshes run on; a job's `warehouse` never sets it | | `refresh_on_build` | `false` | Refresh the model in every build that selects it, not only on its schedule | ## What the plan shows diff --git a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py index f2bbc2c0ee..5d4f757b97 100644 --- a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py +++ b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py @@ -240,7 +240,7 @@ class SnowflakeAdapter( max_identifier_length: ClassVar[int] = 255 supports_session_warehouse: ClassVar[bool] = True connection_routing_keys: ClassVar[frozenset[str]] = frozenset( - {"source", "profile", "target", "project_dir", "profiles_dir"} + {"source", "profile", "target", "project_dir", "profiles_dir", "configured_warehouse"} ) _snapshot_sql_dialect: ClassVar[SnapshotSqlDialect] = SnapshotSqlDialect( timestamp_type="TIMESTAMP", diff --git a/src/sqlbuild/executor/build/classes/build_scheduler.py b/src/sqlbuild/executor/build/classes/build_scheduler.py index 31928ed4d8..60112f2f56 100644 --- a/src/sqlbuild/executor/build/classes/build_scheduler.py +++ b/src/sqlbuild/executor/build/classes/build_scheduler.py @@ -120,6 +120,7 @@ configured_warehouse, with_session_warehouse, ) +from sqlbuild.executor.scheduling.constants import CONFIGURED_WAREHOUSE_KEY from sqlbuild.executor.scheduling.main._run_worker import run_worker_with_completion from sqlbuild.executor.scheduling.types import ExecutionStatus from sqlbuild.executor.seed.constants import SEED_ENTRY_MISSING_CODE @@ -167,6 +168,20 @@ def _resource_result_failed(result: object) -> bool: return getattr(result, "status", None) == ExecutionStatus.FAILED +def _split_configured_warehouse( + connection_config: dict[str, object], +) -> tuple[dict[str, object], tuple[bool, object]]: + """The connection config without the recorded build warehouse, and whether it had one.""" + + return ( + {key: value for key, value in connection_config.items() if key != CONFIGURED_WAREHOUSE_KEY}, + ( + CONFIGURED_WAREHOUSE_KEY in connection_config, + connection_config.get(CONFIGURED_WAREHOUSE_KEY), + ), + ) + + class BuildScheduler: """DAG-aware scheduler that dispatches nodes as their dependencies complete.""" @@ -194,7 +209,9 @@ def __init__( entry.model_name: entry for entry in plan.table_type_entries } self._adapter: BaseAdapter = adapter - self._connection_config: dict[str, object] = connection_config + self._connection_config, self._configured_warehouse = _split_configured_warehouse( + connection_config + ) self._connections: tuple[Any, ...] = connections self._scheduler_connection: Any = scheduler_connection self._schema_prepared: bool = schema_prepared @@ -1084,6 +1101,16 @@ def _execute_node_with_cost_context( error_message=f"unknown executable resource type '{key.resource_type}'", ) + def _build_warehouse(self, connection: Any) -> str | None: + """The target's build warehouse, whichever job's or `--warehouse` `connection` is on now.""" + + overridden, configured = self._configured_warehouse + if overridden: + return configured if isinstance(configured, str) else None + if self._session_warehouses is None: + return self._adapter.session_warehouse(connection=connection) + return self._session_warehouses.build_warehouse(connection=connection) + def _enter_node_warehouse( self, *, key: CompiledObjectKey, connection: Any ) -> ( @@ -1460,6 +1487,7 @@ def _execute_model_node( bound_views=bound_views, fingerprint_writes=self._fingerprint_writes, dynamic_hold=self._runtime.dynamic_hold, + build_warehouse=self._build_warehouse, ), promotion_mode=self._promotion_mode, snapshots=self._snapshots, diff --git a/src/sqlbuild/executor/run/_helpers/materializations/dynamic.py b/src/sqlbuild/executor/run/_helpers/materializations/dynamic.py index 7d104dcbe9..2f1b513e51 100644 --- a/src/sqlbuild/executor/run/_helpers/materializations/dynamic.py +++ b/src/sqlbuild/executor/run/_helpers/materializations/dynamic.py @@ -77,14 +77,18 @@ def _create( entry: ModelPlanEntry = context.entry adapter: BaseAdapter = context.adapter connection: Any = context.connection - warehouse: str | None = entry.refresh_warehouse or adapter.session_warehouse( - connection=connection + build_warehouse: Callable[[Any], str | None] = context.build_warehouse or ( + lambda session: adapter.session_warehouse(connection=session) ) + warehouse: str | None = entry.refresh_warehouse or build_warehouse(connection) if warehouse is None: raise AdapterUserError( f"dynamic model '{entry.name}' has no warehouse to refresh it", code=_NO_WAREHOUSE_CODE, - help="set refresh_warehouse on the model, or a warehouse on the target's connection", + help=( + "set refresh_warehouse on the model, or the target's build warehouse; a job's " + "warehouse never refreshes a dynamic model" + ), ) statements: tuple[str, ...] = adapter.render_create_dynamic( destination=destination, diff --git a/src/sqlbuild/executor/run/models.py b/src/sqlbuild/executor/run/models.py index fc47787c66..cbcc004caa 100644 --- a/src/sqlbuild/executor/run/models.py +++ b/src/sqlbuild/executor/run/models.py @@ -335,6 +335,7 @@ class ModelMaterializationContext: bound_views: BoundViewGuard = field(default_factory=BoundViewGuard) fingerprint_writes: FingerprintWrites | None = None dynamic_hold: DynamicHold | None = None + build_warehouse: Callable[[Any], str | None] | None = None @dataclass(frozen=True, kw_only=True, init=False) diff --git a/src/sqlbuild/executor/scheduling/classes/session_warehouses.py b/src/sqlbuild/executor/scheduling/classes/session_warehouses.py index 052ab18e0b..98cdaefd58 100644 --- a/src/sqlbuild/executor/scheduling/classes/session_warehouses.py +++ b/src/sqlbuild/executor/scheduling/classes/session_warehouses.py @@ -88,6 +88,15 @@ def use(self, *, connection: Any, warehouse: str | None) -> None: with self._lock: self._away[id(connection)] = (connection, target) + def build_warehouse(self, *, connection: Any) -> str | None: + """The build's own warehouse: the configured one, else the one connections start on.""" + + with self._lock: + entry: tuple[Any, str] | None = self._away.get(id(connection)) + if entry is None or entry[0] is not connection: + self._learn_home(connection) + return self._home + def restore(self) -> None: """Return every connection still on a node's warehouse to the build's warehouse.""" diff --git a/src/sqlbuild/executor/scheduling/constants.py b/src/sqlbuild/executor/scheduling/constants.py new file mode 100644 index 0000000000..f7b24db87b --- /dev/null +++ b/src/sqlbuild/executor/scheduling/constants.py @@ -0,0 +1,3 @@ +"""Connection config keys the build scheduler reads.""" + +CONFIGURED_WAREHOUSE_KEY: str = "configured_warehouse" diff --git a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py index f0bb2e2b51..329f424203 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py @@ -1552,3 +1552,12 @@ class JobFlagParityTestCase: expected_function_reasons: tuple[tuple[str, str], ...] expected_full_refresh_counts: tuple[str, ...] expected_exit_code: int + + +@dataclass(frozen=True) +class DynamicRefreshWarehouseTestCase: + """A build of a dynamic model through jobs, and the warehouse its refreshes are given.""" + + description: str + argv: tuple[str, ...] + expected_refresh_warehouses: tuple[str, ...] diff --git a/tests/integration/src/sqlbuild/cli/commands/main/helpers.py b/tests/integration/src/sqlbuild/cli/commands/main/helpers.py index 6ecfeec8de..00eb13a58e 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/helpers.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/helpers.py @@ -1655,3 +1655,31 @@ def unit_test_sql(*models: str) -> str: expected: str = ",\n".join(f"__expected__{model} AS (SELECT 1 AS id)" for model in models) return f"TEST();\n\nWITH\n__ref__orders_daily AS (SELECT 1 AS id),\n{expected}\nSELECT 1\n" + + +_DYNAMIC_FILES: dict[str, str] = { + "models/daily_orders.sql": ( + "MODEL (description 'Daily orders.', materialized dynamic, refresh (lag 1h));\n" + "SELECT 1 AS id\n" + ), + "orchestration/jobs/dynamic_wh_job.toml": ( + 'name = "dynamic_wh_job"\ndescription = "Runs daily_orders."\nselect = "daily_orders"\n\n' + '[settings]\nwarehouse = "DYNAMIC_WH"\n' + ), +} + + +def write_dynamic_job_project(*, project_dir: Path) -> None: + """The job warehouse project plus dynamic model daily_orders, run by job dynamic_wh_job.""" + + write_job_warehouse_project(project_dir=project_dir) + write_project_files(project_dir=project_dir, files=_DYNAMIC_FILES) + + +def dynamic_refresh_warehouses(executed_sql: list[str]) -> tuple[str, ...]: + """The warehouse each created dynamic table is given for its refreshes.""" + + statements: str = "\n".join(executed_sql) + return tuple( + re.findall(r"CREATE OR REPLACE DYNAMIC TABLE .*?WAREHOUSE = (\S+)", statements, re.S) + ) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_job_settings.py b/tests/integration/src/sqlbuild/cli/commands/main/test_job_settings.py index 3cd8eaf94e..9fd535d7bb 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_job_settings.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_job_settings.py @@ -117,6 +117,12 @@ 'name = "fail_job_group"\ndescription = "Fails."\njobs = ["events_job", "fast_fail_job"]\n' ), } +_GRACE_FILES: dict[str, str] = { + "orchestration/jobs/grace_job.toml": ( + 'name = "grace_job"\ndescription = "Takes over."\nselect = "customers"\n\n' + "[settings]\nignore_dynamic_grace = true\n" + ), +} _LENIENT_PAIR_FILES: dict[str, str] = { "models/lenient_peer.sql": ( "MODEL (description 'Lenient peer.', materialized table);\nSELECT 1 AS id\n" @@ -250,6 +256,15 @@ def test_given_a_named_job_when_building_then_its_settings_apply_below_the_comma ), extra_files=_FAIL_FAST_FILES, ), + JobSettingsConflictTestCase( + description="a job ignoring the dynamic grace needs the project to allow it", + command_args=("compile",), + expected_fragment=( + "orchestration/jobs/grace_job.toml:6:24: ignore_dynamic_grace needs [dynamic] " + "allow_ignore_grace = true in sqlbuild_project.toml" + ), + extra_files=_GRACE_FILES, + ), JobSettingsConflictTestCase( description="two named jobs giving one node different full_refresh conflict", command_args=("build", "--select", "job:events_job", "job:steady_events_job"), @@ -575,6 +590,18 @@ def test_given_a_job_concurrency_when_building_then_its_nodes_never_exceed_it( expected_absent=(), extra_files=_LENIENT_PAIR_FILES, ), + JobOutputTestCase( + description="a job may ignore the dynamic grace where the project allows it", + command_args=("compile",), + expected_exit_code=0, + expected_present=("Project compiled",), + expected_absent=("ignore_dynamic_grace",), + extra_files={ + **_GRACE_FILES, + "sqlbuild_project.toml": _PROJECT_FILES["sqlbuild_project.toml"] + + "\n[dynamic]\nallow_ignore_grace = true\n", + }, + ), JobOutputTestCase( description="a command-line option beats a job option it excludes", command_args=("build", "--select", "job:loads_off_job", "--reload"), diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_job_warehouses.py b/tests/integration/src/sqlbuild/cli/commands/main/test_job_warehouses.py index a550d2ea81..4abb6083e3 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_job_warehouses.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_job_warehouses.py @@ -8,14 +8,17 @@ from _pytest.capture import CaptureResult from tests.integration.src.sqlbuild.cli.commands.main._test_types import ( + DynamicRefreshWarehouseTestCase, JobWarehouseOutputTestCase, JobWarehouseSwitchTestCase, ) from tests.integration.src.sqlbuild.cli.commands.main.helpers import ( + dynamic_refresh_warehouses, install_writable_offline_snowflake, run_cli_exit_code, unit_test_sql, warehouse_switches, + write_dynamic_job_project, write_job_warehouse_project, write_project_files, ) @@ -175,3 +178,37 @@ def test_given_named_jobs_with_warehouses_when_planning_or_building_then_output_ test_case.expected_exit_code, True, ), output + + +@pytest.mark.parametrize( + "test_case", + [ + DynamicRefreshWarehouseTestCase( + description="a job switching its nodes to its warehouse leaves refreshes on the build's", + argv=("build", "--select", "stg", "job:orders_wh_job", "job:dynamic_wh_job"), + expected_refresh_warehouses=("BUILD_WH",), + ), + DynamicRefreshWarehouseTestCase( + description="a single job's warehouse, the whole build's, still refreshes on the build's", + argv=("build", "--select", "job:dynamic_wh_job"), + expected_refresh_warehouses=("BUILD_WH",), + ), + ], + ids=lambda case: case.description, +) +def test_given_a_dynamic_model_built_through_a_job_when_created_then_it_refreshes_on_the_build_s( + test_case: DynamicRefreshWarehouseTestCase, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + write_dynamic_job_project(project_dir=tmp_path) + _, executed_sql = install_writable_offline_snowflake(monkeypatch) + + exit_code: int = run_cli_exit_code(test_case.argv, tmp_path) + + captured: CaptureResult[str] = capsys.readouterr() + assert (exit_code, dynamic_refresh_warehouses(executed_sql)) == ( + 0, + test_case.expected_refresh_warehouses, + ), captured.out + captured.err diff --git a/website/src/content/docs/docs/concepts/models/dynamic.mdx b/website/src/content/docs/docs/concepts/models/dynamic.mdx index 9cb53ce8ff..f2434e0b30 100644 --- a/website/src/content/docs/docs/concepts/models/dynamic.mdx +++ b/website/src/content/docs/docs/concepts/models/dynamic.mdx @@ -28,7 +28,7 @@ for the grants and change tracking they need. |-------|---------|-------------| | `refresh (lag )` | none | How far behind its inputs the model may fall: a fixed duration such as `5m`, `1h` or `1d`, at least `1m` on Snowflake and no shorter than the lag of a dynamic model it reads. Without it, the model refreshes when a dynamic model that reads it refreshes | | `refresh_mode` | `incremental` | `incremental`, `full`, `auto` or `adaptive`. Snowflake rejects `incremental` for a query it can't refresh incrementally; use `full` for that query | -| `refresh_warehouse` | the connection's warehouse | Warehouse the scheduled refreshes run on | +| `refresh_warehouse` | the target's build warehouse | Warehouse the scheduled refreshes run on; a job's `warehouse` never sets it | | `refresh_on_build` | `false` | Refresh the model in every build that selects it, not only on its schedule | ## What the plan shows From 33a4035146c0d73bdf0a91737b5898429fc4215f Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 19 Nov 2026 12:31:39 +0000 Subject: [PATCH 16/22] refactor(jobs): share one thread-local scope swap and read job warehouses as a mapping (CHI-1533) --- .../src/plan_output/_helpers/plan_text.rs | 13 ++++++----- .../src/job_settings/_helpers/job_catalog.rs | 23 ++++++++++++------- 2 files changed, 22 insertions(+), 14 deletions(-) diff --git a/crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_text.rs b/crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_text.rs index f1c6d51258..c3cfba3a6d 100644 --- a/crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_text.rs +++ b/crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_text.rs @@ -13,6 +13,7 @@ use sqlbuild_core::cli_text::models::{ SeedPlanText, SourceFreshnessText, SourceLoadPlanText, }; use sqlbuild_core::plan_display::main::format_query_diff::format_query_diff; +use std::collections::BTreeMap; use crate::plan_output::_helpers::cursor_details::CursorDetails; use crate::plan_output::_helpers::cursor_details::DEFERRED; @@ -394,13 +395,13 @@ fn provider_usages( /// The plan metadata's named jobs and the warehouses their nodes run on. pub(crate) fn job_warehouses(metadata: &Bound<'_, PyAny>) -> PyResult> { let value: Bound<'_, PyAny> = mapping_get(metadata, JOB_WAREHOUSES_KEY)?; - let Ok(warehouses) = value.cast::() else { + if value.is_none() { return Ok(Vec::new()); - }; - warehouses - .iter() - .map(|(job, warehouse)| Ok((job.extract()?, warehouse.extract()?))) - .collect() + } + Ok(value + .extract::>()? + .into_iter() + .collect()) } /// `str(item)` of a metadata tuple or list, else nothing. diff --git a/crates/sqlbuild-python-project/src/job_settings/_helpers/job_catalog.rs b/crates/sqlbuild-python-project/src/job_settings/_helpers/job_catalog.rs index c41c30b68b..8b9d2b1757 100644 --- a/crates/sqlbuild-python-project/src/job_settings/_helpers/job_catalog.rs +++ b/crates/sqlbuild-python-project/src/job_settings/_helpers/job_catalog.rs @@ -3,6 +3,7 @@ use std::cell::RefCell; use std::collections::{BTreeMap, BTreeSet}; use std::path::PathBuf; +use std::thread::LocalKey; use pyo3::IntoPyObject; use pyo3::exceptions::PyValueError; @@ -37,14 +38,23 @@ thread_local! { static GIVEN_SETTINGS: RefCell> = const { RefCell::new(BTreeSet::new()) }; } -/// Run one command whose command line gives `given` settings, restoring the enclosing ones after. -pub(crate) fn with_given_settings(given: BTreeSet, run: impl FnOnce() -> T) -> T { - let previous = GIVEN_SETTINGS.with(|current| current.replace(given)); +/// Run `run` with `slot` holding `value`, restoring what it held before. +fn swapped( + slot: &'static LocalKey>, + value: S, + run: impl FnOnce() -> T, +) -> T { + let previous: S = slot.with(|current| current.replace(value)); let result: T = run(); - GIVEN_SETTINGS.with(|current| *current.borrow_mut() = previous); + slot.with(|current| *current.borrow_mut() = previous); result } +/// Run one command whose command line gives `given` settings, restoring the enclosing ones after. +pub(crate) fn with_given_settings(given: BTreeSet, run: impl FnOnce() -> T) -> T { + swapped(&GIVEN_SETTINGS, given, run) +} + /// The settings the running command's command line gives. pub(crate) fn given_settings() -> BTreeSet { GIVEN_SETTINGS.with(|current| current.borrow().clone()) @@ -94,10 +104,7 @@ pub(crate) fn with_frozen_job( } fn within_job_scope(scope: JobScope, run: impl FnOnce() -> T) -> T { - let previous = JOB_SCOPE.with(|current| current.replace(Some(scope))); - let result: T = run(); - JOB_SCOPE.with(|current| *current.borrow_mut() = previous); - result + swapped(&JOB_SCOPE, Some(scope), run) } /// The jobs `selectors` name, as `job:` or as a bare job name. From f356e7fb739558dc988ab293228b845faf0abb0b Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 19 Nov 2026 12:40:57 +0000 Subject: [PATCH 17/22] fix(jobs): fail a microbatch model's warehouse switch with a model result (CHI-1533) --- .../executor/build/classes/build_scheduler.py | 63 ++++++++++--------- 1 file changed, 33 insertions(+), 30 deletions(-) diff --git a/src/sqlbuild/executor/build/classes/build_scheduler.py b/src/sqlbuild/executor/build/classes/build_scheduler.py index 60112f2f56..a52bb4b1aa 100644 --- a/src/sqlbuild/executor/build/classes/build_scheduler.py +++ b/src/sqlbuild/executor/build/classes/build_scheduler.py @@ -708,16 +708,9 @@ def _execute_concurrent_microbatch_node( ledger_path=(self._runtime_dir / "executions" / self._run_id / "statements.jsonl"), on_statement_complete=self._on_statement_complete, ): - failure: ( - ModelExecutionResult - | SeedExecutionResult - | FunctionExecutionResult - | SqlTestExecutionResult - | LoadExecutionResult - | None - ) = self._enter_node_warehouse(key=key, connection=connection) + failure: str | None = self._switch_failure(key=key, connection=connection) if failure is not None: - return failure + return self._failed_model_result(key=key, error_message=failure) return self._execute_model_node( key=key, connection=connection, @@ -1111,6 +1104,22 @@ def _build_warehouse(self, connection: Any) -> str | None: return self._adapter.session_warehouse(connection=connection) return self._session_warehouses.build_warehouse(connection=connection) + def _switch_failure(self, *, key: CompiledObjectKey, connection: Any) -> str | None: + """Put `connection` on the node's job warehouse, or the build's; why it could not.""" + + if self._session_warehouses is None: + return None + warehouse: str | None = self._node_warehouses.get(key) + try: + self._session_warehouses.use(connection=connection, warehouse=warehouse) + except Exception as error: # noqa: BLE001 - the node fails with the reason + return ( + f"cannot run on warehouse {warehouse}: {error}" + if warehouse is not None + else f"cannot return to the build's warehouse: {error}" + ) + return None + def _enter_node_warehouse( self, *, key: CompiledObjectKey, connection: Any ) -> ( @@ -1121,23 +1130,22 @@ def _enter_node_warehouse( | LoadExecutionResult | None ): - """Put `connection` on the node's job warehouse, or the build's; its failure if not.""" + """Put `connection` on the node's warehouse; the node's own kind of failure if not.""" - if self._session_warehouses is None: + failure: str | None = self._switch_failure(key=key, connection=connection) + if failure is None: return None - warehouse: str | None = self._node_warehouses.get(key) - try: - self._session_warehouses.use(connection=connection, warehouse=warehouse) - except Exception as error: # noqa: BLE001 - the node fails with the reason - return self._failed_node_result( - key=key, - error_message=( - f"cannot run on warehouse {warehouse}: {error}" - if warehouse is not None - else f"cannot return to the build's warehouse: {error}" - ), - ) - return None + return self._failed_node_result(key=key, error_message=failure) + + def _failed_model_result( + self, *, key: CompiledObjectKey, error_message: str + ) -> ModelExecutionResult: + return ModelExecutionResult( + model_name=key.name, + status=ExecutionStatus.FAILED, + error_code=BUILD_WORKER_FAILED_CODE, + error_message=error_message, + ) def _failed_node_result( self, *, key: CompiledObjectKey, error_message: str @@ -1182,12 +1190,7 @@ def _failed_node_result( error_code=BUILD_WORKER_FAILED_CODE, error_message=error_message, ) - return ModelExecutionResult( - model_name=key.name, - status=ExecutionStatus.FAILED, - error_code=BUILD_WORKER_FAILED_CODE, - error_message=error_message, - ) + return self._failed_model_result(key=key, error_message=error_message) def _execute_source_node( self, *, key: CompiledObjectKey, connection: Any From 1dc4f8363fa7e8a9257b5ba8714062132e3e707e Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 19 Nov 2026 12:45:11 +0000 Subject: [PATCH 18/22] test(jobs): read a failed node's message through the result helper (CHI-1533) --- tests/unit/src/sqlbuild/executor/build/classes/helpers.py | 7 ++++--- .../build/classes/test_build_scheduler_warehouses.py | 5 ++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/unit/src/sqlbuild/executor/build/classes/helpers.py b/tests/unit/src/sqlbuild/executor/build/classes/helpers.py index f9c8b5bd57..3749bfe7a2 100644 --- a/tests/unit/src/sqlbuild/executor/build/classes/helpers.py +++ b/tests/unit/src/sqlbuild/executor/build/classes/helpers.py @@ -167,12 +167,13 @@ def node_key(*, resource_type: CompiledResourceType, name: str) -> CompiledObjec return CompiledObjectKey(resource_type=resource_type, name=name) -def failed_node_result(result: Any) -> bool: - """Whether a node result of any kind reports failure.""" +def failed_node_result(result: Any) -> tuple[bool, str | None]: + """Whether a node result of any kind reports failure, and its error message.""" return ( getattr(result, "status", None) == ExecutionStatus.FAILED - or getattr(result, "outcome", None) == SqlTestOutcome.ERROR + or getattr(result, "outcome", None) == SqlTestOutcome.ERROR, + getattr(result, "error_message", None), ) diff --git a/tests/unit/src/sqlbuild/executor/build/classes/test_build_scheduler_warehouses.py b/tests/unit/src/sqlbuild/executor/build/classes/test_build_scheduler_warehouses.py index a9182e80ad..211c87026d 100644 --- a/tests/unit/src/sqlbuild/executor/build/classes/test_build_scheduler_warehouses.py +++ b/tests/unit/src/sqlbuild/executor/build/classes/test_build_scheduler_warehouses.py @@ -151,8 +151,7 @@ def test_given_an_unusable_job_warehouse_when_a_node_starts_then_it_fails_naming result: Any = BuildScheduler._enter_node_warehouse(scheduler, key=key, connection=object()) - assert (type(result), failed_node_result(result), result.error_message) == ( + assert (type(result), failed_node_result(result)) == ( test_case.expected_result_type, - True, - test_case.expected_message, + (True, test_case.expected_message), ) From 56100c975068bd7d5f1149246a95e8075374cb52 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 19 Nov 2026 12:50:08 +0000 Subject: [PATCH 19/22] fix(adapters): make the session warehouse switch an optional strict adapter method (CHI-1533) --- .../adapter/contract/classes/base_adapter.py | 12 ------------ .../adapter/contract/classes/strict_adapter.py | 14 ++++++++++++++ .../classes/strict_adapter/test_strict_adapter.py | 2 ++ 3 files changed, 16 insertions(+), 12 deletions(-) diff --git a/src/sqlbuild/adapter/contract/classes/base_adapter.py b/src/sqlbuild/adapter/contract/classes/base_adapter.py index a693bc34e2..143a9e7c76 100644 --- a/src/sqlbuild/adapter/contract/classes/base_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/base_adapter.py @@ -2345,18 +2345,6 @@ def render_create_observations_table_sql(self, *, database: str | None, schema: render_framework_type=self.render_framework_type, ) - def use_session_warehouse(self, *, connection: Any, warehouse: str) -> None: - """Run the session's next statements on `warehouse`, where the adapter has one.""" - - raise AdapterUserError( - f"the '{self.adapter_name}' adapter has no warehouse to select", code="C260" - ) - - def current_session_warehouse(self, *, connection: Any) -> str | None: - """The warehouse the session runs on, as an identifier `use_session_warehouse` accepts.""" - - return None - def render_create_consumption_index_sqls( self, *, diff --git a/src/sqlbuild/adapter/contract/classes/strict_adapter.py b/src/sqlbuild/adapter/contract/classes/strict_adapter.py index 89c03cf168..6b3f4d5b98 100644 --- a/src/sqlbuild/adapter/contract/classes/strict_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/strict_adapter.py @@ -329,6 +329,20 @@ def session_warehouse(self, *, connection: Any) -> str | None: del connection return None + def use_session_warehouse(self, *, connection: Any, warehouse: str) -> None: + """Optional: run the session's next statements on `warehouse`, where the adapter has one.""" + + del connection, warehouse + raise AdapterUserError( + f"the '{self.adapter_name}' adapter has no warehouse to select", code="C260" + ) + + def current_session_warehouse(self, *, connection: Any) -> str | None: + """Optional: the session's warehouse, as an identifier `use_session_warehouse` accepts.""" + + del connection + return None + def render_create_dynamic( self, *, destination: str, sql: str, target_lag: str, refresh_mode: str, warehouse: str ) -> tuple[str, ...]: diff --git a/tests/unit/src/sqlbuild/adapter/contract/classes/strict_adapter/test_strict_adapter.py b/tests/unit/src/sqlbuild/adapter/contract/classes/strict_adapter/test_strict_adapter.py index 6605bc233c..13fe6e5206 100644 --- a/tests/unit/src/sqlbuild/adapter/contract/classes/strict_adapter/test_strict_adapter.py +++ b/tests/unit/src/sqlbuild/adapter/contract/classes/strict_adapter/test_strict_adapter.py @@ -97,6 +97,8 @@ def test_given_first_class_adapters_when_checking_contract_then_no_method_comes_ "render_dynamic_operation", "render_track_changes", "session_warehouse", + "use_session_warehouse", + "current_session_warehouse", } ), expected_output_relation=None, From 5ba1c251447ce910388fbd9d2fbaa7dd87edba80 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 19 Nov 2026 13:28:20 +0000 Subject: [PATCH 20/22] fix(jobs): keep named jobs' concurrency limits under SQLBUILD_CONCURRENCY (CHI-1533) SQLBUILD_CONCURRENCY filled the build's concurrency as if --concurrency were given, which turned off every named job's limit. It now sets the build's limit as a configured one does, below named jobs, and only a --concurrency on the command line turns the job limits off. The offline Snowflake test double records which connection runs each statement, so a connection's own setup is never counted as a warehouse switch when connections open in parallel. Tests that assert exact limits or switch orders clear SQLBUILD_CONCURRENCY, and a new case builds with it set. --- .../_helpers/jobs/job_node_settings.rs | 4 +- .../main/jobs/job_plan_fields.rs | 6 ++- .../references/docs/concepts/orchestration.md | 3 +- .../sqlbuild/cli/commands/main/_test_types.py | 2 + .../src/sqlbuild/cli/commands/main/helpers.py | 45 ++++++++++++++++--- .../cli/commands/main/test_job_settings.py | 16 +++++++ .../cli/commands/main/test_job_warehouses.py | 2 + .../docs/docs/concepts/orchestration.mdx | 3 +- 8 files changed, 71 insertions(+), 10 deletions(-) diff --git a/crates/sqlbuild-python-compile/src/compile_driver/_helpers/jobs/job_node_settings.rs b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/jobs/job_node_settings.rs index 972fb78498..a4f9456ae4 100644 --- a/crates/sqlbuild-python-compile/src/compile_driver/_helpers/jobs/job_node_settings.rs +++ b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/jobs/job_node_settings.rs @@ -285,12 +285,12 @@ impl PlannedJobSettings { pub(crate) fn job_limits<'py>( &self, plan: &Bound<'py, PyAny>, - cli_concurrency: &Bound<'py, PyAny>, + cli_concurrency: bool, ) -> PyResult<(Bound<'py, PyDict>, Bound<'py, PyDict>)> { let py: Python<'py> = plan.py(); let limits: Bound<'py, PyDict> = PyDict::new(py); let node_jobs: Bound<'py, PyDict> = PyDict::new(py); - if !cli_concurrency.is_none() || self.settings.job_limits.is_empty() { + if cli_concurrency || self.settings.job_limits.is_empty() { return Ok((limits, node_jobs)); } for (job, limit) in &self.settings.job_limits { diff --git a/crates/sqlbuild-python-compile/src/compile_driver/main/jobs/job_plan_fields.rs b/crates/sqlbuild-python-compile/src/compile_driver/main/jobs/job_plan_fields.rs index 4b4bc8805a..a9b0dc3e64 100644 --- a/crates/sqlbuild-python-compile/src/compile_driver/main/jobs/job_plan_fields.rs +++ b/crates/sqlbuild-python-compile/src/compile_driver/main/jobs/job_plan_fields.rs @@ -2,6 +2,10 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_project::job_settings::main::given_settings::given_settings; + +/// The option that sets the build's limit and turns the named jobs' limits off when given. +const CONCURRENCY_SETTING: &str = "concurrency"; use crate::compile_driver::models::PlannedJobSettings; use crate::compile_driver::types::PipelineFields; @@ -23,7 +27,7 @@ pub fn job_plan_fields<'py>( ), )?; let (limits, node_jobs) = - job_settings.job_limits(&plan_output, &native_field(options, "concurrency")?)?; + job_settings.job_limits(&plan_output, given_settings().contains(CONCURRENCY_SETTING))?; let (nodes, python) = job_settings.node_warehouses(&plan_output)?; Ok(( plan_output, diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/orchestration.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/orchestration.md index 81c9a21f2a..eff41b9fd8 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/orchestration.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/orchestration.md @@ -113,7 +113,8 @@ A command-line flag replaces every named job's setting rather than merging with settles named jobs that disagree: `--vars` replaces the job's whole `vars`, `--full-refresh` rebuilds every selected model, and `--warehouse` puts every node on its warehouse. An option that rules another out counts as setting both, so `--reload` wins over a job's `load_sources = false`. Any `--concurrency` sets the build's limit and turns off the -per-job limits. The job's settings are added before a `--` on the command line, so selectors after +per-job limits. `SQLBUILD_CONCURRENCY` is not a command-line flag: it sets the build's limit as a +configured one does, and the named jobs' limits still apply. The job's settings are added before a `--` on the command line, so selectors after it still name jobs. `--no-job-settings` leaves out every job's settings. Some settings apply to the job's own nodes only, so a build that names two jobs can treat their diff --git a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py index 329f424203..093b9ac534 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py @@ -1508,6 +1508,8 @@ class JobConcurrencyTestCase: build_nodes: tuple[str, ...] expected_job_peak: int expected_least_build_peak: int + # SQLBUILD_CONCURRENCY for the build; empty leaves it unset. + environment_concurrency: str = "" @dataclass(frozen=True) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/helpers.py b/tests/integration/src/sqlbuild/cli/commands/main/helpers.py index 00eb13a58e..273cda92bd 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/helpers.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/helpers.py @@ -12,7 +12,7 @@ from collections import defaultdict from collections.abc import Callable from datetime import datetime, timedelta -from itertools import accumulate, chain, takewhile +from itertools import accumulate, chain, compress, takewhile from pathlib import Path from types import ModuleType from typing import Any, cast @@ -1485,6 +1485,8 @@ def peak_concurrency(*, events_path: Path, names: tuple[str, ...]) -> int: _CONNECT_MARKER: str = "-- connect" +# The environment variable that sets a build's default concurrency. +CONCURRENCY_VARIABLE: str = "SQLBUILD_CONCURRENCY" def _job_warehouse_model(source: str) -> str: @@ -1555,17 +1557,50 @@ def install_writable_offline_snowflake( def marked_connect(**kwargs: object) -> object: executed_sql.append(_CONNECT_MARKER) - return connect(**kwargs) + raw: Any = connect(**kwargs) + raw._executed_sql = _ConnectionStatements(sink=executed_sql, connection=len(connect_calls)) + return raw monkeypatch.setitem(connector.__dict__, "connect", marked_connect) return connect_calls, executed_sql +class _ConnectionStatements: + """One offline connection's statement log: each statement in order, and which connection.""" + + def __init__(self, *, sink: list[str], connection: int) -> None: + self._sink: list[str] = sink + self._connection: int = connection + + def append(self, sql: str) -> None: + self._sink.append(sql) + _STATEMENT_CONNECTIONS.setdefault(id(self._sink), []).append((self._connection, sql)) + + +# Each shared statement log's statements with the connection that ran them. +_STATEMENT_CONNECTIONS: dict[int, list[tuple[int, str]]] = {} + + def warehouse_switches(executed_sql: list[str]) -> tuple[str, ...]: - """The warehouses statements switch to, leaving out each new connection's own setup.""" + """The warehouses statements switch to, leaving out each connection's first, its own setup.""" - statements: str = "\n".join(executed_sql) - return tuple(re.findall(rf"(? int: diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_job_settings.py b/tests/integration/src/sqlbuild/cli/commands/main/test_job_settings.py index 9fd535d7bb..fa8ec58e87 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_job_settings.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_job_settings.py @@ -16,6 +16,7 @@ JobSettingsConflictTestCase, ) from tests.integration.src.sqlbuild.cli.commands.main.helpers import ( + CONCURRENCY_VARIABLE, PARITY_NODES, job_flag_parity_outcome, peak_concurrency, @@ -210,7 +211,9 @@ def test_given_a_named_job_when_building_then_its_settings_apply_below_the_comma test_case: JobSettingsBuildTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str], + monkeypatch: pytest.MonkeyPatch, ) -> None: + monkeypatch.delenv(CONCURRENCY_VARIABLE, raising=False) write_project_files(project_dir=tmp_path, files=_PROJECT_FILES) exit_code: int = main( @@ -372,6 +375,15 @@ def test_given_a_named_job_with_full_refresh_when_building_then_only_its_models_ expected_job_peak=2, expected_least_build_peak=2, ), + JobConcurrencyTestCase( + description="SQLBUILD_CONCURRENCY sets the build's limit, not the job's", + build_args=("--select", "job:slow_job", "slow_c", "slow_d"), + job_nodes=("slow_a", "slow_b"), + build_nodes=("slow_a", "slow_b", "slow_c", "slow_d"), + expected_job_peak=1, + expected_least_build_peak=2, + environment_concurrency="8", + ), ], ids=lambda case: case.description, ) @@ -379,7 +391,9 @@ def test_given_a_job_concurrency_when_building_then_its_nodes_never_exceed_it( test_case: JobConcurrencyTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str], + monkeypatch: pytest.MonkeyPatch, ) -> None: + monkeypatch.setenv(CONCURRENCY_VARIABLE, test_case.environment_concurrency) write_project_files(project_dir=tmp_path, files=_PROJECT_FILES) events: Path = tmp_path / "events.jsonl" @@ -616,7 +630,9 @@ def test_given_a_named_job_when_planning_or_building_then_the_output_reflects_it test_case: JobOutputTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str], + monkeypatch: pytest.MonkeyPatch, ) -> None: + monkeypatch.delenv(CONCURRENCY_VARIABLE, raising=False) write_project_files(project_dir=tmp_path, files={**_PROJECT_FILES, **test_case.extra_files}) exit_code: int = main(["--project-dir", str(tmp_path), "--no-color", *test_case.command_args]) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_job_warehouses.py b/tests/integration/src/sqlbuild/cli/commands/main/test_job_warehouses.py index 4abb6083e3..3624d3147c 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_job_warehouses.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_job_warehouses.py @@ -13,6 +13,7 @@ JobWarehouseSwitchTestCase, ) from tests.integration.src.sqlbuild.cli.commands.main.helpers import ( + CONCURRENCY_VARIABLE, dynamic_refresh_warehouses, install_writable_offline_snowflake, run_cli_exit_code, @@ -94,6 +95,7 @@ def test_given_named_jobs_with_warehouses_when_building_then_each_node_runs_on_i monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str], ) -> None: + monkeypatch.delenv(CONCURRENCY_VARIABLE, raising=False) write_job_warehouse_project(project_dir=tmp_path) write_project_files(project_dir=tmp_path, files=test_case.extra_files) connect_calls, executed_sql = install_writable_offline_snowflake(monkeypatch) diff --git a/website/src/content/docs/docs/concepts/orchestration.mdx b/website/src/content/docs/docs/concepts/orchestration.mdx index 65ba928b69..9715517ccd 100644 --- a/website/src/content/docs/docs/concepts/orchestration.mdx +++ b/website/src/content/docs/docs/concepts/orchestration.mdx @@ -99,7 +99,8 @@ A command-line flag replaces every named job's setting rather than merging with settles named jobs that disagree: `--vars` replaces the job's whole `vars`, `--full-refresh` rebuilds every selected model, and `--warehouse` puts every node on its warehouse. An option that rules another out counts as setting both, so `--reload` wins over a job's `load_sources = false`. Any `--concurrency` sets the build's limit and turns off the -per-job limits. The job's settings are added before a `--` on the command line, so selectors after +per-job limits. `SQLBUILD_CONCURRENCY` is not a command-line flag: it sets the build's limit as a +configured one does, and the named jobs' limits still apply. The job's settings are added before a `--` on the command line, so selectors after it still name jobs. `--no-job-settings` leaves out every job's settings. Some settings apply to the job's own nodes only, so a build that names two jobs can treat their From 50f9fff7e78a88a28ac20502e35e8588b9900929 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 19 Nov 2026 13:39:19 +0000 Subject: [PATCH 21/22] fix(jobs): read the command line's job-limit setting from the settings table (CHI-1533) --- .../src/compile_driver/main/jobs/job_plan_fields.rs | 13 ++++++++----- .../src/sqlbuild/cli/commands/main/helpers.py | 6 ++++-- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/crates/sqlbuild-python-compile/src/compile_driver/main/jobs/job_plan_fields.rs b/crates/sqlbuild-python-compile/src/compile_driver/main/jobs/job_plan_fields.rs index a9b0dc3e64..9f976c25cd 100644 --- a/crates/sqlbuild-python-compile/src/compile_driver/main/jobs/job_plan_fields.rs +++ b/crates/sqlbuild-python-compile/src/compile_driver/main/jobs/job_plan_fields.rs @@ -1,12 +1,11 @@ //! The plan and pipeline fields the named jobs' settings give a planned build. use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_analysis::job_settings::main::setting_scope::setting_scope; +use sqlbuild_analysis::job_settings::models::SettingScope; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_project::job_settings::main::given_settings::given_settings; -/// The option that sets the build's limit and turns the named jobs' limits off when given. -const CONCURRENCY_SETTING: &str = "concurrency"; - use crate::compile_driver::models::PlannedJobSettings; use crate::compile_driver::types::PipelineFields; @@ -26,8 +25,12 @@ pub fn job_plan_fields<'py>( native_field(options, "ignore_audit_failures")?.is_truthy()?, ), )?; - let (limits, node_jobs) = - job_settings.job_limits(&plan_output, given_settings().contains(CONCURRENCY_SETTING))?; + let (limits, node_jobs) = job_settings.job_limits( + &plan_output, + given_settings() + .iter() + .any(|name| setting_scope(name) == SettingScope::JobLimit), + )?; let (nodes, python) = job_settings.node_warehouses(&plan_output)?; Ok(( plan_output, diff --git a/tests/integration/src/sqlbuild/cli/commands/main/helpers.py b/tests/integration/src/sqlbuild/cli/commands/main/helpers.py index 273cda92bd..8429997502 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/helpers.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/helpers.py @@ -1586,8 +1586,10 @@ def warehouse_switches(executed_sql: list[str]) -> tuple[str, ...]: uses: list[tuple[int, str]] = [ (connection, sql.removeprefix(_USE_WAREHOUSE)) - for connection, sql in _STATEMENT_CONNECTIONS.get(id(executed_sql), []) - if sql.startswith(_USE_WAREHOUSE) + for connection, sql in filter( + lambda statement: statement[1].startswith(_USE_WAREHOUSE), + _STATEMENT_CONNECTIONS.get(id(executed_sql), []), + ) ] first_use: dict[int, int] = { connection: index for index, (connection, _) in reversed(list(enumerate(uses))) From 15232e897dd702c26931a1a77490e9fda52d3e1c Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 19 Nov 2026 16:39:57 +0000 Subject: [PATCH 22/22] refactor(jobs): read the build plan's job fields through one accessor (CHI-1533) --- .../src/bindings/_helpers/build_command/execution.rs | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs index d913efaefb..85c0bb3ec6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs @@ -343,6 +343,7 @@ pub(crate) fn execute_build_plan<'py>( let lifecycle: &DirectPythonLifecycle = preparation.lifecycle.get(); let [start_ts, end_ts, start_int, end_int] = &preparation.cursors; let request = |name: &str| native_field(&context.request, name); + let planned = |name: &str| native_field(pipeline, name); let discovered: Bound<'py, PyAny> = context.get("discovered_inputs")?; let project_dir: Bound<'py, PyAny> = context.get("effective_project_dir")?; let target_config: TargetSettingsValues = @@ -381,12 +382,9 @@ pub(crate) fn execute_build_plan<'py>( ("end_cursor_int", end_int.clone()), ("use_color", boolean(py, context.use_color)), ("providers", providers.clone()), - ("job_limits", native_field(pipeline, "job_limits")?), - ("node_jobs", native_field(pipeline, "node_jobs")?), - ( - "node_warehouses", - native_field(pipeline, "node_warehouses")?, - ), + ("job_limits", planned("job_limits")?), + ("node_jobs", planned("node_jobs")?), + ("node_warehouses", planned("node_warehouses")?), ( "test_timeouts", timeout_record(