Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
23 commits
Select commit Hold shift + click to select a range
9170e45
feat(jobs): resolve the settings named jobs give their nodes (CHI-1533)
kvlonge Nov 18, 2026
4c536d7
feat(jobs): apply named jobs' settings to the nodes they select (CHI-…
kvlonge Nov 18, 2026
c65a468
feat(jobs): take named jobs' settings below the command line's own fl…
kvlonge Nov 18, 2026
848561e
fix(jobs): apply the settings of jobs a job: pattern names (CHI-1533)
kvlonge Nov 18, 2026
4a77931
fix(jobs): count microbatch sub-work against job limits in the build …
kvlonge Nov 18, 2026
96a2c53
fix(jobs): make job settings fail loudly and report per job (CHI-1533)
kvlonge Nov 18, 2026
25c821f
feat(jobs): let named jobs raise the build's limit and run server job…
kvlonge Nov 18, 2026
9c948d6
feat(jobs): run each named job's nodes on the job's warehouse (CHI-1533)
kvlonge Nov 18, 2026
bfb232c
refactor(jobs): move named-job settings into the project and compile …
kvlonge Nov 19, 2026
0ab6ff3
fix(jobs): apply job settings to everything a job selects and let the…
kvlonge Nov 19, 2026
03670fa
refactor(jobs): one job setting table and one resolver for named jobs…
kvlonge Nov 19, 2026
e24d55d
fix(jobs): singular audits weigh nodes outside the jobs, and conflict…
kvlonge Nov 19, 2026
3adfc86
fix(jobs): take --ignore-dynamic-grace into the job setting table (CH…
kvlonge Nov 19, 2026
301ff88
feat(jobs)!: remove the deprecated [on_failure] retry names (CHI-1533)
kvlonge Nov 19, 2026
3263600
fix(jobs): refresh dynamic models on the target's warehouse, check ig…
kvlonge Nov 19, 2026
33a4035
refactor(jobs): share one thread-local scope swap and read job wareho…
kvlonge Nov 19, 2026
f356e7f
fix(jobs): fail a microbatch model's warehouse switch with a model re…
kvlonge Nov 19, 2026
1dc4f83
test(jobs): read a failed node's message through the result helper (C…
kvlonge Nov 19, 2026
56100c9
fix(adapters): make the session warehouse switch an optional strict a…
kvlonge Nov 19, 2026
5ba1c25
fix(jobs): keep named jobs' concurrency limits under SQLBUILD_CONCURR…
kvlonge Nov 19, 2026
50f9fff
fix(jobs): read the command line's job-limit setting from the setting…
kvlonge Nov 19, 2026
feba644
Merge remote-tracking branch 'origin/main' into feat/jobs-phase1
kvlonge Nov 19, 2026
15232e8
refactor(jobs): read the build plan's job fields through one accessor…
kvlonge Nov 19, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
//! The build-wide settings of the named jobs, which must all agree.

use std::collections::BTreeMap;

use serde_json::Value as Json;

use crate::job_settings::errors::JobSettingConflict;
use crate::job_settings::main::conflict_help::conflict_help;
use crate::job_settings::main::setting_scope::setting_scope;
use crate::job_settings::models::{DeclaredRun, SettingScope};

fn settings_of<'a>(
job: &str,
declared: &'a BTreeMap<String, DeclaredRun>,
) -> Option<&'a BTreeMap<String, Json>> {
match declared.get(job) {
Some(DeclaredRun::Job { settings, .. }) => Some(settings),
_ => None,
}
}

fn shown(value: Option<&Json>) -> String {
value.map_or_else(|| "not set".to_owned(), Json::to_string)
}

/// The named jobs' build-wide settings, which must agree; unset differs from set.
pub(crate) fn build_job_settings(
named: &[String],
declared: &BTreeMap<String, DeclaredRun>,
) -> Result<BTreeMap<String, Json>, JobSettingConflict> {
let mut keys: Vec<&String> = named
.iter()
.filter_map(|job| settings_of(job, declared))
.flat_map(BTreeMap::keys)
.filter(|key| setting_scope(key) == SettingScope::Build)
.collect();
keys.sort_unstable();
keys.dedup();
let mut agreed: BTreeMap<String, Json> = BTreeMap::new();
for key in keys {
let mut first: Option<(&String, Option<&Json>)> = None;
for job in named {
let value: Option<&Json> =
settings_of(job, declared).and_then(|settings| settings.get(key));
match first {
None => first = Some((job, value)),
Some((other, seen)) if seen != value => {
return Err(JobSettingConflict {
message: format!(
"jobs {other} and {job} set {key} differently ({} and {}); a build \
applies {key} to everything it runs",
shown(seen),
shown(value)
),
help: conflict_help(key),
});
}
Some(_) => {}
}
}
if let Some((_, Some(value))) = first {
agreed.insert(key.clone(), value.clone());
}
}
Ok(agreed)
}
7 changes: 7 additions & 0 deletions crates/sqlbuild-analysis/src/job_settings/_helpers/mod.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
pub(crate) mod build_job_settings;
pub(crate) mod named_jobs;
pub(crate) mod node_job_settings;
pub(crate) mod node_values;
pub(crate) mod selection;
pub(crate) mod tokens;
pub(crate) mod without_given_settings;
26 changes: 26 additions & 0 deletions crates/sqlbuild-analysis/src/job_settings/_helpers/named_jobs.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
//! The jobs a selection names, as `job:<name>` or a bare job name; naming a group names its jobs.

use std::collections::BTreeMap;

use crate::job_settings::_helpers::tokens::token_jobs;
use crate::job_settings::models::DeclaredRun;

/// The jobs `selectors` name, each once in the order first named.
#[must_use]
pub(crate) fn named_jobs(
selectors: &[String],
declared: &BTreeMap<String, DeclaredRun>,
) -> Vec<String> {
let mut jobs: Vec<String> = Vec::new();
for token in selectors
.iter()
.flat_map(|selector| selector.split_whitespace())
{
for job in token_jobs(token, declared) {
if !jobs.contains(&job) {
jobs.push(job);
}
}
}
jobs
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
//! The settings each selected node takes from the jobs it is selected through.

use std::collections::BTreeMap;

use crate::graph::models::ProjectGraph;
use crate::job_settings::_helpers::node_values::node_values;
use crate::job_settings::_helpers::selection::node_jobs;
use crate::job_settings::errors::NodeJobSettingsError;
use crate::job_settings::main::setting_scope::setting_scope;
use crate::job_settings::models::{DeclaredRun, NodeJobSettings, SettingScope};
use crate::job_settings::types::TokenNodes;

/// Each node's named jobs, the settings they give it, and their limits.
pub(crate) fn node_job_settings(
graph: &ProjectGraph,
selection: (&[String], &[String]),
declared: &BTreeMap<String, DeclaredRun>,
token_nodes: TokenNodes<'_>,
) -> Result<NodeJobSettings, NodeJobSettingsError> {
let mut result = NodeJobSettings {
node_jobs: node_jobs(graph, selection, declared, token_nodes)
.map_err(NodeJobSettingsError::Selector)?,
..NodeJobSettings::default()
};
for (key, jobs) in &result.node_jobs {
let (values, retry) =
node_values(key, jobs, declared).map_err(NodeJobSettingsError::Conflict)?;
if !values.is_empty() {
result.values.insert(key.clone(), values);
}
if let Some(retry) = retry {
result.retries.insert(key.clone(), retry);
}
for job in jobs {
if let Some(DeclaredRun::Job { settings, .. }) = declared.get(job)
&& let Some(limit) = settings
.iter()
.find(|(name, _)| setting_scope(name) == SettingScope::JobLimit)
.and_then(|(_, value)| value.as_i64())
{
result.job_limits.insert(job.clone(), limit);
}
}
}
Ok(result)
}
95 changes: 95 additions & 0 deletions crates/sqlbuild-analysis/src/job_settings/_helpers/node_values.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,95 @@
//! The node-level settings and retry one node takes from the jobs it is selected through.

use std::collections::BTreeMap;

use serde_json::Value as Json;

use crate::assembly::project::types::ObjectKey;
use crate::definition_changes::constants::PYTHON_NODE_KINDS;
use crate::job_settings::constants::{JOB_SETTINGS, RETRY_SETTING};
use crate::job_settings::errors::JobSettingConflict;
use crate::job_settings::main::conflict_help::conflict_help;
use crate::job_settings::models::{DeclaredRun, JobRetry, SettingScope};

/// One node's settings, and its retry, each from whichever of its jobs sets it.
pub(crate) type NodeValues = (BTreeMap<String, Json>, Option<JobRetry>);

fn conflict(
key: &ObjectKey,
(first, second): (&str, &str),
setting: &str,
values: (String, String),
) -> JobSettingConflict {
JobSettingConflict {
message: format!(
"{}:{} is selected through jobs {first} and {second}, which set {setting} differently \
({} and {})",
key.0, key.1, values.0, values.1
),
help: conflict_help(setting),
}
}

fn shown(retry: JobRetry) -> String {
format!(
"{} attempts, {}s apart",
retry.attempts, retry.delay_seconds
)
}

/// The settings `jobs` give `key`; two jobs giving it different values of one setting conflict.
pub(crate) fn node_values(
key: &ObjectKey,
jobs: &[String],
declared: &BTreeMap<String, DeclaredRun>,
) -> Result<NodeValues, JobSettingConflict> {
let python: bool = PYTHON_NODE_KINDS.contains(&key.0.as_str());
let node_settings: Vec<&str> = JOB_SETTINGS
.iter()
.filter(|spec| spec.scope == SettingScope::Node && (spec.python_nodes || !python))
.map(|spec| spec.name)
.collect();
let mut values: BTreeMap<String, (&String, &Json)> = BTreeMap::new();
let mut retry: Option<(&String, JobRetry)> = None;
for job in jobs {
let Some(DeclaredRun::Job {
settings,
retry: job_retry,
}) = declared.get(job)
else {
continue;
};
for &setting in &node_settings {
let Some(value) = settings.get(setting) else {
continue;
};
match values.get(setting) {
Some((other, seen)) if *seen != value => {
let shown_values = (seen.to_string(), value.to_string());
return Err(conflict(key, (other, job), setting, shown_values));
}
Some(_) => {}
None => {
values.insert(setting.to_owned(), (job, value));
}
}
}
match (retry, job_retry.as_ref().filter(|_| !python)) {
(Some((other, seen)), Some(own)) if seen != *own => {
return Err(conflict(
key,
(other, job),
RETRY_SETTING,
(shown(seen), shown(*own)),
));
}
(None, Some(own)) => retry = Some((job, *own)),
_ => {}
}
}
let values: BTreeMap<String, Json> = values
.into_iter()
.map(|(name, (_, value))| (name, value.clone()))
.collect();
Ok((values, retry.map(|(_, retry)| retry)))
}
57 changes: 57 additions & 0 deletions crates/sqlbuild-analysis/src/job_settings/_helpers/selection.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
//! The named jobs each selected node is selected through.

use std::collections::{BTreeMap, BTreeSet};

use crate::assembly::project::types::ObjectKey;
use crate::graph::errors::SelectorError;
use crate::graph::models::ProjectGraph;
use crate::job_settings::_helpers::tokens::token_jobs;
use crate::job_settings::errors::JobSelectorError;
use crate::job_settings::models::DeclaredRun;
use crate::job_settings::types::TokenNodes;

/// Each node a `--select` token, less `--exclude`, selects through a job it names that holds it.
pub(crate) fn node_jobs(
graph: &ProjectGraph,
(select, exclude): (&[String], &[String]),
declared: &BTreeMap<String, DeclaredRun>,
token_nodes: TokenNodes<'_>,
) -> Result<BTreeMap<ObjectKey, Vec<String>>, JobSelectorError> {
let failed = |jobs: &[String], selector: &str| {
let (jobs, selector) = (jobs.to_vec(), selector.to_owned());
move |error: SelectorError| JobSelectorError {
jobs,
selector,
error,
}
};
let mut node_jobs: BTreeMap<ObjectKey, Vec<String>> = BTreeMap::new();
let resolved = graph.jobs.as_ref();
for token in select
.iter()
.flat_map(|selector| selector.split_whitespace())
{
let jobs: Vec<String> = token_jobs(token, declared);
if jobs.is_empty() {
continue;
}
let selected: BTreeSet<ObjectKey> =
token_nodes(token, exclude).map_err(failed(&jobs, token))?;
for job in &jobs {
let own: &BTreeSet<ObjectKey> = match resolved.and_then(|resolved| resolved.get(job)) {
Some(Ok(own)) => own,
Some(Err(error)) => {
return Err(failed(std::slice::from_ref(job), token)(error.clone()));
}
None => continue,
};
for key in selected.intersection(own) {
let jobs: &mut Vec<String> = node_jobs.entry(key.clone()).or_default();
if !jobs.contains(job) {
jobs.push(job.clone());
}
}
}
}
Ok(node_jobs)
}
38 changes: 38 additions & 0 deletions crates/sqlbuild-analysis/src/job_settings/_helpers/tokens.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
//! The jobs one selector token names.

use std::collections::BTreeMap;

use sqlbuild_core::text::main::glob_match::glob_match;

use crate::graph::constants::{INTERSECTION_SEPARATOR, JOB_KIND, NAME_KIND, PATTERN_CHARACTERS};
use crate::graph::main::parse_selector::parse_selector;
use crate::graph::models::ParsedSelector;
use crate::job_settings::models::DeclaredRun;

/// The jobs one selector token names, by name, `job:` pattern or group, each once.
#[must_use]
pub(crate) fn token_jobs(token: &str, declared: &BTreeMap<String, DeclaredRun>) -> Vec<String> {
let mut jobs: Vec<String> = Vec::new();
for part in token.split(INTERSECTION_SEPARATOR) {
let Ok(ParsedSelector::Kind { kind, value, .. }) = parse_selector(part) else {
continue;
};
if kind != JOB_KIND && kind != NAME_KIND {
continue;
}
let pattern: bool = kind == JOB_KIND && value.contains(PATTERN_CHARACTERS);
let named = declared
.iter()
.filter(|(name, _)| **name == value || (pattern && glob_match(name, &value)))
.flat_map(|(name, run)| match run {
DeclaredRun::Job { .. } => vec![name.clone()],
DeclaredRun::Group { members } => members.clone(),
});
for job in named {
if !jobs.contains(&job) {
jobs.push(job);
}
}
}
jobs
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
//! The declared jobs less the settings the command line gives, which replace theirs.

use std::collections::{BTreeMap, BTreeSet};

use crate::job_settings::models::DeclaredRun;

/// `declared` with each setting named in `given` removed from every job, so it never conflicts.
#[must_use]
pub(crate) fn without_given_settings(
mut declared: BTreeMap<String, DeclaredRun>,
given: &BTreeSet<String>,
) -> BTreeMap<String, DeclaredRun> {
for run in declared.values_mut() {
if let DeclaredRun::Job { settings, .. } = run {
settings.retain(|setting, _| !given.contains(setting));
}
}
declared
}
Loading
Loading