diff --git a/crates/sqlbuild-analysis/src/job_settings/_helpers/build_job_settings.rs b/crates/sqlbuild-analysis/src/job_settings/_helpers/build_job_settings.rs new file mode 100644 index 0000000000..b97d16cc20 --- /dev/null +++ b/crates/sqlbuild-analysis/src/job_settings/_helpers/build_job_settings.rs @@ -0,0 +1,66 @@ +//! The build-wide settings of the named jobs, which must all agree. + +use std::collections::BTreeMap; + +use serde_json::Value as Json; + +use crate::job_settings::errors::JobSettingConflict; +use crate::job_settings::main::conflict_help::conflict_help; +use crate::job_settings::main::setting_scope::setting_scope; +use crate::job_settings::models::{DeclaredRun, SettingScope}; + +fn settings_of<'a>( + job: &str, + declared: &'a BTreeMap, +) -> Option<&'a BTreeMap> { + match declared.get(job) { + Some(DeclaredRun::Job { settings, .. }) => Some(settings), + _ => None, + } +} + +fn shown(value: Option<&Json>) -> String { + value.map_or_else(|| "not set".to_owned(), Json::to_string) +} + +/// The named jobs' build-wide settings, which must agree; unset differs from set. +pub(crate) fn build_job_settings( + named: &[String], + declared: &BTreeMap, +) -> Result, JobSettingConflict> { + let mut keys: Vec<&String> = named + .iter() + .filter_map(|job| settings_of(job, declared)) + .flat_map(BTreeMap::keys) + .filter(|key| setting_scope(key) == SettingScope::Build) + .collect(); + keys.sort_unstable(); + keys.dedup(); + let mut agreed: BTreeMap = BTreeMap::new(); + for key in keys { + let mut first: Option<(&String, Option<&Json>)> = None; + for job in named { + let value: Option<&Json> = + settings_of(job, declared).and_then(|settings| settings.get(key)); + match first { + None => first = Some((job, value)), + Some((other, seen)) if seen != value => { + return Err(JobSettingConflict { + message: format!( + "jobs {other} and {job} set {key} differently ({} and {}); a build \ + applies {key} to everything it runs", + shown(seen), + shown(value) + ), + help: conflict_help(key), + }); + } + Some(_) => {} + } + } + if let Some((_, Some(value))) = first { + agreed.insert(key.clone(), value.clone()); + } + } + Ok(agreed) +} diff --git a/crates/sqlbuild-analysis/src/job_settings/_helpers/mod.rs b/crates/sqlbuild-analysis/src/job_settings/_helpers/mod.rs new file mode 100644 index 0000000000..69e5ae86ec --- /dev/null +++ b/crates/sqlbuild-analysis/src/job_settings/_helpers/mod.rs @@ -0,0 +1,7 @@ +pub(crate) mod build_job_settings; +pub(crate) mod named_jobs; +pub(crate) mod node_job_settings; +pub(crate) mod node_values; +pub(crate) mod selection; +pub(crate) mod tokens; +pub(crate) mod without_given_settings; diff --git a/crates/sqlbuild-analysis/src/job_settings/_helpers/named_jobs.rs b/crates/sqlbuild-analysis/src/job_settings/_helpers/named_jobs.rs new file mode 100644 index 0000000000..634c1e316c --- /dev/null +++ b/crates/sqlbuild-analysis/src/job_settings/_helpers/named_jobs.rs @@ -0,0 +1,26 @@ +//! The jobs a selection names, as `job:` or a bare job name; naming a group names its jobs. + +use std::collections::BTreeMap; + +use crate::job_settings::_helpers::tokens::token_jobs; +use crate::job_settings::models::DeclaredRun; + +/// The jobs `selectors` name, each once in the order first named. +#[must_use] +pub(crate) fn named_jobs( + selectors: &[String], + declared: &BTreeMap, +) -> Vec { + let mut jobs: Vec = Vec::new(); + for token in selectors + .iter() + .flat_map(|selector| selector.split_whitespace()) + { + for job in token_jobs(token, declared) { + if !jobs.contains(&job) { + jobs.push(job); + } + } + } + jobs +} diff --git a/crates/sqlbuild-analysis/src/job_settings/_helpers/node_job_settings.rs b/crates/sqlbuild-analysis/src/job_settings/_helpers/node_job_settings.rs new file mode 100644 index 0000000000..17900baec4 --- /dev/null +++ b/crates/sqlbuild-analysis/src/job_settings/_helpers/node_job_settings.rs @@ -0,0 +1,46 @@ +//! The settings each selected node takes from the jobs it is selected through. + +use std::collections::BTreeMap; + +use crate::graph::models::ProjectGraph; +use crate::job_settings::_helpers::node_values::node_values; +use crate::job_settings::_helpers::selection::node_jobs; +use crate::job_settings::errors::NodeJobSettingsError; +use crate::job_settings::main::setting_scope::setting_scope; +use crate::job_settings::models::{DeclaredRun, NodeJobSettings, SettingScope}; +use crate::job_settings::types::TokenNodes; + +/// Each node's named jobs, the settings they give it, and their limits. +pub(crate) fn node_job_settings( + graph: &ProjectGraph, + selection: (&[String], &[String]), + declared: &BTreeMap, + token_nodes: TokenNodes<'_>, +) -> Result { + let mut result = NodeJobSettings { + node_jobs: node_jobs(graph, selection, declared, token_nodes) + .map_err(NodeJobSettingsError::Selector)?, + ..NodeJobSettings::default() + }; + for (key, jobs) in &result.node_jobs { + let (values, retry) = + node_values(key, jobs, declared).map_err(NodeJobSettingsError::Conflict)?; + if !values.is_empty() { + result.values.insert(key.clone(), values); + } + if let Some(retry) = retry { + result.retries.insert(key.clone(), retry); + } + for job in jobs { + if let Some(DeclaredRun::Job { settings, .. }) = declared.get(job) + && let Some(limit) = settings + .iter() + .find(|(name, _)| setting_scope(name) == SettingScope::JobLimit) + .and_then(|(_, value)| value.as_i64()) + { + result.job_limits.insert(job.clone(), limit); + } + } + } + Ok(result) +} diff --git a/crates/sqlbuild-analysis/src/job_settings/_helpers/node_values.rs b/crates/sqlbuild-analysis/src/job_settings/_helpers/node_values.rs new file mode 100644 index 0000000000..db2f2befd8 --- /dev/null +++ b/crates/sqlbuild-analysis/src/job_settings/_helpers/node_values.rs @@ -0,0 +1,95 @@ +//! The node-level settings and retry one node takes from the jobs it is selected through. + +use std::collections::BTreeMap; + +use serde_json::Value as Json; + +use crate::assembly::project::types::ObjectKey; +use crate::definition_changes::constants::PYTHON_NODE_KINDS; +use crate::job_settings::constants::{JOB_SETTINGS, RETRY_SETTING}; +use crate::job_settings::errors::JobSettingConflict; +use crate::job_settings::main::conflict_help::conflict_help; +use crate::job_settings::models::{DeclaredRun, JobRetry, SettingScope}; + +/// One node's settings, and its retry, each from whichever of its jobs sets it. +pub(crate) type NodeValues = (BTreeMap, Option); + +fn conflict( + key: &ObjectKey, + (first, second): (&str, &str), + setting: &str, + values: (String, String), +) -> JobSettingConflict { + JobSettingConflict { + message: format!( + "{}:{} is selected through jobs {first} and {second}, which set {setting} differently \ + ({} and {})", + key.0, key.1, values.0, values.1 + ), + help: conflict_help(setting), + } +} + +fn shown(retry: JobRetry) -> String { + format!( + "{} attempts, {}s apart", + retry.attempts, retry.delay_seconds + ) +} + +/// The settings `jobs` give `key`; two jobs giving it different values of one setting conflict. +pub(crate) fn node_values( + key: &ObjectKey, + jobs: &[String], + declared: &BTreeMap, +) -> Result { + let python: bool = PYTHON_NODE_KINDS.contains(&key.0.as_str()); + let node_settings: Vec<&str> = JOB_SETTINGS + .iter() + .filter(|spec| spec.scope == SettingScope::Node && (spec.python_nodes || !python)) + .map(|spec| spec.name) + .collect(); + let mut values: BTreeMap = BTreeMap::new(); + let mut retry: Option<(&String, JobRetry)> = None; + for job in jobs { + let Some(DeclaredRun::Job { + settings, + retry: job_retry, + }) = declared.get(job) + else { + continue; + }; + for &setting in &node_settings { + let Some(value) = settings.get(setting) else { + continue; + }; + match values.get(setting) { + Some((other, seen)) if *seen != value => { + let shown_values = (seen.to_string(), value.to_string()); + return Err(conflict(key, (other, job), setting, shown_values)); + } + Some(_) => {} + None => { + values.insert(setting.to_owned(), (job, value)); + } + } + } + match (retry, job_retry.as_ref().filter(|_| !python)) { + (Some((other, seen)), Some(own)) if seen != *own => { + return Err(conflict( + key, + (other, job), + RETRY_SETTING, + (shown(seen), shown(*own)), + )); + } + (None, Some(own)) => retry = Some((job, *own)), + _ => {} + } + } + let values: BTreeMap = values + .into_iter() + .map(|(name, (_, value))| (name, value.clone())) + .collect(); + Ok((values, retry.map(|(_, retry)| retry))) +} diff --git a/crates/sqlbuild-analysis/src/job_settings/_helpers/selection.rs b/crates/sqlbuild-analysis/src/job_settings/_helpers/selection.rs new file mode 100644 index 0000000000..bee329397c --- /dev/null +++ b/crates/sqlbuild-analysis/src/job_settings/_helpers/selection.rs @@ -0,0 +1,57 @@ +//! The named jobs each selected node is selected through. + +use std::collections::{BTreeMap, BTreeSet}; + +use crate::assembly::project::types::ObjectKey; +use crate::graph::errors::SelectorError; +use crate::graph::models::ProjectGraph; +use crate::job_settings::_helpers::tokens::token_jobs; +use crate::job_settings::errors::JobSelectorError; +use crate::job_settings::models::DeclaredRun; +use crate::job_settings::types::TokenNodes; + +/// Each node a `--select` token, less `--exclude`, selects through a job it names that holds it. +pub(crate) fn node_jobs( + graph: &ProjectGraph, + (select, exclude): (&[String], &[String]), + declared: &BTreeMap, + token_nodes: TokenNodes<'_>, +) -> Result>, JobSelectorError> { + let failed = |jobs: &[String], selector: &str| { + let (jobs, selector) = (jobs.to_vec(), selector.to_owned()); + move |error: SelectorError| JobSelectorError { + jobs, + selector, + error, + } + }; + let mut node_jobs: BTreeMap> = BTreeMap::new(); + let resolved = graph.jobs.as_ref(); + for token in select + .iter() + .flat_map(|selector| selector.split_whitespace()) + { + let jobs: Vec = token_jobs(token, declared); + if jobs.is_empty() { + continue; + } + let selected: BTreeSet = + token_nodes(token, exclude).map_err(failed(&jobs, token))?; + for job in &jobs { + let own: &BTreeSet = match resolved.and_then(|resolved| resolved.get(job)) { + Some(Ok(own)) => own, + Some(Err(error)) => { + return Err(failed(std::slice::from_ref(job), token)(error.clone())); + } + None => continue, + }; + for key in selected.intersection(own) { + let jobs: &mut Vec = node_jobs.entry(key.clone()).or_default(); + if !jobs.contains(job) { + jobs.push(job.clone()); + } + } + } + } + Ok(node_jobs) +} diff --git a/crates/sqlbuild-analysis/src/job_settings/_helpers/tokens.rs b/crates/sqlbuild-analysis/src/job_settings/_helpers/tokens.rs new file mode 100644 index 0000000000..5a62fa37b9 --- /dev/null +++ b/crates/sqlbuild-analysis/src/job_settings/_helpers/tokens.rs @@ -0,0 +1,38 @@ +//! The jobs one selector token names. + +use std::collections::BTreeMap; + +use sqlbuild_core::text::main::glob_match::glob_match; + +use crate::graph::constants::{INTERSECTION_SEPARATOR, JOB_KIND, NAME_KIND, PATTERN_CHARACTERS}; +use crate::graph::main::parse_selector::parse_selector; +use crate::graph::models::ParsedSelector; +use crate::job_settings::models::DeclaredRun; + +/// The jobs one selector token names, by name, `job:` pattern or group, each once. +#[must_use] +pub(crate) fn token_jobs(token: &str, declared: &BTreeMap) -> Vec { + let mut jobs: Vec = Vec::new(); + for part in token.split(INTERSECTION_SEPARATOR) { + let Ok(ParsedSelector::Kind { kind, value, .. }) = parse_selector(part) else { + continue; + }; + if kind != JOB_KIND && kind != NAME_KIND { + continue; + } + let pattern: bool = kind == JOB_KIND && value.contains(PATTERN_CHARACTERS); + let named = declared + .iter() + .filter(|(name, _)| **name == value || (pattern && glob_match(name, &value))) + .flat_map(|(name, run)| match run { + DeclaredRun::Job { .. } => vec![name.clone()], + DeclaredRun::Group { members } => members.clone(), + }); + for job in named { + if !jobs.contains(&job) { + jobs.push(job); + } + } + } + jobs +} diff --git a/crates/sqlbuild-analysis/src/job_settings/_helpers/without_given_settings.rs b/crates/sqlbuild-analysis/src/job_settings/_helpers/without_given_settings.rs new file mode 100644 index 0000000000..7889a73ffd --- /dev/null +++ b/crates/sqlbuild-analysis/src/job_settings/_helpers/without_given_settings.rs @@ -0,0 +1,19 @@ +//! The declared jobs less the settings the command line gives, which replace theirs. + +use std::collections::{BTreeMap, BTreeSet}; + +use crate::job_settings::models::DeclaredRun; + +/// `declared` with each setting named in `given` removed from every job, so it never conflicts. +#[must_use] +pub(crate) fn without_given_settings( + mut declared: BTreeMap, + given: &BTreeSet, +) -> BTreeMap { + for run in declared.values_mut() { + if let DeclaredRun::Job { settings, .. } = run { + settings.retain(|setting, _| !given.contains(setting)); + } + } + declared +} diff --git a/crates/sqlbuild-analysis/src/job_settings/constants.rs b/crates/sqlbuild-analysis/src/job_settings/constants.rs new file mode 100644 index 0000000000..566e04e0f7 --- /dev/null +++ b/crates/sqlbuild-analysis/src/job_settings/constants.rs @@ -0,0 +1,548 @@ +//! Every build option as a job setting, and the wording of job setting conflicts. + +use crate::job_settings::models::GroupAgreement::{EachBuild, OneBuild}; +use crate::job_settings::models::SettingScope::{Build, JobLimit, Node, RunRequest}; +use crate::job_settings::models::{CommandLineValues, JobSettingSpec, NodeConsumer}; + +/// The warehouse the job's nodes run on. +pub const WAREHOUSE_SETTING: &str = "warehouse"; +/// The help of two named jobs giving one node different `[retry]` tables, which no flag overrides. +pub const RETRY_CONFLICT_HELP: &str = + "make the jobs agree, set [retry] in only one of them, or build with --no-job-settings"; +/// The error code of job settings that cannot apply together. +pub const JOB_SETTING_CONFLICT_CODE: &str = "S027"; +/// How a conflict names a job's `[retry]` table. +pub const RETRY_SETTING: &str = "[retry]"; +/// Every option of `sqb build`, in the parser's order, as a job setting. +pub const JOB_SETTINGS: [JobSettingSpec; 53] = [ + JobSettingSpec { + name: "help", + options: &["-h", "--help"], + scope: RunRequest, + python_nodes: false, + whole_run: false, + command_line: CommandLineValues::Any, + consumer: None, + group: OneBuild, + }, + JobSettingSpec { + name: "no_sql_validation", + options: &["--no-sql-analysis", "--no-sql-validation"], + scope: Build, + python_nodes: false, + whole_run: false, + command_line: CommandLineValues::OnlyTrue, + consumer: None, + group: OneBuild, + }, + JobSettingSpec { + name: "no_cache", + options: &["--no-cache"], + scope: Build, + python_nodes: false, + whole_run: false, + command_line: CommandLineValues::OnlyTrue, + consumer: None, + group: OneBuild, + }, + JobSettingSpec { + name: "defer_to", + options: &["--defer-to"], + scope: Build, + python_nodes: false, + whole_run: false, + command_line: CommandLineValues::Any, + consumer: None, + group: OneBuild, + }, + JobSettingSpec { + name: "defer_clone_from", + options: &["--defer-clone-from"], + scope: Build, + python_nodes: false, + whole_run: false, + command_line: CommandLineValues::Any, + consumer: None, + group: OneBuild, + }, + JobSettingSpec { + name: "defer_sources_to", + options: &["--defer-sources-to"], + scope: Build, + python_nodes: false, + whole_run: false, + command_line: CommandLineValues::Any, + consumer: None, + group: OneBuild, + }, + JobSettingSpec { + name: "plan_from", + options: &["--plan-from"], + scope: Build, + python_nodes: false, + whole_run: false, + command_line: CommandLineValues::Any, + consumer: None, + group: OneBuild, + }, + JobSettingSpec { + name: "target", + options: &["--target"], + scope: RunRequest, + python_nodes: false, + whole_run: false, + command_line: CommandLineValues::Any, + consumer: None, + group: OneBuild, + }, + JobSettingSpec { + name: "json", + options: &["--json"], + scope: RunRequest, + python_nodes: false, + whole_run: false, + command_line: CommandLineValues::OnlyTrue, + consumer: None, + group: OneBuild, + }, + JobSettingSpec { + name: "selection_diagnostics", + options: &["--selection-diagnostics"], + scope: Build, + python_nodes: false, + whole_run: false, + command_line: CommandLineValues::OnlyTrue, + consumer: None, + group: OneBuild, + }, + JobSettingSpec { + name: "include_python", + options: &["--no-python"], + scope: Build, + python_nodes: false, + whole_run: false, + command_line: CommandLineValues::OnlyFalse, + consumer: None, + group: OneBuild, + }, + JobSettingSpec { + name: "run_tests", + options: &["--no-tests"], + scope: Node, + python_nodes: false, + whole_run: false, + command_line: CommandLineValues::OnlyFalse, + consumer: Some(NodeConsumer::TestEntries), + group: OneBuild, + }, + JobSettingSpec { + name: "run_audits", + options: &["--no-audits"], + scope: Build, + python_nodes: false, + whole_run: false, + command_line: CommandLineValues::OnlyFalse, + consumer: None, + group: OneBuild, + }, + JobSettingSpec { + name: "ignore_audit_failures", + options: &["--ignore-audit-failures", "--no-ignore-audit-failures"], + scope: Node, + python_nodes: false, + whole_run: false, + command_line: CommandLineValues::Any, + consumer: Some(NodeConsumer::AuditEntries), + group: OneBuild, + }, + JobSettingSpec { + name: "manifest", + options: &["--manifest"], + scope: RunRequest, + python_nodes: false, + whole_run: false, + command_line: CommandLineValues::OnlyTrue, + consumer: None, + group: OneBuild, + }, + JobSettingSpec { + name: "json_output", + options: &["--json-output"], + scope: RunRequest, + python_nodes: false, + whole_run: false, + command_line: CommandLineValues::Any, + consumer: None, + group: OneBuild, + }, + JobSettingSpec { + name: "event_output", + options: &["--event-output"], + scope: RunRequest, + python_nodes: false, + whole_run: false, + command_line: CommandLineValues::Any, + consumer: None, + group: OneBuild, + }, + JobSettingSpec { + name: "start_cursor", + options: &["--start-cursor"], + scope: Build, + python_nodes: false, + whole_run: false, + command_line: CommandLineValues::Any, + consumer: None, + group: OneBuild, + }, + JobSettingSpec { + name: "end_cursor", + options: &["--end-cursor"], + scope: Build, + python_nodes: false, + whole_run: false, + command_line: CommandLineValues::Any, + consumer: None, + group: OneBuild, + }, + JobSettingSpec { + name: "start_cursor_ts", + options: &["--start-cursor-ts"], + scope: Build, + python_nodes: false, + whole_run: false, + command_line: CommandLineValues::Any, + consumer: None, + group: OneBuild, + }, + JobSettingSpec { + name: "end_cursor_ts", + options: &["--end-cursor-ts"], + scope: Build, + python_nodes: false, + whole_run: false, + command_line: CommandLineValues::Any, + consumer: None, + group: OneBuild, + }, + JobSettingSpec { + name: "start_cursor_int", + options: &["--start-cursor-int"], + scope: Build, + python_nodes: false, + whole_run: false, + command_line: CommandLineValues::Any, + consumer: None, + group: OneBuild, + }, + JobSettingSpec { + name: "end_cursor_int", + options: &["--end-cursor-int"], + scope: Build, + python_nodes: false, + whole_run: false, + command_line: CommandLineValues::Any, + consumer: None, + group: OneBuild, + }, + JobSettingSpec { + name: "max_microbatches", + options: &["--max-microbatches"], + scope: Build, + python_nodes: false, + whole_run: false, + command_line: CommandLineValues::Any, + consumer: None, + group: OneBuild, + }, + JobSettingSpec { + name: "load_sources", + options: &["--load", "--no-load"], + scope: Build, + python_nodes: false, + whole_run: false, + command_line: CommandLineValues::Any, + consumer: None, + group: OneBuild, + }, + JobSettingSpec { + name: "reload", + options: &["--reload"], + scope: Build, + python_nodes: false, + whole_run: false, + command_line: CommandLineValues::OnlyTrue, + consumer: None, + group: OneBuild, + }, + JobSettingSpec { + name: "fail_fast", + options: &["--fail-fast"], + scope: Build, + python_nodes: false, + whole_run: false, + command_line: CommandLineValues::OnlyTrue, + consumer: None, + group: OneBuild, + }, + JobSettingSpec { + name: "full_refresh", + options: &["--full-refresh"], + scope: Node, + python_nodes: false, + whole_run: false, + command_line: CommandLineValues::OnlyTrue, + consumer: Some(NodeConsumer::RefreshOverrides), + group: OneBuild, + }, + JobSettingSpec { + name: "job_settings", + options: &["--no-job-settings"], + scope: RunRequest, + python_nodes: false, + whole_run: false, + command_line: CommandLineValues::OnlyFalse, + consumer: None, + group: OneBuild, + }, + JobSettingSpec { + name: "changed", + options: &["--changed"], + scope: Build, + python_nodes: false, + whole_run: false, + command_line: CommandLineValues::Any, + consumer: None, + group: EachBuild, + }, + JobSettingSpec { + name: "reach", + options: &["--reach"], + scope: Build, + python_nodes: false, + whole_run: false, + command_line: CommandLineValues::Any, + consumer: None, + group: OneBuild, + }, + JobSettingSpec { + name: "allow_snapshot_full_refresh", + options: &["--allow-snapshot-full-refresh"], + scope: Build, + python_nodes: false, + whole_run: false, + command_line: CommandLineValues::OnlyTrue, + consumer: None, + group: OneBuild, + }, + JobSettingSpec { + name: "allow_table_type_downgrade", + options: &["--allow-table-type-downgrade"], + scope: Build, + python_nodes: false, + whole_run: false, + command_line: CommandLineValues::OnlyTrue, + consumer: None, + group: OneBuild, + }, + JobSettingSpec { + name: "allow_retention_decrease", + options: &["--allow-retention-decrease"], + scope: Build, + python_nodes: false, + whole_run: false, + command_line: CommandLineValues::OnlyTrue, + consumer: None, + group: OneBuild, + }, + JobSettingSpec { + name: "allow_missing_migration_origin", + options: &["--allow-missing-migration-origin"], + scope: Build, + python_nodes: false, + whole_run: false, + command_line: CommandLineValues::OnlyTrue, + consumer: None, + group: OneBuild, + }, + JobSettingSpec { + name: "allow_snapshot_schema_change", + options: &["--allow-snapshot-schema-change"], + scope: Build, + python_nodes: false, + whole_run: false, + command_line: CommandLineValues::OnlyTrue, + consumer: None, + group: OneBuild, + }, + JobSettingSpec { + name: "ignore_dynamic_grace", + options: &["--ignore-dynamic-grace"], + scope: Build, + python_nodes: false, + whole_run: false, + command_line: CommandLineValues::OnlyTrue, + consumer: None, + group: OneBuild, + }, + JobSettingSpec { + name: "concurrency", + options: &["--concurrency"], + scope: JobLimit, + python_nodes: false, + whole_run: false, + command_line: CommandLineValues::Any, + consumer: None, + group: OneBuild, + }, + JobSettingSpec { + name: "test_timeout", + options: &["--test-timeout"], + scope: Build, + python_nodes: false, + whole_run: false, + command_line: CommandLineValues::Any, + consumer: None, + group: OneBuild, + }, + JobSettingSpec { + name: "verbose", + options: &["--verbose", "-v"], + scope: Build, + python_nodes: false, + whole_run: false, + command_line: CommandLineValues::OnlyTrue, + consumer: None, + group: OneBuild, + }, + JobSettingSpec { + name: "select", + options: &["--select", "-s"], + scope: RunRequest, + python_nodes: false, + whole_run: false, + command_line: CommandLineValues::Any, + consumer: None, + group: OneBuild, + }, + JobSettingSpec { + name: "select_file", + options: &["--select-file"], + scope: RunRequest, + python_nodes: false, + whole_run: false, + command_line: CommandLineValues::Any, + consumer: None, + group: OneBuild, + }, + JobSettingSpec { + name: "exclude", + options: &["--exclude"], + scope: RunRequest, + python_nodes: false, + whole_run: false, + command_line: CommandLineValues::Any, + consumer: None, + group: OneBuild, + }, + JobSettingSpec { + name: "positional_select", + options: &[], + scope: RunRequest, + python_nodes: false, + whole_run: false, + command_line: CommandLineValues::Any, + consumer: None, + group: OneBuild, + }, + JobSettingSpec { + name: "vars", + options: &["--vars"], + scope: Build, + python_nodes: false, + whole_run: false, + command_line: CommandLineValues::Any, + consumer: None, + group: EachBuild, + }, + JobSettingSpec { + name: "dbt_project_dir", + options: &["--dbt-project-dir"], + scope: RunRequest, + python_nodes: false, + whole_run: false, + command_line: CommandLineValues::Any, + consumer: None, + group: OneBuild, + }, + JobSettingSpec { + name: "dbt_profiles_dir", + options: &["--dbt-profiles-dir"], + scope: RunRequest, + python_nodes: false, + whole_run: false, + command_line: CommandLineValues::Any, + consumer: None, + group: OneBuild, + }, + JobSettingSpec { + name: "dbt_target", + options: &["--dbt-target"], + scope: RunRequest, + python_nodes: false, + whole_run: false, + command_line: CommandLineValues::Any, + consumer: None, + group: OneBuild, + }, + JobSettingSpec { + name: "dbt_target_path", + options: &["--dbt-target-path"], + scope: RunRequest, + python_nodes: false, + whole_run: false, + command_line: CommandLineValues::Any, + consumer: None, + group: OneBuild, + }, + JobSettingSpec { + name: WAREHOUSE_SETTING, + options: &["--warehouse"], + scope: Node, + python_nodes: true, + whole_run: true, + command_line: CommandLineValues::Any, + consumer: Some(NodeConsumer::NodeWarehouses), + group: OneBuild, + }, + JobSettingSpec { + name: "project_dir", + options: &["--project-dir", "--sqb-project-dir"], + scope: RunRequest, + python_nodes: false, + whole_run: false, + command_line: CommandLineValues::Any, + consumer: None, + group: OneBuild, + }, + JobSettingSpec { + name: "no_color", + options: &["--no-color"], + scope: Build, + python_nodes: false, + whole_run: false, + command_line: CommandLineValues::OnlyTrue, + consumer: None, + group: OneBuild, + }, + JobSettingSpec { + name: "debug", + options: &["--debug"], + scope: Build, + python_nodes: false, + whole_run: false, + command_line: CommandLineValues::OnlyTrue, + consumer: None, + group: OneBuild, + }, +]; diff --git a/crates/sqlbuild-analysis/src/job_settings/errors.rs b/crates/sqlbuild-analysis/src/job_settings/errors.rs new file mode 100644 index 0000000000..6c131cbc82 --- /dev/null +++ b/crates/sqlbuild-analysis/src/job_settings/errors.rs @@ -0,0 +1,25 @@ +//! Job settings that cannot apply together. + +use crate::graph::errors::SelectorError; + +/// Two named jobs give one node, or the whole build, different values of one setting. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct JobSettingConflict { + pub message: String, + pub help: String, +} + +/// A selector token that names jobs, and why it failed to select their nodes. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct JobSelectorError { + pub jobs: Vec, + pub selector: String, + pub error: SelectorError, +} + +/// Why the jobs a selection names cannot give their nodes settings. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum NodeJobSettingsError { + Selector(JobSelectorError), + Conflict(JobSettingConflict), +} diff --git a/crates/sqlbuild-analysis/src/job_settings/main/conflict_help.rs b/crates/sqlbuild-analysis/src/job_settings/main/conflict_help.rs new file mode 100644 index 0000000000..4f94ec4eb7 --- /dev/null +++ b/crates/sqlbuild-analysis/src/job_settings/main/conflict_help.rs @@ -0,0 +1,29 @@ +//! The help a job setting conflict gives, from the setting's row in the table. + +use crate::job_settings::constants::{RETRY_CONFLICT_HELP, RETRY_SETTING}; +use crate::job_settings::main::job_setting_spec::job_setting_spec; +use crate::job_settings::models::{CommandLineValues, SettingScope}; + +/// How to settle named jobs disagreeing on `setting`; a flag only for the value it can give. +#[must_use] +pub fn conflict_help(setting: &str) -> String { + let Some(spec) = job_setting_spec(setting) else { + return if setting == RETRY_SETTING { + RETRY_CONFLICT_HELP.to_owned() + } else { + "make the jobs agree, or build with --no-job-settings".to_owned() + }; + }; + let only_one: &str = if spec.scope == SettingScope::Node { + "set it in only one of them, " + } else { + "" + }; + let options: String = spec.options.join(" or "); + let flag: String = match spec.command_line { + CommandLineValues::Any => format!("override it with {options}, "), + CommandLineValues::OnlyTrue => format!("pass {options} to make it true for all, "), + CommandLineValues::OnlyFalse => format!("pass {options} to make it false for all, "), + }; + format!("make the jobs agree, {only_one}{flag}or build with --no-job-settings") +} diff --git a/crates/sqlbuild-analysis/src/job_settings/main/consumed_setting.rs b/crates/sqlbuild-analysis/src/job_settings/main/consumed_setting.rs new file mode 100644 index 0000000000..8665eae167 --- /dev/null +++ b/crates/sqlbuild-analysis/src/job_settings/main/consumed_setting.rs @@ -0,0 +1,13 @@ +//! The node setting one plan consumer applies. + +use crate::job_settings::constants::JOB_SETTINGS; +use crate::job_settings::models::NodeConsumer; + +/// The node setting `consumer` applies, from the table; `None` if no row names it. +#[must_use] +pub fn consumed_setting(consumer: NodeConsumer) -> Option<&'static str> { + JOB_SETTINGS + .iter() + .find(|spec| spec.consumer == Some(consumer)) + .map(|spec| spec.name) +} diff --git a/crates/sqlbuild-analysis/src/job_settings/main/declared_runs.rs b/crates/sqlbuild-analysis/src/job_settings/main/declared_runs.rs new file mode 100644 index 0000000000..0ddafefb85 --- /dev/null +++ b/crates/sqlbuild-analysis/src/job_settings/main/declared_runs.rs @@ -0,0 +1,45 @@ +//! The jobs and groups a target declares, with each job's settings and retry. + +use std::collections::BTreeMap; + +use sqlbuild_discovery::orchestration_files::models::OrchestrationInventory; + +use crate::job_settings::models::{DeclaredRun, JobRetry}; + +/// Each job's settings and retry, and each group's jobs, declared for `target`. +#[must_use] +pub fn declared_runs( + inventory: &OrchestrationInventory, + target: Option<&str>, +) -> BTreeMap { + let declared_for = |targets: &[String]| { + targets.is_empty() || target.is_some_and(|target| targets.iter().any(|name| name == target)) + }; + let mut declared: BTreeMap = BTreeMap::new(); + for (_, job) in &inventory.jobs { + if declared_for(&job.run.targets) { + declared.insert( + job.run.name.value.clone(), + DeclaredRun::Job { + settings: job.settings.clone(), + retry: job.run.retry.map(|retry| JobRetry { + attempts: retry.attempts, + delay_seconds: retry.delay_seconds, + }), + }, + ); + } + } + for (_, group) in &inventory.groups { + if declared_for(&group.run.targets) { + let members: Vec = group + .jobs + .iter() + .filter(|job| declared.contains_key(&job.value)) + .map(|job| job.value.clone()) + .collect(); + declared.insert(group.run.name.value.clone(), DeclaredRun::Group { members }); + } + } + declared +} diff --git a/crates/sqlbuild-analysis/src/job_settings/main/group_setting_mismatch.rs b/crates/sqlbuild-analysis/src/job_settings/main/group_setting_mismatch.rs new file mode 100644 index 0000000000..33fdd84323 --- /dev/null +++ b/crates/sqlbuild-analysis/src/job_settings/main/group_setting_mismatch.rs @@ -0,0 +1,64 @@ +//! A group's jobs must agree on `vars` and `changed` on each target that declares them. + +use std::collections::{BTreeMap, BTreeSet}; + +use sqlbuild_discovery::orchestration_files::models::{GroupFile, OrchestrationInventory}; + +use crate::job_settings::_helpers::build_job_settings::build_job_settings; +use crate::job_settings::main::declared_runs::declared_runs; +use crate::job_settings::main::job_setting_spec::job_setting_spec; +use crate::job_settings::models::{DeclaredRun, GroupAgreement}; + +/// `run` keeping only the settings a group's jobs must agree on. +fn agreed_settings_only(run: DeclaredRun) -> DeclaredRun { + match run { + DeclaredRun::Job { settings, retry } => DeclaredRun::Job { + settings: settings + .into_iter() + .filter(|(name, _)| { + job_setting_spec(name) + .is_some_and(|spec| spec.group == GroupAgreement::EachBuild) + }) + .collect(), + retry, + }, + group => group, + } +} + +/// Why `group` cannot run as one build on some target, as building it would say; `None` if never. +#[must_use] +pub fn group_setting_mismatch( + inventory: &OrchestrationInventory, + group: &GroupFile, +) -> Option { + let mut targets: BTreeSet> = BTreeSet::from([None]); + targets.extend(group.run.targets.iter().map(|target| Some(target.as_str()))); + for (_, job) in &inventory.jobs { + if group + .jobs + .iter() + .any(|member| member.value == job.run.name.value) + { + targets.extend(job.run.targets.iter().map(|target| Some(target.as_str()))); + } + } + for target in targets { + let declared: BTreeMap = declared_runs(inventory, target) + .into_iter() + .map(|(name, run)| (name, agreed_settings_only(run))) + .collect(); + let Some(DeclaredRun::Group { members }) = declared.get(&group.run.name.value) else { + continue; + }; + if let Err(conflict) = build_job_settings(members, &declared) { + let on: String = + target.map_or_else(String::new, |target| format!(" on target {target}")); + return Some(format!( + "{}{on}; group {} builds its jobs together", + conflict.message, group.run.name.value + )); + } + } + None +} diff --git a/crates/sqlbuild-analysis/src/job_settings/main/job_setting_spec.rs b/crates/sqlbuild-analysis/src/job_settings/main/job_setting_spec.rs new file mode 100644 index 0000000000..081b254f7f --- /dev/null +++ b/crates/sqlbuild-analysis/src/job_settings/main/job_setting_spec.rs @@ -0,0 +1,10 @@ +//! The table row of one job setting. + +use crate::job_settings::constants::JOB_SETTINGS; +use crate::job_settings::models::JobSettingSpec; + +/// The row of setting `name`; `None` for a name no build option has. +#[must_use] +pub fn job_setting_spec(name: &str) -> Option<&'static JobSettingSpec> { + JOB_SETTINGS.iter().find(|spec| spec.name == name) +} diff --git a/crates/sqlbuild-analysis/src/job_settings/main/mod.rs b/crates/sqlbuild-analysis/src/job_settings/main/mod.rs new file mode 100644 index 0000000000..f98a28f2a8 --- /dev/null +++ b/crates/sqlbuild-analysis/src/job_settings/main/mod.rs @@ -0,0 +1,7 @@ +pub mod conflict_help; +pub mod consumed_setting; +pub mod declared_runs; +pub mod group_setting_mismatch; +pub mod job_setting_spec; +pub mod resolve_job_settings; +pub mod setting_scope; diff --git a/crates/sqlbuild-analysis/src/job_settings/main/resolve_job_settings.rs b/crates/sqlbuild-analysis/src/job_settings/main/resolve_job_settings.rs new file mode 100644 index 0000000000..b57c24ab3a --- /dev/null +++ b/crates/sqlbuild-analysis/src/job_settings/main/resolve_job_settings.rs @@ -0,0 +1,67 @@ +//! The one place named jobs' settings meet the command line's: command line > named job. + +use std::collections::{BTreeMap, BTreeSet}; + +use serde_json::Value as Json; + +use crate::job_settings::_helpers::build_job_settings::build_job_settings; +use crate::job_settings::_helpers::named_jobs::named_jobs; +use crate::job_settings::_helpers::node_job_settings::node_job_settings; +use crate::job_settings::_helpers::without_given_settings::without_given_settings; +use crate::job_settings::constants::JOB_SETTINGS; +use crate::job_settings::errors::NodeJobSettingsError; +use crate::job_settings::models::{DeclaredRun, ResolvedJobSettings}; +use crate::job_settings::types::NodeSelection; + +/// The value of whole-run setting `name` every named job sets, if they all set the same one. +fn shared_value( + name: &str, + named: &[String], + declared: &BTreeMap, +) -> Option { + let mut values = named.iter().map(|job| match declared.get(job) { + Some(DeclaredRun::Job { settings, .. }) => settings.get(name), + _ => None, + }); + let first: &Json = values.next()??; + values + .all(|value| value == Some(first)) + .then(|| first.clone()) +} + +/// The named jobs' settings less those `given`, then checked for conflicts; per node with `nodes`. +pub fn resolve_job_settings( + declared: BTreeMap, + (selectors, given): (&[String], &BTreeSet), + nodes: Option>, +) -> Result { + let declared: BTreeMap = without_given_settings(declared, given); + let named: Vec = named_jobs(selectors, &declared); + if named.is_empty() { + return Ok(ResolvedJobSettings { + declared, + ..ResolvedJobSettings::default() + }); + } + let build: BTreeMap = + build_job_settings(&named, &declared).map_err(NodeJobSettingsError::Conflict)?; + let mut whole_run: BTreeMap = BTreeMap::new(); + for spec in JOB_SETTINGS.iter().filter(|spec| spec.whole_run) { + if let Some(value) = shared_value(spec.name, &named, &declared) { + whole_run.insert(spec.name.to_owned(), value); + } + } + let nodes = match nodes { + Some((graph, selection, token_nodes)) => { + Some(node_job_settings(graph, selection, &declared, token_nodes)?) + } + None => None, + }; + Ok(ResolvedJobSettings { + declared, + named, + build, + whole_run, + nodes, + }) +} diff --git a/crates/sqlbuild-analysis/src/job_settings/main/setting_scope.rs b/crates/sqlbuild-analysis/src/job_settings/main/setting_scope.rs new file mode 100644 index 0000000000..2b6bfbd5d8 --- /dev/null +++ b/crates/sqlbuild-analysis/src/job_settings/main/setting_scope.rs @@ -0,0 +1,10 @@ +//! How far one job setting reaches. + +use crate::job_settings::main::job_setting_spec::job_setting_spec; +use crate::job_settings::models::SettingScope; + +/// The setting's scope from the table; build-wide for a name the parser then rejects. +#[must_use] +pub fn setting_scope(name: &str) -> SettingScope { + job_setting_spec(name).map_or(SettingScope::Build, |spec| spec.scope) +} diff --git a/crates/sqlbuild-analysis/src/job_settings/mod.rs b/crates/sqlbuild-analysis/src/job_settings/mod.rs new file mode 100644 index 0000000000..7e40b4e8fa --- /dev/null +++ b/crates/sqlbuild-analysis/src/job_settings/mod.rs @@ -0,0 +1,11 @@ +//! Jobs as named selections: which jobs a selection names and the settings they give its nodes. + +pub(crate) mod _helpers; +pub mod constants; +pub mod errors; +pub mod main; +pub mod models; +pub mod types; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-analysis/src/job_settings/models.rs b/crates/sqlbuild-analysis/src/job_settings/models.rs new file mode 100644 index 0000000000..859f11e232 --- /dev/null +++ b/crates/sqlbuild-analysis/src/job_settings/models.rs @@ -0,0 +1,119 @@ +//! Declared jobs and groups, and the settings a selection gives its nodes. + +use std::collections::BTreeMap; + +use serde_json::Value as Json; + +use crate::assembly::project::types::ObjectKey; + +/// A job's `[retry]`: at most `attempts` runs in all, each rerun `delay_seconds` after the last. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct JobRetry { + pub attempts: u8, + pub delay_seconds: u32, +} + +/// A declared job's `[settings]` and `[retry]`, or a group's member jobs. +#[derive(Clone, Debug, PartialEq)] +pub enum DeclaredRun { + Job { + settings: BTreeMap, + retry: Option, + }, + Group { + members: Vec, + }, +} + +/// How far one job setting reaches, which also decides how named jobs' values merge. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum SettingScope { + /// Only the job's own nodes; two jobs giving one node different values conflict, unset not. + Node, + /// A limit on how many of the job's own nodes run at once; each job keeps its own. + JobLimit, + /// The whole build; every named job must set it the same way, unset differing from set. + Build, + /// Not a job setting: the run request owns it. + RunRequest, +} + +/// When a group's jobs must agree on a build-wide setting. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum GroupAgreement { + /// Already while each member runs as its own build, as it changes the compiled SQL. + EachBuild, + /// Once a group runs as one build. + OneBuild, +} + +/// One build option as a job setting: how far it reaches and how named jobs merge it. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct JobSettingSpec { + /// The setting's name, the build parser's destination of its options. + pub name: &'static str, + /// The command-line options that give it, which replace every named job's value. + pub options: &'static [&'static str], + pub scope: SettingScope, + /// Whether a Python node takes it; the other node settings act on SQL nodes only. + pub python_nodes: bool, + /// Whether one value every named job shares becomes the whole build's, setup included. + pub whole_run: bool, + /// The values its command-line options can give it. + pub command_line: CommandLineValues, + /// For a node setting, what applies it to the plan. + pub consumer: Option, + pub group: GroupAgreement, +} + +/// The values a setting's command-line options can give it. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum CommandLineValues { + /// Any value it takes, or both `true` and `false`. + Any, + /// Only `true`, as a flag such as `--full-refresh`. + OnlyTrue, + /// Only `false`, as a flag such as `--no-tests`. + OnlyFalse, +} + +/// What applies a node setting to the planned build. +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub enum NodeConsumer { + /// The planner's `full_refresh_models` and `full_refresh_functions` overrides. + RefreshOverrides, + /// The plan's SQL test entries, dropping the tests a job turns off. + TestEntries, + /// Each audit entry's `ignore_failure`. + AuditEntries, + /// The warehouse each node runs on. + NodeWarehouses, +} + +/// What the jobs a selection names give each of its nodes. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct NodeJobSettings { + /// The named jobs each node is selected through, in the order the jobs are named. + pub node_jobs: BTreeMap>, + /// Each node's node-level settings, from whichever named job sets them. + pub values: BTreeMap>, + /// Each node's `[retry]`, from whichever named job sets it. + pub retries: BTreeMap, + /// Each named job's limit on how many of its nodes run at once. + pub job_limits: BTreeMap, +} + +/// What the named jobs give a build once the command line's own settings replace theirs. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct ResolvedJobSettings { + /// The declared jobs less the settings the command line gives. + pub declared: BTreeMap, + /// The jobs the selectors name. + pub named: Vec, + /// The build-wide settings the named jobs agree on. + pub build: BTreeMap, + /// The whole-run node settings every named job sets to one value. + pub whole_run: BTreeMap, + /// Each selected node's settings and the jobs' limits, when the nodes were resolved. + pub nodes: Option, +} diff --git a/crates/sqlbuild-analysis/src/job_settings/tests.rs b/crates/sqlbuild-analysis/src/job_settings/tests.rs new file mode 100644 index 0000000000..1fc45957d0 --- /dev/null +++ b/crates/sqlbuild-analysis/src/job_settings/tests.rs @@ -0,0 +1,10 @@ +#[path = "tests/test_build_job_settings.rs"] +mod build_job_settings; +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_job_setting_table.rs"] +mod job_setting_table; +#[path = "tests/test_node_job_settings.rs"] +mod node_job_settings; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-analysis/src/job_settings/tests/helpers.rs b/crates/sqlbuild-analysis/src/job_settings/tests/helpers.rs new file mode 100644 index 0000000000..bb76ab991c --- /dev/null +++ b/crates/sqlbuild-analysis/src/job_settings/tests/helpers.rs @@ -0,0 +1,265 @@ +use std::collections::{BTreeMap, BTreeSet}; + +use serde_json::{Value as Json, json}; + +use crate::assembly::project::types::ObjectKey; +use crate::graph::main::attach_jobs::attach_jobs; +use crate::graph::main::build_project_graph::build_project_graph; +use crate::graph::main::resolve_selector_tokens::resolve_selector_tokens; +use crate::graph::models::{GraphResource, JobSelection, ProjectGraph}; +use crate::job_settings::_helpers::node_job_settings::node_job_settings; +use crate::job_settings::_helpers::node_values::node_values; +use crate::job_settings::constants::JOB_SETTINGS; +use crate::job_settings::errors::NodeJobSettingsError; +use crate::job_settings::main::resolve_job_settings::resolve_job_settings; +use crate::job_settings::models::{ + DeclaredRun, JobRetry, JobSettingSpec, NodeConsumer, NodeJobSettings, SettingScope, +}; + +fn key(kind: &str, name: &str) -> ObjectKey { + (kind.to_owned(), name.to_owned()) +} + +fn texts(values: &[&str]) -> Vec { + values.iter().map(|value| (*value).to_owned()).collect() +} + +/// Staged orders feed orders and refunds; customers stands alone. +fn graph() -> ProjectGraph { + let model = |name: &str, deps: &[&str]| GraphResource { + key: key("model", name), + deps: deps.iter().map(|dep| key("model", dep)).collect(), + tags: Vec::new(), + folder: None, + }; + build_project_graph(&[ + model("stg_orders", &[]), + model("orders", &["stg_orders"]), + model("refunds", &["stg_orders"]), + model("customers", &[]), + ]) +} + +/// Jobs over the graph with their settings, and a group of two of them. +pub(super) fn declared() -> BTreeMap { + let job = |settings: Json, retry: Option| DeclaredRun::Job { + settings: settings + .as_object() + .map(|object| object.clone().into_iter().collect()) + .unwrap_or_default(), + retry, + }; + BTreeMap::from([ + ( + "orders_job".to_owned(), + job( + json!({"concurrency": 2, "run_tests": false, "full_refresh": true}), + None, + ), + ), + ( + "refunds_job".to_owned(), + job(json!({"concurrency": 4}), None), + ), + ( + "staging_job".to_owned(), + job(json!({"run_tests": true}), None), + ), + ( + "quiet_job".to_owned(), + job(json!({"run_tests": false}), None), + ), + ( + "crm_job".to_owned(), + job(json!({"vars": {"region": "eu"}}), None), + ), + ( + "crm_us_job".to_owned(), + job(json!({"vars": {"region": "us"}}), None), + ), + ( + "retried_job".to_owned(), + job( + json!({}), + Some(JobRetry { + attempts: 3, + delay_seconds: 60, + }), + ), + ), + ( + "retried_fast_job".to_owned(), + job( + json!({}), + Some(JobRetry { + attempts: 2, + delay_seconds: 0, + }), + ), + ), + ( + "broken_job".to_owned(), + job(json!({"full_refresh": true}), None), + ), + ( + "orders_wh_job".to_owned(), + job( + json!({"warehouse": "ORDERS_WH", "run_tests": false}), + Some(JobRetry { + attempts: 2, + delay_seconds: 0, + }), + ), + ), + ( + "refunds_wh_job".to_owned(), + job(json!({"warehouse": "REFUNDS_WH"}), None), + ), + ( + "daily_group".to_owned(), + DeclaredRun::Group { + members: texts(&["orders_job", "refunds_job"]), + }, + ), + ]) +} + +fn selections() -> BTreeMap { + let job = |select: &[&str]| JobSelection::Job { + select: texts(select), + exclude: Vec::new(), + }; + BTreeMap::from([ + ("orders_job".to_owned(), job(&["+orders"])), + ("refunds_job".to_owned(), job(&["+refunds"])), + ("staging_job".to_owned(), job(&["stg_orders"])), + ("quiet_job".to_owned(), job(&["stg_orders"])), + ("crm_job".to_owned(), job(&["customers"])), + ("crm_us_job".to_owned(), job(&["customers"])), + ("retried_job".to_owned(), job(&["stg_orders"])), + ("retried_fast_job".to_owned(), job(&["stg_orders"])), + ("broken_job".to_owned(), job(&["missing_model"])), + ("orders_wh_job".to_owned(), job(&["+orders"])), + ("refunds_wh_job".to_owned(), job(&["+refunds"])), + ( + "daily_group".to_owned(), + JobSelection::Group { + members: texts(&["orders_job", "refunds_job"]), + }, + ), + ]) +} + +/// The graph with the jobs attached, as a command that names a job resolves it. +pub(super) fn graph_with_jobs() -> ProjectGraph { + attach_jobs(graph(), selections()) +} + +pub(super) fn strings(values: &[&str]) -> Vec { + texts(values) +} + +/// What selecting `select` minus `exclude` gives each node of the graph's jobs. +pub(super) fn resolved_settings( + select: &[&str], + exclude: &[&str], +) -> Result { + let graph: ProjectGraph = graph_with_jobs(); + let token_nodes = |token: &str, exclude: &[String]| { + let excluded: BTreeSet = resolve_selector_tokens(&graph, exclude)? + .into_iter() + .collect(); + let selected = resolve_selector_tokens(&graph, &[token.to_owned()])?; + Ok(selected + .into_iter() + .filter(|key| !excluded.contains(key)) + .collect()) + }; + node_job_settings( + &graph, + (&texts(select), &texts(exclude)), + &declared(), + &token_nodes, + ) +} + +/// Jobs `a_job`, `b_job` and `c_job` setting `name` to each of `values`, unset where `None`. +pub(super) fn jobs_setting(name: &str, values: [Option; 3]) -> BTreeMap { + ["a_job", "b_job", "c_job"] + .into_iter() + .zip(values) + .map(|(job, value)| { + let settings: BTreeMap = value + .map(|value| (name.to_owned(), value)) + .into_iter() + .collect(); + ( + job.to_owned(), + DeclaredRun::Job { + settings, + retry: None, + }, + ) + }) + .collect() +} + +/// The `job:` selectors of `jobs`. +pub(super) fn job_selectors(jobs: &[&str]) -> Vec { + jobs.iter().map(|job| format!("job:{job}")).collect() +} + +/// Whether naming `named`, `given` on the command line, conflicts on `name`, and the build takes it. +pub(super) fn resolves_build_wide(name: &str, named: &[&str], given: &[&str]) -> (bool, bool) { + let declared = jobs_setting(name, [Some(json!("first")), Some(json!("second")), None]); + let given: BTreeSet = given.iter().map(|setting| (*setting).to_owned()).collect(); + resolve_job_settings(declared, (&job_selectors(named), &given), None) + .map_or((true, false), |resolved| { + (false, resolved.build.contains_key(name)) + }) +} + +/// Whether `a_job` and `b_job` giving one model `first` and `second` for `name` conflict. +pub(super) fn conflicts_on_a_node(name: &str) -> bool { + let declared = jobs_setting(name, [Some(json!("first")), Some(json!("second")), None]); + let model: ObjectKey = key("model", "orders"); + node_values(&model, &["a_job".to_owned(), "b_job".to_owned()], &declared).is_err() +} + +/// The whole-run value of `name` when `a_job` and `b_job` set it to `values`. +pub(super) fn whole_run_value(name: &str, values: [Option; 3]) -> Option { + let declared = jobs_setting(name, values); + resolve_job_settings( + declared, + (&job_selectors(&["a_job", "b_job"]), &BTreeSet::new()), + None, + ) + .ok() + .and_then(|resolved| resolved.whole_run.get(name).cloned()) +} + +/// Node settings without a consumer, other settings with one, and consumers named twice. +pub(super) fn consumer_faults() -> (Vec<&'static str>, Vec<&'static str>, usize) { + let node = |spec: &&JobSettingSpec| spec.scope == SettingScope::Node; + let unconsumed: Vec<&str> = JOB_SETTINGS + .iter() + .filter(node) + .filter(|spec| spec.consumer.is_none()) + .map(|spec| spec.name) + .collect(); + let consumed_others: Vec<&str> = JOB_SETTINGS + .iter() + .filter(|spec| spec.scope != SettingScope::Node && spec.consumer.is_some()) + .map(|spec| spec.name) + .collect(); + let consumers: Vec = JOB_SETTINGS + .iter() + .filter_map(|spec| spec.consumer) + .collect(); + let distinct: BTreeSet = consumers.iter().copied().collect(); + ( + unconsumed, + consumed_others, + consumers.len() - distinct.len(), + ) +} diff --git a/crates/sqlbuild-analysis/src/job_settings/tests/test_build_job_settings.rs b/crates/sqlbuild-analysis/src/job_settings/tests/test_build_job_settings.rs new file mode 100644 index 0000000000..e34ce90a5d --- /dev/null +++ b/crates/sqlbuild-analysis/src/job_settings/tests/test_build_job_settings.rs @@ -0,0 +1,83 @@ +use std::collections::BTreeMap; + +use serde_json::json; + +use crate::job_settings::_helpers::build_job_settings::build_job_settings; +use crate::job_settings::_helpers::named_jobs::named_jobs; +use crate::job_settings::tests::helpers::{declared, strings}; +use crate::job_settings::tests::test_types::BuildJobSettingsTestCase; + +#[test] +fn given_named_jobs_when_merging_build_settings_then_they_must_agree() { + let test_cases = [ + BuildJobSettingsTestCase { + description: "a job that leaves vars unset differs from one that sets them", + select: &["crm_job", "job:orders_job"], + expected_named: vec!["crm_job", "orders_job"], + expected_settings: Err( + "jobs crm_job and orders_job set vars differently ({\"region\":\"eu\"} \ + and not set); a build applies vars to everything it runs", + ), + }, + BuildJobSettingsTestCase { + description: "a single job's vars", + select: &["job:crm_job,customers"], + expected_named: vec!["crm_job"], + expected_settings: Ok(BTreeMap::from([("vars", json!({"region": "eu"}))])), + }, + BuildJobSettingsTestCase { + description: "two jobs with different vars", + select: &["job:crm_job job:crm_us_job"], + expected_named: vec!["crm_job", "crm_us_job"], + expected_settings: Err( + "jobs crm_job and crm_us_job set vars differently ({\"region\":\"eu\"} \ + and {\"region\":\"us\"}); a build applies vars to everything it runs", + ), + }, + BuildJobSettingsTestCase { + description: "a job: pattern names every job it matches", + select: &["job:crm_*"], + expected_named: vec!["crm_job", "crm_us_job"], + expected_settings: Err( + "jobs crm_job and crm_us_job set vars differently ({\"region\":\"eu\"} \ + and {\"region\":\"us\"}); a build applies vars to everything it runs", + ), + }, + BuildJobSettingsTestCase { + description: "jobs on different warehouses build together: warehouse is per node", + select: &["job:orders_wh_job job:refunds_wh_job"], + expected_named: vec!["orders_wh_job", "refunds_wh_job"], + expected_settings: Ok(BTreeMap::new()), + }, + BuildJobSettingsTestCase { + description: "a group's jobs without build-wide settings", + select: &["job:daily_group"], + expected_named: vec!["orders_job", "refunds_job"], + expected_settings: Ok(BTreeMap::new()), + }, + BuildJobSettingsTestCase { + description: "tags, paths and node names name no job", + select: &["tag:daily models/marts orders"], + expected_named: vec![], + expected_settings: Ok(BTreeMap::new()), + }, + ]; + for test_case in test_cases { + let named: Vec = named_jobs(&strings(test_case.select), &declared()); + assert_eq!(named, test_case.expected_named, "{}", test_case.description); + let merged = build_job_settings(&named, &declared()); + let merged = merged + .map(|settings| settings.into_iter().collect::>()) + .map_err(|conflict| conflict.message); + let expected = test_case + .expected_settings + .map(|settings| { + settings + .into_iter() + .map(|(key, value)| (key.to_owned(), value)) + .collect() + }) + .map_err(str::to_owned); + assert_eq!(merged, expected, "{}", test_case.description); + } +} diff --git a/crates/sqlbuild-analysis/src/job_settings/tests/test_job_setting_table.rs b/crates/sqlbuild-analysis/src/job_settings/tests/test_job_setting_table.rs new file mode 100644 index 0000000000..e3c339ad9c --- /dev/null +++ b/crates/sqlbuild-analysis/src/job_settings/tests/test_job_setting_table.rs @@ -0,0 +1,189 @@ +use std::collections::BTreeSet; + +use serde_json::json; + +use crate::job_settings::constants::JOB_SETTINGS; +use crate::job_settings::main::conflict_help::conflict_help; +use crate::job_settings::models::SettingScope; +use crate::job_settings::tests::helpers::{ + conflicts_on_a_node, consumer_faults, resolves_build_wide, whole_run_value, +}; +use crate::job_settings::tests::test_types::{ + ConflictHelpTestCase, JobSettingScopeTestCase, NodeConsumerTestCase, WholeRunTestCase, +}; + +#[test] +fn given_every_job_setting_when_named_jobs_merge_it_then_its_scope_decides_and_the_command_line_wins() + { + let test_cases = [ + JobSettingScopeTestCase { + description: "a build-wide setting must agree, unset included, and applies to all", + scope: SettingScope::Build, + expected_build_conflict: true, + expected_unset_conflict: true, + expected_build_value: true, + expected_node_conflict: false, + }, + JobSettingScopeTestCase { + description: "a node setting stays per node and conflicts on a shared node", + scope: SettingScope::Node, + expected_build_conflict: false, + expected_unset_conflict: false, + expected_build_value: false, + expected_node_conflict: true, + }, + JobSettingScopeTestCase { + description: "a job limit stays with each job", + scope: SettingScope::JobLimit, + expected_build_conflict: false, + expected_unset_conflict: false, + expected_build_value: false, + expected_node_conflict: false, + }, + ]; + let job_scopes: BTreeSet = JOB_SETTINGS + .iter() + .filter(|spec| spec.scope != SettingScope::RunRequest) + .map(|spec| format!("{:?}", spec.scope)) + .collect(); + let case_scopes: BTreeSet = test_cases + .iter() + .map(|case| format!("{:?}", case.scope)) + .collect(); + assert_eq!( + job_scopes, case_scopes, + "every job setting scope has a case" + ); + for test_case in test_cases { + for spec in JOB_SETTINGS + .iter() + .filter(|spec| spec.scope == test_case.scope) + { + assert_eq!( + ( + resolves_build_wide(spec.name, &["a_job", "b_job"], &[]).0, + resolves_build_wide(spec.name, &["a_job", "c_job"], &[]).0, + resolves_build_wide(spec.name, &["a_job"], &[]).1, + conflicts_on_a_node(spec.name), + resolves_build_wide(spec.name, &["a_job", "b_job"], &[spec.name]), + ), + ( + test_case.expected_build_conflict, + test_case.expected_unset_conflict, + test_case.expected_build_value, + test_case.expected_node_conflict, + (false, false), + ), + "{}: {}", + test_case.description, + spec.name + ); + } + } +} + +#[test] +fn given_a_whole_run_setting_when_named_jobs_share_its_value_then_it_is_the_build_s() { + let test_cases = [ + WholeRunTestCase { + description: "every named job sets one value", + values: [Some(json!("ONE")), Some(json!("ONE")), None], + expected_value: Some(json!("ONE")), + }, + WholeRunTestCase { + description: "the named jobs set different values", + values: [Some(json!("ONE")), Some(json!("TWO")), None], + expected_value: None, + }, + WholeRunTestCase { + description: "one named job leaves it unset", + values: [Some(json!("ONE")), None, None], + expected_value: None, + }, + ]; + for test_case in test_cases { + for spec in JOB_SETTINGS.iter().filter(|spec| spec.whole_run) { + assert_eq!( + whole_run_value(spec.name, test_case.values.clone()), + test_case.expected_value, + "{}: {}", + test_case.description, + spec.name + ); + } + } +} + +#[test] +fn given_two_jobs_disagreeing_on_a_setting_when_reporting_then_the_help_offers_only_flags_that_settle_it() + { + let test_cases = [ + ConflictHelpTestCase { + description: "--no-tests can only turn tests off", + setting: "run_tests", + expected_help: "make the jobs agree, set it in only one of them, pass --no-tests to make \ + it false for all, or build with --no-job-settings", + }, + ConflictHelpTestCase { + description: "--full-refresh can only turn full refresh on", + setting: "full_refresh", + expected_help: "make the jobs agree, set it in only one of them, pass --full-refresh to \ + make it true for all, or build with --no-job-settings", + }, + ConflictHelpTestCase { + description: "a build-wide flag cannot be left unset by one job", + setting: "fail_fast", + expected_help: "make the jobs agree, pass --fail-fast to make it true for all, or build \ + with --no-job-settings", + }, + ConflictHelpTestCase { + description: "a flag pair gives either value", + setting: "ignore_audit_failures", + expected_help: "make the jobs agree, set it in only one of them, override it with \ + --ignore-audit-failures or --no-ignore-audit-failures, or build with \ + --no-job-settings", + }, + ConflictHelpTestCase { + description: "an option taking a value gives any", + setting: "vars", + expected_help: "make the jobs agree, override it with --vars, or build with \ + --no-job-settings", + }, + ConflictHelpTestCase { + description: "no flag gives a retry table", + setting: "[retry]", + expected_help: "make the jobs agree, set [retry] in only one of them, or build with \ + --no-job-settings", + }, + ]; + for test_case in test_cases { + assert_eq!( + conflict_help(test_case.setting), + test_case.expected_help, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_the_job_setting_table_when_checked_then_each_node_setting_names_its_own_consumer() { + let test_cases = [NodeConsumerTestCase { + description: "a node setting without a consumer would never reach the plan", + expected_unconsumed_node_settings: Vec::new(), + expected_consumed_other_settings: Vec::new(), + expected_shared_consumers: 0, + }]; + for test_case in test_cases { + assert_eq!( + consumer_faults(), + ( + test_case.expected_unconsumed_node_settings, + test_case.expected_consumed_other_settings, + test_case.expected_shared_consumers, + ), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/job_settings/tests/test_node_job_settings.rs b/crates/sqlbuild-analysis/src/job_settings/tests/test_node_job_settings.rs new file mode 100644 index 0000000000..efc9f24ad8 --- /dev/null +++ b/crates/sqlbuild-analysis/src/job_settings/tests/test_node_job_settings.rs @@ -0,0 +1,272 @@ +use std::collections::BTreeMap; + +use serde_json::json; + +use crate::job_settings::_helpers::node_values::node_values; +use crate::job_settings::errors::NodeJobSettingsError; +use crate::job_settings::models::NodeJobSettings; +use crate::job_settings::tests::helpers::{declared, resolved_settings, strings}; +use crate::job_settings::tests::test_types::{ + JobSelectorFailureTestCase, NodeJobConflictTestCase, NodeJobSettingsTestCase, + PythonNodeValuesTestCase, +}; + +#[test] +fn given_named_jobs_when_resolving_node_settings_then_only_their_own_selected_nodes_take_them() { + let orders_values = + || BTreeMap::from([("run_tests", json!(false)), ("full_refresh", json!(true))]); + let orders_wh_values = || { + BTreeMap::from([ + ("run_tests", json!(false)), + ("warehouse", json!("ORDERS_WH")), + ]) + }; + let test_cases = [ + NodeJobSettingsTestCase { + description: "a named job gives each of its nodes its settings and limit", + select: &["job:orders_job"], + exclude: &[], + expected_node_jobs: BTreeMap::from([ + ("orders", vec!["orders_job"]), + ("stg_orders", vec!["orders_job"]), + ]), + expected_values: BTreeMap::from([ + ("orders", orders_values()), + ("stg_orders", orders_values()), + ]), + expected_job_limits: BTreeMap::from([("orders_job", 2)]), + }, + NodeJobSettingsTestCase { + description: "an intersection gives the job's settings to just the nodes it selects", + select: &["job:orders_job,orders"], + exclude: &[], + expected_node_jobs: BTreeMap::from([("orders", vec!["orders_job"])]), + expected_values: BTreeMap::from([("orders", orders_values())]), + expected_job_limits: BTreeMap::from([("orders_job", 2)]), + }, + NodeJobSettingsTestCase { + description: "nodes added beside the job, or by + around it, take none of its settings", + select: &["job:orders_job customers", "job:orders_job+"], + exclude: &[], + expected_node_jobs: BTreeMap::from([ + ("orders", vec!["orders_job"]), + ("stg_orders", vec!["orders_job"]), + ]), + expected_values: BTreeMap::from([ + ("orders", orders_values()), + ("stg_orders", orders_values()), + ]), + expected_job_limits: BTreeMap::from([("orders_job", 2)]), + }, + NodeJobSettingsTestCase { + description: "an excluded node takes nothing", + select: &["orders_job"], + exclude: &["stg_orders"], + expected_node_jobs: BTreeMap::from([("orders", vec!["orders_job"])]), + expected_values: BTreeMap::from([("orders", orders_values())]), + expected_job_limits: BTreeMap::from([("orders_job", 2)]), + }, + NodeJobSettingsTestCase { + description: "naming a group names its jobs; a shared node keeps both jobs' limits", + select: &["job:daily_group"], + exclude: &[], + expected_node_jobs: BTreeMap::from([ + ("orders", vec!["orders_job"]), + ("refunds", vec!["refunds_job"]), + ("stg_orders", vec!["orders_job", "refunds_job"]), + ]), + expected_values: BTreeMap::from([ + ("orders", orders_values()), + ("stg_orders", orders_values()), + ]), + expected_job_limits: BTreeMap::from([("orders_job", 2), ("refunds_job", 4)]), + }, + NodeJobSettingsTestCase { + description: "jobs that agree, or where only one sets a setting, do not conflict", + select: &["job:quiet_job job:orders_job"], + exclude: &[], + expected_node_jobs: BTreeMap::from([ + ("orders", vec!["orders_job"]), + ("stg_orders", vec!["quiet_job", "orders_job"]), + ]), + expected_values: BTreeMap::from([ + ("orders", orders_values()), + ("stg_orders", orders_values()), + ]), + expected_job_limits: BTreeMap::from([("orders_job", 2)]), + }, + NodeJobSettingsTestCase { + description: "jobs on different warehouses give each its own nodes its warehouse", + select: &["job:orders_wh_job", "refunds"], + exclude: &[], + expected_node_jobs: BTreeMap::from([ + ("orders", vec!["orders_wh_job"]), + ("stg_orders", vec!["orders_wh_job"]), + ]), + expected_values: BTreeMap::from([ + ("orders", orders_wh_values()), + ("stg_orders", orders_wh_values()), + ]), + expected_job_limits: BTreeMap::new(), + }, + NodeJobSettingsTestCase { + description: "a selection that names no job takes no settings", + select: &["+orders"], + exclude: &[], + expected_node_jobs: BTreeMap::new(), + expected_values: BTreeMap::new(), + expected_job_limits: BTreeMap::new(), + }, + ]; + for test_case in test_cases { + let resolved: NodeJobSettings = + resolved_settings(test_case.select, test_case.exclude).expect(test_case.description); + let node_jobs: BTreeMap<&str, Vec<&str>> = resolved + .node_jobs + .iter() + .map(|(key, jobs)| (key.1.as_str(), jobs.iter().map(String::as_str).collect())) + .collect(); + let values: BTreeMap<&str, BTreeMap<&str, serde_json::Value>> = resolved + .values + .iter() + .map(|(key, values)| { + ( + key.1.as_str(), + values + .iter() + .map(|(name, value)| (name.as_str(), value.clone())) + .collect(), + ) + }) + .collect(); + let limits: BTreeMap<&str, i64> = resolved + .job_limits + .iter() + .map(|(job, limit)| (job.as_str(), *limit)) + .collect(); + assert_eq!( + node_jobs, test_case.expected_node_jobs, + "{}", + test_case.description + ); + assert_eq!( + values, test_case.expected_values, + "{}", + test_case.description + ); + assert_eq!( + limits, test_case.expected_job_limits, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_two_named_jobs_setting_one_node_differently_when_resolving_then_it_is_a_conflict() { + let test_cases = [ + NodeJobConflictTestCase { + description: "a node-level setting", + select: &["job:staging_job", "job:orders_job"], + expected_message: "model:stg_orders is selected through jobs staging_job and orders_job, \ + which set run_tests differently (true and false)", + }, + NodeJobConflictTestCase { + description: "a warehouse", + select: &["job:orders_wh_job job:refunds_wh_job"], + expected_message: "model:stg_orders is selected through jobs orders_wh_job and \ + refunds_wh_job, which set warehouse differently (\"ORDERS_WH\" and \ + \"REFUNDS_WH\")", + }, + NodeJobConflictTestCase { + description: "a retry policy", + select: &["job:retried_job job:retried_fast_job"], + expected_message: "model:stg_orders is selected through jobs retried_job and \ + retried_fast_job, which set [retry] differently (3 attempts, 60s apart and 2 \ + attempts, 0s apart)", + }, + ]; + for test_case in test_cases { + let Err(NodeJobSettingsError::Conflict(conflict)) = + resolved_settings(test_case.select, &[]) + else { + panic!("{}: expected a conflict", test_case.description); + }; + assert_eq!( + conflict.message, test_case.expected_message, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_a_named_job_whose_selection_fails_when_resolving_then_it_fails_naming_the_job() { + let test_cases = [JobSelectorFailureTestCase { + description: "a job selecting a node that does not exist", + select: &["job:broken_job"], + expected_failure: (vec!["broken_job"], "job:broken_job", "S007"), + }]; + for test_case in test_cases { + let Err(NodeJobSettingsError::Selector(failure)) = resolved_settings(test_case.select, &[]) + else { + panic!("{}: expected a selector failure", test_case.description); + }; + let jobs: Vec<&str> = failure.jobs.iter().map(String::as_str).collect(); + assert_eq!( + (jobs, failure.selector.as_str(), failure.error.code), + test_case.expected_failure, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_a_python_node_in_named_jobs_when_resolving_then_it_takes_only_their_warehouse() { + let test_cases = [ + PythonNodeValuesTestCase { + description: "the warehouse applies; run_tests and [retry] do not", + jobs: &["orders_wh_job"], + expected_values: Some(BTreeMap::from([("warehouse", json!("ORDERS_WH"))])), + expected_conflict: None, + }, + PythonNodeValuesTestCase { + description: "jobs that agree on everything Python nodes take do not conflict", + jobs: &["orders_wh_job", "orders_job"], + expected_values: Some(BTreeMap::from([("warehouse", json!("ORDERS_WH"))])), + expected_conflict: None, + }, + PythonNodeValuesTestCase { + description: "two warehouses conflict", + jobs: &["orders_wh_job", "refunds_wh_job"], + expected_values: None, + expected_conflict: Some( + "asset:daily_report is selected through jobs orders_wh_job and refunds_wh_job, \ + which set warehouse differently (\"ORDERS_WH\" and \"REFUNDS_WH\")", + ), + }, + ]; + let key = ("asset".to_owned(), "daily_report".to_owned()); + for test_case in test_cases { + let resolved = node_values(&key, &strings(test_case.jobs), &declared()); + let values: Option> = + resolved.as_ref().ok().map(|(values, _)| { + values + .iter() + .map(|(name, value)| (name.as_str(), value.clone())) + .collect() + }); + let retry = resolved.as_ref().ok().and_then(|(_, retry)| *retry); + let conflict: Option<&str> = resolved + .as_ref() + .err() + .map(|conflict| conflict.message.as_str()); + assert_eq!( + (values, retry, conflict), + (test_case.expected_values, None, test_case.expected_conflict), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/job_settings/tests/test_types.rs b/crates/sqlbuild-analysis/src/job_settings/tests/test_types.rs new file mode 100644 index 0000000000..be39b98642 --- /dev/null +++ b/crates/sqlbuild-analysis/src/job_settings/tests/test_types.rs @@ -0,0 +1,77 @@ +use std::collections::BTreeMap; + +use serde_json::Value as Json; + +pub(super) struct NodeJobSettingsTestCase { + pub(super) description: &'static str, + pub(super) select: &'static [&'static str], + pub(super) exclude: &'static [&'static str], + /// Each node's named jobs, as `name -> jobs`. + pub(super) expected_node_jobs: BTreeMap<&'static str, Vec<&'static str>>, + /// Each node's node-level settings, as `name -> setting -> value`. + pub(super) expected_values: BTreeMap<&'static str, BTreeMap<&'static str, Json>>, + pub(super) expected_job_limits: BTreeMap<&'static str, i64>, +} + +pub(super) struct JobSelectorFailureTestCase { + pub(super) description: &'static str, + pub(super) select: &'static [&'static str], + pub(super) expected_failure: (Vec<&'static str>, &'static str, &'static str), +} + +pub(super) struct NodeJobConflictTestCase { + pub(super) description: &'static str, + pub(super) select: &'static [&'static str], + pub(super) expected_message: &'static str, +} + +pub(super) struct PythonNodeValuesTestCase { + pub(super) description: &'static str, + /// The named jobs the Python node is selected through. + pub(super) jobs: &'static [&'static str], + pub(super) expected_values: Option>, + pub(super) expected_conflict: Option<&'static str>, +} + +pub(super) struct BuildJobSettingsTestCase { + pub(super) description: &'static str, + pub(super) select: &'static [&'static str], + pub(super) expected_named: Vec<&'static str>, + pub(super) expected_settings: Result, &'static str>, +} + +/// How a setting of one scope merges across two named jobs and against the command line. +pub(super) struct JobSettingScopeTestCase { + pub(super) description: &'static str, + pub(super) scope: crate::job_settings::models::SettingScope, + /// Whether two named jobs giving different values fail as a build-wide conflict. + pub(super) expected_build_conflict: bool, + /// Whether a named job leaving it unset fails against one that sets it. + pub(super) expected_unset_conflict: bool, + /// Whether a single named job's value becomes the build's. + pub(super) expected_build_value: bool, + /// Whether two jobs giving one SQL node different values conflict on that node. + pub(super) expected_node_conflict: bool, +} + +/// A whole-run setting's values in two named jobs, and the build's value. +pub(super) struct WholeRunTestCase { + pub(super) description: &'static str, + pub(super) values: [Option; 3], + pub(super) expected_value: Option, +} + +/// A setting two named jobs disagree on, and the help the conflict gives. +pub(super) struct ConflictHelpTestCase { + pub(super) description: &'static str, + pub(super) setting: &'static str, + pub(super) expected_help: &'static str, +} + +/// The table's node settings, which must each name one distinct consumer, and the others none. +pub(super) struct NodeConsumerTestCase { + pub(super) description: &'static str, + pub(super) expected_unconsumed_node_settings: Vec<&'static str>, + pub(super) expected_consumed_other_settings: Vec<&'static str>, + pub(super) expected_shared_consumers: usize, +} diff --git a/crates/sqlbuild-analysis/src/job_settings/types.rs b/crates/sqlbuild-analysis/src/job_settings/types.rs new file mode 100644 index 0000000000..2365b5d9ae --- /dev/null +++ b/crates/sqlbuild-analysis/src/job_settings/types.rs @@ -0,0 +1,16 @@ +//! Type aliases of named job settings. + +use std::collections::BTreeSet; + +use crate::assembly::project::types::ObjectKey; +use crate::graph::errors::SelectorError; + +/// The nodes one selector token selects, less those `--exclude` removes, as the build resolves them. +pub type TokenNodes<'r> = &'r dyn Fn(&str, &[String]) -> Result, SelectorError>; + +/// The graph, the `--select` and `--exclude` selectors and their token resolution of a build's nodes. +pub type NodeSelection<'r> = ( + &'r crate::graph::models::ProjectGraph, + (&'r [String], &'r [String]), + TokenNodes<'r>, +); diff --git a/crates/sqlbuild-analysis/src/lib.rs b/crates/sqlbuild-analysis/src/lib.rs index 14f19e6e7b..a0ab57d89f 100644 --- a/crates/sqlbuild-analysis/src/lib.rs +++ b/crates/sqlbuild-analysis/src/lib.rs @@ -20,6 +20,7 @@ pub mod fingerprint_storage; pub mod graph; pub mod identifier_fitting; pub mod janitor_planning; +pub mod job_settings; pub mod lineage; pub mod lineage_command; pub mod lineage_graph; diff --git a/crates/sqlbuild-cli/src/argv/_helpers/job_settings.rs b/crates/sqlbuild-cli/src/argv/_helpers/job_settings.rs index d867bc0438..f05e843f7c 100644 --- a/crates/sqlbuild-cli/src/argv/_helpers/job_settings.rs +++ b/crates/sqlbuild-cli/src/argv/_helpers/job_settings.rs @@ -1,36 +1,23 @@ -//! Render one authored setting and check emitted options against the build parser. +//! Render job settings as options, check them against the build parser and parse without Python. +use sqlbuild_analysis::job_settings::main::setting_scope::setting_scope; +use sqlbuild_analysis::job_settings::models::SettingScope; use std::collections::BTreeMap; use serde_json::Value as Json; +use sqlbuild_core::text::main::python_str_repr::python_str_repr; use crate::argv::_helpers::table::root_parser; use crate::argv::constants::CONCURRENCY_DEST; -use crate::argv::models::{ActionKind, ActionSpec, Default, Nargs, ParserSpec, Value}; +use crate::argv::models::{ActionKind, ActionSpec, Default, Nargs, ParserSpec, TypeFailure, Value}; +use crate::argv::types::ArgvHost; pub(crate) fn setting_argument( parser: &ParserSpec, name: &str, value: &Json, ) -> Result, String> { - if matches!( - name, - "help" - | "target" - | "project_dir" - | "select" - | "exclude" - | "select_file" - | "positional_select" - | "json" - | "json_output" - | "event_output" - | "manifest" - | "dbt_project_dir" - | "dbt_profiles_dir" - | "dbt_target" - | "dbt_target_path" - ) { + if setting_scope(name) == SettingScope::RunRequest { return Err(format!("job setting {name} is owned by the run request")); } let actions: Vec<&ActionSpec> = parser @@ -148,3 +135,23 @@ pub(crate) fn validate_mutex(parser: &ParserSpec, arguments: &[String]) -> Resul } Ok(()) } + +/// Converts `int`-like options so the parse validates them, and keeps every other value as text. +pub(crate) struct IntegerHost; + +impl ArgvHost for IntegerHost { + fn convert(&mut self, _converter: &str, text: &str) -> Result { + Ok(text.trim().parse::().map_or_else( + |_| Value::Str(text.to_owned()), + |number| Value::Int(number.to_string()), + )) + } + + fn repr(&self, text: &str) -> String { + python_str_repr(text) + } + + fn program_version(&mut self) -> String { + String::new() + } +} diff --git a/crates/sqlbuild-cli/src/argv/_helpers/parsing.rs b/crates/sqlbuild-cli/src/argv/_helpers/parsing.rs index cb7c78461e..d672165d0c 100644 --- a/crates/sqlbuild-cli/src/argv/_helpers/parsing.rs +++ b/crates/sqlbuild-cli/src/argv/_helpers/parsing.rs @@ -1,6 +1,6 @@ //! `ArgumentParser.parse_known_args`, reproduced over the command table. -use std::collections::{HashMap, HashSet}; +use std::collections::{BTreeSet, HashMap, HashSet}; use sqlbuild_core::text::main::is_decimal_digit::is_decimal_digit; @@ -25,6 +25,8 @@ pub(crate) struct Run<'a> { /// The `prog` of the deepest sub-command parser entered so far. pub(crate) command_prog: Option, pub(crate) usage_failure: Option, + /// The destination of every option the arguments wrote or excluded, at any parser level. + pub(crate) given: BTreeSet, } /// `SystemExit` with this status. @@ -363,6 +365,17 @@ impl<'p, 'r, 'h> Level<'p, 'r, 'h> { let parser: &ParserSpec = self.parser; let action: &ActionSpec = &parser.actions[index]; self.seen.insert(action.id); + if !action.option_strings.is_empty() { + let excluded = parser.actions.iter().filter(|other| { + action.mutex_group.is_some() && other.mutex_group == action.mutex_group + }); + for dest in std::iter::once(action) + .chain(excluded) + .filter_map(|given| given.dest.as_ref()) + { + self.run.given.insert(dest.clone()); + } + } let values: Option = self .run .action_values(action, &args) diff --git a/crates/sqlbuild-cli/src/argv/command_table.json b/crates/sqlbuild-cli/src/argv/command_table.json index 6f19fa1d5d..322ddc5b85 100644 --- a/crates/sqlbuild-cli/src/argv/command_table.json +++ b/crates/sqlbuild-cli/src/argv/command_table.json @@ -1888,6 +1888,18 @@ "default": false, "group": 1 }, + { + "kind": "store_false", + "options": [ + "--no-job-settings" + ], + "dest": "job_settings", + "nargs": 0, + "const": false, + "default": true, + "help": "ignore the settings of the jobs the selection names, using only the command line's, the target's and the project's", + "group": 1 + }, { "kind": "store", "options": [ @@ -2548,6 +2560,18 @@ "default": false, "group": 1 }, + { + "kind": "store_false", + "options": [ + "--no-job-settings" + ], + "dest": "job_settings", + "nargs": 0, + "const": false, + "default": true, + "help": "ignore the settings of the jobs the selection names, using only the command line's, the target's and the project's", + "group": 1 + }, { "kind": "store", "options": [ diff --git a/crates/sqlbuild-cli/src/argv/constants.rs b/crates/sqlbuild-cli/src/argv/constants.rs index c99a0c09a5..a7d78b7319 100644 --- a/crates/sqlbuild-cli/src/argv/constants.rs +++ b/crates/sqlbuild-cli/src/argv/constants.rs @@ -77,7 +77,7 @@ pub(crate) const INT_CONVERTER: &str = "int"; pub(crate) const PATH_CONVERTER: &str = "Path"; /// Every `CliNamespace` attribute with a default, in declaration order. -pub(crate) const CLI_NAMESPACE_DEFAULTS: [(&str, NamespaceDefault); 193] = [ +pub(crate) const CLI_NAMESPACE_DEFAULTS: [(&str, NamespaceDefault); 194] = [ ("command", NamespaceDefault::None), ("project_dir", NamespaceDefault::None), ("dbt_project_dir", NamespaceDefault::None), @@ -135,6 +135,7 @@ pub(crate) const CLI_NAMESPACE_DEFAULTS: [(&str, NamespaceDefault); 193] = [ ("no_color", NamespaceDefault::Bool(false)), ("fail_fast", NamespaceDefault::Bool(false)), ("full_refresh", NamespaceDefault::Bool(false)), + ("job_settings", NamespaceDefault::Bool(true)), ("changed", NamespaceDefault::Bool(false)), ("changed_counts", NamespaceDefault::None), ("reach", NamespaceDefault::None), diff --git a/crates/sqlbuild-cli/src/argv/main/given_setting_dests.rs b/crates/sqlbuild-cli/src/argv/main/given_setting_dests.rs new file mode 100644 index 0000000000..419b6de573 --- /dev/null +++ b/crates/sqlbuild-cli/src/argv/main/given_setting_dests.rs @@ -0,0 +1,22 @@ +//! The settings a command line gives itself, which the settings of a job it names never override. + +use std::collections::BTreeSet; + +use crate::argv::_helpers::job_settings::IntegerHost; +use crate::argv::main::parse_cli_arguments::parse_cli_arguments; +use crate::argv::models::ArgvSettings; + +/// The destination of every option `argv` writes, however abbreviated; `None` if it fails to parse. +#[must_use] +pub fn given_setting_dests(argv: &[String]) -> Option> { + let mut host: IntegerHost = IntegerHost; + let settings: ArgvSettings = ArgvSettings { + columns: 80, + use_color: false, + }; + let outcome = parse_cli_arguments(argv, Vec::new(), settings, &mut host); + match outcome.parsed { + Ok(_) => Some(outcome.given), + Err(_exit_status) => None, + } +} diff --git a/crates/sqlbuild-cli/src/argv/main/job_run_arguments.rs b/crates/sqlbuild-cli/src/argv/main/job_run_arguments.rs new file mode 100644 index 0000000000..15e6520191 --- /dev/null +++ b/crates/sqlbuild-cli/src/argv/main/job_run_arguments.rs @@ -0,0 +1,27 @@ +//! The build arguments a server job run passes, leaving the job's own settings to the build. + +use std::collections::BTreeMap; + +use serde_json::Value as Json; +use sqlbuild_analysis::job_settings::main::job_setting_spec::job_setting_spec; +use sqlbuild_analysis::job_settings::models::SettingScope; + +use crate::argv::main::job_settings_arguments::job_settings_arguments; + +/// `--select job:` and its build-wide settings and warehouse, the run's own, as flags. +pub fn job_run_arguments( + job: &str, + settings: &BTreeMap, +) -> Result, String> { + let build_wide: BTreeMap = settings + .iter() + .filter(|(name, _)| { + job_setting_spec(name) + .is_some_and(|spec| spec.scope == SettingScope::Build || spec.whole_run) + }) + .map(|(name, value)| (name.clone(), value.clone())) + .collect(); + let mut arguments: Vec = job_settings_arguments(&build_wide)?; + arguments.extend(["--select".to_owned(), format!("job:{job}")]); + Ok(arguments) +} diff --git a/crates/sqlbuild-cli/src/argv/main/mod.rs b/crates/sqlbuild-cli/src/argv/main/mod.rs index 10cdc0c9bf..62fed59be5 100644 --- a/crates/sqlbuild-cli/src/argv/main/mod.rs +++ b/crates/sqlbuild-cli/src/argv/main/mod.rs @@ -1,6 +1,8 @@ pub mod accepted_settings_arguments; pub mod cli_namespace_defaults; pub mod command_settings_arguments; +pub mod given_setting_dests; +pub mod job_run_arguments; pub mod job_settings_arguments; pub mod parse_cli_arguments; pub mod parse_dbt_execution_arguments; diff --git a/crates/sqlbuild-cli/src/argv/main/parse_cli_arguments.rs b/crates/sqlbuild-cli/src/argv/main/parse_cli_arguments.rs index c0fce019da..e3632f3c38 100644 --- a/crates/sqlbuild-cli/src/argv/main/parse_cli_arguments.rs +++ b/crates/sqlbuild-cli/src/argv/main/parse_cli_arguments.rs @@ -1,5 +1,7 @@ //! Parse `sqb`'s argv as its `argparse` parser and `parse_cli_invocation` did. +use std::collections::BTreeSet; + use crate::argv::_helpers::parsing::Run; use crate::argv::_helpers::table::root_parser; use crate::argv::models::{ArgvOutcome, ArgvSettings, Namespace, ParserSpec, Value}; @@ -19,6 +21,7 @@ pub fn parse_cli_arguments( writes: Vec::new(), command_prog: None, usage_failure: None, + given: BTreeSet::new(), }; let namespace: Namespace = Namespace { entries: defaults, @@ -32,5 +35,6 @@ pub fn parse_cli_arguments( writes: run.writes, parsed, usage_failure: run.usage_failure, + given: run.given, } } diff --git a/crates/sqlbuild-cli/src/argv/main/parse_dbt_execution_arguments.rs b/crates/sqlbuild-cli/src/argv/main/parse_dbt_execution_arguments.rs index 40a24ce62f..cf17f5871b 100644 --- a/crates/sqlbuild-cli/src/argv/main/parse_dbt_execution_arguments.rs +++ b/crates/sqlbuild-cli/src/argv/main/parse_dbt_execution_arguments.rs @@ -1,5 +1,7 @@ //! Parse `sqb dbt plan|run|build` arguments as their `argparse` parser did. +use std::collections::BTreeSet; + use sqlbuild_core::dbt_cli::models::DbtParsedFlags; use crate::argv::_helpers::dbt_execution::{PlainHost, parsed_flags}; @@ -29,6 +31,7 @@ pub fn parse_dbt_execution_arguments( writes: Vec::new(), command_prog: None, usage_failure: None, + given: BTreeSet::new(), }; let known: Result = run.known_args(&parser, head.to_vec(), Namespace::default()); diff --git a/crates/sqlbuild-cli/src/argv/models.rs b/crates/sqlbuild-cli/src/argv/models.rs index 35da9d6276..e3b7a7f63a 100644 --- a/crates/sqlbuild-cli/src/argv/models.rs +++ b/crates/sqlbuild-cli/src/argv/models.rs @@ -1,6 +1,6 @@ //! The `sqb` command table, parsed values and the outcome of one argv parse. -use std::collections::HashMap; +use std::collections::{BTreeSet, HashMap}; use sqlbuild_core::dbt_cli::models::DbtParsedFlags; @@ -174,6 +174,8 @@ pub struct ArgvOutcome { pub parsed: Result, /// The usage failure that ended the parse, if one did. pub usage_failure: Option, + /// The destination of every option the arguments wrote, or excluded by writing another. + pub given: BTreeSet, } /// Why a `sqb dbt` execution parse stopped before producing flags. diff --git a/crates/sqlbuild-cli/src/argv/tests.rs b/crates/sqlbuild-cli/src/argv/tests.rs index 0289a8fd3d..c8b4187dfd 100644 --- a/crates/sqlbuild-cli/src/argv/tests.rs +++ b/crates/sqlbuild-cli/src/argv/tests.rs @@ -1,5 +1,11 @@ +#[path = "tests/test_given_setting_dests.rs"] +mod given_setting_dests; #[path = "tests/helpers.rs"] mod helpers; +#[path = "tests/test_job_run_arguments.rs"] +mod job_run_arguments; +#[path = "tests/test_job_setting_table.rs"] +mod job_setting_table; #[path = "tests/test_parse_cli_arguments.rs"] mod parse_cli_arguments; #[path = "tests/test_parse_dbt_execution_arguments.rs"] diff --git a/crates/sqlbuild-cli/src/argv/tests/helpers.rs b/crates/sqlbuild-cli/src/argv/tests/helpers.rs index b735659aeb..8e4abc656f 100644 --- a/crates/sqlbuild-cli/src/argv/tests/helpers.rs +++ b/crates/sqlbuild-cli/src/argv/tests/helpers.rs @@ -1,4 +1,14 @@ +use std::collections::{BTreeMap, BTreeSet}; + +use serde_json::{Value as Json, json}; +use sqlbuild_analysis::job_settings::constants::JOB_SETTINGS; +use sqlbuild_analysis::job_settings::models::{CommandLineValues, SettingScope}; + +use crate::argv::_helpers::table::command_parser; +use crate::argv::main::command_settings_arguments::command_settings_arguments; +use crate::argv::main::job_run_arguments::job_run_arguments; use crate::argv::main::parse_cli_arguments::parse_cli_arguments; +use crate::argv::models::{ActionKind, ActionSpec, Nargs}; use crate::argv::models::{ArgvOutcome, ArgvSettings, Value}; use crate::argv::tests::test_types::FakeHost; @@ -26,3 +36,149 @@ pub(super) fn strings(items: &[&str]) -> Value { pub(super) fn owned(items: &[&str]) -> Vec { items.iter().map(|item| (*item).to_owned()).collect() } + +/// Each destination of `command`'s parser with all its option strings, in the parser's order. +pub(super) fn parser_options(command: &str) -> Vec<(String, Vec)> { + let actions: Vec<&ActionSpec> = command_parser(command) + .map(|parser| parser.actions.iter().collect()) + .unwrap_or_default(); + let mut seen: BTreeSet<&str> = BTreeSet::new(); + let dests: Vec<&str> = actions + .iter() + .filter_map(|action| action.dest.as_deref()) + .filter(|dest| seen.insert(dest)) + .collect(); + dests + .into_iter() + .map(|dest| { + let strings: Vec = actions + .iter() + .filter(|action| action.dest.as_deref() == Some(dest)) + .flat_map(|action| action.option_strings.iter().cloned()) + .collect(); + (dest.to_owned(), strings) + }) + .collect() +} + +/// Each job setting with its listed options, in the table's order. +pub(super) fn table_options() -> Vec<(String, Vec)> { + JOB_SETTINGS + .iter() + .map(|spec| { + let strings = spec + .options + .iter() + .map(|option| (*option).to_owned()) + .collect(); + (spec.name.to_owned(), strings) + }) + .collect() +} + +/// A value the option of `action` accepts, as a job would write it. +fn sample(action: &ActionSpec) -> Json { + let converter: Option<&str> = action.converter.as_deref(); + let candidates: [(bool, Json); 6] = [ + ( + action.kind == ActionKind::StoreTrue || action.nargs == Nargs::Attached, + json!(true), + ), + (action.kind == ActionKind::StoreFalse, json!(false)), + ( + action.choices.is_some(), + json!(action.choices.as_ref().and_then(|choices| choices.first())), + ), + ( + matches!(converter, Some("int" | "_positive_integer")), + json!(3), + ), + (converter == Some("parse_cli_vars"), json!({"region": "eu"})), + (true, json!("2026-01-01T00:00:00+00:00")), + ]; + candidates + .into_iter() + .find_map(|(applies, value)| applies.then_some(value)) + .unwrap_or_default() +} + +/// The job settings of `scope` and `whole_run` that do not render as one of their listed options. +pub(super) fn unrendered_settings(scope: SettingScope, whole_run: bool) -> Vec<&'static str> { + JOB_SETTINGS + .iter() + .filter(|spec| spec.scope == scope && spec.whole_run == whole_run) + .filter(|spec| { + !rendered_option(spec.name) + .is_some_and(|option| spec.options.contains(&option.as_str())) + }) + .map(|spec| spec.name) + .collect() +} + +/// The option a `build` job setting `name` renders as, from a sample value. +fn rendered_option(name: &str) -> Option { + let parser = command_parser("build").ok()?; + let action = parser + .actions + .iter() + .find(|action| action.dest.as_deref() == Some(name))?; + let settings = BTreeMap::from([(name.to_owned(), sample(action))]); + let rendered: Vec = command_settings_arguments("build", &settings).ok()?; + rendered + .first() + .and_then(|argument| argument.split('=').next()) + .map(str::to_owned) +} + +/// The job settings whose `command_line` disagrees with what the build parser's options give. +pub(super) fn command_line_mismatches() -> Vec<&'static str> { + let actions: Vec<&ActionSpec> = command_parser("build") + .map(|parser| parser.actions.iter().collect()) + .unwrap_or_default(); + JOB_SETTINGS + .iter() + .filter(|spec| { + let kinds: Vec = actions + .iter() + .filter(|action| action.dest.as_deref() == Some(spec.name)) + .map(|action| action.kind) + .collect(); + let only = + |kind: ActionKind| !kinds.is_empty() && kinds.iter().all(|seen| *seen == kind); + let expected: CommandLineValues = [ + (only(ActionKind::StoreTrue), CommandLineValues::OnlyTrue), + (only(ActionKind::StoreFalse), CommandLineValues::OnlyFalse), + ] + .into_iter() + .find_map(|(applies, values)| applies.then_some(values)) + .unwrap_or(CommandLineValues::Any); + spec.command_line != expected + }) + .map(|spec| spec.name) + .collect() +} + +/// The node settings that are not whole-run but still reach a job run's build arguments. +pub(super) fn job_run_leaks() -> Vec<&'static str> { + let parser = command_parser("build").ok(); + JOB_SETTINGS + .iter() + .filter(|spec| spec.scope == SettingScope::Node && !spec.whole_run) + .filter(|spec| { + let value: Json = parser + .and_then(|parser| { + parser + .actions + .iter() + .find(|action| action.dest.as_deref() == Some(spec.name)) + }) + .map(sample) + .unwrap_or_default(); + let settings = BTreeMap::from([(spec.name.to_owned(), value)]); + job_run_arguments("orders_job", &settings).map_or(true, |arguments| { + arguments != ["--select", "job:orders_job"] + }) + }) + .map(|spec| spec.name) + .collect() +} diff --git a/crates/sqlbuild-cli/src/argv/tests/test_given_setting_dests.rs b/crates/sqlbuild-cli/src/argv/tests/test_given_setting_dests.rs new file mode 100644 index 0000000000..c42d877671 --- /dev/null +++ b/crates/sqlbuild-cli/src/argv/tests/test_given_setting_dests.rs @@ -0,0 +1,59 @@ +use crate::argv::main::given_setting_dests::given_setting_dests; +use crate::argv::tests::test_types::GivenSettingsTestCase; + +#[test] +fn given_a_command_line_when_reading_its_own_settings_then_every_written_option_counts() { + let test_cases = [ + GivenSettingsTestCase { + description: "long options, with = or a separate value, and global options", + argv: &[ + "--no-color", + "build", + "--vars={\"a\": \"1\"}", + "--concurrency", + "3", + "orders_job", + ], + expected_given: Some(&["concurrency", "no_color", "vars"]), + }, + GivenSettingsTestCase { + description: "an abbreviated option names its full option's setting", + argv: &["build", "--concur", "3", "--select", "job:orders_job"], + expected_given: Some(&["concurrency", "select"]), + }, + GivenSettingsTestCase { + description: "a negative flag gives the setting it shares", + argv: &["plan", "--no-ignore-audit-failures", "--no-job-settings"], + expected_given: Some(&["ignore_audit_failures", "job_settings"]), + }, + GivenSettingsTestCase { + description: "an option gives every setting of its mutually exclusive group", + argv: &["build", "--reload", "orders_job"], + expected_given: Some(&["load_sources", "reload"]), + }, + GivenSettingsTestCase { + description: "a selection alone gives no other setting", + argv: &["build", "orders_job"], + expected_given: Some(&[]), + }, + GivenSettingsTestCase { + description: "a command line that does not parse", + argv: &["build", "--no-such-option"], + expected_given: None, + }, + ]; + for test_case in test_cases { + let argv: Vec = test_case.argv.iter().map(|arg| (*arg).to_owned()).collect(); + let given: Option> = + given_setting_dests(&argv).map(|dests| dests.into_iter().collect()); + let given: Option> = given + .as_ref() + .map(|dests| dests.iter().map(String::as_str).collect()); + assert_eq!( + given.as_deref(), + test_case.expected_given, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-cli/src/argv/tests/test_job_run_arguments.rs b/crates/sqlbuild-cli/src/argv/tests/test_job_run_arguments.rs new file mode 100644 index 0000000000..e5de2b5f29 --- /dev/null +++ b/crates/sqlbuild-cli/src/argv/tests/test_job_run_arguments.rs @@ -0,0 +1,48 @@ +use std::collections::BTreeMap; + +use serde_json::{Value as Json, json}; + +use crate::argv::main::job_run_arguments::job_run_arguments; +use crate::argv::tests::test_types::JobRunArgumentsTestCase; + +#[test] +fn given_a_server_job_run_when_building_its_arguments_then_it_names_the_job_and_leaves_its_own_settings() + { + let test_cases = [ + JobRunArgumentsTestCase { + description: "concurrency and node-level settings are left to the build", + settings: json!({"concurrency": 8, "run_tests": false, "full_refresh": true}), + expected_arguments: &["--select", "job:orders_job"], + }, + JobRunArgumentsTestCase { + description: "the job's warehouse is the whole run's, setup and planning included", + settings: json!({"concurrency": 8, "run_tests": false, "warehouse": "ORDERS_WH"}), + expected_arguments: &["--warehouse=ORDERS_WH", "--select", "job:orders_job"], + }, + JobRunArgumentsTestCase { + description: "build-wide settings still travel as flags", + settings: json!({"concurrency": 4, "vars": {"region": "eu"}, "fail_fast": true}), + expected_arguments: &[ + "--fail-fast", + "--vars={\"region\":\"eu\"}", + "--select", + "job:orders_job", + ], + }, + ]; + for test_case in test_cases { + let settings: BTreeMap = test_case + .settings + .as_object() + .map(|object| object.clone().into_iter().collect()) + .unwrap_or_default(); + let arguments: Vec = + job_run_arguments("orders_job", &settings).expect(test_case.description); + let arguments: Vec<&str> = arguments.iter().map(String::as_str).collect(); + assert_eq!( + arguments, test_case.expected_arguments, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-cli/src/argv/tests/test_job_setting_table.rs b/crates/sqlbuild-cli/src/argv/tests/test_job_setting_table.rs new file mode 100644 index 0000000000..14e78ac48f --- /dev/null +++ b/crates/sqlbuild-cli/src/argv/tests/test_job_setting_table.rs @@ -0,0 +1,75 @@ +use sqlbuild_analysis::job_settings::models::SettingScope; + +use crate::argv::tests::helpers::{ + command_line_mismatches, job_run_leaks, parser_options, table_options, unrendered_settings, +}; +use crate::argv::tests::test_types::{ + JobSettingFaultsTestCase, JobSettingTableTestCase, RenderedSettingTestCase, +}; + +#[test] +fn given_the_build_parser_when_compared_with_the_job_setting_table_then_every_option_has_one_row() { + let test_cases = [JobSettingTableTestCase { + description: "add a JOB_SETTINGS row for each new build option", + command: "build", + expected_rows: table_options(), + }]; + for test_case in test_cases { + assert_eq!( + parser_options(test_case.command), + test_case.expected_rows, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_a_job_setting_the_build_takes_as_a_flag_when_rendered_then_it_is_one_of_its_options() { + let test_cases = [ + RenderedSettingTestCase { + description: "a build-wide setting travels as its own option", + scope: SettingScope::Build, + whole_run: false, + expected_unrendered: Vec::new(), + }, + RenderedSettingTestCase { + description: "a whole-run node setting travels as its own option", + scope: SettingScope::Node, + whole_run: true, + expected_unrendered: Vec::new(), + }, + ]; + for test_case in test_cases { + assert_eq!( + unrendered_settings(test_case.scope, test_case.whole_run), + test_case.expected_unrendered, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_the_job_setting_table_when_checked_against_the_parser_and_job_runs_then_nothing_leaks() { + let test_cases = [ + JobSettingFaultsTestCase { + description: "each row's command_line is what its options can give", + faults: command_line_mismatches, + expected_faults: Vec::new(), + }, + JobSettingFaultsTestCase { + description: "a node setting that is not whole-run never travels as a job run flag", + faults: job_run_leaks, + expected_faults: Vec::new(), + }, + ]; + for test_case in test_cases { + assert_eq!( + (test_case.faults)(), + test_case.expected_faults, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-cli/src/argv/tests/test_types.rs b/crates/sqlbuild-cli/src/argv/tests/test_types.rs index 793e4a87a3..5ec630d3a1 100644 --- a/crates/sqlbuild-cli/src/argv/tests/test_types.rs +++ b/crates/sqlbuild-cli/src/argv/tests/test_types.rs @@ -81,3 +81,39 @@ pub(super) struct JobSettingsRejectionTestCase { pub(super) settings: BTreeMap, pub(super) expected_error: &'static str, } + +/// A command line and the settings it gives itself. +pub(super) struct GivenSettingsTestCase { + pub(super) description: &'static str, + pub(super) argv: &'static [&'static str], + pub(super) expected_given: Option<&'static [&'static str]>, +} + +/// A server job run's frozen settings and the build arguments it passes. +pub(super) struct JobRunArgumentsTestCase { + pub(super) description: &'static str, + pub(super) settings: serde_json::Value, + pub(super) expected_arguments: &'static [&'static str], +} + +/// A command's parser and the job setting table rows it must match. +pub(super) struct JobSettingTableTestCase { + pub(super) description: &'static str, + pub(super) command: &'static str, + pub(super) expected_rows: Vec<(String, Vec)>, +} + +/// The job settings of some scopes, each of which must render as one of its listed options. +pub(super) struct RenderedSettingTestCase { + pub(super) description: &'static str, + pub(super) scope: sqlbuild_analysis::job_settings::models::SettingScope, + pub(super) whole_run: bool, + pub(super) expected_unrendered: Vec<&'static str>, +} + +/// Settings whose table row disagrees with the parser, or leaks into a job run's arguments. +pub(super) struct JobSettingFaultsTestCase { + pub(super) description: &'static str, + pub(super) faults: fn() -> Vec<&'static str>, + pub(super) expected_faults: Vec<&'static str>, +} diff --git a/crates/sqlbuild-cli/src/dispatch/constants.rs b/crates/sqlbuild-cli/src/dispatch/constants.rs index 953ee7474c..97bd005245 100644 --- a/crates/sqlbuild-cli/src/dispatch/constants.rs +++ b/crates/sqlbuild-cli/src/dispatch/constants.rs @@ -110,7 +110,7 @@ pub(crate) const DAG_ROUTE: Route = routed( }, ); -const PLAN_FIELDS: [Field; 25] = [ +const PLAN_FIELDS: [Field; 26] = [ PROJECT_DIR, NO_SQL_VALIDATION, same("no_cache"), @@ -121,6 +121,7 @@ const PLAN_FIELDS: [Field; 25] = [ CURSOR_OVERRIDES, JSON_OUTPUT, same("full_refresh"), + same("job_settings"), same("changed"), same("changed_counts"), same("reach"), @@ -145,7 +146,7 @@ pub(crate) const PLAN_ROUTE: Route = routed( REQUEST_KEYWORD_SHAPE, ); -const BUILD_FIELDS: [Field; 41] = [ +const BUILD_FIELDS: [Field; 42] = [ PROJECT_DIR, NO_SQL_VALIDATION, same("no_cache"), @@ -158,6 +159,7 @@ const BUILD_FIELDS: [Field; 41] = [ NO_COLOR, same("fail_fast"), same("full_refresh"), + same("job_settings"), same("changed"), same("changed_counts"), same("reach"), diff --git a/crates/sqlbuild-config/src/command_connection/constants.rs b/crates/sqlbuild-config/src/command_connection/constants.rs index 2d6591fd76..bdf1b6aae9 100644 --- a/crates/sqlbuild-config/src/command_connection/constants.rs +++ b/crates/sqlbuild-config/src/command_connection/constants.rs @@ -4,6 +4,8 @@ pub const CLI_WAREHOUSE_SOURCE: &str = "--warehouse"; pub const CONNECTION_WAREHOUSE_SOURCE: &str = "connection warehouse"; pub const CONNECTION_DEFAULT_WAREHOUSE_SOURCE: &str = "connection default"; pub const CONNECTION_WAREHOUSE_KEY: &str = "warehouse"; +/// The target's own build warehouse, recorded beside a `--warehouse` override; never connected with. +pub const CONFIGURED_WAREHOUSE_KEY: &str = "configured_warehouse"; pub const CONNECTION_SOURCE_KEY: &str = "source"; pub const DBT_PROFILE_CONNECTION_SOURCE: &str = "dbt_profile"; pub const DUCKDB_ADAPTER: &str = "duckdb"; diff --git a/crates/sqlbuild-config/src/command_connection/main/check_job_warehouse.rs b/crates/sqlbuild-config/src/command_connection/main/check_job_warehouse.rs new file mode 100644 index 0000000000..016b9a2645 --- /dev/null +++ b/crates/sqlbuild-config/src/command_connection/main/check_job_warehouse.rs @@ -0,0 +1,32 @@ +//! Validate the `warehouse` a job's `[settings]` gives its nodes. + +use sqlbuild_core::text::main::python_str_repr::python_str_repr; +use sqlbuild_core::text::main::python_strip::python_strip; + +use crate::command_connection::_helpers::identifiers::is_warehouse_identifier; +use crate::command_connection::constants::WAREHOUSE_ERROR_CODE; +use crate::command_connection::errors::CommandConnectionError; + +/// The warehouse name of `job`'s `warehouse` setting; `value` is `None` when the setting is not text. +pub fn check_job_warehouse( + job: &str, + value: Option<&str>, +) -> Result { + let problem: String = match value { + Some(name) if is_warehouse_identifier(name) => return Ok(name.to_owned()), + Some(name) if python_strip(name).is_empty() => "an empty warehouse name".to_owned(), + Some(name) => format!( + "warehouse {}, which is not a valid Snowflake warehouse identifier", + python_str_repr(name) + ), + None => "a warehouse that is not text".to_owned(), + }; + Err(CommandConnectionError::User { + code: Some(WAREHOUSE_ERROR_CODE), + message: format!("job {job} sets {problem}"), + help: Some( + "set warehouse in the job's [settings] to an unquoted identifier such as \"ADHOC_WH\", or a double-quoted one such as '\"Adhoc WH\"'" + .to_owned(), + ), + }) +} diff --git a/crates/sqlbuild-config/src/command_connection/main/mod.rs b/crates/sqlbuild-config/src/command_connection/main/mod.rs index 0cfa5f9805..6a36e4e96f 100644 --- a/crates/sqlbuild-config/src/command_connection/main/mod.rs +++ b/crates/sqlbuild-config/src/command_connection/main/mod.rs @@ -1,7 +1,9 @@ pub mod authored_warehouse; pub mod check_cli_warehouse; pub mod check_expanded_warehouse; +pub mod check_job_warehouse; pub mod check_preview_target; pub mod missing_connection_error; pub mod snowflake_key_warning; pub mod unsupported_cli_warehouse; +pub mod unsupported_job_warehouse; diff --git a/crates/sqlbuild-config/src/command_connection/main/unsupported_job_warehouse.rs b/crates/sqlbuild-config/src/command_connection/main/unsupported_job_warehouse.rs new file mode 100644 index 0000000000..1490235b07 --- /dev/null +++ b/crates/sqlbuild-config/src/command_connection/main/unsupported_job_warehouse.rs @@ -0,0 +1,23 @@ +//! A job `warehouse` for an adapter without a session warehouse. + +use crate::command_connection::constants::UNSUPPORTED_WAREHOUSE_CODE; +use crate::command_connection::errors::CommandConnectionError; + +/// The error for a job's `warehouse` the adapter cannot select. +#[must_use] +pub fn unsupported_job_warehouse( + job: &str, + warehouse: &str, + adapter_name: &str, +) -> CommandConnectionError { + CommandConnectionError::User { + code: Some(UNSUPPORTED_WAREHOUSE_CODE), + message: format!( + "job {job} sets warehouse {warehouse}, but the '{adapter_name}' adapter has no warehouse to select" + ), + help: Some( + "remove warehouse from the job's [settings], or build with --no-job-settings; it needs an adapter with a session warehouse, such as snowflake or an adapter extending it" + .to_owned(), + ), + } +} diff --git a/crates/sqlbuild-config/src/command_connection/tests/test_command_connection.rs b/crates/sqlbuild-config/src/command_connection/tests/test_command_connection.rs index 4c4c02071a..699d552ce4 100644 --- a/crates/sqlbuild-config/src/command_connection/tests/test_command_connection.rs +++ b/crates/sqlbuild-config/src/command_connection/tests/test_command_connection.rs @@ -2,10 +2,11 @@ use crate::command_connection::errors::CommandConnectionError; use crate::command_connection::main::authored_warehouse::authored_warehouse; use crate::command_connection::main::check_cli_warehouse::check_cli_warehouse; use crate::command_connection::main::check_expanded_warehouse::check_expanded_warehouse; +use crate::command_connection::main::check_job_warehouse::check_job_warehouse; use crate::command_connection::main::snowflake_key_warning::snowflake_key_warning; use crate::command_connection::tests::helpers::message; use crate::command_connection::tests::test_types::{ - CliWarehouseTestCase, ExpandedWarehouseTestCase, SnowflakeKeyTestCase, + CliWarehouseTestCase, ExpandedWarehouseTestCase, JobWarehouseTestCase, SnowflakeKeyTestCase, }; #[test] @@ -135,3 +136,47 @@ fn given_connection_keys_when_warning_then_only_duckdb_snowflake_keys_are_listed ); } } + +#[test] +fn given_a_job_warehouse_when_checking_then_it_names_the_job_and_follows_identifier_rules() { + let test_cases = [ + JobWarehouseTestCase { + description: "unquoted identifier", + value: Some("ORDERS_WH"), + expected_name: Some("ORDERS_WH"), + expected_message: None, + }, + JobWarehouseTestCase { + description: "blank value", + value: Some(" "), + expected_name: None, + expected_message: Some("job orders_job sets an empty warehouse name"), + }, + JobWarehouseTestCase { + description: "not an identifier", + value: Some("bad wh"), + expected_name: None, + expected_message: Some( + "job orders_job sets warehouse 'bad wh', which is not a valid Snowflake warehouse identifier", + ), + }, + JobWarehouseTestCase { + description: "not text", + value: None, + expected_name: None, + expected_message: Some("job orders_job sets a warehouse that is not text"), + }, + ]; + + for test_case in test_cases { + let result = check_job_warehouse("orders_job", test_case.value); + let name: Option<&str> = result.as_deref().ok(); + let message: Option<&str> = result.as_ref().err().map(|error| error.message()); + assert_eq!( + (name, message), + (test_case.expected_name, test_case.expected_message), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-config/src/command_connection/tests/test_types.rs b/crates/sqlbuild-config/src/command_connection/tests/test_types.rs index bc3e6c8362..c636c5d2c5 100644 --- a/crates/sqlbuild-config/src/command_connection/tests/test_types.rs +++ b/crates/sqlbuild-config/src/command_connection/tests/test_types.rs @@ -6,6 +6,13 @@ pub(super) struct CliWarehouseTestCase { pub(super) expected_message: Option<&'static str>, } +pub(super) struct JobWarehouseTestCase { + pub(super) description: &'static str, + pub(super) value: Option<&'static str>, + pub(super) expected_name: Option<&'static str>, + pub(super) expected_message: Option<&'static str>, +} + pub(super) struct SnowflakeKeyTestCase { pub(super) description: &'static str, pub(super) adapter_name: &'static str, diff --git a/crates/sqlbuild-config/src/effective/models.rs b/crates/sqlbuild-config/src/effective/models.rs index cb01766a63..b1b11cd89c 100644 --- a/crates/sqlbuild-config/src/effective/models.rs +++ b/crates/sqlbuild-config/src/effective/models.rs @@ -112,6 +112,8 @@ pub enum ConcurrencySource { Target(String), LocalSettings, ProjectSettings, + /// The named jobs' `concurrency` limits added together, when larger than the configured limit. + NamedJobs, } impl std::fmt::Display for ConcurrencySource { @@ -121,6 +123,7 @@ impl std::fmt::Display for ConcurrencySource { Self::Target(name) => write!(formatter, "targets.{name}.concurrency"), Self::LocalSettings => formatter.write_str("sqlbuild_local.toml settings.concurrency"), Self::ProjectSettings => formatter.write_str("settings.concurrency"), + Self::NamedJobs => formatter.write_str("named jobs' concurrency"), } } } diff --git a/crates/sqlbuild-core/src/build_command/main/render_run_context.rs b/crates/sqlbuild-core/src/build_command/main/render_run_context.rs index a87bd1e6d0..7be1e19f18 100644 --- a/crates/sqlbuild-core/src/build_command/main/render_run_context.rs +++ b/crates/sqlbuild-core/src/build_command/main/render_run_context.rs @@ -6,6 +6,16 @@ use crate::build_command::models::RunContextFacts; use crate::cli_text::models::{CliDocument, CliStyle}; use crate::number_text::main::format_thousands::format_thousands; +/// `true`, the nodes of the named jobs that rebuild theirs from scratch, or `false`. +fn full_refresh(facts: &RunContextFacts) -> String { + match facts.full_refresh_jobs.as_slice() { + _ if facts.full_refresh => true.to_string(), + [] => false.to_string(), + [job] => format!("nodes of job {}", sanitize(job)), + jobs => format!("nodes of jobs {}", sanitize(&jobs.join(", "))), + } +} + fn date_vars(facts: &RunContextFacts) -> String { match &facts.date_vars { (None, None) => NOT_SET.to_owned(), @@ -18,10 +28,17 @@ fn date_vars(facts: &RunContextFacts) -> String { /// The resolved placement, mode and selection, then the selector files. #[must_use] pub fn render_run_context(facts: &RunContextFacts, style: CliStyle) -> String { - let warehouse: String = facts.warehouse.as_ref().map_or_else( + let mut warehouse: String = facts.warehouse.as_ref().map_or_else( || NOT_SET.to_owned(), |(warehouse, source)| format!("{} ({source})", sanitize(warehouse)), ); + for (job, job_warehouse) in &facts.job_warehouses { + warehouse.push_str(&format!( + "; nodes of job {} on {}", + sanitize(job), + sanitize(job_warehouse) + )); + } let rows: Vec<(String, String)> = [ ("command", sanitize(&facts.command)), ("run_id", display_value(facts.run_id.as_deref())), @@ -41,7 +58,7 @@ pub fn render_run_context(facts: &RunContextFacts, style: CliStyle) -> String { ) }, ), - ("full_refresh", facts.full_refresh.to_string()), + ("full_refresh", full_refresh(facts)), ( "selected", format!( diff --git a/crates/sqlbuild-core/src/build_command/models.rs b/crates/sqlbuild-core/src/build_command/models.rs index bf8b0d29e7..1ab51524d8 100644 --- a/crates/sqlbuild-core/src/build_command/models.rs +++ b/crates/sqlbuild-core/src/build_command/models.rs @@ -182,6 +182,10 @@ pub struct RunContextFacts { /// The configuration layer the concurrency came from. pub concurrency_source: String, pub full_refresh: bool, + /// The named jobs whose own models the build rebuilds from scratch. + pub full_refresh_jobs: Vec, + /// Each named job whose nodes run on its own warehouse, and that warehouse. + pub job_warehouses: Vec<(String, String)>, pub selected_count: usize, pub total_count: usize, /// Valid ISO `start_date` and `end_date` variables. diff --git a/crates/sqlbuild-core/src/build_command/tests/test_build_command.rs b/crates/sqlbuild-core/src/build_command/tests/test_build_command.rs index f9bb0b89ad..eec9cd311d 100644 --- a/crates/sqlbuild-core/src/build_command/tests/test_build_command.rs +++ b/crates/sqlbuild-core/src/build_command/tests/test_build_command.rs @@ -151,6 +151,19 @@ fn given_run_context_when_rendering_then_sanitizes_and_groups_counts() { }, expected_text: "Execution\n command sqb build\n run_id not set\n target dev\n database not set\n schema not set\n warehouse small (connection warehouse)\n concurrency 0 configured limit\n full_refresh true\n selected 0 of 0 build resources\n date vars end_date=2026-01-31\n\nSelection files\n selector_file a.txt (1 selector)\n selector_file b.txt (2 selectors)\n\n", }, + ContextTestCase { + description: "named jobs whose nodes run on their own warehouses", + facts: RunContextFacts { + command: "sqb build".to_owned(), + warehouse: Some(("small".to_owned(), "connection warehouse".to_owned())), + job_warehouses: vec![ + ("orders_job".to_owned(), "ORDERS_WH".to_owned()), + ("refunds_job".to_owned(), "REFUNDS_WH".to_owned()), + ], + ..RunContextFacts::default() + }, + expected_text: "Execution\n command sqb build\n run_id not set\n target not set\n database not set\n schema not set\n warehouse small (connection warehouse); nodes of job orders_job on ORDERS_WH; nodes of job refunds_job on REFUNDS_WH\n concurrency 0 configured limit\n full_refresh false\n selected 0 of 0 build resources\n date vars not set\n\n", + }, ]; for test_case in test_cases { diff --git a/crates/sqlbuild-core/src/cli_text/_helpers/plan_document.rs b/crates/sqlbuild-core/src/cli_text/_helpers/plan_document.rs index 5331766ced..9a4edd216c 100644 --- a/crates/sqlbuild-core/src/cli_text/_helpers/plan_document.rs +++ b/crates/sqlbuild-core/src/cli_text/_helpers/plan_document.rs @@ -13,6 +13,12 @@ use crate::cli_text::constants::{ }; use crate::cli_text::models::{CliStyle, ModelPlanText, PlanTextReport}; +/// `job a` or `jobs a, b`. +fn job_list(jobs: &[String]) -> String { + let noun: &str = if jobs.len() == 1 { "job" } else { "jobs" }; + format!("{noun} {}", jobs.join(", ")) +} + fn header_lines(report: &PlanTextReport, full_refresh: bool) -> Vec { let style: CliStyle = CliStyle { use_color: true }; let mut lines: Vec = Vec::new(); @@ -25,6 +31,14 @@ fn header_lines(report: &PlanTextReport, full_refresh: bool) -> Vec { } if report.ignore_audit_failures { lines.push(style.warning(IGNORED_AUDIT_FAILURES_NOTICE)); + } else if !report.ignore_audit_failure_jobs.is_empty() { + lines.push(style.warning(&format!( + "{IGNORED_AUDIT_FAILURES_NOTICE} for the audits of {}", + job_list(&report.ignore_audit_failure_jobs) + ))); + } + for (job, warehouse) in &report.job_warehouses { + lines.push(format!("Nodes of job {job} run on warehouse {warehouse}")); } lines } diff --git a/crates/sqlbuild-core/src/cli_text/models.rs b/crates/sqlbuild-core/src/cli_text/models.rs index 189916b4e5..473141944c 100644 --- a/crates/sqlbuild-core/src/cli_text/models.rs +++ b/crates/sqlbuild-core/src/cli_text/models.rs @@ -430,6 +430,10 @@ pub struct PlanTextReport { pub pre_build_lines: Vec, /// Audit failures hold nothing back in the build this plan previews. pub ignore_audit_failures: bool, + /// The named jobs whose own audit failures hold nothing back. + pub ignore_audit_failure_jobs: Vec, + /// Each named job whose nodes run on its own warehouse, and that warehouse. + pub job_warehouses: Vec<(String, String)>, } /// The structural kind of a rendered plan line. diff --git a/crates/sqlbuild-core/src/cli_text/tests/helpers.rs b/crates/sqlbuild-core/src/cli_text/tests/helpers.rs index 85d07a22b4..441a5d4032 100644 --- a/crates/sqlbuild-core/src/cli_text/tests/helpers.rs +++ b/crates/sqlbuild-core/src/cli_text/tests/helpers.rs @@ -56,6 +56,8 @@ pub(super) fn plan_report(models: Vec) -> PlanTextReport { pruned_models: Vec::new(), pre_build_lines: Vec::new(), ignore_audit_failures: false, + ignore_audit_failure_jobs: Vec::new(), + job_warehouses: Vec::new(), } } diff --git a/crates/sqlbuild-discovery/src/orchestration_files/_helpers/raw.rs b/crates/sqlbuild-discovery/src/orchestration_files/_helpers/raw.rs index f3bbf3c737..bdf744b190 100644 --- a/crates/sqlbuild-discovery/src/orchestration_files/_helpers/raw.rs +++ b/crates/sqlbuild-discovery/src/orchestration_files/_helpers/raw.rs @@ -18,14 +18,12 @@ pub(crate) struct RawSchedule { pub(crate) default_status: InitialStatus, } -/// `[retry]`, or `[on_failure]`; each key may still use its deprecated name. +/// `[retry]`. #[derive(Deserialize)] #[serde(deny_unknown_fields)] pub(crate) struct RawRetry { pub(crate) attempts: Option>, - pub(crate) retries: Option>, pub(crate) delay_seconds: Option>, - pub(crate) retry_delay_seconds: Option>, } #[derive(Deserialize)] @@ -41,7 +39,6 @@ pub(crate) struct RawRun { pub(crate) pools: BTreeMap>, pub(crate) schedule: Option>, pub(crate) retry: Option>, - pub(crate) on_failure: Option>, } /// The job-only fields; the shared ones are read as `RawRun` from the same text. diff --git a/crates/sqlbuild-discovery/src/orchestration_files/_helpers/reading.rs b/crates/sqlbuild-discovery/src/orchestration_files/_helpers/reading.rs index d0924df8e3..dab5e5a4a2 100644 --- a/crates/sqlbuild-discovery/src/orchestration_files/_helpers/reading.rs +++ b/crates/sqlbuild-discovery/src/orchestration_files/_helpers/reading.rs @@ -8,12 +8,12 @@ use toml::de::DeTable; use crate::orchestration_files::_helpers::raw::{RawRetry, RawRun, RawSchedule}; use crate::orchestration_files::constants::{ - ATTEMPTS_RANGE, CONCURRENCY_KEY, MAX_POOL_AMOUNT, RETRIES_RANGE, RETRY_ATTEMPTS_KEY, + ATTEMPTS_RANGE, CONCURRENCY_KEY, MAX_POOL_AMOUNT, REMOVED_RETRY_TABLE, RETRY_ATTEMPTS_KEY, RETRY_DELAY_KEY, RETRY_DELAY_RANGE, RETRY_TABLE, SUGGESTION_CUTOFF, }; use crate::orchestration_files::errors::FileError; use crate::orchestration_files::models::{ - Deprecation, FilePosition, Positioned, RetryPolicy, RunFile, ScheduleFile, ScheduleSource, + FilePosition, Positioned, RetryPolicy, RunFile, ScheduleFile, ScheduleSource, }; /// The 1-based line and character column of byte `offset` in `text`. @@ -63,6 +63,8 @@ pub(crate) fn check_keys(text: &str, allowed: &[&str], kind: &str) -> Result<(), } let help: String = if name == CONCURRENCY_KEY { format!("move it under [settings] as `{CONCURRENCY_KEY} = ...`") + } else if name == REMOVED_RETRY_TABLE.0 { + REMOVED_RETRY_TABLE.1.to_owned() } else if let Some(nearest) = close_matches(name, allowed, 1, SUGGESTION_CUTOFF).first() { format!("did you mean '{nearest}'?") } else { @@ -122,39 +124,14 @@ fn schedule(text: &str, raw: Spanned) -> Result>, Option>); - -/// A retry number with whether it was set, and the deprecation its name records. -type RetryNumber = (Option<(T, bool)>, Option); - -/// One retry number within its name's range, and the deprecation a deprecated name records. +/// One `[retry]` number within `range`, if set. fn retry_number>( text: &str, - (table, keys, values): (&str, (&str, &str), RetryValues), - ranges: ((i64, i64), (i64, i64)), -) -> Result, FileError> { - let mut deprecation: Option = None; - let (key, value, range) = match values { - (Some(_), Some(deprecated)) => { - return Err(error_at( - text, - deprecated.span().start, - format!( - "{table} sets both `{}` and its deprecated name `{}`; keep only `{}`", - keys.0, keys.1, keys.0 - ), - )); - } - (Some(current), None) => (keys.0, current, ranges.0), - (None, Some(deprecated)) => { - deprecation = Some(Deprecation { - message: format!("`{}` is deprecated; rename it to `{}`", keys.1, keys.0), - position: position(text, deprecated.span().start), - }); - (keys.1, deprecated, ranges.1) - } - (None, None) => return Ok((None, None)), + (key, value): (&str, Option>), + range: (i64, i64), +) -> Result, FileError> { + let Some(value) = value else { + return Ok(None); }; let start: usize = value.span().start; let number: i64 = value.into_inner(); @@ -163,7 +140,7 @@ fn retry_number>( text, start, format!( - "{table} {key} must be a whole number from {} to {}", + "{RETRY_TABLE} {key} must be a whole number from {} to {}", range.0, range.1 ), ) @@ -171,82 +148,40 @@ fn retry_number>( if !(range.0..=range.1).contains(&number) { return Err(invalid()); } - T::try_from(number) - .map(|number| (Some((number, key == keys.1)), deprecation)) - .map_err(|_| invalid()) + T::try_from(number).map(Some).map_err(|_| invalid()) } -/// `[retry]`, or the deprecated `[on_failure]`, checked, with every deprecated name it uses. -fn retry( - text: &str, - (current, deprecated): (Option>, Option>), -) -> Result<(Option, Vec), FileError> { - let mut deprecations: Vec = Vec::new(); - let (table, raw) = match (current, deprecated) { - (Some(_), Some(deprecated)) => { - return Err(error_at( - text, - deprecated.span().start, - format!( - "`[{}]` and its deprecated name `[{}]` are both set; keep only `[{}]`", - RETRY_TABLE.0, RETRY_TABLE.1, RETRY_TABLE.0 - ), - )); - } - (Some(current), None) => (RETRY_TABLE.0, current), - (None, Some(deprecated)) => { - deprecations.push(Deprecation { - message: format!( - "`[{}]` is deprecated; rename it to `[{}]`", - RETRY_TABLE.1, RETRY_TABLE.0 - ), - position: position(text, deprecated.span().start), - }); - (RETRY_TABLE.1, deprecated) - } - (None, None) => return Ok((None, deprecations)), +/// `[retry]`, checked. +fn retry(text: &str, raw: Option>) -> Result, FileError> { + let Some(raw) = raw else { + return Ok(None); }; let start: usize = raw.span().start; let raw: RawRetry = raw.into_inner(); - let ranges = |range: (u8, u8)| (i64::from(range.0), i64::from(range.1)); - let (attempts, attempts_deprecation) = retry_number::( + let attempts: Option = retry_number( text, - (table, RETRY_ATTEMPTS_KEY, (raw.attempts, raw.retries)), - (ranges(ATTEMPTS_RANGE), ranges(RETRIES_RANGE)), + (RETRY_ATTEMPTS_KEY, raw.attempts), + (i64::from(ATTEMPTS_RANGE.0), i64::from(ATTEMPTS_RANGE.1)), )?; - let attempts: Option = - attempts.map(|(number, reruns)| if reruns { number + 1 } else { number }); - let (delay_seconds, delay_deprecation) = retry_number::( + let delay_seconds: Option = retry_number( text, + (RETRY_DELAY_KEY, raw.delay_seconds), ( - table, - RETRY_DELAY_KEY, - (raw.delay_seconds, raw.retry_delay_seconds), + i64::from(RETRY_DELAY_RANGE.0), + i64::from(RETRY_DELAY_RANGE.1), ), - { - let delay = ( - i64::from(RETRY_DELAY_RANGE.0), - i64::from(RETRY_DELAY_RANGE.1), - ); - (delay, delay) - }, )?; - let delay_seconds: Option = delay_seconds.map(|(seconds, _)| seconds); - deprecations.extend(attempts_deprecation.into_iter().chain(delay_deprecation)); let attempts: u8 = attempts.ok_or_else(|| { error_at( text, start, - format!("{table} needs `{}`", RETRY_ATTEMPTS_KEY.0), + format!("{RETRY_TABLE} needs `{RETRY_ATTEMPTS_KEY}`"), ) })?; - Ok(( - Some(RetryPolicy { - attempts, - delay_seconds: delay_seconds.unwrap_or(RETRY_DELAY_RANGE.0), - }), - deprecations, - )) + Ok(Some(RetryPolicy { + attempts, + delay_seconds: delay_seconds.unwrap_or(RETRY_DELAY_RANGE.0), + })) } /// The fields jobs and groups share, checked. @@ -278,7 +213,7 @@ pub(crate) fn run_file(text: &str, raw: RawRun) -> Result { .schedule .map(|value| schedule(text, value)) .transpose()?; - let (retry, deprecations) = retry(text, (raw.retry, raw.on_failure))?; + let retry: Option = retry(text, raw.retry)?; Ok(RunFile { name, description, @@ -288,6 +223,5 @@ pub(crate) fn run_file(text: &str, raw: RawRun) -> Result { pools, schedule: scheduled, retry, - deprecations, }) } diff --git a/crates/sqlbuild-discovery/src/orchestration_files/constants.rs b/crates/sqlbuild-discovery/src/orchestration_files/constants.rs index f9af4272d7..9fbb9d80f8 100644 --- a/crates/sqlbuild-discovery/src/orchestration_files/constants.rs +++ b/crates/sqlbuild-discovery/src/orchestration_files/constants.rs @@ -1,7 +1,7 @@ //! The keys job and job group files accept, and the bounds on their run controls. /// Top-level keys of a job file. -pub const JOB_KEYS: [&str; 13] = [ +pub const JOB_KEYS: [&str; 12] = [ "name", "description", "select", @@ -12,12 +12,11 @@ pub const JOB_KEYS: [&str; 13] = [ "pools", "schedule", "retry", - "on_failure", "resources", "settings", ]; /// Top-level keys of a job group file. -pub const GROUP_KEYS: [&str; 11] = [ +pub const GROUP_KEYS: [&str; 10] = [ "name", "description", "jobs", @@ -27,7 +26,6 @@ pub const GROUP_KEYS: [&str; 11] = [ "pools", "schedule", "retry", - "on_failure", "dependencies", ]; /// The group table that sets how members wait for and react to the jobs they depend on. @@ -40,16 +38,18 @@ pub(crate) const DEPENDENCY_FAILURE_KEY: &str = "on_failure"; pub(crate) const DEPENDENCY_FAILURE_VALUES: [&str; 3] = ["partial", "skip", "run"]; /// The fewest and most runs `retry.attempts` allows, the first run included. pub const ATTEMPTS_RANGE: (u8, u8) = (2, 11); -/// The fewest and most reruns the deprecated `on_failure.retries` allows; `attempts` is one more. -pub const RETRIES_RANGE: (u8, u8) = (1, 10); /// The shortest and longest `retry.delay_seconds`, a day. pub const RETRY_DELAY_RANGE: (u32, u32) = (0, 86_400); -/// The retry table, and the deprecated name it replaces. -pub(crate) const RETRY_TABLE: (&str, &str) = ("retry", "on_failure"); -/// The retry count key, and the deprecated name it replaces. -pub(crate) const RETRY_ATTEMPTS_KEY: (&str, &str) = ("attempts", "retries"); -/// The retry delay key, and the deprecated name it replaces. -pub(crate) const RETRY_DELAY_KEY: (&str, &str) = ("delay_seconds", "retry_delay_seconds"); +/// The retry table, and the keys of its run count and delay. +pub(crate) const RETRY_TABLE: &str = "retry"; +pub(crate) const RETRY_ATTEMPTS_KEY: &str = "attempts"; +pub(crate) const RETRY_DELAY_KEY: &str = "delay_seconds"; +/// The removed retry table, and how to write it as `[retry]`. +pub(crate) const REMOVED_RETRY_TABLE: (&str, &str) = ( + "on_failure", + "`[on_failure]` is now `[retry]`; `attempts = retries + 1`, \ + `delay_seconds = retry_delay_seconds`", +); /// The job setting that moved under `[settings]`. pub(crate) const CONCURRENCY_KEY: &str = "concurrency"; /// The most slots a job or job group may take of one pool. diff --git a/crates/sqlbuild-discovery/src/orchestration_files/models.rs b/crates/sqlbuild-discovery/src/orchestration_files/models.rs index e27348a7a8..afec0707b2 100644 --- a/crates/sqlbuild-discovery/src/orchestration_files/models.rs +++ b/crates/sqlbuild-discovery/src/orchestration_files/models.rs @@ -34,13 +34,6 @@ pub struct RetryPolicy { pub delay_seconds: u32, } -/// A deprecated name a file still uses, where it is written and what replaces it. -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct Deprecation { - pub message: String, - pub position: FilePosition, -} - /// When a group member starts: once the jobs it reads from end (`job`). #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] pub enum WaitFor { @@ -116,10 +109,7 @@ pub struct RunFile { /// Each pool the run takes, with how many of its slots. pub pools: BTreeMap, pub schedule: Option, - /// `[retry]`, or the deprecated `[on_failure]` it replaces. pub retry: Option, - /// Deprecated names the file uses, in file order. - pub deprecations: Vec, } /// One `orchestration/jobs/**.toml` file. diff --git a/crates/sqlbuild-discovery/src/orchestration_files/tests/test_read_files.rs b/crates/sqlbuild-discovery/src/orchestration_files/tests/test_read_files.rs index ecbbd4c247..b5bdcb4b33 100644 --- a/crates/sqlbuild-discovery/src/orchestration_files/tests/test_read_files.rs +++ b/crates/sqlbuild-discovery/src/orchestration_files/tests/test_read_files.rs @@ -23,8 +23,8 @@ overlap = "skip" pools = { warehouse = 2, supplier_api = 1 } schedule = { calendar = "trading_days_calendar", timezone = "Europe/London", default_status = "paused" } -[on_failure] -retries = 2 +[retry] +attempts = 3 [resources] memory = "2g" @@ -50,16 +50,6 @@ fn given_job_files_when_reading_then_types_every_field_and_keeps_positions() { attempts: 3, delay_seconds: 0, }), - expected_deprecations: vec![ - ( - "`[on_failure]` is deprecated; rename it to `[retry]`", - at(11, 1), - ), - ( - "`retries` is deprecated; rename it to `attempts`", - at(12, 11), - ), - ], expected_schedule: Some(ScheduleFile { source: ScheduleSource::Calendar(written("trading_days_calendar", 9, 25)), timezone: written("Europe/London", 9, 61), @@ -81,7 +71,6 @@ fn given_job_files_when_reading_then_types_every_field_and_keeps_positions() { expected_overlap: Overlap::Queue, expected_pools: &[], expected_retry: None, - expected_deprecations: Vec::new(), expected_schedule: None, expected_cpus: None, expected_settings: BTreeMap::new(), @@ -98,33 +87,6 @@ fn given_job_files_when_reading_then_types_every_field_and_keeps_positions() { attempts: 3, delay_seconds: 900, }), - expected_deprecations: Vec::new(), - expected_schedule: None, - expected_cpus: None, - expected_settings: BTreeMap::new(), - }, - JobReadTestCase { - description: "a retry table may still use the deprecated key names", - text: "name = \"a_job\"\ndescription = \"d\"\nselect = \"orders\"\nretry = { retries = 1, retry_delay_seconds = 60 }\n", - expected_name: written("a_job", 1, 8), - expected_select: written("orders", 3, 10), - expected_exclude: None, - expected_overlap: Overlap::Queue, - expected_pools: &[], - expected_retry: Some(RetryPolicy { - attempts: 2, - delay_seconds: 60, - }), - expected_deprecations: vec![ - ( - "`retries` is deprecated; rename it to `attempts`", - at(4, 21), - ), - ( - "`retry_delay_seconds` is deprecated; rename it to `delay_seconds`", - at(4, 46), - ), - ], expected_schedule: None, expected_cpus: None, expected_settings: BTreeMap::new(), @@ -168,16 +130,6 @@ fn given_job_files_when_reading_then_types_every_field_and_keeps_positions() { "{}", test_case.description ); - assert_eq!( - job.run - .deprecations - .iter() - .map(|deprecation| (deprecation.message.as_str(), deprecation.position)) - .collect::>(), - test_case.expected_deprecations, - "{}", - test_case.description - ); assert_eq!( job.run.schedule, test_case.expected_schedule, "{}", @@ -215,6 +167,16 @@ fn given_invalid_job_files_when_reading_then_reports_message_position_and_help() expected_position: at(4, 1), expected_help: Some("move it under [settings] as `concurrency = ...`"), }, + FileErrorTestCase { + description: "the removed on_failure table names its replacement", + text: "name = \"a_job\"\ndescription = \"d\"\nselect = \"orders\"\n\n[on_failure]\nretries = 2\n", + expected_message: "unknown job key 'on_failure'", + expected_position: at(5, 2), + expected_help: Some( + "`[on_failure]` is now `[retry]`; `attempts = retries + 1`, \ + `delay_seconds = retry_delay_seconds`", + ), + }, FileErrorTestCase { description: "a pool amount of zero", text: "name = \"a_job\"\ndescription = \"d\"\nselect = \"orders\"\npools = { supplier_api = 0 }\n", @@ -243,18 +205,11 @@ fn given_invalid_job_files_when_reading_then_reports_message_position_and_help() expected_position: at(4, 12), expected_help: None, }, - FileErrorTestCase { - description: "retries out of range", - text: "name = \"a_job\"\ndescription = \"d\"\nselect = \"orders\"\non_failure = { retries = 0 }\n", - expected_message: "on_failure retries must be a whole number from 1 to 10", - expected_position: at(4, 26), - expected_help: None, - }, FileErrorTestCase { description: "retry delay beyond a day", - text: "name = \"a_job\"\ndescription = \"d\"\nselect = \"orders\"\non_failure = { retries = 1, retry_delay_seconds = 86401 }\n", - expected_message: "on_failure retry_delay_seconds must be a whole number from 0 to 86400", - expected_position: at(4, 51), + text: "name = \"a_job\"\ndescription = \"d\"\nselect = \"orders\"\nretry = { attempts = 2, delay_seconds = 86401 }\n", + expected_message: "retry delay_seconds must be a whole number from 0 to 86400", + expected_position: at(4, 41), expected_help: None, }, FileErrorTestCase { @@ -279,23 +234,16 @@ fn given_invalid_job_files_when_reading_then_reports_message_position_and_help() expected_help: None, }, FileErrorTestCase { - description: "a key and its deprecated name together", - text: "name = \"a_job\"\ndescription = \"d\"\nselect = \"orders\"\nretry = { attempts = 1, retries = 2 }\n", - expected_message: "retry sets both `attempts` and its deprecated name `retries`; keep only `attempts`", - expected_position: at(4, 35), - expected_help: None, - }, - FileErrorTestCase { - description: "the retry table and its deprecated name together", - text: "name = \"a_job\"\ndescription = \"d\"\nselect = \"orders\"\nretry = { attempts = 1 }\non_failure = { retries = 2 }\n", - expected_message: "`[retry]` and its deprecated name `[on_failure]` are both set; keep only `[retry]`", - expected_position: at(5, 14), + description: "the removed retry key names fail", + text: "name = \"a_job\"\ndescription = \"d\"\nselect = \"orders\"\nretry = { retries = 2 }\n", + expected_message: "unknown field `retries`, expected `attempts` or `delay_seconds`", + expected_position: at(4, 11), expected_help: None, }, FileErrorTestCase { description: "unknown retry key", text: "name = \"a_job\"\ndescription = \"d\"\nselect = \"orders\"\nretry = { attempts = 1, delay = 5 }\n", - expected_message: "unknown field `delay`, expected one of `attempts`, `retries`, `delay_seconds`, `retry_delay_seconds`", + expected_message: "unknown field `delay`, expected `attempts` or `delay_seconds`", expected_position: at(4, 25), expected_help: None, }, diff --git a/crates/sqlbuild-discovery/src/orchestration_files/tests/test_types.rs b/crates/sqlbuild-discovery/src/orchestration_files/tests/test_types.rs index 428ed58fce..7c99a04bec 100644 --- a/crates/sqlbuild-discovery/src/orchestration_files/tests/test_types.rs +++ b/crates/sqlbuild-discovery/src/orchestration_files/tests/test_types.rs @@ -13,8 +13,6 @@ pub(super) struct JobReadTestCase { pub(super) expected_overlap: Overlap, pub(super) expected_pools: &'static [(&'static str, u32)], pub(super) expected_retry: Option, - /// Each deprecation message with its line and column. - pub(super) expected_deprecations: Vec<(&'static str, FilePosition)>, pub(super) expected_schedule: Option, pub(super) expected_cpus: Option, pub(super) expected_settings: BTreeMap, diff --git a/crates/sqlbuild-python-base/src/records/_helpers/compiler_records/constants.rs b/crates/sqlbuild-python-base/src/records/_helpers/compiler_records/constants.rs index ca785c24e1..7b590fc4c3 100644 --- a/crates/sqlbuild-python-base/src/records/_helpers/compiler_records/constants.rs +++ b/crates/sqlbuild-python-base/src/records/_helpers/compiler_records/constants.rs @@ -400,6 +400,8 @@ const R_COMPILE_PIPELINE_OPTIONS: RecordSpec = RecordSpec { FieldSpec::new("selection_diagnostics", FieldDefault::False), FieldSpec::new("plan_sql_tests", FieldDefault::True), FieldSpec::new("ignore_audit_failures", FieldDefault::False), + FieldSpec::new("cli_ignore_audit_failures", FieldDefault::None), + FieldSpec::new("job_settings", FieldDefault::True), FieldSpec::new("record_migration_fingerprints", FieldDefault::True), FieldSpec::new("accepts_unit_test_selectors", FieldDefault::False), FieldSpec::new( @@ -873,6 +875,10 @@ const R_COMPILE_PIPELINE_RESULT: RecordSpec = RecordSpec { FieldSpec::new("custom_materializations", FieldDefault::EmptyDict), FieldSpec::new("python_node_names", FieldDefault::EmptyFrozenset), FieldSpec::new("python_plan_entries", FieldDefault::EmptyTuple), + FieldSpec::new("job_limits", FieldDefault::EmptyDict), + FieldSpec::new("node_jobs", FieldDefault::EmptyDict), + FieldSpec::new("node_warehouses", FieldDefault::EmptyDict), + FieldSpec::new("python_node_warehouses", FieldDefault::EmptyDict), FieldSpec::new("compile_seconds", FieldDefault::None), FieldSpec::new("planning_seconds", FieldDefault::None), ], diff --git a/crates/sqlbuild-python-base/src/records/_helpers/planner_records/constants.rs b/crates/sqlbuild-python-base/src/records/_helpers/planner_records/constants.rs index be7daa312e..6c629f4d2b 100644 --- a/crates/sqlbuild-python-base/src/records/_helpers/planner_records/constants.rs +++ b/crates/sqlbuild-python-base/src/records/_helpers/planner_records/constants.rs @@ -85,6 +85,8 @@ const R_PLANNER_OVERRIDES: RecordSpec = RecordSpec { fields: &[ FieldSpec::new("cursor_overrides", FieldDefault::None), FieldSpec::new("full_refresh", FieldDefault::False), + FieldSpec::new("full_refresh_models", FieldDefault::EmptyFrozenset), + FieldSpec::new("full_refresh_functions", FieldDefault::EmptyFrozenset), FieldSpec::new("reload_sources", FieldDefault::False), FieldSpec::new("forced_stale_model_names", FieldDefault::EmptyTuple), FieldSpec::new("external_blocked_model_names", FieldDefault::EmptyTuple), diff --git a/crates/sqlbuild-python-base/src/records/constants.rs b/crates/sqlbuild-python-base/src/records/constants.rs index d2b0e21e1b..643e4877df 100644 --- a/crates/sqlbuild-python-base/src/records/constants.rs +++ b/crates/sqlbuild-python-base/src/records/constants.rs @@ -244,6 +244,7 @@ const BUILD_COMMAND_REQUEST: RecordSpec = RecordSpec { FieldSpec::new("no_color", FieldDefault::False), FieldSpec::new("fail_fast", FieldDefault::False), FieldSpec::new("full_refresh", FieldDefault::False), + FieldSpec::new("job_settings", FieldDefault::True), FieldSpec::new("changed", FieldDefault::False), FieldSpec::new("changed_counts", FieldDefault::None), FieldSpec::new("reach", FieldDefault::None), @@ -568,6 +569,7 @@ const PLAN_COMMAND_REQUEST: RecordSpec = RecordSpec { FieldSpec::new("cursor_overrides", FieldDefault::None), FieldSpec::new("json_output", FieldDefault::False), FieldSpec::new("full_refresh", FieldDefault::False), + FieldSpec::new("job_settings", FieldDefault::True), FieldSpec::new("changed", FieldDefault::False), FieldSpec::new("changed_counts", FieldDefault::None), FieldSpec::new("reach", FieldDefault::None), diff --git a/crates/sqlbuild-python-compile/src/compile_driver/_helpers/jobs/job_limits.rs b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/jobs/job_limits.rs new file mode 100644 index 0000000000..c74c98a000 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/jobs/job_limits.rs @@ -0,0 +1,18 @@ +//! The named jobs' limits a planned build carries. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::types::{PyDict, PyDictMethods}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +/// The named jobs' `concurrency` limits added together; none without a command-line override. +pub(crate) fn job_limit_total(pipeline: &Bound<'_, PyAny>) -> PyResult> { + let limits: Bound<'_, PyDict> = native_field(pipeline, "job_limits")?.cast_into()?; + if limits.is_empty() { + return Ok(None); + } + let mut total: i64 = 0; + for value in limits.values() { + total = total.saturating_add(value.extract::()?); + } + Ok(Some(total)) +} diff --git a/crates/sqlbuild-python-compile/src/compile_driver/_helpers/jobs/job_node_settings.rs b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/jobs/job_node_settings.rs new file mode 100644 index 0000000000..a4f9456ae4 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/jobs/job_node_settings.rs @@ -0,0 +1,542 @@ +//! The settings each planned node takes from the jobs its selection names, applied to the plan. + +use std::collections::{BTreeMap, BTreeSet}; + +use pyo3::IntoPyObject; +use pyo3::PyErr; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyBool, PyDict, PyDictMethods, PyFrozenSet, PyTuple}; +use serde_json::Value; +use sqlbuild_analysis::assembly::project::types::ObjectKey; +use sqlbuild_analysis::definition_changes::constants::PYTHON_NODE_KINDS; +use sqlbuild_analysis::graph::main::selectors_need_jobs::selectors_need_jobs; +use sqlbuild_analysis::job_settings::constants::WAREHOUSE_SETTING; +use sqlbuild_analysis::job_settings::errors::{JobSettingConflict, NodeJobSettingsError}; +use sqlbuild_analysis::job_settings::main::conflict_help::conflict_help; +use sqlbuild_analysis::job_settings::main::consumed_setting::consumed_setting; +use sqlbuild_analysis::job_settings::main::resolve_job_settings::resolve_job_settings; +use sqlbuild_analysis::job_settings::models::{ + DeclaredRun, NodeConsumer, NodeJobSettings, ResolvedJobSettings, +}; +use sqlbuild_config::command_connection::errors::CommandConnectionError; +use sqlbuild_config::command_connection::main::check_job_warehouse::check_job_warehouse; +use sqlbuild_config::command_connection::main::unsupported_job_warehouse::unsupported_job_warehouse; +use sqlbuild_python_base::records::main::construct::new_error::new_error; +use sqlbuild_python_base::records::types::ErrorId; +use sqlbuild_python_project::job_settings::main::given_settings::given_settings; + +use crate::compile_driver::main::setup::project_graph::project_graph; +use crate::compile_driver::models::PlannedJobSettings; +use sqlbuild_python_base::boundary::main::compiler_guard::compiler_guard; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::main::fields::replaced::replaced; +use sqlbuild_python_front::compile_driver::main::session_warehouse_support::session_warehouse_support; +use sqlbuild_python_front::contracts::main::resolve_effective_adapter_name::resolve_effective_adapter_name; +use sqlbuild_python_project::compiled_project::constants::{ + JOB_FULL_REFRESH_KEY, JOB_IGNORED_AUDIT_FAILURES_KEY, JOB_WAREHOUSES_KEY, +}; +use sqlbuild_python_project::compiled_views::main::project::audits::audits; +use sqlbuild_python_project::graph::main::nodes::native_key::native_key; +use sqlbuild_python_project::graph::main::python_nodes::python_node_graph::python_node_graph; +use sqlbuild_python_project::graph::models::KeyCache; +use sqlbuild_python_project::graph::models::{ + CompiledProjectGraph, NativeProjectGraph, PythonAwareSelection, +}; +use sqlbuild_python_project::job_settings::main::job_declarations::job_declarations; +use sqlbuild_python_project::job_settings::main::job_scope_project_dir::job_scope_project_dir; +use sqlbuild_python_project::job_settings::main::job_selector_error::job_selector_error; +use sqlbuild_python_project::job_settings::main::job_setting_conflict::job_setting_conflict; +use sqlbuild_python_project::resource_views::main::audit_reads::attached_target_kind::attached_target_kind; +use sqlbuild_python_project::resource_views::main::audit_reads::attached_target_name::attached_target_name; +use sqlbuild_python_project::resource_views::main::audit_reads::key::key as audit_key; +use sqlbuild_python_project::resource_views::main::audit_reads::scope_deps::scope_deps; + +const MODEL_KIND: &str = "model"; +const FUNCTION_KINDS: [&str; 2] = ["udf", "table_fn"]; +/// The node setting `consumer` applies, as the job setting table names it. +pub(crate) fn node_setting(consumer: NodeConsumer) -> &'static str { + consumed_setting(consumer).unwrap_or_default() +} + +/// The `CliUserError` of a job warehouse that cannot be used. +fn cli_error(py: Python<'_>, error: &CommandConnectionError) -> PyErr { + let (code, help) = match error { + CommandConnectionError::User { code, help, .. } => (*code, help.as_deref()), + CommandConnectionError::ProjectConfig(_) => (None, None), + }; + new_error(py, ErrorId::CliUser, error.message(), (code, help)) +} + +fn is_python(key: &ObjectKey) -> bool { + PYTHON_NODE_KINDS.contains(&key.0.as_str()) +} + +/// The jobs whose `warehouse` a node runs on, checked like `--warehouse` (`C261`, `C260`). +fn job_warehouses( + (project, discovered): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), + settings: &NodeJobSettings, + declared: &BTreeMap, +) -> PyResult> { + let py: Python<'_> = project.py(); + let mut warehouses: BTreeMap = BTreeMap::new(); + for (key, jobs) in &settings.node_jobs { + if !settings + .values + .get(key) + .is_some_and(|values| values.contains_key(node_setting(NodeConsumer::NodeWarehouses))) + { + continue; + } + for job in jobs { + if warehouses.contains_key(job) { + continue; + } + if let Some(DeclaredRun::Job { settings, .. }) = declared.get(job) + && let Some(value) = settings.get(node_setting(NodeConsumer::NodeWarehouses)) + { + let name: String = check_job_warehouse(job, value.as_str()) + .map_err(|error| cli_error(py, &error))?; + warehouses.insert(job.clone(), name); + } + } + } + let Some((job, warehouse)) = warehouses.iter().next() else { + return Ok(warehouses); + }; + let adapter_name: Bound<'_, PyAny> = resolve_effective_adapter_name( + &native_field(discovered, "project_config")?, + &native_field(discovered, "local_config")?, + )?; + let project_dir: Bound<'_, PyAny> = job_scope_project_dir().into_pyobject(py)?.into_any(); + if !session_warehouse_support(py, &adapter_name, &project_dir)? { + let adapter: String = adapter_name.str()?.extract()?; + return Err(cli_error( + py, + &unsupported_job_warehouse(job, warehouse, &adapter), + )); + } + Ok(warehouses) +} + +/// The named jobs of `settings` that set `setting` to true for a selected node of `kind`, if given. +fn jobs_setting( + settings: &NodeJobSettings, + declared: &BTreeMap, + (setting, kinds): (&str, Option<&[&str]>), +) -> Vec { + let mut jobs: BTreeSet = BTreeSet::new(); + for (key, node_jobs) in &settings.node_jobs { + if is_python(key) || kinds.is_some_and(|kinds| !kinds.contains(&key.0.as_str())) { + continue; + } + for job in node_jobs { + if let Some(DeclaredRun::Job { settings, .. }) = declared.get(job) + && settings.get(setting) == Some(&Value::Bool(true)) + { + jobs.insert(job.clone()); + } + } + } + jobs.into_iter().collect() +} + +/// Whether every read node ignores, by job value or else `build_ignores`; `None` without job values. +pub(crate) fn singular_audit_ignores( + job_values: impl Iterator>, + build_ignores: bool, +) -> Option { + let values: Vec> = job_values.collect(); + values + .iter() + .any(Option::is_some) + .then(|| values.iter().all(|value| value.unwrap_or(build_ignores))) +} + +/// The nodes each audit attached to no node reads, by the audit's key. +fn unattached_audit_reads( + project: &Bound<'_, PyAny>, +) -> PyResult>> { + let mut reads: BTreeMap> = BTreeMap::new(); + for audit in audits(project)?.try_iter()? { + let audit: Bound<'_, PyAny> = audit?; + if !attached_target_kind(&audit)?.is_none() && !attached_target_name(&audit)?.is_none() { + continue; + } + let deps: Vec = scope_deps(&audit)? + .try_iter()? + .map(|dep| native_key(&dep?)) + .collect::>()?; + reads.insert(native_key(&audit_key(&audit)?)?, deps); + } + Ok(reads) +} + +fn strings(value: &Bound<'_, PyAny>) -> PyResult> { + value.try_iter()?.map(|item| item?.extract()).collect() +} + +/// The named jobs' settings per selected node; `None` when ignored, unnamed or unselectable. +pub(crate) fn planned_job_settings( + (project, discovered): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), + (options, cli_warehouse): (&Bound<'_, PyAny>, bool), +) -> PyResult> { + let py: Python<'_> = project.py(); + if !native_field(options, "job_settings")?.is_truthy()? { + return Ok(None); + } + let select: Vec = strings(&native_field(options, "select")?)?; + let native: Bound<'_, NativeProjectGraph> = + native_field(project, "lineage_graph")?.cast_into()?; + let native = native.get(); + if !selectors_need_jobs(native.graph(), &select, &[]) { + return Ok(None); + } + let exclude: Vec = strings(&native_field(options, "exclude")?)?; + let Some(declared): Option> = job_declarations(py)? else { + return Ok(None); + }; + let mut given: BTreeSet = given_settings(); + if cli_warehouse { + given.insert(WAREHOUSE_SETTING.to_owned()); + } + let all: Vec = select.iter().chain(&exclude).cloned().collect(); + let compiled: Bound<'_, CompiledProjectGraph> = project_graph(project)?.cast_into()?; + let selection = PythonAwareSelection::new( + (&compiled, native.graph()), + &python_node_graph(discovered)?, + &all, + )?; + let token_nodes = + |token: &str, exclude: &[String]| selection.resolve_keys(&[token.to_owned()], exclude); + let resolved = compiler_guard(|| { + Ok(resolve_job_settings( + declared, + (&select, &given), + Some((selection.graph(), (&select, &exclude), &token_nodes)), + )) + })?; + match resolved { + Ok(ResolvedJobSettings { + nodes: Some(settings), + declared, + .. + }) => Ok(Some(PlannedJobSettings { + ignoring_jobs: jobs_setting( + &settings, + &declared, + (node_setting(NodeConsumer::AuditEntries), None), + ), + refreshing_jobs: jobs_setting( + &settings, + &declared, + ( + node_setting(NodeConsumer::RefreshOverrides), + Some(&[MODEL_KIND, FUNCTION_KINDS[0], FUNCTION_KINDS[1]]), + ), + ), + job_warehouses: job_warehouses((project, discovered), &settings, &declared)?, + unattached_audit_reads: unattached_audit_reads(project)?, + settings, + })), + Ok(ResolvedJobSettings { nodes: None, .. }) => Ok(None), + Err(NodeJobSettingsError::Selector(failure)) => Err(job_selector_error(py, &failure)), + Err(NodeJobSettingsError::Conflict(conflict)) => Err(job_setting_conflict(py, &conflict)), + } +} + +impl PlannedJobSettings { + fn value(&self, key: &ObjectKey, setting: &str) -> Option<&Value> { + self.settings + .values + .get(key) + .and_then(|values| values.get(setting)) + } + + /// The names of the nodes of `kinds` the named jobs rebuild from scratch. + fn refreshed<'py>(&self, py: Python<'py>, kinds: &[&str]) -> PyResult> { + let names: Vec<&String> = self + .settings + .values + .iter() + .filter(|(key, values)| { + kinds.contains(&key.0.as_str()) + && values.get(node_setting(NodeConsumer::RefreshOverrides)) + == Some(&Value::Bool(true)) + }) + .map(|(key, _)| &key.1) + .collect(); + PyFrozenSet::new(py, names) + } + + /// The models the named jobs rebuild from scratch. + pub fn full_refresh_models<'py>(&self, py: Python<'py>) -> PyResult> { + self.refreshed(py, &[MODEL_KIND]) + } + + /// The functions the named jobs recreate. + pub fn full_refresh_functions<'py>( + &self, + py: Python<'py>, + ) -> PyResult> { + self.refreshed(py, &FUNCTION_KINDS) + } + + /// Each job's limit and each planned node's limited jobs; none under `--concurrency`. + pub(crate) fn job_limits<'py>( + &self, + plan: &Bound<'py, PyAny>, + cli_concurrency: bool, + ) -> PyResult<(Bound<'py, PyDict>, Bound<'py, PyDict>)> { + let py: Python<'py> = plan.py(); + let limits: Bound<'py, PyDict> = PyDict::new(py); + let node_jobs: Bound<'py, PyDict> = PyDict::new(py); + if cli_concurrency || self.settings.job_limits.is_empty() { + return Ok((limits, node_jobs)); + } + for (job, limit) in &self.settings.job_limits { + limits.set_item(job, limit)?; + } + let limited = |keys: &[ObjectKey]| -> Vec { + let mut jobs: Vec = Vec::new(); + for job in keys + .iter() + .filter_map(|key| self.settings.node_jobs.get(key)) + .flatten() + .filter(|job| self.settings.job_limits.contains_key(*job)) + { + if !jobs.contains(job) { + jobs.push(job.clone()); + } + } + jobs + }; + let cache = KeyCache::new(py)?; + for key in self.settings.node_jobs.keys().filter(|key| !is_python(key)) { + let jobs: Vec = limited(std::slice::from_ref(key)); + if !jobs.is_empty() { + node_jobs.set_item(cache.key(key)?, PyTuple::new(py, jobs)?)?; + } + } + for entry in native_field(plan, "test_entries")?.try_iter()? { + let entry = entry?; + let mut keys: Vec = vec![native_key(&native_field(&entry, "key")?)?]; + for dep in native_field(&entry, "scope_deps")?.try_iter()? { + keys.push(native_key(&dep?)?); + } + let jobs: Vec = limited(&keys); + if !jobs.is_empty() { + node_jobs.set_item(native_field(&entry, "key")?, PyTuple::new(py, jobs)?)?; + } + } + Ok((limits, node_jobs)) + } + + /// The warehouse of a node's named job, when one applies to it. + fn warehouse(&self, key: &ObjectKey) -> Option<&str> { + if self.job_warehouses.is_empty() { + return None; + } + self.value(key, node_setting(NodeConsumer::NodeWarehouses)) + .and_then(Value::as_str) + } + + /// Each SQL node's job warehouse, a test's only if all its nodes share it, and Python nodes'. + pub(crate) fn node_warehouses<'py>( + &self, + plan: &Bound<'py, PyAny>, + ) -> PyResult<(Bound<'py, PyDict>, Bound<'py, PyDict>)> { + let py: Python<'py> = plan.py(); + let nodes: Bound<'py, PyDict> = PyDict::new(py); + let python: Bound<'py, PyDict> = PyDict::new(py); + if self.job_warehouses.is_empty() { + return Ok((nodes, python)); + } + for key in native_field(plan, "execution_order")?.try_iter()? { + let key: Bound<'py, PyAny> = key?; + if let Some(warehouse) = self.warehouse(&native_key(&key)?) { + nodes.set_item(key, warehouse)?; + } + } + for entry in native_field(plan, "test_entries")?.try_iter()? { + let entry: Bound<'py, PyAny> = entry?; + let mut covered: BTreeSet> = BTreeSet::new(); + for dep in native_field(&entry, "scope_deps")?.try_iter()? { + covered.insert(self.warehouse(&native_key(&dep?)?)); + } + let key: Bound<'py, PyAny> = native_field(&entry, "key")?; + match covered.into_iter().collect::>().as_slice() { + [Some(warehouse)] => nodes.set_item(key, *warehouse)?, + _ if nodes.contains(&key)? => nodes.del_item(key)?, + _ => {} + } + } + for key in self.settings.values.keys().filter(|key| is_python(key)) { + if let Some(warehouse) = self.warehouse(key) { + python.set_item(&key.1, warehouse)?; + } + } + Ok((nodes, python)) + } + + /// The plan with its jobs' `run_tests` and, unless the command line decided, audit failures. + pub(crate) fn apply_to_plan<'py>( + &self, + plan: &Bound<'py, PyAny>, + (cli_ignore_audit_failures, build_ignores): (&Bound<'py, PyAny>, bool), + ) -> PyResult> { + let py: Python<'py> = plan.py(); + let mut skipped: BTreeSet = BTreeSet::new(); + let mut tests: Vec> = Vec::new(); + for entry in native_field(plan, "test_entries")?.try_iter()? { + let entry = entry?; + let key: ObjectKey = native_key(&native_field(&entry, "key")?)?; + if self.skips_test(&key, &native_field(&entry, "scope_deps")?)? { + skipped.insert(key); + } else { + tests.push(entry); + } + } + let mut changes: Vec<(&str, Bound<'py, PyAny>)> = Vec::new(); + if !skipped.is_empty() { + let order: Vec> = native_field(plan, "execution_order")? + .try_iter()? + .filter_map(|key| match key { + Ok(key) => match native_key(&key) { + Ok(native) if skipped.contains(&native) => None, + Ok(_) => Some(Ok(key)), + Err(error) => Some(Err(error)), + }, + Err(error) => Some(Err(error)), + }) + .collect::>()?; + changes.push(("test_entries", PyTuple::new(py, tests)?.into_any())); + changes.push(("execution_order", PyTuple::new(py, order)?.into_any())); + } + if cli_ignore_audit_failures.is_none() { + let mut audits: Vec> = Vec::new(); + let mut changed = false; + for entry in native_field(plan, "audit_entries")?.try_iter()? { + let entry = entry?; + match self.audit_ignores(&entry, build_ignores)? { + Some(ignored) + if native_field(&entry, "ignore_failure")?.is_truthy()? != ignored => + { + changed = true; + audits.push(replaced( + &entry, + &[( + "ignore_failure", + PyBool::new(py, ignored).to_owned().into_any(), + )], + )?); + } + _ => audits.push(entry), + } + } + if changed { + changes.push(("audit_entries", PyTuple::new(py, audits)?.into_any())); + } + } + let ignoring: &[String] = if cli_ignore_audit_failures.is_none() { + &self.ignoring_jobs + } else { + &[] + }; + if !ignoring.is_empty() + || !self.refreshing_jobs.is_empty() + || !self.job_warehouses.is_empty() + { + let metadata: Bound<'py, PyDict> = native_field(plan, "metadata")? + .cast_into::()? + .copy()?; + if !ignoring.is_empty() { + metadata.set_item(JOB_IGNORED_AUDIT_FAILURES_KEY, PyTuple::new(py, ignoring)?)?; + } + if !self.refreshing_jobs.is_empty() { + metadata.set_item( + JOB_FULL_REFRESH_KEY, + PyTuple::new(py, &self.refreshing_jobs)?, + )?; + } + if !self.job_warehouses.is_empty() { + let warehouses: Bound<'py, PyDict> = PyDict::new(py); + for (job, warehouse) in &self.job_warehouses { + warehouses.set_item(job, warehouse)?; + } + metadata.set_item(JOB_WAREHOUSES_KEY, warehouses)?; + } + changes.push(("metadata", metadata.into_any())); + } + if changes.is_empty() { + return Ok(plan.clone()); + } + replaced(plan, &changes) + } + + /// Whether a SQL test's models' jobs turn tests off; jobs that disagree conflict. + fn skips_test(&self, test: &ObjectKey, scope_deps: &Bound<'_, PyAny>) -> PyResult { + let mut keys: Vec = vec![test.clone()]; + for dep in scope_deps.try_iter()? { + keys.push(native_key(&dep?)?); + } + let mut seen: Option<(&String, &Value)> = None; + for key in &keys { + let Some(value) = self.value(key, node_setting(NodeConsumer::TestEntries)) else { + continue; + }; + let job: &String = self + .settings + .node_jobs + .get(key) + .and_then(|jobs| jobs.first()) + .unwrap_or(&key.1); + match seen { + Some((other, earlier)) if earlier != value => { + return Err(job_setting_conflict( + scope_deps.py(), + &JobSettingConflict { + message: format!( + "test {} covers nodes of jobs {other} and {job}, which set \ + run_tests differently ({earlier} and {value})", + test.1 + ), + help: conflict_help(node_setting(NodeConsumer::TestEntries)), + }, + )); + } + Some(_) => {} + None => seen = Some((job, value)), + } + } + Ok(seen.is_some_and(|(_, value)| value == &Value::Bool(false))) + } + + /// The audited node's `ignore_audit_failures`, or for a singular audit, its read nodes' together. + fn audit_ignores( + &self, + entry: &Bound<'_, PyAny>, + build_ignores: bool, + ) -> PyResult> { + let kind: Bound<'_, PyAny> = native_field(entry, "attached_target_kind")?; + let name: Bound<'_, PyAny> = native_field(entry, "attached_target_name")?; + if kind.is_none() || name.is_none() { + let audit: ObjectKey = native_key(&native_field(entry, "key")?)?; + let reads: &[ObjectKey] = self + .unattached_audit_reads + .get(&audit) + .map_or(&[], Vec::as_slice); + return Ok(singular_audit_ignores( + reads.iter().map(|key| { + self.value(key, node_setting(NodeConsumer::AuditEntries)) + .and_then(Value::as_bool) + }), + build_ignores, + )); + } + let kind: String = native_field(&kind, "value")?.extract()?; + let key: ObjectKey = (kind, name.extract()?); + Ok(self + .value(&key, node_setting(NodeConsumer::AuditEntries)) + .and_then(Value::as_bool)) + } +} diff --git a/crates/sqlbuild-python-compile/src/compile_driver/_helpers/jobs/mod.rs b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/jobs/mod.rs new file mode 100644 index 0000000000..15e1f2d769 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/jobs/mod.rs @@ -0,0 +1,2 @@ +pub(crate) mod job_limits; +pub(crate) mod job_node_settings; diff --git a/crates/sqlbuild-python-compile/src/compile_driver/_helpers/mod.rs b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/mod.rs index 4b77187d1e..1e8b92dab8 100644 --- a/crates/sqlbuild-python-compile/src/compile_driver/_helpers/mod.rs +++ b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/mod.rs @@ -1,3 +1,4 @@ pub(crate) mod assembly; +pub(crate) mod jobs; pub(crate) mod setup; pub(crate) mod stages; diff --git a/crates/sqlbuild-python-compile/src/compile_driver/main/jobs/build_concurrency.rs b/crates/sqlbuild-python-compile/src/compile_driver/main/jobs/build_concurrency.rs new file mode 100644 index 0000000000..3ad7861e98 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_driver/main/jobs/build_concurrency.rs @@ -0,0 +1,20 @@ +//! The concurrency limit of a build that names jobs. + +use pyo3::prelude::{Bound, PyAny, PyResult}; +use sqlbuild_config::effective::models::{ConcurrencySource, EffectiveConcurrency}; + +use crate::compile_driver::_helpers::jobs::job_limits::job_limit_total; + +/// The build's limit: the configured one, raised to the named jobs' limits added together. +pub fn build_concurrency( + configured: EffectiveConcurrency, + pipeline: &Bound<'_, PyAny>, +) -> PyResult { + Ok(match job_limit_total(pipeline)? { + Some(total) if total > configured.limit => EffectiveConcurrency { + limit: total, + source: ConcurrencySource::NamedJobs, + }, + _ => configured, + }) +} diff --git a/crates/sqlbuild-python-compile/src/compile_driver/main/jobs/connection_limit.rs b/crates/sqlbuild-python-compile/src/compile_driver/main/jobs/connection_limit.rs new file mode 100644 index 0000000000..49a52d2902 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_driver/main/jobs/connection_limit.rs @@ -0,0 +1,29 @@ +//! The connections a build that names limited jobs opens. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::types::{PyDict, PyDictMethods}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; + +use crate::compile_driver::_helpers::jobs::job_limits::job_limit_total; +use crate::compile_driver::main::jobs::build_concurrency::build_concurrency; +use sqlbuild_config::effective::models::EffectiveConcurrency; + +/// The build's limit, or the named jobs' limits added together when every planned node has one. +pub fn connection_limit( + configured: EffectiveConcurrency, + pipeline: &Bound<'_, PyAny>, +) -> PyResult { + let limit: i64 = build_concurrency(configured, pipeline)?.limit; + let Some(total) = job_limit_total(pipeline)? else { + return Ok(limit); + }; + let node_jobs: Bound<'_, PyDict> = native_field(pipeline, "node_jobs")?.cast_into()?; + for key in + native_field(&native_field(pipeline, "plan_output")?, "execution_order")?.try_iter()? + { + if !node_jobs.contains(key?)? { + return Ok(limit); + } + } + Ok(limit.min(total.max(1))) +} diff --git a/crates/sqlbuild-python-compile/src/compile_driver/main/jobs/job_override_fields.rs b/crates/sqlbuild-python-compile/src/compile_driver/main/jobs/job_override_fields.rs new file mode 100644 index 0000000000..a175e313ec --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_driver/main/jobs/job_override_fields.rs @@ -0,0 +1,26 @@ +//! The planner overrides the named jobs' settings add. + +use pyo3::prelude::{PyResult, Python}; + +use crate::compile_driver::models::PlannedJobSettings; +use crate::compile_driver::types::PipelineFields; + +/// The models and functions the named jobs rebuild, as the planner's `full_refresh_*` overrides. +pub fn job_override_fields<'py>( + py: Python<'py>, + job_settings: Option<&PlannedJobSettings>, +) -> PyResult> { + let Some(job_settings) = job_settings else { + return Ok(Vec::new()); + }; + Ok(vec![ + ( + "full_refresh_models", + job_settings.full_refresh_models(py)?.into_any(), + ), + ( + "full_refresh_functions", + job_settings.full_refresh_functions(py)?.into_any(), + ), + ]) +} diff --git a/crates/sqlbuild-python-compile/src/compile_driver/main/jobs/job_plan_fields.rs b/crates/sqlbuild-python-compile/src/compile_driver/main/jobs/job_plan_fields.rs new file mode 100644 index 0000000000..9f976c25cd --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_driver/main/jobs/job_plan_fields.rs @@ -0,0 +1,44 @@ +//! The plan and pipeline fields the named jobs' settings give a planned build. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_analysis::job_settings::main::setting_scope::setting_scope; +use sqlbuild_analysis::job_settings::models::SettingScope; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_project::job_settings::main::given_settings::given_settings; + +use crate::compile_driver::models::PlannedJobSettings; +use crate::compile_driver::types::PipelineFields; + +/// The plan with the jobs' settings applied, and their limits and warehouses as pipeline fields. +pub fn job_plan_fields<'py>( + job_settings: Option<&PlannedJobSettings>, + plan_output: Bound<'py, PyAny>, + options: &Bound<'py, PyAny>, +) -> PyResult<(Bound<'py, PyAny>, PipelineFields<'py>)> { + let Some(job_settings) = job_settings else { + return Ok((plan_output, Vec::new())); + }; + let plan_output: Bound<'py, PyAny> = job_settings.apply_to_plan( + &plan_output, + ( + &native_field(options, "cli_ignore_audit_failures")?, + native_field(options, "ignore_audit_failures")?.is_truthy()?, + ), + )?; + let (limits, node_jobs) = job_settings.job_limits( + &plan_output, + given_settings() + .iter() + .any(|name| setting_scope(name) == SettingScope::JobLimit), + )?; + let (nodes, python) = job_settings.node_warehouses(&plan_output)?; + Ok(( + plan_output, + vec![ + ("job_limits", limits.into_any()), + ("node_jobs", node_jobs.into_any()), + ("node_warehouses", nodes.into_any()), + ("python_node_warehouses", python.into_any()), + ], + )) +} diff --git a/crates/sqlbuild-python-compile/src/compile_driver/main/jobs/job_run_context.rs b/crates/sqlbuild-python-compile/src/compile_driver/main/jobs/job_run_context.rs new file mode 100644 index 0000000000..65856ffa6d --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_driver/main/jobs/job_run_context.rs @@ -0,0 +1,34 @@ +//! What a build's run context says about its named jobs. + +use crate::compile_driver::types::JobRunContext; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::types::{PyDict, PyDictMethods}; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; +use sqlbuild_python_project::compiled_project::constants::{ + JOB_FULL_REFRESH_KEY, JOB_WAREHOUSES_KEY, +}; + +/// The jobs whose models the planned build rebuilds, and each job's warehouse its nodes run on. +pub fn job_run_context(pipeline: &Bound<'_, PyAny>) -> PyResult { + let metadata: Bound<'_, PyAny> = + native_field(&native_field(pipeline, "plan_output")?, "metadata")?; + let refreshing: Bound<'_, PyAny> = mapping_get(&metadata, JOB_FULL_REFRESH_KEY)?; + let jobs: Vec = if refreshing.is_none() { + Vec::new() + } else { + refreshing + .try_iter()? + .map(|job| job?.extract()) + .collect::>()? + }; + let warehouses: Bound<'_, PyAny> = mapping_get(&metadata, JOB_WAREHOUSES_KEY)?; + let warehouses: Vec<(String, String)> = match warehouses.cast::() { + Ok(warehouses) => warehouses + .iter() + .map(|(job, warehouse)| Ok((job.extract()?, warehouse.extract()?))) + .collect::>()?, + Err(_) => Vec::new(), + }; + Ok((jobs, warehouses)) +} diff --git a/crates/sqlbuild-python-compile/src/compile_driver/main/jobs/mod.rs b/crates/sqlbuild-python-compile/src/compile_driver/main/jobs/mod.rs new file mode 100644 index 0000000000..97102c5ad6 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_driver/main/jobs/mod.rs @@ -0,0 +1,6 @@ +pub mod build_concurrency; +pub mod connection_limit; +pub mod job_override_fields; +pub mod job_plan_fields; +pub mod job_run_context; +pub mod planned_job_settings; diff --git a/crates/sqlbuild-python-compile/src/compile_driver/main/jobs/planned_job_settings.rs b/crates/sqlbuild-python-compile/src/compile_driver/main/jobs/planned_job_settings.rs new file mode 100644 index 0000000000..28eded5b53 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_driver/main/jobs/planned_job_settings.rs @@ -0,0 +1,15 @@ +//! The settings each planned node takes from the jobs its selection names. + +use pyo3::prelude::{Bound, PyAny, PyResult}; + +use crate::compile_driver::_helpers::jobs::job_node_settings; +use crate::compile_driver::models::PlannedJobSettings; + +/// The named jobs' settings per selected node; `None` when ignored, unnamed or unselectable. +#[inline] +pub fn planned_job_settings( + inputs: (&Bound<'_, PyAny>, &Bound<'_, PyAny>), + options: (&Bound<'_, PyAny>, bool), +) -> PyResult> { + job_node_settings::planned_job_settings(inputs, options) +} diff --git a/crates/sqlbuild-python-compile/src/compile_driver/main/mod.rs b/crates/sqlbuild-python-compile/src/compile_driver/main/mod.rs index 2f69896363..57a7fde978 100644 --- a/crates/sqlbuild-python-compile/src/compile_driver/main/mod.rs +++ b/crates/sqlbuild-python-compile/src/compile_driver/main/mod.rs @@ -1,2 +1,3 @@ +pub mod jobs; pub mod setup; pub mod stages; diff --git a/crates/sqlbuild-python-compile/src/compile_driver/mod.rs b/crates/sqlbuild-python-compile/src/compile_driver/mod.rs index cb9dd32021..88823c59ec 100644 --- a/crates/sqlbuild-python-compile/src/compile_driver/mod.rs +++ b/crates/sqlbuild-python-compile/src/compile_driver/mod.rs @@ -3,4 +3,6 @@ pub(crate) mod _helpers; pub mod main; pub mod models; +#[cfg(test)] +mod tests; pub mod types; diff --git a/crates/sqlbuild-python-compile/src/compile_driver/models.rs b/crates/sqlbuild-python-compile/src/compile_driver/models.rs index 5838cd685d..bdc58049f7 100644 --- a/crates/sqlbuild-python-compile/src/compile_driver/models.rs +++ b/crates/sqlbuild-python-compile/src/compile_driver/models.rs @@ -8,10 +8,13 @@ use pyo3::prelude::PyAny; use pyo3::prelude::PyResult; use pyo3::prelude::Python; use pyo3::types::PyBool; +use sqlbuild_analysis::assembly::project::types::ObjectKey; use sqlbuild_analysis::assembly::target_checks::constants::PRESERVE_TARGET_VALUE; +use sqlbuild_analysis::job_settings::models::NodeJobSettings; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::types::RecordId; +use std::collections::BTreeMap; /// Semantic analysis cache outcome counts for one invocation. #[derive(Debug)] @@ -148,3 +151,17 @@ impl TargetSettingsValues { ) } } + +/// What the planned build takes from its named jobs, or nothing when it names none. +#[derive(Debug)] +pub struct PlannedJobSettings { + pub(crate) settings: NodeJobSettings, + /// The named jobs that set `ignore_audit_failures = true` for nodes they select. + pub(crate) ignoring_jobs: Vec, + /// The named jobs that set `full_refresh = true` for models they select. + pub(crate) refreshing_jobs: Vec, + /// Each named job whose `warehouse` its own nodes run on, and that warehouse. + pub(crate) job_warehouses: BTreeMap, + /// The nodes each audit attached to no node reads, by the audit's key. + pub(crate) unattached_audit_reads: BTreeMap>, +} diff --git a/crates/sqlbuild-python-compile/src/compile_driver/tests.rs b/crates/sqlbuild-python-compile/src/compile_driver/tests.rs new file mode 100644 index 0000000000..bb7b53d2e8 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_driver/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/test_node_setting_consumers.rs"] +mod node_setting_consumers; +#[path = "tests/test_singular_audit_ignores.rs"] +mod singular_audit_ignores; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-python-compile/src/compile_driver/tests/test_node_setting_consumers.rs b/crates/sqlbuild-python-compile/src/compile_driver/tests/test_node_setting_consumers.rs new file mode 100644 index 0000000000..8abaa6af4b --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_driver/tests/test_node_setting_consumers.rs @@ -0,0 +1,33 @@ +use std::collections::BTreeSet; + +use sqlbuild_analysis::job_settings::constants::JOB_SETTINGS; +use sqlbuild_analysis::job_settings::models::{NodeConsumer, SettingScope}; + +use crate::compile_driver::_helpers::jobs::job_node_settings::node_setting; +use crate::compile_driver::tests::test_types::NodeSettingConsumersTestCase; + +#[test] +fn given_every_node_setting_in_the_table_when_planning_then_a_consumer_applies_it() { + let test_cases = [NodeSettingConsumersTestCase { + description: "each node setting is read through the consumer its table row names", + consumers: vec![ + NodeConsumer::RefreshOverrides, + NodeConsumer::TestEntries, + NodeConsumer::AuditEntries, + NodeConsumer::NodeWarehouses, + ], + expected_node_settings: JOB_SETTINGS + .iter() + .filter(|spec| spec.scope == SettingScope::Node) + .map(|spec| spec.name) + .collect(), + }]; + for test_case in test_cases { + let read: BTreeSet<&str> = test_case.consumers.into_iter().map(node_setting).collect(); + assert_eq!( + read, test_case.expected_node_settings, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-python-compile/src/compile_driver/tests/test_singular_audit_ignores.rs b/crates/sqlbuild-python-compile/src/compile_driver/tests/test_singular_audit_ignores.rs new file mode 100644 index 0000000000..c9b3773600 --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_driver/tests/test_singular_audit_ignores.rs @@ -0,0 +1,52 @@ +use crate::compile_driver::_helpers::jobs::job_node_settings::singular_audit_ignores; +use crate::compile_driver::tests::test_types::SingularAuditIgnoresTestCase; + +#[test] +fn given_a_singular_audit_when_its_nodes_have_job_and_build_values_then_all_must_ignore() { + let test_cases = [ + SingularAuditIgnoresTestCase { + description: "a job node ignores, a node outside the jobs takes the build's false", + job_values: vec![Some(true), None], + build_ignores: false, + expected_ignores: Some(false), + }, + SingularAuditIgnoresTestCase { + description: "a job node ignores, a node outside the jobs takes the build's true", + job_values: vec![Some(true), None], + build_ignores: true, + expected_ignores: Some(true), + }, + SingularAuditIgnoresTestCase { + description: "a job node that refuses wins over the build's true", + job_values: vec![Some(false), None], + build_ignores: true, + expected_ignores: Some(false), + }, + SingularAuditIgnoresTestCase { + description: "two job nodes that both ignore", + job_values: vec![Some(true), Some(true)], + build_ignores: false, + expected_ignores: Some(true), + }, + SingularAuditIgnoresTestCase { + description: "two job nodes that disagree", + job_values: vec![Some(true), Some(false)], + build_ignores: true, + expected_ignores: Some(false), + }, + SingularAuditIgnoresTestCase { + description: "no read node has a job value, so the build decides", + job_values: vec![None, None], + build_ignores: true, + expected_ignores: None, + }, + ]; + for test_case in test_cases { + assert_eq!( + singular_audit_ignores(test_case.job_values.into_iter(), test_case.build_ignores), + test_case.expected_ignores, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-python-compile/src/compile_driver/tests/test_types.rs b/crates/sqlbuild-python-compile/src/compile_driver/tests/test_types.rs new file mode 100644 index 0000000000..4bb86a239e --- /dev/null +++ b/crates/sqlbuild-python-compile/src/compile_driver/tests/test_types.rs @@ -0,0 +1,18 @@ +use std::collections::BTreeSet; + +use sqlbuild_analysis::job_settings::models::NodeConsumer; + +/// The consumers the planner reads node settings through, and the settings they must name. +pub(super) struct NodeSettingConsumersTestCase { + pub(super) description: &'static str, + pub(super) consumers: Vec, + pub(super) expected_node_settings: BTreeSet<&'static str>, +} + +/// A singular audit's read nodes' job values, the build's value, and whether it ignores failures. +pub(super) struct SingularAuditIgnoresTestCase { + pub(super) description: &'static str, + pub(super) job_values: Vec>, + pub(super) build_ignores: bool, + pub(super) expected_ignores: Option, +} diff --git a/crates/sqlbuild-python-compile/src/compile_driver/types.rs b/crates/sqlbuild-python-compile/src/compile_driver/types.rs index 8e30752bcd..245e1cab39 100644 --- a/crates/sqlbuild-python-compile/src/compile_driver/types.rs +++ b/crates/sqlbuild-python-compile/src/compile_driver/types.rs @@ -6,3 +6,8 @@ use pyo3::prelude::PyAny; /// One compile's outcome, input reads and artifact writes, before it is stored and printed. pub type PhasesOutcome<'py> = (CommandOutcome<'py>, Bound<'py, PyAny>, Bound<'py, PyAny>); + +/// Pipeline result fields by name. +pub type PipelineFields<'py> = Vec<(&'static str, pyo3::Bound<'py, pyo3::PyAny>)>; +/// The jobs whose models a build rebuilds, and each job with the warehouse its nodes run on. +pub type JobRunContext = (Vec, Vec<(String, String)>); diff --git a/crates/sqlbuild-python-front/src/command_runtime/main/published_concurrency.rs b/crates/sqlbuild-python-front/src/command_runtime/main/published_concurrency.rs index 391ab27b93..a9a9bff66f 100644 --- a/crates/sqlbuild-python-front/src/command_runtime/main/published_concurrency.rs +++ b/crates/sqlbuild-python-front/src/command_runtime/main/published_concurrency.rs @@ -32,7 +32,7 @@ pub fn published_concurrency( Ok(keys.iter().any(|key| key == CONCURRENCY)) }; let source = match effective.source { - Layer::CommandLine => ConcurrencySource::Job, + Layer::CommandLine | Layer::NamedJobs => ConcurrencySource::Job, Layer::Target(_) => ConcurrencySource::Target, Layer::LocalSettings => ConcurrencySource::Local, Layer::ProjectSettings if project_sets()? => ConcurrencySource::Project, diff --git a/crates/sqlbuild-python-front/src/discovery/_helpers/project/project_validation.rs b/crates/sqlbuild-python-front/src/discovery/_helpers/project/project_validation.rs index 16cbfbccca..83fd905355 100644 --- a/crates/sqlbuild-python-front/src/discovery/_helpers/project/project_validation.rs +++ b/crates/sqlbuild-python-front/src/discovery/_helpers/project/project_validation.rs @@ -6,10 +6,8 @@ use crate::discovery::main::sources::orchestration_declarations::orchestration_d use crate::discovery::models::NodeRow; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::pyfunction; -use sqlbuild_discovery::orchestration_files::models::{FilePosition, OrchestrationInventory}; use sqlbuild_discovery::project_validation::main::validate_project_inputs::validate_project_inputs; use sqlbuild_discovery::project_validation::models::ProjectFacts; -use sqlbuild_python_base::native_host::main::write_console::write_console; use sqlbuild_python_base::records::main::fields::field::field as native_field; pub(crate) const CSV_MODULE: &str = "csv"; @@ -39,36 +37,7 @@ pub(crate) fn validate_discovered_project_inputs( let project_dir: Bound<'_, PyAny> = native_field(&inputs, "project_dir")?; if !project_dir.is_none() { let inventory = orchestration_declarations(&project_dir)?; - warn_deprecated_orchestration(py, &inventory)?; facts.orchestration = inventory.names(); } validate_project_inputs(&facts).map_err(|failure| validation_error(py, failure)) } - -/// Write a warning line to stderr for each deprecated name a job or job group file still uses. -fn warn_deprecated_orchestration( - py: Python<'_>, - inventory: &OrchestrationInventory, -) -> PyResult<()> { - let runs = inventory - .jobs - .iter() - .map(|(path, job)| (path, &job.run)) - .chain( - inventory - .groups - .iter() - .map(|(path, group)| (path, &group.run)), - ); - for (path, run) in runs { - for deprecation in &run.deprecations { - let FilePosition { line, column } = deprecation.position; - write_console( - py, - "stderr", - &format!("Warning: {path}:{line}:{column}: {}\n", deprecation.message), - )?; - } - } - Ok(()) -} diff --git a/crates/sqlbuild-python-front/src/discovery/_helpers/sources/orchestration_values.rs b/crates/sqlbuild-python-front/src/discovery/_helpers/sources/orchestration_values.rs index fbc3692ebc..5dc7a777cb 100644 --- a/crates/sqlbuild-python-front/src/discovery/_helpers/sources/orchestration_values.rs +++ b/crates/sqlbuild-python-front/src/discovery/_helpers/sources/orchestration_values.rs @@ -7,6 +7,7 @@ use chrono_tz::Tz; use pyo3::prelude::Python; use serde_json::Value; use sqlbuild_analysis::graph::main::parse_selector::parse_selector; +use sqlbuild_analysis::job_settings::main::group_setting_mismatch::group_setting_mismatch; use sqlbuild_cli::argv::main::job_settings_arguments::job_settings_arguments; use sqlbuild_discovery::models::{DiscoveryFailure, FailureKind}; use sqlbuild_discovery::orchestration_files::models::{ @@ -17,13 +18,16 @@ use sqlbuild_orchestration::store::errors::ControlError; use sqlbuild_python_base::orchestration_reads::main::run_resources::run_resources; +/// The job setting that takes over held dynamic models, which `[dynamic]` must allow. +const IGNORE_DYNAMIC_GRACE: &str = "ignore_dynamic_grace"; + /// A `D017` at `position` of `path`, worded as discovery words its own. type ValueError = (FilePosition, String); /// The first invalid schedule, setting, resource limit or selector, as `D017` at its file, line and column. pub(crate) fn check_orchestration_values( py: Python<'_>, - inventory: &OrchestrationInventory, + (inventory, allow_ignore_grace): (&OrchestrationInventory, bool), ) -> Result<(), DiscoveryFailure> { let runs = inventory .jobs @@ -48,6 +52,18 @@ pub(crate) fn check_orchestration_values( job_settings_arguments(&BTreeMap::from([(key.clone(), value.clone())])) .map_err(|message| failure(path, (job.setting_positions[key], message)))?; } + if !allow_ignore_grace && job.settings.get(IGNORE_DYNAMIC_GRACE) == Some(&Value::Bool(true)) + { + return Err(failure( + path, + ( + job.setting_positions[IGNORE_DYNAMIC_GRACE], + "ignore_dynamic_grace needs [dynamic] allow_ignore_grace = true in \ + sqlbuild_project.toml" + .to_owned(), + ), + )); + } if let Some(first) = job.setting_positions.values().min() { job_settings_arguments(&settings) .map_err(|message| failure(path, (*first, message)))?; @@ -58,6 +74,11 @@ pub(crate) fn check_orchestration_values( })?; } } + for (path, group) in &inventory.groups { + if let Some(message) = group_setting_mismatch(inventory, group) { + return Err(failure(path, (group.run.name.position, message))); + } + } Ok(()) } diff --git a/crates/sqlbuild-python-front/src/discovery/main/sources/orchestration_declarations.rs b/crates/sqlbuild-python-front/src/discovery/main/sources/orchestration_declarations.rs index f508e7dd0d..76a4f4d99f 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/sources/orchestration_declarations.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/sources/orchestration_declarations.rs @@ -1,10 +1,11 @@ //! Every job, job group, calendar and sensor of `project_dir`, or `OrchestrationDeclarationError` (D017). use crate::discovery::_helpers::failures::raised::failure_error; -use crate::discovery::_helpers::project::project_config::project_pool_limits; +use crate::discovery::_helpers::project::project_config::{configured, project_pool_limits}; use crate::discovery::_helpers::sources::orchestration_files::orchestration_error; use crate::discovery::_helpers::sources::orchestration_values::check_orchestration_values; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use sqlbuild_config::project::main::load_dynamic_config::load_dynamic_config; use sqlbuild_discovery::orchestration_files::main::check_pools::check_pools; use sqlbuild_discovery::orchestration_files::main::orchestration_inventory::orchestration_inventory; use sqlbuild_discovery::orchestration_files::models::OrchestrationInventory; @@ -25,6 +26,8 @@ pub fn orchestration_declarations( let limits = project_pool_limits(py, project_dir)?; check_pools(&inventory, &limits) .map_err(|failure| orchestration_error(&tree, project_dir, failure))?; - check_orchestration_values(py, &inventory).map_err(|failure| failure_error(py, failure))?; + let allow_ignore_grace: bool = configured(project_dir, load_dynamic_config)?.allow_ignore_grace; + check_orchestration_values(py, (&inventory, allow_ignore_grace)) + .map_err(|failure| failure_error(py, failure))?; Ok(inventory) } diff --git a/crates/sqlbuild-python-planner/src/consumption/main/changed_policy.rs b/crates/sqlbuild-python-planner/src/consumption/main/changed_policy.rs index ca236b42d0..5181e3b661 100644 --- a/crates/sqlbuild-python-planner/src/consumption/main/changed_policy.rs +++ b/crates/sqlbuild-python-planner/src/consumption/main/changed_policy.rs @@ -10,8 +10,8 @@ use pyo3::exceptions::PyValueError; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyRef, PyResult}; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_front::discovery::main::project::configured_changes::configured_changes; -use sqlbuild_python_project::compiled_project::main::triggered_jobs::triggered_jobs; use sqlbuild_python_project::graph::models::NativeProjectGraph; +use sqlbuild_python_project::job_settings::main::triggered_jobs::triggered_jobs; use sqlbuild_state::consumption::models::{ChangeKind, ChangedPolicy, Reach}; use std::collections::BTreeSet; diff --git a/crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_text.rs b/crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_text.rs index 9dcef5903c..c3cfba3a6d 100644 --- a/crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_text.rs +++ b/crates/sqlbuild-python-planner/src/plan_output/_helpers/plan_text.rs @@ -13,6 +13,7 @@ use sqlbuild_core::cli_text::models::{ SeedPlanText, SourceFreshnessText, SourceLoadPlanText, }; use sqlbuild_core::plan_display::main::format_query_diff::format_query_diff; +use std::collections::BTreeMap; use crate::plan_output::_helpers::cursor_details::CursorDetails; use crate::plan_output::_helpers::cursor_details::DEFERRED; @@ -34,6 +35,9 @@ use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::cursor::main::scalars::sentinel_to_token::sentinel_to_token; +use sqlbuild_python_project::compiled_project::constants::{ + JOB_IGNORED_AUDIT_FAILURES_KEY, JOB_WAREHOUSES_KEY, +}; const SKIP_ACTION: &str = "skip"; const QUERY_CHANGED: &str = "query_changed"; @@ -388,6 +392,18 @@ fn provider_usages( Ok(usages) } +/// The plan metadata's named jobs and the warehouses their nodes run on. +pub(crate) fn job_warehouses(metadata: &Bound<'_, PyAny>) -> PyResult> { + let value: Bound<'_, PyAny> = mapping_get(metadata, JOB_WAREHOUSES_KEY)?; + if value.is_none() { + return Ok(Vec::new()); + } + Ok(value + .extract::>()? + .into_iter() + .collect()) +} + /// `str(item)` of a metadata tuple or list, else nothing. fn metadata_names(value: Option>) -> PyResult> { match value { @@ -529,5 +545,10 @@ pub(crate) fn report(arguments: &KeywordArguments<'_>) -> PyResult, options: Option>) -> P )?; let metadata: Bound<'_, PyAny> = native_field(&plan, "metadata")?; if metadata.is_truthy()? { - result.set_item("metadata", metadata)?; + result.set_item("metadata", &metadata)?; } let synced_copy: Bound<'_, PyAny> = arguments.value(SYNCED_COPY_KEY)?; if !synced_copy.is_none() { @@ -119,5 +125,24 @@ pub fn format_plan_json(py: Python<'_>, options: Option>) -> P if native_field(&plan, IGNORE_AUDIT_FAILURES_KEY)?.is_truthy()? { result.set_item(IGNORE_AUDIT_FAILURES_KEY, true)?; } + for (key, label) in [ + ( + JOB_IGNORED_AUDIT_FAILURES_KEY, + IGNORE_AUDIT_FAILURES_JOBS_KEY, + ), + (JOB_FULL_REFRESH_KEY, FULL_REFRESH_JOBS_KEY), + ] { + let jobs: Bound<'_, PyAny> = mapping_get(&metadata, key)?; + if !jobs.is_none() { + result.set_item( + label, + PyList::new(py, jobs.try_iter()?.collect::>>()?)?, + )?; + } + } + let warehouses: Bound<'_, PyAny> = mapping_get(&metadata, JOB_WAREHOUSES_KEY)?; + if !warehouses.is_none() { + result.set_item(WAREHOUSE_JOBS_KEY, warehouses)?; + } indented_json(result.as_any()) } diff --git a/crates/sqlbuild-python-planner/src/planner_changes/_helpers/detect.rs b/crates/sqlbuild-python-planner/src/planner_changes/_helpers/detect.rs index bd6322ca35..f16c3da16f 100644 --- a/crates/sqlbuild-python-planner/src/planner_changes/_helpers/detect.rs +++ b/crates/sqlbuild-python-planner/src/planner_changes/_helpers/detect.rs @@ -1,5 +1,7 @@ //! Detect how each selected model and function changed since its recorded build. +use std::collections::HashSet; + use crate::planner_changes::main::native_enum::native_enum; use crate::planner_changes::main::planner_enum::planner_enum; use crate::planner_changes::models::ModelPass; @@ -85,7 +87,7 @@ pub(crate) fn detect_changes_of<'py>( &arguments.value("scope")?, &arguments.value("snapshot")?, ), - arguments.flag("full_refresh")?, + (arguments.flag("full_refresh")?, &HashSet::new()), ( arguments.value("expected_version_hashes")?, arguments.value("expected_metadata_jsons")?, @@ -93,10 +95,10 @@ pub(crate) fn detect_changes_of<'py>( ) } -/// Selected model and function changes against expected version hashes and metadata JSON. +/// Selected model and function changes; functions in `refreshed_functions` are recreated too. pub(crate) fn detect_changes<'py>( (project, scope, snapshot): (&Bound<'py, PyAny>, &Bound<'py, PyAny>, &Bound<'py, PyAny>), - full_refresh: bool, + (full_refresh, refreshed_functions): (bool, &HashSet), expected: (Bound<'py, PyAny>, Bound<'py, PyAny>), ) -> PyResult> { let py: Python<'py> = project.py(); @@ -124,7 +126,9 @@ pub(crate) fn detect_changes<'py>( if selected.selected.contains( &sqlbuild_python_project::graph::main::nodes::native_key::native_key(&key(&function)?)?, ) { - let reason: &str = if full_refresh { + let reason: &str = if full_refresh + || refreshed_functions.contains(&name.extract::()?) + { "full_refresh" } else { let fingerprint: Bound<'py, PyAny> = mapping_get(&recorded, &name)?; diff --git a/crates/sqlbuild-python-planner/src/planner_changes/main/detect_changes.rs b/crates/sqlbuild-python-planner/src/planner_changes/main/detect_changes.rs index fee967c759..e95626a336 100644 --- a/crates/sqlbuild-python-planner/src/planner_changes/main/detect_changes.rs +++ b/crates/sqlbuild-python-planner/src/planner_changes/main/detect_changes.rs @@ -1,5 +1,7 @@ //! Selected model and function changes against expected version hashes and metadata JSON. +use std::collections::HashSet; + use crate::planner_changes::_helpers::detect; use pyo3::prelude::{Bound, PyAny, PyResult}; @@ -7,7 +9,7 @@ use pyo3::prelude::{Bound, PyAny, PyResult}; #[inline] pub fn detect_changes<'py>( (project, scope, snapshot): (&Bound<'py, PyAny>, &Bound<'py, PyAny>, &Bound<'py, PyAny>), - full_refresh: bool, + full_refresh: (bool, &HashSet), expected: (Bound<'py, PyAny>, Bound<'py, PyAny>), ) -> PyResult> { detect::detect_changes((project, scope, snapshot), full_refresh, expected) diff --git a/crates/sqlbuild-python-planner/src/planner_entries/_helpers/entries/model_entries.rs b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/entries/model_entries.rs index bb20834b7b..3885247bdc 100644 --- a/crates/sqlbuild-python-planner/src/planner_entries/_helpers/entries/model_entries.rs +++ b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/entries/model_entries.rs @@ -478,7 +478,8 @@ pub(crate) fn plan_entries<'py>( &cursor_overrides, (start_override.clone(), end_override.clone()), )?; - let effective_full_refresh = resolve_model_full_refresh(&model, full_refresh)?; + let effective_full_refresh = + resolve_model_full_refresh(&model, full_refresh.applies(name))?; let (mut entry, entry_warnings) = plan_model_from_change(&ModelPlanRequest { model: model.clone(), snapshot: snapshot.clone(), diff --git a/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/execution_plan.rs b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/execution_plan.rs index a36dd6b897..8e86841d98 100644 --- a/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/execution_plan.rs +++ b/crates/sqlbuild-python-planner/src/planner_entries/_helpers/plans/execution_plan.rs @@ -32,6 +32,7 @@ use crate::planner_phases::main::outputs::unselected_upstream_models::unselected use crate::planner_phases::main::rebuilds::build_planner_entry_results::build_planner_entry_results; use crate::planner_phases::main::rebuilds::defer_to_migration_origins::defer_to_migration_origins; use crate::planner_phases::main::rebuilds::reconcile_execution_changes::reconcile_execution_changes; +use crate::planner_phases::models::FullRefreshRequest; use crate::planner_phases::models::PlannerIdentityContext; use crate::planner_phases::models::{ PlanOutputContext, PlanWarningContext, PlannedModelEntries, StoragePolicyContext, @@ -88,13 +89,14 @@ fn change_results<'py>( let versions = class_field(identities, |i: &PlannerIdentityContext| { &i.version_identities })?; + let full_refresh: FullRefreshRequest = FullRefreshRequest::of(overrides)?; let changes = detect_changes( ( project, &class_field(scopes, |s: &PlannerScopeResolution| &s.inspection_scope)?, snapshot, ), - native_field(overrides, "full_refresh")?.is_truthy()?, + (full_refresh.all, &full_refresh.functions), ( native_field(&versions, "model_version_hashes")?, native_field(&versions, "model_metadata_jsons")?, diff --git a/crates/sqlbuild-python-planner/src/planner_entries/models.rs b/crates/sqlbuild-python-planner/src/planner_entries/models.rs index 367bb18a6f..4136195d77 100644 --- a/crates/sqlbuild-python-planner/src/planner_entries/models.rs +++ b/crates/sqlbuild-python-planner/src/planner_entries/models.rs @@ -1,5 +1,6 @@ //! Public records of planner entries. +use crate::planner_phases::models::FullRefreshRequest; use crate::planner_warehouse::models::PlannerReads; use pyo3::prelude::{Bound, Py, PyAny}; use pyo3::pyclass; @@ -31,7 +32,7 @@ pub struct PlanEntriesRequest<'py> { pub(crate) relations: Bound<'py, PyAny>, pub(crate) resolved_models: Bound<'py, PyAny>, pub(crate) cursor_overrides: Bound<'py, PyAny>, - pub(crate) full_refresh: bool, + pub(crate) full_refresh: FullRefreshRequest, pub(crate) build_inputs: Bound<'py, PyAny>, /// The effective worker limit execution will use. pub(crate) concurrency: i64, diff --git a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/planning/planning_boundary.rs b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/planning/planning_boundary.rs index 7e12a8e95a..edfd40fd26 100644 --- a/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/planning/planning_boundary.rs +++ b/crates/sqlbuild-python-planner/src/planner_migrations/_helpers/planning/planning_boundary.rs @@ -7,6 +7,7 @@ use crate::planner_phases::main::inputs::runtime_field::runtime_field; use crate::planner_phases::main::inputs::runtime_reads::runtime_reads; use crate::planner_phases::main::rebuilds::effective_target_config::effective_target_config; use crate::planner_phases::main::rebuilds::effectively_full_refreshed_model_names::effectively_full_refreshed_model_names; +use crate::planner_phases::models::FullRefreshRequest; use crate::planner_warehouse::main::inspection::gather_redirected_cursor_snapshots::gather_redirected_cursor_snapshots; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use sqlbuild_python_base::boundary::main::keywords::keywords; @@ -60,10 +61,8 @@ pub(crate) fn redirected_cursor_snapshots<'py>( ), ], )?; - let full_refresh = effectively_full_refreshed_model_names( - &project, - native_field(overrides, "full_refresh")?.is_truthy()?, - )?; + let full_refresh = + effectively_full_refreshed_model_names(&project, &FullRefreshRequest::of(overrides)?)?; gather_redirected_cursor_snapshots( py, Some(keywords( diff --git a/crates/sqlbuild-python-planner/src/planner_phases/_helpers/inputs/warehouse_state.rs b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/inputs/warehouse_state.rs index 4c5adbe3cd..dbc53ca41c 100644 --- a/crates/sqlbuild-python-planner/src/planner_phases/_helpers/inputs/warehouse_state.rs +++ b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/inputs/warehouse_state.rs @@ -12,6 +12,7 @@ use crate::planner_phases::_helpers::inputs::runtime::runtime_field; use crate::planner_phases::main::inputs::runtime_dialect::runtime_dialect; use crate::planner_phases::main::inputs::runtime_reads::runtime_reads; use crate::planner_phases::main::rebuilds::effectively_full_refreshed_model_names::effectively_full_refreshed_model_names; +use crate::planner_phases::models::FullRefreshRequest; use crate::planner_phases::models::{PlannerRuntime, PlannerWarehouseState}; use crate::planner_warehouse::main::inspection::warehouse_snapshot::warehouse_snapshot; use crate::planner_warehouse::models::MigrationFingerprintCache; @@ -51,7 +52,7 @@ pub(crate) fn gather_planner_warehouse_state<'py>( .into_any(); progress(runtime, "Inspecting warehouse state...")?; let selected_scope = class_field(scopes, |s: &PlannerScopeResolution| &s.selected_scope)?; - let full_refresh: bool = native_field(overrides, "full_refresh")?.extract()?; + let full_refresh: FullRefreshRequest = FullRefreshRequest::of(overrides)?; let config = runtime_field(runtime, |runtime| &runtime.project_config)?; let cursor_scope = native( py, @@ -102,7 +103,7 @@ pub(crate) fn gather_planner_warehouse_state<'py>( reads: reads.clone(), selected_keys: stale_keys, full_refresh: false, - full_refresh_model_names: effectively_full_refreshed_model_names(&project, full_refresh)?, + full_refresh_model_names: effectively_full_refreshed_model_names(&project, &full_refresh)?, on_progress: runtime_field(runtime, |runtime| &runtime.on_progress)?, deferred_locations: native_field(deferral, "deferred_locations")?, cursor_scope, @@ -155,7 +156,7 @@ pub(crate) fn gather_planner_warehouse_state<'py>( &ColumnMigrationInputs { full_refresh_model_names: effectively_full_refreshed_model_names( &project, - full_refresh, + &full_refresh, )?, physical_relations: physical.into_any(), source_columns: native_field(&inspection_relations, "source_warehouse_columns")?, diff --git a/crates/sqlbuild-python-planner/src/planner_phases/_helpers/rebuilds/execution_changes.rs b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/rebuilds/execution_changes.rs index 03d6b014b0..0bcc447d06 100644 --- a/crates/sqlbuild-python-planner/src/planner_phases/_helpers/rebuilds/execution_changes.rs +++ b/crates/sqlbuild-python-planner/src/planner_phases/_helpers/rebuilds/execution_changes.rs @@ -8,6 +8,7 @@ use crate::planner_phases::_helpers::inputs::runtime::runtime_field; use crate::planner_phases::_helpers::inputs::runtime::warehouse_field; use crate::planner_phases::main::inputs::runtime_dialect::runtime_dialect; use crate::planner_phases::models::ChangeReconciliation; +use crate::planner_phases::models::FullRefreshRequest; use crate::planner_phases::models::ModelEntryResults; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::PyFrozenSet; @@ -120,7 +121,7 @@ pub(crate) fn build_planner_entry_results<'py>( relations, resolved_models: native_field(&reconciliation.resolved_actions, "models")?, cursor_overrides: native_field(overrides, "cursor_overrides")?, - full_refresh: native_field(overrides, "full_refresh")?.is_truthy()?, + full_refresh: FullRefreshRequest::of(overrides)?, build_inputs, concurrency, }) diff --git a/crates/sqlbuild-python-planner/src/planner_phases/constants.rs b/crates/sqlbuild-python-planner/src/planner_phases/constants.rs index 3524cae4a7..d44d8393a1 100644 --- a/crates/sqlbuild-python-planner/src/planner_phases/constants.rs +++ b/crates/sqlbuild-python-planner/src/planner_phases/constants.rs @@ -111,3 +111,9 @@ pub const WRITING: &str = "Writing the plan"; /// Time inside the plan that no step covers. pub const OTHER: &str = "Other"; +/// The planner override asking every model to rebuild from scratch. +pub const FULL_REFRESH_KEY: &str = "full_refresh"; +/// The planner override naming the models the named jobs rebuild from scratch. +pub const FULL_REFRESH_MODELS_KEY: &str = "full_refresh_models"; +/// The planner override naming the functions the named jobs recreate. +pub const FULL_REFRESH_FUNCTIONS_KEY: &str = "full_refresh_functions"; diff --git a/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/effectively_full_refreshed_model_names.rs b/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/effectively_full_refreshed_model_names.rs index 2e93d90235..335058e260 100644 --- a/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/effectively_full_refreshed_model_names.rs +++ b/crates/sqlbuild-python-planner/src/planner_phases/main/rebuilds/effectively_full_refreshed_model_names.rs @@ -1,7 +1,9 @@ //! Names of models effectively full refreshed for this invocation. use crate::planner_phases::main::rebuilds::resolve_model_full_refresh::resolve_model_full_refresh; +use crate::planner_phases::models::FullRefreshRequest; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_python_project::compiled_views::main::compiled_models::model_name::model_name; use sqlbuild_python_project::compiled_views::main::compiled_models::name_object::name_object; use sqlbuild_python_project::compiled_views::main::project::compiled_models::compiled_models; use sqlbuild_python_project::planner_phases::main::frozenset::frozenset; @@ -9,12 +11,12 @@ use sqlbuild_python_project::planner_phases::main::frozenset::frozenset; /// Names of models effectively full refreshed for this invocation. pub fn effectively_full_refreshed_model_names<'py>( project: &Bound<'py, PyAny>, - cli_full_refresh: bool, + requested: &FullRefreshRequest, ) -> PyResult> { let mut names: Vec<_> = Vec::new(); for model in compiled_models(project)?.try_iter()? { let model = model?; - if resolve_model_full_refresh(&model, cli_full_refresh)? { + if resolve_model_full_refresh(&model, requested.applies(&model_name(&model)?))? { names.push(name_object(&model)?); } } diff --git a/crates/sqlbuild-python-planner/src/planner_phases/models.rs b/crates/sqlbuild-python-planner/src/planner_phases/models.rs index ebe7df27dc..7f3191446d 100644 --- a/crates/sqlbuild-python-planner/src/planner_phases/models.rs +++ b/crates/sqlbuild-python-planner/src/planner_phases/models.rs @@ -4,12 +4,17 @@ use crate::planner_phases::_helpers::timing::timings::STEPS; use crate::planner_phases::_helpers::timing::timings::timings_of; use crate::planner_phases::constants::MILLISECONDS; use crate::planner_phases::constants::PERCENT; +use crate::planner_phases::constants::{ + FULL_REFRESH_FUNCTIONS_KEY, FULL_REFRESH_KEY, FULL_REFRESH_MODELS_KEY, +}; use crate::planner_warehouse::models::PlannerReads; use pyo3::prelude::{Bound, Py, PyAny}; +use pyo3::prelude::{PyAnyMethods, PyResult}; use pyo3::pyclass; use serde_json::{Value, json}; use sqlbuild_python_base::native_host::models::Dialect; -use std::collections::BTreeMap; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use std::collections::{BTreeMap, HashSet}; use std::time::Instant; /// The plan output's fields and metadata entries, each set before the output is built. @@ -198,3 +203,33 @@ impl PlanTimings { }) } } + +/// The models and functions to rebuild: all with `--full-refresh`, else those of `full_refresh` jobs. +#[derive(Clone, Debug, Default)] +pub struct FullRefreshRequest { + pub all: bool, + pub models: HashSet, + pub functions: HashSet, +} + +impl FullRefreshRequest { + /// The request of planner overrides' `full_refresh` and `full_refresh_*` names. + pub fn of(overrides: &Bound<'_, PyAny>) -> PyResult { + let names = |key: &str| -> PyResult> { + native_field(overrides, key)? + .try_iter()? + .map(|name| name?.extract()) + .collect() + }; + Ok(Self { + all: native_field(overrides, FULL_REFRESH_KEY)?.is_truthy()?, + models: names(FULL_REFRESH_MODELS_KEY)?, + functions: names(FULL_REFRESH_FUNCTIONS_KEY)?, + }) + } + + /// Whether the invocation asks to rebuild `model` from scratch. + pub fn applies(&self, model: &str) -> bool { + self.all || self.models.contains(model) + } +} diff --git a/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/snapshot.rs b/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/snapshot.rs index a580b7064f..ed63ed5361 100644 --- a/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/snapshot.rs +++ b/crates/sqlbuild-python-planner/src/planner_warehouse/_helpers/inspection/snapshot.rs @@ -1,6 +1,7 @@ //! Gather a frozen point-in-time warehouse snapshot for planning. use crate::planner_phases::main::rebuilds::effectively_full_refreshed_model_names::effectively_full_refreshed_model_names; +use crate::planner_phases::models::FullRefreshRequest; use crate::planner_warehouse::_helpers::inspection::cursors::{CursorInputs, cursor_snapshots}; use crate::planner_warehouse::constants::{ FUNCTION_NODE_TYPES, METADATA_NAME_FILTER_LIMIT, MIGRATE_FROM_CONFIG_KEY, @@ -705,7 +706,13 @@ pub(crate) fn warehouse_snapshot<'py>( (&states.fingerprints, &relevant), )?; let full_refresh_names = if request.full_refresh_model_names.is_none() { - effectively_full_refreshed_model_names(project, request.full_refresh)? + effectively_full_refreshed_model_names( + project, + &FullRefreshRequest { + all: request.full_refresh, + ..FullRefreshRequest::default() + }, + )? } else { request.full_refresh_model_names.clone() }; diff --git a/crates/sqlbuild-python-project/src/compiled_project/_helpers/mod.rs b/crates/sqlbuild-python-project/src/compiled_project/_helpers/mod.rs index bf19c152a0..0bf4bc4a65 100644 --- a/crates/sqlbuild-python-project/src/compiled_project/_helpers/mod.rs +++ b/crates/sqlbuild-python-project/src/compiled_project/_helpers/mod.rs @@ -1,5 +1,4 @@ pub(crate) mod attachment_inputs; -pub(crate) mod job_catalog; pub(crate) mod project; pub(crate) mod schema_facts; pub(crate) mod values; diff --git a/crates/sqlbuild-python-project/src/compiled_project/constants.rs b/crates/sqlbuild-python-project/src/compiled_project/constants.rs index d72ce96a40..71ee41fc3c 100644 --- a/crates/sqlbuild-python-project/src/compiled_project/constants.rs +++ b/crates/sqlbuild-python-project/src/compiled_project/constants.rs @@ -10,10 +10,10 @@ pub const REPORT_ATTEMPTS: usize = 3; pub const STORE_BUSY_STATUS: u16 = 503; /// Selector kinds that can name a job: `job:` and a bare name. -pub(crate) const JOB_SELECTOR_KINDS: [&str; 2] = ["job", "name"]; +pub const JOB_SELECTOR_KINDS: [&str; 2] = ["job", "name"]; /// Separates the parts of an intersection selector. -pub(crate) const SELECTOR_INTERSECTION: char = ','; +pub const SELECTOR_INTERSECTION: char = ','; /// The key prefix of a SQL unit test's result, as the snapshot names it. pub const SQL_TEST_KEY_PREFIX: &str = "sql_test:"; @@ -24,3 +24,17 @@ pub(crate) const SENSOR_RUN_ARGUMENT: &str = "run"; pub const CONTRACT_KEY: &str = "contract"; pub const INCREMENTAL_MODE_KEY: &str = "incremental_mode"; + +/// The build settings a job run's `--changed` reach reads, and the reach every job run uses. +pub const CHANGED_SETTING: &str = "changed"; +pub const REACH_SETTING: &str = "reach"; +/// A job run reaches its whole selection: `reach` only stops builds that are not a job's run. +pub const JOB_RUN_REACH: &str = "all"; +/// What a job run with `changed: True` counts when `[changes] counts` is not set: every change. +pub const JOB_RUN_COUNTS: [&str; 2] = ["definition", "source_data"]; +/// Plan metadata naming the jobs whose own audits a build lets fail. +pub const JOB_IGNORED_AUDIT_FAILURES_KEY: &str = "job_ignored_audit_failures"; +/// Plan metadata naming the jobs whose own models a build rebuilds from scratch. +pub const JOB_FULL_REFRESH_KEY: &str = "job_full_refresh"; +/// Plan metadata mapping each named job whose warehouse its nodes run on to that warehouse. +pub const JOB_WAREHOUSES_KEY: &str = "job_warehouses"; diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/mod.rs b/crates/sqlbuild-python-project/src/compiled_project/main/mod.rs index efde8565a4..7495ef0105 100644 --- a/crates/sqlbuild-python-project/src/compiled_project/main/mod.rs +++ b/crates/sqlbuild-python-project/src/compiled_project/main/mod.rs @@ -1,19 +1,12 @@ pub mod config_values; pub mod declarations; -pub mod declared_jobs; pub mod entry_schema_fact; pub mod family_fact; -pub mod graph_with_jobs; pub mod install; -pub mod job_names; pub mod materialization; pub mod model_facts; -pub mod named_jobs; -pub mod needed_jobs; pub mod plain_value; pub mod python_column_fact; pub mod python_column_facts; pub mod reference_row_tuple; pub mod register_project; -pub mod triggered_jobs; -pub mod with_job_scope; diff --git a/crates/sqlbuild-python-project/src/compiled_project/mod.rs b/crates/sqlbuild-python-project/src/compiled_project/mod.rs index 4af01a464c..15f3d76eeb 100644 --- a/crates/sqlbuild-python-project/src/compiled_project/mod.rs +++ b/crates/sqlbuild-python-project/src/compiled_project/mod.rs @@ -1,4 +1,4 @@ -//! The compiled project: its rows, attachment inputs, schema facts, values and job catalog. +//! The compiled project: its rows, attachment inputs, schema facts, and values. pub(crate) mod _helpers; pub mod constants; diff --git a/crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/python_selection.rs b/crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/python_selection.rs index 6a23dbe2a3..1a37690a10 100644 --- a/crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/python_selection.rs +++ b/crates/sqlbuild-python-project/src/graph/_helpers/python_nodes/python_selection.rs @@ -4,6 +4,7 @@ use crate::graph::main::python_nodes::python_node_names::python_node_names; use sqlbuild_python_base::positional_record; use sqlbuild_python_base::records::main::construct::new_error::new_error; use std::borrow::Cow; +use std::collections::{BTreeMap, BTreeSet}; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::pyfunction; @@ -19,8 +20,8 @@ use sqlbuild_analysis::python_selection::models::{ SqlSelectionGraph, }; -use crate::compiled_project::main::needed_jobs::needed_jobs; -use crate::graph::models::{CompiledProjectGraph, NativeProjectGraph}; +use crate::graph::models::{CompiledProjectGraph, NativeProjectGraph, PythonAwareSelection}; +use crate::job_settings::main::needed_jobs::needed_jobs; use sqlbuild_python_base::boundary::main::compiler_guard::compiler_guard; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_items::mapping_items; @@ -153,6 +154,85 @@ pub(crate) fn resolve_python_node_selection<'py>( python_node_names(graph, (select, exclude)) } +impl<'g> PythonAwareSelection<'g> { + /// The graph of `native` with the jobs `selectors` need attached over `python_graph`. + pub fn new( + (project_graph, native): (&Bound<'_, CompiledProjectGraph>, &'g ProjectGraph), + python_graph: &Bound<'_, PyAny>, + selectors: &[String], + ) -> PyResult { + let py: Python<'_> = project_graph.py(); + let sources: Vec<(String, Option)> = + project_sources(project_graph.get().project(py))?; + let allow_warehouse_round_trips: bool = + allows_round_trips(project_graph.get().project(py))?; + let nodes: PythonSelectionGraph = selection_graph(python_graph)?; + let node_names: Vec<&str> = nodes.nodes.iter().map(|node| node.name.as_str()).collect(); + let graph: Cow<'g, ProjectGraph> = match needed_jobs(py, native, selectors, &node_names)? { + Some(catalog) => Cow::Owned(attach_python_jobs( + SqlSelectionGraph { + graph: native, + sources: &sources, + allow_warehouse_round_trips, + }, + &nodes, + catalog, + )), + None => Cow::Borrowed(native), + }; + Ok(Self { + graph, + sources, + allow_warehouse_round_trips, + nodes, + }) + } + + /// The graph with the needed jobs attached. + pub fn graph(&self) -> &ProjectGraph { + &self.graph + } + + /// The SQL keys and Python node `(kind, name)` keys `select` minus `exclude` resolve to. + pub fn resolve_keys( + &self, + select: &[String], + exclude: &[String], + ) -> Result, SelectorError> { + let RunSelection { + sql_keys, + python_node_names, + } = self.resolve(select, exclude)?; + let kinds: BTreeMap<&str, &str> = self + .nodes + .nodes + .iter() + .map(|node| (node.name.as_str(), node.kind.as_str())) + .collect(); + let mut keys: BTreeSet = sql_keys.into_iter().collect(); + for name in python_node_names { + if let Some(kind) = kinds.get(name.as_str()) { + keys.insert(((*kind).to_owned(), name)); + } + } + Ok(keys) + } + + /// The SQL keys and Python node names `select` minus `exclude` resolve to. + pub fn resolve( + &self, + select: &[String], + exclude: &[String], + ) -> Result { + let sql = SqlSelectionGraph { + graph: &self.graph, + sources: &self.sources, + allow_warehouse_round_trips: self.allow_warehouse_round_trips, + }; + run_selection(sql, &self.nodes, select, exclude) + } +} + /// `(sql keys, python node names)` run selectors resolve to across SQL resources and Python nodes. #[pyfunction] #[pyo3(signature = (*, project_graph, python_graph, select, exclude))] @@ -165,37 +245,14 @@ pub(crate) fn resolve_python_sql_run_selection( let py: Python<'_> = project_graph.py(); let project_graph: &Bound<'_, CompiledProjectGraph> = project_graph.cast()?; let native: &Bound<'_, NativeProjectGraph> = project_graph.get().native(py); - let sources: Vec<(String, Option)> = project_sources(project_graph.get().project(py))?; - let allow_warehouse_round_trips: bool = allows_round_trips(project_graph.get().project(py))?; - let nodes: PythonSelectionGraph = selection_graph(python_graph)?; - let node_names: Vec<&str> = nodes.nodes.iter().map(|node| node.name.as_str()).collect(); let selectors: Vec = select.iter().chain(&exclude).cloned().collect(); - let graph: Cow<'_, ProjectGraph> = - match needed_jobs(py, native.get().graph(), &selectors, &node_names)? { - Some(catalog) => Cow::Owned(attach_python_jobs( - SqlSelectionGraph { - graph: native.get().graph(), - sources: &sources, - allow_warehouse_round_trips, - }, - &nodes, - catalog, - )), - None => Cow::Borrowed(native.get().graph()), - }; - let selection: RunSelection = compiler_guard(|| { - Ok(run_selection( - SqlSelectionGraph { - graph: &graph, - sources: &sources, - allow_warehouse_round_trips, - }, - &nodes, - &select, - &exclude, - )) - })? - .map_err(|error| planner_error(py, error))?; + let resolution = PythonAwareSelection::new( + (project_graph, native.get().graph()), + python_graph, + &selectors, + )?; + let selection: RunSelection = compiler_guard(|| Ok(resolution.resolve(&select, &exclude)))? + .map_err(|error| planner_error(py, error))?; Ok((selection.sql_keys, selection.python_node_names)) } diff --git a/crates/sqlbuild-python-project/src/graph/main/python_nodes/resolve_node_sets.rs b/crates/sqlbuild-python-project/src/graph/main/python_nodes/resolve_node_sets.rs index a494b6d3c8..9f037cc092 100644 --- a/crates/sqlbuild-python-project/src/graph/main/python_nodes/resolve_node_sets.rs +++ b/crates/sqlbuild-python-project/src/graph/main/python_nodes/resolve_node_sets.rs @@ -2,13 +2,13 @@ use std::collections::BTreeMap; -use crate::compiled_project::main::declared_jobs::declared_jobs; use crate::graph::_helpers::python_nodes::python_node_graph::explicit_python_node_graph; use crate::graph::_helpers::python_nodes::python_selection::{ allows_round_trips, project_sources, selection_graph, }; use crate::graph::_helpers::python_nodes::set_dependencies::dependants; use crate::graph::models::NativeProjectGraph; +use crate::job_settings::main::declared_jobs::declared_jobs; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use sqlbuild_analysis::graph::models::JobSelection; use sqlbuild_analysis::node_set_dependencies::main::resolve_node_set_dependencies::resolve_node_set_dependencies; diff --git a/crates/sqlbuild-python-project/src/graph/models.rs b/crates/sqlbuild-python-project/src/graph/models.rs index 76de4d0736..cb48954aa7 100644 --- a/crates/sqlbuild-python-project/src/graph/models.rs +++ b/crates/sqlbuild-python-project/src/graph/models.rs @@ -52,6 +52,7 @@ use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::discovery::main::records::stored_node_sets::stored_node_sets; +use std::borrow::Cow; use std::cell::RefCell; use std::collections::BTreeMap; use std::collections::BTreeSet; @@ -566,3 +567,12 @@ pub struct GraphViewMappings<'a, 'py> { pub tag_index: &'a Bound<'py, PyAny>, pub path_index: &'a Bound<'py, PyAny>, } + +/// A project graph with the named jobs attached, Python nodes included, and the facts to resolve. +#[derive(Debug)] +pub struct PythonAwareSelection<'g> { + pub(crate) graph: Cow<'g, ProjectGraph>, + pub(crate) sources: Vec<(String, Option)>, + pub(crate) allow_warehouse_round_trips: bool, + pub(crate) nodes: PythonSelectionGraph, +} diff --git a/crates/sqlbuild-python-project/src/job_settings/_helpers/argv.rs b/crates/sqlbuild-python-project/src/job_settings/_helpers/argv.rs new file mode 100644 index 0000000000..370edbf3ab --- /dev/null +++ b/crates/sqlbuild-python-project/src/job_settings/_helpers/argv.rs @@ -0,0 +1,101 @@ +//! A build or plan naming jobs takes their build-wide settings where its command line is silent. + +use std::collections::{BTreeMap, BTreeSet}; +use std::path::{Path, PathBuf}; + +use pyo3::exceptions::{PyOSError, PyRuntimeError, PyValueError}; +use pyo3::prelude::{Bound, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods}; +use serde_json::Value; +use sqlbuild_cli::argv::main::accepted_settings_arguments::accepted_settings_arguments; +use sqlbuild_cli::argv::main::given_setting_dests::given_setting_dests; +use sqlbuild_cli::dispatch::main::selector_file_selectors::selector_file_selectors; +use sqlbuild_discovery::constants::ORCHESTRATION_DIRECTORY; +use sqlbuild_python_base::records::main::construct::error_class::error_class; +use sqlbuild_python_base::records::types::ErrorId; + +use crate::job_settings::_helpers::could_name_job::could_name_job; +use crate::job_settings::_helpers::named_job_build_settings::named_job_build_settings; +use crate::job_settings::constants::{JOB_SETTING_COMMANDS, OPTIONS_TERMINATOR}; + +/// One parsed argument, `None` when unset. +fn argument<'py, T: pyo3::FromPyObject<'py>>( + namespace: &Bound<'py, PyDict>, + name: &str, +) -> PyResult> { + Ok(namespace + .get_item(name)? + .map(|value| value.extract::>()) + .transpose()? + .flatten()) +} + +/// The `--select` selectors and those of each `--select-file`. +fn selectors(namespace: &Bound<'_, PyDict>) -> PyResult> { + let mut selectors: Vec = argument(namespace, "select")?.unwrap_or_default(); + let files: Vec = argument(namespace, "select_file")?.unwrap_or_default(); + for path in files { + let text: String = std::fs::read_to_string(&path) + .map_err(|error| PyOSError::new_err(error.to_string()))?; + selectors.extend(selector_file_selectors(&text)); + } + Ok(selectors) +} + +/// The project directory a command runs in, resolved as dispatch resolves it. +fn project_dir(namespace: &Bound<'_, PyDict>) -> PyResult { + let os_error = |error: std::io::Error| PyOSError::new_err(error.to_string()); + let Some(raw): Option = argument(namespace, "project_dir")? else { + return std::env::current_dir().map_err(os_error); + }; + std::fs::canonicalize(&raw).or_else(|_| std::path::absolute(&raw).map_err(os_error)) +} + +/// The settings `argv` gives, and `argv` plus its named jobs' other build-wide flags, if any. +pub(crate) fn named_job_argv( + py: Python<'_>, + argv: &[String], + namespace: &Bound<'_, PyDict>, +) -> PyResult<(BTreeSet, Option>)> { + let none = |given: BTreeSet| Ok((given, None)); + let Some(command): Option = argument(namespace, "command")? else { + return none(BTreeSet::new()); + }; + let job_settings: bool = argument(namespace, "job_settings")?.unwrap_or(false); + if !JOB_SETTING_COMMANDS.contains(&command.as_str()) || !job_settings { + return none(BTreeSet::new()); + } + let given: BTreeSet = given_setting_dests(argv).ok_or_else(|| { + PyRuntimeError::new_err("cannot tell which settings the command line gives") + })?; + let selectors: Vec = selectors(namespace)?; + if !selectors.iter().any(|selector| could_name_job(selector)) { + return none(given); + } + let directory: PathBuf = project_dir(namespace)?; + if !Path::new(&directory).join(ORCHESTRATION_DIRECTORY).is_dir() { + return none(given); + } + let target: Option = argument(namespace, "target")?; + let settings: BTreeMap = + named_job_build_settings(py, (&selectors, &given), (&directory, target))?; + let flags: Vec = + accepted_settings_arguments(&command, &settings).map_err(PyValueError::new_err)?; + if flags.is_empty() { + return none(given); + } + let terminator: usize = argv + .iter() + .position(|argument| argument == OPTIONS_TERMINATOR) + .unwrap_or(argv.len()); + let mut job_argv: Vec = argv.to_vec(); + job_argv.splice(terminator..terminator, flags); + Ok((given, Some(job_argv))) +} + +/// Whether the command meets `error` again, and reports it, when it runs. +pub(crate) fn reported_when_run(py: Python<'_>, error: &PyErr) -> PyResult { + Ok(error.is_instance(py, &error_class(py, ErrorId::CliUser)?) + || error.is_instance_of::(py) + || error.is_instance_of::(py)) +} diff --git a/crates/sqlbuild-python-project/src/job_settings/_helpers/could_name_job.rs b/crates/sqlbuild-python-project/src/job_settings/_helpers/could_name_job.rs new file mode 100644 index 0000000000..cfaca441f0 --- /dev/null +++ b/crates/sqlbuild-python-project/src/job_settings/_helpers/could_name_job.rs @@ -0,0 +1,17 @@ +//! Whether a selector can name a job. + +use sqlbuild_analysis::graph::main::parse_selector::parse_selector; +use sqlbuild_analysis::graph::models::ParsedSelector; + +use crate::compiled_project::constants::{JOB_SELECTOR_KINDS, SELECTOR_INTERSECTION}; + +/// Whether a selector has an atom that can name a job: `job:`, or a bare name. +pub(crate) fn could_name_job(selector: &str) -> bool { + selector + .split_whitespace() + .flat_map(|token| token.split(SELECTOR_INTERSECTION)) + .any(|atom| match parse_selector(atom) { + Ok(ParsedSelector::Kind { kind, .. }) => JOB_SELECTOR_KINDS.contains(&kind.as_str()), + _ => false, + }) +} diff --git a/crates/sqlbuild-python-project/src/compiled_project/_helpers/job_catalog.rs b/crates/sqlbuild-python-project/src/job_settings/_helpers/job_catalog.rs similarity index 65% rename from crates/sqlbuild-python-project/src/compiled_project/_helpers/job_catalog.rs rename to crates/sqlbuild-python-project/src/job_settings/_helpers/job_catalog.rs index ce70fdee71..8b9d2b1757 100644 --- a/crates/sqlbuild-python-project/src/compiled_project/_helpers/job_catalog.rs +++ b/crates/sqlbuild-python-project/src/job_settings/_helpers/job_catalog.rs @@ -3,12 +3,16 @@ use std::cell::RefCell; use std::collections::{BTreeMap, BTreeSet}; use std::path::PathBuf; +use std::thread::LocalKey; use pyo3::IntoPyObject; use pyo3::exceptions::PyValueError; use pyo3::prelude::{PyResult, Python}; +use serde_json::Value; use sqlbuild_analysis::graph::main::parse_selector::parse_selector; use sqlbuild_analysis::graph::models::{JobSelection, ParsedSelector}; +use sqlbuild_analysis::job_settings::main::declared_runs::declared_runs; +use sqlbuild_analysis::job_settings::models::DeclaredRun; use sqlbuild_cli::native_commands::main::effective_target::effective_target; use sqlbuild_discovery::orchestration_files::models::{OrchestrationInventory, RunFile}; @@ -20,16 +24,40 @@ use sqlbuild_python_front::discovery::main::sources::orchestration_declarations: /// Each job and group's selection, and the names of those a schedule or sensor starts. type Catalog = (BTreeMap, BTreeSet); -/// The running command's project directory, its `--target` and, once loaded, its jobs. +/// The running command's project directory, its `--target` and, once loaded, its jobs and their settings. struct JobScope { project_dir: PathBuf, selected_target: Option, - catalog: Option, + catalog: Option<(Catalog, BTreeMap)>, } thread_local! { /// Per thread, since a command resolves its selectors on the thread that dispatches it. static JOB_SCOPE: RefCell> = const { RefCell::new(None) }; + /// The settings the running command's own command line gives, which replace named jobs'. + static GIVEN_SETTINGS: RefCell> = const { RefCell::new(BTreeSet::new()) }; +} + +/// Run `run` with `slot` holding `value`, restoring what it held before. +fn swapped( + slot: &'static LocalKey>, + value: S, + run: impl FnOnce() -> T, +) -> T { + let previous: S = slot.with(|current| current.replace(value)); + let result: T = run(); + slot.with(|current| *current.borrow_mut() = previous); + result +} + +/// Run one command whose command line gives `given` settings, restoring the enclosing ones after. +pub(crate) fn with_given_settings(given: BTreeSet, run: impl FnOnce() -> T) -> T { + swapped(&GIVEN_SETTINGS, given, run) +} + +/// The settings the running command's command line gives. +pub(crate) fn given_settings() -> BTreeSet { + GIVEN_SETTINGS.with(|current| current.borrow().clone()) } /// Run one command with jobs selectable from `project_dir`, restoring the enclosing scope after. @@ -37,16 +65,46 @@ pub(crate) fn with_job_scope( (project_dir, selected_target): (PathBuf, Option), run: impl FnOnce() -> T, ) -> T { - let previous = JOB_SCOPE.with(|scope| { - scope.replace(Some(JobScope { + within_job_scope( + JobScope { project_dir, selected_target, catalog: None, - })) - }); - let result: T = run(); - JOB_SCOPE.with(|scope| *scope.borrow_mut() = previous); - result + }, + run, + ) +} + +/// Run one command with only `job` declared, as its frozen run pins it: its nodes and settings. +pub(crate) fn with_frozen_job( + (project_dir, selected_target): (PathBuf, Option), + (job, select, settings): (&str, Vec, BTreeMap), + run: impl FnOnce() -> T, +) -> T { + let selection = JobSelection::Job { + select, + exclude: Vec::new(), + }; + let declared = DeclaredRun::Job { + settings, + retry: None, + }; + let catalog: Catalog = ( + BTreeMap::from([(job.to_owned(), selection)]), + BTreeSet::new(), + ); + within_job_scope( + JobScope { + project_dir, + selected_target, + catalog: Some((catalog, BTreeMap::from([(job.to_owned(), declared)]))), + }, + run, + ) +} + +fn within_job_scope(scope: JobScope, run: impl FnOnce() -> T) -> T { + swapped(&JOB_SCOPE, Some(scope), run) } /// The jobs `selectors` name, as `job:` or as a bare job name. @@ -82,29 +140,51 @@ pub(crate) fn ordered_jobs( } pub(crate) fn job_catalog(py: Python<'_>) -> PyResult> { + Ok(loaded_jobs(py)?.map(|(catalog, _)| catalog)) +} + +/// The project directory of the running command's job scope, if it has one. +pub(crate) fn job_scope_project_dir() -> Option { + JOB_SCOPE.with(|scope| { + scope + .borrow() + .as_ref() + .map(|scope| scope.project_dir.clone()) + }) +} + +/// Each job's `[settings]` and `[retry]` and each group's jobs declared for the command's target; none outside a command. +pub(crate) fn job_declarations(py: Python<'_>) -> PyResult>> { + Ok(loaded_jobs(py)?.map(|(_, declared)| declared)) +} + +fn loaded_jobs(py: Python<'_>) -> PyResult)>> { let Some(scope) = JOB_SCOPE.with(|scope| { scope.borrow().as_ref().map(|scope| match &scope.catalog { - Some(catalog) => Ok(catalog.clone()), + Some(loaded) => Ok(loaded.clone()), None => Err((scope.project_dir.clone(), scope.selected_target.clone())), }) }) else { return Ok(None); }; let (project_dir, selected_target): (PathBuf, Option) = match scope { - Ok(catalog) => return Ok(Some(catalog)), + Ok(loaded) => return Ok(Some(loaded)), Err(scope) => scope, }; let target: Option = effective_target(&project_dir, selected_target).map_err(PyValueError::new_err)?; let inventory: OrchestrationInventory = orchestration_declarations(project_dir.clone().into_pyobject(py)?.as_any())?; - let catalog: Catalog = inventory_catalog(&inventory, target.as_deref()); + let loaded = ( + inventory_catalog(&inventory, target.as_deref()), + declared_runs(&inventory, target.as_deref()), + ); JOB_SCOPE.with(|scope| { if let Some(scope) = scope.borrow_mut().as_mut() { - scope.catalog = Some(catalog.clone()); + scope.catalog = Some(loaded.clone()); } }); - Ok(Some(catalog)) + Ok(Some(loaded)) } /// Each job's selector tokens and each group's members declared for `target`, and the names a schedule or sensor starts there. diff --git a/crates/sqlbuild-python-project/src/job_settings/_helpers/job_run_settings.rs b/crates/sqlbuild-python-project/src/job_settings/_helpers/job_run_settings.rs new file mode 100644 index 0000000000..9c39d3454f --- /dev/null +++ b/crates/sqlbuild-python-project/src/job_settings/_helpers/job_run_settings.rs @@ -0,0 +1,32 @@ +//! A job run's `--changed` settings. + +use std::collections::BTreeMap; + +use pyo3::prelude::{Bound, PyAny, PyResult}; +use serde_json::Value; +use sqlbuild_python_front::discovery::main::project::configured_changes::configured_changes; + +use crate::compiled_project::constants::{ + CHANGED_SETTING, JOB_RUN_COUNTS, JOB_RUN_REACH, REACH_SETTING, +}; + +/// A job run's `--changed` settings: its whole selection, and source data counted unless configured. +pub(crate) fn job_run_settings( + project_dir: &Bound<'_, PyAny>, + mut settings: BTreeMap, +) -> PyResult> { + let Some(changed) = settings.get(CHANGED_SETTING).cloned() else { + return Ok(settings); + }; + if changed == Value::Bool(false) { + return Ok(settings); + } + settings.insert(REACH_SETTING.to_owned(), Value::from(JOB_RUN_REACH)); + if changed == Value::Bool(true) && configured_changes(project_dir)?.counts.is_none() { + settings.insert( + CHANGED_SETTING.to_owned(), + Value::from(JOB_RUN_COUNTS.to_vec()), + ); + } + Ok(settings) +} diff --git a/crates/sqlbuild-python-project/src/job_settings/_helpers/mod.rs b/crates/sqlbuild-python-project/src/job_settings/_helpers/mod.rs new file mode 100644 index 0000000000..e799fed501 --- /dev/null +++ b/crates/sqlbuild-python-project/src/job_settings/_helpers/mod.rs @@ -0,0 +1,5 @@ +pub(crate) mod argv; +pub(crate) mod could_name_job; +pub(crate) mod job_catalog; +pub(crate) mod job_run_settings; +pub(crate) mod named_job_build_settings; diff --git a/crates/sqlbuild-python-project/src/job_settings/_helpers/named_job_build_settings.rs b/crates/sqlbuild-python-project/src/job_settings/_helpers/named_job_build_settings.rs new file mode 100644 index 0000000000..fb701a05ae --- /dev/null +++ b/crates/sqlbuild-python-project/src/job_settings/_helpers/named_job_build_settings.rs @@ -0,0 +1,70 @@ +//! The build-wide settings the named jobs give a build. + +use std::collections::{BTreeMap, BTreeSet}; +use std::path::Path; + +use pyo3::IntoPyObject; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use serde_json::Value; +use sqlbuild_analysis::job_settings::constants::WAREHOUSE_SETTING; +use sqlbuild_analysis::job_settings::errors::NodeJobSettingsError; +use sqlbuild_analysis::job_settings::main::resolve_job_settings::resolve_job_settings; +use sqlbuild_analysis::job_settings::models::ResolvedJobSettings; +use sqlbuild_config::command_connection::main::check_job_warehouse::check_job_warehouse; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_front::compile_driver::main::session_warehouse_support::session_warehouse_support; +use sqlbuild_python_front::contracts::main::resolve_effective_adapter_name::resolve_effective_adapter_name; +use sqlbuild_python_front::discovery::main::project::discover_project_configuration::discover_project_configuration; + +use crate::job_settings::_helpers::job_run_settings::job_run_settings; +use crate::job_settings::main::job_declarations::job_declarations; +use crate::job_settings::main::job_selector_error::job_selector_error; +use crate::job_settings::main::job_setting_conflict::job_setting_conflict; +use crate::job_settings::main::with_job_scope::with_job_scope; + +/// Whether whole-run `name` = `value` suits a whole build: a valid warehouse the adapter can use. +fn applies_to_whole_run( + py: Python<'_>, + (name, value): (&str, &Value), + directory: &Bound<'_, PyAny>, +) -> PyResult { + if name != WAREHOUSE_SETTING || check_job_warehouse("", value.as_str()).is_err() { + return Ok(false); + } + let discovered: Bound<'_, PyAny> = discover_project_configuration(py, directory)?; + let adapter_name: Bound<'_, PyAny> = resolve_effective_adapter_name( + &native_field(&discovered, "project_config")?, + &native_field(&discovered, "local_config")?, + )?; + session_warehouse_support(py, &adapter_name, directory) +} + +/// The named jobs' build-wide and shared whole-run settings, less `given`; or `S027`. +pub(crate) fn named_job_build_settings( + py: Python<'_>, + (selectors, given): (&[String], &BTreeSet), + (directory, selected_target): (&Path, Option), +) -> PyResult> { + let scope = (directory.to_path_buf(), selected_target); + let Some(declared) = with_job_scope(scope, || job_declarations(py))? else { + return Ok(BTreeMap::new()); + }; + let resolved: ResolvedJobSettings = + match resolve_job_settings(declared, (selectors, given), None) { + Ok(resolved) => resolved, + Err(NodeJobSettingsError::Conflict(conflict)) => { + return Err(job_setting_conflict(py, &conflict)); + } + Err(NodeJobSettingsError::Selector(failure)) => { + return Err(job_selector_error(py, &failure)); + } + }; + let directory: Bound<'_, PyAny> = directory.into_pyobject(py)?.into_any(); + let mut agreed: BTreeMap = resolved.build; + for (name, value) in resolved.whole_run { + if applies_to_whole_run(py, (&name, &value), &directory)? { + agreed.insert(name, value); + } + } + job_run_settings(&directory, agreed) +} diff --git a/crates/sqlbuild-python-project/src/job_settings/constants.rs b/crates/sqlbuild-python-project/src/job_settings/constants.rs new file mode 100644 index 0000000000..bf826724b1 --- /dev/null +++ b/crates/sqlbuild-python-project/src/job_settings/constants.rs @@ -0,0 +1,17 @@ +//! Constants of named job settings. + +/// Ends the options; job flags go before it so they are never read as selectors. +pub const OPTIONS_TERMINATOR: &str = "--"; +/// The commands whose selections take the settings of the jobs they name. +pub const JOB_SETTING_COMMANDS: [&str; 2] = ["build", "plan"]; +/// A SQL unit test's published kind; a frozen run selects it back with `test:`. +pub const SQL_TEST_NODE_KIND: &str = "sql_test"; +/// The cursor settings a frozen run's window replaces. +pub const WINDOW_CURSOR_SETTINGS: [&str; 6] = [ + "start_cursor", + "end_cursor", + "start_cursor_ts", + "end_cursor_ts", + "start_cursor_int", + "end_cursor_int", +]; diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/declared_jobs.rs b/crates/sqlbuild-python-project/src/job_settings/main/declared_jobs.rs similarity index 91% rename from crates/sqlbuild-python-project/src/compiled_project/main/declared_jobs.rs rename to crates/sqlbuild-python-project/src/job_settings/main/declared_jobs.rs index 6510eb7a92..cbbe081979 100644 --- a/crates/sqlbuild-python-project/src/compiled_project/main/declared_jobs.rs +++ b/crates/sqlbuild-python-project/src/job_settings/main/declared_jobs.rs @@ -1,6 +1,6 @@ //! Every job and group a project declares, for resolving `job(...)` dependencies. -use crate::compiled_project::_helpers::job_catalog::catalog_where; +use crate::job_settings::_helpers::job_catalog::catalog_where; use pyo3::prelude::{Bound, PyAny, PyResult}; use sqlbuild_analysis::graph::models::JobSelection; use sqlbuild_discovery::orchestration_files::models::OrchestrationInventory; diff --git a/crates/sqlbuild-python-project/src/job_settings/main/frozen_run_flags.rs b/crates/sqlbuild-python-project/src/job_settings/main/frozen_run_flags.rs new file mode 100644 index 0000000000..b44d399673 --- /dev/null +++ b/crates/sqlbuild-python-project/src/job_settings/main/frozen_run_flags.rs @@ -0,0 +1,49 @@ +//! The command-line flags of a frozen run's settings. + +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use sqlbuild_cli::argv::main::command_settings_arguments::command_settings_arguments; +use sqlbuild_cli::argv::main::job_run_arguments::job_run_arguments; +use sqlbuild_cli::argv::main::job_settings_arguments::job_settings_arguments; +use sqlbuild_orchestration::runs::models::{RunCommand, RunRequest, RunWindow}; + +use crate::job_settings::_helpers::job_run_settings::job_run_settings; +use crate::job_settings::constants::WINDOW_CURSOR_SETTINGS; +use crate::job_settings::models::FrozenRunFlags; + +/// A frozen run's settings with its window's cursors, and the flags a worker passes for them. +pub fn frozen_run_flags( + frozen: &RunRequest, + project_dir: &Bound<'_, PyAny>, +) -> PyResult { + let mut settings = frozen.settings.clone(); + if let Some(window) = &frozen.window { + for key in WINDOW_CURSOR_SETTINGS { + settings.remove(key); + } + let (kind, start, end) = match window { + RunWindow::Timestamp { start, end } => ("ts", start.to_rfc3339(), end.to_rfc3339()), + RunWindow::Integer { start, end } => ("int", start.to_string(), end.to_string()), + }; + settings.insert(format!("start_cursor_{kind}"), serde_json::json!(start)); + settings.insert(format!("end_cursor_{kind}"), serde_json::json!(end)); + } + if frozen.job_key.is_some() { + settings = job_run_settings(project_dir, settings)?; + } + let job_build: Option = frozen + .job_key + .clone() + .filter(|_| frozen.command == RunCommand::Build); + let flags: Vec = match &job_build { + Some(job) => job_run_arguments(job, &settings), + None if frozen.command == RunCommand::Build => job_settings_arguments(&settings), + None => command_settings_arguments(frozen.command.as_str(), &settings), + } + .map_err(PyValueError::new_err)?; + Ok(FrozenRunFlags { + settings, + flags, + job_build, + }) +} diff --git a/crates/sqlbuild-python-project/src/job_settings/main/frozen_selectors.rs b/crates/sqlbuild-python-project/src/job_settings/main/frozen_selectors.rs new file mode 100644 index 0000000000..fb994a27e4 --- /dev/null +++ b/crates/sqlbuild-python-project/src/job_settings/main/frozen_selectors.rs @@ -0,0 +1,35 @@ +//! The selectors that select a frozen run's nodes back. + +use std::collections::BTreeMap; + +use pyo3::exceptions::PyValueError; +use pyo3::prelude::PyResult; +use sqlbuild_analysis::definition_changes::constants::PYTHON_NODE_KINDS; +use sqlbuild_orchestration::runs::models::RunRequest; + +use crate::job_settings::constants::SQL_TEST_NODE_KIND; + +/// One selector per frozen node the run has not skipped. +pub fn frozen_selectors( + frozen: &RunRequest, + skipped_nodes: &BTreeMap, +) -> PyResult> { + let mut selectors: Vec = Vec::new(); + for key in frozen + .selection + .iter() + .filter(|key| !skipped_nodes.contains_key(*key)) + { + let (kind, name) = key + .split_once(':') + .ok_or_else(|| PyValueError::new_err("frozen run contains an invalid node key"))?; + selectors.push(if PYTHON_NODE_KINDS.contains(&kind) { + key.clone() + } else if kind == SQL_TEST_NODE_KIND { + format!("test:{name}") + } else { + name.to_owned() + }); + } + Ok(selectors) +} diff --git a/crates/sqlbuild-python-project/src/job_settings/main/given_settings.rs b/crates/sqlbuild-python-project/src/job_settings/main/given_settings.rs new file mode 100644 index 0000000000..08cd78b9cb --- /dev/null +++ b/crates/sqlbuild-python-project/src/job_settings/main/given_settings.rs @@ -0,0 +1,11 @@ +//! The settings the running command's own command line gives. + +use std::collections::BTreeSet; + +use crate::job_settings::_helpers::job_catalog; + +/// The settings the running command's command line gives, which replace named jobs' settings. +#[inline] +pub fn given_settings() -> BTreeSet { + job_catalog::given_settings() +} diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/graph_with_jobs.rs b/crates/sqlbuild-python-project/src/job_settings/main/graph_with_jobs.rs similarity index 92% rename from crates/sqlbuild-python-project/src/compiled_project/main/graph_with_jobs.rs rename to crates/sqlbuild-python-project/src/job_settings/main/graph_with_jobs.rs index e8fda5ea27..ea1a300df4 100644 --- a/crates/sqlbuild-python-project/src/compiled_project/main/graph_with_jobs.rs +++ b/crates/sqlbuild-python-project/src/job_settings/main/graph_with_jobs.rs @@ -1,6 +1,6 @@ //! `graph` with the command's jobs attached when `selectors` name a job, otherwise `graph` itself. -use crate::compiled_project::main::needed_jobs::needed_jobs; +use crate::job_settings::main::needed_jobs::needed_jobs; use pyo3::prelude::{PyResult, Python}; use sqlbuild_analysis::graph::main::attach_jobs::attach_jobs; use sqlbuild_analysis::graph::models::ProjectGraph; diff --git a/crates/sqlbuild-python-project/src/job_settings/main/job_declarations.rs b/crates/sqlbuild-python-project/src/job_settings/main/job_declarations.rs new file mode 100644 index 0000000000..9e114f56d5 --- /dev/null +++ b/crates/sqlbuild-python-project/src/job_settings/main/job_declarations.rs @@ -0,0 +1,12 @@ +//! Each job's `[settings]` and `[retry]` and each group's jobs declared for the command's target. + +use crate::job_settings::_helpers::job_catalog; +use pyo3::prelude::{PyResult, Python}; +use sqlbuild_analysis::job_settings::models::DeclaredRun; +use std::collections::BTreeMap; + +/// Each job's `[settings]` and `[retry]` and each group's jobs declared for the command's target; none outside a command. +#[inline] +pub fn job_declarations(py: Python<'_>) -> PyResult>> { + job_catalog::job_declarations(py) +} diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/job_names.rs b/crates/sqlbuild-python-project/src/job_settings/main/job_names.rs similarity index 85% rename from crates/sqlbuild-python-project/src/compiled_project/main/job_names.rs rename to crates/sqlbuild-python-project/src/job_settings/main/job_names.rs index bd3875b222..742094bf8d 100644 --- a/crates/sqlbuild-python-project/src/compiled_project/main/job_names.rs +++ b/crates/sqlbuild-python-project/src/job_settings/main/job_names.rs @@ -1,6 +1,6 @@ //! The command's job and job group names, loaded on first use; empty outside a command. -use crate::compiled_project::_helpers::job_catalog::job_catalog; +use crate::job_settings::_helpers::job_catalog::job_catalog; use pyo3::prelude::{PyResult, Python}; /// The command's job and job group names, loaded on first use; empty outside a command. diff --git a/crates/sqlbuild-python-project/src/job_settings/main/job_scope_project_dir.rs b/crates/sqlbuild-python-project/src/job_settings/main/job_scope_project_dir.rs new file mode 100644 index 0000000000..a06e3ecf74 --- /dev/null +++ b/crates/sqlbuild-python-project/src/job_settings/main/job_scope_project_dir.rs @@ -0,0 +1,10 @@ +//! The project directory of the running command's job scope. + +use crate::job_settings::_helpers::job_catalog; +use std::path::PathBuf; + +/// The project directory of the running command's job scope, if it has one. +#[inline] +pub fn job_scope_project_dir() -> Option { + job_catalog::job_scope_project_dir() +} diff --git a/crates/sqlbuild-python-project/src/job_settings/main/job_selector_error.rs b/crates/sqlbuild-python-project/src/job_settings/main/job_selector_error.rs new file mode 100644 index 0000000000..88e5b7870a --- /dev/null +++ b/crates/sqlbuild-python-project/src/job_settings/main/job_selector_error.rs @@ -0,0 +1,25 @@ +//! The error of a selector that cannot resolve a named job's nodes. + +use pyo3::PyErr; +use pyo3::prelude::Python; +use sqlbuild_analysis::job_settings::errors::JobSelectorError; +use sqlbuild_python_base::records::main::construct::new_error::new_error; +use sqlbuild_python_base::records::types::ErrorId; + +/// A `CliUserError` naming the jobs whose selection a selector failed to resolve, with its code. +pub fn job_selector_error(py: Python<'_>, failure: &JobSelectorError) -> PyErr { + let jobs: String = if failure.jobs.is_empty() { + "the named jobs".to_owned() + } else { + format!("job {}", failure.jobs.join(", ")) + }; + new_error( + py, + ErrorId::CliUser, + &format!( + "cannot apply the settings of {jobs}: selector '{}' failed: {}", + failure.selector, failure.error.message + ), + (Some(failure.error.code), failure.error.help.as_deref()), + ) +} diff --git a/crates/sqlbuild-python-project/src/job_settings/main/job_setting_conflict.rs b/crates/sqlbuild-python-project/src/job_settings/main/job_setting_conflict.rs new file mode 100644 index 0000000000..23fa8512e6 --- /dev/null +++ b/crates/sqlbuild-python-project/src/job_settings/main/job_setting_conflict.rs @@ -0,0 +1,21 @@ +//! The error of job settings that cannot apply together. + +use pyo3::PyErr; +use pyo3::prelude::Python; +use sqlbuild_analysis::job_settings::constants::JOB_SETTING_CONFLICT_CODE; +use sqlbuild_analysis::job_settings::errors::JobSettingConflict; +use sqlbuild_python_base::records::main::construct::new_error::new_error; +use sqlbuild_python_base::records::types::ErrorId; + +/// `CliUserError` for job settings that cannot apply together. +pub fn job_setting_conflict(py: Python<'_>, conflict: &JobSettingConflict) -> PyErr { + new_error( + py, + ErrorId::CliUser, + &conflict.message, + ( + Some(JOB_SETTING_CONFLICT_CODE), + Some(conflict.help.as_str()), + ), + ) +} diff --git a/crates/sqlbuild-python-project/src/job_settings/main/job_settings_argv.rs b/crates/sqlbuild-python-project/src/job_settings/main/job_settings_argv.rs new file mode 100644 index 0000000000..d4ff004767 --- /dev/null +++ b/crates/sqlbuild-python-project/src/job_settings/main/job_settings_argv.rs @@ -0,0 +1,40 @@ +//! The argv a build or plan naming jobs parses again with their build-wide flags. + +use std::collections::BTreeSet; + +use pyo3::exceptions::PyUnicodeError; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyString}; +use sqlbuild_cli::argv::main::given_setting_dests::given_setting_dests; + +use crate::job_settings::_helpers::argv::{named_job_argv, reported_when_run}; +use crate::job_settings::types::JobSettingsArgv; + +/// The settings the command line gives, and the argv to parse again with its jobs' flags, if any. +pub fn job_settings_argv<'py>( + argv: &Bound<'py, PyAny>, + namespace: &Bound<'py, PyDict>, +) -> PyResult> { + let py: Python<'py> = argv.py(); + let arguments: Vec = match argv.extract() { + Ok(arguments) => arguments, + Err(undecodable) if undecodable.is_instance_of::(py) => { + return Ok((BTreeSet::new(), None)); + } + Err(error) => return Err(error), + }; + let (given, job_argv) = match named_job_argv(py, &arguments, namespace) { + Err(error) if reported_when_run(py, &error)? => { + (given_setting_dests(&arguments).unwrap_or_default(), None) + } + result => result?, + }; + let Some(job_argv) = job_argv else { + return Ok((given, None)); + }; + let strings: Vec> = job_argv + .iter() + .map(|argument| PyString::new(py, argument)) + .collect(); + Ok((given, Some(strings))) +} diff --git a/crates/sqlbuild-python-project/src/job_settings/main/mod.rs b/crates/sqlbuild-python-project/src/job_settings/main/mod.rs new file mode 100644 index 0000000000..d9222cb453 --- /dev/null +++ b/crates/sqlbuild-python-project/src/job_settings/main/mod.rs @@ -0,0 +1,17 @@ +pub mod declared_jobs; +pub mod frozen_run_flags; +pub mod frozen_selectors; +pub mod given_settings; +pub mod graph_with_jobs; +pub mod job_declarations; +pub mod job_names; +pub mod job_scope_project_dir; +pub mod job_selector_error; +pub mod job_setting_conflict; +pub mod job_settings_argv; +pub mod named_jobs; +pub mod needed_jobs; +pub mod triggered_jobs; +pub mod with_frozen_job; +pub mod with_given_settings; +pub mod with_job_scope; diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/named_jobs.rs b/crates/sqlbuild-python-project/src/job_settings/main/named_jobs.rs similarity index 76% rename from crates/sqlbuild-python-project/src/compiled_project/main/named_jobs.rs rename to crates/sqlbuild-python-project/src/job_settings/main/named_jobs.rs index b79ebfbee1..5fc7884c95 100644 --- a/crates/sqlbuild-python-project/src/compiled_project/main/named_jobs.rs +++ b/crates/sqlbuild-python-project/src/job_settings/main/named_jobs.rs @@ -1,7 +1,7 @@ //! The jobs a command's selectors name, in the order named. -use crate::compiled_project::_helpers::job_catalog::job_catalog; -use crate::compiled_project::_helpers::job_catalog::ordered_jobs; +use crate::job_settings::_helpers::job_catalog::job_catalog; +use crate::job_settings::_helpers::job_catalog::ordered_jobs; use pyo3::prelude::{PyResult, Python}; /// The jobs `selectors` name, as `job:` or a bare job name, each once in the order named; none outside a command. diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/needed_jobs.rs b/crates/sqlbuild-python-project/src/job_settings/main/needed_jobs.rs similarity index 91% rename from crates/sqlbuild-python-project/src/compiled_project/main/needed_jobs.rs rename to crates/sqlbuild-python-project/src/job_settings/main/needed_jobs.rs index 949d525f66..25378c194d 100644 --- a/crates/sqlbuild-python-project/src/compiled_project/main/needed_jobs.rs +++ b/crates/sqlbuild-python-project/src/job_settings/main/needed_jobs.rs @@ -1,6 +1,6 @@ //! The command's jobs when `selectors` name one and `graph` has none attached yet. -use crate::compiled_project::_helpers::job_catalog::job_catalog; +use crate::job_settings::_helpers::job_catalog::job_catalog; use pyo3::prelude::{PyResult, Python}; use sqlbuild_analysis::graph::main::selectors_need_jobs::selectors_need_jobs; use sqlbuild_analysis::graph::models::{JobSelection, ProjectGraph}; diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/triggered_jobs.rs b/crates/sqlbuild-python-project/src/job_settings/main/triggered_jobs.rs similarity index 91% rename from crates/sqlbuild-python-project/src/compiled_project/main/triggered_jobs.rs rename to crates/sqlbuild-python-project/src/job_settings/main/triggered_jobs.rs index 741f09e695..075d6e339c 100644 --- a/crates/sqlbuild-python-project/src/compiled_project/main/triggered_jobs.rs +++ b/crates/sqlbuild-python-project/src/job_settings/main/triggered_jobs.rs @@ -1,7 +1,7 @@ //! Each job a schedule or sensor starts with the nodes it refreshes, less those of jobs `selectors` run. -use crate::compiled_project::_helpers::job_catalog::job_catalog; -use crate::compiled_project::_helpers::job_catalog::selected_jobs; +use crate::job_settings::_helpers::job_catalog::job_catalog; +use crate::job_settings::_helpers::job_catalog::selected_jobs; use pyo3::prelude::{PyResult, Python}; use sqlbuild_analysis::assembly::project::types::ObjectKey; use sqlbuild_analysis::graph::main::attach_jobs::attach_jobs; diff --git a/crates/sqlbuild-python-project/src/job_settings/main/with_frozen_job.rs b/crates/sqlbuild-python-project/src/job_settings/main/with_frozen_job.rs new file mode 100644 index 0000000000..9c8d428b6e --- /dev/null +++ b/crates/sqlbuild-python-project/src/job_settings/main/with_frozen_job.rs @@ -0,0 +1,16 @@ +//! Run one command with only one job declared, as its frozen run pins it. + +use crate::job_settings::_helpers::job_catalog; +use serde_json::Value; +use std::collections::BTreeMap; +use std::path::PathBuf; + +/// Run one command with only `job` declared, as its frozen run pins it: its nodes and settings. +#[inline] +pub fn with_frozen_job( + directory: (PathBuf, Option), + frozen: (&str, Vec, BTreeMap), + run: impl FnOnce() -> T, +) -> T { + job_catalog::with_frozen_job(directory, frozen, run) +} diff --git a/crates/sqlbuild-python-project/src/job_settings/main/with_given_settings.rs b/crates/sqlbuild-python-project/src/job_settings/main/with_given_settings.rs new file mode 100644 index 0000000000..fe9957d53f --- /dev/null +++ b/crates/sqlbuild-python-project/src/job_settings/main/with_given_settings.rs @@ -0,0 +1,11 @@ +//! Run one command whose command line gives some settings itself. + +use std::collections::BTreeSet; + +use crate::job_settings::_helpers::job_catalog; + +/// Run one command whose command line gives `given` settings, restoring the enclosing ones after. +#[inline] +pub fn with_given_settings(given: BTreeSet, run: impl FnOnce() -> T) -> T { + job_catalog::with_given_settings(given, run) +} diff --git a/crates/sqlbuild-python-project/src/compiled_project/main/with_job_scope.rs b/crates/sqlbuild-python-project/src/job_settings/main/with_job_scope.rs similarity index 89% rename from crates/sqlbuild-python-project/src/compiled_project/main/with_job_scope.rs rename to crates/sqlbuild-python-project/src/job_settings/main/with_job_scope.rs index bc8fde79ad..0e33891f3e 100644 --- a/crates/sqlbuild-python-project/src/compiled_project/main/with_job_scope.rs +++ b/crates/sqlbuild-python-project/src/job_settings/main/with_job_scope.rs @@ -1,6 +1,6 @@ //! Run one command with jobs selectable from `project_dir`, restoring the enclosing scope after. -use crate::compiled_project::_helpers::job_catalog; +use crate::job_settings::_helpers::job_catalog; use std::path::PathBuf; /// Run one command with jobs selectable from `project_dir`, restoring the enclosing scope after. diff --git a/crates/sqlbuild-python-project/src/job_settings/mod.rs b/crates/sqlbuild-python-project/src/job_settings/mod.rs new file mode 100644 index 0000000000..66a086ca40 --- /dev/null +++ b/crates/sqlbuild-python-project/src/job_settings/mod.rs @@ -0,0 +1,7 @@ +//! Named jobs: the catalog a command's selectors name jobs from, and the settings jobs give it. + +pub(crate) mod _helpers; +pub mod constants; +pub mod main; +pub mod models; +pub mod types; diff --git a/crates/sqlbuild-python-project/src/job_settings/models.rs b/crates/sqlbuild-python-project/src/job_settings/models.rs new file mode 100644 index 0000000000..0a6c3ff976 --- /dev/null +++ b/crates/sqlbuild-python-project/src/job_settings/models.rs @@ -0,0 +1,13 @@ +//! Models of named job settings. + +use std::collections::BTreeMap; + +use serde_json::Value; + +/// A frozen run's settings, the command-line flags they become, and the job a job's build names. +#[derive(Debug)] +pub struct FrozenRunFlags { + pub settings: BTreeMap, + pub flags: Vec, + pub job_build: Option, +} diff --git a/crates/sqlbuild-python-project/src/job_settings/types.rs b/crates/sqlbuild-python-project/src/job_settings/types.rs new file mode 100644 index 0000000000..5873fec138 --- /dev/null +++ b/crates/sqlbuild-python-project/src/job_settings/types.rs @@ -0,0 +1,9 @@ +//! Type aliases of named-job settings. + +use std::collections::BTreeSet; + +use pyo3::prelude::Bound; +use pyo3::types::PyString; + +/// The settings a command line gives, and its argv with its named jobs' flags when they add any. +pub type JobSettingsArgv<'py> = (BTreeSet, Option>>); diff --git a/crates/sqlbuild-python-project/src/lib.rs b/crates/sqlbuild-python-project/src/lib.rs index 7af797d10d..fb136cbe39 100644 --- a/crates/sqlbuild-python-project/src/lib.rs +++ b/crates/sqlbuild-python-project/src/lib.rs @@ -11,6 +11,7 @@ pub mod compiled_views; pub mod contracts; pub mod fingerprints; pub mod graph; +pub mod job_settings; pub mod lineage; pub mod native_host; pub mod planner_identity; diff --git a/crates/sqlbuild-python-project/src/native_host/_helpers/runs_state.rs b/crates/sqlbuild-python-project/src/native_host/_helpers/runs_state.rs index effdc06fcc..e5e40d9abd 100644 --- a/crates/sqlbuild-python-project/src/native_host/_helpers/runs_state.rs +++ b/crates/sqlbuild-python-project/src/native_host/_helpers/runs_state.rs @@ -1,6 +1,6 @@ //! Helpers of the runs table recorder: its DDL, the run's identity facts and its row cells. -use crate::compiled_project::main::named_jobs::named_jobs; +use crate::job_settings::main::named_jobs::named_jobs; use crate::native_host::models::RunCommandScope; use crate::native_host::models::RunIdentity; use pyo3::exceptions::{PyKeyError, PyOSError, PyValueError}; diff --git a/crates/sqlbuild-python-project/src/planner_selection/_helpers/unit_tests.rs b/crates/sqlbuild-python-project/src/planner_selection/_helpers/unit_tests.rs index 600f939727..7f205bcb85 100644 --- a/crates/sqlbuild-python-project/src/planner_selection/_helpers/unit_tests.rs +++ b/crates/sqlbuild-python-project/src/planner_selection/_helpers/unit_tests.rs @@ -13,7 +13,7 @@ use sqlbuild_analysis::planner_selection::models::{ SelectorGroups, UnitTestSelectorNames, UnitTestSelectorSplit, }; -use crate::compiled_project::main::job_names::job_names; +use crate::job_settings::main::job_names::job_names; use crate::planner_selection::models::Carrier; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; use sqlbuild_python_base::records::main::fields::field::field as native_field; diff --git a/crates/sqlbuild-python-project/src/planner_selection/main/match_project_selector.rs b/crates/sqlbuild-python-project/src/planner_selection/main/match_project_selector.rs index fd2804e314..b579d93dbb 100644 --- a/crates/sqlbuild-python-project/src/planner_selection/main/match_project_selector.rs +++ b/crates/sqlbuild-python-project/src/planner_selection/main/match_project_selector.rs @@ -1,8 +1,8 @@ //! The keys one parsed selector matches before `+` graph expansion. -use crate::compiled_project::main::graph_with_jobs::graph_with_jobs; use crate::graph::models::GraphViewMappings; use crate::graph::models::NativeProjectGraph; +use crate::job_settings::main::graph_with_jobs::graph_with_jobs; use crate::planner_selection::_helpers::selectors::graph_names; use crate::planner_selection::_helpers::selectors::selected; use crate::planner_selection::_helpers::selectors::view_graph; diff --git a/crates/sqlbuild-python-project/src/planner_selection/main/resolve_graph_keys.rs b/crates/sqlbuild-python-project/src/planner_selection/main/resolve_graph_keys.rs index c52bfcc3c6..a0471e7c14 100644 --- a/crates/sqlbuild-python-project/src/planner_selection/main/resolve_graph_keys.rs +++ b/crates/sqlbuild-python-project/src/planner_selection/main/resolve_graph_keys.rs @@ -1,6 +1,6 @@ //! Resolve graph keys for native planning without building a public frozenset. -use crate::compiled_project::main::graph_with_jobs::graph_with_jobs; +use crate::job_settings::main::graph_with_jobs::graph_with_jobs; use crate::planner_selection::_helpers::selectors::graph_names; use crate::planner_selection::main::py_strings::py_strings; use crate::planner_selection::models::Carrier; diff --git a/crates/sqlbuild-python-project/src/planner_selection/main/resolve_project_selector_tokens.rs b/crates/sqlbuild-python-project/src/planner_selection/main/resolve_project_selector_tokens.rs index b480f0018a..77d5c3e88e 100644 --- a/crates/sqlbuild-python-project/src/planner_selection/main/resolve_project_selector_tokens.rs +++ b/crates/sqlbuild-python-project/src/planner_selection/main/resolve_project_selector_tokens.rs @@ -1,7 +1,7 @@ //! The keys whitespace-separated selector tokens match, without required build resources. -use crate::compiled_project::main::graph_with_jobs::graph_with_jobs; use crate::graph::models::NativeProjectGraph; +use crate::job_settings::main::graph_with_jobs::graph_with_jobs; use crate::planner_selection::_helpers::selectors::VIEW_OPTIONS; use crate::planner_selection::_helpers::selectors::arguments_graph; use crate::planner_selection::_helpers::selectors::graph_names; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs index cba13215c6..85c0bb3ec6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs @@ -43,6 +43,9 @@ use sqlbuild_python_base::records::main::construct::new_record::new_record; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_base::records::types::RecordId; +use sqlbuild_python_compile::compile_driver::main::jobs::build_concurrency::build_concurrency; +use sqlbuild_python_compile::compile_driver::main::jobs::connection_limit::connection_limit; +use sqlbuild_python_compile::compile_driver::main::jobs::job_run_context::job_run_context; use sqlbuild_python_compile::compile_driver::main::setup::build_project_relation_targets::build_project_relation_targets; use sqlbuild_python_compile::compile_driver::main::setup::project_target_settings::project_target_settings; use sqlbuild_python_compile::compile_driver::models::TargetSettingsValues; @@ -165,6 +168,7 @@ fn run_context_facts( selector_count: native_field(&file, "selector_count")?.extract()?, }); } + let (full_refresh_jobs, job_warehouses) = job_run_context(pipeline)?; Ok(RunContextFacts { command: EXECUTION_COMMAND.to_owned(), run_id: display_text(&native_field(&project, "run_id")?)?, @@ -175,6 +179,8 @@ fn run_context_facts( concurrency: concurrency.limit, concurrency_source: concurrency.source.to_string(), full_refresh: context.request_flag("full_refresh")?, + full_refresh_jobs, + job_warehouses, selected_count, total_count, date_vars: ( @@ -190,7 +196,7 @@ pub(crate) fn write_run_context( context: &BuildContext<'_>, pipeline: &Bound<'_, PyAny>, ) -> PyResult<()> { - let concurrency: EffectiveConcurrency = context.concurrency()?; + let concurrency: EffectiveConcurrency = build_concurrency(context.concurrency()?, pipeline)?; let facts: RunContextFacts = run_context_facts(context, pipeline, &concurrency)?; emit( &context.stream, @@ -255,7 +261,7 @@ pub(crate) fn prepare_build_execution<'py>( }) .and_then(|value| pyo3::Bound::new(py, value)) .map(pyo3::Bound::into_any)?; - let concurrency = context.concurrency()?.limit; + let concurrency = connection_limit(context.concurrency()?, pipeline)?; if context.verbose { write_run_context(context, pipeline)?; } else { @@ -337,6 +343,7 @@ pub(crate) fn execute_build_plan<'py>( let lifecycle: &DirectPythonLifecycle = preparation.lifecycle.get(); let [start_ts, end_ts, start_int, end_int] = &preparation.cursors; let request = |name: &str| native_field(&context.request, name); + let planned = |name: &str| native_field(pipeline, name); let discovered: Bound<'py, PyAny> = context.get("discovered_inputs")?; let project_dir: Bound<'py, PyAny> = context.get("effective_project_dir")?; let target_config: TargetSettingsValues = @@ -375,6 +382,9 @@ pub(crate) fn execute_build_plan<'py>( ("end_cursor_int", end_int.clone()), ("use_color", boolean(py, context.use_color)), ("providers", providers.clone()), + ("job_limits", planned("job_limits")?), + ("node_jobs", planned("node_jobs")?), + ("node_warehouses", planned("node_warehouses")?), ( "test_timeouts", timeout_record( @@ -676,6 +686,10 @@ fn check_runtime<'py>( .and_then(|value| pyo3::IntoPyObjectExt::into_bound_py_any(value, py))?, ), ("providers", providers.clone()), + ( + "node_warehouses", + native_field(pipeline, "python_node_warehouses")?, + ), ], ) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs index f5e23ede68..4719e205d5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs @@ -1,6 +1,6 @@ //! Ingress Python before the build and read-side tasks and assets dispatched as SQL completes. -use std::collections::{HashMap, HashSet}; +use std::collections::{BTreeMap, HashMap, HashSet}; use std::sync::Mutex; use pyo3::exceptions::PyRuntimeError; @@ -344,6 +344,8 @@ struct RuntimeParts<'a, 'py> { target_config: TargetSettingsValues, relations: (Bound<'py, PyAny>, Bound<'py, PyAny>), inputs: &'a LifecycleInputs<'py>, + /// Each Python node's job warehouse, by name. + warehouses: BTreeMap, } fn node_runtime<'py>( @@ -375,6 +377,10 @@ fn node_runtime<'py>( ("relation_targets", parts.relations.0.clone()), ("project_relations", parts.relations.1.clone()), ("providers", parts.inputs.providers.clone()), + ( + "node_warehouses", + pyo3::IntoPyObject::into_pyobject(parts.warehouses.clone(), py)?.into_any(), + ), ], ) } @@ -528,6 +534,7 @@ pub(crate) fn prepare_direct_python_lifecycle<'py>( target_config, relations: (relation_targets, project_relations), inputs, + warehouses: native_field(pipeline, "python_node_warehouses")?.extract()?, }; let mut ingress_python: Bound<'py, PyAny> = tuple(py, &[])?; let mut ingress_loads: Bound<'py, PyAny> = tuple(py, &[])?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/planning.rs index 9cbf89ba27..65595cdf52 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/planning.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/planning.rs @@ -96,6 +96,8 @@ fn request_options<'py>(context: &BuildContext<'py>) -> PyResult) -> PyResult> { } } +/// The attributes a parse set on its namespace. +fn namespace_of<'py>(args: &Bound<'py, PyAny>) -> PyResult> { + Ok(args.getattr("__dict__")?.cast_into::()?) +} + fn parsed_invocation<'py>( argv: &Bound<'py, PyAny>, handlers: Option>, @@ -805,7 +812,7 @@ fn parsed_invocation<'py>( let py: Python<'py> = argv.py(); let use_color: bool = !argv.contains(NO_COLOR_OPTION)? && stdout_supports_color(py)?; let parsed: Bound<'py, PyAny> = parse_cli_invocation(py, argv.extract()?, use_color)?; - let args: Bound<'py, PyAny> = native_field(&parsed, "args")?; + let mut args: Bound<'py, PyAny> = native_field(&parsed, "args")?; if args.is_none() { let exit_code: Bound<'py, PyAny> = native_field(&parsed, "exit_code")?; return if exit_code.is_none() { @@ -814,7 +821,15 @@ fn parsed_invocation<'py>( Ok(exit_code) }; } - let namespace = args.getattr("__dict__")?.cast_into::()?; + let mut namespace: Bound<'py, PyDict> = namespace_of(&args)?; + let (given, job_argv) = job_settings_argv(argv, &namespace)?; + if let Some(job_argv) = job_argv { + args = native_field(&parse_cli_invocation(py, job_argv, use_color)?, "args")?; + if args.is_none() { + return exit(py, 1); + } + namespace = namespace_of(&args)?; + } let command: Option = namespace .get_item("command")? .map(|value| value.extract::>()) @@ -830,7 +845,9 @@ fn parsed_invocation<'py>( execute_parsed_invocation(args, handlers, identity, use_color) }); } - execute_parsed_invocation(args, handlers, identity, use_color) + with_given_settings(given, || { + execute_parsed_invocation(args, handlers, identity, use_color) + }) } /// Runtime stream routing must cover extension discovery, invocation scopes and their teardown. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouses.rs b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouses.rs index 52ccd14ab5..8a2596b594 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouses.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouses.rs @@ -6,8 +6,8 @@ use pyo3::prelude::{ use pyo3::types::{PyDict, PyDictMethods, PyMapping, PyString}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_config::command_connection::constants::{ - CLI_WAREHOUSE_SOURCE, CONNECTION_DEFAULT_WAREHOUSE_SOURCE, CONNECTION_WAREHOUSE_KEY, - CONNECTION_WAREHOUSE_SOURCE, + CLI_WAREHOUSE_SOURCE, CONFIGURED_WAREHOUSE_KEY, CONNECTION_DEFAULT_WAREHOUSE_SOURCE, + CONNECTION_WAREHOUSE_KEY, CONNECTION_WAREHOUSE_SOURCE, }; use sqlbuild_config::command_connection::main::authored_warehouse::authored_warehouse; use sqlbuild_config::command_connection::main::check_cli_warehouse::check_cli_warehouse; @@ -384,6 +384,10 @@ pub(crate) fn apply_warehouse<'py>( let applied: Bound<'py, PyDict> = PyDict::new(py); applied.update(&config.cast::()?.clone())?; applied.set_item(CONNECTION_WAREHOUSE_KEY, warehouse)?; + if scope.cli_warehouse.is_some() { + let own = resolve_warehouse(inputs, (&scope.group, None), (config, &scope.cli_vars))?; + applied.set_item(CONFIGURED_WAREHOUSE_KEY, own.0)?; + } Ok(applied.into_any()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs index c9b9c1fcc7..4ec8868c8b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs @@ -10,6 +10,11 @@ use sqlbuild_python_compile::compile_driver::main::stages::resolve_deferred_targ use sqlbuild_python_compile::compile_driver::main::setup::build_project_graph_from_compiled_project::build_project_graph_from_compiled_project; use sqlbuild_python_compile::compile_driver::main::setup::build_python_run_plan_outputs::build_python_run_plan_outputs; use sqlbuild_python_compile::compile_driver::main::setup::python_read_source_names_for::python_read_source_names_for; +use crate::bindings::_helpers::command_runtime::warehouses::command_scope; +use sqlbuild_python_compile::compile_driver::main::jobs::job_override_fields::job_override_fields; +use sqlbuild_python_compile::compile_driver::main::jobs::job_plan_fields::job_plan_fields; +use sqlbuild_python_compile::compile_driver::main::jobs::planned_job_settings::planned_job_settings; +use sqlbuild_python_compile::compile_driver::models::PlannedJobSettings; use sqlbuild_analysis::planner_scope::constants::SOURCE; use sqlbuild_python_base::boundary::main::keywords::keywords; use sqlbuild_python_base::records::main::construct::new_record_with::new_record_with; @@ -155,20 +160,22 @@ pub(crate) fn pipeline_result<'py>( ("sql_test_selection", sql_test_selection), ], )?; - let overrides: Bound<'py, PyAny> = new_record_with( - py, - RecordId::PlannerOverrides, - &option_fields( - options, - &[ - "cursor_overrides", - "full_refresh", - "reload_sources", - "max_microbatches", - "concurrency", - ], - )?, + let cli_warehouse: bool = command_scope(py)?.cli_warehouse.is_some(); + let job_settings: Option = + planned_job_settings((project, discovered), (options, cli_warehouse))?; + let mut override_fields: Vec<(&str, Bound<'py, PyAny>)> = option_fields( + options, + &[ + "cursor_overrides", + "full_refresh", + "reload_sources", + "max_microbatches", + "concurrency", + ], )?; + override_fields.extend(job_override_fields(py, job_settings.as_ref())?); + let overrides: Bound<'py, PyAny> = + new_record_with(py, RecordId::PlannerOverrides, &override_fields)?; let mut deferral_fields: Vec<(&str, Bound<'py, PyAny>)> = vec![ ("deferred_locations", deferred_locations), ("deferred_relations", deferred_relations), @@ -218,6 +225,7 @@ pub(crate) fn pipeline_result<'py>( ], )?), )?; + let (plan_output, job_fields) = job_plan_fields(job_settings.as_ref(), plan_output, options)?; let custom_materializations: Bound<'py, PyAny> = host_entry(py, UserCodeHost::LoadCustomMaterializations)? .call1((native_field(discovered, "materialization_files")?,))?; @@ -233,23 +241,21 @@ pub(crate) fn pipeline_result<'py>( ], )?), )?; - new_record_with( - py, - RecordId::CompilePipelineResult, - &[ - ("project", project.clone()), - ("plan_output", native_field(&python_outputs, "plan_output")?), - ("custom_materializations", custom_materializations), - ( - "python_node_names", - native_field(&python_outputs, "selected_python_node_names")?, - ), - ( - "python_plan_entries", - native_field(&python_outputs, "python_plan_entries")?, - ), - ], - ) + let mut result_fields: Vec<(&str, Bound<'py, PyAny>)> = vec![ + ("project", project.clone()), + ("plan_output", native_field(&python_outputs, "plan_output")?), + ("custom_materializations", custom_materializations), + ( + "python_node_names", + native_field(&python_outputs, "selected_python_node_names")?, + ), + ( + "python_plan_entries", + native_field(&python_outputs, "python_plan_entries")?, + ), + ]; + result_fields.extend(job_fields); + new_record_with(py, RecordId::CompilePipelineResult, &result_fields) } /// `(name, options.name)` for each of `names`. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/constants.rs index ef9d973459..10a6c67240 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/constants.rs @@ -22,16 +22,6 @@ pub(crate) const SOURCE_NODE_KIND: &str = "source"; /// DuckDB's message when another process, such as a running build, holds the database file. pub(crate) const DUCKDB_LOCK_CONFLICT: &str = "Conflicting lock is held"; -/// The build settings a job run's `--changed` reach reads, and the reach every job run uses. -pub(crate) const CHANGED_SETTING: &str = "changed"; -pub(crate) const REACH_SETTING: &str = "reach"; -/// A job run reaches its whole selection: `reach` only stops builds that are not a job's run. -pub(crate) const JOB_RUN_REACH: &str = "all"; -/// What a job run with `changed: True` counts when `[changes] counts` is not set: every change. -pub(crate) const JOB_RUN_COUNTS: [&str; 2] = ["definition", "source_data"]; - -/// A SQL unit test's published kind; a frozen run selects it back with `test:`. -pub(crate) const SQL_TEST_NODE_KIND: &str = "sql_test"; /// The setting-row field the published providers document leaves out: secret matching reads it, the UI does not. pub(crate) const SNAPSHOT_HIDDEN_SETTING: &str = "alias"; /// The runtime request key of a plan preview: `{"plan": {"argv", "copy"}}`. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs index f55b4f5f2f..8005156b99 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs @@ -1,6 +1,7 @@ //! Execute only the API's frozen request after compiling and verifying the unmodified baseline. use std::collections::BTreeMap; +use std::path::PathBuf; use std::sync::{Arc, mpsc}; use std::time::Duration; @@ -8,14 +9,11 @@ use pyo3::exceptions::{PyRuntimeError, PyValueError}; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::PyString; use serde_json::Value; -use sqlbuild_analysis::definition_changes::constants::PYTHON_NODE_KINDS; -use sqlbuild_cli::argv::main::command_settings_arguments::command_settings_arguments; -use sqlbuild_cli::argv::main::job_settings_arguments::job_settings_arguments; use sqlbuild_cli::dispatch::main::command_dispatch::command_dispatch; use sqlbuild_cli::dispatch::models::Dispatch; use sqlbuild_host::entries::models::ExecutorHost; use sqlbuild_orchestration::runs::models::{ - RunChange, RunCommand, RunKind, RunOutcome, RunRequest, RunWindow, + RunChange, RunCommand, RunKind, RunOutcome, RunRequest, }; use sqlbuild_orchestration::snapshots::main::reads::node_hashes::node_hashes; use sqlbuild_orchestration::snapshots::models::SnapshotInput; @@ -32,12 +30,7 @@ use crate::bindings::_helpers::compile_driver::rules_at_load::RulesAtLoadScope; use crate::bindings::_helpers::cost_report::deferred::collect_deferred_cost_after_finish; use crate::bindings::_helpers::freshness::freshness_command::run_freshness_command; use crate::bindings::_helpers::location_command::command::compile_snapshot; -use crate::bindings::_helpers::location_command::constants::CHANGED_SETTING; use crate::bindings::_helpers::location_command::constants::COMMAND_IMPLEMENTATIONS; -use crate::bindings::_helpers::location_command::constants::JOB_RUN_COUNTS; -use crate::bindings::_helpers::location_command::constants::JOB_RUN_REACH; -use crate::bindings::_helpers::location_command::constants::REACH_SETTING; -use crate::bindings::_helpers::location_command::constants::SQL_TEST_NODE_KIND; use crate::bindings::_helpers::query_command::project::ProjectAdapter; use crate::bindings::_helpers::seed_command::command::run_seed; use crate::bindings::_helpers::sql_test_command::test_command::run_test; @@ -49,11 +42,15 @@ use sqlbuild_python_base::native_host::main::host_entry::host_entry as entry; use sqlbuild_python_base::records::main::fields::field::field; use sqlbuild_python_base::records::main::fields::replaced::replaced; use sqlbuild_python_front::compile_attachment::main::build_effective_connection_config::build_effective_connection_config; -use sqlbuild_python_front::discovery::main::project::configured_changes::configured_changes; use sqlbuild_python_output::recall::main::write_stderr::write_stderr; use sqlbuild_python_output::recall::models::InvocationRecall; use sqlbuild_python_project::compiled_project::constants::SQL_TEST_KEY_PREFIX; use sqlbuild_python_project::compiled_views::main::project::sql_tests::sql_tests; +use sqlbuild_python_project::job_settings::constants::SQL_TEST_NODE_KIND; +use sqlbuild_python_project::job_settings::main::frozen_run_flags::frozen_run_flags; +use sqlbuild_python_project::job_settings::main::frozen_selectors::frozen_selectors; +use sqlbuild_python_project::job_settings::main::with_frozen_job::with_frozen_job; +use sqlbuild_python_project::job_settings::models::FrozenRunFlags; use sqlbuild_python_project::project_snapshot::models::NativeOrchestrationSnapshot; use sqlbuild_python_project::project_snapshot::models::NativeOrchestrationWorker; use sqlbuild_python_project::project_snapshot::models::WorkerBuild; @@ -127,65 +124,24 @@ fn upstream_nodes(snapshot: &NativeOrchestrationSnapshot) -> BTreeMap, - mut settings: BTreeMap, -) -> PyResult> { - let Some(changed) = settings.get(CHANGED_SETTING).cloned() else { - return Ok(settings); - }; - if changed == Value::Bool(false) { - return Ok(settings); - } - settings.insert(REACH_SETTING.to_owned(), Value::from(JOB_RUN_REACH)); - if changed == Value::Bool(true) && configured_changes(project_dir)?.counts.is_none() { - settings.insert( - CHANGED_SETTING.to_owned(), - Value::from(JOB_RUN_COUNTS.to_vec()), - ); - } - Ok(settings) -} +/// A job run's job, the selectors of its frozen nodes and its settings, as its build declares it. +type FrozenJob = (String, Vec, BTreeMap); -/// The frozen command's request and warehouse scope; build carries the verified inputs and compiled project in its request. +/// The frozen command's request, warehouse scope and, for a job's build, the frozen job it names. fn command_request<'py>( py: Python<'py>, discovered: &Bound<'py, PyAny>, arguments: &KeywordArguments<'py>, (worker, compiled): (&Bound<'py, NativeOrchestrationWorker>, &Bound<'py, PyAny>), -) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { +) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>, Option)> { let execution = worker.get().execution(py)?; let frozen = execution.claim.request; - let mut settings = frozen.settings; - if let Some(window) = frozen.window { - for key in [ - "start_cursor", - "end_cursor", - "start_cursor_ts", - "end_cursor_ts", - "start_cursor_int", - "end_cursor_int", - ] { - settings.remove(key); - } - let (kind, start, end) = match window { - RunWindow::Timestamp { start, end } => ("ts", start.to_rfc3339(), end.to_rfc3339()), - RunWindow::Integer { start, end } => ("int", start.to_string(), end.to_string()), - }; - settings.insert(format!("start_cursor_{kind}"), serde_json::json!(start)); - settings.insert(format!("end_cursor_{kind}"), serde_json::json!(end)); - } + let FrozenRunFlags { + settings, + flags, + job_build, + } = frozen_run_flags(&frozen, &arguments.value("project_dir")?)?; let command: RunCommand = frozen.command; - if frozen.job_key.is_some() { - settings = job_run_settings(&arguments.value("project_dir")?, settings)?; - } - let flags: Vec = if command == RunCommand::Build { - job_settings_arguments(&settings) - } else { - command_settings_arguments(command.as_str(), &settings) - } - .map_err(PyValueError::new_err)?; let mut argv: Vec> = std::iter::once(command.as_str().to_owned()) .chain(flags) .map(|argument| PyString::new(py, &argument)) @@ -198,21 +154,11 @@ fn command_request<'py>( PyString::new(py, "--no-color"), PyString::new(py, "--json"), ]); - for key in &frozen.selection { - if execution.skipped_nodes.contains_key(key) { - continue; + let selectors: Vec = frozen_selectors(&frozen, &execution.skipped_nodes)?; + if job_build.is_none() { + for selector in &selectors { + argv.extend([PyString::new(py, "--select"), PyString::new(py, selector)]); } - let (kind, name) = key - .split_once(':') - .ok_or_else(|| PyValueError::new_err("frozen run contains an invalid node key"))?; - let selector: String = if PYTHON_NODE_KINDS.contains(&kind) { - key.clone() - } else if kind == SQL_TEST_NODE_KIND { - format!("test:{name}") - } else { - name.to_owned() - }; - argv.extend([PyString::new(py, "--select"), PyString::new(py, &selector)]); } let parsed = parse_cli_invocation(py, argv, false)?; let args = field(&parsed, "args")?; @@ -239,7 +185,8 @@ fn command_request<'py>( ], )?; } - Ok((request, warehouse_scope(&args)?)) + let frozen_job: Option = job_build.map(|job| (job, selectors, settings)); + Ok((request, warehouse_scope(&args)?, frozen_job)) } /// Run the frozen command from the verified inputs, reporting its progress and outcomes to `worker`. @@ -396,11 +343,21 @@ pub(super) fn execute_worker<'py>( return Err(PyValueError::new_err(reason)); } record_compiled_vars(py, (&owner, &frozen), (arguments, input, &project))?; - let (request, scope) = command_request(py, &discovered, arguments, (&owner, compiled))?; + let (request, scope, frozen_job) = + command_request(py, &discovered, arguments, (&owner, compiled))?; write_stderr(py, "Run inputs verified. Executing frozen selection...\n")?; let _rules = RulesAtLoadScope::enter(); - within(&scope, |_| { - run_cancellable(py, &worker, (&request, &discovered)) + let directory: (PathBuf, Option) = ( + arguments.value("project_dir")?.extract()?, + Some(frozen.target.clone()), + ); + within(&scope, |_| match frozen_job { + Some((job, selectors, settings)) => { + with_frozen_job(directory, (&job, selectors, settings), || { + run_cancellable(py, &worker, (&request, &discovered)) + }) + } + None => run_cancellable(py, &worker, (&request, &discovered)), }) })(); let code = result.as_ref().copied().unwrap_or(1); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs b/crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs index d9fc02c4d7..56da07d274 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs @@ -340,6 +340,11 @@ fn compile_pipeline<'py>( "ignore_audit_failures", resolve_ignore_audit_failures(request, &invocation.discovered, &planned_target)?, )?; + options.set_item( + "cli_ignore_audit_failures", + native_field(request, "ignore_audit_failures")?, + )?; + options.set_item("job_settings", native_field(request, "job_settings")?)?; let pipeline_options: Bound<'py, PyAny> = new_record(py, RecordId::CompilePipelineOptions, (), Some(&options))?; let adapter: Bound<'py, PyAny> = native_field(&invocation.context, "adapter")?; diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/build.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/build.md index ab495b4a1d..30c069f116 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/build.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/build.md @@ -60,6 +60,7 @@ own option. `--version` works only directly after `sqb`, as in `sqb --version`: | `--no-sql-analysis` | Disable compile-time SQL analysis (`--no-sql-validation` is an alias) | | `--no-cache` | Bypass the reusable compile-analysis cache for this invocation | | `--full-refresh` | Drop and rebuild selected models unless a model sets `full_refresh false`; `full_refresh true` forces a model even without this flag | +| `--no-job-settings` | Ignore the settings of the jobs the selection names; see [Settings when a build names a job](../concepts/orchestration.md#settings-when-a-build-names-a-job) | | `--changed` | Build only the selected models and seeds that changed, and what they reach downstream; `--changed=definition,source_data` picks what counts. See [Building only what changed](#building-only-what-changed) | | `--reach` | With `--changed`: `job` stops at models another scheduled job refreshes, `all` builds everything downstream | | `--defer-to` | Resolve unselected model references against another target | diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/plan.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/plan.md index 66d69eead9..2cce0a9bf2 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/plan.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/plan.md @@ -41,6 +41,7 @@ Positional `SELECTOR` arguments work like `--select`, and both forms can be comb | `--ignore-audit-failures`, `--no-ignore-audit-failures` | Plan the build `sqb build` would run with the same flag; a plan whose audit failures are ignored starts with `Audit failures won't stop this build`. See [Ignoring audit failures](../concepts/audits.md#ignoring-audit-failures) | | `--selection-diagnostics` | Warn when a selected model will build on changed upstream models that are not selected; see [Selection and staleness](../concepts/planning/selection-and-staleness.md) | | `--full-refresh` | Plan a full rebuild for selected models unless a model sets `full_refresh false`; `full_refresh true` forces a model without this flag | +| `--no-job-settings` | Ignore the settings of the jobs the selection names; see [Settings when a build names a job](../concepts/orchestration.md#settings-when-a-build-names-a-job) | | `--changed` | Show what `sqb build --changed` would run, which nodes it holds back and why; takes `--changed=definition,source_data` and `--reach` like [`sqb build`](build.md#building-only-what-changed). In `--json`, the `changed` object lists `runs`, `held_back` and `unchanged` | | `--start-cursor` | Override the start cursor, typed from the selected incremental models' cursor type; fails with `S303` if the selection mixes timestamp and integer cursors. With no cursor model selected, an integer value is read as an integer cursor and anything else as an ISO timestamp | | `--end-cursor` | Override the end cursor, typed the same way as `--start-cursor` | diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md index 2e46bc706b..7e900811e0 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md @@ -780,6 +780,10 @@ warehouse_heavy = 1 `max_workers`, write claims, `overlap` and pools are compared on [Concurrency](../concepts/concurrency.md). +A job's run builds `--select job:`, as `sqb build --select job:` would on the command +line, so the job's `concurrency`, `full_refresh`, `run_tests` and `ignore_audit_failures` apply to +its nodes the same way. A group runs each of its jobs as its own run. + ## Run ad-hoc commands People with `run.adhoc` can run a `sqb` command on the current publication without declaring a diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/snowflake.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/snowflake.md index 85c44a8187..4ce91a45d2 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/snowflake.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/snowflake.md @@ -90,9 +90,10 @@ query = "ADHOC_WH" Each command uses the first of these that is set: 1. `--warehouse `, for that run only -2. The target's warehouse for the command's group -3. The connection's `warehouse` -4. The Snowflake user's default warehouse +2. For `sqb build`, the `warehouse` of a named job, for that job's nodes only; see below +3. The target's warehouse for the command's group +4. The connection's `warehouse` +5. The Snowflake user's default warehouse A location's syncs use the target's `sync` warehouse, else its `query` warehouse, else the connection's `warehouse`, so a sync never wakes the target's `build` warehouse just to read @@ -108,10 +109,44 @@ sync = "TINY_WH" Runs that a location executes, such as `sqb build`, use their own command's group. -Every connection a command opens uses the same warehouse. Tests and audits that run inside +Every connection a command opens starts on the same warehouse. Tests and audits that run inside `sqb build` use the `build` warehouse. Under `sqb dbt`, dbt's own nodes use the warehouse in their dbt profile. +### A job's own warehouse + +A job can run its nodes on a warehouse of its own: + +```toml +# orchestration/jobs/nightly_orders_job.toml +[settings] +warehouse = "ORDERS_WH" +``` + +A server run of the job, and a build whose named jobs all set the same warehouse, such as +`sqb build --select job:nightly_orders_job`, runs entirely on `ORDERS_WH`, as if passed +`--warehouse ORDERS_WH`: setup, planning and bookkeeping included. Nodes the build selects outside +the named jobs, such as upstream models in `--select +job:nightly_orders_job`, run there too. + +When the named jobs set different warehouses, or only some set one, each job's nodes run on its +own warehouse. Its models' audits, its SQL tests, microbatch batches, source loads and Python nodes +go with them, and loaders and Python code that open their own connections from the context's +`connection_config` get the job's warehouse there too. A SQL test runs on a job warehouse only +when every node it covers runs there. Everything else in the build, including nodes selected +without naming a job and the build's own bookkeeping, stays on the `build` warehouse. + +The connection running a node switches with `USE WAREHOUSE` only when the node needs a different +warehouse than the connection is on, and switches back before it runs a node outside the job. +Connections return to the `build` warehouse when the build ends. Without a configured warehouse, +SQLBuild reads the session's warehouse once before the first switch, so it can switch back. + +`--warehouse` puts the whole build on one warehouse and leaves the jobs' warehouses out. Two named +jobs that give one node different warehouses fail with `S027`; see +[Settings when a build names a job](../orchestration.md#settings-when-a-build-names-a-job). +A job's warehouse must be a Snowflake identifier like a group warehouse (`C261`), and it needs the +Snowflake adapter or one built on it (`C260`). Cost estimates read each query's warehouse, so a +build's cost names every warehouse it used. + A group warehouse wins over a connection `warehouse` from either file. To change one for yourself, set the group in `sqlbuild_local.toml`; the other group keeps its project value: diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/concurrency.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/concurrency.md index 957db31cb5..e7cc29868c 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/concurrency.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/concurrency.md @@ -46,7 +46,15 @@ concurrency = 2 ``` `--concurrency` on the command line wins over all of them; see -[Target concurrency](project-configuration.md#target-concurrency) for the full order. The job's page shows the value its runs use on the target and where it came from. +[Target concurrency](project-configuration.md#target-concurrency) for the full order. +A job's `concurrency` limits how many of that job's nodes build at once in a build that names the +job. Without `--concurrency`, the build may run as many nodes at once as the larger of the +configured limit and the named jobs' limits added together; jobs without a `concurrency` add +nothing. With a project limit of 1 and a job limit of 4, the build runs up to 4 nodes at once; with +a project limit of 16, the build allows 16 and the job's nodes still run at most 4 at once. When +one job is at its limit, a ready node of another job starts instead. Microbatch sub-work counts +against the job too. A server runs a job the same way. `--concurrency` on the command line sets +the build's limit and turns the per-job limits off; see [Settings when a build names a job](orchestration.md#settings-when-a-build-names-a-job). The job's page shows the value its runs use on the target and where it came from. ## Runs at once on a server: `max_workers` diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/dynamic.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/dynamic.md index 53cc1d2d88..4d3c304492 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/dynamic.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/dynamic.md @@ -31,7 +31,7 @@ for the grants and change tracking they need. |-------|---------|-------------| | `refresh (lag )` | none | How far behind its inputs the model may fall: a fixed duration such as `5m`, `1h` or `1d`, at least `1m` on Snowflake and no shorter than the lag of a dynamic model it reads. Without it, the model refreshes when a dynamic model that reads it refreshes | | `refresh_mode` | `incremental` | `incremental`, `full`, `auto` or `adaptive`. Snowflake rejects `incremental` for a query it can't refresh incrementally; use `full` for that query | -| `refresh_warehouse` | the connection's warehouse | Warehouse the scheduled refreshes run on | +| `refresh_warehouse` | the target's build warehouse | Warehouse the scheduled refreshes run on; a job's `warehouse` never sets it | | `refresh_on_build` | `false` | Refresh the model in every build that selects it, not only on its schedule | ## What the plan shows diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/orchestration.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/orchestration.md index 4b6cc3d451..eff41b9fd8 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/orchestration.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/orchestration.md @@ -98,6 +98,54 @@ cpus = 2 A job's `[settings]` cannot set `select` or `exclude`: the job's own selection is the run's selection. +### Settings when a build names a job + +A build or plan whose selection names a job, as `job:nightly_orders_job`, the bare name or a +`job:` pattern such as `job:nightly_*`, takes the job's `[settings]`. Selecting the same nodes some other way, such as by tag, takes none of them. +Naming a group names each of its jobs. The first value set wins: + +1. A flag on the command line, such as `--vars` or `--concurrency` +2. The named job's setting +3. The target's setting +4. The project's setting + +A command-line flag replaces every named job's setting rather than merging with it, so it also +settles named jobs that disagree: `--vars` replaces the job's whole `vars`, `--full-refresh` +rebuilds every selected model, and `--warehouse` puts every node on its warehouse. An option that rules another out counts as setting both, so `--reload` wins +over a job's `load_sources = false`. Any `--concurrency` sets the build's limit and turns off the +per-job limits. `SQLBUILD_CONCURRENCY` is not a command-line flag: it sets the build's limit as a +configured one does, and the named jobs' limits still apply. The job's settings are added before a `--` on the command line, so selectors after +it still name jobs. `--no-job-settings` leaves out every job's settings. + +Some settings apply to the job's own nodes only, so a build that names two jobs can treat their +nodes differently: + +| Setting | Applies to | +| --- | --- | +| `full_refresh` | The job's incremental models, rebuilt from scratch, and its functions, recreated | +| `run_tests` | The SQL unit tests of the job's models | +| `ignore_audit_failures` | The audits of the job's nodes, and singular audits that read them; a singular audit is ignored only when every node it reads ignores failures, by its job's setting or else the build's | +| `warehouse` | The Snowflake warehouse the job's nodes run on, Python nodes included; see [A job's own warehouse](adapters/snowflake.md#a-jobs-own-warehouse) | +| `concurrency` | How many of the job's nodes build at once, and a node in two named jobs fits within both. The build's own limit becomes the larger of the configured limit and the named jobs' limits added together | +| `[retry]` | The job's nodes | + +A job's settings reach only the nodes its own selection holds. `--select job:nightly_orders_job,orders` +gives them to `orders` alone, and `+job:nightly_orders_job` builds the job's upstream nodes without +them. + +Every other setting, such as `vars`, `changed` or `fail_fast`, applies to the whole build, so every +named job must set it the same way; a job that leaves it unset disagrees with one that sets it. +Pass the flag to settle a disagreement. When every named job sets the same `warehouse`, it is the +whole build's warehouse, as if passed as `--warehouse`; see +[A job's own warehouse](adapters/snowflake.md#a-jobs-own-warehouse). + +Each job in a group still runs as its own build, so a group's jobs must agree only on `vars` and +`changed`, which change the compiled SQL. A group whose jobs set either differently on a target +that declares them together fails to compile, with the error a build of the group would give. + +Two named jobs that give one node different values of the same setting fail the build with `S027`, +naming the node, both jobs and the setting. A setting only one of them sets is no conflict. + Select a pipeline by its folder or a tag rather than listing its nodes. Rule `SQBRORCHESTRATION101` reports a job or sensor `select` that names more than three nodes one by one; see [Orchestration Rules](rules/orchestration-rules.md). @@ -283,6 +331,7 @@ Every problem with an orchestration file is a `D017` error at its file, line and | `job nightly_orders_job resources: cpus must be a positive number` | Set `cpus` above zero | | `job 'nightly_orders_job' uses pool 'warehouse_hevy', which [pools] in sqlbuild_project.toml does not declare` | Fix the name, or declare the pool in [`[pools]`](project-configuration.md#pools) | | `job 'nightly_orders_job' takes 2 slots of pool 'supplier_api', more than its limit of 1 on target 'production'` | Take fewer slots, or raise the limit in `[targets.production.pools]` | +| `jobs orders_job and crm_job set vars differently ({"amount":"5"} and {"amount":"9"}); a build applies vars to everything it runs; group daily_job_group builds its jobs together` | Give the group's jobs the same `vars` and `changed`, or move one job to another group | | `sensor 'orders_sensor' is declared here, but importing the module does not define it` | Assign the sensor to a module-level name or list, not a dictionary or function body | | `group dependencies contain a cycle in job group daily_orders_job_group: orders_marts_job waits for staging_orders_job (model:orders reads model:stg_orders); staging_orders_job waits for orders_marts_job (model:stg_orders reads seed:regions)` | Move each node to one job, and give seeds and nodes that several members read their own job that runs first | @@ -292,6 +341,11 @@ Details: a job selector never imports a calendar or sensor module. - A job or group declared with `targets` is selectable only when the command's target is one of them, as on the server. +- Job settings that cannot apply together fail the build or plan with `S027`, such as `model:orders + is selected through jobs orders_job and finance_job, which set run_tests differently (false and + true)`. The help names the way out: make the jobs agree, set a node setting in only one of + them, or use `--no-job-settings`. It offers a flag only for the value the flag gives, such as + `--no-tests` to turn tests off for all; no flag turns them back on. - Selectors are checked for syntax at compile. A selector that names a node the project does not have fails when a location publishes, such as `job nightly_orders_job: unknown selector name 'order_total'`. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md index 539bf642a3..2de6859ef1 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md @@ -275,9 +275,12 @@ concurrency = 8 value set in this order: 1. `--concurrency` on the command line, for that invocation only -2. The selected target's `concurrency`, from `[targets.]` in either file -3. `[settings] concurrency` in `sqlbuild_local.toml` -4. `[settings] concurrency` in `sqlbuild_project.toml` (default `1`) +2. For `sqb build`, the named jobs' `concurrency` added together, when larger than the limit the + layers below give; each job's own limit still applies to its nodes. See + [Settings when a build names a job](orchestration.md#settings-when-a-build-names-a-job) +3. The selected target's `concurrency`, from `[targets.]` in either file +4. `[settings] concurrency` in `sqlbuild_local.toml` +5. `[settings] concurrency` in `sqlbuild_project.toml` (default `1`) The value must be an integer of at least `1`; anything else fails with `D001`. `sqb build --verbose` shows the effective limit and the setting it came from, for example diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/selectors.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/selectors.md index 9a25d769ef..e5af77df77 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/selectors.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/selectors.md @@ -108,6 +108,8 @@ sqb build --select "job:nightly_orders_job,tag:finance" ``` A job selects its own `select` minus its own `exclude`. A job group selects the union of its jobs. +Naming a job also applies its settings to its nodes; see +[Settings when a build names a job](orchestration.md#settings-when-a-build-names-a-job). A job or job group declared with `targets` is selectable only when the command's target is one of them, as on the server. Job selectors compose with `+`, `,` and `--exclude` like any other selector. A job's own selection diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 2626a77c8d..d3384c5a85 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -6583,6 +6583,7 @@ class BuildCommandRequest(NativeRecord): no_color: bool fail_fast: bool full_refresh: bool + job_settings: bool changed: bool changed_counts: tuple[str, ...] | None reach: str | None @@ -6628,6 +6629,7 @@ class BuildCommandRequest(NativeRecord): no_color: bool = False, fail_fast: bool = False, full_refresh: bool = False, + job_settings: bool = True, changed: bool = False, changed_counts: tuple[str, ...] | None = None, reach: str | None = None, @@ -6998,6 +7000,7 @@ class PlanCommandRequest(NativeRecord): cursor_overrides: CursorOverrides | None json_output: bool full_refresh: bool + job_settings: bool changed: bool changed_counts: tuple[str, ...] | None reach: str | None @@ -7025,6 +7028,7 @@ class PlanCommandRequest(NativeRecord): cursor_overrides: CursorOverrides | None = None, json_output: bool = False, full_refresh: bool = False, + job_settings: bool = True, changed: bool = False, changed_counts: tuple[str, ...] | None = None, reach: str | None = None, @@ -10259,6 +10263,8 @@ class CompilePipelineOptions(NativeRecord): source_lexical_syntax: SqlLexicalSyntax | None plan_from: str | None ignore_audit_failures: bool + cli_ignore_audit_failures: bool | None + job_settings: bool def __init__( self, selected_target: str | None = ..., @@ -10288,6 +10294,8 @@ class CompilePipelineOptions(NativeRecord): source_lexical_syntax: SqlLexicalSyntax | None = ..., plan_from: str | None = ..., ignore_audit_failures: bool = ..., + cli_ignore_audit_failures: bool | None = ..., + job_settings: bool = ..., ) -> None: ... class PythonPlanEntry(NativeRecord): @@ -10505,6 +10513,8 @@ class PlannerOverrides(NativeRecord): cursor_overrides: CursorOverrides | None full_refresh: bool + full_refresh_models: frozenset[str] + full_refresh_functions: frozenset[str] reload_sources: bool forced_stale_model_names: tuple[str, ...] external_blocked_model_names: tuple[str, ...] @@ -10514,6 +10524,8 @@ class PlannerOverrides(NativeRecord): self, cursor_overrides: CursorOverrides | None = ..., full_refresh: bool = ..., + full_refresh_models: frozenset[str] = ..., + full_refresh_functions: frozenset[str] = ..., reload_sources: bool = ..., forced_stale_model_names: tuple[str, ...] = ..., external_blocked_model_names: tuple[str, ...] = ..., @@ -15321,6 +15333,10 @@ class CompilePipelineResult(NativeRecord): custom_materializations: dict[str, Callable[..., Any]] python_node_names: frozenset[str] python_plan_entries: tuple[PythonPlanEntry, ...] + job_limits: dict[str, int] + node_jobs: dict[CompiledObjectKey, tuple[str, ...]] + node_warehouses: dict[CompiledObjectKey, str] + python_node_warehouses: dict[str, str] compile_seconds: float | None planning_seconds: float | None def __init__( @@ -15330,6 +15346,10 @@ class CompilePipelineResult(NativeRecord): custom_materializations: dict[str, Callable[..., Any]] = ..., python_node_names: frozenset[str] = ..., python_plan_entries: tuple[PythonPlanEntry, ...] = ..., + job_limits: dict[str, int] = ..., + node_jobs: dict[CompiledObjectKey, tuple[str, ...]] = ..., + node_warehouses: dict[CompiledObjectKey, str] = ..., + python_node_warehouses: dict[str, str] = ..., compile_seconds: float | None = ..., planning_seconds: float | None = ..., ) -> None: ... diff --git a/src/sqlbuild/adapter/contract/classes/strict_adapter.py b/src/sqlbuild/adapter/contract/classes/strict_adapter.py index 89c03cf168..6b3f4d5b98 100644 --- a/src/sqlbuild/adapter/contract/classes/strict_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/strict_adapter.py @@ -329,6 +329,20 @@ def session_warehouse(self, *, connection: Any) -> str | None: del connection return None + def use_session_warehouse(self, *, connection: Any, warehouse: str) -> None: + """Optional: run the session's next statements on `warehouse`, where the adapter has one.""" + + del connection, warehouse + raise AdapterUserError( + f"the '{self.adapter_name}' adapter has no warehouse to select", code="C260" + ) + + def current_session_warehouse(self, *, connection: Any) -> str | None: + """Optional: the session's warehouse, as an identifier `use_session_warehouse` accepts.""" + + del connection + return None + def render_create_dynamic( self, *, destination: str, sql: str, target_lag: str, refresh_mode: str, warehouse: str ) -> tuple[str, ...]: diff --git a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py index 6d62b24af0..5d4f757b97 100644 --- a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py +++ b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py @@ -200,6 +200,7 @@ _SHOW_OBJECT_RELATION_TYPES: dict[str, str] = {"TABLE": "base table", "VIEW": "view"} _COST_LOGIN_TIMEOUT_SECONDS: int = 5 _COST_NETWORK_TIMEOUT_SECONDS: int = 15 +_UNQUOTED_WAREHOUSE: re.Pattern[str] = re.compile(r"[A-Z_][A-Z0-9_$]*") _SHOW_COLUMN_KIND: str = "COLUMN" _SHOW_COLUMN_KINDS: frozenset[str] = frozenset({_SHOW_COLUMN_KIND, "VIRTUAL_COLUMN"}) _UNRESOLVED: object = object() @@ -239,7 +240,7 @@ class SnowflakeAdapter( max_identifier_length: ClassVar[int] = 255 supports_session_warehouse: ClassVar[bool] = True connection_routing_keys: ClassVar[frozenset[str]] = frozenset( - {"source", "profile", "target", "project_dir", "profiles_dir"} + {"source", "profile", "target", "project_dir", "profiles_dir", "configured_warehouse"} ) _snapshot_sql_dialect: ClassVar[SnapshotSqlDialect] = SnapshotSqlDialect( timestamp_type="TIMESTAMP", @@ -1323,6 +1324,20 @@ def render_current_utc_timestamp(self) -> str: def render_current_warehouse_user(self) -> str | None: return "CURRENT_USER()" + def use_session_warehouse(self, *, connection: Any, warehouse: str) -> None: + self.execute(connection=connection, sql=f"USE WAREHOUSE {warehouse}") + + def current_session_warehouse(self, *, connection: Any) -> str | None: + rows: tuple[tuple[object, ...], ...] = self.query( + connection=connection, sql="SELECT CURRENT_WAREHOUSE()", limit=None + ).rows + name: object = rows[0][0] if rows else None + if not isinstance(name, str) or not name: + return None + if _UNQUOTED_WAREHOUSE.fullmatch(name): + return name + return '"' + name.replace('"', '""') + '"' + def create_table_as( self, *, diff --git a/src/sqlbuild/executor/build/classes/build_scheduler.py b/src/sqlbuild/executor/build/classes/build_scheduler.py index 395f78de8b..a52bb4b1aa 100644 --- a/src/sqlbuild/executor/build/classes/build_scheduler.py +++ b/src/sqlbuild/executor/build/classes/build_scheduler.py @@ -115,6 +115,12 @@ ) from sqlbuild.executor.run.types import MicrobatchBatchExecutor from sqlbuild.executor.scheduling.classes.recovering_connection_pool import RecoveringConnectionPool +from sqlbuild.executor.scheduling.classes.session_warehouses import ( + SessionWarehouses, + configured_warehouse, + with_session_warehouse, +) +from sqlbuild.executor.scheduling.constants import CONFIGURED_WAREHOUSE_KEY from sqlbuild.executor.scheduling.main._run_worker import run_worker_with_completion from sqlbuild.executor.scheduling.types import ExecutionStatus from sqlbuild.executor.seed.constants import SEED_ENTRY_MISSING_CODE @@ -162,6 +168,20 @@ def _resource_result_failed(result: object) -> bool: return getattr(result, "status", None) == ExecutionStatus.FAILED +def _split_configured_warehouse( + connection_config: dict[str, object], +) -> tuple[dict[str, object], tuple[bool, object]]: + """The connection config without the recorded build warehouse, and whether it had one.""" + + return ( + {key: value for key, value in connection_config.items() if key != CONFIGURED_WAREHOUSE_KEY}, + ( + CONFIGURED_WAREHOUSE_KEY in connection_config, + connection_config.get(CONFIGURED_WAREHOUSE_KEY), + ), + ) + + class BuildScheduler: """DAG-aware scheduler that dispatches nodes as their dependencies complete.""" @@ -189,7 +209,9 @@ def __init__( entry.model_name: entry for entry in plan.table_type_entries } self._adapter: BaseAdapter = adapter - self._connection_config: dict[str, object] = connection_config + self._connection_config, self._configured_warehouse = _split_configured_warehouse( + connection_config + ) self._connections: tuple[Any, ...] = connections self._scheduler_connection: Any = scheduler_connection self._schema_prepared: bool = schema_prepared @@ -242,6 +264,15 @@ def __init__( callbacks.python_identity_recorder ) self._max_concurrency: int = len(connections) + self._init_job_limits(runtime) + self._node_warehouses: Mapping[CompiledObjectKey, str] = runtime.node_warehouses + self._session_warehouses: SessionWarehouses | None = ( + SessionWarehouses( + adapter=adapter, home=configured_warehouse(connection_config=connection_config) + ) + if runtime.node_warehouses + else None + ) self._blocked_keys: set[CompiledObjectKey] = set() self._unbuilt_keys: set[CompiledObjectKey] = set(initial_state.initial_failed_keys) self._completed_keys: set[CompiledObjectKey] = set(initial_state.precompleted_keys) @@ -305,6 +336,8 @@ def run( else: self._run_concurrent() finally: + if self._session_warehouses is not None: + self._session_warehouses.restore() self._close_replacement_connections() enforce_execution_deadline() @@ -468,19 +501,15 @@ def _run_concurrent(self) -> None: if self._stop and not self._in_flight: break - with self._microbatch_coordinator_lock: - queued_coordinators: int = sum( - 1 for key in tuple(self._ready) if self._is_concurrent_microbatch(key) - ) - self._microbatch_coordinator_demand = min( - self._max_concurrency, - len(self._microbatch_coordinators) + queued_coordinators, - ) + self._update_microbatch_coordinator_demand() while self._ready and len(self._in_flight) < self._max_concurrency: if self._stop: break - key: CompiledObjectKey = self._ready.popleft() + next_key: CompiledObjectKey | None = self._take_ready_key() + if next_key is None: + break + key: CompiledObjectKey = next_key if key.resource_type == CompiledResourceType.SOURCE: if key in self._blocked_keys: self._record_skipped(key) @@ -489,7 +518,7 @@ def _run_concurrent(self) -> None: if not self._pre_dispatch(key): self._mark_complete(key) continue - self._in_flight.add(key) + self._start_in_flight(key) pool.submit(copy_context().run, self._worker, key) continue if key in self._blocked_keys: @@ -497,7 +526,9 @@ def _run_concurrent(self) -> None: self._mark_complete(key) continue if self._is_concurrent_microbatch(key) and any( - not self._is_concurrent_microbatch(candidate) for candidate in self._ready + not self._is_concurrent_microbatch(candidate) + and self._has_job_room(candidate) + for candidate in self._ready ): self._ready.append(key) continue @@ -513,7 +544,7 @@ def _run_concurrent(self) -> None: self._ready.appendleft(key) break self._microbatch_coordinators.add(key) - self._in_flight.add(key) + self._start_in_flight(key) pool.submit( copy_context().run, self._concurrent_microbatch_worker, @@ -521,7 +552,7 @@ def _run_concurrent(self) -> None: pool=pool, ) continue - self._in_flight.add(key) + self._start_in_flight(key) pool.submit(copy_context().run, self._worker, key) self._report_scheduler_state() @@ -529,7 +560,7 @@ def _run_concurrent(self) -> None: break completion: NodeCompletion = self._completion_queue.get() - self._in_flight.discard(completion.key) + self._finish_in_flight(completion.key) if completion.error is not None: if escaped is None: escaped = completion.error @@ -541,6 +572,74 @@ def _run_concurrent(self) -> None: if escaped is not None: raise escaped + def _update_microbatch_coordinator_demand(self) -> None: + """Count the running coordinators and the queued ones whose jobs would let them start.""" + + with self._microbatch_coordinator_lock: + queued_coordinators: int = sum( + 1 + for key in tuple(self._ready) + if self._is_concurrent_microbatch(key) and self._has_job_room(key) + ) + self._microbatch_coordinator_demand = min( + self._max_concurrency, + len(self._microbatch_coordinators) + queued_coordinators, + ) + + def _init_job_limits(self, runtime: BuildRuntimeParams) -> None: + """Job limits, each node's jobs, and the lock coordinator threads take to count them.""" + + self._job_limits: Mapping[str, int] = runtime.job_limits + self._node_jobs: Mapping[CompiledObjectKey, tuple[str, ...]] = runtime.node_jobs + self._job_running: dict[str, int] = {} + self._job_lock: threading.Lock = threading.Lock() + + def _take_ready_key(self) -> CompiledObjectKey | None: + """Remove and return the first ready node whose named jobs all have a free slot.""" + + if not self._job_limits or self._has_job_room(self._ready[0]): + return self._ready.popleft() + index: int + key: CompiledObjectKey + with self._job_lock: + for index, key in enumerate(self._ready): + if key in self._blocked_keys or self._jobs_have_room(self._node_jobs.get(key, ())): + del self._ready[index] + return key + return None + + def _has_job_room(self, key: CompiledObjectKey) -> bool: + """Whether each job limiting `key` has a free slot; a node to skip never waits.""" + + if key in self._blocked_keys: + return True + with self._job_lock: + return self._jobs_have_room(self._node_jobs.get(key, ())) + + def _jobs_have_room(self, jobs: tuple[str, ...]) -> bool: + """Whether each of `jobs` runs fewer nodes than its limit; the caller holds `_job_lock`.""" + + return all(self._job_running.get(job, 0) < self._job_limits[job] for job in jobs) + + def _add_job_running(self, *, jobs: tuple[str, ...], change: int) -> None: + """Count `change` more running nodes in each of `jobs`; the caller holds `_job_lock`.""" + + job: str + for job in jobs: + self._job_running[job] = self._job_running.get(job, 0) + change + + def _start_in_flight(self, key: CompiledObjectKey) -> None: + self._in_flight.add(key) + with self._job_lock: + self._add_job_running(jobs=self._node_jobs.get(key, ()), change=1) + + def _finish_in_flight(self, key: CompiledObjectKey) -> None: + if key not in self._in_flight: + return + self._in_flight.discard(key) + with self._job_lock: + self._add_job_running(jobs=self._node_jobs.get(key, ()), change=-1) + def _report_scheduler_state(self) -> None: callback: Callable[[SchedulerState], None] | None = self._callbacks.on_scheduler_state if callback is None: @@ -609,12 +708,17 @@ def _execute_concurrent_microbatch_node( ledger_path=(self._runtime_dir / "executions" / self._run_id / "statements.jsonl"), on_statement_complete=self._on_statement_complete, ): + failure: str | None = self._switch_failure(key=key, connection=connection) + if failure is not None: + return self._failed_model_result(key=key, error_message=failure) return self._execute_model_node( key=key, connection=connection, microbatch_batch_runner=lambda batches, concurrency, execute: ( self._run_microbatch_subwork( pool=pool, + jobs=self._node_jobs.get(key, ()), + warehouse=self._node_warehouses.get(key), coordinator_connection=connection, batches=batches, concurrency=concurrency, @@ -630,6 +734,8 @@ def _run_microbatch_subwork( self, *, pool: ThreadPoolExecutor, + jobs: tuple[str, ...], + warehouse: str | None = None, coordinator_connection: Any, batches: tuple[BatchWindow, ...], concurrency: int, @@ -652,12 +758,14 @@ def submit_one() -> tuple[Future[MicrobatchPhaseOutcome], BatchWindow]: pool.submit( copy_context().run, self._run_microbatch_batch_worker, + jobs=jobs, + warehouse=warehouse, batch=batch, execute=execute, ), ) except BaseException: - self._release_microbatch_subworker() + self._release_microbatch_subworker(jobs) raise return future, batch @@ -666,7 +774,7 @@ def submit_one() -> tuple[Future[MicrobatchPhaseOutcome], BatchWindow]: while ( len(pending_batches) > 1 and len(in_flight) < model_subworker_limit - and self._reserve_microbatch_subworker() + and self._reserve_microbatch_subworker(jobs) ): submitted_future, submitted_batch = submit_one() in_flight[submitted_future] = submitted_batch @@ -678,7 +786,7 @@ def submit_one() -> tuple[Future[MicrobatchPhaseOutcome], BatchWindow]: failed: bool = coordinator_result.failure is not None if failed: for future in in_flight: - self._cancel_microbatch_subworker(future=future) + self._cancel_microbatch_subworker(future=future, jobs=jobs) first_exception: BaseException | None = None while in_flight: completed, _ = wait(tuple(in_flight), return_when=FIRST_COMPLETED) @@ -693,18 +801,18 @@ def submit_one() -> tuple[Future[MicrobatchPhaseOutcome], BatchWindow]: first_exception = exc failed = True for sibling in in_flight: - self._cancel_microbatch_subworker(future=sibling) + self._cancel_microbatch_subworker(future=sibling, jobs=jobs) continue results[batch.index] = outcome if outcome.failure is not None: failed = True for sibling in in_flight: - self._cancel_microbatch_subworker(future=sibling) + self._cancel_microbatch_subworker(future=sibling, jobs=jobs) while ( not failed and pending_batches and len(in_flight) < model_subworker_limit - and self._reserve_microbatch_subworker() + and self._reserve_microbatch_subworker(jobs) ): submitted_future, submitted_batch = submit_one() in_flight[submitted_future] = submitted_batch @@ -720,19 +828,25 @@ def submit_one() -> tuple[Future[MicrobatchPhaseOutcome], BatchWindow]: def _run_microbatch_batch_worker( self, *, + jobs: tuple[str, ...], + warehouse: str | None = None, batch: BatchWindow, execute: MicrobatchBatchExecutor, ) -> MicrobatchPhaseOutcome: connection: Any = self._connection_pool.get() try: + if self._session_warehouses is not None: + self._session_warehouses.use(connection=connection, warehouse=warehouse) return execute(batch, connection) finally: try: self._connection_pool.put(connection) finally: - self._release_microbatch_subworker() + self._release_microbatch_subworker(jobs) + + def _reserve_microbatch_subworker(self, jobs: tuple[str, ...]) -> bool: + """Take a sub-worker slot, and one in each of its coordinator's jobs, if all are free.""" - def _reserve_microbatch_subworker(self) -> bool: with self._microbatch_coordinator_lock: if ( max( @@ -743,16 +857,24 @@ def _reserve_microbatch_subworker(self) -> bool: >= self._max_concurrency ): return False + with self._job_lock: + if not self._jobs_have_room(jobs): + return False + self._add_job_running(jobs=jobs, change=1) self._microbatch_subworkers += 1 return True - def _release_microbatch_subworker(self) -> None: + def _release_microbatch_subworker(self, jobs: tuple[str, ...]) -> None: with self._microbatch_coordinator_lock: self._microbatch_subworkers -= 1 + with self._job_lock: + self._add_job_running(jobs=jobs, change=-1) - def _cancel_microbatch_subworker(self, *, future: Future[MicrobatchPhaseOutcome]) -> None: + def _cancel_microbatch_subworker( + self, *, future: Future[MicrobatchPhaseOutcome], jobs: tuple[str, ...] + ) -> None: if not future.cancelled() and future.cancel(): - self._release_microbatch_subworker() + self._release_microbatch_subworker(jobs) def _pre_dispatch(self, key: CompiledObjectKey) -> bool: """Run pre-dispatch checks. Returns False if the node should be skipped.""" @@ -945,6 +1067,16 @@ def _execute_node_with_cost_context( ): """Execute a single node. Runs in a worker thread (or scheduler thread for serial).""" + failure: ( + ModelExecutionResult + | SeedExecutionResult + | FunctionExecutionResult + | SqlTestExecutionResult + | LoadExecutionResult + | None + ) = self._enter_node_warehouse(key=key, connection=connection) + if failure is not None: + return failure if key.resource_type == CompiledResourceType.SEED: return self._execute_seed_node(key=key, connection=connection) if key.resource_type == CompiledResourceType.SOURCE: @@ -962,6 +1094,104 @@ def _execute_node_with_cost_context( error_message=f"unknown executable resource type '{key.resource_type}'", ) + def _build_warehouse(self, connection: Any) -> str | None: + """The target's build warehouse, whichever job's or `--warehouse` `connection` is on now.""" + + overridden, configured = self._configured_warehouse + if overridden: + return configured if isinstance(configured, str) else None + if self._session_warehouses is None: + return self._adapter.session_warehouse(connection=connection) + return self._session_warehouses.build_warehouse(connection=connection) + + def _switch_failure(self, *, key: CompiledObjectKey, connection: Any) -> str | None: + """Put `connection` on the node's job warehouse, or the build's; why it could not.""" + + if self._session_warehouses is None: + return None + warehouse: str | None = self._node_warehouses.get(key) + try: + self._session_warehouses.use(connection=connection, warehouse=warehouse) + except Exception as error: # noqa: BLE001 - the node fails with the reason + return ( + f"cannot run on warehouse {warehouse}: {error}" + if warehouse is not None + else f"cannot return to the build's warehouse: {error}" + ) + return None + + def _enter_node_warehouse( + self, *, key: CompiledObjectKey, connection: Any + ) -> ( + ModelExecutionResult + | SeedExecutionResult + | FunctionExecutionResult + | SqlTestExecutionResult + | LoadExecutionResult + | None + ): + """Put `connection` on the node's warehouse; the node's own kind of failure if not.""" + + failure: str | None = self._switch_failure(key=key, connection=connection) + if failure is None: + return None + return self._failed_node_result(key=key, error_message=failure) + + def _failed_model_result( + self, *, key: CompiledObjectKey, error_message: str + ) -> ModelExecutionResult: + return ModelExecutionResult( + model_name=key.name, + status=ExecutionStatus.FAILED, + error_code=BUILD_WORKER_FAILED_CODE, + error_message=error_message, + ) + + def _failed_node_result( + self, *, key: CompiledObjectKey, error_message: str + ) -> ( + ModelExecutionResult + | SeedExecutionResult + | FunctionExecutionResult + | SqlTestExecutionResult + | LoadExecutionResult + ): + """The failed result of the node's own kind, so its kind's failure handling applies.""" + + if key.resource_type == CompiledResourceType.SEED: + return SeedExecutionResult( + seed_name=key.name, + status=ExecutionStatus.FAILED, + error_code=BUILD_WORKER_FAILED_CODE, + error_message=error_message, + ) + if key.resource_type == CompiledResourceType.SOURCE: + source: SourceEntry = self._plan.source_map[key.name] + return LoadExecutionResult( + source_name=source.name, + loader_name=source.loader or "", + status=ExecutionStatus.FAILED, + target=source.table or source.name, + resource_kind=load_resource_kind(source), + error_message=error_message, + ) + if key.resource_type in {CompiledResourceType.UDF, CompiledResourceType.TABLE_FN}: + return FunctionExecutionResult( + function_name=key.name, + status=ExecutionStatus.FAILED, + function_kind=str(key.resource_type), + error_code=BUILD_WORKER_FAILED_CODE, + error_message=error_message, + ) + if key.resource_type == CompiledResourceType.SQL_TEST: + return SqlTestExecutionResult( + test_name=key.name, + outcome=SqlTestOutcome.ERROR, + error_code=BUILD_WORKER_FAILED_CODE, + error_message=error_message, + ) + return self._failed_model_result(key=key, error_message=error_message) + def _execute_source_node( self, *, key: CompiledObjectKey, connection: Any ) -> LoadExecutionResult: @@ -977,7 +1207,10 @@ def _execute_source_node( loader_functions_by_name=self._loader_functions_by_name, loader_ref_entries=self._loader_ref_entries, adapter=self._adapter, - connection_config=self._connection_config, + connection_config=with_session_warehouse( + connection_config=self._connection_config, + warehouse=self._node_warehouses.get(key), + ), connection=connection, runtime=self._runtime, callbacks=self._callbacks, @@ -1257,6 +1490,7 @@ def _execute_model_node( bound_views=bound_views, fingerprint_writes=self._fingerprint_writes, dynamic_hold=self._runtime.dynamic_hold, + build_warehouse=self._build_warehouse, ), promotion_mode=self._promotion_mode, snapshots=self._snapshots, diff --git a/src/sqlbuild/executor/build/models.py b/src/sqlbuild/executor/build/models.py index d245ce43a3..0f3ede6d6a 100644 --- a/src/sqlbuild/executor/build/models.py +++ b/src/sqlbuild/executor/build/models.py @@ -107,6 +107,9 @@ class BuildRuntimeParams: allow_ignore_dynamic_grace: bool = False ignore_dynamic_grace: bool = False dynamic_hold: DynamicHold | None = None + job_limits: Mapping[str, int] = field(default_factory=dict) + node_jobs: Mapping[CompiledObjectKey, tuple[str, ...]] = field(default_factory=dict) + node_warehouses: Mapping[CompiledObjectKey, str] = field(default_factory=dict) @dataclass(frozen=True) diff --git a/src/sqlbuild/executor/python_nodes/_helpers/execution.py b/src/sqlbuild/executor/python_nodes/_helpers/execution.py index 4501569516..aa979c659d 100644 --- a/src/sqlbuild/executor/python_nodes/_helpers/execution.py +++ b/src/sqlbuild/executor/python_nodes/_helpers/execution.py @@ -19,6 +19,7 @@ from sqlbuild.executor.node_results.main._direct_store import build_direct_node_result_store from sqlbuild.executor.node_results.models import NodeResultRecord from sqlbuild.executor.node_results.types import NodeResultStatus +from sqlbuild.executor.python_nodes._helpers.node_warehouse import node_runtime from sqlbuild.executor.python_nodes._helpers.relation_guard import build_python_node_relation_guard from sqlbuild.executor.python_nodes._helpers.results import ( build_python_node_failure_result, @@ -180,6 +181,16 @@ def _execute_ready_node_in_cost_scope( ) _persist_python_node_result(result_store=result_store, result=result, run_id=run_id) return result, None + runtime, warehouse_failure = node_runtime(runtime=runtime, name=node.name) + if warehouse_failure is not None: + result = PythonNodeExecutionResult( + node_name=node.name, + kind=node_kind, + status=PythonNodeStatus.FAILED, + error_message=warehouse_failure, + ) + _persist_python_node_result(result_store=result_store, result=result, run_id=run_id) + return result, None warnings: list[str] = [] context: TaskContext | AssetContext = _build_context( node=node, diff --git a/src/sqlbuild/executor/python_nodes/_helpers/ingress_execution.py b/src/sqlbuild/executor/python_nodes/_helpers/ingress_execution.py index 383e915f9e..82e990217e 100644 --- a/src/sqlbuild/executor/python_nodes/_helpers/ingress_execution.py +++ b/src/sqlbuild/executor/python_nodes/_helpers/ingress_execution.py @@ -31,6 +31,7 @@ try_write_python_node_identity_fingerprint, ) from sqlbuild.executor.python_nodes._helpers.lifecycle_nodes import build_ingress_lifecycle_nodes +from sqlbuild.executor.python_nodes._helpers.node_warehouse import node_runtime from sqlbuild.executor.python_nodes.classes.ingress_results import IngressResultAccumulator from sqlbuild.executor.python_nodes.main._ready import run_ready_python_node from sqlbuild.executor.python_nodes.models import ( @@ -275,35 +276,47 @@ def _execute_ingress_loader( resource_name=loader.name, phase="load", ): - result: LoadExecutionResult = execute_source_load( - source_entry=source_entry, - loader_function=loader, - adapter=runtime.adapter, - connection_config=runtime.connection_config, - connection=runtime.connection, - runtime=LoadRuntimeParams( - run_id=runtime.run_id, - target=runtime.target, - target_config=runtime.target_config, - default_database=runtime.default_database, - default_schema=runtime.default_schema, - vars=runtime.vars, - is_reload=runtime.is_reload, - start_cursor_ts=runtime.start_cursor_ts, - end_cursor_ts=runtime.end_cursor_ts, - start_cursor_int=runtime.start_cursor_int, - end_cursor_int=runtime.end_cursor_int, - use_color=callbacks.use_color, - providers=runtime.providers, - result_store=runtime.result_store, - project_relations=runtime.project_relations, - ), - statement_recorder=StatementRecorder(), - loader_ref_entries=_loader_ref_entries( - loader_by_name=loader_by_name, - source_by_loader_name=source_by_loader_name, - ), - source_ref_entries=source_map, + runtime, warehouse_failure = node_runtime(runtime=runtime, name=node.name) + result: LoadExecutionResult = ( + LoadExecutionResult( + source_name=source_entry.name, + loader_name=loader.name, + status=ExecutionStatus.FAILED, + target=source_entry.table or source_entry.name, + resource_kind=resource_kind, + error_message=warehouse_failure, + ) + if warehouse_failure is not None + else execute_source_load( + source_entry=source_entry, + loader_function=loader, + adapter=runtime.adapter, + connection_config=runtime.connection_config, + connection=runtime.connection, + runtime=LoadRuntimeParams( + run_id=runtime.run_id, + target=runtime.target, + target_config=runtime.target_config, + default_database=runtime.default_database, + default_schema=runtime.default_schema, + vars=runtime.vars, + is_reload=runtime.is_reload, + start_cursor_ts=runtime.start_cursor_ts, + end_cursor_ts=runtime.end_cursor_ts, + start_cursor_int=runtime.start_cursor_int, + end_cursor_int=runtime.end_cursor_int, + use_color=callbacks.use_color, + providers=runtime.providers, + result_store=runtime.result_store, + project_relations=runtime.project_relations, + ), + statement_recorder=StatementRecorder(), + loader_ref_entries=_loader_ref_entries( + loader_by_name=loader_by_name, + source_by_loader_name=source_by_loader_name, + ), + source_ref_entries=source_map, + ) ) results.record_load_result(name=node.name, result=result) persist_loader_result( diff --git a/src/sqlbuild/executor/python_nodes/_helpers/node_warehouse.py b/src/sqlbuild/executor/python_nodes/_helpers/node_warehouse.py new file mode 100644 index 0000000000..8d6da355d6 --- /dev/null +++ b/src/sqlbuild/executor/python_nodes/_helpers/node_warehouse.py @@ -0,0 +1,39 @@ +"""The runtime one Python node runs with: on its job's warehouse, or on the build's.""" + +from __future__ import annotations + +from dataclasses import replace + +from sqlbuild.executor.python_nodes.models import PythonNodeRuntime +from sqlbuild.executor.scheduling.classes.session_warehouses import ( + SessionWarehouses, + with_session_warehouse, +) + + +def node_runtime(*, runtime: PythonNodeRuntime, name: str) -> tuple[PythonNodeRuntime, str | None]: + """The runtime of node `name` on its warehouse, and why its connection cannot move there.""" + + session_warehouses: SessionWarehouses | None = runtime.session_warehouses + if session_warehouses is None: + return runtime, None + warehouse: str | None = runtime.node_warehouses.get(name) + try: + session_warehouses.use(connection=runtime.connection, warehouse=warehouse) + except Exception as error: # noqa: BLE001 - the node fails with the reason + return runtime, ( + f"cannot run on warehouse {warehouse}: {error}" + if warehouse is not None + else f"cannot return to the build's warehouse: {error}" + ) + if warehouse is None: + return runtime, None + return ( + replace( + runtime, + connection_config=with_session_warehouse( + connection_config=runtime.connection_config, warehouse=warehouse + ), + ), + None, + ) diff --git a/src/sqlbuild/executor/python_nodes/_helpers/python_checks.py b/src/sqlbuild/executor/python_nodes/_helpers/python_checks.py index 70715affd9..472ed512aa 100644 --- a/src/sqlbuild/executor/python_nodes/_helpers/python_checks.py +++ b/src/sqlbuild/executor/python_nodes/_helpers/python_checks.py @@ -24,6 +24,7 @@ from sqlbuild.executor.python_nodes._helpers.fingerprinting import ( try_write_python_node_identity_fingerprint, ) +from sqlbuild.executor.python_nodes._helpers.node_warehouse import node_runtime from sqlbuild.executor.python_nodes._helpers.relation_guard import build_python_node_relation_guard from sqlbuild.executor.python_nodes._helpers.results import normalize_python_check_return from sqlbuild.executor.python_nodes.models import ( @@ -134,6 +135,18 @@ def _execute_one_python_check( result_store: Any | None, callbacks: PythonCheckCallbacks, ) -> PythonCheckExecutionResult: + if blocked is None: + runtime, warehouse_failure = node_runtime(runtime=runtime, name=check_function.name) + blocked = ( + None + if warehouse_failure is None + else PythonCheckExecutionResult( + node_name=check_function.name, + passed=False, + severity=PythonCheckSeverity.ERROR, + error_message=warehouse_failure, + ) + ) adapter: BaseAdapter = runtime.adapter providers: ProviderContainer | None = runtime.providers with ( diff --git a/src/sqlbuild/executor/python_nodes/models.py b/src/sqlbuild/executor/python_nodes/models.py index 661af2edc7..d1fbbbec14 100644 --- a/src/sqlbuild/executor/python_nodes/models.py +++ b/src/sqlbuild/executor/python_nodes/models.py @@ -3,7 +3,7 @@ from __future__ import annotations import logging -from collections.abc import Callable +from collections.abc import Callable, Mapping from dataclasses import dataclass, field from datetime import datetime from typing import Any @@ -28,6 +28,10 @@ PYTHON_NODE_RELATION_QUALIFIER_SEPARATOR, ) from sqlbuild.executor.python_nodes.types import PythonIdentityRecorder +from sqlbuild.executor.scheduling.classes.session_warehouses import ( + SessionWarehouses, + configured_warehouse, +) from sqlbuild.provider.main.runtime import ProviderContainer, _empty_provider_container from sqlbuild.python_nodes.models import SqlResourceRef from sqlbuild.python_nodes.types import PythonCheckSeverity @@ -147,6 +151,19 @@ class PythonNodeRuntime: providers: ProviderContainer | None = None result_store: Any | None = None persist_node_results: bool = True + node_warehouses: Mapping[str, str] = field(default_factory=dict) + session_warehouses: SessionWarehouses | None = None + + def __post_init__(self) -> None: + if self.node_warehouses and self.session_warehouses is None: + object.__setattr__( + self, + "session_warehouses", + SessionWarehouses( + adapter=self.adapter, + home=configured_warehouse(connection_config=self.connection_config), + ), + ) @property def resolved_relation_targets(self) -> dict[SqlResourceRef, str]: diff --git a/src/sqlbuild/executor/run/_helpers/materializations/dynamic.py b/src/sqlbuild/executor/run/_helpers/materializations/dynamic.py index 7d104dcbe9..2f1b513e51 100644 --- a/src/sqlbuild/executor/run/_helpers/materializations/dynamic.py +++ b/src/sqlbuild/executor/run/_helpers/materializations/dynamic.py @@ -77,14 +77,18 @@ def _create( entry: ModelPlanEntry = context.entry adapter: BaseAdapter = context.adapter connection: Any = context.connection - warehouse: str | None = entry.refresh_warehouse or adapter.session_warehouse( - connection=connection + build_warehouse: Callable[[Any], str | None] = context.build_warehouse or ( + lambda session: adapter.session_warehouse(connection=session) ) + warehouse: str | None = entry.refresh_warehouse or build_warehouse(connection) if warehouse is None: raise AdapterUserError( f"dynamic model '{entry.name}' has no warehouse to refresh it", code=_NO_WAREHOUSE_CODE, - help="set refresh_warehouse on the model, or a warehouse on the target's connection", + help=( + "set refresh_warehouse on the model, or the target's build warehouse; a job's " + "warehouse never refreshes a dynamic model" + ), ) statements: tuple[str, ...] = adapter.render_create_dynamic( destination=destination, diff --git a/src/sqlbuild/executor/run/models.py b/src/sqlbuild/executor/run/models.py index fc47787c66..cbcc004caa 100644 --- a/src/sqlbuild/executor/run/models.py +++ b/src/sqlbuild/executor/run/models.py @@ -335,6 +335,7 @@ class ModelMaterializationContext: bound_views: BoundViewGuard = field(default_factory=BoundViewGuard) fingerprint_writes: FingerprintWrites | None = None dynamic_hold: DynamicHold | None = None + build_warehouse: Callable[[Any], str | None] | None = None @dataclass(frozen=True, kw_only=True, init=False) diff --git a/src/sqlbuild/executor/scheduling/classes/session_warehouses.py b/src/sqlbuild/executor/scheduling/classes/session_warehouses.py new file mode 100644 index 0000000000..98cdaefd58 --- /dev/null +++ b/src/sqlbuild/executor/scheduling/classes/session_warehouses.py @@ -0,0 +1,133 @@ +"""The warehouse each build connection runs on, switched only when a node needs another.""" + +from __future__ import annotations + +import logging +import threading +from collections.abc import Mapping +from typing import Any + +from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter +from sqlbuild.errors.contracts.exceptions import ExecutorInputError + +_CONNECTION_WAREHOUSE_KEY: str = "warehouse" +_UNSET_PREFIX: str = " str | None: + """The warehouse a connection opened with `connection_config` starts on, if it names one.""" + + value: object | None = connection_config.get(_CONNECTION_WAREHOUSE_KEY) + if not isinstance(value, str): + return None + stripped: str = value.strip() + if not stripped or stripped.startswith(_UNSET_PREFIX): + return None + return stripped + + +def with_session_warehouse( + *, connection_config: dict[str, object], warehouse: str | None +) -> dict[str, object]: + """`connection_config` for code that opens its own connections on `warehouse`.""" + + if warehouse is None: + return connection_config + return {**connection_config, _CONNECTION_WAREHOUSE_KEY: warehouse} + + +def _same_warehouse(*, first: str, second: str) -> bool: + """Unquoted warehouse names match case-insensitively, as Snowflake resolves them.""" + + if first.startswith('"') or second.startswith('"'): + return first == second + return first.upper() == second.upper() + + +class SessionWarehouses: + """Moves a connection to a node's warehouse, and back to the build's, only when it changes.""" + + def __init__(self, *, adapter: BaseAdapter, home: str | None) -> None: + self._adapter: BaseAdapter = adapter + self._home: str | None = home + self._home_known: bool = home is not None + self._lock: threading.Lock = threading.Lock() + self._away: dict[int, tuple[Any, str]] = {} + + def use(self, *, connection: Any, warehouse: str | None) -> None: + """Put `connection` on `warehouse`, or on the build's warehouse when it is None.""" + + with self._lock: + entry: tuple[Any, str] | None = self._away.get(id(connection)) + current: str | None = None if entry is None or entry[0] is not connection else entry[1] + target: str | None = warehouse + if ( + target is not None + and self._home is not None + and _same_warehouse(first=target, second=self._home) + ): + target = None + if current == target or ( + current is not None + and target is not None + and _same_warehouse(first=current, second=target) + ): + return + if target is None: + self._adapter.use_session_warehouse(connection=connection, warehouse=self._return_to()) + with self._lock: + self._away.pop(id(connection), None) + return + if current is None: + self._learn_home(connection) + if self._home is not None and _same_warehouse(first=target, second=self._home): + return + self._adapter.use_session_warehouse(connection=connection, warehouse=target) + with self._lock: + self._away[id(connection)] = (connection, target) + + def build_warehouse(self, *, connection: Any) -> str | None: + """The build's own warehouse: the configured one, else the one connections start on.""" + + with self._lock: + entry: tuple[Any, str] | None = self._away.get(id(connection)) + if entry is None or entry[0] is not connection: + self._learn_home(connection) + return self._home + + def restore(self) -> None: + """Return every connection still on a node's warehouse to the build's warehouse.""" + + with self._lock: + away: list[tuple[Any, str]] = list(self._away.values()) + self._away.clear() + if not away or self._home is None: + return + connection: Any + for connection, _ in away: + try: + self._adapter.use_session_warehouse(connection=connection, warehouse=self._home) + except Exception as error: # noqa: BLE001 - its owner closes or replaces it next + _LOGGER.warning( + "could not return a connection to warehouse %s: %s", self._home, error + ) + + def _learn_home(self, connection: Any) -> None: + """Read the warehouse the build's connections start on before one first leaves it.""" + + with self._lock: + if self._home_known: + return + self._home = self._adapter.current_session_warehouse(connection=connection) + self._home_known = True + + def _return_to(self) -> str: + if self._home is None: + raise ExecutorInputError( + "cannot return a connection to the build's warehouse: the target sets no " + "warehouse and the session had none; set the target's warehouse to build nodes " + "outside the named jobs beside them" + ) + return self._home diff --git a/src/sqlbuild/executor/scheduling/constants.py b/src/sqlbuild/executor/scheduling/constants.py new file mode 100644 index 0000000000..f7b24db87b --- /dev/null +++ b/src/sqlbuild/executor/scheduling/constants.py @@ -0,0 +1,3 @@ +"""Connection config keys the build scheduler reads.""" + +CONFIGURED_WAREHOUSE_KEY: str = "configured_warehouse" diff --git a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py index a07f1a708e..093b9ac534 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py @@ -1475,3 +1475,91 @@ class SeedRunRecordTestCase: description: str # Each row as (phase, command, select_json, named_jobs_json, var_keys_json, status, counts). expected_rows: list[tuple[str, str, str, str, str, str | None, str | None]] + + +@dataclass(frozen=True) +class JobSettingsBuildTestCase: + description: str + build_args: tuple[str, ...] + expected_amount: int + expected_fragments: tuple[str, ...] + + +@dataclass(frozen=True) +class JobSettingsConflictTestCase: + description: str + command_args: tuple[str, ...] + expected_fragment: str + extra_files: dict[str, str] = field(default_factory=dict) + + +@dataclass(frozen=True) +class JobFullRefreshTestCase: + description: str + build_args: tuple[str, ...] + expected_counts: tuple[tuple[int, int], ...] + + +@dataclass(frozen=True) +class JobConcurrencyTestCase: + description: str + build_args: tuple[str, ...] + job_nodes: tuple[str, ...] + build_nodes: tuple[str, ...] + expected_job_peak: int + expected_least_build_peak: int + # SQLBUILD_CONCURRENCY for the build; empty leaves it unset. + environment_concurrency: str = "" + + +@dataclass(frozen=True) +class JobOutputTestCase: + description: str + command_args: tuple[str, ...] + expected_exit_code: int + expected_present: tuple[str, ...] + expected_absent: tuple[str, ...] + extra_files: dict[str, str] = field(default_factory=dict) + + +@dataclass(frozen=True) +class JobWarehouseSwitchTestCase: + """A build naming jobs, and the warehouses its connections switch to after connecting.""" + + description: str + argv: tuple[str, ...] + expected_connect_warehouses: frozenset[str] + expected_switches: tuple[str, ...] + extra_files: dict[str, str] = field(default_factory=dict) + + +@dataclass(frozen=True) +class JobWarehouseOutputTestCase: + """A command naming jobs with warehouses, its exit code and what its output says.""" + + description: str + argv: tuple[str, ...] + expected_exit_code: int + expected_fragment: str + + +@dataclass(frozen=True) +class JobFlagParityTestCase: + """A single named job's run, and the global flags its node settings used to be.""" + + description: str + job_args: tuple[str, ...] + flag_plan_args: tuple[str, ...] + flag_build_args: tuple[str, ...] + expected_function_reasons: tuple[tuple[str, str], ...] + expected_full_refresh_counts: tuple[str, ...] + expected_exit_code: int + + +@dataclass(frozen=True) +class DynamicRefreshWarehouseTestCase: + """A build of a dynamic model through jobs, and the warehouse its refreshes are given.""" + + description: str + argv: tuple[str, ...] + expected_refresh_warehouses: tuple[str, ...] diff --git a/tests/integration/src/sqlbuild/cli/commands/main/helpers.py b/tests/integration/src/sqlbuild/cli/commands/main/helpers.py index 0be7c535d1..8429997502 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/helpers.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/helpers.py @@ -2,14 +2,17 @@ import json import os +import re import shutil import signal import subprocess import sys import tempfile import time +from collections import defaultdict from collections.abc import Callable -from itertools import takewhile +from datetime import datetime, timedelta +from itertools import accumulate, chain, compress, takewhile from pathlib import Path from types import ModuleType from typing import Any, cast @@ -55,6 +58,9 @@ ) +_COMPLETED_EVENT: str = "resource_attempt_completed" + + def artifact_tree(root: Path) -> dict[str, bytes]: """Return each file under root by its POSIX path relative to root, or nothing when missing.""" @@ -1455,3 +1461,262 @@ def prepare_warehouse(*, project_dir: Path, sql: str) -> None: with duckdb.connect(str(project_dir / "demo.duckdb")) as warehouse: warehouse.execute(sql) + + +def peak_concurrency(*, events_path: Path, names: tuple[str, ...]) -> int: + """The most of `names` a build's event output shows running at once.""" + + events: dict[tuple[str, str], list[dict[str, Any]]] = defaultdict(list) + for line in events_path.read_text(encoding="utf-8").splitlines(): + event: dict[str, Any] = json.loads(line) + events[(event["event_type"], str(event.get("resource_name")))].append(event) + completions: list[dict[str, Any]] = list( + chain.from_iterable(events[(_COMPLETED_EVENT, name)] for name in names) + ) + ends: list[tuple[datetime, float]] = [ + (datetime.fromisoformat(event["occurred_at"]), event["duration_ms"]) + for event in completions + ] + changes: list[tuple[datetime, int]] = sorted( + [(end - timedelta(milliseconds=duration), 1) for end, duration in ends] + + [(end, -1) for end, _ in ends] + ) + return max(accumulate(change for _, change in changes), default=0) + + +_CONNECT_MARKER: str = "-- connect" +# The environment variable that sets a build's default concurrency. +CONCURRENCY_VARIABLE: str = "SQLBUILD_CONCURRENCY" + + +def _job_warehouse_model(source: str) -> str: + return f"MODEL (description 'Step.', materialized table);\nSELECT 1 AS id{source}\n" + + +def _job_file(name: str, select: str, warehouse: str) -> str: + return ( + f'name = "{name}"\ndescription = "Runs {select}."\nselect = "{select}"\n\n' + f'[settings]\nwarehouse = "{warehouse}"\n' + ) + + +def write_job_warehouse_project(*, project_dir: Path) -> None: + """A Snowflake chain stg -> orders -> orders_daily -> refunds -> summary and a task, with jobs + on their own warehouses: orders_wh_job, orders_copy_job (orders_daily on the same warehouse), + refunds_wh_job, refunds_alt_job (also refunds), tasks_wh_job and bad_wh_job, whose warehouse + is not an identifier.""" + + write_snowflake_warehouse_project( + project_dir=project_dir, + warehouses_section='[targets.dev.warehouses]\nbuild = "BUILD_WH"\n', + local_config="", + ) + write_project_files( + project_dir=project_dir, + files={ + "models/stg.sql": _job_warehouse_model(""), + "models/orders.sql": _job_warehouse_model(' FROM __ref("stg")'), + "models/orders_daily.sql": _job_warehouse_model(' FROM __ref("orders")'), + "models/refunds.sql": _job_warehouse_model(' FROM __ref("orders_daily")'), + "models/summary.sql": _job_warehouse_model(' FROM __ref("refunds")'), + "python/tasks.py": ( + "from sqlbuild.tasks import task\n\n\n@task\ndef refresh_marker(ctx):\n" + ' """Refresh."""\n ctx.execute_sql("SELECT \'task marker\'")\n' + ), + "orchestration/jobs/orders_wh_job.toml": _job_file( + "orders_wh_job", "orders orders_daily", "ORDERS_WH" + ), + "orchestration/jobs/refunds_wh_job.toml": _job_file( + "refunds_wh_job", "refunds", "REFUNDS_WH" + ), + "orchestration/jobs/orders_copy_job.toml": _job_file( + "orders_copy_job", "orders_daily", "ORDERS_WH" + ), + "orchestration/jobs/refunds_alt_job.toml": _job_file( + "refunds_alt_job", "refunds", "ALT_WH" + ), + "orchestration/jobs/tasks_wh_job.toml": _job_file( + "tasks_wh_job", "refresh_marker", "TASKS_WH" + ), + "orchestration/jobs/bad_wh_job.toml": _job_file("bad_wh_job", "stg", "bad wh"), + }, + ) + + +def install_writable_offline_snowflake( + monkeypatch: pytest.MonkeyPatch, +) -> tuple[list[dict[str, object]], list[str]]: + """The offline connector, accepting writes, with a marker before each connection's setup.""" + + connect_calls, executed_sql = install_offline_snowflake_connector(monkeypatch) + monkeypatch.setattr( + sys.modules[__name__], "_refuse_offline_snowflake_write", _accept_offline_snowflake_read + ) + connector: ModuleType = sys.modules["snowflake.connector"] + connect: Callable[..., object] = connector.__dict__["connect"] + + def marked_connect(**kwargs: object) -> object: + executed_sql.append(_CONNECT_MARKER) + raw: Any = connect(**kwargs) + raw._executed_sql = _ConnectionStatements(sink=executed_sql, connection=len(connect_calls)) + return raw + + monkeypatch.setitem(connector.__dict__, "connect", marked_connect) + return connect_calls, executed_sql + + +class _ConnectionStatements: + """One offline connection's statement log: each statement in order, and which connection.""" + + def __init__(self, *, sink: list[str], connection: int) -> None: + self._sink: list[str] = sink + self._connection: int = connection + + def append(self, sql: str) -> None: + self._sink.append(sql) + _STATEMENT_CONNECTIONS.setdefault(id(self._sink), []).append((self._connection, sql)) + + +# Each shared statement log's statements with the connection that ran them. +_STATEMENT_CONNECTIONS: dict[int, list[tuple[int, str]]] = {} + + +def warehouse_switches(executed_sql: list[str]) -> tuple[str, ...]: + """The warehouses statements switch to, leaving out each connection's first, its own setup.""" + + uses: list[tuple[int, str]] = [ + (connection, sql.removeprefix(_USE_WAREHOUSE)) + for connection, sql in filter( + lambda statement: statement[1].startswith(_USE_WAREHOUSE), + _STATEMENT_CONNECTIONS.get(id(executed_sql), []), + ) + ] + first_use: dict[int, int] = { + connection: index for index, (connection, _) in reversed(list(enumerate(uses))) + } + return tuple( + compress( + (warehouse for _, warehouse in uses), + (first_use[connection] != index for index, (connection, _) in enumerate(uses)), + ) + ) + + +_USE_WAREHOUSE: str = "USE WAREHOUSE " + + +def run_cli_exit_code(argv: tuple[str, ...], project_dir: Path) -> int: + """The exit code of one CLI invocation in `project_dir`, returned or raised.""" + + try: + return int(main(["--no-color", "--project-dir", str(project_dir), *argv]) or 0) + except SystemExit as exited: + return int(exited.code or 0) + + +_PARITY_PROJECT_FILES: dict[str, str] = { + "sqlbuild_project.toml": ( + 'name = "demo"\nadapter = "duckdb"\ndefault_target = "dev"\n\n' + '[connection]\ndatabase = "demo.duckdb"\n\n[targets.dev]\n' + ), + "functions/sql/udf__band.sql": ( + "FUNCTION (description 'Band.', arguments (amount INT), returns VARCHAR);\n" + "CASE WHEN amount >= 20 THEN 'large' ELSE 'small' END\n" + ), + "models/parity_lookup.sql": ( + "MODEL (description 'Lookup.', materialized table, " + "columns (id (type INTEGER, audits [not_null])));\nSELECT CAST(NULL AS INTEGER) AS id\n" + ), + "models/parity_events.sql": ( + "MODEL (description 'Events.', materialized incremental, incremental_strategy append);\n" + 'SELECT 1 AS event_id, __udf("udf__band")(5) AS band\n' + ), + "tests/unit/test_parity_events.sql": ( + "TEST();\n\nWITH\n__ref__parity_lookup AS (SELECT 1 AS id),\n" + "__expected__parity_events AS (SELECT 99 AS event_id, 'x' AS band)\nSELECT 1\n" + ), + "audits/singular/events_have_lookups.sql": ( + 'AUDIT (name "events_have_lookups", severity error);\n' + 'SELECT e.event_id FROM __ref("parity_events") e ' + 'LEFT JOIN __ref("parity_lookup") l ON e.event_id = l.id WHERE l.id IS NULL\n' + ), + "orchestration/jobs/parity_job.toml": ( + 'name = "parity_job"\ndescription = "Parity."\n' + 'select = "udf__band parity_lookup parity_events"\n\n' + "[settings]\nfull_refresh = true\nrun_tests = false\nignore_audit_failures = true\n" + ), +} +PARITY_NODES: tuple[str, ...] = ("udf__band", "parity_lookup", "parity_events") + + +def job_flag_parity_outcome( + *, + project_dir: Path, + args: tuple[tuple[str, ...], tuple[str, ...]], + capsys: pytest.CaptureFixture[str], +) -> tuple[object, ...]: + """Build a parity project once without its job, then plan and build it with `args`: the + function reasons, full-refresh count, exit code, summary and ignored audits they report.""" + + plan_args, build_args = args + + write_project_files(project_dir=project_dir, files=_PARITY_PROJECT_FILES) + base: list[str] = ["--no-color", "--project-dir", str(project_dir)] + main( + [ + *base, + "build", + "--no-job-settings", + "--no-tests", + "--ignore-audit-failures", + "--select", + *PARITY_NODES, + ] + ) + capsys.readouterr() + main([*base, "plan", *plan_args]) + plan: str = capsys.readouterr().out + exit_code: int = main([*base, "build", *build_args]) + build: str = capsys.readouterr().out + return ( + tuple(re.findall(r"^ (\S+) +sql udf\n +reason: (.*)$", plan, re.M)), + tuple(re.findall(r"^Full refresh \((\d+)\)$", plan, re.M)), + exit_code, + tuple(re.findall(r"Completed [^\n]*?TOTAL=\d+", build)), + tuple(re.findall(r"audit .+ returned \d+ rows? \(ignored\)", build)), + ) + + +def unit_test_sql(*models: str) -> str: + """A unit test mocking orders_daily that expects one row from each of `models`.""" + + expected: str = ",\n".join(f"__expected__{model} AS (SELECT 1 AS id)" for model in models) + return f"TEST();\n\nWITH\n__ref__orders_daily AS (SELECT 1 AS id),\n{expected}\nSELECT 1\n" + + +_DYNAMIC_FILES: dict[str, str] = { + "models/daily_orders.sql": ( + "MODEL (description 'Daily orders.', materialized dynamic, refresh (lag 1h));\n" + "SELECT 1 AS id\n" + ), + "orchestration/jobs/dynamic_wh_job.toml": ( + 'name = "dynamic_wh_job"\ndescription = "Runs daily_orders."\nselect = "daily_orders"\n\n' + '[settings]\nwarehouse = "DYNAMIC_WH"\n' + ), +} + + +def write_dynamic_job_project(*, project_dir: Path) -> None: + """The job warehouse project plus dynamic model daily_orders, run by job dynamic_wh_job.""" + + write_job_warehouse_project(project_dir=project_dir) + write_project_files(project_dir=project_dir, files=_DYNAMIC_FILES) + + +def dynamic_refresh_warehouses(executed_sql: list[str]) -> tuple[str, ...]: + """The warehouse each created dynamic table is given for its refreshes.""" + + statements: str = "\n".join(executed_sql) + return tuple( + re.findall(r"CREATE OR REPLACE DYNAMIC TABLE .*?WAREHOUSE = (\S+)", statements, re.S) + ) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_job_settings.py b/tests/integration/src/sqlbuild/cli/commands/main/test_job_settings.py new file mode 100644 index 0000000000..fa8ec58e87 --- /dev/null +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_job_settings.py @@ -0,0 +1,701 @@ +"""A build that names jobs takes their settings: build-wide ones once, node-level ones per node.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from sqlbuild._native import run_cli as main +from tests.integration.src.sqlbuild.cli.commands.main._test_types import ( + JobConcurrencyTestCase, + JobFlagParityTestCase, + JobFullRefreshTestCase, + JobOutputTestCase, + JobSettingsBuildTestCase, + JobSettingsConflictTestCase, +) +from tests.integration.src.sqlbuild.cli.commands.main.helpers import ( + CONCURRENCY_VARIABLE, + PARITY_NODES, + job_flag_parity_outcome, + peak_concurrency, + query_duckdb_rows, + write_project_files, +) + +_PROJECT_FILES: dict[str, str] = { + "sqlbuild_project.toml": ( + 'name = "demo"\nadapter = "duckdb"\ndefault_target = "dev"\n\n' + '[connection]\ndatabase = "demo.duckdb"\n\n[targets.dev]\n\n[targets.prod]\n\n' + '[settings]\nconcurrency = 4\n\n[vars]\namount = "1"\n' + ), + "models/stg_orders.sql": ( + "MODEL (description 'Staged orders.', materialized table);\n" + "SELECT CAST(@@amount AS INTEGER) AS order_id\n" + ), + "models/orders.sql": ( + "MODEL (description 'Orders.', materialized table);\n" + 'SELECT order_id FROM __ref("stg_orders")\n' + ), + "models/customers.sql": ( + "MODEL (description 'Customers.', materialized table);\nSELECT 7 AS customer_id\n" + ), + "models/order_events.sql": ( + "MODEL (description 'Order events.', materialized incremental, " + "incremental_strategy append);\nSELECT 1 AS event_id\n" + ), + "models/refund_events.sql": ( + "MODEL (description 'Refund events.', materialized incremental, " + "incremental_strategy append);\nSELECT 1 AS event_id\n" + ), + "tests/unit/test_orders.sql": ( + "TEST();\n\nWITH\n__ref__stg_orders AS (SELECT 4 AS order_id),\n" + "__expected__orders AS (SELECT 4 AS order_id)\nSELECT 1\n" + ), + "orchestration/jobs/orders_job.toml": ( + 'name = "orders_job"\ndescription = "Orders."\nselect = "+orders"\n\n' + '[settings]\nconcurrency = 1\nrun_tests = false\nvars = { amount = "5" }\n' + ), + "orchestration/jobs/checked_orders_job.toml": ( + 'name = "checked_orders_job"\ndescription = "Orders, tested."\nselect = "orders"\n\n' + '[settings]\nrun_tests = true\nvars = { amount = "5" }\n' + ), + "orchestration/jobs/crm_job.toml": ( + 'name = "crm_job"\ndescription = "Customers."\nselect = "customers"\n\n' + '[settings]\nvars = { amount = "9" }\n' + ), + "orchestration/jobs/events_job.toml": ( + 'name = "events_job"\ndescription = "Events."\nselect = "order_events"\n\n' + "[settings]\nfull_refresh = true\n" + ), + "orchestration/jobs/refund_events_job.toml": ( + 'name = "refund_events_job"\ndescription = "Refund events."\nselect = "refund_events"\n' + ), + "orchestration/job_groups/events_job_group.toml": ( + 'name = "events_job_group"\ndescription = "Events."\n' + 'jobs = ["events_job", "refund_events_job"]\n' + ), + "models/lenient_model.sql": ( + "MODEL (description 'Lenient.', materialized table, " + "columns (id (type INTEGER, audits [not_null])));\nSELECT CAST(NULL AS INTEGER) AS id\n" + ), + "orchestration/jobs/lenient_job.toml": ( + 'name = "lenient_job"\ndescription = "Lenient."\nselect = "lenient_model"\n\n' + "[settings]\nignore_audit_failures = true\n" + ), + "orchestration/jobs/loads_off_job.toml": ( + 'name = "loads_off_job"\ndescription = "No loads."\nselect = "customers"\n\n' + "[settings]\nload_sources = false\n" + ), + **{ + f"models/slow_{name}.sql": ( + f"MODEL (description 'Slow {name}.', materialized table);\n" + "SELECT sum(i * i % 7) AS total FROM range(25000000) AS t(i)\n" + ) + for name in ("a", "b", "c", "d") + }, + "orchestration/jobs/warehouse_job.toml": ( + 'name = "warehouse_job"\ndescription = "On a warehouse."\nselect = "customers"\n\n' + '[settings]\nwarehouse = "ORDERS_WH"\n' + ), + "orchestration/jobs/wide_job.toml": ( + 'name = "wide_job"\ndescription = "Wide."\nselect = "customers"\n\n' + '[settings]\nconcurrency = 6\nvars = { amount = "5" }\n' + ), + "orchestration/jobs/slow_job.toml": ( + 'name = "slow_job"\ndescription = "Slow."\nselect = "slow_a slow_b"\n\n' + "[settings]\nconcurrency = 1\n" + ), +} + +_FAIL_FAST_FILES: dict[str, str] = { + "orchestration/jobs/fast_fail_job.toml": ( + 'name = "fast_fail_job"\ndescription = "Fails fast."\nselect = "customers"\n\n' + "[settings]\nfail_fast = true\n" + ), + "orchestration/job_groups/fail_job_group.toml": ( + 'name = "fail_job_group"\ndescription = "Fails."\njobs = ["events_job", "fast_fail_job"]\n' + ), +} +_GRACE_FILES: dict[str, str] = { + "orchestration/jobs/grace_job.toml": ( + 'name = "grace_job"\ndescription = "Takes over."\nselect = "customers"\n\n' + "[settings]\nignore_dynamic_grace = true\n" + ), +} +_LENIENT_PAIR_FILES: dict[str, str] = { + "models/lenient_peer.sql": ( + "MODEL (description 'Lenient peer.', materialized table);\nSELECT 1 AS id\n" + ), + "audits/singular/lenient_pair.sql": ( + 'AUDIT (name "lenient_pair", severity error);\n' + 'SELECT p.id FROM __ref("lenient_peer") p ' + 'LEFT JOIN __ref("lenient_model") m ON p.id = m.id WHERE m.id IS NULL\n' + ), + "orchestration/jobs/lenient_peer_job.toml": ( + 'name = "lenient_peer_job"\ndescription = "Lenient peer."\nselect = "lenient_peer"\n\n' + "[settings]\nignore_audit_failures = true\n" + ), +} +_STEADY_EVENTS_FILES: dict[str, str] = { + "orchestration/jobs/steady_events_job.toml": ( + 'name = "steady_events_job"\ndescription = "Events, kept."\nselect = "order_events"\n\n' + "[settings]\nfull_refresh = false\n" + ), +} + + +@pytest.mark.parametrize( + "test_case", + [ + JobSettingsBuildTestCase( + description="a named job's vars, concurrency and run_tests apply", + build_args=("--select", "job:orders_job"), + expected_amount=5, + expected_fragments=("(concurrency: 1)", "TOTAL=2"), + ), + JobSettingsBuildTestCase( + description="a job: pattern names the jobs it matches", + build_args=("--select", "job:orders_*"), + expected_amount=5, + expected_fragments=("(concurrency: 1)", "TOTAL=2"), + ), + JobSettingsBuildTestCase( + description="job flags go before -- so a selector after it still names the job", + build_args=("--", "orders_job"), + expected_amount=5, + expected_fragments=("(concurrency: 1)", "TOTAL=2"), + ), + JobSettingsBuildTestCase( + description="a bare job name names the job", + build_args=("--select", "orders_job"), + expected_amount=5, + expected_fragments=("(concurrency: 1)", "TOTAL=2"), + ), + JobSettingsBuildTestCase( + description="command-line flags beat the job's settings", + build_args=( + "--select", + "job:orders_job", + "--vars", + '{"amount": "2"}', + "--concurrency", + "3", + ), + expected_amount=2, + expected_fragments=("(concurrency: 3)", "TOTAL=2"), + ), + JobSettingsBuildTestCase( + description="an abbreviated command-line flag still beats the job's setting", + build_args=("--select", "job:orders_job", "--concur", "2"), + expected_amount=5, + expected_fragments=("(concurrency: 2)", "TOTAL=2"), + ), + JobSettingsBuildTestCase( + description="--no-job-settings builds with the project's settings", + build_args=("--select", "job:orders_job", "--no-job-settings"), + expected_amount=1, + expected_fragments=("(concurrency: 4)", "TOTAL=3"), + ), + JobSettingsBuildTestCase( + description="selecting the job's nodes without naming it takes none of its settings", + build_args=("--select", "+orders"), + expected_amount=1, + expected_fragments=("(concurrency: 4)", "TOTAL=3"), + ), + ], + ids=lambda case: case.description, +) +def test_given_a_named_job_when_building_then_its_settings_apply_below_the_command_line( + test_case: JobSettingsBuildTestCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.delenv(CONCURRENCY_VARIABLE, raising=False) + write_project_files(project_dir=tmp_path, files=_PROJECT_FILES) + + exit_code: int = main( + ["--project-dir", str(tmp_path), "--no-color", "build", *test_case.build_args] + ) + captured: str = "".join(capsys.readouterr()) + + assert exit_code == 0, captured + assert query_duckdb_rows( + db_path=tmp_path / "demo.duckdb", sql="SELECT order_id FROM orders" + ) == ((test_case.expected_amount,),) + assert {fragment: fragment in captured for fragment in test_case.expected_fragments} == ( + dict.fromkeys(test_case.expected_fragments, True) + ), captured + + +@pytest.mark.parametrize( + "test_case", + [ + JobSettingsConflictTestCase( + description="two named jobs with different vars cannot build together", + command_args=("build", "--select", "job:orders_job", "job:crm_job"), + expected_fragment=( + 'error[S027]: jobs orders_job and crm_job set vars differently ({"amount":"5"} ' + 'and {"amount":"9"}); a build applies vars to everything it runs' + ), + ), + JobSettingsConflictTestCase( + description="a plan naming jobs with different vars fails the same way", + command_args=("plan", "--select", "job:orders_job", "job:crm_job"), + expected_fragment=( + 'error[S027]: jobs orders_job and crm_job set vars differently ({"amount":"5"} ' + 'and {"amount":"9"}); a build applies vars to everything it runs' + ), + ), + JobSettingsConflictTestCase( + description="two named jobs setting a build-wide setting differently cannot build", + command_args=("build", "--select", "job:events_job", "job:fast_fail_job"), + expected_fragment=( + "jobs events_job and fast_fail_job set fail_fast differently (not set and true); " + "a build applies fail_fast to everything it runs = help: make the jobs agree, " + "pass --fail-fast to make it true for all, or build with --no-job-settings" + ), + extra_files=_FAIL_FAST_FILES, + ), + JobSettingsConflictTestCase( + description="a job ignoring the dynamic grace needs the project to allow it", + command_args=("compile",), + expected_fragment=( + "orchestration/jobs/grace_job.toml:6:24: ignore_dynamic_grace needs [dynamic] " + "allow_ignore_grace = true in sqlbuild_project.toml" + ), + extra_files=_GRACE_FILES, + ), + JobSettingsConflictTestCase( + description="two named jobs giving one node different full_refresh conflict", + command_args=("build", "--select", "job:events_job", "job:steady_events_job"), + expected_fragment=( + "error[S027]: model:order_events is selected through jobs events_job and " + "steady_events_job, which set full_refresh differently (true and false)" + ), + extra_files=_STEADY_EVENTS_FILES, + ), + JobSettingsConflictTestCase( + description="two named jobs giving one node different run_tests conflict", + command_args=("build", "--select", "job:orders_job", "job:checked_orders_job"), + expected_fragment=( + "error[S027]: model:orders is selected through jobs orders_job and " + "checked_orders_job, which set run_tests differently (false and true)" + ), + ), + JobSettingsConflictTestCase( + description="a group whose jobs set different vars does not compile", + command_args=("compile",), + expected_fragment=( + 'jobs orders_job and crm_job set vars differently ({"amount":"5"} and ' + '{"amount":"9"}); a build applies vars to everything it runs; group ' + "daily_job_group builds its jobs together" + ), + extra_files={ + "orchestration/job_groups/daily_job_group.toml": ( + 'name = "daily_job_group"\ndescription = "Daily."\n' + 'jobs = ["orders_job", "crm_job"]\n' + ), + }, + ), + ], + ids=lambda case: case.description, +) +def test_given_conflicting_job_settings_when_running_then_the_command_fails_naming_them( + test_case: JobSettingsConflictTestCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + write_project_files(project_dir=tmp_path, files={**_PROJECT_FILES, **test_case.extra_files}) + + exit_code: int = main(["--project-dir", str(tmp_path), "--no-color", *test_case.command_args]) + captured: str = "".join(capsys.readouterr()) + + assert exit_code == 1, captured + assert test_case.expected_fragment in captured.replace("\n", " ").replace(" ", " "), captured + + +@pytest.mark.parametrize( + "test_case", + [ + JobFullRefreshTestCase( + description="a named job's full_refresh rebuilds only its own incremental models", + build_args=("--select", "job:events_job", "refund_events"), + expected_counts=((1, 3),), + ), + JobFullRefreshTestCase( + description="--full-refresh rebuilds every selected incremental model", + build_args=("--select", "job:events_job", "refund_events", "--full-refresh"), + expected_counts=((1, 1),), + ), + ], + ids=lambda case: case.description, +) +def test_given_a_named_job_with_full_refresh_when_building_then_only_its_models_start_over( + test_case: JobFullRefreshTestCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + write_project_files(project_dir=tmp_path, files=_PROJECT_FILES) + arguments: list[str] = ["--project-dir", str(tmp_path), "--no-color", "build", "--no-tests"] + first: int = main([*arguments, "--select", "order_events", "refund_events"]) + second: int = main([*arguments, "--select", "order_events", "refund_events"]) + + exit_code: int = main([*arguments, *test_case.build_args]) + captured: str = "".join(capsys.readouterr()) + + assert (first, second, exit_code) == (0, 0, 0), captured + assert ( + query_duckdb_rows( + db_path=tmp_path / "demo.duckdb", + sql="SELECT (SELECT count(*) FROM order_events), (SELECT count(*) FROM refund_events)", + ) + == test_case.expected_counts + ) + + +@pytest.mark.parametrize( + "test_case", + [ + JobConcurrencyTestCase( + description="a job's concurrency limits its own nodes while other nodes run beside them", + build_args=("--select", "job:slow_job", "slow_c", "slow_d"), + job_nodes=("slow_a", "slow_b"), + build_nodes=("slow_a", "slow_b", "slow_c", "slow_d"), + expected_job_peak=1, + expected_least_build_peak=2, + ), + JobConcurrencyTestCase( + description="without job settings the job's nodes run together", + build_args=("--select", "job:slow_job", "slow_c", "slow_d", "--no-job-settings"), + job_nodes=("slow_a", "slow_b"), + build_nodes=("slow_a", "slow_b", "slow_c", "slow_d"), + expected_job_peak=2, + expected_least_build_peak=2, + ), + JobConcurrencyTestCase( + description="SQLBUILD_CONCURRENCY sets the build's limit, not the job's", + build_args=("--select", "job:slow_job", "slow_c", "slow_d"), + job_nodes=("slow_a", "slow_b"), + build_nodes=("slow_a", "slow_b", "slow_c", "slow_d"), + expected_job_peak=1, + expected_least_build_peak=2, + environment_concurrency="8", + ), + ], + ids=lambda case: case.description, +) +def test_given_a_job_concurrency_when_building_then_its_nodes_never_exceed_it( + test_case: JobConcurrencyTestCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv(CONCURRENCY_VARIABLE, test_case.environment_concurrency) + write_project_files(project_dir=tmp_path, files=_PROJECT_FILES) + events: Path = tmp_path / "events.jsonl" + + exit_code: int = main( + [ + "--project-dir", + str(tmp_path), + "--no-color", + "build", + *test_case.build_args, + "--event-output", + str(events), + ] + ) + captured: str = "".join(capsys.readouterr()) + + assert exit_code == 0, captured + assert ( + peak_concurrency(events_path=events, names=test_case.job_nodes), + peak_concurrency(events_path=events, names=test_case.build_nodes) + >= test_case.expected_least_build_peak, + ) == (test_case.expected_job_peak, True), captured + + +@pytest.mark.parametrize( + "test_case", + [ + JobOutputTestCase( + description="a named job's concurrency above the configured limit raises the build's", + command_args=("build", "--select", "job:wide_job"), + expected_exit_code=0, + expected_present=("(concurrency: 6)",), + expected_absent=(), + ), + JobOutputTestCase( + description="the named jobs' limits add up", + command_args=("build", "--select", "job:wide_job", "job:orders_job"), + expected_exit_code=0, + expected_present=("(concurrency: 7)",), + expected_absent=(), + ), + JobOutputTestCase( + description="the run context says the named jobs set the limit", + command_args=("build", "--verbose", "--select", "job:wide_job", "+orders"), + expected_exit_code=0, + expected_present=("6 configured limit (named jobs' concurrency)",), + expected_absent=(), + ), + JobOutputTestCase( + description="a configured limit above the named jobs' stays the build's limit", + command_args=("build", "--verbose", "--select", "job:orders_job", "customers"), + expected_exit_code=0, + expected_present=("4 configured limit (settings.concurrency)",), + expected_absent=(), + ), + JobOutputTestCase( + description="an explicit --concurrency wins over the named jobs", + command_args=("build", "--select", "job:wide_job", "--concurrency", "2"), + expected_exit_code=0, + expected_present=("(concurrency: 2)",), + expected_absent=(), + ), + JobOutputTestCase( + description="a job warehouse on an adapter without one fails, naming the job", + command_args=("build", "--select", "job:warehouse_job"), + expected_exit_code=1, + expected_present=( + "error[C260]: job warehouse_job sets warehouse ORDERS_WH, but the 'duckdb' " + "adapter has no warehouse to select", + ), + expected_absent=(), + ), + JobOutputTestCase( + description="--no-job-settings leaves a job warehouse out on such an adapter", + command_args=("build", "--no-job-settings", "--select", "job:warehouse_job"), + expected_exit_code=0, + expected_present=(), + expected_absent=("C260",), + ), + JobOutputTestCase( + description="a plan shows the job's full refresh", + command_args=("plan", "--select", "job:events_job"), + expected_exit_code=0, + expected_present=("Plan ready full refresh, 1 selected",), + expected_absent=(), + ), + JobOutputTestCase( + description="plan --no-job-settings leaves out the job's full refresh", + command_args=("plan", "--select", "job:events_job", "--no-job-settings"), + expected_exit_code=0, + expected_present=("Plan ready 1 selected",), + expected_absent=("full refresh",), + ), + JobOutputTestCase( + description="plan JSON names the jobs that full refresh their models", + command_args=("plan", "--json", "--select", "job:events_job"), + expected_exit_code=0, + expected_present=('"full_refresh_jobs": [\n "events_job"\n ]',), + expected_absent=(), + ), + JobOutputTestCase( + description="a plan says whose audit failures will not stop the build", + command_args=("plan", "--select", "job:lenient_job"), + expected_exit_code=0, + expected_present=( + "Audit failures won't stop this build for the audits of job lenient_job", + ), + expected_absent=(), + ), + JobOutputTestCase( + description="a named job's ignore_audit_failures lets its audit failures through", + command_args=("build", "--select", "job:lenient_job"), + expected_exit_code=0, + expected_present=("1 audit failure ignored",), + expected_absent=(), + ), + JobOutputTestCase( + description="the same node built without naming the job fails its audit", + command_args=("build", "--select", "lenient_model"), + expected_exit_code=1, + expected_present=("not_null (id)",), + expected_absent=("ignored",), + ), + JobOutputTestCase( + description="the run context names the jobs whose models are fully refreshed", + command_args=("build", "--verbose", "--select", "job:events_job"), + expected_exit_code=0, + expected_present=("full_refresh nodes of job events_job",), + expected_absent=(), + ), + JobOutputTestCase( + description="naming a group applies each job's settings to its own models", + command_args=("plan", "--json", "--select", "job:events_job_group"), + expected_exit_code=0, + expected_present=('"full_refresh_jobs": [\n "events_job"\n ]',), + expected_absent=("refund_events_job",), + ), + JobOutputTestCase( + description="a group compiles when its disagreeing jobs never share a target", + command_args=("compile",), + expected_exit_code=0, + expected_present=("Project compiled",), + expected_absent=("D017",), + extra_files={ + "orchestration/jobs/prod_fail_job.toml": ( + 'name = "prod_fail_job"\ndescription = "Prod."\nselect = "customers"\n' + 'targets = ["prod"]\n\n[settings]\nfail_fast = true\n' + ), + "orchestration/jobs/dev_job.toml": ( + 'name = "dev_job"\ndescription = "Dev."\nselect = "customers"\n' + 'targets = ["dev"]\n' + ), + "orchestration/job_groups/split_job_group.toml": ( + 'name = "split_job_group"\ndescription = "Split."\n' + 'jobs = ["prod_fail_job", "dev_job"]\n' + ), + }, + ), + JobOutputTestCase( + description="a group whose jobs differ only in fail_fast compiles, each job a build", + command_args=("compile",), + expected_exit_code=0, + expected_present=("Project compiled",), + expected_absent=("D017",), + extra_files=_FAIL_FAST_FILES, + ), + JobOutputTestCase( + description="--fail-fast settles the named jobs' fail_fast disagreement", + command_args=( + "build", + "--fail-fast", + "--select", + "job:events_job", + "job:fast_fail_job", + ), + expected_exit_code=0, + expected_present=("Completed successfully",), + expected_absent=("S027",), + extra_files=_FAIL_FAST_FILES, + ), + JobOutputTestCase( + description="--full-refresh settles two jobs' full_refresh disagreement on one node", + command_args=( + "build", + "--full-refresh", + "--select", + "job:events_job", + "job:steady_events_job", + ), + expected_exit_code=0, + expected_present=("Completed successfully",), + expected_absent=("S027",), + extra_files=_STEADY_EVENTS_FILES, + ), + JobOutputTestCase( + description="a singular audit over a lenient job node and a node outside it fails", + command_args=("build", "--select", "job:lenient_job", "lenient_peer"), + expected_exit_code=1, + expected_present=("lenient_pair",), + expected_absent=("lenient_pair returned 1 row (ignored)",), + extra_files=_LENIENT_PAIR_FILES, + ), + JobOutputTestCase( + description="a singular audit whose nodes' jobs all ignore failures is ignored", + command_args=("build", "--select", "job:lenient_job", "job:lenient_peer_job"), + expected_exit_code=0, + expected_present=("audit lenient_pair returned 1 row (ignored)",), + expected_absent=(), + extra_files=_LENIENT_PAIR_FILES, + ), + JobOutputTestCase( + description="a job may ignore the dynamic grace where the project allows it", + command_args=("compile",), + expected_exit_code=0, + expected_present=("Project compiled",), + expected_absent=("ignore_dynamic_grace",), + extra_files={ + **_GRACE_FILES, + "sqlbuild_project.toml": _PROJECT_FILES["sqlbuild_project.toml"] + + "\n[dynamic]\nallow_ignore_grace = true\n", + }, + ), + JobOutputTestCase( + description="a command-line option beats a job option it excludes", + command_args=("build", "--select", "job:loads_off_job", "--reload"), + expected_exit_code=0, + expected_present=("Completed successfully",), + expected_absent=("not allowed with",), + ), + ], + ids=lambda case: case.description, +) +def test_given_a_named_job_when_planning_or_building_then_the_output_reflects_its_settings( + test_case: JobOutputTestCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.delenv(CONCURRENCY_VARIABLE, raising=False) + write_project_files(project_dir=tmp_path, files={**_PROJECT_FILES, **test_case.extra_files}) + + exit_code: int = main(["--project-dir", str(tmp_path), "--no-color", *test_case.command_args]) + captured: str = "".join(capsys.readouterr()) + + assert exit_code == test_case.expected_exit_code, captured + assert {fragment: fragment in captured for fragment in test_case.expected_present} == ( + dict.fromkeys(test_case.expected_present, True) + ), captured + assert {fragment: fragment in captured for fragment in test_case.expected_absent} == ( + dict.fromkeys(test_case.expected_absent, False) + ), captured + + +@pytest.mark.parametrize( + "test_case", + [ + JobFlagParityTestCase( + description="full_refresh, run_tests and ignore_audit_failures act as the flags did", + job_args=("--select", "job:parity_job"), + flag_plan_args=( + "--no-job-settings", + "--full-refresh", + "--ignore-audit-failures", + "--select", + *PARITY_NODES, + ), + flag_build_args=( + "--no-job-settings", + "--full-refresh", + "--no-tests", + "--ignore-audit-failures", + "--select", + *PARITY_NODES, + ), + expected_function_reasons=(("udf__band", "full refresh"),), + expected_full_refresh_counts=("2",), + expected_exit_code=0, + ), + ], + ids=lambda case: case.description, +) +def test_given_a_single_named_job_when_building_then_it_matches_its_settings_as_global_flags( + test_case: JobFlagParityTestCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + job_outcome: tuple[object, ...] = job_flag_parity_outcome( + project_dir=tmp_path / "job", + args=(test_case.job_args, test_case.job_args), + capsys=capsys, + ) + flag_outcome: tuple[object, ...] = job_flag_parity_outcome( + project_dir=tmp_path / "flags", + args=(test_case.flag_plan_args, test_case.flag_build_args), + capsys=capsys, + ) + + assert (job_outcome == flag_outcome, job_outcome[:3]) == ( + True, + ( + test_case.expected_function_reasons, + test_case.expected_full_refresh_counts, + test_case.expected_exit_code, + ), + ) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_job_warehouses.py b/tests/integration/src/sqlbuild/cli/commands/main/test_job_warehouses.py new file mode 100644 index 0000000000..3624d3147c --- /dev/null +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_job_warehouses.py @@ -0,0 +1,216 @@ +"""A build naming jobs runs each job's nodes on the job's warehouse, switching only on change.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest +from _pytest.capture import CaptureResult + +from tests.integration.src.sqlbuild.cli.commands.main._test_types import ( + DynamicRefreshWarehouseTestCase, + JobWarehouseOutputTestCase, + JobWarehouseSwitchTestCase, +) +from tests.integration.src.sqlbuild.cli.commands.main.helpers import ( + CONCURRENCY_VARIABLE, + dynamic_refresh_warehouses, + install_writable_offline_snowflake, + run_cli_exit_code, + unit_test_sql, + warehouse_switches, + write_dynamic_job_project, + write_job_warehouse_project, + write_project_files, +) + +_BOTH_JOBS: tuple[str, ...] = ( + "--select", + "stg", + "job:orders_wh_job", + "job:refunds_wh_job", + "summary", +) + + +@pytest.mark.parametrize( + "test_case", + [ + JobWarehouseSwitchTestCase( + description="two jobs' nodes run on their own warehouses, switching only on change", + argv=("build", *_BOTH_JOBS), + expected_connect_warehouses=frozenset({"BUILD_WH"}), + expected_switches=("ORDERS_WH", "REFUNDS_WH", "BUILD_WH"), + ), + JobWarehouseSwitchTestCase( + description="a single named job's warehouse is the whole build's, its task included", + argv=("build", "--select", "job:tasks_wh_job"), + expected_connect_warehouses=frozenset({"TASKS_WH"}), + expected_switches=(), + ), + JobWarehouseSwitchTestCase( + description="a warehouse every named job shares runs upstream nodes outside them too", + argv=("build", "--select", "stg", "job:orders_wh_job", "job:orders_copy_job"), + expected_connect_warehouses=frozenset({"ORDERS_WH"}), + expected_switches=(), + ), + JobWarehouseSwitchTestCase( + description="--warehouse wins over every named job, even ones that would conflict", + argv=("build", "--warehouse", "CLI_WH", *_BOTH_JOBS, "job:refunds_alt_job"), + expected_connect_warehouses=frozenset({"CLI_WH"}), + expected_switches=(), + ), + JobWarehouseSwitchTestCase( + description="a SQL test covering nodes with and without a job warehouse stays put", + argv=("build", *_BOTH_JOBS), + expected_connect_warehouses=frozenset({"BUILD_WH"}), + expected_switches=("ORDERS_WH", "REFUNDS_WH", "BUILD_WH"), + extra_files={"tests/unit/test_refunds.sql": unit_test_sql("refunds", "summary")}, + ), + JobWarehouseSwitchTestCase( + description="a SQL test covering only one job's nodes runs on its warehouse", + argv=("build", *_BOTH_JOBS), + expected_connect_warehouses=frozenset({"BUILD_WH"}), + expected_switches=("REFUNDS_WH", "ORDERS_WH", "REFUNDS_WH", "BUILD_WH"), + extra_files={"tests/unit/test_refunds.sql": unit_test_sql("refunds")}, + ), + JobWarehouseSwitchTestCase( + description="--no-job-settings keeps every node on the build's warehouse", + argv=("build", "--no-job-settings", *_BOTH_JOBS), + expected_connect_warehouses=frozenset({"BUILD_WH"}), + expected_switches=(), + ), + JobWarehouseSwitchTestCase( + description="nodes selected without naming their job stay on the build's warehouse", + argv=("build", "--select", "+refunds"), + expected_connect_warehouses=frozenset({"BUILD_WH"}), + expected_switches=(), + ), + ], + ids=lambda case: case.description, +) +def test_given_named_jobs_with_warehouses_when_building_then_each_node_runs_on_its_jobs( + test_case: JobWarehouseSwitchTestCase, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + monkeypatch.delenv(CONCURRENCY_VARIABLE, raising=False) + write_job_warehouse_project(project_dir=tmp_path) + write_project_files(project_dir=tmp_path, files=test_case.extra_files) + connect_calls, executed_sql = install_writable_offline_snowflake(monkeypatch) + + exit_code: int = run_cli_exit_code(test_case.argv, tmp_path) + + captured: CaptureResult[str] = capsys.readouterr() + assert ( + exit_code, + {call.get("warehouse") for call in connect_calls}, + warehouse_switches(executed_sql), + ) == ( + 0, + test_case.expected_connect_warehouses, + test_case.expected_switches, + ), captured.out + captured.err + + +@pytest.mark.parametrize( + "test_case", + [ + JobWarehouseOutputTestCase( + description="a plan names each job's warehouse", + argv=("plan", *_BOTH_JOBS), + expected_exit_code=0, + expected_fragment="Nodes of job orders_wh_job run on warehouse ORDERS_WH", + ), + JobWarehouseOutputTestCase( + description="plan JSON maps each job to its warehouse", + argv=("plan", "--json", *_BOTH_JOBS), + expected_exit_code=0, + expected_fragment=( + '"job_warehouses": {\n "orders_wh_job": "ORDERS_WH",\n' + ' "refunds_wh_job": "REFUNDS_WH"\n }' + ), + ), + JobWarehouseOutputTestCase( + description="the run context shows the jobs' warehouses beside the build's", + argv=("build", "--verbose", *_BOTH_JOBS), + expected_exit_code=0, + expected_fragment=( + "warehouse BUILD_WH (sqlbuild_project.toml [targets.dev.warehouses] build); " + "nodes of job orders_wh_job on ORDERS_WH; nodes of job refunds_wh_job on " + "REFUNDS_WH" + ), + ), + JobWarehouseOutputTestCase( + description="two jobs giving one node different warehouses conflict", + argv=("build", "--select", "job:refunds_wh_job", "job:refunds_alt_job"), + expected_exit_code=1, + expected_fragment=( + "error[S027]: model:refunds is selected through jobs refunds_wh_job and " + 'refunds_alt_job, which set warehouse differently ("REFUNDS_WH" and "ALT_WH")' + ), + ), + JobWarehouseOutputTestCase( + description="a job warehouse that is not an identifier fails before building", + argv=("plan", "--select", "job:bad_wh_job"), + expected_exit_code=1, + expected_fragment=( + "error[C261]: job bad_wh_job sets warehouse 'bad wh', which is not a valid " + "Snowflake warehouse identifier" + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_named_jobs_with_warehouses_when_planning_or_building_then_output_names_them( + test_case: JobWarehouseOutputTestCase, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + write_job_warehouse_project(project_dir=tmp_path) + install_writable_offline_snowflake(monkeypatch) + + exit_code: int = run_cli_exit_code(test_case.argv, tmp_path) + + captured: CaptureResult[str] = capsys.readouterr() + output: str = captured.out + captured.err + assert (exit_code, test_case.expected_fragment in output) == ( + test_case.expected_exit_code, + True, + ), output + + +@pytest.mark.parametrize( + "test_case", + [ + DynamicRefreshWarehouseTestCase( + description="a job switching its nodes to its warehouse leaves refreshes on the build's", + argv=("build", "--select", "stg", "job:orders_wh_job", "job:dynamic_wh_job"), + expected_refresh_warehouses=("BUILD_WH",), + ), + DynamicRefreshWarehouseTestCase( + description="a single job's warehouse, the whole build's, still refreshes on the build's", + argv=("build", "--select", "job:dynamic_wh_job"), + expected_refresh_warehouses=("BUILD_WH",), + ), + ], + ids=lambda case: case.description, +) +def test_given_a_dynamic_model_built_through_a_job_when_created_then_it_refreshes_on_the_build_s( + test_case: DynamicRefreshWarehouseTestCase, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + write_dynamic_job_project(project_dir=tmp_path) + _, executed_sql = install_writable_offline_snowflake(monkeypatch) + + exit_code: int = run_cli_exit_code(test_case.argv, tmp_path) + + captured: CaptureResult[str] = capsys.readouterr() + assert (exit_code, dynamic_refresh_warehouses(executed_sql)) == ( + 0, + test_case.expected_refresh_warehouses, + ), captured.out + captured.err diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py index a2f2f71aec..4cf8ae8513 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py @@ -435,6 +435,52 @@ def test_given_a_scheduled_group_when_workers_execute_then_dependencies_and_fail ("udf:udf__order_band", "success", None, None, None), ], ), + WorkerExecutionTestCase( + "a job's full_refresh, run_tests and ignore_audit_failures act as the global flags did", + "SELECT 7 AS amount", + "SELECT count(*) FROM parity_events", + "success", + [(1,)], + files={ + "functions/sql/udf__band.sql": ( + "FUNCTION (description 'Band.', arguments (amount INT), returns VARCHAR);\n" + "CASE WHEN amount >= 20 THEN 'large' ELSE 'small' END\n" + ), + "models/parity_lookup.sql": ( + "MODEL (description 'Lookup.', materialized table, " + "columns (id (type INTEGER, audits [not_null])));\n" + "SELECT CAST(NULL AS INTEGER) AS id\n" + ), + "models/parity_events.sql": ( + "MODEL (description 'Events.', materialized incremental, " + "incremental_strategy append);\n" + 'SELECT 1 AS event_id, __udf("udf__band")(5) AS band\n' + ), + "tests/unit/test_parity_events.sql": ( + "TEST();\n\nWITH\n__ref__parity_lookup AS (SELECT 1 AS id),\n" + "__expected__parity_events AS (SELECT 99 AS event_id, 'x' AS band)\nSELECT 1\n" + ), + "audits/singular/events_have_lookups.sql": ( + 'AUDIT (name "events_have_lookups", severity error);\n' + 'SELECT e.event_id FROM __ref("parity_events") e ' + 'LEFT JOIN __ref("parity_lookup") l ON e.event_id = l.id WHERE l.id IS NULL\n' + ), + "orchestration/jobs/annual_orders_job.toml": ( + 'name = "annual_orders_job"\n' + 'description = "Build orders."\n' + 'select = "orders udf__band parity_lookup parity_events"\n' + 'schedule = { cron = "0 0 * * *", timezone = "UTC", starts_at = "2000-01-01T00:00:00Z" }\n' + "\n[settings]\nfull_refresh = true\nrun_tests = false\n" + "ignore_audit_failures = true\n" + ), + }, + expected_run_nodes=[ + ("model:orders", "success", None, None, None), + ("model:parity_events", "success", None, None, None), + ("model:parity_lookup", "success", None, None, None), + ("udf:udf__band", "success", None, None, None), + ], + ), ], ids=lambda case: case.description, ) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py index d38ad1f21e..d83cd55041 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py @@ -901,7 +901,7 @@ def snapshot(schedule: str) -> dict[str, Any]: 'overlap = "skip"\n' "pools = { order_reporting = 2 }\n" f"schedule = {schedule}\n" - "\n[on_failure]\nretries = 2\nretry_delay_seconds = 600\n" + "\n[retry]\nattempts = 3\ndelay_seconds = 600\n" '\n[settings]\nrun_tests = false\nvars = { batch = 3 }\nwarehouse = "reporting"\n' "concurrency = 4\n" ), @@ -1067,23 +1067,10 @@ def test_given_concurrency_layers_when_publishing_then_each_job_and_group_carrie expected_on_failure={"retries": 2, "retry_delay_seconds": 900}, expected_warnings=(), ), - SnapshotRetryTestCase( - description="deprecated on_failure names", - tables="[on_failure]\nretries = 2\nretry_delay_seconds = 600\n", - expected_on_failure={"retries": 2, "retry_delay_seconds": 600}, - expected_warnings=( - "Warning: orchestration/jobs/daily_build_job.toml:6:1: " - "`[on_failure]` is deprecated; rename it to `[retry]`", - "Warning: orchestration/jobs/daily_build_job.toml:7:11: " - "`retries` is deprecated; rename it to `attempts`", - "Warning: orchestration/jobs/daily_build_job.toml:8:23: " - "`retry_delay_seconds` is deprecated; rename it to `delay_seconds`", - ), - ), ], ids=lambda case: case.description, ) -def test_given_a_job_retry_table_when_compiling_then_both_names_publish_one_policy_and_old_ones_warn( +def test_given_a_job_retry_table_when_compiling_then_it_publishes_one_policy_without_warnings( tmp_path: Path, capfd: pytest.CaptureFixture[str], test_case: SnapshotRetryTestCase ) -> None: text: str = "".join(f"{key} = {value}\n" for key, value in _DAILY_BUILD.items()) @@ -1212,19 +1199,24 @@ def test_given_a_job_retry_table_when_compiling_then_both_names_publish_one_poli expected_message="unknown variant `coalesce`, expected `queue` or `skip`", ), SnapshotDeclarationRejectionTestCase( - description="no retries", - tables="[on_failure]\nretries = 0\n", - expected_message="retries must be a whole number from 1 to 10", + description="a single attempt", + tables="[retry]\nattempts = 1\n", + expected_message="attempts must be a whole number from 2 to 11", ), SnapshotDeclarationRejectionTestCase( - description="too many retries", - tables="[on_failure]\nretries = 11\n", - expected_message="retries must be a whole number from 1 to 10", + description="too many attempts", + tables="[retry]\nattempts = 12\n", + expected_message="attempts must be a whole number from 2 to 11", ), SnapshotDeclarationRejectionTestCase( description="retry delay over a day", - tables="[on_failure]\nretries = 1\nretry_delay_seconds = 86401\n", - expected_message="retry_delay_seconds must be a whole number from 0 to 86400", + tables="[retry]\nattempts = 2\ndelay_seconds = 86401\n", + expected_message="delay_seconds must be a whole number from 0 to 86400", + ), + SnapshotDeclarationRejectionTestCase( + description="the removed on_failure table", + tables="[on_failure]\nretries = 2\n", + expected_message="unknown job key 'on_failure'", ), SnapshotDeclarationRejectionTestCase( description="concurrency belongs under settings", diff --git a/tests/unit/src/sqlbuild/adapter/contract/classes/strict_adapter/test_strict_adapter.py b/tests/unit/src/sqlbuild/adapter/contract/classes/strict_adapter/test_strict_adapter.py index 6605bc233c..13fe6e5206 100644 --- a/tests/unit/src/sqlbuild/adapter/contract/classes/strict_adapter/test_strict_adapter.py +++ b/tests/unit/src/sqlbuild/adapter/contract/classes/strict_adapter/test_strict_adapter.py @@ -97,6 +97,8 @@ def test_given_first_class_adapters_when_checking_contract_then_no_method_comes_ "render_dynamic_operation", "render_track_changes", "session_warehouse", + "use_session_warehouse", + "current_session_warehouse", } ), expected_output_relation=None, diff --git a/tests/unit/src/sqlbuild/executor/build/classes/_test_types.py b/tests/unit/src/sqlbuild/executor/build/classes/_test_types.py index 357a1fdaae..5ca1560d89 100644 --- a/tests/unit/src/sqlbuild/executor/build/classes/_test_types.py +++ b/tests/unit/src/sqlbuild/executor/build/classes/_test_types.py @@ -2,6 +2,9 @@ from dataclasses import dataclass +from sqlbuild._native import CompiledResourceType +from sqlbuild.executor.run.models import ModelExecutionResult + @dataclass(frozen=True) class MicrobatchBoundedFutureTestCase: @@ -37,3 +40,53 @@ class SchedulerIdentityTestCase: description: str expected_invocation_id: str expected_run_id: str + + +@dataclass(frozen=True) +class JobLimitedBuildTestCase: + """A small build graph run through the concurrent dispatch loop with per-job limits.""" + + description: str + # Each node and the nodes it waits for. + upstream: dict[str, tuple[str, ...]] + node_jobs: dict[str, tuple[str, ...]] + job_limits: dict[str, int] + max_concurrency: int + failing: frozenset[str] + # The most nodes of each named job, or of the whole build under "*", seen running at once. + expected_peaks: dict[str, int] + expected_skipped: tuple[str, ...] + + +@dataclass(frozen=True) +class SubworkerReservationTestCase: + """Running and queued microbatch coordinators, and the sub-worker slots one of them gets.""" + + description: str + job_limits: dict[str, int] + # Nodes of each job already running, the coordinator's included. + job_running: dict[str, int] + coordinator_jobs: tuple[str, ...] + queued_coordinator_jobs: tuple[tuple[str, ...], ...] + max_concurrency: int + expected_demand: int + expected_reservations: tuple[bool, ...] + + +@dataclass(frozen=True) +class SubworkerWarehouseTestCase: + """A microbatch sub-worker taking a pooled connection a previous node may have moved.""" + + description: str + coordinator_warehouse: str | None + earlier_warehouse: str | None + expected_statements: tuple[str, ...] + + +@dataclass(frozen=True) +class NodeWarehouseFailureTestCase: + description: str + node_warehouse: str | None + expected_message: str + resource_type: CompiledResourceType = CompiledResourceType.MODEL + expected_result_type: type = ModelExecutionResult diff --git a/tests/unit/src/sqlbuild/executor/build/classes/helpers.py b/tests/unit/src/sqlbuild/executor/build/classes/helpers.py new file mode 100644 index 0000000000..3749bfe7a2 --- /dev/null +++ b/tests/unit/src/sqlbuild/executor/build/classes/helpers.py @@ -0,0 +1,202 @@ +"""Drive the build scheduler's job-limited dispatch with stub workers that record concurrency.""" + +from __future__ import annotations + +import threading +import time +from collections import deque +from queue import Queue +from types import SimpleNamespace +from typing import Any, cast + +from sqlbuild._native import CompiledObjectKey, CompiledResourceType +from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter +from sqlbuild.executor.build.classes.build_scheduler import BuildScheduler +from sqlbuild.executor.build.models import BuildRuntimeParams, NodeCompletion +from sqlbuild.executor.run.models import ModelExecutionResult +from sqlbuild.executor.scheduling.classes.session_warehouses import SessionWarehouses +from sqlbuild.executor.scheduling.types import ExecutionStatus +from sqlbuild.executor.testing.types import SqlTestOutcome +from tests.unit.src.sqlbuild.executor.build.classes._test_types import ( + JobLimitedBuildTestCase, + SubworkerReservationTestCase, +) +from tests.unit.src.sqlbuild.executor.scheduling.classes.helpers import RecordingWarehouseAdapter + +# How long each stub node runs, long enough for the loop to start every node it may. +_NODE_SECONDS: float = 0.05 +_ALL_NODES: str = "*" +_STATUS_WHEN_FAILING: dict[bool, ExecutionStatus] = { + True: ExecutionStatus.FAILED, + False: ExecutionStatus.SUCCESS, +} + + +def model_key(name: str) -> CompiledObjectKey: + """The key of model `name`.""" + + return CompiledObjectKey(resource_type=CompiledResourceType.MODEL, name=name) + + +def _bare_scheduler( + *, job_limits: dict[str, int], node_jobs: dict[str, tuple[str, ...]], max_concurrency: int +) -> Any: + scheduler: Any = object.__new__(BuildScheduler) + scheduler._ready = deque() + scheduler._in_flight = set() + scheduler._blocked_keys = set() + scheduler._max_concurrency = max_concurrency + scheduler._stop = False + scheduler._microbatch_coordinator_lock = threading.Lock() + scheduler._microbatch_coordinators = set() + scheduler._microbatch_coordinator_demand = 0 + scheduler._microbatch_subworkers = 0 + scheduler._completion_queue = Queue() + scheduler._session_warehouses = None + scheduler._node_warehouses = {} + scheduler._init_job_limits( + BuildRuntimeParams( + run_id="run", + job_limits=job_limits, + node_jobs={model_key(name): jobs for name, jobs in node_jobs.items()}, + ) + ) + return scheduler + + +def run_job_limited_build( + test_case: JobLimitedBuildTestCase, +) -> tuple[dict[str, int], tuple[str, ...]]: + """Each job's, and the whole build's, peak running nodes, and the nodes skipped.""" + + scheduler: Any = _bare_scheduler( + job_limits=test_case.job_limits, + node_jobs=test_case.node_jobs, + max_concurrency=test_case.max_concurrency, + ) + lock: threading.Lock = threading.Lock() + running: dict[str, int] = {} + peaks: dict[str, int] = {} + finished: set[str] = set() + unbuilt: set[str] = set() + skipped: list[str] = [] + scheduler._ready.extend( + model_key(name) + for name in filter(lambda node: not test_case.upstream[node], test_case.upstream) + ) + + dependents: dict[str, tuple[str, ...]] = { + name: tuple(filter(lambda node: name in test_case.upstream[node], test_case.upstream)) + for name in test_case.upstream + } + + def release(name: str) -> None: + finished.add(name) + ready: tuple[str, ...] = tuple( + filter(lambda node: set(test_case.upstream[node]) <= finished, dependents[name]) + ) + scheduler._blocked_keys.update( + model_key(node) + for node in filter(lambda node: bool(unbuilt & set(test_case.upstream[node])), ready) + ) + scheduler._ready.extend(model_key(node) for node in ready) + + def count(name: str, change: int) -> None: + with lock: + for group in (*test_case.node_jobs.get(name, ()), _ALL_NODES): + running[group] = running.get(group, 0) + change + peaks[group] = max(peaks.get(group, 0), running[group]) + + def worker(key: CompiledObjectKey) -> None: + count(key.name, 1) + time.sleep(_NODE_SECONDS) + count(key.name, -1) + status: ExecutionStatus = _STATUS_WHEN_FAILING[key.name in test_case.failing] + scheduler._completion_queue.put( + NodeCompletion(key=key, result=ModelExecutionResult(model_name=key.name, status=status)) + ) + + def handle_completion(*, key: CompiledObjectKey, result: ModelExecutionResult) -> None: + unbuilt.update({key.name} & test_case.failing) + release(key.name) + + def mark_complete(key: CompiledObjectKey) -> None: + unbuilt.add(key.name) + release(key.name) + + scheduler._worker = worker + scheduler._handle_completion = handle_completion + scheduler._mark_complete = mark_complete + scheduler._record_skipped = lambda key: skipped.append(key.name) + scheduler._pre_dispatch = lambda _key: True + scheduler._is_concurrent_microbatch = lambda _key: False + scheduler._report_scheduler_state = lambda: None + scheduler._run_concurrent() + return ( + {group: peaks.get(group, 0) for group in test_case.expected_peaks}, + tuple(sorted(skipped)), + ) + + +def reserve_subworkers(test_case: SubworkerReservationTestCase) -> tuple[int, tuple[bool, ...]]: + """The coordinator demand with the queued coordinators, then each reservation's outcome.""" + + queued: dict[str, tuple[str, ...]] = { + f"queued_{index}": jobs for index, jobs in enumerate(test_case.queued_coordinator_jobs) + } + scheduler: Any = _bare_scheduler( + job_limits=test_case.job_limits, + node_jobs={"coordinator": test_case.coordinator_jobs, **queued}, + max_concurrency=test_case.max_concurrency, + ) + scheduler._job_running = dict(test_case.job_running) + scheduler._microbatch_coordinators = {model_key("coordinator")} + scheduler._ready.extend(model_key(name) for name in queued) + scheduler._is_concurrent_microbatch = lambda _key: True + scheduler._update_microbatch_coordinator_demand() + reservations: tuple[bool, ...] = tuple( + scheduler._reserve_microbatch_subworker(test_case.coordinator_jobs) + for _ in test_case.expected_reservations + ) + return scheduler._microbatch_coordinator_demand, reservations + + +def node_key(*, resource_type: CompiledResourceType, name: str) -> CompiledObjectKey: + """The key of node `name` of `resource_type`.""" + + return CompiledObjectKey(resource_type=resource_type, name=name) + + +def failed_node_result(result: Any) -> tuple[bool, str | None]: + """Whether a node result of any kind reports failure, and its error message.""" + + return ( + getattr(result, "status", None) == ExecutionStatus.FAILED + or getattr(result, "outcome", None) == SqlTestOutcome.ERROR, + getattr(result, "error_message", None), + ) + + +def source_plan(name: str) -> SimpleNamespace: + """A plan with only source `name`'s entry, as a failed load result reads it.""" + + return SimpleNamespace( + source_map={name: SimpleNamespace(name=name, loader=None, table=None, meta={})} + ) + + +def warehouse_scheduler(adapter: RecordingWarehouseAdapter) -> Any: + """A scheduler with only the state warehouse switching and sub-worker slots read.""" + + scheduler: Any = object.__new__(BuildScheduler) + scheduler._session_warehouses = SessionWarehouses( + adapter=cast(BaseAdapter, adapter), home="BUILD_WH" + ) + scheduler._node_warehouses = {} + scheduler._microbatch_coordinator_lock = threading.Lock() + scheduler._job_lock = threading.Lock() + scheduler._job_limits = {} + scheduler._job_running = {} + scheduler._microbatch_subworkers = 1 + scheduler._connection_pool = Queue() + return scheduler diff --git a/tests/unit/src/sqlbuild/executor/build/classes/test_build_scheduler_identity.py b/tests/unit/src/sqlbuild/executor/build/classes/test_build_scheduler_identity.py index 6baf9f4afa..8335747e25 100644 --- a/tests/unit/src/sqlbuild/executor/build/classes/test_build_scheduler_identity.py +++ b/tests/unit/src/sqlbuild/executor/build/classes/test_build_scheduler_identity.py @@ -43,6 +43,10 @@ def test_given_regular_primary_work_when_scheduler_submits_then_identity_is_prop scheduler._ready = deque((key,)) scheduler._in_flight = set() scheduler._blocked_keys = set() + scheduler._node_jobs = {} + scheduler._job_limits = {} + scheduler._job_running = {} + scheduler._job_lock = threading.Lock() scheduler._max_concurrency = 1 scheduler._stop = False scheduler._microbatch_coordinator_lock = threading.Lock() @@ -98,6 +102,10 @@ def test_given_microbatch_primary_work_when_scheduler_submits_then_identity_is_p scheduler._ready = deque((key,)) scheduler._in_flight = set() scheduler._blocked_keys = set() + scheduler._node_jobs = {} + scheduler._job_limits = {} + scheduler._job_running = {} + scheduler._job_lock = threading.Lock() scheduler._max_concurrency = 2 scheduler._stop = False scheduler._microbatch_coordinator_lock = threading.Lock() diff --git a/tests/unit/src/sqlbuild/executor/build/classes/test_build_scheduler_job_limits.py b/tests/unit/src/sqlbuild/executor/build/classes/test_build_scheduler_job_limits.py new file mode 100644 index 0000000000..528d0db7c1 --- /dev/null +++ b/tests/unit/src/sqlbuild/executor/build/classes/test_build_scheduler_job_limits.py @@ -0,0 +1,123 @@ +"""The build scheduler keeps each named job within its concurrency without holding up other jobs.""" + +from __future__ import annotations + +import pytest + +from tests.unit.src.sqlbuild.executor.build.classes._test_types import ( + JobLimitedBuildTestCase, + SubworkerReservationTestCase, +) +from tests.unit.src.sqlbuild.executor.build.classes.helpers import ( + reserve_subworkers, + run_job_limited_build, +) + + +@pytest.mark.parametrize( + "test_case", + [ + JobLimitedBuildTestCase( + description="a full job lets ready nodes of other work start", + upstream={"orders_a": (), "orders_b": (), "orders_c": (), "refunds": (), "crm": ()}, + node_jobs={ + "orders_a": ("orders_job",), + "orders_b": ("orders_job",), + "orders_c": ("orders_job",), + }, + job_limits={"orders_job": 1}, + max_concurrency=4, + failing=frozenset(), + expected_peaks={"orders_job": 1, "*": 3}, + expected_skipped=(), + ), + JobLimitedBuildTestCase( + description="a node in two jobs fits within the lower limit", + upstream={"shared": (), "orders_a": (), "orders_b": (), "refunds_a": ()}, + node_jobs={ + "shared": ("orders_job", "refunds_job"), + "orders_a": ("orders_job",), + "orders_b": ("orders_job",), + "refunds_a": ("refunds_job",), + }, + job_limits={"orders_job": 2, "refunds_job": 1}, + max_concurrency=4, + failing=frozenset(), + expected_peaks={"orders_job": 2, "refunds_job": 1}, + expected_skipped=(), + ), + JobLimitedBuildTestCase( + description="a failed node frees its slot and its dependents are skipped", + upstream={ + "orders_a": (), + "orders_b": (), + "orders_mart": ("orders_a",), + "orders_report": ("orders_mart",), + }, + node_jobs={ + "orders_a": ("orders_job",), + "orders_b": ("orders_job",), + "orders_mart": ("orders_job",), + "orders_report": ("orders_job",), + }, + job_limits={"orders_job": 1}, + max_concurrency=4, + failing=frozenset({"orders_a"}), + expected_peaks={"orders_job": 1, "*": 1}, + expected_skipped=("orders_mart", "orders_report"), + ), + ], + ids=lambda case: case.description, +) +def test_given_job_limits_when_running_the_dispatch_loop_then_each_job_stays_within_its_limit( + test_case: JobLimitedBuildTestCase, +) -> None: + assert run_job_limited_build(test_case) == ( + test_case.expected_peaks, + test_case.expected_skipped, + ), test_case.description + + +@pytest.mark.parametrize( + "test_case", + [ + SubworkerReservationTestCase( + description="sub-workers count against the coordinator's job", + job_limits={"events_job": 2}, + job_running={"events_job": 1}, + coordinator_jobs=("events_job",), + queued_coordinator_jobs=(), + max_concurrency=8, + expected_demand=1, + expected_reservations=(True, False), + ), + SubworkerReservationTestCase( + description="a queued coordinator whose job is full claims no sub-worker slots", + job_limits={"events_job": 3, "refunds_job": 1}, + job_running={"events_job": 1, "refunds_job": 1}, + coordinator_jobs=("events_job",), + queued_coordinator_jobs=(("refunds_job",), ("refunds_job",)), + max_concurrency=3, + expected_demand=1, + expected_reservations=(True, True, False), + ), + SubworkerReservationTestCase( + description="a queued coordinator that could start keeps its slot", + job_limits={"events_job": 3}, + job_running={"events_job": 1}, + coordinator_jobs=("events_job",), + queued_coordinator_jobs=((),), + max_concurrency=3, + expected_demand=2, + expected_reservations=(True, False), + ), + ], + ids=lambda case: case.description, +) +def test_given_microbatch_coordinators_when_reserving_sub_workers_then_job_limits_apply( + test_case: SubworkerReservationTestCase, +) -> None: + assert reserve_subworkers(test_case) == ( + test_case.expected_demand, + test_case.expected_reservations, + ), test_case.description diff --git a/tests/unit/src/sqlbuild/executor/build/classes/test_build_scheduler_microbatch.py b/tests/unit/src/sqlbuild/executor/build/classes/test_build_scheduler_microbatch.py index 48f68815cb..130c580f92 100644 --- a/tests/unit/src/sqlbuild/executor/build/classes/test_build_scheduler_microbatch.py +++ b/tests/unit/src/sqlbuild/executor/build/classes/test_build_scheduler_microbatch.py @@ -74,10 +74,14 @@ def test_given_large_microbatch_phase_when_workers_are_blocked_then_future_submi scheduler: Any = object.__new__(BuildScheduler) scheduler._max_concurrency = test_case.global_concurrency scheduler._microbatch_coordinator_lock = threading.Lock() + scheduler._job_lock = threading.Lock() + scheduler._job_limits = {} + scheduler._job_running = {} scheduler._microbatch_coordinators = {"orders"} scheduler._microbatch_coordinator_demand = 1 scheduler._microbatch_subworkers = 0 scheduler._connection_pool = Queue() + scheduler._session_warehouses = None for index in range(test_case.expected_max_subworker_futures): scheduler._connection_pool.put(f"worker-{index}") @@ -113,6 +117,7 @@ def run_phase() -> None: results.append( BuildScheduler._run_microbatch_subwork( scheduler, + jobs=(), pool=pool, coordinator_connection="coordinator", batches=batches, @@ -162,10 +167,14 @@ def test_given_coordinator_failure_when_subworkers_are_running_then_pending_batc scheduler: Any = object.__new__(BuildScheduler) scheduler._max_concurrency = test_case.global_concurrency scheduler._microbatch_coordinator_lock = threading.Lock() + scheduler._job_lock = threading.Lock() + scheduler._job_limits = {} + scheduler._job_running = {} scheduler._microbatch_coordinators = {"orders"} scheduler._microbatch_coordinator_demand = 1 scheduler._microbatch_subworkers = 0 scheduler._connection_pool = Queue() + scheduler._session_warehouses = None for index in range(test_case.expected_max_subworker_futures): scheduler._connection_pool.put(f"worker-{index}") @@ -202,6 +211,7 @@ def execute(batch: BatchWindow, connection: object) -> MicrobatchPhaseOutcome: with _TrackingThreadPoolExecutor(max_workers=test_case.expected_max_subworker_futures) as pool: outcomes: tuple[MicrobatchPhaseOutcome, ...] = BuildScheduler._run_microbatch_subwork( scheduler, + jobs=(), pool=pool, coordinator_connection="coordinator", batches=batches, diff --git a/tests/unit/src/sqlbuild/executor/build/classes/test_build_scheduler_warehouses.py b/tests/unit/src/sqlbuild/executor/build/classes/test_build_scheduler_warehouses.py new file mode 100644 index 0000000000..211c87026d --- /dev/null +++ b/tests/unit/src/sqlbuild/executor/build/classes/test_build_scheduler_warehouses.py @@ -0,0 +1,157 @@ +"""Build connections follow each node's job warehouse, microbatch sub-workers included.""" + +from __future__ import annotations + +from typing import Any + +import pytest + +from sqlbuild._native import CompiledObjectKey, CompiledResourceType +from sqlbuild.executor.build.classes.build_scheduler import BuildScheduler +from sqlbuild.executor.build.models import FunctionExecutionResult, SeedExecutionResult +from sqlbuild.executor.load.models import LoadExecutionResult +from sqlbuild.executor.run.models import BatchWindow +from sqlbuild.executor.testing.models import SqlTestExecutionResult +from tests.unit.src.sqlbuild.executor.build.classes._test_types import ( + NodeWarehouseFailureTestCase, + SubworkerWarehouseTestCase, +) +from tests.unit.src.sqlbuild.executor.build.classes.helpers import ( + failed_node_result, + node_key, + source_plan, + warehouse_scheduler, +) +from tests.unit.src.sqlbuild.executor.scheduling.classes.helpers import ( + MissingWarehouseAdapter, + RecordingWarehouseAdapter, +) + + +@pytest.mark.parametrize( + "test_case", + [ + SubworkerWarehouseTestCase( + description="a sub-worker moves its connection to the coordinator's job warehouse", + coordinator_warehouse="ORDERS_WH", + earlier_warehouse=None, + expected_statements=("ORDERS_WH",), + ), + SubworkerWarehouseTestCase( + description="a plain model's sub-worker returns a connection a job left elsewhere", + coordinator_warehouse=None, + earlier_warehouse="REFUNDS_WH", + expected_statements=("REFUNDS_WH", "BUILD_WH"), + ), + SubworkerWarehouseTestCase( + description="a connection already on the warehouse issues nothing more", + coordinator_warehouse="ORDERS_WH", + earlier_warehouse="ORDERS_WH", + expected_statements=("ORDERS_WH",), + ), + ], + ids=lambda case: case.description, +) +def test_given_a_coordinator_warehouse_when_a_subworker_runs_a_batch_then_its_connection_follows( + test_case: SubworkerWarehouseTestCase, +) -> None: + adapter: RecordingWarehouseAdapter = RecordingWarehouseAdapter(session_warehouse=None) + scheduler: Any = warehouse_scheduler(adapter) + pooled: object = object() + scheduler._session_warehouses.use(connection=pooled, warehouse=test_case.earlier_warehouse) + scheduler._connection_pool.put(pooled) + executed_on: list[object] = [] + + def execute(_batch: BatchWindow, connection: object) -> Any: + executed_on.append(connection) + return "done" + + outcome: Any = BuildScheduler._run_microbatch_batch_worker( + scheduler, + jobs=(), + warehouse=test_case.coordinator_warehouse, + batch=BatchWindow(start="0", end="1", index=0), + execute=execute, + ) + + assert ( + outcome, + executed_on == [pooled], + tuple(warehouse for _, warehouse in adapter.statements), + scheduler._connection_pool.get_nowait() is pooled, + ) == ("done", True, test_case.expected_statements, True) + + +@pytest.mark.parametrize( + "test_case", + [ + NodeWarehouseFailureTestCase( + description="a job warehouse the session cannot use fails the node with the reason", + node_warehouse="MISSING_WH", + expected_message=( + "cannot run on warehouse MISSING_WH: Object does not exist: warehouse MISSING_WH" + ), + ), + NodeWarehouseFailureTestCase( + description="a seed fails with its own kind's result", + node_warehouse="MISSING_WH", + expected_message=( + "cannot run on warehouse MISSING_WH: Object does not exist: warehouse MISSING_WH" + ), + resource_type=CompiledResourceType.SEED, + expected_result_type=SeedExecutionResult, + ), + NodeWarehouseFailureTestCase( + description="a source fails with its own kind's result", + node_warehouse="MISSING_WH", + expected_message=( + "cannot run on warehouse MISSING_WH: Object does not exist: warehouse MISSING_WH" + ), + resource_type=CompiledResourceType.SOURCE, + expected_result_type=LoadExecutionResult, + ), + NodeWarehouseFailureTestCase( + description="a udf fails with its own kind's result", + node_warehouse="MISSING_WH", + expected_message=( + "cannot run on warehouse MISSING_WH: Object does not exist: warehouse MISSING_WH" + ), + resource_type=CompiledResourceType.UDF, + expected_result_type=FunctionExecutionResult, + ), + NodeWarehouseFailureTestCase( + description="a table_fn fails with its own kind's result", + node_warehouse="MISSING_WH", + expected_message=( + "cannot run on warehouse MISSING_WH: Object does not exist: warehouse MISSING_WH" + ), + resource_type=CompiledResourceType.TABLE_FN, + expected_result_type=FunctionExecutionResult, + ), + NodeWarehouseFailureTestCase( + description="a sql_test fails with its own kind's result", + node_warehouse="MISSING_WH", + expected_message=( + "cannot run on warehouse MISSING_WH: Object does not exist: warehouse MISSING_WH" + ), + resource_type=CompiledResourceType.SQL_TEST, + expected_result_type=SqlTestExecutionResult, + ), + ], + ids=lambda case: case.description, +) +def test_given_an_unusable_job_warehouse_when_a_node_starts_then_it_fails_naming_the_warehouse( + test_case: NodeWarehouseFailureTestCase, +) -> None: + adapter: MissingWarehouseAdapter = MissingWarehouseAdapter(session_warehouse=None) + scheduler: Any = warehouse_scheduler(adapter) + key: CompiledObjectKey = node_key(resource_type=test_case.resource_type, name="orders") + scheduler._node_warehouses = {key: test_case.node_warehouse} + scheduler._plan = source_plan("orders") + + result: Any = BuildScheduler._enter_node_warehouse(scheduler, key=key, connection=object()) + + assert (type(result), failed_node_result(result)) == ( + test_case.expected_result_type, + (True, test_case.expected_message), + ) diff --git a/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/_test_types.py b/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/_test_types.py index bfd9e54a11..a1bc5d099c 100644 --- a/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/_test_types.py @@ -245,3 +245,24 @@ class PythonIdentityFingerprintWriteTestCase: expected_sql_count: int expected_fragments: tuple[str, ...] unexpected_fragments: tuple[str, ...] = () + + +@dataclass(frozen=True) +class IngressWarehouseFailureTestCase: + """An ingress node whose job warehouse the session cannot use, and what the run records.""" + + description: str + missing_warehouse_node: str + expected_python: tuple[tuple[str, PythonNodeStatus, str | None], ...] + expected_loads: tuple[tuple[str, str, str | None], ...] + expected_call_order: tuple[str, ...] + + +@dataclass(frozen=True) +class CheckWarehouseFailureTestCase: + """A check on a job warehouse the session cannot use, after an upstream of some status.""" + + description: str + upstream_status: PythonNodeStatus + expected_error_message: str | None + expected_switch_attempts: int diff --git a/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/helpers.py b/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/helpers.py index 6a7767909d..cfe319463c 100644 --- a/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/helpers.py @@ -46,6 +46,7 @@ PythonNodeRuntime, ) from sqlbuild.executor.python_nodes.types import ExecutablePythonNode +from sqlbuild.executor.scheduling.classes.session_warehouses import SessionWarehouses from sqlbuild.executor.scheduling.main._unlock_downstream import unlock_downstream_python_nodes from sqlbuild.executor.scheduling.models import LifecycleExecutionNode from sqlbuild.observability import LifecycleEvent @@ -671,3 +672,24 @@ def _raise_missing_result( (False, False): _read_default_result, (False, True): _raise_missing_result, } + + +MISSING_WAREHOUSE_FAILURE: str = ( + "cannot run on warehouse MISSING_WH: Object does not exist: warehouse MISSING_WH" +) + + +def missing_warehouse_runtime(*, adapter: Any, node: str) -> PythonNodeRuntime: + """A runtime putting `node` on MISSING_WH through `adapter`'s session warehouses.""" + + return PythonNodeRuntime( + adapter=PythonNodeContextTestAdapter(), + connection_config={}, + connection=object(), + run_id="warehouse-run", + target="dev", + vars={}, + is_reload=False, + node_warehouses={node: "MISSING_WH"}, + session_warehouses=SessionWarehouses(adapter=cast(BaseAdapter, adapter), home="BUILD_WH"), + ) diff --git a/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_node_warehouse_failures.py b/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_node_warehouse_failures.py new file mode 100644 index 0000000000..68a05f2839 --- /dev/null +++ b/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_node_warehouse_failures.py @@ -0,0 +1,134 @@ +"""A Python node whose job warehouse the session cannot use fails alone, with its own result.""" + +from __future__ import annotations + +from typing import Any + +import pytest + +from sqlbuild._native import DiscoveredCheckFunction, PythonNodeGraph +from sqlbuild.compiler.python_nodes.types import PythonNodeKind, PythonNodeStatus +from sqlbuild.executor.python_nodes._helpers.ingress_execution import ( + execute_ingress_python_loader_nodes, +) +from sqlbuild.executor.python_nodes._helpers.python_checks import execute_python_check_nodes +from sqlbuild.executor.python_nodes.models import ( + IngressCallbacks, + PythonCheckCallbacks, + PythonCheckExecutionResult, + PythonIngressLoaderExecutorResult, + PythonNodeExecutionResult, + PythonNodeRunState, +) +from tests.unit.src.sqlbuild.executor.python_nodes._helpers._test_types import ( + CheckWarehouseFailureTestCase, + IngressWarehouseFailureTestCase, +) +from tests.unit.src.sqlbuild.executor.python_nodes._helpers.helpers import ( + MISSING_WAREHOUSE_FAILURE, + build_ingress_task_loader_graph, + build_python_check_graph, + ingress_calls, + ingress_loader_function, + ingress_source_map, + missing_warehouse_runtime, + python_check_function_for_case, + reset_ingress_calls, +) +from tests.unit.src.sqlbuild.executor.scheduling.classes.helpers import MissingWarehouseAdapter + + +@pytest.mark.parametrize( + "test_case", + [ + IngressWarehouseFailureTestCase( + description="a loader that cannot switch fails as a load, after its task ran", + missing_warehouse_node="load_ingress_orders", + expected_python=(("prepare_ingress_orders", PythonNodeStatus.SUCCESS, None),), + expected_loads=(("raw_orders", "failed", MISSING_WAREHOUSE_FAILURE),), + expected_call_order=("prepare_ingress_orders",), + ), + IngressWarehouseFailureTestCase( + description="a task that cannot switch fails, and its loader never runs", + missing_warehouse_node="prepare_ingress_orders", + expected_python=( + ("prepare_ingress_orders", PythonNodeStatus.FAILED, MISSING_WAREHOUSE_FAILURE), + ), + expected_loads=(("raw_orders", "skipped", None),), + expected_call_order=(), + ), + ], + ids=lambda case: case.description, +) +def test_given_an_unusable_job_warehouse_when_running_ingress_then_only_that_node_fails( + test_case: IngressWarehouseFailureTestCase, +) -> None: + reset_ingress_calls() + graph: PythonNodeGraph = build_ingress_task_loader_graph() + adapter: MissingWarehouseAdapter = MissingWarehouseAdapter(session_warehouse=None) + + result: PythonIngressLoaderExecutorResult = execute_ingress_python_loader_nodes( + python_graph=graph, + selected_python_names=frozenset({"prepare_ingress_orders", "load_ingress_orders"}), + loader_functions=(ingress_loader_function(),), + source_map=ingress_source_map(), + runtime=missing_warehouse_runtime(adapter=adapter, node=test_case.missing_warehouse_node), + callbacks=IngressCallbacks(), + ) + + assert ( + tuple((node.node_name, node.status, node.error_message) for node in result.python_results), + tuple( + (load.source_name, load.status.value, load.error_message) + for load in result.load_results + ), + ingress_calls(), + ) == (test_case.expected_python, test_case.expected_loads, test_case.expected_call_order) + + +@pytest.mark.parametrize( + "test_case", + [ + CheckWarehouseFailureTestCase( + description="a ready check that cannot switch fails with the reason", + upstream_status=PythonNodeStatus.SUCCESS, + expected_error_message=MISSING_WAREHOUSE_FAILURE, + expected_switch_attempts=1, + ), + CheckWarehouseFailureTestCase( + description="a blocked check keeps its block and never switches", + upstream_status=PythonNodeStatus.FAILED, + expected_error_message="Upstream Python node failed: upstream_task", + expected_switch_attempts=0, + ), + ], + ids=lambda case: case.description, +) +def test_given_an_unusable_job_warehouse_when_running_a_check_then_it_fails_alone( + test_case: CheckWarehouseFailureTestCase, +) -> None: + check_function: DiscoveredCheckFunction = python_check_function_for_case("passes") + adapter: MissingWarehouseAdapter = MissingWarehouseAdapter(session_warehouse=None) + + results: tuple[PythonCheckExecutionResult, ...] = execute_python_check_nodes( + check_functions=(check_function,), + python_graph=build_python_check_graph(check_function=check_function), + upstream_python_results=( + PythonNodeExecutionResult( + node_name="upstream_task", + kind=PythonNodeKind.TASK, + status=test_case.upstream_status, + ), + ), + upstream_load_results=(), + run_state=PythonNodeRunState(), + runtime=missing_warehouse_runtime(adapter=adapter, node=check_function.name), + callbacks=PythonCheckCallbacks(), + ) + + checked: Any = results[0] + assert ( + checked.failed, + checked.error_message, + len(adapter.statements), + ) == (True, test_case.expected_error_message, test_case.expected_switch_attempts) diff --git a/tests/unit/src/sqlbuild/executor/scheduling/classes/_test_types.py b/tests/unit/src/sqlbuild/executor/scheduling/classes/_test_types.py index 8125446930..dd1df2ec22 100644 --- a/tests/unit/src/sqlbuild/executor/scheduling/classes/_test_types.py +++ b/tests/unit/src/sqlbuild/executor/scheduling/classes/_test_types.py @@ -10,3 +10,24 @@ class UnreplaceableConnectionTestCase: expected_completions: tuple[tuple[str, str], ...] expected_reconnect_attempts: int expected_maximum_seconds: float + + +@dataclass(frozen=True) +class SessionWarehousesTestCase: + """Nodes taking connections in turn, each on its job's warehouse or the build's (None).""" + + description: str + configured: str | None + session_warehouse: str | None + steps: tuple[tuple[str, str | None], ...] + expected_statements: tuple[tuple[str, str], ...] + expected_session_reads: int + + +@dataclass(frozen=True) +class SessionWithoutWarehouseTestCase: + """A connection moved to a job's warehouse when the session had no warehouse to return to.""" + + description: str + job_warehouse: str + expected_error: str diff --git a/tests/unit/src/sqlbuild/executor/scheduling/classes/helpers.py b/tests/unit/src/sqlbuild/executor/scheduling/classes/helpers.py index e18022c9d4..176fb5ed73 100644 --- a/tests/unit/src/sqlbuild/executor/scheduling/classes/helpers.py +++ b/tests/unit/src/sqlbuild/executor/scheduling/classes/helpers.py @@ -54,3 +54,28 @@ def success_completion(*, key: str, result: str) -> tuple[str, str]: def failure_completion(*, key: str, error: BaseException) -> tuple[str, str]: return key, f"failed: {error}" + + +class RecordingWarehouseAdapter: + """Records each warehouse switch; reports `session_warehouse` as the session's warehouse.""" + + def __init__(self, *, session_warehouse: str | None) -> None: + self.session_warehouse: str | None = session_warehouse + self.statements: list[tuple[Any, str]] = [] + self.session_reads: int = 0 + + def use_session_warehouse(self, *, connection: Any, warehouse: str) -> None: + self.statements.append((connection, warehouse)) + + def current_session_warehouse(self, *, connection: Any) -> str | None: + del connection + self.session_reads += 1 + return self.session_warehouse + + +class MissingWarehouseAdapter(RecordingWarehouseAdapter): + """An adapter whose session cannot use any warehouse it is asked for; records each attempt.""" + + def use_session_warehouse(self, *, connection: Any, warehouse: str) -> None: + self.statements.append((connection, warehouse)) + raise RuntimeError(f"Object does not exist: warehouse {warehouse}") diff --git a/tests/unit/src/sqlbuild/executor/scheduling/classes/test_session_warehouses.py b/tests/unit/src/sqlbuild/executor/scheduling/classes/test_session_warehouses.py new file mode 100644 index 0000000000..8784802905 --- /dev/null +++ b/tests/unit/src/sqlbuild/executor/scheduling/classes/test_session_warehouses.py @@ -0,0 +1,131 @@ +"""Build connections move to a node's warehouse and back only when the warehouse changes.""" + +from __future__ import annotations + +from typing import Any, cast + +import pytest + +from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter +from sqlbuild.errors.contracts.exceptions import ExecutorInputError +from sqlbuild.executor.scheduling.classes.session_warehouses import SessionWarehouses +from tests.unit.src.sqlbuild.executor.scheduling.classes._test_types import ( + SessionWarehousesTestCase, + SessionWithoutWarehouseTestCase, +) +from tests.unit.src.sqlbuild.executor.scheduling.classes.helpers import ( + RecordingWarehouseAdapter, +) + + +@pytest.mark.parametrize( + "test_case", + [ + SessionWarehousesTestCase( + description="nodes on the build's warehouse issue nothing, however it is spelled", + configured="BUILD_WH", + session_warehouse=None, + steps=(("a", None), ("a", "BUILD_WH"), ("a", "build_wh")), + expected_statements=(), + expected_session_reads=0, + ), + SessionWarehousesTestCase( + description="a job's nodes switch once, the next job's again, a plain node back", + configured="BUILD_WH", + session_warehouse=None, + steps=(("a", "ORDERS_WH"), ("a", "ORDERS_WH"), ("a", "REFUNDS_WH"), ("a", None)), + expected_statements=(("a", "ORDERS_WH"), ("a", "REFUNDS_WH"), ("a", "BUILD_WH")), + expected_session_reads=0, + ), + SessionWarehousesTestCase( + description="each connection keeps its own warehouse", + configured="BUILD_WH", + session_warehouse=None, + steps=(("a", "ORDERS_WH"), ("b", "ORDERS_WH"), ("b", None), ("a", "ORDERS_WH")), + expected_statements=( + ("a", "ORDERS_WH"), + ("b", "ORDERS_WH"), + ("b", "BUILD_WH"), + ("a", "BUILD_WH"), + ), + expected_session_reads=0, + ), + SessionWarehousesTestCase( + description="restoring returns every connection still on a job's warehouse", + configured="BUILD_WH", + session_warehouse=None, + steps=(("a", "ORDERS_WH"), ("b", "REFUNDS_WH")), + expected_statements=( + ("a", "ORDERS_WH"), + ("b", "REFUNDS_WH"), + ("a", "BUILD_WH"), + ("b", "BUILD_WH"), + ), + expected_session_reads=0, + ), + SessionWarehousesTestCase( + description="without a configured warehouse the session's is read once, then used", + configured=None, + session_warehouse="USER_WH", + steps=(("a", "ORDERS_WH"), ("b", "ORDERS_WH"), ("a", None)), + expected_statements=( + ("a", "ORDERS_WH"), + ("b", "ORDERS_WH"), + ("a", "USER_WH"), + ("b", "USER_WH"), + ), + expected_session_reads=1, + ), + SessionWarehousesTestCase( + description="a job on the session's own warehouse issues nothing", + configured=None, + session_warehouse="ORDERS_WH", + steps=(("a", "ORDERS_WH"), ("a", None)), + expected_statements=(), + expected_session_reads=1, + ), + ], + ids=lambda case: case.description, +) +def test_given_nodes_on_job_warehouses_when_taking_connections_then_switches_only_on_change( + test_case: SessionWarehousesTestCase, +) -> None: + adapter: RecordingWarehouseAdapter = RecordingWarehouseAdapter( + session_warehouse=test_case.session_warehouse + ) + warehouses: SessionWarehouses = SessionWarehouses( + adapter=cast(BaseAdapter, adapter), home=test_case.configured + ) + connections: dict[str, Any] = {"a": object(), "b": object()} + for name, warehouse in test_case.steps: + warehouses.use(connection=connections[name], warehouse=warehouse) + warehouses.restore() + names: dict[int, str] = {id(connection): name for name, connection in connections.items()} + + assert ( + tuple((names[id(connection)], warehouse) for connection, warehouse in adapter.statements), + adapter.session_reads, + ) == (test_case.expected_statements, test_case.expected_session_reads) + + +@pytest.mark.parametrize( + "test_case", + [ + SessionWithoutWarehouseTestCase( + description="a session without any warehouse cannot be returned to one", + job_warehouse="ORDERS_WH", + expected_error="cannot return a connection to the build's warehouse", + ), + ], + ids=lambda case: case.description, +) +def test_given_a_session_without_any_warehouse_when_returning_to_it_then_it_fails( + test_case: SessionWithoutWarehouseTestCase, +) -> None: + adapter: RecordingWarehouseAdapter = RecordingWarehouseAdapter(session_warehouse=None) + warehouses: SessionWarehouses = SessionWarehouses(adapter=cast(BaseAdapter, adapter), home=None) + connection: object = object() + warehouses.use(connection=connection, warehouse=test_case.job_warehouse) + + with pytest.raises(ExecutorInputError, match=test_case.expected_error): + warehouses.use(connection=connection, warehouse=None) diff --git a/website/src/content/docs/docs/cli/build.mdx b/website/src/content/docs/docs/cli/build.mdx index 12e7916aca..5aecdf4af2 100644 --- a/website/src/content/docs/docs/cli/build.mdx +++ b/website/src/content/docs/docs/cli/build.mdx @@ -43,6 +43,7 @@ own option. `--version` works only directly after `sqb`, as in `sqb --version`: | `--no-sql-analysis` | Disable compile-time SQL analysis (`--no-sql-validation` is an alias) | | `--no-cache` | Bypass the reusable compile-analysis cache for this invocation | | `--full-refresh` | Drop and rebuild selected models unless a model sets `full_refresh false`; `full_refresh true` forces a model even without this flag | +| `--no-job-settings` | Ignore the settings of the jobs the selection names; see [Settings when a build names a job](/docs/concepts/orchestration#settings-when-a-build-names-a-job) | | `--changed` | Build only the selected models and seeds that changed, and what they reach downstream; `--changed=definition,source_data` picks what counts. See [Building only what changed](#building-only-what-changed) | | `--reach` | With `--changed`: `job` stops at models another scheduled job refreshes, `all` builds everything downstream | | `--defer-to` | Resolve unselected model references against another target | diff --git a/website/src/content/docs/docs/cli/plan.mdx b/website/src/content/docs/docs/cli/plan.mdx index 55f7910385..4fee526405 100644 --- a/website/src/content/docs/docs/cli/plan.mdx +++ b/website/src/content/docs/docs/cli/plan.mdx @@ -30,6 +30,7 @@ Positional `SELECTOR` arguments work like `--select`, and both forms can be comb | `--ignore-audit-failures`, `--no-ignore-audit-failures` | Plan the build `sqb build` would run with the same flag; a plan whose audit failures are ignored starts with `Audit failures won't stop this build`. See [Ignoring audit failures](/docs/concepts/audits#ignoring-audit-failures) | | `--selection-diagnostics` | Warn when a selected model will build on changed upstream models that are not selected; see [Selection and staleness](/docs/concepts/planning/selection-and-staleness) | | `--full-refresh` | Plan a full rebuild for selected models unless a model sets `full_refresh false`; `full_refresh true` forces a model without this flag | +| `--no-job-settings` | Ignore the settings of the jobs the selection names; see [Settings when a build names a job](/docs/concepts/orchestration#settings-when-a-build-names-a-job) | | `--changed` | Show what `sqb build --changed` would run, which nodes it holds back and why; takes `--changed=definition,source_data` and `--reach` like [`sqb build`](/docs/cli/build#building-only-what-changed). In `--json`, the `changed` object lists `runs`, `held_back` and `unchanged` | | `--start-cursor` | Override the start cursor, typed from the selected incremental models' cursor type; fails with `S303` if the selection mixes timestamp and integer cursors. With no cursor model selected, an integer value is read as an integer cursor and anything else as an ISO timestamp | | `--end-cursor` | Override the end cursor, typed the same way as `--start-cursor` | diff --git a/website/src/content/docs/docs/cli/server.mdx b/website/src/content/docs/docs/cli/server.mdx index 12b838eb33..2cc6e0df5c 100644 --- a/website/src/content/docs/docs/cli/server.mdx +++ b/website/src/content/docs/docs/cli/server.mdx @@ -759,6 +759,10 @@ warehouse_heavy = 1 `max_workers`, write claims, `overlap` and pools are compared on [Concurrency](/docs/concepts/concurrency). +A job's run builds `--select job:`, as `sqb build --select job:` would on the command +line, so the job's `concurrency`, `full_refresh`, `run_tests` and `ignore_audit_failures` apply to +its nodes the same way. A group runs each of its jobs as its own run. + ## Run ad-hoc commands People with `run.adhoc` can run a `sqb` command on the current publication without declaring a diff --git a/website/src/content/docs/docs/concepts/adapters/snowflake.mdx b/website/src/content/docs/docs/concepts/adapters/snowflake.mdx index 29194563a5..5f0b448ef5 100644 --- a/website/src/content/docs/docs/concepts/adapters/snowflake.mdx +++ b/website/src/content/docs/docs/concepts/adapters/snowflake.mdx @@ -74,9 +74,10 @@ query = "ADHOC_WH" Each command uses the first of these that is set: 1. `--warehouse `, for that run only -2. The target's warehouse for the command's group -3. The connection's `warehouse` -4. The Snowflake user's default warehouse +2. For `sqb build`, the `warehouse` of a named job, for that job's nodes only; see below +3. The target's warehouse for the command's group +4. The connection's `warehouse` +5. The Snowflake user's default warehouse A location's syncs use the target's `sync` warehouse, else its `query` warehouse, else the connection's `warehouse`, so a sync never wakes the target's `build` warehouse just to read @@ -92,10 +93,44 @@ sync = "TINY_WH" Runs that a location executes, such as `sqb build`, use their own command's group. -Every connection a command opens uses the same warehouse. Tests and audits that run inside +Every connection a command opens starts on the same warehouse. Tests and audits that run inside `sqb build` use the `build` warehouse. Under `sqb dbt`, dbt's own nodes use the warehouse in their dbt profile. +### A job's own warehouse + +A job can run its nodes on a warehouse of its own: + +```toml +# orchestration/jobs/nightly_orders_job.toml +[settings] +warehouse = "ORDERS_WH" +``` + +A server run of the job, and a build whose named jobs all set the same warehouse, such as +`sqb build --select job:nightly_orders_job`, runs entirely on `ORDERS_WH`, as if passed +`--warehouse ORDERS_WH`: setup, planning and bookkeeping included. Nodes the build selects outside +the named jobs, such as upstream models in `--select +job:nightly_orders_job`, run there too. + +When the named jobs set different warehouses, or only some set one, each job's nodes run on its +own warehouse. Its models' audits, its SQL tests, microbatch batches, source loads and Python nodes +go with them, and loaders and Python code that open their own connections from the context's +`connection_config` get the job's warehouse there too. A SQL test runs on a job warehouse only +when every node it covers runs there. Everything else in the build, including nodes selected +without naming a job and the build's own bookkeeping, stays on the `build` warehouse. + +The connection running a node switches with `USE WAREHOUSE` only when the node needs a different +warehouse than the connection is on, and switches back before it runs a node outside the job. +Connections return to the `build` warehouse when the build ends. Without a configured warehouse, +SQLBuild reads the session's warehouse once before the first switch, so it can switch back. + +`--warehouse` puts the whole build on one warehouse and leaves the jobs' warehouses out. Two named +jobs that give one node different warehouses fail with `S027`; see +[Settings when a build names a job](/docs/concepts/orchestration#settings-when-a-build-names-a-job). +A job's warehouse must be a Snowflake identifier like a group warehouse (`C261`), and it needs the +Snowflake adapter or one built on it (`C260`). Cost estimates read each query's warehouse, so a +build's cost names every warehouse it used. + A group warehouse wins over a connection `warehouse` from either file. To change one for yourself, set the group in `sqlbuild_local.toml`; the other group keeps its project value: diff --git a/website/src/content/docs/docs/concepts/concurrency.mdx b/website/src/content/docs/docs/concepts/concurrency.mdx index 14089df20c..4eb27e70ef 100644 --- a/website/src/content/docs/docs/concepts/concurrency.mdx +++ b/website/src/content/docs/docs/concepts/concurrency.mdx @@ -35,7 +35,15 @@ concurrency = 2 ``` `--concurrency` on the command line wins over all of them; see -[Target concurrency](/docs/concepts/project-configuration#target-concurrency) for the full order. The job's page shows the value its runs use on the target and where it came from. +[Target concurrency](/docs/concepts/project-configuration#target-concurrency) for the full order. +A job's `concurrency` limits how many of that job's nodes build at once in a build that names the +job. Without `--concurrency`, the build may run as many nodes at once as the larger of the +configured limit and the named jobs' limits added together; jobs without a `concurrency` add +nothing. With a project limit of 1 and a job limit of 4, the build runs up to 4 nodes at once; with +a project limit of 16, the build allows 16 and the job's nodes still run at most 4 at once. When +one job is at its limit, a ready node of another job starts instead. Microbatch sub-work counts +against the job too. A server runs a job the same way. `--concurrency` on the command line sets +the build's limit and turns the per-job limits off; see [Settings when a build names a job](/docs/concepts/orchestration#settings-when-a-build-names-a-job). The job's page shows the value its runs use on the target and where it came from. ## Runs at once on a server: `max_workers` diff --git a/website/src/content/docs/docs/concepts/models/dynamic.mdx b/website/src/content/docs/docs/concepts/models/dynamic.mdx index 9cb53ce8ff..f2434e0b30 100644 --- a/website/src/content/docs/docs/concepts/models/dynamic.mdx +++ b/website/src/content/docs/docs/concepts/models/dynamic.mdx @@ -28,7 +28,7 @@ for the grants and change tracking they need. |-------|---------|-------------| | `refresh (lag )` | none | How far behind its inputs the model may fall: a fixed duration such as `5m`, `1h` or `1d`, at least `1m` on Snowflake and no shorter than the lag of a dynamic model it reads. Without it, the model refreshes when a dynamic model that reads it refreshes | | `refresh_mode` | `incremental` | `incremental`, `full`, `auto` or `adaptive`. Snowflake rejects `incremental` for a query it can't refresh incrementally; use `full` for that query | -| `refresh_warehouse` | the connection's warehouse | Warehouse the scheduled refreshes run on | +| `refresh_warehouse` | the target's build warehouse | Warehouse the scheduled refreshes run on; a job's `warehouse` never sets it | | `refresh_on_build` | `false` | Refresh the model in every build that selects it, not only on its schedule | ## What the plan shows diff --git a/website/src/content/docs/docs/concepts/orchestration.mdx b/website/src/content/docs/docs/concepts/orchestration.mdx index 27b50e0a87..9715517ccd 100644 --- a/website/src/content/docs/docs/concepts/orchestration.mdx +++ b/website/src/content/docs/docs/concepts/orchestration.mdx @@ -84,6 +84,54 @@ cpus = 2 A job's `[settings]` cannot set `select` or `exclude`: the job's own selection is the run's selection. +### Settings when a build names a job + +A build or plan whose selection names a job, as `job:nightly_orders_job`, the bare name or a +`job:` pattern such as `job:nightly_*`, takes the job's `[settings]`. Selecting the same nodes some other way, such as by tag, takes none of them. +Naming a group names each of its jobs. The first value set wins: + +1. A flag on the command line, such as `--vars` or `--concurrency` +2. The named job's setting +3. The target's setting +4. The project's setting + +A command-line flag replaces every named job's setting rather than merging with it, so it also +settles named jobs that disagree: `--vars` replaces the job's whole `vars`, `--full-refresh` +rebuilds every selected model, and `--warehouse` puts every node on its warehouse. An option that rules another out counts as setting both, so `--reload` wins +over a job's `load_sources = false`. Any `--concurrency` sets the build's limit and turns off the +per-job limits. `SQLBUILD_CONCURRENCY` is not a command-line flag: it sets the build's limit as a +configured one does, and the named jobs' limits still apply. The job's settings are added before a `--` on the command line, so selectors after +it still name jobs. `--no-job-settings` leaves out every job's settings. + +Some settings apply to the job's own nodes only, so a build that names two jobs can treat their +nodes differently: + +| Setting | Applies to | +| --- | --- | +| `full_refresh` | The job's incremental models, rebuilt from scratch, and its functions, recreated | +| `run_tests` | The SQL unit tests of the job's models | +| `ignore_audit_failures` | The audits of the job's nodes, and singular audits that read them; a singular audit is ignored only when every node it reads ignores failures, by its job's setting or else the build's | +| `warehouse` | The Snowflake warehouse the job's nodes run on, Python nodes included; see [A job's own warehouse](/docs/concepts/adapters/snowflake#a-jobs-own-warehouse) | +| `concurrency` | How many of the job's nodes build at once, and a node in two named jobs fits within both. The build's own limit becomes the larger of the configured limit and the named jobs' limits added together | +| `[retry]` | The job's nodes | + +A job's settings reach only the nodes its own selection holds. `--select job:nightly_orders_job,orders` +gives them to `orders` alone, and `+job:nightly_orders_job` builds the job's upstream nodes without +them. + +Every other setting, such as `vars`, `changed` or `fail_fast`, applies to the whole build, so every +named job must set it the same way; a job that leaves it unset disagrees with one that sets it. +Pass the flag to settle a disagreement. When every named job sets the same `warehouse`, it is the +whole build's warehouse, as if passed as `--warehouse`; see +[A job's own warehouse](/docs/concepts/adapters/snowflake#a-jobs-own-warehouse). + +Each job in a group still runs as its own build, so a group's jobs must agree only on `vars` and +`changed`, which change the compiled SQL. A group whose jobs set either differently on a target +that declares them together fails to compile, with the error a build of the group would give. + +Two named jobs that give one node different values of the same setting fail the build with `S027`, +naming the node, both jobs and the setting. A setting only one of them sets is no conflict. + Select a pipeline by its folder or a tag rather than listing its nodes. Rule `SQBRORCHESTRATION101` reports a job or sensor `select` that names more than three nodes one by one; see [Orchestration Rules](/docs/concepts/rules/orchestration-rules). @@ -270,6 +318,7 @@ Every problem with an orchestration file is a `D017` error at its file, line and | `job nightly_orders_job resources: cpus must be a positive number` | Set `cpus` above zero | | `job 'nightly_orders_job' uses pool 'warehouse_hevy', which [pools] in sqlbuild_project.toml does not declare` | Fix the name, or declare the pool in [`[pools]`](/docs/concepts/project-configuration#pools) | | `job 'nightly_orders_job' takes 2 slots of pool 'supplier_api', more than its limit of 1 on target 'production'` | Take fewer slots, or raise the limit in `[targets.production.pools]` | +| `jobs orders_job and crm_job set vars differently ({"amount":"5"} and {"amount":"9"}); a build applies vars to everything it runs; group daily_job_group builds its jobs together` | Give the group's jobs the same `vars` and `changed`, or move one job to another group | | `sensor 'orders_sensor' is declared here, but importing the module does not define it` | Assign the sensor to a module-level name or list, not a dictionary or function body | | `group dependencies contain a cycle in job group daily_orders_job_group: orders_marts_job waits for staging_orders_job (model:orders reads model:stg_orders); staging_orders_job waits for orders_marts_job (model:stg_orders reads seed:regions)` | Move each node to one job, and give seeds and nodes that several members read their own job that runs first | @@ -279,6 +328,11 @@ Details: a job selector never imports a calendar or sensor module. - A job or group declared with `targets` is selectable only when the command's target is one of them, as on the server. +- Job settings that cannot apply together fail the build or plan with `S027`, such as `model:orders + is selected through jobs orders_job and finance_job, which set run_tests differently (false and + true)`. The help names the way out: make the jobs agree, set a node setting in only one of + them, or use `--no-job-settings`. It offers a flag only for the value the flag gives, such as + `--no-tests` to turn tests off for all; no flag turns them back on. - Selectors are checked for syntax at compile. A selector that names a node the project does not have fails when a location publishes, such as `job nightly_orders_job: unknown selector name 'order_total'`. diff --git a/website/src/content/docs/docs/concepts/project-configuration.mdx b/website/src/content/docs/docs/concepts/project-configuration.mdx index 827f300ec7..b6e4e12f66 100644 --- a/website/src/content/docs/docs/concepts/project-configuration.mdx +++ b/website/src/content/docs/docs/concepts/project-configuration.mdx @@ -248,9 +248,12 @@ concurrency = 8 value set in this order: 1. `--concurrency` on the command line, for that invocation only -2. The selected target's `concurrency`, from `[targets.]` in either file -3. `[settings] concurrency` in `sqlbuild_local.toml` -4. `[settings] concurrency` in `sqlbuild_project.toml` (default `1`) +2. For `sqb build`, the named jobs' `concurrency` added together, when larger than the limit the + layers below give; each job's own limit still applies to its nodes. See + [Settings when a build names a job](/docs/concepts/orchestration#settings-when-a-build-names-a-job) +3. The selected target's `concurrency`, from `[targets.]` in either file +4. `[settings] concurrency` in `sqlbuild_local.toml` +5. `[settings] concurrency` in `sqlbuild_project.toml` (default `1`) The value must be an integer of at least `1`; anything else fails with `D001`. `sqb build --verbose` shows the effective limit and the setting it came from, for example diff --git a/website/src/content/docs/docs/concepts/selectors.mdx b/website/src/content/docs/docs/concepts/selectors.mdx index e8ec7d9eaa..6d6f2cff26 100644 --- a/website/src/content/docs/docs/concepts/selectors.mdx +++ b/website/src/content/docs/docs/concepts/selectors.mdx @@ -95,6 +95,8 @@ sqb build --select "job:nightly_orders_job,tag:finance" ``` A job selects its own `select` minus its own `exclude`. A job group selects the union of its jobs. +Naming a job also applies its settings to its nodes; see +[Settings when a build names a job](/docs/concepts/orchestration#settings-when-a-build-names-a-job). A job or job group declared with `targets` is selectable only when the command's target is one of them, as on the server. Job selectors compose with `+`, `,` and `--exclude` like any other selector. A job's own selection