diff --git a/clients/typescript/src/schema.d.ts b/clients/typescript/src/schema.d.ts index 238a3410c8..9e47129e2d 100644 --- a/clients/typescript/src/schema.d.ts +++ b/clients/typescript/src/schema.d.ts @@ -1890,11 +1890,6 @@ export interface components { /** Format: uuid */ snapshot_id?: string | null; }; - /** - * @description The layer that set a run's concurrency. - * @enum {string} - */ - ConcurrencySource: "job" | "target" | "local" | "project" | "default"; /** @description A row whose key was already stored with different content; the first row is kept. */ ConflictEvidence: { existing_sha256: string; @@ -3183,11 +3178,10 @@ export interface components { /** Format: uuid */ source_run_id: string; }; - /** @description The models a run builds at once on its target, and the layer that set it. */ + /** @description The most of a job's nodes a run of it builds at once on its target. */ ResolvedConcurrency: { /** Format: int64 */ limit: number; - source: components["schemas"]["ConcurrencySource"]; }; RoleAssignmentRequest: { /** Format: uuid */ diff --git a/crates/sqlbuild-cli/tests/native_commands/test_control.rs b/crates/sqlbuild-cli/tests/native_commands/test_control.rs index f163a688d1..197e53199f 100644 --- a/crates/sqlbuild-cli/tests/native_commands/test_control.rs +++ b/crates/sqlbuild-cli/tests/native_commands/test_control.rs @@ -379,13 +379,13 @@ fn given_a_control_store_when_running_server_migrate_then_dry_run_lists_and_migr StoreMigrationCommandTestCase { description: "migrate an empty store", arguments: &["server", "migrate"], - expected_stdout: "The control store is empty: created schema version 3.\n", + expected_stdout: "The control store is empty: created schema version 4.\n", expected_store: true, }, StoreMigrationCommandTestCase { description: "dry run on a current store", arguments: &["server", "migrate", "--dry-run"], - expected_stdout: "The control store is at schema version 3: nothing to migrate.\n", + expected_stdout: "The control store is at schema version 4: nothing to migrate.\n", expected_store: true, }, ]; diff --git a/crates/sqlbuild-config/src/effective/main/effective_concurrency.rs b/crates/sqlbuild-config/src/effective/main/effective_concurrency.rs index 16a6aa60eb..7aeb732401 100644 --- a/crates/sqlbuild-config/src/effective/main/effective_concurrency.rs +++ b/crates/sqlbuild-config/src/effective/main/effective_concurrency.rs @@ -1,28 +1,14 @@ -//! `effective_concurrency`: the worker limit of one command and the layer it came from. +//! `effective_concurrency`: the worker limit of one command. -use crate::effective::models::{ConcurrencyLayers, ConcurrencySource, EffectiveConcurrency}; +use crate::effective::models::{ConcurrencyLayers, EffectiveConcurrency}; /// `--concurrency`, else the target's `concurrency`, else the local-then-project setting. #[must_use] -pub fn effective_concurrency(layers: &ConcurrencyLayers<'_>) -> EffectiveConcurrency { - if let Some(limit) = layers.requested { - return EffectiveConcurrency { - limit, - source: ConcurrencySource::CommandLine, - }; - } - if let Some((target, limit)) = layers.target { - return EffectiveConcurrency { - limit, - source: ConcurrencySource::Target(target.to_owned()), - }; - } +pub fn effective_concurrency(layers: &ConcurrencyLayers) -> EffectiveConcurrency { EffectiveConcurrency { - limit: layers.settings, - source: if layers.local_override { - ConcurrencySource::LocalSettings - } else { - ConcurrencySource::ProjectSettings - }, + limit: layers + .requested + .or(layers.target) + .unwrap_or(layers.settings), } } diff --git a/crates/sqlbuild-config/src/effective/models.rs b/crates/sqlbuild-config/src/effective/models.rs index b1b11cd89c..3e380de013 100644 --- a/crates/sqlbuild-config/src/effective/models.rs +++ b/crates/sqlbuild-config/src/effective/models.rs @@ -83,15 +83,13 @@ pub struct TargetChoice { /// The concurrency each configuration layer sets, highest precedence first. #[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub struct ConcurrencyLayers<'a> { +pub struct ConcurrencyLayers { /// `--concurrency` on the command line. pub requested: Option, - /// The selected target's name and its `concurrency`, when it sets one. - pub target: Option<(&'a str, i64)>, + /// The selected target's `concurrency`, when it sets one. + pub target: Option, /// `[settings] concurrency` after any `sqlbuild_local.toml` override. pub settings: i64, - /// Whether `sqlbuild_local.toml` `[settings]` overrides `concurrency`. - pub local_override: bool, } /// Whether audit failures are ignored at each configuration layer, highest precedence first. @@ -105,32 +103,8 @@ pub struct AuditFailureLayers { pub settings: bool, } -/// Where the effective concurrency came from. -#[derive(Clone, Debug, PartialEq, Eq)] -pub enum ConcurrencySource { - CommandLine, - Target(String), - LocalSettings, - ProjectSettings, - /// The named jobs' `concurrency` limits added together, when larger than the configured limit. - NamedJobs, -} - -impl std::fmt::Display for ConcurrencySource { - fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - Self::CommandLine => formatter.write_str("--concurrency"), - Self::Target(name) => write!(formatter, "targets.{name}.concurrency"), - Self::LocalSettings => formatter.write_str("sqlbuild_local.toml settings.concurrency"), - Self::ProjectSettings => formatter.write_str("settings.concurrency"), - Self::NamedJobs => formatter.write_str("named jobs' concurrency"), - } - } -} - -/// One command's worker limit and the layer that set it. +/// One command's worker limit. #[derive(Clone, Debug, PartialEq, Eq)] pub struct EffectiveConcurrency { pub limit: i64, - pub source: ConcurrencySource, } diff --git a/crates/sqlbuild-config/src/effective/tests/test_effective.rs b/crates/sqlbuild-config/src/effective/tests/test_effective.rs index a9a9f07c96..d25081d82a 100644 --- a/crates/sqlbuild-config/src/effective/tests/test_effective.rs +++ b/crates/sqlbuild-config/src/effective/tests/test_effective.rs @@ -305,38 +305,23 @@ fn given_concurrency_layers_when_resolving_then_cli_beats_target_beats_settings( ConcurrencyTestCase { description: "the command line beats the target", requested: Some(2), - target: Some(("prod", 8)), + target: Some(8), settings: 16, - local_override: true, expected_limit: 2, - expected_source: "--concurrency", }, ConcurrencyTestCase { - description: "the target beats a local settings override", + description: "the target beats the settings", requested: None, - target: Some(("prod", 8)), + target: Some(8), settings: 4, - local_override: true, expected_limit: 8, - expected_source: "targets.prod.concurrency", }, ConcurrencyTestCase { - description: "a local settings override applies without a target value", - requested: None, - target: None, - settings: 4, - local_override: true, - expected_limit: 4, - expected_source: "sqlbuild_local.toml settings.concurrency", - }, - ConcurrencyTestCase { - description: "the project setting is the default", + description: "the settings apply without a target value", requested: None, target: None, settings: 16, - local_override: false, expected_limit: 16, - expected_source: "settings.concurrency", }, ]; @@ -345,11 +330,9 @@ fn given_concurrency_layers_when_resolving_then_cli_beats_target_beats_settings( requested: test_case.requested, target: test_case.target, settings: test_case.settings, - local_override: test_case.local_override, }); assert_eq!( - (resolved.limit, resolved.source.to_string().as_str()), - (test_case.expected_limit, test_case.expected_source), + resolved.limit, test_case.expected_limit, "{}", test_case.description ); diff --git a/crates/sqlbuild-config/src/effective/tests/test_types.rs b/crates/sqlbuild-config/src/effective/tests/test_types.rs index eb1276509b..e08d48f274 100644 --- a/crates/sqlbuild-config/src/effective/tests/test_types.rs +++ b/crates/sqlbuild-config/src/effective/tests/test_types.rs @@ -52,11 +52,9 @@ pub(super) struct MessageTestCase { pub(super) struct ConcurrencyTestCase { pub(super) description: &'static str, pub(super) requested: Option, - pub(super) target: Option<(&'static str, i64)>, + pub(super) target: Option, pub(super) settings: i64, - pub(super) local_override: bool, pub(super) expected_limit: i64, - pub(super) expected_source: &'static str, } pub(super) struct AuditFailureTestCase { diff --git a/crates/sqlbuild-config/src/project/_helpers/documents.rs b/crates/sqlbuild-config/src/project/_helpers/documents.rs index 9b85455043..3c940a56aa 100644 --- a/crates/sqlbuild-config/src/project/_helpers/documents.rs +++ b/crates/sqlbuild-config/src/project/_helpers/documents.rs @@ -518,15 +518,6 @@ pub(crate) fn pool_limits_config(paths: &ConfigPaths) -> Result Result, LoadFailure> { - let file = resolve(paths, false)?; - let table = load_toml_mapping(paths, file)?; - let settings = - get(table.as_map().unwrap_or_default(), "settings").and_then(ConfigValue::as_map); - Ok(settings.map(setting_overrides).unwrap_or_default()) -} - /// `[git] base_ref` of `sqlbuild_project.toml`. pub(crate) fn git_base_ref(paths: &ConfigPaths) -> Result, LoadFailure> { project_section(paths, "git", load_git) diff --git a/crates/sqlbuild-config/src/project/main/load_project_setting_keys.rs b/crates/sqlbuild-config/src/project/main/load_project_setting_keys.rs deleted file mode 100644 index 3795ac8257..0000000000 --- a/crates/sqlbuild-config/src/project/main/load_project_setting_keys.rs +++ /dev/null @@ -1,9 +0,0 @@ -//! Read which `[settings]` keys `sqlbuild_project.toml` sets itself, rather than leaving to defaults. - -use crate::project::errors::LoadFailure; -use crate::project::models::ConfigPaths; - -/// The `[settings]` keys the project file sets, under their current names. -pub fn load_project_setting_keys(paths: &ConfigPaths) -> Result, LoadFailure> { - crate::project::_helpers::documents::project_setting_keys(paths) -} diff --git a/crates/sqlbuild-config/src/project/main/mod.rs b/crates/sqlbuild-config/src/project/main/mod.rs index abf7b235dc..5785c9a6fc 100644 --- a/crates/sqlbuild-config/src/project/main/mod.rs +++ b/crates/sqlbuild-config/src/project/main/mod.rs @@ -7,7 +7,6 @@ pub mod load_git_base_ref; pub mod load_local_config; pub mod load_pool_limits; pub mod load_project_config; -pub mod load_project_setting_keys; pub mod load_review_config; pub mod local_sql_analysis_failure; pub mod microbatch_guidance; diff --git a/crates/sqlbuild-core/src/build_command/main/render_run_context.rs b/crates/sqlbuild-core/src/build_command/main/render_run_context.rs index 7be1e19f18..fcbbebbdb9 100644 --- a/crates/sqlbuild-core/src/build_command/main/render_run_context.rs +++ b/crates/sqlbuild-core/src/build_command/main/render_run_context.rs @@ -48,15 +48,7 @@ pub fn render_run_context(facts: &RunContextFacts, style: CliStyle) -> String { ("warehouse", warehouse), ( "concurrency", - if facts.concurrency_source.is_empty() { - format!("{} configured limit", facts.concurrency) - } else { - format!( - "{} configured limit ({})", - facts.concurrency, - sanitize(&facts.concurrency_source) - ) - }, + format!("{} configured limit", facts.concurrency), ), ("full_refresh", full_refresh(facts)), ( diff --git a/crates/sqlbuild-core/src/build_command/models.rs b/crates/sqlbuild-core/src/build_command/models.rs index 1ab51524d8..463ed782e8 100644 --- a/crates/sqlbuild-core/src/build_command/models.rs +++ b/crates/sqlbuild-core/src/build_command/models.rs @@ -179,8 +179,6 @@ pub struct RunContextFacts { /// The warehouse and where it came from. pub warehouse: Option<(String, String)>, pub concurrency: i64, - /// The configuration layer the concurrency came from. - pub concurrency_source: String, pub full_refresh: bool, /// The named jobs whose own models the build rebuilds from scratch. pub full_refresh_jobs: Vec, diff --git a/crates/sqlbuild-core/src/build_command/tests/test_build_command.rs b/crates/sqlbuild-core/src/build_command/tests/test_build_command.rs index eec9cd311d..ed737c0512 100644 --- a/crates/sqlbuild-core/src/build_command/tests/test_build_command.rs +++ b/crates/sqlbuild-core/src/build_command/tests/test_build_command.rs @@ -121,13 +121,12 @@ fn given_run_context_when_rendering_then_sanitizes_and_groups_counts() { command: "sqb build".to_owned(), run_id: Some("run\u{1b}1".to_owned()), concurrency: 4, - concurrency_source: "targets.prod.concurrency".to_owned(), selected_count: 1200, total_count: 1_234_567, date_vars: (Some("2026-01-01".to_owned()), Some("2026-01-31".to_owned())), ..RunContextFacts::default() }, - expected_text: "Execution\n command sqb build\n run_id run\\x1b1\n target not set\n database not set\n schema not set\n warehouse not set\n concurrency 4 configured limit (targets.prod.concurrency)\n full_refresh false\n selected 1,200 of 1,234,567 build resources\n date vars 2026-01-01 to 2026-01-31\n\n", + expected_text: "Execution\n command sqb build\n run_id run\\x1b1\n target not set\n database not set\n schema not set\n warehouse not set\n concurrency 4 configured limit\n full_refresh false\n selected 1,200 of 1,234,567 build resources\n date vars 2026-01-01 to 2026-01-31\n\n", }, ContextTestCase { description: "a warehouse, one date and selector files", diff --git a/crates/sqlbuild-orchestration/src/declarations/models.rs b/crates/sqlbuild-orchestration/src/declarations/models.rs index bddd2e927a..c57d6a13d2 100644 --- a/crates/sqlbuild-orchestration/src/declarations/models.rs +++ b/crates/sqlbuild-orchestration/src/declarations/models.rs @@ -38,7 +38,7 @@ pub struct JobDefinition { /// Each pool the job takes, with its amount and the pool's limit on the target. #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] pub pools: BTreeMap, - /// Models a run builds at once on the target, and the layer that set it; absent from older snapshots. + /// The most of the job's nodes a run builds at once on the target; absent from older snapshots. #[serde(default, skip_serializing_if = "Option::is_none")] pub resolved_concurrency: Option, /// The selectors as written, for display; `selection` is what runs. @@ -181,28 +181,11 @@ pub struct WaitReason { pub to_node: String, } -/// The layer that set a run's concurrency. -#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] -#[serde(rename_all = "snake_case")] -pub enum ConcurrencySource { - /// The job's own `[settings] concurrency`. - Job, - /// The target's `concurrency` in `[targets.]`. - Target, - /// `[settings] concurrency` in the location's `sqlbuild_local.toml`. - Local, - /// `[settings] concurrency` in `sqlbuild_project.toml`. - Project, - /// SQLBuild's built-in default. - Default, -} - -/// The models a run builds at once on its target, and the layer that set it. +/// The most of a job's nodes a run of it builds at once on its target. #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] #[serde(deny_unknown_fields)] pub struct ResolvedConcurrency { pub limit: u64, - pub source: ConcurrencySource, } /// How many slots of a pool a job takes, and the pool's limit on the job's target. @@ -324,7 +307,7 @@ pub struct JobRecord { /// The job's own `ignore_audit_failures` setting; absent uses the target's, then the project's. #[serde(default, skip_serializing_if = "Option::is_none")] pub ignore_audit_failures: Option, - /// The concurrency a run uses on the snapshot's target and where it came from; absent for older snapshots. + /// The most of the job's nodes a run builds at once on the snapshot's target; absent for older snapshots. pub resolved_concurrency: Option, pub snapshot_id: uuid::Uuid, /// A job's selectors as written; absent for a group and for snapshots published before they were recorded. @@ -359,7 +342,7 @@ pub struct JobDetailRecord { /// The job's own `ignore_audit_failures` setting; absent uses the target's, then the project's. #[serde(default, skip_serializing_if = "Option::is_none")] pub ignore_audit_failures: Option, - /// The concurrency a run uses on the snapshot's target and where it came from; absent for older snapshots. + /// The most of the job's nodes a run builds at once on the snapshot's target; absent for older snapshots. pub resolved_concurrency: Option, /// Command settings; `vars` and a `defer_*` value that is not a target name show as `[REDACTED]`, and text values pass through secret redaction. pub settings: BTreeMap, diff --git a/crates/sqlbuild-orchestration/src/store_migrations/_helpers/migration.rs b/crates/sqlbuild-orchestration/src/store_migrations/_helpers/migration.rs index de16a20674..0c3885b33b 100644 --- a/crates/sqlbuild-orchestration/src/store_migrations/_helpers/migration.rs +++ b/crates/sqlbuild-orchestration/src/store_migrations/_helpers/migration.rs @@ -25,6 +25,11 @@ pub(crate) const MIGRATIONS: &[StoreMigration] = &[ sqlite: include_str!("../sql/0003_attention_acknowledgement_index.sqlite.sql"), postgres: include_str!("../sql/0003_attention_acknowledgement_index.postgres.sql"), }, + StoreMigration { + name: "drop_concurrency_source", + sqlite: include_str!("../sql/0004_drop_concurrency_source.sqlite.sql"), + postgres: include_str!("../sql/0004_drop_concurrency_source.postgres.sql"), + }, ]; /// The schema version a store created by this build has. diff --git a/crates/sqlbuild-orchestration/src/store_migrations/sql/0004_drop_concurrency_source.postgres.sql b/crates/sqlbuild-orchestration/src/store_migrations/sql/0004_drop_concurrency_source.postgres.sql new file mode 100644 index 0000000000..d7e07a9250 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store_migrations/sql/0004_drop_concurrency_source.postgres.sql @@ -0,0 +1,3 @@ +-- Published jobs no longer record which layer set their run concurrency: drop the removed +-- resolved_concurrency.source from each job's stored definition, keeping its limit. +UPDATE sqb_jobs SET definition_json = definition_json #- ARRAY['resolved_concurrency', 'source'] WHERE definition_json -> 'resolved_concurrency' -> 'source' IS NOT NULL; diff --git a/crates/sqlbuild-orchestration/src/store_migrations/sql/0004_drop_concurrency_source.sqlite.sql b/crates/sqlbuild-orchestration/src/store_migrations/sql/0004_drop_concurrency_source.sqlite.sql new file mode 100644 index 0000000000..0d72b77ddf --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store_migrations/sql/0004_drop_concurrency_source.sqlite.sql @@ -0,0 +1,3 @@ +-- Published jobs no longer record which layer set their run concurrency: drop the removed +-- resolved_concurrency.source from each job's stored definition, keeping its limit. +UPDATE sqb_jobs SET definition_json = json_remove(definition_json, '$.resolved_concurrency.source') WHERE json_extract(definition_json, '$.resolved_concurrency.source') IS NOT NULL; diff --git a/crates/sqlbuild-orchestration/tests/runs/test_job_detail.rs b/crates/sqlbuild-orchestration/tests/runs/test_job_detail.rs index ebc26a0c8a..b1c3439ec7 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_job_detail.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_job_detail.rs @@ -72,7 +72,7 @@ fn given_published_jobs_when_people_read_and_launch_them_then_listing_view_and_r "declared_at": {"path": "orchestration/jobs.py", "line": 7}, "owner": "Orders team", "overlap": "skip", "pools": {"warehouse": {"amount": 3, "limit": 4}}, - "resolved_concurrency": {"limit": 4, "source": "job"}, + "resolved_concurrency": {"limit": 4}, "on_failure": {"retries": 2, "retry_delay_seconds": 300}}, "customers": {"description": "Build customers", "selection": ["customers"]}, })); @@ -231,10 +231,10 @@ fn given_published_jobs_when_people_read_and_launch_them_then_listing_view_and_r json!([{"upstream_job": "daily_orders", "from_node": "orders", "to_node": "customers"}]), json!([{"pool": "warehouse", "amount": 3, "limit": 4, "cap": 2, "server_limit": 2, "clamped": true, "shared_with": ["orders_group"]}]), - (Some(4), json!({"limit": 4, "source": "job"})), + (Some(4), json!({"limit": 4})), json!([{"pool": "warehouse", "amount": 3, "limit": 4, "cap": 2, "server_limit": 2, "clamped": true, "shared_with": ["orders_group"]}]), - (Some(4), json!({"limit": 4, "source": "job"})), + (Some(4), json!({"limit": 4})), json!([{"pool": "warehouse", "amount": 1, "limit": 4, "cap": 2, "server_limit": 2, "clamped": false, "shared_with": ["daily_orders"]}]), json!(null) diff --git a/crates/sqlbuild-orchestration/tests/scheduler/helpers.rs b/crates/sqlbuild-orchestration/tests/scheduler/helpers.rs index b892b9daaa..8f86616f18 100644 --- a/crates/sqlbuild-orchestration/tests/scheduler/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler/helpers.rs @@ -99,3 +99,184 @@ pub(crate) async fn leadership_contract(backend: Backend) -> Option { close_test_store(test_store).await; Some(epoch) } + +/// The operator a test's manual run requests come from. +pub(crate) fn operator() -> crate::credentials::models::AuditContext { + crate::credentials::models::AuditContext { + actor: "test-operator".into(), + granting_role: "control-store-owner".into(), + request_id: Uuid::now_v7(), + ip_address: None, + } +} + +/// A manual request for one run of `job_key` on the development target. +pub(crate) fn manual_run(location_id: Uuid, job_key: &str) -> crate::runs::models::JobRunRequest { + crate::runs::models::JobRunRequest { + run_id: Uuid::now_v7(), + location_id, + target: "development".into(), + job_key: job_key.into(), + reason: "Build it now".into(), + priority: 0, + window: None, + full_refresh: false, + select: Vec::new(), + exclude: Vec::new(), + } +} + +/// Write every published job's definition back with the concurrency source an older release recorded. +pub(crate) async fn store_old_job_definitions(store: &ControlStore, source: &str) { + use crate::runs::tests::helpers::{json_column, rows}; + use crate::store::main::{begin::begin, commit::commit, query::query}; + use crate::store::models::{Parameter, TransactionMode}; + + let jobs = rows( + store, + "SELECT job_key, definition_json FROM sqb_jobs", + vec![], + ) + .await; + let mut transaction = begin(store, TransactionMode::Write).await.expect("begin"); + for job in jobs { + let mut definition: serde_json::Value = json_column(&job["definition_json"]); + definition["resolved_concurrency"]["source"] = serde_json::json!(source); + (transaction, _) = query( + transaction, + "UPDATE sqb_jobs SET definition_json=$1 WHERE job_key=$2", + vec![ + Parameter::Json(definition), + Parameter::Text(job["job_key"].as_str().expect("job key").to_owned()), + ], + ) + .await + .expect("store the old definition"); + } + commit(transaction).await.expect("commit"); +} + +/// Point each snapshot's `jobs` and `groups` documents at old-form bodies under hashes of their own, as an older release stored them. +pub(crate) async fn store_old_documents(store: &ControlStore, source: &str) { + use crate::runs::tests::helpers::rows; + use crate::store::main::{ + begin::begin, commit::commit, compress::compress, decompress::decompress, query::query, + }; + use crate::store::models::{Parameter, TransactionMode}; + + let documents = rows( + store, + "SELECT b.document_hash, b.body FROM sqb_snapshot_documents d \ + JOIN sqb_documents b ON b.document_hash=d.document_hash WHERE d.kind IN ('jobs', 'groups')", + vec![], + ) + .await; + let mut transaction = begin(store, TransactionMode::Write).await.expect("begin"); + for document in documents { + let mut body: serde_json::Value = + serde_json::from_slice(&decompress(&document["body"]).expect("body")).expect("JSON"); + for job in body.as_object_mut().expect("jobs or groups").values_mut() { + job["resolved_concurrency"]["source"] = serde_json::json!(source); + } + let hash = document["document_hash"].as_str().expect("hash").to_owned(); + let old_hash = format!("{hash}-published-before-0004"); + (transaction, _) = query( + transaction, + "INSERT INTO sqb_documents (document_hash, body) VALUES ($1, $2)", + vec![ + Parameter::Text(old_hash.clone()), + Parameter::Bytes( + compress(&serde_json::to_vec(&body).expect("JSON")).expect("gzip"), + ), + ], + ) + .await + .expect("store the old document"); + (transaction, _) = query( + transaction, + "UPDATE sqb_snapshot_documents SET document_hash=$1 WHERE document_hash=$2", + vec![Parameter::Text(old_hash), Parameter::Text(hash)], + ) + .await + .expect("point the snapshots at it"); + } + commit(transaction).await.expect("commit"); +} + +/// A control-store result as a test compares it: `ok`, or the error as `Invalid("...")` and the like. +pub(crate) fn outcome(result: Result) -> String { + result.map_or_else(|error| format!("{error:?}"), |_| "ok".to_owned()) +} + +/// The scheduled orders snapshot with job `orders`, a group `nightly_group` of it and `triggers`, as a release before 0004 published it once its source is stored. +pub(crate) fn upgrade_snapshot( + location_id: Uuid, + triggers: serde_json::Value, +) -> crate::snapshots::models::SnapshotInput { + let mut input = crate::scheduler::tests::contracts::ticks::helpers::snapshot(location_id); + input.documents.insert( + "jobs".into(), + serde_json::json!({"orders": {"description": "Build orders", "selection": ["orders"], + "resolved_concurrency": {"limit": 4}}}), + ); + input.documents.insert( + "groups".into(), + serde_json::json!({"nightly_group": {"description": "Everything nightly", + "selection": ["orders"], "jobs": {"orders": []}, "resolved_concurrency": {"limit": 4}}}), + ); + input.documents.insert("triggers".into(), triggers); + input +} + +/// Report a claimed run as its worker does: started, each node with `node`, then finished with `run`; each report's outcome. +pub(crate) async fn report_run( + store: &ControlStore, + claim: &crate::runs::models::RunClaim, + (node, run): ( + crate::runs::models::NodeOutcome, + crate::runs::models::RunOutcome, + ), +) -> Vec { + use crate::runs::main::execution::change_run::change_run; + use crate::runs::models::{RunAuthority, RunChange, RunChangeRequest}; + + let mut changes = vec![RunChange::Started]; + changes.extend( + claim + .request + .selection + .iter() + .map(|node_key| RunChange::NodeFinished { + node_key: node_key.clone(), + outcome: node, + rows_affected: Some(1), + error_code: None, + skipped_because_of: None, + error_message: None, + }), + ); + changes.push(RunChange::Finished { + outcome: run, + reason: "the worker finished".into(), + exit_code: Some(1), + }); + let mut outcomes = Vec::new(); + for change in changes { + let reported = change_run( + store, + RunAuthority::Worker { + run_id: claim.request.run_id, + claim_id: claim.claim_id, + }, + &RunChangeRequest { + event_id: Uuid::now_v7(), + run_id: claim.request.run_id, + claim_id: Some(claim.claim_id), + change, + }, + ) + .await; + outcomes.push(outcome(reported)); + } + outcomes +} diff --git a/crates/sqlbuild-orchestration/tests/scheduler/test_concurrency_source.rs b/crates/sqlbuild-orchestration/tests/scheduler/test_concurrency_source.rs new file mode 100644 index 0000000000..629a8e84fd --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/scheduler/test_concurrency_source.rs @@ -0,0 +1,410 @@ +//! Migration 0004 drops the removed `resolved_concurrency.source` from stored job definitions; until a location publishes again, reads of its snapshot documents still fail. + +use std::collections::BTreeMap; + +use chrono::Utc; +use serde_json::json; +use uuid::Uuid; + +use crate::credentials::models::DefaultRole; +use crate::credentials::tests::location_helpers::create_location; +use crate::credentials::tests::role_helpers::{credentials, role_fixture}; +use crate::declarations::main::read_person_job::read_person_job; +use crate::declarations::models::ServerPoolCaps; +use crate::runs::main::{ + execution::advance_groups::advance_groups, execution::claim_next_run::claim_next_run, + requests::request_job_run::request_job_run, +}; +use crate::runs::models::{NodeOutcome, RunOutcome}; +use crate::runs::tests::helpers::{json_column, rows, snapshot}; +use crate::scheduler::models::SchedulerState; +use crate::scheduler::tests::contracts::helpers::{ + acquire_lease, manual_run, operator, outcome, report_run, scheduler_cycle, store_old_documents, + store_old_job_definitions, upgrade_snapshot, +}; +use crate::scheduler::tests::contracts::test_types::{ + FailedJobRunOldDocumentTestCase, GroupScheduleOldDocumentTestCase, + MigratedConcurrencySourceTestCase, PinnedOldDocumentTestCase, +}; +use crate::scheduler::tests::contracts::ticks::helpers::calendar_text; +use crate::sensors::tests::helpers::append_events; +use crate::snapshots::main::publication::publish_snapshot::publish_snapshot; +use crate::store::errors::ControlError; +use crate::store::main::migrate_store::migrate_store; +use crate::store::tests::lease_helpers::expire_lease; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; +use crate::store_migrations::_helpers::migration::MIGRATIONS; +use crate::store_migrations::_helpers::migration_run::migrate_with; + +/// How reading a document an older release published, with `source`, fails. +const OLD_FORM: &str = "Invalid(\"jobs: unknown field `source`, expected `limit`\")"; + +#[test] +fn given_jobs_published_before_0004_when_migrating_then_plain_job_runs_work_and_snapshot_reads_wait_for_a_republish() + { + let test_cases = [ + MigratedConcurrencySourceTestCase { + description: "SQLite store", + backend: Backend::Sqlite, + stored_source: "project", + expected_refused_before: true, + expected_resolved_concurrency: json!({"limit": 4}), + expected_ticks: 1, + expected_runs: &["manual 1", "schedule 1", "sensor 1"], + expected_until_republished: [OLD_FORM, OLD_FORM], + expected_after_republish: ["ok", "ok"], + }, + MigratedConcurrencySourceTestCase { + description: "PostgreSQL store", + backend: Backend::Postgres, + stored_source: "target", + expected_refused_before: true, + expected_resolved_concurrency: json!({"limit": 4}), + expected_ticks: 1, + expected_runs: &["manual 1", "schedule 1", "sensor 1"], + expected_until_republished: [OLD_FORM, OLD_FORM], + expected_after_republish: ["ok", "ok"], + }, + ]; + for test_case in test_cases { + run(async { + let fixture = open_test_store(test_case.backend) + .await + .expect("required backend"); + let store = &fixture.store; + migrate_with(store, &MIGRATIONS[..2]) + .await + .expect("a store at version 3"); + let roles = role_fixture(store).await; + let location = roles.location_id; + let mut input = upgrade_snapshot( + location, + json!({ + "daily_orders": {"description": "Refresh orders", "job_key": "orders", + "starts_at": calendar_text(0), + "schedule": {"kind": "calendar", "times": [calendar_text(0), calendar_text(1), + calendar_text(2), "2030-01-01T00:00:00Z"], "refresh_at": "2029-01-01T00:00:00Z"}}, + "health_orders": {"description": "Refresh orders on overdue ticks", "job_key": "orders", + "when": {"kind": "platform_event", "kinds": ["tick_overdue"]}, + "minimum_interval_seconds": 1} + }), + ); + input.generation = 2; + publish_snapshot(store, &input).await.expect("publish"); + store_old_job_definitions(store, test_case.stored_source).await; + store_old_documents(store, test_case.stored_source).await; + let before = + request_job_run(store, (&operator(), None), &manual_run(location, "orders")).await; + let caps = ServerPoolCaps::default(); + let administrator = (&roles.context, credentials(&roles.administrator)); + let page_and_group = || async { + [ + outcome( + read_person_job( + store, + administrator, + (location, "development", "orders"), + (DefaultRole::None, &caps), + ) + .await, + ), + outcome( + request_job_run( + store, + (&operator(), None), + &manual_run(location, "nightly_group"), + ) + .await, + ), + ] + }; + + migrate_store(store).await.expect("migrate to version 4"); + let stored = rows( + store, + "SELECT definition_json FROM sqb_jobs WHERE job_key='orders'", + vec![], + ) + .await; + append_events(store, 0..1, Utc::now()).await; + let (_, cycle) = scheduler_cycle(store, SchedulerState::new(Uuid::now_v7())) + .await + .expect("a scheduler cycle over the migrated jobs"); + request_job_run(store, (&operator(), None), &manual_run(location, "orders")) + .await + .expect("a manual run of the migrated job"); + let runs: Vec = rows( + store, + "SELECT origin, COUNT(*) AS runs FROM sqb_runs GROUP BY origin ORDER BY origin", + vec![], + ) + .await + .iter() + .map(|row| { + format!( + "{} {}", + row["origin"].as_str().unwrap_or_default(), + row["runs"] + ) + }) + .collect(); + let until_republished = page_and_group().await; + input.generation = 3; + publish_snapshot(store, &input) + .await + .expect("the location publishes again"); + let after_republish = page_and_group().await; + + assert_eq!( + ( + matches!(&before, Err(ControlError::Invalid(message)) if message.contains("unknown field `source`")), + json_column(&stored[0]["definition_json"])["resolved_concurrency"].clone(), + cycle.ticks, + runs, + until_republished, + after_republish, + ), + ( + test_case.expected_refused_before, + test_case.expected_resolved_concurrency.clone(), + test_case.expected_ticks, + test_case + .expected_runs + .iter() + .map(|run| (*run).to_owned()) + .collect::>(), + test_case.expected_until_republished.map(str::to_owned), + test_case.expected_after_republish.map(str::to_owned), + ), + "{}: refused before 0004; plain job runs after it; the job page and a group run once the location publishes again", + test_case.description + ); + close_test_store(fixture).await; + }); + } +} + +// Until its location publishes again, a schedule of a group reads the current snapshot's groups +// document, which 0004 can't rewrite, and its `ControlError::Invalid` fails the whole scheduler +// cycle. Making this a per-run failure is tracked separately, and that change replaces this +// expectation. +#[test] +fn given_a_group_schedule_published_before_0004_when_the_scheduler_cycles_then_it_fails_until_the_location_publishes_again() + { + let test_cases = [ + GroupScheduleOldDocumentTestCase { + description: "SQLite store", + backend: Backend::Sqlite, + stored_source: "project", + expected_until_republished: OLD_FORM, + expected_ticks_after_republish: 1, + }, + GroupScheduleOldDocumentTestCase { + description: "PostgreSQL store", + backend: Backend::Postgres, + stored_source: "local", + expected_until_republished: OLD_FORM, + expected_ticks_after_republish: 1, + }, + ]; + for test_case in test_cases { + run(async { + let fixture = open_test_store(test_case.backend) + .await + .expect("required backend"); + let store = &fixture.store; + migrate_with(store, &MIGRATIONS[..2]) + .await + .expect("a store at version 3"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let mut input = upgrade_snapshot( + location, + json!({"nightly_orders": {"description": "Run the group", "job_key": "nightly_group", + "starts_at": calendar_text(0), + "schedule": {"kind": "calendar", "times": [calendar_text(0), calendar_text(1), + calendar_text(2), "2030-01-01T00:00:00Z"], "refresh_at": "2029-01-01T00:00:00Z"}}}), + ); + publish_snapshot(store, &input).await.expect("publish"); + store_old_job_definitions(store, test_case.stored_source).await; + store_old_documents(store, test_case.stored_source).await; + migrate_store(store).await.expect("migrate to version 4"); + + let stopped = scheduler_cycle(store, SchedulerState::new(Uuid::now_v7())).await; + input.generation = 2; + publish_snapshot(store, &input) + .await + .expect("the location publishes again"); + // The stopped scheduler's lease runs out before a restarted one takes over. + expire_lease(store).await; + let (_, cycle) = scheduler_cycle(store, SchedulerState::new(Uuid::now_v7())) + .await + .expect("a scheduler cycle once the location has published again"); + + assert_eq!( + (outcome(stopped), cycle.ticks), + ( + test_case.expected_until_republished.to_owned(), + test_case.expected_ticks_after_republish + ), + "{}: the cycle fails Invalid until the location publishes again, then ticks", + test_case.description + ); + close_test_store(fixture).await; + }); + } +} + +// A job run that fails reads its snapshot's documents to decide on a retry before it checks for a +// retry policy, so until the location publishes again its failure report is refused and the run +// stays running until it is found unresponsive. Tracked separately with the other reads, and that +// change replaces this expectation. +#[test] +fn given_a_job_published_before_0004_when_its_run_fails_before_a_republish_then_the_failure_is_refused() + { + let test_cases = [FailedJobRunOldDocumentTestCase { + description: "SQLite store", + backend: Backend::Sqlite, + stored_source: "project", + expected_reports: ["ok", "ok", OLD_FORM], + expected_status: "running", + }]; + for test_case in test_cases { + run(async { + let fixture = open_test_store(test_case.backend) + .await + .expect("required backend"); + let store = &fixture.store; + migrate_with(store, &MIGRATIONS[..2]) + .await + .expect("a store at version 3"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + publish_snapshot(store, &upgrade_snapshot(location, json!({}))) + .await + .expect("publish"); + store_old_job_definitions(store, test_case.stored_source).await; + store_old_documents(store, test_case.stored_source).await; + migrate_store(store).await.expect("migrate to version 4"); + request_job_run(store, (&operator(), None), &manual_run(location, "orders")) + .await + .expect("a manual run of the migrated job"); + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + let claim = claim_next_run( + store, + &lease, + &BTreeMap::new(), + Some((location, "development")), + ) + .await + .expect("claim") + .expect("the queued run"); + + let reports = + report_run(store, &claim, (NodeOutcome::Failed, RunOutcome::Failed)).await; + let status = rows(store, "SELECT status FROM sqb_runs", vec![]).await; + + assert_eq!( + ( + reports, + status[0]["status"].as_str().unwrap_or_default().to_owned() + ), + ( + test_case.expected_reports.map(str::to_owned).to_vec(), + test_case.expected_status.to_owned() + ), + "{}: the failure report is refused and the run stays running", + test_case.description + ); + close_test_store(fixture).await; + }); + } +} + +// Runs pinned to a snapshot published before 0004 read its immutable documents, which 0004 can't +// rewrite. They still fail with `ControlError::Invalid`, which stops the scheduler; operators let +// such runs finish or cancel them before upgrading. Turning this into a per-run failure is tracked +// separately, and that change replaces this expectation. +#[test] +fn given_a_group_pinned_to_a_snapshot_document_in_the_old_form_when_advancing_it_then_it_fails_invalid() + { + let test_cases = [ + PinnedOldDocumentTestCase { + description: "SQLite store", + backend: Backend::Sqlite, + stored_source: "project", + expected_error: OLD_FORM, + }, + PinnedOldDocumentTestCase { + description: "PostgreSQL store", + backend: Backend::Postgres, + stored_source: "default", + expected_error: OLD_FORM, + }, + ]; + for test_case in test_cases { + run(async { + let fixture = open_test_store(test_case.backend) + .await + .expect("required backend"); + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let mut input = snapshot(location); + input.nodes.get_mut("orders").expect("orders").definition["deps"] = + json!(["customers"]); + input.documents.insert( + "jobs".into(), + json!({ + "staging_job": {"description": "Stage customers", "selection": ["customers"], + "resolved_concurrency": {"limit": 4}}, + "marts_job": {"description": "Build orders", "selection": ["orders"], + "resolved_concurrency": {"limit": 4}} + }), + ); + input.documents.insert( + "groups".into(), + json!({"nightly_job_group": { + "description": "Stage then build orders", "selection": ["customers", "orders"], + "jobs": {"staging_job": [], "marts_job": ["staging_job"]} + }}), + ); + publish_snapshot(store, &input).await.expect("publish"); + request_job_run( + store, + (&operator(), None), + &manual_run(location, "nightly_job_group"), + ) + .await + .expect("request the group"); + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + let limits = BTreeMap::new(); + let scope = (location, "development"); + advance_groups(store, &lease, (&limits, scope)) + .await + .expect("start the group"); + store_old_documents(store, test_case.stored_source).await; + + let advanced = advance_groups(store, &lease, (&limits, scope)).await; + + assert_eq!( + outcome(advanced), + test_case.expected_error, + "{}: the group pinned to the old document fails Invalid", + test_case.description + ); + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/scheduler/test_contracts.rs b/crates/sqlbuild-orchestration/tests/scheduler/test_contracts.rs index c70bd73153..b248cb5ca8 100644 --- a/crates/sqlbuild-orchestration/tests/scheduler/test_contracts.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler/test_contracts.rs @@ -2,6 +2,8 @@ #[path = "helpers.rs"] pub(crate) mod helpers; +#[path = "test_concurrency_source.rs"] +mod test_concurrency_source; #[path = "test_contract.rs"] mod test_contract; #[path = "test_types.rs"] diff --git a/crates/sqlbuild-orchestration/tests/scheduler/test_types.rs b/crates/sqlbuild-orchestration/tests/scheduler/test_types.rs index fd5543086b..2734da4600 100644 --- a/crates/sqlbuild-orchestration/tests/scheduler/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler/test_types.rs @@ -8,3 +8,51 @@ pub(crate) struct LeadershipTestCase { pub(crate) backend: Backend, pub(crate) expected_epoch: i64, } + +pub(crate) struct MigratedConcurrencySourceTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + /// The concurrency source an older release stored in each job's definition. + pub(crate) stored_source: &'static str, + /// Whether a manual run of the job is refused before migration 0004. + pub(crate) expected_refused_before: bool, + pub(crate) expected_resolved_concurrency: serde_json::Value, + pub(crate) expected_ticks: usize, + /// Runs queued after 0004, as `origin count`. + pub(crate) expected_runs: &'static [&'static str], + /// A job page read and a group run request after 0004, before the location publishes again. + pub(crate) expected_until_republished: [&'static str; 2], + /// The same once the location has published again. + pub(crate) expected_after_republish: [&'static str; 2], +} + +pub(crate) struct GroupScheduleOldDocumentTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + /// The concurrency source an older release stored with each job. + pub(crate) stored_source: &'static str, + /// The scheduler cycle due to queue the group after 0004, before the location publishes again. + pub(crate) expected_until_republished: &'static str, + /// Ticks the cycle takes once the location has published again. + pub(crate) expected_ticks_after_republish: usize, +} + +pub(crate) struct PinnedOldDocumentTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + /// The concurrency source an older release stored in the snapshot's jobs document. + pub(crate) stored_source: &'static str, + /// The `ControlError::Invalid` message advancing the group returns. + pub(crate) expected_error: &'static str, +} + +pub(crate) struct FailedJobRunOldDocumentTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + /// The concurrency source an older release stored with each job. + pub(crate) stored_source: &'static str, + /// The outcome of each worker report: started, the node finished, the run failed. + pub(crate) expected_reports: [&'static str; 3], + /// The run's status afterwards. + pub(crate) expected_status: &'static str, +} diff --git a/crates/sqlbuild-orchestration/tests/store/schema/test_store_migrations.rs b/crates/sqlbuild-orchestration/tests/store/schema/test_store_migrations.rs index 041225eeed..2db3f1cfd3 100644 --- a/crates/sqlbuild-orchestration/tests/store/schema/test_store_migrations.rs +++ b/crates/sqlbuild-orchestration/tests/store/schema/test_store_migrations.rs @@ -41,6 +41,11 @@ const SQLITE_CHECKSUMS: &[(i64, &str)] = &[ 3, "734163e22863ff86fbcf1329540736f279806d4ce5ea04f6d6ecb1703b3468c8", ), + // 0004 rewrites stored job definitions and leaves the schema as it is. + ( + 4, + "734163e22863ff86fbcf1329540736f279806d4ce5ea04f6d6ecb1703b3468c8", + ), ]; const POSTGRES_CHECKSUMS: &[(i64, &str)] = &[ ( @@ -55,6 +60,11 @@ const POSTGRES_CHECKSUMS: &[(i64, &str)] = &[ 3, "b1250935a5adf1d4691836a2125bb57360c1ff0a2a296a77e4d11bf6a80a99d9", ), + // 0004 rewrites stored job definitions and leaves the schema as it is. + ( + 4, + "b1250935a5adf1d4691836a2125bb57360c1ff0a2a296a77e4d11bf6a80a99d9", + ), ]; /// A version 1 store mid-run: a child run holding a concurrency slot, a jobs document and an audit result. const VERSION_1_LIVE_WORK: &str = "ALTER TABLE sqb_schema_version ADD COLUMN version BIGINT; \ diff --git a/crates/sqlbuild-python-compile/src/compile_driver/main/jobs/build_concurrency.rs b/crates/sqlbuild-python-compile/src/compile_driver/main/jobs/build_concurrency.rs index 3ad7861e98..98a53f1601 100644 --- a/crates/sqlbuild-python-compile/src/compile_driver/main/jobs/build_concurrency.rs +++ b/crates/sqlbuild-python-compile/src/compile_driver/main/jobs/build_concurrency.rs @@ -1,7 +1,7 @@ //! The concurrency limit of a build that names jobs. use pyo3::prelude::{Bound, PyAny, PyResult}; -use sqlbuild_config::effective::models::{ConcurrencySource, EffectiveConcurrency}; +use sqlbuild_config::effective::models::EffectiveConcurrency; use crate::compile_driver::_helpers::jobs::job_limits::job_limit_total; @@ -11,10 +11,7 @@ pub fn build_concurrency( pipeline: &Bound<'_, PyAny>, ) -> PyResult { Ok(match job_limit_total(pipeline)? { - Some(total) if total > configured.limit => EffectiveConcurrency { - limit: total, - source: ConcurrencySource::NamedJobs, - }, + Some(total) if total > configured.limit => EffectiveConcurrency { limit: total }, _ => configured, }) } diff --git a/crates/sqlbuild-python-front/src/command_runtime/main/published_concurrency.rs b/crates/sqlbuild-python-front/src/command_runtime/main/published_concurrency.rs index a9a9bff66f..0dbf658477 100644 --- a/crates/sqlbuild-python-front/src/command_runtime/main/published_concurrency.rs +++ b/crates/sqlbuild-python-front/src/command_runtime/main/published_concurrency.rs @@ -1,21 +1,14 @@ -//! The concurrency a published job without its own setting runs with on the compiled `target`, and its layer. - -use std::path::Path; +//! The concurrency a published job without its own setting runs with on the compiled `target`. use pyo3::IntoPyObject; use pyo3::exceptions::PyValueError; use pyo3::prelude::{Bound, PyAny, PyResult}; -use sqlbuild_config::effective::models::ConcurrencySource as Layer; -use sqlbuild_config::project::main::load_project_setting_keys::load_project_setting_keys; -use sqlbuild_config::project::main::project_config_paths::project_config_paths; -use sqlbuild_orchestration::declarations::models::{ConcurrencySource, ResolvedConcurrency}; +use sqlbuild_orchestration::declarations::models::ResolvedConcurrency; -use crate::command_runtime::_helpers::concurrency::CONCURRENCY; use crate::command_runtime::main::resolve_concurrency::resolve_concurrency; -/// The concurrency a published job without its own setting runs with on the compiled `target`, and its layer. +/// The concurrency a published job without its own setting runs with on the compiled `target`. pub fn published_concurrency( - project_dir: &Path, discovered: &Bound<'_, PyAny>, target: Option<&str>, ) -> PyResult { @@ -25,20 +18,7 @@ pub fn published_concurrency( discovered, target.into_pyobject(py)?.as_any(), )?; - let project_sets = || -> PyResult { - let keys = load_project_setting_keys(&project_config_paths(project_dir)).map_err(|_| { - PyValueError::new_err("cannot read [settings] of sqlbuild_project.toml") - })?; - Ok(keys.iter().any(|key| key == CONCURRENCY)) - }; - let source = match effective.source { - Layer::CommandLine | Layer::NamedJobs => ConcurrencySource::Job, - Layer::Target(_) => ConcurrencySource::Target, - Layer::LocalSettings => ConcurrencySource::Local, - Layer::ProjectSettings if project_sets()? => ConcurrencySource::Project, - Layer::ProjectSettings => ConcurrencySource::Default, - }; let limit = u64::try_from(effective.limit) .map_err(|_| PyValueError::new_err("concurrency must be at least 1"))?; - Ok(ResolvedConcurrency { limit, source }) + Ok(ResolvedConcurrency { limit }) } diff --git a/crates/sqlbuild-python-front/src/command_runtime/main/resolve_concurrency.rs b/crates/sqlbuild-python-front/src/command_runtime/main/resolve_concurrency.rs index da83e1347d..09c12c7e81 100644 --- a/crates/sqlbuild-python-front/src/command_runtime/main/resolve_concurrency.rs +++ b/crates/sqlbuild-python-front/src/command_runtime/main/resolve_concurrency.rs @@ -1,11 +1,11 @@ -//! The effective concurrency of a command on `selected_target`, and the layer that set it. +//! The effective concurrency of a command on `selected_target`. use crate::command_runtime::main::resolve_configured_concurrency::resolve_configured_concurrency; use pyo3::prelude::{Bound, PyAny, PyResult}; use sqlbuild_config::effective::models::EffectiveConcurrency; use sqlbuild_python_base::records::main::fields::field::field as native_field; -/// The effective concurrency of a command on `selected_target`, and the layer that set it. +/// The effective concurrency of a command on `selected_target`. pub fn resolve_concurrency( requested: &Bound<'_, PyAny>, discovered: &Bound<'_, PyAny>, diff --git a/crates/sqlbuild-python-front/src/command_runtime/main/resolve_configured_concurrency.rs b/crates/sqlbuild-python-front/src/command_runtime/main/resolve_configured_concurrency.rs index 5a40e5d3a8..cda9ed3e57 100644 --- a/crates/sqlbuild-python-front/src/command_runtime/main/resolve_configured_concurrency.rs +++ b/crates/sqlbuild-python-front/src/command_runtime/main/resolve_configured_concurrency.rs @@ -16,28 +16,23 @@ pub fn resolve_configured_concurrency( selected_target: &Bound<'_, PyAny>, ) -> PyResult { let py = project_config.py(); - let target: Option<(String, i64)> = + let target: Option = match resolve_target_name(py, project_config, local_config, selected_target)? { Some(name) => { let config: Bound<'_, PyAny> = resolve_target_config(py, project_config, local_config, name.as_any())?; - let limit: Option = native_field(&config, CONCURRENCY)?.extract()?; - limit.map(|limit| Ok::<_, pyo3::PyErr>((name.extract::()?, limit))) + native_field(&config, CONCURRENCY)?.extract()? } None => None, - } - .transpose()?; + }; let settings: i64 = native_field( &build_effective_settings(project_config, local_config)?, CONCURRENCY, )? .extract()?; - let local_override: bool = - native_field(local_config, "setting_overrides")?.contains(CONCURRENCY)?; Ok(effective_concurrency(&ConcurrencyLayers { requested: requested.extract()?, - target: target.as_ref().map(|(name, limit)| (name.as_str(), *limit)), + target, settings, - local_override, })) } diff --git a/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_declarations.rs b/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_declarations.rs index d03fc007b2..c4d89a51e5 100644 --- a/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_declarations.rs +++ b/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_declarations.rs @@ -19,8 +19,8 @@ use sqlbuild_orchestration::declarations::main::group_dependencies::group_depend use sqlbuild_orchestration::declarations::main::group_wait_reasons::group_wait_reasons; use sqlbuild_orchestration::declarations::main::snapshot_definitions::snapshot_definitions; use sqlbuild_orchestration::declarations::models::{ - ConcurrencySource, DeclaredAt, Definitions, GroupDefinition, JobDefinition, PoolSlots, - ResolvedConcurrency, Schedule, TriggerDeclaration, TriggerDefinition, TriggerStatus, + DeclaredAt, Definitions, GroupDefinition, JobDefinition, PoolSlots, ResolvedConcurrency, + Schedule, TriggerDeclaration, TriggerDefinition, TriggerStatus, }; use sqlbuild_orchestration::snapshots::models::SnapshotInput; @@ -156,10 +156,7 @@ fn job_concurrency( target: ResolvedConcurrency, ) -> ResolvedConcurrency { match settings.get("concurrency").and_then(Value::as_u64) { - Some(limit) => ResolvedConcurrency { - limit, - source: ConcurrencySource::Job, - }, + Some(limit) => ResolvedConcurrency { limit }, None => target, } } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs index 85c0bb3ec6..2f237f69c7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs @@ -177,7 +177,6 @@ fn run_context_facts( schema: display_text(&native_field(&project, "effective_target_schema")?)?, warehouse, concurrency: concurrency.limit, - concurrency_source: concurrency.source.to_string(), full_refresh: context.request_flag("full_refresh")?, full_refresh_jobs, job_warehouses, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs index f2e7efbe18..381eb77170 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs @@ -120,7 +120,6 @@ pub(crate) fn orchestration_snapshot<'py>( validate_orchestration_names(&facts).map_err(|failure| validation_error(py, failure))?; let declarations = python_declarations(py, &project_dir, &inventory)?; let concurrency = published_concurrency( - &project_dir, &discovered_inputs, project.effective_target_name().as_deref(), )?; diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md index 255a91828e..de4a6f5324 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md @@ -314,6 +314,20 @@ afterwards, so back it up before you upgrade (see [Back up the control database](#back-up-the-control-database)). To roll back, restore that backup; runs recorded since the upgrade are lost. +A run stays pinned to the snapshot it started on, and a snapshot an earlier release published can +hold job definitions this release no longer reads, which fail with +`` jobs: unknown field `source`, expected `limit` ``. Upgrade in this order: + +1. Pause every schedule and sensor, and start no manual runs. Keep the scheduler running: it + launches queued runs and advances group runs. +2. Let queued and running runs finish, or cancel them. +3. Upgrade the server. +4. Upgrade and restart every location, so each publishes its project again with the new release. +5. Once every location has published again, resume schedules and sensors. + +Until a location has published again, its job pages, group runs and ad hoc runs fail, a group's +schedule or sensor stops the scheduler, and a job run that fails can't record its failure. + To see what an upgrade changes before you start the server, run the new release with `--dry-run`: ```bash diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md index 2de6859ef1..a15b81ec2b 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md @@ -283,9 +283,8 @@ value set in this order: 5. `[settings] concurrency` in `sqlbuild_project.toml` (default `1`) The value must be an integer of at least `1`; anything else fails with `D001`. `sqb build ---verbose` shows the effective limit and the setting it came from, for example -`8 configured limit (targets.prod.concurrency)`. `sqb scenario test` has no `--target` option and -uses the active target's value. +--verbose` shows the build's effective limit, for example `8 configured limit`. `sqb scenario +test` has no `--target` option and uses the active target's value. ### Planning from synced state diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_selector_surface.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_selector_surface.py index 44f4bc3ab1..69ec9e5be0 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_selector_surface.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_selector_surface.py @@ -136,7 +136,7 @@ def test_given_selector_commands_when_running_cli_then_behavior_matches_expectat "run_id ", "target not set", "warehouse not set", - "concurrency 3 configured limit", + "concurrency 3 configured limit\n", "full_refresh false", "selected 2 of 19 build resources", "date vars 1970-01-01 to 2030-12-31", diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_target_concurrency.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_target_concurrency.py index 871c5bbeca..3c6ef44909 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_target_concurrency.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_target_concurrency.py @@ -31,42 +31,42 @@ target_concurrency="concurrency = 3\n", local_settings="", command=("--no-color", "build", "--verbose", "--target", "ci"), - expected_fragment="3 configured limit (targets.ci.concurrency)", + expected_fragment="concurrency 3 configured limit\n", ), TargetConcurrencyE2ETestCase( description="--concurrency overrides the target", target_concurrency="concurrency = 3\n", local_settings="", command=("--no-color", "build", "--verbose", "--target", "ci", "--concurrency", "1"), - expected_fragment="1 configured limit (--concurrency)", + expected_fragment="concurrency 1 configured limit\n", ), TargetConcurrencyE2ETestCase( description="the target overrides a local settings override", target_concurrency="concurrency = 3\n", local_settings="[settings]\nconcurrency = 5\n", command=("--no-color", "build", "--verbose", "--target", "ci"), - expected_fragment="3 configured limit (targets.ci.concurrency)", + expected_fragment="concurrency 3 configured limit\n", ), TargetConcurrencyE2ETestCase( description="a target without concurrency keeps the local settings override", target_concurrency="", local_settings="[settings]\nconcurrency = 5\n", command=("--no-color", "build", "--verbose", "--target", "ci"), - expected_fragment="5 configured limit (sqlbuild_local.toml settings.concurrency)", + expected_fragment="concurrency 5 configured limit\n", ), TargetConcurrencyE2ETestCase( description="a local target concurrency overrides the project target", target_concurrency="concurrency = 3\n", local_settings="[targets.ci]\nconcurrency = 4\n", command=("--no-color", "build", "--verbose", "--target", "ci"), - expected_fragment="4 configured limit (targets.ci.concurrency)", + expected_fragment="concurrency 4 configured limit\n", ), TargetConcurrencyE2ETestCase( description="other targets keep the project setting", target_concurrency="concurrency = 3\n", local_settings="", command=("--no-color", "build", "--verbose"), - expected_fragment="2 configured limit (settings.concurrency)", + expected_fragment="concurrency 2 configured limit\n", ), TargetConcurrencyE2ETestCase( description="seed uses the target's concurrency", diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_job_settings.py b/tests/integration/src/sqlbuild/cli/commands/main/test_job_settings.py index fa8ec58e87..0028592a07 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_job_settings.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_job_settings.py @@ -439,14 +439,14 @@ def test_given_a_job_concurrency_when_building_then_its_nodes_never_exceed_it( description="the run context says the named jobs set the limit", command_args=("build", "--verbose", "--select", "job:wide_job", "+orders"), expected_exit_code=0, - expected_present=("6 configured limit (named jobs' concurrency)",), + expected_present=("concurrency 6 configured limit\n",), expected_absent=(), ), JobOutputTestCase( description="a configured limit above the named jobs' stays the build's limit", command_args=("build", "--verbose", "--select", "job:orders_job", "customers"), expected_exit_code=0, - expected_present=("4 configured limit (settings.concurrency)",), + expected_present=("concurrency 4 configured limit\n",), expected_absent=(), ), JobOutputTestCase( diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py index d83cd55041..5ff1e15611 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py @@ -935,7 +935,7 @@ def snapshot(schedule: str) -> dict[str, Any]: "warehouse": "reporting", }, "pools": {"order_reporting": {"amount": 2, "limit": 2}}, - "resolved_concurrency": {"limit": 4, "source": "job"}, + "resolved_concurrency": {"limit": 4}, "select": "+orders", "exclude": "stg_orders", "targets": ["development"], @@ -971,8 +971,8 @@ def snapshot(schedule: str) -> dict[str, Any]: local_files={}, job_settings="", selected_target=None, - expected_job={"limit": 1, "source": "default"}, - expected_group={"limit": 1, "source": "default"}, + expected_job={"limit": 1}, + expected_group={"limit": 1}, ), SnapshotConcurrencyTestCase( description="the project's setting", @@ -980,8 +980,8 @@ def snapshot(schedule: str) -> dict[str, Any]: local_files={}, job_settings="", selected_target=None, - expected_job={"limit": 3, "source": "project"}, - expected_group={"limit": 3, "source": "project"}, + expected_job={"limit": 3}, + expected_group={"limit": 3}, ), SnapshotConcurrencyTestCase( description="the location's local setting over the project's", @@ -989,8 +989,8 @@ def snapshot(schedule: str) -> dict[str, Any]: local_files={"sqlbuild_local.toml": "[settings]\nconcurrency = 5\n"}, job_settings="", selected_target=None, - expected_job={"limit": 5, "source": "local"}, - expected_group={"limit": 5, "source": "local"}, + expected_job={"limit": 5}, + expected_group={"limit": 5}, ), SnapshotConcurrencyTestCase( description="the target's concurrency over the settings", @@ -998,8 +998,8 @@ def snapshot(schedule: str) -> dict[str, Any]: local_files={}, job_settings="", selected_target="development", - expected_job={"limit": 6, "source": "target"}, - expected_group={"limit": 6, "source": "target"}, + expected_job={"limit": 6}, + expected_group={"limit": 6}, ), SnapshotConcurrencyTestCase( description="the job's own setting over the target's; its group keeps the target's", @@ -1007,8 +1007,8 @@ def snapshot(schedule: str) -> dict[str, Any]: local_files={}, job_settings="\n[settings]\nconcurrency = 2\n", selected_target="development", - expected_job={"limit": 2, "source": "job"}, - expected_group={"limit": 6, "source": "target"}, + expected_job={"limit": 2}, + expected_group={"limit": 6}, ), ], ids=lambda case: case.description, diff --git a/ui/e2e/concurrency.spec.ts b/ui/e2e/concurrency.spec.ts index 937dee9f10..c99948b9f4 100644 --- a/ui/e2e/concurrency.spec.ts +++ b/ui/e2e/concurrency.spec.ts @@ -1,4 +1,5 @@ import { expect, test, type Page } from '@playwright/test'; +import { FakeControl } from './support/fake-control'; // The Concurrency panel on job and group pages: plain facts about what runs alongside what. With // SHOTS set, each page is photographed: SHOTS=/some/dir npx playwright test e2e/concurrency.spec.ts @@ -17,10 +18,10 @@ async function panel(page: Page, job: string, group = false): Promise return (await lines.allInnerTexts()).map((t) => t.replace(/\s+/g, ' ').replace(/ Running now: .*$/, '').trim()); } -test('a job states its run concurrency and where it comes from, overlap, and its pool', async ({ page }) => { +test('a job states its run concurrency limit, overlap, and its pool', async ({ page }) => { const lines = await panel(page, 'core_build'); expect(lines).toEqual([ - 'Inside a run: 8 at once (job setting)', + "Inside a run: at most 8 of this job's nodes at once", 'Never alongside runs writing the same tables (automatic)', 'If triggered while running: is skipped', 'warehouse_heavy: takes 1 of 1 (capped on this server; the project allows 3), shared with reporting_exports, close_of_business' @@ -29,12 +30,12 @@ test('a job states its run concurrency and where it comes from, overlap, and its await expect(page.locator('ul.concurrency a', { hasText: 'reporting_exports' })).toHaveAttribute('href', /\/jobs\/reporting_exports/); }); -test('each layer a run concurrency can come from is named', async ({ page }) => { +test('a job without its own setting states the limit its runs get, without naming where it is set', async ({ page }) => { for (const [job, line] of [ - ['marts_only', 'Inside a run: 6 at once (target prod)'], - ['activity_microbatch', 'Inside a run: 2 at once (local setting)'], - ['revenue_refresh', 'Inside a run: 4 at once (project setting)'], - ['load_sources', 'Inside a run: 4 at once (SQLBuild default)'] + ['marts_only', "Inside a run: at most 6 of this job's nodes at once"], + ['activity_microbatch', "Inside a run: at most 2 of this job's nodes at once"], + ['revenue_refresh', "Inside a run: at most 4 of this job's nodes at once"], + ['load_sources', "Inside a run: at most 4 of this job's nodes at once"] ] as const) { expect((await panel(page, job))[0], job).toBe(line); } @@ -54,7 +55,9 @@ test('a group capped below what it asks takes the whole pool, shown as a warning }); test('a group in no pool says so', async ({ page }) => { - expect(await panel(page, 'nightly_all', true)).toContain('Pools: none'); + const lines = await panel(page, 'nightly_all', true); + expect(lines).toContain('Pools: none'); + expect(lines[0]).toBe('Members without their own limit: at most 4 nodes at once'); }); test("the location shows the server's pool-cap warning", async ({ page }) => { @@ -65,3 +68,26 @@ test("the location shows the server's pool-cap warning", async ({ page }) => { await expect(caps.getByRole('link', { name: 'close_of_business' })).toHaveAttribute('href', /\/jobs\/close_of_business/); if (shots) await page.screenshot({ path: `${shots}/concurrency-panel-location.png` }); }); + +test("a published job's limit comes through the server's job read as a plain limit", async ({ page }) => { + const server = new FakeControl({ + shop: { + jobs: [ + { key: 'orders_job', kind: 'job', concurrency: 2, limit: 2 }, + { key: 'refunds_job', kind: 'job', concurrency: null, limit: 16 }, + { key: 'nightly', kind: 'group', concurrency: null, limit: 16 } + ] + } + }); + await server.install(page); + for (const [job, line, group] of [ + ['orders_job', "Inside a run: at most 2 of this job's nodes at once", false], + ['refunds_job', "Inside a run: at most 16 of this job's nodes at once", false], + ['nightly', 'Members without their own limit: at most 16 nodes at once', true] + ] as const) { + await page.goto(`/jobs/${job}`); + if (group) await page.getByRole('button', { name: 'Toggle details' }).click({ timeout: 20_000 }); + const first = page.locator('ul.concurrency > li').first(); + await expect(first, job).toHaveText(line, { timeout: 20_000 }); + } +}); diff --git a/ui/e2e/support/fake-control.ts b/ui/e2e/support/fake-control.ts index 358becdbe9..9ab02dbb25 100644 --- a/ui/e2e/support/fake-control.ts +++ b/ui/e2e/support/fake-control.ts @@ -32,8 +32,13 @@ export type TargetState = { failWithdrawals: boolean; /** Delays the sync status, so it arrives after the page's first read has begun. */ healthDelayMs: number; + /** Published jobs and groups, with the concurrency limit their runs get. */ + jobs: FakeJob[]; }; +/** A published job or group: its key, kind, own `concurrency` setting and the limit its runs get; a group's members are the target's jobs. */ +export type FakeJob = { key: string; kind: 'job' | 'group'; concurrency: number | null; limit: number }; + const MODELS = ['orders', 'order_items', 'customers', 'payments', 'refunds']; const json = (route: Route, body: unknown, status = 200) => route.fulfill({ status, contentType: 'application/json', body: JSON.stringify(body) }); @@ -49,7 +54,7 @@ export class FakeControl { #lease = new Date(Date.now() - 60_000).toISOString(); constructor(overrides: Partial>> = {}) { - const base = (): TargetState => ({ changed: [], sync: 'settled', schedulerLeader: true, acks: [], failedRun: false, readDelayMs: 0, failReads: false, failWithdrawals: false, healthDelayMs: 0 }); + const base = (): TargetState => ({ changed: [], sync: 'settled', schedulerLeader: true, acks: [], failedRun: false, readDelayMs: 0, failReads: false, failWithdrawals: false, healthDelayMs: 0, jobs: [] }); this.targets = { shop: { ...base(), ...overrides.shop }, returns: { ...base(), ...overrides.returns } }; } @@ -75,6 +80,12 @@ export class FakeControl { await page.route('**/api/**', (route) => this.#answer(route)); } + /** A job as the server lists it: every model, and the concurrency limit its runs get. */ + #job(job: FakeJob) { + const selection = MODELS.map((m) => `model:${m}`); + return { job_key: job.key, kind: job.kind, description: `Builds ${job.key}.`, selection, pools: [], concurrency: job.concurrency, resolved_concurrency: { limit: job.limit }, overlap: 'queue', snapshot_id: SNAPSHOT }; + } + async #answer(route: Route) { const request = route.request(); const url = new URL(request.url()); @@ -120,7 +131,12 @@ export class FakeControl { return json(route, { snapshot_id: SNAPSHOT, version: this.#states[name].version, states }); } if (rest.startsWith('snapshots')) return json(route, [{ snapshot_id: SNAPSHOT, generation: 1, git_revision: null, git_dirty: false, git_author: null, environment_hash: 'e'.repeat(64), sqb_version: '0.0.0', published_at: '2026-11-01T00:00:00Z', changes: [], connection: null }]); - if (rest === 'jobs' || rest === 'triggers' || rest === 'checks' || rest === 'acknowledgements') return json(route, []); + if (rest === 'jobs') return json(route, target.jobs.map((job) => this.#job(job))); + const job = rest.startsWith('jobs/') ? target.jobs.find((j) => `jobs/${j.key}` === rest) : undefined; + // A group's members are the target's jobs, all starting together. + const members = job?.kind === 'group' ? target.jobs.filter((j) => j.kind === 'job').map((j) => ({ job_key: j.key, waits_for: [], reasons: [] })) : []; + if (job) return json(route, { ...this.#job(job), critical_nodes: [], member_of: [], members, settings: job.concurrency === null ? {} : { concurrency: job.concurrency }, targets: [] }); + if (rest === 'triggers' || rest === 'checks' || rest === 'acknowledgements') return json(route, []); if (rest.startsWith('cost')) return json(route, { since: '2026-11-01T00:00:00Z', credits: 0, usd: null, runs: 0, incomplete_runs: 0, by_job: [], by_warehouse_size: [], by_node: [] }); // The Overview's own reads: answered as things stood when the request arrived, then slowed or // refused as the test asks. diff --git a/ui/src/lib/api/http-jobs.test.ts b/ui/src/lib/api/http-jobs.test.ts index acbe4d9211..1cb3bb4d61 100644 --- a/ui/src/lib/api/http-jobs.test.ts +++ b/ui/src/lib/api/http-jobs.test.ts @@ -115,7 +115,7 @@ describe('job mapping', () => { description: 'Build orders', selection: ['model:customers', 'model:orders'], pools: [{ pool: 'warehouse_heavy', amount: 3, limit: 4, cap: 2, server_limit: 2, clamped: true, shared_with: ['nightly'] }], - resolved_concurrency: { limit: 4, source: 'job' }, + resolved_concurrency: { limit: 4 }, concurrency: 4, snapshot_id: SNAPSHOT, select: '+orders', @@ -130,7 +130,7 @@ describe('job mapping', () => { expect(toJob({ ...wire, owner: ' ', overlap: 'queue', on_failure: null }, triggers, [])).toMatchObject({ owner: null, overlap: 'queue', onFailure: null }); expect(job.lastRun?.id).toBe('r-ord'); expect(job.pools).toEqual([{ pool: 'warehouse_heavy', amount: 3, limit: 4, cap: 2, serverLimit: 2, clamped: true, sharedWith: ['nightly'] }]); - expect(job.resolvedConcurrency).toEqual({ limit: 4, source: 'job' }); + expect(job.resolvedConcurrency).toEqual({ limit: 4 }); expect(toJob({ ...wire, resolved_concurrency: null }, triggers, []).resolvedConcurrency).toBeNull(); expect(toJob({ ...wire, concurrency: null }, triggers, []).concurrency).toBeNull(); const detail = jobDetailOf(job, { ...wire, settings: { vars: '[REDACTED]', concurrency: 4, empty_ok: true, models: ['a', 'b'] }, members: [], member_of: ['nightly'], critical_nodes: ['model:orders'], exclude: ' stg_orders audit_orders', targets: ['production'], cursor_type: 'integer' }); diff --git a/ui/src/lib/api/mock/operate.ts b/ui/src/lib/api/mock/operate.ts index d788bd5f14..f1e07c9f71 100644 --- a/ui/src/lib/api/mock/operate.ts +++ b/ui/src/lib/api/mock/operate.ts @@ -53,11 +53,10 @@ export const POOL_CAPS: Record = { warehouse_heavy: 1 }; /** A pool's limit on this server: the project's, lowered to the cap. */ export const serverLimitOf = (pool: string) => Math.min(POOL_LIMITS[pool] ?? 1, POOL_CAPS[pool] ?? Infinity); -/** Where each job's run concurrency comes from, when not its own setting: one of every layer. */ +/** The run concurrency of jobs without their own setting, where it differs from the mock project's 4. */ const CONCURRENCY_FROM: Record = { - marts_only: { limit: 6, source: 'target' }, - activity_microbatch: { limit: 2, source: 'local' }, - load_sources: { limit: 4, source: 'default' } + marts_only: { limit: 6 }, + activity_microbatch: { limit: 2 } }; const decl = (sourceFile: string, line: number, code: string) => ({ sourceFile, line, code }); @@ -410,10 +409,10 @@ export function poolHoldersOf(job: JobDef | undefined): PoolHolders[] { return Object.keys(job?.pools ?? {}).map((pool) => ({ pool, limit: serverLimitOf(pool), holders: [] })); } -/** What a run of the job uses: its own setting, else the layer the mock gives it, else the mock project's 4. */ +/** What a run of the job uses: its own setting, else the limit the mock gives it, else the mock project's 4. */ export function resolvedConcurrencyOf(job: JobDef | undefined): ResolvedConcurrency { const own = runConcurrencyOf(job); - return own ? { limit: own, source: 'job' } : (CONCURRENCY_FROM[job?.key ?? ''] ?? { limit: 4, source: 'project' }); + return own ? { limit: own } : (CONCURRENCY_FROM[job?.key ?? ''] ?? { limit: 4 }); } /** The jobs and groups asking a pool for more than this server allows, as the server's pool_capped warning names them. */ diff --git a/ui/src/lib/api/types/operate.ts b/ui/src/lib/api/types/operate.ts index e836626ba2..5a9e8e6f77 100644 --- a/ui/src/lib/api/types/operate.ts +++ b/ui/src/lib/api/types/operate.ts @@ -929,8 +929,8 @@ export type PoolUse = { sharedWith: string[]; }; -/** Models a run builds at once, and the layer that set it. */ -export type ResolvedConcurrency = { limit: number; source: 'job' | 'target' | 'local' | 'project' | 'default' }; +/** The most of a job's nodes a run of it builds at once. */ +export type ResolvedConcurrency = { limit: number }; /** One reason a member waits: `fromNode` of `upstreamJob` feeds its `toNode`, directly or through nodes no member selects. */ export type WaitReason = { upstreamJob: string; fromNode: string; toNode: string }; diff --git a/ui/src/lib/operate/labels.test.ts b/ui/src/lib/operate/labels.test.ts index d0d2b2247b..184e9b6276 100644 --- a/ui/src/lib/operate/labels.test.ts +++ b/ui/src/lib/operate/labels.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest'; -import { concurrencySource, describeWait, onFailureText, overlapPlain, overlapWhileRunning, poolText, triggerKindLabel, waitTechnical } from './labels'; +import { describeWait, onFailureText, overlapPlain, overlapWhileRunning, poolText, runConcurrencyText, triggerKindLabel, waitTechnical } from './labels'; import type { QueueWait } from '$lib/api'; const text = (w: QueueWait) => describeWait(w).map((p) => p.text).join(''); @@ -49,13 +49,10 @@ describe('concurrency in plain words', () => { expect(`${clamped.text} (${clamped.detail})`).toBe('Takes the whole warehouse_heavy pool on this server (capped at 1; the project asks for 3)'); }); - it("names the layer that set a run's concurrency", () => { - expect(concurrencySource('job', 'dev')).toBe('job setting'); - expect(concurrencySource('target', 'dev')).toBe('target dev'); - expect(concurrencySource('target', null)).toBe('target setting'); - expect(concurrencySource('local', 'dev')).toBe('local setting'); - expect(concurrencySource('project', 'dev')).toBe('project setting'); - expect(concurrencySource('default', 'dev')).toBe('SQLBuild default'); + it("states a run's concurrency as the limit it applies, for a job and for a group's members", () => { + expect(runConcurrencyText(4, 'job')).toBe("Inside a run: at most 4 of this job's nodes at once"); + expect(runConcurrencyText(1, 'job')).toBe("Inside a run: at most 1 of this job's nodes at once"); + expect(runConcurrencyText(6, 'group')).toBe('Members without their own limit: at most 6 nodes at once'); }); it('says what a job triggered while it runs does', () => { diff --git a/ui/src/lib/operate/labels.ts b/ui/src/lib/operate/labels.ts index 03869bb5e4..f6194b009c 100644 --- a/ui/src/lib/operate/labels.ts +++ b/ui/src/lib/operate/labels.ts @@ -7,7 +7,6 @@ import type { NodeCounts, OnFailure, OverlapPolicy, - ResolvedConcurrency, PoolUse, QueueWait, RunCost, @@ -112,20 +111,14 @@ export const SHARED_SHOWN = 3; /** The docs page the Concurrency panel links to. */ export const CONCURRENCY_DOCS = 'https://sqlbuild.com/docs/concepts/concurrency/'; -/** Which layer set a run's concurrency, as the panel names it; a target by its name when known. */ -export function concurrencySource(source: ResolvedConcurrency['source'], target: string | null): string { - switch (source) { - case 'job': - return 'job setting'; - case 'target': - return target ? `target ${target}` : 'target setting'; - case 'local': - return 'local setting'; - case 'project': - return 'project setting'; - case 'default': - return 'SQLBuild default'; - } +/** + * A run's concurrency as the panel states it. A group's limit is the target's, which each member + * job's run uses unless the job sets its own. + */ +export function runConcurrencyText(limit: number, kind: 'job' | 'group'): string { + return kind === 'group' + ? `Members without their own limit: at most ${limit} nodes at once` + : `Inside a run: at most ${limit} of this job's nodes at once`; } /** What a job does when triggered while a run of it is active, finishing "If triggered while running: …". */ diff --git a/ui/src/routes/jobs/[name]/job-view.svelte b/ui/src/routes/jobs/[name]/job-view.svelte index 1a3a9ee017..ee5e75ce76 100644 --- a/ui/src/routes/jobs/[name]/job-view.svelte +++ b/ui/src/routes/jobs/[name]/job-view.svelte @@ -23,7 +23,7 @@ import { formatNodeCounts, isActiveRun, nodeOutcomeLabel, nodeOutcomeTone, runOriginLabel, runStatusLabel, runStatusTone } from '$lib/status'; import { kindLine, shortName } from '$lib/build-area/labels'; import { planHref } from '$lib/run/command'; - import { CONCURRENCY_DOCS, concurrencySource, formatCost, narrowingText, onFailureText, overlapHelp, overlapWhileRunning, poolText, shortRunId } from '$lib/operate/labels'; + import { CONCURRENCY_DOCS, formatCost, narrowingText, onFailureText, overlapHelp, overlapWhileRunning, poolText, runConcurrencyText, shortRunId } from '$lib/operate/labels'; import ScopeGraph from '$lib/lineage/scope-graph.svelte'; import MemberGraph from '$lib/lineage/member-graph.svelte'; import { memberLinks, publishedLinks, waitReasons, type MemberInfo, type MemberSelection } from '$lib/lineage/member-graph'; @@ -313,7 +313,7 @@ {@const resolved = def.resolvedConcurrency}
    {#if resolved} -
  • Inside a run: {resolved.limit} at once ({concurrencySource(resolved.source, app.target?.name ?? null)})
  • +
  • {runConcurrencyText(resolved.limit, def.kind)}
  • {/if}
  • Never alongside runs writing the same tables (automatic)
  • {#if def.overlap} @@ -531,7 +531,7 @@ .picked li { padding: 2px 0; } - /* The Concurrency panel: one plain fact per line, where it came from quieter beside it. */ + /* The Concurrency panel: one plain fact per line, any detail quieter beside it. */ .concurrency { display: flex; flex-direction: column; diff --git a/website/src/content/docs/docs/cli/server.mdx b/website/src/content/docs/docs/cli/server.mdx index 2cea042e05..e134269e7c 100644 --- a/website/src/content/docs/docs/cli/server.mdx +++ b/website/src/content/docs/docs/cli/server.mdx @@ -293,6 +293,20 @@ afterwards, so back it up before you upgrade (see [Back up the control database](#back-up-the-control-database)). To roll back, restore that backup; runs recorded since the upgrade are lost. +A run stays pinned to the snapshot it started on, and a snapshot an earlier release published can +hold job definitions this release no longer reads, which fail with +`` jobs: unknown field `source`, expected `limit` ``. Upgrade in this order: + +1. Pause every schedule and sensor, and start no manual runs. Keep the scheduler running: it + launches queued runs and advances group runs. +2. Let queued and running runs finish, or cancel them. +3. Upgrade the server. +4. Upgrade and restart every location, so each publishes its project again with the new release. +5. Once every location has published again, resume schedules and sensors. + +Until a location has published again, its job pages, group runs and ad hoc runs fail, a group's +schedule or sensor stops the scheduler, and a job run that fails can't record its failure. + To see what an upgrade changes before you start the server, run the new release with `--dry-run`: ```bash diff --git a/website/src/content/docs/docs/concepts/project-configuration.mdx b/website/src/content/docs/docs/concepts/project-configuration.mdx index b6e4e12f66..03f4dcd7bd 100644 --- a/website/src/content/docs/docs/concepts/project-configuration.mdx +++ b/website/src/content/docs/docs/concepts/project-configuration.mdx @@ -256,9 +256,8 @@ value set in this order: 5. `[settings] concurrency` in `sqlbuild_project.toml` (default `1`) The value must be an integer of at least `1`; anything else fails with `D001`. `sqb build ---verbose` shows the effective limit and the setting it came from, for example -`8 configured limit (targets.prod.concurrency)`. `sqb scenario test` has no `--target` option and -uses the active target's value. +--verbose` shows the build's effective limit, for example `8 configured limit`. `sqb scenario +test` has no `--target` option and uses the active target's value. ### Planning from synced state