From 135016a3d9968423b1107683fe3b5d9d592354cb Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 19 Nov 2026 23:43:07 +0000 Subject: [PATCH 1/4] feat!: drop the concurrency source from the CLI and published jobs A build's concurrency is no longer described by the layer that set it: since named jobs' limits combine with the target and project cap, one source label no longer describes the limit a build uses. - The verbose run context reads `concurrency 16 configured limit`. RunContextFacts.concurrency_source and the config ConcurrencySource are removed; EffectiveConcurrency keeps only its limit. - A published job's resolved_concurrency is `{limit}`; its `source` and the ConcurrencySource schema are removed from the API. - Control store migration 0004 removes resolved_concurrency.source from each stored job definition, so schedules, sensors and manual runs read jobs published before the upgrade. --- .../tests/native_commands/test_control.rs | 4 +- .../effective/main/effective_concurrency.rs | 28 +- .../sqlbuild-config/src/effective/models.rs | 34 +-- .../src/effective/tests/test_effective.rs | 27 +- .../src/effective/tests/test_types.rs | 4 +- .../build_command/main/render_run_context.rs | 10 +- .../sqlbuild-core/src/build_command/models.rs | 2 - .../build_command/tests/test_build_command.rs | 3 +- .../src/declarations/models.rs | 19 +- .../store_migrations/_helpers/migration.rs | 5 + .../0004_drop_concurrency_source.postgres.sql | 3 + .../0004_drop_concurrency_source.sqlite.sql | 3 + .../tests/runs/test_job_detail.rs | 6 +- .../scheduler/test_concurrency_source.rs | 281 ++++++++++++++++++ .../tests/scheduler/test_contracts.rs | 2 + .../tests/scheduler/test_types.rs | 5 + .../store/schema/test_store_migrations.rs | 10 + .../main/jobs/build_concurrency.rs | 7 +- .../main/published_concurrency.rs | 28 +- .../main/resolve_configured_concurrency.rs | 13 +- .../snapshot/snapshot_declarations.rs | 9 +- .../_helpers/build_command/execution.rs | 1 - .../_helpers/location_command/snapshot.rs | 1 - .../main/build/test_selector_surface.py | 2 +- .../main/build/test_target_concurrency.py | 12 +- .../cli/commands/main/test_job_settings.py | 4 +- .../test_orchestration_snapshot.py | 22 +- 27 files changed, 367 insertions(+), 178 deletions(-) create mode 100644 crates/sqlbuild-orchestration/src/store_migrations/sql/0004_drop_concurrency_source.postgres.sql create mode 100644 crates/sqlbuild-orchestration/src/store_migrations/sql/0004_drop_concurrency_source.sqlite.sql create mode 100644 crates/sqlbuild-orchestration/tests/scheduler/test_concurrency_source.rs diff --git a/crates/sqlbuild-cli/tests/native_commands/test_control.rs b/crates/sqlbuild-cli/tests/native_commands/test_control.rs index f163a688d1..197e53199f 100644 --- a/crates/sqlbuild-cli/tests/native_commands/test_control.rs +++ b/crates/sqlbuild-cli/tests/native_commands/test_control.rs @@ -379,13 +379,13 @@ fn given_a_control_store_when_running_server_migrate_then_dry_run_lists_and_migr StoreMigrationCommandTestCase { description: "migrate an empty store", arguments: &["server", "migrate"], - expected_stdout: "The control store is empty: created schema version 3.\n", + expected_stdout: "The control store is empty: created schema version 4.\n", expected_store: true, }, StoreMigrationCommandTestCase { description: "dry run on a current store", arguments: &["server", "migrate", "--dry-run"], - expected_stdout: "The control store is at schema version 3: nothing to migrate.\n", + expected_stdout: "The control store is at schema version 4: nothing to migrate.\n", expected_store: true, }, ]; diff --git a/crates/sqlbuild-config/src/effective/main/effective_concurrency.rs b/crates/sqlbuild-config/src/effective/main/effective_concurrency.rs index 16a6aa60eb..7aeb732401 100644 --- a/crates/sqlbuild-config/src/effective/main/effective_concurrency.rs +++ b/crates/sqlbuild-config/src/effective/main/effective_concurrency.rs @@ -1,28 +1,14 @@ -//! `effective_concurrency`: the worker limit of one command and the layer it came from. +//! `effective_concurrency`: the worker limit of one command. -use crate::effective::models::{ConcurrencyLayers, ConcurrencySource, EffectiveConcurrency}; +use crate::effective::models::{ConcurrencyLayers, EffectiveConcurrency}; /// `--concurrency`, else the target's `concurrency`, else the local-then-project setting. #[must_use] -pub fn effective_concurrency(layers: &ConcurrencyLayers<'_>) -> EffectiveConcurrency { - if let Some(limit) = layers.requested { - return EffectiveConcurrency { - limit, - source: ConcurrencySource::CommandLine, - }; - } - if let Some((target, limit)) = layers.target { - return EffectiveConcurrency { - limit, - source: ConcurrencySource::Target(target.to_owned()), - }; - } +pub fn effective_concurrency(layers: &ConcurrencyLayers) -> EffectiveConcurrency { EffectiveConcurrency { - limit: layers.settings, - source: if layers.local_override { - ConcurrencySource::LocalSettings - } else { - ConcurrencySource::ProjectSettings - }, + limit: layers + .requested + .or(layers.target) + .unwrap_or(layers.settings), } } diff --git a/crates/sqlbuild-config/src/effective/models.rs b/crates/sqlbuild-config/src/effective/models.rs index b1b11cd89c..3e380de013 100644 --- a/crates/sqlbuild-config/src/effective/models.rs +++ b/crates/sqlbuild-config/src/effective/models.rs @@ -83,15 +83,13 @@ pub struct TargetChoice { /// The concurrency each configuration layer sets, highest precedence first. #[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub struct ConcurrencyLayers<'a> { +pub struct ConcurrencyLayers { /// `--concurrency` on the command line. pub requested: Option, - /// The selected target's name and its `concurrency`, when it sets one. - pub target: Option<(&'a str, i64)>, + /// The selected target's `concurrency`, when it sets one. + pub target: Option, /// `[settings] concurrency` after any `sqlbuild_local.toml` override. pub settings: i64, - /// Whether `sqlbuild_local.toml` `[settings]` overrides `concurrency`. - pub local_override: bool, } /// Whether audit failures are ignored at each configuration layer, highest precedence first. @@ -105,32 +103,8 @@ pub struct AuditFailureLayers { pub settings: bool, } -/// Where the effective concurrency came from. -#[derive(Clone, Debug, PartialEq, Eq)] -pub enum ConcurrencySource { - CommandLine, - Target(String), - LocalSettings, - ProjectSettings, - /// The named jobs' `concurrency` limits added together, when larger than the configured limit. - NamedJobs, -} - -impl std::fmt::Display for ConcurrencySource { - fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - Self::CommandLine => formatter.write_str("--concurrency"), - Self::Target(name) => write!(formatter, "targets.{name}.concurrency"), - Self::LocalSettings => formatter.write_str("sqlbuild_local.toml settings.concurrency"), - Self::ProjectSettings => formatter.write_str("settings.concurrency"), - Self::NamedJobs => formatter.write_str("named jobs' concurrency"), - } - } -} - -/// One command's worker limit and the layer that set it. +/// One command's worker limit. #[derive(Clone, Debug, PartialEq, Eq)] pub struct EffectiveConcurrency { pub limit: i64, - pub source: ConcurrencySource, } diff --git a/crates/sqlbuild-config/src/effective/tests/test_effective.rs b/crates/sqlbuild-config/src/effective/tests/test_effective.rs index a9a9f07c96..d25081d82a 100644 --- a/crates/sqlbuild-config/src/effective/tests/test_effective.rs +++ b/crates/sqlbuild-config/src/effective/tests/test_effective.rs @@ -305,38 +305,23 @@ fn given_concurrency_layers_when_resolving_then_cli_beats_target_beats_settings( ConcurrencyTestCase { description: "the command line beats the target", requested: Some(2), - target: Some(("prod", 8)), + target: Some(8), settings: 16, - local_override: true, expected_limit: 2, - expected_source: "--concurrency", }, ConcurrencyTestCase { - description: "the target beats a local settings override", + description: "the target beats the settings", requested: None, - target: Some(("prod", 8)), + target: Some(8), settings: 4, - local_override: true, expected_limit: 8, - expected_source: "targets.prod.concurrency", }, ConcurrencyTestCase { - description: "a local settings override applies without a target value", - requested: None, - target: None, - settings: 4, - local_override: true, - expected_limit: 4, - expected_source: "sqlbuild_local.toml settings.concurrency", - }, - ConcurrencyTestCase { - description: "the project setting is the default", + description: "the settings apply without a target value", requested: None, target: None, settings: 16, - local_override: false, expected_limit: 16, - expected_source: "settings.concurrency", }, ]; @@ -345,11 +330,9 @@ fn given_concurrency_layers_when_resolving_then_cli_beats_target_beats_settings( requested: test_case.requested, target: test_case.target, settings: test_case.settings, - local_override: test_case.local_override, }); assert_eq!( - (resolved.limit, resolved.source.to_string().as_str()), - (test_case.expected_limit, test_case.expected_source), + resolved.limit, test_case.expected_limit, "{}", test_case.description ); diff --git a/crates/sqlbuild-config/src/effective/tests/test_types.rs b/crates/sqlbuild-config/src/effective/tests/test_types.rs index eb1276509b..e08d48f274 100644 --- a/crates/sqlbuild-config/src/effective/tests/test_types.rs +++ b/crates/sqlbuild-config/src/effective/tests/test_types.rs @@ -52,11 +52,9 @@ pub(super) struct MessageTestCase { pub(super) struct ConcurrencyTestCase { pub(super) description: &'static str, pub(super) requested: Option, - pub(super) target: Option<(&'static str, i64)>, + pub(super) target: Option, pub(super) settings: i64, - pub(super) local_override: bool, pub(super) expected_limit: i64, - pub(super) expected_source: &'static str, } pub(super) struct AuditFailureTestCase { diff --git a/crates/sqlbuild-core/src/build_command/main/render_run_context.rs b/crates/sqlbuild-core/src/build_command/main/render_run_context.rs index 7be1e19f18..fcbbebbdb9 100644 --- a/crates/sqlbuild-core/src/build_command/main/render_run_context.rs +++ b/crates/sqlbuild-core/src/build_command/main/render_run_context.rs @@ -48,15 +48,7 @@ pub fn render_run_context(facts: &RunContextFacts, style: CliStyle) -> String { ("warehouse", warehouse), ( "concurrency", - if facts.concurrency_source.is_empty() { - format!("{} configured limit", facts.concurrency) - } else { - format!( - "{} configured limit ({})", - facts.concurrency, - sanitize(&facts.concurrency_source) - ) - }, + format!("{} configured limit", facts.concurrency), ), ("full_refresh", full_refresh(facts)), ( diff --git a/crates/sqlbuild-core/src/build_command/models.rs b/crates/sqlbuild-core/src/build_command/models.rs index 1ab51524d8..463ed782e8 100644 --- a/crates/sqlbuild-core/src/build_command/models.rs +++ b/crates/sqlbuild-core/src/build_command/models.rs @@ -179,8 +179,6 @@ pub struct RunContextFacts { /// The warehouse and where it came from. pub warehouse: Option<(String, String)>, pub concurrency: i64, - /// The configuration layer the concurrency came from. - pub concurrency_source: String, pub full_refresh: bool, /// The named jobs whose own models the build rebuilds from scratch. pub full_refresh_jobs: Vec, diff --git a/crates/sqlbuild-core/src/build_command/tests/test_build_command.rs b/crates/sqlbuild-core/src/build_command/tests/test_build_command.rs index eec9cd311d..ed737c0512 100644 --- a/crates/sqlbuild-core/src/build_command/tests/test_build_command.rs +++ b/crates/sqlbuild-core/src/build_command/tests/test_build_command.rs @@ -121,13 +121,12 @@ fn given_run_context_when_rendering_then_sanitizes_and_groups_counts() { command: "sqb build".to_owned(), run_id: Some("run\u{1b}1".to_owned()), concurrency: 4, - concurrency_source: "targets.prod.concurrency".to_owned(), selected_count: 1200, total_count: 1_234_567, date_vars: (Some("2026-01-01".to_owned()), Some("2026-01-31".to_owned())), ..RunContextFacts::default() }, - expected_text: "Execution\n command sqb build\n run_id run\\x1b1\n target not set\n database not set\n schema not set\n warehouse not set\n concurrency 4 configured limit (targets.prod.concurrency)\n full_refresh false\n selected 1,200 of 1,234,567 build resources\n date vars 2026-01-01 to 2026-01-31\n\n", + expected_text: "Execution\n command sqb build\n run_id run\\x1b1\n target not set\n database not set\n schema not set\n warehouse not set\n concurrency 4 configured limit\n full_refresh false\n selected 1,200 of 1,234,567 build resources\n date vars 2026-01-01 to 2026-01-31\n\n", }, ContextTestCase { description: "a warehouse, one date and selector files", diff --git a/crates/sqlbuild-orchestration/src/declarations/models.rs b/crates/sqlbuild-orchestration/src/declarations/models.rs index bddd2e927a..91ad53a7fd 100644 --- a/crates/sqlbuild-orchestration/src/declarations/models.rs +++ b/crates/sqlbuild-orchestration/src/declarations/models.rs @@ -181,28 +181,11 @@ pub struct WaitReason { pub to_node: String, } -/// The layer that set a run's concurrency. -#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] -#[serde(rename_all = "snake_case")] -pub enum ConcurrencySource { - /// The job's own `[settings] concurrency`. - Job, - /// The target's `concurrency` in `[targets.]`. - Target, - /// `[settings] concurrency` in the location's `sqlbuild_local.toml`. - Local, - /// `[settings] concurrency` in `sqlbuild_project.toml`. - Project, - /// SQLBuild's built-in default. - Default, -} - -/// The models a run builds at once on its target, and the layer that set it. +/// The most of a job's nodes a run of it builds at once on its target. #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] #[serde(deny_unknown_fields)] pub struct ResolvedConcurrency { pub limit: u64, - pub source: ConcurrencySource, } /// How many slots of a pool a job takes, and the pool's limit on the job's target. diff --git a/crates/sqlbuild-orchestration/src/store_migrations/_helpers/migration.rs b/crates/sqlbuild-orchestration/src/store_migrations/_helpers/migration.rs index de16a20674..0c3885b33b 100644 --- a/crates/sqlbuild-orchestration/src/store_migrations/_helpers/migration.rs +++ b/crates/sqlbuild-orchestration/src/store_migrations/_helpers/migration.rs @@ -25,6 +25,11 @@ pub(crate) const MIGRATIONS: &[StoreMigration] = &[ sqlite: include_str!("../sql/0003_attention_acknowledgement_index.sqlite.sql"), postgres: include_str!("../sql/0003_attention_acknowledgement_index.postgres.sql"), }, + StoreMigration { + name: "drop_concurrency_source", + sqlite: include_str!("../sql/0004_drop_concurrency_source.sqlite.sql"), + postgres: include_str!("../sql/0004_drop_concurrency_source.postgres.sql"), + }, ]; /// The schema version a store created by this build has. diff --git a/crates/sqlbuild-orchestration/src/store_migrations/sql/0004_drop_concurrency_source.postgres.sql b/crates/sqlbuild-orchestration/src/store_migrations/sql/0004_drop_concurrency_source.postgres.sql new file mode 100644 index 0000000000..d7e07a9250 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store_migrations/sql/0004_drop_concurrency_source.postgres.sql @@ -0,0 +1,3 @@ +-- Published jobs no longer record which layer set their run concurrency: drop the removed +-- resolved_concurrency.source from each job's stored definition, keeping its limit. +UPDATE sqb_jobs SET definition_json = definition_json #- ARRAY['resolved_concurrency', 'source'] WHERE definition_json -> 'resolved_concurrency' -> 'source' IS NOT NULL; diff --git a/crates/sqlbuild-orchestration/src/store_migrations/sql/0004_drop_concurrency_source.sqlite.sql b/crates/sqlbuild-orchestration/src/store_migrations/sql/0004_drop_concurrency_source.sqlite.sql new file mode 100644 index 0000000000..0d72b77ddf --- /dev/null +++ b/crates/sqlbuild-orchestration/src/store_migrations/sql/0004_drop_concurrency_source.sqlite.sql @@ -0,0 +1,3 @@ +-- Published jobs no longer record which layer set their run concurrency: drop the removed +-- resolved_concurrency.source from each job's stored definition, keeping its limit. +UPDATE sqb_jobs SET definition_json = json_remove(definition_json, '$.resolved_concurrency.source') WHERE json_extract(definition_json, '$.resolved_concurrency.source') IS NOT NULL; diff --git a/crates/sqlbuild-orchestration/tests/runs/test_job_detail.rs b/crates/sqlbuild-orchestration/tests/runs/test_job_detail.rs index ebc26a0c8a..b1c3439ec7 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_job_detail.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_job_detail.rs @@ -72,7 +72,7 @@ fn given_published_jobs_when_people_read_and_launch_them_then_listing_view_and_r "declared_at": {"path": "orchestration/jobs.py", "line": 7}, "owner": "Orders team", "overlap": "skip", "pools": {"warehouse": {"amount": 3, "limit": 4}}, - "resolved_concurrency": {"limit": 4, "source": "job"}, + "resolved_concurrency": {"limit": 4}, "on_failure": {"retries": 2, "retry_delay_seconds": 300}}, "customers": {"description": "Build customers", "selection": ["customers"]}, })); @@ -231,10 +231,10 @@ fn given_published_jobs_when_people_read_and_launch_them_then_listing_view_and_r json!([{"upstream_job": "daily_orders", "from_node": "orders", "to_node": "customers"}]), json!([{"pool": "warehouse", "amount": 3, "limit": 4, "cap": 2, "server_limit": 2, "clamped": true, "shared_with": ["orders_group"]}]), - (Some(4), json!({"limit": 4, "source": "job"})), + (Some(4), json!({"limit": 4})), json!([{"pool": "warehouse", "amount": 3, "limit": 4, "cap": 2, "server_limit": 2, "clamped": true, "shared_with": ["orders_group"]}]), - (Some(4), json!({"limit": 4, "source": "job"})), + (Some(4), json!({"limit": 4})), json!([{"pool": "warehouse", "amount": 1, "limit": 4, "cap": 2, "server_limit": 2, "clamped": false, "shared_with": ["daily_orders"]}]), json!(null) diff --git a/crates/sqlbuild-orchestration/tests/scheduler/test_concurrency_source.rs b/crates/sqlbuild-orchestration/tests/scheduler/test_concurrency_source.rs new file mode 100644 index 0000000000..dab6199995 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/scheduler/test_concurrency_source.rs @@ -0,0 +1,281 @@ +//! Migration 0004 drops the removed `resolved_concurrency.source` from stored job definitions, and a pinned snapshot document still in the old form fails. + +use std::collections::BTreeMap; + +use chrono::Utc; +use serde_json::{Value, json}; +use uuid::Uuid; + +use crate::credentials::models::AuditContext; +use crate::credentials::tests::location_helpers::create_location; +use crate::runs::main::{ + execution::advance_groups::advance_groups, requests::request_job_run::request_job_run, +}; +use crate::runs::models::JobRunRequest; +use crate::runs::tests::helpers::{json_column, rows, snapshot}; +use crate::scheduler::main::_scheduler_cycle::scheduler_cycle; +use crate::scheduler::models::SchedulerState; +use crate::scheduler::tests::contracts::helpers::acquire_lease; +use crate::scheduler::tests::contracts::test_types::StoredConcurrencySourceTestCase; +use crate::scheduler::tests::contracts::ticks::helpers::snapshot as scheduled_snapshot; +use crate::sensors::tests::helpers::append_events; +use crate::snapshots::main::publication::publish_snapshot::publish_snapshot; +use crate::store::errors::ControlError; +use crate::store::main::compress::compress; +use crate::store::main::decompress::decompress; +use crate::store::main::query::query; +use crate::store::main::{begin::begin, commit::commit, migrate_store::migrate_store}; +use crate::store::models::{ControlStore, Parameter, TransactionMode}; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; +use crate::store_migrations::_helpers::migration::MIGRATIONS; +use crate::store_migrations::_helpers::migration_run::migrate_with; + +/// The field 0004 removes, as releases before it stored it. +const OLD_SOURCE: &str = "project"; + +const BACKENDS: [StoredConcurrencySourceTestCase; 2] = [ + StoredConcurrencySourceTestCase { + description: "SQLite store", + backend: Backend::Sqlite, + }, + StoredConcurrencySourceTestCase { + description: "PostgreSQL store", + backend: Backend::Postgres, + }, +]; + +fn operator() -> AuditContext { + AuditContext { + actor: "test-operator".into(), + granting_role: "control-store-owner".into(), + request_id: Uuid::now_v7(), + ip_address: None, + } +} + +fn manual_run(location_id: Uuid, job_key: &str) -> JobRunRequest { + JobRunRequest { + run_id: Uuid::now_v7(), + location_id, + target: "development".into(), + job_key: job_key.into(), + reason: "Build it now".into(), + priority: 0, + window: None, + full_refresh: false, + select: Vec::new(), + exclude: Vec::new(), + } +} + +/// Write every published job's definition back with the source an older release recorded. +async fn store_old_job_definitions(store: &ControlStore) { + let jobs = rows( + store, + "SELECT job_key, definition_json FROM sqb_jobs", + vec![], + ) + .await; + let mut transaction = begin(store, TransactionMode::Write).await.expect("begin"); + for job in jobs { + let mut definition: Value = json_column(&job["definition_json"]); + definition["resolved_concurrency"]["source"] = json!(OLD_SOURCE); + (transaction, _) = query( + transaction, + "UPDATE sqb_jobs SET definition_json=$1 WHERE job_key=$2", + vec![ + Parameter::Json(definition), + Parameter::Text(job["job_key"].as_str().expect("job key").to_owned()), + ], + ) + .await + .expect("store the old definition"); + } + commit(transaction).await.expect("commit"); +} + +/// Rewrite the stored `jobs` document of every snapshot as an older release published it. +async fn store_old_jobs_documents(store: &ControlStore) { + let documents = rows( + store, + "SELECT b.document_hash, b.body FROM sqb_snapshot_documents d \ + JOIN sqb_documents b ON b.document_hash=d.document_hash WHERE d.kind='jobs'", + vec![], + ) + .await; + let mut transaction = begin(store, TransactionMode::Write).await.expect("begin"); + for document in documents { + let mut body: Value = + serde_json::from_slice(&decompress(&document["body"]).expect("body")).expect("JSON"); + for job in body.as_object_mut().expect("jobs").values_mut() { + job["resolved_concurrency"]["source"] = json!(OLD_SOURCE); + } + (transaction, _) = query( + transaction, + "UPDATE sqb_documents SET body=$1 WHERE document_hash=$2", + vec![ + Parameter::Bytes( + compress(&serde_json::to_vec(&body).expect("JSON")).expect("gzip"), + ), + Parameter::Text(document["document_hash"].as_str().expect("hash").to_owned()), + ], + ) + .await + .expect("store the old document"); + } + commit(transaction).await.expect("commit"); +} + +#[test] +fn given_a_job_stored_with_its_concurrency_source_when_migrating_then_schedules_sensors_and_manual_runs_read_it() + { + let mut completed = 0; + for test_case in BACKENDS { + run(async { + let fixture = open_test_store(test_case.backend) + .await + .expect("required backend"); + let store = &fixture.store; + migrate_with(store, &MIGRATIONS[..2]) + .await + .expect("a store at version 3"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let mut input = scheduled_snapshot(location); + input.documents.insert( + "jobs".into(), + json!({"orders": {"description": "Build orders", "selection": ["orders"], + "resolved_concurrency": {"limit": 4}}}), + ); + input.documents.get_mut("triggers").expect("triggers")["health_orders"] = json!({ + "description": "Refresh orders on overdue ticks", "job_key": "orders", + "when": {"kind": "platform_event", "kinds": ["tick_overdue"]}, + "minimum_interval_seconds": 1 + }); + publish_snapshot(store, &input).await.expect("publish"); + store_old_job_definitions(store).await; + let before = + request_job_run(store, (&operator(), None), &manual_run(location, "orders")) + .await + .map(|_| ()); + + migrate_store(store).await.expect("migrate to version 4"); + let stored = rows(store, "SELECT definition_json FROM sqb_jobs", vec![]).await; + append_events(store, 0..1, Utc::now()).await; + let (_, cycle) = scheduler_cycle(store, SchedulerState::new(Uuid::now_v7())) + .await + .expect("a scheduler cycle over the migrated jobs"); + request_job_run(store, (&operator(), None), &manual_run(location, "orders")) + .await + .expect("a manual run of the migrated job"); + let runs = rows( + store, + "SELECT origin, COUNT(*) AS runs FROM sqb_runs GROUP BY origin ORDER BY origin", + vec![], + ) + .await; + + assert_eq!( + ( + matches!(&before, Err(ControlError::Invalid(message)) if message.contains("unknown field `source`")), + json_column(&stored[0]["definition_json"])["resolved_concurrency"].clone(), + cycle.ticks, + runs.iter() + .map(|row| format!( + "{} {}", + row["origin"].as_str().unwrap_or_default(), + row["runs"] + )) + .collect::>(), + ), + ( + true, + json!({"limit": 4}), + 1, + vec![ + "manual 1".to_owned(), + "schedule 1".to_owned(), + "sensor 1".to_owned() + ], + ), + "{}: refused before 0004, then read by the scheduler, its sensor and a manual run", + test_case.description + ); + close_test_store(fixture).await; + completed += 1; + }); + } + assert_eq!(completed, BACKENDS.len(), "every backend ran"); +} + +// Runs pinned to a snapshot published before 0004 read its immutable documents, which 0004 can't +// rewrite. They still fail with `ControlError::Invalid`, which stops the scheduler; operators let +// such runs finish or cancel them before upgrading. Turning this into a per-run failure is tracked +// separately, and that change replaces this expectation. +#[test] +fn given_a_group_pinned_to_a_snapshot_document_in_the_old_form_when_advancing_it_then_it_fails_invalid() + { + let mut completed = 0; + for test_case in BACKENDS { + run(async { + let fixture = open_test_store(test_case.backend) + .await + .expect("required backend"); + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let mut input = snapshot(location); + input.nodes.get_mut("orders").expect("orders").definition["deps"] = + json!(["customers"]); + input.documents.insert( + "jobs".into(), + json!({ + "staging_job": {"description": "Stage customers", "selection": ["customers"], + "resolved_concurrency": {"limit": 4}}, + "marts_job": {"description": "Build orders", "selection": ["orders"], + "resolved_concurrency": {"limit": 4}} + }), + ); + input.documents.insert( + "groups".into(), + json!({"nightly_job_group": { + "description": "Stage then build orders", "selection": ["customers", "orders"], + "jobs": {"staging_job": [], "marts_job": ["staging_job"]} + }}), + ); + publish_snapshot(store, &input).await.expect("publish"); + request_job_run( + store, + (&operator(), None), + &manual_run(location, "nightly_job_group"), + ) + .await + .expect("request the group"); + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + let limits = BTreeMap::new(); + let scope = (location, "development"); + advance_groups(store, &lease, (&limits, scope)) + .await + .expect("start the group"); + store_old_jobs_documents(store).await; + + let advanced = advance_groups(store, &lease, (&limits, scope)).await; + + assert!( + matches!(&advanced, Err(ControlError::Invalid(message)) if message == "jobs: unknown field `source`, expected `limit`"), + "{}: {advanced:?}", + test_case.description + ); + close_test_store(fixture).await; + completed += 1; + }); + } + assert_eq!(completed, BACKENDS.len(), "every backend ran"); +} diff --git a/crates/sqlbuild-orchestration/tests/scheduler/test_contracts.rs b/crates/sqlbuild-orchestration/tests/scheduler/test_contracts.rs index c70bd73153..b248cb5ca8 100644 --- a/crates/sqlbuild-orchestration/tests/scheduler/test_contracts.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler/test_contracts.rs @@ -2,6 +2,8 @@ #[path = "helpers.rs"] pub(crate) mod helpers; +#[path = "test_concurrency_source.rs"] +mod test_concurrency_source; #[path = "test_contract.rs"] mod test_contract; #[path = "test_types.rs"] diff --git a/crates/sqlbuild-orchestration/tests/scheduler/test_types.rs b/crates/sqlbuild-orchestration/tests/scheduler/test_types.rs index fd5543086b..97ead9e28b 100644 --- a/crates/sqlbuild-orchestration/tests/scheduler/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler/test_types.rs @@ -8,3 +8,8 @@ pub(crate) struct LeadershipTestCase { pub(crate) backend: Backend, pub(crate) expected_epoch: i64, } + +pub(crate) struct StoredConcurrencySourceTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, +} diff --git a/crates/sqlbuild-orchestration/tests/store/schema/test_store_migrations.rs b/crates/sqlbuild-orchestration/tests/store/schema/test_store_migrations.rs index 041225eeed..2db3f1cfd3 100644 --- a/crates/sqlbuild-orchestration/tests/store/schema/test_store_migrations.rs +++ b/crates/sqlbuild-orchestration/tests/store/schema/test_store_migrations.rs @@ -41,6 +41,11 @@ const SQLITE_CHECKSUMS: &[(i64, &str)] = &[ 3, "734163e22863ff86fbcf1329540736f279806d4ce5ea04f6d6ecb1703b3468c8", ), + // 0004 rewrites stored job definitions and leaves the schema as it is. + ( + 4, + "734163e22863ff86fbcf1329540736f279806d4ce5ea04f6d6ecb1703b3468c8", + ), ]; const POSTGRES_CHECKSUMS: &[(i64, &str)] = &[ ( @@ -55,6 +60,11 @@ const POSTGRES_CHECKSUMS: &[(i64, &str)] = &[ 3, "b1250935a5adf1d4691836a2125bb57360c1ff0a2a296a77e4d11bf6a80a99d9", ), + // 0004 rewrites stored job definitions and leaves the schema as it is. + ( + 4, + "b1250935a5adf1d4691836a2125bb57360c1ff0a2a296a77e4d11bf6a80a99d9", + ), ]; /// A version 1 store mid-run: a child run holding a concurrency slot, a jobs document and an audit result. const VERSION_1_LIVE_WORK: &str = "ALTER TABLE sqb_schema_version ADD COLUMN version BIGINT; \ diff --git a/crates/sqlbuild-python-compile/src/compile_driver/main/jobs/build_concurrency.rs b/crates/sqlbuild-python-compile/src/compile_driver/main/jobs/build_concurrency.rs index 3ad7861e98..98a53f1601 100644 --- a/crates/sqlbuild-python-compile/src/compile_driver/main/jobs/build_concurrency.rs +++ b/crates/sqlbuild-python-compile/src/compile_driver/main/jobs/build_concurrency.rs @@ -1,7 +1,7 @@ //! The concurrency limit of a build that names jobs. use pyo3::prelude::{Bound, PyAny, PyResult}; -use sqlbuild_config::effective::models::{ConcurrencySource, EffectiveConcurrency}; +use sqlbuild_config::effective::models::EffectiveConcurrency; use crate::compile_driver::_helpers::jobs::job_limits::job_limit_total; @@ -11,10 +11,7 @@ pub fn build_concurrency( pipeline: &Bound<'_, PyAny>, ) -> PyResult { Ok(match job_limit_total(pipeline)? { - Some(total) if total > configured.limit => EffectiveConcurrency { - limit: total, - source: ConcurrencySource::NamedJobs, - }, + Some(total) if total > configured.limit => EffectiveConcurrency { limit: total }, _ => configured, }) } diff --git a/crates/sqlbuild-python-front/src/command_runtime/main/published_concurrency.rs b/crates/sqlbuild-python-front/src/command_runtime/main/published_concurrency.rs index a9a9bff66f..0dbf658477 100644 --- a/crates/sqlbuild-python-front/src/command_runtime/main/published_concurrency.rs +++ b/crates/sqlbuild-python-front/src/command_runtime/main/published_concurrency.rs @@ -1,21 +1,14 @@ -//! The concurrency a published job without its own setting runs with on the compiled `target`, and its layer. - -use std::path::Path; +//! The concurrency a published job without its own setting runs with on the compiled `target`. use pyo3::IntoPyObject; use pyo3::exceptions::PyValueError; use pyo3::prelude::{Bound, PyAny, PyResult}; -use sqlbuild_config::effective::models::ConcurrencySource as Layer; -use sqlbuild_config::project::main::load_project_setting_keys::load_project_setting_keys; -use sqlbuild_config::project::main::project_config_paths::project_config_paths; -use sqlbuild_orchestration::declarations::models::{ConcurrencySource, ResolvedConcurrency}; +use sqlbuild_orchestration::declarations::models::ResolvedConcurrency; -use crate::command_runtime::_helpers::concurrency::CONCURRENCY; use crate::command_runtime::main::resolve_concurrency::resolve_concurrency; -/// The concurrency a published job without its own setting runs with on the compiled `target`, and its layer. +/// The concurrency a published job without its own setting runs with on the compiled `target`. pub fn published_concurrency( - project_dir: &Path, discovered: &Bound<'_, PyAny>, target: Option<&str>, ) -> PyResult { @@ -25,20 +18,7 @@ pub fn published_concurrency( discovered, target.into_pyobject(py)?.as_any(), )?; - let project_sets = || -> PyResult { - let keys = load_project_setting_keys(&project_config_paths(project_dir)).map_err(|_| { - PyValueError::new_err("cannot read [settings] of sqlbuild_project.toml") - })?; - Ok(keys.iter().any(|key| key == CONCURRENCY)) - }; - let source = match effective.source { - Layer::CommandLine | Layer::NamedJobs => ConcurrencySource::Job, - Layer::Target(_) => ConcurrencySource::Target, - Layer::LocalSettings => ConcurrencySource::Local, - Layer::ProjectSettings if project_sets()? => ConcurrencySource::Project, - Layer::ProjectSettings => ConcurrencySource::Default, - }; let limit = u64::try_from(effective.limit) .map_err(|_| PyValueError::new_err("concurrency must be at least 1"))?; - Ok(ResolvedConcurrency { limit, source }) + Ok(ResolvedConcurrency { limit }) } diff --git a/crates/sqlbuild-python-front/src/command_runtime/main/resolve_configured_concurrency.rs b/crates/sqlbuild-python-front/src/command_runtime/main/resolve_configured_concurrency.rs index 5a40e5d3a8..cda9ed3e57 100644 --- a/crates/sqlbuild-python-front/src/command_runtime/main/resolve_configured_concurrency.rs +++ b/crates/sqlbuild-python-front/src/command_runtime/main/resolve_configured_concurrency.rs @@ -16,28 +16,23 @@ pub fn resolve_configured_concurrency( selected_target: &Bound<'_, PyAny>, ) -> PyResult { let py = project_config.py(); - let target: Option<(String, i64)> = + let target: Option = match resolve_target_name(py, project_config, local_config, selected_target)? { Some(name) => { let config: Bound<'_, PyAny> = resolve_target_config(py, project_config, local_config, name.as_any())?; - let limit: Option = native_field(&config, CONCURRENCY)?.extract()?; - limit.map(|limit| Ok::<_, pyo3::PyErr>((name.extract::()?, limit))) + native_field(&config, CONCURRENCY)?.extract()? } None => None, - } - .transpose()?; + }; let settings: i64 = native_field( &build_effective_settings(project_config, local_config)?, CONCURRENCY, )? .extract()?; - let local_override: bool = - native_field(local_config, "setting_overrides")?.contains(CONCURRENCY)?; Ok(effective_concurrency(&ConcurrencyLayers { requested: requested.extract()?, - target: target.as_ref().map(|(name, limit)| (name.as_str(), *limit)), + target, settings, - local_override, })) } diff --git a/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_declarations.rs b/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_declarations.rs index d03fc007b2..c4d89a51e5 100644 --- a/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_declarations.rs +++ b/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_declarations.rs @@ -19,8 +19,8 @@ use sqlbuild_orchestration::declarations::main::group_dependencies::group_depend use sqlbuild_orchestration::declarations::main::group_wait_reasons::group_wait_reasons; use sqlbuild_orchestration::declarations::main::snapshot_definitions::snapshot_definitions; use sqlbuild_orchestration::declarations::models::{ - ConcurrencySource, DeclaredAt, Definitions, GroupDefinition, JobDefinition, PoolSlots, - ResolvedConcurrency, Schedule, TriggerDeclaration, TriggerDefinition, TriggerStatus, + DeclaredAt, Definitions, GroupDefinition, JobDefinition, PoolSlots, ResolvedConcurrency, + Schedule, TriggerDeclaration, TriggerDefinition, TriggerStatus, }; use sqlbuild_orchestration::snapshots::models::SnapshotInput; @@ -156,10 +156,7 @@ fn job_concurrency( target: ResolvedConcurrency, ) -> ResolvedConcurrency { match settings.get("concurrency").and_then(Value::as_u64) { - Some(limit) => ResolvedConcurrency { - limit, - source: ConcurrencySource::Job, - }, + Some(limit) => ResolvedConcurrency { limit }, None => target, } } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs index 85c0bb3ec6..2f237f69c7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs @@ -177,7 +177,6 @@ fn run_context_facts( schema: display_text(&native_field(&project, "effective_target_schema")?)?, warehouse, concurrency: concurrency.limit, - concurrency_source: concurrency.source.to_string(), full_refresh: context.request_flag("full_refresh")?, full_refresh_jobs, job_warehouses, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs index f2e7efbe18..381eb77170 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs @@ -120,7 +120,6 @@ pub(crate) fn orchestration_snapshot<'py>( validate_orchestration_names(&facts).map_err(|failure| validation_error(py, failure))?; let declarations = python_declarations(py, &project_dir, &inventory)?; let concurrency = published_concurrency( - &project_dir, &discovered_inputs, project.effective_target_name().as_deref(), )?; diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_selector_surface.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_selector_surface.py index 44f4bc3ab1..69ec9e5be0 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_selector_surface.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_selector_surface.py @@ -136,7 +136,7 @@ def test_given_selector_commands_when_running_cli_then_behavior_matches_expectat "run_id ", "target not set", "warehouse not set", - "concurrency 3 configured limit", + "concurrency 3 configured limit\n", "full_refresh false", "selected 2 of 19 build resources", "date vars 1970-01-01 to 2030-12-31", diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_target_concurrency.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_target_concurrency.py index 871c5bbeca..3c6ef44909 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_target_concurrency.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_target_concurrency.py @@ -31,42 +31,42 @@ target_concurrency="concurrency = 3\n", local_settings="", command=("--no-color", "build", "--verbose", "--target", "ci"), - expected_fragment="3 configured limit (targets.ci.concurrency)", + expected_fragment="concurrency 3 configured limit\n", ), TargetConcurrencyE2ETestCase( description="--concurrency overrides the target", target_concurrency="concurrency = 3\n", local_settings="", command=("--no-color", "build", "--verbose", "--target", "ci", "--concurrency", "1"), - expected_fragment="1 configured limit (--concurrency)", + expected_fragment="concurrency 1 configured limit\n", ), TargetConcurrencyE2ETestCase( description="the target overrides a local settings override", target_concurrency="concurrency = 3\n", local_settings="[settings]\nconcurrency = 5\n", command=("--no-color", "build", "--verbose", "--target", "ci"), - expected_fragment="3 configured limit (targets.ci.concurrency)", + expected_fragment="concurrency 3 configured limit\n", ), TargetConcurrencyE2ETestCase( description="a target without concurrency keeps the local settings override", target_concurrency="", local_settings="[settings]\nconcurrency = 5\n", command=("--no-color", "build", "--verbose", "--target", "ci"), - expected_fragment="5 configured limit (sqlbuild_local.toml settings.concurrency)", + expected_fragment="concurrency 5 configured limit\n", ), TargetConcurrencyE2ETestCase( description="a local target concurrency overrides the project target", target_concurrency="concurrency = 3\n", local_settings="[targets.ci]\nconcurrency = 4\n", command=("--no-color", "build", "--verbose", "--target", "ci"), - expected_fragment="4 configured limit (targets.ci.concurrency)", + expected_fragment="concurrency 4 configured limit\n", ), TargetConcurrencyE2ETestCase( description="other targets keep the project setting", target_concurrency="concurrency = 3\n", local_settings="", command=("--no-color", "build", "--verbose"), - expected_fragment="2 configured limit (settings.concurrency)", + expected_fragment="concurrency 2 configured limit\n", ), TargetConcurrencyE2ETestCase( description="seed uses the target's concurrency", diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_job_settings.py b/tests/integration/src/sqlbuild/cli/commands/main/test_job_settings.py index fa8ec58e87..0028592a07 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_job_settings.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_job_settings.py @@ -439,14 +439,14 @@ def test_given_a_job_concurrency_when_building_then_its_nodes_never_exceed_it( description="the run context says the named jobs set the limit", command_args=("build", "--verbose", "--select", "job:wide_job", "+orders"), expected_exit_code=0, - expected_present=("6 configured limit (named jobs' concurrency)",), + expected_present=("concurrency 6 configured limit\n",), expected_absent=(), ), JobOutputTestCase( description="a configured limit above the named jobs' stays the build's limit", command_args=("build", "--verbose", "--select", "job:orders_job", "customers"), expected_exit_code=0, - expected_present=("4 configured limit (settings.concurrency)",), + expected_present=("concurrency 4 configured limit\n",), expected_absent=(), ), JobOutputTestCase( diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py index d83cd55041..5ff1e15611 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py @@ -935,7 +935,7 @@ def snapshot(schedule: str) -> dict[str, Any]: "warehouse": "reporting", }, "pools": {"order_reporting": {"amount": 2, "limit": 2}}, - "resolved_concurrency": {"limit": 4, "source": "job"}, + "resolved_concurrency": {"limit": 4}, "select": "+orders", "exclude": "stg_orders", "targets": ["development"], @@ -971,8 +971,8 @@ def snapshot(schedule: str) -> dict[str, Any]: local_files={}, job_settings="", selected_target=None, - expected_job={"limit": 1, "source": "default"}, - expected_group={"limit": 1, "source": "default"}, + expected_job={"limit": 1}, + expected_group={"limit": 1}, ), SnapshotConcurrencyTestCase( description="the project's setting", @@ -980,8 +980,8 @@ def snapshot(schedule: str) -> dict[str, Any]: local_files={}, job_settings="", selected_target=None, - expected_job={"limit": 3, "source": "project"}, - expected_group={"limit": 3, "source": "project"}, + expected_job={"limit": 3}, + expected_group={"limit": 3}, ), SnapshotConcurrencyTestCase( description="the location's local setting over the project's", @@ -989,8 +989,8 @@ def snapshot(schedule: str) -> dict[str, Any]: local_files={"sqlbuild_local.toml": "[settings]\nconcurrency = 5\n"}, job_settings="", selected_target=None, - expected_job={"limit": 5, "source": "local"}, - expected_group={"limit": 5, "source": "local"}, + expected_job={"limit": 5}, + expected_group={"limit": 5}, ), SnapshotConcurrencyTestCase( description="the target's concurrency over the settings", @@ -998,8 +998,8 @@ def snapshot(schedule: str) -> dict[str, Any]: local_files={}, job_settings="", selected_target="development", - expected_job={"limit": 6, "source": "target"}, - expected_group={"limit": 6, "source": "target"}, + expected_job={"limit": 6}, + expected_group={"limit": 6}, ), SnapshotConcurrencyTestCase( description="the job's own setting over the target's; its group keeps the target's", @@ -1007,8 +1007,8 @@ def snapshot(schedule: str) -> dict[str, Any]: local_files={}, job_settings="\n[settings]\nconcurrency = 2\n", selected_target="development", - expected_job={"limit": 2, "source": "job"}, - expected_group={"limit": 6, "source": "target"}, + expected_job={"limit": 2}, + expected_group={"limit": 6}, ), ], ids=lambda case: case.description, From 600409adf39cd74bfa50ce1b129f078f02bf0ccd Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 20 Nov 2026 02:03:02 +0000 Subject: [PATCH 2/4] feat(ui)!: state a job's run concurrency as a plain limit The job page's Concurrency panel reads "Inside a run: at most 4 of this job's nodes at once" for a job, and "Members without their own limit: at most 6 nodes at once" for a group, without naming the layer that set it. A test reads it through the server's job read on the fake control server. The TypeScript control client is regenerated without ConcurrencySource, and the server upgrade docs say to let runs finish or cancel them before upgrading, then restart every location. The migration and pinned-document tests follow the scheduler test layout, and load_project_setting_keys, whose only caller named the project layer, is removed. BREAKING CHANGE: a published job's `resolved_concurrency` is now `{limit}`. Its `source` field and the `ConcurrencySource` schema are removed from the control API, and `sqb build --verbose` no longer names the setting the concurrency came from. Control store migration 0004 removes the field from each stored job definition. Let queued and running runs finish, or cancel them, before upgrading: a group run or retry pinned to a snapshot published by an earlier release fails with `jobs: unknown field 'source'` and stops the scheduler. After upgrading the server, restart every location so each publishes its project again. --- clients/typescript/src/schema.d.ts | 8 +- .../src/project/_helpers/documents.rs | 9 - .../project/main/load_project_setting_keys.rs | 9 - .../sqlbuild-config/src/project/main/mod.rs | 1 - .../tests/scheduler/helpers.rs | 94 ++++++++ .../scheduler/test_concurrency_source.rs | 215 ++++++------------ .../tests/scheduler/test_types.rs | 19 +- .../sqlbuild/references/docs/cli/server.md | 6 + .../docs/concepts/project-configuration.md | 3 +- ui/e2e/concurrency.spec.ts | 42 +++- ui/e2e/support/fake-control.ts | 20 +- ui/src/lib/api/http-jobs.test.ts | 4 +- ui/src/lib/api/mock/operate.ts | 11 +- ui/src/lib/api/types/operate.ts | 4 +- ui/src/lib/operate/labels.test.ts | 13 +- ui/src/lib/operate/labels.ts | 23 +- ui/src/routes/jobs/[name]/job-view.svelte | 4 +- website/src/content/docs/docs/cli/server.mdx | 6 + .../docs/concepts/project-configuration.mdx | 3 +- 19 files changed, 278 insertions(+), 216 deletions(-) delete mode 100644 crates/sqlbuild-config/src/project/main/load_project_setting_keys.rs diff --git a/clients/typescript/src/schema.d.ts b/clients/typescript/src/schema.d.ts index 238a3410c8..9e47129e2d 100644 --- a/clients/typescript/src/schema.d.ts +++ b/clients/typescript/src/schema.d.ts @@ -1890,11 +1890,6 @@ export interface components { /** Format: uuid */ snapshot_id?: string | null; }; - /** - * @description The layer that set a run's concurrency. - * @enum {string} - */ - ConcurrencySource: "job" | "target" | "local" | "project" | "default"; /** @description A row whose key was already stored with different content; the first row is kept. */ ConflictEvidence: { existing_sha256: string; @@ -3183,11 +3178,10 @@ export interface components { /** Format: uuid */ source_run_id: string; }; - /** @description The models a run builds at once on its target, and the layer that set it. */ + /** @description The most of a job's nodes a run of it builds at once on its target. */ ResolvedConcurrency: { /** Format: int64 */ limit: number; - source: components["schemas"]["ConcurrencySource"]; }; RoleAssignmentRequest: { /** Format: uuid */ diff --git a/crates/sqlbuild-config/src/project/_helpers/documents.rs b/crates/sqlbuild-config/src/project/_helpers/documents.rs index 9b85455043..3c940a56aa 100644 --- a/crates/sqlbuild-config/src/project/_helpers/documents.rs +++ b/crates/sqlbuild-config/src/project/_helpers/documents.rs @@ -518,15 +518,6 @@ pub(crate) fn pool_limits_config(paths: &ConfigPaths) -> Result Result, LoadFailure> { - let file = resolve(paths, false)?; - let table = load_toml_mapping(paths, file)?; - let settings = - get(table.as_map().unwrap_or_default(), "settings").and_then(ConfigValue::as_map); - Ok(settings.map(setting_overrides).unwrap_or_default()) -} - /// `[git] base_ref` of `sqlbuild_project.toml`. pub(crate) fn git_base_ref(paths: &ConfigPaths) -> Result, LoadFailure> { project_section(paths, "git", load_git) diff --git a/crates/sqlbuild-config/src/project/main/load_project_setting_keys.rs b/crates/sqlbuild-config/src/project/main/load_project_setting_keys.rs deleted file mode 100644 index 3795ac8257..0000000000 --- a/crates/sqlbuild-config/src/project/main/load_project_setting_keys.rs +++ /dev/null @@ -1,9 +0,0 @@ -//! Read which `[settings]` keys `sqlbuild_project.toml` sets itself, rather than leaving to defaults. - -use crate::project::errors::LoadFailure; -use crate::project::models::ConfigPaths; - -/// The `[settings]` keys the project file sets, under their current names. -pub fn load_project_setting_keys(paths: &ConfigPaths) -> Result, LoadFailure> { - crate::project::_helpers::documents::project_setting_keys(paths) -} diff --git a/crates/sqlbuild-config/src/project/main/mod.rs b/crates/sqlbuild-config/src/project/main/mod.rs index abf7b235dc..5785c9a6fc 100644 --- a/crates/sqlbuild-config/src/project/main/mod.rs +++ b/crates/sqlbuild-config/src/project/main/mod.rs @@ -7,7 +7,6 @@ pub mod load_git_base_ref; pub mod load_local_config; pub mod load_pool_limits; pub mod load_project_config; -pub mod load_project_setting_keys; pub mod load_review_config; pub mod local_sql_analysis_failure; pub mod microbatch_guidance; diff --git a/crates/sqlbuild-orchestration/tests/scheduler/helpers.rs b/crates/sqlbuild-orchestration/tests/scheduler/helpers.rs index b892b9daaa..2d6671b35e 100644 --- a/crates/sqlbuild-orchestration/tests/scheduler/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler/helpers.rs @@ -99,3 +99,97 @@ pub(crate) async fn leadership_contract(backend: Backend) -> Option { close_test_store(test_store).await; Some(epoch) } + +/// The operator a test's manual run requests come from. +pub(crate) fn operator() -> crate::credentials::models::AuditContext { + crate::credentials::models::AuditContext { + actor: "test-operator".into(), + granting_role: "control-store-owner".into(), + request_id: Uuid::now_v7(), + ip_address: None, + } +} + +/// A manual request for one run of `job_key` on the development target. +pub(crate) fn manual_run(location_id: Uuid, job_key: &str) -> crate::runs::models::JobRunRequest { + crate::runs::models::JobRunRequest { + run_id: Uuid::now_v7(), + location_id, + target: "development".into(), + job_key: job_key.into(), + reason: "Build it now".into(), + priority: 0, + window: None, + full_refresh: false, + select: Vec::new(), + exclude: Vec::new(), + } +} + +/// Write every published job's definition back with the concurrency source an older release recorded. +pub(crate) async fn store_old_job_definitions(store: &ControlStore, source: &str) { + use crate::runs::tests::helpers::{json_column, rows}; + use crate::store::main::{begin::begin, commit::commit, query::query}; + use crate::store::models::{Parameter, TransactionMode}; + + let jobs = rows( + store, + "SELECT job_key, definition_json FROM sqb_jobs", + vec![], + ) + .await; + let mut transaction = begin(store, TransactionMode::Write).await.expect("begin"); + for job in jobs { + let mut definition: serde_json::Value = json_column(&job["definition_json"]); + definition["resolved_concurrency"]["source"] = serde_json::json!(source); + (transaction, _) = query( + transaction, + "UPDATE sqb_jobs SET definition_json=$1 WHERE job_key=$2", + vec![ + Parameter::Json(definition), + Parameter::Text(job["job_key"].as_str().expect("job key").to_owned()), + ], + ) + .await + .expect("store the old definition"); + } + commit(transaction).await.expect("commit"); +} + +/// Rewrite every snapshot's stored `jobs` document as an older release published it, with each job's concurrency source. +pub(crate) async fn store_old_jobs_documents(store: &ControlStore, source: &str) { + use crate::runs::tests::helpers::rows; + use crate::store::main::{ + begin::begin, commit::commit, compress::compress, decompress::decompress, query::query, + }; + use crate::store::models::{Parameter, TransactionMode}; + + let documents = rows( + store, + "SELECT b.document_hash, b.body FROM sqb_snapshot_documents d \ + JOIN sqb_documents b ON b.document_hash=d.document_hash WHERE d.kind='jobs'", + vec![], + ) + .await; + let mut transaction = begin(store, TransactionMode::Write).await.expect("begin"); + for document in documents { + let mut body: serde_json::Value = + serde_json::from_slice(&decompress(&document["body"]).expect("body")).expect("JSON"); + for job in body.as_object_mut().expect("jobs").values_mut() { + job["resolved_concurrency"]["source"] = serde_json::json!(source); + } + (transaction, _) = query( + transaction, + "UPDATE sqb_documents SET body=$1 WHERE document_hash=$2", + vec![ + Parameter::Bytes( + compress(&serde_json::to_vec(&body).expect("JSON")).expect("gzip"), + ), + Parameter::Text(document["document_hash"].as_str().expect("hash").to_owned()), + ], + ) + .await + .expect("store the old document"); + } + commit(transaction).await.expect("commit"); +} diff --git a/crates/sqlbuild-orchestration/tests/scheduler/test_concurrency_source.rs b/crates/sqlbuild-orchestration/tests/scheduler/test_concurrency_source.rs index dab6199995..086ba57a6c 100644 --- a/crates/sqlbuild-orchestration/tests/scheduler/test_concurrency_source.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler/test_concurrency_source.rs @@ -3,135 +3,56 @@ use std::collections::BTreeMap; use chrono::Utc; -use serde_json::{Value, json}; +use serde_json::json; use uuid::Uuid; -use crate::credentials::models::AuditContext; use crate::credentials::tests::location_helpers::create_location; use crate::runs::main::{ execution::advance_groups::advance_groups, requests::request_job_run::request_job_run, }; -use crate::runs::models::JobRunRequest; use crate::runs::tests::helpers::{json_column, rows, snapshot}; -use crate::scheduler::main::_scheduler_cycle::scheduler_cycle; use crate::scheduler::models::SchedulerState; -use crate::scheduler::tests::contracts::helpers::acquire_lease; -use crate::scheduler::tests::contracts::test_types::StoredConcurrencySourceTestCase; +use crate::scheduler::tests::contracts::helpers::{ + acquire_lease, manual_run, operator, scheduler_cycle, store_old_job_definitions, + store_old_jobs_documents, +}; +use crate::scheduler::tests::contracts::test_types::{ + MigratedConcurrencySourceTestCase, PinnedOldDocumentTestCase, +}; use crate::scheduler::tests::contracts::ticks::helpers::snapshot as scheduled_snapshot; use crate::sensors::tests::helpers::append_events; use crate::snapshots::main::publication::publish_snapshot::publish_snapshot; use crate::store::errors::ControlError; -use crate::store::main::compress::compress; -use crate::store::main::decompress::decompress; -use crate::store::main::query::query; -use crate::store::main::{begin::begin, commit::commit, migrate_store::migrate_store}; -use crate::store::models::{ControlStore, Parameter, TransactionMode}; +use crate::store::main::migrate_store::migrate_store; use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; use crate::store::tests::store_test_types::Backend; use crate::store_migrations::_helpers::migration::MIGRATIONS; use crate::store_migrations::_helpers::migration_run::migrate_with; -/// The field 0004 removes, as releases before it stored it. -const OLD_SOURCE: &str = "project"; - -const BACKENDS: [StoredConcurrencySourceTestCase; 2] = [ - StoredConcurrencySourceTestCase { - description: "SQLite store", - backend: Backend::Sqlite, - }, - StoredConcurrencySourceTestCase { - description: "PostgreSQL store", - backend: Backend::Postgres, - }, -]; - -fn operator() -> AuditContext { - AuditContext { - actor: "test-operator".into(), - granting_role: "control-store-owner".into(), - request_id: Uuid::now_v7(), - ip_address: None, - } -} - -fn manual_run(location_id: Uuid, job_key: &str) -> JobRunRequest { - JobRunRequest { - run_id: Uuid::now_v7(), - location_id, - target: "development".into(), - job_key: job_key.into(), - reason: "Build it now".into(), - priority: 0, - window: None, - full_refresh: false, - select: Vec::new(), - exclude: Vec::new(), - } -} - -/// Write every published job's definition back with the source an older release recorded. -async fn store_old_job_definitions(store: &ControlStore) { - let jobs = rows( - store, - "SELECT job_key, definition_json FROM sqb_jobs", - vec![], - ) - .await; - let mut transaction = begin(store, TransactionMode::Write).await.expect("begin"); - for job in jobs { - let mut definition: Value = json_column(&job["definition_json"]); - definition["resolved_concurrency"]["source"] = json!(OLD_SOURCE); - (transaction, _) = query( - transaction, - "UPDATE sqb_jobs SET definition_json=$1 WHERE job_key=$2", - vec![ - Parameter::Json(definition), - Parameter::Text(job["job_key"].as_str().expect("job key").to_owned()), - ], - ) - .await - .expect("store the old definition"); - } - commit(transaction).await.expect("commit"); -} - -/// Rewrite the stored `jobs` document of every snapshot as an older release published it. -async fn store_old_jobs_documents(store: &ControlStore) { - let documents = rows( - store, - "SELECT b.document_hash, b.body FROM sqb_snapshot_documents d \ - JOIN sqb_documents b ON b.document_hash=d.document_hash WHERE d.kind='jobs'", - vec![], - ) - .await; - let mut transaction = begin(store, TransactionMode::Write).await.expect("begin"); - for document in documents { - let mut body: Value = - serde_json::from_slice(&decompress(&document["body"]).expect("body")).expect("JSON"); - for job in body.as_object_mut().expect("jobs").values_mut() { - job["resolved_concurrency"]["source"] = json!(OLD_SOURCE); - } - (transaction, _) = query( - transaction, - "UPDATE sqb_documents SET body=$1 WHERE document_hash=$2", - vec![ - Parameter::Bytes( - compress(&serde_json::to_vec(&body).expect("JSON")).expect("gzip"), - ), - Parameter::Text(document["document_hash"].as_str().expect("hash").to_owned()), - ], - ) - .await - .expect("store the old document"); - } - commit(transaction).await.expect("commit"); -} - #[test] fn given_a_job_stored_with_its_concurrency_source_when_migrating_then_schedules_sensors_and_manual_runs_read_it() { - let mut completed = 0; - for test_case in BACKENDS { + let test_cases = [ + MigratedConcurrencySourceTestCase { + description: "SQLite store", + backend: Backend::Sqlite, + stored_source: "project", + expected_refused_before: true, + expected_resolved_concurrency: json!({"limit": 4}), + expected_ticks: 1, + expected_runs: &["manual 1", "schedule 1", "sensor 1"], + }, + MigratedConcurrencySourceTestCase { + description: "PostgreSQL store", + backend: Backend::Postgres, + stored_source: "target", + expected_refused_before: true, + expected_resolved_concurrency: json!({"limit": 4}), + expected_ticks: 1, + expected_runs: &["manual 1", "schedule 1", "sensor 1"], + }, + ]; + for test_case in test_cases { run(async { let fixture = open_test_store(test_case.backend) .await @@ -155,11 +76,9 @@ fn given_a_job_stored_with_its_concurrency_source_when_migrating_then_schedules_ "minimum_interval_seconds": 1 }); publish_snapshot(store, &input).await.expect("publish"); - store_old_job_definitions(store).await; + store_old_job_definitions(store, test_case.stored_source).await; let before = - request_job_run(store, (&operator(), None), &manual_run(location, "orders")) - .await - .map(|_| ()); + request_job_run(store, (&operator(), None), &manual_run(location, "orders")).await; migrate_store(store).await.expect("migrate to version 4"); let stored = rows(store, "SELECT definition_json FROM sqb_jobs", vec![]).await; @@ -170,44 +89,45 @@ fn given_a_job_stored_with_its_concurrency_source_when_migrating_then_schedules_ request_job_run(store, (&operator(), None), &manual_run(location, "orders")) .await .expect("a manual run of the migrated job"); - let runs = rows( + let runs: Vec = rows( store, "SELECT origin, COUNT(*) AS runs FROM sqb_runs GROUP BY origin ORDER BY origin", vec![], ) - .await; + .await + .iter() + .map(|row| { + format!( + "{} {}", + row["origin"].as_str().unwrap_or_default(), + row["runs"] + ) + }) + .collect(); assert_eq!( ( matches!(&before, Err(ControlError::Invalid(message)) if message.contains("unknown field `source`")), json_column(&stored[0]["definition_json"])["resolved_concurrency"].clone(), cycle.ticks, - runs.iter() - .map(|row| format!( - "{} {}", - row["origin"].as_str().unwrap_or_default(), - row["runs"] - )) - .collect::>(), + runs, ), ( - true, - json!({"limit": 4}), - 1, - vec![ - "manual 1".to_owned(), - "schedule 1".to_owned(), - "sensor 1".to_owned() - ], + test_case.expected_refused_before, + test_case.expected_resolved_concurrency.clone(), + test_case.expected_ticks, + test_case + .expected_runs + .iter() + .map(|run| (*run).to_owned()) + .collect::>(), ), "{}: refused before 0004, then read by the scheduler, its sensor and a manual run", test_case.description ); close_test_store(fixture).await; - completed += 1; }); } - assert_eq!(completed, BACKENDS.len(), "every backend ran"); } // Runs pinned to a snapshot published before 0004 read its immutable documents, which 0004 can't @@ -217,8 +137,21 @@ fn given_a_job_stored_with_its_concurrency_source_when_migrating_then_schedules_ #[test] fn given_a_group_pinned_to_a_snapshot_document_in_the_old_form_when_advancing_it_then_it_fails_invalid() { - let mut completed = 0; - for test_case in BACKENDS { + let test_cases = [ + PinnedOldDocumentTestCase { + description: "SQLite store", + backend: Backend::Sqlite, + stored_source: "project", + expected_error: "jobs: unknown field `source`, expected `limit`", + }, + PinnedOldDocumentTestCase { + description: "PostgreSQL store", + backend: Backend::Postgres, + stored_source: "default", + expected_error: "jobs: unknown field `source`, expected `limit`", + }, + ]; + for test_case in test_cases { run(async { let fixture = open_test_store(test_case.backend) .await @@ -264,18 +197,20 @@ fn given_a_group_pinned_to_a_snapshot_document_in_the_old_form_when_advancing_it advance_groups(store, &lease, (&limits, scope)) .await .expect("start the group"); - store_old_jobs_documents(store).await; + store_old_jobs_documents(store, test_case.stored_source).await; let advanced = advance_groups(store, &lease, (&limits, scope)).await; - assert!( - matches!(&advanced, Err(ControlError::Invalid(message)) if message == "jobs: unknown field `source`, expected `limit`"), - "{}: {advanced:?}", + assert_eq!( + advanced.map_err(|error| match error { + ControlError::Invalid(message) => message, + other => format!("not Invalid: {other:?}"), + }), + Err(test_case.expected_error.to_owned()), + "{}: the group pinned to the old document fails Invalid", test_case.description ); close_test_store(fixture).await; - completed += 1; }); } - assert_eq!(completed, BACKENDS.len(), "every backend ran"); } diff --git a/crates/sqlbuild-orchestration/tests/scheduler/test_types.rs b/crates/sqlbuild-orchestration/tests/scheduler/test_types.rs index 97ead9e28b..35138c2262 100644 --- a/crates/sqlbuild-orchestration/tests/scheduler/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler/test_types.rs @@ -9,7 +9,24 @@ pub(crate) struct LeadershipTestCase { pub(crate) expected_epoch: i64, } -pub(crate) struct StoredConcurrencySourceTestCase { +pub(crate) struct MigratedConcurrencySourceTestCase { pub(crate) description: &'static str, pub(crate) backend: Backend, + /// The concurrency source an older release stored in each job's definition. + pub(crate) stored_source: &'static str, + /// Whether a manual run of the job is refused before migration 0004. + pub(crate) expected_refused_before: bool, + pub(crate) expected_resolved_concurrency: serde_json::Value, + pub(crate) expected_ticks: usize, + /// Runs queued after 0004, as `origin count`. + pub(crate) expected_runs: &'static [&'static str], +} + +pub(crate) struct PinnedOldDocumentTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + /// The concurrency source an older release stored in the snapshot's jobs document. + pub(crate) stored_source: &'static str, + /// The `ControlError::Invalid` message advancing the group returns. + pub(crate) expected_error: &'static str, } diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md index 255a91828e..ccae22cf69 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md @@ -314,6 +314,12 @@ afterwards, so back it up before you upgrade (see [Back up the control database](#back-up-the-control-database)). To roll back, restore that backup; runs recorded since the upgrade are lost. +Before you upgrade, let queued and running runs finish, or cancel them. A run stays pinned to the +snapshot it started on, and a snapshot an earlier release published can hold job definitions this +release no longer reads: a group run or a retry from it fails with an error such as +``jobs: unknown field `source`, expected `limit` ``, which stops the server's scheduler. After the server, upgrade and +restart every location, so each publishes its project again with the new release. + To see what an upgrade changes before you start the server, run the new release with `--dry-run`: ```bash diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md index 2de6859ef1..fb829aaf21 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md @@ -283,8 +283,7 @@ value set in this order: 5. `[settings] concurrency` in `sqlbuild_project.toml` (default `1`) The value must be an integer of at least `1`; anything else fails with `D001`. `sqb build ---verbose` shows the effective limit and the setting it came from, for example -`8 configured limit (targets.prod.concurrency)`. `sqb scenario test` has no `--target` option and +--verbose` shows the build's effective limit, for example `8 configured limit`. `sqb scenario test` has no `--target` option and uses the active target's value. ### Planning from synced state diff --git a/ui/e2e/concurrency.spec.ts b/ui/e2e/concurrency.spec.ts index 937dee9f10..c99948b9f4 100644 --- a/ui/e2e/concurrency.spec.ts +++ b/ui/e2e/concurrency.spec.ts @@ -1,4 +1,5 @@ import { expect, test, type Page } from '@playwright/test'; +import { FakeControl } from './support/fake-control'; // The Concurrency panel on job and group pages: plain facts about what runs alongside what. With // SHOTS set, each page is photographed: SHOTS=/some/dir npx playwright test e2e/concurrency.spec.ts @@ -17,10 +18,10 @@ async function panel(page: Page, job: string, group = false): Promise return (await lines.allInnerTexts()).map((t) => t.replace(/\s+/g, ' ').replace(/ Running now: .*$/, '').trim()); } -test('a job states its run concurrency and where it comes from, overlap, and its pool', async ({ page }) => { +test('a job states its run concurrency limit, overlap, and its pool', async ({ page }) => { const lines = await panel(page, 'core_build'); expect(lines).toEqual([ - 'Inside a run: 8 at once (job setting)', + "Inside a run: at most 8 of this job's nodes at once", 'Never alongside runs writing the same tables (automatic)', 'If triggered while running: is skipped', 'warehouse_heavy: takes 1 of 1 (capped on this server; the project allows 3), shared with reporting_exports, close_of_business' @@ -29,12 +30,12 @@ test('a job states its run concurrency and where it comes from, overlap, and its await expect(page.locator('ul.concurrency a', { hasText: 'reporting_exports' })).toHaveAttribute('href', /\/jobs\/reporting_exports/); }); -test('each layer a run concurrency can come from is named', async ({ page }) => { +test('a job without its own setting states the limit its runs get, without naming where it is set', async ({ page }) => { for (const [job, line] of [ - ['marts_only', 'Inside a run: 6 at once (target prod)'], - ['activity_microbatch', 'Inside a run: 2 at once (local setting)'], - ['revenue_refresh', 'Inside a run: 4 at once (project setting)'], - ['load_sources', 'Inside a run: 4 at once (SQLBuild default)'] + ['marts_only', "Inside a run: at most 6 of this job's nodes at once"], + ['activity_microbatch', "Inside a run: at most 2 of this job's nodes at once"], + ['revenue_refresh', "Inside a run: at most 4 of this job's nodes at once"], + ['load_sources', "Inside a run: at most 4 of this job's nodes at once"] ] as const) { expect((await panel(page, job))[0], job).toBe(line); } @@ -54,7 +55,9 @@ test('a group capped below what it asks takes the whole pool, shown as a warning }); test('a group in no pool says so', async ({ page }) => { - expect(await panel(page, 'nightly_all', true)).toContain('Pools: none'); + const lines = await panel(page, 'nightly_all', true); + expect(lines).toContain('Pools: none'); + expect(lines[0]).toBe('Members without their own limit: at most 4 nodes at once'); }); test("the location shows the server's pool-cap warning", async ({ page }) => { @@ -65,3 +68,26 @@ test("the location shows the server's pool-cap warning", async ({ page }) => { await expect(caps.getByRole('link', { name: 'close_of_business' })).toHaveAttribute('href', /\/jobs\/close_of_business/); if (shots) await page.screenshot({ path: `${shots}/concurrency-panel-location.png` }); }); + +test("a published job's limit comes through the server's job read as a plain limit", async ({ page }) => { + const server = new FakeControl({ + shop: { + jobs: [ + { key: 'orders_job', kind: 'job', concurrency: 2, limit: 2 }, + { key: 'refunds_job', kind: 'job', concurrency: null, limit: 16 }, + { key: 'nightly', kind: 'group', concurrency: null, limit: 16 } + ] + } + }); + await server.install(page); + for (const [job, line, group] of [ + ['orders_job', "Inside a run: at most 2 of this job's nodes at once", false], + ['refunds_job', "Inside a run: at most 16 of this job's nodes at once", false], + ['nightly', 'Members without their own limit: at most 16 nodes at once', true] + ] as const) { + await page.goto(`/jobs/${job}`); + if (group) await page.getByRole('button', { name: 'Toggle details' }).click({ timeout: 20_000 }); + const first = page.locator('ul.concurrency > li').first(); + await expect(first, job).toHaveText(line, { timeout: 20_000 }); + } +}); diff --git a/ui/e2e/support/fake-control.ts b/ui/e2e/support/fake-control.ts index 358becdbe9..9ab02dbb25 100644 --- a/ui/e2e/support/fake-control.ts +++ b/ui/e2e/support/fake-control.ts @@ -32,8 +32,13 @@ export type TargetState = { failWithdrawals: boolean; /** Delays the sync status, so it arrives after the page's first read has begun. */ healthDelayMs: number; + /** Published jobs and groups, with the concurrency limit their runs get. */ + jobs: FakeJob[]; }; +/** A published job or group: its key, kind, own `concurrency` setting and the limit its runs get; a group's members are the target's jobs. */ +export type FakeJob = { key: string; kind: 'job' | 'group'; concurrency: number | null; limit: number }; + const MODELS = ['orders', 'order_items', 'customers', 'payments', 'refunds']; const json = (route: Route, body: unknown, status = 200) => route.fulfill({ status, contentType: 'application/json', body: JSON.stringify(body) }); @@ -49,7 +54,7 @@ export class FakeControl { #lease = new Date(Date.now() - 60_000).toISOString(); constructor(overrides: Partial>> = {}) { - const base = (): TargetState => ({ changed: [], sync: 'settled', schedulerLeader: true, acks: [], failedRun: false, readDelayMs: 0, failReads: false, failWithdrawals: false, healthDelayMs: 0 }); + const base = (): TargetState => ({ changed: [], sync: 'settled', schedulerLeader: true, acks: [], failedRun: false, readDelayMs: 0, failReads: false, failWithdrawals: false, healthDelayMs: 0, jobs: [] }); this.targets = { shop: { ...base(), ...overrides.shop }, returns: { ...base(), ...overrides.returns } }; } @@ -75,6 +80,12 @@ export class FakeControl { await page.route('**/api/**', (route) => this.#answer(route)); } + /** A job as the server lists it: every model, and the concurrency limit its runs get. */ + #job(job: FakeJob) { + const selection = MODELS.map((m) => `model:${m}`); + return { job_key: job.key, kind: job.kind, description: `Builds ${job.key}.`, selection, pools: [], concurrency: job.concurrency, resolved_concurrency: { limit: job.limit }, overlap: 'queue', snapshot_id: SNAPSHOT }; + } + async #answer(route: Route) { const request = route.request(); const url = new URL(request.url()); @@ -120,7 +131,12 @@ export class FakeControl { return json(route, { snapshot_id: SNAPSHOT, version: this.#states[name].version, states }); } if (rest.startsWith('snapshots')) return json(route, [{ snapshot_id: SNAPSHOT, generation: 1, git_revision: null, git_dirty: false, git_author: null, environment_hash: 'e'.repeat(64), sqb_version: '0.0.0', published_at: '2026-11-01T00:00:00Z', changes: [], connection: null }]); - if (rest === 'jobs' || rest === 'triggers' || rest === 'checks' || rest === 'acknowledgements') return json(route, []); + if (rest === 'jobs') return json(route, target.jobs.map((job) => this.#job(job))); + const job = rest.startsWith('jobs/') ? target.jobs.find((j) => `jobs/${j.key}` === rest) : undefined; + // A group's members are the target's jobs, all starting together. + const members = job?.kind === 'group' ? target.jobs.filter((j) => j.kind === 'job').map((j) => ({ job_key: j.key, waits_for: [], reasons: [] })) : []; + if (job) return json(route, { ...this.#job(job), critical_nodes: [], member_of: [], members, settings: job.concurrency === null ? {} : { concurrency: job.concurrency }, targets: [] }); + if (rest === 'triggers' || rest === 'checks' || rest === 'acknowledgements') return json(route, []); if (rest.startsWith('cost')) return json(route, { since: '2026-11-01T00:00:00Z', credits: 0, usd: null, runs: 0, incomplete_runs: 0, by_job: [], by_warehouse_size: [], by_node: [] }); // The Overview's own reads: answered as things stood when the request arrived, then slowed or // refused as the test asks. diff --git a/ui/src/lib/api/http-jobs.test.ts b/ui/src/lib/api/http-jobs.test.ts index acbe4d9211..1cb3bb4d61 100644 --- a/ui/src/lib/api/http-jobs.test.ts +++ b/ui/src/lib/api/http-jobs.test.ts @@ -115,7 +115,7 @@ describe('job mapping', () => { description: 'Build orders', selection: ['model:customers', 'model:orders'], pools: [{ pool: 'warehouse_heavy', amount: 3, limit: 4, cap: 2, server_limit: 2, clamped: true, shared_with: ['nightly'] }], - resolved_concurrency: { limit: 4, source: 'job' }, + resolved_concurrency: { limit: 4 }, concurrency: 4, snapshot_id: SNAPSHOT, select: '+orders', @@ -130,7 +130,7 @@ describe('job mapping', () => { expect(toJob({ ...wire, owner: ' ', overlap: 'queue', on_failure: null }, triggers, [])).toMatchObject({ owner: null, overlap: 'queue', onFailure: null }); expect(job.lastRun?.id).toBe('r-ord'); expect(job.pools).toEqual([{ pool: 'warehouse_heavy', amount: 3, limit: 4, cap: 2, serverLimit: 2, clamped: true, sharedWith: ['nightly'] }]); - expect(job.resolvedConcurrency).toEqual({ limit: 4, source: 'job' }); + expect(job.resolvedConcurrency).toEqual({ limit: 4 }); expect(toJob({ ...wire, resolved_concurrency: null }, triggers, []).resolvedConcurrency).toBeNull(); expect(toJob({ ...wire, concurrency: null }, triggers, []).concurrency).toBeNull(); const detail = jobDetailOf(job, { ...wire, settings: { vars: '[REDACTED]', concurrency: 4, empty_ok: true, models: ['a', 'b'] }, members: [], member_of: ['nightly'], critical_nodes: ['model:orders'], exclude: ' stg_orders audit_orders', targets: ['production'], cursor_type: 'integer' }); diff --git a/ui/src/lib/api/mock/operate.ts b/ui/src/lib/api/mock/operate.ts index d788bd5f14..f1e07c9f71 100644 --- a/ui/src/lib/api/mock/operate.ts +++ b/ui/src/lib/api/mock/operate.ts @@ -53,11 +53,10 @@ export const POOL_CAPS: Record = { warehouse_heavy: 1 }; /** A pool's limit on this server: the project's, lowered to the cap. */ export const serverLimitOf = (pool: string) => Math.min(POOL_LIMITS[pool] ?? 1, POOL_CAPS[pool] ?? Infinity); -/** Where each job's run concurrency comes from, when not its own setting: one of every layer. */ +/** The run concurrency of jobs without their own setting, where it differs from the mock project's 4. */ const CONCURRENCY_FROM: Record = { - marts_only: { limit: 6, source: 'target' }, - activity_microbatch: { limit: 2, source: 'local' }, - load_sources: { limit: 4, source: 'default' } + marts_only: { limit: 6 }, + activity_microbatch: { limit: 2 } }; const decl = (sourceFile: string, line: number, code: string) => ({ sourceFile, line, code }); @@ -410,10 +409,10 @@ export function poolHoldersOf(job: JobDef | undefined): PoolHolders[] { return Object.keys(job?.pools ?? {}).map((pool) => ({ pool, limit: serverLimitOf(pool), holders: [] })); } -/** What a run of the job uses: its own setting, else the layer the mock gives it, else the mock project's 4. */ +/** What a run of the job uses: its own setting, else the limit the mock gives it, else the mock project's 4. */ export function resolvedConcurrencyOf(job: JobDef | undefined): ResolvedConcurrency { const own = runConcurrencyOf(job); - return own ? { limit: own, source: 'job' } : (CONCURRENCY_FROM[job?.key ?? ''] ?? { limit: 4, source: 'project' }); + return own ? { limit: own } : (CONCURRENCY_FROM[job?.key ?? ''] ?? { limit: 4 }); } /** The jobs and groups asking a pool for more than this server allows, as the server's pool_capped warning names them. */ diff --git a/ui/src/lib/api/types/operate.ts b/ui/src/lib/api/types/operate.ts index e836626ba2..5a9e8e6f77 100644 --- a/ui/src/lib/api/types/operate.ts +++ b/ui/src/lib/api/types/operate.ts @@ -929,8 +929,8 @@ export type PoolUse = { sharedWith: string[]; }; -/** Models a run builds at once, and the layer that set it. */ -export type ResolvedConcurrency = { limit: number; source: 'job' | 'target' | 'local' | 'project' | 'default' }; +/** The most of a job's nodes a run of it builds at once. */ +export type ResolvedConcurrency = { limit: number }; /** One reason a member waits: `fromNode` of `upstreamJob` feeds its `toNode`, directly or through nodes no member selects. */ export type WaitReason = { upstreamJob: string; fromNode: string; toNode: string }; diff --git a/ui/src/lib/operate/labels.test.ts b/ui/src/lib/operate/labels.test.ts index d0d2b2247b..184e9b6276 100644 --- a/ui/src/lib/operate/labels.test.ts +++ b/ui/src/lib/operate/labels.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest'; -import { concurrencySource, describeWait, onFailureText, overlapPlain, overlapWhileRunning, poolText, triggerKindLabel, waitTechnical } from './labels'; +import { describeWait, onFailureText, overlapPlain, overlapWhileRunning, poolText, runConcurrencyText, triggerKindLabel, waitTechnical } from './labels'; import type { QueueWait } from '$lib/api'; const text = (w: QueueWait) => describeWait(w).map((p) => p.text).join(''); @@ -49,13 +49,10 @@ describe('concurrency in plain words', () => { expect(`${clamped.text} (${clamped.detail})`).toBe('Takes the whole warehouse_heavy pool on this server (capped at 1; the project asks for 3)'); }); - it("names the layer that set a run's concurrency", () => { - expect(concurrencySource('job', 'dev')).toBe('job setting'); - expect(concurrencySource('target', 'dev')).toBe('target dev'); - expect(concurrencySource('target', null)).toBe('target setting'); - expect(concurrencySource('local', 'dev')).toBe('local setting'); - expect(concurrencySource('project', 'dev')).toBe('project setting'); - expect(concurrencySource('default', 'dev')).toBe('SQLBuild default'); + it("states a run's concurrency as the limit it applies, for a job and for a group's members", () => { + expect(runConcurrencyText(4, 'job')).toBe("Inside a run: at most 4 of this job's nodes at once"); + expect(runConcurrencyText(1, 'job')).toBe("Inside a run: at most 1 of this job's nodes at once"); + expect(runConcurrencyText(6, 'group')).toBe('Members without their own limit: at most 6 nodes at once'); }); it('says what a job triggered while it runs does', () => { diff --git a/ui/src/lib/operate/labels.ts b/ui/src/lib/operate/labels.ts index 03869bb5e4..f6194b009c 100644 --- a/ui/src/lib/operate/labels.ts +++ b/ui/src/lib/operate/labels.ts @@ -7,7 +7,6 @@ import type { NodeCounts, OnFailure, OverlapPolicy, - ResolvedConcurrency, PoolUse, QueueWait, RunCost, @@ -112,20 +111,14 @@ export const SHARED_SHOWN = 3; /** The docs page the Concurrency panel links to. */ export const CONCURRENCY_DOCS = 'https://sqlbuild.com/docs/concepts/concurrency/'; -/** Which layer set a run's concurrency, as the panel names it; a target by its name when known. */ -export function concurrencySource(source: ResolvedConcurrency['source'], target: string | null): string { - switch (source) { - case 'job': - return 'job setting'; - case 'target': - return target ? `target ${target}` : 'target setting'; - case 'local': - return 'local setting'; - case 'project': - return 'project setting'; - case 'default': - return 'SQLBuild default'; - } +/** + * A run's concurrency as the panel states it. A group's limit is the target's, which each member + * job's run uses unless the job sets its own. + */ +export function runConcurrencyText(limit: number, kind: 'job' | 'group'): string { + return kind === 'group' + ? `Members without their own limit: at most ${limit} nodes at once` + : `Inside a run: at most ${limit} of this job's nodes at once`; } /** What a job does when triggered while a run of it is active, finishing "If triggered while running: …". */ diff --git a/ui/src/routes/jobs/[name]/job-view.svelte b/ui/src/routes/jobs/[name]/job-view.svelte index 1a3a9ee017..cc5a3a6d47 100644 --- a/ui/src/routes/jobs/[name]/job-view.svelte +++ b/ui/src/routes/jobs/[name]/job-view.svelte @@ -23,7 +23,7 @@ import { formatNodeCounts, isActiveRun, nodeOutcomeLabel, nodeOutcomeTone, runOriginLabel, runStatusLabel, runStatusTone } from '$lib/status'; import { kindLine, shortName } from '$lib/build-area/labels'; import { planHref } from '$lib/run/command'; - import { CONCURRENCY_DOCS, concurrencySource, formatCost, narrowingText, onFailureText, overlapHelp, overlapWhileRunning, poolText, shortRunId } from '$lib/operate/labels'; + import { CONCURRENCY_DOCS, formatCost, narrowingText, onFailureText, overlapHelp, overlapWhileRunning, poolText, runConcurrencyText, shortRunId } from '$lib/operate/labels'; import ScopeGraph from '$lib/lineage/scope-graph.svelte'; import MemberGraph from '$lib/lineage/member-graph.svelte'; import { memberLinks, publishedLinks, waitReasons, type MemberInfo, type MemberSelection } from '$lib/lineage/member-graph'; @@ -313,7 +313,7 @@ {@const resolved = def.resolvedConcurrency}
    {#if resolved} -
  • Inside a run: {resolved.limit} at once ({concurrencySource(resolved.source, app.target?.name ?? null)})
  • +
  • {runConcurrencyText(resolved.limit, def.kind)}
  • {/if}
  • Never alongside runs writing the same tables (automatic)
  • {#if def.overlap} diff --git a/website/src/content/docs/docs/cli/server.mdx b/website/src/content/docs/docs/cli/server.mdx index 2cea042e05..43ccaa5c67 100644 --- a/website/src/content/docs/docs/cli/server.mdx +++ b/website/src/content/docs/docs/cli/server.mdx @@ -293,6 +293,12 @@ afterwards, so back it up before you upgrade (see [Back up the control database](#back-up-the-control-database)). To roll back, restore that backup; runs recorded since the upgrade are lost. +Before you upgrade, let queued and running runs finish, or cancel them. A run stays pinned to the +snapshot it started on, and a snapshot an earlier release published can hold job definitions this +release no longer reads: a group run or a retry from it fails with an error such as +``jobs: unknown field `source`, expected `limit` ``, which stops the server's scheduler. After the server, upgrade and +restart every location, so each publishes its project again with the new release. + To see what an upgrade changes before you start the server, run the new release with `--dry-run`: ```bash diff --git a/website/src/content/docs/docs/concepts/project-configuration.mdx b/website/src/content/docs/docs/concepts/project-configuration.mdx index b6e4e12f66..20eba60287 100644 --- a/website/src/content/docs/docs/concepts/project-configuration.mdx +++ b/website/src/content/docs/docs/concepts/project-configuration.mdx @@ -256,8 +256,7 @@ value set in this order: 5. `[settings] concurrency` in `sqlbuild_project.toml` (default `1`) The value must be an integer of at least `1`; anything else fails with `D001`. `sqb build ---verbose` shows the effective limit and the setting it came from, for example -`8 configured limit (targets.prod.concurrency)`. `sqb scenario test` has no `--target` option and +--verbose` shows the build's effective limit, for example `8 configured limit`. `sqb scenario test` has no `--target` option and uses the active target's value. ### Planning from synced state From 55315758c1580e6b61ca344d5af94772481b65c2 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 20 Nov 2026 04:20:58 +0000 Subject: [PATCH 3/4] docs(server)!: state what fails until a location publishes again after upgrading Correction to 135016a3d9: migration 0004 does not make every read of jobs published before the upgrade work. It removes resolved_concurrency.source from sqb_jobs, so job runs from schedules, sensors and manual requests work after it. The snapshot documents those jobs were published in keep the field, and 0004 can't rewrite them: until the location publishes again, its job pages, group runs and ad hoc runs fail with `jobs: unknown field 'source'`, and a group's schedule stops the scheduler. Runs pinned to a snapshot published before the upgrade fail the same way when they read it. The server upgrade docs now say to pause schedules and sensors, or keep the scheduler off, and to drain or cancel runs before upgrading, then to upgrade and restart every location after upgrading the server. The migration test stores the old form in sqb_jobs and the snapshot documents, and checks both what works after 0004 alone and that the rest works once the location publishes again. A new test shows a group's schedule failing the scheduler cycle until then. Comments no longer say the published concurrency names the layer that set it. BREAKING CHANGE: a published job's `resolved_concurrency` is now `{limit}`; its `source` field and the `ConcurrencySource` schema are removed from the control API, and `sqb build --verbose` no longer names the setting the concurrency came from. Before upgrading, pause schedules and sensors (or set `scheduler = false` under `[server]`) and let queued and running runs finish, or cancel them. Control store migration 0004 removes the field from each stored job definition. After upgrading the server, upgrade and restart every location: until a location has published again, its job pages, group runs and ad hoc runs fail with `jobs: unknown field 'source'`, and a group's schedule stops the scheduler. --- .../src/declarations/models.rs | 6 +- .../tests/scheduler/helpers.rs | 46 ++++- .../scheduler/test_concurrency_source.rs | 179 +++++++++++++++--- .../tests/scheduler/test_types.rs | 15 ++ .../main/resolve_concurrency.rs | 4 +- .../sqlbuild/references/docs/cli/server.md | 16 +- .../docs/concepts/project-configuration.md | 4 +- website/src/content/docs/docs/cli/server.mdx | 16 +- .../docs/concepts/project-configuration.mdx | 4 +- 9 files changed, 236 insertions(+), 54 deletions(-) diff --git a/crates/sqlbuild-orchestration/src/declarations/models.rs b/crates/sqlbuild-orchestration/src/declarations/models.rs index 91ad53a7fd..c57d6a13d2 100644 --- a/crates/sqlbuild-orchestration/src/declarations/models.rs +++ b/crates/sqlbuild-orchestration/src/declarations/models.rs @@ -38,7 +38,7 @@ pub struct JobDefinition { /// Each pool the job takes, with its amount and the pool's limit on the target. #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] pub pools: BTreeMap, - /// Models a run builds at once on the target, and the layer that set it; absent from older snapshots. + /// The most of the job's nodes a run builds at once on the target; absent from older snapshots. #[serde(default, skip_serializing_if = "Option::is_none")] pub resolved_concurrency: Option, /// The selectors as written, for display; `selection` is what runs. @@ -307,7 +307,7 @@ pub struct JobRecord { /// The job's own `ignore_audit_failures` setting; absent uses the target's, then the project's. #[serde(default, skip_serializing_if = "Option::is_none")] pub ignore_audit_failures: Option, - /// The concurrency a run uses on the snapshot's target and where it came from; absent for older snapshots. + /// The most of the job's nodes a run builds at once on the snapshot's target; absent for older snapshots. pub resolved_concurrency: Option, pub snapshot_id: uuid::Uuid, /// A job's selectors as written; absent for a group and for snapshots published before they were recorded. @@ -342,7 +342,7 @@ pub struct JobDetailRecord { /// The job's own `ignore_audit_failures` setting; absent uses the target's, then the project's. #[serde(default, skip_serializing_if = "Option::is_none")] pub ignore_audit_failures: Option, - /// The concurrency a run uses on the snapshot's target and where it came from; absent for older snapshots. + /// The most of the job's nodes a run builds at once on the snapshot's target; absent for older snapshots. pub resolved_concurrency: Option, /// Command settings; `vars` and a `defer_*` value that is not a target name show as `[REDACTED]`, and text values pass through secret redaction. pub settings: BTreeMap, diff --git a/crates/sqlbuild-orchestration/tests/scheduler/helpers.rs b/crates/sqlbuild-orchestration/tests/scheduler/helpers.rs index 2d6671b35e..468aa61bd2 100644 --- a/crates/sqlbuild-orchestration/tests/scheduler/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler/helpers.rs @@ -156,8 +156,8 @@ pub(crate) async fn store_old_job_definitions(store: &ControlStore, source: &str commit(transaction).await.expect("commit"); } -/// Rewrite every snapshot's stored `jobs` document as an older release published it, with each job's concurrency source. -pub(crate) async fn store_old_jobs_documents(store: &ControlStore, source: &str) { +/// Point each snapshot's `jobs` and `groups` documents at old-form bodies under hashes of their own, as an older release stored them. +pub(crate) async fn store_old_documents(store: &ControlStore, source: &str) { use crate::runs::tests::helpers::rows; use crate::store::main::{ begin::begin, commit::commit, compress::compress, decompress::decompress, query::query, @@ -167,7 +167,7 @@ pub(crate) async fn store_old_jobs_documents(store: &ControlStore, source: &str) let documents = rows( store, "SELECT b.document_hash, b.body FROM sqb_snapshot_documents d \ - JOIN sqb_documents b ON b.document_hash=d.document_hash WHERE d.kind='jobs'", + JOIN sqb_documents b ON b.document_hash=d.document_hash WHERE d.kind IN ('jobs', 'groups')", vec![], ) .await; @@ -175,21 +175,55 @@ pub(crate) async fn store_old_jobs_documents(store: &ControlStore, source: &str) for document in documents { let mut body: serde_json::Value = serde_json::from_slice(&decompress(&document["body"]).expect("body")).expect("JSON"); - for job in body.as_object_mut().expect("jobs").values_mut() { + for job in body.as_object_mut().expect("jobs or groups").values_mut() { job["resolved_concurrency"]["source"] = serde_json::json!(source); } + let hash = document["document_hash"].as_str().expect("hash").to_owned(); + let old_hash = format!("{hash}-published-before-0004"); (transaction, _) = query( transaction, - "UPDATE sqb_documents SET body=$1 WHERE document_hash=$2", + "INSERT INTO sqb_documents (document_hash, body) VALUES ($1, $2)", vec![ + Parameter::Text(old_hash.clone()), Parameter::Bytes( compress(&serde_json::to_vec(&body).expect("JSON")).expect("gzip"), ), - Parameter::Text(document["document_hash"].as_str().expect("hash").to_owned()), ], ) .await .expect("store the old document"); + (transaction, _) = query( + transaction, + "UPDATE sqb_snapshot_documents SET document_hash=$1 WHERE document_hash=$2", + vec![Parameter::Text(old_hash), Parameter::Text(hash)], + ) + .await + .expect("point the snapshots at it"); } commit(transaction).await.expect("commit"); } + +/// A control-store result as a test compares it: `ok`, or the error as `Invalid("...")` and the like. +pub(crate) fn outcome(result: Result) -> String { + result.map_or_else(|error| format!("{error:?}"), |_| "ok".to_owned()) +} + +/// The scheduled orders snapshot with job `orders`, a group `nightly_group` of it and `triggers`, as a release before 0004 published it once its source is stored. +pub(crate) fn upgrade_snapshot( + location_id: Uuid, + triggers: serde_json::Value, +) -> crate::snapshots::models::SnapshotInput { + let mut input = crate::scheduler::tests::contracts::ticks::helpers::snapshot(location_id); + input.documents.insert( + "jobs".into(), + serde_json::json!({"orders": {"description": "Build orders", "selection": ["orders"], + "resolved_concurrency": {"limit": 4}}}), + ); + input.documents.insert( + "groups".into(), + serde_json::json!({"nightly_group": {"description": "Everything nightly", + "selection": ["orders"], "jobs": {"orders": []}, "resolved_concurrency": {"limit": 4}}}), + ); + input.documents.insert("triggers".into(), triggers); + input +} diff --git a/crates/sqlbuild-orchestration/tests/scheduler/test_concurrency_source.rs b/crates/sqlbuild-orchestration/tests/scheduler/test_concurrency_source.rs index 086ba57a6c..df3c5aae5c 100644 --- a/crates/sqlbuild-orchestration/tests/scheduler/test_concurrency_source.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler/test_concurrency_source.rs @@ -1,4 +1,4 @@ -//! Migration 0004 drops the removed `resolved_concurrency.source` from stored job definitions, and a pinned snapshot document still in the old form fails. +//! Migration 0004 drops the removed `resolved_concurrency.source` from stored job definitions; until a location publishes again, reads of its snapshot documents still fail. use std::collections::BTreeMap; @@ -6,31 +6,39 @@ use chrono::Utc; use serde_json::json; use uuid::Uuid; +use crate::credentials::models::DefaultRole; use crate::credentials::tests::location_helpers::create_location; +use crate::credentials::tests::role_helpers::{credentials, role_fixture}; +use crate::declarations::main::read_person_job::read_person_job; +use crate::declarations::models::ServerPoolCaps; use crate::runs::main::{ execution::advance_groups::advance_groups, requests::request_job_run::request_job_run, }; use crate::runs::tests::helpers::{json_column, rows, snapshot}; use crate::scheduler::models::SchedulerState; use crate::scheduler::tests::contracts::helpers::{ - acquire_lease, manual_run, operator, scheduler_cycle, store_old_job_definitions, - store_old_jobs_documents, + acquire_lease, manual_run, operator, outcome, scheduler_cycle, store_old_documents, + store_old_job_definitions, upgrade_snapshot, }; use crate::scheduler::tests::contracts::test_types::{ - MigratedConcurrencySourceTestCase, PinnedOldDocumentTestCase, + GroupScheduleOldDocumentTestCase, MigratedConcurrencySourceTestCase, PinnedOldDocumentTestCase, }; -use crate::scheduler::tests::contracts::ticks::helpers::snapshot as scheduled_snapshot; +use crate::scheduler::tests::contracts::ticks::helpers::calendar_text; use crate::sensors::tests::helpers::append_events; use crate::snapshots::main::publication::publish_snapshot::publish_snapshot; use crate::store::errors::ControlError; use crate::store::main::migrate_store::migrate_store; +use crate::store::tests::lease_helpers::expire_lease; use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; use crate::store::tests::store_test_types::Backend; use crate::store_migrations::_helpers::migration::MIGRATIONS; use crate::store_migrations::_helpers::migration_run::migrate_with; +/// How reading a document an older release published, with `source`, fails. +const OLD_FORM: &str = "Invalid(\"jobs: unknown field `source`, expected `limit`\")"; + #[test] -fn given_a_job_stored_with_its_concurrency_source_when_migrating_then_schedules_sensors_and_manual_runs_read_it() +fn given_jobs_published_before_0004_when_migrating_then_plain_job_runs_work_and_snapshot_reads_wait_for_a_republish() { let test_cases = [ MigratedConcurrencySourceTestCase { @@ -41,6 +49,8 @@ fn given_a_job_stored_with_its_concurrency_source_when_migrating_then_schedules_ expected_resolved_concurrency: json!({"limit": 4}), expected_ticks: 1, expected_runs: &["manual 1", "schedule 1", "sensor 1"], + expected_until_republished: [OLD_FORM, OLD_FORM], + expected_after_republish: ["ok", "ok"], }, MigratedConcurrencySourceTestCase { description: "PostgreSQL store", @@ -50,6 +60,8 @@ fn given_a_job_stored_with_its_concurrency_source_when_migrating_then_schedules_ expected_resolved_concurrency: json!({"limit": 4}), expected_ticks: 1, expected_runs: &["manual 1", "schedule 1", "sensor 1"], + expected_until_republished: [OLD_FORM, OLD_FORM], + expected_after_republish: ["ok", "ok"], }, ]; for test_case in test_cases { @@ -61,27 +73,57 @@ fn given_a_job_stored_with_its_concurrency_source_when_migrating_then_schedules_ migrate_with(store, &MIGRATIONS[..2]) .await .expect("a store at version 3"); - let location = create_location(store, "orders", "test-admin") - .await - .expect("location"); - let mut input = scheduled_snapshot(location); - input.documents.insert( - "jobs".into(), - json!({"orders": {"description": "Build orders", "selection": ["orders"], - "resolved_concurrency": {"limit": 4}}}), + let roles = role_fixture(store).await; + let location = roles.location_id; + let mut input = upgrade_snapshot( + location, + json!({ + "daily_orders": {"description": "Refresh orders", "job_key": "orders", + "starts_at": calendar_text(0), + "schedule": {"kind": "calendar", "times": [calendar_text(0), calendar_text(1), + calendar_text(2), "2030-01-01T00:00:00Z"], "refresh_at": "2029-01-01T00:00:00Z"}}, + "health_orders": {"description": "Refresh orders on overdue ticks", "job_key": "orders", + "when": {"kind": "platform_event", "kinds": ["tick_overdue"]}, + "minimum_interval_seconds": 1} + }), ); - input.documents.get_mut("triggers").expect("triggers")["health_orders"] = json!({ - "description": "Refresh orders on overdue ticks", "job_key": "orders", - "when": {"kind": "platform_event", "kinds": ["tick_overdue"]}, - "minimum_interval_seconds": 1 - }); + input.generation = 2; publish_snapshot(store, &input).await.expect("publish"); store_old_job_definitions(store, test_case.stored_source).await; + store_old_documents(store, test_case.stored_source).await; let before = request_job_run(store, (&operator(), None), &manual_run(location, "orders")).await; + let caps = ServerPoolCaps::default(); + let administrator = (&roles.context, credentials(&roles.administrator)); + let page_and_group = || async { + [ + outcome( + read_person_job( + store, + administrator, + (location, "development", "orders"), + (DefaultRole::None, &caps), + ) + .await, + ), + outcome( + request_job_run( + store, + (&operator(), None), + &manual_run(location, "nightly_group"), + ) + .await, + ), + ] + }; migrate_store(store).await.expect("migrate to version 4"); - let stored = rows(store, "SELECT definition_json FROM sqb_jobs", vec![]).await; + let stored = rows( + store, + "SELECT definition_json FROM sqb_jobs WHERE job_key='orders'", + vec![], + ) + .await; append_events(store, 0..1, Utc::now()).await; let (_, cycle) = scheduler_cycle(store, SchedulerState::new(Uuid::now_v7())) .await @@ -104,6 +146,12 @@ fn given_a_job_stored_with_its_concurrency_source_when_migrating_then_schedules_ ) }) .collect(); + let until_republished = page_and_group().await; + input.generation = 3; + publish_snapshot(store, &input) + .await + .expect("the location publishes again"); + let after_republish = page_and_group().await; assert_eq!( ( @@ -111,6 +159,8 @@ fn given_a_job_stored_with_its_concurrency_source_when_migrating_then_schedules_ json_column(&stored[0]["definition_json"])["resolved_concurrency"].clone(), cycle.ticks, runs, + until_republished, + after_republish, ), ( test_case.expected_refused_before, @@ -121,8 +171,82 @@ fn given_a_job_stored_with_its_concurrency_source_when_migrating_then_schedules_ .iter() .map(|run| (*run).to_owned()) .collect::>(), + test_case.expected_until_republished.map(str::to_owned), + test_case.expected_after_republish.map(str::to_owned), ), - "{}: refused before 0004, then read by the scheduler, its sensor and a manual run", + "{}: refused before 0004; plain job runs after it; the job page and a group run once the location publishes again", + test_case.description + ); + close_test_store(fixture).await; + }); + } +} + +// Until its location publishes again, a schedule of a group reads the current snapshot's groups +// document, which 0004 can't rewrite, and its `ControlError::Invalid` fails the whole scheduler +// cycle. Making this a per-run failure is tracked separately, and that change replaces this +// expectation. +#[test] +fn given_a_group_schedule_published_before_0004_when_the_scheduler_cycles_then_it_fails_until_the_location_publishes_again() + { + let test_cases = [ + GroupScheduleOldDocumentTestCase { + description: "SQLite store", + backend: Backend::Sqlite, + stored_source: "project", + expected_until_republished: OLD_FORM, + expected_ticks_after_republish: 1, + }, + GroupScheduleOldDocumentTestCase { + description: "PostgreSQL store", + backend: Backend::Postgres, + stored_source: "local", + expected_until_republished: OLD_FORM, + expected_ticks_after_republish: 1, + }, + ]; + for test_case in test_cases { + run(async { + let fixture = open_test_store(test_case.backend) + .await + .expect("required backend"); + let store = &fixture.store; + migrate_with(store, &MIGRATIONS[..2]) + .await + .expect("a store at version 3"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let mut input = upgrade_snapshot( + location, + json!({"nightly_orders": {"description": "Run the group", "job_key": "nightly_group", + "starts_at": calendar_text(0), + "schedule": {"kind": "calendar", "times": [calendar_text(0), calendar_text(1), + calendar_text(2), "2030-01-01T00:00:00Z"], "refresh_at": "2029-01-01T00:00:00Z"}}}), + ); + publish_snapshot(store, &input).await.expect("publish"); + store_old_job_definitions(store, test_case.stored_source).await; + store_old_documents(store, test_case.stored_source).await; + migrate_store(store).await.expect("migrate to version 4"); + + let stopped = scheduler_cycle(store, SchedulerState::new(Uuid::now_v7())).await; + input.generation = 2; + publish_snapshot(store, &input) + .await + .expect("the location publishes again"); + // The stopped scheduler's lease runs out before a restarted one takes over. + expire_lease(store).await; + let (_, cycle) = scheduler_cycle(store, SchedulerState::new(Uuid::now_v7())) + .await + .expect("a scheduler cycle once the location has published again"); + + assert_eq!( + (outcome(stopped), cycle.ticks), + ( + test_case.expected_until_republished.to_owned(), + test_case.expected_ticks_after_republish + ), + "{}: the cycle fails Invalid until the location publishes again, then ticks", test_case.description ); close_test_store(fixture).await; @@ -142,13 +266,13 @@ fn given_a_group_pinned_to_a_snapshot_document_in_the_old_form_when_advancing_it description: "SQLite store", backend: Backend::Sqlite, stored_source: "project", - expected_error: "jobs: unknown field `source`, expected `limit`", + expected_error: OLD_FORM, }, PinnedOldDocumentTestCase { description: "PostgreSQL store", backend: Backend::Postgres, stored_source: "default", - expected_error: "jobs: unknown field `source`, expected `limit`", + expected_error: OLD_FORM, }, ]; for test_case in test_cases { @@ -197,16 +321,13 @@ fn given_a_group_pinned_to_a_snapshot_document_in_the_old_form_when_advancing_it advance_groups(store, &lease, (&limits, scope)) .await .expect("start the group"); - store_old_jobs_documents(store, test_case.stored_source).await; + store_old_documents(store, test_case.stored_source).await; let advanced = advance_groups(store, &lease, (&limits, scope)).await; assert_eq!( - advanced.map_err(|error| match error { - ControlError::Invalid(message) => message, - other => format!("not Invalid: {other:?}"), - }), - Err(test_case.expected_error.to_owned()), + outcome(advanced), + test_case.expected_error, "{}: the group pinned to the old document fails Invalid", test_case.description ); diff --git a/crates/sqlbuild-orchestration/tests/scheduler/test_types.rs b/crates/sqlbuild-orchestration/tests/scheduler/test_types.rs index 35138c2262..8e3bb74770 100644 --- a/crates/sqlbuild-orchestration/tests/scheduler/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler/test_types.rs @@ -20,6 +20,21 @@ pub(crate) struct MigratedConcurrencySourceTestCase { pub(crate) expected_ticks: usize, /// Runs queued after 0004, as `origin count`. pub(crate) expected_runs: &'static [&'static str], + /// A job page read and a group run request after 0004, before the location publishes again. + pub(crate) expected_until_republished: [&'static str; 2], + /// The same once the location has published again. + pub(crate) expected_after_republish: [&'static str; 2], +} + +pub(crate) struct GroupScheduleOldDocumentTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + /// The concurrency source an older release stored with each job. + pub(crate) stored_source: &'static str, + /// The scheduler cycle due to queue the group after 0004, before the location publishes again. + pub(crate) expected_until_republished: &'static str, + /// Ticks the cycle takes once the location has published again. + pub(crate) expected_ticks_after_republish: usize, } pub(crate) struct PinnedOldDocumentTestCase { diff --git a/crates/sqlbuild-python-front/src/command_runtime/main/resolve_concurrency.rs b/crates/sqlbuild-python-front/src/command_runtime/main/resolve_concurrency.rs index da83e1347d..09c12c7e81 100644 --- a/crates/sqlbuild-python-front/src/command_runtime/main/resolve_concurrency.rs +++ b/crates/sqlbuild-python-front/src/command_runtime/main/resolve_concurrency.rs @@ -1,11 +1,11 @@ -//! The effective concurrency of a command on `selected_target`, and the layer that set it. +//! The effective concurrency of a command on `selected_target`. use crate::command_runtime::main::resolve_configured_concurrency::resolve_configured_concurrency; use pyo3::prelude::{Bound, PyAny, PyResult}; use sqlbuild_config::effective::models::EffectiveConcurrency; use sqlbuild_python_base::records::main::fields::field::field as native_field; -/// The effective concurrency of a command on `selected_target`, and the layer that set it. +/// The effective concurrency of a command on `selected_target`. pub fn resolve_concurrency( requested: &Bound<'_, PyAny>, discovered: &Bound<'_, PyAny>, diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md index ccae22cf69..0f121a5bb6 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md @@ -314,11 +314,17 @@ afterwards, so back it up before you upgrade (see [Back up the control database](#back-up-the-control-database)). To roll back, restore that backup; runs recorded since the upgrade are lost. -Before you upgrade, let queued and running runs finish, or cancel them. A run stays pinned to the -snapshot it started on, and a snapshot an earlier release published can hold job definitions this -release no longer reads: a group run or a retry from it fails with an error such as -``jobs: unknown field `source`, expected `limit` ``, which stops the server's scheduler. After the server, upgrade and -restart every location, so each publishes its project again with the new release. +Before you upgrade, pause your schedules and sensors, or keep the scheduler off (set +`scheduler = false` under `[server]` and stop any `sqb scheduler`), then let queued and running +runs finish, or cancel them. A run stays pinned to the snapshot it started on, and a snapshot an +earlier release published can hold job definitions this release no longer reads: a group run or a +retry from it fails with `` jobs: unknown field `source`, expected `limit` ``, which stops the +scheduler. + +After upgrading the server, upgrade and restart every location, so each publishes its project again +with the new release. Until a location has published again, its job pages, group runs and ad hoc +runs fail with the same error, and a group's schedule stops the scheduler; its job runs from +schedules, sensors and manual requests work. To see what an upgrade changes before you start the server, run the new release with `--dry-run`: diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md index fb829aaf21..a15b81ec2b 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md @@ -283,8 +283,8 @@ value set in this order: 5. `[settings] concurrency` in `sqlbuild_project.toml` (default `1`) The value must be an integer of at least `1`; anything else fails with `D001`. `sqb build ---verbose` shows the build's effective limit, for example `8 configured limit`. `sqb scenario test` has no `--target` option and -uses the active target's value. +--verbose` shows the build's effective limit, for example `8 configured limit`. `sqb scenario +test` has no `--target` option and uses the active target's value. ### Planning from synced state diff --git a/website/src/content/docs/docs/cli/server.mdx b/website/src/content/docs/docs/cli/server.mdx index 43ccaa5c67..f2bdd5f30c 100644 --- a/website/src/content/docs/docs/cli/server.mdx +++ b/website/src/content/docs/docs/cli/server.mdx @@ -293,11 +293,17 @@ afterwards, so back it up before you upgrade (see [Back up the control database](#back-up-the-control-database)). To roll back, restore that backup; runs recorded since the upgrade are lost. -Before you upgrade, let queued and running runs finish, or cancel them. A run stays pinned to the -snapshot it started on, and a snapshot an earlier release published can hold job definitions this -release no longer reads: a group run or a retry from it fails with an error such as -``jobs: unknown field `source`, expected `limit` ``, which stops the server's scheduler. After the server, upgrade and -restart every location, so each publishes its project again with the new release. +Before you upgrade, pause your schedules and sensors, or keep the scheduler off (set +`scheduler = false` under `[server]` and stop any `sqb scheduler`), then let queued and running +runs finish, or cancel them. A run stays pinned to the snapshot it started on, and a snapshot an +earlier release published can hold job definitions this release no longer reads: a group run or a +retry from it fails with `` jobs: unknown field `source`, expected `limit` ``, which stops the +scheduler. + +After upgrading the server, upgrade and restart every location, so each publishes its project again +with the new release. Until a location has published again, its job pages, group runs and ad hoc +runs fail with the same error, and a group's schedule stops the scheduler; its job runs from +schedules, sensors and manual requests work. To see what an upgrade changes before you start the server, run the new release with `--dry-run`: diff --git a/website/src/content/docs/docs/concepts/project-configuration.mdx b/website/src/content/docs/docs/concepts/project-configuration.mdx index 20eba60287..03f4dcd7bd 100644 --- a/website/src/content/docs/docs/concepts/project-configuration.mdx +++ b/website/src/content/docs/docs/concepts/project-configuration.mdx @@ -256,8 +256,8 @@ value set in this order: 5. `[settings] concurrency` in `sqlbuild_project.toml` (default `1`) The value must be an integer of at least `1`; anything else fails with `D001`. `sqb build ---verbose` shows the build's effective limit, for example `8 configured limit`. `sqb scenario test` has no `--target` option and -uses the active target's value. +--verbose` shows the build's effective limit, for example `8 configured limit`. `sqb scenario +test` has no `--target` option and uses the active target's value. ### Planning from synced state From d34f0ab3f8377809428bbb369cb135fe99025652 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 20 Nov 2026 04:45:28 +0000 Subject: [PATCH 4/4] docs(server)!: upgrade in order, keeping the scheduler running to drain runs The server upgrade note no longer offers `scheduler = false`: the scheduler loop launches queued runs and advances group runs, so with it off nothing drains. It now gives the order to upgrade in and when to resume schedules and sensors, and says that until a location has published again a group's schedule or sensor stops the scheduler and a job run that fails can't record its failure. A test shows that failure: the worker's report that a job run failed is refused, because deciding on a retry reads the snapshot's documents before it checks for a retry policy, and the run stays running. BREAKING CHANGE: a published job's `resolved_concurrency` is now `{limit}`; its `source` field and the `ConcurrencySource` schema are removed from the control API, and `sqb build --verbose` no longer names the setting the concurrency came from. Control store migration 0004 removes the field from each stored job definition, but not from the snapshot documents jobs were published in. Upgrade in this order: 1. Pause every schedule and sensor, and start no manual runs. Keep the scheduler running. 2. Let queued and running runs finish, or cancel them. 3. Upgrade the server. 4. Upgrade and restart every location. 5. Once every location has published again, resume schedules and sensors. Until a location has published again, its job pages, group runs and ad hoc runs fail, a group's schedule or sensor stops the scheduler, and a job run that fails can't record its failure. --- .../tests/scheduler/helpers.rs | 53 +++++++++++++ .../scheduler/test_concurrency_source.rs | 79 ++++++++++++++++++- .../tests/scheduler/test_types.rs | 11 +++ .../sqlbuild/references/docs/cli/server.md | 24 +++--- ui/src/routes/jobs/[name]/job-view.svelte | 2 +- website/src/content/docs/docs/cli/server.mdx | 24 +++--- 6 files changed, 167 insertions(+), 26 deletions(-) diff --git a/crates/sqlbuild-orchestration/tests/scheduler/helpers.rs b/crates/sqlbuild-orchestration/tests/scheduler/helpers.rs index 468aa61bd2..8f86616f18 100644 --- a/crates/sqlbuild-orchestration/tests/scheduler/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler/helpers.rs @@ -227,3 +227,56 @@ pub(crate) fn upgrade_snapshot( input.documents.insert("triggers".into(), triggers); input } + +/// Report a claimed run as its worker does: started, each node with `node`, then finished with `run`; each report's outcome. +pub(crate) async fn report_run( + store: &ControlStore, + claim: &crate::runs::models::RunClaim, + (node, run): ( + crate::runs::models::NodeOutcome, + crate::runs::models::RunOutcome, + ), +) -> Vec { + use crate::runs::main::execution::change_run::change_run; + use crate::runs::models::{RunAuthority, RunChange, RunChangeRequest}; + + let mut changes = vec![RunChange::Started]; + changes.extend( + claim + .request + .selection + .iter() + .map(|node_key| RunChange::NodeFinished { + node_key: node_key.clone(), + outcome: node, + rows_affected: Some(1), + error_code: None, + skipped_because_of: None, + error_message: None, + }), + ); + changes.push(RunChange::Finished { + outcome: run, + reason: "the worker finished".into(), + exit_code: Some(1), + }); + let mut outcomes = Vec::new(); + for change in changes { + let reported = change_run( + store, + RunAuthority::Worker { + run_id: claim.request.run_id, + claim_id: claim.claim_id, + }, + &RunChangeRequest { + event_id: Uuid::now_v7(), + run_id: claim.request.run_id, + claim_id: Some(claim.claim_id), + change, + }, + ) + .await; + outcomes.push(outcome(reported)); + } + outcomes +} diff --git a/crates/sqlbuild-orchestration/tests/scheduler/test_concurrency_source.rs b/crates/sqlbuild-orchestration/tests/scheduler/test_concurrency_source.rs index df3c5aae5c..629a8e84fd 100644 --- a/crates/sqlbuild-orchestration/tests/scheduler/test_concurrency_source.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler/test_concurrency_source.rs @@ -12,16 +12,19 @@ use crate::credentials::tests::role_helpers::{credentials, role_fixture}; use crate::declarations::main::read_person_job::read_person_job; use crate::declarations::models::ServerPoolCaps; use crate::runs::main::{ - execution::advance_groups::advance_groups, requests::request_job_run::request_job_run, + execution::advance_groups::advance_groups, execution::claim_next_run::claim_next_run, + requests::request_job_run::request_job_run, }; +use crate::runs::models::{NodeOutcome, RunOutcome}; use crate::runs::tests::helpers::{json_column, rows, snapshot}; use crate::scheduler::models::SchedulerState; use crate::scheduler::tests::contracts::helpers::{ - acquire_lease, manual_run, operator, outcome, scheduler_cycle, store_old_documents, + acquire_lease, manual_run, operator, outcome, report_run, scheduler_cycle, store_old_documents, store_old_job_definitions, upgrade_snapshot, }; use crate::scheduler::tests::contracts::test_types::{ - GroupScheduleOldDocumentTestCase, MigratedConcurrencySourceTestCase, PinnedOldDocumentTestCase, + FailedJobRunOldDocumentTestCase, GroupScheduleOldDocumentTestCase, + MigratedConcurrencySourceTestCase, PinnedOldDocumentTestCase, }; use crate::scheduler::tests::contracts::ticks::helpers::calendar_text; use crate::sensors::tests::helpers::append_events; @@ -254,6 +257,76 @@ fn given_a_group_schedule_published_before_0004_when_the_scheduler_cycles_then_i } } +// A job run that fails reads its snapshot's documents to decide on a retry before it checks for a +// retry policy, so until the location publishes again its failure report is refused and the run +// stays running until it is found unresponsive. Tracked separately with the other reads, and that +// change replaces this expectation. +#[test] +fn given_a_job_published_before_0004_when_its_run_fails_before_a_republish_then_the_failure_is_refused() + { + let test_cases = [FailedJobRunOldDocumentTestCase { + description: "SQLite store", + backend: Backend::Sqlite, + stored_source: "project", + expected_reports: ["ok", "ok", OLD_FORM], + expected_status: "running", + }]; + for test_case in test_cases { + run(async { + let fixture = open_test_store(test_case.backend) + .await + .expect("required backend"); + let store = &fixture.store; + migrate_with(store, &MIGRATIONS[..2]) + .await + .expect("a store at version 3"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + publish_snapshot(store, &upgrade_snapshot(location, json!({}))) + .await + .expect("publish"); + store_old_job_definitions(store, test_case.stored_source).await; + store_old_documents(store, test_case.stored_source).await; + migrate_store(store).await.expect("migrate to version 4"); + request_job_run(store, (&operator(), None), &manual_run(location, "orders")) + .await + .expect("a manual run of the migrated job"); + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + let claim = claim_next_run( + store, + &lease, + &BTreeMap::new(), + Some((location, "development")), + ) + .await + .expect("claim") + .expect("the queued run"); + + let reports = + report_run(store, &claim, (NodeOutcome::Failed, RunOutcome::Failed)).await; + let status = rows(store, "SELECT status FROM sqb_runs", vec![]).await; + + assert_eq!( + ( + reports, + status[0]["status"].as_str().unwrap_or_default().to_owned() + ), + ( + test_case.expected_reports.map(str::to_owned).to_vec(), + test_case.expected_status.to_owned() + ), + "{}: the failure report is refused and the run stays running", + test_case.description + ); + close_test_store(fixture).await; + }); + } +} + // Runs pinned to a snapshot published before 0004 read its immutable documents, which 0004 can't // rewrite. They still fail with `ControlError::Invalid`, which stops the scheduler; operators let // such runs finish or cancel them before upgrading. Turning this into a per-run failure is tracked diff --git a/crates/sqlbuild-orchestration/tests/scheduler/test_types.rs b/crates/sqlbuild-orchestration/tests/scheduler/test_types.rs index 8e3bb74770..2734da4600 100644 --- a/crates/sqlbuild-orchestration/tests/scheduler/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/scheduler/test_types.rs @@ -45,3 +45,14 @@ pub(crate) struct PinnedOldDocumentTestCase { /// The `ControlError::Invalid` message advancing the group returns. pub(crate) expected_error: &'static str, } + +pub(crate) struct FailedJobRunOldDocumentTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + /// The concurrency source an older release stored with each job. + pub(crate) stored_source: &'static str, + /// The outcome of each worker report: started, the node finished, the run failed. + pub(crate) expected_reports: [&'static str; 3], + /// The run's status afterwards. + pub(crate) expected_status: &'static str, +} diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md index 0f121a5bb6..de4a6f5324 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md @@ -314,17 +314,19 @@ afterwards, so back it up before you upgrade (see [Back up the control database](#back-up-the-control-database)). To roll back, restore that backup; runs recorded since the upgrade are lost. -Before you upgrade, pause your schedules and sensors, or keep the scheduler off (set -`scheduler = false` under `[server]` and stop any `sqb scheduler`), then let queued and running -runs finish, or cancel them. A run stays pinned to the snapshot it started on, and a snapshot an -earlier release published can hold job definitions this release no longer reads: a group run or a -retry from it fails with `` jobs: unknown field `source`, expected `limit` ``, which stops the -scheduler. - -After upgrading the server, upgrade and restart every location, so each publishes its project again -with the new release. Until a location has published again, its job pages, group runs and ad hoc -runs fail with the same error, and a group's schedule stops the scheduler; its job runs from -schedules, sensors and manual requests work. +A run stays pinned to the snapshot it started on, and a snapshot an earlier release published can +hold job definitions this release no longer reads, which fail with +`` jobs: unknown field `source`, expected `limit` ``. Upgrade in this order: + +1. Pause every schedule and sensor, and start no manual runs. Keep the scheduler running: it + launches queued runs and advances group runs. +2. Let queued and running runs finish, or cancel them. +3. Upgrade the server. +4. Upgrade and restart every location, so each publishes its project again with the new release. +5. Once every location has published again, resume schedules and sensors. + +Until a location has published again, its job pages, group runs and ad hoc runs fail, a group's +schedule or sensor stops the scheduler, and a job run that fails can't record its failure. To see what an upgrade changes before you start the server, run the new release with `--dry-run`: diff --git a/ui/src/routes/jobs/[name]/job-view.svelte b/ui/src/routes/jobs/[name]/job-view.svelte index cc5a3a6d47..ee5e75ce76 100644 --- a/ui/src/routes/jobs/[name]/job-view.svelte +++ b/ui/src/routes/jobs/[name]/job-view.svelte @@ -531,7 +531,7 @@ .picked li { padding: 2px 0; } - /* The Concurrency panel: one plain fact per line, where it came from quieter beside it. */ + /* The Concurrency panel: one plain fact per line, any detail quieter beside it. */ .concurrency { display: flex; flex-direction: column; diff --git a/website/src/content/docs/docs/cli/server.mdx b/website/src/content/docs/docs/cli/server.mdx index f2bdd5f30c..e134269e7c 100644 --- a/website/src/content/docs/docs/cli/server.mdx +++ b/website/src/content/docs/docs/cli/server.mdx @@ -293,17 +293,19 @@ afterwards, so back it up before you upgrade (see [Back up the control database](#back-up-the-control-database)). To roll back, restore that backup; runs recorded since the upgrade are lost. -Before you upgrade, pause your schedules and sensors, or keep the scheduler off (set -`scheduler = false` under `[server]` and stop any `sqb scheduler`), then let queued and running -runs finish, or cancel them. A run stays pinned to the snapshot it started on, and a snapshot an -earlier release published can hold job definitions this release no longer reads: a group run or a -retry from it fails with `` jobs: unknown field `source`, expected `limit` ``, which stops the -scheduler. - -After upgrading the server, upgrade and restart every location, so each publishes its project again -with the new release. Until a location has published again, its job pages, group runs and ad hoc -runs fail with the same error, and a group's schedule stops the scheduler; its job runs from -schedules, sensors and manual requests work. +A run stays pinned to the snapshot it started on, and a snapshot an earlier release published can +hold job definitions this release no longer reads, which fail with +`` jobs: unknown field `source`, expected `limit` ``. Upgrade in this order: + +1. Pause every schedule and sensor, and start no manual runs. Keep the scheduler running: it + launches queued runs and advances group runs. +2. Let queued and running runs finish, or cancel them. +3. Upgrade the server. +4. Upgrade and restart every location, so each publishes its project again with the new release. +5. Once every location has published again, resume schedules and sensors. + +Until a location has published again, its job pages, group runs and ad hoc runs fail, a group's +schedule or sensor stops the scheduler, and a job run that fails can't record its failure. To see what an upgrade changes before you start the server, run the new release with `--dry-run`: