Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 1 addition & 7 deletions clients/typescript/src/schema.d.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1890,11 +1890,6 @@ export interface components {
/** Format: uuid */
snapshot_id?: string | null;
};
/**
* @description The layer that set a run's concurrency.
* @enum {string}
*/
ConcurrencySource: "job" | "target" | "local" | "project" | "default";
/** @description A row whose key was already stored with different content; the first row is kept. */
ConflictEvidence: {
existing_sha256: string;
Expand Down Expand Up @@ -3183,11 +3178,10 @@ export interface components {
/** Format: uuid */
source_run_id: string;
};
/** @description The models a run builds at once on its target, and the layer that set it. */
/** @description The most of a job's nodes a run of it builds at once on its target. */
ResolvedConcurrency: {
/** Format: int64 */
limit: number;
source: components["schemas"]["ConcurrencySource"];
};
RoleAssignmentRequest: {
/** Format: uuid */
Expand Down
4 changes: 2 additions & 2 deletions crates/sqlbuild-cli/tests/native_commands/test_control.rs
Original file line number Diff line number Diff line change
Expand Up @@ -379,13 +379,13 @@ fn given_a_control_store_when_running_server_migrate_then_dry_run_lists_and_migr
StoreMigrationCommandTestCase {
description: "migrate an empty store",
arguments: &["server", "migrate"],
expected_stdout: "The control store is empty: created schema version 3.\n",
expected_stdout: "The control store is empty: created schema version 4.\n",
expected_store: true,
},
StoreMigrationCommandTestCase {
description: "dry run on a current store",
arguments: &["server", "migrate", "--dry-run"],
expected_stdout: "The control store is at schema version 3: nothing to migrate.\n",
expected_stdout: "The control store is at schema version 4: nothing to migrate.\n",
expected_store: true,
},
];
Expand Down
28 changes: 7 additions & 21 deletions crates/sqlbuild-config/src/effective/main/effective_concurrency.rs
Original file line number Diff line number Diff line change
@@ -1,28 +1,14 @@
//! `effective_concurrency`: the worker limit of one command and the layer it came from.
//! `effective_concurrency`: the worker limit of one command.

use crate::effective::models::{ConcurrencyLayers, ConcurrencySource, EffectiveConcurrency};
use crate::effective::models::{ConcurrencyLayers, EffectiveConcurrency};

/// `--concurrency`, else the target's `concurrency`, else the local-then-project setting.
#[must_use]
pub fn effective_concurrency(layers: &ConcurrencyLayers<'_>) -> EffectiveConcurrency {
if let Some(limit) = layers.requested {
return EffectiveConcurrency {
limit,
source: ConcurrencySource::CommandLine,
};
}
if let Some((target, limit)) = layers.target {
return EffectiveConcurrency {
limit,
source: ConcurrencySource::Target(target.to_owned()),
};
}
pub fn effective_concurrency(layers: &ConcurrencyLayers) -> EffectiveConcurrency {
EffectiveConcurrency {
limit: layers.settings,
source: if layers.local_override {
ConcurrencySource::LocalSettings
} else {
ConcurrencySource::ProjectSettings
},
limit: layers
.requested
.or(layers.target)
.unwrap_or(layers.settings),
}
}
34 changes: 4 additions & 30 deletions crates/sqlbuild-config/src/effective/models.rs
Original file line number Diff line number Diff line change
Expand Up @@ -83,15 +83,13 @@ pub struct TargetChoice<K> {

/// The concurrency each configuration layer sets, highest precedence first.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct ConcurrencyLayers<'a> {
pub struct ConcurrencyLayers {
/// `--concurrency` on the command line.
pub requested: Option<i64>,
/// The selected target's name and its `concurrency`, when it sets one.
pub target: Option<(&'a str, i64)>,
/// The selected target's `concurrency`, when it sets one.
pub target: Option<i64>,
/// `[settings] concurrency` after any `sqlbuild_local.toml` override.
pub settings: i64,
/// Whether `sqlbuild_local.toml` `[settings]` overrides `concurrency`.
pub local_override: bool,
}

/// Whether audit failures are ignored at each configuration layer, highest precedence first.
Expand All @@ -105,32 +103,8 @@ pub struct AuditFailureLayers {
pub settings: bool,
}

/// Where the effective concurrency came from.
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum ConcurrencySource {
CommandLine,
Target(String),
LocalSettings,
ProjectSettings,
/// The named jobs' `concurrency` limits added together, when larger than the configured limit.
NamedJobs,
}

impl std::fmt::Display for ConcurrencySource {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Self::CommandLine => formatter.write_str("--concurrency"),
Self::Target(name) => write!(formatter, "targets.{name}.concurrency"),
Self::LocalSettings => formatter.write_str("sqlbuild_local.toml settings.concurrency"),
Self::ProjectSettings => formatter.write_str("settings.concurrency"),
Self::NamedJobs => formatter.write_str("named jobs' concurrency"),
}
}
}

/// One command's worker limit and the layer that set it.
/// One command's worker limit.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct EffectiveConcurrency {
pub limit: i64,
pub source: ConcurrencySource,
}
27 changes: 5 additions & 22 deletions crates/sqlbuild-config/src/effective/tests/test_effective.rs
Original file line number Diff line number Diff line change
Expand Up @@ -305,38 +305,23 @@ fn given_concurrency_layers_when_resolving_then_cli_beats_target_beats_settings(
ConcurrencyTestCase {
description: "the command line beats the target",
requested: Some(2),
target: Some(("prod", 8)),
target: Some(8),
settings: 16,
local_override: true,
expected_limit: 2,
expected_source: "--concurrency",
},
ConcurrencyTestCase {
description: "the target beats a local settings override",
description: "the target beats the settings",
requested: None,
target: Some(("prod", 8)),
target: Some(8),
settings: 4,
local_override: true,
expected_limit: 8,
expected_source: "targets.prod.concurrency",
},
ConcurrencyTestCase {
description: "a local settings override applies without a target value",
requested: None,
target: None,
settings: 4,
local_override: true,
expected_limit: 4,
expected_source: "sqlbuild_local.toml settings.concurrency",
},
ConcurrencyTestCase {
description: "the project setting is the default",
description: "the settings apply without a target value",
requested: None,
target: None,
settings: 16,
local_override: false,
expected_limit: 16,
expected_source: "settings.concurrency",
},
];

Expand All @@ -345,11 +330,9 @@ fn given_concurrency_layers_when_resolving_then_cli_beats_target_beats_settings(
requested: test_case.requested,
target: test_case.target,
settings: test_case.settings,
local_override: test_case.local_override,
});
assert_eq!(
(resolved.limit, resolved.source.to_string().as_str()),
(test_case.expected_limit, test_case.expected_source),
resolved.limit, test_case.expected_limit,
"{}",
test_case.description
);
Expand Down
4 changes: 1 addition & 3 deletions crates/sqlbuild-config/src/effective/tests/test_types.rs
Original file line number Diff line number Diff line change
Expand Up @@ -52,11 +52,9 @@ pub(super) struct MessageTestCase {
pub(super) struct ConcurrencyTestCase {
pub(super) description: &'static str,
pub(super) requested: Option<i64>,
pub(super) target: Option<(&'static str, i64)>,
pub(super) target: Option<i64>,
pub(super) settings: i64,
pub(super) local_override: bool,
pub(super) expected_limit: i64,
pub(super) expected_source: &'static str,
}

pub(super) struct AuditFailureTestCase {
Expand Down
9 changes: 0 additions & 9 deletions crates/sqlbuild-config/src/project/_helpers/documents.rs
Original file line number Diff line number Diff line change
Expand Up @@ -518,15 +518,6 @@ pub(crate) fn pool_limits_config(paths: &ConfigPaths) -> Result<PoolLimits, Load
Ok(pool_limits(table.as_map().unwrap_or_default())?)
}

/// The `[settings]` keys `sqlbuild_project.toml` sets, under their current names.
pub(crate) fn project_setting_keys(paths: &ConfigPaths) -> Result<Vec<String>, LoadFailure> {
let file = resolve(paths, false)?;
let table = load_toml_mapping(paths, file)?;
let settings =
get(table.as_map().unwrap_or_default(), "settings").and_then(ConfigValue::as_map);
Ok(settings.map(setting_overrides).unwrap_or_default())
}

/// `[git] base_ref` of `sqlbuild_project.toml`.
pub(crate) fn git_base_ref(paths: &ConfigPaths) -> Result<Option<String>, LoadFailure> {
project_section(paths, "git", load_git)
Expand Down

This file was deleted.

1 change: 0 additions & 1 deletion crates/sqlbuild-config/src/project/main/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@ pub mod load_git_base_ref;
pub mod load_local_config;
pub mod load_pool_limits;
pub mod load_project_config;
pub mod load_project_setting_keys;
pub mod load_review_config;
pub mod local_sql_analysis_failure;
pub mod microbatch_guidance;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -48,15 +48,7 @@ pub fn render_run_context(facts: &RunContextFacts, style: CliStyle) -> String {
("warehouse", warehouse),
(
"concurrency",
if facts.concurrency_source.is_empty() {
format!("{} configured limit", facts.concurrency)
} else {
format!(
"{} configured limit ({})",
facts.concurrency,
sanitize(&facts.concurrency_source)
)
},
format!("{} configured limit", facts.concurrency),
),
("full_refresh", full_refresh(facts)),
(
Expand Down
2 changes: 0 additions & 2 deletions crates/sqlbuild-core/src/build_command/models.rs
Original file line number Diff line number Diff line change
Expand Up @@ -179,8 +179,6 @@ pub struct RunContextFacts {
/// The warehouse and where it came from.
pub warehouse: Option<(String, String)>,
pub concurrency: i64,
/// The configuration layer the concurrency came from.
pub concurrency_source: String,
pub full_refresh: bool,
/// The named jobs whose own models the build rebuilds from scratch.
pub full_refresh_jobs: Vec<String>,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -121,13 +121,12 @@ fn given_run_context_when_rendering_then_sanitizes_and_groups_counts() {
command: "sqb build".to_owned(),
run_id: Some("run\u{1b}1".to_owned()),
concurrency: 4,
concurrency_source: "targets.prod.concurrency".to_owned(),
selected_count: 1200,
total_count: 1_234_567,
date_vars: (Some("2026-01-01".to_owned()), Some("2026-01-31".to_owned())),
..RunContextFacts::default()
},
expected_text: "Execution\n command sqb build\n run_id run\\x1b1\n target not set\n database not set\n schema not set\n warehouse not set\n concurrency 4 configured limit (targets.prod.concurrency)\n full_refresh false\n selected 1,200 of 1,234,567 build resources\n date vars 2026-01-01 to 2026-01-31\n\n",
expected_text: "Execution\n command sqb build\n run_id run\\x1b1\n target not set\n database not set\n schema not set\n warehouse not set\n concurrency 4 configured limit\n full_refresh false\n selected 1,200 of 1,234,567 build resources\n date vars 2026-01-01 to 2026-01-31\n\n",
},
ContextTestCase {
description: "a warehouse, one date and selector files",
Expand Down
25 changes: 4 additions & 21 deletions crates/sqlbuild-orchestration/src/declarations/models.rs
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ pub struct JobDefinition {
/// Each pool the job takes, with its amount and the pool's limit on the target.
#[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
pub pools: BTreeMap<String, PoolSlots>,
/// Models a run builds at once on the target, and the layer that set it; absent from older snapshots.
/// The most of the job's nodes a run builds at once on the target; absent from older snapshots.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub resolved_concurrency: Option<ResolvedConcurrency>,
/// The selectors as written, for display; `selection` is what runs.
Expand Down Expand Up @@ -181,28 +181,11 @@ pub struct WaitReason {
pub to_node: String,
}

/// The layer that set a run's concurrency.
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)]
#[serde(rename_all = "snake_case")]
pub enum ConcurrencySource {
/// The job's own `[settings] concurrency`.
Job,
/// The target's `concurrency` in `[targets.<name>]`.
Target,
/// `[settings] concurrency` in the location's `sqlbuild_local.toml`.
Local,
/// `[settings] concurrency` in `sqlbuild_project.toml`.
Project,
/// SQLBuild's built-in default.
Default,
}

/// The models a run builds at once on its target, and the layer that set it.
/// The most of a job's nodes a run of it builds at once on its target.
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)]
#[serde(deny_unknown_fields)]
pub struct ResolvedConcurrency {
pub limit: u64,
pub source: ConcurrencySource,
}

/// How many slots of a pool a job takes, and the pool's limit on the job's target.
Expand Down Expand Up @@ -324,7 +307,7 @@ pub struct JobRecord {
/// The job's own `ignore_audit_failures` setting; absent uses the target's, then the project's.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub ignore_audit_failures: Option<bool>,
/// The concurrency a run uses on the snapshot's target and where it came from; absent for older snapshots.
/// The most of the job's nodes a run builds at once on the snapshot's target; absent for older snapshots.
pub resolved_concurrency: Option<ResolvedConcurrency>,
pub snapshot_id: uuid::Uuid,
/// A job's selectors as written; absent for a group and for snapshots published before they were recorded.
Expand Down Expand Up @@ -359,7 +342,7 @@ pub struct JobDetailRecord {
/// The job's own `ignore_audit_failures` setting; absent uses the target's, then the project's.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub ignore_audit_failures: Option<bool>,
/// The concurrency a run uses on the snapshot's target and where it came from; absent for older snapshots.
/// The most of the job's nodes a run builds at once on the snapshot's target; absent for older snapshots.
pub resolved_concurrency: Option<ResolvedConcurrency>,
/// Command settings; `vars` and a `defer_*` value that is not a target name show as `[REDACTED]`, and text values pass through secret redaction.
pub settings: BTreeMap<String, serde_json::Value>,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,11 @@ pub(crate) const MIGRATIONS: &[StoreMigration] = &[
sqlite: include_str!("../sql/0003_attention_acknowledgement_index.sqlite.sql"),
postgres: include_str!("../sql/0003_attention_acknowledgement_index.postgres.sql"),
},
StoreMigration {
name: "drop_concurrency_source",
sqlite: include_str!("../sql/0004_drop_concurrency_source.sqlite.sql"),
postgres: include_str!("../sql/0004_drop_concurrency_source.postgres.sql"),
},
];

/// The schema version a store created by this build has.
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
-- Published jobs no longer record which layer set their run concurrency: drop the removed
-- resolved_concurrency.source from each job's stored definition, keeping its limit.
UPDATE sqb_jobs SET definition_json = definition_json #- ARRAY['resolved_concurrency', 'source'] WHERE definition_json -> 'resolved_concurrency' -> 'source' IS NOT NULL;
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
-- Published jobs no longer record which layer set their run concurrency: drop the removed
-- resolved_concurrency.source from each job's stored definition, keeping its limit.
UPDATE sqb_jobs SET definition_json = json_remove(definition_json, '$.resolved_concurrency.source') WHERE json_extract(definition_json, '$.resolved_concurrency.source') IS NOT NULL;
6 changes: 3 additions & 3 deletions crates/sqlbuild-orchestration/tests/runs/test_job_detail.rs
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ fn given_published_jobs_when_people_read_and_launch_them_then_listing_view_and_r
"declared_at": {"path": "orchestration/jobs.py", "line": 7},
"owner": "Orders team", "overlap": "skip",
"pools": {"warehouse": {"amount": 3, "limit": 4}},
"resolved_concurrency": {"limit": 4, "source": "job"},
"resolved_concurrency": {"limit": 4},
"on_failure": {"retries": 2, "retry_delay_seconds": 300}},
"customers": {"description": "Build customers", "selection": ["customers"]},
}));
Expand Down Expand Up @@ -231,10 +231,10 @@ fn given_published_jobs_when_people_read_and_launch_them_then_listing_view_and_r
json!([{"upstream_job": "daily_orders", "from_node": "orders", "to_node": "customers"}]),
json!([{"pool": "warehouse", "amount": 3, "limit": 4, "cap": 2, "server_limit": 2,
"clamped": true, "shared_with": ["orders_group"]}]),
(Some(4), json!({"limit": 4, "source": "job"})),
(Some(4), json!({"limit": 4})),
json!([{"pool": "warehouse", "amount": 3, "limit": 4, "cap": 2, "server_limit": 2,
"clamped": true, "shared_with": ["orders_group"]}]),
(Some(4), json!({"limit": 4, "source": "job"})),
(Some(4), json!({"limit": 4})),
json!([{"pool": "warehouse", "amount": 1, "limit": 4, "cap": 2, "server_limit": 2,
"clamped": false, "shared_with": ["daily_orders"]}]),
json!(null)
Expand Down
Loading
Loading