Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 33 additions & 4 deletions crates/sqlbuild-core/src/plan_display/_helpers/plan_pages.rs
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,8 @@ use serde_json::Value;
use crate::plan_display::_helpers::diff_lines::diff_stats;
use crate::plan_display::_helpers::warning_groups::warning_groups;
use crate::plan_display::constants::{
DIFF_STATS_FIELD, FIRST_RUN_REASON, NO_CHANGE_REASON, SKIP_ACTION,
DIFF_STATS_FIELD, FIRST_RUN_REASON, IS_RENAME_FIELD, NO_CHANGE_REASON, ORIGIN_MODEL_FIELD,
SKIP_ACTION,
};
use crate::plan_display::models::PlanPageSection;

Expand Down Expand Up @@ -81,6 +82,21 @@ impl<'document> PlanRows<'document> {
Self { document, runs }
}

/// The migration steps: every migration-like row except renamed models' own migrations.
pub(crate) fn migration_steps(&self) -> usize {
let migrations = self
.list("migrations")
.into_iter()
.filter(|migration| {
migration.get(IS_RENAME_FIELD).and_then(Value::as_bool) != Some(true)
})
.count();
migrations
+ self.list("column_migrations").len()
+ self.list("table_type_conversions").len()
+ self.list("retention").len()
}

/// The section's rows in plan order.
pub(crate) fn rows(&self, section: PlanPageSection) -> Vec<&'document Value> {
match section {
Expand Down Expand Up @@ -345,13 +361,26 @@ fn model_section(model: &Value) -> PlanPageSection {
.map_or(PlanPageSection::RoutineModels, |(_, section)| *section)
}

/// The name a row is found by; a warning is found by its model and message.
/// The name a row is found by; renames match old names and retention changes their models.
pub(crate) fn row_name(section: PlanPageSection, row: &Value) -> String {
match section {
PlanPageSection::Renamed => {
format!("{} {}", text(row, "name"), text(row, ORIGIN_MODEL_FIELD))
}
PlanPageSection::Migrations
| PlanPageSection::ColumnMigrations
| PlanPageSection::TableTypeConversions
| PlanPageSection::Retention => text(row, "model").to_owned(),
| PlanPageSection::TableTypeConversions => text(row, "model").to_owned(),
PlanPageSection::Retention => {
let mut names: Vec<&str> = row
.get("models")
.and_then(Value::as_array)
.into_iter()
.flatten()
.filter_map(Value::as_str)
.collect();
names.push(text(row, "name"));
names.join(" ")
}
PlanPageSection::HeldBack => key_name(text(row, "node")).to_owned(),
PlanPageSection::Unchanged => key_name(row.as_str().unwrap_or_default()).to_owned(),
PlanPageSection::Warnings => {
Expand Down
8 changes: 8 additions & 0 deletions crates/sqlbuild-core/src/plan_display/constants.rs
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,14 @@ pub(crate) const SKIP_ACTION: &str = "skip";
pub(crate) const NO_CHANGE_REASON: &str = "no_change";
/// The reason of a model or seed without a recorded build.
pub(crate) const FIRST_RUN_REASON: &str = "first_run";
/// The field naming the model a renamed model was renamed from, on its row and on its migration's.
pub const ORIGIN_MODEL_FIELD: &str = "origin_model";
/// The field saying what stays at a moved model's old name, on its migration's row and a renamed model's.
pub const OLD_NAME_FIELD: &str = "old_name";
/// The migration row field saying the migration is a renamed model's own.
pub const IS_RENAME_FIELD: &str = "is_rename";
/// The summary field counting migration steps other than renamed models' own migrations.
pub(crate) const MIGRATION_STEPS_FIELD: &str = "migration_steps";
/// The plan document field of per-model metadata, which a summary leaves out.
pub(crate) const METADATA_FIELD: &str = "metadata";
/// The plan document's row lists, which a summary empties and counts by section instead.
Expand Down
4 changes: 3 additions & 1 deletion crates/sqlbuild-core/src/plan_display/main/plan_summary.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,9 @@ use serde_json::Value;

use crate::plan_display::_helpers::plan_pages::PlanRows;
use crate::plan_display::_helpers::plan_sections::{emptied, section_counts, with_warning_groups};
use crate::plan_display::constants::MIGRATION_STEPS_FIELD;

/// The document with every row list emptied, each section counted and its warnings grouped; a failure report is returned unchanged.
/// The document with its row lists emptied and counted by section; a failure report is unchanged.
pub fn plan_summary(document: &Value) -> Value {
let Some(fields) = document
.as_object()
Expand All @@ -18,5 +19,6 @@ pub fn plan_summary(document: &Value) -> Value {
let mut summary = with_warning_groups(&plan, emptied(fields));
summary.insert("sections".into(), Value::Array(sections));
summary.insert("pages".into(), Value::Object(pages));
summary.insert(MIGRATION_STEPS_FIELD.into(), plan.migration_steps().into());
Value::Object(summary)
}
12 changes: 9 additions & 3 deletions crates/sqlbuild-core/src/plan_display/tests/helpers.rs
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,9 @@ pub(super) fn full_plan() -> serde_json::Value {
let mut payments = planned_model("payments", ("create_table", "query_changed"));
payments["query_diff"] = serde_json::json!(["-select id", "+SELECT id"]);
payments["config_diff"] = serde_json::json!(["- \"tags\": []", "+ \"tags\": [\"finance\"]"]);
let mut old_orders = planned_model("old_orders", ("create_table", "renamed"));
old_orders["origin_model"] = serde_json::json!("orders_v1");
old_orders["old_name"] = serde_json::json!({"action": "view_only", "view": "main.orders_v1", "expires_at": "2026-12-18T00:00:00Z"});
let mut totals = planned_model("totals", ("create_table", "config_changed"));
totals["config_diff"] = serde_json::json!([
"--- a",
Expand Down Expand Up @@ -85,7 +88,7 @@ pub(super) fn full_plan() -> serde_json::Value {
ledger,
daily_orders,
daily_payments,
planned_model("old_orders", ("create_table", "renamed")),
old_orders,
planned_model("archive", ("create_table", "full_refresh")),
planned_model("customers", ("create_view", "no_change")),
planned_model("Customer_Notes", ("create_view", "no_change")),
Expand Down Expand Up @@ -118,10 +121,13 @@ pub(super) fn full_plan() -> serde_json::Value {
{"severity": "warning", "message": "model stg__customers_3 has no tests."},
{"severity": "warning", "message": "scenario 'returns' step `refund` is stale", "code": "W1"},
],
"migrations": [{"model": "old_orders", "origin_model": "orders_v1", "decision": "rename", "completed_at": null}],
"migrations": [
{"model": "old_orders", "origin_model": "orders_v1", "decision": "renamed", "completed_at": null, "is_rename": true},
{"model": "totals", "origin_model": "totals_legacy", "decision": "forced_replace", "completed_at": null, "is_rename": false},
],
"column_migrations": [{"model": "ledger", "origin_column": "amt", "destination_column": "amount", "decision": "rename"}],
"table_type_conversions": [{"model": "totals", "actual_type": "view", "desired_type": "table"}],
"retention": [{"model": "events", "action": "set"}],
"retention": [{"scope": "table", "name": "main.events", "models": ["events"], "desired_days": 7, "actual_days": 1, "direction": "increase"}],
"metadata": {"orders": {"columns": 3}},
"synced_copy": {"synced_at": null, "cursor_window_starts": {"events": "2026-11-01"}, "unknown_until_run": []},
"changed": {
Expand Down
112 changes: 110 additions & 2 deletions crates/sqlbuild-core/src/plan_display/tests/test_plan_pages.rs
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ fn given_a_plan_when_paging_each_section_then_every_section_pages_its_rows_in_pl
SectionTestCase {
description: "migrations",
section: PlanPageSection::Migrations,
expected_rows: &["old_orders"],
expected_rows: &["old_orders", "totals"],
},
SectionTestCase {
description: "column migrations",
Expand All @@ -72,7 +72,7 @@ fn given_a_plan_when_paging_each_section_then_every_section_pages_its_rows_in_pl
SectionTestCase {
description: "retention",
section: PlanPageSection::Retention,
expected_rows: &["events"],
expected_rows: &["main.events"],
},
SectionTestCase {
description: "loader reloads",
Expand Down Expand Up @@ -220,6 +220,24 @@ fn given_a_name_filter_when_paging_then_matching_rows_page_with_their_count() {
expected_total: 2,
expected_section_total: 6,
},
PageTestCase {
description: "a retention change matches by its models",
section: PlanPageSection::Retention,
window: (0, 50),
name: Some("EVENTS"),
expected_rows: &["main.events"],
expected_total: 1,
expected_section_total: 1,
},
PageTestCase {
description: "a renamed model matches by its old name",
section: PlanPageSection::Renamed,
window: (0, 50),
name: Some("orders_v1"),
expected_rows: &["old_orders"],
expected_total: 1,
expected_section_total: 1,
},
PageTestCase {
description: "nothing matching is an empty page",
section: PlanPageSection::FirstRun,
Expand Down Expand Up @@ -404,6 +422,11 @@ fn given_a_plan_when_summarised_then_sections_are_counts_and_no_row_list_is_left
{"template": "already-built dates keep their old results", "count": 1, "subject": null},
]),
},
SummaryFieldTestCase {
description: "migration steps leave out the renamed model's own migration",
pointer: "/migration_steps",
expected_value: json!(4),
},
SummaryFieldTestCase {
description: "--changed facts stay",
pointer: "/changed/reach",
Expand Down Expand Up @@ -712,3 +735,88 @@ fn given_config_diffs_when_paged_whole_or_detached_then_short_ones_read_as_one_c
);
}
}

#[test]
fn given_renames_when_paged_or_summarised_then_rows_keep_their_marks_and_steps_leave_renames_out() {
let test_cases = [
SummaryFieldTestCase {
description: "a renamed row names the model it was renamed from",
pointer: "/renamed/origin_model",
expected_value: json!("orders_v1"),
},
SummaryFieldTestCase {
description: "a renamed row says what stays at its old name",
pointer: "/renamed/old_name/action",
expected_value: json!("view_only"),
},
SummaryFieldTestCase {
description: "a rename's own migration is marked",
pointer: "/migrations/0/is_rename",
expected_value: json!(true),
},
SummaryFieldTestCase {
description: "another migration is not",
pointer: "/migrations/1/is_rename",
expected_value: json!(false),
},
];
let plan = full_plan();
let observed = json!({
"renamed": plan_section_page(&plan, PlanPageSection::Renamed, ((0, 1), None)).rows[0],
"migrations": plan_section_page(&plan, PlanPageSection::Migrations, ((0, 50), None)).rows,
});
for test_case in test_cases {
assert_eq!(
observed
.pointer(test_case.pointer)
.cloned()
.unwrap_or_default(),
test_case.expected_value,
"{}",
test_case.description
);
}
}

#[test]
fn given_a_schema_retention_change_without_models_when_filtered_then_its_schema_name_matches() {
let test_cases = [
PageTestCase {
description: "a schema's retention change matches by the schema's name",
section: PlanPageSection::Retention,
window: (0, 50),
name: Some("ARCHIVE"),
expected_rows: &["analytics_archive"],
expected_total: 1,
expected_section_total: 2,
},
PageTestCase {
description: "a table's retention change still matches by its models",
section: PlanPageSection::Retention,
window: (0, 50),
name: Some("events"),
expected_rows: &["main.events"],
expected_total: 1,
expected_section_total: 2,
},
];
let mut plan = full_plan();
plan["retention"]
.as_array_mut()
.expect("retention rows")
.push(json!({"scope": "schema", "name": "analytics_archive", "models": [], "desired_days": 30, "actual_days": 7, "direction": "increase"}));
for test_case in test_cases {
let page = plan_section_page(&plan, test_case.section, (test_case.window, test_case.name));

assert_eq!(
(labels(&page.rows), page.total, page.section_total),
(
owned(test_case.expected_rows),
test_case.expected_total,
test_case.expected_section_total
),
"{}",
test_case.description
);
}
}
2 changes: 2 additions & 0 deletions crates/sqlbuild-python-planner/src/plan_output/constants.rs
Original file line number Diff line number Diff line change
Expand Up @@ -8,3 +8,5 @@ pub const IGNORE_AUDIT_FAILURES_JOBS_KEY: &str = "ignore_audit_failures_jobs";
pub const FULL_REFRESH_JOBS_KEY: &str = "full_refresh_jobs";
/// The plan JSON mapping each named job to the warehouse its nodes run on.
pub const WAREHOUSE_JOBS_KEY: &str = "job_warehouses";
/// The plan JSON reason of a model built under a new name.
pub const RENAMED_REASON: &str = "renamed";
Original file line number Diff line number Diff line change
Expand Up @@ -15,22 +15,26 @@ use crate::plan_output::_helpers::plan_json::table_type_json;
use crate::plan_output::_helpers::plan_json::warning_json;
use crate::plan_output::_helpers::values::mapped;
use crate::plan_output::constants::{
CHANGED_KEY, FULL_REFRESH_JOBS_KEY, IGNORE_AUDIT_FAILURES_JOBS_KEY, WAREHOUSE_JOBS_KEY,
CHANGED_KEY, FULL_REFRESH_JOBS_KEY, IGNORE_AUDIT_FAILURES_JOBS_KEY, RENAMED_REASON,
WAREHOUSE_JOBS_KEY,
};
use crate::plan_output::main::any::any;
use crate::plan_output::main::dict::dict;
use crate::plan_output::main::items::items;
use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python};
use pyo3::pyfunction;
use pyo3::types::{PyDict, PyDictMethods, PyList};
use sqlbuild_core::plan_display::constants::SYNCED_COPY_KEY;
use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods};
use sqlbuild_core::plan_display::constants::{
IS_RENAME_FIELD, OLD_NAME_FIELD, ORIGIN_MODEL_FIELD, SYNCED_COPY_KEY,
};
use sqlbuild_python_base::boundary::models::KeywordArguments;
use sqlbuild_python_base::compile_outputs::main::indented_json::indented_json;
use sqlbuild_python_base::records::main::fields::field::field as native_field;
use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get;
use sqlbuild_python_project::compiled_project::constants::{
JOB_FULL_REFRESH_KEY, JOB_IGNORED_AUDIT_FAILURES_KEY, JOB_WAREHOUSES_KEY,
};
use std::collections::HashMap;

/// The plan as one JSON document, `json.dumps(indent=2)`.
#[pyfunction]
Expand Down Expand Up @@ -111,6 +115,7 @@ pub fn format_plan_json(py: Python<'_>, options: Option<Bound<'_, PyDict>>) -> P
),
],
)?;
mark_renames(&result)?;
let metadata: Bound<'_, PyAny> = native_field(&plan, "metadata")?;
if metadata.is_truthy()? {
result.set_item("metadata", &metadata)?;
Expand Down Expand Up @@ -146,3 +151,43 @@ pub fn format_plan_json(py: Python<'_>, options: Option<Bound<'_, PyDict>>) -> P
}
indented_json(result.as_any())
}

/// Marks renamed models' own migrations and copies their origin and old name onto the model rows.
fn mark_renames(result: &Bound<'_, PyDict>) -> PyResult<()> {
let (Some(models), Some(migrations)) =
(result.get_item("models")?, result.get_item("migrations")?)
else {
return Ok(());
};
let mut renamed: HashMap<String, Bound<'_, PyDict>> = HashMap::new();
for model in models.downcast::<PyList>()?.iter() {
let model: Bound<'_, PyDict> = model.downcast_into::<PyDict>()?;
let reason: Option<String> = model.get_item("reason")?.map(|r| r.extract()).transpose()?;
if reason.as_deref() == Some(RENAMED_REASON)
&& let Some(name) = model.get_item("name")?
{
renamed.insert(name.extract()?, model);
}
}
for migration in migrations.downcast::<PyList>()?.iter() {
let migration: Bound<'_, PyDict> = migration.downcast_into::<PyDict>()?;
let model: Option<String> = migration
.get_item("model")?
.map(|m| m.extract())
.transpose()?;
let row: Option<&Bound<'_, PyDict>> = model.as_deref().and_then(|name| renamed.get(name));
migration.set_item(IS_RENAME_FIELD, row.is_some())?;
let Some(row) = row else {
continue;
};
for field in [ORIGIN_MODEL_FIELD, OLD_NAME_FIELD] {
if let Some(value) = migration.get_item(field)?
&& !value.is_none()
&& !row.contains(field)?
{
row.set_item(field, value)?;
}
}
}
Ok(())
}
Original file line number Diff line number Diff line change
Expand Up @@ -125,7 +125,7 @@ Seeds (1)

When query, function, or schema changes are detected, the plan lists each affected model under its own change with its backfill action. A model that calls a changed function directly appears under `Function changed` with `cause: function <name> changed`; models further downstream continue forward. See [Replay decisions](../concepts/planning/replay-decisions.md).

In `sqb plan --json`, each model's `reason` is its own change. A direct caller of a changed function has `"reason": "function_changed"` and a `changed_functions` list, and a renamed model whose query also changed has `"reason": "renamed"` and `"query_changed": true`. A changed model also carries the detail the text plan prints under it: `query_diff` when its query changed and the previous query is known, and `config_diff` when its config changed, each as unified diff lines, and `schema_findings` for added, removed, or retyped columns.
In `sqb plan --json`, each model's `reason` is its own change. A direct caller of a changed function has `"reason": "function_changed"` and a `changed_functions` list, and a renamed model whose query also changed has `"reason": "renamed"` and `"query_changed": true`. A renamed model's row names the model it was renamed from in `origin_model` and what stays at the old name in `old_name` (the same object as its migration's; see [Model migrations](../concepts/models/migrations.md); absent when the migration has none), and that rename's own entry in `migrations` has `"is_rename": true`; every other migration has `"is_rename": false`. A changed model also carries the detail the text plan prints under it: `query_diff` when its query changed and the previous query is known, and `config_diff` when its config changed, each as unified diff lines, and `schema_findings` for added, removed, or retyped columns.

Warnings and maintenance notices, such as `migrate_from can be removed`, are listed in a `Warnings` section before the final `Plan complete` line. `sqb plan --json` output is unchanged.

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -246,6 +246,9 @@ migration decisions using your current connection. It only inspects; nothing is
- A recorded view that no longer exists is not re-created by a build; janitor records it as dropped.
- Run the same SQLBuild version for `sqb janitor` as for builds. An older janitor doesn't recognise
compatibility views and archives them as stale relations.
- In `sqb plan --json`, a migration that moves a renamed model has `"is_rename": true`, and the renamed
model's own row names its old model in `origin_model` and carries the same `old_name` object as that
migration, or no `old_name` when the migration has none. Other migrations have `"is_rename": false`.
- In `sqb plan --json`, each migration has an `old_name` object with `action` (`archive_and_view`,
`view_only`, `live` or `none`), `view`, `reads`, `expires_at`, `column_aliases`, `grants_copied`
and `reason`. The top-level `old_names` list also includes steps that resume an earlier move.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@ class AutomaticMigrationTestCase:
description: str
expected_migrations: tuple[tuple[str | None, str, str, str], ...] = ()
expected_events: tuple[tuple[str, str, str], ...] = ()
expected_renames: tuple[tuple[str, str], ...] = ()
expected_reason: str = ""
expected_warning: str = ""

Expand Down
Loading
Loading