diff --git a/crates/sqlbuild-core/src/plan_display/_helpers/plan_pages.rs b/crates/sqlbuild-core/src/plan_display/_helpers/plan_pages.rs index b754ee080d..bf15c5a3eb 100644 --- a/crates/sqlbuild-core/src/plan_display/_helpers/plan_pages.rs +++ b/crates/sqlbuild-core/src/plan_display/_helpers/plan_pages.rs @@ -7,7 +7,8 @@ use serde_json::Value; use crate::plan_display::_helpers::diff_lines::diff_stats; use crate::plan_display::_helpers::warning_groups::warning_groups; use crate::plan_display::constants::{ - DIFF_STATS_FIELD, FIRST_RUN_REASON, NO_CHANGE_REASON, SKIP_ACTION, + DIFF_STATS_FIELD, FIRST_RUN_REASON, IS_RENAME_FIELD, NO_CHANGE_REASON, ORIGIN_MODEL_FIELD, + SKIP_ACTION, }; use crate::plan_display::models::PlanPageSection; @@ -81,6 +82,21 @@ impl<'document> PlanRows<'document> { Self { document, runs } } + /// The migration steps: every migration-like row except renamed models' own migrations. + pub(crate) fn migration_steps(&self) -> usize { + let migrations = self + .list("migrations") + .into_iter() + .filter(|migration| { + migration.get(IS_RENAME_FIELD).and_then(Value::as_bool) != Some(true) + }) + .count(); + migrations + + self.list("column_migrations").len() + + self.list("table_type_conversions").len() + + self.list("retention").len() + } + /// The section's rows in plan order. pub(crate) fn rows(&self, section: PlanPageSection) -> Vec<&'document Value> { match section { @@ -345,13 +361,26 @@ fn model_section(model: &Value) -> PlanPageSection { .map_or(PlanPageSection::RoutineModels, |(_, section)| *section) } -/// The name a row is found by; a warning is found by its model and message. +/// The name a row is found by; renames match old names and retention changes their models. pub(crate) fn row_name(section: PlanPageSection, row: &Value) -> String { match section { + PlanPageSection::Renamed => { + format!("{} {}", text(row, "name"), text(row, ORIGIN_MODEL_FIELD)) + } PlanPageSection::Migrations | PlanPageSection::ColumnMigrations - | PlanPageSection::TableTypeConversions - | PlanPageSection::Retention => text(row, "model").to_owned(), + | PlanPageSection::TableTypeConversions => text(row, "model").to_owned(), + PlanPageSection::Retention => { + let mut names: Vec<&str> = row + .get("models") + .and_then(Value::as_array) + .into_iter() + .flatten() + .filter_map(Value::as_str) + .collect(); + names.push(text(row, "name")); + names.join(" ") + } PlanPageSection::HeldBack => key_name(text(row, "node")).to_owned(), PlanPageSection::Unchanged => key_name(row.as_str().unwrap_or_default()).to_owned(), PlanPageSection::Warnings => { diff --git a/crates/sqlbuild-core/src/plan_display/constants.rs b/crates/sqlbuild-core/src/plan_display/constants.rs index 7642d785e7..a2e9173661 100644 --- a/crates/sqlbuild-core/src/plan_display/constants.rs +++ b/crates/sqlbuild-core/src/plan_display/constants.rs @@ -49,6 +49,14 @@ pub(crate) const SKIP_ACTION: &str = "skip"; pub(crate) const NO_CHANGE_REASON: &str = "no_change"; /// The reason of a model or seed without a recorded build. pub(crate) const FIRST_RUN_REASON: &str = "first_run"; +/// The field naming the model a renamed model was renamed from, on its row and on its migration's. +pub const ORIGIN_MODEL_FIELD: &str = "origin_model"; +/// The field saying what stays at a moved model's old name, on its migration's row and a renamed model's. +pub const OLD_NAME_FIELD: &str = "old_name"; +/// The migration row field saying the migration is a renamed model's own. +pub const IS_RENAME_FIELD: &str = "is_rename"; +/// The summary field counting migration steps other than renamed models' own migrations. +pub(crate) const MIGRATION_STEPS_FIELD: &str = "migration_steps"; /// The plan document field of per-model metadata, which a summary leaves out. pub(crate) const METADATA_FIELD: &str = "metadata"; /// The plan document's row lists, which a summary empties and counts by section instead. diff --git a/crates/sqlbuild-core/src/plan_display/main/plan_summary.rs b/crates/sqlbuild-core/src/plan_display/main/plan_summary.rs index 72c1bfd08d..15ff93de0f 100644 --- a/crates/sqlbuild-core/src/plan_display/main/plan_summary.rs +++ b/crates/sqlbuild-core/src/plan_display/main/plan_summary.rs @@ -4,8 +4,9 @@ use serde_json::Value; use crate::plan_display::_helpers::plan_pages::PlanRows; use crate::plan_display::_helpers::plan_sections::{emptied, section_counts, with_warning_groups}; +use crate::plan_display::constants::MIGRATION_STEPS_FIELD; -/// The document with every row list emptied, each section counted and its warnings grouped; a failure report is returned unchanged. +/// The document with its row lists emptied and counted by section; a failure report is unchanged. pub fn plan_summary(document: &Value) -> Value { let Some(fields) = document .as_object() @@ -18,5 +19,6 @@ pub fn plan_summary(document: &Value) -> Value { let mut summary = with_warning_groups(&plan, emptied(fields)); summary.insert("sections".into(), Value::Array(sections)); summary.insert("pages".into(), Value::Object(pages)); + summary.insert(MIGRATION_STEPS_FIELD.into(), plan.migration_steps().into()); Value::Object(summary) } diff --git a/crates/sqlbuild-core/src/plan_display/tests/helpers.rs b/crates/sqlbuild-core/src/plan_display/tests/helpers.rs index d3bf33a6d2..19d4b4be91 100644 --- a/crates/sqlbuild-core/src/plan_display/tests/helpers.rs +++ b/crates/sqlbuild-core/src/plan_display/tests/helpers.rs @@ -38,6 +38,9 @@ pub(super) fn full_plan() -> serde_json::Value { let mut payments = planned_model("payments", ("create_table", "query_changed")); payments["query_diff"] = serde_json::json!(["-select id", "+SELECT id"]); payments["config_diff"] = serde_json::json!(["- \"tags\": []", "+ \"tags\": [\"finance\"]"]); + let mut old_orders = planned_model("old_orders", ("create_table", "renamed")); + old_orders["origin_model"] = serde_json::json!("orders_v1"); + old_orders["old_name"] = serde_json::json!({"action": "view_only", "view": "main.orders_v1", "expires_at": "2026-12-18T00:00:00Z"}); let mut totals = planned_model("totals", ("create_table", "config_changed")); totals["config_diff"] = serde_json::json!([ "--- a", @@ -85,7 +88,7 @@ pub(super) fn full_plan() -> serde_json::Value { ledger, daily_orders, daily_payments, - planned_model("old_orders", ("create_table", "renamed")), + old_orders, planned_model("archive", ("create_table", "full_refresh")), planned_model("customers", ("create_view", "no_change")), planned_model("Customer_Notes", ("create_view", "no_change")), @@ -118,10 +121,13 @@ pub(super) fn full_plan() -> serde_json::Value { {"severity": "warning", "message": "model stg__customers_3 has no tests."}, {"severity": "warning", "message": "scenario 'returns' step `refund` is stale", "code": "W1"}, ], - "migrations": [{"model": "old_orders", "origin_model": "orders_v1", "decision": "rename", "completed_at": null}], + "migrations": [ + {"model": "old_orders", "origin_model": "orders_v1", "decision": "renamed", "completed_at": null, "is_rename": true}, + {"model": "totals", "origin_model": "totals_legacy", "decision": "forced_replace", "completed_at": null, "is_rename": false}, + ], "column_migrations": [{"model": "ledger", "origin_column": "amt", "destination_column": "amount", "decision": "rename"}], "table_type_conversions": [{"model": "totals", "actual_type": "view", "desired_type": "table"}], - "retention": [{"model": "events", "action": "set"}], + "retention": [{"scope": "table", "name": "main.events", "models": ["events"], "desired_days": 7, "actual_days": 1, "direction": "increase"}], "metadata": {"orders": {"columns": 3}}, "synced_copy": {"synced_at": null, "cursor_window_starts": {"events": "2026-11-01"}, "unknown_until_run": []}, "changed": { diff --git a/crates/sqlbuild-core/src/plan_display/tests/test_plan_pages.rs b/crates/sqlbuild-core/src/plan_display/tests/test_plan_pages.rs index bf67a9aed7..ace9bf29fd 100644 --- a/crates/sqlbuild-core/src/plan_display/tests/test_plan_pages.rs +++ b/crates/sqlbuild-core/src/plan_display/tests/test_plan_pages.rs @@ -57,7 +57,7 @@ fn given_a_plan_when_paging_each_section_then_every_section_pages_its_rows_in_pl SectionTestCase { description: "migrations", section: PlanPageSection::Migrations, - expected_rows: &["old_orders"], + expected_rows: &["old_orders", "totals"], }, SectionTestCase { description: "column migrations", @@ -72,7 +72,7 @@ fn given_a_plan_when_paging_each_section_then_every_section_pages_its_rows_in_pl SectionTestCase { description: "retention", section: PlanPageSection::Retention, - expected_rows: &["events"], + expected_rows: &["main.events"], }, SectionTestCase { description: "loader reloads", @@ -220,6 +220,24 @@ fn given_a_name_filter_when_paging_then_matching_rows_page_with_their_count() { expected_total: 2, expected_section_total: 6, }, + PageTestCase { + description: "a retention change matches by its models", + section: PlanPageSection::Retention, + window: (0, 50), + name: Some("EVENTS"), + expected_rows: &["main.events"], + expected_total: 1, + expected_section_total: 1, + }, + PageTestCase { + description: "a renamed model matches by its old name", + section: PlanPageSection::Renamed, + window: (0, 50), + name: Some("orders_v1"), + expected_rows: &["old_orders"], + expected_total: 1, + expected_section_total: 1, + }, PageTestCase { description: "nothing matching is an empty page", section: PlanPageSection::FirstRun, @@ -404,6 +422,11 @@ fn given_a_plan_when_summarised_then_sections_are_counts_and_no_row_list_is_left {"template": "already-built dates keep their old results", "count": 1, "subject": null}, ]), }, + SummaryFieldTestCase { + description: "migration steps leave out the renamed model's own migration", + pointer: "/migration_steps", + expected_value: json!(4), + }, SummaryFieldTestCase { description: "--changed facts stay", pointer: "/changed/reach", @@ -712,3 +735,88 @@ fn given_config_diffs_when_paged_whole_or_detached_then_short_ones_read_as_one_c ); } } + +#[test] +fn given_renames_when_paged_or_summarised_then_rows_keep_their_marks_and_steps_leave_renames_out() { + let test_cases = [ + SummaryFieldTestCase { + description: "a renamed row names the model it was renamed from", + pointer: "/renamed/origin_model", + expected_value: json!("orders_v1"), + }, + SummaryFieldTestCase { + description: "a renamed row says what stays at its old name", + pointer: "/renamed/old_name/action", + expected_value: json!("view_only"), + }, + SummaryFieldTestCase { + description: "a rename's own migration is marked", + pointer: "/migrations/0/is_rename", + expected_value: json!(true), + }, + SummaryFieldTestCase { + description: "another migration is not", + pointer: "/migrations/1/is_rename", + expected_value: json!(false), + }, + ]; + let plan = full_plan(); + let observed = json!({ + "renamed": plan_section_page(&plan, PlanPageSection::Renamed, ((0, 1), None)).rows[0], + "migrations": plan_section_page(&plan, PlanPageSection::Migrations, ((0, 50), None)).rows, + }); + for test_case in test_cases { + assert_eq!( + observed + .pointer(test_case.pointer) + .cloned() + .unwrap_or_default(), + test_case.expected_value, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_a_schema_retention_change_without_models_when_filtered_then_its_schema_name_matches() { + let test_cases = [ + PageTestCase { + description: "a schema's retention change matches by the schema's name", + section: PlanPageSection::Retention, + window: (0, 50), + name: Some("ARCHIVE"), + expected_rows: &["analytics_archive"], + expected_total: 1, + expected_section_total: 2, + }, + PageTestCase { + description: "a table's retention change still matches by its models", + section: PlanPageSection::Retention, + window: (0, 50), + name: Some("events"), + expected_rows: &["main.events"], + expected_total: 1, + expected_section_total: 2, + }, + ]; + let mut plan = full_plan(); + plan["retention"] + .as_array_mut() + .expect("retention rows") + .push(json!({"scope": "schema", "name": "analytics_archive", "models": [], "desired_days": 30, "actual_days": 7, "direction": "increase"})); + for test_case in test_cases { + let page = plan_section_page(&plan, test_case.section, (test_case.window, test_case.name)); + + assert_eq!( + (labels(&page.rows), page.total, page.section_total), + ( + owned(test_case.expected_rows), + test_case.expected_total, + test_case.expected_section_total + ), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-python-planner/src/plan_output/constants.rs b/crates/sqlbuild-python-planner/src/plan_output/constants.rs index 56360156e9..50ad542d78 100644 --- a/crates/sqlbuild-python-planner/src/plan_output/constants.rs +++ b/crates/sqlbuild-python-planner/src/plan_output/constants.rs @@ -8,3 +8,5 @@ pub const IGNORE_AUDIT_FAILURES_JOBS_KEY: &str = "ignore_audit_failures_jobs"; pub const FULL_REFRESH_JOBS_KEY: &str = "full_refresh_jobs"; /// The plan JSON mapping each named job to the warehouse its nodes run on. pub const WAREHOUSE_JOBS_KEY: &str = "job_warehouses"; +/// The plan JSON reason of a model built under a new name. +pub const RENAMED_REASON: &str = "renamed"; diff --git a/crates/sqlbuild-python-planner/src/plan_output/main/format_plan_json.rs b/crates/sqlbuild-python-planner/src/plan_output/main/format_plan_json.rs index 756e459d9c..8c4a0aa218 100644 --- a/crates/sqlbuild-python-planner/src/plan_output/main/format_plan_json.rs +++ b/crates/sqlbuild-python-planner/src/plan_output/main/format_plan_json.rs @@ -15,15 +15,18 @@ use crate::plan_output::_helpers::plan_json::table_type_json; use crate::plan_output::_helpers::plan_json::warning_json; use crate::plan_output::_helpers::values::mapped; use crate::plan_output::constants::{ - CHANGED_KEY, FULL_REFRESH_JOBS_KEY, IGNORE_AUDIT_FAILURES_JOBS_KEY, WAREHOUSE_JOBS_KEY, + CHANGED_KEY, FULL_REFRESH_JOBS_KEY, IGNORE_AUDIT_FAILURES_JOBS_KEY, RENAMED_REASON, + WAREHOUSE_JOBS_KEY, }; use crate::plan_output::main::any::any; use crate::plan_output::main::dict::dict; use crate::plan_output::main::items::items; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::pyfunction; -use pyo3::types::{PyDict, PyDictMethods, PyList}; -use sqlbuild_core::plan_display::constants::SYNCED_COPY_KEY; +use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods}; +use sqlbuild_core::plan_display::constants::{ + IS_RENAME_FIELD, OLD_NAME_FIELD, ORIGIN_MODEL_FIELD, SYNCED_COPY_KEY, +}; use sqlbuild_python_base::boundary::models::KeywordArguments; use sqlbuild_python_base::compile_outputs::main::indented_json::indented_json; use sqlbuild_python_base::records::main::fields::field::field as native_field; @@ -31,6 +34,7 @@ use sqlbuild_python_base::records::main::fields::mapping_get::mapping_get; use sqlbuild_python_project::compiled_project::constants::{ JOB_FULL_REFRESH_KEY, JOB_IGNORED_AUDIT_FAILURES_KEY, JOB_WAREHOUSES_KEY, }; +use std::collections::HashMap; /// The plan as one JSON document, `json.dumps(indent=2)`. #[pyfunction] @@ -111,6 +115,7 @@ pub fn format_plan_json(py: Python<'_>, options: Option>) -> P ), ], )?; + mark_renames(&result)?; let metadata: Bound<'_, PyAny> = native_field(&plan, "metadata")?; if metadata.is_truthy()? { result.set_item("metadata", &metadata)?; @@ -146,3 +151,43 @@ pub fn format_plan_json(py: Python<'_>, options: Option>) -> P } indented_json(result.as_any()) } + +/// Marks renamed models' own migrations and copies their origin and old name onto the model rows. +fn mark_renames(result: &Bound<'_, PyDict>) -> PyResult<()> { + let (Some(models), Some(migrations)) = + (result.get_item("models")?, result.get_item("migrations")?) + else { + return Ok(()); + }; + let mut renamed: HashMap> = HashMap::new(); + for model in models.downcast::()?.iter() { + let model: Bound<'_, PyDict> = model.downcast_into::()?; + let reason: Option = model.get_item("reason")?.map(|r| r.extract()).transpose()?; + if reason.as_deref() == Some(RENAMED_REASON) + && let Some(name) = model.get_item("name")? + { + renamed.insert(name.extract()?, model); + } + } + for migration in migrations.downcast::()?.iter() { + let migration: Bound<'_, PyDict> = migration.downcast_into::()?; + let model: Option = migration + .get_item("model")? + .map(|m| m.extract()) + .transpose()?; + let row: Option<&Bound<'_, PyDict>> = model.as_deref().and_then(|name| renamed.get(name)); + migration.set_item(IS_RENAME_FIELD, row.is_some())?; + let Some(row) = row else { + continue; + }; + for field in [ORIGIN_MODEL_FIELD, OLD_NAME_FIELD] { + if let Some(value) = migration.get_item(field)? + && !value.is_none() + && !row.contains(field)? + { + row.set_item(field, value)?; + } + } + } + Ok(()) +} diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/plan.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/plan.md index 2cce0a9bf2..159f636f8d 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/plan.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/plan.md @@ -125,7 +125,7 @@ Seeds (1) When query, function, or schema changes are detected, the plan lists each affected model under its own change with its backfill action. A model that calls a changed function directly appears under `Function changed` with `cause: function changed`; models further downstream continue forward. See [Replay decisions](../concepts/planning/replay-decisions.md). -In `sqb plan --json`, each model's `reason` is its own change. A direct caller of a changed function has `"reason": "function_changed"` and a `changed_functions` list, and a renamed model whose query also changed has `"reason": "renamed"` and `"query_changed": true`. A changed model also carries the detail the text plan prints under it: `query_diff` when its query changed and the previous query is known, and `config_diff` when its config changed, each as unified diff lines, and `schema_findings` for added, removed, or retyped columns. +In `sqb plan --json`, each model's `reason` is its own change. A direct caller of a changed function has `"reason": "function_changed"` and a `changed_functions` list, and a renamed model whose query also changed has `"reason": "renamed"` and `"query_changed": true`. A renamed model's row names the model it was renamed from in `origin_model` and what stays at the old name in `old_name` (the same object as its migration's; see [Model migrations](../concepts/models/migrations.md); absent when the migration has none), and that rename's own entry in `migrations` has `"is_rename": true`; every other migration has `"is_rename": false`. A changed model also carries the detail the text plan prints under it: `query_diff` when its query changed and the previous query is known, and `config_diff` when its config changed, each as unified diff lines, and `schema_findings` for added, removed, or retyped columns. Warnings and maintenance notices, such as `migrate_from can be removed`, are listed in a `Warnings` section before the final `Plan complete` line. `sqb plan --json` output is unchanged. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/migrations.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/migrations.md index 8ac221da44..b3e3a6a7dc 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/migrations.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/migrations.md @@ -246,6 +246,9 @@ migration decisions using your current connection. It only inspects; nothing is - A recorded view that no longer exists is not re-created by a build; janitor records it as dropped. - Run the same SQLBuild version for `sqb janitor` as for builds. An older janitor doesn't recognise compatibility views and archives them as stale relations. +- In `sqb plan --json`, a migration that moves a renamed model has `"is_rename": true`, and the renamed + model's own row names its old model in `origin_model` and carries the same `old_name` object as that + migration, or no `old_name` when the migration has none. Other migrations have `"is_rename": false`. - In `sqb plan --json`, each migration has an `old_name` object with `action` (`archive_and_view`, `view_only`, `live` or `none`), `view`, `reads`, `expires_at`, `column_aliases`, `grants_copied` and `reason`. The top-level `old_names` list also includes steps that resume an earlier move. diff --git a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/_test_types.py b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/_test_types.py index 42aa220ac6..5cfcfa5add 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/_test_types.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/_test_types.py @@ -60,6 +60,7 @@ class AutomaticMigrationTestCase: description: str expected_migrations: tuple[tuple[str | None, str, str, str], ...] = () expected_events: tuple[tuple[str, str, str], ...] = () + expected_renames: tuple[tuple[str, str], ...] = () expected_reason: str = "" expected_warning: str = "" diff --git a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/helpers.py b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/helpers.py index 2cb80efbe6..0491c81693 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/helpers.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/helpers.py @@ -338,6 +338,57 @@ def planned_migrations(plan: dict[str, Any]) -> tuple[tuple[str | None, str, str ) +def is_renamed_model(model: dict[str, Any]) -> bool: + """Return whether the plan builds this model under a new name.""" + + return model["reason"] == "renamed" + + +def is_rename_migration(migration: dict[str, Any]) -> bool: + """Return whether this migration is a renamed model's own.""" + + return bool(migration["is_rename"]) + + +def renamed_models(plan: dict[str, Any]) -> tuple[tuple[str, str], ...]: + """Return sorted (origin_model, name) for every model the plan builds under a new name.""" + + return tuple( + sorted( + (model["origin_model"], model["name"]) + for model in filter(is_renamed_model, plan["models"]) + ) + ) + + +def rename_migrations(plan: dict[str, Any]) -> tuple[tuple[str, str], ...]: + """Return sorted (origin_model, model) for every migration marked as a renamed model's own.""" + + return tuple( + sorted( + (migration["origin_model"], migration["model"]) + for migration in filter(is_rename_migration, plan["migrations"]) + ) + ) + + +def renamed_old_names(plan: dict[str, Any]) -> dict[str, Any]: + """Return each renamed model's old_name, by model, as its own row carries it.""" + + return { + model["name"]: model.get("old_name") for model in filter(is_renamed_model, plan["models"]) + } + + +def rename_migration_old_names(plan: dict[str, Any]) -> dict[str, Any]: + """Return each rename migration's old_name, by model.""" + + return { + migration["model"]: migration["old_name"] + for migration in filter(is_rename_migration, plan["migrations"]) + } + + def warning_messages(plan: dict[str, Any]) -> tuple[str, ...]: """Return every plan warning message.""" diff --git a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_automatic_migrations.py b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_automatic_migrations.py index 6cac8cd3c3..f3cacecaac 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_automatic_migrations.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_automatic_migrations.py @@ -30,6 +30,10 @@ plan_json, planned_migrations, relation_names, + rename_migration_old_names, + rename_migrations, + renamed_models, + renamed_old_names, renamed_order_models, row_count, run_sqb, @@ -55,6 +59,9 @@ ("orders_enriched", "customer_orders_enriched", "renamed"), ("stg_orders", "stg_customer_orders", "migrate"), ), + # Only the view is built under its new name; the incremental models take their data + # over, so their migrations are steps, not renames. + expected_renames=(("orders_enriched", "customer_orders_enriched"),), ) ], ids=lambda case: case.description, @@ -71,6 +78,12 @@ def test_given_coordinated_renames_when_building_then_history_moves_automaticall rebuild_plan: dict[str, Any] = plan_json(project_dir=tmp_path, capsys=capsys) assert planned_migrations(plan) == test_case.expected_migrations + # Each renamed model's row names its old model, and its own migration is marked as the rename. + assert renamed_models(plan) == test_case.expected_renames + assert rename_migrations(plan) == test_case.expected_renames + # What stays at the old name is on the renamed row too, as its migration has it. + assert renamed_old_names(plan) == rename_migration_old_names(plan) + assert all(old_name is not None for old_name in renamed_old_names(plan).values()) assert model_entry(plan=plan, name="stg_customer_orders")["reason"] != _FIRST_RUN assert model_entry(plan=plan, name="customer_daily_order_totals")["reason"] != _FIRST_RUN assert "Migrated main.stg_orders -> main.stg_customer_orders" in result.output diff --git a/ui/README.md b/ui/README.md index 85d7309d61..82c13b9d4e 100644 --- a/ui/README.md +++ b/ui/README.md @@ -60,3 +60,13 @@ browser saving a HAR, then ask the server for every read the pages made: node scripts/record-demo.mjs demo/recording.json.gz --server http://127.0.0.1: \ --replace =/home/demo/duckshop --replace =duckshop session.har ``` + +To record only the plan previews again, through the server's plan summary, rows and diff reads, +keep the rest of the recording and plan each recorded command anew on `sqb dev`: + +```sh +node scripts/record-demo-plans.mjs demo/recording.json.gz --server http://127.0.0.1: \ + --replace =/home/demo/duckshop --ask job:marts_build=job:marts_build_job +``` + +`--ask` rewrites only the command sent to the server, for a job renamed since the recording. diff --git a/ui/demo/recording.json.gz b/ui/demo/recording.json.gz index a5c6adaa17..711e791d02 100644 Binary files a/ui/demo/recording.json.gz and b/ui/demo/recording.json.gz differ diff --git a/ui/e2e/plan-browser.spec.ts b/ui/e2e/plan-browser.spec.ts index 390f52399b..4d563f1f1a 100644 --- a/ui/e2e/plan-browser.spec.ts +++ b/ui/e2e/plan-browser.spec.ts @@ -29,8 +29,24 @@ async function select(page: Page, key: string) { test('the index lists the sections in order, opens on the first, and the address keeps the one picked', async ({ page }) => { await openPlan(page); await expect(page).toHaveTitle('Plan · sqb build --changed'); - await expect(page.locator('.index .summary')).toHaveText('94 changed, 672 new, 121 renames, 64 loads, 433 Python, 40 held back · 1,519 unchanged, 300 skipped'); - expect(await page.locator('.index .entry').evaluateAll((es) => es.map((e) => (e as HTMLElement).dataset.section))).toEqual(['changed', 'renames', 'first_run', 'loads', 'python', 'held_back', 'warnings', 'unchanged', 'skipped']); + // Grouped under what will run and what won't, each heading counting nodes once, with no summary sentence. + await expect(page.locator('.index .summary')).toHaveCount(0); + await expect(page.getByTestId('index-will-run')).toHaveText('Will run2,770'); + await expect(page.getByTestId('index-wont-run')).toHaveText('Won’t run340'); + expect(await page.locator('.index .entry').evaluateAll((es) => es.map((e) => [(e as HTMLElement).dataset.section, e.querySelector('.count')?.textContent]))).toEqual([ + ['changed', '94'], + // The mock's migrations act on models outside the selection: no renamed nodes, only steps, which + // the section's header counts. + ['renames', '121 steps'], + ['first_run', '672'], + ['loads', '64'], + ['python', '433'], + // 1,507 models; the section also lists 12 unchanged functions, which are not nodes. + ['unchanged', '1,507'], + ['held_back', '40'], + ['skipped', '300'], + ['warnings', '74'] + ]); await expect(page.locator('.index .entry[aria-current="true"]')).toHaveAttribute('data-section', 'changed'); await select(page, 'first_run'); await expect(page).toHaveURL(/[?&]section=first_run/); @@ -210,6 +226,9 @@ test('seeds sit with the models of the same reason: first runs, changes and unch await expect(pane(page).locator('.subtitle')).toHaveText('87 query · 6 config · 1 function · 1 seed'); await select(page, 'unchanged'); await expect(pane(page).locator('.subtitle')).toHaveText('1,507 models · 12 functions · 7 seeds'); + // The header counts nodes as the index does, and the functions listed with them apart. + await expect(pane(page).locator('h3')).toHaveText('Unchanged 1,514 nodes · 12 functions'); + await expect(page.locator('.index .entry[data-section="unchanged"] .count')).toHaveText('1,514'); await expect(rows(page)).toHaveCount(50); if (shots) await page.screenshot({ path: `${shots}/plan-b-seeds.png` }); }); @@ -227,7 +246,37 @@ test('photographs every section of the large plan', async ({ page }) => { await openPlan(page); for (const key of ['changed', 'renames', 'first_run', 'loads', 'python', 'held_back', 'skipped']) { await select(page, key); - await expect(rows(page).first()).toBeVisible(); + await expect(pane(page).locator('.plan-row, [data-testid="plan-renames"] .row').first()).toBeVisible(); await page.screenshot({ path: `${shots}/plan-b-${key}.png` }); } }); + +test('a failed read is not repeated on its own: it says so, and Retry reads again', async ({ page }) => { + await page.addInitScript(() => ((window as unknown as { __sqbFailPlanRows?: boolean }).__sqbFailPlanRows = true)); + await page.goto(`${LARGE}§ion=unchanged`); + await expect(page.getByTestId('plan-rows-error')).toContainText('The plan has expired', { timeout: 30_000 }); + await page.waitForTimeout(1500); + expect((await calls(page)).filter((c) => c.section === 'routine_models'), 'one read, not a retry loop').toHaveLength(1); + await page.evaluate(() => ((window as unknown as { __sqbFailPlanRows?: boolean }).__sqbFailPlanRows = false)); + await page.getByTestId('plan-rows-error').getByRole('button', { name: 'Retry' }).click(); + await expect(rows(page)).toHaveCount(50); + await expect(page.getByTestId('plan-rows-error')).toHaveCount(0); +}); + +test('a failed read blocks only its own group, and a re-plan reads afresh', async ({ page }) => { + const fail = (on: boolean) => page.evaluate((v) => ((window as unknown as { __sqbFailPlanRows?: boolean }).__sqbFailPlanRows = v), on); + await page.addInitScript(() => ((window as unknown as { __sqbFailPlanRows?: boolean }).__sqbFailPlanRows = true)); + await page.goto(`${LARGE}§ion=unchanged`); + await expect(page.getByTestId('plan-rows-error')).toBeVisible({ timeout: 30_000 }); + await fail(false); + // Another group reads its rows, though Unchanged's read failed. + await select(page, 'first_run'); + await expect(rows(page).first()).toBeVisible(); + await expect(page.getByTestId('plan-rows-error')).toHaveCount(0); + // Back on Unchanged the failure stands until Retry, or until the plan is made again. + await select(page, 'unchanged'); + await expect(page.getByTestId('plan-rows-error')).toBeVisible(); + await page.getByRole('button', { name: 'Plan again' }).click(); + await expect(rows(page)).toHaveCount(50, { timeout: 30_000 }); + await expect(page.getByTestId('plan-rows-error')).toHaveCount(0); +}); diff --git a/ui/e2e/plan-renames.spec.ts b/ui/e2e/plan-renames.spec.ts new file mode 100644 index 0000000000..1189c7965f --- /dev/null +++ b/ui/e2e/plan-renames.spec.ts @@ -0,0 +1,113 @@ +import { expect, test, type Page } from '@playwright/test'; + +// The plan index under Will run and Won't run, the Renames and migrations section and the plan's width +// (CHI-1563), on a plan shaped like a large real one's (`?mock&summary`). +test.use({ viewport: { width: 1600, height: 1000 } }); + +const CMD = `cmd=${encodeURIComponent('sqb build --changed')}`; +const renames = (page: Page) => page.getByTestId('plan-renames'); +type Call = { section: string; offset: number; limit: number; name?: string }; +const calls = (page: Page) => page.evaluate(() => (window as unknown as { __sqbPlanRows?: Call[] }).__sqbPlanRows ?? []); + +test('the index counts what will run and what won’t once each, renames as nodes', async ({ page }) => { + await page.goto(`/plan?mock&summary&${CMD}`); + await expect(page.locator('.plan-browser')).toBeVisible({ timeout: 30_000 }); + await expect(page.getByTestId('index-will-run')).toHaveText('Will run2,877'); + await expect(page.getByTestId('index-wont-run')).toHaveText('Won’t run97'); + await expect(page.locator('.index .entry[data-section="renames"] .count')).toHaveText('58'); + await expect(page.locator('.index')).not.toContainText('migration steps'); +}); + +test('Renamed lists old name → new name, and migration steps by kind say only what their heading doesn’t', async ({ page }) => { + await page.goto(`/plan?mock&summary§ion=renames&${CMD}`); + await expect(renames(page).locator('h3')).toHaveText('Renames and migrations 58 nodes · 63 migration steps', { timeout: 30_000 }); + const renamed = renames(page).locator('[data-part="renamed"] .row'); + await expect(renamed).toHaveCount(10); + await expect(renamed.first().locator('.from')).toHaveText(/_old$/); + // A rename's usual rebuild goes unsaid; what stays at the old name is said. + await expect(renames(page).locator('[data-part="renamed"]')).not.toContainText('rebuilt'); + await expect(renamed.nth(1).locator('.detail')).toHaveText('old table archived; a view at the old name until 2026-12-18'); + await expect(renames(page)).not.toContainText('On nodes counted elsewhere'); + expect(await renames(page).locator('.kind h5').allTextContents()).toEqual(['Data moved between models 2', 'Column renames 41', 'Table type changes 12', 'Time travel retention 8']); + const columns = renames(page).locator('[data-kind="column_migrations"] .row .detail'); + await expect(columns.first()).toHaveText('cust_id → customer_id'); + await expect(renames(page).locator('[data-kind="column_migrations"]')).not.toContainText('renamed in place'); + await expect(columns.nth(4)).toHaveText('sku_code → product_sku·already renamed; recorded'); + // Each kind shows a few rows and loads the rest of its own section when asked. + await renames(page).locator('[data-kind="column_migrations"]').getByRole('button', { name: /^Show more/ }).click(); + await expect(renames(page).locator('[data-kind="column_migrations"] .row')).toHaveCount(41); +}); + +test('the filter finds a renamed model by its old name, and a retention change by its models', async ({ page }) => { + await page.goto(`/plan?mock&summary§ion=renames&${CMD}`); + await expect(renames(page).locator('h3')).toHaveText('Renames and migrations 58 nodes · 63 migration steps', { timeout: 30_000 }); + const filter = renames(page).getByRole('textbox', { name: 'Filter renames and migrations' }); + const old = (await renames(page).locator('[data-part="renamed"] .row .from').first().textContent()) ?? ''; + await filter.fill(old); + await expect(renames(page).locator('[data-part="renamed"] .row')).toHaveCount(1); + await expect(renames(page).locator('[data-part="steps"]')).toHaveCount(0); + // A retention change lists its models, not a model: it is found by any of them. + await filter.fill('summary_334'); + await expect(renames(page).locator('[data-part="renamed"]')).toHaveCount(0); + await expect(renames(page).locator('[data-kind="retention"] .row')).toHaveCount(1); + await expect(renames(page).locator('[data-kind="retention"] .row').first()).toContainText('raised: 1 day → 14 days'); + await filter.fill('nowhere'); + await expect(renames(page)).toContainText('No renames or migrations match “nowhere”.'); + await renames(page).getByRole('button', { name: 'Clear the filter' }).click(); + await expect(renames(page).locator('[data-part="renamed"] .row')).toHaveCount(10); +}); + +test('the plan stops at its capped width, its toolbar with it, and the graph stays full width', async ({ page }) => { + await page.setViewportSize({ width: 2000, height: 1000 }); + await page.goto(`/plan?mock&summary&${CMD}`); + await expect(page.locator('.plan-browser')).toBeVisible({ timeout: 30_000 }); + const right = async (selector: string) => { + const box = await page.locator(selector).first().boundingBox(); + return box ? box.x + box.width : 0; + }; + const pane = await right('.plan-browser'); + expect(pane - (await page.locator('.plan-browser').boundingBox())!.x).toBeLessThanOrEqual(230 + 24 + 1100 + 1); + expect(Math.abs((await right('.summary')) - pane)).toBeLessThanOrEqual(1); + await page.getByRole('tablist', { name: 'View' }).getByRole('tab', { name: 'Graph' }).click(); + expect(await right('.summary')).toBeGreaterThan(pane + 200); +}); + +test('a filter typed before a re-plan reads the new plan’s rows', async ({ page }) => { + await page.goto(`/plan?mock&summary§ion=renames&${CMD}`); + await expect(renames(page).locator('h3')).toHaveText(/58 nodes/, { timeout: 30_000 }); + await renames(page).getByRole('textbox', { name: 'Filter renames and migrations' }).fill('summary_334'); + await expect(renames(page).locator('[data-kind="retention"] .row')).toHaveCount(1); + const asked = (await calls(page)).filter((c) => c.name === 'summary_334').length; + await page.getByRole('button', { name: 'Plan again' }).click(); + // The new plan is asked for the same name, and its rows are shown. + await expect.poll(async () => (await calls(page)).filter((c) => c.name === 'summary_334').length, { timeout: 30_000 }).toBeGreaterThan(asked); + await expect(renames(page).locator('[data-kind="retention"] .row')).toHaveCount(1); + await expect(renames(page).getByRole('textbox', { name: 'Filter renames and migrations' })).toHaveValue('summary_334'); +}); + +test('a failed read in Renames and migrations is not repeated on its own; Retry reads again', async ({ page }) => { + await page.addInitScript(() => ((window as unknown as { __sqbFailPlanRows?: boolean }).__sqbFailPlanRows = true)); + await page.goto(`/plan?mock&summary§ion=renames&${CMD}`); + await expect(renames(page).getByTestId('renames-error')).toContainText('The plan has expired', { timeout: 30_000 }); + await page.waitForTimeout(1500); + expect((await calls(page)).filter((c) => c.section === 'renamed'), 'one read, not a retry loop').toHaveLength(1); + await page.evaluate(() => ((window as unknown as { __sqbFailPlanRows?: boolean }).__sqbFailPlanRows = false)); + await renames(page).getByRole('button', { name: 'Retry' }).click(); + await expect(renames(page).locator('[data-part="renamed"] .row')).toHaveCount(10); + await expect(renames(page).getByTestId('renames-error')).toHaveCount(0); +}); + +test('a failed page read in Renames and migrations leaves the filter working, and a re-plan reads afresh', async ({ page }) => { + const fail = (on: boolean) => page.evaluate((v) => ((window as unknown as { __sqbFailPlanRows?: boolean }).__sqbFailPlanRows = v), on); + await page.addInitScript(() => ((window as unknown as { __sqbFailPlanRows?: boolean }).__sqbFailPlanRows = true)); + await page.goto(`/plan?mock&summary§ion=renames&${CMD}`); + await expect(renames(page).getByTestId('renames-error')).toBeVisible({ timeout: 30_000 }); + await fail(false); + await renames(page).getByRole('textbox', { name: 'Filter renames and migrations' }).fill('summary_334'); + await expect(renames(page).locator('[data-kind="retention"] .row')).toHaveCount(1); + await renames(page).getByRole('button', { name: 'Clear the filter' }).click(); + await expect(renames(page).getByTestId('renames-error')).toBeVisible(); + await page.getByRole('button', { name: 'Plan again' }).click(); + await expect(renames(page).locator('[data-part="renamed"] .row')).toHaveCount(10, { timeout: 30_000 }); + await expect(renames(page).getByTestId('renames-error')).toHaveCount(0); +}); diff --git a/ui/e2e/plan-server.spec.ts b/ui/e2e/plan-server.spec.ts new file mode 100644 index 0000000000..8b87575fc7 --- /dev/null +++ b/ui/e2e/plan-server.spec.ts @@ -0,0 +1,47 @@ +import { expect, test } from '@playwright/test'; +import { FakeControl } from './support/fake-control'; +import { FakePlans, renamesPlan } from './support/fake-plans'; + +// The plan page against a faked control server, through the real server client: a field the client +// drops between the server's summary and the page shows here, where the mock client would hide it. +// Covers the summary's migration_steps and the rows' origin_model, old_name and is_rename. +test.use({ viewport: { width: 1600, height: 1000 } }); + +async function open(page: import('@playwright/test').Page, plans: FakePlans) { + await new FakeControl().install(page); + await plans.install(page); + await page.goto(`/plan?cmd=${encodeURIComponent('sqb build')}§ion=renames`); + await expect(page.getByTestId('plan-renames')).toBeVisible({ timeout: 30_000 }); +} + +test('the Renames and migrations section reads the server’s step count and rename fields', async ({ page }) => { + const plans = new FakePlans(renamesPlan()); + await open(page, plans); + const renames = page.getByTestId('plan-renames'); + // migration_steps from the summary: the forced move and the column rename, not the two renames' own migrations. + await expect(renames.locator('h3')).toHaveText('Renames and migrations 2 nodes · 2 migration steps'); + await expect(page.getByTestId('index-will-run')).toHaveText('Will run3'); + await expect(page.locator('.index .entry[data-section="renames"] .count')).toHaveText('2'); + // origin_model and old_name from the renamed rows. + const renamed = renames.locator('[data-part="renamed"] .row'); + await expect(renamed).toHaveCount(2); + await expect(renamed.first().locator('.from')).toHaveText('customers'); + await expect(renamed.first().locator('.detail')).toHaveText('old table archived; a view at the old name until 2026-12-18'); + await expect(renamed.nth(1).locator('.from')).toHaveText('payments'); + // is_rename leaves the renames' own migrations out of the steps. + const moved = renames.locator('[data-kind="migrations"]'); + await expect(moved.locator('h5')).toHaveText('Data moved between models 1'); + await expect(moved.locator('.row')).toHaveCount(1); + await expect(moved.locator('.row')).toContainText('replaced by the data of orders_legacy (forced)'); + await expect(renames.locator('[data-kind="column_migrations"] .row')).toContainText('amt → amount'); +}); + +test('the filter asks the server, which finds a renamed model by its old name', async ({ page }) => { + const plans = new FakePlans(renamesPlan()); + await open(page, plans); + const renames = page.getByTestId('plan-renames'); + await renames.getByRole('textbox', { name: 'Filter renames and migrations' }).fill('payments'); + await expect(renames.locator('[data-part="renamed"] .row')).toHaveCount(1); + await expect(renames.locator('[data-part="renamed"] .row .from')).toHaveText('payments'); + expect(plans.reads.some((r) => r.section === 'renamed' && r.name === 'payments')).toBe(true); +}); diff --git a/ui/e2e/support/fake-plans.ts b/ui/e2e/support/fake-plans.ts new file mode 100644 index 0000000000..2ad86056e8 --- /dev/null +++ b/ui/e2e/support/fake-plans.ts @@ -0,0 +1,92 @@ +// The control server's plan endpoints, faked inside a browser test over `FakeControl`, so the plan page +// reads a plan through the real server client: the dry run, the plan request, its summary and its +// section pages. The summary and rows are shaped as the server's `plan_summary` and plan rows read +// write them. Lives only in the tests. + +import type { Page, Route } from '@playwright/test'; + +/** A plan the fake serves: the summary document, and each section's rows. */ +export type FakePlan = { summary: Record; rows: Record[]>; selection: string[] }; + +/** One section page read the page asked for. */ +export type RowsRead = { section: string; offset: number; limit: number; name: string | null }; + +const json = (route: Route, body: unknown, status = 200) => route.fulfill({ status, contentType: 'application/json', body: JSON.stringify(body) }); + +/** The name a row is found by, as the server finds it. */ +function rowName(section: string, row: Record): string { + if (section === 'renamed') return `${row.name} ${row.origin_model ?? ''}`; + if (section === 'retention') return [...((row.models as string[] | undefined) ?? []), row.name].join(' '); + return String(row.name ?? row.model ?? ''); +} + +export class FakePlans { + readonly reads: RowsRead[] = []; + constructor(readonly plan: FakePlan) {} + + /** Answers the plan endpoints; install after `FakeControl.install`, so these routes come first. */ + async install(page: Page) { + await page.route('**/api/control/runs/adhoc/check', (route) => + json(route, { verb: 'build', selection: this.plan.selection, write_set: this.plan.selection, settings: {}, permissions: [], error: null, launchable: true }) + ); + await page.route(/\/api\/control\/locations\/[^/]+\/targets\/dev\/plans(\/.*)?(\?.*)?$/, (route) => this.#answer(route)); + } + + #answer(route: Route) { + const url = new URL(route.request().url()); + if (route.request().method() === 'POST') return json(route, { applied: true }); + if (url.pathname.endsWith('/summary')) return json(route, { expires_at: new Date(Date.now() + 600_000).toISOString(), outcome: { kind: 'plan', document: this.plan.summary }, timings: null }); + if (url.pathname.endsWith('/diff')) return json(route, { error: 'no diff' }, 404); + if (url.pathname.endsWith('/rows')) { + const section = url.searchParams.get('section') ?? ''; + const offset = Number(url.searchParams.get('offset') ?? 0); + const limit = Number(url.searchParams.get('limit') ?? 50); + const name = url.searchParams.get('name'); + this.reads.push({ section, offset, limit, name }); + const all = this.plan.rows[section] ?? []; + const found = name ? all.filter((row) => rowName(section, row).toLowerCase().includes(name.toLowerCase())) : all; + return json(route, { total: found.length, section_total: all.length, rows: found.slice(offset, offset + limit), runs: [], warnings: [], cursor_window_starts: {} }); + } + return json(route, { error: 'not found' }, 404); + } +} + +/** + * A small plan with renames, as the server summarises it: one changed model, two renamed (one keeping a + * view at its old name), their two rename migrations, one migration into a model that exists and one + * column rename, so two migration steps. + */ +export function renamesPlan(): FakePlan { + const model = (name: string, reason: string, extra: Record = {}) => ({ name, materialization_type: 'table', action: 'create_table', reason, backfill: { action: 'forward', duration: null }, ...extra }); + const rows = { + query_changed: [model('orders', 'query_changed', { diff_stats: { query: { added: 2, removed: 1, whitespace_only: false } } })], + renamed: [ + model('customers_v2', 'renamed', { origin_model: 'customers', old_name: { action: 'archive_and_view', view: 'dev.customers', reads: 'dev.customers_v2', expires_at: '2026-12-18T00:00:00Z', column_aliases: {}, grants_copied: false, reason: null } }), + model('payments_v2', 'renamed', { origin_model: 'payments' }) + ], + migrations: [ + { kind: 'model_migration', model: 'customers_v2', origin_model: 'customers', decision: 'renamed', completed_at: null, is_rename: true, old_name: null }, + { kind: 'model_migration', model: 'payments_v2', origin_model: 'payments', decision: 'renamed', completed_at: null, is_rename: true, old_name: null }, + { kind: 'model_migration', model: 'orders', origin_model: 'orders_legacy', decision: 'forced_replace', completed_at: null, is_rename: false, old_name: null } + ], + column_migrations: [{ kind: 'column_migration', model: 'orders', origin_column: 'amt', destination_column: 'amount', decision: 'rename', completed_at: null }] + }; + const sections = Object.entries(rows).map(([section, list]) => ({ section, total: list.length, notes: [] })); + const empty = { models: [], seeds: [], source_loads: [], functions: [], python_nodes: [], warnings: [], migrations: [], column_migrations: [], table_type_conversions: [], retention: [], old_names: [] }; + return { + selection: ['model:orders', 'model:customers_v2', 'model:payments_v2'], + rows, + summary: { + selected_count: 3, + source_load_count: 0, + python_node_count: 0, + ...empty, + synced_copy: null, + changed: null, + sections, + pages: Object.fromEntries(sections.map((s) => [s.section, s.total])), + warning_groups: [], + migration_steps: 2 + } + }; +} diff --git a/ui/scripts/record-demo-plans.mjs b/ui/scripts/record-demo-plans.mjs new file mode 100644 index 0000000000..54a139076e --- /dev/null +++ b/ui/scripts/record-demo-plans.mjs @@ -0,0 +1,96 @@ +// Records the static demo's plan previews again from DuckShop on `sqb dev`, through the server's real +// plan reads: each recorded plan command is planned anew, and its summary, every page of its sections +// and its models' diffs replace the recorded answers. The rest of the recording is kept. +// +// node scripts/record-demo-plans.mjs --server [--replace FROM=TO]... [--ask FROM=TO]... +// +// The server's location and request ids are written as the recording's, so the replay matches the +// demo's plan requests as before. `--replace` rewrites text everywhere, as record-demo.mjs does; +// `--ask` rewrites only the command sent to the server, such as a job renamed since the recording. + +import { readFileSync, writeFileSync } from 'node:fs'; +import { gunzipSync, gzipSync } from 'node:zlib'; +import { randomUUID } from 'node:crypto'; + +const USAGE = 'usage: node scripts/record-demo-plans.mjs --server [--replace FROM=TO]... [--ask FROM=TO]...'; +const [path, ...options] = process.argv.slice(2); +let server = null; +const replacements = []; +const asked = []; +for (let i = 0; i < options.length; i++) { + const value = options[i + 1]; + if (options[i] === '--server' && value) server = options[++i].replace(/\/$/, ''); + else if (options[i] === '--replace' && value?.includes('=')) replacements.push([value.slice(0, value.indexOf('=')), options[++i].slice(value.indexOf('=') + 1)]); + else if (options[i] === '--ask' && value?.includes('=')) asked.push([value.slice(0, value.indexOf('=')), options[++i].slice(value.indexOf('=') + 1)]); + else { + console.error(USAGE); + process.exit(2); + } +} +if (!path || !server) { + console.error(USAGE); + process.exit(2); +} + +/** Rows each section page read asks for: the plan page's pages, and the Checks tab's larger reads. */ +const PAGE_SIZES = [50, 1000]; +const PLANS = /^\/api\/control\/locations\/([^/]+)\/targets\/([^/]+)\/plans$/; +const POLL_MS = 500; + +const recording = JSON.parse(gunzipSync(readFileSync(path)).toString('utf8')); +const asks = recording.responses.filter((r) => r.method === 'POST' && PLANS.test(r.url)); +if (!asks.length) throw new Error('The recording has no plan requests to record again'); + +const get = async (url) => { + const response = await fetch(server + url); + if (response.status >= 500) throw new Error(`${url} answered ${response.status}`); + return { status: response.status, contentType: response.headers.get('content-type') ?? 'application/json', text: await response.text() }; +}; +const bootstrap = JSON.parse((await get('/api/control/bootstrap')).text); +const location = bootstrap.access.locations[0].location_id; + +const fresh = []; +for (const ask of asks) { + const [, recordedLocation, target] = PLANS.exec(ask.url); + const recordedId = ask.body.request_id; + const id = randomUUID(); + const base = `/api/control/locations/${location}/targets/${target}/plans`; + const posted = await fetch(server + base, { method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify({ ...ask.body, command: ask.body.command.map((token) => asked.reduce((out, [from, to]) => (out === from ? to : out), token)), request_id: id }) }); + if (!posted.ok) throw new Error(`planning ${JSON.stringify(ask.body.command)} answered ${posted.status}: ${await posted.text()}`); + let summary; + for (;;) { + summary = await get(`${base}/${id}/summary`); + if (summary.status !== 200) throw new Error(`the summary of ${JSON.stringify(ask.body.command)} answered ${summary.status}: ${summary.text}`); + if (JSON.parse(summary.text).outcome) break; + await new Promise((resolve) => setTimeout(resolve, POLL_MS)); + } + // Written as the recording's location and request, so the replay's id mapping finds them. + const ours = (text) => replacements.reduce((out, [from, to]) => out.split(from).join(to), text.split(location).join(recordedLocation).split(id).join(recordedId)); + const keep = (url, answer) => fresh.push({ method: 'GET', url: ours(url), ...answer, text: ours(answer.text) }); + keep(`${base}/${id}/summary`, summary); + const document = JSON.parse(summary.text).outcome.document ?? {}; + if (document.error || !document.sections) throw new Error(`${ask.body.command.join(' ')} planned no summary: ${summary.text.slice(0, 400)}`); + const models = new Set(); + for (const { section, total } of document.sections ?? []) { + for (const limit of PAGE_SIZES) { + for (let offset = 0; offset < total; offset += limit) { + const url = `${base}/${id}/rows?section=${section}&offset=${offset}&limit=${limit}`; + const page = await get(url); + keep(url, page); + for (const row of JSON.parse(page.text).rows ?? []) if (row && typeof row === 'object' && row.diff_stats) models.add(row.name); + } + } + } + for (const model of models) { + const url = `${base}/${id}/diff?model=${encodeURIComponent(model)}`; + keep(url, await get(url)); + } + console.log(`Planned ${ask.body.command.join(' ')}: ${(document.sections ?? []).length} sections, ${models.size} diffs.`); +} + +// The plan reads the page no longer makes go; every plan read is the fresh one. +const planRead = /\/plans\/[^/?]+(\/(summary|rows|diff))?(\?|$)/; +const kept = recording.responses.filter((r) => !(r.method === 'GET' && planRead.test(r.url))); +const responses = [...kept, ...fresh].sort((a, b) => a.url.localeCompare(b.url) || a.method.localeCompare(b.method)); +writeFileSync(path, gzipSync(`${JSON.stringify({ recordedAt: recording.recordedAt, responses })}\n`, { level: 9 })); +console.log(`Recorded ${fresh.length} plan reads again; ${responses.length} responses in ${path}.`); diff --git a/ui/src/lib/api/http-runs.test.ts b/ui/src/lib/api/http-runs.test.ts index 33f1c12863..7a67c1d023 100644 --- a/ui/src/lib/api/http-runs.test.ts +++ b/ui/src/lib/api/http-runs.test.ts @@ -339,7 +339,8 @@ describe('group run nodes', () => { describe('planResultOf', () => { it.each([ - { description: 'a plan keeps its models and fills missing sections', outcome: { kind: 'plan', document: { selected_count: 1, models: [{ name: 'orders', reason: 'first_run' }] } }, expected: { plan: 'orders', error: null } }, + { description: 'a plan keeps its models and fills missing sections', outcome: { kind: 'plan', document: { selected_count: 1, models: [{ name: 'orders', reason: 'first_run' }], sections: [], migration_steps: 0 } }, expected: { plan: 'orders', error: null } }, + { description: 'a plan without its section counts is refused', outcome: { kind: 'plan', document: { selected_count: 1, models: [{ name: 'orders', reason: 'first_run' }] } }, expected: { plan: null, error: 'The server sent a plan without its section counts' } }, { description: "sqb plan's own error is the result", outcome: { kind: 'plan', document: { error: { code: 'S007', message: "unknown selector name 'ordrs'" } } }, expected: { plan: null, error: "S007: unknown selector name 'ordrs'" } }, { description: 'the help line follows the error', outcome: { kind: 'plan', document: { error: { code: 'S000', message: 'no cursor interval', help: 'pass --start-cursor-ts' } } }, expected: { plan: null, error: 'S000: no cursor interval\nhelp: pass --start-cursor-ts' } }, { description: 'a superseded plan asks to plan again', outcome: { kind: 'superseded' }, expected: { plan: null, error: 'The project changed on the location while it was planning; plan again' } } @@ -351,12 +352,31 @@ describe('planResultOf', () => { }); }); +describe('planResultOf with renames', () => { + it('keeps the migration step count and the rename fields on rows', () => { + const plan = planResultOf({ + kind: 'plan', + document: { + models: [{ name: 'orders', action: 'create_table', reason: 'renamed', origin_model: 'orders_v1', old_name: { action: 'view_only', expires_at: '2026-12-18T00:00:00Z' } }], + migrations: [{ model: 'orders', origin_model: 'orders_v1', decision: 'renamed', completed_at: null, is_rename: true }], + sections: [{ section: 'renamed', total: 1, notes: [] }], + migration_steps: 3 + } + } as never).plan!; + expect(plan.migration_steps).toBe(3); + expect(plan.models[0]).toMatchObject({ origin_model: 'orders_v1', old_name: { action: 'view_only' } }); + expect(plan.migrations[0].is_rename).toBe(true); + }); +}); + describe('mergePlanPage', () => { const summary = planResultOf({ kind: 'plan', document: { models: [{ name: 'orders', action: 'create_table', reason: 'query_changed' }], pages: { routine_models: 2, unchanged: 3 }, + sections: [{ section: 'routine_models', total: 2, notes: [] }], + migration_steps: 0, changed: { counts: ['definition'], reach: 'all', runs: [{ node: 'model:orders', reason: 'definition changed', downstream_of: null }], held_back: [], unchanged: [] } } } as never).plan!; diff --git a/ui/src/lib/api/http-runs.ts b/ui/src/lib/api/http-runs.ts index 45cb21f668..3a0664cab5 100644 --- a/ui/src/lib/api/http-runs.ts +++ b/ui/src/lib/api/http-runs.ts @@ -22,6 +22,7 @@ export function planResultOf(outcome: Wire['LocationWorkOutcome']): PlanResult { return { plan: null, error: [code, message].filter(Boolean).join(': ') + (help ? `\nhelp: ${help}` : '') }; } const list = (value: T[] | undefined): T[] => value ?? []; + if (!document.sections || typeof document.migration_steps !== 'number') return { plan: null, error: 'The server sent a plan without its section counts' }; return { plan: { selected_count: document.selected_count ?? 0, @@ -39,7 +40,8 @@ export function planResultOf(outcome: Wire['LocationWorkOutcome']): PlanResult { changed: document.changed ?? null, ...(document.ignore_audit_failures ? { ignore_audit_failures: true } : {}), ...(document.pages ? { pages: document.pages } : {}), - ...(document.sections ? { sections: document.sections } : {}), + sections: document.sections, + migration_steps: document.migration_steps, ...(document.warning_groups ? { warning_groups: document.warning_groups } : {}) }, error: null diff --git a/ui/src/lib/api/index.ts b/ui/src/lib/api/index.ts index f4f2d510b0..a571c36561 100644 --- a/ui/src/lib/api/index.ts +++ b/ui/src/lib/api/index.ts @@ -53,7 +53,9 @@ export function createClient(search: URLSearchParams, onUnauthorized: () => void connection: LIVE.find((l) => l === live), still: search.has('still') }); - const sized = search.has('huge') ? hugeProject(mock) : search.has('large') ? largeProject(mock) : mock; + // `?summary`: the large project with a plan shaped like a large real one's summary (CHI-1563). + const summary = search.has('summary'); + const sized = search.has('huge') ? hugeProject(mock) : search.has('large') || summary ? largeProject(mock, { summary }) : mock; return countedReads(search.has('awkward') ? awkward(sized) : sized); } diff --git a/ui/src/lib/api/mock/large-plan.ts b/ui/src/lib/api/mock/large-plan.ts index e4846346c5..88c631dcb2 100644 --- a/ui/src/lib/api/mock/large-plan.ts +++ b/ui/src/lib/api/mock/large-plan.ts @@ -7,18 +7,18 @@ import { groupWarnings } from '$lib/run/plan-index'; import { seedSection } from '$lib/run/plan-text'; import type { PlanDiff, PlanDiffStats, PlanDocument, PlanModel, PlanPage, PlanPageSection, PlanSectionCount, PlanSectionNote, PlanWarning } from '../types'; -const DOMAINS = ['orders', 'customers', 'inventory', 'payments', 'shipping', 'returns', 'pricing', 'suppliers', 'regions', 'finance', 'marketing', 'support']; +export const DOMAINS = ['orders', 'customers', 'inventory', 'payments', 'shipping', 'returns', 'pricing', 'suppliers', 'regions', 'finance', 'marketing', 'support']; const LAYERS = ['stg', 'int', 'mart']; const NOUNS = ['daily_rollup', 'events', 'snapshot', 'curated', 'enriched', 'mapped', 'totals', 'history', 'latest', 'summary']; /** A neutral model name, distinct for each `i`. */ -const name = (i: number) => `${DOMAINS[i % DOMAINS.length]}__${LAYERS[Math.floor(i / DOMAINS.length) % LAYERS.length]}__${NOUNS[Math.floor(i / (DOMAINS.length * LAYERS.length)) % NOUNS.length]}_${i}`; +export const name = (i: number) => `${DOMAINS[i % DOMAINS.length]}__${LAYERS[Math.floor(i / DOMAINS.length) % LAYERS.length]}__${NOUNS[Math.floor(i / (DOMAINS.length * LAYERS.length)) % NOUNS.length]}_${i}`; -const HEAD = ['--- previous', '+++ current']; +export const HEAD = ['--- previous', '+++ current']; /** A query diff of `added` and `removed` lines. */ -function queryDiff(i: number, added: number, removed: number): string[] { +export function queryDiff(i: number, added: number, removed: number): string[] { return [...HEAD, `@@ -1,${removed + 2} +1,${added + 2} @@`, ' select', ...Array.from({ length: removed }, (_, k) => `- old_column_${k},`), ...Array.from({ length: added }, (_, k) => `+ new_column_${k}_${i},`), ` from ${name(i + 1)}`]; } -const model = (i: number, reason: string, extra: Partial = {}): PlanModel => ({ +export const model = (i: number, reason: string, extra: Partial = {}): PlanModel => ({ name: name(i), materialization_type: 'table', action: 'create_table', @@ -27,7 +27,7 @@ const model = (i: number, reason: string, extra: Partial = {}): PlanM qualified_name: `analytics.${name(i)}`, ...extra }); -const incremental = (extra: Partial = {}): Partial => ({ +export const incremental = (extra: Partial = {}): Partial => ({ materialization_type: 'incremental', action: 'incremental_merge', incremental_strategy: 'merge', @@ -67,7 +67,7 @@ export function largePlanDocument(changed: boolean): PlanDocument { ...Array.from({ length: 71 }, (_, k) => ({ severity: 'warning', message: `Scenario 'test_${DOMAINS[k % 12]}_${k % 24}' provides fixture for unknown model '${name(k + 3000)}'.` })), ...Array.from({ length: 3 }, (_, k) => ({ severity: 'warning', message: `Scenario 'test_returns_${k}' requires source 'raw.returns_${k}', but no fixture was provided. Add a CTE named __source__raw__returns_${k} AS (...).` })) ], - migrations: Array.from({ length: 121 }, (_, k) => ({ model: name(5000 + k), origin_model: name(6000 + k), decision: 'renamed', completed_at: null })), + migrations: Array.from({ length: 121 }, (_, k) => ({ model: name(5000 + k), origin_model: name(6000 + k), decision: 'migrate', completed_at: null, is_rename: false })), column_migrations: [], table_type_conversions: [], retention: [], @@ -98,7 +98,9 @@ function sectionRows(doc: PlanDocument, section: PlanPageSection): { rows: unkno const byModel = (rows: { model: string }[]) => ({ rows, name: (row: unknown) => (row as { model: string }).model }); if (MODEL_SECTIONS.includes(section) || section === 'routine_models') { const built = doc.models.filter((m) => m.action !== 'skip' && runsNode(`model:${m.name}`)); - return named(built.filter((m) => (MODEL_SECTIONS.includes(m.reason as PlanPageSection) ? m.reason === section : section === 'routine_models'))); + const rows = built.filter((m) => (MODEL_SECTIONS.includes(m.reason as PlanPageSection) ? m.reason === section : section === 'routine_models')); + // A renamed model is found by its old name too, as the server finds it. + return section === 'renamed' ? { rows, name: (row) => `${(row as PlanModel).name} ${(row as PlanModel).origin_model ?? ''}` } : named(rows); } const loads = (kind: string, reload: boolean) => named(doc.source_loads.filter((l) => l.kind === kind && l.is_reload === reload)); const changedFunction = (f: PlanDocument['functions'][number]) => !!f.reason && f.reason !== 'no_change'; @@ -110,7 +112,7 @@ function sectionRows(doc: PlanDocument, section: PlanPageSection): { rows: unkno case 'table_type_conversions': return byModel(doc.table_type_conversions); case 'retention': - return byModel(doc.retention); + return { rows: doc.retention, name: (row) => ((row as PlanDocument['retention'][number]).models ?? []).join(' ') }; case 'loader_reloads': return loads('loader', true); case 'loader_loads': @@ -172,6 +174,7 @@ export function planSummary(doc: PlanDocument): PlanDocument { ...empty, sections, pages: Object.fromEntries(sections.map((s) => [s.section, s.total])), + migration_steps: doc.migrations.filter((m) => !m.is_rename).length + doc.column_migrations.length + doc.table_type_conversions.length + doc.retention.length, warning_groups: groupWarnings(doc.warnings).map((g) => ({ template: g.template, count: g.count, subject: subjectOf(g.label) })), changed: doc.changed && { ...doc.changed, runs: [], held_back: [], unchanged: [] }, synced_copy: doc.synced_copy && { ...doc.synced_copy, cursor_window_starts: {} } diff --git a/ui/src/lib/api/mock/large-project.ts b/ui/src/lib/api/mock/large-project.ts index d4fa7a7c74..549fef1250 100644 --- a/ui/src/lib/api/mock/large-project.ts +++ b/ui/src/lib/api/mock/large-project.ts @@ -7,6 +7,7 @@ import type { ApiClient } from '../client'; import type { PlanDocument, PlanPage, PlanResult } from '../types'; import { largePlanDocument, planModelDiff, planPage, planSummary } from './large-plan'; import { recordPlanDiff, recordPlanRows } from './plan-rows-log'; +import { summaryPlanDocument } from './plan-summary-mockup'; import type { CatalogNode } from '../types'; import { newRequestId } from '$lib/browser/request-id'; @@ -44,8 +45,11 @@ export function largeProjectNodes(template: CatalogNode): CatalogNode[] { /** The large plans planned so far, by request, for their pages. */ const plans = new Map(); -/** The client with its catalog replaced by the large project, and a build planned as the large plan. */ -export function largeProject(client: ApiClient): ApiClient { +/** + * The client with its catalog replaced by the large project, and a build planned as the large plan; + * `summary` plans the plan summary's mock plan instead (CHI-1563). + */ +export function largeProject(client: ApiClient, { summary = false }: { summary?: boolean } = {}): ApiClient { return new Proxy(client, { get(target, prop, receiver) { const value = Reflect.get(target, prop, receiver); @@ -53,7 +57,8 @@ export function largeProject(client: ApiClient): ApiClient { return async (...args: Parameters): Promise => { const result = await target.planBuild(...args); if (!result.plan) return result; - const doc = largePlanDocument(args[1].some((t) => t === '--changed' || t.startsWith('--changed='))); + const changed = args[1].some((t) => t === '--changed' || t.startsWith('--changed=')); + const doc = summary ? summaryPlanDocument(changed) : largePlanDocument(changed); const requestId = newRequestId(); plans.set(requestId, doc); return { plan: { ...planSummary(doc), timings: result.plan.timings }, error: null, requestId }; @@ -65,6 +70,8 @@ export function largeProject(client: ApiClient): ApiClient { if (!doc) return target.planRows(scope, requestId, section, offset, limit, name); recordPlanRows(section, offset, limit, name); await new Promise((resolve) => setTimeout(resolve, 120)); + // The browser suite sets `window.__sqbFailPlanRows` to make every read fail, as an expired plan's do. + if ((globalThis as { __sqbFailPlanRows?: boolean }).__sqbFailPlanRows) throw new Error('The plan has expired; plan again.'); return planPage(doc, section, offset, limit, name); }; } diff --git a/ui/src/lib/api/mock/plan-summary-mockup.ts b/ui/src/lib/api/mock/plan-summary-mockup.ts new file mode 100644 index 0000000000..90a4ca40a9 --- /dev/null +++ b/ui/src/lib/api/mock/plan-summary-mockup.ts @@ -0,0 +1,83 @@ +// The plan summary's mock plan (CHI-1563, `?mock&summary`): a `sqb build --changed` plan +// shaped like a large real one: 134 changed, 4 new, 58 renamed, 79 loads, 418 Python, 2,184 unchanged, +// 97 skipped and 83 warnings, and 63 migration steps on models counted in those: 2 model migrations into +// models that exist, 41 column renames, 12 table type changes and 8 retention changes. Each renamed +// model has its own rename migration, as a real plan's does. Pure. + +import type { PlanDocument, PlanModel } from '../types'; +import { DOMAINS, HEAD, incremental, model, name, queryDiff } from './large-plan'; + +/** The mockup's plan; `changed` adds the `--changed` decision (97 skipped). */ +export function summaryPlanDocument(changed: boolean): PlanDocument { + let i = 0; + const models: PlanModel[] = []; + // 134 changed: 120 query, 8 config, 4 through a changed function, 2 schema. + for (let k = 0; k < 120; k++) models.push(model(i++, 'query_changed', { backfill: { action: k % 3 ? 'forward' : 'full', duration: null }, query_diff: queryDiff(i, 1 + ((k * 7) % 60), (k * 5) % 40) })); + for (let k = 0; k < 8; k++) models.push(model(i++, 'config_changed', { config_diff: [...HEAD, '@@ -1,3 +1,3 @@', ' "materialized": "table",', '- "full_refresh": true,', '+ "full_refresh": false,'], config_change: 'full_refresh true → false' })); + for (let k = 0; k < 4; k++) models.push(model(i++, 'function_changed', { changed_functions: ['parse_order_codes'] })); + for (let k = 0; k < 2; k++) models.push(model(i++, 'schema_changed', { schema_findings: [{ kind: 'column_added', column: 'region_code', expected_type: 'varchar', actual_type: null }] })); + const changedFrom = 0; + const changedTo = i; + // 4 first runs. + for (let k = 0; k < 4; k++) models.push(model(i++, 'first_run', k % 2 ? incremental() : {})); + // 58 renamed models, each with its rename migration below. + const renamedFrom = i; + /** What stays at the `k`th renamed model's old name: a view there until a date, for two in three. */ + const oldName = (k: number) => (k % 3 ? { action: 'archive_and_view', view: `analytics.${name(renamedFrom + k)}_old`, reads: `analytics.${name(renamedFrom + k)}`, expires_at: '2026-12-18T00:00:00Z' } : null); + for (let k = 0; k < 58; k++, i++) models.push(model(i, 'renamed', { origin_model: `${name(i)}_old`, old_name: oldName(k) })); + // 2,170 models with no change of their own that still build: incremental runs and tables downstream of a change. + for (let k = 0; k < 2170; k++) models.push(model(i++, k % 4 === 0 ? 'normal_incremental' : 'no_change', k % 4 === 0 ? incremental({ cursor: null }) : {})); + const keys = models.map((m) => `model:${m.name}`); + // The models --changed skips are planned like the others; only the decision leaves them unbuilt. + const skipped = changed ? Array.from({ length: 97 }, (_, k) => model(8000 + k, 'no_change')) : []; + models.push(...skipped); + const seeds = Array.from({ length: 14 }, (_, k) => ({ name: `reference_codes_${k}`, reason: 'no_change' })); + return { + // As sqb plan counts it: models, seeds and functions; loads and Python nodes are counted apart. + selected_count: models.length + seeds.length, + models, + seeds, + source_loads: [ + ...Array.from({ length: 70 }, (_, k) => ({ name: `${DOMAINS[k % DOMAINS.length]}_feed_${k}`, kind: 'source', write_strategy: 'append', cursor_column: 'loaded_at', is_reload: false })), + ...Array.from({ length: 9 }, (_, k) => ({ name: `${DOMAINS[k % DOMAINS.length]}_api_${k}`, kind: 'loader', write_strategy: 'merge', cursor_column: 'updated_at', is_reload: false })) + ], + functions: [], + python_nodes: Array.from({ length: 418 }, (_, k) => ({ name: `${DOMAINS[k % DOMAINS.length]}_export_${k}`, kind: 'asset', phase: 'after_models', identity_status: 'known' })), + warnings: [ + ...Array.from({ length: 71 }, (_, k) => ({ severity: 'warning', message: `Scenario 'test_${DOMAINS[k % 12]}_${k % 24}' provides fixture for unknown model '${name(k + 9000)}'.` })), + ...Array.from({ length: 9 }, (_, k) => ({ severity: 'warning', message: `Model '${name(changedFrom + k)}' reads column 'legacy_code_${k}', which is deprecated.` })), + ...Array.from({ length: 3 }, (_, k) => ({ severity: 'warning', message: `Scenario 'test_returns_${k}' requires source 'raw.returns_${k}', but no fixture was provided. Add a CTE named __source__raw__returns_${k} AS (...).` })) + ], + migrations: [ + ...Array.from({ length: 58 }, (_, k) => ({ + model: name(renamedFrom + k), + origin_model: `${name(renamedFrom + k)}_old`, + decision: 'renamed', + completed_at: null, + is_rename: true, + compatibility: 'not_checked', + transfer: 'rebuild', + old_name: oldName(k) + })), + { model: name(changedTo + 10), origin_model: 'orders__mart__totals_legacy', decision: 'forced_replace', completed_at: null, is_rename: false, compatibility: 'compatible', transfer: null, old_name: null }, + { model: name(changedTo + 11), origin_model: 'customers__mart__summary_v1', decision: 'superseded_replace', completed_at: null, is_rename: false, compatibility: 'compatible', transfer: null, old_name: null } + ], + column_migrations: Array.from({ length: 41 }, (_, k) => ({ model: name(k % 7 === 0 ? changedFrom + k : renamedFrom + 60 + k), origin_column: ['cust_id', 'amt', 'created', 'region', 'sku_code'][k % 5], destination_column: ['customer_id', 'amount', 'created_at', 'region_code', 'product_sku'][k % 5], decision: k % 9 === 4 ? 'record' : 'rename' })), + table_type_conversions: Array.from({ length: 12 }, (_, k) => ({ model: name(changedTo + 100 + k * 3), actual_type: k < 9 ? 'permanent' : 'transient', desired_type: k < 9 ? 'transient' : 'permanent', downgrade: k < 9, irreversible_warning: k < 9 ? 'Time travel beyond 1 day is lost when the table becomes transient.' : null })), + retention: Array.from({ length: 8 }, (_, k) => + k === 7 + ? { scope: 'schema', name: 'analytics_archive', models: [name(changedTo + 300), name(changedTo + 301), name(changedTo + 302)], desired_days: 30, actual_days: 7, direction: 'increase', phase: 'post', irreversible_warning: null } + : { scope: 'table', name: `analytics.${name(changedTo + 200 + k)}`, models: [name(changedTo + 200 + k)], desired_days: k < 5 ? 14 : 1, actual_days: k < 5 ? 1 : 7, direction: k < 5 ? 'increase' : 'decrease', phase: k < 5 ? 'post' : 'pre', irreversible_warning: k < 5 ? null : 'Lowering retention drops history older than 1 day; it cannot be restored.' } + ), + synced_copy: null, + changed: changed + ? { + counts: ['definition'], + reach: 'all', + runs: [...keys, ...seeds.map((s) => `seed:${s.name}`)].map((node, k) => ({ node, reason: k < changedTo ? 'definition changed' : 'no completed record', downstream_of: null })), + held_back: [], + unchanged: skipped.map((m) => `model:${m.name}`) + } + : null + }; +} diff --git a/ui/src/lib/api/types/operate.ts b/ui/src/lib/api/types/operate.ts index 4d3dc595c0..30b4b6fa96 100644 --- a/ui/src/lib/api/types/operate.ts +++ b/ui/src/lib/api/types/operate.ts @@ -669,6 +669,12 @@ export type PlanModel = { /** A config change of at most two plain keys in words, such as `full_refresh true → false`. */ config_change?: string; schema_findings?: { kind: string; column: string; expected_type: string | null; actual_type: string | null }[]; + /** The query changed as well as the name, on a renamed model. */ + query_changed?: boolean; + /** The model a renamed model was renamed from. */ + origin_model?: string; + /** What stays at a renamed model's old name. */ + old_name?: PlanOldName | null; /** A paged row's diff counts in place of its diff text, which `planDiff` reads on demand. */ diff_stats?: { query?: PlanDiffStats; config?: PlanDiffStats }; }; @@ -684,8 +690,63 @@ export type PlanWarning = { severity: string; message: string; model_name?: stri /** One warning template and how many warnings fill it in; `subject` counts the named things it opens with, such as scenarios, when they repeat. */ export type PlanWarningGroup = { template: string; count: number; subject: { noun: string; count: number } | null }; -/** `sqb plan --json` as the location printed it; sections the page doesn't show are kept as they are. */ -export type PlanDocument = { +/** What a build does at a moved model's old name: keep a view there (`view`) reading the new one, until `expires_at`. */ +export type PlanOldName = { action: string; view?: string | null; reads?: string | null; expires_at?: string | null }; + +/** + * A model migration: `model` takes its data from `origin_model` (a rename, or a move into a model that + * exists). `decision` says what this run does; `transfer` how a pending rename moves the data. + */ +export type PlanMigration = { + model: string; + origin_model?: string; + decision: string; + completed_at: string | null; + /** Whether this is a renamed model's own migration, which its Renamed row says, not a migration step. */ + is_rename: boolean; + compatibility?: string; + transfer?: string | null; + old_name?: PlanOldName | null; +}; + +/** A column of `model` renamed from `origin_column` to `destination_column`. */ +export type PlanColumnMigration = { model: string; origin_column: string; destination_column: string; decision: string; completed_at?: string | null }; + +/** A model's table type changing from `actual_type` to `desired_type`. */ +export type PlanTableTypeConversion = { model: string; actual_type?: string | null; desired_type?: string | null; downgrade?: boolean; irreversible_warning?: string | null }; + +/** A time-travel retention change on a relation (`scope` `table` or `schema`), for the models it holds. */ +export type PlanRetention = { + scope?: string; + name?: string | null; + models?: string[]; + desired_days?: number | null; + actual_days?: number | null; + direction?: string; + phase?: string; + irreversible_warning?: string | null; +}; + +/** + * `sqb plan --json` as the location printed it; sections the page doesn't show are kept as they are. A + * server summary has its sections counted and its migration steps with them; only the default mock + * serves a whole plan, without either. + */ +export type PlanDocument = PlanFields & (WholePlan | PlanSummaryCounts); + +/** The default mock's whole plan: every row, nothing counted. */ +type WholePlan = { sections?: undefined; migration_steps?: undefined }; + +/** A server summary's counts. */ +type PlanSummaryCounts = { + /** The nonempty sections in plan order, each with the notes its rows share most. */ + sections: PlanSectionCount[]; + /** The migration steps: migrations other than renamed models' own, column migrations, table type and retention changes. */ + migration_steps: number; +}; + +/** The fields of every plan document. */ +type PlanFields = { selected_count: number; models: PlanModel[]; seeds: { name: string; reason: string; qualified_name?: string }[]; @@ -693,10 +754,10 @@ export type PlanDocument = { functions: { name: string; language: string; reason?: string }[]; python_nodes: { name: string; kind: string; phase: string; identity_status: string }[]; warnings: PlanWarning[]; - migrations: { model: string; origin_model?: string; decision: string; completed_at: string | null }[]; - column_migrations: { model: string; origin_column: string; destination_column: string; decision: string }[]; - table_type_conversions: { model: string; actual_type?: string; desired_type?: string }[]; - retention: { model: string; action?: string }[]; + migrations: PlanMigration[]; + column_migrations: PlanColumnMigration[]; + table_type_conversions: PlanTableTypeConversion[]; + retention: PlanRetention[]; /** What a location's preview planned from instead of the warehouse; `null` for a live plan. */ synced_copy: PlanSyncedCopy | null; /** What `--changed` runs, holds back and skips; `null` without `--changed`. */ @@ -709,8 +770,6 @@ export type PlanDocument = { pages?: Partial>; /** A server summary's warnings by message template, largest first; the warnings section pages them in this order. */ warning_groups?: PlanWarningGroup[]; - /** A server summary's nonempty sections in plan order, each with the notes its rows share most. */ - sections?: PlanSectionCount[]; /** Rows of each paged section loaded so far, which is the offset of its next page. */ loaded?: Partial>; }; diff --git a/ui/src/lib/run/plan-browser.svelte b/ui/src/lib/run/plan-browser.svelte index 5f7b0a4583..1db3803536 100644 --- a/ui/src/lib/run/plan-browser.svelte +++ b/ui/src/lib/run/plan-browser.svelte @@ -8,13 +8,19 @@ import { to } from '$lib/paths'; import { mergePlanPage, type PlanDiff, type PlanDocument, type PlanPage, type PlanPageSection, type PlanWarning } from '$lib/api'; import { warningText } from './plan-text'; - import { AFTER_DIVIDER, WARNINGS_SHOWN, diffPreview, planGroups, planSummaryLine, selectedGroup, withoutRows, type PlanGroupKey, type PlanRow, type WarningGroup } from './plan-index'; + import { plural } from '$lib/format/plural'; + import { planSectionCounts, planTotals } from './plan-totals'; + import { migrationSteps } from './plan-steps'; + import { WARNINGS_SHOWN, diffPreview, planGroups, selectedGroup, withoutRows, type PlanGroupKey, type PlanRow, type WarningGroup } from './plan-index'; + import PlanRenames from './plan-renames.svelte'; /** * `section` is the group shown, kept in the address by the page; `more` adds the next page of a * section to `plan`, of `limit` rows; `load` reads a page of rows whose name contains `name`, for the * filter; `diff` reads a model's diff, null when the server has none to give; `refresh` words what * refreshes a held-back job's nodes. + * The index groups the sections under Will run and Won't run, each heading the nodes counted once + * (plan-totals.ts), with warnings apart. */ let { plan, @@ -55,6 +61,8 @@ const groups = $derived(planGroups(plan, refresh, diffs)); const group = $derived(selectedGroup(groups, section)); + /** The Renames and migrations pane lists renamed models and migration steps apart, and loads its own sections. */ + const renamesPane = $derived(group?.key === 'renames'); const warningGroups = $derived(group?.warningGroups ?? []); /** A server summary's groups come counted; each one's warnings are read when it opens, by where they start. */ let readWarnings = $state>({}); @@ -90,7 +98,7 @@ }) .catch((e: unknown) => { if (read === readWarnings) readingWarnings.delete(w.offset); - loadError = e instanceof Error ? e.message : String(e); + if (read === readWarnings) loadErrors.warnings = e instanceof Error ? e.message : String(e); }); } function toggleWarnings(w: WarningGroup) { @@ -109,11 +117,26 @@ let askedOf = $state>({}); let openDiff = $state>({}); let loading = $state(false); - let loadError = $state(null); + /** Failed reads, by group: shown there with Retry, and that group reads nothing more until then. */ + let loadErrors = $state>({}); + const failed = (e: unknown) => (e instanceof Error ? e.message : String(e)); + // Another plan (a re-plan) starts with no failed reads: its rows are read from scratch. + let erroredFor: unknown = null; + $effect.pre(() => { + if (plan.sections === erroredFor) return; + erroredFor = plan.sections; + loadErrors = {}; + }); + /** Reads a group's rows again after a failure; a filter's view starts over. */ + function retry(key: string) { + delete loadErrors[key]; + if (view?.key === key) view = null; + if (key === 'warnings') for (const w of warningGroups) if (openWarnings[w.offset]) readGroup(w, allWarnings[w.offset] ? WARNINGS_READ : WARNINGS_SHOWN); + } const shown = $derived(group ? (shownOf[group.key] ?? PAGE) : PAGE); const filter = $derived(group ? (filterOf[group.key] ?? '') : ''); - /** The server finds rows by name, a page at a time; a whole plan, with every row, is filtered here. */ + /** The server finds rows by name, a page at a time; the default mock's whole plan, with every row, is filtered here. */ const serverFilter = $derived(!!plan.sections && !!load); const asked = $derived(group ? (askedOf[group.key] ?? '') : ''); @@ -135,10 +158,9 @@ */ type View = { key: string; name: string; doc: PlanDocument; matches: Partial>; sections: PlanDocument['sections'] }; let view = $state.raw(null); - const viewDoc = (matches: View['matches'], doc: PlanDocument): PlanDocument => ({ - ...doc, - sections: (plan.sections ?? []).map((c) => ({ section: c.section, total: matches[c.section] ?? 0, notes: [] })) - }); + /** Only a server summary is filtered by the server, so a view's document is always a summary. */ + const viewDoc = (matches: View['matches'], doc: PlanDocument): PlanDocument => + doc.sections ? { ...doc, sections: (plan.sections ?? []).map((c) => ({ section: c.section, total: matches[c.section] ?? 0, notes: [] })) } : doc; // A new filter starts its view: the first page of each of the group's sections, for how many each matches. $effect(() => { @@ -149,7 +171,7 @@ const start: View = { key: g.key, name, doc: viewDoc({}, withoutRows(plan)), matches: {}, sections: plan.sections }; view = start; loading = true; - loadError = null; + delete loadErrors[g.key]; (async () => { let current = start; for (const s of g.sections) { @@ -160,7 +182,7 @@ view = current; } })() - .catch((e: unknown) => (loadError = e instanceof Error ? e.message : String(e))) + .catch((e: unknown) => (loadErrors[g.key] = failed(e))) .finally(() => (loading = false)); }); @@ -182,7 +204,8 @@ $effect(() => { const g = shownGroup; const page = nextPage; - if (!g || !page || loading || g.key === 'warnings' || g.rows.length >= Math.min(shown, g.count)) return; + // After a failed read nothing more is read until Retry, so a lasting failure doesn't repeat its request. + if (!g || !page || loading || loadErrors[g.key] || renamesPane || g.key === 'warnings' || g.rows.length >= Math.min(shown, g.count)) return; if (filtering && !serverFilter) return; const current = view; const step = filtering @@ -195,8 +218,7 @@ : (more?.(page.section, PAGE) ?? null); if (!step) return; loading = true; - loadError = null; - step.catch((e: unknown) => (loadError = e instanceof Error ? e.message : String(e))).finally(() => (loading = false)); + step.catch((e: unknown) => (loadErrors[g.key] = failed(e))).finally(() => (loading = false)); }); function showMore() { @@ -212,29 +234,71 @@ if (r.fetch === 'diff') readDiff(r.name); } } - /** The index's divider goes before the first quiet group, unless nothing comes before it. */ - const dividerBefore = $derived(groups.findIndex((g) => g.key === AFTER_DIVIDER || g.key === 'unchanged' || g.key === 'skipped')); const href = (r: PlanRow) => (r.linked && !/^\d+$/.test(r.name) ? to(`/catalog/model:${r.name}`) : null); const remaining = $derived(rowCount - visible.length); + + // The index under Will run and Won't run, each entry counting nodes once (plan-totals.ts), warnings apart. + const counts = $derived(planSectionCounts(plan)); + const totals = $derived(planTotals(counts, migrationSteps(plan))); + const WONT_GROUPS = new Set(['held_back', 'skipped']); + const FUNCTION_SECTIONS = new Set(['functions', 'changed_functions']); + const runGroups = $derived(groups.filter((g) => !WONT_GROUPS.has(g.key) && g.key !== 'warnings')); + const wontGroups = $derived(groups.filter((g) => WONT_GROUPS.has(g.key))); + const warningGroup = $derived(groups.find((g) => g.key === 'warnings')); + /** A group's nodes, as its heading counts them; warnings count themselves. */ + const nodesOf = (g: (typeof groups)[number]) => [...totals.runParts, ...totals.wontParts].filter((p) => p.group === g.key).reduce((n, p) => n + p.count, 0); + /** A group's functions, which are listed but are not nodes the build runs. */ + const functionsOf = (g: (typeof groups)[number]) => counts.filter((c) => FUNCTION_SECTIONS.has(c.section) && g.sections.includes(c.section)).reduce((n, c) => n + c.total, 0); + /** + * What an index entry counts: its nodes, or when it has none, muted, what it lists instead: the + * Renames group's migration steps, or a group's functions. + */ + function entryCount(g: (typeof groups)[number]): { text: string; muted: boolean } { + if (g.key === 'warnings') return { text: g.count.toLocaleString(), muted: false }; + const nodes = nodesOf(g); + if (nodes) return { text: nodes.toLocaleString(), muted: false }; + if (g.key === 'renames' && totals.steps) return { text: plural(totals.steps, 'step'), muted: true }; + const functions = functionsOf(g); + return { text: functions ? plural(functions, 'function') : '', muted: true }; + } + /** A pane's count: its nodes, and its functions after them, muted. */ + const paneCount = (g: (typeof groups)[number]) => ({ nodes: nodesOf(g), functions: functionsOf(g) }); +{#snippet entry(g: (typeof groups)[number], title = g.title)} + {@const selected = g.key === group?.key} + {@const count = entryCount(g)} + + +{/snippet} +
{#if group}
+ {#if renamesPane} + + {:else} + {@const c = group.key === 'warnings' ? null : paneCount(group)}
-

{group.title} {group.count.toLocaleString()}

+

{group.title} {#if c}{#if c.nodes || !c.functions}{plural(c.nodes, 'node')}{/if}{#if c.functions}{c.nodes ? ' · ' : ''}{plural(c.functions, 'function')}{/if}{:else}{group.count.toLocaleString()}{/if}

{#if group.subtitle}

{group.subtitle}

{/if}
@@ -260,6 +324,7 @@ {/if}
{/each} + {#if loadErrors.warnings}

Couldn't read the warnings: {loadErrors.warnings}

{/if} {:else}