From 999916e8581cb7435069011eacfe31d190b34c1b Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 20 Nov 2026 16:33:01 +0000 Subject: [PATCH 1/2] feat(loader): create a new staged Python table at its first commit A Python table planned from its data was created on its first batch, and columns that first held values on a later page were added after the _sqlload_* columns. A table whose rows load through a staging table (merge, delete_insert, and an append with checks, logged state or child tables) is now created at its first commit, with every column seen by then and SQLLoad's columns last; its staging table still grows as columns arrive. A first load that fails before its first commit leaves no table behind. Appends written straight into the table and replace builds still create it at the first batch. A column first seen after the table exists is still appended; the docs say so, and that --reload rebuilds it in order. --- .../src/loader/_helpers/destination.rs | 23 ++++++ crates/sqlload/src/loader/_helpers/publish.rs | 37 ++++++--- crates/sqlload/src/loader/_helpers/python.rs | 47 +++++------ .../sqlload/src/loader/_helpers/table_plan.rs | 16 ++++ crates/sqlload/src/loader/models.rs | 4 + packages/sqlload/tests/test_batched_python.py | 81 +++++++++++++++++++ packages/sqlload/tests/test_python_sources.py | 4 +- .../content/docs/docs/concepts/projects.mdx | 9 +++ 8 files changed, 187 insertions(+), 34 deletions(-) diff --git a/crates/sqlload/src/loader/_helpers/destination.rs b/crates/sqlload/src/loader/_helpers/destination.rs index 5deb3fa3f..391a98c53 100644 --- a/crates/sqlload/src/loader/_helpers/destination.rs +++ b/crates/sqlload/src/loader/_helpers/destination.rs @@ -54,6 +54,7 @@ use crate::loader::_helpers::publish::{staged_columns, write_target}; use crate::loader::_helpers::resolve::display_name; use crate::loader::_helpers::sources::{ReadPart, read_source}; use crate::loader::_helpers::table_plan::blocked_plan; +use crate::loader::_helpers::table_plan::{change_lines, schema_statements}; use crate::loader::constants::{CATALOG_CONFLICT, CATALOG_CONFLICT_TRIES, COALESCED_BYTES}; use crate::loader::errors::LoaderError; use crate::loader::models::SourceReader; @@ -662,6 +663,28 @@ pub(crate) async fn replace_files( } impl Publisher { + /// The lines not yet reported: the table's own `created ...` (once [created at its first commit](Self::create_table)), then the `created ...`, `altered ...` and `loaded ...` lines of the child tables filled so far. + pub(crate) fn take_changes(&mut self) -> Vec { + let mut changes = std::mem::take(&mut self.changes); + if let Some(children) = self.children.as_mut() { + changes.append(&mut children.changes); + } + changes + } + + /// Create a [new table](Publisher::new_table) before its first commit writes to it (or at the end of a load that wrote none), with the columns planned by then and its `created ...` line kept for the load to report; nothing once it exists. + pub(in crate::loader) async fn create_table(&mut self) -> Result<(), LoaderError> { + if !self.new_table || self.table.schema.action != TableAction::Create { + return Ok(()); + } + if let Some(statements) = schema_statements(&self.table) { + execute_ddl(&self.destination, statements).await?; + self.changes.extend(change_lines(&self.table)); + } + self.table.schema.action = TableAction::Unchanged; + Ok(()) + } + /// Add a batch to the commit's files, starting them (and marking the commit as writing) on its first batch. pub(crate) async fn stage_files(&mut self, batch: RecordBatch) -> Result<(), LoaderError> { let writer = match self.files.as_mut().and_then(|sink| sink.writer.take()) { diff --git a/crates/sqlload/src/loader/_helpers/publish.rs b/crates/sqlload/src/loader/_helpers/publish.rs index 36600478f..90caa6140 100644 --- a/crates/sqlload/src/loader/_helpers/publish.rs +++ b/crates/sqlload/src/loader/_helpers/publish.rs @@ -147,6 +147,7 @@ use crate::schema_plan::constants::{ use crate::schema_plan::models::ColumnOrigin; use crate::schema_plan::models::DesiredColumn; use crate::schema_plan::models::LiveColumn; +use crate::schema_plan::models::TableAction; use crate::snowflake_source::main::select_query::select_query; use crate::table_lease::main::lease_now::lease_now; use crate::table_lease::main::take_lease::take_lease; @@ -177,6 +178,25 @@ fn writes_directly(table: &TablePlan, writer: &Writer) -> bool { && (!table.route.as_ref().is_some_and(Route::pulls_unloaded) || unloads_directly(writer)) } +/// Whether `table` is a table planned from its data (a Python table, or one of its child tables) that its load creates in a SQL destination, and whose rows load through a staging table anyway: a merge, a `delete_insert`, or an append that stages (logged state, checks, child tables, or a merge parent's child table). It is created at its first commit rather than its first batch, so the columns its later batches bring come before the `_sqlload_*` metadata. Other appends (including ones a warehouse writer stages internally) and `replace` (`--reload`) builds keep creating it at the first batch. +pub(crate) fn created_at_commit(table: &TablePlan) -> bool { + let spec = &table.spec; + let stages = match table.mode { + WriteMode::Merge { .. } | WriteMode::DeleteInsert { .. } => true, + WriteMode::Append => { + table.logged.is_some() + || spec.checks.is_some() + || spec.unnest != Unnest::None + || spec.root_rows + } + _ => false, + }; + stages + && table.schema.action == TableAction::Create + && spec.python.is_some() + && table.output.is_none() +} + /// Where the batches go, from the publisher's fields: `staging` when there is one, and `columns` as the staged batches carry them. pub(in crate::loader) fn write_target<'a>( table: &'a TablePlan, @@ -547,6 +567,7 @@ impl Publisher { } _ => None, }; + let new_table = created_at_commit(&table); let direct = files.is_none() && table.spec.checks.is_none() && table.spec.unnest == Unnest::None @@ -626,6 +647,8 @@ impl Publisher { at_end, builds, indexes: shape.map(|shape| shape.indexes).unwrap_or_default(), + new_table, + changes: Vec::new(), table, destination, run, @@ -792,14 +815,6 @@ impl Publisher { } } - /// The `created ...`, `altered ...` and `loaded ...` lines of the child tables filled so far. - pub(crate) fn child_changes(&mut self) -> Vec { - self.children - .as_mut() - .map(|children| std::mem::take(&mut children.changes)) - .unwrap_or_default() - } - /// Whether the batch holds about 1 GB (Parquet staged on Snowflake, with the zstd files `send = "at_commit"` holds for it, which come within a few percent of that Parquet, Arrow elsewhere), where a size-cut table may end it. pub(crate) fn full(&self) -> bool { let bytes = match (self.writer.staged_bytes(), &self.spool) { @@ -853,6 +868,9 @@ impl Publisher { || matches!(self.table.mode, WriteMode::MarkMissing(_)) || self.leftover_deletes() || self.pieces.unfinished; + if !pending { + self.create_table().await?; + } if let Some(children) = self.children.as_mut().filter(|_| self.direct || !pending) { Box::pin(children.finish( (&self.table, (&self.load_id, &self.loaded_at)), @@ -1261,6 +1279,7 @@ impl Publisher { } _ => None, }; + self.create_table().await?; if let Some(children) = &mut self.children { match last { true => { @@ -1574,7 +1593,7 @@ pub(crate) async fn publish_table( }; Ok(TableLoaded { rows, - changes: publisher.child_changes(), + changes: publisher.take_changes(), note: marked(table, (rows, publisher.marked)) .or_else(|| older_note(table, publisher.older)), after_load: report, diff --git a/crates/sqlload/src/loader/_helpers/python.rs b/crates/sqlload/src/loader/_helpers/python.rs index 88f49784f..764eca9dd 100644 --- a/crates/sqlload/src/loader/_helpers/python.rs +++ b/crates/sqlload/src/loader/_helpers/python.rs @@ -34,6 +34,7 @@ use crate::load_naming::main::metadata_name::metadata_name; use crate::load_naming::main::name_lines::name_lines; use crate::load_naming::main::named_batch::named_batch; use crate::load_naming::models::{Compat, LoadedNames, MetadataRole, NameMemo, RenameColumn}; +use crate::loader::_helpers::publish::created_at_commit; use crate::loader::_helpers::publish::replaces_roots; use crate::loader::_helpers::publish::staged_columns; use crate::loader::_helpers::resolve::{KnownNames, host_rename, resolve_known}; @@ -41,6 +42,7 @@ use crate::loader::_helpers::run_log::{log_empty_run, record_numbered, run_entry use crate::loader::_helpers::specs::is_child_column; use crate::loader::_helpers::specs::null_cursor_problem; use crate::loader::_helpers::table_plan::append_problem; +use crate::loader::_helpers::table_plan::change_lines; use crate::loader::_helpers::table_plan::replan_columns; use crate::loader::_helpers::table_plan::schema_statements; use crate::loader::constants::DROPPED_CHUNK; @@ -498,22 +500,6 @@ pub(crate) fn cursor_type_name(table: &TablePlan, column: &str) -> String { } } -/// The schema changes the load made, one line each, as `apply` would print them. -pub(crate) fn change_lines(table: &TablePlan) -> Vec { - let name = sql_adapter(table.spec.warehouse).render_display_name(&table.spec.destination); - match &table.schema.action { - TableAction::Create => vec![format!( - "created {name} {} columns", - table.schema.write_columns.len() - )], - TableAction::Alter(changes) => changes - .iter() - .map(|change| format!("altered {name} {change}")) - .collect(), - TableAction::Recreate | TableAction::Unchanged => Vec::new(), - } -} - /// Fail when writing no rows would empty a table that has rows: `replace` or `--reload` without `allow_empty` (an empty result must not wipe a table by accident). pub(crate) async fn refuse_emptying( table: &TablePlan, @@ -827,9 +813,9 @@ impl PythonFeed { let state = self.state.clone(); let publisher = self.publisher(load).await?; let rows = publisher.finish(state).await?; - let children = publisher.child_changes(); + let changes = publisher.take_changes(); let replaced_keys = publisher.replaced.clone(); - self.changes.extend(children); + self.changes.extend(changes); Box::pin(record_numbered(&self.table, load.destination)).await?; Ok(TableLoaded { rows, @@ -975,7 +961,7 @@ impl PythonFeed { } } - /// Plan the table for the data seen so far and bring the table (and the staging table) in line: created on the first batch, altered as later batches add or widen columns; columns that only held NULL wait for a typed value, or become text at the end (`settle`). + /// Plan the table for the data seen so far and bring the table (and the staging table) in line: created at its first commit with the columns seen by then (see [`created_at_commit`]), altered as later batches add or widen columns; columns that only held NULL wait for a typed value, or become text at the end (`settle`). pub(crate) async fn plan_data( &mut self, load: &PythonLoad<'_>, @@ -1020,6 +1006,9 @@ impl PythonFeed { .is_none() .then_some(self.table.live.as_deref()); let replanned = replan(load, &self.table, (&data, known, &self.names), destination).await?; + if let Some(publisher) = self.publisher.as_mut() { + self.changes.append(&mut publisher.changes); + } let old = staged_columns(&self.table); let changes = bring_in_line( (load.destination, self.publisher.as_mut()), @@ -1288,7 +1277,7 @@ fn table_read( } } -/// Bring a table planned from its data in line with `replanned`, through `publisher`'s connection once it is open (`destination` before): its schema changes run, and an open publisher's rows so far are settled and its staging table altered from the `old` staged columns, before it takes the new plan. Returns the `created ...` and `altered ...` lines. +/// Bring a table planned from its data in line with `replanned`, through `publisher`'s connection once it is open (`destination` before): its schema changes run (a table the load creates waits for its first commit, see [`created_at_commit`]), and an open publisher's rows so far are settled and its staging table altered from the `old` staged columns, before it takes the new plan. Returns the `created ...` and `altered ...` lines. pub(crate) async fn bring_in_line( (destination, publisher): (&Destination, Option<&mut Publisher>), (old, replanned): (&[DesiredColumn], &TablePlan), @@ -1298,7 +1287,13 @@ pub(crate) async fn bring_in_line( Some(publisher) => &publisher.destination, None => destination, }; - if let Some(statements) = schema_statements(replanned) { + if created_at_commit(replanned) { + if publisher.is_none() { + let adapter = sql_adapter(replanned.spec.warehouse); + let schema = adapter.render_create_schema(&replanned.spec.destination); + execute_ddl(destination, vec![schema]).await?; + } + } else if let Some(statements) = schema_statements(replanned) { execute_ddl(destination, statements).await?; changes.extend(change_lines(replanned)); } @@ -1592,6 +1587,7 @@ impl ChildTables { .filter_map(|feed| feed.publisher.as_mut()) { publisher.cut(CommitRange::Planned).await?; + self.changes.append(&mut publisher.changes); } Ok(()) } @@ -1657,6 +1653,7 @@ impl ChildTables { } if let Some(publisher) = &mut feed.publisher { publisher.finish(None).await?; + self.changes.append(&mut publisher.changes); } } Ok(()) @@ -1983,9 +1980,13 @@ impl ChildFeed { schema: columns.schema, ..self.table.clone() }; + let mut changes = (self.publisher.as_mut()) + .map(|publisher| std::mem::take(&mut publisher.changes)) + .unwrap_or_default(); let old = staged_columns(&self.table); - let changes = - bring_in_line((destination, self.publisher.as_mut()), (&old, &replanned)).await?; + changes.extend( + bring_in_line((destination, self.publisher.as_mut()), (&old, &replanned)).await?, + ); self.table = TablePlan { live: replanned.live.clone().or(self.table.live.clone()), ..replanned diff --git a/crates/sqlload/src/loader/_helpers/table_plan.rs b/crates/sqlload/src/loader/_helpers/table_plan.rs index aa148e24e..f4fb33165 100644 --- a/crates/sqlload/src/loader/_helpers/table_plan.rs +++ b/crates/sqlload/src/loader/_helpers/table_plan.rs @@ -897,6 +897,22 @@ async fn plan_table_once( }) } +/// The schema changes the load made, one line each, as `apply` would print them. +pub(crate) fn change_lines(table: &TablePlan) -> Vec { + let name = sql_adapter(table.spec.warehouse).render_display_name(&table.spec.destination); + match &table.schema.action { + TableAction::Create => vec![format!( + "created {name} {} columns", + table.schema.write_columns.len() + )], + TableAction::Alter(changes) => changes + .iter() + .map(|change| format!("altered {name} {change}")) + .collect(), + TableAction::Recreate | TableAction::Unchanged => Vec::new(), + } +} + /// The statements that bring the table's schema in line with its plan, a created or rebuilt table getting its description as its comment; `None` when there is nothing to run: unchanged, rebuilt by the load itself, or a files target. pub(crate) fn schema_statements(table: &TablePlan) -> Option> { let spec = &table.spec; diff --git a/crates/sqlload/src/loader/models.rs b/crates/sqlload/src/loader/models.rs index ab7f0c6a8..3b6a0aeed 100644 --- a/crates/sqlload/src/loader/models.rs +++ b/crates/sqlload/src/loader/models.rs @@ -1142,6 +1142,10 @@ pub struct Publisher { pub builds: bool, /// The indexes to re-create on the build table once it is swapped in. pub indexes: Vec, + /// A table planned from its data that this load creates (see [`created_at_commit`](crate::loader::_helpers::publish::created_at_commit)): it is created at the first commit with every column seen by then. + pub new_table: bool, + /// The `created ...` line of a table created at its first commit, until its load reports it. + pub changes: Vec, pub staging: Option, /// The batch being filled, from 1. pub batch: u64, diff --git a/packages/sqlload/tests/test_batched_python.py b/packages/sqlload/tests/test_batched_python.py index 0928e0781..6cdc50224 100644 --- a/packages/sqlload/tests/test_batched_python.py +++ b/packages/sqlload/tests/test_batched_python.py @@ -188,6 +188,22 @@ def growing(ctx): yield pa.table({"id": [3], "b": [2], "c": [1]}) +def late(ctx): + for page in range(1, 9): + rows = [{"id": page * 10 + n, "a": page, "c": page if page >= 7 else None} for n in range(3)] + yield pa.Table.from_pylist([{**row, "b": page} if page >= 3 else row for row in rows]) + + +for strategy, keys in [("merge", {"unique_key": ["id"]}), ("delete_insert", {"replace_key": ["id"]}), ("append", {"checks": {"not_null": ["id"]}}), ("append_direct", {})]: + api.table(description="Test table.", name=f"late_{strategy}", write_strategy=strategy.removesuffix("_direct"), **keys)(late) + + +@api.table(description="Test table.", write_strategy="append", cursor_column="id", cursor_start="0", batch_size=3) +def late_windowed(ctx): + for i in range(max(int(ctx.cursor_value), 1), min(int(ctx.cursor_end), 7)): + yield {"id": i, "late": i} if i >= 4 else {"id": i} + + def wait_for(name): deadline = time.monotonic() + 20 while not (HERE / "data" / name).exists(): @@ -442,6 +458,36 @@ def test_given_append_without_a_cursor_when_a_rerun_follows_a_failure_then_no_ro assert sorted(rows_loaded(project, "appended")) == sorted(EVERY_ROW) +def column_names(project: Path, table: str) -> list[str]: + return [name for (name,) in query(project, f"SELECT column_name FROM information_schema.columns WHERE table_name = 'api__{table}' ORDER BY ordinal_position")] + + +@pytest.mark.parametrize("strategy", ["merge", "delete_insert", "append"]) +def test_given_a_new_staged_table_whose_columns_first_hold_values_on_later_pages_when_loading_in_one_commit_then_the_metadata_columns_come_last( + project: Path, strategy: str +) -> None: + done = sqlload(project, "load", f"api__late_{strategy}") + assert done.returncode == 0, done.stdout + done.stderr + assert column_names(project, f"late_{strategy}") == ["id", "a", "c", "b", "_sqlload_load_id", "_sqlload_loaded_at"] + assert query(project, f'SELECT count(*), count(b), count(c) FROM "RAW"."api__late_{strategy}"') == [(24, 18, 6)] + assert f"created RAW.api__late_{strategy} 6 columns" in done.stdout, done.stdout + assert "altered" not in done.stdout, done.stdout + + +def test_given_an_append_written_straight_into_its_new_table_when_columns_first_hold_values_later_then_they_follow_the_metadata(project: Path) -> None: + done = sqlload(project, "load", "api__late_append_direct") + assert done.returncode == 0, done.stdout + done.stderr + assert column_names(project, "late_append_direct") == ["id", "a", "_sqlload_load_id", "_sqlload_loaded_at", "b", "c"] + assert query(project, 'SELECT count(*), count(b), count(c) FROM "RAW"."api__late_append_direct"') == [(24, 18, 6)] + + +def test_given_a_windowed_first_load_when_a_column_first_shows_after_the_first_commit_then_it_is_appended_after_the_metadata(project: Path) -> None: + done = sqlload(project, "load", "--end-cursor", "7", "api__late_windowed") + assert done.returncode == 0, done.stdout + done.stderr + assert column_names(project, "late_windowed") == ["id", "_sqlload_load_id", "_sqlload_loaded_at", "late"] + assert query(project, 'SELECT id, late FROM "RAW"."api__late_windowed" ORDER BY id') == [(1, None), (2, None), (3, None), (4, 4), (5, 5), (6, 6)] + + def test_given_columns_added_over_batches_when_loading_then_each_change_reads_the_live_columns_once(project: Path) -> None: done = sqlload(project, "load", "api__growing", env={"RUST_LOG": "sqlload=debug"}) assert done.returncode == 0, done.stdout + done.stderr @@ -593,6 +639,41 @@ def orders(ctx): """ +STAGED_UNNEST_MODULE = """ +import sqlload as sl + +api = sl.source("api") + + +@api.table(description="Test table.", write_strategy="merge", unique_key=["id"], unnest=["items"]) +def merged(ctx): + yield [{"id": 1, "items": [{"sku": "a"}]}] + yield [{"id": 2, "items": [{"sku": "b"}], "late": 2}] + + +@api.table(description="Test table.", write_strategy="delete_insert", cursor_column="id", cursor_start="0", unnest=["items"]) +def replaced(ctx): + yield [{"id": 1, "items": [{"sku": "a"}]}] + yield [{"id": 2, "items": [{"sku": "b"}], "late": 2}] +""" + + +@pytest.mark.parametrize("table", ["merged", "replaced"]) +def test_given_a_staged_parent_dropped_while_its_child_table_remains_when_loading_again_then_the_parent_is_created_with_its_metadata_last( + project: Path, table: str +) -> None: + (project / "sources" / "api.py").write_text(STAGED_UNNEST_MODULE) + first = sqlload(project, "load", f"api__{table}") + assert first.returncode == 0, first.stdout + first.stderr + with duckdb.connect(str(project / "wh.duckdb")) as connection: + connection.execute(f'DROP TABLE "RAW".api__{table}') + done = sqlload(project, "load", f"api__{table}") + assert done.returncode == 0, done.stdout + done.stderr + assert column_names(project, table) == ["id", "late", "_sqlload_load_id", "_sqlload_loaded_at", "_sqlload_row_id"] + linked = query(project, f'SELECT o.id, i.sku FROM "RAW".api__{table} o JOIN "RAW".api__{table}__items i ON i._sqlload_parent_id = o._sqlload_row_id ORDER BY o.id') + assert linked == [(1, "a"), (2, "b")] + + def test_given_unnest_when_loading_a_python_table_then_its_lists_become_child_tables_linked_by_id(project: Path) -> None: (project / "sources" / "api.py").write_text(UNNEST_MODULE) done = sqlload(project, "load", "api__orders") diff --git a/packages/sqlload/tests/test_python_sources.py b/packages/sqlload/tests/test_python_sources.py index d24502602..4cd67aaba 100644 --- a/packages/sqlload/tests/test_python_sources.py +++ b/packages/sqlload/tests/test_python_sources.py @@ -1616,7 +1616,7 @@ def test_given_a_python_table_repeating_a_merge_key_when_it_must_fail_then_the_l done = sqlload(project, "load") assert done.returncode == 1, done.stdout + done.stderr assert "the loaded rows repeat its unique_key: 1 value of (id) appears in more than one row, such as (1)" in done.stdout + done.stderr, done.stdout + done.stderr - assert query(project, 'SELECT count(*) FROM "RAW"."shop__rows"') == [(0,)] + assert query(project, "SELECT count(*) FROM information_schema.tables WHERE table_name = 'shop__rows'") == [(0,)] @pytest.mark.parametrize( @@ -1673,7 +1673,7 @@ def test_given_source_level_dedupe_settings_when_loading_then_only_its_merge_tab assert "the loaded rows repeat its unique_key" in done.stdout + done.stderr, done.stdout + done.stderr assert query(project, 'SELECT count(*) FROM "RAW"."shop__log"') == [(3,)] assert query(project, 'SELECT id, v FROM "RAW"."shop__kept"') == [(1, "c")] - assert query(project, 'SELECT count(*) FROM "RAW"."shop__strict"') == [(0,)] + assert query(project, "SELECT count(*) FROM information_schema.tables WHERE table_name = 'shop__strict'") == [(0,)] def test_given_a_source_that_fails_on_repeated_keys_when_a_table_sets_newest_by_then_that_table_keeps_its_newest_row(tmp_path: Path) -> None: diff --git a/website/src/content/docs/docs/concepts/projects.mdx b/website/src/content/docs/docs/concepts/projects.mdx index d09f7818f..0f82a0eed 100644 --- a/website/src/content/docs/docs/concepts/projects.mdx +++ b/website/src/content/docs/docs/concepts/projects.mdx @@ -131,6 +131,15 @@ Every loaded table gets columns that say where each row came from: | `_sqlload_source_row` | files sources | The row's number in its file, from 1. BigQuery doesn't report a row's position in its file, so a BigQuery table that writes it has SQLLoad read the files; [exclude it](/docs/targets/bigquery/#parquet-in-google-cloud-storage) to have BigQuery load them itself. | | `_sqlload_source_row` | Google Sheets tables | The row's number in the sheet, as Google Sheets numbers it: a range starting at `A3` starts at 3, and a skipped empty row leaves a gap. Not on a table with a `transform`. | +They come after the table's own columns when SQLLoad creates the table. A Python table planned +from its data is created at its first commit, with every column its rows brought by then, when its +rows load through a staging table: `merge`, `delete_insert`, and an `append` with checks, logged +state or child tables. An `append` written straight into the table, and a `replace`, create it at +the first batch. A column first seen after the table exists, such as on a later page of a direct +`append`, in a later `batch_size` window, after a commit of about 1 GB, or on a later run, is added +after SQLLoad's columns: column order follows the order columns arrived in, and +`sqlload load --reload` rebuilds the table with SQLLoad's columns last. + `plan` lists them as `+ _sqlload_load_id, _loaded_at`. Files sources use `_sqlload_source_file` to load each file once under `append`. SQLLoad's own columns never count as differences in [`sqlload contract`](/docs/cli/contract/). From 582ea23a8a064d668f67576bc796ebfa9583105c Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 20 Nov 2026 17:22:45 +0000 Subject: [PATCH 2/2] feat(project): rename a source's schema key to destination_schema A source's schema key set where its tables land, while include patterns and {schema} in table_name name the schema they are read from. The key is now destination_schema, on TOML sources and on sl.source(...), and copy --print-toml writes it. plan refuses schema on a source, naming the new key and include; sl.source(schema=...) fails the same way. {schema} in table_name and in files paths, include and exclude, and the target's loader_schema are unchanged. BREAKING CHANGE: a source's schema key is now destination_schema, in sqlload.toml and in sl.source(...). A source that still sets schema fails plan with a message naming the new key. --- crates/sqlload/src/copy/_helpers/project.rs | 2 +- crates/sqlload/src/loader/_helpers/resolve.rs | 2 +- .../sqlload/src/project/_helpers/targets.rs | 4 ++-- .../src/project/_helpers/unknown_keys.rs | 14 +++++++------ crates/sqlload/src/project/constants.rs | 7 +++++-- crates/sqlload/src/project/models.rs | 4 ++-- .../src/project/tests/test_parse_project.rs | 11 +++++++--- .../tests/test_parse_python_project.rs | 4 ++-- .../loader/test_postgres_to_clickhouse.rs | 2 +- .../tests/loader/test_snowflake_routes.rs | 2 +- packages/sqlload/python/sqlload/_declare.py | 4 +++- packages/sqlload/tests/test_batched_python.py | 13 ++++++++++-- packages/sqlload/tests/test_cli.py | 21 +++++++++++++++++++ website/src/content/docs/docs/cli/copy.mdx | 2 +- .../content/docs/docs/concepts/projects.mdx | 2 +- .../src/content/docs/docs/configuration.mdx | 8 +++---- .../src/content/docs/docs/guides/from-dlt.mdx | 2 +- .../src/content/docs/docs/sources/odbc.mdx | 3 ++- .../src/content/docs/docs/sources/python.mdx | 4 ++-- .../content/docs/docs/targets/bigquery.mdx | 2 +- .../content/docs/docs/targets/clickhouse.mdx | 4 ++-- .../src/content/docs/docs/targets/index.mdx | 4 ++-- .../content/docs/docs/targets/postgres.mdx | 2 +- 23 files changed, 83 insertions(+), 40 deletions(-) diff --git a/crates/sqlload/src/copy/_helpers/project.rs b/crates/sqlload/src/copy/_helpers/project.rs index 9fa555320..0f5b50dc4 100644 --- a/crates/sqlload/src/copy/_helpers/project.rs +++ b/crates/sqlload/src/copy/_helpers/project.rs @@ -172,7 +172,7 @@ fn source_settings( "description".to_owned(), Value::String(format!("Copied from {}.", source.described)), ); - settings.extend(schema.map(|schema| ("schema".to_owned(), Value::String(schema)))); + settings.extend(schema.map(|schema| ("destination_schema".to_owned(), Value::String(schema)))); settings.extend(table_settings(request, !FILE_KINDS.contains(&kind(source)))); match kind(source) { "postgres" => { diff --git a/crates/sqlload/src/loader/_helpers/resolve.rs b/crates/sqlload/src/loader/_helpers/resolve.rs index 1f8309e29..4aa5f9331 100644 --- a/crates/sqlload/src/loader/_helpers/resolve.rs +++ b/crates/sqlload/src/loader/_helpers/resolve.rs @@ -847,7 +847,7 @@ pub(crate) fn source_target( }); } let schema = source - .schema + .destination_schema .clone() .or_else(|| target.loader_schema.clone()) .ok_or_else(|| format!("target {name:?} has no loader_schema"))?; diff --git a/crates/sqlload/src/project/_helpers/targets.rs b/crates/sqlload/src/project/_helpers/targets.rs index 0cd2503e6..a38ed2e94 100644 --- a/crates/sqlload/src/project/_helpers/targets.rs +++ b/crates/sqlload/src/project/_helpers/targets.rs @@ -190,9 +190,9 @@ pub(crate) fn destination_kind_problems( "{place} destination_path is for targets on an s3, gcs or filesystem connection" )); } - if source.schema.is_some() && files_target { + if source.destination_schema.is_some() && files_target { problems.push(format!( - "{place} schema is for warehouse targets; a files target places tables with path or destination_path" + "{place} destination_schema is for warehouse targets; a files target places tables with path or destination_path" )); } problems diff --git a/crates/sqlload/src/project/_helpers/unknown_keys.rs b/crates/sqlload/src/project/_helpers/unknown_keys.rs index 3afe9215c..3464d6785 100644 --- a/crates/sqlload/src/project/_helpers/unknown_keys.rs +++ b/crates/sqlload/src/project/_helpers/unknown_keys.rs @@ -1,7 +1,8 @@ use toml::Value; use crate::project::constants::{ - PYTHON_SETTING_KEYS, PYTHON_SOURCE_KEYS, SOURCE_ONLY_KEYS, TABLE_ONLY_KEYS, TABLE_SETTING_KEYS, + PYTHON_SETTING_KEYS, PYTHON_SOURCE_KEYS, RENAMED_SCHEMA_KEY, SOURCE_ONLY_KEYS, TABLE_ONLY_KEYS, + TABLE_SETTING_KEYS, }; use crate::similar_names::main::nearest_name::nearest_name; @@ -54,11 +55,12 @@ fn keys_problems( let Some(source) = source.as_table() else { continue; }; - problems.extend(unknown_in( - source.keys(), - source_keys, - &place(source_name, None), - )); + let source_place = place(source_name, None); + problems.extend(source.keys().filter(|key| *key == RENAMED_SCHEMA_KEY).map(|_| { + format!("{source_place}: schema was renamed destination_schema (the schema tables land in); the source schema goes in include, such as \"dbo.*\"") + })); + let keys = source.keys().filter(|key| *key != RENAMED_SCHEMA_KEY); + problems.extend(unknown_in(keys, source_keys, &source_place)); let tables = source.get("tables").and_then(Value::as_table); for (table_name, table) in tables.into_iter().flatten() { let keys = table.as_table().map(|table| table.keys()); diff --git a/crates/sqlload/src/project/constants.rs b/crates/sqlload/src/project/constants.rs index b7df53288..c58dfb316 100644 --- a/crates/sqlload/src/project/constants.rs +++ b/crates/sqlload/src/project/constants.rs @@ -48,13 +48,16 @@ pub const TABLE_SETTING_KEYS: &[&str] = &[ "test_loader_route", ]; +/// The source key `destination_schema` replaced; `plan` refuses it, naming the new key. +pub const RENAMED_SCHEMA_KEY: &str = "schema"; + /// Keys only a source may use. pub const SOURCE_ONLY_KEYS: &[&str] = &[ "kind", "connection", "description", "destination", - "schema", + "destination_schema", "include", "exclude", "new_tables", @@ -103,7 +106,7 @@ pub const PYTHON_SOURCE_KEYS: &[&str] = &[ "connection", "description", "destination", - "schema", + "destination_schema", "table_name", "destination_path", "exclude_columns", diff --git a/crates/sqlload/src/project/models.rs b/crates/sqlload/src/project/models.rs index e52d8b9c8..a9f0233ba 100644 --- a/crates/sqlload/src/project/models.rs +++ b/crates/sqlload/src/project/models.rs @@ -497,9 +497,9 @@ pub struct SourceConfig { /// The target to load into; `default_target` when omitted. #[serde(default)] pub destination: Option, - /// The destination schema; the target's `loader_schema` when omitted. + /// The schema the source's tables land in; the target's `loader_schema` when omitted. The source schema is chosen by `include`. #[serde(default)] - pub schema: Option, + pub destination_schema: Option, /// Source tables to sync, as `schema.table` patterns (database sources). #[serde(default)] pub include: Vec, diff --git a/crates/sqlload/src/project/tests/test_parse_project.rs b/crates/sqlload/src/project/tests/test_parse_project.rs index 8dc395c32..c3c016812 100644 --- a/crates/sqlload/src/project/tests/test_parse_project.rs +++ b/crates/sqlload/src/project/tests/test_parse_project.rs @@ -695,12 +695,17 @@ fn given_an_invalid_project_when_parsing_then_every_problem_is_named() { expected_message: "[sources.app] destination_path is for targets on an s3, gcs or filesystem connection", }, ParseProjectErrorTestCase { - description: "a source schema is for warehouse targets", + description: "a source's destination schema is for warehouse targets", text: concat!( "[connections.lake]\nkind = \"filesystem\"\nroot = \"/data\"\n[targets.lake]\nconnection = \"lake\"\n", - "[sources.app]\nkind = \"postgres\"\nconnection = \"pg\"\ndestination = \"lake\"\ninclude = [\"*\"]\nschema = \"RAW\"\n" + "[sources.app]\nkind = \"postgres\"\nconnection = \"pg\"\ndestination = \"lake\"\ninclude = [\"*\"]\ndestination_schema = \"RAW\"\n" ), - expected_message: "[sources.app] schema is for warehouse targets", + expected_message: "[sources.app] destination_schema is for warehouse targets", + }, + ParseProjectErrorTestCase { + description: "a source's schema key is gone: the error names destination_schema, and include for the source schema", + text: "[sources.erp]\nkind = \"postgres\"\nconnection = \"pg\"\ninclude = [\"*\"]\nschema = \"raw_erp\"\n", + expected_message: "[sources.erp]: schema was renamed destination_schema (the schema tables land in); the source schema goes in include, such as \"dbo.*\"", }, ParseProjectErrorTestCase { description: "a sheet range the source's column_names reach that keeps its header row is refused", diff --git a/crates/sqlload/src/project/tests/test_parse_python_project.rs b/crates/sqlload/src/project/tests/test_parse_python_project.rs index da656e1e5..422965340 100644 --- a/crates/sqlload/src/project/tests/test_parse_python_project.rs +++ b/crates/sqlload/src/project/tests/test_parse_python_project.rs @@ -114,9 +114,9 @@ fn given_python_sources_when_parsing_the_project_then_they_join_it_under_the_sam ParsePythonProjectTestCase { description: "a None setting from Python is refused", toml: "", - sources: r#"[{"name": "a", "module": "sources/a.py", "settings": {"schema": null, "tables": {}}}]"#, + sources: r#"[{"name": "a", "module": "sources/a.py", "settings": {"destination_schema": null, "tables": {}}}]"#, read: |_| String::new(), - expected_value: Err("schema is None; leave the setting out instead"), + expected_value: Err("destination_schema is None; leave the setting out instead"), }, ]; for test_case in test_cases { diff --git a/crates/sqlload/tests/loader/test_postgres_to_clickhouse.rs b/crates/sqlload/tests/loader/test_postgres_to_clickhouse.rs index e3ecae44e..b96b09a16 100644 --- a/crates/sqlload/tests/loader/test_postgres_to_clickhouse.rs +++ b/crates/sqlload/tests/loader/test_postgres_to_clickhouse.rs @@ -119,7 +119,7 @@ write_strategy = "replace" kind = "postgres" connection = "app_pg" include = ["public.stores"] -schema = "legacy" +destination_schema = "legacy" write_strategy = "replace" [sources.old.tables."public.stores"] diff --git a/crates/sqlload/tests/loader/test_snowflake_routes.rs b/crates/sqlload/tests/loader/test_snowflake_routes.rs index ccddc217e..96eaf0833 100644 --- a/crates/sqlload/tests/loader/test_snowflake_routes.rs +++ b/crates/sqlload/tests/loader/test_snowflake_routes.rs @@ -401,7 +401,7 @@ fn given_a_snowflake_target_when_planning_and_loading_then_rows_take_the_route_i RouteTestCase { description: "a stage goes in the target's loader schema, whatever schema the table loads into, and apply creates only that one for it", stage: MANAGED, - settings: "schema = \"OTHER\"\nwrite_strategy = \"append\"\ntest_loader_route = true", + settings: "destination_schema = \"OTHER\"\nwrite_strategy = \"append\"\ntest_loader_route = true", prepare: integration, expected_plan: &["\"DB\".\"RAW\".\"_SQLLOAD_STAGE\""], expected_sent: &[ diff --git a/packages/sqlload/python/sqlload/_declare.py b/packages/sqlload/python/sqlload/_declare.py index d1b875da1..0d621921c 100644 --- a/packages/sqlload/python/sqlload/_declare.py +++ b/packages/sqlload/python/sqlload/_declare.py @@ -75,10 +75,10 @@ "connection", "destination", "destination_path", + "destination_schema", "exclude_columns", "mask_key", "parallel_tables", - "schema", "table_name", } ) @@ -166,6 +166,8 @@ def _docstring(function: TableFunction) -> str | None: def source(name: str, *, description: str | None = None, **settings: Any) -> Source: """A source tables attach to with `@.table(...)`; `settings` are its connection, destination and the defaults its tables share.""" + if "schema" in settings: + raise TypeError("sl.source() got schema=; use destination_schema=") _check_settings(settings, TABLE_SETTINGS | SOURCE_SETTINGS, "source") if not isinstance(settings.get("parallel_tables", False), bool): raise TypeError(f"source {name}: parallel_tables must be True or False") diff --git a/packages/sqlload/tests/test_batched_python.py b/packages/sqlload/tests/test_batched_python.py index 6cdc50224..7cf92339e 100644 --- a/packages/sqlload/tests/test_batched_python.py +++ b/packages/sqlload/tests/test_batched_python.py @@ -229,7 +229,7 @@ def c_small(ctx): yield {"id": 3} -fresh = sl.source("fresh", schema="NEW_SCHEMA", write_strategy="append") +fresh = sl.source("fresh", destination_schema="NEW_SCHEMA", write_strategy="append") for number in range(8): def _one(ctx, number=number): yield {"id": number} @@ -601,7 +601,7 @@ def test_given_a_state_changed_in_place_without_assigning_when_the_run_ends_then def test_given_a_first_load_with_no_rows_when_its_schema_is_new_then_no_schema_is_created(project: Path) -> None: (project / "sources" / "empty.py").write_text( "import sqlload as sl\n" - "empty = sl.source('empty', schema='NEVER_MADE')\n" + "empty = sl.source('empty', destination_schema='NEVER_MADE')\n" "@empty.table(description='Test table.', write_strategy='append')\n" "def nothing(ctx):\n" " return\n" @@ -612,6 +612,15 @@ def test_given_a_first_load_with_no_rows_when_its_schema_is_new_then_no_schema_i assert query(project, "SELECT schema_name FROM information_schema.schemata WHERE schema_name = 'NEVER_MADE'") == [] +def test_given_a_source_setting_schema_when_planning_then_it_fails_naming_destination_schema(project: Path) -> None: + (project / "sources" / "old.py").write_text( + "import sqlload as sl\nold = sl.source('old', schema='RAW_OLD')\n@old.table(description='Test table.')\ndef t(ctx):\n yield {'id': 1}\n" + ) + done = sqlload(project, "plan", "old__t") + assert done.returncode != 0, done.stdout + assert "sl.source() got schema=; use destination_schema=" in done.stdout + done.stderr + + def test_given_shaping_when_declaring_a_python_table_then_it_is_refused_as_an_unknown_setting(project: Path) -> None: (project / "sources" / "shaped.py").write_text( "import sqlload as sl\n" diff --git a/packages/sqlload/tests/test_cli.py b/packages/sqlload/tests/test_cli.py index 9afc28eee..a3500b6c0 100644 --- a/packages/sqlload/tests/test_cli.py +++ b/packages/sqlload/tests/test_cli.py @@ -126,6 +126,27 @@ def test_given_no_project_when_copying_a_folder_then_its_rows_load_and_print_tom assert copied(tmp_path) == [(1, "open"), (2, "shut")] +def test_given_destination_schema_on_a_source_when_loading_then_tables_land_there_named_by_their_source_schema(tmp_path: Path) -> None: + import duckdb + + with duckdb.connect(str(tmp_path / "shop.duckdb")) as connection: + connection.execute("CREATE SCHEMA erp; CREATE TABLE erp.orders AS SELECT 1 AS id") + project = ( + f'[connections.src]\nkind = "duckdb"\ndatabase = "{tmp_path / "shop.duckdb"}"\n[connections.wh]\nkind = "duckdb"\ndatabase = "{tmp_path / "wh.duckdb"}"\n' + '[targets.dev]\nconnection = "wh"\nloader_schema = "RAW"\n' + '[sources.erp]\nkind = "duckdb"\nconnection = "src"\ninclude = ["erp.*"]\ndestination_schema = "raw_erp"\ntable_name = "{schema}__{table}"\nwrite_strategy = "replace"\n' + ) + (tmp_path / "sqlload.toml").write_text(project) + done = run("load", "-c", str(tmp_path / "sqlload.toml")) + assert done.returncode == 0, done.stdout + done.stderr + with duckdb.connect(str(tmp_path / "wh.duckdb"), read_only=True) as connection: + assert connection.execute("SELECT id FROM raw_erp.erp__orders").fetchall() == [(1,)] + (tmp_path / "sqlload.toml").write_text(project.replace("destination_schema =", "schema =")) + planned = run("plan", "-c", str(tmp_path / "sqlload.toml")) + assert planned.returncode != 0, planned.stdout + assert 'schema was renamed destination_schema (the schema tables land in); the source schema goes in include, such as "dbo.*"' in planned.stdout + planned.stderr + + def test_given_a_literal_password_in_a_uri_when_the_copy_fails_then_no_output_shows_it(tmp_path: Path) -> None: done = run("copy", "postgresql://app:hunter2-secret@127.0.0.1:1/shop#public.accounts", f"duckdb://{tmp_path / 'wh.duckdb'}#RAW.ACCOUNTS", "-c", str(tmp_path / "sqlload.toml")) assert done.returncode == 75, "a refused connection is worth retrying" diff --git a/website/src/content/docs/docs/cli/copy.mdx b/website/src/content/docs/docs/cli/copy.mdx index f38282f2a..afb003cad 100644 --- a/website/src/content/docs/docs/cli/copy.mdx +++ b/website/src/content/docs/docs/cli/copy.mdx @@ -90,7 +90,7 @@ description = "Copied from folder orders/." destination = "copy_orders" kind = "files" path = "" -schema = "raw" +destination_schema = "raw" table_name = "orders" write_strategy = "replace" diff --git a/website/src/content/docs/docs/concepts/projects.mdx b/website/src/content/docs/docs/concepts/projects.mdx index 0f82a0eed..024d4de79 100644 --- a/website/src/content/docs/docs/concepts/projects.mdx +++ b/website/src/content/docs/docs/concepts/projects.mdx @@ -105,7 +105,7 @@ Destination tables are named from a template with `{source}`, `{schema}` and `{t | Files | `{source}` (a files source is one table) | `events` | Set `table_name` on the source, or `name` on one table, to change it. The table goes into the -source's `schema`, or the target's `loader_schema`. +source's `destination_schema`, or the target's `loader_schema`. ## Descriptions diff --git a/website/src/content/docs/docs/configuration.mdx b/website/src/content/docs/docs/configuration.mdx index 617de88b2..935a0c966 100644 --- a/website/src/content/docs/docs/configuration.mdx +++ b/website/src/content/docs/docs/configuration.mdx @@ -264,7 +264,7 @@ http shop rate_limit 10/s, max_in_flight 4, retries 5, auth bearer | Key | Applies to | Meaning | |-----|------------|---------| | `connection` | all | A `snowflake`, `databricks`, `bigquery`, `clickhouse`, `postgres`, `duckdb`, `s3` or `filesystem` connection, or a `motherduck` connection. | -| `loader_schema` | warehouses, required | The schema for tables whose source sets no `schema`; a BigQuery target's dataset, a ClickHouse target's database. | +| `loader_schema` | warehouses, required | The schema for tables whose source sets no `destination_schema`; a BigQuery target's dataset, a ClickHouse target's database. | | `database` | warehouses except ClickHouse | Overrides the connection's database. | | `path` | files | Each table's folder template, with `{source}`, `{schema}`, `{table}`, `{cursor}` and `{cursor:date}`. Default `{source}/{table}/`. | | `format` | files | `parquet` (default), or `csv`. | @@ -286,8 +286,8 @@ http shop rate_limit 10/s, max_in_flight 4, retries 5, auth bearer | `connection` | all | The connection the source reads. | | `description` | all | The source's description; a files source's table description. | | `destination` | all | The target; `default_target` when omitted. | -| `schema` | warehouse targets | The destination schema; the target's `loader_schema` when omitted. | -| `table_name` | all | The destination name template, with `{source}`, `{schema}` and `{table}`. Default `{source}__{table}`, or `{source}` for files. | +| `destination_schema` | warehouse targets | The schema the source's tables land in; the target's `loader_schema` when omitted. The schema they are read from is chosen by `include`. | +| `table_name` | all | The destination name template, with `{source}`, `{schema}` (the source schema) and `{table}`. Default `{source}__{table}`, or `{source}` for files. | | `destination_path` | files targets | The folder template for this source's tables, overriding the target's `path`. | | `include`, `exclude` | postgres, postgres_cdc, snowflake, odbc, bigquery, databricks | `schema.table` patterns to sync and to leave out (`dataset.table` for bigquery). | | `new_tables` | postgres | `sync` (default), `skip` or `fail`. See [Sources](/docs/sources/postgres/#choosing-tables). | @@ -339,7 +339,7 @@ matches rows on the primary key, and a `unique_key` must lie within it. ## Table settings -Set on a source for all its tables, or in a `[sources.NAME.tables."schema.table"]` entry for one. +Set on a source for all its tables, or in a `[sources.NAME.tables."schema.table"]` entry for one. The `schema` in an entry's name is the source schema; where tables land is the source's `destination_schema`. | Key | Default | Meaning | |-----|---------|---------| diff --git a/website/src/content/docs/docs/guides/from-dlt.mdx b/website/src/content/docs/docs/guides/from-dlt.mdx index e9ebbc9d4..5b8aafe1b 100644 --- a/website/src/content/docs/docs/guides/from-dlt.mdx +++ b/website/src/content/docs/docs/guides/from-dlt.mdx @@ -23,7 +23,7 @@ what dlt 1.32.0 loads from the same files. For how the two differ in design, see | dlt | SQLLoad | | --- | --- | -| `dlt.pipeline(destination=..., dataset_name="shop")` | A [target](/docs/configuration/#targets) in `sqlload.toml`, with `loader_schema = "shop"`. There is no pipeline working directory. | +| `dlt.pipeline(destination=..., dataset_name="shop")` | A [target](/docs/configuration/#targets) in `sqlload.toml`, with `loader_schema = "shop"`; a source's `destination_schema` puts its tables in another. There is no pipeline working directory. | | `@dlt.source` | `sl.source(...)` in Python, or a `[sources.]` entry for a database or files. | | `@dlt.resource` | `@source.table` in Python, or a table of a source. | | A resource's table name | `{source}__{table}` by default. `table_name = "{table}"` on the source names it as dlt does. | diff --git a/website/src/content/docs/docs/sources/odbc.mdx b/website/src/content/docs/docs/sources/odbc.mdx index e33839824..2a6ba720d 100644 --- a/website/src/content/docs/docs/sources/odbc.mdx +++ b/website/src/content/docs/docs/sources/odbc.mdx @@ -250,7 +250,8 @@ When both ways fail, the error shows both of the driver's errors. `include` and `exclude` are `schema.table` patterns, matched without regard to case (`*` matches any text). SQLLoad loads the tables and views the driver reports that match. For a database without schemas, the patterns match the table name alone. A `[sources.erp.tables."schema.table"]` entry sets -options for one table; on its own, it doesn't add the table. +options for one table; on its own, it doesn't add the table. These patterns choose what is read; the +schema the tables land in is the source's `destination_schema`, or the target's `loader_schema`. A source needs at least one `include` pattern or one query table. Every table needs a `write_strategy`, from its entry or the source, and a `merge` table a `unique_key`: its columns, or diff --git a/website/src/content/docs/docs/sources/python.mdx b/website/src/content/docs/docs/sources/python.mdx index cd452eaed..3cdbcb45d 100644 --- a/website/src/content/docs/docs/sources/python.mdx +++ b/website/src/content/docs/docs/sources/python.mdx @@ -37,7 +37,7 @@ def orders(ctx: sl.Context): | Function | Returns | |----------|---------| -| `sl.source(name, *, description=None, **settings)` | A `Source`. `settings` are its `connection`, `destination`, `schema` and the defaults its tables share. | +| `sl.source(name, *, description=None, **settings)` | A `Source`. `settings` are its `connection`, `destination`, `destination_schema` and the defaults its tables share. | | `@source.table` or `@source.table(*, name=None, description=None, **settings)` | A `Table` attached to the source. | | `@sl.table` or `@sl.table(...)` | A standalone `Table`, loaded into the target's schema; it sets its own `write_strategy`. | | `@source.client` | Declares a client the source's tables share as `ctx.client`, built once per run. | @@ -66,7 +66,7 @@ Unknown settings raise `TypeError` when the module is imported. | `unnest`, `max_depth` | table | As in [`sqlload.toml`](/docs/configuration/#table-settings): the list fields whose items become child tables, such as `unnest=["items"]` or `"all"`, and how many levels deep. See [Nested data](/docs/concepts/nested-data/). | | `mask`, `exclude_columns`, `allow_columns` | table (the last two also source) | As for other sources. | | `table_name`, `destination_path` | source or table | Destination name template, and folder in a files target. | -| `connection`, `destination`, `schema`, `mask_key`, `parallel_tables` | source | Its connection, target, schema, HMAC key, and whether tables sharing a client may load at once. | +| `connection`, `destination`, `destination_schema`, `mask_key`, `parallel_tables` | source | Its connection, target, the schema its tables land in, HMAC key, and whether tables sharing a client may load at once. | ## What a function yields diff --git a/website/src/content/docs/docs/targets/bigquery.mdx b/website/src/content/docs/docs/targets/bigquery.mdx index ceafca25f..d988c684f 100644 --- a/website/src/content/docs/docs/targets/bigquery.mdx +++ b/website/src/content/docs/docs/targets/bigquery.mdx @@ -35,7 +35,7 @@ loader_schema = "raw" | `service_account_key` | The key file's JSON text, not its path. Required. | | `project` | The project tables load into and jobs run in. Default: the key's own project. | -The target's `loader_schema` is the dataset tables load into; a source's `schema` overrides it. +The target's `loader_schema` is the dataset tables load into; a source's `destination_schema` overrides it. `sqlload apply` creates the dataset when it doesn't exist. Pass the key through the environment, so it stays out of `sqlload.toml`: diff --git a/website/src/content/docs/docs/targets/clickhouse.mdx b/website/src/content/docs/docs/targets/clickhouse.mdx index ee0e224be..c2a8a2313 100644 --- a/website/src/content/docs/docs/targets/clickhouse.mdx +++ b/website/src/content/docs/docs/targets/clickhouse.mdx @@ -32,8 +32,8 @@ loader_schema = "raw" | `password` | The user's password. Use `${ENV:NAME}`. | | `secure` | `true` for HTTPS. Default: `false`. | -The target's `loader_schema` is the ClickHouse database tables load into; a source's `schema` -overrides it. ClickHouse has no level above its databases, so a ClickHouse target takes no +The target's `loader_schema` is the ClickHouse database tables load into; a source's +`destination_schema` overrides it. ClickHouse has no level above its databases, so a ClickHouse target takes no `database`, and `plan` refuses one. `sqlload apply` creates the database when it doesn't exist. SQLLoad is tested against ClickHouse 25.3, 25.8 and 26.10. diff --git a/website/src/content/docs/docs/targets/index.mdx b/website/src/content/docs/docs/targets/index.mdx index a2a374b84..19ae38605 100644 --- a/website/src/content/docs/docs/targets/index.mdx +++ b/website/src/content/docs/docs/targets/index.mdx @@ -64,10 +64,10 @@ loader_schema = "raw" | Key | Meaning | |-----|---------| | `connection` | A `snowflake`, `databricks`, `bigquery`, `clickhouse`, `postgres` or `duckdb` connection, or a `motherduck` connection. | -| `loader_schema` | The schema for tables whose source sets no `schema`. Required. | +| `loader_schema` | The schema for tables whose source sets no `destination_schema`. Required. | | `database` | Overrides the connection's database. Not for ClickHouse. | -A source's `schema` overrides `loader_schema` for its tables. A DuckDB connection's `database` is +A source's `destination_schema` overrides `loader_schema` for its tables. A DuckDB connection's `database` is a file path, or `:memory:`. A Snowflake connection also needs a `load_method`; see [Snowflake](/docs/targets/snowflake/). A Postgres target uses the same connection kind as a Postgres source. diff --git a/website/src/content/docs/docs/targets/postgres.mdx b/website/src/content/docs/docs/targets/postgres.mdx index a7a600f75..366f6666c 100644 --- a/website/src/content/docs/docs/targets/postgres.mdx +++ b/website/src/content/docs/docs/targets/postgres.mdx @@ -33,7 +33,7 @@ loader_schema = "reporting" | `password` | Use `${ENV:NAME}`. | | `sslmode`, `sslrootcert`, `sslcert`, `sslkey` | TLS, as libpq takes it. See [TLS](/docs/sources/postgres/#tls). | -The target's `loader_schema` is the schema tables load into; a source's `schema` overrides it. The +The target's `loader_schema` is the schema tables load into; a source's `destination_schema` overrides it. The target's `database` overrides the connection's. `sqlload debug` shows the target's user, database, server version and what protects the