Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion crates/sqlload/src/copy/_helpers/project.rs
Original file line number Diff line number Diff line change
Expand Up @@ -172,7 +172,7 @@ fn source_settings(
"description".to_owned(),
Value::String(format!("Copied from {}.", source.described)),
);
settings.extend(schema.map(|schema| ("schema".to_owned(), Value::String(schema))));
settings.extend(schema.map(|schema| ("destination_schema".to_owned(), Value::String(schema))));
settings.extend(table_settings(request, !FILE_KINDS.contains(&kind(source))));
match kind(source) {
"postgres" => {
Expand Down
23 changes: 23 additions & 0 deletions crates/sqlload/src/loader/_helpers/destination.rs
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,7 @@ use crate::loader::_helpers::publish::{staged_columns, write_target};
use crate::loader::_helpers::resolve::display_name;
use crate::loader::_helpers::sources::{ReadPart, read_source};
use crate::loader::_helpers::table_plan::blocked_plan;
use crate::loader::_helpers::table_plan::{change_lines, schema_statements};
use crate::loader::constants::{CATALOG_CONFLICT, CATALOG_CONFLICT_TRIES, COALESCED_BYTES};
use crate::loader::errors::LoaderError;
use crate::loader::models::SourceReader;
Expand Down Expand Up @@ -662,6 +663,28 @@ pub(crate) async fn replace_files(
}

impl Publisher {
/// The lines not yet reported: the table's own `created ...` (once [created at its first commit](Self::create_table)), then the `created ...`, `altered ...` and `loaded ...` lines of the child tables filled so far.
pub(crate) fn take_changes(&mut self) -> Vec<String> {
let mut changes = std::mem::take(&mut self.changes);
if let Some(children) = self.children.as_mut() {
changes.append(&mut children.changes);
}
changes
}

/// Create a [new table](Publisher::new_table) before its first commit writes to it (or at the end of a load that wrote none), with the columns planned by then and its `created ...` line kept for the load to report; nothing once it exists.
pub(in crate::loader) async fn create_table(&mut self) -> Result<(), LoaderError> {
if !self.new_table || self.table.schema.action != TableAction::Create {
return Ok(());
}
if let Some(statements) = schema_statements(&self.table) {
execute_ddl(&self.destination, statements).await?;
self.changes.extend(change_lines(&self.table));
}
self.table.schema.action = TableAction::Unchanged;
Ok(())
}

/// Add a batch to the commit's files, starting them (and marking the commit as writing) on its first batch.
pub(crate) async fn stage_files(&mut self, batch: RecordBatch) -> Result<(), LoaderError> {
let writer = match self.files.as_mut().and_then(|sink| sink.writer.take()) {
Expand Down
37 changes: 28 additions & 9 deletions crates/sqlload/src/loader/_helpers/publish.rs
Original file line number Diff line number Diff line change
Expand Up @@ -147,6 +147,7 @@ use crate::schema_plan::constants::{
use crate::schema_plan::models::ColumnOrigin;
use crate::schema_plan::models::DesiredColumn;
use crate::schema_plan::models::LiveColumn;
use crate::schema_plan::models::TableAction;
use crate::snowflake_source::main::select_query::select_query;
use crate::table_lease::main::lease_now::lease_now;
use crate::table_lease::main::take_lease::take_lease;
Expand Down Expand Up @@ -177,6 +178,25 @@ fn writes_directly(table: &TablePlan, writer: &Writer) -> bool {
&& (!table.route.as_ref().is_some_and(Route::pulls_unloaded) || unloads_directly(writer))
}

/// Whether `table` is a table planned from its data (a Python table, or one of its child tables) that its load creates in a SQL destination, and whose rows load through a staging table anyway: a merge, a `delete_insert`, or an append that stages (logged state, checks, child tables, or a merge parent's child table). It is created at its first commit rather than its first batch, so the columns its later batches bring come before the `_sqlload_*` metadata. Other appends (including ones a warehouse writer stages internally) and `replace` (`--reload`) builds keep creating it at the first batch.
pub(crate) fn created_at_commit(table: &TablePlan) -> bool {
let spec = &table.spec;
let stages = match table.mode {
WriteMode::Merge { .. } | WriteMode::DeleteInsert { .. } => true,
WriteMode::Append => {
table.logged.is_some()
|| spec.checks.is_some()
|| spec.unnest != Unnest::None
|| spec.root_rows
}
_ => false,
};
stages
&& table.schema.action == TableAction::Create
&& spec.python.is_some()
&& table.output.is_none()
}

/// Where the batches go, from the publisher's fields: `staging` when there is one, and `columns` as the staged batches carry them.
pub(in crate::loader) fn write_target<'a>(
table: &'a TablePlan,
Expand Down Expand Up @@ -547,6 +567,7 @@ impl Publisher {
}
_ => None,
};
let new_table = created_at_commit(&table);
let direct = files.is_none()
&& table.spec.checks.is_none()
&& table.spec.unnest == Unnest::None
Expand Down Expand Up @@ -626,6 +647,8 @@ impl Publisher {
at_end,
builds,
indexes: shape.map(|shape| shape.indexes).unwrap_or_default(),
new_table,
changes: Vec::new(),
table,
destination,
run,
Expand Down Expand Up @@ -792,14 +815,6 @@ impl Publisher {
}
}

/// The `created ...`, `altered ...` and `loaded ...` lines of the child tables filled so far.
pub(crate) fn child_changes(&mut self) -> Vec<String> {
self.children
.as_mut()
.map(|children| std::mem::take(&mut children.changes))
.unwrap_or_default()
}

/// Whether the batch holds about 1 GB (Parquet staged on Snowflake, with the zstd files `send = "at_commit"` holds for it, which come within a few percent of that Parquet, Arrow elsewhere), where a size-cut table may end it.
pub(crate) fn full(&self) -> bool {
let bytes = match (self.writer.staged_bytes(), &self.spool) {
Expand Down Expand Up @@ -853,6 +868,9 @@ impl Publisher {
|| matches!(self.table.mode, WriteMode::MarkMissing(_))
|| self.leftover_deletes()
|| self.pieces.unfinished;
if !pending {
self.create_table().await?;
}
if let Some(children) = self.children.as_mut().filter(|_| self.direct || !pending) {
Box::pin(children.finish(
(&self.table, (&self.load_id, &self.loaded_at)),
Expand Down Expand Up @@ -1261,6 +1279,7 @@ impl Publisher {
}
_ => None,
};
self.create_table().await?;
if let Some(children) = &mut self.children {
match last {
true => {
Expand Down Expand Up @@ -1574,7 +1593,7 @@ pub(crate) async fn publish_table(
};
Ok(TableLoaded {
rows,
changes: publisher.child_changes(),
changes: publisher.take_changes(),
note: marked(table, (rows, publisher.marked))
.or_else(|| older_note(table, publisher.older)),
after_load: report,
Expand Down
47 changes: 24 additions & 23 deletions crates/sqlload/src/loader/_helpers/python.rs
Original file line number Diff line number Diff line change
Expand Up @@ -34,13 +34,15 @@ use crate::load_naming::main::metadata_name::metadata_name;
use crate::load_naming::main::name_lines::name_lines;
use crate::load_naming::main::named_batch::named_batch;
use crate::load_naming::models::{Compat, LoadedNames, MetadataRole, NameMemo, RenameColumn};
use crate::loader::_helpers::publish::created_at_commit;
use crate::loader::_helpers::publish::replaces_roots;
use crate::loader::_helpers::publish::staged_columns;
use crate::loader::_helpers::resolve::{KnownNames, host_rename, resolve_known};
use crate::loader::_helpers::run_log::{log_empty_run, record_numbered, run_entry};
use crate::loader::_helpers::specs::is_child_column;
use crate::loader::_helpers::specs::null_cursor_problem;
use crate::loader::_helpers::table_plan::append_problem;
use crate::loader::_helpers::table_plan::change_lines;
use crate::loader::_helpers::table_plan::replan_columns;
use crate::loader::_helpers::table_plan::schema_statements;
use crate::loader::constants::DROPPED_CHUNK;
Expand Down Expand Up @@ -498,22 +500,6 @@ pub(crate) fn cursor_type_name(table: &TablePlan, column: &str) -> String {
}
}

/// The schema changes the load made, one line each, as `apply` would print them.
pub(crate) fn change_lines(table: &TablePlan) -> Vec<String> {
let name = sql_adapter(table.spec.warehouse).render_display_name(&table.spec.destination);
match &table.schema.action {
TableAction::Create => vec![format!(
"created {name} {} columns",
table.schema.write_columns.len()
)],
TableAction::Alter(changes) => changes
.iter()
.map(|change| format!("altered {name} {change}"))
.collect(),
TableAction::Recreate | TableAction::Unchanged => Vec::new(),
}
}

/// Fail when writing no rows would empty a table that has rows: `replace` or `--reload` without `allow_empty` (an empty result must not wipe a table by accident).
pub(crate) async fn refuse_emptying(
table: &TablePlan,
Expand Down Expand Up @@ -827,9 +813,9 @@ impl PythonFeed {
let state = self.state.clone();
let publisher = self.publisher(load).await?;
let rows = publisher.finish(state).await?;
let children = publisher.child_changes();
let changes = publisher.take_changes();
let replaced_keys = publisher.replaced.clone();
self.changes.extend(children);
self.changes.extend(changes);
Box::pin(record_numbered(&self.table, load.destination)).await?;
Ok(TableLoaded {
rows,
Expand Down Expand Up @@ -975,7 +961,7 @@ impl PythonFeed {
}
}

/// Plan the table for the data seen so far and bring the table (and the staging table) in line: created on the first batch, altered as later batches add or widen columns; columns that only held NULL wait for a typed value, or become text at the end (`settle`).
/// Plan the table for the data seen so far and bring the table (and the staging table) in line: created at its first commit with the columns seen by then (see [`created_at_commit`]), altered as later batches add or widen columns; columns that only held NULL wait for a typed value, or become text at the end (`settle`).
pub(crate) async fn plan_data(
&mut self,
load: &PythonLoad<'_>,
Expand Down Expand Up @@ -1020,6 +1006,9 @@ impl PythonFeed {
.is_none()
.then_some(self.table.live.as_deref());
let replanned = replan(load, &self.table, (&data, known, &self.names), destination).await?;
if let Some(publisher) = self.publisher.as_mut() {
self.changes.append(&mut publisher.changes);
}
let old = staged_columns(&self.table);
let changes = bring_in_line(
(load.destination, self.publisher.as_mut()),
Expand Down Expand Up @@ -1288,7 +1277,7 @@ fn table_read(
}
}

/// Bring a table planned from its data in line with `replanned`, through `publisher`'s connection once it is open (`destination` before): its schema changes run, and an open publisher's rows so far are settled and its staging table altered from the `old` staged columns, before it takes the new plan. Returns the `created ...` and `altered ...` lines.
/// Bring a table planned from its data in line with `replanned`, through `publisher`'s connection once it is open (`destination` before): its schema changes run (a table the load creates waits for its first commit, see [`created_at_commit`]), and an open publisher's rows so far are settled and its staging table altered from the `old` staged columns, before it takes the new plan. Returns the `created ...` and `altered ...` lines.
pub(crate) async fn bring_in_line(
(destination, publisher): (&Destination, Option<&mut Publisher>),
(old, replanned): (&[DesiredColumn], &TablePlan),
Expand All @@ -1298,7 +1287,13 @@ pub(crate) async fn bring_in_line(
Some(publisher) => &publisher.destination,
None => destination,
};
if let Some(statements) = schema_statements(replanned) {
if created_at_commit(replanned) {
if publisher.is_none() {
let adapter = sql_adapter(replanned.spec.warehouse);
let schema = adapter.render_create_schema(&replanned.spec.destination);
execute_ddl(destination, vec![schema]).await?;
}
} else if let Some(statements) = schema_statements(replanned) {
execute_ddl(destination, statements).await?;
changes.extend(change_lines(replanned));
}
Expand Down Expand Up @@ -1592,6 +1587,7 @@ impl ChildTables {
.filter_map(|feed| feed.publisher.as_mut())
{
publisher.cut(CommitRange::Planned).await?;
self.changes.append(&mut publisher.changes);
}
Ok(())
}
Expand Down Expand Up @@ -1657,6 +1653,7 @@ impl ChildTables {
}
if let Some(publisher) = &mut feed.publisher {
publisher.finish(None).await?;
self.changes.append(&mut publisher.changes);
}
}
Ok(())
Expand Down Expand Up @@ -1983,9 +1980,13 @@ impl ChildFeed {
schema: columns.schema,
..self.table.clone()
};
let mut changes = (self.publisher.as_mut())
.map(|publisher| std::mem::take(&mut publisher.changes))
.unwrap_or_default();
let old = staged_columns(&self.table);
let changes =
bring_in_line((destination, self.publisher.as_mut()), (&old, &replanned)).await?;
changes.extend(
bring_in_line((destination, self.publisher.as_mut()), (&old, &replanned)).await?,
);
self.table = TablePlan {
live: replanned.live.clone().or(self.table.live.clone()),
..replanned
Expand Down
2 changes: 1 addition & 1 deletion crates/sqlload/src/loader/_helpers/resolve.rs
Original file line number Diff line number Diff line change
Expand Up @@ -847,7 +847,7 @@ pub(crate) fn source_target(
});
}
let schema = source
.schema
.destination_schema
.clone()
.or_else(|| target.loader_schema.clone())
.ok_or_else(|| format!("target {name:?} has no loader_schema"))?;
Expand Down
16 changes: 16 additions & 0 deletions crates/sqlload/src/loader/_helpers/table_plan.rs
Original file line number Diff line number Diff line change
Expand Up @@ -897,6 +897,22 @@ async fn plan_table_once(
})
}

/// The schema changes the load made, one line each, as `apply` would print them.
pub(crate) fn change_lines(table: &TablePlan) -> Vec<String> {
let name = sql_adapter(table.spec.warehouse).render_display_name(&table.spec.destination);
match &table.schema.action {
TableAction::Create => vec![format!(
"created {name} {} columns",
table.schema.write_columns.len()
)],
TableAction::Alter(changes) => changes
.iter()
.map(|change| format!("altered {name} {change}"))
.collect(),
TableAction::Recreate | TableAction::Unchanged => Vec::new(),
}
}

/// The statements that bring the table's schema in line with its plan, a created or rebuilt table getting its description as its comment; `None` when there is nothing to run: unchanged, rebuilt by the load itself, or a files target.
pub(crate) fn schema_statements(table: &TablePlan) -> Option<Vec<String>> {
let spec = &table.spec;
Expand Down
4 changes: 4 additions & 0 deletions crates/sqlload/src/loader/models.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1142,6 +1142,10 @@ pub struct Publisher {
pub builds: bool,
/// The indexes to re-create on the build table once it is swapped in.
pub indexes: Vec<String>,
/// A table planned from its data that this load creates (see [`created_at_commit`](crate::loader::_helpers::publish::created_at_commit)): it is created at the first commit with every column seen by then.
pub new_table: bool,
/// The `created ...` line of a table created at its first commit, until its load reports it.
pub changes: Vec<String>,
pub staging: Option<TableName>,
/// The batch being filled, from 1.
pub batch: u64,
Expand Down
4 changes: 2 additions & 2 deletions crates/sqlload/src/project/_helpers/targets.rs
Original file line number Diff line number Diff line change
Expand Up @@ -190,9 +190,9 @@ pub(crate) fn destination_kind_problems(
"{place} destination_path is for targets on an s3, gcs or filesystem connection"
));
}
if source.schema.is_some() && files_target {
if source.destination_schema.is_some() && files_target {
problems.push(format!(
"{place} schema is for warehouse targets; a files target places tables with path or destination_path"
"{place} destination_schema is for warehouse targets; a files target places tables with path or destination_path"
));
}
problems
Expand Down
14 changes: 8 additions & 6 deletions crates/sqlload/src/project/_helpers/unknown_keys.rs
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
use toml::Value;

use crate::project::constants::{
PYTHON_SETTING_KEYS, PYTHON_SOURCE_KEYS, SOURCE_ONLY_KEYS, TABLE_ONLY_KEYS, TABLE_SETTING_KEYS,
PYTHON_SETTING_KEYS, PYTHON_SOURCE_KEYS, RENAMED_SCHEMA_KEY, SOURCE_ONLY_KEYS, TABLE_ONLY_KEYS,
TABLE_SETTING_KEYS,
};
use crate::similar_names::main::nearest_name::nearest_name;

Expand Down Expand Up @@ -54,11 +55,12 @@ fn keys_problems(
let Some(source) = source.as_table() else {
continue;
};
problems.extend(unknown_in(
source.keys(),
source_keys,
&place(source_name, None),
));
let source_place = place(source_name, None);
problems.extend(source.keys().filter(|key| *key == RENAMED_SCHEMA_KEY).map(|_| {
format!("{source_place}: schema was renamed destination_schema (the schema tables land in); the source schema goes in include, such as \"dbo.*\"")
}));
let keys = source.keys().filter(|key| *key != RENAMED_SCHEMA_KEY);
problems.extend(unknown_in(keys, source_keys, &source_place));
let tables = source.get("tables").and_then(Value::as_table);
for (table_name, table) in tables.into_iter().flatten() {
let keys = table.as_table().map(|table| table.keys());
Expand Down
7 changes: 5 additions & 2 deletions crates/sqlload/src/project/constants.rs
Original file line number Diff line number Diff line change
Expand Up @@ -48,13 +48,16 @@ pub const TABLE_SETTING_KEYS: &[&str] = &[
"test_loader_route",
];

/// The source key `destination_schema` replaced; `plan` refuses it, naming the new key.
pub const RENAMED_SCHEMA_KEY: &str = "schema";

/// Keys only a source may use.
pub const SOURCE_ONLY_KEYS: &[&str] = &[
"kind",
"connection",
"description",
"destination",
"schema",
"destination_schema",
"include",
"exclude",
"new_tables",
Expand Down Expand Up @@ -103,7 +106,7 @@ pub const PYTHON_SOURCE_KEYS: &[&str] = &[
"connection",
"description",
"destination",
"schema",
"destination_schema",
"table_name",
"destination_path",
"exclude_columns",
Expand Down
4 changes: 2 additions & 2 deletions crates/sqlload/src/project/models.rs
Original file line number Diff line number Diff line change
Expand Up @@ -497,9 +497,9 @@ pub struct SourceConfig {
/// The target to load into; `default_target` when omitted.
#[serde(default)]
pub destination: Option<String>,
/// The destination schema; the target's `loader_schema` when omitted.
/// The schema the source's tables land in; the target's `loader_schema` when omitted. The source schema is chosen by `include`.
#[serde(default)]
pub schema: Option<String>,
pub destination_schema: Option<String>,
/// Source tables to sync, as `schema.table` patterns (database sources).
#[serde(default)]
pub include: Vec<String>,
Expand Down
Loading
Loading