diff --git a/crates/sqlquack-semantics/src/bigquery/profile/models.rs b/crates/sqlquack-semantics/src/bigquery/profile/models.rs index e012b65ab..fd03fd0a0 100644 --- a/crates/sqlquack-semantics/src/bigquery/profile/models.rs +++ b/crates/sqlquack-semantics/src/bigquery/profile/models.rs @@ -10,7 +10,7 @@ use crate::bigquery::profile::_helpers::typing::{ use crate::bigquery::profile::constants::{BIGQUERY_DATE_PART_FUNCTIONS, BIGQUERY_WEEKDAY_PARTS}; use crate::dialects::types::DialectProfile; use crate::lineage::constants::SNOWFLAKE_VALUES_COLUMN; -use crate::lineage::models::Engine; +use crate::lineage::models::{Engine, JoinConditionReach}; use crate::lineage::models::{Lineage, LineageContext, TransformKind}; use crate::lineage::types::DialectRules; use crate::lineage::types::ValueType; @@ -58,6 +58,22 @@ impl DialectRules for BigQueryRules { false } + fn window_keys_read_outputs(&self) -> bool { + false + } + + fn select_list_reads_outputs(&self) -> bool { + true + } + + fn merged_keys_hide_other_items(&self) -> bool { + true + } + + fn join_conditions_read(&self) -> JoinConditionReach { + JoinConditionReach::Every + } + fn checks_call_signatures(&self) -> bool { true } diff --git a/crates/sqlquack-semantics/src/bigquery/profile/tests/test_checks.rs b/crates/sqlquack-semantics/src/bigquery/profile/tests/test_checks.rs index 0a58c00ba..532bdd5d7 100644 --- a/crates/sqlquack-semantics/src/bigquery/profile/tests/test_checks.rs +++ b/crates/sqlquack-semantics/src/bigquery/profile/tests/test_checks.rs @@ -6,6 +6,11 @@ use crate::lineage::tests::test_types::QueryFindingsTestCase; #[test] fn given_a_bigquery_query_when_checking_it_then_findings_follow_googlesql() { let test_cases = [ + QueryFindingsTestCase { + description: "an inline window ordered by another output's alias", + sql: "SELECT quantity AS q, sum(quantity) OVER (ORDER BY q) AS x FROM orders", + expected_findings: &[("E201", "Unknown column 'q' in table 'orders'", 51)], + }, QueryFindingsTestCase { description: "a TIMESTAMP_TRUNC date part is not a column (analyzer: accepted)", sql: "SELECT TIMESTAMP_TRUNC(ordered_at, DAY) AS v FROM orders", diff --git a/crates/sqlquack-semantics/src/clickhouse/profile/models.rs b/crates/sqlquack-semantics/src/clickhouse/profile/models.rs index c5c5bed61..4a813dd22 100644 --- a/crates/sqlquack-semantics/src/clickhouse/profile/models.rs +++ b/crates/sqlquack-semantics/src/clickhouse/profile/models.rs @@ -14,7 +14,7 @@ use crate::lineage::constants::{ DATE_PART_FUNCTIONS, SNOWFLAKE_VALUES_COLUMN, WINDOW_ONLY_FUNCTIONS, }; use crate::lineage::main::rendered_type::rendered_type; -use crate::lineage::models::Engine; +use crate::lineage::models::{Engine, JoinConditionReach}; use crate::lineage::models::{Lineage, LineageContext}; use crate::lineage::types::DialectRules; use crate::lineage::types::ValueType; @@ -73,6 +73,22 @@ impl DialectRules for ClickHouseRules { false } + fn window_keys_read_outputs(&self) -> bool { + true + } + + fn select_list_reads_outputs(&self) -> bool { + true + } + + fn merged_keys_hide_other_items(&self) -> bool { + true + } + + fn join_conditions_read(&self) -> JoinConditionReach { + JoinConditionReach::Every + } + fn checks_call_signatures(&self) -> bool { true } diff --git a/crates/sqlquack-semantics/src/databricks/profile/models.rs b/crates/sqlquack-semantics/src/databricks/profile/models.rs index cdc35fc72..dc1b394b6 100644 --- a/crates/sqlquack-semantics/src/databricks/profile/models.rs +++ b/crates/sqlquack-semantics/src/databricks/profile/models.rs @@ -11,7 +11,7 @@ use crate::databricks::profile::_helpers::typing::{ use crate::databricks::profile::constants::DATABRICKS_VALUES_COLUMN; use crate::dialects::types::DialectProfile; use crate::lineage::constants::DATE_PART_FUNCTIONS; -use crate::lineage::models::Engine; +use crate::lineage::models::{Engine, JoinConditionReach}; use crate::lineage::models::{Lineage, LineageContext}; use crate::lineage::types::DialectRules; use crate::lineage::types::ValueType; @@ -63,6 +63,22 @@ impl DialectRules for DatabricksRules { false } + fn window_keys_read_outputs(&self) -> bool { + true + } + + fn select_list_reads_outputs(&self) -> bool { + true + } + + fn merged_keys_hide_other_items(&self) -> bool { + true + } + + fn join_conditions_read(&self) -> JoinConditionReach { + JoinConditionReach::Every + } + fn checks_call_signatures(&self) -> bool { true } diff --git a/crates/sqlquack-semantics/src/duckdb/profile/models.rs b/crates/sqlquack-semantics/src/duckdb/profile/models.rs index f1cc261d8..fef814aa6 100644 --- a/crates/sqlquack-semantics/src/duckdb/profile/models.rs +++ b/crates/sqlquack-semantics/src/duckdb/profile/models.rs @@ -16,7 +16,7 @@ use crate::duckdb::profile::constants::{ DUCKDB_VALUES_COLUMN, }; use crate::lineage::constants::{DATE_PART_FUNCTIONS, WINDOW_ONLY_FUNCTIONS}; -use crate::lineage::models::Engine; +use crate::lineage::models::{Engine, JoinConditionReach}; use crate::lineage::models::{Lineage, LineageContext}; use crate::lineage::types::DialectRules; use crate::lineage::types::ValueType; @@ -64,6 +64,22 @@ impl DialectRules for DuckDbRules { false } + fn window_keys_read_outputs(&self) -> bool { + true + } + + fn select_list_reads_outputs(&self) -> bool { + true + } + + fn merged_keys_hide_other_items(&self) -> bool { + true + } + + fn join_conditions_read(&self) -> JoinConditionReach { + JoinConditionReach::Earlier + } + fn checks_call_signatures(&self) -> bool { true } diff --git a/crates/sqlquack-semantics/src/duckdb/profile/tests/test_statement_findings.rs b/crates/sqlquack-semantics/src/duckdb/profile/tests/test_statement_findings.rs index 2d8651296..fa825ca53 100644 --- a/crates/sqlquack-semantics/src/duckdb/profile/tests/test_statement_findings.rs +++ b/crates/sqlquack-semantics/src/duckdb/profile/tests/test_statement_findings.rs @@ -683,6 +683,60 @@ fn given_named_windows_when_resolving_names_then_only_the_windows_an_over_reads_ sql: "SELECT sum(amount) OVER w AS x FROM orders WINDOW w AS (ORDER BY nope)", expected_findings: &[("E201", "Unknown column 'nope' in table 'orders'", 65)], }, + QueryFindingsTestCase { + description: "a window named in another case, which DuckDB resolves to it", + sql: "SELECT sum(amount) OVER W AS x FROM orders WINDOW w AS (PARTITION BY nope)", + expected_findings: &[("E201", "Unknown column 'nope' in table 'orders'", 69)], + }, + QueryFindingsTestCase { + description: "a used window extending an unknown window", + sql: "SELECT sum(amount) OVER w2 AS x FROM orders WINDOW w AS (ORDER BY id), w2 AS (zz ORDER BY id)", + expected_findings: &[("E232", "Unknown named window 'zz'", 78)], + }, + QueryFindingsTestCase { + description: "an unknown window two steps along the used window's chain", + sql: "SELECT sum(amount) OVER w3 AS x FROM orders WINDOW w2 AS (zz ORDER BY id), w3 AS (w2)", + expected_findings: &[("E232", "Unknown named window 'zz'", 58)], + }, + QueryFindingsTestCase { + description: "an unused window extending an unknown window", + sql: "SELECT amount FROM orders WINDOW w2 AS (zz ORDER BY id)", + expected_findings: &[], + }, + QueryFindingsTestCase { + description: "an unknown column in a used window's frame offset", + sql: "SELECT sum(amount) OVER w AS x FROM orders WINDOW w AS (ORDER BY id ROWS BETWEEN nope PRECEDING AND CURRENT ROW)", + expected_findings: &[("E201", "Unknown column 'nope' in table 'orders'", 81)], + }, + QueryFindingsTestCase { + description: "the same offset written inline", + sql: "SELECT sum(amount) OVER (ORDER BY id ROWS BETWEEN nope PRECEDING AND CURRENT ROW) AS x FROM orders", + expected_findings: &[("E201", "Unknown column 'nope' in table 'orders'", 50)], + }, + QueryFindingsTestCase { + description: "an unknown qualifier in a used window's end offset", + sql: "SELECT sum(amount) OVER w AS x FROM orders WINDOW w AS (ORDER BY id ROWS BETWEEN 1 PRECEDING AND zz.id FOLLOWING)", + expected_findings: &[( + "E222", + "Unknown table or alias 'zz' referenced by column 'id'", + 97, + )], + }, + QueryFindingsTestCase { + description: "a used window ordered by its own projection's alias, as inline (2.0: Binder Error)", + sql: "SELECT sum(amount) OVER w AS x FROM orders WINDOW w AS (ORDER BY x)", + expected_findings: &[("E201", "Unknown column 'x' in table 'orders'", 65)], + }, + QueryFindingsTestCase { + description: "a window shared by two projections, ordered by one's alias", + sql: "SELECT sum(amount) OVER w AS x, avg(amount) OVER w AS y FROM orders WINDOW w AS (ORDER BY y)", + expected_findings: &[("E201", "Unknown column 'y' in table 'orders'", 90)], + }, + QueryFindingsTestCase { + description: "an unknown column in a window shared by two projections is reported once", + sql: "SELECT sum(amount) OVER w AS x, avg(amount) OVER w AS y FROM orders WINDOW w AS (ORDER BY nope)", + expected_findings: &[("E201", "Unknown column 'nope' in table 'orders'", 90)], + }, QueryFindingsTestCase { description: "a window that the used window extends", sql: "SELECT sum(amount) OVER w2 AS x FROM orders WINDOW w AS (PARTITION BY nope), w2 AS (w ORDER BY id)", @@ -716,6 +770,58 @@ fn given_a_comma_after_a_joined_item_when_resolving_names_then_the_join_reads_on sql: "SELECT * FROM orders a JOIN orders b USING (status), customers c", expected_findings: &[], }, + QueryFindingsTestCase { + description: "an ON condition reading a USING key only a later join merges (2.0: column not found)", + sql: "SELECT 1 FROM exports a JOIN exports b ON id IS NOT NULL JOIN orders c ON true JOIN customers d USING (id)", + expected_findings: &[( + "E201", + "Unknown column 'id' (not found in any referenced table)", + 42, + )], + }, + QueryFindingsTestCase { + description: "an ON condition reading a key a later join widens, beside an item offering the name (2.0: accepted)", + sql: "SELECT 1 FROM orders x JOIN (orders a JOIN customers b USING (id)) ON true JOIN exports c ON id IS NOT NULL JOIN customers d USING (id)", + expected_findings: &[], + }, + QueryFindingsTestCase { + description: "an ON condition reading a USING key a later join merges again (2.0: accepted)", + sql: "SELECT 1 FROM orders a JOIN customers b USING (id) JOIN exports c ON id IS NOT NULL JOIN customers d USING (id)", + expected_findings: &[], + }, + QueryFindingsTestCase { + description: "an ON condition reading a later comma group's item (2.0: table not found)", + sql: "SELECT 1 FROM orders a JOIN customers c ON b.id = c.id, customers b", + expected_findings: &[( + "E222", + "Unknown table or alias 'b' referenced by column 'id'", + 43, + )], + }, + QueryFindingsTestCase { + description: "an ON condition reading a later item of its own join chain (2.0: table not found)", + sql: "SELECT 1 FROM orders a JOIN customers b ON c.id = a.id JOIN customers c ON true", + expected_findings: &[( + "E222", + "Unknown table or alias 'c' referenced by column 'id'", + 43, + )], + }, + QueryFindingsTestCase { + description: "an ON condition reading an earlier comma group's item (2.0: accepted)", + sql: "SELECT 1 FROM orders a, customers b JOIN customers c ON a.id = c.id", + expected_findings: &[], + }, + QueryFindingsTestCase { + description: "an ON condition reading an earlier comma group's column (2.0: accepted)", + sql: "SELECT 1 FROM orders a, customers b JOIN customers c ON customer_id = c.id", + expected_findings: &[], + }, + QueryFindingsTestCase { + description: "a parenthesised join's ON condition reading an earlier comma item (2.0: accepted)", + sql: "SELECT 1 FROM orders a, (customers b JOIN customers c ON a.id = c.id)", + expected_findings: &[], + }, QueryFindingsTestCase { description: "an unqualified USING key beside a later item's column of its name, which DuckDB binds", sql: "SELECT id FROM orders a JOIN customers b USING (id), exports c CROSS JOIN customers d", @@ -732,6 +838,55 @@ fn given_a_comma_after_a_joined_item_when_resolving_names_then_the_join_reads_on } } +#[test] +fn given_a_natural_join_when_resolving_names_then_sides_sharing_no_column_are_refused() { + let test_cases = [ + QueryFindingsTestCase { + description: "a NATURAL join of tables sharing no column (2.0: no columns found to join on)", + sql: "SELECT * FROM orders NATURAL JOIN shop.products", + expected_findings: &[("E201", "NATURAL JOIN has no column both sides offer", 39)], + }, + QueryFindingsTestCase { + description: "a NATURAL LEFT join sharing no column", + sql: "SELECT * FROM customers NATURAL LEFT JOIN exports", + expected_findings: &[("E201", "NATURAL JOIN has no column both sides offer", 42)], + }, + QueryFindingsTestCase { + description: "a NATURAL join whose comma neighbour, not its left item, shares a column", + sql: "SELECT * FROM customers c, shop.products p NATURAL JOIN orders o", + expected_findings: &[("E201", "NATURAL JOIN has no column both sides offer", 56)], + }, + QueryFindingsTestCase { + description: "a NATURAL join of a derived table sharing no column", + sql: "SELECT * FROM orders NATURAL JOIN (SELECT 1 AS z) s", + expected_findings: &[("E201", "NATURAL JOIN has no column both sides offer", 34)], + }, + QueryFindingsTestCase { + description: "a second NATURAL join whose right shares no column with the joined left", + sql: "SELECT * FROM orders o NATURAL JOIN customers c NATURAL JOIN exports e", + expected_findings: &[("E201", "NATURAL JOIN has no column both sides offer", 61)], + }, + QueryFindingsTestCase { + description: "a NATURAL join of tables sharing a column", + sql: "SELECT * FROM orders NATURAL JOIN customers", + expected_findings: &[], + }, + QueryFindingsTestCase { + description: "a NATURAL join sharing a column spelled in another case", + sql: "SELECT * FROM orders NATURAL JOIN (SELECT 1 AS ID) s", + expected_findings: &[], + }, + ]; + for test_case in test_cases { + assert_eq!( + duckdb_parser_findings(test_case.sql), + owned_findings(test_case.expected_findings), + "{}", + test_case.description + ); + } +} + #[test] fn given_joined_and_comma_items_when_expanding_a_star_then_the_columns_follow_the_written_order() { let test_cases = [ diff --git a/crates/sqlquack-semantics/src/lineage/_helpers/checks/scope_rules.rs b/crates/sqlquack-semantics/src/lineage/_helpers/checks/scope_rules.rs index 267cd2ead..7fe3cfb0d 100644 --- a/crates/sqlquack-semantics/src/lineage/_helpers/checks/scope_rules.rs +++ b/crates/sqlquack-semantics/src/lineage/_helpers/checks/scope_rules.rs @@ -279,6 +279,40 @@ fn output_aliases<'s>( aliases } +/// An unknown base of a window an OVER in `read` reaches, as `w2 AS (zz ORDER BY b)`. +fn unknown_bases<'s>( + (node, select): (&Expression, &'s Select), + mut read: Vec<&'s Identifier>, + level: RuleLevel<'_, '_>, + known: &dyn Fn(&Identifier) -> bool, +) -> Vec { + let mut findings: Vec = Vec::new(); + let mut seen: HashSet = HashSet::new(); + while let Some(name) = read.pop() { + if !seen.insert(level.key(name)) { + continue; + } + let definitions = select.windows.iter().flatten().filter(|window| { + level + .context + .profile + .same_identifier(&level.key(&window.name), &level.key(name)) + }); + for base in definitions.filter_map(|window| window.spec.window_name.as_ref()) { + match known(base) { + true => read.push(base), + false => findings.push(issue( + ("E232", SCOPE_PHASE), + format!("Unknown named window '{}'", base.name), + base.span, + node, + )), + } + } + } + findings +} + /// Duplicate names, unknown named windows, QUALIFY without a window, and bad LIMIT or OFFSET. fn structure_rules( (node, select): (&Expression, &Select), @@ -334,28 +368,38 @@ fn structure_rules( )); } } - let windows: HashSet<&str> = level + let windows: Vec<&str> = level .scope .windows .iter() .map(|(name, _)| name.as_str()) .collect(); + let known = |name: &Identifier| { + windows.iter().any(|known| { + level + .context + .profile + .same_identifier(known, &level.key(name)) + }) + }; let mut has_window: bool = false; + let mut read: Vec<&Identifier> = Vec::new(); for expression in in_scope(node) { if let Expression::WindowFunction(window) = expression { has_window = true; - if let Some(name) = &window.over.window_name - && !windows.contains(level.key(name).as_str()) - { - findings.push(rule( + match &window.over.window_name { + Some(name) if !known(name) => findings.push(rule( "E232", format!("Unknown named window '{}'", name.name), expression, node, - )); + )), + Some(name) => read.push(name), + None => {} } } } + findings.extend(unknown_bases((node, select), read, level, &known)); match engine { Engine::Postgres | Engine::Sqlite | Engine::Snowflake | Engine::DuckDB => { let rules = dialect_rules(level.context.profile); diff --git a/crates/sqlquack-semantics/src/lineage/_helpers/findings/checks.rs b/crates/sqlquack-semantics/src/lineage/_helpers/findings/checks.rs index 29ce9a360..f8a83d830 100644 --- a/crates/sqlquack-semantics/src/lineage/_helpers/findings/checks.rs +++ b/crates/sqlquack-semantics/src/lineage/_helpers/findings/checks.rs @@ -11,7 +11,7 @@ use crate::lineage::_helpers::checks::scope_rules::output_width; use crate::lineage::_helpers::tracing::queries::empty_scope; use crate::lineage::_helpers::tracing::resolution::key; use crate::lineage::_helpers::tracing::reuse::holds_unpivot; -use crate::lineage::_helpers::tracing::sources::{from_parts, item_sources, sided_parts}; +use crate::lineage::_helpers::tracing::sources::{from_parts, item_sources, natural, sided_parts}; use crate::lineage::constants::{ ALIAS_COUNT_CODE, AMBIGUOUS_COLUMN_CODE, COLUMNS_NO_MATCH, CONNECT_BY_LEVEL, NAME_PHASE, UNKNOWN_COLUMN_CODE, UNKNOWN_QUALIFIER_CODE, @@ -19,8 +19,8 @@ use crate::lineage::constants::{ use crate::lineage::constants::{ METADATA_PREFIX, POSITIONAL_PREFIX, SEQUENCE_COLUMNS, SINGLE_JOIN_SOURCES, SQL_STAR, }; -use crate::lineage::models::NameFinding; -use crate::lineage::models::{LineageContext, LineageScope, NamedQuery, ScopeSource}; +use crate::lineage::models::{Engine, NameFinding}; +use crate::lineage::models::{LineageContext, LineageScope, MergedKey, NamedQuery, ScopeSource}; /// The level a clause is read in: its scope, the aliases it may read, CONNECT BY and filter joins. #[derive(Clone, Copy)] @@ -32,6 +32,8 @@ pub(crate) struct Level<'s, 'a> { pub(crate) own: &'s [String], /// The name of the projection being read, which hides own aliases spelled the same. pub(crate) hidden: Option<&'s str>, + /// Whether the keys of a window the clause reads may read this level's output aliases. + pub(crate) windows_read_outputs: bool, pub(crate) connect: bool, /// Whether a SEMI or ANTI join filters the level, its right side hidden from the outputs. pub(crate) filtering_join: bool, @@ -102,11 +104,7 @@ pub(crate) fn column_finding( } let mut scope: Option<&LineageScope<'_>> = Some(level.scope); while let Some(current) = scope { - if current - .merged - .iter() - .any(|merged| context.profile.same_identifier(&merged.column.key, &wanted)) - { + if merged_answers(current, &wanted, context) { return None; } let matches: usize = offering_count(current, &wanted, context); @@ -392,7 +390,7 @@ fn unknown_message(written: &str, scope: &LineageScope<'_>, context: LineageCont } } -/// A USING key a closed side of its join lacks. +/// A USING key a closed side of its join lacks, or on DuckDB a NATURAL join of closed sides sharing no name. pub(crate) fn using_findings( select: &Select, scope: &LineageScope<'_>, @@ -413,6 +411,18 @@ pub(crate) fn using_findings( continue; }; let (left, right) = (sources(part.left), sources(part.right)); + if natural(join.kind) + && Engine::of(context) == Engine::DuckDB + && !shares_name(left, right, context) + { + findings.push(NameFinding { + code: UNKNOWN_COLUMN_CODE, + message: "NATURAL JOIN has no column both sides offer".to_owned(), + span: join.this.source_span().or(at), + phase: NAME_PHASE, + ..NameFinding::default() + }); + } for name in &join.using { let wanted: String = key(name, context); if lacks(left, &wanted, context) || lacks(right, &wanted, context) { @@ -430,6 +440,15 @@ pub(crate) fn using_findings( findings } +/// Whether a side of a join is open or offers a column the other side offers. +fn shares_name(left: &[ScopeSource], right: &[ScopeSource], context: LineageContext<'_>) -> bool { + left.iter().chain(right).any(unjudged) + || right + .iter() + .flat_map(|source| source.query.columns.iter()) + .any(|column| !lacks(left, &column.key, context)) +} + /// Whether a closed side of a join offers no column named `wanted`. fn lacks(side: &[ScopeSource], wanted: &str, context: LineageContext<'_>) -> bool { !side.iter().any(unjudged) && !side.iter().any(|source| offers(source, wanted, context)) @@ -487,6 +506,35 @@ fn offers(source: &ScopeSource, wanted: &str, context: LineageContext<'_>) -> bo .contains(&wanted)) } +/// Whether a USING or NATURAL key named `wanted` answers it: where other items may share the name, alone. +fn merged_answers(scope: &LineageScope<'_>, wanted: &str, context: LineageContext<'_>) -> bool { + let named = |merged: &&MergedKey| context.profile.same_identifier(&merged.column.key, wanted); + if !scope.merged.iter().any(|merged| named(&merged)) { + return false; + } + if dialect_rules(context.profile).merged_keys_hide_other_items() { + return true; + } + let keys: Vec<&std::ops::Range> = scope + .merged + .iter() + .filter(named) + .map(|merged| &merged.sources) + .collect(); + let uncovered: usize = scope + .sources + .iter() + .enumerate() + .filter(|(index, source)| !covered(&keys, *index) && offers(source, wanted, context)) + .count(); + keys.len() + uncovered <= 1 +} + +/// Whether one of the merged keys' joins holds the `index`th source. +fn covered(keys: &[&std::ops::Range], index: usize) -> bool { + keys.iter().any(|range| range.contains(&index)) +} + /// How many FROM items offer `wanted`, two under one name counted once. fn offering_count(scope: &LineageScope<'_>, wanted: &str, context: LineageContext<'_>) -> usize { let mut names: Vec<&str> = Vec::new(); diff --git a/crates/sqlquack-semantics/src/lineage/_helpers/findings/levels.rs b/crates/sqlquack-semantics/src/lineage/_helpers/findings/levels.rs index 0afe8fc55..74ec3748b 100644 --- a/crates/sqlquack-semantics/src/lineage/_helpers/findings/levels.rs +++ b/crates/sqlquack-semantics/src/lineage/_helpers/findings/levels.rs @@ -5,7 +5,7 @@ use std::collections::HashMap; use sqlquack_syntax::Expression; use sqlquack_syntax::ExpressionWalk; use sqlquack_syntax::expressions::queries::models::{ - JoinKind, NamedWindow, OrderBy, Select, Subquery, With, + JoinKind, NamedWindow, OrderBy, Over, Select, Subquery, WindowFrameBound, With, }; use crate::dialects::main::dialect_rules::dialect_rules; @@ -26,11 +26,11 @@ use crate::lineage::_helpers::tracing::queries::{ }; use crate::lineage::_helpers::tracing::resolution::key; use crate::lineage::_helpers::tracing::reuse::{Reach, Reuse, reusing}; -use crate::lineage::_helpers::tracing::sources::{every_join, from_parts}; +use crate::lineage::_helpers::tracing::sources::{every_join, from_parts, sided_parts}; use crate::lineage::constants::{COLUMNS_FUNCTION, OUTER_RIGHT_JOINS}; -use crate::lineage::models::Engine; use crate::lineage::models::NameFinding; -use crate::lineage::models::{Lineage, LineageContext, LineageScope, NamedQuery}; +use crate::lineage::models::{Engine, JoinConditionReach}; +use crate::lineage::models::{Lineage, LineageContext, LineageScope, MergedKey, NamedQuery}; use crate::lineage::types::DialectRules; /// The findings of `query` and its inner queries; `aliases` are enclosing levels' output aliases. @@ -205,6 +205,67 @@ fn left_scope<'a>( }) } +/// Each join condition of `select`, with the FROM items `reach` lets it read. +fn condition_scopes<'s, 'a>( + select: &'s Select, + scope: &LineageScope<'a>, + reach: JoinConditionReach, +) -> Vec<(&'s Expression, LineageScope<'a>)> { + let at = |part: usize| { + scope.starts.get(part).map_or(scope.sources.len(), |start| { + (*start).min(scope.sources.len()) + }) + }; + let mut scopes: Vec<(&'s Expression, LineageScope<'a>)> = Vec::new(); + for part in sided_parts(select) { + let Some(join) = part.join else { + continue; + }; + let first: usize = match reach { + JoinConditionReach::OwnGroup => at(part.left.start), + _ => 0, + }; + let reached: std::ops::Range = first..at(part.right.end); + for condition in join.on.iter().chain(join.match_condition.iter()) { + scopes.push((condition, reached_scope(scope, reached.clone()))); + } + } + scopes +} + +/// `scope` narrowed to the sources in `reached`, with each merged key as it stood within them. +fn reached_scope<'a>( + scope: &LineageScope<'a>, + reached: std::ops::Range, +) -> LineageScope<'a> { + LineageScope { + sources: scope.sources[reached.clone()].to_vec(), + merged: scope + .merged + .iter() + .filter_map(|merged| { + let range = reached_range(merged, &reached)?; + Some(MergedKey { + sources: range.start - reached.start..range.end - reached.start, + earlier: Vec::new(), + ..merged.clone() + }) + }) + .collect(), + ..empty_scope(scope.ctes, scope.outer) + } +} + +/// The latest sources a merged key stood for wholly inside `reached`, if it was merged by then. +fn reached_range<'k>( + merged: &'k MergedKey, + reached: &std::ops::Range, +) -> Option<&'k std::ops::Range> { + std::iter::once(&merged.sources) + .chain(merged.earlier.iter().rev()) + .find(|range| reached.start <= range.start && range.end <= reached.end) +} + /// The key of a derived table in FROM, by its alias; none when `item` is not a derived table. fn derived_key(item: &Expression, context: LineageContext<'_>) -> Option> { match item { @@ -314,6 +375,7 @@ fn select_findings( enclosing: aliases, own: &own_outputs, hidden: None, + windows_read_outputs: true, connect: select.connect.is_some(), filtering_join: every_join(select).into_iter().any(|join| { matches!( @@ -336,8 +398,14 @@ fn select_findings( let own_name: Option = anywhere.then(|| output_key(unannotated(projection), context)); let projected = match &own_name { + Some(_) if !rules.select_list_reads_outputs() => Level { + own: &[], + windows_read_outputs: true, + ..level + }, Some(own_name) => Level { hidden: Some(own_name), + windows_read_outputs: rules.window_keys_read_outputs(), ..level }, None => unnamed, @@ -348,6 +416,10 @@ fn select_findings( } let mut read: Vec<&Expression> = clauses(select); read.extend(orders.iter().flat_map(|order| order_keys(order))); + let conditions: Vec<(&Expression, LineageScope<'_>)> = match rules.join_conditions_read() { + JoinConditionReach::Every => Vec::new(), + reach => condition_scopes(select, &scope, reach), + }; for clause in read { let named_place: bool = ordered_name_place(select, &orders, clause) || (rules.group_keys_read_outputs() && group_key_place(select, clause)); @@ -359,6 +431,13 @@ fn select_findings( }, (false, false) => unnamed, }; + let read = match conditions.iter().find(|(on, _)| std::ptr::eq(*on, clause)) { + Some((_, reached)) => Level { + scope: reached, + ..read + }, + None => read, + }; let (found, inner) = clause_findings(clause, &read, context); own.extend(found); nested.extend(inner); @@ -371,13 +450,9 @@ fn select_findings( let reads: Vec<&Expression> = std::iter::once(node) .chain(orders.iter().flat_map(|order| order_keys(order))) .collect(); - let windowed = if anywhere { level } else { unnamed }; - let used: Vec<&NamedWindow> = used_windows(windows, &reads, context); - for key in used.into_iter().flat_map(window_keys) { - let (found, inner) = clause_findings(key, &windowed, context); - own.extend(found); - nested.extend(inner); - } + let (found, inner) = window_findings(select, (windows, &reads), (level, anywhere), context); + own.extend(found); + nested.extend(inner); own.sort_by_key(|finding| finding.span.map(|span| span.start)); own.extend(nested); own @@ -417,6 +492,81 @@ fn ordered_name_place(select: &Select, after: &[&OrderBy], clause: &Expression) .any(|ordered| std::ptr::eq(&ordered.this, clause)) } +/// The findings in the windows OVERs in `reads` reach, each read as the projection reaching it reads. +fn window_findings( + select: &Select, + (windows, reads): (Vec<&NamedWindow>, &[&Expression]), + (level, anywhere): (Level<'_, '_>, bool), + context: LineageContext<'_>, +) -> (Vec, Vec) { + let used: Vec<&NamedWindow> = used_windows(windows.clone(), reads, context); + let names: Vec = select + .expressions + .iter() + .map(|projection| output_key(unannotated(projection), context)) + .collect(); + let unnamed = Level { + enclosing: &[], + own: &[], + ..level + }; + let rules: &dyn DialectRules = dialect_rules(context.profile); + let projected_reads: bool = rules.window_keys_read_outputs(); + let unprojected = match rules.select_list_reads_outputs() { + true => unnamed, + false => Level { own: &[], ..level }, + }; + let mut readers: Vec<(Level<'_, '_>, Vec<&NamedWindow>)> = match anywhere { + true => select + .expressions + .iter() + .zip(&names) + .map(|(projection, name)| { + let reached = used_windows(windows.clone(), &[projection], context); + let reader = match projected_reads { + true => Level { + hidden: Some(name.as_str()), + ..level + }, + false => unprojected, + }; + (reader, reached) + }) + .collect(), + false => Vec::new(), + }; + let projected: Vec<&NamedWindow> = readers + .iter() + .flat_map(|(_, reached)| reached.iter().copied()) + .collect(); + let mut rest: Vec<&NamedWindow> = Vec::new(); + for window in used { + if !projected.iter().any(|read| std::ptr::eq(*read, window)) { + rest.push(window); + } + } + readers.push((if anywhere { level } else { unnamed }, rest)); + let (mut own, mut nested): (Vec, Vec) = (Vec::new(), Vec::new()); + for (reader, reached) in readers { + for key in reached.into_iter().flat_map(window_keys) { + let (found, inner) = clause_findings(key, &reader, context); + own.extend( + found + .into_iter() + .filter(|finding| !own.contains(finding)) + .collect::>(), + ); + nested.extend( + inner + .into_iter() + .filter(|finding| !nested.contains(finding)) + .collect::>(), + ); + } + } + (own, nested) +} + /// The windows of `windows` an OVER in `reads` names, directly or through a window extending it. fn used_windows<'w>( windows: Vec<&'w NamedWindow>, @@ -462,10 +612,23 @@ fn used_windows<'w>( .collect() } -/// The partition and order keys of `window`. +/// The partition and order keys of `window`, and its frame's offsets. fn window_keys(window: &NamedWindow) -> impl Iterator { - let orders = window.spec.order_by.iter().map(|ordered| &ordered.this); - window.spec.partition_by.iter().chain(orders) + over_keys(&window.spec) +} + +/// The partition and order keys of `over`, and its frame's offsets. +fn over_keys(over: &Over) -> impl Iterator { + let orders = over.order_by.iter().map(|ordered| &ordered.this); + let frame = over.frame.iter(); + let bounds = frame.flat_map(|frame| std::iter::once(&frame.start).chain(&frame.end)); + let offsets = bounds.filter_map(|bound| match bound { + WindowFrameBound::Preceding(offset) + | WindowFrameBound::Following(offset) + | WindowFrameBound::Value(offset) => Some(offset.as_ref()), + _ => None, + }); + over.partition_by.iter().chain(orders).chain(offsets) } /// The keys of `order`. @@ -579,6 +742,27 @@ fn clause_findings( bound.extend(lambda.parameters.iter().map(|name| key(name, context))); pending.push(&lambda.body); } + Expression::WindowFunction(window) if !level.windows_read_outputs => { + pending.push(&window.this); + pending.extend( + window + .keep + .iter() + .flat_map(|keep| &keep.order_by) + .map(|ordered| &ordered.this), + ); + let keys = Level { + enclosing: &[], + own: &[], + windows_read_outputs: true, + ..*level + }; + for key in over_keys(&window.over) { + let (found, inner) = clause_findings(key, &keys, context); + own.extend(found); + nested.extend(inner); + } + } _ => { for child in node.children() { if !date_part_argument(node, child, context) { diff --git a/crates/sqlquack-semantics/src/lineage/_helpers/tracing/sources.rs b/crates/sqlquack-semantics/src/lineage/_helpers/tracing/sources.rs index 9e9eee1e3..a0deb2885 100644 --- a/crates/sqlquack-semantics/src/lineage/_helpers/tracing/sources.rs +++ b/crates/sqlquack-semantics/src/lineage/_helpers/tracing/sources.rs @@ -285,6 +285,7 @@ fn joined( expression: None, }, sources: reach.clone(), + earlier: Vec::new(), }; match merged_keys.iter_mut().find(|merged| { overlaps(&merged.sources, &reach) @@ -295,7 +296,13 @@ fn joined( Some(merged) => { let sources: Range = merged.sources.start.min(reach.start)..merged.sources.end.max(reach.end); - *merged = MergedKey { sources, ..column }; + let mut earlier: Vec> = std::mem::take(&mut merged.earlier); + earlier.push(merged.sources.clone()); + *merged = MergedKey { + sources, + earlier, + ..column + }; } None => merged_keys.push(column), } @@ -303,7 +310,8 @@ fn joined( (merged_keys, rows) } -fn natural(kind: JoinKind) -> bool { +/// Whether a join is NATURAL, joining on the column names both sides offer. +pub(crate) fn natural(kind: JoinKind) -> bool { matches!( kind, JoinKind::Natural | JoinKind::NaturalLeft | JoinKind::NaturalRight | JoinKind::NaturalFull diff --git a/crates/sqlquack-semantics/src/lineage/models.rs b/crates/sqlquack-semantics/src/lineage/models.rs index 702825e77..c6316e569 100644 --- a/crates/sqlquack-semantics/src/lineage/models.rs +++ b/crates/sqlquack-semantics/src/lineage/models.rs @@ -161,6 +161,8 @@ pub(crate) struct MergedKey { pub(crate) column: ScopedColumn, /// The sources of the join, whose columns of the key's name it stands in for. pub(crate) sources: Range, + /// The sources it stood for before later joins merged it again, oldest first. + pub(crate) earlier: Vec>, } /// A FROM item: its key, its known columns, and whether it offers others of unknown name. @@ -282,6 +284,22 @@ impl DialectRules for SharedRules { false } + fn window_keys_read_outputs(&self) -> bool { + true + } + + fn select_list_reads_outputs(&self) -> bool { + true + } + + fn merged_keys_hide_other_items(&self) -> bool { + true + } + + fn join_conditions_read(&self) -> JoinConditionReach { + JoinConditionReach::Every + } + fn checks_call_signatures(&self) -> bool { true } @@ -415,6 +433,17 @@ impl DialectRules for SharedRules { } } +/// Which FROM items a join's ON condition may read. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum JoinConditionReach { + /// Every FROM item of its query level. + Every, + /// Every item before the join, and the join's own item. + Earlier, + /// The items of its own comma group before the join, and the join's own item. + OwnGroup, +} + /// The engine whose binder the checks follow. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(crate) enum Engine { diff --git a/crates/sqlquack-semantics/src/lineage/types.rs b/crates/sqlquack-semantics/src/lineage/types.rs index 02e6b064e..65fc5fa89 100644 --- a/crates/sqlquack-semantics/src/lineage/types.rs +++ b/crates/sqlquack-semantics/src/lineage/types.rs @@ -8,7 +8,7 @@ use sqlquack_syntax::{DataType, Expression}; use crate::dialects::types::DialectProfile; use crate::lineage::_helpers::checks::window_only::named_window_only; use crate::lineage::main::rendered_type::rendered_type; -use crate::lineage::models::{Engine, Nullability}; +use crate::lineage::models::{Engine, JoinConditionReach, Nullability}; use crate::lineage::models::{ Lineage, LineageContext, NullabilityRule, SchemaColumn, SourceColumn, }; @@ -72,6 +72,18 @@ pub(crate) trait DialectRules: Sync { /// Where outputs are not read anywhere, whether GROUP BY keys may read them as ORDER BY keys do. fn group_keys_read_outputs(&self) -> bool; + /// Where outputs are read anywhere, whether the keys of a window the select list reads may read them. + fn window_keys_read_outputs(&self) -> bool; + + /// Where outputs are read anywhere, whether the select list may read them too. + fn select_list_reads_outputs(&self) -> bool; + + /// Whether an unqualified USING or NATURAL key reads its merged column though an item outside its join offers the name. + fn merged_keys_hide_other_items(&self) -> bool; + + /// Which FROM items a join's ON condition may read. + fn join_conditions_read(&self) -> JoinConditionReach; + /// Whether calls are checked against the signatures of the functions they name. fn checks_call_signatures(&self) -> bool; diff --git a/crates/sqlquack-semantics/src/postgres/profile/models.rs b/crates/sqlquack-semantics/src/postgres/profile/models.rs index 76d74ac21..5006d5bd3 100644 --- a/crates/sqlquack-semantics/src/postgres/profile/models.rs +++ b/crates/sqlquack-semantics/src/postgres/profile/models.rs @@ -8,7 +8,7 @@ use crate::dialects::types::DialectProfile; use crate::lineage::constants::{ DATE_PART_FUNCTIONS, SNOWFLAKE_VALUES_COLUMN, WINDOW_ONLY_FUNCTIONS, }; -use crate::lineage::models::Engine; +use crate::lineage::models::{Engine, JoinConditionReach}; use crate::lineage::models::{Lineage, LineageContext}; use crate::lineage::types::DialectRules; use crate::lineage::types::ValueType; @@ -65,6 +65,22 @@ impl DialectRules for PostgresRules { true } + fn window_keys_read_outputs(&self) -> bool { + false + } + + fn select_list_reads_outputs(&self) -> bool { + true + } + + fn merged_keys_hide_other_items(&self) -> bool { + false + } + + fn join_conditions_read(&self) -> JoinConditionReach { + JoinConditionReach::OwnGroup + } + fn checks_call_signatures(&self) -> bool { true } diff --git a/crates/sqlquack-semantics/src/postgres/profile/tests/test_statement_findings.rs b/crates/sqlquack-semantics/src/postgres/profile/tests/test_statement_findings.rs index c819ac4bd..105f4ed65 100644 --- a/crates/sqlquack-semantics/src/postgres/profile/tests/test_statement_findings.rs +++ b/crates/sqlquack-semantics/src/postgres/profile/tests/test_statement_findings.rs @@ -289,6 +289,21 @@ fn given_a_named_window_when_resolving_names_then_its_keys_read_the_input_column sql: "SELECT amount AS q, sum(amount) OVER w AS x FROM orders WINDOW w AS (ORDER BY q)", expected_findings: &[("E201", "Unknown column 'q' in table 'orders'", 78)], }, + QueryFindingsTestCase { + description: "a quoted window name keeps its case, so W is not w (18.6: 42704)", + sql: "SELECT sum(amount) OVER \"W\" AS x FROM orders WINDOW w AS (ORDER BY id)", + expected_findings: &[("E232", "Unknown named window 'W'", 11)], + }, + QueryFindingsTestCase { + description: "a quoted window name read by the same quoted name", + sql: "SELECT sum(amount) OVER \"W\" AS x FROM orders WINDOW \"W\" AS (ORDER BY id)", + expected_findings: &[], + }, + QueryFindingsTestCase { + description: "an unquoted name folds to the quoted lower-case window", + sql: "SELECT sum(amount) OVER (W ORDER BY id) AS x FROM orders WINDOW \"w\" AS (PARTITION BY nope)", + expected_findings: &[("E201", "Unknown column 'nope' in table 'orders'", 85)], + }, ]; let schema = duckdb_shop_schema(); for test_case in test_cases { @@ -343,6 +358,153 @@ fn given_a_comma_after_a_joined_item_when_resolving_names_then_the_join_reads_on sql: "SELECT * FROM orders a JOIN orders b USING (status), customers c", expected_findings: &[], }, + QueryFindingsTestCase { + description: "an ON condition reading a USING key only a later join merges (18: column does not exist)", + sql: "SELECT 1 FROM exports a JOIN exports b ON id IS NOT NULL JOIN orders c ON true JOIN customers d USING (id)", + expected_findings: &[( + "E201", + "Unknown column 'id' (not found in any referenced table)", + 42, + )], + }, + QueryFindingsTestCase { + description: "an ON condition reading a key a later join widens past an item offering the name (18: ambiguous)", + sql: "SELECT 1 FROM orders x JOIN (orders a JOIN customers b USING (id)) ON true JOIN exports c ON id IS NOT NULL JOIN customers d USING (id)", + expected_findings: &[( + "E221", + "Ambiguous unqualified column 'id' found in 3 referenced tables", + 93, + )], + }, + QueryFindingsTestCase { + description: "an ON condition reading a USING key a later join merges again (18: accepted)", + sql: "SELECT 1 FROM orders a JOIN customers b USING (id) JOIN exports c ON id IS NOT NULL JOIN customers d USING (id)", + expected_findings: &[], + }, + QueryFindingsTestCase { + description: "an ON condition reading a NATURAL key a later join merges again (18: accepted)", + sql: "SELECT 1 FROM orders a NATURAL JOIN customers b JOIN exports c ON id IS NOT NULL NATURAL JOIN customers d", + expected_findings: &[], + }, + QueryFindingsTestCase { + description: "an ON condition reading an earlier comma group's item (18: invalid reference)", + sql: "SELECT 1 FROM orders a, customers b JOIN customers c ON a.id = c.id", + expected_findings: &[( + "E222", + "Unknown table or alias 'a' referenced by column 'id'", + 56, + )], + }, + QueryFindingsTestCase { + description: "an ON condition reading an earlier comma group's column (18: cannot be referenced)", + sql: "SELECT 1 FROM orders a, customers b JOIN customers c ON customer_id = c.id", + expected_findings: &[( + "E201", + "Unknown column 'customer_id' (not found in any referenced table)", + 56, + )], + }, + QueryFindingsTestCase { + description: "an ON condition reading a later comma group's item (18: missing FROM entry)", + sql: "SELECT 1 FROM orders a JOIN customers c ON b.id = c.id, customers b", + expected_findings: &[( + "E222", + "Unknown table or alias 'b' referenced by column 'id'", + 43, + )], + }, + QueryFindingsTestCase { + description: "an ON condition reading a later item of its own join chain (18: missing FROM entry)", + sql: "SELECT 1 FROM orders a JOIN customers b ON c.id = a.id JOIN customers c ON true", + expected_findings: &[( + "E222", + "Unknown table or alias 'c' referenced by column 'id'", + 43, + )], + }, + QueryFindingsTestCase { + description: "a parenthesised join's ON condition reading an earlier comma item (18: invalid reference)", + sql: "SELECT 1 FROM orders a, (customers b JOIN customers c ON a.id = c.id)", + expected_findings: &[( + "E222", + "Unknown table or alias 'a' referenced by column 'id'", + 57, + )], + }, + QueryFindingsTestCase { + description: "a subquery in an ON condition reading an earlier comma group's item (18: invalid reference)", + sql: "SELECT 1 FROM orders a, customers b JOIN customers c ON (SELECT a.id) = c.id", + expected_findings: &[( + "E222", + "Unknown table or alias 'a' referenced by column 'id'", + 64, + )], + }, + QueryFindingsTestCase { + description: "an ON condition reading its own comma group (18: accepted)", + sql: "SELECT 1 FROM orders a, customers b JOIN customers c ON b.id = c.id", + expected_findings: &[], + }, + QueryFindingsTestCase { + description: "an ON condition reading its group's merged key, an earlier group offering the name (18: accepted)", + sql: "SELECT 1 FROM customers a, customers b JOIN customers c USING (name) JOIN orders d ON name = d.status", + expected_findings: &[], + }, + QueryFindingsTestCase { + description: "a USING key read unqualified beside a comma item offering it (18: ambiguous)", + sql: "SELECT id FROM orders a JOIN customers b USING (id), customers c", + expected_findings: &[( + "E221", + "Ambiguous unqualified column 'id' found in 3 referenced tables", + 7, + )], + }, + QueryFindingsTestCase { + description: "a USING key read unqualified beside a later ON join offering it (18: ambiguous)", + sql: "SELECT id FROM orders a JOIN customers b USING (id) JOIN customers c ON c.id = a.id", + expected_findings: &[( + "E221", + "Ambiguous unqualified column 'id' found in 3 referenced tables", + 7, + )], + }, + QueryFindingsTestCase { + description: "a USING key read unqualified beside an earlier comma item offering it (18: ambiguous)", + sql: "SELECT id FROM orders a, customers b JOIN customers c USING (id)", + expected_findings: &[( + "E221", + "Ambiguous unqualified column 'id' found in 3 referenced tables", + 7, + )], + }, + QueryFindingsTestCase { + description: "a NATURAL key read unqualified beside a comma item offering it (18: ambiguous)", + sql: "SELECT id FROM orders a NATURAL JOIN customers b, customers c", + expected_findings: &[( + "E221", + "Ambiguous unqualified column 'id' found in 3 referenced tables", + 7, + )], + }, + QueryFindingsTestCase { + description: "a USING key of a parenthesised join beside a later item offering it (18: ambiguous)", + sql: "SELECT id FROM (orders a JOIN customers b USING (id)) JOIN customers c ON true", + expected_findings: &[( + "E221", + "Ambiguous unqualified column 'id' found in 3 referenced tables", + 7, + )], + }, + QueryFindingsTestCase { + description: "a USING key a later USING join merges again (18: accepted)", + sql: "SELECT id FROM orders a JOIN customers b USING (id) JOIN customers c USING (id)", + expected_findings: &[], + }, + QueryFindingsTestCase { + description: "a USING key outer joins merge again (18: accepted)", + sql: "SELECT id FROM orders a LEFT JOIN customers b USING (id) RIGHT JOIN customers c USING (id)", + expected_findings: &[], + }, ]; let schema = duckdb_shop_schema(); for test_case in test_cases { diff --git a/crates/sqlquack-semantics/src/snowflake/profile/models.rs b/crates/sqlquack-semantics/src/snowflake/profile/models.rs index ed8a9a79e..d80152c28 100644 --- a/crates/sqlquack-semantics/src/snowflake/profile/models.rs +++ b/crates/sqlquack-semantics/src/snowflake/profile/models.rs @@ -8,7 +8,7 @@ use crate::dialects::types::DialectProfile; use crate::lineage::constants::{ DATE_PART_FUNCTIONS, SNOWFLAKE_VALUES_COLUMN, WINDOW_ONLY_FUNCTIONS, }; -use crate::lineage::models::Engine; +use crate::lineage::models::{Engine, JoinConditionReach}; use crate::lineage::models::{Lineage, LineageContext}; use crate::lineage::types::DialectRules; use crate::lineage::types::ValueType; @@ -103,6 +103,22 @@ impl DialectRules for SnowflakeRules { false } + fn window_keys_read_outputs(&self) -> bool { + true + } + + fn select_list_reads_outputs(&self) -> bool { + true + } + + fn merged_keys_hide_other_items(&self) -> bool { + true + } + + fn join_conditions_read(&self) -> JoinConditionReach { + JoinConditionReach::Every + } + fn checks_call_signatures(&self) -> bool { true } diff --git a/crates/sqlquack-semantics/src/sqlite/profile/models.rs b/crates/sqlquack-semantics/src/sqlite/profile/models.rs index 047ce4706..37ac788f5 100644 --- a/crates/sqlquack-semantics/src/sqlite/profile/models.rs +++ b/crates/sqlquack-semantics/src/sqlite/profile/models.rs @@ -6,7 +6,7 @@ use sqlquack_syntax::{DataType, Dialect, Expression}; use crate::dialects::types::DialectProfile; use crate::lineage::constants::{DATE_PART_FUNCTIONS, WINDOW_ONLY_FUNCTIONS}; -use crate::lineage::models::Engine; +use crate::lineage::models::{Engine, JoinConditionReach}; use crate::lineage::models::{Lineage, LineageContext, Nullability}; use crate::lineage::types::DialectRules; use crate::lineage::types::ValueType; @@ -58,6 +58,22 @@ impl DialectRules for SqliteRules { false } + fn window_keys_read_outputs(&self) -> bool { + false + } + + fn select_list_reads_outputs(&self) -> bool { + false + } + + fn merged_keys_hide_other_items(&self) -> bool { + false + } + + fn join_conditions_read(&self) -> JoinConditionReach { + JoinConditionReach::Every + } + fn checks_call_signatures(&self) -> bool { false } diff --git a/crates/sqlquack-semantics/src/sqlite/profile/tests/test_lineage.rs b/crates/sqlquack-semantics/src/sqlite/profile/tests/test_lineage.rs index 9f14a7315..601e34fad 100644 --- a/crates/sqlquack-semantics/src/sqlite/profile/tests/test_lineage.rs +++ b/crates/sqlquack-semantics/src/sqlite/profile/tests/test_lineage.rs @@ -41,6 +41,71 @@ fn given_a_sqlite_query_sqlite_rejects_when_checking_then_the_broken_rule_is_fou 0, )], }, + QueryFindingsTestCase { + description: "an inline window partitioned by another output's alias (3.40: no such column)", + sql: "SELECT quantity AS q, sum(quantity) OVER (PARTITION BY q) AS x FROM orders", + expected_findings: &[("E201", "Unknown column 'q' in table 'orders'", 55)], + }, + QueryFindingsTestCase { + description: "a named window ordered by another output's alias (3.40: no such column)", + sql: "SELECT quantity AS q, sum(quantity) OVER w AS x FROM orders WINDOW w AS (ORDER BY q)", + expected_findings: &[("E201", "Unknown column 'q' in table 'orders'", 82)], + }, + QueryFindingsTestCase { + description: "a projection reading an earlier output's alias (3.40: no such column)", + sql: "SELECT quantity AS q, q + 1 AS r FROM orders", + expected_findings: &[("E201", "Unknown column 'q' in table 'orders'", 22)], + }, + QueryFindingsTestCase { + description: "a projection reading a later output's alias (3.40: no such column)", + sql: "SELECT q + 1 AS r, quantity AS q FROM orders", + expected_findings: &[("E201", "Unknown column 'q' in table 'orders'", 7)], + }, + QueryFindingsTestCase { + description: "a subquery in the select list reading an output alias (3.40: no such column)", + sql: "SELECT quantity AS q, (SELECT q) AS r FROM orders", + expected_findings: &[( + "E201", + "Unknown column 'q' (not found in any referenced table)", + 30, + )], + }, + QueryFindingsTestCase { + description: "a USING key read unqualified beside a comma item offering it (3.40: ambiguous)", + sql: "SELECT id FROM orders a JOIN customers b USING (id), customers c", + expected_findings: &[( + "E221", + "Ambiguous unqualified column 'id' found in 3 referenced tables", + 7, + )], + }, + QueryFindingsTestCase { + description: "a USING key read unqualified beside a later ON join offering it (3.40: ambiguous)", + sql: "SELECT id FROM orders a JOIN customers b USING (id) JOIN customers c ON c.id = a.id", + expected_findings: &[( + "E221", + "Ambiguous unqualified column 'id' found in 3 referenced tables", + 7, + )], + }, + QueryFindingsTestCase { + description: "a NATURAL key read unqualified beside a comma item offering it (3.40: ambiguous)", + sql: "SELECT id FROM orders a NATURAL JOIN customers b, customers c", + expected_findings: &[( + "E221", + "Ambiguous unqualified column 'id' found in 3 referenced tables", + 7, + )], + }, + QueryFindingsTestCase { + description: "a USING key of a parenthesised join beside a later item offering it (3.40: ambiguous)", + sql: "SELECT id FROM (orders a JOIN customers b USING (id)) JOIN customers c ON true", + expected_findings: &[( + "E221", + "Ambiguous unqualified column 'id' found in 3 referenced tables", + 7, + )], + }, QueryFindingsTestCase { description: "a ranking function without OVER", sql: "SELECT ROW_NUMBER() AS r FROM orders", @@ -84,6 +149,56 @@ fn given_a_sqlite_query_sqlite_rejects_when_checking_then_the_broken_rule_is_fou #[test] fn given_a_sqlite_query_sqlite_accepts_when_checking_then_nothing_is_found() { let test_cases = [ + QueryFindingsTestCase { + description: "a nested select list's named window reading the enclosing query's alias (3.40: accepted)", + sql: "SELECT quantity AS q FROM orders WHERE EXISTS (SELECT sum(1) OVER w FROM customers WINDOW w AS (ORDER BY q))", + expected_findings: &[], + }, + QueryFindingsTestCase { + description: "an ON condition reading a later comma group's item (3.40: accepted)", + sql: "SELECT 1 FROM orders a JOIN customers c ON b.id = c.id, customers b", + expected_findings: &[], + }, + QueryFindingsTestCase { + description: "an ON condition reading a later item of its own join chain (3.40: accepted)", + sql: "SELECT 1 FROM orders a JOIN customers b ON c.id = a.id JOIN customers c ON true", + expected_findings: &[], + }, + QueryFindingsTestCase { + description: "a USING key a later USING join merges again (3.40: accepted)", + sql: "SELECT id FROM orders a JOIN customers b USING (id) JOIN customers c USING (id)", + expected_findings: &[], + }, + QueryFindingsTestCase { + description: "a USING key outer joins merge again (3.40: accepted)", + sql: "SELECT id FROM orders a LEFT JOIN customers b USING (id) RIGHT JOIN customers c USING (id)", + expected_findings: &[], + }, + QueryFindingsTestCase { + description: "an output alias read in WHERE, GROUP BY and HAVING (3.40: accepted)", + sql: "SELECT quantity AS q, count(*) AS n FROM orders WHERE q > 0 GROUP BY q HAVING q > 0", + expected_findings: &[], + }, + QueryFindingsTestCase { + description: "a subquery in WHERE reading an output alias (3.40: accepted)", + sql: "SELECT quantity AS q FROM orders WHERE (SELECT q) > 0", + expected_findings: &[], + }, + QueryFindingsTestCase { + description: "a subquery's select list reading the enclosing query's alias (3.40: accepted)", + sql: "SELECT quantity AS q FROM orders WHERE EXISTS (SELECT q)", + expected_findings: &[], + }, + QueryFindingsTestCase { + description: "a window in ORDER BY reading an output alias (3.40: accepted)", + sql: "SELECT quantity AS q FROM orders ORDER BY sum(quantity) OVER (ORDER BY q)", + expected_findings: &[], + }, + QueryFindingsTestCase { + description: "a named window only ORDER BY reads, ordered by an output alias (3.40: accepted)", + sql: "SELECT quantity AS q FROM orders WINDOW w AS (ORDER BY q) ORDER BY sum(quantity) OVER w", + expected_findings: &[], + }, QueryFindingsTestCase { description: "a set operation's ORDER BY naming an output in upper case", sql: "SELECT id FROM orders UNION ALL SELECT id FROM customers ORDER BY ID", diff --git a/crates/sqlquack-semantics/src/sqlserver/profile/models.rs b/crates/sqlquack-semantics/src/sqlserver/profile/models.rs index 0e3b0ddbe..32c634b52 100644 --- a/crates/sqlquack-semantics/src/sqlserver/profile/models.rs +++ b/crates/sqlquack-semantics/src/sqlserver/profile/models.rs @@ -9,7 +9,7 @@ use crate::lineage::constants::{ DATE_PART_FUNCTIONS, SNOWFLAKE_VALUES_COLUMN, WINDOW_ONLY_FUNCTIONS, }; use crate::lineage::main::rendered_type::rendered_type; -use crate::lineage::models::Engine; +use crate::lineage::models::{Engine, JoinConditionReach}; use crate::lineage::models::{Lineage, LineageContext}; use crate::lineage::types::DialectRules; use crate::lineage::types::ValueType; @@ -65,6 +65,22 @@ impl DialectRules for SqlServerRules { false } + fn window_keys_read_outputs(&self) -> bool { + false + } + + fn select_list_reads_outputs(&self) -> bool { + true + } + + fn merged_keys_hide_other_items(&self) -> bool { + true + } + + fn join_conditions_read(&self) -> JoinConditionReach { + JoinConditionReach::Every + } + fn checks_call_signatures(&self) -> bool { false } diff --git a/crates/sqlquack-syntax-databricks/README.md b/crates/sqlquack-syntax-databricks/README.md index 33c0cc7f4..ba93de75a 100644 --- a/crates/sqlquack-syntax-databricks/README.md +++ b/crates/sqlquack-syntax-databricks/README.md @@ -164,9 +164,9 @@ These parse without a gap, but the shared tree keeps less than the source said: - `TIMESTAMP_LTZ` and `TIMESTAMP_NTZ` literals become a `Cast` to a custom type; collations on string types are dropped. - `COPY *` on a view becomes `copy_grants`. -- A function's `RETURNS TABLE (columns)` keeps the column list as `returns_table_body` text; its - `COMMENT`, `SPECIFIC` and `DEFAULT COLLATION` become `options` properties, and parameter - comments and `NOT NULL` are dropped. +- A function's `RETURNS TABLE (columns)` keeps the column list as `returns_table_body`, its + tokens separated by single spaces; its `COMMENT`, `SPECIFIC` and `DEFAULT COLLATION` become + `options` properties, and parameter comments and `NOT NULL` are dropped. - `SHOW TBLPROPERTIES t (key)` holds the key as the `scope` of `scope_kind` `PROPERTY`, a key written as a string literal as a string and any other as its dotted identifier: the shared `Show` has no field for a property key. diff --git a/crates/sqlquack-syntax-databricks/src/parse/_helpers/commands/routines.rs b/crates/sqlquack-syntax-databricks/src/parse/_helpers/commands/routines.rs index aae17a33d..ccd3abe7f 100644 --- a/crates/sqlquack-syntax-databricks/src/parse/_helpers/commands/routines.rs +++ b/crates/sqlquack-syntax-databricks/src/parse/_helpers/commands/routines.rs @@ -182,9 +182,11 @@ impl Parser<'_> { let first: usize = self.at; if self.is_char('(') { self.parenthesized(|parser| parser.function_parameter())?; - let from: usize = self.tokens[first].start; - node.returns_table_body = - Some(self.sql[from..self.tokens[self.at - 1].end].to_owned()); + let written: Vec<&str> = self.tokens[first..self.at] + .iter() + .map(|token| &self.sql[token.start..token.end]) + .collect(); + node.returns_table_body = Some(written.join(" ")); } } else { node.return_type = Some(self.data_type()?); diff --git a/crates/sqlquack-syntax-databricks/src/parse/_helpers/tree/forms.rs b/crates/sqlquack-syntax-databricks/src/parse/_helpers/tree/forms.rs index 1c6a9a31d..8cfdc8ec7 100644 --- a/crates/sqlquack-syntax-databricks/src/parse/_helpers/tree/forms.rs +++ b/crates/sqlquack-syntax-databricks/src/parse/_helpers/tree/forms.rs @@ -409,12 +409,16 @@ impl Parser<'_> { }))) } - /// `resource`: tokens with no space between them as written, as Spark splits resources, or a string. + /// `resource`: unspaced tokens as written, as Spark splits resources, or a string; `r'x'` stays written. fn resource(&mut self) -> Parsing { if self.ends_at(0) { return Err(self.error()); } - if self.string_at(0) && !self.adjacent(0) { + let quoted: bool = matches!( + self.peek().map(|token| token.kind), + Some(TokenKind::String { raw: false }) + ); + if quoted && !self.adjacent(0) { return Ok(string(&self.advance()?.value)); } let first: usize = self.at; diff --git a/crates/sqlquack-syntax-databricks/src/parse/_helpers/values/primary.rs b/crates/sqlquack-syntax-databricks/src/parse/_helpers/values/primary.rs index 09476ac8b..00c15058f 100644 --- a/crates/sqlquack-syntax-databricks/src/parse/_helpers/values/primary.rs +++ b/crates/sqlquack-syntax-databricks/src/parse/_helpers/values/primary.rs @@ -895,7 +895,7 @@ impl Parser<'_> { fn path_bracket(&mut self, names: bool) -> Parsing { self.expect_char('[')?; let index: Expression = match self.peek().map(|token| token.kind) { - Some(TokenKind::String { .. }) => string(&self.advance()?.value), + Some(TokenKind::String { .. }) => literal(self.one_string()?), Some(TokenKind::Number(NumberKind::Integer)) => { literal(Literal::Number(self.advance()?.value)) } diff --git a/crates/sqlquack-syntax-databricks/src/parse/tests/test_written_forms.rs b/crates/sqlquack-syntax-databricks/src/parse/tests/test_written_forms.rs index 9e667f9c9..fa2bed5fa 100644 --- a/crates/sqlquack-syntax-databricks/src/parse/tests/test_written_forms.rs +++ b/crates/sqlquack-syntax-databricks/src/parse/tests/test_written_forms.rs @@ -28,6 +28,36 @@ fn given_one_form_written_in_different_ways_when_parsing_then_each_tree_keeps_wh written: &["ADD FILE v1", "ADD FILE `v1`", "ADD FILE 'v1'"], expected_distinct_trees: 3, }, + DistinctTreesTestCase { + description: "a returned table's columns with different spacing and comments", + written: &[ + "CREATE FUNCTION f1 ( ) RETURNS TABLE ( c1 INT , c2 STRING ) RETURN SELECT 1 , 'a'", + "CREATE FUNCTION f1 ( ) RETURNS TABLE(c1 INT, c2 STRING) RETURN SELECT 1 , 'a'", + "CREATE FUNCTION f1 ( ) RETURNS TABLE (\n c1 INT, -- x\n c2 STRING\n) RETURN SELECT 1 , 'a'", + "CREATE FUNCTION f1 ( ) RETURNS TABLE ( c1 BIGINT , c2 STRING ) RETURN SELECT 1 , 'a'", + ], + expected_distinct_trees: 2, + }, + DistinctTreesTestCase { + description: "a resource written as a raw string and as a string", + written: &[ + "ADD FILE r'v1'", + "ADD FILE 'v1'", + "ADD JAR r'v1'", + "ADD JAR 'v1'", + "LIST FILE r'v1'", + "LIST FILE 'v1'", + ], + expected_distinct_trees: 6, + }, + DistinctTreesTestCase { + description: "a path key written as a raw string and as a string", + written: &[ + "SELECT c1 : [ r'k' ] FROM t1", + "SELECT c1 : [ 'k' ] FROM t1", + ], + expected_distinct_trees: 2, + }, DistinctTreesTestCase { description: "a listed resource written backquoted and as a string holding the backquotes", written: &["LIST FILE `v1 Q`", "LIST FILE '`v1 Q`'"], diff --git a/crates/sqlquack-syntax-snowflake/src/parser/_helpers/functions/generic.rs b/crates/sqlquack-syntax-snowflake/src/parser/_helpers/functions/generic.rs index 44f2d425f..2c4f0300b 100644 --- a/crates/sqlquack-syntax-snowflake/src/parser/_helpers/functions/generic.rs +++ b/crates/sqlquack-syntax-snowflake/src/parser/_helpers/functions/generic.rs @@ -1255,7 +1255,7 @@ impl Parser { /// Parse OVER clause; inside a semantic view, which has no named windows, a window never starts with a name. pub(crate) fn parse_over_clause(&mut self) -> Result { if !self.check(TokenType::LParen) { - let window_name = self.parse_unquoted_identifier(Self::expect_identifier_or_keyword)?; + let window_name = self.parse_quoting_identifier(Self::expect_identifier_or_keyword)?; return Ok(Over { window_name: Some(window_name), partition_by: Vec::new(), @@ -1270,6 +1270,7 @@ impl Parser { let window_name = if !self.config.in_semantic_view && (self.check(TokenType::Identifier) || self.check(TokenType::Var) + || self.check(TokenType::QuotedIdentifier) || self.check_keyword()) && !self.check(TokenType::Partition) && !self.check(TokenType::Order) @@ -1280,7 +1281,8 @@ impl Parser { && !self.check(TokenType::Sort) { let pos = self.current; - let name = identifier_from_token(self.advance()?, false); + let quoted = self.check(TokenType::QuotedIdentifier); + let name = identifier_from_token(self.advance()?, quoted); if self.check(TokenType::Order) || self.check(TokenType::Partition) || self.check(TokenType::Rows) diff --git a/crates/sqlquack-syntax-snowflake/src/parser/_helpers/queries/clauses.rs b/crates/sqlquack-syntax-snowflake/src/parser/_helpers/queries/clauses.rs index 8aca194eb..a352e2567 100644 --- a/crates/sqlquack-syntax-snowflake/src/parser/_helpers/queries/clauses.rs +++ b/crates/sqlquack-syntax-snowflake/src/parser/_helpers/queries/clauses.rs @@ -1814,7 +1814,7 @@ impl Parser { let mut windows: Vec = Vec::new(); loop { - let name = self.parse_unquoted_identifier(Self::expect_identifier)?; + let name = self.parse_quoting_identifier(Self::expect_identifier)?; self.expect_token(TokenType::As)?; self.expect_token(TokenType::LParen)?; @@ -1835,7 +1835,7 @@ impl Parser { | TokenType::Comma ) }) { - Some(self.parse_unquoted_identifier(Self::expect_identifier)?) + Some(self.parse_quoting_identifier(Self::expect_identifier)?) } else { None }; diff --git a/crates/sqlquack-syntax-snowflake/src/parser/_helpers/queries/select.rs b/crates/sqlquack-syntax-snowflake/src/parser/_helpers/queries/select.rs index a31534f63..53a63163a 100644 --- a/crates/sqlquack-syntax-snowflake/src/parser/_helpers/queries/select.rs +++ b/crates/sqlquack-syntax-snowflake/src/parser/_helpers/queries/select.rs @@ -100,7 +100,12 @@ impl Parser { && !(self.is_snowflake_reserved_keyword() && !self.check(TokenType::Window)) && !(self.check(TokenType::Window) && self.peek_nth(1).is_some_and(|token| { - matches!(token.token_type, TokenType::Var | TokenType::Identifier) + matches!( + token.token_type, + TokenType::Var + | TokenType::Identifier + | TokenType::QuotedIdentifier + ) })) { let alias = self.parse_projection_alias_identifier(false)?; diff --git a/crates/sqlquack-syntax-snowflake/src/parser/_helpers/statements/analyze.rs b/crates/sqlquack-syntax-snowflake/src/parser/_helpers/statements/analyze.rs index 2f6abfb0a..64dfc0d78 100644 --- a/crates/sqlquack-syntax-snowflake/src/parser/_helpers/statements/analyze.rs +++ b/crates/sqlquack-syntax-snowflake/src/parser/_helpers/statements/analyze.rs @@ -280,8 +280,10 @@ impl Parser { || self.check(TokenType::Window) && { let next_pos = self.current + 1; next_pos >= self.tokens.len() - || self.tokens[next_pos].token_type != TokenType::Var - && self.tokens[next_pos].token_type != TokenType::Identifier + || !matches!( + self.tokens[next_pos].token_type, + TokenType::Var | TokenType::Identifier | TokenType::QuotedIdentifier + ) } } diff --git a/crates/sqlquack-syntax-snowflake/src/parser/_helpers/statements/entry.rs b/crates/sqlquack-syntax-snowflake/src/parser/_helpers/statements/entry.rs index e8e41c366..3cab85392 100644 --- a/crates/sqlquack-syntax-snowflake/src/parser/_helpers/statements/entry.rs +++ b/crates/sqlquack-syntax-snowflake/src/parser/_helpers/statements/entry.rs @@ -1677,8 +1677,10 @@ impl Parser { if self.check(TokenType::Window) && { let next_pos = self.current + 1; next_pos < self.tokens.len() - && (self.tokens[next_pos].token_type == TokenType::Var - || self.tokens[next_pos].token_type == TokenType::Identifier) + && matches!( + self.tokens[next_pos].token_type, + TokenType::Var | TokenType::Identifier | TokenType::QuotedIdentifier + ) } { self.skip(); Some(self.parse_named_windows()?) diff --git a/crates/sqlquack-syntax-snowflake/src/parser/tests/test_parser.rs b/crates/sqlquack-syntax-snowflake/src/parser/tests/test_parser.rs index 6fca0d86d..d51d534bf 100644 --- a/crates/sqlquack-syntax-snowflake/src/parser/tests/test_parser.rs +++ b/crates/sqlquack-syntax-snowflake/src/parser/tests/test_parser.rs @@ -1131,3 +1131,53 @@ fn given_a_snowflake_cte_whose_body_is_a_bare_values_list_when_parsing_then_pars ); } } + +#[test] +fn given_quoted_window_names_when_parsing_then_each_keeps_its_quotes() { + let test_cases = [ + ParseValueTestCase { + description: "an OVER name keeps its quotes", + dialect: Some(DialectType::PostgreSQL), + sql: "SELECT sum(x) OVER \"W\" AS a, sum(x) OVER (\"V\" ORDER BY y) AS b FROM t WINDOW \"W\" AS (ORDER BY y), \"V\" AS (\"W\")", + pointer: "/0/select/expressions/0/alias/this/window_function/over/window_name/quoted", + expected_value: Ok(json!(true)), + }, + ParseValueTestCase { + description: "a base named inside OVER keeps its quotes", + dialect: Some(DialectType::PostgreSQL), + sql: "SELECT sum(x) OVER \"W\" AS a, sum(x) OVER (\"V\" ORDER BY y) AS b FROM t WINDOW \"W\" AS (ORDER BY y), \"V\" AS (\"W\")", + pointer: "/0/select/expressions/1/alias/this/window_function/over/window_name/quoted", + expected_value: Ok(json!(true)), + }, + ParseValueTestCase { + description: "a WINDOW clause name keeps its quotes", + dialect: Some(DialectType::PostgreSQL), + sql: "SELECT sum(x) OVER \"W\" AS a, sum(x) OVER (\"V\" ORDER BY y) AS b FROM t WINDOW \"W\" AS (ORDER BY y), \"V\" AS (\"W\")", + pointer: "/0/select/windows/0/name/quoted", + expected_value: Ok(json!(true)), + }, + ParseValueTestCase { + description: "a base named in a WINDOW definition keeps its quotes", + dialect: Some(DialectType::PostgreSQL), + sql: "SELECT sum(x) OVER \"W\" AS a, sum(x) OVER (\"V\" ORDER BY y) AS b FROM t WINDOW \"W\" AS (ORDER BY y), \"V\" AS (\"W\")", + pointer: "/0/select/windows/1/spec/window_name/quoted", + expected_value: Ok(json!(true)), + }, + ParseValueTestCase { + description: "an unquoted WINDOW clause name is not quoted", + dialect: Some(DialectType::PostgreSQL), + sql: "SELECT sum(x) OVER w AS a FROM t WINDOW w AS (ORDER BY y)", + pointer: "/0/select/windows/0/name/quoted", + expected_value: Ok(json!(false)), + }, + ]; + + for test_case in test_cases { + assert_eq!( + value_at(test_case.dialect, test_case.sql, test_case.pointer), + test_case.expected_value, + "{}", + test_case.description + ); + } +}