From 48bc86cbe9b816da9c0ad4b67f27e2cca7176973 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 19 Nov 2026 05:48:20 +0000 Subject: [PATCH] feat!: pin SQLQuack 0.22.0 and document the 512 nesting ceiling SQLQuack 0.22.0 makes max_function_call_depth the exact limit in every linted dialect and refuses any statement nested past 512 levels in Postgres, BigQuery and Databricks too, as it already did in Snowflake and DuckDB. At 512 a refusal of SQL that calls something explains the ceiling; call-free SQL gets the bare refusal. - format passes whether the SQL calls anything to SQLQuack's refusal hint, so a format refusal at 512 explains the ceiling as check does. - Join facts read each comma item's joins, which SQLQuack now keeps with the item in Postgres and Databricks, so facts match 0.10.0. - Configuration and embedding docs say the statement and every expression around the calls count toward 512. - CLI tests cover the limits in Databricks, BigQuery and Postgres. BREAKING CHANGE: Postgres, BigQuery and Databricks files nested more than 512 levels deep, such as 600 UNION ALL operands, are now a SQLCLEAN999 finding and are left as written by format. --- Cargo.lock | 44 ++--- Cargo.toml | 12 +- .../skill/references/docs/configuration.md | 15 +- .../skill/references/docs/embedding.md | 4 +- .../src/sql_facts/_helpers/collection.rs | 111 ++++++++--- .../src/sql_facts/tests/test_file_facts.rs | 122 +++++++++++- .../src/sql_facts/tests/test_types.rs | 1 + .../src/sql_lint/_helpers/formatter.rs | 7 + crates/sqlclean/src/sql_lint/tests/helpers.rs | 8 + .../src/sql_lint/tests/test_function_depth.rs | 108 ++++++++++ .../sqlclean/src/sql_lint/tests/test_types.rs | 10 + tests/e2e/src/sqlclean/hosting/_test_types.py | 21 ++ tests/e2e/src/sqlclean/hosting/helpers.py | 12 ++ .../sqlclean/hosting/test_sqlclean_command.py | 185 +++++++++++++++++- .../src/content/docs/docs/configuration.mdx | 14 +- website/src/content/docs/docs/embedding.mdx | 4 +- 16 files changed, 616 insertions(+), 62 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c59fb80..72cad88 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -612,8 +612,8 @@ dependencies = [ [[package]] name = "sqlquack-derive" -version = "0.21.0" -source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=b938ec518b6ab5491090c55832f7d5a4059059bf#b938ec518b6ab5491090c55832f7d5a4059059bf" +version = "0.22.0" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=3543c055634ec2e3a0c58d732e55f570dafbe277#3543c055634ec2e3a0c58d732e55f570dafbe277" dependencies = [ "proc-macro2", "quote", @@ -622,8 +622,8 @@ dependencies = [ [[package]] name = "sqlquack-fingerprint" -version = "0.21.0" -source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=b938ec518b6ab5491090c55832f7d5a4059059bf#b938ec518b6ab5491090c55832f7d5a4059059bf" +version = "0.22.0" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=3543c055634ec2e3a0c58d732e55f570dafbe277#3543c055634ec2e3a0c58d732e55f570dafbe277" dependencies = [ "sha2", "sqlquack-parse", @@ -632,8 +632,8 @@ dependencies = [ [[package]] name = "sqlquack-format" -version = "0.21.0" -source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=b938ec518b6ab5491090c55832f7d5a4059059bf#b938ec518b6ab5491090c55832f7d5a4059059bf" +version = "0.22.0" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=3543c055634ec2e3a0c58d732e55f570dafbe277#3543c055634ec2e3a0c58d732e55f570dafbe277" dependencies = [ "serde_json", "sqlquack-parse", @@ -642,8 +642,8 @@ dependencies = [ [[package]] name = "sqlquack-lint" -version = "0.21.0" -source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=b938ec518b6ab5491090c55832f7d5a4059059bf#b938ec518b6ab5491090c55832f7d5a4059059bf" +version = "0.22.0" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=3543c055634ec2e3a0c58d732e55f570dafbe277#3543c055634ec2e3a0c58d732e55f570dafbe277" dependencies = [ "rayon", "rustc-hash", @@ -655,8 +655,8 @@ dependencies = [ [[package]] name = "sqlquack-parse" -version = "0.21.0" -source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=b938ec518b6ab5491090c55832f7d5a4059059bf#b938ec518b6ab5491090c55832f7d5a4059059bf" +version = "0.22.0" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=3543c055634ec2e3a0c58d732e55f570dafbe277#3543c055634ec2e3a0c58d732e55f570dafbe277" dependencies = [ "sqlquack-syntax", "sqlquack-syntax-bigquery", @@ -668,8 +668,8 @@ dependencies = [ [[package]] name = "sqlquack-syntax" -version = "0.21.0" -source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=b938ec518b6ab5491090c55832f7d5a4059059bf#b938ec518b6ab5491090c55832f7d5a4059059bf" +version = "0.22.0" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=3543c055634ec2e3a0c58d732e55f570dafbe277#3543c055634ec2e3a0c58d732e55f570dafbe277" dependencies = [ "serde", "serde_json", @@ -681,8 +681,8 @@ dependencies = [ [[package]] name = "sqlquack-syntax-bigquery" -version = "0.21.0" -source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=b938ec518b6ab5491090c55832f7d5a4059059bf#b938ec518b6ab5491090c55832f7d5a4059059bf" +version = "0.22.0" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=3543c055634ec2e3a0c58d732e55f570dafbe277#3543c055634ec2e3a0c58d732e55f570dafbe277" dependencies = [ "sqlquack-syntax", "stacker", @@ -690,8 +690,8 @@ dependencies = [ [[package]] name = "sqlquack-syntax-databricks" -version = "0.21.0" -source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=b938ec518b6ab5491090c55832f7d5a4059059bf#b938ec518b6ab5491090c55832f7d5a4059059bf" +version = "0.22.0" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=3543c055634ec2e3a0c58d732e55f570dafbe277#3543c055634ec2e3a0c58d732e55f570dafbe277" dependencies = [ "sqlquack-syntax", "stacker", @@ -699,8 +699,8 @@ dependencies = [ [[package]] name = "sqlquack-syntax-postgres" -version = "0.21.0" -source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=b938ec518b6ab5491090c55832f7d5a4059059bf#b938ec518b6ab5491090c55832f7d5a4059059bf" +version = "0.22.0" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=3543c055634ec2e3a0c58d732e55f570dafbe277#3543c055634ec2e3a0c58d732e55f570dafbe277" dependencies = [ "sqlquack-syntax", "stacker", @@ -708,8 +708,8 @@ dependencies = [ [[package]] name = "sqlquack-syntax-snowflake" -version = "0.21.0" -source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=b938ec518b6ab5491090c55832f7d5a4059059bf#b938ec518b6ab5491090c55832f7d5a4059059bf" +version = "0.22.0" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=3543c055634ec2e3a0c58d732e55f570dafbe277#3543c055634ec2e3a0c58d732e55f570dafbe277" dependencies = [ "serde", "serde_json", @@ -721,8 +721,8 @@ dependencies = [ [[package]] name = "sqlquack-syntax-sqlserver" -version = "0.21.0" -source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=b938ec518b6ab5491090c55832f7d5a4059059bf#b938ec518b6ab5491090c55832f7d5a4059059bf" +version = "0.22.0" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=3543c055634ec2e3a0c58d732e55f570dafbe277#3543c055634ec2e3a0c58d732e55f570dafbe277" dependencies = [ "sqlquack-syntax", "stacker", diff --git a/Cargo.toml b/Cargo.toml index 717214f..49045c8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -30,12 +30,12 @@ sha2 = "0.10" similar = "2" sqlclean = { path = "crates/sqlclean" } # SQLQuack's formatter and lint engine, and the tokens and trees they read, which replace Polyglot. -sqlquack-fingerprint = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "b938ec518b6ab5491090c55832f7d5a4059059bf", features = ["bigquery", "databricks", "duckdb", "postgres", "snowflake", "sqlite", "sqlserver"] } -sqlquack-format = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "b938ec518b6ab5491090c55832f7d5a4059059bf", features = ["bigquery", "databricks", "duckdb", "postgres", "snowflake", "sqlserver"] } -sqlquack-lint = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "b938ec518b6ab5491090c55832f7d5a4059059bf", features = ["bigquery", "databricks", "duckdb", "postgres", "snowflake", "sqlite"] } -sqlquack-parse = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "b938ec518b6ab5491090c55832f7d5a4059059bf" } -sqlquack-syntax = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "b938ec518b6ab5491090c55832f7d5a4059059bf" } -sqlquack-syntax-sqlserver = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "b938ec518b6ab5491090c55832f7d5a4059059bf" } +sqlquack-fingerprint = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "3543c055634ec2e3a0c58d732e55f570dafbe277", features = ["bigquery", "databricks", "duckdb", "postgres", "snowflake", "sqlite", "sqlserver"] } +sqlquack-format = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "3543c055634ec2e3a0c58d732e55f570dafbe277", features = ["bigquery", "databricks", "duckdb", "postgres", "snowflake", "sqlserver"] } +sqlquack-lint = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "3543c055634ec2e3a0c58d732e55f570dafbe277", features = ["bigquery", "databricks", "duckdb", "postgres", "snowflake", "sqlite"] } +sqlquack-parse = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "3543c055634ec2e3a0c58d732e55f570dafbe277" } +sqlquack-syntax = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "3543c055634ec2e3a0c58d732e55f570dafbe277" } +sqlquack-syntax-sqlserver = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "3543c055634ec2e3a0c58d732e55f570dafbe277" } [workspace.lints.rust] unsafe_code = "forbid" diff --git a/crates/sqlclean-cli/skill/references/docs/configuration.md b/crates/sqlclean-cli/skill/references/docs/configuration.md index 7c78dec..28fea3d 100644 --- a/crates/sqlclean-cli/skill/references/docs/configuration.md +++ b/crates/sqlclean-cli/skill/references/docs/configuration.md @@ -10,6 +10,7 @@ Online: https://sqlclean.io/docs/configuration/ - Finding the file - Keys +- Nesting limit - Choosing files - Dialects - Validation @@ -72,7 +73,7 @@ directory is the root. | `include` | `["**/*.sql"]` | Glob patterns of the files to read | | `exclude` | `[]` | Glob patterns of files never to read | | `[[dialects]]` | none | `paths` and `dialect`: a different dialect for matching files | -| `max_function_call_depth` | `256` | The deepest function-call nesting SQLClean parses, from `1` to `512`; a file nested deeper is a `SQLCLEAN999` finding and is not formatted | +| `max_function_call_depth` | `256` | The deepest function-call nesting SQLClean parses, from `1` to `512`; see [Nesting limit](#nesting-limit) | | `[format] line_width` | `100` | The line width the formatter fills; see [Line width](formatting.md#line-width) | | `[format] indent` | `4` | Spaces per indent step, from `1` to `8`; see [Indent](formatting.md#indent) | | `[rules] select` | Rules enabled by default | Rule codes or prefixes to run | @@ -86,6 +87,18 @@ directory is the root. Selection, exceptions and ignores are described in [Rules](rules/overview.md). +## Nesting limit + +`max_function_call_depth` sets how deeply function calls may nest. A file nested deeper is a +`SQLCLEAN999` finding, and `format` leaves it as written. Below 512 the message names the key. + +The maximum is 512. The statement and every expression around the calls also count toward that +limit, so at 512 a bare SELECT can nest at most 511 calls, and fewer fit inside an alias, comparison +or CTE. + +A statement nested more than 512 levels deep is refused the same way in every dialect `check` reads, +whatever the setting and even without function calls, for example 600 `UNION ALL` operands. + ## Choosing files With no paths on the command line, a command reads every file under the configuration root whose diff --git a/crates/sqlclean-cli/skill/references/docs/embedding.md b/crates/sqlclean-cli/skill/references/docs/embedding.md index df67cd0..a240081 100644 --- a/crates/sqlclean-cli/skill/references/docs/embedding.md +++ b/crates/sqlclean-cli/skill/references/docs/embedding.md @@ -152,7 +152,9 @@ println!("{}", format_json(request)?); before the `sqlclean` command chose four; pass `"indent": 4` to match `sqlclean format`. `max_function_call_depth` bounds nested function calls as it does in a lint request: `256` by -default, at most `512`. +default. The maximum is 512. The statement and every expression around the calls also count toward +that limit, so at 512 a bare SELECT can nest at most 511 calls, and fewer fit inside an alias, +comparison or CTE. When the formatter cannot format the SQL, `formatted` is `false`, `sql` is the input unchanged, and `reason` says why. diff --git a/crates/sqlclean/src/sql_facts/_helpers/collection.rs b/crates/sqlclean/src/sql_facts/_helpers/collection.rs index c9cfd01..8864729 100644 --- a/crates/sqlclean/src/sql_facts/_helpers/collection.rs +++ b/crates/sqlclean/src/sql_facts/_helpers/collection.rs @@ -239,7 +239,15 @@ fn node_joins<'a>(node: &'a Expression, tokens: &[Token]) -> Vec<&'a Join> { return Vec::new(); }; let mut joins: Vec<&Join> = Vec::new(); - let mut pending: Vec<&Join> = select.joins.iter().rev().collect(); + let mut pending: Vec<&Join> = select + .joins + .iter() + .chain(from_item_joins( + select.from.iter().flat_map(|from| &from.expressions), + tokens, + )) + .collect(); + pending.reverse(); while let Some(join) = pending.pop() { joins.push(join); if let Expression::JoinedTable(joined) = &join.this @@ -251,12 +259,72 @@ fn node_joins<'a>(node: &'a Expression, tokens: &[Token]) -> Vec<&'a Join> { joins } +/// The joins each item of a FROM list of several items keeps with it, unless the item is written in parentheses. +fn from_item_joins<'a>( + items: impl Iterator, + tokens: &[Token], +) -> impl Iterator { + items + .filter_map(|item| match item { + Expression::JoinedTable(joined) if !parenthesized_item(joined, tokens) => { + Some(joined.joins.iter()) + } + _ => None, + }) + .flatten() +} + +/// Whether a FROM item is written inside one pair of parentheses, which closes after its last join. +fn parenthesized_item(joined: &JoinedTable, tokens: &[Token]) -> bool { + let (Some(first), Some(last)) = ( + located_spans(&joined.left) + .chain(joined.joins.iter().flat_map(join_spans)) + .map(|span| span.start) + .min(), + joined + .joins + .iter() + .flat_map(join_spans) + .map(|span| span.end) + .max(), + ) else { + return false; + }; + tokens + .iter() + .rposition(|token| token.span.end <= first) + .filter(|&open| tokens[open].token_type == TokenType::LParen) + .and_then(|open| closing_parenthesis(tokens, open)) + .is_some_and(|close| last <= close.start) +} + +/// The spans of the tables and columns a join names in its relation and its ON clause. +fn join_spans(join: &Join) -> impl Iterator + '_ { + located_spans(&join.this).chain(join.on.iter().flat_map(located_spans)) +} + +/// The span of the `)` that closes the `(` at `open`. +fn closing_parenthesis(tokens: &[Token], open: usize) -> Option { + let mut depth: usize = 0; + for token in &tokens[open..] { + match token.token_type { + TokenType::LParen => depth += 1, + TokenType::RParen => { + depth -= 1; + if depth == 0 { + return Some(token.span); + } + } + _ => {} + } + } + None +} + /// Whether the `(` after a join group's JOIN keyword closes after everything the group names. fn parenthesized(joined: &JoinedTable, tokens: &[Token]) -> bool { let spans: Vec = located_spans(&joined.left) - .chain(joined.joins.iter().flat_map(|join| { - located_spans(&join.this).chain(join.on.iter().flat_map(located_spans)) - })) + .chain(joined.joins.iter().flat_map(join_spans)) .collect(); let (Some(first), Some(last)) = ( spans.iter().map(|span| span.start).min(), @@ -278,20 +346,7 @@ fn parenthesized(joined: &JoinedTable, tokens: &[Token]) -> bool { else { return false; }; - let mut depth: usize = 0; - for token in &tokens[open..] { - match token.token_type { - TokenType::LParen => depth += 1, - TokenType::RParen => { - depth -= 1; - if depth == 0 { - return last <= token.span.start; - } - } - _ => {} - } - } - false + closing_parenthesis(tokens, open).is_some_and(|close| last <= close.start) } /// The spans of the tables and columns `expression` names. @@ -308,14 +363,24 @@ fn implicit_join_nodes(node: &Expression, tokens: &[Token], characters: &[char]) let Expression::Select(select) = node else { return Vec::new(); }; - let (Some(from), Some(first_join)) = ( - select.from.as_ref(), - select.joins.first().and_then(|join| first_span(&join.this)), - ) else { + let Some(from) = select.from.as_ref() else { + return Vec::new(); + }; + let Some(first_join) = select + .joins + .iter() + .chain(from_item_joins(from.expressions.iter(), tokens)) + .filter_map(|join| first_span(&join.this)) + .min_by_key(|span| span.start) + else { return Vec::new(); }; let mut joins: Vec = Vec::new(); - for relation in &from.expressions { + for item in &from.expressions { + let relation: &Expression = match item { + Expression::JoinedTable(joined) if !parenthesized_item(joined, tokens) => &joined.left, + _ => item, + }; let Some(span) = first_span(relation).filter(|span| span.start > first_join.start) else { continue; }; diff --git a/crates/sqlclean/src/sql_facts/tests/test_file_facts.rs b/crates/sqlclean/src/sql_facts/tests/test_file_facts.rs index 633a03a..4df1732 100644 --- a/crates/sqlclean/src/sql_facts/tests/test_file_facts.rs +++ b/crates/sqlclean/src/sql_facts/tests/test_file_facts.rs @@ -209,33 +209,83 @@ FROM recent AS r LEFT JOIN customers AS c ON r.customer_id = c.customer_id", } #[test] -fn given_comma_and_nested_joins_when_collecting_postgres_facts_then_joins_read_as_polyglot_did() +fn given_comma_and_nested_joins_when_collecting_postgres_and_databricks_facts_then_joins_read_as_polyglot_did() -> Result<(), String> { let test_cases = [ JoinFactsTestCase { - description: "a comma relation after an explicit join is an implicit join", + description: "postgres: a comma relation after an explicit join is an implicit join", sql: "SELECT * FROM a FULL JOIN b USING (x), c AS cc", + dialect: "postgres", expected_joins: &[ ("FULL", "b", true, "FULL JOIN b USING (x)"), ("IMPLICIT", "c", false, "c AS cc"), ], }, JoinFactsTestCase { - description: "a comma relation before the first join is not", + description: "postgres: each comma item keeps its own joins, and a later item is an implicit join", + sql: "SELECT * FROM a JOIN b ON true, c JOIN d ON true", + dialect: "postgres", + expected_joins: &[ + ("INNER", "b", true, "JOIN b ON true"), + ("IMPLICIT", "c", false, "c"), + ("INNER", "d", true, "JOIN d ON true"), + ], + }, + JoinFactsTestCase { + description: "postgres: a comma relation before the first join is not", sql: "SELECT * FROM a, b JOIN c ON true", + dialect: "postgres", expected_joins: &[("INNER", "c", true, "JOIN c ON true")], }, JoinFactsTestCase { - description: "a join nested by its ON clauses reports both joins", + description: "postgres: a join nested by its ON clauses reports both joins", sql: "SELECT * FROM t2 LEFT JOIN t3 FULL JOIN t4 ON false ON false", + dialect: "postgres", expected_joins: &[ ("LEFT", "t3", true, "LEFT JOIN t3"), ("FULL", "t4", true, "FULL JOIN t4 ON false ON false"), ], }, JoinFactsTestCase { - description: "a parenthesized join group reports only the join to it", + description: "postgres: a parenthesized join group reports only the join to it", + sql: "SELECT * FROM t1 LEFT JOIN (t2 LEFT JOIN t3 ON true) ON true", + dialect: "postgres", + expected_joins: &[( + "LEFT", + "", + true, + "LEFT JOIN (t2 LEFT JOIN t3 ON true) ON true", + )], + }, + JoinFactsTestCase { + description: "databricks: a comma relation after an explicit join is an implicit join", + sql: "SELECT * FROM a FULL JOIN b USING (x), c AS cc", + dialect: "databricks", + expected_joins: &[ + ("FULL", "b", true, "FULL JOIN b USING (x)"), + ("IMPLICIT", "c", false, "c AS cc"), + ], + }, + JoinFactsTestCase { + description: "databricks: each comma item keeps its own joins, and a later item is an implicit join", + sql: "SELECT * FROM a JOIN b ON true, c JOIN d ON true", + dialect: "databricks", + expected_joins: &[ + ("INNER", "b", true, "JOIN b ON true"), + ("IMPLICIT", "c", false, "c"), + ("INNER", "d", true, "JOIN d ON true"), + ], + }, + JoinFactsTestCase { + description: "databricks: a comma relation before the first join is not", + sql: "SELECT * FROM a, b JOIN c ON true", + dialect: "databricks", + expected_joins: &[("INNER", "c", true, "JOIN c ON true")], + }, + JoinFactsTestCase { + description: "databricks: a parenthesized join group reports only the join to it", sql: "SELECT * FROM t1 LEFT JOIN (t2 LEFT JOIN t3 ON true) ON true", + dialect: "databricks", expected_joins: &[( "LEFT", "", @@ -243,9 +293,69 @@ fn given_comma_and_nested_joins_when_collecting_postgres_facts_then_joins_read_a "LEFT JOIN (t2 LEFT JOIN t3 ON true) ON true", )], }, + JoinFactsTestCase { + description: "postgres: a comma item whose relation is a table function keeps its joins", + sql: "SELECT * FROM a, unnest(arr) u JOIN b ON true", + dialect: "postgres", + expected_joins: &[("INNER", "b", true, "JOIN b ON true")], + }, + JoinFactsTestCase { + description: "postgres: a comma item that opens with a parenthesized group keeps the joins after it", + sql: "SELECT * FROM a, (b JOIN c ON true) JOIN d ON true", + dialect: "postgres", + expected_joins: &[("INNER", "d", true, "JOIN d ON true")], + }, + JoinFactsTestCase { + description: "postgres: a comma item headed by a subquery is not an implicit join of its own join", + sql: "SELECT * FROM a JOIN b ON true, (SELECT 1) s JOIN c ON true", + dialect: "postgres", + expected_joins: &[ + ("INNER", "b", true, "JOIN b ON true"), + ("INNER", "c", true, "JOIN c ON true"), + ], + }, + JoinFactsTestCase { + description: "postgres: a wholly parenthesized comma item is an unnamed implicit join", + sql: "SELECT * FROM a JOIN x ON true, (b JOIN c ON true)", + dialect: "postgres", + expected_joins: &[ + ("INNER", "x", true, "JOIN x ON true"), + ("IMPLICIT", "", false, "b"), + ], + }, + JoinFactsTestCase { + description: "databricks: a comma item whose relation is a table function keeps its joins", + sql: "SELECT * FROM a, unnest(arr) u JOIN b ON true", + dialect: "databricks", + expected_joins: &[("INNER", "b", true, "JOIN b ON true")], + }, + JoinFactsTestCase { + description: "databricks: a comma item that opens with a parenthesized group keeps the joins after it", + sql: "SELECT * FROM a, (b JOIN c ON true) JOIN d ON true", + dialect: "databricks", + expected_joins: &[("INNER", "d", true, "JOIN d ON true")], + }, + JoinFactsTestCase { + description: "databricks: a comma item headed by a subquery is not an implicit join of its own join", + sql: "SELECT * FROM a JOIN b ON true, (SELECT 1) s JOIN c ON true", + dialect: "databricks", + expected_joins: &[ + ("INNER", "b", true, "JOIN b ON true"), + ("INNER", "c", true, "JOIN c ON true"), + ], + }, + JoinFactsTestCase { + description: "databricks: a wholly parenthesized comma item is an unnamed implicit join", + sql: "SELECT * FROM a JOIN x ON true, (b JOIN c ON true)", + dialect: "databricks", + expected_joins: &[ + ("INNER", "x", true, "JOIN x ON true"), + ("IMPLICIT", "", false, "b"), + ], + }, ]; for test_case in &test_cases { - let facts: FileFacts = file_facts(test_case.sql, "postgres", None)?; + let facts: FileFacts = file_facts(test_case.sql, test_case.dialect, None)?; let joins: Vec<(&str, &str, bool, String)> = facts.statements[0] .joins .iter() diff --git a/crates/sqlclean/src/sql_facts/tests/test_types.rs b/crates/sqlclean/src/sql_facts/tests/test_types.rs index 841dabe..f6dc213 100644 --- a/crates/sqlclean/src/sql_facts/tests/test_types.rs +++ b/crates/sqlclean/src/sql_facts/tests/test_types.rs @@ -35,5 +35,6 @@ pub(crate) struct StatementEndsTestCase { pub(crate) struct JoinFactsTestCase { pub(crate) description: &'static str, pub(crate) sql: &'static str, + pub(crate) dialect: &'static str, pub(crate) expected_joins: &'static [(&'static str, &'static str, bool, &'static str)], } diff --git a/crates/sqlclean/src/sql_lint/_helpers/formatter.rs b/crates/sqlclean/src/sql_lint/_helpers/formatter.rs index 35de174..e531848 100644 --- a/crates/sqlclean/src/sql_lint/_helpers/formatter.rs +++ b/crates/sqlclean/src/sql_lint/_helpers/formatter.rs @@ -4,6 +4,7 @@ use sqlquack_parse::parsing::constants::MAX_FUNCTION_CALL_DEPTH; use sqlquack_parse::parsing::main::call_depth_hint::call_depth_hint; use sqlquack_parse::parsing::main::checked_call_depth::checked_call_depth; use sqlquack_syntax::Dialect as SqlQuackDialect; +use sqlquack_syntax::guard::main::has_function_call::has_function_call; use crate::constants::INTERPOLATION_PLACEHOLDER_PREFIX; use crate::sql_lint::_helpers::cte_macros::{take_out_cte_macros, wrap_query_placeholders}; @@ -105,6 +106,12 @@ fn format_with_sqlquack( reason, max_function_call_depth, Some("max_function_call_depth"), + || { + macros.is_some_and(|dialect| { + sqlquack_parse::dialect_tokens(&request.sql, dialect) + .is_ok_and(|tokens| has_function_call(&tokens)) + }) + }, ) })?; if request.sql.contains(INTERPOLATION_PLACEHOLDER_PREFIX) diff --git a/crates/sqlclean/src/sql_lint/tests/helpers.rs b/crates/sqlclean/src/sql_lint/tests/helpers.rs index 31f5d3a..d07dc96 100644 --- a/crates/sqlclean/src/sql_lint/tests/helpers.rs +++ b/crates/sqlclean/src/sql_lint/tests/helpers.rs @@ -22,6 +22,14 @@ pub(crate) fn nested_function_sql(depth: usize) -> String { format!("SELECT {}1{}", "F(".repeat(depth), ")".repeat(depth)) } +/// `operands` selects joined by UNION ALL: a statement that nests deeply without calling anything. +pub(crate) fn union_all_sql(operands: usize) -> String { + (0..operands) + .map(|index| format!("SELECT {index} AS n")) + .collect::>() + .join(" UNION ALL ") +} + pub(crate) fn diagnostics(sql: &str) -> Result, String> { let response = lint_json( &json!({ diff --git a/crates/sqlclean/src/sql_lint/tests/test_function_depth.rs b/crates/sqlclean/src/sql_lint/tests/test_function_depth.rs index da448b9..6f359c4 100644 --- a/crates/sqlclean/src/sql_lint/tests/test_function_depth.rs +++ b/crates/sqlclean/src/sql_lint/tests/test_function_depth.rs @@ -1,5 +1,7 @@ use serde_json::{Value, json}; +use sqlquack_parse::parsing::constants::CALL_DEPTH_CEILING_NOTE; + use crate::sql_lint::main::engine::lint_json; use crate::sql_lint::main::formatter::format_json; use crate::sql_lint::tests::{helpers, test_types}; @@ -60,6 +62,112 @@ fn given_nested_function_calls_when_linting_then_the_requested_or_default_depth_ } } +#[test] +fn given_sql_past_the_512_ceiling_when_formatting_then_only_sql_with_calls_explains_the_ceiling() { + // The shared parser refuses with its guard code; the native dialects' formatter declines in its own words. + let test_cases = [ + test_types::FormatCeilingTestCase { + description: "snowflake: 512 nested calls are declined, explaining the ceiling once", + dialect: "snowflake", + sql: helpers::nested_function_sql(512), + expected_reason: "value 513 exceeds configured limit 512", + expected_notes: 1, + }, + test_types::FormatCeilingTestCase { + description: "snowflake: 600 UNION ALL operands are declined without the ceiling note", + dialect: "snowflake", + sql: helpers::union_all_sql(600), + expected_reason: "value 513 exceeds configured limit 512", + expected_notes: 0, + }, + test_types::FormatCeilingTestCase { + description: "duckdb: 512 nested calls are declined, explaining the ceiling once", + dialect: "duckdb", + sql: helpers::nested_function_sql(512), + expected_reason: "value 513 exceeds configured limit 512", + expected_notes: 1, + }, + test_types::FormatCeilingTestCase { + description: "duckdb: 600 UNION ALL operands are declined without the ceiling note", + dialect: "duckdb", + sql: helpers::union_all_sql(600), + expected_reason: "value 513 exceeds configured limit 512", + expected_notes: 0, + }, + test_types::FormatCeilingTestCase { + description: "postgres: 512 nested calls are declined by the native formatter", + dialect: "postgres", + sql: helpers::nested_function_sql(512), + expected_reason: "512 levels deep", + expected_notes: 0, + }, + test_types::FormatCeilingTestCase { + description: "postgres: 600 UNION ALL operands are declined without the ceiling note", + dialect: "postgres", + sql: helpers::union_all_sql(600), + expected_reason: "512 levels deep", + expected_notes: 0, + }, + test_types::FormatCeilingTestCase { + description: "bigquery: 512 nested calls are declined by the native formatter", + dialect: "bigquery", + sql: helpers::nested_function_sql(512), + expected_reason: "512 levels deep", + expected_notes: 0, + }, + test_types::FormatCeilingTestCase { + description: "bigquery: 600 UNION ALL operands are declined without the ceiling note", + dialect: "bigquery", + sql: helpers::union_all_sql(600), + expected_reason: "512 levels deep", + expected_notes: 0, + }, + test_types::FormatCeilingTestCase { + description: "databricks: 512 nested calls are declined by the native formatter", + dialect: "databricks", + sql: helpers::nested_function_sql(512), + expected_reason: "512 levels deep", + expected_notes: 0, + }, + test_types::FormatCeilingTestCase { + description: "databricks: 600 UNION ALL operands are declined without the ceiling note", + dialect: "databricks", + sql: helpers::union_all_sql(600), + expected_reason: "512 levels deep", + expected_notes: 0, + }, + ]; + for test_case in &test_cases { + let request = json!({ + "version": 1, + "sql": test_case.sql, + "dialect": test_case.dialect, + "max_function_call_depth": 512, + }); + let response: Value = + serde_json::from_str(&format_json(&request.to_string()).expect("a format response")) + .expect("JSON"); + let reason: &str = response["reason"].as_str().unwrap_or_default(); + assert_eq!( + response["formatted"], + json!(false), + "{}: {reason}", + test_case.description + ); + assert!( + reason.contains(test_case.expected_reason), + "{}: {reason}", + test_case.description + ); + assert_eq!( + reason.matches(CALL_DEPTH_CEILING_NOTE).count(), + test_case.expected_notes, + "{}: {reason}", + test_case.description + ); + } +} + #[test] fn given_nested_function_calls_when_formatting_then_the_requested_or_default_depth_bounds_them() { let test_cases = [ diff --git a/crates/sqlclean/src/sql_lint/tests/test_types.rs b/crates/sqlclean/src/sql_lint/tests/test_types.rs index 7a5c785..814a1cf 100644 --- a/crates/sqlclean/src/sql_lint/tests/test_types.rs +++ b/crates/sqlclean/src/sql_lint/tests/test_types.rs @@ -147,6 +147,16 @@ pub(crate) struct FunctionDepthTestCase { pub expected_error: Option<&'static str>, } +pub(crate) struct FormatCeilingTestCase { + pub description: &'static str, + pub dialect: &'static str, + pub sql: String, + /// Text the refusal must contain. + pub expected_reason: &'static str, + /// How many times the refusal explains the 512 ceiling. + pub expected_notes: usize, +} + pub(crate) struct CrossJoinTestCase { pub description: &'static str, pub sql: &'static str, diff --git a/tests/e2e/src/sqlclean/hosting/_test_types.py b/tests/e2e/src/sqlclean/hosting/_test_types.py index 299295c..1963ea7 100644 --- a/tests/e2e/src/sqlclean/hosting/_test_types.py +++ b/tests/e2e/src/sqlclean/hosting/_test_types.py @@ -32,6 +32,27 @@ class CallDepthTestCase: files: dict[str, str] = field(default_factory=dict) +@dataclass(frozen=True) +class DepthAcceptedTestCase: + """A configuration under which SQL nested up to a limit is checked and formatted.""" + + description: str + config: str + sql: str + expected_codes: tuple[str, ...] + + +@dataclass(frozen=True) +class DepthRefusedTestCase: + """A dialect and a configuration under which SQL nested past a limit is refused, and how.""" + + description: str + config: str + sql: str + expected_message: str + expected_ceiling_notes: int + + @dataclass(frozen=True) class CallDepthFormatTestCase: """A configuration and what formatting 300 nested function calls twice must give.""" diff --git a/tests/e2e/src/sqlclean/hosting/helpers.py b/tests/e2e/src/sqlclean/hosting/helpers.py index a4c972f..9af5378 100644 --- a/tests/e2e/src/sqlclean/hosting/helpers.py +++ b/tests/e2e/src/sqlclean/hosting/helpers.py @@ -43,3 +43,15 @@ def run( check=False, env={**os.environ, **environment}, ) + + +def nested_calls(depth: int) -> str: + """Return depth nested calls around one column.""" + + return "lower(" * depth + "name" + ")" * depth + + +def union_all(operands: int) -> str: + """Return operands selects joined by UNION ALL: deep nesting without any call.""" + + return " union all ".join(f"select {index} as n" for index in range(operands)) diff --git a/tests/e2e/src/sqlclean/hosting/test_sqlclean_command.py b/tests/e2e/src/sqlclean/hosting/test_sqlclean_command.py index f962963..5613310 100644 --- a/tests/e2e/src/sqlclean/hosting/test_sqlclean_command.py +++ b/tests/e2e/src/sqlclean/hosting/test_sqlclean_command.py @@ -13,6 +13,8 @@ CallDepthFormatTestCase, CallDepthTestCase, CheckTestCase, + DepthAcceptedTestCase, + DepthRefusedTestCase, IdempotencyTestCase, ManyFilesTestCase, RewriteTestCase, @@ -20,7 +22,7 @@ SkillsTestCase, SummaryTestCase, ) -from tests.e2e.src.sqlclean.hosting.helpers import run, write_project +from tests.e2e.src.sqlclean.hosting.helpers import nested_calls, run, union_all, write_project from tests.integration.src.sqlclean.rule_testing.helpers import SAMPLE_RULES DUCKDB: str = 'dialect = "duckdb"\n' @@ -40,6 +42,17 @@ NULL_AND_COMMA: str = "select a.id from a, b where a.x = null\n" # SQLCLEAN001 is a default Rule; SQLCLEAN034 and SQLCLEAN050 are opt-in. DEFAULT_AND_OPT_IN: str = "insert into orders select a.id from a where a.x = null\n" +CHECK_NOTHING: str = '[rules]\nselect = ["SQLCLEAN038"]\n' +CEILING_NOTE: str = ( + "The maximum is 512. The statement and every expression around the calls also count toward" + " that limit, so at 512 a bare SELECT can nest at most 511 calls, and fewer fit inside an" + " alias, comparison or CTE." +) +CEILING_REFUSAL: str = "value 513 exceeds configured limit 512" +LIMIT_64_REFUSAL: str = ( + "value 65 exceeds configured limit 64; raise max_function_call_depth, up to 512" +) +DEFAULT_LIMIT_REFUSAL: str = "value 257 exceeds configured limit 256; raise max_function_call_depth" @pytest.mark.parametrize( @@ -231,6 +244,176 @@ def test_given_deeply_nested_calls_when_checking_then_the_configured_depth_bound assert all(test_case.expected_message in str(finding["message"]) for finding in findings) +@pytest.mark.parametrize( + "test_case", + [ + DepthAcceptedTestCase( + description="databricks: a limit of 64 reads 64 nested calls", + config='max_function_call_depth = 64\ndialect = "databricks"\n' + CHECK_NOTHING, + sql=f"select {nested_calls(64)} as n from a\n", + expected_codes=(), + ), + DepthAcceptedTestCase( + description="databricks: at 512 a bare select reads 511 nested calls", + config='max_function_call_depth = 512\ndialect = "databricks"\n' + CHECK_NOTHING, + sql=f"select {nested_calls(511)}\n", + expected_codes=(), + ), + DepthAcceptedTestCase( + description="bigquery: a limit of 64 reads 64 nested calls", + config='max_function_call_depth = 64\ndialect = "bigquery"\n' + CHECK_NOTHING, + sql=f"select {nested_calls(64)} as n from a\n", + expected_codes=(), + ), + DepthAcceptedTestCase( + description="bigquery: at 512 a bare select reads 511 nested calls", + config='max_function_call_depth = 512\ndialect = "bigquery"\n' + CHECK_NOTHING, + sql=f"select {nested_calls(511)}\n", + expected_codes=(), + ), + DepthAcceptedTestCase( + description="postgres: a limit of 64 reads 64 nested calls", + config='max_function_call_depth = 64\ndialect = "postgres"\n' + CHECK_NOTHING, + sql=f"select {nested_calls(64)} as n from a\n", + expected_codes=(), + ), + DepthAcceptedTestCase( + description="postgres: at 512 a bare select reads 511 nested calls", + config='max_function_call_depth = 512\ndialect = "postgres"\n' + CHECK_NOTHING, + sql=f"select {nested_calls(511)}\n", + expected_codes=(), + ), + ], + ids=lambda case: case.description, +) +def test_given_sql_nested_up_to_a_limit_when_checking_and_formatting_then_it_is_read( + tmp_path: Path, test_case: DepthAcceptedTestCase +) -> None: + write_project(tmp_path, config=test_case.config, files={"models/q.sql": test_case.sql}) + + checked: subprocess.CompletedProcess[str] = run( + tmp_path, "check", "--output-format", "json", environment={} + ) + formatted: subprocess.CompletedProcess[str] = run(tmp_path, "format", environment={}) + text: str = (tmp_path / "models/q.sql").read_text(encoding="utf-8") + findings: list[dict[str, object]] = json.loads(checked.stdout)["findings"] + + assert tuple(finding["code"] for finding in findings) == test_case.expected_codes, findings + assert formatted.returncode == 0, formatted.stderr + assert text != test_case.sql + + +@pytest.mark.parametrize( + "test_case", + [ + DepthRefusedTestCase( + description="databricks: a limit of 64 refuses 65 nested calls and names the key", + config='max_function_call_depth = 64\ndialect = "databricks"\n' + CHECK_NOTHING, + sql=f"select {nested_calls(65)} as n from a\n", + expected_message=LIMIT_64_REFUSAL, + expected_ceiling_notes=0, + ), + DepthRefusedTestCase( + description="databricks: the default limit refuses 257 nested calls", + config='dialect = "databricks"\n' + CHECK_NOTHING, + sql=f"select {nested_calls(257)} as n from a\n", + expected_message=DEFAULT_LIMIT_REFUSAL, + expected_ceiling_notes=0, + ), + DepthRefusedTestCase( + description="databricks: at 512 a bare select of 512 nested calls explains the ceiling", + config='max_function_call_depth = 512\ndialect = "databricks"\n' + CHECK_NOTHING, + sql=f"select {nested_calls(512)}\n", + expected_message=CEILING_REFUSAL + ". " + CEILING_NOTE, + expected_ceiling_notes=1, + ), + DepthRefusedTestCase( + description="databricks: 600 union all operands are refused without the ceiling note", + config='max_function_call_depth = 512\ndialect = "databricks"\n' + CHECK_NOTHING, + sql=union_all(600) + "\n", + expected_message=CEILING_REFUSAL, + expected_ceiling_notes=0, + ), + DepthRefusedTestCase( + description="bigquery: a limit of 64 refuses 65 nested calls and names the key", + config='max_function_call_depth = 64\ndialect = "bigquery"\n' + CHECK_NOTHING, + sql=f"select {nested_calls(65)} as n from a\n", + expected_message=LIMIT_64_REFUSAL, + expected_ceiling_notes=0, + ), + DepthRefusedTestCase( + description="bigquery: the default limit refuses 257 nested calls", + config='dialect = "bigquery"\n' + CHECK_NOTHING, + sql=f"select {nested_calls(257)} as n from a\n", + expected_message=DEFAULT_LIMIT_REFUSAL, + expected_ceiling_notes=0, + ), + DepthRefusedTestCase( + description="bigquery: at 512 a bare select of 512 nested calls explains the ceiling", + config='max_function_call_depth = 512\ndialect = "bigquery"\n' + CHECK_NOTHING, + sql=f"select {nested_calls(512)}\n", + expected_message=CEILING_REFUSAL + ". " + CEILING_NOTE, + expected_ceiling_notes=1, + ), + DepthRefusedTestCase( + description="bigquery: 600 union all operands are refused without the ceiling note", + config='max_function_call_depth = 512\ndialect = "bigquery"\n' + CHECK_NOTHING, + sql=union_all(600) + "\n", + expected_message=CEILING_REFUSAL, + expected_ceiling_notes=0, + ), + DepthRefusedTestCase( + description="postgres: a limit of 64 refuses 65 nested calls and names the key", + config='max_function_call_depth = 64\ndialect = "postgres"\n' + CHECK_NOTHING, + sql=f"select {nested_calls(65)} as n from a\n", + expected_message=LIMIT_64_REFUSAL, + expected_ceiling_notes=0, + ), + DepthRefusedTestCase( + description="postgres: the default limit refuses 257 nested calls", + config='dialect = "postgres"\n' + CHECK_NOTHING, + sql=f"select {nested_calls(257)} as n from a\n", + expected_message=DEFAULT_LIMIT_REFUSAL, + expected_ceiling_notes=0, + ), + DepthRefusedTestCase( + description="postgres: at 512 a bare select of 512 nested calls explains the ceiling", + config='max_function_call_depth = 512\ndialect = "postgres"\n' + CHECK_NOTHING, + sql=f"select {nested_calls(512)}\n", + expected_message=CEILING_REFUSAL + ". " + CEILING_NOTE, + expected_ceiling_notes=1, + ), + DepthRefusedTestCase( + description="postgres: 600 union all operands are refused without the ceiling note", + config='max_function_call_depth = 512\ndialect = "postgres"\n' + CHECK_NOTHING, + sql=union_all(600) + "\n", + expected_message=CEILING_REFUSAL, + expected_ceiling_notes=0, + ), + ], + ids=lambda case: case.description, +) +def test_given_sql_nested_past_a_limit_when_checking_and_formatting_then_it_is_refused_as_written( + tmp_path: Path, test_case: DepthRefusedTestCase +) -> None: + write_project(tmp_path, config=test_case.config, files={"models/q.sql": test_case.sql}) + + checked: subprocess.CompletedProcess[str] = run( + tmp_path, "check", "--output-format", "json", environment={} + ) + findings: list[dict[str, object]] = json.loads(checked.stdout)["findings"] + formatted: subprocess.CompletedProcess[str] = run(tmp_path, "format", environment={}) + text: str = (tmp_path / "models/q.sql").read_text(encoding="utf-8") + message: str = " ".join(str(finding["message"]) for finding in findings) + + assert [finding["code"] for finding in findings] == ["SQLCLEAN999"], message + assert test_case.expected_message in message + assert message.count(CEILING_NOTE) == test_case.expected_ceiling_notes + assert formatted.returncode == 1, formatted.stderr + assert text == test_case.sql + assert test_case.expected_message in formatted.stdout + formatted.stderr + + @pytest.mark.parametrize( "test_case", [ diff --git a/website/src/content/docs/docs/configuration.mdx b/website/src/content/docs/docs/configuration.mdx index fb73454..c28aba1 100644 --- a/website/src/content/docs/docs/configuration.mdx +++ b/website/src/content/docs/docs/configuration.mdx @@ -61,7 +61,7 @@ directory is the root. | `include` | `["**/*.sql"]` | Glob patterns of the files to read | | `exclude` | `[]` | Glob patterns of files never to read | | `[[dialects]]` | none | `paths` and `dialect`: a different dialect for matching files | -| `max_function_call_depth` | `256` | The deepest function-call nesting SQLClean parses, from `1` to `512`; a file nested deeper is a `SQLCLEAN999` finding and is not formatted | +| `max_function_call_depth` | `256` | The deepest function-call nesting SQLClean parses, from `1` to `512`; see [Nesting limit](#nesting-limit) | | `[format] line_width` | `100` | The line width the formatter fills; see [Line width](/docs/formatting/#line-width) | | `[format] indent` | `4` | Spaces per indent step, from `1` to `8`; see [Indent](/docs/formatting/#indent) | | `[rules] select` | Rules enabled by default | Rule codes or prefixes to run | @@ -75,6 +75,18 @@ directory is the root. Selection, exceptions and ignores are described in [Rules](/docs/rules/overview/). +## Nesting limit + +`max_function_call_depth` sets how deeply function calls may nest. A file nested deeper is a +`SQLCLEAN999` finding, and `format` leaves it as written. Below 512 the message names the key. + +The maximum is 512. The statement and every expression around the calls also count toward that +limit, so at 512 a bare SELECT can nest at most 511 calls, and fewer fit inside an alias, comparison +or CTE. + +A statement nested more than 512 levels deep is refused the same way in every dialect `check` reads, +whatever the setting and even without function calls, for example 600 `UNION ALL` operands. + ## Choosing files With no paths on the command line, a command reads every file under the configuration root whose diff --git a/website/src/content/docs/docs/embedding.mdx b/website/src/content/docs/docs/embedding.mdx index 99462ec..cc1317b 100644 --- a/website/src/content/docs/docs/embedding.mdx +++ b/website/src/content/docs/docs/embedding.mdx @@ -138,7 +138,9 @@ println!("{}", format_json(request)?); before the `sqlclean` command chose four; pass `"indent": 4` to match `sqlclean format`. `max_function_call_depth` bounds nested function calls as it does in a lint request: `256` by -default, at most `512`. +default. The maximum is 512. The statement and every expression around the calls also count toward +that limit, so at 512 a bare SELECT can nest at most 511 calls, and fewer fit inside an alias, +comparison or CTE. When the formatter cannot format the SQL, `formatted` is `false`, `sql` is the input unchanged, and `reason` says why.