diff --git a/Cargo.lock b/Cargo.lock index c59fb80..72cad88 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -612,8 +612,8 @@ dependencies = [ [[package]] name = "sqlquack-derive" -version = "0.21.0" -source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=b938ec518b6ab5491090c55832f7d5a4059059bf#b938ec518b6ab5491090c55832f7d5a4059059bf" +version = "0.22.0" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=3543c055634ec2e3a0c58d732e55f570dafbe277#3543c055634ec2e3a0c58d732e55f570dafbe277" dependencies = [ "proc-macro2", "quote", @@ -622,8 +622,8 @@ dependencies = [ [[package]] name = "sqlquack-fingerprint" -version = "0.21.0" -source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=b938ec518b6ab5491090c55832f7d5a4059059bf#b938ec518b6ab5491090c55832f7d5a4059059bf" +version = "0.22.0" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=3543c055634ec2e3a0c58d732e55f570dafbe277#3543c055634ec2e3a0c58d732e55f570dafbe277" dependencies = [ "sha2", "sqlquack-parse", @@ -632,8 +632,8 @@ dependencies = [ [[package]] name = "sqlquack-format" -version = "0.21.0" -source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=b938ec518b6ab5491090c55832f7d5a4059059bf#b938ec518b6ab5491090c55832f7d5a4059059bf" +version = "0.22.0" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=3543c055634ec2e3a0c58d732e55f570dafbe277#3543c055634ec2e3a0c58d732e55f570dafbe277" dependencies = [ "serde_json", "sqlquack-parse", @@ -642,8 +642,8 @@ dependencies = [ [[package]] name = "sqlquack-lint" -version = "0.21.0" -source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=b938ec518b6ab5491090c55832f7d5a4059059bf#b938ec518b6ab5491090c55832f7d5a4059059bf" +version = "0.22.0" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=3543c055634ec2e3a0c58d732e55f570dafbe277#3543c055634ec2e3a0c58d732e55f570dafbe277" dependencies = [ "rayon", "rustc-hash", @@ -655,8 +655,8 @@ dependencies = [ [[package]] name = "sqlquack-parse" -version = "0.21.0" -source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=b938ec518b6ab5491090c55832f7d5a4059059bf#b938ec518b6ab5491090c55832f7d5a4059059bf" +version = "0.22.0" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=3543c055634ec2e3a0c58d732e55f570dafbe277#3543c055634ec2e3a0c58d732e55f570dafbe277" dependencies = [ "sqlquack-syntax", "sqlquack-syntax-bigquery", @@ -668,8 +668,8 @@ dependencies = [ [[package]] name = "sqlquack-syntax" -version = "0.21.0" -source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=b938ec518b6ab5491090c55832f7d5a4059059bf#b938ec518b6ab5491090c55832f7d5a4059059bf" +version = "0.22.0" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=3543c055634ec2e3a0c58d732e55f570dafbe277#3543c055634ec2e3a0c58d732e55f570dafbe277" dependencies = [ "serde", "serde_json", @@ -681,8 +681,8 @@ dependencies = [ [[package]] name = "sqlquack-syntax-bigquery" -version = "0.21.0" -source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=b938ec518b6ab5491090c55832f7d5a4059059bf#b938ec518b6ab5491090c55832f7d5a4059059bf" +version = "0.22.0" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=3543c055634ec2e3a0c58d732e55f570dafbe277#3543c055634ec2e3a0c58d732e55f570dafbe277" dependencies = [ "sqlquack-syntax", "stacker", @@ -690,8 +690,8 @@ dependencies = [ [[package]] name = "sqlquack-syntax-databricks" -version = "0.21.0" -source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=b938ec518b6ab5491090c55832f7d5a4059059bf#b938ec518b6ab5491090c55832f7d5a4059059bf" +version = "0.22.0" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=3543c055634ec2e3a0c58d732e55f570dafbe277#3543c055634ec2e3a0c58d732e55f570dafbe277" dependencies = [ "sqlquack-syntax", "stacker", @@ -699,8 +699,8 @@ dependencies = [ [[package]] name = "sqlquack-syntax-postgres" -version = "0.21.0" -source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=b938ec518b6ab5491090c55832f7d5a4059059bf#b938ec518b6ab5491090c55832f7d5a4059059bf" +version = "0.22.0" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=3543c055634ec2e3a0c58d732e55f570dafbe277#3543c055634ec2e3a0c58d732e55f570dafbe277" dependencies = [ "sqlquack-syntax", "stacker", @@ -708,8 +708,8 @@ dependencies = [ [[package]] name = "sqlquack-syntax-snowflake" -version = "0.21.0" -source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=b938ec518b6ab5491090c55832f7d5a4059059bf#b938ec518b6ab5491090c55832f7d5a4059059bf" +version = "0.22.0" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=3543c055634ec2e3a0c58d732e55f570dafbe277#3543c055634ec2e3a0c58d732e55f570dafbe277" dependencies = [ "serde", "serde_json", @@ -721,8 +721,8 @@ dependencies = [ [[package]] name = "sqlquack-syntax-sqlserver" -version = "0.21.0" -source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=b938ec518b6ab5491090c55832f7d5a4059059bf#b938ec518b6ab5491090c55832f7d5a4059059bf" +version = "0.22.0" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=3543c055634ec2e3a0c58d732e55f570dafbe277#3543c055634ec2e3a0c58d732e55f570dafbe277" dependencies = [ "sqlquack-syntax", "stacker", diff --git a/Cargo.toml b/Cargo.toml index 717214f..49045c8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -30,12 +30,12 @@ sha2 = "0.10" similar = "2" sqlclean = { path = "crates/sqlclean" } # SQLQuack's formatter and lint engine, and the tokens and trees they read, which replace Polyglot. -sqlquack-fingerprint = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "b938ec518b6ab5491090c55832f7d5a4059059bf", features = ["bigquery", "databricks", "duckdb", "postgres", "snowflake", "sqlite", "sqlserver"] } -sqlquack-format = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "b938ec518b6ab5491090c55832f7d5a4059059bf", features = ["bigquery", "databricks", "duckdb", "postgres", "snowflake", "sqlserver"] } -sqlquack-lint = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "b938ec518b6ab5491090c55832f7d5a4059059bf", features = ["bigquery", "databricks", "duckdb", "postgres", "snowflake", "sqlite"] } -sqlquack-parse = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "b938ec518b6ab5491090c55832f7d5a4059059bf" } -sqlquack-syntax = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "b938ec518b6ab5491090c55832f7d5a4059059bf" } -sqlquack-syntax-sqlserver = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "b938ec518b6ab5491090c55832f7d5a4059059bf" } +sqlquack-fingerprint = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "3543c055634ec2e3a0c58d732e55f570dafbe277", features = ["bigquery", "databricks", "duckdb", "postgres", "snowflake", "sqlite", "sqlserver"] } +sqlquack-format = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "3543c055634ec2e3a0c58d732e55f570dafbe277", features = ["bigquery", "databricks", "duckdb", "postgres", "snowflake", "sqlserver"] } +sqlquack-lint = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "3543c055634ec2e3a0c58d732e55f570dafbe277", features = ["bigquery", "databricks", "duckdb", "postgres", "snowflake", "sqlite"] } +sqlquack-parse = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "3543c055634ec2e3a0c58d732e55f570dafbe277" } +sqlquack-syntax = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "3543c055634ec2e3a0c58d732e55f570dafbe277" } +sqlquack-syntax-sqlserver = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "3543c055634ec2e3a0c58d732e55f570dafbe277" } [workspace.lints.rust] unsafe_code = "forbid" diff --git a/crates/sqlclean-cli/skill/references/docs/configuration.md b/crates/sqlclean-cli/skill/references/docs/configuration.md index 7c78dec..28fea3d 100644 --- a/crates/sqlclean-cli/skill/references/docs/configuration.md +++ b/crates/sqlclean-cli/skill/references/docs/configuration.md @@ -10,6 +10,7 @@ Online: https://sqlclean.io/docs/configuration/ - Finding the file - Keys +- Nesting limit - Choosing files - Dialects - Validation @@ -72,7 +73,7 @@ directory is the root. | `include` | `["**/*.sql"]` | Glob patterns of the files to read | | `exclude` | `[]` | Glob patterns of files never to read | | `[[dialects]]` | none | `paths` and `dialect`: a different dialect for matching files | -| `max_function_call_depth` | `256` | The deepest function-call nesting SQLClean parses, from `1` to `512`; a file nested deeper is a `SQLCLEAN999` finding and is not formatted | +| `max_function_call_depth` | `256` | The deepest function-call nesting SQLClean parses, from `1` to `512`; see [Nesting limit](#nesting-limit) | | `[format] line_width` | `100` | The line width the formatter fills; see [Line width](formatting.md#line-width) | | `[format] indent` | `4` | Spaces per indent step, from `1` to `8`; see [Indent](formatting.md#indent) | | `[rules] select` | Rules enabled by default | Rule codes or prefixes to run | @@ -86,6 +87,18 @@ directory is the root. Selection, exceptions and ignores are described in [Rules](rules/overview.md). +## Nesting limit + +`max_function_call_depth` sets how deeply function calls may nest. A file nested deeper is a +`SQLCLEAN999` finding, and `format` leaves it as written. Below 512 the message names the key. + +The maximum is 512. The statement and every expression around the calls also count toward that +limit, so at 512 a bare SELECT can nest at most 511 calls, and fewer fit inside an alias, comparison +or CTE. + +A statement nested more than 512 levels deep is refused the same way in every dialect `check` reads, +whatever the setting and even without function calls, for example 600 `UNION ALL` operands. + ## Choosing files With no paths on the command line, a command reads every file under the configuration root whose diff --git a/crates/sqlclean-cli/skill/references/docs/embedding.md b/crates/sqlclean-cli/skill/references/docs/embedding.md index df67cd0..a240081 100644 --- a/crates/sqlclean-cli/skill/references/docs/embedding.md +++ b/crates/sqlclean-cli/skill/references/docs/embedding.md @@ -152,7 +152,9 @@ println!("{}", format_json(request)?); before the `sqlclean` command chose four; pass `"indent": 4` to match `sqlclean format`. `max_function_call_depth` bounds nested function calls as it does in a lint request: `256` by -default, at most `512`. +default. The maximum is 512. The statement and every expression around the calls also count toward +that limit, so at 512 a bare SELECT can nest at most 511 calls, and fewer fit inside an alias, +comparison or CTE. When the formatter cannot format the SQL, `formatted` is `false`, `sql` is the input unchanged, and `reason` says why. diff --git a/crates/sqlclean/src/sql_facts/_helpers/collection.rs b/crates/sqlclean/src/sql_facts/_helpers/collection.rs index c9cfd01..8864729 100644 --- a/crates/sqlclean/src/sql_facts/_helpers/collection.rs +++ b/crates/sqlclean/src/sql_facts/_helpers/collection.rs @@ -239,7 +239,15 @@ fn node_joins<'a>(node: &'a Expression, tokens: &[Token]) -> Vec<&'a Join> { return Vec::new(); }; let mut joins: Vec<&Join> = Vec::new(); - let mut pending: Vec<&Join> = select.joins.iter().rev().collect(); + let mut pending: Vec<&Join> = select + .joins + .iter() + .chain(from_item_joins( + select.from.iter().flat_map(|from| &from.expressions), + tokens, + )) + .collect(); + pending.reverse(); while let Some(join) = pending.pop() { joins.push(join); if let Expression::JoinedTable(joined) = &join.this @@ -251,12 +259,72 @@ fn node_joins<'a>(node: &'a Expression, tokens: &[Token]) -> Vec<&'a Join> { joins } +/// The joins each item of a FROM list of several items keeps with it, unless the item is written in parentheses. +fn from_item_joins<'a>( + items: impl Iterator, + tokens: &[Token], +) -> impl Iterator { + items + .filter_map(|item| match item { + Expression::JoinedTable(joined) if !parenthesized_item(joined, tokens) => { + Some(joined.joins.iter()) + } + _ => None, + }) + .flatten() +} + +/// Whether a FROM item is written inside one pair of parentheses, which closes after its last join. +fn parenthesized_item(joined: &JoinedTable, tokens: &[Token]) -> bool { + let (Some(first), Some(last)) = ( + located_spans(&joined.left) + .chain(joined.joins.iter().flat_map(join_spans)) + .map(|span| span.start) + .min(), + joined + .joins + .iter() + .flat_map(join_spans) + .map(|span| span.end) + .max(), + ) else { + return false; + }; + tokens + .iter() + .rposition(|token| token.span.end <= first) + .filter(|&open| tokens[open].token_type == TokenType::LParen) + .and_then(|open| closing_parenthesis(tokens, open)) + .is_some_and(|close| last <= close.start) +} + +/// The spans of the tables and columns a join names in its relation and its ON clause. +fn join_spans(join: &Join) -> impl Iterator + '_ { + located_spans(&join.this).chain(join.on.iter().flat_map(located_spans)) +} + +/// The span of the `)` that closes the `(` at `open`. +fn closing_parenthesis(tokens: &[Token], open: usize) -> Option { + let mut depth: usize = 0; + for token in &tokens[open..] { + match token.token_type { + TokenType::LParen => depth += 1, + TokenType::RParen => { + depth -= 1; + if depth == 0 { + return Some(token.span); + } + } + _ => {} + } + } + None +} + /// Whether the `(` after a join group's JOIN keyword closes after everything the group names. fn parenthesized(joined: &JoinedTable, tokens: &[Token]) -> bool { let spans: Vec = located_spans(&joined.left) - .chain(joined.joins.iter().flat_map(|join| { - located_spans(&join.this).chain(join.on.iter().flat_map(located_spans)) - })) + .chain(joined.joins.iter().flat_map(join_spans)) .collect(); let (Some(first), Some(last)) = ( spans.iter().map(|span| span.start).min(), @@ -278,20 +346,7 @@ fn parenthesized(joined: &JoinedTable, tokens: &[Token]) -> bool { else { return false; }; - let mut depth: usize = 0; - for token in &tokens[open..] { - match token.token_type { - TokenType::LParen => depth += 1, - TokenType::RParen => { - depth -= 1; - if depth == 0 { - return last <= token.span.start; - } - } - _ => {} - } - } - false + closing_parenthesis(tokens, open).is_some_and(|close| last <= close.start) } /// The spans of the tables and columns `expression` names. @@ -308,14 +363,24 @@ fn implicit_join_nodes(node: &Expression, tokens: &[Token], characters: &[char]) let Expression::Select(select) = node else { return Vec::new(); }; - let (Some(from), Some(first_join)) = ( - select.from.as_ref(), - select.joins.first().and_then(|join| first_span(&join.this)), - ) else { + let Some(from) = select.from.as_ref() else { + return Vec::new(); + }; + let Some(first_join) = select + .joins + .iter() + .chain(from_item_joins(from.expressions.iter(), tokens)) + .filter_map(|join| first_span(&join.this)) + .min_by_key(|span| span.start) + else { return Vec::new(); }; let mut joins: Vec = Vec::new(); - for relation in &from.expressions { + for item in &from.expressions { + let relation: &Expression = match item { + Expression::JoinedTable(joined) if !parenthesized_item(joined, tokens) => &joined.left, + _ => item, + }; let Some(span) = first_span(relation).filter(|span| span.start > first_join.start) else { continue; }; diff --git a/crates/sqlclean/src/sql_facts/tests/test_file_facts.rs b/crates/sqlclean/src/sql_facts/tests/test_file_facts.rs index 633a03a..4df1732 100644 --- a/crates/sqlclean/src/sql_facts/tests/test_file_facts.rs +++ b/crates/sqlclean/src/sql_facts/tests/test_file_facts.rs @@ -209,33 +209,83 @@ FROM recent AS r LEFT JOIN customers AS c ON r.customer_id = c.customer_id", } #[test] -fn given_comma_and_nested_joins_when_collecting_postgres_facts_then_joins_read_as_polyglot_did() +fn given_comma_and_nested_joins_when_collecting_postgres_and_databricks_facts_then_joins_read_as_polyglot_did() -> Result<(), String> { let test_cases = [ JoinFactsTestCase { - description: "a comma relation after an explicit join is an implicit join", + description: "postgres: a comma relation after an explicit join is an implicit join", sql: "SELECT * FROM a FULL JOIN b USING (x), c AS cc", + dialect: "postgres", expected_joins: &[ ("FULL", "b", true, "FULL JOIN b USING (x)"), ("IMPLICIT", "c", false, "c AS cc"), ], }, JoinFactsTestCase { - description: "a comma relation before the first join is not", + description: "postgres: each comma item keeps its own joins, and a later item is an implicit join", + sql: "SELECT * FROM a JOIN b ON true, c JOIN d ON true", + dialect: "postgres", + expected_joins: &[ + ("INNER", "b", true, "JOIN b ON true"), + ("IMPLICIT", "c", false, "c"), + ("INNER", "d", true, "JOIN d ON true"), + ], + }, + JoinFactsTestCase { + description: "postgres: a comma relation before the first join is not", sql: "SELECT * FROM a, b JOIN c ON true", + dialect: "postgres", expected_joins: &[("INNER", "c", true, "JOIN c ON true")], }, JoinFactsTestCase { - description: "a join nested by its ON clauses reports both joins", + description: "postgres: a join nested by its ON clauses reports both joins", sql: "SELECT * FROM t2 LEFT JOIN t3 FULL JOIN t4 ON false ON false", + dialect: "postgres", expected_joins: &[ ("LEFT", "t3", true, "LEFT JOIN t3"), ("FULL", "t4", true, "FULL JOIN t4 ON false ON false"), ], }, JoinFactsTestCase { - description: "a parenthesized join group reports only the join to it", + description: "postgres: a parenthesized join group reports only the join to it", + sql: "SELECT * FROM t1 LEFT JOIN (t2 LEFT JOIN t3 ON true) ON true", + dialect: "postgres", + expected_joins: &[( + "LEFT", + "", + true, + "LEFT JOIN (t2 LEFT JOIN t3 ON true) ON true", + )], + }, + JoinFactsTestCase { + description: "databricks: a comma relation after an explicit join is an implicit join", + sql: "SELECT * FROM a FULL JOIN b USING (x), c AS cc", + dialect: "databricks", + expected_joins: &[ + ("FULL", "b", true, "FULL JOIN b USING (x)"), + ("IMPLICIT", "c", false, "c AS cc"), + ], + }, + JoinFactsTestCase { + description: "databricks: each comma item keeps its own joins, and a later item is an implicit join", + sql: "SELECT * FROM a JOIN b ON true, c JOIN d ON true", + dialect: "databricks", + expected_joins: &[ + ("INNER", "b", true, "JOIN b ON true"), + ("IMPLICIT", "c", false, "c"), + ("INNER", "d", true, "JOIN d ON true"), + ], + }, + JoinFactsTestCase { + description: "databricks: a comma relation before the first join is not", + sql: "SELECT * FROM a, b JOIN c ON true", + dialect: "databricks", + expected_joins: &[("INNER", "c", true, "JOIN c ON true")], + }, + JoinFactsTestCase { + description: "databricks: a parenthesized join group reports only the join to it", sql: "SELECT * FROM t1 LEFT JOIN (t2 LEFT JOIN t3 ON true) ON true", + dialect: "databricks", expected_joins: &[( "LEFT", "", @@ -243,9 +293,69 @@ fn given_comma_and_nested_joins_when_collecting_postgres_facts_then_joins_read_a "LEFT JOIN (t2 LEFT JOIN t3 ON true) ON true", )], }, + JoinFactsTestCase { + description: "postgres: a comma item whose relation is a table function keeps its joins", + sql: "SELECT * FROM a, unnest(arr) u JOIN b ON true", + dialect: "postgres", + expected_joins: &[("INNER", "b", true, "JOIN b ON true")], + }, + JoinFactsTestCase { + description: "postgres: a comma item that opens with a parenthesized group keeps the joins after it", + sql: "SELECT * FROM a, (b JOIN c ON true) JOIN d ON true", + dialect: "postgres", + expected_joins: &[("INNER", "d", true, "JOIN d ON true")], + }, + JoinFactsTestCase { + description: "postgres: a comma item headed by a subquery is not an implicit join of its own join", + sql: "SELECT * FROM a JOIN b ON true, (SELECT 1) s JOIN c ON true", + dialect: "postgres", + expected_joins: &[ + ("INNER", "b", true, "JOIN b ON true"), + ("INNER", "c", true, "JOIN c ON true"), + ], + }, + JoinFactsTestCase { + description: "postgres: a wholly parenthesized comma item is an unnamed implicit join", + sql: "SELECT * FROM a JOIN x ON true, (b JOIN c ON true)", + dialect: "postgres", + expected_joins: &[ + ("INNER", "x", true, "JOIN x ON true"), + ("IMPLICIT", "", false, "b"), + ], + }, + JoinFactsTestCase { + description: "databricks: a comma item whose relation is a table function keeps its joins", + sql: "SELECT * FROM a, unnest(arr) u JOIN b ON true", + dialect: "databricks", + expected_joins: &[("INNER", "b", true, "JOIN b ON true")], + }, + JoinFactsTestCase { + description: "databricks: a comma item that opens with a parenthesized group keeps the joins after it", + sql: "SELECT * FROM a, (b JOIN c ON true) JOIN d ON true", + dialect: "databricks", + expected_joins: &[("INNER", "d", true, "JOIN d ON true")], + }, + JoinFactsTestCase { + description: "databricks: a comma item headed by a subquery is not an implicit join of its own join", + sql: "SELECT * FROM a JOIN b ON true, (SELECT 1) s JOIN c ON true", + dialect: "databricks", + expected_joins: &[ + ("INNER", "b", true, "JOIN b ON true"), + ("INNER", "c", true, "JOIN c ON true"), + ], + }, + JoinFactsTestCase { + description: "databricks: a wholly parenthesized comma item is an unnamed implicit join", + sql: "SELECT * FROM a JOIN x ON true, (b JOIN c ON true)", + dialect: "databricks", + expected_joins: &[ + ("INNER", "x", true, "JOIN x ON true"), + ("IMPLICIT", "", false, "b"), + ], + }, ]; for test_case in &test_cases { - let facts: FileFacts = file_facts(test_case.sql, "postgres", None)?; + let facts: FileFacts = file_facts(test_case.sql, test_case.dialect, None)?; let joins: Vec<(&str, &str, bool, String)> = facts.statements[0] .joins .iter() diff --git a/crates/sqlclean/src/sql_facts/tests/test_types.rs b/crates/sqlclean/src/sql_facts/tests/test_types.rs index 841dabe..f6dc213 100644 --- a/crates/sqlclean/src/sql_facts/tests/test_types.rs +++ b/crates/sqlclean/src/sql_facts/tests/test_types.rs @@ -35,5 +35,6 @@ pub(crate) struct StatementEndsTestCase { pub(crate) struct JoinFactsTestCase { pub(crate) description: &'static str, pub(crate) sql: &'static str, + pub(crate) dialect: &'static str, pub(crate) expected_joins: &'static [(&'static str, &'static str, bool, &'static str)], } diff --git a/crates/sqlclean/src/sql_lint/_helpers/formatter.rs b/crates/sqlclean/src/sql_lint/_helpers/formatter.rs index 35de174..e531848 100644 --- a/crates/sqlclean/src/sql_lint/_helpers/formatter.rs +++ b/crates/sqlclean/src/sql_lint/_helpers/formatter.rs @@ -4,6 +4,7 @@ use sqlquack_parse::parsing::constants::MAX_FUNCTION_CALL_DEPTH; use sqlquack_parse::parsing::main::call_depth_hint::call_depth_hint; use sqlquack_parse::parsing::main::checked_call_depth::checked_call_depth; use sqlquack_syntax::Dialect as SqlQuackDialect; +use sqlquack_syntax::guard::main::has_function_call::has_function_call; use crate::constants::INTERPOLATION_PLACEHOLDER_PREFIX; use crate::sql_lint::_helpers::cte_macros::{take_out_cte_macros, wrap_query_placeholders}; @@ -105,6 +106,12 @@ fn format_with_sqlquack( reason, max_function_call_depth, Some("max_function_call_depth"), + || { + macros.is_some_and(|dialect| { + sqlquack_parse::dialect_tokens(&request.sql, dialect) + .is_ok_and(|tokens| has_function_call(&tokens)) + }) + }, ) })?; if request.sql.contains(INTERPOLATION_PLACEHOLDER_PREFIX) diff --git a/crates/sqlclean/src/sql_lint/tests/helpers.rs b/crates/sqlclean/src/sql_lint/tests/helpers.rs index 31f5d3a..d07dc96 100644 --- a/crates/sqlclean/src/sql_lint/tests/helpers.rs +++ b/crates/sqlclean/src/sql_lint/tests/helpers.rs @@ -22,6 +22,14 @@ pub(crate) fn nested_function_sql(depth: usize) -> String { format!("SELECT {}1{}", "F(".repeat(depth), ")".repeat(depth)) } +/// `operands` selects joined by UNION ALL: a statement that nests deeply without calling anything. +pub(crate) fn union_all_sql(operands: usize) -> String { + (0..operands) + .map(|index| format!("SELECT {index} AS n")) + .collect::>() + .join(" UNION ALL ") +} + pub(crate) fn diagnostics(sql: &str) -> Result, String> { let response = lint_json( &json!({ diff --git a/crates/sqlclean/src/sql_lint/tests/test_function_depth.rs b/crates/sqlclean/src/sql_lint/tests/test_function_depth.rs index da448b9..6f359c4 100644 --- a/crates/sqlclean/src/sql_lint/tests/test_function_depth.rs +++ b/crates/sqlclean/src/sql_lint/tests/test_function_depth.rs @@ -1,5 +1,7 @@ use serde_json::{Value, json}; +use sqlquack_parse::parsing::constants::CALL_DEPTH_CEILING_NOTE; + use crate::sql_lint::main::engine::lint_json; use crate::sql_lint::main::formatter::format_json; use crate::sql_lint::tests::{helpers, test_types}; @@ -60,6 +62,112 @@ fn given_nested_function_calls_when_linting_then_the_requested_or_default_depth_ } } +#[test] +fn given_sql_past_the_512_ceiling_when_formatting_then_only_sql_with_calls_explains_the_ceiling() { + // The shared parser refuses with its guard code; the native dialects' formatter declines in its own words. + let test_cases = [ + test_types::FormatCeilingTestCase { + description: "snowflake: 512 nested calls are declined, explaining the ceiling once", + dialect: "snowflake", + sql: helpers::nested_function_sql(512), + expected_reason: "value 513 exceeds configured limit 512", + expected_notes: 1, + }, + test_types::FormatCeilingTestCase { + description: "snowflake: 600 UNION ALL operands are declined without the ceiling note", + dialect: "snowflake", + sql: helpers::union_all_sql(600), + expected_reason: "value 513 exceeds configured limit 512", + expected_notes: 0, + }, + test_types::FormatCeilingTestCase { + description: "duckdb: 512 nested calls are declined, explaining the ceiling once", + dialect: "duckdb", + sql: helpers::nested_function_sql(512), + expected_reason: "value 513 exceeds configured limit 512", + expected_notes: 1, + }, + test_types::FormatCeilingTestCase { + description: "duckdb: 600 UNION ALL operands are declined without the ceiling note", + dialect: "duckdb", + sql: helpers::union_all_sql(600), + expected_reason: "value 513 exceeds configured limit 512", + expected_notes: 0, + }, + test_types::FormatCeilingTestCase { + description: "postgres: 512 nested calls are declined by the native formatter", + dialect: "postgres", + sql: helpers::nested_function_sql(512), + expected_reason: "512 levels deep", + expected_notes: 0, + }, + test_types::FormatCeilingTestCase { + description: "postgres: 600 UNION ALL operands are declined without the ceiling note", + dialect: "postgres", + sql: helpers::union_all_sql(600), + expected_reason: "512 levels deep", + expected_notes: 0, + }, + test_types::FormatCeilingTestCase { + description: "bigquery: 512 nested calls are declined by the native formatter", + dialect: "bigquery", + sql: helpers::nested_function_sql(512), + expected_reason: "512 levels deep", + expected_notes: 0, + }, + test_types::FormatCeilingTestCase { + description: "bigquery: 600 UNION ALL operands are declined without the ceiling note", + dialect: "bigquery", + sql: helpers::union_all_sql(600), + expected_reason: "512 levels deep", + expected_notes: 0, + }, + test_types::FormatCeilingTestCase { + description: "databricks: 512 nested calls are declined by the native formatter", + dialect: "databricks", + sql: helpers::nested_function_sql(512), + expected_reason: "512 levels deep", + expected_notes: 0, + }, + test_types::FormatCeilingTestCase { + description: "databricks: 600 UNION ALL operands are declined without the ceiling note", + dialect: "databricks", + sql: helpers::union_all_sql(600), + expected_reason: "512 levels deep", + expected_notes: 0, + }, + ]; + for test_case in &test_cases { + let request = json!({ + "version": 1, + "sql": test_case.sql, + "dialect": test_case.dialect, + "max_function_call_depth": 512, + }); + let response: Value = + serde_json::from_str(&format_json(&request.to_string()).expect("a format response")) + .expect("JSON"); + let reason: &str = response["reason"].as_str().unwrap_or_default(); + assert_eq!( + response["formatted"], + json!(false), + "{}: {reason}", + test_case.description + ); + assert!( + reason.contains(test_case.expected_reason), + "{}: {reason}", + test_case.description + ); + assert_eq!( + reason.matches(CALL_DEPTH_CEILING_NOTE).count(), + test_case.expected_notes, + "{}: {reason}", + test_case.description + ); + } +} + #[test] fn given_nested_function_calls_when_formatting_then_the_requested_or_default_depth_bounds_them() { let test_cases = [ diff --git a/crates/sqlclean/src/sql_lint/tests/test_types.rs b/crates/sqlclean/src/sql_lint/tests/test_types.rs index 7a5c785..814a1cf 100644 --- a/crates/sqlclean/src/sql_lint/tests/test_types.rs +++ b/crates/sqlclean/src/sql_lint/tests/test_types.rs @@ -147,6 +147,16 @@ pub(crate) struct FunctionDepthTestCase { pub expected_error: Option<&'static str>, } +pub(crate) struct FormatCeilingTestCase { + pub description: &'static str, + pub dialect: &'static str, + pub sql: String, + /// Text the refusal must contain. + pub expected_reason: &'static str, + /// How many times the refusal explains the 512 ceiling. + pub expected_notes: usize, +} + pub(crate) struct CrossJoinTestCase { pub description: &'static str, pub sql: &'static str, diff --git a/tests/e2e/src/sqlclean/hosting/_test_types.py b/tests/e2e/src/sqlclean/hosting/_test_types.py index 299295c..1963ea7 100644 --- a/tests/e2e/src/sqlclean/hosting/_test_types.py +++ b/tests/e2e/src/sqlclean/hosting/_test_types.py @@ -32,6 +32,27 @@ class CallDepthTestCase: files: dict[str, str] = field(default_factory=dict) +@dataclass(frozen=True) +class DepthAcceptedTestCase: + """A configuration under which SQL nested up to a limit is checked and formatted.""" + + description: str + config: str + sql: str + expected_codes: tuple[str, ...] + + +@dataclass(frozen=True) +class DepthRefusedTestCase: + """A dialect and a configuration under which SQL nested past a limit is refused, and how.""" + + description: str + config: str + sql: str + expected_message: str + expected_ceiling_notes: int + + @dataclass(frozen=True) class CallDepthFormatTestCase: """A configuration and what formatting 300 nested function calls twice must give.""" diff --git a/tests/e2e/src/sqlclean/hosting/helpers.py b/tests/e2e/src/sqlclean/hosting/helpers.py index a4c972f..9af5378 100644 --- a/tests/e2e/src/sqlclean/hosting/helpers.py +++ b/tests/e2e/src/sqlclean/hosting/helpers.py @@ -43,3 +43,15 @@ def run( check=False, env={**os.environ, **environment}, ) + + +def nested_calls(depth: int) -> str: + """Return depth nested calls around one column.""" + + return "lower(" * depth + "name" + ")" * depth + + +def union_all(operands: int) -> str: + """Return operands selects joined by UNION ALL: deep nesting without any call.""" + + return " union all ".join(f"select {index} as n" for index in range(operands)) diff --git a/tests/e2e/src/sqlclean/hosting/test_sqlclean_command.py b/tests/e2e/src/sqlclean/hosting/test_sqlclean_command.py index f962963..5613310 100644 --- a/tests/e2e/src/sqlclean/hosting/test_sqlclean_command.py +++ b/tests/e2e/src/sqlclean/hosting/test_sqlclean_command.py @@ -13,6 +13,8 @@ CallDepthFormatTestCase, CallDepthTestCase, CheckTestCase, + DepthAcceptedTestCase, + DepthRefusedTestCase, IdempotencyTestCase, ManyFilesTestCase, RewriteTestCase, @@ -20,7 +22,7 @@ SkillsTestCase, SummaryTestCase, ) -from tests.e2e.src.sqlclean.hosting.helpers import run, write_project +from tests.e2e.src.sqlclean.hosting.helpers import nested_calls, run, union_all, write_project from tests.integration.src.sqlclean.rule_testing.helpers import SAMPLE_RULES DUCKDB: str = 'dialect = "duckdb"\n' @@ -40,6 +42,17 @@ NULL_AND_COMMA: str = "select a.id from a, b where a.x = null\n" # SQLCLEAN001 is a default Rule; SQLCLEAN034 and SQLCLEAN050 are opt-in. DEFAULT_AND_OPT_IN: str = "insert into orders select a.id from a where a.x = null\n" +CHECK_NOTHING: str = '[rules]\nselect = ["SQLCLEAN038"]\n' +CEILING_NOTE: str = ( + "The maximum is 512. The statement and every expression around the calls also count toward" + " that limit, so at 512 a bare SELECT can nest at most 511 calls, and fewer fit inside an" + " alias, comparison or CTE." +) +CEILING_REFUSAL: str = "value 513 exceeds configured limit 512" +LIMIT_64_REFUSAL: str = ( + "value 65 exceeds configured limit 64; raise max_function_call_depth, up to 512" +) +DEFAULT_LIMIT_REFUSAL: str = "value 257 exceeds configured limit 256; raise max_function_call_depth" @pytest.mark.parametrize( @@ -231,6 +244,176 @@ def test_given_deeply_nested_calls_when_checking_then_the_configured_depth_bound assert all(test_case.expected_message in str(finding["message"]) for finding in findings) +@pytest.mark.parametrize( + "test_case", + [ + DepthAcceptedTestCase( + description="databricks: a limit of 64 reads 64 nested calls", + config='max_function_call_depth = 64\ndialect = "databricks"\n' + CHECK_NOTHING, + sql=f"select {nested_calls(64)} as n from a\n", + expected_codes=(), + ), + DepthAcceptedTestCase( + description="databricks: at 512 a bare select reads 511 nested calls", + config='max_function_call_depth = 512\ndialect = "databricks"\n' + CHECK_NOTHING, + sql=f"select {nested_calls(511)}\n", + expected_codes=(), + ), + DepthAcceptedTestCase( + description="bigquery: a limit of 64 reads 64 nested calls", + config='max_function_call_depth = 64\ndialect = "bigquery"\n' + CHECK_NOTHING, + sql=f"select {nested_calls(64)} as n from a\n", + expected_codes=(), + ), + DepthAcceptedTestCase( + description="bigquery: at 512 a bare select reads 511 nested calls", + config='max_function_call_depth = 512\ndialect = "bigquery"\n' + CHECK_NOTHING, + sql=f"select {nested_calls(511)}\n", + expected_codes=(), + ), + DepthAcceptedTestCase( + description="postgres: a limit of 64 reads 64 nested calls", + config='max_function_call_depth = 64\ndialect = "postgres"\n' + CHECK_NOTHING, + sql=f"select {nested_calls(64)} as n from a\n", + expected_codes=(), + ), + DepthAcceptedTestCase( + description="postgres: at 512 a bare select reads 511 nested calls", + config='max_function_call_depth = 512\ndialect = "postgres"\n' + CHECK_NOTHING, + sql=f"select {nested_calls(511)}\n", + expected_codes=(), + ), + ], + ids=lambda case: case.description, +) +def test_given_sql_nested_up_to_a_limit_when_checking_and_formatting_then_it_is_read( + tmp_path: Path, test_case: DepthAcceptedTestCase +) -> None: + write_project(tmp_path, config=test_case.config, files={"models/q.sql": test_case.sql}) + + checked: subprocess.CompletedProcess[str] = run( + tmp_path, "check", "--output-format", "json", environment={} + ) + formatted: subprocess.CompletedProcess[str] = run(tmp_path, "format", environment={}) + text: str = (tmp_path / "models/q.sql").read_text(encoding="utf-8") + findings: list[dict[str, object]] = json.loads(checked.stdout)["findings"] + + assert tuple(finding["code"] for finding in findings) == test_case.expected_codes, findings + assert formatted.returncode == 0, formatted.stderr + assert text != test_case.sql + + +@pytest.mark.parametrize( + "test_case", + [ + DepthRefusedTestCase( + description="databricks: a limit of 64 refuses 65 nested calls and names the key", + config='max_function_call_depth = 64\ndialect = "databricks"\n' + CHECK_NOTHING, + sql=f"select {nested_calls(65)} as n from a\n", + expected_message=LIMIT_64_REFUSAL, + expected_ceiling_notes=0, + ), + DepthRefusedTestCase( + description="databricks: the default limit refuses 257 nested calls", + config='dialect = "databricks"\n' + CHECK_NOTHING, + sql=f"select {nested_calls(257)} as n from a\n", + expected_message=DEFAULT_LIMIT_REFUSAL, + expected_ceiling_notes=0, + ), + DepthRefusedTestCase( + description="databricks: at 512 a bare select of 512 nested calls explains the ceiling", + config='max_function_call_depth = 512\ndialect = "databricks"\n' + CHECK_NOTHING, + sql=f"select {nested_calls(512)}\n", + expected_message=CEILING_REFUSAL + ". " + CEILING_NOTE, + expected_ceiling_notes=1, + ), + DepthRefusedTestCase( + description="databricks: 600 union all operands are refused without the ceiling note", + config='max_function_call_depth = 512\ndialect = "databricks"\n' + CHECK_NOTHING, + sql=union_all(600) + "\n", + expected_message=CEILING_REFUSAL, + expected_ceiling_notes=0, + ), + DepthRefusedTestCase( + description="bigquery: a limit of 64 refuses 65 nested calls and names the key", + config='max_function_call_depth = 64\ndialect = "bigquery"\n' + CHECK_NOTHING, + sql=f"select {nested_calls(65)} as n from a\n", + expected_message=LIMIT_64_REFUSAL, + expected_ceiling_notes=0, + ), + DepthRefusedTestCase( + description="bigquery: the default limit refuses 257 nested calls", + config='dialect = "bigquery"\n' + CHECK_NOTHING, + sql=f"select {nested_calls(257)} as n from a\n", + expected_message=DEFAULT_LIMIT_REFUSAL, + expected_ceiling_notes=0, + ), + DepthRefusedTestCase( + description="bigquery: at 512 a bare select of 512 nested calls explains the ceiling", + config='max_function_call_depth = 512\ndialect = "bigquery"\n' + CHECK_NOTHING, + sql=f"select {nested_calls(512)}\n", + expected_message=CEILING_REFUSAL + ". " + CEILING_NOTE, + expected_ceiling_notes=1, + ), + DepthRefusedTestCase( + description="bigquery: 600 union all operands are refused without the ceiling note", + config='max_function_call_depth = 512\ndialect = "bigquery"\n' + CHECK_NOTHING, + sql=union_all(600) + "\n", + expected_message=CEILING_REFUSAL, + expected_ceiling_notes=0, + ), + DepthRefusedTestCase( + description="postgres: a limit of 64 refuses 65 nested calls and names the key", + config='max_function_call_depth = 64\ndialect = "postgres"\n' + CHECK_NOTHING, + sql=f"select {nested_calls(65)} as n from a\n", + expected_message=LIMIT_64_REFUSAL, + expected_ceiling_notes=0, + ), + DepthRefusedTestCase( + description="postgres: the default limit refuses 257 nested calls", + config='dialect = "postgres"\n' + CHECK_NOTHING, + sql=f"select {nested_calls(257)} as n from a\n", + expected_message=DEFAULT_LIMIT_REFUSAL, + expected_ceiling_notes=0, + ), + DepthRefusedTestCase( + description="postgres: at 512 a bare select of 512 nested calls explains the ceiling", + config='max_function_call_depth = 512\ndialect = "postgres"\n' + CHECK_NOTHING, + sql=f"select {nested_calls(512)}\n", + expected_message=CEILING_REFUSAL + ". " + CEILING_NOTE, + expected_ceiling_notes=1, + ), + DepthRefusedTestCase( + description="postgres: 600 union all operands are refused without the ceiling note", + config='max_function_call_depth = 512\ndialect = "postgres"\n' + CHECK_NOTHING, + sql=union_all(600) + "\n", + expected_message=CEILING_REFUSAL, + expected_ceiling_notes=0, + ), + ], + ids=lambda case: case.description, +) +def test_given_sql_nested_past_a_limit_when_checking_and_formatting_then_it_is_refused_as_written( + tmp_path: Path, test_case: DepthRefusedTestCase +) -> None: + write_project(tmp_path, config=test_case.config, files={"models/q.sql": test_case.sql}) + + checked: subprocess.CompletedProcess[str] = run( + tmp_path, "check", "--output-format", "json", environment={} + ) + findings: list[dict[str, object]] = json.loads(checked.stdout)["findings"] + formatted: subprocess.CompletedProcess[str] = run(tmp_path, "format", environment={}) + text: str = (tmp_path / "models/q.sql").read_text(encoding="utf-8") + message: str = " ".join(str(finding["message"]) for finding in findings) + + assert [finding["code"] for finding in findings] == ["SQLCLEAN999"], message + assert test_case.expected_message in message + assert message.count(CEILING_NOTE) == test_case.expected_ceiling_notes + assert formatted.returncode == 1, formatted.stderr + assert text == test_case.sql + assert test_case.expected_message in formatted.stdout + formatted.stderr + + @pytest.mark.parametrize( "test_case", [ diff --git a/website/src/content/docs/docs/configuration.mdx b/website/src/content/docs/docs/configuration.mdx index fb73454..c28aba1 100644 --- a/website/src/content/docs/docs/configuration.mdx +++ b/website/src/content/docs/docs/configuration.mdx @@ -61,7 +61,7 @@ directory is the root. | `include` | `["**/*.sql"]` | Glob patterns of the files to read | | `exclude` | `[]` | Glob patterns of files never to read | | `[[dialects]]` | none | `paths` and `dialect`: a different dialect for matching files | -| `max_function_call_depth` | `256` | The deepest function-call nesting SQLClean parses, from `1` to `512`; a file nested deeper is a `SQLCLEAN999` finding and is not formatted | +| `max_function_call_depth` | `256` | The deepest function-call nesting SQLClean parses, from `1` to `512`; see [Nesting limit](#nesting-limit) | | `[format] line_width` | `100` | The line width the formatter fills; see [Line width](/docs/formatting/#line-width) | | `[format] indent` | `4` | Spaces per indent step, from `1` to `8`; see [Indent](/docs/formatting/#indent) | | `[rules] select` | Rules enabled by default | Rule codes or prefixes to run | @@ -75,6 +75,18 @@ directory is the root. Selection, exceptions and ignores are described in [Rules](/docs/rules/overview/). +## Nesting limit + +`max_function_call_depth` sets how deeply function calls may nest. A file nested deeper is a +`SQLCLEAN999` finding, and `format` leaves it as written. Below 512 the message names the key. + +The maximum is 512. The statement and every expression around the calls also count toward that +limit, so at 512 a bare SELECT can nest at most 511 calls, and fewer fit inside an alias, comparison +or CTE. + +A statement nested more than 512 levels deep is refused the same way in every dialect `check` reads, +whatever the setting and even without function calls, for example 600 `UNION ALL` operands. + ## Choosing files With no paths on the command line, a command reads every file under the configuration root whose diff --git a/website/src/content/docs/docs/embedding.mdx b/website/src/content/docs/docs/embedding.mdx index 99462ec..cc1317b 100644 --- a/website/src/content/docs/docs/embedding.mdx +++ b/website/src/content/docs/docs/embedding.mdx @@ -138,7 +138,9 @@ println!("{}", format_json(request)?); before the `sqlclean` command chose four; pass `"indent": 4` to match `sqlclean format`. `max_function_call_depth` bounds nested function calls as it does in a lint request: `256` by -default, at most `512`. +default. The maximum is 512. The statement and every expression around the calls also count toward +that limit, so at 512 a bare SELECT can nest at most 511 calls, and fewer fit inside an alias, +comparison or CTE. When the formatter cannot format the SQL, `formatted` is `false`, `sql` is the input unchanged, and `reason` says why.