Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 6 additions & 6 deletions coverage/syntax-bigquery/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -490,18 +490,18 @@ parser reports every case as `no_parser`, which must fail rather than compare eq

Where the local parser and BigQuery differ by design rather than by a missing form:

- **Nesting.** The parser counts nested levels against a budget of 200 units and refuses deeper
- **Nesting.** The parser counts nested levels against a budget of 600 units and refuses deeper
statements with "the statement nests too deeply", a parser check rather than a syntax error.
A level can cost more than one unit: the budget allows 98 nested scalar, `EXISTS` and `IN`
A level can cost more than one unit: the budget allows 298 nested scalar, `EXISTS` and `IN`
subqueries, parenthesised joins and set operations, lambdas and parenthesised table-valued
function arguments, 197 parentheses, calls and `CASE` expressions, and fewer levels of mixed
constructs (39 of `(SELECT f(-( ... )))`). It also refuses a statement whose tree nests more
function arguments, 597 parentheses, calls and `CASE` expressions, and fewer levels of mixed
constructs (119 of `(SELECT f(-( ... )))`). It also refuses a statement whose tree nests more
than 8,000 levels, which bounds flat chains and chains nested in another chain's operand.
GoogleSQL's parser accepts all of these at 20,000 levels.
- **Nested keyword-led calls.** A keyword reading of a call such as `TRIM` or `POSITION` is tried
at most once at each position, so nested calls take about quadratic time in their depth; the
nesting budget caps it at about 0.4 s in a debug build and 31 ms in a release build at 195
levels.
nesting budget caps it at about 2.6 s in a debug build and 0.37 s in a release build at 590
levels of `TRIM(`.
- **A string date part.** `EXTRACT('HOUR' FROM ts)` is refused by the local parser as a syntax
error. GoogleSQL's parser takes it and its analysis refuses it ("A valid date part name is
required"), so the verdict matches BigQuery's and only the syntax flag differs.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ pub(crate) fn bigquery_profile() -> LintProfile {
.tokenize(sql)
.map_err(|error| error.to_string())
},
statements: |sql, _| native_statements(sql, strict_parse),
statements: |sql, options| native_statements(sql, strict_parse, options),
predicates: |text, _| native_predicates(text, strict_parse),
own_predicate_parser: true,
foreign_aggregates: &FOREIGN_AGGREGATE_CALLS,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ pub(crate) fn databricks_profile() -> LintProfile {
.tokenize(sql)
.map_err(|error| error.to_string())
},
statements: |sql, _| native_statements(sql, strict_parse),
statements: |sql, options| native_statements(sql, strict_parse, options),
predicates: |text, _| native_predicates(text, strict_parse),
own_predicate_parser: true,
foreign_aggregates: &FOREIGN_AGGREGATE_CALLS,
Expand Down
25 changes: 20 additions & 5 deletions crates/sqlquack-lint/src/dialects/main/native_statements.rs
Original file line number Diff line number Diff line change
@@ -1,14 +1,29 @@
//! `native_statements`: the statements of SQL from a dialect's own strict parser.

use sqlquack_syntax::Expression;
use sqlquack_syntax::guard::main::enforce_ast::enforce_ast;
use sqlquack_syntax::{ComplexityGuardOptions, Expression, ParseOptions};

use crate::dialects::_helpers::parse_errors::{parse_error, uncovered};
use crate::dialects::types::StrictParse;

/// The statements `parse` reads from `sql`, refusing what that engine refuses with the line and column where it stopped.
pub(crate) fn native_statements(sql: &str, parse: StrictParse) -> Result<Vec<Expression>, String> {
parse(sql).map_err(|(position, message)| match message {
/// The statements `parse` reads from `sql`, refusing what that engine refuses and trees past `options`' AST depth.
pub(crate) fn native_statements(
sql: &str,
parse: StrictParse,
options: &ParseOptions,
) -> Result<Vec<Expression>, String> {
let statements: Vec<Expression> = parse(sql).map_err(|(position, message)| match message {
Some(message) => parse_error(sql, position, &message),
None => uncovered(sql, position),
})
})?;
let depth_only = ComplexityGuardOptions {
max_ast_nodes: None,
max_ast_depth: options
.complexity_guard
.as_ref()
.and_then(|guard| guard.max_ast_depth),
..Default::default()
};
enforce_ast(&statements, &depth_only).map_err(|error| error.to_string())?;
Ok(statements)
}
49 changes: 39 additions & 10 deletions crates/sqlquack-lint/src/lint/_helpers/parsing.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,11 @@

use std::collections::HashSet;

use sqlquack_parse::parsing::constants::MAX_FUNCTION_CALL_DEPTH;
use sqlquack_parse::parsing::constants::{MAX_CALL_DEPTH_CEILING, MAX_FUNCTION_CALL_DEPTH};
use sqlquack_parse::parsing::main::call_depth_hint::call_depth_hint;
use sqlquack_parse::parsing::main::checked_call_depth::checked_call_depth;
use sqlquack_syntax::guard::main::enforce_lexed_tokens::enforce_lexed_tokens;
use sqlquack_syntax::guard::main::has_function_call::has_function_call;
use sqlquack_syntax::tokens::models::{Token, TokenType};
use sqlquack_syntax::{ComplexityGuardOptions, Expression, ExpressionWalk, ParseOptions};

Expand All @@ -23,23 +25,50 @@ pub(crate) fn parse(
.map_err(|reason| format!("max_function_call_depth: {reason}"))?;
let profile: LintProfile = profile(dialect);
let tokens: Vec<Token> = (profile.tokens)(sql)?;
let parse_text = empty_relation
.map(|name| without_empty_relation_parentheses(sql, &tokens, name))
.unwrap_or_default();
let (parse_text, parse_tokens): (&str, Vec<Token>) = if parse_text.is_empty() {
(sql, Vec::new())
} else {
(&parse_text, (profile.tokens)(&parse_text)?)
};
let guarded_tokens: &[Token] = if parse_tokens.is_empty() {
&tokens
} else {
&parse_tokens
};
let depth_guard = ComplexityGuardOptions {
max_parser_depth: None,
max_input_bytes: None,
max_tokens: None,
max_ast_nodes: None,
max_ast_depth: None,
max_parenthesis_depth: Some(MAX_CALL_DEPTH_CEILING),
max_function_call_depth: Some(max_call_depth),
};
enforce_lexed_tokens(guarded_tokens, &depth_guard).map_err(|error| {
call_depth_hint(
error.to_string(),
max_call_depth,
Some("max_function_call_depth"),
|| has_function_call(guarded_tokens),
)
})?;
let options = ParseOptions {
complexity_guard: Some(ComplexityGuardOptions {
max_function_call_depth: Some(max_call_depth),
..Default::default()
}),
};
let parse_text = empty_relation
.map(|name| without_empty_relation_parentheses(sql, &tokens, name))
.unwrap_or_default();
let parse_text = if parse_text.is_empty() {
sql
} else {
&parse_text
};
let statements: Vec<Expression> =
(profile.statements)(parse_text, &options).map_err(|message| {
call_depth_hint(message, max_call_depth, Some("max_function_call_depth"))
call_depth_hint(
message,
max_call_depth,
Some("max_function_call_depth"),
|| has_function_call(guarded_tokens),
)
})?;
for statement in &statements {
if statement
Expand Down
69 changes: 68 additions & 1 deletion crates/sqlquack-lint/src/lint/tests/helpers.rs
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,47 @@ pub(crate) fn nested_function_sql(depth: usize) -> String {
format!("SELECT {}1{}", "F(".repeat(depth), ")".repeat(depth))
}

/// `depth` nested calls under an output alias.
pub(crate) fn aliased_calls_sql(depth: usize) -> String {
format!(
"SELECT {}1{} AS n FROM t",
"F(".repeat(depth),
")".repeat(depth)
)
}

/// `depth` nested calls compared in a WHERE clause.
pub(crate) fn compared_calls_sql(depth: usize) -> String {
format!(
"SELECT * FROM t WHERE {}1{} = 1",
"F(".repeat(depth),
")".repeat(depth)
)
}

/// `depth` nested calls under an alias inside a CTE.
pub(crate) fn cte_calls_sql(depth: usize) -> String {
format!(
"WITH c AS (SELECT {}1{} AS n FROM t) SELECT n FROM c",
"F(".repeat(depth),
")".repeat(depth)
)
}

/// `depth` nested calls around a subquery reading the fixture harness's empty relation.
pub(crate) fn calls_around_empty_relation_sql(depth: usize) -> String {
format!(
"SELECT {}(SELECT 1 FROM __empty_fixture()){}",
"F(".repeat(depth),
")".repeat(depth)
)
}

/// `depth` UNION ALL operands after a first SELECT, with no call.
pub(crate) fn union_chain_sql(depth: usize) -> String {
format!("SELECT 1{}", " UNION ALL SELECT 1".repeat(depth))
}

pub(crate) fn nested_from_sql(depth: usize) -> String {
format!(
"SELECT * FROM {}orders{}",
Expand All @@ -26,17 +67,43 @@ pub(crate) fn diagnostics(sql: &str) -> Result<Vec<Value>, String> {
pub(crate) fn diagnostics_within(
sql: &str,
max_function_call_depth: Option<usize>,
) -> Result<Vec<Value>, String> {
dialect_diagnostics_within(sql, "snowflake", max_function_call_depth)
}

/// The `dialect` diagnostics of `sql`, with the request's `max_function_call_depth`, null when absent.
pub(crate) fn dialect_diagnostics_within(
sql: &str,
dialect: &str,
max_function_call_depth: Option<usize>,
) -> Result<Vec<Value>, String> {
let request = json!({
"version": 1,
"sql": sql,
"dialect": "snowflake",
"dialect": dialect,
"max_function_call_depth": max_function_call_depth
});
let response = lint_json(&request.to_string())?;
diagnostic_values(&response)
}

/// The `dialect` diagnostics of `sql` in the fixture harness, with the request's `max_function_call_depth`.
pub(crate) fn harness_diagnostics_within(
sql: &str,
dialect: &str,
max_function_call_depth: Option<usize>,
) -> Result<Vec<Value>, String> {
let request = json!({
"version": 1,
"sql": sql,
"dialect": dialect,
"max_function_call_depth": max_function_call_depth,
"fixture_harness": fixture_harness()
});
let response = lint_json(&request.to_string())?;
diagnostic_values(&response)
}

pub(crate) fn diagnostics_for_rules(sql: &str, rules: &[&str]) -> Result<Vec<Value>, String> {
diagnostics_for_dialect(sql, "snowflake", rules)
}
Expand Down
Loading
Loading