diff --git a/apple/Plugins/FormatPluginTests/SQLImportBatchTests.swift b/apple/Plugins/FormatPluginTests/SQLImportBatchTests.swift new file mode 100644 index 0000000000..5c712ad158 --- /dev/null +++ b/apple/Plugins/FormatPluginTests/SQLImportBatchTests.swift @@ -0,0 +1,307 @@ +// +// SQLImportBatchTests.swift +// FormatPluginTests +// + +import Foundation +import TableProPluginKit +import Testing + +private struct Refused: LocalizedError { + var errorDescription: String? { "duplicate key value" } +} + +private struct Unreadable: LocalizedError { + var errorDescription: String? { "The string that starts on line 3 is longer than the import reads." } +} + +private struct RulesChanged: Error {} + +private final class BatchingSink: PluginImportDataSink, @unchecked Sendable { + let databaseTypeId = "PostgreSQL" + var events: [String] = [] + var failing: Set = [] + var refusesBatches = false + var changesReadingRules = false + var stopsDuringBatch: Progress? + + func execute(statement: String) async throws { + events.append(statement) + if failing.contains(statement) { throw Refused() } + } + + func batches(_ statement: String) -> Bool { + statement.hasPrefix("INSERT") && !statement.contains("\\") + } + + func execute(statements: [PluginImportStatement], readAhead: Bool) async throws { + events.append("batch \(statements.count)\(readAhead ? " read ahead" : "")") + if refusesBatches { throw Refused() } + stopsDuringBatch?.cancel() + for (index, statement) in statements.enumerated() where failing.contains(statement.text) { + throw PluginImportBatchFailure(index: index, underlying: Refused()) + } + if changesReadingRules, readAhead { + throw PluginImportBatchFailure(index: statements.count, underlying: RulesChanged()) + } + } + + func beginTransaction() async throws { events.append("begin") } + func commitTransaction() async throws { events.append("commit") } + func rollbackTransaction() async throws { events.append("rollback") } +} + +private final class LineSource: PluginImportSource, @unchecked Sendable { + private let lines: [String] + private let failsAfterLines: Bool + + init(_ lines: [String], failsAfterLines: Bool = false) { + self.lines = lines + self.failsAfterLines = failsAfterLines + } + + func statements() async throws -> AsyncThrowingStream<(statement: String, lineNumber: Int), Error> { + let lines = self.lines + let fails = failsAfterLines + return AsyncThrowingStream { continuation in + for (index, line) in lines.enumerated() { + continuation.yield((statement: line, lineNumber: index + 1)) + } + continuation.finish(throwing: fails ? Unreadable() : nil) + } + } + + func fileURL() -> URL { URL(fileURLWithPath: "/dev/null") } + + func fileSizeBytes() -> Int64 { 0 } +} + +@Suite(.serialized) +struct SQLImportBatchTests { + private static func inserts(_ count: Int) -> [String] { + (1...count).map { "INSERT INTO t VALUES (\($0))" } + } + + private func runImport( + _ statements: [String], + errorHandling: ImportErrorHandling = .stopAndRollback, + wrapInTransaction: Bool = true, + sink: BatchingSink, + source: LineSource? = nil, + progress: Progress = Progress() + ) async -> Result { + let plugin = SQLImportPlugin() + let original = plugin.settings + defer { plugin.settings = original } + plugin.settings.errorHandling = errorHandling + plugin.settings.wrapInTransaction = wrapInTransaction + plugin.settings.disableForeignKeyChecks = false + do { + return .success(try await plugin.performImport( + source: source ?? LineSource(statements), + sink: sink, + progress: PluginImportProgress(progress: progress) + )) + } catch { + return .failure(error) + } + } + + private static func failedLine(_ outcome: Result) -> (String, Int)? { + guard case .failure(let error) = outcome, + case .statementFailed(let statement, let line, _) = error as? PluginImportError else { return nil } + return (statement, line) + } + + @Test("Statements inside the import's transaction go out in batches") + func sendsBatches() async throws { + let sink = BatchingSink() + + let result = try await runImport(Self.inserts(3), sink: sink).get() + + #expect(sink.events == ["begin", "batch 3", "commit"]) + #expect(result.executedStatements == 3) + } + + @Test("A batch stops the import at its failed statement and names that statement's line") + func failureNamesItsLine() async { + let sink = BatchingSink() + let statements = Self.inserts(3) + sink.failing = [statements[1]] + + let outcome = await runImport(statements, sink: sink) + + #expect(Self.failedLine(outcome)?.0 == statements[1]) + #expect(Self.failedLine(outcome)?.1 == 2) + #expect(sink.events == ["begin", "batch 3", "rollback"]) + } + + @Test("A batch refused before any statement ran fails as its first statement would have") + func refusedBatchNamesItsFirstStatement() async { + let sink = BatchingSink() + sink.refusesBatches = true + + let outcome = await runImport(Self.inserts(3), sink: sink) + + #expect(Self.failedLine(outcome)?.1 == 1) + #expect(sink.events == ["begin", "batch 3", "rollback"]) + } + + @Test("A Stop that lands while the last batch runs rolls the import back instead of committing it") + func stopDuringLastBatchRollsBack() async { + let sink = BatchingSink() + let progress = Progress() + sink.stopsDuringBatch = progress + + let outcome = await runImport(Self.inserts(3), sink: sink, progress: progress) + + guard case .failure = outcome else { + Issue.record("A stopped import should not succeed") + return + } + #expect(sink.events == ["begin", "batch 3", "rollback"]) + } + + @Test("Stop and commit commits what ran before a batched failure") + func stopAndCommitCommits() async { + let sink = BatchingSink() + let statements = Self.inserts(3) + sink.failing = [statements[2]] + + let outcome = await runImport(statements, errorHandling: .stopAndCommit, sink: sink) + + #expect(Self.failedLine(outcome)?.1 == 3) + #expect(sink.events == ["begin", "batch 3", "commit"]) + } + + @Test("A statement the connection keeps alone first sends the batch waiting ahead of it, saying the file was read past it") + func loneStatementFlushesFirst() async throws { + let sink = BatchingSink() + let statements = [Self.inserts(1)[0], "SET search_path = app", "INSERT INTO t VALUES (9)"] + + _ = try await runImport(statements, sink: sink).get() + + #expect(sink.events == ["begin", "batch 1 read ahead", "SET search_path = app", "batch 1", "commit"]) + } + + @Test("A statement holding a backslash runs alone, after the batch read ahead of it") + func backslashRunsAlone() async throws { + let sink = BatchingSink() + let escaped = #"INSERT INTO t VALUES ('a\'b')"# + let statements = [Self.inserts(1)[0], escaped, "INSERT INTO t VALUES (9)"] + + _ = try await runImport(statements, sink: sink).get() + + #expect(sink.events == ["begin", "batch 1 read ahead", escaped, "batch 1", "commit"]) + } + + @Test("A batch that changed how the file reads stops the import at the statement read past it") + func changedReadingRulesStopAtTheNextStatement() async { + let sink = BatchingSink() + sink.changesReadingRules = true + let escaped = #"INSERT INTO t VALUES ('a\'b')"# + + let outcome = await runImport(Self.inserts(2) + [escaped], sink: sink) + + #expect(Self.failedLine(outcome)?.0 == escaped) + #expect(Self.failedLine(outcome)?.1 == 3) + #expect(sink.events == ["begin", "batch 2 read ahead", "rollback"]) + } + + @Test("Stop and commit keeps the statements that ran before the one the import could no longer read") + func changedReadingRulesUnderStopAndCommit() async { + let sink = BatchingSink() + sink.changesReadingRules = true + + let outcome = await runImport(Self.inserts(2) + ["SET x = 1"], errorHandling: .stopAndCommit, sink: sink) + + #expect(Self.failedLine(outcome)?.1 == 3) + #expect(sink.events == ["begin", "batch 2 read ahead", "commit"]) + } + + @Test("A statement that fails ahead of text the file cannot read is the failure reported, as it was one at a time") + func failureAheadOfReadFailureIsReported() async { + let sink = BatchingSink() + let statements = Self.inserts(2) + sink.failing = [statements[1]] + + let outcome = await runImport(statements, sink: sink, source: LineSource(statements, failsAfterLines: true)) + + #expect(Self.failedLine(outcome)?.1 == 2) + #expect(sink.events == ["begin", "batch 2 read ahead", "rollback"]) + } + + @Test("A read failure after a batch rolls back in every stop mode, as one read one statement at a time does", arguments: [ + ImportErrorHandling.stopAndRollback, .stopAndCommit, + ]) + func readFailureRollsBack(errorHandling: ImportErrorHandling) async { + let sink = BatchingSink() + let statements = Self.inserts(2) + + let outcome = await runImport( + statements, errorHandling: errorHandling, sink: sink, source: LineSource(statements, failsAfterLines: true) + ) + + guard case .failure(let error) = outcome, + case .importFailed(let message) = error as? PluginImportError else { + Issue.record("The read failure should end the import") + return + } + #expect(message == Unreadable().errorDescription) + #expect(sink.events == ["begin", "batch 2 read ahead", "rollback"]) + } + + @Test("A batch that changed how the file reads, ahead of a read failure, still rolls back under Stop and Commit") + func changedReadingRulesBeforeReadFailureRollBack() async { + let sink = BatchingSink() + sink.changesReadingRules = true + let statements = Self.inserts(2) + + let outcome = await runImport( + statements, errorHandling: .stopAndCommit, sink: sink, source: LineSource(statements, failsAfterLines: true) + ) + + guard case .failure(let error) = outcome, case .importFailed = error as? PluginImportError else { + Issue.record("The import should fail as a read failure does") + return + } + #expect(sink.events == ["begin", "batch 2 read ahead", "rollback"]) + } + + @Test("Skip and continue runs outside a transaction, so every statement goes alone") + func skipModeNeverBatches() async throws { + let sink = BatchingSink() + let statements = Self.inserts(3) + sink.failing = [statements[1]] + + let result = try await runImport(statements, errorHandling: .skipAndContinue, sink: sink).get() + + #expect(sink.events == statements) + #expect(result.executedStatements == 2) + #expect(result.skippedStatements == 1) + } + + @Test("Without the import's transaction every statement goes alone", arguments: [ + ImportErrorHandling.stopAndRollback, .stopAndCommit, + ]) + func unwrappedImportNeverBatches(errorHandling: ImportErrorHandling) async { + let sink = BatchingSink() + let statements = Self.inserts(3) + sink.failing = [statements[1]] + + let outcome = await runImport(statements, errorHandling: errorHandling, wrapInTransaction: false, sink: sink) + + #expect(Self.failedLine(outcome)?.1 == 2) + #expect(sink.events == Array(statements.prefix(2))) + } + + @Test("A batch never grows past its statement limit") + func batchesAreBounded() async throws { + let sink = BatchingSink() + let limit = SQLImportBatch.statementLimit + + _ = try await runImport(Self.inserts(limit * 2 + 7), sink: sink).get() + + #expect(sink.events == ["begin", "batch \(limit)", "batch \(limit)", "batch 7", "commit"]) + } +} diff --git a/apple/Plugins/PostgreSQLDriverPlugin/LibPQDriverCore.swift b/apple/Plugins/PostgreSQLDriverPlugin/LibPQDriverCore.swift index cfba761901..3e13376dcd 100644 --- a/apple/Plugins/PostgreSQLDriverPlugin/LibPQDriverCore.swift +++ b/apple/Plugins/PostgreSQLDriverPlugin/LibPQDriverCore.swift @@ -447,6 +447,10 @@ final class LibPQDriverCore: @unchecked Sendable { ) } + func executePipeline(_ statements: [LibPQPipelineStatement]) async throws -> [PluginBatchStatementOutcome]? { + try await connection().executePipeline(statements) + } + func executeReadOnly( query: String, parameters: [PluginCellValue]?, @@ -796,6 +800,54 @@ extension LibPQBackedDriver { return try await readOnlyRead(query: query, parameters: parameters, rowCap: nil, context: context, scope: readOnly) } + // Each stretch a pipeline admits goes as one, every other statement alone, and the first failure ends the run. + func executeStatements(_ statements: [PluginBatchStatement]) async throws -> [PluginBatchStatementOutcome]? { + guard capabilities.contains(.statementBatches), + statements.allSatisfy({ readOnlyScope($0.context) == nil && !$0.context.readsCatalog && $0.context.origin == .app }), + await sessionTransactionState() == .inTransaction + else { return nil } + var outcomes: [PluginBatchStatementOutcome] = [] + var index = 0 + while index < statements.count { + // Read again each time: a statement sent alone may have changed how the session reads a literal. + let admission = LibPQPipelineAdmission(session: sessionLexicalState) + var end = index + while end < statements.count, admission.admits(statements[end].query) { end += 1 } + if end - index > 1 { + let stretch = statements[index.. LibPQReadOnlyTransaction.Scope? { LibPQReadOnlyTransaction.scope(for: context, capabilities: capabilities) } diff --git a/apple/Plugins/PostgreSQLDriverPlugin/LibPQPipeline.swift b/apple/Plugins/PostgreSQLDriverPlugin/LibPQPipeline.swift new file mode 100644 index 0000000000..9d2b8d88e0 --- /dev/null +++ b/apple/Plugins/PostgreSQLDriverPlugin/LibPQPipeline.swift @@ -0,0 +1,137 @@ +// +// LibPQPipeline.swift +// PostgreSQLDriverPlugin +// + +import Foundation +import TableProPluginKit +import TableProSQLGrammar + +nonisolated struct LibPQPipelineStatement: Sendable { + let query: String + let parameters: [PluginCellValue] +} + +// libpq hands a pipeline's results back in send order, each statement's ending with a null, and an error aborts every +// statement after it up to the Sync. +nonisolated struct LibPQPipelineTally { + enum Answer { + case completed(rowsAffected: Int, commandTag: String?) + case failed(any Error) + case aborted + } + + private(set) var answers: [Answer] = [] + private(set) var times: [TimeInterval] = [] + private var pending: Answer? + private var rowsSoFar = 0 + private var lastArrival: Date + private let total: Int + var sent = 0 + private(set) var receivedSync = false + + init(total: Int, startedAt: Date) { + self.total = total + lastArrival = startedAt + } + + // Asking libpq for a result past the last one sent would read a null that ends no statement. + var awaitsStatement: Bool { + answers.count < sent + } + + var isComplete: Bool { + receivedSync + } + + mutating func take(_ answer: Answer) { + guard pending == nil else { return } + if case .completed(let rowsAffected, let commandTag) = answer { + pending = .completed(rowsAffected: rowsAffected + rowsSoFar, commandTag: commandTag) + } else { + pending = answer + } + } + + // A row read alone, counted and dropped, as a lone read past its limit counts and drops it. + mutating func takeRow() { + rowsSoFar += 1 + } + + mutating func takeSync() { + receivedSync = true + } + + // False for a null with no statement behind it, which only a broken connection produces. + mutating func endStatement(at arrival: Date = Date()) -> Bool { + guard awaitsStatement, let answer = pending else { return false } + answers.append(answer) + times.append(arrival.timeIntervalSince(lastArrival)) + lastArrival = arrival + pending = nil + rowsSoFar = 0 + return true + } + + // Stops at the first failure; an aborted statement never ran, so it has no outcome. `cutShort` is why the exchange + // ended early: the failure of the first statement with no answer, or of the last one when its Sync never came. + func outcomes(cutShort: (any Error)?) -> [PluginBatchStatementOutcome] { + var outcomes: [PluginBatchStatementOutcome] = [] + answers: for (answer, time) in zip(answers, times) { + switch answer { + case .completed(let rowsAffected, _): + outcomes.append(.succeeded(rowsAffected: rowsAffected, executionTime: time)) + case .failed(let error): + outcomes.append(.failed(error)) + return outcomes + case .aborted: + break answers + } + } + guard let cutShort else { return outcomes } + if outcomes.count < total { + outcomes.append(.failed(cutShort)) + } else if !outcomes.isEmpty { + outcomes[outcomes.count - 1] = .failed(cutShort) + } + return outcomes + } + + var commandTags: [String] { + answers.compactMap { + guard case .completed(_, let tag) = $0 else { return nil } + return tag + } + } +} + +nonisolated struct LibPQPipelineAdmission { + // A pipeline runs on the extended protocol, where these either fail or end the block the batch relies on. + private static let refusedKeywords: Set = [ + "BEGIN", "START", "COMMIT", "END", "ROLLBACK", "ABORT", "SAVEPOINT", "RELEASE", "PREPARE", "COPY", + "VACUUM", "CHECKPOINT", "CALL", "DO", "LISTEN", "UNLISTEN", "NOTIFY", "DISCARD", "RESET", "SET", + ] + + private let grammar: SQLLexicalGrammar + + // The session decides only whether a backslash escapes in a plain literal (`standard_conforming_strings`). + init(session: PluginSessionLexicalState?) { + let facts = session.map { + SQLSessionLexicalFacts( + determined: .backslashEscapesInSingleQuotes, + enabled: $0.enabled.contains(.backslashEscapesInSingleQuotes) ? .backslashEscapesInSingleQuotes : [] + ) + } + grammar = SQLLexicalReadings.resolve( + databaseTypeId: PostgreSQLPlugin.databaseTypeId, declared: nil, session: facts + ).execution + } + + // The extended protocol also refuses a text holding a second statement, which a lone simple Query runs whole. + func admits(_ query: String) -> Bool { + guard let keyword = SQLStatementRowLimit.leadingKeyword(query, grammar: grammar), + !Self.refusedKeywords.contains(keyword) + else { return false } + return SQLStatementScanner.executableStatements(in: query, grammar: grammar).count == 1 + } +} diff --git a/apple/Plugins/PostgreSQLDriverPlugin/LibPQPluginConnection.swift b/apple/Plugins/PostgreSQLDriverPlugin/LibPQPluginConnection.swift index 4fc5aefc4b..cfa19375b4 100644 --- a/apple/Plugins/PostgreSQLDriverPlugin/LibPQPluginConnection.swift +++ b/apple/Plugins/PostgreSQLDriverPlugin/LibPQPluginConnection.swift @@ -2475,3 +2475,159 @@ extension LibPQPluginConnection { return Array(results.dropFirst()) } } + +// MARK: - Statement Pipelines + +extension LibPQPluginConnection { + private static let pipelineWaitMicroseconds: Int64 = 100_000 + + // Inside a transaction block only: there an error aborts every statement after it until the Sync, as a caller + // stopping at the failure would. Outside one the pipeline is one implicit transaction, and a failure would roll + // back statements that, sent alone, commit. Nil also when libpq will not enter pipeline mode. + func executePipeline(_ statements: [LibPQPipelineStatement]) async throws -> [PluginBatchStatementOutcome]? { + try await pluginDispatchAsync(on: queue) { [self] () throws -> [PluginBatchStatementOutcome]? in + guard !isShuttingDown, let conn = connectionHandle else { throw LibPQPluginError.notConnected } + guard activeConnectDeadline == nil, !statements.isEmpty, transactionStateOnQueue() == .inTransaction else { + return nil + } + return try pipelineSync(statements, conn: conn) + } + } + + fileprivate func pipelineSync( + _ statements: [LibPQPipelineStatement], + conn: OpaquePointer + ) throws -> [PluginBatchStatementOutcome]? { + let generation = cancellationGate.beginQuery() + defer { cancellationGate.endQuery(generation) } + defer { refreshStandardConformingStrings(from: conn) } + + finishAbandonedResults(conn, cancellingOutput: cancelsAbandonedOutput(conn)) + if let ended = sessionEndedBeforeSending(conn) { return [.failed(ended)] } + guard PQenterPipelineMode(conn) == 1 else { return nil } + // Non-blocking sends, with replies read between them: a server whose output nobody reads stops reading. + guard PQsetnonblocking(conn, 1) == 0 else { + _ = PQexitPipelineMode(conn) + return nil + } + + var tally = LibPQPipelineTally(total: statements.count, startedAt: Date()) + var cutShort: (any Error)? + var sending = true + for statement in statements where sending { + // Stop holds back what is not sent yet; the statement it lands on fails with the cancel. + if cancellationGate.isCancelled(generation) { + cutShort = CancellationError() + break + } + let sent: Int32 + do { + sent = try LibPQParameterBuffers.with(statement.parameters) { values, lengths, formats in + statement.query.withCString { queryPtr in + PQsendQueryParams(conn, queryPtr, Int32(statement.parameters.count), nil, values, lengths, formats, 0) + } + } + } catch { + // The pipeline is already open, so it still gets its Sync. + cutShort = error + break + } + guard sent != 0 else { + cutShort = getError(from: conn) + break + } + tally.sent += 1 + readsRowByRow(conn) + sending = flushPipeline(conn, tally: &tally, generation: generation) + } + // Without its Sync the server holds every reply, so a read would wait for ever. + let synced = sending && PQsendPipelineSync(conn) == 1 && flushPipeline(conn, tally: &tally, generation: generation) + if synced, PQsetnonblocking(conn, 0) == 0 { + while PQstatus(conn) == CONNECTION_OK, !tally.isComplete { + guard let result = PQgetResult(conn) else { + guard tally.endStatement() else { break } + readsRowByRow(conn) + continue + } + take(result, into: &tally, conn: conn, generation: generation) + } + } + recordTransactionState(of: conn) + + if PQstatus(conn) != CONNECTION_OK || !tally.isComplete { + let reason = (cutShort as? LibPQPluginError) ?? getError(from: conn) + if PQstatus(conn) == CONNECTION_OK { + // Its replies can no longer be told apart, so the session is not used again. + Self.logger.fault("A statement pipeline ended before its Sync; dropping the connection") + let loss = LibPQConnectionLoss(sent: true, recordedState: lastTransactionState) + stateLock.withLock { + _lastTransactionState = .unknown + _hasLostConnection = true + } + disconnect() + cutShort = LibPQConnectionLostError(loss: loss, underlying: reason) + } else { + cutShort = lostConnection(reason, on: conn, sent: true) + } + } else if PQexitPipelineMode(conn) != 1 { + Self.logger.fault("libpq stayed in pipeline mode; dropping the connection") + stateLock.withLock { _hasLostConnection = true } + disconnect() + } else { + for tag in Set(tally.commandTags) { noteCommandTag(tag, conn: conn) } + } + return tally.outcomes(cutShort: cutShort) + } + + // False once the connection fails. Each wait also reads, so the server's replies never back up behind ours. + private func flushPipeline(_ conn: OpaquePointer, tally: inout LibPQPipelineTally, generation: Int) -> Bool { + while true { + let flushed = PQflush(conn) + if flushed == 0 { return true } + guard flushed == 1, !isShuttingDown else { return false } + let socket = PQsocket(conn) + guard socket >= 0 else { return false } + let deadline = pg_usec_time_t(Int64(PQgetCurrentTimeUSec()) + Self.pipelineWaitMicroseconds) + guard PQsocketPoll(socket, 1, 1, deadline) >= 0 || errno == EINTR else { return false } + guard PQconsumeInput(conn) == 1 else { return false } + while tally.awaitsStatement, PQisBusy(conn) == 0 { + guard let result = PQgetResult(conn) else { + guard tally.endStatement() else { return false } + readsRowByRow(conn) + continue + } + take(result, into: &tally, conn: conn, generation: generation) + } + } + } + + // For the statement whose replies come next, before any of them is read; libpq refuses once one has been, and an + // aborted statement has none to read. + private func readsRowByRow(_ conn: OpaquePointer) { + _ = PQsetSingleRowMode(conn) + } + + private func take(_ result: OpaquePointer, into tally: inout LibPQPipelineTally, conn: OpaquePointer, generation: Int) { + defer { PQclear(result) } + let status = PQresultStatus(result) + switch status { + case PGRES_SINGLE_TUPLE: + tally.takeRow() + case PGRES_PIPELINE_SYNC: + tally.takeSync() + case PGRES_PIPELINE_ABORTED: + tally.take(.aborted) + case PGRES_COMMAND_OK: + tally.take(.completed(rowsAffected: getAffectedRows(from: result), commandTag: getCommandTag(from: result))) + case PGRES_TUPLES_OK: + tally.take(.completed(rowsAffected: Int(PQntuples(result)), commandTag: getCommandTag(from: result))) + default: + let error = getResultError(from: result) + if cancellationGate.isCancelled(generation) { + tally.take(.failed(CancellationError())) + } else { + tally.take(.failed(lostConnection(error, on: conn, sent: true))) + } + } + } +} diff --git a/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLPluginDriver.swift b/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLPluginDriver.swift index 71c04e61a9..1aeedb7718 100644 --- a/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLPluginDriver.swift +++ b/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLPluginDriver.swift @@ -36,7 +36,7 @@ class PostgreSQLPluginDriver: LibPQBackedDriver, @unchecked Sendable { } var capabilities: PluginCapabilities { - [ + var capabilities: PluginCapabilities = [ .parameterizedQueries, .transactions, .alterTableDDL, @@ -55,6 +55,9 @@ class PostgreSQLPluginDriver: LibPQBackedDriver, @unchecked Sendable { .enforcesReadOnlyMode, .copyFromStandardInput, ] + // Pipelines were measured on PostgreSQL alone, not on PGlite or KingbaseES. + if tunesCommunitySession { capabilities.insert(.statementBatches) } + return capabilities } // In the class body so PGlite and KingbaseES, neither measured with these session settings, can turn them off. diff --git a/apple/Plugins/PostgreSQLDriverTests/LibPQPipelineTests.swift b/apple/Plugins/PostgreSQLDriverTests/LibPQPipelineTests.swift new file mode 100644 index 0000000000..471e6f1f98 --- /dev/null +++ b/apple/Plugins/PostgreSQLDriverTests/LibPQPipelineTests.swift @@ -0,0 +1,327 @@ +// +// LibPQPipelineTests.swift +// PostgreSQLDriverTests +// + +import Foundation +import TableProPluginKit +import Testing + +@Suite(.timeLimit(.minutes(1))) +struct LibPQPipelineTests { + private static let inserts = (1...3).map { "INSERT INTO events VALUES (\($0))" } + + private static func server(failing: String? = nil, closingAfter: String? = nil) throws -> LibPQScriptedServer { + try LibPQScriptedServer( + status: { query, earlier in + (earlier + [query]).contains("BEGIN") ? LibPQScriptedServer.inTransaction : LibPQScriptedServer.idle + }, + answer: { query in + guard query.hasPrefix("INSERT") else { return nil } + if query == failing { + return .init(failure: .init(sqlState: "23505", message: "duplicate key value violates unique constraint")) + } + return .init(completion: "INSERT 0 1", closesSessionAtSync: query == closingAfter) + } + ) + } + + private static func connected(to server: LibPQScriptedServer) async throws -> LibPQPluginConnection { + let connection = LibPQPluginConnection( + host: "127.0.0.1", port: server.port, user: "postgres", password: nil, database: "postgres" + ) + try await connection.connect() + try await connection.finishConnecting() + return connection + } + + private static func driver(on server: LibPQScriptedServer) async throws -> PostgreSQLPluginDriver { + let driver = PostgreSQLPluginDriver(config: DriverConnectionConfig( + host: "127.0.0.1", port: server.port, username: "postgres", password: "", database: "postgres" + )) + try await driver.connect() + return driver + } + + private static func statements(_ queries: [String]) -> [LibPQPipelineStatement] { + queries.map { LibPQPipelineStatement(query: $0, parameters: []) } + } + + private static func rowCounts(_ outcomes: [PluginBatchStatementOutcome]?) -> [Int?] { + (outcomes ?? []).map { + guard case .succeeded(let rowsAffected, _) = $0 else { return nil } + return rowsAffected + } + } + + @Test("Inside a transaction block the statements go out as one pipeline and each reports its own count") + func pipelinesInsideTransaction() async throws { + let server = try Self.server() + defer { server.close() } + server.start() + let connection = try await Self.connected(to: server) + defer { connection.disconnect() } + _ = try await connection.executeQuery("BEGIN") + + let outcomes = try await connection.executePipeline(Self.statements(Self.inserts)) + + #expect(Self.rowCounts(outcomes) == [1, 1, 1]) + #expect(server.parsedStatements == Self.inserts) + #expect(await connection.transactionState() == .inTransaction) + _ = try await connection.executeQuery("COMMIT") + #expect(server.queries.last == "COMMIT") + } + + @Test("A failure ends the run at that statement, and the server runs nothing after it") + func failureStopsTheRun() async throws { + let server = try Self.server(failing: Self.inserts[1]) + defer { server.close() } + server.start() + let connection = try await Self.connected(to: server) + defer { connection.disconnect() } + _ = try await connection.executeQuery("BEGIN") + + let outcomes = try #require(try await connection.executePipeline(Self.statements(Self.inserts))) + + #expect(outcomes.count == 2) + #expect(Self.rowCounts([outcomes[0]]) == [1]) + guard case .failed(let error) = outcomes[1] else { + Issue.record("The second statement should have failed") + return + } + #expect(error.localizedDescription.contains("duplicate key")) + #expect(server.parsedStatements == Array(Self.inserts.prefix(2))) + #expect(await connection.transactionState() == .inError) + _ = try await connection.executeQuery("ROLLBACK") + } + + @Test("Rows a statement returns are counted one at a time, and a failure after some of them fails it") + func rowsAreCountedAndDropped() async throws { + let read = "SELECT * FROM events" + let broken = "SELECT * FROM broken" + let columns = [LibPQScriptedServer.Column(name: "id", type: 23)] + let server = try LibPQScriptedServer( + status: { query, earlier in + (earlier + [query]).contains("BEGIN") ? LibPQScriptedServer.inTransaction : LibPQScriptedServer.idle + }, + answer: { query in + let rows: [[String?]] = (0..<500).map { [String($0)] } + if query == read { return .init(columns: columns, rows: rows) } + if query == broken { + return .init(columns: columns, rows: [["1"]], failureAfterRows: .init(sqlState: "22012", message: "division by zero")) + } + return nil + } + ) + defer { server.close() } + server.start() + let connection = try await Self.connected(to: server) + defer { connection.disconnect() } + _ = try await connection.executeQuery("BEGIN") + + let outcomes = try #require(try await connection.executePipeline(Self.statements([read, read, broken, read]))) + + #expect(outcomes.count == 3) + #expect(Self.rowCounts(Array(outcomes.prefix(2))) == [500, 500]) + guard case .failed(let error) = outcomes[2] else { + Issue.record("The read that failed after a row should fail") + return + } + #expect(error.localizedDescription.contains("division by zero")) + } + + @Test("A session lost after the last answer but before the Sync fails that last statement") + func lostBeforeSyncFailsTheLastStatement() async throws { + let server = try Self.server(closingAfter: Self.inserts[2]) + defer { server.close() } + server.start() + let connection = try await Self.connected(to: server) + defer { connection.disconnect() } + _ = try await connection.executeQuery("BEGIN") + + let outcomes = try #require(try await connection.executePipeline(Self.statements(Self.inserts))) + + #expect(outcomes.count == 3) + #expect(Self.rowCounts(Array(outcomes.prefix(2))) == [1, 1]) + guard case .failed = outcomes[2] else { + Issue.record("The statement whose Sync never came should fail") + return + } + #expect(connection.hasLostConnection) + } + + @Test("Outside a transaction block the pipeline is declined and nothing is sent") + func declinedOutsideTransaction() async throws { + let server = try Self.server() + defer { server.close() } + server.start() + let connection = try await Self.connected(to: server) + defer { connection.disconnect() } + let sentBefore = server.queries + + let outcomes = try await connection.executePipeline(Self.statements(Self.inserts)) + + #expect(outcomes == nil) + #expect(server.queries == sentBefore) + } + + @Test("Each statement binds its own parameters") + func bindsParameters() async throws { + let server = try Self.server() + defer { server.close() } + server.start() + let connection = try await Self.connected(to: server) + defer { connection.disconnect() } + _ = try await connection.executeQuery("BEGIN") + + let outcomes = try await connection.executePipeline([ + LibPQPipelineStatement(query: "INSERT INTO events VALUES ($1)", parameters: [.text("a")]), + LibPQPipelineStatement(query: "INSERT INTO events VALUES ($1, $2)", parameters: [.text("b"), .null]), + ]) + + #expect(Self.rowCounts(outcomes) == [1, 1]) + #expect(server.boundParameters.suffix(2) == [["a"], ["b", nil]]) + } + + @Test("The driver sends a statement a pipeline does not admit alone, and the stretches around it as pipelines") + func driverSplitsAroundRefusedStatements() async throws { + let server = try Self.server() + defer { server.close() } + server.start() + let driver = try await Self.driver(on: server) + defer { driver.disconnect() } + _ = try await driver.execute(query: "BEGIN") + let queries = [Self.inserts[0], "SET search_path = app", Self.inserts[1], Self.inserts[2]] + + let outcomes = try await driver.executeStatements(queries.map { + PluginBatchStatement(query: $0, parameters: nil, context: PluginStatementContext()) + }) + + #expect(outcomes?.count == 4) + #expect(server.parsedStatements.suffix(2) == Array(Self.inserts.suffix(2))) + #expect(!server.parsedStatements.contains("SET search_path = app")) + #expect(server.queries.contains("SET search_path = app")) + } + + @Test("The driver declines a batch outside a transaction block or one holding a read-only statement") + func driverDeclines() async throws { + let server = try Self.server() + defer { server.close() } + server.start() + let driver = try await Self.driver(on: server) + defer { driver.disconnect() } + let plain = Self.inserts.map { PluginBatchStatement(query: $0, parameters: nil, context: PluginStatementContext()) } + + #expect(try await driver.executeStatements(plain) == nil) + + _ = try await driver.execute(query: "BEGIN") + var readOnly = PluginStatementContext() + readOnly.enforcesReadOnly = true + let withRead = plain + [PluginBatchStatement(query: "SELECT 1", parameters: nil, context: readOnly)] + #expect(try await driver.executeStatements(withRead) == nil) + } + + @Test("Only PostgreSQL itself declares statement batches") + func capabilityIsPostgreSQLOnly() { + let config = DriverConnectionConfig(host: "h", port: 5_432, username: "u", password: "", database: "d") + #expect(PostgreSQLPluginDriver(config: config).capabilities.contains(.statementBatches)) + #expect(!PGlitePluginDriver(config: config).capabilities.contains(.statementBatches)) + } + + private static let conforming = LibPQPipelineAdmission( + session: PluginSessionLexicalState(determined: .backslashEscapesInSingleQuotes, enabled: []) + ) + private static let escaping = LibPQPipelineAdmission( + session: PluginSessionLexicalState( + determined: .backslashEscapesInSingleQuotes, enabled: .backslashEscapesInSingleQuotes + ) + ) + + @Test("Transaction control, COPY and session settings are never pipelined, past any comment", arguments: [ + "BEGIN", "commit", "ROLLBACK TO SAVEPOINT a", "SAVEPOINT a", "COPY t FROM STDIN", "SET search_path = a", + "VACUUM t", "CALL p()", "DO $$ BEGIN END $$", "-- note\rCOMMIT\nWORK", "/* a /* nested */ b */ SET x = 1", + ]) + func refusesSessionStatements(text: String) { + #expect(!Self.conforming.admits(text)) + } + + @Test("A text holding a second statement goes alone, as a lone simple Query runs it whole", arguments: [ + "INSERT INTO t VALUES (1); INSERT INTO t VALUES (2)", "INSERT INTO t VALUES ('a'); -- x\nDELETE FROM t", + "UPDATE t SET a = 1 /* ; */; SELECT 1", + ]) + func refusesSeveralStatements(text: String) { + #expect(!Self.conforming.admits(text)) + } + + @Test("A `;` inside a literal, a quoted name, a dollar quote or a comment leaves one statement", arguments: [ + "INSERT INTO t VALUES ('a;b')", #"INSERT INTO "a;b" VALUES (1)"#, "INSERT INTO t VALUES ($$a;b$$)", + "INSERT INTO t VALUES ($q$a;b$q$)", "DELETE FROM t; -- done", "UPDATE t SET a = 1 /* x; y */", + "INSERT INTO t VALUES (1) -- a; b\r", + ]) + func admitsSemicolonsInsideTokens(text: String) { + #expect(Self.conforming.admits(text)) + } + + @Test("Where a literal ends follows the session's standard_conforming_strings") + func literalsFollowTheSession() { + let text = #"INSERT INTO t VALUES ('a\'); DELETE FROM t; --')"# + #expect(!Self.conforming.admits(text)) + #expect(Self.escaping.admits(text)) + #expect(Self.conforming.admits(#"INSERT INTO t VALUES (E'a\'; b')"#)) + } + + @Test("Ordinary reads and writes are pipelined", arguments: [ + "INSERT INTO t VALUES (1);", "DELETE FROM t ; \n", + "INSERT INTO t VALUES (1)", "UPDATE t SET a = 1", "DELETE FROM t", "CREATE TABLE t (a int)", + "SELECT pg_catalog.setval('s', 1, true)", "WITH x AS (SELECT 1) INSERT INTO t SELECT * FROM x", + "-- note\r\nINSERT INTO t VALUES (1)", + ]) + func admitsOrdinaryStatements(text: String) { + #expect(Self.conforming.admits(text)) + } + + @Test("A tally whose every statement answered but whose exchange broke fails the last statement") + func tallyFailsLastWhenSyncIsLost() { + var tally = LibPQPipelineTally(total: 2, startedAt: Date()) + tally.sent = 2 + for _ in 0..<2 { + tally.take(.completed(rowsAffected: 1, commandTag: "INSERT 0 1")) + _ = tally.endStatement() + } + struct Lost: Error {} + + let outcomes = tally.outcomes(cutShort: Lost()) + + #expect(outcomes.count == 2) + #expect(Self.rowCounts([outcomes[0]]) == [1]) + guard case .failed(let error) = outcomes[1] else { + Issue.record("The last statement should carry the loss") + return + } + #expect(error is Lost) + } + + @Test("A tally stops at the first failure and names a cut-short statement only when one was never answered") + func tallyOutcomes() { + var tally = LibPQPipelineTally(total: 3, startedAt: Date()) + tally.sent = 3 + tally.take(.completed(rowsAffected: 2, commandTag: "UPDATE 2")) + let firstEnded = tally.endStatement() + tally.take(.aborted) + let secondEnded = tally.endStatement() + let strayEnded = tally.endStatement() + struct Lost: Error {} + + let outcomes = tally.outcomes(cutShort: Lost()) + + #expect(outcomes.count == 2) + #expect(Self.rowCounts([outcomes[0]]) == [2]) + guard case .failed(let error) = outcomes[1] else { + Issue.record("The aborted statement should report the cut") + return + } + #expect(error is Lost) + #expect(firstEnded && secondEnded) + #expect(!strayEnded) + } +} diff --git a/apple/Plugins/PostgreSQLDriverTests/LibPQScriptedServer.swift b/apple/Plugins/PostgreSQLDriverTests/LibPQScriptedServer.swift index d7efab3d16..c017a12bcf 100644 --- a/apple/Plugins/PostgreSQLDriverTests/LibPQScriptedServer.swift +++ b/apple/Plugins/PostgreSQLDriverTests/LibPQScriptedServer.swift @@ -39,6 +39,10 @@ final class LibPQScriptedServer: @unchecked Sendable { // Sent after the rows in place of the completion, as a statement that fails part way through its result. var failureAfterRows: Failure? var copyIn: CopyIn? + // The command tag in place of `SELECT `, such as `INSERT 0 1`. + var completion: String? + // Closes the session at the next Sync instead of answering it, so every reply but the ReadyForQuery arrives. + var closesSessionAtSync = false } struct CopyIn: Sendable { @@ -80,6 +84,7 @@ final class LibPQScriptedServer: @unchecked Sendable { private let failingStatement: String? private let lock = NSLock() private var received: [String] = [] + private var parsed: [String] = [] private var bindings: [[String?]] = [] private var cancels = 0 private var isClosed = false @@ -90,6 +95,12 @@ final class LibPQScriptedServer: @unchecked Sendable { lock.withLock { received } } + // The statements the extended protocol's Parse brought in, in order, leaving out the ones a failure passed over + // until the next Sync, as PostgreSQL does. + var parsedStatements: [String] { + lock.withLock { parsed } + } + // Each Bind's parameters as text, nil for a null. var boundParameters: [[String?]] { lock.withLock { bindings } @@ -166,6 +177,7 @@ final class LibPQScriptedServer: @unchecked Sendable { var statement: Statement? var failed = false + var closesAtSync = false while let type = receive(client, 1)?.first, let body = receiveBody(client) { var reply: [UInt8] = [] switch type { @@ -191,6 +203,7 @@ final class LibPQScriptedServer: @unchecked Sendable { case UInt8(ascii: "P"): let query = Self.cString(body, from: Self.cString(body, from: 0).next).text let earlier = record(query) + if !failed { lock.withLock { parsed.append(query) } } onQuery(query) statement = Statement(query: query, earlier: earlier, answer: answer(query) ?? Answer()) if !failed { reply = Self.message("1", []) } @@ -204,7 +217,9 @@ final class LibPQScriptedServer: @unchecked Sendable { case UInt8(ascii: "E"): guard !failed, let statement else { break } guard execute(statement.answer, on: client, failed: &failed) else { return } + closesAtSync = closesAtSync || statement.answer.closesSessionAtSync case UInt8(ascii: "S"): + if closesAtSync { return } var state = statement.map { status($0.query, $0.earlier) } ?? Self.idle if failed, state == Self.inTransaction { state = Self.failedTransaction } reply = Self.message("Z", [state]) @@ -245,7 +260,8 @@ final class LibPQScriptedServer: @unchecked Sendable { sqlState: "57014", message: "canceling statement due to user request" ))) } - return send(client, Self.message("C", Array("SELECT \(answer.rows.count)".utf8) + [0])) + let completion = answer.completion ?? "SELECT \(answer.rows.count)" + return send(client, Self.message("C", Array(completion.utf8) + [0])) } // CopyInResponse with every column in the copy's format, then CopyData until CopyDone or CopyFail. diff --git a/apple/Plugins/SQLImportPlugin/SQLImportBatch.swift b/apple/Plugins/SQLImportPlugin/SQLImportBatch.swift new file mode 100644 index 0000000000..4490e81091 --- /dev/null +++ b/apple/Plugins/SQLImportPlugin/SQLImportBatch.swift @@ -0,0 +1,37 @@ +// +// SQLImportBatch.swift +// SQLImportPlugin +// + +import Foundation +import TableProPluginKit + +// Bounded so Stop and Pause still land between exchanges, and a file of large INSERTs is not held whole. +nonisolated struct SQLImportBatch { + static let statementLimit = 200 + static let byteLimit = 4 << 20 + + private var statements: [PluginImportStatement] = [] + private var bytes = 0 + + var isEmpty: Bool { + statements.isEmpty + } + + var isFull: Bool { + statements.count >= Self.statementLimit || bytes >= Self.byteLimit + } + + mutating func append(_ statement: PluginImportStatement) { + statements.append(statement) + bytes += statement.text.utf8.count + } + + mutating func take() -> [PluginImportStatement] { + defer { + statements = [] + bytes = 0 + } + return statements + } +} diff --git a/apple/Plugins/SQLImportPlugin/SQLImportPlugin.swift b/apple/Plugins/SQLImportPlugin/SQLImportPlugin.swift index a5b7266c1a..2dbaa098a5 100644 --- a/apple/Plugins/SQLImportPlugin/SQLImportPlugin.swift +++ b/apple/Plugins/SQLImportPlugin/SQLImportPlugin.swift @@ -67,67 +67,122 @@ nonisolated final class SQLImportPlugin: ObservableObject, ImportFormatPlugin, S let stream = try await source.elements() - for try await element in stream { + // What one failed statement does to the run, the same whether it ran alone or in a batch. + func handleFailure(_ error: any Error, statement: String, lineNumber: Int) async throws { + if error is PluginImportCancellationError { throw PluginImportCancellationError() } + if error is CancellationError { throw CancellationError() } + switch errorMode { + case .stopAndRollback: + throw PluginImportError.statementFailed( + statement: statement, + line: lineNumber, + underlyingError: error + ) + + case .stopAndCommit: + let statementError = error + if useTransaction { + do { + try await sink.commitTransaction() + didCommit = true + } catch { + Self.logger.warning("Failed to commit partial import") + } + } + if options.disableForeignKeyChecks { + do { + try await sink.enableForeignKeyChecks() + } catch { + Self.logger.warning("Failed to re-enable foreign key checks") + } + } + throw PluginImportError.statementFailed( + statement: statement, + line: lineNumber, + underlyingError: statementError + ) + + case .skipAndContinue: + skippedCount += 1 + if errors.count < maxErrors { + let snippet = (statement as NSString).length > 200 + ? String(statement.prefix(200)) + "..." + : statement + errors.append(.init( + statement: snippet, + line: lineNumber, + errorMessage: error.localizedDescription + )) + } + progress.incrementStatement() + } + } + + // Only inside the import's transaction, never in Skip mode: a stop rolls back or commits what ran before the + // failure, and the server runs nothing after it. + var pending = SQLImportBatch() + + // `next` is the element the file was read for while the batch waited, nil when reading it failed. + func flush(readAhead: Bool, next: PluginImportStatement? = nil) async throws { + guard !pending.isEmpty else { return } + try progress.checkCancellation() + let statements = pending.take() + do { + try await sink.execute(statements: statements, readAhead: readAhead) + executedCount += statements.count + for _ in statements { progress.incrementStatement() } + // A Stop during the batch could not reach the statements in it; the rollback still can. + try progress.checkCancellation() + } catch let failure as PluginImportBatchFailure { + let ran = min(max(failure.index, 0), statements.count) + executedCount += ran + for _ in 0.. 200 - ? String(statement.prefix(200)) + "..." - : statement - errors.append(.init( - statement: snippet, - line: lineNumber, - errorMessage: error.localizedDescription - )) - } - progress.incrementStatement() - } + try await handleFailure(error, statement: statement, lineNumber: lineNumber) } } + try await flush(readAhead: false) if useTransaction { try await sink.commitTransaction() diff --git a/apple/Plugins/TableProPluginKit/PluginBatchStatement.swift b/apple/Plugins/TableProPluginKit/PluginBatchStatement.swift new file mode 100644 index 0000000000..f42df9621c --- /dev/null +++ b/apple/Plugins/TableProPluginKit/PluginBatchStatement.swift @@ -0,0 +1,20 @@ +import Foundation + +public struct PluginBatchStatement: Sendable { + public let query: String + // Nil sends the text as `execute(query:context:)` does, an array binds it as `executeParameterized` does. + public let parameters: [PluginCellValue]? + public let context: PluginStatementContext + + public init(query: String, parameters: [PluginCellValue]?, context: PluginStatementContext) { + self.query = query + self.parameters = parameters + self.context = context + } +} + +// A statement's rows are read and dropped: a caller that needs them runs the statement on its own. +public enum PluginBatchStatementOutcome: Sendable { + case succeeded(rowsAffected: Int, executionTime: TimeInterval) + case failed(any Error) +} diff --git a/apple/Plugins/TableProPluginKit/PluginCapabilities.swift b/apple/Plugins/TableProPluginKit/PluginCapabilities.swift index 0eb929ecbc..4a717fb05b 100644 --- a/apple/Plugins/TableProPluginKit/PluginCapabilities.swift +++ b/apple/Plugins/TableProPluginKit/PluginCapabilities.swift @@ -50,4 +50,6 @@ public struct PluginCapabilities: OptionSet, Sendable { // The driver runs a `COPY ... FROM STDIN` and sends its rows through // `PluginDatabaseDriver.copyFromStandardInput(statement:rows:context:)`. public static let copyFromStandardInput = PluginCapabilities(rawValue: 1 << 25) + // `PluginDatabaseDriver.executeStatements(_:)` can answer, though it may still decline a given batch. + public static let statementBatches = PluginCapabilities(rawValue: 1 << 26) } diff --git a/apple/Plugins/TableProPluginKit/PluginDatabaseDriver.swift b/apple/Plugins/TableProPluginKit/PluginDatabaseDriver.swift index 1bba4542eb..fd5a72617d 100644 --- a/apple/Plugins/TableProPluginKit/PluginDatabaseDriver.swift +++ b/apple/Plugins/TableProPluginKit/PluginDatabaseDriver.swift @@ -145,6 +145,10 @@ public protocol PluginDatabaseDriver: AnyObject, Sendable { /// statements after it may still have run and returned results. func executeBatch(query: String, rowCap: Int?, parameters: [PluginCellValue]?) async throws -> PluginBatchResult? + // In order, in one exchange, stopping at the first failure: nothing after it reaches the server. One outcome per + // statement that ran, a failure last. Nil when this batch cannot be held to that, and the host sends each alone. + func executeStatements(_ statements: [PluginBatchStatement]) async throws -> [PluginBatchStatementOutcome]? + /// Runs `statement`, a `COPY ... FROM STDIN`, and sends it `rows` a piece at a time through the engine's copy /// protocol, holding the session until the copy ends. A driver declares `PluginCapabilities.copyFromStandardInput` /// for it; the default throws `PluginDriverUnsupportedOperation.copyFromStandardInput`. @@ -1634,6 +1638,10 @@ public extension PluginDatabaseDriver { nil } + func executeStatements(_ statements: [PluginBatchStatement]) async throws -> [PluginBatchStatementOutcome]? { + nil + } + func copyFromStandardInput( statement: String, rows: PluginCopyData, diff --git a/apple/Plugins/TableProPluginKit/PluginImportDataSink.swift b/apple/Plugins/TableProPluginKit/PluginImportDataSink.swift index 440057949f..f18c87c2e9 100644 --- a/apple/Plugins/TableProPluginKit/PluginImportDataSink.swift +++ b/apple/Plugins/TableProPluginKit/PluginImportDataSink.swift @@ -18,6 +18,15 @@ public protocol PluginImportDataSink: AnyObject, Sendable { /// Runs a `COPY ... FROM STDIN` read from the file's `line` and sends it `rows`. func copyFromStandardInput(statement: String, line: Int, rows: PluginCopyData) async throws + // True only when the connection can send several statements in one exchange and `statement` may wait for the + // ones after it: its text reads the same whatever the statements before it change, so the file can be read past it. + func batches(_ statement: String) -> Bool + + // Runs each as `execute(statement:line:)` does, in order. The first that fails throws `PluginImportBatchFailure` + // naming it, and none after it runs. `readAhead` says the file was read past them before they ran: when they + // changed how the session reads text, it throws one whose index is `statements.count`. + func execute(statements: [PluginImportStatement], readAhead: Bool) async throws + func insertRow(_ values: [String: PluginCellValue]) async throws func insertRows(_ rows: [[String: PluginCellValue]]) async throws func deleteAllRowsFromTargetTable() async throws @@ -38,6 +47,16 @@ public extension PluginImportDataSink { String(localized: "This connection cannot load the rows of COPY FROM STDIN.") ) } + func batches(_ statement: String) -> Bool { false } + func execute(statements: [PluginImportStatement], readAhead: Bool) async throws { + for (index, statement) in statements.enumerated() { + do { + try await execute(statement: statement.text, line: statement.line) + } catch { + throw PluginImportBatchFailure(index: index, underlying: error) + } + } + } func insertRow(_ values: [String: PluginCellValue]) async throws { throw PluginImportError.importFailed("Row-based import is not supported by this connection") } @@ -52,3 +71,24 @@ public extension PluginImportDataSink { func disableForeignKeyChecks() async throws {} func enableForeignKeyChecks() async throws {} } + +public struct PluginImportStatement: Sendable { + public let text: String + public let line: Int + + public init(text: String, line: Int) { + self.text = text + self.line = line + } +} + +public struct PluginImportBatchFailure: Error { + // `statements.count` when every statement ran and the file cannot be read on past them. + public let index: Int + public let underlying: any Error + + public init(index: Int, underlying: any Error) { + self.index = index + self.underlying = underlying + } +} diff --git a/apple/TablePro/Core/Database/DatabaseDriver.swift b/apple/TablePro/Core/Database/DatabaseDriver.swift index 2855ef8141..d3b4e0c65c 100644 --- a/apple/TablePro/Core/Database/DatabaseDriver.swift +++ b/apple/TablePro/Core/Database/DatabaseDriver.swift @@ -144,6 +144,11 @@ protocol DatabaseDriver: AnyObject, Sendable { /// text statement by statement instead. A server error inside the batch is part of the answer, not a throw. func executeBatch(query: String, rowCap: Int?, parameters: [Any?]?) async throws -> QueryBatchResult? + var supportsStatementBatches: Bool { get } + + // In order, stopping at the first failure. Nil when the driver sends these one at a time. + func executeStatements(_ statements: [BatchedStatement]) async throws -> [BatchedStatementOutcome]? + /// Whether ``copyFromStandardInput(statement:rows:context:)`` sends a `COPY ... FROM STDIN` its rows. var supportsCopyFromStandardInput: Bool { get } @@ -648,6 +653,10 @@ extension DatabaseDriver { func executeBatch(query: String, rowCap: Int?, parameters: [Any?]?) async throws -> QueryBatchResult? { nil } + var supportsStatementBatches: Bool { false } + + func executeStatements(_ statements: [BatchedStatement]) async throws -> [BatchedStatementOutcome]? { nil } + var supportsCopyFromStandardInput: Bool { false } func copyFromStandardInput( diff --git a/apple/TablePro/Core/Plugins/ImportDataSinkAdapter.swift b/apple/TablePro/Core/Plugins/ImportDataSinkAdapter.swift index 4ce287d2dc..98e7765d28 100644 --- a/apple/TablePro/Core/Plugins/ImportDataSinkAdapter.swift +++ b/apple/TablePro/Core/Plugins/ImportDataSinkAdapter.swift @@ -260,6 +260,82 @@ final class ImportDataSinkAdapter: PluginImportDataSink, @unchecked Sendable { } } + // A write or definition alone: no transaction boundary, no setting, one statement as this session splits it. + func batches(_ statement: String) -> Bool { + guard driver.supportsStatementBatches, Self.readsAlike(statement, session: driver.sessionLexicalState) else { + return false + } + let grammar = SQLLexicalResolver.readings(for: databaseType, session: driver.sessionLexicalState).execution + guard !grammar.contains(.batchSeparatorLines), + Self.batchedKeywords.contains(QueryClassifier.leadingKeyword(of: statement, grammar: grammar)), + !CatalogChangeClassifier.effect(of: statement, databaseType: databaseType, grammar: grammar).endsTransaction, + statement.range(of: "set_config", options: .caseInsensitive) == nil + else { return false } + return SQLStatementScanner.executableStatements(in: statement, grammar: grammar).count == 1 + } + + // A read is left out because a call in one is how a dump changes a setting. + private static let batchedKeywords: Set = [ + "INSERT", "UPDATE", "DELETE", "CREATE", "ALTER", "DROP", "COMMENT", "GRANT", "REVOKE", "TRUNCATE", + ] + + private static let backslashEscapes: SQLLexicalFeatures = [ + .backslashEscapesInSingleQuotes, .backslashEscapesInDoubleQuotes, .backslashEscapesInBackticks, + ] + + // The file is split ahead of what runs, so a waiting statement must split the same under every reading the session + // can switch to. The only switches a session reports are backslash escapes, and text without a backslash reads + // alike under each. + private static func readsAlike(_ statement: String, session: PluginSessionLexicalState?) -> Bool { + guard let determined = session?.determined, !determined.isEmpty else { return true } + return backslashEscapes.isSuperset(of: determined) && !statement.utf8.contains(UInt8(ascii: "\\")) + } + + func execute(statements: [PluginImportStatement], readAhead: Bool) async throws { + let readingRules = driver.sessionLexicalState + try await run(statements) + // The element the file was read for was split under the rules in force before these ran. + guard !readAhead || driver.sessionLexicalState == readingRules else { + throw PluginImportBatchFailure(index: statements.count, underlying: ImportReadingRulesChanged()) + } + } + + private func run(_ statements: [PluginImportStatement]) async throws { + let grammar = SQLLexicalResolver.readings(for: databaseType, session: driver.sessionLexicalState).execution + let texts = statements.map(\.text) + let driver = self.driver + let send: () async throws -> [BatchedStatementOutcome]? = { + try await driver.executeStatements(texts.map { BatchedStatement(sql: $0) }) + } + let answered = if let importControl { + try await importControl.statements(texts, operation: send) + } else { + try await send() + } + // Declined, as outside a transaction block: each goes alone, through its own admission. + guard let outcomes = answered else { + for (index, statement) in statements.enumerated() { + do { + try await execute(statement: statement.text, line: statement.line) + } catch { + throw PluginImportBatchFailure(index: index, underlying: error) + } + } + return + } + for (index, (statement, outcome)) in zip(statements, outcomes).enumerated() { + noteNameHazards(in: statement.text, grammar: grammar) + if case .failed(let error) = outcome { throw PluginImportBatchFailure(index: index, underlying: error) } + await noteTransactionEnd(after: statement.text, grammar: grammar) + } + guard outcomes.count >= statements.count else { + throw PluginImportBatchFailure( + index: outcomes.count, + underlying: DatabaseError.queryFailed(String(localized: "The driver did not report every statement it was sent.")) + ) + } + } + // The driver's Stop ends a copy at once (a CopyFail, and a cancel request when the server takes no data). func copyFromStandardInput(statement: String, line: Int, rows: PluginCopyData) async throws { guard driver.supportsCopyFromStandardInput else { throw SQLFileParseError.copyFromStandardInput(line: line) } @@ -494,3 +570,12 @@ final class ImportDataSinkAdapter: PluginImportDataSink, @unchecked Sendable { return try await operation() } } + +internal struct ImportReadingRulesChanged: LocalizedError { + var errorDescription: String? { + String(localized: """ + Statements sent together changed how the database reads text after the import had read past them, so the \ + import stopped there. Import the file again with Wrap in transaction turned off. + """) + } +} diff --git a/apple/TablePro/Core/Plugins/ImportSinkRun.swift b/apple/TablePro/Core/Plugins/ImportSinkRun.swift index d0f50b8eaa..f6caf00617 100644 --- a/apple/TablePro/Core/Plugins/ImportSinkRun.swift +++ b/apple/TablePro/Core/Plugins/ImportSinkRun.swift @@ -139,6 +139,55 @@ internal final class ImportSinkRun: @unchecked Sendable { } } + // One admission for statements sent together; each that ran is recorded as `data` records one. Nil when the driver + // declined them, and nothing was sent. Not interruptible: a Pause waits for the batch, as it waits for a statement. + internal func statements( + _ texts: [String], + operation: () async throws -> [BatchedStatementOutcome]? + ) async throws -> [BatchedStatementOutcome]? { + try await admitted(.data) { + if self.commitMode == .manual { try self.refuseWriteOutsideTransaction() } + let before = self.run.snapshot.currentState + let hadAuthority = self.lock.withLock { self.ownsTransaction } + let answered: [BatchedStatementOutcome]? + do { + answered = try await operation() + } catch { + // A driver that throws may have sent some of them, as `data` assumes of a write that throws. + self.run.record { + $0.recordError(error) + if let index = self.fileIndex { + $0.effects[index]?.admittedWrite = true + $0.effects[index]?.failedWrite = true + } + } + await self.observe() + throw error + } + guard let outcomes = answered else { return nil } + if let index = self.fileIndex { + self.run.record { $0.effects[index]?.admittedWrite = true } + } + for (text, outcome) in zip(texts, outcomes) { + self.noteStatement(text) + switch outcome { + case .succeeded: + if let index = self.fileIndex { self.run.record { $0.effects[index]?.statements += 1 } } + case .failed(let error): + self.run.record { + $0.recordError(error) + if let index = self.fileIndex { $0.effects[index]?.failedWrite = true } + } + } + } + await self.observe() + for (text, outcome) in zip(texts, outcomes) where outcome.succeeded { + self.noteCompletedStatement(text, before: before, wasOwned: hadAuthority) + } + return outcomes + } + } + // Decided at the first write, before a GO line resets the holder: outside a transaction each statement commits. private func refuseWriteOutsideTransaction() throws { let holder = run.snapshot.holder diff --git a/apple/TablePro/Core/Plugins/PluginDriverAdapter.swift b/apple/TablePro/Core/Plugins/PluginDriverAdapter.swift index 0aab414335..6ff19d9058 100644 --- a/apple/TablePro/Core/Plugins/PluginDriverAdapter.swift +++ b/apple/TablePro/Core/Plugins/PluginDriverAdapter.swift @@ -487,6 +487,54 @@ final class PluginDriverAdapter: DatabaseDriver, SchemaSwitchable, DatabaseRepor ) } + var supportsStatementBatches: Bool { + pluginDriver.capabilities.contains(.statementBatches) + } + + // Each statement is checked and described as its own call would, and one that would be refused ends the batch there. + func executeStatements(_ statements: [BatchedStatement]) async throws -> [BatchedStatementOutcome]? { + var items: [PluginBatchStatement] = [] + var refusal: (any Error)? + for statement in statements { + do { + try StatementTextValidator.validate(statement.sql) + try refuseUnenforcedPolicy(statement.context ?? PluginStatementContext()) + } catch { + refusal = error + break + } + items.append(pluginStatement(for: statement)) + } + guard !items.isEmpty else { return refusal.map { [.failed($0)] } } + guard let answered = try await pluginDriver.executeStatements(items) else { return nil } + var outcomes: [BatchedStatementOutcome] = [] + for answer in answered.prefix(items.count) { + outcomes.append(BatchedStatementOutcome(answer)) + guard outcomes.last?.succeeded == true else { return outcomes } + } + guard outcomes.count == items.count else { + return outcomes + [.failed(DatabaseError.queryFailed( + String(localized: "The driver did not report every statement it was sent.") + ))] + } + return outcomes + (refusal.map { [.failed($0)] } ?? []) + } + + private func pluginStatement(for statement: BatchedStatement) -> PluginBatchStatement { + guard let parameters = statement.parameters else { + let context = statement.context.map { statementContext(statement.sql, approvedAs: $0) } + ?? statementContext(statement.sql) + return PluginBatchStatement(query: statement.sql, parameters: nil, context: context) + } + return PluginBatchStatement( + query: statement.sql, + parameters: parameters.map(Self.cellValue(for:)), + context: statementContext( + statement.sql, approvedAs: statement.context ?? PluginStatementContext(), parameters: parameters + ) + ) + } + func explainPlan(statement: String, variantId: String) async throws -> QueryResult { try StatementTextValidator.validate(statement) return mapQueryResult(try await pluginDriver.explainPlan(statement: statement, variantId: variantId)) diff --git a/apple/TablePro/Core/Services/Execution/DatabaseDriver+StatementBatches.swift b/apple/TablePro/Core/Services/Execution/DatabaseDriver+StatementBatches.swift new file mode 100644 index 0000000000..eed661eb0f --- /dev/null +++ b/apple/TablePro/Core/Services/Execution/DatabaseDriver+StatementBatches.swift @@ -0,0 +1,44 @@ +// +// DatabaseDriver+StatementBatches.swift +// TablePro +// + +import Foundation +import TableProPluginKit + +struct BatchedStatement: @unchecked Sendable { + let sql: String + // Nil runs the text as `execute(query:)` does, an array binds it as `executeParameterized` does. + let parameters: [Any?]? + // Nil runs it as the call without a context does. + let context: PluginStatementContext? + + init(sql: String, parameters: [Any?]? = nil, context: PluginStatementContext? = nil) { + self.sql = sql + self.parameters = parameters + self.context = context + } +} + +enum BatchedStatementOutcome: Sendable { + case succeeded(rowsAffected: Int, executionTime: TimeInterval) + case failed(any Error) + + init(_ outcome: PluginBatchStatementOutcome) { + switch outcome { + case .succeeded(let rowsAffected, let executionTime): + self = .succeeded(rowsAffected: rowsAffected, executionTime: executionTime) + case .failed(let error): + self = .failed(error) + @unknown default: + self = .failed(DatabaseError.queryFailed( + String(localized: "The driver answered a statement in a way this version of TablePro cannot read.") + )) + } + } + + var succeeded: Bool { + if case .succeeded = self { return true } + return false + } +} diff --git a/apple/TablePro/Resources/Localizable.xcstrings b/apple/TablePro/Resources/Localizable.xcstrings index 4a14daeb6f..f3ca32d926 100644 --- a/apple/TablePro/Resources/Localizable.xcstrings +++ b/apple/TablePro/Resources/Localizable.xcstrings @@ -178860,6 +178860,9 @@ } } } + }, + "Statements sent together changed how the database reads text after the import had read past them, so the import stopped there. Import the file again with Wrap in transaction turned off." : { + }, "Statements TablePro generated and ran" : { "localizations" : { @@ -192709,6 +192712,9 @@ } } } + }, + "The driver answered a statement in a way this version of TablePro cannot read." : { + }, "The driver cannot delete %lld rows marked for deletion." : { "localizations" : { @@ -192769,6 +192775,9 @@ } } } + }, + "The driver did not report every statement it was sent." : { + }, "The driver returned part of the listing." : { "localizations" : { diff --git a/apple/TableProTests/Core/Plugins/ImportDataSinkAdapterBatchTests.swift b/apple/TableProTests/Core/Plugins/ImportDataSinkAdapterBatchTests.swift new file mode 100644 index 0000000000..b259c4102b --- /dev/null +++ b/apple/TableProTests/Core/Plugins/ImportDataSinkAdapterBatchTests.swift @@ -0,0 +1,212 @@ +// +// ImportDataSinkAdapterBatchTests.swift +// TableProTests +// + +import Foundation +@testable import TablePro +import TableProPluginKit +import Testing + +private final class PipeliningDriver: PluginDriverStub, @unchecked Sendable { + struct Refused: Error {} + + static let backslashesEscape = PluginSessionLexicalState( + determined: .backslashEscapesInSingleQuotes, enabled: .backslashEscapesInSingleQuotes + ) + static let backslashesAreText = PluginSessionLexicalState(determined: .backslashEscapesInSingleQuotes, enabled: []) + + let declared: PluginCapabilities + let failing: Set + var declines = false + var lexical: PluginSessionLexicalState? = PipeliningDriver.backslashesAreText + // Where a statement leaves the session's reading rules, as a function that sets them would. + var rulesAfter: [String: PluginSessionLexicalState] = [:] + private(set) var events: [String] = [] + private var inTransaction = false + + init(declared: PluginCapabilities = [.statementBatches], failing: Set = []) { + self.declared = declared + self.failing = failing + } + + var capabilities: PluginCapabilities { declared } + var supportsTransactions: Bool { true } + var supportsTransactionalDDL: Bool { true } + var sessionLexicalState: PluginSessionLexicalState? { lexical } + + func execute(query: String) async throws -> PluginQueryResult { + events.append(query) + if failing.contains(query) { throw Refused() } + if let rules = rulesAfter[query] { lexical = rules } + return .empty + } + + func executeStatements(_ statements: [PluginBatchStatement]) async throws -> [PluginBatchStatementOutcome]? { + guard !declines else { return nil } + events.append("batch \(statements.count)") + var outcomes: [PluginBatchStatementOutcome] = [] + for statement in statements { + if failing.contains(statement.query) { return outcomes + [.failed(Refused())] } + if let rules = rulesAfter[statement.query] { lexical = rules } + outcomes.append(.succeeded(rowsAffected: 1, executionTime: 0)) + } + return outcomes + } + + func sessionTransactionState() async -> PluginSessionTransactionState { inTransaction ? .inTransaction : .idle } + func beginTransaction(mode: PluginTransactionAccessMode) async throws { + events.append("BEGIN") + inTransaction = true + } + func rollbackTransaction() async throws { + events.append("ROLLBACK") + inTransaction = false + } +} + +struct ImportDataSinkAdapterBatchTests { + private func sink(on driver: PipeliningDriver, importRun: ImportRun? = nil) -> ImportDataSinkAdapter { + let adapter = PluginDriverAdapter(connection: TestFixtures.makeConnection(type: .postgresql), pluginDriver: driver) + return ImportDataSinkAdapter(driver: adapter, databaseType: .postgresql, importRun: importRun) + } + + private static func statements(_ count: Int) -> [PluginImportStatement] { + (1...count).map { PluginImportStatement(text: "INSERT INTO t VALUES (\($0))", line: $0) } + } + + @Test("Writes and definitions may wait for a batch", arguments: [ + "INSERT INTO t VALUES (1)", "update t set a = 1", "DELETE FROM t", "CREATE TABLE t (a int)", + "ALTER TABLE t ADD b int", "COMMENT ON TABLE t IS 'x'", "INSERT INTO t VALUES ($$a;b$$)", + ]) + func batchesWrites(statement: String) { + #expect(sink(on: PipeliningDriver()).batches(statement)) + } + + @Test("A transaction boundary, a setting, a read, a call that may set one, or a text of two statements runs alone", arguments: [ + "BEGIN", "COMMIT", "ROLLBACK", "SAVEPOINT a", "SET standard_conforming_strings = off", + "SELECT pg_catalog.set_config('search_path', '', false)", "INSERT INTO t VALUES (1); INSERT INTO t VALUES (2)", + "COPY t FROM STDIN", "SELECT pg_catalog.setval('s', 1, true)", + "INSERT INTO t VALUES (set_config('standard_conforming_strings', 'off', false))", + ]) + func keepsSessionStatementsAlone(statement: String) { + #expect(!sink(on: PipeliningDriver()).batches(statement)) + } + + @Test("Text holding a backslash runs alone, since the session's rules decide where it ends", arguments: [ + #"INSERT INTO t VALUES ('a\'b')"#, #"INSERT INTO t VALUES (E'\n')"#, #"COMMENT ON TABLE t IS 'C:\dir'"#, + #"INSERT INTO t SELECT pg_catalog.U&"set\005fconfig"('standard_conforming_strings', 'off', false)"#, + ]) + func keepsBackslashesAlone(statement: String) { + #expect(!sink(on: PipeliningDriver()).batches(statement)) + } + + @Test("A connection that cannot batch never holds a statement back") + func driverWithoutBatchesRunsAlone() { + #expect(!sink(on: PipeliningDriver(declared: [])).batches("INSERT INTO t VALUES (1)")) + } + + @Test("A session that decides more than backslash escapes never batches, since other text could split differently") + func sessionDecidingOtherRulesRunsAlone() { + let driver = PipeliningDriver() + driver.lexical = PluginSessionLexicalState( + determined: [.backslashEscapesInSingleQuotes, .hashLineComments], enabled: [] + ) + + #expect(!sink(on: driver).batches("INSERT INTO t VALUES (1)")) + } + + @Test("A batch the driver declines runs each statement alone, through the import's own admission") + func declinedBatchRunsAlone() async throws { + let driver = PipeliningDriver() + driver.declines = true + let sink = sink(on: driver, importRun: ImportRun()) + try await sink.prepareImportRun() + let statements = Self.statements(2) + + try await sink.execute(statements: statements, readAhead: false) + + let texts = statements.map(\.text) + #expect(driver.events == texts) + } + + @Test("A batch that changed how the session reads text fails at the statement read past it, and rolls nothing back itself") + func changedRulesStopTheReadAhead() async throws { + let statements = Self.statements(2) + let driver = PipeliningDriver() + driver.rulesAfter = [statements[0].text: PipeliningDriver.backslashesEscape] + let sink = sink(on: driver) + try await sink.beginTransaction() + + do { + try await sink.execute(statements: statements, readAhead: true) + Issue.record("The statement read past the batch should be refused") + } catch let failure as PluginImportBatchFailure { + #expect(failure.index == statements.count) + #expect(failure.underlying is ImportReadingRulesChanged) + } + #expect(driver.events == ["BEGIN", "batch 2"]) + } + + @Test("Changed rules leave a batch alone when the file was not read past it") + func changedRulesWithoutReadAhead() async throws { + let statements = Self.statements(2) + let driver = PipeliningDriver() + driver.rulesAfter = [statements[0].text: PipeliningDriver.backslashesEscape] + let sink = sink(on: driver) + try await sink.beginTransaction() + + try await sink.execute(statements: statements, readAhead: false) + + #expect(driver.events == ["BEGIN", "batch 2"]) + } + + @Test("Rules a batch changes and changes back leave the statement read past it as it was read") + func restoredRulesKeepTheReadAhead() async throws { + let statements = Self.statements(2) + let driver = PipeliningDriver() + driver.rulesAfter = [ + statements[0].text: PipeliningDriver.backslashesEscape, + statements[1].text: PipeliningDriver.backslashesAreText, + ] + let sink = sink(on: driver) + try await sink.beginTransaction() + + try await sink.execute(statements: statements, readAhead: true) + + #expect(driver.events == ["BEGIN", "batch 2"]) + } + + @Test("Statements the driver declined and ran alone are held to the same rule") + func declinedBatchThatChangedRules() async throws { + let statements = Self.statements(2) + let driver = PipeliningDriver() + driver.declines = true + driver.rulesAfter = [statements[1].text: PipeliningDriver.backslashesEscape] + + do { + try await sink(on: driver).execute(statements: statements, readAhead: true) + Issue.record("The statement read past the batch should be refused") + } catch let failure as PluginImportBatchFailure { + #expect(failure.index == statements.count) + #expect(failure.underlying is ImportReadingRulesChanged) + } + let texts = statements.map(\.text) + #expect(driver.events == texts) + } + + @Test("A batch goes to the driver in one call, and its first failure names that statement") + func failureNamesItsStatement() async throws { + let statements = (1...3).map { PluginImportStatement(text: "INSERT INTO t VALUES (\($0))", line: $0 * 10) } + let driver = PipeliningDriver(failing: [statements[1].text]) + + do { + try await sink(on: driver).execute(statements: statements, readAhead: true) + Issue.record("The batch should have failed at its second statement") + } catch let failure as PluginImportBatchFailure { + #expect(failure.index == 1) + #expect(failure.underlying is PipeliningDriver.Refused) + } + #expect(driver.events == ["batch 3"]) + } +} diff --git a/apple/TableProTests/Core/Plugins/PluginDriverAdapterStatementBatchTests.swift b/apple/TableProTests/Core/Plugins/PluginDriverAdapterStatementBatchTests.swift new file mode 100644 index 0000000000..cbd04f51df --- /dev/null +++ b/apple/TableProTests/Core/Plugins/PluginDriverAdapterStatementBatchTests.swift @@ -0,0 +1,107 @@ +// +// PluginDriverAdapterStatementBatchTests.swift +// TableProTests +// + +import Foundation +@testable import TablePro +import TableProPluginKit +import Testing + +private final class ReportingDriver: PluginDriverStub, @unchecked Sendable { + struct Refused: Error {} + + var answersFewer = false + var declines = false + var failing: Set = [] + private(set) var batches: [[PluginBatchStatement]] = [] + private(set) var alone: [String] = [] + + var capabilities: PluginCapabilities { [.statementBatches] } + + func execute(query: String) async throws -> PluginQueryResult { + alone.append(query) + if failing.contains(query) { throw Refused() } + return .empty + } + + func executeParameterized(query: String, parameters: [PluginCellValue]) async throws -> PluginQueryResult { + try await execute(query: query) + } + + func executeStatements(_ statements: [PluginBatchStatement]) async throws -> [PluginBatchStatementOutcome]? { + guard !declines else { return nil } + batches.append(statements) + let answered = answersFewer ? statements.dropLast() : statements[...] + return answered.map { _ in .succeeded(rowsAffected: 1, executionTime: 0) } + } +} + +struct PluginDriverAdapterStatementBatchTests { + private func adapter(_ driver: ReportingDriver) -> PluginDriverAdapter { + PluginDriverAdapter(connection: TestFixtures.makeConnection(type: .postgresql), pluginDriver: driver) + } + + @Test("A statement the host would refuse ends the batch there, and only the ones before it are sent") + func refusedStatementEndsTheBatch() async throws { + let driver = ReportingDriver() + let statements = [ + BatchedStatement(sql: "INSERT INTO t VALUES (1)"), + BatchedStatement(sql: "INSERT INTO t VALUES ('a\u{0}b')"), + BatchedStatement(sql: "INSERT INTO t VALUES (3)"), + ] + + let outcomes = try await adapter(driver).executeStatements(statements) ?? [] + + #expect(driver.batches.map { $0.map(\.query) } == [["INSERT INTO t VALUES (1)"]]) + #expect(outcomes.count == 2) + #expect(outcomes.first?.succeeded == true) + guard case .failed(let error) = outcomes.last else { + Issue.record("The refused statement should fail") + return + } + #expect((error as? DatabaseError)?.errorDescription == DatabaseError.statementContainsNulCharacter.errorDescription) + } + + @Test("A driver that answers fewer statements than it was sent fails the first one it left out") + func missingAnswerFails() async throws { + let driver = ReportingDriver() + driver.answersFewer = true + + let outcomes = try await adapter(driver).executeStatements([ + BatchedStatement(sql: "DELETE FROM t WHERE id = 1"), + BatchedStatement(sql: "DELETE FROM t WHERE id = 2"), + ]) ?? [] + + #expect(outcomes.count == 2) + #expect(outcomes[0].succeeded) + #expect(!outcomes[1].succeeded) + } + + @Test("Parameters reach the driver bound, and a text without them goes as text") + func carriesParameters() async throws { + let driver = ReportingDriver() + + _ = try await adapter(driver).executeStatements([ + BatchedStatement(sql: "UPDATE t SET a = $1", parameters: ["x"]), + BatchedStatement(sql: "DELETE FROM t"), + ]) + + let sent = driver.batches.first ?? [] + #expect(sent.first?.parameters == [.text("x")]) + #expect(sent.last?.parameters == nil) + } + + @Test("A batch the driver declines comes back nil, with nothing sent") + func declinedBatchIsNil() async throws { + let driver = ReportingDriver() + driver.declines = true + + let outcomes = try await adapter(driver).executeStatements( + (1...3).map { BatchedStatement(sql: "INSERT INTO t VALUES (\($0))") } + ) + + #expect(outcomes == nil) + #expect(driver.alone.isEmpty) + } +} diff --git a/apple/changelog.d/4356.changed.1.md b/apple/changelog.d/4356.changed.1.md new file mode 100644 index 0000000000..a57019e3ae --- /dev/null +++ b/apple/changelog.d/4356.changed.1.md @@ -0,0 +1 @@ +A PostgreSQL SQL import stops at a statement read before an earlier write changed standard_conforming_strings. diff --git a/apple/changelog.d/4356.changed.md b/apple/changelog.d/4356.changed.md new file mode 100644 index 0000000000..f9f6ed61dd --- /dev/null +++ b/apple/changelog.d/4356.changed.md @@ -0,0 +1 @@ +A PostgreSQL SQL import with Wrap in transaction on sends its writes in batches, not one statement at a time. diff --git a/apple/project.yml b/apple/project.yml index 1234b6ec26..845c44bbce 100644 --- a/apple/project.yml +++ b/apple/project.yml @@ -195,7 +195,7 @@ targetTemplates: PostgreSQLLinkage: dependencies: - package: TableProCore - products: [TableProCoreTypes, TableProGeometry, TableProKerberos, TableProLogRedaction, TableProPostgreSQLCore, TableProSQLVocabulary, TableProTLSClientIdentity] + products: [TableProCoreTypes, TableProGeometry, TableProKerberos, TableProLogRedaction, TableProPostgreSQLCore, TableProSQLGrammar, TableProSQLVocabulary, TableProTLSClientIdentity] settings: base: HEADER_SEARCH_PATHS: