From df32cc00496affb7e6b22beb32a53987853bda2f Mon Sep 17 00:00:00 2001 From: Andrew Tierney Date: Tue, 17 Nov 2026 10:00:35 +1100 Subject: [PATCH 1/2] fix: atomic ExecuteBatchSQL, upsert / INSERT OR conflict clauses, subquery ORDER BY collation, quoted batch SET columns - ExecuteBatchSQL: SELECT-bearing and mixed batches ran statement by statement without a transaction; they now run as one transaction with the PageBased undo scope, and every grouped batch transaction uses the undo scope too, so a failing batch leaves every table unchanged. - INSERT OR IGNORE|REPLACE|ABORT|FAIL|ROLLBACK, REPLACE INTO and ON CONFLICT ... DO NOTHING | DO UPDATE SET ... [WHERE ...] run on the SQL INSERT path (VALUES and SELECT, single and composite keys); DO UPDATE assignments are expressions with excluded.col. Composite conflict lookups compare with the column collation. - AST ORDER BY (derived tables, IN-subqueries) honours COLLATE and NULLS FIRST/LAST and compares text as BINARY by default; COLLATE NOCASE DESC no longer loses its direction. - Batch UPDATEs with a quoted SET column take the structured fast path again. - Composite UNIQUE check registers the column's auto index before falling back to a scan. --- docs/CHANGELOG.md | 6 + docs/sharpcoredb-known-issues.md | 9 +- src/SharpCoreDB/DataStructures/Table.CRUD.cs | 13 +- .../DataStructures/Table.UniqueKeys.cs | 7 +- .../Database/Execution/Database.Batch.cs | 104 +++++-- src/SharpCoreDB/Execution/SubqueryExecutor.cs | 5 +- .../Services/EnhancedSqlParser.Select.cs | 44 ++- src/SharpCoreDB/Services/SqlAst.Nodes.cs | 9 + src/SharpCoreDB/Services/SqlOrderBy.cs | 32 ++ src/SharpCoreDB/Services/SqlParser.DML.cs | 48 +-- src/SharpCoreDB/Services/SqlParser.Upsert.cs | 280 ++++++++++++++++++ .../Services/SqlScalarExpression.cs | 11 + .../SharpCoreDB.Tests/BatchAtomicityTests.cs | 140 +++++++++ .../BatchUpdateClassificationTests.cs | 20 ++ .../SubqueryOrderByCollationTests.cs | 118 ++++++++ tests/SharpCoreDB.Tests/UpsertTests.cs | 174 +++++++++++ 16 files changed, 968 insertions(+), 52 deletions(-) create mode 100644 src/SharpCoreDB/Services/SqlParser.Upsert.cs create mode 100644 tests/SharpCoreDB.Tests/BatchAtomicityTests.cs create mode 100644 tests/SharpCoreDB.Tests/SubqueryOrderByCollationTests.cs create mode 100644 tests/SharpCoreDB.Tests/UpsertTests.cs diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index b924ba17..e1c59213 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -71,6 +71,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **A transaction rollback did not undo PageBased tables (known issue §13).** `CancelBatchUpdate()` — an ADO.NET `DbTransaction.Rollback()`, and so linq2db's and EF Core's — left a PageBased UPDATE or INSERT in place, also after a reopen: the PageBased engine writes into its page cache and page files directly, outside the storage transaction buffer, and its `Rollback()` was a stub. Each page manager now keeps a page's image on its first change inside the transaction and puts the images back on rollback (pages the transaction allocated return to the free list); every table then rebuilds its primary-key index and lets its hash indexes rebuild, which also fixes a columnar rollback that left the hash index holding the rolled-back value. A write outside a transaction pays one null check. Pinned by `TransactionRollbackTests`. `ExecuteBatchSQL`'s own failure rollback does not use the undo scope yet (recorded in §13). - **Inside a transaction, a columnar full scan did not see the transaction's own UPDATEs and DELETEs (known issue §12).** The scan read the data file as it was on disk, while the transaction's in-place overwrites and its deletes were buffered until commit: it returned an updated row's old value, still listed a deleted row, and `WHERE n = 50` after `SET n = 50` found nothing; a primary-key lookup still returned a deleted row. The buffered overwrites and tombstones are laid over the scanned records now (only while a transaction holds them for that file), and a point read of a deleted record returns nothing. Pinned by `InTransactionReadTests` (encrypted and `NoEncryptMode`). Suites: core **2489/0 failed (6 skipped)**, VectorSearch 255/0, EF Core 116/0, linq2DB 96/0, regression canaries 51/0; write-path gate: GATE PASSED (nothing slower than baseline × 1.50; raw READ 1.30×, default READ 1.27×, default DELETE 1.31× in the watch band, as in the previous run on this box). +- **`ExecuteBatchSQL` was not atomic.** A batch with a SELECT, or one mixing statement kinds, ran its statements one at a time without a transaction, so a failing statement kept every statement before it on every storage mode; and the grouped executor's transaction rolled back only the storage buffer, so a PageBased table kept a failed batch's earlier changes. Both paths now run as one transaction with the PageBased undo scope (inside a caller's transaction, that transaction decides). Pinned by `BatchAtomicityTests`. +- **Upsert and INSERT conflict clauses.** `INSERT OR IGNORE|REPLACE|ABORT|FAIL|ROLLBACK` failed with "Unsupported SQL statement", `REPLACE INTO` likewise, and an `ON CONFLICT … DO NOTHING | DO UPDATE SET … [WHERE …]` clause was ignored, so an upsert onto an existing key failed as a duplicate — the conflict handlers existed but nothing called them. They run on the SQL INSERT path now (VALUES and SELECT, single-column and composite keys, inside `ExecuteBatchSQL` too); a DO UPDATE assignment and its WHERE are expressions over the existing row with `excluded.col` the incoming value. The composite conflict lookup compares with each column's declared collation, and its "same row" check with the key's — it was always case-insensitive. Pinned by `UpsertTests`. +- **ORDER BY on derived tables and IN-subqueries ignored COLLATE and NULLS FIRST/LAST, and sorted text case-insensitively.** The AST parser read only ASC/DESC after a key, so `COLLATE NOCASE DESC` lost its direction too; a descending sort moved NULLs to the end whatever the statement said; and text without a COLLATE compared case-insensitively instead of SQLite's BINARY. Pinned by `SubqueryOrderByCollationTests`. +- **Batch UPDATEs with a quoted SET column took the slow path.** `SET "n" = 5` / `SET [n] = 5` (linq2db's and EF Core's form) was looked up with its quotes and left the structured fast path; the name is unquoted first now (no allocation for an unquoted name). Pinned by `BatchUpdateClassificationTests.BatchUpdate_QuotedSetColumn_TakesTheStructuredFastPath`. +- **Composite UNIQUE check without its index.** When the first column of a multi-column constraint has no hash index, the check registers the column's auto index (as reads do) and scans only as a last resort. The constraint's own index cannot be dropped (as in SQLite), so this is hardening. Suites: core **2523/0 failed (6 skipped)**, VectorSearch 255/0, EF Core 116/0, linq2DB 96/0, regression canaries 51/0; write-path gate: GATE PASSED (nothing slower than baseline × 1.50; raw READ 1.39×, default READ 1.42×, default DELETE 1.33× in the watch band on an un-quieted box). + ## [2.1.4] - 2026-11-13 > The issue #497 follow-up: a quoted identifier in `CREATE TABLE` is catalogued without its quoting, so a schema diff --git a/docs/sharpcoredb-known-issues.md b/docs/sharpcoredb-known-issues.md index 1e05f4d6..149375d9 100644 --- a/docs/sharpcoredb-known-issues.md +++ b/docs/sharpcoredb-known-issues.md @@ -561,10 +561,9 @@ scan, so it needs the write-path gate and a read benchmark; not attempted in thi > pages the transaction allocated to the free list, then every table rebuilds its primary-key index from storage and > marks its hash indexes stale — which also fixes a columnar rollback leaving the hash index on the rolled-back value. > Outside a transaction a write pays one null check. Pinned by `TransactionRollbackTests` (including an ADO.NET -> `DbTransaction.Rollback()`). **Still open:** `ExecuteBatchSQL` rolls its own internal transaction back on a failing -> statement without the undo scope, so a PageBased table keeps the statements of a failed batch that ran before the -> failure; hooking it adds a page copy per touched page to the most-measured write path and needs its own gate run. -> And without a page WAL, a crash in the middle of a transaction can still leave pages the cache wrote early. +> `DbTransaction.Rollback()`). `ExecuteBatchSQL` uses the undo scope too since `fix/batch-rollback-and-sql-gaps` +> (`BatchAtomicityTests`), so a failing batch leaves a PageBased table unchanged. **Still open:** without a page WAL, a +> crash in the middle of a transaction can still leave pages the cache wrote early. `CancelBatchUpdate()` (an ADO.NET `DbTransaction.Rollback()`) does not undo a PageBased table's changes: a rolled-back UPDATE or INSERT is still there — also after a reopen — and a rolled-back DELETE stays deleted. Columnar tables roll @@ -598,4 +597,4 @@ in this change. | 10 | Single-file ignores `COLLATE` | 🟡 Low | — | **Open, owner decision** — needs a column-entry format extension (§10) | | 11 | PK uniqueness / composite keys | 🔴 High | — | **Fixed on directory storage for new tables** (composite key = UNIQUE over its columns); **open, owner decision** for single-file storage, which enforces no PK uniqueness (§11) | | 12 | Columnar scan inside a transaction misses its own UPDATE/DELETE | 🟠 Medium | — | **Fixed** — buffered overwrites and tombstones are laid over the scan (§12) | -| 13 | PageBased transaction rollback | 🔴 High | — | **Fixed** for `CancelBatchUpdate` (ADO.NET, linq2db, EF Core) via page undo images; `ExecuteBatchSQL`'s internal failure rollback still open (§13) | \ No newline at end of file +| 13 | PageBased transaction rollback | 🔴 High | — | **Fixed** for `CancelBatchUpdate` (ADO.NET, linq2db, EF Core) and `ExecuteBatchSQL` via page undo images; crash atomicity without a page WAL still open (§13) | \ No newline at end of file diff --git a/src/SharpCoreDB/DataStructures/Table.CRUD.cs b/src/SharpCoreDB/DataStructures/Table.CRUD.cs index c4e3889f..6b0ca0af 100644 --- a/src/SharpCoreDB/DataStructures/Table.CRUD.cs +++ b/src/SharpCoreDB/DataStructures/Table.CRUD.cs @@ -5302,7 +5302,9 @@ private bool TryFindCompositeUniqueConflict( candidateRow.TryGetValue(constraintColumn, out var existingValue); row.TryGetValue(constraintColumn, out var incomingValue); - if (!SqlParser.AreValuesEqual(existingValue, incomingValue)) + // The column's declared collation decides, as it does for INSERT and UPDATE (Table.UniqueKeys): a + // NOCASE column conflicts on 'ABC' against 'abc', a BINARY one does not. + if (!CompositeComponentEquals(constraintColumn, existingValue, incomingValue)) { allColumnsMatch = false; break; @@ -5321,7 +5323,7 @@ private bool TryFindCompositeUniqueConflict( return false; } - private static bool TryResolveConflictingPrimaryKey( + private bool TryResolveConflictingPrimaryKey( Dictionary existingRow, string primaryKeyColumn, object? currentPrimaryKeyValue, @@ -5334,10 +5336,11 @@ private static bool TryResolveConflictingPrimaryKey( if (currentPrimaryKeyValue is not null && currentPrimaryKeyValue is not DBNull) { - var samePrimaryKey = string.Equals( + // "The same row" is decided with the primary key's collation; up to 2.1.4 it was always case-insensitive, + // so on a BINARY text key a row 'ABC' was taken for the row 'abc' and its UNIQUE conflict was missed. + var samePrimaryKey = UniqueKeyComparer(primaryKeyColumn).Equals( existingPrimaryKeyValue.ToString(), - currentPrimaryKeyValue.ToString(), - StringComparison.OrdinalIgnoreCase); + currentPrimaryKeyValue.ToString()); if (samePrimaryKey) return false; diff --git a/src/SharpCoreDB/DataStructures/Table.UniqueKeys.cs b/src/SharpCoreDB/DataStructures/Table.UniqueKeys.cs index 01eaee86..0b78c787 100644 --- a/src/SharpCoreDB/DataStructures/Table.UniqueKeys.cs +++ b/src/SharpCoreDB/DataStructures/Table.UniqueKeys.cs @@ -484,7 +484,12 @@ private List> FindCompositeKeyHolders(List co { var holders = new List>(); var seed = constraint[0]; - foreach (var candidate in this.registeredIndexes.ContainsKey(seed) + + // The constraint's own index (EnsureUniqueConstraintIndexes) normally exists; if it was dropped, the column's + // auto index is registered again, which ordinary reads do too. Only a column that cannot be indexed is scanned — + // a full scan per checked row, so it stays the last resort. (The table lock allows recursion, so neither path + // can deadlock under the write lock the callers hold.) + foreach (var candidate in EnsureAutoHashIndexRegistered(seed) ? FindByIndex(seed, values[seed]) : SelectInternal(null, orderBy: null, asc: true, noEncrypt: false)) { diff --git a/src/SharpCoreDB/Database/Execution/Database.Batch.cs b/src/SharpCoreDB/Database/Execution/Database.Batch.cs index 1dae2a2c..2ab7a112 100644 --- a/src/SharpCoreDB/Database/Execution/Database.Batch.cs +++ b/src/SharpCoreDB/Database/Execution/Database.Batch.cs @@ -397,17 +397,20 @@ public long[] InsertBatchTyped(string tableName, Optimizations.TypedRowBuffer.Co lock (_walLock) { + BeginTablesUndo(); storage.BeginTransaction(); try { var positions = table.InsertBatch(rows); storage.CommitSync(); + CommitTablesUndo(); return positions; } catch { storage.Rollback(); + RollbackTablesUndo(); throw; } } @@ -456,17 +459,20 @@ public long[] InsertBatch(string tableName, List> row lock (_walLock) { + BeginTablesUndo(); storage.BeginTransaction(); try { var positions = table.InsertBatch(rows); storage.CommitSync(); + CommitTablesUndo(); return positions; } catch { storage.Rollback(); + RollbackTablesUndo(); throw; } } @@ -498,17 +504,20 @@ public async Task InsertBatchAsync( { lock (_walLock) { + BeginTablesUndo(); storage.BeginTransaction(); try { var positions = table.InsertBatch(rows); storage.CommitSync(); + CommitTablesUndo(); return positions; } catch { storage.Rollback(); + RollbackTablesUndo(); throw; } } @@ -534,12 +543,14 @@ public long Insert(string tableName, Dictionary row) lock (_walLock) { + BeginTablesUndo(); storage.BeginTransaction(); try { table.Insert(row); storage.CommitSync(); + CommitTablesUndo(); // Return -1 as we can't easily get position from ITable interface // Caller can use InsertBatch for position tracking @@ -548,6 +559,7 @@ public long Insert(string tableName, Dictionary row) catch { storage.Rollback(); + RollbackTablesUndo(); throw; } } @@ -678,17 +690,20 @@ public long[] InsertBatch(string tableName, IReadOnlyList rows, IReadO lock (_walLock) { + BeginTablesUndo(); storage.BeginTransaction(); try { var positions = concreteTable.InsertBatch(rowArrays, columnOrder); storage.CommitSync(); + CommitTablesUndo(); return positions; } catch { storage.Rollback(); + RollbackTablesUndo(); throw; } } @@ -708,6 +723,7 @@ private int RunInBatchTransaction(Func body) if (!isInTransactionBefore) { + BeginTablesUndo(); storage.BeginTransaction(); } @@ -719,6 +735,7 @@ private int RunInBatchTransaction(Func body) { long commitStart = Diagnostics.WritePathProfiler.Stamp(); storage.CommitSync(); + CommitTablesUndo(); Diagnostics.WritePathProfiler.Add(Diagnostics.WritePathProfiler.Stage.Commit, commitStart); long commitBufferStart = Diagnostics.WritePathProfiler.Stamp(); @@ -733,6 +750,7 @@ private int RunInBatchTransaction(Func body) if (!isInTransactionBefore) { storage.Rollback(); + RollbackTablesUndo(); } throw; @@ -1109,6 +1127,9 @@ private bool TryParseUpdateForBatch( return false; } + // A quoted SET column (`SET "n" = 5`, `SET [n] = 5` — linq2db's and EF Core's form) is matched by its name. + // Up to 2.1.4 it was looked up with its quotes, missed, and the statement left this fast path. + setCol = UnquoteSetColumn(setCol); int colIdx = fastTableMeta.Columns.IndexOf(setCol); if (colIdx < 0 || SqlParser.IsComputedSetValue(setValRaw, fastTableMeta.Columns, out _)) { @@ -1177,7 +1198,7 @@ private bool TryParseUpdateForBatch( var eqIdx = setPart.IndexOf('='); if (eqIdx < 0) return false; - var colName = setPart[..eqIdx].Trim(); + var colName = UnquoteSetColumn(setPart[..eqIdx].Trim()); var valStr = setPart[(eqIdx + 1)..].Trim(); var fallbackColIdx = table.Columns.IndexOf(colName); @@ -1196,6 +1217,10 @@ private bool TryParseUpdateForBatch( return true; } + /// A SET column name without its quoting; an unquoted name is returned as is, without allocating. + private static string UnquoteSetColumn(string column) => + column.Length > 1 && column[0] is '"' or '[' or '`' ? SqlIdentifierScanner.IdentifierText(column) : column; + /// /// True when a scanned identifier is already in the form the table layer matches its column names against: /// no alias qualifier and no quoting or bracketing. @@ -1260,6 +1285,49 @@ private bool TryParseDeleteForBatch(string sql, out string tableName, out string return tables.ContainsKey(tableName) && !string.IsNullOrWhiteSpace(where); } + /// + /// Runs a batch's statements one at a time, in the order given, as one transaction: a statement that fails rolls back + /// every statement of the batch before it (PageBased tables through their undo scope). Inside a transaction the + /// caller opened, that transaction owns the outcome. This is the path for a batch with a SELECT and for a mixed batch, + /// which the grouped executor cannot take; up to 2.1.4 it ran the statements without any transaction, so a failing + /// statement kept every statement before it. + /// + private void ExecuteStatementsAtomically(string[] statements) + { + lock (_walLock) + { + if (storage.IsInTransaction) + { + foreach (var sql in statements) + { + ExecuteSQL(sql); + } + + return; + } + + BeginTablesUndo(); + storage.BeginTransaction(); + try + { + foreach (var sql in statements) + { + ExecuteSQL(sql); + } + + storage.CommitSync(); + CommitTablesUndo(); + storage.FlushTransactionBuffer(); + } + catch + { + storage.Rollback(); + RollbackTablesUndo(); + throw; + } + } + } + /// /// ✅ PHASE 3: Optimized batch SQL execution with prepared statement caching. /// Uses cached parsers for repeated INSERT schemas, reducing parsing overhead by ~40%. @@ -1281,10 +1349,7 @@ private bool TryParseDeleteForBatch(string sql, out string tableName, out string if (hasSelect) { - foreach (var sql in statements) - { - ExecuteSQL(sql); - } + ExecuteStatementsAtomically(statements); return; } @@ -1300,10 +1365,7 @@ private bool TryParseDeleteForBatch(string sql, out string tableName, out string // only a homogeneous batch may take that path (2026-11-13). if (!batch.IsHomogeneous) { - foreach (var sql in statements) - { - ExecuteSQL(sql); - } + ExecuteStatementsAtomically(statements); return; } @@ -1317,6 +1379,7 @@ private bool TryParseDeleteForBatch(string sql, out string tableName, out string if (!isInTransactionBefore) { + BeginTablesUndo(); storage.BeginTransaction(); } @@ -1336,6 +1399,7 @@ private bool TryParseDeleteForBatch(string sql, out string tableName, out string // The Add order matters: Add closes the checkpoint its Stamp opened, so each pair stays closed. long commitStart = Diagnostics.WritePathProfiler.Stamp(); storage.CommitSync(); + CommitTablesUndo(); Diagnostics.WritePathProfiler.Add(Diagnostics.WritePathProfiler.Stage.Commit, commitStart); long commitBufferStart = Diagnostics.WritePathProfiler.Stamp(); @@ -1353,6 +1417,7 @@ private bool TryParseDeleteForBatch(string sql, out string tableName, out string if (!isInTransactionBefore) { storage.Rollback(); + RollbackTablesUndo(); } throw; } @@ -2003,10 +2068,8 @@ public async Task ExecuteBatchSQLAsync(IEnumerable sqlStatements, Cancel if (hasSelect) { - foreach (var sql in statements) - { - await ExecuteSQLAsync(sql, cancellationToken); - } + cancellationToken.ThrowIfCancellationRequested(); + ExecuteStatementsAtomically(statements); return; } @@ -2020,10 +2083,8 @@ public async Task ExecuteBatchSQLAsync(IEnumerable sqlStatements, Cancel // because the grouped executor's fixed bucket order is only valid for one kind of statement. if (!batch.IsHomogeneous) { - foreach (var sql in statements) - { - await ExecuteSQLAsync(sql, cancellationToken).ConfigureAwait(false); - } + cancellationToken.ThrowIfCancellationRequested(); + ExecuteStatementsAtomically(statements); return; } @@ -2042,6 +2103,7 @@ public async Task ExecuteBatchSQLAsync(IEnumerable sqlStatements, Cancel if (startedTransaction) { + BeginTablesUndo(); storage.BeginTransaction(); } @@ -2055,6 +2117,7 @@ public async Task ExecuteBatchSQLAsync(IEnumerable sqlStatements, Cancel { commitStart = SharpCoreDB.Diagnostics.WritePathProfiler.Stamp(); commitTask = storage.CommitAsync(); + CommitTablesUndo(); } } catch @@ -2062,6 +2125,7 @@ public async Task ExecuteBatchSQLAsync(IEnumerable sqlStatements, Cancel if (startedTransaction) { storage.Rollback(); + RollbackTablesUndo(); } throw; @@ -2108,6 +2172,7 @@ await Task.Run(() => { lock (_walLock) { + BeginTablesUndo(); storage.BeginTransaction(); try @@ -2128,10 +2193,12 @@ await Task.Run(() => } storage.CommitAsync().GetAwaiter().GetResult(); + CommitTablesUndo(); } catch { storage.Rollback(); + RollbackTablesUndo(); throw; } } @@ -2154,6 +2221,7 @@ await Task.Run(() => { lock (_walLock) { + BeginTablesUndo(); storage.BeginTransaction(); try @@ -2236,10 +2304,12 @@ await Task.Run(() => } storage.CommitAsync().GetAwaiter().GetResult(); + CommitTablesUndo(); } catch { storage.Rollback(); + RollbackTablesUndo(); throw; } } diff --git a/src/SharpCoreDB/Execution/SubqueryExecutor.cs b/src/SharpCoreDB/Execution/SubqueryExecutor.cs index 8aceacf9..8e658478 100644 --- a/src/SharpCoreDB/Execution/SubqueryExecutor.cs +++ b/src/SharpCoreDB/Execution/SubqueryExecutor.cs @@ -459,9 +459,8 @@ private List> ApplyOrderBy( foreach (var item in orderBy.Items) { var key = AstExecutor.OrderKeySelector(item); - ordered = ordered is null - ? item.IsAscending ? rows.OrderBy(key, SqlOrderBy.KeyComparer.Instance) : rows.OrderByDescending(key, SqlOrderBy.KeyComparer.Instance) - : item.IsAscending ? ordered.ThenBy(key, SqlOrderBy.KeyComparer.Instance) : ordered.ThenByDescending(key, SqlOrderBy.KeyComparer.Instance); + var itemComparer = SqlOrderBy.ItemComparer(item, SqlOrderBy.KeyComparer.Instance); + ordered = ordered is null ? rows.OrderBy(key, itemComparer) : ordered.ThenBy(key, itemComparer); } return [.. ordered!]; diff --git a/src/SharpCoreDB/Services/EnhancedSqlParser.Select.cs b/src/SharpCoreDB/Services/EnhancedSqlParser.Select.cs index 8dabb03e..9d12da7e 100644 --- a/src/SharpCoreDB/Services/EnhancedSqlParser.Select.cs +++ b/src/SharpCoreDB/Services/EnhancedSqlParser.Select.cs @@ -981,6 +981,26 @@ private HavingNode ParseHaving() return node; } + /// + /// Reads what may follow an ORDER BY key, in SQLite's order: COLLATE name, ASC|DESC, + /// NULLS FIRST|LAST. Up to 2.1.4 only ASC/DESC was read, so COLLATE NOCASE DESC lost its direction as + /// well and the rest was left as unparsed trailing content. + /// + private void ParseOrderItemSuffix(OrderByItem item) + { + if (MatchKeyword("COLLATE")) + { + item.Collation = ConsumeIdentifier()?.ToUpperInvariant(); + } + + item.IsAscending = !MatchKeyword("DESC"); + MatchKeyword("ASC"); + if (MatchKeyword("NULLS")) + { + item.NullsFirst = MatchKeyword("FIRST") ? true : MatchKeyword("LAST") ? false : null; + } + } + /// Returns true when the text after the current position continues an expression (a call, an operator). private bool ContinuesAsExpression() { @@ -1002,6 +1022,14 @@ private bool TryParseExpressionOrderItem(int itemStart, out OrderByItem item) item = null!; int end = FindOrderItemEnd(itemStart); var text = _sql[itemStart..end].Trim(); + bool? nullsFirst = null; + var nulls = Regex.Match(text, @"\s+NULLS\s+(FIRST|LAST)$", RegexOptions.IgnoreCase, TimeSpan.FromSeconds(1)); + if (nulls.Success) + { + nullsFirst = nulls.Groups[1].Value.Equals("FIRST", StringComparison.OrdinalIgnoreCase); + text = text[..nulls.Index].TrimEnd(); + } + bool ascending = true; var direction = Regex.Match(text, @"\s+(ASC|DESC)$", RegexOptions.IgnoreCase, TimeSpan.FromSeconds(1)); if (direction.Success) @@ -1010,6 +1038,14 @@ private bool TryParseExpressionOrderItem(int itemStart, out OrderByItem item) text = text[..direction.Index].TrimEnd(); } + string? collation = null; + var collate = Regex.Match(text, @"\s+COLLATE\s+(\w+)$", RegexOptions.IgnoreCase, TimeSpan.FromSeconds(1)); + if (collate.Success) + { + collation = collate.Groups[1].Value.ToUpperInvariant(); + text = text[..collate.Index].TrimEnd(); + } + if (text.Length == 0 || !SqlScalarExpression.TryParse(text, out _)) { return false; @@ -1020,6 +1056,8 @@ private bool TryParseExpressionOrderItem(int itemStart, out OrderByItem item) Column = new ColumnReferenceNode { ColumnName = text }, Expression = text, IsAscending = ascending, + Collation = collation, + NullsFirst = nullsFirst, }; _position = end; return true; @@ -1087,9 +1125,8 @@ private OrderByNode ParseOrderBy() { Column = new ColumnReferenceNode { ColumnName = ordinal.ToString() }, OrdinalPosition = ordinal, - IsAscending = !MatchKeyword("DESC") }; - MatchKeyword("ASC"); + ParseOrderItemSuffix(item); node.Items.Add(item); } else @@ -1130,10 +1167,9 @@ private OrderByNode ParseOrderBy() var item = new OrderByItem { Column = new ColumnReferenceNode { TableAlias = tableAlias, ColumnName = columnName }, - IsAscending = !MatchKeyword("DESC") }; - MatchKeyword("ASC"); + ParseOrderItemSuffix(item); node.Items.Add(item); } } while (MatchToken(",")); diff --git a/src/SharpCoreDB/Services/SqlAst.Nodes.cs b/src/SharpCoreDB/Services/SqlAst.Nodes.cs index 6326333f..c933d429 100644 --- a/src/SharpCoreDB/Services/SqlAst.Nodes.cs +++ b/src/SharpCoreDB/Services/SqlAst.Nodes.cs @@ -394,6 +394,15 @@ public class OrderByItem /// (ORDER BY LENGTH(name), ORDER BY price * qty); null for a column or an ordinal. /// public string? Expression { get; set; } + + /// Gets or sets the key's COLLATE name (upper case: NOCASE, RTRIM, BINARY); null when none is written. + public string? Collation { get; set; } + + /// + /// Gets or sets NULLS FIRST (true) or NULLS LAST (false); null when not written, which means SQLite's + /// default — NULLs first ascending, last descending. + /// + public bool? NullsFirst { get; set; } } /// diff --git a/src/SharpCoreDB/Services/SqlOrderBy.cs b/src/SharpCoreDB/Services/SqlOrderBy.cs index 988157f9..088e8f78 100644 --- a/src/SharpCoreDB/Services/SqlOrderBy.cs +++ b/src/SharpCoreDB/Services/SqlOrderBy.cs @@ -24,6 +24,38 @@ namespace SharpCoreDB.Services; /// internal static class SqlOrderBy { + /// + /// The comparer for one AST ORDER BY item: its direction, its NULL placement (NULLS FIRST/LAST, else SQLite's default + /// — first ascending, last descending) and its COLLATE, over for the non-NULL values. The + /// caller sorts ascending with it; the direction is inside. + /// + internal static IComparer ItemComparer(OrderByItem item, IComparer values) + { + bool descending = !item.IsAscending; + bool nullsFirst = item.NullsFirst ?? !descending; + IComparer? text = item.Collation switch + { + "NOCASE" => StringComparer.OrdinalIgnoreCase, + "RTRIM" => RTrimComparer, + "BINARY" => StringComparer.Ordinal, + _ => null, + }; + + return Comparer.Create((x, y) => + { + bool xNull = x is null or DBNull; + bool yNull = y is null or DBNull; + if (xNull || yNull) + { + return xNull == yNull ? 0 : xNull == nullsFirst ? -1 : 1; + } + + int c = text is not null && x is string xs && y is string ys ? text.Compare(xs, ys) : values.Compare(x, y); + c = Math.Sign(c); + return descending ? -c : c; + }); + } + /// /// Compares sort keys as SQLite does: NULL first, numbers and text by . /// diff --git a/src/SharpCoreDB/Services/SqlParser.DML.cs b/src/SharpCoreDB/Services/SqlParser.DML.cs index 4bfc131f..f68cb7c2 100644 --- a/src/SharpCoreDB/Services/SqlParser.DML.cs +++ b/src/SharpCoreDB/Services/SqlParser.DML.cs @@ -165,9 +165,15 @@ private void ExecuteInternal(string sql, string[] parts, IWAL? wal = null, bool break; case (SqlConstants.INSERT, SqlConstants.INTO): + case (SqlConstants.INSERT, "OR"): ExecuteInsert(sql, wal); break; + case ("REPLACE", SqlConstants.INTO): + // SQLite: REPLACE INTO is INSERT OR REPLACE INTO. + ExecuteInsert("INSERT OR " + sql.TrimStart(), wal); + break; + case ("UPDATE", _): ExecuteUpdate(sql, wal); break; @@ -314,6 +320,8 @@ private void ExecuteInsert(string sql, IWAL? wal) var restTrimmed = rest.Trim(); if (restTrimmed.StartsWith("SELECT", StringComparison.OrdinalIgnoreCase)) { + var selectConflict = ReadInsertConflictClause(sqlWithoutReturning, ref restTrimmed); + int selectChanges = 0; var selectSql = restTrimmed; var selectParts = selectSql.Split(' ', StringSplitOptions.RemoveEmptyEntries); var sourceRows = ExecuteSelectQuery(selectSql, selectParts, noEncrypt: false); @@ -335,13 +343,18 @@ private void ExecuteInsert(string sql, IWAL? wal) foreach (var kv in sourceRow) newRow[kv.Key] = kv.Value; } + if (selectConflict is not null && ResolveInsertConflict(tableName, table, newRow, selectConflict, null, ref selectChanges)) + { + continue; + } + FireTriggers(tableName, TriggerTiming.Before, TriggerEvent.Insert, newRow: newRow); table.Insert(newRow); FireTriggers(tableName, TriggerTiming.After, TriggerEvent.Insert, newRow: newRow); insertedFromSelect.Add(new Dictionary(newRow, StringComparer.OrdinalIgnoreCase)); } - _lastChanges = insertedFromSelect.Count; - _totalChanges += insertedFromSelect.Count; + _lastChanges = insertedFromSelect.Count + selectChanges; + _totalChanges += insertedFromSelect.Count + selectChanges; if (insertedFromSelect.Count > 0) _lastInsertRowId++; if (returningColumns is not null) _pendingQueryResults = ProjectReturningRows(insertedFromSelect, returningColumns); // §2 instrumentation: the INSERT … SELECT branch's per-statement WAL call, stamped like the @@ -354,6 +367,7 @@ private void ExecuteInsert(string sql, IWAL? wal) // ────────────────────────────────────────────────────────────────── var valuesStart = rest.IndexOf("VALUES", StringComparison.OrdinalIgnoreCase) + "VALUES".Length; var valuesRest = rest[valuesStart..].Trim(); + var conflictClause = ReadInsertConflictClause(sqlWithoutReturning, ref valuesRest); // §2 instrumentation (2026-09-15): the statement text had no attribution on this path at all. This // covers the one-pass VALUES scan; the per-row, per-column literal conversion is RowBuild. long parseStart = Diagnostics.WritePathProfiler.Stamp(); @@ -496,16 +510,23 @@ Dictionary BuildRowFromValues(List rowValues) // any, which makes the statement atomic. Up to 2.1.4 it took the per-row loop, so a failing row // left the rows before it in place. bool useBatchedInsert = allRowValues.Count >= MinRowsForBatchedInsert + && conflictClause is null // a conflict clause decides row by row && (tableAsTable is { CanUseBatchedInsert: true } || table is SingleFileTable) && !HasTriggersFor(tableName, TriggerEvent.Insert); List>? batchedRows = useBatchedInsert ? new List>(allRowValues.Count) : null; + int conflictChanges = 0; foreach (var rowValues in allRowValues) { var row = BuildRowFromValues(rowValues); + if (conflictClause is not null && ResolveInsertConflict(tableName, table, row, conflictClause, returningRows, ref conflictChanges)) + { + continue; + } + if (batchedRows is not null) { batchedRows.Add(row); @@ -579,8 +600,8 @@ Dictionary BuildRowFromValues(List rowValues) } } } - _lastChanges = insertedCount; - _totalChanges += insertedCount; + _lastChanges = insertedCount + conflictChanges; + _totalChanges += insertedCount + conflictChanges; if (insertedCount > 0) { // SQLite compatibility: last_insert_rowid returns the first INTEGER column value @@ -3393,25 +3414,18 @@ private static List> ApplyOrderBy(List.Create(CompareValues); IOrderedEnumerable>? ordered = null; for (int i = 0; i < orderBy.Items.Count; i++) { var item = orderBy.Items[i]; var key = OrderKeySelector(item); - if (i == 0) - { - ordered = item.IsAscending - ? rows.OrderBy(key, comparer) - : rows.OrderByDescending(key, comparer); - } - else - { - ordered = item.IsAscending - ? ordered.ThenBy(key, comparer) - : ordered.ThenByDescending(key, comparer); - } + + // Direction, NULLS FIRST/LAST and COLLATE live in the item's comparer, so the sort itself is ascending. Text + // without a COLLATE compares as BINARY, SQLite's default; up to 2.1.4 this path compared it case-insensitively, + // so `ORDER BY name` over a derived table put 'a' before 'C'. + var itemComparer = SqlOrderBy.ItemComparer(item, SqlOrderBy.KeyComparer.Instance); + ordered = i == 0 ? rows.OrderBy(key, itemComparer) : ordered!.ThenBy(key, itemComparer); } return [.. ordered]; diff --git a/src/SharpCoreDB/Services/SqlParser.Upsert.cs b/src/SharpCoreDB/Services/SqlParser.Upsert.cs new file mode 100644 index 00000000..7a1e612c --- /dev/null +++ b/src/SharpCoreDB/Services/SqlParser.Upsert.cs @@ -0,0 +1,280 @@ +// +// Copyright (c) 2026 MPCoreDeveloper. All rights reserved. +// Licensed under the MIT License. See LICENSE file in the project root for full license information. +// + +namespace SharpCoreDB.Services; + +using System; +using System.Collections.Generic; +using System.Text.RegularExpressions; +using SharpCoreDB.DataStructures; +using SharpCoreDB.Interfaces; + +/// +/// SqlParser partial — SQLite's INSERT conflict clauses: INSERT OR REPLACE|IGNORE|ABORT|FAIL|ROLLBACK, +/// REPLACE INTO and the upsert clause ON CONFLICT [(cols)] DO NOTHING | DO UPDATE SET … [WHERE …]. +/// +/// Up to 2.1.4 none of them ran on the SQL path: INSERT OR … failed with "Unsupported SQL statement", and an +/// ON CONFLICT clause was ignored, so an upsert onto an existing key failed as a duplicate. The conflict lookups +/// and handlers in SqlParser.DML.InsertConflict existed but nothing called them. A DO UPDATE assignment is +/// an expression over the existing row, with excluded.col the value the INSERT tried to write. +/// +/// +public partial class SqlParser +{ + private static readonly Regex InsertOrPolicyRegex = new( + @"^\s*INSERT\s+OR\s+(REPLACE|IGNORE|ABORT|FAIL|ROLLBACK)\b", + RegexOptions.IgnoreCase | RegexOptions.CultureInvariant | RegexOptions.Compiled, + TimeSpan.FromSeconds(1)); + + private static readonly Regex OnConflictRegex = new( + @"^ON\s+CONFLICT\s*(?:\((?[^)]*)\))?\s*DO\s+(?:(?NOTHING)|UPDATE\s+SET\s+(?.+))\s*;?\s*$", + RegexOptions.IgnoreCase | RegexOptions.CultureInvariant | RegexOptions.Singleline | RegexOptions.Compiled, + TimeSpan.FromSeconds(1)); + + /// An INSERT's conflict handling: its OR policy and its ON CONFLICT clause. + private sealed class InsertConflictClause + { + public InsertConflictPolicy Policy { get; init; } + + public InsertOnConflictAction Action { get; init; } + + public List Targets { get; init; } = []; + + public List> Assignments { get; init; } = []; + + public string? Where { get; init; } + } + + /// + /// Reads the statement's INSERT OR … policy and cuts a trailing ON CONFLICT … clause off + /// (the text after VALUES, or the SELECT). Returns null when the statement has + /// neither, which keeps every other INSERT on its existing paths. + /// + private static InsertConflictClause? ReadInsertConflictClause(string statement, ref string valuesText) + { + var policy = InsertConflictPolicy.None; + var policyMatch = InsertOrPolicyRegex.Match(statement); + if (policyMatch.Success) + { + policy = policyMatch.Groups[1].Value.ToUpperInvariant() switch + { + "REPLACE" => InsertConflictPolicy.Replace, + "IGNORE" => InsertConflictPolicy.Ignore, + "FAIL" => InsertConflictPolicy.Fail, + _ => InsertConflictPolicy.Abort, // ABORT, and ROLLBACK (no statement-level transaction here) + }; + } + + int onAt = FindTopLevelOnConflict(valuesText); + if (onAt < 0) + { + return policy == InsertConflictPolicy.None ? null : new InsertConflictClause { Policy = policy }; + } + + var clauseText = valuesText[onAt..].Trim(); + valuesText = valuesText[..onAt].TrimEnd(); + var match = OnConflictRegex.Match(clauseText); + if (!match.Success) + { + throw new InvalidOperationException($"Unrecognised ON CONFLICT clause: {clauseText}"); + } + + var targets = new List(); + if (match.Groups["targets"].Success) + { + foreach (var target in SqlInPredicate.SplitTopLevelComma(match.Groups["targets"].Value)) + { + if (!string.IsNullOrWhiteSpace(target)) + { + targets.Add(SqlIdentifierScanner.IdentifierText(target.Trim())); + } + } + } + + if (match.Groups["nothing"].Success) + { + return new InsertConflictClause { Policy = policy, Action = InsertOnConflictAction.DoNothing, Targets = targets }; + } + + var (setList, where) = SplitUpdateSetAndWhere(match.Groups["set"].Value); + var assignments = new List>(); + foreach (var assignment in SqlInPredicate.SplitTopLevelComma(setList)) + { + int eq = assignment.IndexOf('='); + if (eq <= 0) + { + throw new InvalidOperationException($"Invalid DO UPDATE assignment: {assignment}"); + } + + assignments.Add(new(SqlIdentifierScanner.IdentifierText(assignment[..eq].Trim()), assignment[(eq + 1)..].Trim())); + } + + return new InsertConflictClause + { + Policy = policy, + Action = InsertOnConflictAction.DoUpdate, + Targets = targets, + Assignments = assignments, + Where = where.Length == 0 ? null : where, + }; + } + + /// The position of a top-level ON CONFLICT (outside quotes and parentheses), or -1. + private static int FindTopLevelOnConflict(string text) + { + int depth = 0; + for (int i = 0; i < text.Length; i++) + { + char c = text[i]; + if (c is '\'' or '"' or '`' or '[') + { + char close = c == '[' ? ']' : c; + for (i++; i < text.Length; i++) + { + if (text[i] != close) + { + continue; + } + + if (close != ']' && i + 1 < text.Length && text[i + 1] == close) + { + i++; + continue; + } + + break; + } + + continue; + } + + if (c == '(') + { + depth++; + } + else if (c == ')') + { + depth--; + } + else if (depth == 0 && (c is 'O' or 'o') && (i == 0 || char.IsWhiteSpace(text[i - 1]) || text[i - 1] == ')') && + Regex.IsMatch(text[i..], @"^ON\s+CONFLICT\b", RegexOptions.IgnoreCase, TimeSpan.FromSeconds(1))) + { + return i; + } + } + + return -1; + } + + /// + /// Applies the conflict clause to one row about to be inserted. Returns true when the row was handled and must + /// not be inserted (skipped, or merged into the existing row by DO UPDATE); false to insert it — no conflict, + /// or REPLACE removed the conflicting rows. ABORT/FAIL, and a conflict the clause does not cover, raise the error. + /// + private bool ResolveInsertConflict( + string tableName, + ITable table, + Dictionary row, + InsertConflictClause clause, + List>? returningRows, + ref int changes) + { + if (!TryDetectInsertConflict(table, row, clause.Targets, out var conflictingKey, out var message)) + { + return false; + } + + switch (clause.Action) + { + case InsertOnConflictAction.DoNothing: + return true; + case InsertOnConflictAction.DoUpdate when conflictingKey is not null: + if (ApplyDoUpdate(table, conflictingKey, row, clause)) + { + changes++; + if (returningRows is not null && table.FindByPrimaryKey(conflictingKey) is { } updated) + { + returningRows.Add(new Dictionary(updated, StringComparer.OrdinalIgnoreCase)); + } + } + + return true; + } + + switch (clause.Policy) + { + case InsertConflictPolicy.Ignore: + return true; + case InsertConflictPolicy.Replace when conflictingKey is not null: + // Every row the new one conflicts with is deleted (a PRIMARY KEY and a UNIQUE conflict can be two rows). + for (int guard = 0; guard < 64 && conflictingKey is not null; guard++) + { + if (!table.DeleteByPrimaryKey(conflictingKey)) + { + break; + } + + TryDetectInsertConflict(table, row, [], out conflictingKey, out _); + } + + return false; + default: + throw new InvalidOperationException(string.IsNullOrEmpty(message) ? "UNIQUE constraint failed" : message); + } + } + + /// + /// Runs a DO UPDATE on the conflicting row: each assignment is an expression over the existing row, with + /// excluded.col the incoming value; the optional WHERE decides whether the row is updated at all. + /// + private static bool ApplyDoUpdate(ITable table, object conflictingKey, Dictionary incoming, InsertConflictClause clause) + { + var existing = table.FindByPrimaryKey(conflictingKey); + if (existing is null) + { + return false; + } + + var scope = new Dictionary(existing, StringComparer.OrdinalIgnoreCase); + foreach (var (column, value) in incoming) + { + scope["excluded." + column] = value; + } + + if (clause.Where is { } where && + (!SqlScalarExpression.TryEvaluateFilter(where, scope, out var matches) ? !EvaluateDoUpdateWhere(where, existing, incoming, table) : !matches)) + { + return false; + } + + var updates = new Dictionary(); + foreach (var (column, expression) in clause.Assignments) + { + int index = table.Columns.FindIndex(c => c.Equals(column, StringComparison.OrdinalIgnoreCase)); + if (index < 0) + { + throw new InvalidOperationException($"Column '{column}' does not exist (ON CONFLICT DO UPDATE)."); + } + + var type = table.ColumnTypes[index]; + updates[table.Columns[index]] = (SqlScalarExpression.TryParse(expression, out var node) + ? ConvertComputedSetValue(SqlScalarExpression.Evaluate(node, scope), type) + : ParseSetLiteral(expression, type)) ?? DBNull.Value; + } + + if (updates.Count == 0) + { + return false; + } + + if (table is Table concrete) + { + return concrete.UpdateByPrimaryKey(conflictingKey, updates); + } + + var pkColumn = table.Columns[table.PrimaryKeyIndex]; + return table.UpdateAffectedCount(Table.BuildStructuredWhereText(pkColumn, conflictingKey), updates) > 0; + } +} diff --git a/src/SharpCoreDB/Services/SqlScalarExpression.cs b/src/SharpCoreDB/Services/SqlScalarExpression.cs index 516757cb..87b985b2 100644 --- a/src/SharpCoreDB/Services/SqlScalarExpression.cs +++ b/src/SharpCoreDB/Services/SqlScalarExpression.cs @@ -110,6 +110,17 @@ internal static bool TryEvaluateCheck(string expression, Dictionary + /// Evaluates a filter condition (WHERE semantics: only a true result matches; NULL does not). Returns false — + /// "not mine" — when the condition uses syntax this evaluator does not cover. + /// + internal static bool TryEvaluateFilter(string condition, Dictionary row, out bool matches) + { + var compiled = GetOrAdd(CheckCache, condition, static c => ConditionCompiler.CompileAny(c)); + matches = compiled is not null && compiled.Evaluate(row) == true; + return compiled is not null; + } + /// /// Returns true when the expression reads a column for which holds, or calls a /// function or CAST — that is, when its value can differ from row to row. Constant arithmetic such as diff --git a/tests/SharpCoreDB.Tests/BatchAtomicityTests.cs b/tests/SharpCoreDB.Tests/BatchAtomicityTests.cs new file mode 100644 index 00000000..50399f4c --- /dev/null +++ b/tests/SharpCoreDB.Tests/BatchAtomicityTests.cs @@ -0,0 +1,140 @@ +// +// Copyright (c) 2026 MPCoreDeveloper. All rights reserved. +// Licensed under the MIT License. See LICENSE file in the project root for full license information. +// + +namespace SharpCoreDB.Tests; + +using System; +using System.Globalization; +using System.IO; +using System.Linq; +using Microsoft.Extensions.DependencyInjection; +using SharpCoreDB.Interfaces; +using Xunit; + +/// +/// ExecuteBatchSQL is one atomic unit: a statement that fails leaves every table as it was before the batch. +/// +/// Up to 2.1.4 two paths broke that. A batch with a SELECT, or one mixing statement kinds, ran its statements one at a +/// time without a transaction, so the statements before a failure stayed (on every storage mode). And the grouped +/// executor's own transaction rolled back the storage buffer only, which PageBased tables do not write through, so a +/// PageBased table kept the batch's earlier changes. +/// +/// +public sealed class BatchAtomicityTests : IDisposable +{ + private readonly DatabaseFactory _factory; + private readonly string _root = Path.Combine(Path.GetTempPath(), $"SCDB_BatchAtomic_{Guid.NewGuid():N}"); + + public BatchAtomicityTests() + { + var services = new ServiceCollection(); + services.AddSharpCoreDB(); + _factory = services.BuildServiceProvider().GetRequiredService(); + Directory.CreateDirectory(_root); + } + + public static TheoryData Modes => new() { "columnar", "page-based" }; + + public void Dispose() + { + try + { + if (Directory.Exists(_root)) + { + Directory.Delete(_root, true); + } + } + catch (IOException) + { + // Best-effort cleanup of a temp directory. + } + } + + private IDatabase Open(string mode, string ddl) + { + var db = _factory.Create(Path.Combine(_root, $"db_{Guid.NewGuid():N}"), "pw"); + db.ExecuteSQL(mode == "page-based" ? ddl + " STORAGE = PAGE_BASED" : ddl); + return db; + } + + private static string[] Rows(IDatabase db, string sql) => + [.. db.ExecuteQuery(sql).Select(r => string.Join(",", r.Values.Select(v => v is null or DBNull ? "NULL" : Convert.ToString(v, CultureInfo.InvariantCulture))))]; + + private IDatabase Seed(string mode) + { + var db = Open(mode, "CREATE TABLE t (id INTEGER PRIMARY KEY, n INTEGER CHECK (n < 10), name TEXT)"); + db.ExecuteSQL("INSERT INTO t (id, n, name) VALUES (1, 1, 'a'), (2, 2, 'b')"); + return db; + } + + [Theory] + [MemberData(nameof(Modes))] + public void UpdateBatch_FailingOnItsSecondStatement_ChangesNothing(string mode) + { + var db = Seed(mode); + + Assert.ThrowsAny(() => db.ExecuteBatchSQL(["UPDATE t SET n = 5 WHERE id = 1", "UPDATE t SET n = 50 WHERE id = 2"])); + + Assert.Equal(["1,1", "2,2"], Rows(db, "SELECT id, n FROM t").Order()); + } + + [Theory] + [MemberData(nameof(Modes))] + public void MixedBatch_FailingOnItsLastStatement_ChangesNothing(string mode) + { + var db = Seed(mode); + + Assert.ThrowsAny(() => db.ExecuteBatchSQL([ + "UPDATE t SET n = 6 WHERE id = 1", + "INSERT INTO t (id, n, name) VALUES (3, 3, 'c')", + "DELETE FROM t WHERE id = 2", + "UPDATE t SET n = n + 100 WHERE id = 1"])); + + Assert.Equal(["1,1", "2,2"], Rows(db, "SELECT id, n FROM t").Order()); + } + + [Theory] + [MemberData(nameof(Modes))] + public void BatchWithASelect_FailingAfterAWrite_ChangesNothing(string mode) + { + var db = Seed(mode); + + Assert.ThrowsAny(() => db.ExecuteBatchSQL([ + "UPDATE t SET n = 7 WHERE id = 1", + "SELECT id FROM t", + "INSERT INTO t (id, n, name) VALUES (1, 1, 'duplicate key')"])); + + Assert.Equal(["1,1", "2,2"], Rows(db, "SELECT id, n FROM t").Order()); + } + + [Theory] + [MemberData(nameof(Modes))] + public void MixedBatch_WithoutAFailure_AppliesEveryStatementInOrder(string mode) + { + var db = Seed(mode); + + db.ExecuteBatchSQL([ + "DELETE FROM t WHERE id = 2", + "INSERT INTO t (id, n, name) VALUES (2, 9, 'again')", + "UPDATE t SET n = n + 1 WHERE id = 1", + "SELECT id FROM t"]); + + Assert.Equal(["1,2", "2,9"], Rows(db, "SELECT id, n FROM t").Order()); + } + + [Theory] + [MemberData(nameof(Modes))] + public void FailingBatch_InsideAnOuterTransaction_IsUndoneByTheOuterRollback(string mode) + { + var db = Seed(mode); + + db.BeginBatchUpdate(); + db.ExecuteSQL("UPDATE t SET n = 3 WHERE id = 2"); + Assert.ThrowsAny(() => db.ExecuteBatchSQL(["UPDATE t SET n = 4 WHERE id = 1", "UPDATE t SET n = 40 WHERE id = 1"])); + db.CancelBatchUpdate(); + + Assert.Equal(["1,1", "2,2"], Rows(db, "SELECT id, n FROM t").Order()); + } +} diff --git a/tests/SharpCoreDB.Tests/BatchUpdateClassificationTests.cs b/tests/SharpCoreDB.Tests/BatchUpdateClassificationTests.cs index 5e7722b7..36649b69 100644 --- a/tests/SharpCoreDB.Tests/BatchUpdateClassificationTests.cs +++ b/tests/SharpCoreDB.Tests/BatchUpdateClassificationTests.cs @@ -78,6 +78,26 @@ public void BatchUpdate_QuotedSetColumn_UpdatesTheRowsAndDeletesNone(string mode Assert.Equal(["1,5,ann", "2,2,z"], Rows(db, "SELECT id, n, name FROM t").Order()); } + [Theory] + [MemberData(nameof(Modes))] + public void BatchUpdate_QuotedSetColumn_TakesTheStructuredFastPath(string mode) + { + // Up to 2.1.4 a quoted SET column was looked up with its quotes, missed, and the statement left the fast path. + var (db, _) = Open(mode, "CREATE TABLE t (id INTEGER PRIMARY KEY, n INTEGER)"); + db.ExecuteSQL("INSERT INTO t (id, n) VALUES (1, 1), (2, 2)"); + db.TryGetTable("t", out var table); + var concrete = (SharpCoreDB.DataStructures.Table)table; + + // The fast path counts its operations (the columnar fixed-width route as bulk batches); the shared parser's + // UPDATE path, where a declined statement goes, counts neither. + long before = concrete.StructuredUpdateOperations + concrete.BulkContiguousUpdateBatches; + + db.ExecuteBatchSQL(["UPDATE t SET \"n\" = 5 WHERE id = 1", "UPDATE t SET [n] = 6 WHERE id = 2"]); + + Assert.True(concrete.StructuredUpdateOperations + concrete.BulkContiguousUpdateBatches > before); + Assert.Equal(["1,5", "2,6"], Rows(db, "SELECT id, n FROM t").Order()); + } + [Theory] [MemberData(nameof(Modes))] public void BatchUpdate_QuotedSetColumn_IsKeptAfterReopen(string mode) diff --git a/tests/SharpCoreDB.Tests/SubqueryOrderByCollationTests.cs b/tests/SharpCoreDB.Tests/SubqueryOrderByCollationTests.cs new file mode 100644 index 00000000..3269aaa6 --- /dev/null +++ b/tests/SharpCoreDB.Tests/SubqueryOrderByCollationTests.cs @@ -0,0 +1,118 @@ +// +// Copyright (c) 2026 MPCoreDeveloper. All rights reserved. +// Licensed under the MIT License. See LICENSE file in the project root for full license information. +// + +namespace SharpCoreDB.Tests; + +using System; +using System.Globalization; +using System.IO; +using System.Linq; +using Microsoft.Extensions.DependencyInjection; +using SharpCoreDB.Interfaces; +using Xunit; + +/// +/// ORDER BY on the AST path (derived tables, IN-subqueries) honours COLLATE and NULLS FIRST|LAST, as the +/// single-table and JOIN paths do. +/// +/// Up to 2.1.4 the AST parser read only ASC/DESC after a key: COLLATE NOCASE was left as unparsed trailing content +/// — and took the DESC after it along — and NULLS LAST was ignored; a descending sort also moved NULLs to the end +/// regardless of what the statement said. +/// +/// +public sealed class SubqueryOrderByCollationTests : IDisposable +{ + private readonly DatabaseFactory _factory; + private readonly string _root = Path.Combine(Path.GetTempPath(), $"SCDB_SubOrder_{Guid.NewGuid():N}"); + + public SubqueryOrderByCollationTests() + { + var services = new ServiceCollection(); + services.AddSharpCoreDB(); + _factory = services.BuildServiceProvider().GetRequiredService(); + Directory.CreateDirectory(_root); + } + + public static TheoryData Modes => new() { "columnar", "page-based" }; + + public void Dispose() + { + try + { + if (Directory.Exists(_root)) + { + Directory.Delete(_root, true); + } + } + catch (IOException) + { + // Best-effort cleanup of a temp directory. + } + } + + private IDatabase Open(string mode, string ddl) + { + var db = _factory.Create(Path.Combine(_root, $"db_{Guid.NewGuid():N}"), "pw"); + db.ExecuteSQL(mode == "page-based" ? ddl + " STORAGE = PAGE_BASED" : ddl); + return db; + } + + private static string[] Rows(IDatabase db, string sql) => + [.. db.ExecuteQuery(sql).Select(r => string.Join(",", r.Values.Select(v => v is null or DBNull ? "NULL" : Convert.ToString(v, CultureInfo.InvariantCulture))))]; + + private IDatabase Seed() + { + var db = Open("columnar", "CREATE TABLE p (id INTEGER PRIMARY KEY, name TEXT, score INTEGER)"); + db.ExecuteSQL("INSERT INTO p (id, name, score) VALUES (1, 'b', 5), (2, 'A', NULL), (3, 'C', 1), (4, 'a', 9)"); + return db; + } + + private static string[] Names(IDatabase db, string sql) => + [.. db.ExecuteQuery(sql).Select(r => Convert.ToString(r["name"], CultureInfo.InvariantCulture)!)]; + + [Fact] + public void DerivedTable_OrderByTextWithoutCollate_SortsAsBinary() + { + // SQLite's default collation is BINARY: upper case sorts before lower case. The AST path compared text + // case-insensitively, so 'a' came before 'C'. + var db = Seed(); + Assert.Equal(["A", "C", "a", "b"], Names(db, "SELECT name FROM (SELECT name FROM p) ORDER BY name")); + } + + [Fact] + public void DerivedTable_OrderByCollateNoCase_SortsCaseInsensitively() + { + var db = Seed(); + Assert.Equal(["A", "a", "b", "C"], Names(db, "SELECT name FROM (SELECT name, id FROM p) ORDER BY name COLLATE NOCASE, id")); + } + + [Fact] + public void InSubquery_OrderByCollateNoCaseDescending_KeepsTheDirection() + { + var db = Seed(); + Assert.Equal(["C", "b", "A", "a"], Names(db, "SELECT name FROM p WHERE id IN (SELECT id FROM p) ORDER BY name COLLATE NOCASE DESC, id")); + } + + [Fact] + public void DerivedTable_OrderByNullsLast_PutsNullsAtTheEnd() + { + var db = Seed(); + Assert.Equal(["C", "b", "a", "A"], Names(db, "SELECT name FROM (SELECT name, score FROM p) ORDER BY score NULLS LAST")); + } + + [Fact] + public void DerivedTable_OrderByDescendingNullsFirst_PutsNullsFirst() + { + var db = Seed(); + Assert.Equal(["A", "a", "b", "C"], Names(db, "SELECT name FROM (SELECT name, score FROM p) ORDER BY score DESC NULLS FIRST")); + } + + [Fact] + public void DerivedTable_OrderByDescending_PutsNullsLastByDefault() + { + var db = Seed(); + Assert.Equal(["a", "b", "C", "A"], Names(db, "SELECT name FROM (SELECT name, score FROM p) ORDER BY score DESC")); + } +} diff --git a/tests/SharpCoreDB.Tests/UpsertTests.cs b/tests/SharpCoreDB.Tests/UpsertTests.cs new file mode 100644 index 00000000..af6bef09 --- /dev/null +++ b/tests/SharpCoreDB.Tests/UpsertTests.cs @@ -0,0 +1,174 @@ +// +// Copyright (c) 2026 MPCoreDeveloper. All rights reserved. +// Licensed under the MIT License. See LICENSE file in the project root for full license information. +// + +namespace SharpCoreDB.Tests; + +using System; +using System.Globalization; +using System.IO; +using System.Linq; +using Microsoft.Extensions.DependencyInjection; +using SharpCoreDB.Interfaces; +using Xunit; + +/// +/// SQLite's INSERT conflict clauses: the upsert ON CONFLICT [(cols)] DO NOTHING | DO UPDATE SET … [WHERE …], +/// INSERT OR IGNORE|REPLACE|ABORT|FAIL and REPLACE INTO, on single-column and composite keys. +/// +/// Up to 2.1.4 none of them ran on the SQL path: INSERT OR … failed with "Unsupported SQL statement" and an +/// ON CONFLICT clause was ignored, so an upsert onto an existing key failed as a duplicate — the conflict handlers +/// existed but nothing called them. The composite lookup compared values without the column's collation, and the "same +/// row" check was always case-insensitive. +/// +/// +public sealed class UpsertTests : IDisposable +{ + private readonly DatabaseFactory _factory; + private readonly string _root = Path.Combine(Path.GetTempPath(), $"SCDB_Upsert_{Guid.NewGuid():N}"); + + public UpsertTests() + { + var services = new ServiceCollection(); + services.AddSharpCoreDB(); + _factory = services.BuildServiceProvider().GetRequiredService(); + Directory.CreateDirectory(_root); + } + + public static TheoryData Modes => new() { "columnar", "page-based" }; + + public void Dispose() + { + try + { + if (Directory.Exists(_root)) + { + Directory.Delete(_root, true); + } + } + catch (IOException) + { + // Best-effort cleanup of a temp directory. + } + } + + private IDatabase Open(string mode, string ddl) + { + var db = _factory.Create(Path.Combine(_root, $"db_{Guid.NewGuid():N}"), "pw"); + db.ExecuteSQL(mode == "page-based" ? ddl + " STORAGE = PAGE_BASED" : ddl); + return db; + } + + private static string[] Rows(IDatabase db, string sql) => + [.. db.ExecuteQuery(sql).Select(r => string.Join(",", r.Values.Select(v => v is null or DBNull ? "NULL" : Convert.ToString(v, CultureInfo.InvariantCulture))))]; + + [Theory] + [MemberData(nameof(Modes))] + public void OnConflictDoUpdate_SingleColumnKey_MergesIntoTheExistingRow(string mode) + { + var db = Open(mode, "CREATE TABLE s (code TEXT PRIMARY KEY, n INTEGER)"); + db.ExecuteSQL("INSERT INTO s (code, n) VALUES ('a', 1)"); + + db.ExecuteSQL("INSERT INTO s (code, n) VALUES ('a', 5) ON CONFLICT (code) DO UPDATE SET n = n + excluded.n"); + db.ExecuteSQL("INSERT INTO s (code, n) VALUES ('b', 7) ON CONFLICT (code) DO UPDATE SET n = n + excluded.n"); + + Assert.Equal(["a,6", "b,7"], Rows(db, "SELECT code, n FROM s").Order()); + } + + [Theory] + [MemberData(nameof(Modes))] + public void OnConflictDoUpdate_WithAFalseWhere_LeavesTheRow(string mode) + { + var db = Open(mode, "CREATE TABLE s (code TEXT PRIMARY KEY, n INTEGER)"); + db.ExecuteSQL("INSERT INTO s (code, n) VALUES ('a', 1)"); + + db.ExecuteSQL("INSERT INTO s (code, n) VALUES ('a', 2) ON CONFLICT (code) DO UPDATE SET n = excluded.n WHERE excluded.n > 3"); + Assert.Equal(["a,1"], Rows(db, "SELECT code, n FROM s")); + + db.ExecuteSQL("INSERT INTO s (code, n) VALUES ('a', 9) ON CONFLICT (code) DO UPDATE SET n = excluded.n WHERE excluded.n > 3"); + Assert.Equal(["a,9"], Rows(db, "SELECT code, n FROM s")); + } + + [Theory] + [MemberData(nameof(Modes))] + public void OnConflict_CompositeKey_DoUpdateAndDoNothing(string mode) + { + var db = Open(mode, "CREATE TABLE u (a INTEGER NOT NULL, b INTEGER NOT NULL, v TEXT, PRIMARY KEY (a, b))"); + db.ExecuteSQL("INSERT INTO u (a, b, v) VALUES (1, 1, 'x'), (1, 2, 'y')"); + + db.ExecuteSQL("INSERT INTO u (a, b, v) VALUES (1, 1, 'new') ON CONFLICT (a, b) DO UPDATE SET v = excluded.v"); + db.ExecuteSQL("INSERT INTO u (a, b, v) VALUES (1, 2, 'kept?') ON CONFLICT (a, b) DO NOTHING"); + db.ExecuteSQL("INSERT INTO u (a, b, v) VALUES (2, 2, 'z') ON CONFLICT DO NOTHING"); + + Assert.Equal(["1,1,new", "1,2,y", "2,2,z"], Rows(db, "SELECT a, b, v FROM u").Order()); + } + + [Theory] + [MemberData(nameof(Modes))] + public void InsertOrIgnore_AndInsertOrReplace_AndReplaceInto(string mode) + { + var db = Open(mode, "CREATE TABLE s (code TEXT PRIMARY KEY, n INTEGER)"); + db.ExecuteSQL("INSERT INTO s (code, n) VALUES ('a', 1), ('b', 2)"); + + db.ExecuteSQL("INSERT OR IGNORE INTO s (code, n) VALUES ('a', 100), ('c', 3)"); + db.ExecuteSQL("INSERT OR REPLACE INTO s (code, n) VALUES ('b', 20)"); + db.ExecuteSQL("REPLACE INTO s (code, n) VALUES ('c', 30)"); + + Assert.Equal(["a,1", "b,20", "c,30"], Rows(db, "SELECT code, n FROM s").Order()); + } + + [Theory] + [MemberData(nameof(Modes))] + public void InsertOrAbort_OnAConflict_Throws(string mode) + { + var db = Open(mode, "CREATE TABLE s (code TEXT PRIMARY KEY, n INTEGER)"); + db.ExecuteSQL("INSERT INTO s (code, n) VALUES ('a', 1)"); + + Assert.ThrowsAny(() => db.ExecuteSQL("INSERT OR ABORT INTO s (code, n) VALUES ('a', 2)")); + Assert.Equal(["a,1"], Rows(db, "SELECT code, n FROM s")); + } + + [Theory] + [MemberData(nameof(Modes))] + public void OnConflict_CompositeUnique_RespectsTheColumnCollation(string mode) + { + var db = Open(mode, "CREATE TABLE n (id INTEGER PRIMARY KEY, a TEXT COLLATE NOCASE, b TEXT, v TEXT, UNIQUE (a, b))"); + db.ExecuteSQL("INSERT INTO n (id, a, b, v) VALUES (1, 'abc', 'k', 'x')"); + + // NOCASE column: 'ABC' is the same key, so the existing row is updated. + db.ExecuteSQL("INSERT INTO n (id, a, b, v) VALUES (2, 'ABC', 'k', 'merged') ON CONFLICT (a, b) DO UPDATE SET v = excluded.v"); + // BINARY column: 'K' is a different key, so the row is inserted. + db.ExecuteSQL("INSERT INTO n (id, a, b, v) VALUES (3, 'abc', 'K', 'new') ON CONFLICT (a, b) DO NOTHING"); + + Assert.Equal(["1,abc,k,merged", "3,abc,K,new"], Rows(db, "SELECT id, a, b, v FROM n").Order()); + } + + [Theory] + [MemberData(nameof(Modes))] + public void InsertOrIgnore_FromASelect_SkipsExistingKeys(string mode) + { + var db = Open(mode, "CREATE TABLE s (code TEXT PRIMARY KEY, n INTEGER)"); + db.ExecuteSQL("CREATE TABLE src (code TEXT PRIMARY KEY, n INTEGER)"); + db.ExecuteSQL("INSERT INTO s (code, n) VALUES ('a', 1)"); + db.ExecuteSQL("INSERT INTO src (code, n) VALUES ('a', 100), ('b', 2)"); + + db.ExecuteSQL("INSERT OR IGNORE INTO s (code, n) SELECT code, n FROM src"); + + Assert.Equal(["a,1", "b,2"], Rows(db, "SELECT code, n FROM s").Order()); + } + + [Theory] + [MemberData(nameof(Modes))] + public void Upsert_InABatch_IsApplied(string mode) + { + var db = Open(mode, "CREATE TABLE s (code TEXT PRIMARY KEY, n INTEGER)"); + db.ExecuteSQL("INSERT INTO s (code, n) VALUES ('a', 1)"); + + db.ExecuteBatchSQL([ + "INSERT INTO s (code, n) VALUES ('a', 100) ON CONFLICT (code) DO UPDATE SET n = excluded.n", + "INSERT INTO s (code, n) VALUES ('b', 1) ON CONFLICT (code) DO NOTHING"]); + + Assert.Equal(["a,100", "b,1"], Rows(db, "SELECT code, n FROM s").Order()); + } +} From dd12ce81f84e0c0f0a0a9f22ec5d7957fb8b4eb0 Mon Sep 17 00:00:00 2001 From: Andrew Tierney Date: Tue, 17 Nov 2026 11:32:22 +1100 Subject: [PATCH 2/2] refactor: one top-level scanner for the SET/WHERE, ON CONFLICT and ORDER BY splitters (SonarCloud duplication) SqlLiteralScanner.FindTopLevel / IsKeywordAt replace three copies of the same quote- and parenthesis-aware loop (SplitUpdateSetAndWhere, the ON CONFLICT finder, FindOrderItemEnd), which SonarCloud's quality gate failed on as duplicated new code (3.6% > 3%). Also clears the flagged smells: the conflict-clause reader and resolver split into smaller methods (ReadInsertOrPolicy, ReadDoUpdateAssignments, DeleteConflictingRows), ResolveInsertConflict made static without its unused table-name parameter, and nested ternaries in SqlOrderBy / ParseOrderItemSuffix rewritten. No behaviour change. --- .../Services/EnhancedSqlParser.Select.cs | 54 ++----- src/SharpCoreDB/Services/SqlLiteralScanner.cs | 67 +++++++++ src/SharpCoreDB/Services/SqlOrderBy.cs | 49 +++++-- src/SharpCoreDB/Services/SqlParser.DML.cs | 49 +------ src/SharpCoreDB/Services/SqlParser.Upsert.cs | 138 +++++++----------- 5 files changed, 169 insertions(+), 188 deletions(-) diff --git a/src/SharpCoreDB/Services/EnhancedSqlParser.Select.cs b/src/SharpCoreDB/Services/EnhancedSqlParser.Select.cs index 9d12da7e..4bb7cd18 100644 --- a/src/SharpCoreDB/Services/EnhancedSqlParser.Select.cs +++ b/src/SharpCoreDB/Services/EnhancedSqlParser.Select.cs @@ -997,7 +997,14 @@ private void ParseOrderItemSuffix(OrderByItem item) MatchKeyword("ASC"); if (MatchKeyword("NULLS")) { - item.NullsFirst = MatchKeyword("FIRST") ? true : MatchKeyword("LAST") ? false : null; + if (MatchKeyword("FIRST")) + { + item.NullsFirst = true; + } + else if (MatchKeyword("LAST")) + { + item.NullsFirst = false; + } } } @@ -1067,45 +1074,12 @@ private bool TryParseExpressionOrderItem(int itemStart, out OrderByItem item) /// that ends a subquery, LIMIT/OFFSET/a set operator, a semicolon or the end of the statement. private int FindOrderItemEnd(int start) { - int depth = 0; - for (int i = start; i < _sql.Length; i++) - { - char c = _sql[i]; - if (c is '\'' or '"' or '`' or '[') - { - char close = c == '[' ? ']' : c; - for (i++; i < _sql.Length && _sql[i] != close; i++) - { - } - - continue; - } - - if (c == '(') - { - depth++; - } - else if (c == ')') - { - if (depth == 0) - { - return i; - } - - depth--; - } - else if (depth == 0 && (c is ',' or ';')) - { - return i; - } - else if (depth == 0 && char.IsWhiteSpace(c) && - Regex.IsMatch(_sql[(i + 1)..], @"^(LIMIT|OFFSET|UNION|INTERSECT|EXCEPT)\b", RegexOptions.IgnoreCase, TimeSpan.FromSeconds(1))) - { - return i; - } - } - - return _sql.Length; + int end = SqlLiteralScanner.FindTopLevel(_sql, start, static (text, i) => + text[i] is ',' or ';' || + (char.IsWhiteSpace(text[i]) && + Regex.IsMatch(text[(i + 1)..], @"^(LIMIT|OFFSET|UNION|INTERSECT|EXCEPT)\b", RegexOptions.IgnoreCase, TimeSpan.FromSeconds(1))), + stopAtUnmatchedClose: true); + return end < 0 ? _sql.Length : end; } private OrderByNode ParseOrderBy() diff --git a/src/SharpCoreDB/Services/SqlLiteralScanner.cs b/src/SharpCoreDB/Services/SqlLiteralScanner.cs index 720112c3..8a0c2962 100644 --- a/src/SharpCoreDB/Services/SqlLiteralScanner.cs +++ b/src/SharpCoreDB/Services/SqlLiteralScanner.cs @@ -79,6 +79,73 @@ internal static int SkipLiteral(ReadOnlySpan text, int start) return closing < 0 ? -1 : closing + 1; } + /// + /// Finds the first position at parenthesis depth 0, outside quoted literals and quoted identifiers ('…', + /// "…", `…`, […]), where holds. With + /// , a closing parenthesis that has no opening one in the scanned text (the + /// end of an enclosing subquery) also ends the search there. Returns -1 when nothing matches. + /// + /// The text to scan. + /// The index to start at. + /// Tests one top-level position. + /// Whether an unmatched ) ends the search. + internal static int FindTopLevel(string text, int start, Func matchesAt, bool stopAtUnmatchedClose = false) + { + int depth = 0; + int i = start; + while (i < text.Length) + { + char c = text[i]; + if (c is '\'' or '"' or '`') + { + int after = SkipLiteral(text.AsSpan(), i); + i = after < 0 ? text.Length : after; + continue; + } + + if (c == '[') + { + int close = text.IndexOf(']', i + 1); + i = close < 0 ? text.Length : close + 1; + continue; + } + + if (c == '(') + { + depth++; + } + else if (c == ')') + { + if (depth == 0 && stopAtUnmatchedClose) + { + return i; + } + + depth--; + } + else if (depth == 0 && matchesAt(text, i)) + { + return i; + } + + i++; + } + + return -1; + } + + /// + /// Returns true when starts at as a whole word (no letter, + /// digit or underscore on either side). + /// + internal static bool IsKeywordAt(string text, int index, string keyword) => + index + keyword.Length <= text.Length && + (index == 0 || !IsWordChar(text[index - 1])) && + string.Compare(text, index, keyword, 0, keyword.Length, StringComparison.OrdinalIgnoreCase) == 0 && + (index + keyword.Length == text.Length || !IsWordChar(text[index + keyword.Length])); + + private static bool IsWordChar(char c) => char.IsLetterOrDigit(c) || c == '_'; + /// /// Splits a condition on whitespace that is not inside a quoted literal, a parenthesized group or a /// bracketed literal, returning the tokens with their text unaltered. diff --git a/src/SharpCoreDB/Services/SqlOrderBy.cs b/src/SharpCoreDB/Services/SqlOrderBy.cs index 088e8f78..dd011c8a 100644 --- a/src/SharpCoreDB/Services/SqlOrderBy.cs +++ b/src/SharpCoreDB/Services/SqlOrderBy.cs @@ -41,19 +41,39 @@ internal static class SqlOrderBy _ => null, }; - return Comparer.Create((x, y) => + return Comparer.Create((x, y) => CompareItemKeys(x, y, descending, nullsFirst, text, values)); + } + + private static int CompareItemKeys( + object? x, object? y, bool descending, bool nullsFirst, IComparer? text, IComparer values) + { + int nulls = CompareNulls(x, y, nullsFirst); + if (nulls != 2) { - bool xNull = x is null or DBNull; - bool yNull = y is null or DBNull; - if (xNull || yNull) - { - return xNull == yNull ? 0 : xNull == nullsFirst ? -1 : 1; - } + return nulls; + } - int c = text is not null && x is string xs && y is string ys ? text.Compare(xs, ys) : values.Compare(x, y); - c = Math.Sign(c); - return descending ? -c : c; - }); + int c = text is not null && x is string xs && y is string ys ? text.Compare(xs, ys) : values.Compare(x, y); + c = Math.Sign(c); + return descending ? -c : c; + } + + /// The order of two keys when either is NULL (-1, 0, 1), or 2 when neither is. + private static int CompareNulls(object? x, object? y, bool nullsFirst) + { + bool xNull = x is null or DBNull; + bool yNull = y is null or DBNull; + if (!xNull && !yNull) + { + return 2; + } + + if (xNull == yNull) + { + return 0; + } + + return xNull == nullsFirst ? -1 : 1; } /// @@ -65,11 +85,10 @@ internal sealed class KeyComparer : IComparer public int Compare(object? x, object? y) { - bool xNull = x is null or DBNull; - bool yNull = y is null or DBNull; - if (xNull || yNull) + int nulls = CompareNulls(x, y, nullsFirst: true); + if (nulls != 2) { - return xNull == yNull ? 0 : xNull ? -1 : 1; + return nulls; } return x is string xs && y is string ys diff --git a/src/SharpCoreDB/Services/SqlParser.DML.cs b/src/SharpCoreDB/Services/SqlParser.DML.cs index f68cb7c2..7ec78eb5 100644 --- a/src/SharpCoreDB/Services/SqlParser.DML.cs +++ b/src/SharpCoreDB/Services/SqlParser.DML.cs @@ -343,7 +343,7 @@ private void ExecuteInsert(string sql, IWAL? wal) foreach (var kv in sourceRow) newRow[kv.Key] = kv.Value; } - if (selectConflict is not null && ResolveInsertConflict(tableName, table, newRow, selectConflict, null, ref selectChanges)) + if (selectConflict is not null && ResolveInsertConflict(table, newRow, selectConflict, null, ref selectChanges)) { continue; } @@ -522,7 +522,7 @@ Dictionary BuildRowFromValues(List rowValues) { var row = BuildRowFromValues(rowValues); - if (conflictClause is not null && ResolveInsertConflict(tableName, table, row, conflictClause, returningRows, ref conflictChanges)) + if (conflictClause is not null && ResolveInsertConflict(table, row, conflictClause, returningRows, ref conflictChanges)) { continue; } @@ -2573,49 +2573,8 @@ private static List> ApplySetList( /// private static (string SetList, string Where) SplitUpdateSetAndWhere(string text) { - int depth = 0; - for (int i = 0; i < text.Length; i++) - { - char c = text[i]; - if (c is '\'' or '"' or '`' or '[') - { - char close = c == '[' ? ']' : c; - for (i++; i < text.Length; i++) - { - if (text[i] != close) - { - continue; - } - - if (close != ']' && i + 1 < text.Length && text[i + 1] == close) - { - i++; // a doubled quote is one quote of data - continue; - } - - break; - } - - continue; - } - - if (c == '(') - { - depth++; - } - else if (c == ')') - { - depth--; - } - else if (depth == 0 && (c is 'W' or 'w') && i > 0 && char.IsWhiteSpace(text[i - 1]) && - string.Compare(text, i, "WHERE", 0, 5, StringComparison.OrdinalIgnoreCase) == 0 && - (i + 5 == text.Length || char.IsWhiteSpace(text[i + 5]) || text[i + 5] == '(')) - { - return (text[..i].Trim(), text[(i + 5)..].Trim()); - } - } - - return (text.Trim(), string.Empty); + int at = SqlLiteralScanner.FindTopLevel(text, 0, static (t, i) => i > 0 && SqlLiteralScanner.IsKeywordAt(t, i, "WHERE")); + return at < 0 ? (text.Trim(), string.Empty) : (text[..at].Trim(), text[(at + 5)..].Trim()); } /// diff --git a/src/SharpCoreDB/Services/SqlParser.Upsert.cs b/src/SharpCoreDB/Services/SqlParser.Upsert.cs index 7a1e612c..b2c8de59 100644 --- a/src/SharpCoreDB/Services/SqlParser.Upsert.cs +++ b/src/SharpCoreDB/Services/SqlParser.Upsert.cs @@ -54,20 +54,10 @@ private sealed class InsertConflictClause /// private static InsertConflictClause? ReadInsertConflictClause(string statement, ref string valuesText) { - var policy = InsertConflictPolicy.None; - var policyMatch = InsertOrPolicyRegex.Match(statement); - if (policyMatch.Success) - { - policy = policyMatch.Groups[1].Value.ToUpperInvariant() switch - { - "REPLACE" => InsertConflictPolicy.Replace, - "IGNORE" => InsertConflictPolicy.Ignore, - "FAIL" => InsertConflictPolicy.Fail, - _ => InsertConflictPolicy.Abort, // ABORT, and ROLLBACK (no statement-level transaction here) - }; - } - - int onAt = FindTopLevelOnConflict(valuesText); + var policy = ReadInsertOrPolicy(statement); + int onAt = SqlLiteralScanner.FindTopLevel(valuesText, 0, static (t, i) => + SqlLiteralScanner.IsKeywordAt(t, i, "ON") && + Regex.IsMatch(t[i..], @"^ON\s+CONFLICT\b", RegexOptions.IgnoreCase, TimeSpan.FromSeconds(1))); if (onAt < 0) { return policy == InsertConflictPolicy.None ? null : new InsertConflictClause { Policy = policy }; @@ -81,17 +71,11 @@ private sealed class InsertConflictClause throw new InvalidOperationException($"Unrecognised ON CONFLICT clause: {clauseText}"); } - var targets = new List(); - if (match.Groups["targets"].Success) - { - foreach (var target in SqlInPredicate.SplitTopLevelComma(match.Groups["targets"].Value)) - { - if (!string.IsNullOrWhiteSpace(target)) - { - targets.Add(SqlIdentifierScanner.IdentifierText(target.Trim())); - } - } - } + List targets = match.Groups["targets"].Success + ? [.. SqlInPredicate.SplitTopLevelComma(match.Groups["targets"].Value) + .Where(static target => !string.IsNullOrWhiteSpace(target)) + .Select(static target => SqlIdentifierScanner.IdentifierText(target.Trim()))] + : []; if (match.Groups["nothing"].Success) { @@ -99,73 +83,50 @@ private sealed class InsertConflictClause } var (setList, where) = SplitUpdateSetAndWhere(match.Groups["set"].Value); - var assignments = new List>(); - foreach (var assignment in SqlInPredicate.SplitTopLevelComma(setList)) - { - int eq = assignment.IndexOf('='); - if (eq <= 0) - { - throw new InvalidOperationException($"Invalid DO UPDATE assignment: {assignment}"); - } - - assignments.Add(new(SqlIdentifierScanner.IdentifierText(assignment[..eq].Trim()), assignment[(eq + 1)..].Trim())); - } - return new InsertConflictClause { Policy = policy, Action = InsertOnConflictAction.DoUpdate, Targets = targets, - Assignments = assignments, + Assignments = ReadDoUpdateAssignments(setList), Where = where.Length == 0 ? null : where, }; } - /// The position of a top-level ON CONFLICT (outside quotes and parentheses), or -1. - private static int FindTopLevelOnConflict(string text) + /// The statement's INSERT OR … policy, or . + private static InsertConflictPolicy ReadInsertOrPolicy(string statement) { - int depth = 0; - for (int i = 0; i < text.Length; i++) + var policyMatch = InsertOrPolicyRegex.Match(statement); + if (!policyMatch.Success) { - char c = text[i]; - if (c is '\'' or '"' or '`' or '[') - { - char close = c == '[' ? ']' : c; - for (i++; i < text.Length; i++) - { - if (text[i] != close) - { - continue; - } - - if (close != ']' && i + 1 < text.Length && text[i + 1] == close) - { - i++; - continue; - } - - break; - } + return InsertConflictPolicy.None; + } - continue; - } + return policyMatch.Groups[1].Value.ToUpperInvariant() switch + { + "REPLACE" => InsertConflictPolicy.Replace, + "IGNORE" => InsertConflictPolicy.Ignore, + "FAIL" => InsertConflictPolicy.Fail, + _ => InsertConflictPolicy.Abort, // ABORT, and ROLLBACK (no statement-level transaction here) + }; + } - if (c == '(') - { - depth++; - } - else if (c == ')') - { - depth--; - } - else if (depth == 0 && (c is 'O' or 'o') && (i == 0 || char.IsWhiteSpace(text[i - 1]) || text[i - 1] == ')') && - Regex.IsMatch(text[i..], @"^ON\s+CONFLICT\b", RegexOptions.IgnoreCase, TimeSpan.FromSeconds(1))) + /// The col = expression pairs of a DO UPDATE SET list. + private static List> ReadDoUpdateAssignments(string setList) + { + var assignments = new List>(); + foreach (var assignment in SqlInPredicate.SplitTopLevelComma(setList)) + { + int eq = assignment.IndexOf('='); + if (eq <= 0) { - return i; + throw new InvalidOperationException($"Invalid DO UPDATE assignment: {assignment}"); } + + assignments.Add(new(SqlIdentifierScanner.IdentifierText(assignment[..eq].Trim()), assignment[(eq + 1)..].Trim())); } - return -1; + return assignments; } /// @@ -173,8 +134,7 @@ private static int FindTopLevelOnConflict(string text) /// not be inserted (skipped, or merged into the existing row by DO UPDATE); false to insert it — no conflict, /// or REPLACE removed the conflicting rows. ABORT/FAIL, and a conflict the clause does not cover, raise the error. /// - private bool ResolveInsertConflict( - string tableName, + private static bool ResolveInsertConflict( ITable table, Dictionary row, InsertConflictClause clause, @@ -208,23 +168,25 @@ private bool ResolveInsertConflict( case InsertConflictPolicy.Ignore: return true; case InsertConflictPolicy.Replace when conflictingKey is not null: - // Every row the new one conflicts with is deleted (a PRIMARY KEY and a UNIQUE conflict can be two rows). - for (int guard = 0; guard < 64 && conflictingKey is not null; guard++) - { - if (!table.DeleteByPrimaryKey(conflictingKey)) - { - break; - } - - TryDetectInsertConflict(table, row, [], out conflictingKey, out _); - } - + DeleteConflictingRows(table, row, conflictingKey); return false; default: throw new InvalidOperationException(string.IsNullOrEmpty(message) ? "UNIQUE constraint failed" : message); } } + /// + /// REPLACE: deletes every row the new one conflicts with — a PRIMARY KEY and a UNIQUE conflict can be two rows. + /// + private static void DeleteConflictingRows(ITable table, Dictionary row, object conflictingKey) + { + object? next = conflictingKey; + for (int guard = 0; guard < 64 && next is not null && table.DeleteByPrimaryKey(next); guard++) + { + TryDetectInsertConflict(table, row, [], out next, out _); + } + } + /// /// Runs a DO UPDATE on the conflicting row: each assignment is an expression over the existing row, with /// excluded.col the incoming value; the optional WHERE decides whether the row is updated at all.