diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 48c5bb7e..b924ba17 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -68,6 +68,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **`UPDATE … RETURNING` returned no rows when the SET changed a column the WHERE tests.** The returned rows were found by running the WHERE clause again after the update, so `SET status = 'done' WHERE status = 'open' RETURNING id` and `SET n = n + 10 WHERE n < 5 RETURNING n` came back empty. The matched rows are read first and the SET list is applied to them as the update applies it. Pinned by `UpdateReturningTests`. - **CHECK constraints written with IN, NOT IN, BETWEEN, NOT BETWEEN or GLOB accepted every row.** They were left to an older evaluator that passed them: `CHECK (status IN ('a', 'b'))` let `'z'` in. The SQLite-semantics evaluator reads them now (three-valued: a NULL still passes; GLOB case-sensitive with `*`, `?` and character classes), and the same operators over a computed value in a WHERE clause (`length(name) BETWEEN 3 AND 6`, `upper(name) IN (…)`) use it too. Pinned by `CheckConstraintOperatorTests`. Suites: core **2473/0 failed (6 skipped)**, VectorSearch 255/0, EF Core 116/0, linq2DB 96/0, regression canaries 51/0; write-path gate: GATE PASSED (nothing slower than baseline × 1.50; raw UPDATE 1.31×, default READ 1.33×, default DELETE 1.40× in the watch band on an un-quieted box). +- **A transaction rollback did not undo PageBased tables (known issue §13).** `CancelBatchUpdate()` — an ADO.NET `DbTransaction.Rollback()`, and so linq2db's and EF Core's — left a PageBased UPDATE or INSERT in place, also after a reopen: the PageBased engine writes into its page cache and page files directly, outside the storage transaction buffer, and its `Rollback()` was a stub. Each page manager now keeps a page's image on its first change inside the transaction and puts the images back on rollback (pages the transaction allocated return to the free list); every table then rebuilds its primary-key index and lets its hash indexes rebuild, which also fixes a columnar rollback that left the hash index holding the rolled-back value. A write outside a transaction pays one null check. Pinned by `TransactionRollbackTests`. `ExecuteBatchSQL`'s own failure rollback does not use the undo scope yet (recorded in §13). +- **Inside a transaction, a columnar full scan did not see the transaction's own UPDATEs and DELETEs (known issue §12).** The scan read the data file as it was on disk, while the transaction's in-place overwrites and its deletes were buffered until commit: it returned an updated row's old value, still listed a deleted row, and `WHERE n = 50` after `SET n = 50` found nothing; a primary-key lookup still returned a deleted row. The buffered overwrites and tombstones are laid over the scanned records now (only while a transaction holds them for that file), and a point read of a deleted record returns nothing. Pinned by `InTransactionReadTests` (encrypted and `NoEncryptMode`). Suites: core **2489/0 failed (6 skipped)**, VectorSearch 255/0, EF Core 116/0, linq2DB 96/0, regression canaries 51/0; write-path gate: GATE PASSED (nothing slower than baseline × 1.50; raw READ 1.30×, default READ 1.27×, default DELETE 1.31× in the watch band, as in the previous run on this box). + ## [2.1.4] - 2026-11-13 > The issue #497 follow-up: a quoted identifier in `CREATE TABLE` is catalogued without its quoting, so a schema diff --git a/docs/sharpcoredb-known-issues.md b/docs/sharpcoredb-known-issues.md index 3f1d8ad3..1e05f4d6 100644 --- a/docs/sharpcoredb-known-issues.md +++ b/docs/sharpcoredb-known-issues.md @@ -526,10 +526,13 @@ on the tuple — for files written before, there is no way to recover the lost k unenforced), and only then enforce single-column uniqueness on single-file storage. Pinned by the skipped single-file cases in `PrimaryKeyUniquenessTests`. -## 12. 🟠 OPEN — Inside a transaction, a columnar full scan does not see the transaction's own UPDATEs and DELETEs +## 12. ✅ FIXED — Inside a transaction, a columnar full scan did not see the transaction's own UPDATEs and DELETEs -> ⚠️ **STILL TRUE (verified 2026-11-17, `fix/returning-check-in-tx-reads`).** Found while investigating -> read-your-own-writes; the two commit defects found with it are fixed (see the CHANGELOG), this is not. +> ✅ **FIXED (2026-11-17, `fix/transaction-rollback-and-scan-overlay`).** The scan's data source +> (`Storage.ReadBytesWithRecordOffsets`) lays the transaction's buffered in-place overwrites and commit-time tombstones +> over the records it returns — only while a transaction holds such changes for the file, so other scans are untouched +> — and a point read (`ReadBytesFrom`) of a record the transaction deleted returns nothing. Encrypted and +> `NoEncryptMode` files alike. Pinned by `InTransactionReadTests`. The description below is kept as the record. Inside `BeginBatchUpdate()` / `EndBatchUpdate()` — the transaction an ADO.NET `DbTransaction` opens, and so linq2db's `TransactionAsync` and EF Core's transactions — a columnar table answers reads inconsistently until the commit: @@ -549,9 +552,19 @@ where the scan finds them. After the commit every read is correct (`BatchUpdateT record offset, which the scan already carries for its stale-version filter). It is a read-path change on the hottest scan, so it needs the write-path gate and a read benchmark; not attempted in this change. -## 13. 🔴 OPEN — PageBased storage cannot roll a transaction back +## 13. ✅ FIXED — PageBased storage could not roll a transaction back -> ⚠️ **STILL TRUE (verified 2026-11-17, `fix/returning-check-in-tx-reads`).** +> ✅ **FIXED (2026-11-17, `fix/transaction-rollback-and-scan-overlay`) for `BeginBatchUpdate` / `CancelBatchUpdate` and +> the storage-only transaction the EF Core provider uses** — so ADO.NET, linq2db and EF Core rollbacks. Each +> `PageManager` keeps a page's image on its first change inside the transaction (`BeginUndoScope`); rollback restores +> the images (marked dirty, so a page already written to the file during the transaction is written back) and returns +> pages the transaction allocated to the free list, then every table rebuilds its primary-key index from storage and +> marks its hash indexes stale — which also fixes a columnar rollback leaving the hash index on the rolled-back value. +> Outside a transaction a write pays one null check. Pinned by `TransactionRollbackTests` (including an ADO.NET +> `DbTransaction.Rollback()`). **Still open:** `ExecuteBatchSQL` rolls its own internal transaction back on a failing +> statement without the undo scope, so a PageBased table keeps the statements of a failed batch that ran before the +> failure; hooking it adds a page copy per touched page to the most-measured write path and needs its own gate run. +> And without a page WAL, a crash in the middle of a transaction can still leave pages the cache wrote early. `CancelBatchUpdate()` (an ADO.NET `DbTransaction.Rollback()`) does not undo a PageBased table's changes: a rolled-back UPDATE or INSERT is still there — also after a reopen — and a rolled-back DELETE stays deleted. Columnar tables roll @@ -584,5 +597,5 @@ in this change. | 9 | Column `UNIQUE` not enforced | 🟠 Medium | — | **Fixed on directory storage** — automatic unique index, composite `UNIQUE (a, b)` included; single-file still open (format decision, §10/§11) | | 10 | Single-file ignores `COLLATE` | 🟡 Low | — | **Open, owner decision** — needs a column-entry format extension (§10) | | 11 | PK uniqueness / composite keys | 🔴 High | — | **Fixed on directory storage for new tables** (composite key = UNIQUE over its columns); **open, owner decision** for single-file storage, which enforces no PK uniqueness (§11) | -| 12 | Columnar scan inside a transaction misses its own UPDATE/DELETE | 🟠 Medium | Read by primary key, or commit first | **Open** — buffered overwrites/deletes are not overlaid on the scan (§12) | -| 13 | PageBased transaction rollback | 🔴 High | Columnar storage for tables that need rollback | **Open** — `PageBasedEngine.Rollback()` is a stub (§13) | \ No newline at end of file +| 12 | Columnar scan inside a transaction misses its own UPDATE/DELETE | 🟠 Medium | — | **Fixed** — buffered overwrites and tombstones are laid over the scan (§12) | +| 13 | PageBased transaction rollback | 🔴 High | — | **Fixed** for `CancelBatchUpdate` (ADO.NET, linq2db, EF Core) via page undo images; `ExecuteBatchSQL`'s internal failure rollback still open (§13) | \ No newline at end of file diff --git a/src/SharpCoreDB/DataStructures/Table.TransactionUndo.cs b/src/SharpCoreDB/DataStructures/Table.TransactionUndo.cs new file mode 100644 index 00000000..3a4d2733 --- /dev/null +++ b/src/SharpCoreDB/DataStructures/Table.TransactionUndo.cs @@ -0,0 +1,74 @@ +// +// Copyright (c) 2026 MPCoreDeveloper. All rights reserved. +// Licensed under the MIT License. See LICENSE file in the project root for full license information. +// + +namespace SharpCoreDB.DataStructures; + +using SharpCoreDB.Storage.Engines; +using SharpCoreDB.Storage.Hybrid; + +/// +/// Table partial — rolling a PageBased table back with its database transaction (known issue §13). +/// +/// The PageBased engine writes into its page cache and page files directly, outside the storage transaction buffer that +/// columnar tables write through, and up to 2.1.4 nothing undid those writes: CancelBatchUpdate() — an ADO.NET +/// DbTransaction.Rollback() — left a PageBased UPDATE or INSERT in place, also after a reopen. The engine now keeps +/// each changed page's image for the length of the transaction and puts it back on rollback; the table then rebuilds its +/// primary-key index from the restored pages and lets its hash indexes rebuild on next use. +/// +/// +public partial class Table +{ + /// Starts the transaction's undo scope (PageBased tables; a no-op for columnar ones). + internal void BeginTransactionUndo() + { + if (StorageMode == StorageMode.PageBased && GetOrCreateStorageEngine() is PageBasedEngine engine) + { + engine.BeginUndoScope(); + } + } + + /// Ends the undo scope keeping every change (the transaction committed). + internal void CommitTransactionUndo() + { + if (_storageEngine is PageBasedEngine engine) + { + engine.CommitUndoScope(); + } + } + + /// + /// After a rollback: puts every page the transaction changed back (PageBased), then rebuilds the primary-key index + /// from storage and lets the hash indexes rebuild on next use. A columnar table's rows are rolled back by the storage + /// transaction buffer, but its in-memory indexes were not: an UPDATE of an indexed column, rolled back, left the hash + /// index holding the rolled-back value, so WHERE name = 'old' found nothing. Both modes rebuild here, as every + /// commit already rebuilds the primary-key index. + /// + internal void RollbackTransactionUndo() + { + if (_storageEngine is PageBasedEngine engine) + { + engine.RollbackUndoScope(); + } + else if (_storageEngine is null) + { + return; // never touched: nothing in memory to rebuild + } + + this.rwLock.EnterWriteLock(); + try + { + RebuildPrimaryKeyIndexFromDisk(); + foreach (var column in this.registeredIndexes.Keys) + { + this.staleIndexes.Add(column); + this._indexReadyCache.TryRemove(column, out _); + } + } + finally + { + this.rwLock.ExitWriteLock(); + } + } +} diff --git a/src/SharpCoreDB/Database/Transactions/Database.BatchUpdateTransaction.cs b/src/SharpCoreDB/Database/Transactions/Database.BatchUpdateTransaction.cs index 9456a772..1271cba1 100644 --- a/src/SharpCoreDB/Database/Transactions/Database.BatchUpdateTransaction.cs +++ b/src/SharpCoreDB/Database/Transactions/Database.BatchUpdateTransaction.cs @@ -96,6 +96,8 @@ public void BeginBatchUpdate() } } + BeginTablesUndo(); + storage.BeginTransaction(); _batchInsertedRows.Clear(); _batchUpdateActive = true; @@ -118,6 +120,8 @@ public void BeginStorageTransactionOnly() if (isReadOnly) throw new InvalidOperationException("Cannot begin transaction in readonly mode"); + BeginTablesUndo(); + storage.BeginTransaction(); _batchInsertedRows.Clear(); _batchUpdateActive = true; @@ -136,6 +140,7 @@ public void CommitStorageTransaction() return; storage.CommitAsync().GetAwaiter().GetResult(); + CommitTablesUndo(); _batchUpdateActive = false; } } @@ -175,10 +180,47 @@ public void RollbackStorageTransaction() } storage.Rollback(); + RollbackTablesUndo(); _batchUpdateActive = false; } } + /// Starts every PageBased table's undo scope for the transaction (see Table.TransactionUndo). + private void BeginTablesUndo() + { + foreach (var table in tables.Values) + { + if (table is Table t) + { + t.BeginTransactionUndo(); + } + } + } + + /// Ends every PageBased table's undo scope keeping the changes. + private void CommitTablesUndo() + { + foreach (var table in tables.Values) + { + if (table is Table t) + { + t.CommitTransactionUndo(); + } + } + } + + /// Rolls every PageBased table back to its state at the start of the transaction. + private void RollbackTablesUndo() + { + foreach (var table in tables.Values) + { + if (table is Table t) + { + t.RollbackTransactionUndo(); + } + } + } + /// /// Ends the batch UPDATE transaction and commits changes. /// Performs single WAL flush + bulk index rebuild for optimal performance. @@ -194,6 +236,7 @@ public void EndBatchUpdate() try { storage.CommitAsync().GetAwaiter().GetResult(); + CommitTablesUndo(); // Ensure data pages are flushed before index rebuild so subsequent // queries (including those after connection close) see the committed inserts. @@ -226,6 +269,7 @@ public void EndBatchUpdate() catch { storage.Rollback(); + RollbackTablesUndo(); _batchUpdateActive = false; throw; } @@ -272,6 +316,7 @@ public void CancelBatchUpdate() // Now perform the storage-level rollback. storage.Rollback(); + RollbackTablesUndo(); foreach (var table in tables.Values) { diff --git a/src/SharpCoreDB/Services/Storage.Append.cs b/src/SharpCoreDB/Services/Storage.Append.cs index 1acfece3..c569ba6f 100644 --- a/src/SharpCoreDB/Services/Storage.Append.cs +++ b/src/SharpCoreDB/Services/Storage.Append.cs @@ -788,6 +788,17 @@ public bool HasBufferedOverwriteAt(string path, long offset) => /// public bool AreRecordsEncrypted(string path) => UseRecordEncryption && FileHasEncryptedHeader(path); + /// Returns true when the current transaction deleted the record at . + private bool IsBufferedTombstone(string path, long offset) + { + lock (appendLock) + { + return bufferedTombstones.Count > 0 && + bufferedTombstones.TryGetValue(path, out var tombstones) && + tombstones.Contains(offset); + } + } + /// public void BufferTombstoneForCommit(string path, long offset) { @@ -1616,6 +1627,13 @@ private bool FlushOverwritePages(string path, long fileLength, int pageBytes, Di return DecryptBufferedAppendPayload(path, bufferedAppend); } + // Inside a transaction, a record the transaction deleted is gone, although its tombstone is written only at + // commit. Up to 2.1.4 the read returned it, so a primary-key lookup inside the transaction still found the row. + if (IsInTransaction && bufferedTombstones.Count > 0 && IsBufferedTombstone(path, offset)) + { + return null; + } + // B7: inside a transaction, a buffered in-place overwrite takes precedence over the disk // version (the overwrite is written to disk only at commit). The buffer holds the payload // only (its length is the record's stored length). diff --git a/src/SharpCoreDB/Services/Storage.ReadWrite.cs b/src/SharpCoreDB/Services/Storage.ReadWrite.cs index 1654afcb..a87ca6b8 100644 --- a/src/SharpCoreDB/Services/Storage.ReadWrite.cs +++ b/src/SharpCoreDB/Services/Storage.ReadWrite.cs @@ -198,12 +198,115 @@ public void Write(string path, string data) // Plaintext (and every legacy layout): the buffer IS the file, so the buffer offset is already // the physical offset and the caller needs no map. Gated on the magic header (not on // AreRecordsEncrypted) to stay recursion-free with ReadBytes. + byte[]? data; if (noEncrypt || this.noEncryption || !FileHasEncryptedHeader(path)) { - return ReadBytes(path, noEncrypt); + data = ReadBytes(path, noEncrypt); + } + else + { + data = DecryptTableFileToPlaintext(path, out physicalOffsets); + } + + // Inside a transaction the file on disk does not yet hold the transaction's in-place UPDATEs and DELETEs (they + // are buffered until commit), so they are laid over the records here. Up to 2.1.4 a full scan inside a + // transaction returned the old value of an updated row and still listed a deleted one, while a primary-key + // lookup saw the change (known issue §12). + return data is not null && IsInTransaction && HasBufferedChanges(path) + ? OverlayBufferedChanges(path, data, ref physicalOffsets) + : data; + } + + /// Returns true when the transaction holds an in-place overwrite or a tombstone for the file. + private bool HasBufferedChanges(string path) + { + if (!bufferedOverwrites.IsEmpty && bufferedOverwrites.ContainsKey(path)) + { + return true; + } + + lock (appendLock) + { + return bufferedTombstones.TryGetValue(path, out var tombstones) && tombstones.Count > 0; + } + } + + /// + /// Lays the transaction's buffered in-place overwrites and tombstones over a scanned copy of the file: an overwritten + /// record gets its new payload (same length — an in-place overwrite never changes it), a deleted record becomes a + /// tombstone (negative length prefix) and leaves the physical-offset map, which lists live records only. + /// + private byte[] OverlayBufferedChanges(string path, byte[] data, ref long[]? physicalOffsets) + { + Dictionary? overwrites = null; + HashSet? tombstones = null; + lock (appendLock) + { + if (bufferedOverwrites.TryGetValue(path, out var buffered) && buffered.Count > 0) + { + overwrites = new Dictionary(buffered); + } + + if (bufferedTombstones.TryGetValue(path, out var deleted) && deleted.Count > 0) + { + tombstones = [.. deleted]; + } + } + + bool encrypted = UseRecordEncryption && FileHasEncryptedHeader(path); + var copy = (byte[])data.Clone(); + var span = copy.AsSpan(); + var keptOffsets = physicalOffsets is null ? null : new List(physicalOffsets.Length); + int position = 0; + int ordinal = 0; + while (position + 4 <= span.Length) + { + int length = BinaryPrimitives.ReadInt32LittleEndian(span.Slice(position, 4)); + if (length < 0) + { + if (-length < 4) + { + break; + } + + position += -length; // an existing tombstone + continue; + } + + if (position + 4 + length > span.Length) + { + break; + } + + long physical = physicalOffsets is not null && ordinal < physicalOffsets.Length ? physicalOffsets[ordinal] : position; + ordinal++; + + if (tombstones is not null && tombstones.Contains(physical)) + { + BinaryPrimitives.WriteInt32LittleEndian(span.Slice(position, 4), -(4 + length)); + } + else + { + keptOffsets?.Add(physical); + if (length > 0 && overwrites is not null && overwrites.TryGetValue(physical, out var stored)) + { + var payload = encrypted ? DecryptRecord((byte[])stored.Clone()) : stored; + if (payload is not null && payload.Length == length) + { + payload.CopyTo(span.Slice(position + 4, length)); + } + } + } + + position += 4 + length; + } + + if (keptOffsets is not null) + { + physicalOffsets = [.. keptOffsets]; } - return DecryptTableFileToPlaintext(path, out physicalOffsets); + return copy; } /// diff --git a/src/SharpCoreDB/Storage/Engines/PageBasedEngine.cs b/src/SharpCoreDB/Storage/Engines/PageBasedEngine.cs index 5e9cf0eb..61fe2ace 100644 --- a/src/SharpCoreDB/Storage/Engines/PageBasedEngine.cs +++ b/src/SharpCoreDB/Storage/Engines/PageBasedEngine.cs @@ -326,6 +326,45 @@ public void Rollback() } } + private volatile bool undoScopeActive; + + /// + /// Starts a database transaction's undo scope on every page manager (see ). + /// Independent of this engine's own , which the table layer uses around batch inserts. + /// + public void BeginUndoScope() + { + undoScopeActive = true; + foreach (var manager in tableManagers.Values) + { + manager.BeginUndoScope(); + } + } + + /// Ends the undo scope and keeps the changes (the database transaction committed). + public void CommitUndoScope() + { + undoScopeActive = false; + foreach (var manager in tableManagers.Values) + { + manager.CommitUndoScope(); + } + } + + /// Ends the undo scope and puts every changed page back (the database transaction rolled back). + /// true when any page was restored. + public bool RollbackUndoScope() + { + undoScopeActive = false; + bool restored = false; + foreach (var manager in tableManagers.Values) + { + restored |= manager.RollbackUndoScope(); + } + + return restored; + } + /// public bool IsInTransaction { @@ -400,7 +439,13 @@ private PageManager GetOrCreatePageManager(string tableName) return tableManagers.GetOrAdd(tableName, name => { var tableId = GetTableId(name); - return new PageManager(databasePath, tableId, config); // ✅ Pass config! + var manager = new PageManager(databasePath, tableId, config); // ✅ Pass config! + if (undoScopeActive) + { + manager.BeginUndoScope(); // a table first touched inside the transaction rolls back too + } + + return manager; }); } diff --git a/src/SharpCoreDB/Storage/PageManager.cs b/src/SharpCoreDB/Storage/PageManager.cs index e558f19a..d8258257 100644 --- a/src/SharpCoreDB/Storage/PageManager.cs +++ b/src/SharpCoreDB/Storage/PageManager.cs @@ -115,6 +115,13 @@ public enum RecordFlags : byte // O(1) Free list implementation private readonly Queue freePageList = new(); + + // Transaction undo scope (see BeginUndoScope). Null outside a scope, so a write outside a transaction pays one null + // check. Maps each page changed inside the scope to its image before the first change, or to null for a page the + // scope allocated past the end of the file. + private Dictionary? undoImages; + private Dictionary>? undoTablePages; + private ulong[]? undoFreePages; // Cache statistics private long cacheHits = 0; @@ -351,10 +358,15 @@ public virtual PageId AllocatePage(uint tableId, PageType pageType) if (freePageList.Count > 0) { pageIdValue = freePageList.Dequeue(); + CaptureUndo(pageIdValue); } else { pageIdValue = nextPageId++; + if (undoImages is not null) + { + undoImages.TryAdd(pageIdValue, null); + } } var pageId = new PageId(pageIdValue); @@ -398,6 +410,8 @@ public virtual void FreePage(PageId pageId) lock (writeLock) { + CaptureUndo(pageId.Value); + // Add to free list for reuse freePageList.Enqueue(pageId.Value); @@ -455,9 +469,11 @@ public virtual RecordId InsertRecord(PageId pageId, byte[] data) { if (pageId.Value == 0 || data.Length > MAX_RECORD_SIZE) return new(0); - + lock (writeLock) { + CaptureUndo(pageId.Value); + // Get or load page if (!pageCache.TryGetValue(pageId.Value, out var page)) { @@ -530,6 +546,8 @@ public virtual (PageId Page, RecordId Record) UpdateRecord(PageId pageId, Record lock (writeLock) { + CaptureUndo(pageId.Value); + if (!pageCache.TryGetValue(pageId.Value, out var page)) { page = ReadPage(pageId); @@ -648,9 +666,11 @@ public virtual void DeleteRecord(PageId pageId, RecordId recordId) { if (pageId.Value == 0 || recordId.SlotIndex >= ushort.MaxValue) return; - + lock (writeLock) { + CaptureUndo(pageId.Value); + if (!pageCache.TryGetValue(pageId.Value, out var page)) { page = ReadPage(pageId); @@ -864,6 +884,125 @@ private void EvictIfNeeded() } } + /// + /// Starts a transaction undo scope: from here on, the first change to each page keeps a copy of the page as it was, + /// so can put it back. Up to 2.1.4 the PageBased engine had no rollback at all — it + /// writes into this cache and the page file directly, outside the storage transaction buffer — so a rolled-back + /// UPDATE or INSERT stayed, also after a reopen (known issue §13). Starting a scope that is already open is a no-op. + /// + public void BeginUndoScope() + { + lock (writeLock) + { + if (undoImages is not null) + { + return; + } + + undoImages = new Dictionary(); + undoTablePages = tablePagesIndex.ToDictionary(kv => kv.Key, kv => new List(kv.Value)); + undoFreePages = [.. freePageList]; + } + } + + /// Ends the undo scope and keeps every change (commit). + public void CommitUndoScope() + { + lock (writeLock) + { + undoImages = null; + undoTablePages = null; + undoFreePages = null; + } + } + + /// + /// Ends the undo scope and puts every page changed inside it back as it was (rollback). Restored pages are marked + /// dirty, so a page the cache had already written to the file during the scope is written back on the next flush; a + /// page the scope allocated is written as a free page and returned to the free list. + /// + /// true when the scope changed any page. + public bool RollbackUndoScope() + { + lock (writeLock) + { + if (undoImages is null) + { + return false; + } + + var images = undoImages; + undoImages = null; + + tablePagesIndex.Clear(); + foreach (var (tableId, pages) in undoTablePages!) + { + tablePagesIndex[tableId] = pages; + } + + freePageList.Clear(); + foreach (var free in undoFreePages!) + { + freePageList.Enqueue(free); + } + + undoTablePages = null; + undoFreePages = null; + + foreach (var (pageId, image) in images) + { + var restored = image ?? new Page + { + TableId = 0, + Type = PageType.Free, + RecordCount = 0, + FreeSpace = (ushort)(PAGE_SIZE - PAGE_HEADER_SIZE), + PageId = pageId, + Data = new Memory(new byte[PAGE_SIZE - PAGE_HEADER_SIZE]), + Slots = [], + }; + + if (image is null) + { + freePageList.Enqueue(pageId); + } + + restored = ClonePage(restored); + restored.IsDirty = true; + pageCache[pageId] = restored; + UpdateLRU(pageId); + dirtyPages.Add(pageId); + } + + EvictIfNeeded(); + return images.Count > 0; + } + } + + /// Keeps the page's image before its first change inside an undo scope. Called under the write lock. + private void CaptureUndo(ulong pageId) + { + if (undoImages is null || undoImages.ContainsKey(pageId)) + { + return; + } + + if (pageCache.TryGetValue(pageId, out var cached)) + { + undoImages[pageId] = ClonePage(cached); + return; + } + + var onDisk = ReadPage(new PageId(pageId)); + undoImages[pageId] = onDisk.PageId == 0 ? null : ClonePage(onDisk); + } + + private static Page ClonePage(Page page) => page with + { + Data = new Memory(page.Data.ToArray()), + Slots = page.Slots is null ? [] : [.. page.Slots], + }; + public virtual void Dispose() { pagesFile?.Dispose(); diff --git a/tests/SharpCoreDB.Tests/InTransactionReadTests.cs b/tests/SharpCoreDB.Tests/InTransactionReadTests.cs new file mode 100644 index 00000000..e493e7f6 --- /dev/null +++ b/tests/SharpCoreDB.Tests/InTransactionReadTests.cs @@ -0,0 +1,127 @@ +// +// Copyright (c) 2026 MPCoreDeveloper. All rights reserved. +// Licensed under the MIT License. See LICENSE file in the project root for full license information. +// + +namespace SharpCoreDB.Tests; + +using System; +using System.Globalization; +using System.IO; +using System.Linq; +using Microsoft.Extensions.DependencyInjection; +using SharpCoreDB.Data.Provider; +using SharpCoreDB.Interfaces; +using Xunit; + +/// +/// Inside a transaction (BeginBatchUpdate(), which every ADO.NET transaction opens) every read sees the +/// transaction's own changes — a full scan as well as a primary-key or index lookup (known issue §12). +/// +/// Up to 2.1.4 a columnar full scan read the data file as it was on disk, while the transaction's in-place UPDATEs and +/// its DELETEs were held in the storage buffers until commit: the scan returned an updated row's old value, still listed +/// a deleted row, and WHERE n = 50 after SET n = 50 found nothing. A primary-key lookup still returned a +/// deleted row. The scan now lays the buffered overwrites and tombstones over the records, and a point read of a +/// record the transaction deleted returns nothing. +/// +/// +public sealed class InTransactionReadTests : IDisposable +{ + private readonly DatabaseFactory _factory; + private readonly string _root = Path.Combine(Path.GetTempPath(), $"SCDB_TxReads_{Guid.NewGuid():N}"); + + public InTransactionReadTests() + { + var services = new ServiceCollection(); + services.AddSharpCoreDB(); + _factory = services.BuildServiceProvider().GetRequiredService(); + Directory.CreateDirectory(_root); + } + + public static TheoryData Modes => new() { "columnar", "page-based" }; + + public void Dispose() + { + try + { + if (Directory.Exists(_root)) + { + Directory.Delete(_root, true); + } + } + catch (IOException) + { + // Best-effort cleanup of a temp directory. + } + } + + private (IDatabase Db, string Path) Open(string mode, bool noEncrypt = false) + { + var path = Path.Combine(_root, $"db_{Guid.NewGuid():N}"); + var db = _factory.Create(path, "pw", false, new DatabaseConfig { NoEncryptMode = noEncrypt }); + var ddl = "CREATE TABLE t (id INTEGER PRIMARY KEY, n INTEGER, name TEXT)"; + db.ExecuteSQL(mode == "page-based" ? ddl + " STORAGE = PAGE_BASED" : ddl); + db.ExecuteSQL("CREATE INDEX ix_name ON t (name)"); + db.ExecuteSQL("INSERT INTO t (id, n, name) VALUES (1, 1, 'a'), (2, 2, 'b'), (5, 5, 'e')"); + return (db, path); + } + + private static string[] Rows(IDatabase db, string sql) => + [.. db.ExecuteQuery(sql).Select(r => string.Join(",", r.Values.Select(v => v is null or DBNull ? "NULL" : Convert.ToString(v, CultureInfo.InvariantCulture))))]; + + public static TheoryData ModesAndEncryption => new() + { + { "columnar", false }, + { "columnar", true }, + { "page-based", false }, + }; + + [Theory] + [MemberData(nameof(ModesAndEncryption))] + public void Scan_InsideATransaction_SeesTheTransactionsUpdate(string mode, bool noEncrypt) + { + var (db, _) = Open(mode, noEncrypt); + + db.BeginBatchUpdate(); + db.ExecuteSQL("UPDATE t SET n = 50 WHERE id = 1"); + + Assert.Equal(["1,50", "2,2", "5,5"], Rows(db, "SELECT id, n FROM t").Order()); + Assert.Equal(["1"], Rows(db, "SELECT id FROM t WHERE n = 50")); + db.EndBatchUpdate(); + } + + [Theory] + [MemberData(nameof(ModesAndEncryption))] + public void Reads_InsideATransaction_DoNotSeeARowItDeleted(string mode, bool noEncrypt) + { + var (db, _) = Open(mode, noEncrypt); + + db.BeginBatchUpdate(); + db.ExecuteSQL("DELETE FROM t WHERE id = 2"); + + Assert.Equal(["1", "5"], Rows(db, "SELECT id FROM t").Order()); + Assert.Empty(db.ExecuteQuery("SELECT n FROM t WHERE id = 2")); + Assert.Empty(db.ExecuteQuery("SELECT id FROM t WHERE name = 'b'")); + db.EndBatchUpdate(); + + Assert.Equal(["1", "5"], Rows(db, "SELECT id FROM t").Order()); + } + + [Theory] + [MemberData(nameof(ModesAndEncryption))] + public void Scan_InsideATransaction_SeesUpdatesAndInsertsTogether(string mode, bool noEncrypt) + { + var (db, _) = Open(mode, noEncrypt); + + db.BeginBatchUpdate(); + db.ExecuteSQL("UPDATE t SET n = 50 WHERE id = 1"); + db.ExecuteSQL("INSERT INTO t (id, n, name) VALUES (3, 3, 'c')"); + db.ExecuteSQL("UPDATE t SET name = 'z' WHERE id = 5"); + + Assert.Equal(["1,50,a", "2,2,b", "3,3,c", "5,5,z"], Rows(db, "SELECT id, n, name FROM t").Order()); + Assert.Equal(["5"], Rows(db, "SELECT id FROM t WHERE name = 'z'")); + db.EndBatchUpdate(); + + Assert.Equal(["1,50,a", "2,2,b", "3,3,c", "5,5,z"], Rows(db, "SELECT id, n, name FROM t").Order()); + } +} diff --git a/tests/SharpCoreDB.Tests/TransactionRollbackTests.cs b/tests/SharpCoreDB.Tests/TransactionRollbackTests.cs new file mode 100644 index 00000000..4cd8477a --- /dev/null +++ b/tests/SharpCoreDB.Tests/TransactionRollbackTests.cs @@ -0,0 +1,163 @@ +// +// Copyright (c) 2026 MPCoreDeveloper. All rights reserved. +// Licensed under the MIT License. See LICENSE file in the project root for full license information. +// + +namespace SharpCoreDB.Tests; + +using System; +using System.Globalization; +using System.IO; +using System.Linq; +using Microsoft.Extensions.DependencyInjection; +using SharpCoreDB.Data.Provider; +using SharpCoreDB.Interfaces; +using Xunit; + +/// +/// Rolling a transaction back — CancelBatchUpdate(), which an ADO.NET DbTransaction.Rollback() calls — +/// restores every table, PageBased ones included (known issue §13). +/// +/// Up to 2.1.4 the PageBased engine had no rollback: it writes into its page cache and page files directly, outside the +/// storage transaction buffer, and PageBasedEngine.Rollback() was a stub. A rolled-back UPDATE or INSERT stayed, +/// also after a reopen. The engine now keeps each changed page's image for the length of the transaction and puts it +/// back. A columnar rollback restored the rows but not the in-memory hash index, so after rolling back an UPDATE of an +/// indexed column, WHERE name = 'old' found nothing. +/// +/// +public sealed class TransactionRollbackTests : IDisposable +{ + private readonly DatabaseFactory _factory; + private readonly string _root = Path.Combine(Path.GetTempPath(), $"SCDB_TxRollback_{Guid.NewGuid():N}"); + + public TransactionRollbackTests() + { + var services = new ServiceCollection(); + services.AddSharpCoreDB(); + _factory = services.BuildServiceProvider().GetRequiredService(); + Directory.CreateDirectory(_root); + } + + public static TheoryData Modes => new() { "columnar", "page-based" }; + + public void Dispose() + { + try + { + if (Directory.Exists(_root)) + { + Directory.Delete(_root, true); + } + } + catch (IOException) + { + // Best-effort cleanup of a temp directory. + } + } + + private (IDatabase Db, string Path) Open(string mode, bool noEncrypt = false) + { + var path = Path.Combine(_root, $"db_{Guid.NewGuid():N}"); + var db = _factory.Create(path, "pw", false, new DatabaseConfig { NoEncryptMode = noEncrypt }); + var ddl = "CREATE TABLE t (id INTEGER PRIMARY KEY, n INTEGER, name TEXT)"; + db.ExecuteSQL(mode == "page-based" ? ddl + " STORAGE = PAGE_BASED" : ddl); + db.ExecuteSQL("CREATE INDEX ix_name ON t (name)"); + db.ExecuteSQL("INSERT INTO t (id, n, name) VALUES (1, 1, 'a'), (2, 2, 'b'), (5, 5, 'e')"); + return (db, path); + } + + private static string[] Rows(IDatabase db, string sql) => + [.. db.ExecuteQuery(sql).Select(r => string.Join(",", r.Values.Select(v => v is null or DBNull ? "NULL" : Convert.ToString(v, CultureInfo.InvariantCulture))))]; + + [Theory] + [MemberData(nameof(Modes))] + public void CancelBatchUpdate_UpdateInsertAndDelete_RestoresEveryRowAndIndex(string mode) + { + var (db, path) = Open(mode); + + db.BeginBatchUpdate(); + db.ExecuteSQL("UPDATE t SET n = 99, name = 'zz' WHERE id = 2"); + db.ExecuteSQL("INSERT INTO t (id, n, name) VALUES (3, 3, 'c')"); + db.ExecuteSQL("DELETE FROM t WHERE id = 1"); + db.CancelBatchUpdate(); + + Assert.Equal(["1,1,a", "2,2,b", "5,5,e"], Rows(db, "SELECT id, n, name FROM t").Order()); + Assert.Equal(["1"], Rows(db, "SELECT n FROM t WHERE id = 1")); + Assert.Equal(["2"], Rows(db, "SELECT id FROM t WHERE name = 'b'")); + Assert.Empty(db.ExecuteQuery("SELECT id FROM t WHERE name = 'zz'")); + + (db as IDisposable)?.Dispose(); + var reopened = _factory.Create(path, "pw"); + Assert.Equal(["1,1,a", "2,2,b", "5,5,e"], Rows(reopened, "SELECT id, n, name FROM t").Order()); + (reopened as IDisposable)?.Dispose(); + } + + [Theory] + [MemberData(nameof(Modes))] + public void CancelBatchUpdate_ThenNewWrites_AreKeptAfterReopen(string mode) + { + var (db, path) = Open(mode); + + db.BeginBatchUpdate(); + db.ExecuteSQL("UPDATE t SET n = 99 WHERE id = 1"); + db.CancelBatchUpdate(); + db.ExecuteSQL("INSERT INTO t (id, n, name) VALUES (4, 4, 'd')"); + db.ExecuteSQL("UPDATE t SET n = 22 WHERE id = 2"); + + (db as IDisposable)?.Dispose(); + var reopened = _factory.Create(path, "pw"); + Assert.Equal(["1,1,a", "2,22,b", "4,4,d", "5,5,e"], Rows(reopened, "SELECT id, n, name FROM t").Order()); + (reopened as IDisposable)?.Dispose(); + } + + [Fact] + public void CancelBatchUpdate_PageBasedTransactionSpanningManyPages_RestoresTheTable() + { + var (db, path) = Open("page-based"); + + db.BeginBatchUpdate(); + for (int i = 10; i < 600; i++) + { + db.ExecuteSQL($"INSERT INTO t (id, n, name) VALUES ({i}, {i}, 'row-{i}-{new string('x', 40)}')"); + } + + db.ExecuteSQL("UPDATE t SET n = 7 WHERE id = 2"); + db.CancelBatchUpdate(); + + Assert.Equal(["1,1,a", "2,2,b", "5,5,e"], Rows(db, "SELECT id, n, name FROM t").Order()); + + (db as IDisposable)?.Dispose(); + var reopened = _factory.Create(path, "pw"); + Assert.Equal(["1,1,a", "2,2,b", "5,5,e"], Rows(reopened, "SELECT id, n, name FROM t").Order()); + (reopened as IDisposable)?.Dispose(); + } + + [Theory] + [MemberData(nameof(Modes))] + public void AdoNetTransactionRollback_DiscardsTheChanges(string mode) + { + var path = Path.Combine(_root, $"ado_{Guid.NewGuid():N}"); + using var connection = new SharpCoreDBConnection($"Data Source={path};Password=pw"); + connection.Open(); + using (var create = connection.CreateCommand()) + { + create.CommandText = "CREATE TABLE t (id INTEGER PRIMARY KEY, n INTEGER)" + (mode == "page-based" ? " STORAGE = PAGE_BASED" : ""); + create.ExecuteNonQuery(); + create.CommandText = "INSERT INTO t (id, n) VALUES (1, 1)"; + create.ExecuteNonQuery(); + } + + using (var transaction = connection.BeginTransaction()) + { + using var update = connection.CreateCommand(); + update.Transaction = transaction; + update.CommandText = "UPDATE t SET n = 99 WHERE id = 1"; + update.ExecuteNonQuery(); + transaction.Rollback(); + } + + using var select = connection.CreateCommand(); + select.CommandText = "SELECT n FROM t WHERE id = 1"; + Assert.Equal(1L, Convert.ToInt64(select.ExecuteScalar(), CultureInfo.InvariantCulture)); + } +}