From 7e92c8138a0b4ac7bc05728fe68cb9fcc984d2e9 Mon Sep 17 00:00:00 2001 From: hskl18 Date: Tue, 14 Jul 2026 00:47:02 -0700 Subject: [PATCH 1/4] feat: add paged storage engine v2 --- CMakeLists.txt | 68 +- DESIGN.md | 30 +- README.md | 55 +- docs/release-checklist-v2.md | 15 + docs/storage-v2.md | 198 ++++ includes/binary_files/file_record.cpp | 2 +- includes/binary_files/file_record.h | 2 +- includes/storage/page_buffer.cpp | 48 + includes/storage/page_buffer.h | 37 + includes/storage/storage_v2.cpp | 1422 +++++++++++++++++++++++++ includes/storage/storage_v2.h | 78 ++ includes/table/table.cpp | 128 +-- includes/table/table.h | 3 + includes/version.h.in | 6 + run_sql.cpp | 24 +- tests/cli_tests.cpp | 106 ++ tests/migration_tests.cpp | 217 ++++ tests/sql_tests.cpp | 51 +- tests/storage_crash_tests.cpp | 323 ++++++ tests/storage_crash_worker.cpp | 20 + tests/storage_v2_tests.cpp | 321 ++++++ tools/sql_migrate_v1.cpp | 246 +++++ 22 files changed, 3283 insertions(+), 117 deletions(-) create mode 100644 docs/release-checklist-v2.md create mode 100644 docs/storage-v2.md create mode 100644 includes/storage/page_buffer.cpp create mode 100644 includes/storage/page_buffer.h create mode 100644 includes/storage/storage_v2.cpp create mode 100644 includes/storage/storage_v2.h create mode 100644 includes/version.h.in create mode 100644 tests/cli_tests.cpp create mode 100644 tests/migration_tests.cpp create mode 100644 tests/storage_crash_tests.cpp create mode 100644 tests/storage_crash_worker.cpp create mode 100644 tests/storage_v2_tests.cpp create mode 100644 tools/sql_migrate_v1.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 54f6e18..0de66d7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,6 +1,10 @@ cmake_minimum_required(VERSION 3.20) -project(SQLInCPP VERSION 1.0.0 LANGUAGES CXX) +project(SQLInCPP VERSION 2.0.0 LANGUAGES CXX) + +if(NOT UNIX) + message(FATAL_ERROR "Storage format v2 requires POSIX file and process primitives") +endif() set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED ON) @@ -22,6 +26,8 @@ set(SQL_CORE_SOURCES includes/parser/parser.cpp includes/shunting_yard/shunting_yard.cpp includes/sql/sql.cpp + includes/storage/page_buffer.cpp + includes/storage/storage_v2.cpp includes/table/table.cpp includes/tokenizer/ftokenize.cpp includes/tokenizer/state_machine_functions.cpp @@ -29,7 +35,15 @@ set(SQL_CORE_SOURCES ) add_library(sql_core STATIC ${SQL_CORE_SOURCES}) -target_include_directories(sql_core PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}") +configure_file( + "${CMAKE_CURRENT_SOURCE_DIR}/includes/version.h.in" + "${CMAKE_CURRENT_BINARY_DIR}/generated/version.h" + @ONLY +) +target_include_directories(sql_core PUBLIC + "${CMAKE_CURRENT_SOURCE_DIR}" + "${CMAKE_CURRENT_BINARY_DIR}/generated" +) function(sql_enable_warnings target) if(MSVC) @@ -45,6 +59,10 @@ add_executable(run_sql run_sql.cpp) target_link_libraries(run_sql PRIVATE sql_core) sql_enable_warnings(run_sql) +add_executable(sql_migrate_v1 tools/sql_migrate_v1.cpp) +target_link_libraries(sql_migrate_v1 PRIVATE sql_core) +sql_enable_warnings(sql_migrate_v1) + include(CTest) if(BUILD_TESTING) @@ -72,4 +90,50 @@ if(BUILD_TESTING) target_link_libraries(bplustree_tests PRIVATE sql_core) sql_enable_warnings(bplustree_tests) add_test(NAME bplustree_evidence COMMAND bplustree_tests) + + add_executable(storage_v2_tests tests/storage_v2_tests.cpp) + target_link_libraries(storage_v2_tests PRIVATE sql_core) + sql_enable_warnings(storage_v2_tests) + add_test(NAME storage_v2_integrity COMMAND storage_v2_tests) + + add_executable(storage_crash_worker tests/storage_crash_worker.cpp) + target_link_libraries(storage_crash_worker PRIVATE sql_core) + sql_enable_warnings(storage_crash_worker) + + add_executable(storage_crash_tests tests/storage_crash_tests.cpp) + target_link_libraries(storage_crash_tests PRIVATE sql_core) + target_compile_definitions(storage_crash_tests PRIVATE + STORAGE_CRASH_WORKER_PATH="$" + ) + add_dependencies(storage_crash_tests storage_crash_worker) + sql_enable_warnings(storage_crash_tests) + add_test(NAME storage_crash_recovery COMMAND storage_crash_tests) + set_tests_properties(storage_crash_recovery PROPERTIES TIMEOUT 60 LABELS crash) + + add_executable(migration_tests tests/migration_tests.cpp) + target_link_libraries(migration_tests PRIVATE sql_core) + target_compile_definitions(migration_tests PRIVATE + MIGRATION_TOOL_PATH="$" + ) + add_dependencies(migration_tests sql_migrate_v1) + sql_enable_warnings(migration_tests) + add_test(NAME migration_v1_equivalence COMMAND migration_tests) + + add_executable(cli_tests tests/cli_tests.cpp) + target_link_libraries(cli_tests PRIVATE sql_core) + target_compile_definitions(cli_tests PRIVATE + RUN_SQL_PATH="$" + ) + add_dependencies(cli_tests run_sql) + sql_enable_warnings(cli_tests) + add_test(NAME cli_storage_inspection COMMAND cli_tests) + + add_test(NAME cli_version COMMAND run_sql --version) + set_tests_properties(cli_version PROPERTIES + PASS_REGULAR_EXPRESSION "SQL in C\\+\\+ 2.0.0 \\(storage format 2\\)" + ) + add_test(NAME migration_cli_version COMMAND sql_migrate_v1 --version) + set_tests_properties(migration_cli_version PROPERTIES + PASS_REGULAR_EXPRESSION "sql_migrate_v1 2.0.0 \\(storage format 2\\)" + ) endif() diff --git a/DESIGN.md b/DESIGN.md index 1331c2f..9e5f658 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -1,6 +1,6 @@ # Design and invariants -SQL in C++ is an educational database engine with a deliberately small grammar and fixed-width local persistence. +SQL in C++ is an educational database engine with a deliberately small grammar and checksummed paged local persistence. The implementation favors inspectable data structures over production database features. ## Command path @@ -16,35 +16,41 @@ An invalid `WHERE` expression never falls back to an unfiltered query. ## Storage format -Each field occupies one zero-padded 100-byte slot. -The schema file stores the field count followed by one slot per field name. -The data file stores each row as exactly `field_count * 100` bytes. +Version 2 stores a table in one 4096-byte paged `.sql2` file. +Every page has an explicit type, identifier, payload bound, next pointer, and CRC32. +Schema and row streams span checked page chains, so the maximum supported 64-field row can safely exceed one page. +Each mutating statement updates transaction-owned row and B+ tree pages, tracks a deterministic dirty-page set, and builds a complete recovery image before any database page is written. -The format retains the repository's existing fixed-width layout. -Writes now use explicit zero-initialized buffers instead of reading beyond a string allocation. -Values longer than 100 bytes, rows with the wrong number of values, duplicate fields, partial rows, and invalid metadata are rejected before they can be treated as valid data. +The checksummed WAL stores that complete image and a commit footer. +The writer fsyncs the WAL before database pages, fsyncs the database, then truncates and fsyncs the WAL. +Recovery validates the whole WAL before redo and rejects incomplete or corrupt records. +Redo is idempotent and the engine never acknowledges a statement before the WAL, database, and cleared-WAL boundaries complete. + +The legacy v1 fixed-width pair remains available through a read-only compatibility path and the copy-first migration tool. +Migration never modifies or deletes its v1 source. Tables support between 1 and 64 uniquely named fields. Table and field identifiers use letters, digits, and underscores, and cannot begin with a digit. ## Indexes and query semantics -Every field has an in-memory `MMap` backed by the repository's B+ tree implementation. -Indexes are rebuilt from the data file when a table is opened. +Every field has a persistent page-backed B+ tree ordered by `(value, record_id)`. +Inserts mutate the persisted search path incrementally, preserve untouched page identifiers, split bounded leaf and internal pages, and replace roots through the index directory. +Opening and querying a v2 table validates and reads persisted index entries without inserting them into the legacy in-memory `MMap`. Equality and range filters return record numbers, while logical expressions combine those sets with union or intersection. Values are strings. Ordering is lexical rather than numeric, so `"10"` sorts before `"2"`. -The current `lower_bound` and `upper_bound` implementations scan the linked leaf sequence and are not a logarithmic query-planning claim. +The current query layer loads persistent leaves into its compatibility map and is not a query-planning or logarithmic performance claim. ## Deliberate limits -The engine has no transactions, concurrency control, write-ahead log, schema migration, joins, updates, deletes, query optimizer, or SQL compatibility guarantee. +The engine has atomic single-writer statements and crash recovery, but no multi-statement transactions, concurrency control, joins, updates, deletes, query optimizer, or broader SQL compatibility guarantee. It is not designed for untrusted multi-user workloads. The command-line process reads and writes table files in its current working directory. ## Evidence -The CTest suite covers end-to-end SQL behavior, parser safety, and deterministic B+ tree insertion and lookup properties. +The CTest suite covers end-to-end SQL behavior, parser safety, persistent randomized B+ tree images, page and WAL corruption, crash recovery, v1 migration equivalence, large rows, and deterministic in-memory B+ tree properties. CI runs release builds on Linux and macOS plus an AddressSanitizer and UndefinedBehaviorSanitizer build on Linux. The sanitizer job is a memory-safety regression gate, not evidence of production database readiness. diff --git a/README.md b/README.md index dc9220b..8218679 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # SQL in C++ -SQL in C++ is an educational relational database engine and command-line DSL implemented from first principles. -It parses a focused subset of SQL, persists records to binary files, indexes field values with B+ trees, and evaluates compound filters with a shunting-yard pipeline. +SQL in C++ 2.0 is an educational relational database engine and command-line DSL implemented from first principles. +It parses a focused subset of SQL, persists records in checksummed pages, maintains persistent B+ tree indexes, recovers atomic single-writer statements through a WAL, and evaluates compound filters with a shunting-yard pipeline. ![Command-line preview](image/preview.png) @@ -12,11 +12,14 @@ It parses a focused subset of SQL, persists records to binary files, indexes fie - `SELECT *` and projected field lists. - `WHERE` filters using `=`, `!=`, `<`, `<=`, `>`, and `>=`. - Compound `AND` and `OR` expressions with parentheses. -- Binary persistence across separate CLI sessions. +- Checksummed paged persistence across separate CLI sessions. +- Persistent per-field B+ tree indexes. +- Crash recovery for atomic single-writer statements. +- Copy-first migration from the v1 fixed-width format. - Batch execution from the sample command files in `batch/`. This project is intentionally smaller than a production SQL database. -It does not implement joins, transactions, schemas, query planning, updates, or deletes. +It does not implement joins, multi-statement transactions, schemas, query planning, updates, or deletes. Values are stored and ordered as strings, so numeric-looking values use lexical comparison semantics. ## Architecture @@ -27,7 +30,8 @@ Values are stored and ordered as strings, so numeric-looking values use lexical | `includes/parser` | Validates the supported grammar and builds a parse tree. | | `includes/shunting_yard` | Converts compound filter expressions to postfix form. | | `includes/map` | Implements B+ tree-backed maps and multimaps. | -| `includes/binary_files` | Reads and writes fixed-format binary records. | +| `includes/binary_files` | Reads the legacy v1 fixed-width format. | +| `includes/storage` | Owns v2 pages, persistent indexes, checksums, WAL, and recovery. | | `includes/table` | Owns table persistence, indexes, projections, and selections. | | `includes/sql` | Coordinates parsing and table operations through the public `SQL` interface. | @@ -51,7 +55,7 @@ The build has no downloaded test framework or other third-party dependency. ctest --test-dir build --output-on-failure ``` -The test executables verify inserts, projections, compound filters, persistence, table and row validation, corrupt-file handling, quote and parenthesis safety, deterministic B+ tree properties, and both repository batch fixtures. +The test executables verify SQL behavior, persistent incremental B+ tree splits, stable untouched pages, WAL ordering, dynamically enumerated crash boundaries, corruption rejection, idempotent recovery, migration equivalence, maximum-width rows, CLI inspection, parser safety, and both repository batch fixtures. Tests run in an isolated build directory, so they do not leave database files in the source tree. To run the same AddressSanitizer and UndefinedBehaviorSanitizer gate used by CI: @@ -73,8 +77,8 @@ ctest --test-dir build-sanitized --output-on-failure ``` Type `end` to exit and `cls` to clear the terminal. -The CLI writes each table to `
.bin` and `
_fields.bin` in the current working directory. -It reports missing tables, invalid projections, wrong row arity, duplicate schemas, malformed filters, and corrupt fixed-width files without terminating the session. +The CLI writes each table to `
.sql2` and uses `
.sql2-wal` during a mutating statement. +It reports missing tables, invalid projections, wrong row arity, duplicate schemas, malformed filters, corrupt pages, and corrupt WAL records without terminating the session. Example session: @@ -99,17 +103,46 @@ select <* | field [, field ...]> from
Quote values that contain spaces, such as `"Mary Ann"`. Quoted values may contain spaces, numbers, and punctuation, but escaped double quotes are not supported. +## Version and storage inspection + +```bash +./build/run_sql --version +./build/run_sql --inspect-storage student +``` + +Inspection validates checksums and structural bounds without applying or clearing a pending WAL. + +## Migrate a v1 table + +Inspect a legacy table without writing anything: + +```bash +./build/sql_migrate_v1 --dry-run student +``` + +Migrate it to `student.sql2`: + +```bash +./build/sql_migrate_v1 student +``` + +The migration reads the v1 `.bin` and `_fields.bin` files read-only, writes an exclusively owned temporary v2 image, reopens and compares every row, rechecks the source bytes, then atomically links the verified destination without replacement. +It never modifies or deletes the v1 source. +Use `--destination ` to choose another destination and note that existing destinations are rejected. + ## Persistence boundaries - Tables contain between 1 and 64 uniquely named fields. - Identifiers contain letters, digits, and underscores and cannot start with a digit. -- Each stored field occupies one zero-padded 100-byte slot. +- Each value is at most 100 bytes, and rows may cross page boundaries. - An insert must provide exactly one value per field. - Values longer than 100 bytes are rejected before a write. - Existing tables cannot be overwritten by another `CREATE TABLE` command. -- Partial rows and invalid schema metadata fail closed when a table is opened. +- Page, index, schema, row, and WAL corruption fail closed when a table is opened. +- V1 tables can be queried read-only and must be migrated before new inserts. +- The durability scope is one process, one writer, and one atomic statement at a time. -The fixed-width layout is intentionally simple and is not a transactional or crash-recovery format. +The exact byte layout, recovery ordering, checksums, and compatibility boundary are documented in [docs/storage-v2.md](docs/storage-v2.md). ## License diff --git a/docs/release-checklist-v2.md b/docs/release-checklist-v2.md new file mode 100644 index 0000000..ca5b193 --- /dev/null +++ b/docs/release-checklist-v2.md @@ -0,0 +1,15 @@ +# Version 2.0.0 release checklist + +This checklist prepares the repository for a later release without creating a tag or GitHub Release. + +- [ ] Confirm the CMake project version is `2.0.0`. +- [ ] Confirm `run_sql --version` reports CLI `2.0.0` and storage format `2`. +- [ ] Run the complete Release CTest suite on Linux and macOS. +- [ ] Run AddressSanitizer and UndefinedBehaviorSanitizer with warnings as errors. +- [ ] Migrate the canonical v1 fixtures and compare schemas, rows, and indexed range-query results. +- [ ] Run every deterministic write and recovery crash point. +- [ ] Run superblock, page, WAL, child-pointer, and partial-record corruption tests. +- [ ] Confirm persistent indexes reopen without a row-derived rebuild. +- [ ] Confirm the v1 source files remain byte-identical after migration. +- [ ] Review the documented single-process and single-writer compatibility boundary. +- [ ] Review the pull request CI evidence before a maintainer creates `v2.0.0`. diff --git a/docs/storage-v2.md b/docs/storage-v2.md new file mode 100644 index 0000000..c9f5000 --- /dev/null +++ b/docs/storage-v2.md @@ -0,0 +1,198 @@ +# Storage format v2 + +## Scope + +Storage format v2 is an inspectable, checksummed, paged format for the existing SQL in C++ command set. +It provides atomic single-statement `CREATE TABLE` and `INSERT INTO` persistence for one process and one writer. +It does not provide multi-process locking, concurrent transactions, rollback, update, delete, or broader SQL compatibility. + +The v1 fixed-width files remain readable by the migration tool and are never modified by migration. +The SQL runtime opens a v2 table when `
.sql2` exists. +It otherwise opens a v1 table read-only through the compatibility path. + +## Integer encoding and limits + +All multi-byte integers are unsigned and encoded in little-endian byte order. +The page size is 4096 bytes. +Page identifiers are unsigned 32-bit integers. +Record identifiers and generations are unsigned 64-bit integers. +Strings are byte strings with an unsigned 16-bit length and a maximum length of 100 bytes. +Tables contain between 1 and 64 fields. + +## Page envelope + +Every page begins with a 32-byte header. + +| Offset | Size | Meaning | +| --- | ---: | --- | +| 0 | 4 | ASCII magic `SQL2` | +| 4 | 1 | Page type | +| 5 | 1 | Flags, currently zero | +| 6 | 2 | Format version, currently 2 | +| 8 | 4 | Page identifier | +| 12 | 4 | Payload byte count | +| 16 | 4 | Next page or `0xffffffff` | +| 20 | 4 | CRC32 of the full page with this field zeroed | +| 24 | 8 | Reserved, must be zero | + +Readers reject a wrong magic, version, page identifier, reserved value, payload bound, page type, or checksum. +Reads must be page-aligned and bounded by the page count in the superblock and by the physical file size. +The numeric page types are `1` for the superblock, `2` for schema, `3` for rows, `4` for the index directory, `5` for an index leaf, and `6` for an index internal node. +Unused payload bytes through byte 4095 are zero. +The physical database size must equal `superblock.page_count * 4096` with no partial or trailing pages. + +The checksum is reflected IEEE CRC32. +It uses polynomial `0xedb88320`, initial value `0xffffffff`, and final XOR `0xffffffff`. +The checksum covers all 4096 page bytes after bytes 20 through 23 are set to zero. + +## Superblock + +Page zero has type `superblock`. +Its payload contains the database generation, page size, total page count, schema chain head, row chain head, index-directory chain head, free-page head, field count, and row count. +The free-page head is `0xffffffff` when no reusable page exists. +Page allocation is deterministic and append-only. +The initial image allocates schema, row, index-directory, leaf, and internal pages in that order. +Incremental inserts retain existing page identifiers and append only overflow row pages and split index pages. +Because v2 has no update or delete command, no live page becomes free and the free list remains empty. + +The WAL carries a complete target image for simple idempotent recovery, while normal statement application writes only the buffer's dirty page set. +This makes buffer ownership, flush ordering, crash recovery, and corruption behavior directly inspectable without claiming a production cache or eviction policy. + +The superblock payload begins at physical byte 32 and is exactly 44 bytes. + +| Payload offset | Size | Meaning | +| --- | ---: | --- | +| 0 | 8 | Generation | +| 8 | 4 | Page size, exactly 4096 | +| 12 | 4 | Physical page count | +| 16 | 4 | Schema chain head | +| 20 | 4 | Row chain head | +| 24 | 4 | Index-directory chain head | +| 28 | 4 | Free-page head, currently `0xffffffff` | +| 32 | 4 | Field count | +| 36 | 8 | Row count | + +The in-memory transaction buffer owns the target image and an explicit ordered dirty-page set until the WAL record is durable. +No database page is written from that buffer before WAL fsync. +The crash suite observes the database bytes at the durable-WAL boundary to verify this no-steal flush behavior. + +## Chained payload pages + +Schema, row, and index-directory values are encoded as logical byte streams and split across chained pages. +Each chain is acyclic, terminates with `0xffffffff`, and may contain only its declared page type. +Readers reject cycles, out-of-range links, duplicate page ownership, missing pages, and trailing or truncated logical records. + +The schema stream contains the field count followed by length-prefixed field names. +The row stream contains the row count followed by each row's field count and length-prefixed values. +Rows may cross page boundaries, including a 64-field row containing 100-byte values. + +The index directory stores one root page identifier per field. +An empty field index uses `0xffffffff` as its root. + +The schema stream encoding is `u32 field_count`, followed by exactly `field_count` values encoded as `u16 byte_length` and that many bytes. +The row stream encoding is `u64 row_count`, followed for each row by `u32 field_count`, then exactly that many `u16 byte_length` and byte-string values. +Each sequential row encoding is a logical record slot whose zero-based position is its stable record identifier. +One logical slot may span chained row pages, so the 64-field maximum row does not depend on fitting inside one physical page. +The index-directory stream encoding is `u32 field_count`, followed by exactly that many `u32 root_page_id` values. +Any missing or trailing logical bytes are corruption. + +## Persistent B+ tree pages + +Every indexed field has an independent B+ tree. +Leaf entries are ordered by `(key, record_id)` and contain a length-prefixed key plus a record identifier. +Duplicate keys therefore remain individually addressable and deterministic. +Leaf pages store their previous sibling in the payload and their next sibling in the common page header. + +Internal pages store their tree level, a first child, and ordered composite-separator and right-child pairs. +A separator is the smallest `(key, record_id)` entry reachable through its right child. +The root page identifier is stored in the index directory. +Initial construction fills bounded leaf pages, links siblings, then repeatedly creates bounded parent levels until one root remains. +An insert descends through persisted internal pages, modifies one leaf, splits an overflowing leaf by encoded bytes, repairs adjacent sibling links, propagates composite separators, splits overflowing internal pages, and replaces the root when required. +Untouched page identifiers and bytes remain stable. + +An index-leaf payload starts with `u16 key_count`, `u32 previous_leaf`, and a zero `u16 reserved` value. +Each leaf entry is `u16 key_byte_length`, the key bytes, and `u64 record_id`. +The next-leaf identifier is the common page header's next field. + +An index-internal payload starts with `u16 level`, `u16 key_count`, and `u32 first_child`. +It then contains `key_count` entries encoded as `u16 separator_byte_length`, separator bytes, `u64 separator_record_id`, and `u32 right_child`. +All children of an internal page have level exactly one less than their parent. +Separators may compare equal when duplicate values span leaves, but child ranges may never move backward. + +On open, the runtime validates key counts, entry bounds, ordering, child identifiers, separator ranges, levels, sibling symmetry, leaf-chain completeness, and the exact set of record identifiers. +Queries load the persisted leaf entries and do not rebuild indexes from table rows. + +## Write-ahead log + +The WAL path is `
.sql2-wal`. +A WAL transaction is one complete database after-image with this structure: + +1. A fixed header containing magic `SQL2WAL`, WAL version 1, image byte count, target generation, and an image CRC32. +2. The complete page-aligned database image. +3. A commit footer containing magic `COMMIT2` and a CRC32 of the header and image. + +The WAL header is exactly 40 bytes. + +| Offset | Size | Meaning | +| --- | ---: | --- | +| 0 | 8 | `SQL2WAL` plus a zero byte | +| 8 | 4 | WAL version, exactly 1 | +| 12 | 4 | Header size, exactly 40 | +| 16 | 8 | Image byte count | +| 24 | 8 | Target database generation | +| 32 | 4 | Image CRC32 | +| 36 | 4 | Reserved, exactly zero | + +The WAL footer is exactly 16 bytes at offset `40 + image_byte_count`. +It contains `COMMIT2` plus a zero byte, a CRC32 of the complete header and image, and a zero `u32` reserved value. +The database image must be non-empty and page aligned, and every contained page must validate before redo begins. + +The writer performs these boundaries in order: + +1. Truncate and write the complete WAL record. +2. Flush the stream and fsync the WAL file. +3. Fsync the parent directory if the WAL was newly created. +4. Extend the database to the target length when pages were appended and write dirty pages in page-id order. +5. Flush the stream and fsync the database file. +6. Truncate and fsync the WAL file. + +The statement is acknowledged only after step 6. +Deterministic crash injection is available at the partial-WAL, WAL-write, WAL-fsync, per-page, data-fsync, WAL-truncate, and WAL-clear boundaries. +The crash harness uses abrupt process termination and proves recovery logic and write ordering with the kernel page cache intact. +It does not simulate physical media failure or loss of the operating system page cache, so power-loss durability rests on the documented fsync and directory-fsync contract plus Linux and macOS CI behavior. + +## Recovery + +An empty or absent WAL needs no recovery. +A structurally complete committed WAL with valid checksums is redone by writing every page in page-id order, fsyncing the database, and clearing the WAL. +Redo is idempotent because the WAL contains the complete target image and generation. +A WAL target generation lower than a separately valid database generation is rejected so recovery cannot roll a database backward. +A WAL generation equal to or higher than the valid database generation is safe to redo. +A torn database image is replaced from a fully validated committed WAL without trusting its damaged superblock. + +An incomplete WAL record, invalid commit footer, invalid WAL checksum, non-page-aligned image, or corrupt contained page fails closed with an explicit corruption error. +Recovery never interprets an incomplete record as committed and never silently discards it. + +## Migration and compatibility + +`sql_migrate_v1` accepts a v1 table name and supports dry-run inspection. +Migration validates both v1 files, reads empty strings and exact 100-byte values, writes a v2 image to an exclusively created temporary destination, reopens it, compares every field and row, and rechecks the source bytes. +It then creates the final destination with a same-filesystem hard link that fails if the destination exists, fsyncs the directory, removes the temporary link, and fsyncs the directory again. +The source `.bin` and `_fields.bin` files are never opened for writing, renamed, truncated, or deleted. + +The v2 runtime prefers `
.sql2` when both formats exist. +The migration tool refuses to replace an existing destination. +There is no in-place upgrade and no downgrade writer. + +## Inspection + +`run_sql --version` reports the CLI and storage format versions. +`run_sql --inspect-storage
` reports the format, generation, page count, row count, field count, persistent index roots, and WAL state without changing the table. +Inspection applies the same bounded-read and checksum validation as a normal open. +When a valid committed WAL is pending, inspection validates and reports the target image from the WAL without applying it or clearing the WAL. + +## Durability boundary + +The fsync implementation uses the native POSIX file descriptor on Linux and macOS. +Directory fsync is performed after first database creation and migration promotion so the directory entry is durable. +The format makes no durability claim for unsupported filesystems or platforms where these primitives are unavailable. diff --git a/includes/binary_files/file_record.cpp b/includes/binary_files/file_record.cpp index 0c6bd6a..9a96ccb 100644 --- a/includes/binary_files/file_record.cpp +++ b/includes/binary_files/file_record.cpp @@ -31,7 +31,7 @@ long FileRecord::write(fstream &outs) { } // Write the record to the end of the file -long FileRecord::read(fstream &ins, long recno) { +long FileRecord::read(istream &ins, long recno) { if (_records.empty() || recno < 0) { throw std::invalid_argument("Record shape and number must be valid"); } diff --git a/includes/binary_files/file_record.h b/includes/binary_files/file_record.h index d9e7fd0..6de35aa 100644 --- a/includes/binary_files/file_record.h +++ b/includes/binary_files/file_record.h @@ -17,7 +17,7 @@ class FileRecord{ explicit FileRecord(vector values); long write(fstream& outs); - long read(fstream& ins, long recno); + long read(istream& ins, long recno); std::size_t encoded_size() const { return MAX_VALUE_LENGTH * _records.size(); } vector get_records_string(){ return this->_records;} diff --git a/includes/storage/page_buffer.cpp b/includes/storage/page_buffer.cpp new file mode 100644 index 0000000..ae12c02 --- /dev/null +++ b/includes/storage/page_buffer.cpp @@ -0,0 +1,48 @@ +#include "page_buffer.h" + +#include +#include + +namespace storage_v2 { + +PageBuffer::PageBuffer( + std::vector image, + std::set dirty_pages +) + : image_(std::move(image)), dirty_pages_(std::move(dirty_pages)) { + if (image_.empty() || image_.size() % PAGE_BYTES != 0) { + throw std::invalid_argument("Page buffer requires a non-empty complete page image"); + } + if (page_count() > std::numeric_limits::max()) { + throw std::length_error("Page buffer contains too many pages"); + } + if (dirty_pages_.empty()) { + for (std::uint32_t page = 0; page < page_count(); ++page) dirty_pages_.insert(page); + } + if (*dirty_pages_.rbegin() >= page_count()) { + throw std::out_of_range("Dirty page is outside the owned image"); + } +} + +std::size_t PageBuffer::page_count() const { + return image_.size() / PAGE_BYTES; +} + +const std::uint8_t* PageBuffer::page_data(std::uint32_t page_id) const { + if (page_id >= page_count()) { + throw std::out_of_range("Page buffer read is outside the owned image"); + } + return image_.data() + static_cast(page_id) * PAGE_BYTES; +} + +void PageBuffer::visit_pages(const Visitor& visitor) const { + for (std::uint32_t page_id = 0; page_id < page_count(); ++page_id) { + visitor(page_id, page_data(page_id)); + } +} + +void PageBuffer::visit_dirty_pages(const Visitor& visitor) const { + for (const auto page_id : dirty_pages_) visitor(page_id, page_data(page_id)); +} + +} // namespace storage_v2 diff --git a/includes/storage/page_buffer.h b/includes/storage/page_buffer.h new file mode 100644 index 0000000..d92cf15 --- /dev/null +++ b/includes/storage/page_buffer.h @@ -0,0 +1,37 @@ +#ifndef STORAGE_PAGE_BUFFER_H +#define STORAGE_PAGE_BUFFER_H + +#include +#include +#include +#include +#include +#include + +namespace storage_v2 { + +class PageBuffer { +public: + static constexpr std::size_t PAGE_BYTES = 4096; + using Visitor = std::function; + + explicit PageBuffer( + std::vector image, + std::set dirty_pages = {} + ); + + std::size_t page_count() const; + const std::uint8_t* page_data(std::uint32_t page_id) const; + const std::vector& bytes() const { return image_; } + void visit_pages(const Visitor& visitor) const; + void visit_dirty_pages(const Visitor& visitor) const; + const std::set& dirty_pages() const { return dirty_pages_; } + +private: + std::vector image_; + std::set dirty_pages_; +}; + +} // namespace storage_v2 + +#endif diff --git a/includes/storage/storage_v2.cpp b/includes/storage/storage_v2.cpp new file mode 100644 index 0000000..b960701 --- /dev/null +++ b/includes/storage/storage_v2.cpp @@ -0,0 +1,1422 @@ +#include "storage_v2.h" +#include "page_buffer.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace storage_v2 { +namespace { + +constexpr std::size_t PAGE_HEADER_SIZE = 32; +constexpr std::size_t PAGE_PAYLOAD_SIZE = PAGE_SIZE - PAGE_HEADER_SIZE; +constexpr std::uint32_t NO_PAGE = std::numeric_limits::max(); +constexpr std::size_t MAX_FIELDS = 64; +constexpr std::size_t MAX_VALUE_LENGTH = 100; + +enum class PageType : std::uint8_t { + Superblock = 1, + Schema = 2, + Rows = 3, + IndexDirectory = 4, + IndexLeaf = 5, + IndexInternal = 6, +}; + +using Bytes = std::vector; +using PageBytes = std::array; + +struct Page { + PageType type; + std::uint32_t id; + std::uint32_t next = NO_PAGE; + Bytes payload; +}; + +struct Superblock { + std::uint64_t generation = 0; + std::uint32_t page_count = 0; + std::uint32_t schema_head = NO_PAGE; + std::uint32_t rows_head = NO_PAGE; + std::uint32_t index_directory_head = NO_PAGE; + std::uint32_t free_head = NO_PAGE; + std::uint32_t field_count = 0; + std::uint64_t row_count = 0; +}; + +struct ChildDescriptor { + std::uint32_t id; + std::string minimum_key; + std::uint64_t minimum_record; + std::uint16_t level; +}; + +struct ParsedImage { + LoadedTable table; + std::vector pages; +}; + +std::uint32_t crc32(const std::uint8_t* data, std::size_t size) { + std::uint32_t crc = 0xffffffffU; + for (std::size_t i = 0; i < size; ++i) { + crc ^= data[i]; + for (int bit = 0; bit < 8; ++bit) { + const std::uint32_t mask = 0U - (crc & 1U); + crc = (crc >> 1U) ^ (0xedb88320U & mask); + } + } + return ~crc; +} + +void append_u16(Bytes& output, std::uint16_t value) { + output.push_back(static_cast(value & 0xffU)); + output.push_back(static_cast((value >> 8U) & 0xffU)); +} + +void append_u32(Bytes& output, std::uint32_t value) { + for (int shift = 0; shift < 32; shift += 8) { + output.push_back(static_cast((value >> shift) & 0xffU)); + } +} + +void append_u64(Bytes& output, std::uint64_t value) { + for (int shift = 0; shift < 64; shift += 8) { + output.push_back(static_cast((value >> shift) & 0xffU)); + } +} + +void append_string(Bytes& output, const std::string& value) { + if (value.size() > MAX_VALUE_LENGTH) { + throw std::length_error("Values must not exceed 100 bytes"); + } + append_u16(output, static_cast(value.size())); + output.insert(output.end(), value.begin(), value.end()); +} + +class Reader { +public: + Reader(const Bytes& bytes, const std::string& context) + : bytes_(bytes), context_(context) {} + + std::uint16_t u16() { + require(2); + const auto value = static_cast(bytes_[position_]) | + (static_cast(bytes_[position_ + 1]) << 8U); + position_ += 2; + return value; + } + + std::uint32_t u32() { + require(4); + std::uint32_t value = 0; + for (int shift = 0; shift < 32; shift += 8) { + value |= static_cast(bytes_[position_++]) << shift; + } + return value; + } + + std::uint64_t u64() { + require(8); + std::uint64_t value = 0; + for (int shift = 0; shift < 64; shift += 8) { + value |= static_cast(bytes_[position_++]) << shift; + } + return value; + } + + std::string string() { + const auto length = u16(); + if (length > MAX_VALUE_LENGTH) { + throw CorruptionError(context_ + " string exceeds 100 bytes"); + } + require(length); + std::string value( + bytes_.begin() + static_cast(position_), + bytes_.begin() + static_cast(position_ + length) + ); + position_ += length; + return value; + } + + std::size_t remaining() const { return bytes_.size() - position_; } + std::size_t position() const { return position_; } + +private: + void require(std::size_t count) const { + if (count > bytes_.size() - position_) { + throw CorruptionError(context_ + " contains a partial record"); + } + } + + const Bytes& bytes_; + std::string context_; + std::size_t position_ = 0; +}; + +void validate_schema( + const std::vector& fields, + const std::vector>& rows +) { + if (fields.empty() || fields.size() > MAX_FIELDS) { + throw std::invalid_argument("A table must have between 1 and 64 fields"); + } + std::set unique; + for (const auto& field : fields) { + if (field.empty() || field.size() > MAX_VALUE_LENGTH) { + throw std::invalid_argument("Field names must be between 1 and 100 bytes"); + } + if (!unique.insert(field).second) { + throw std::invalid_argument("Duplicate field name: " + field); + } + } + for (const auto& row : rows) { + if (row.size() != fields.size()) { + throw std::invalid_argument("Stored row field count does not match schema"); + } + for (const auto& value : row) { + if (value.size() > MAX_VALUE_LENGTH) { + throw std::length_error("Values must not exceed 100 bytes"); + } + } + } +} + +PageBytes encode_page(const Page& page) { + if (page.payload.size() > PAGE_PAYLOAD_SIZE) { + throw std::logic_error("Page payload exceeds its bound"); + } + PageBytes output{}; + output[0] = 'S'; + output[1] = 'Q'; + output[2] = 'L'; + output[3] = '2'; + output[4] = static_cast(page.type); + output[5] = 0; + output[6] = static_cast(FORMAT_VERSION & 0xffU); + output[7] = static_cast(FORMAT_VERSION >> 8U); + const auto put32 = [&output](std::size_t offset, std::uint32_t value) { + for (int shift = 0; shift < 32; shift += 8) { + output[offset++] = static_cast((value >> shift) & 0xffU); + } + }; + put32(8, page.id); + put32(12, static_cast(page.payload.size())); + put32(16, page.next); + std::copy(page.payload.begin(), page.payload.end(), output.begin() + PAGE_HEADER_SIZE); + put32(20, crc32(output.data(), output.size())); + return output; +} + +std::uint32_t read_u32_at(const std::uint8_t* bytes) { + std::uint32_t value = 0; + for (int shift = 0; shift < 32; shift += 8) { + value |= static_cast(*bytes++) << shift; + } + return value; +} + +Page decode_page(const std::uint8_t* bytes, std::uint32_t expected_id) { + if (!(bytes[0] == 'S' && bytes[1] == 'Q' && bytes[2] == 'L' && bytes[3] == '2')) { + throw CorruptionError("page " + std::to_string(expected_id) + " has invalid magic"); + } + const auto version = static_cast(bytes[6]) | + (static_cast(bytes[7]) << 8U); + if (version != FORMAT_VERSION) { + throw CorruptionError("page " + std::to_string(expected_id) + " has unsupported version"); + } + if (bytes[5] != 0) { + throw CorruptionError("page " + std::to_string(expected_id) + " has unsupported flags"); + } + for (std::size_t i = 24; i < PAGE_HEADER_SIZE; ++i) { + if (bytes[i] != 0) { + throw CorruptionError("page " + std::to_string(expected_id) + " reserved bytes are not zero"); + } + } + if (read_u32_at(bytes + 8) != expected_id) { + throw CorruptionError("page identifier does not match its file offset"); + } + const auto payload_size = read_u32_at(bytes + 12); + if (payload_size > PAGE_PAYLOAD_SIZE) { + throw CorruptionError("page payload exceeds its bound"); + } + for (std::size_t i = PAGE_HEADER_SIZE + payload_size; i < PAGE_SIZE; ++i) { + if (bytes[i] != 0) { + throw CorruptionError("page " + std::to_string(expected_id) + " unused bytes are not zero"); + } + } + PageBytes checksum_page{}; + std::copy(bytes, bytes + PAGE_SIZE, checksum_page.begin()); + const auto expected_checksum = read_u32_at(bytes + 20); + std::fill(checksum_page.begin() + 20, checksum_page.begin() + 24, 0); + if (crc32(checksum_page.data(), checksum_page.size()) != expected_checksum) { + throw CorruptionError("page " + std::to_string(expected_id) + " checksum mismatch"); + } + const auto raw_type = bytes[4]; + if (raw_type < static_cast(PageType::Superblock) || + raw_type > static_cast(PageType::IndexInternal)) { + throw CorruptionError("page " + std::to_string(expected_id) + " has invalid type"); + } + Page page{ + static_cast(raw_type), + expected_id, + read_u32_at(bytes + 16), + {}, + }; + page.payload.assign(bytes + PAGE_HEADER_SIZE, bytes + PAGE_HEADER_SIZE + payload_size); + return page; +} + +std::uint32_t add_chain( + std::vector& pages, + PageType type, + const Bytes& bytes +) { + if (bytes.empty()) { + Page page{type, static_cast(pages.size()), NO_PAGE, {}}; + pages.push_back(page); + return page.id; + } + const auto first = static_cast(pages.size()); + std::size_t offset = 0; + while (offset < bytes.size()) { + const auto size = std::min(PAGE_PAYLOAD_SIZE, bytes.size() - offset); + Page page{ + type, + static_cast(pages.size()), + NO_PAGE, + Bytes(bytes.begin() + static_cast(offset), + bytes.begin() + static_cast(offset + size)), + }; + if (!pages.empty() && pages.back().type == type && pages.back().next == NO_PAGE && + pages.back().id >= first) { + pages.back().next = page.id; + } + pages.push_back(std::move(page)); + offset += size; + } + return first; +} + +std::uint32_t build_index( + std::vector& pages, + const std::vector>& entries +) { + if (entries.empty()) return NO_PAGE; + + std::vector level; + std::vector leaf_positions; + std::size_t cursor = 0; + while (cursor < entries.size()) { + Bytes payload; + append_u16(payload, 0); + append_u32(payload, NO_PAGE); + append_u16(payload, 0); + std::uint16_t count = 0; + const auto first_entry = entries[cursor]; + while (cursor < entries.size()) { + const auto entry_size = 2U + entries[cursor].first.size() + 8U; + if (payload.size() + entry_size > PAGE_PAYLOAD_SIZE) break; + append_string(payload, entries[cursor].first); + append_u64(payload, entries[cursor].second); + ++count; + ++cursor; + } + if (count == 0) throw std::logic_error("Index entry does not fit in a page"); + payload[0] = static_cast(count & 0xffU); + payload[1] = static_cast(count >> 8U); + const auto id = static_cast(pages.size()); + leaf_positions.push_back(pages.size()); + pages.push_back(Page{PageType::IndexLeaf, id, NO_PAGE, std::move(payload)}); + level.push_back(ChildDescriptor{id, first_entry.first, first_entry.second, 0}); + } + for (std::size_t i = 0; i < leaf_positions.size(); ++i) { + auto& leaf = pages[leaf_positions[i]]; + leaf.next = i + 1 < leaf_positions.size() + ? pages[leaf_positions[i + 1]].id + : NO_PAGE; + const auto previous = i == 0 ? NO_PAGE : pages[leaf_positions[i - 1]].id; + for (int shift = 0; shift < 32; shift += 8) { + leaf.payload[2 + static_cast(shift / 8)] = + static_cast((previous >> shift) & 0xffU); + } + } + + while (level.size() > 1) { + std::vector parents; + std::size_t child = 0; + while (child < level.size()) { + const auto begin = child; + std::size_t end = child + 1; + std::size_t encoded_size = 8; + while (end < level.size()) { + const auto entry_size = 2U + level[end].minimum_key.size() + 8U + 4U; + if (encoded_size + entry_size > PAGE_PAYLOAD_SIZE) break; + encoded_size += entry_size; + ++end; + } + if (level.size() - end == 1 && end - begin > 2) --end; + if (end - begin < 2) { + throw std::logic_error("Internal index page must contain at least two children"); + } + Bytes payload; + append_u16(payload, static_cast(level[child].level + 1)); + append_u16(payload, 0); + append_u32(payload, level[child].id); + ++child; + std::uint16_t key_count = 0; + while (child < end) { + append_string(payload, level[child].minimum_key); + append_u64(payload, level[child].minimum_record); + append_u32(payload, level[child].id); + ++key_count; + ++child; + } + payload[2] = static_cast(key_count & 0xffU); + payload[3] = static_cast(key_count >> 8U); + const auto id = static_cast(pages.size()); + pages.push_back(Page{PageType::IndexInternal, id, NO_PAGE, std::move(payload)}); + parents.push_back(ChildDescriptor{ + id, + level[begin].minimum_key, + level[begin].minimum_record, + static_cast(level[begin].level + 1), + }); + } + level = std::move(parents); + } + return level.front().id; +} + +Bytes build_image( + const std::vector& fields, + const std::vector>& rows, + std::uint64_t generation +) { + validate_schema(fields, rows); + std::vector pages; + pages.push_back(Page{PageType::Superblock, 0, NO_PAGE, {}}); + + Bytes schema; + append_u32(schema, static_cast(fields.size())); + for (const auto& field : fields) append_string(schema, field); + const auto schema_head = add_chain(pages, PageType::Schema, schema); + + Bytes row_bytes; + append_u64(row_bytes, static_cast(rows.size())); + for (const auto& row : rows) { + append_u32(row_bytes, static_cast(row.size())); + for (const auto& value : row) append_string(row_bytes, value); + } + const auto rows_head = add_chain(pages, PageType::Rows, row_bytes); + + const auto index_directory_position = pages.size(); + const auto index_directory_head = static_cast(pages.size()); + pages.push_back(Page{PageType::IndexDirectory, index_directory_head, NO_PAGE, {}}); + + std::vector roots; + roots.reserve(fields.size()); + for (std::size_t field = 0; field < fields.size(); ++field) { + std::vector> entries; + entries.reserve(rows.size()); + for (std::size_t row = 0; row < rows.size(); ++row) { + entries.emplace_back(rows[row][field], static_cast(row)); + } + std::sort(entries.begin(), entries.end()); + roots.push_back(build_index(pages, entries)); + } + + Bytes index_directory; + append_u32(index_directory, static_cast(roots.size())); + for (const auto root : roots) append_u32(index_directory, root); + pages[index_directory_position].payload = std::move(index_directory); + + Bytes superblock; + append_u64(superblock, generation); + append_u32(superblock, PAGE_SIZE); + append_u32(superblock, static_cast(pages.size())); + append_u32(superblock, schema_head); + append_u32(superblock, rows_head); + append_u32(superblock, index_directory_head); + append_u32(superblock, NO_PAGE); + append_u32(superblock, static_cast(fields.size())); + append_u64(superblock, static_cast(rows.size())); + pages[0].payload = std::move(superblock); + + Bytes image; + image.reserve(pages.size() * PAGE_SIZE); + for (const auto& page : pages) { + const auto encoded = encode_page(page); + image.insert(image.end(), encoded.begin(), encoded.end()); + } + return image; +} + +Bytes read_file(const std::string& path) { + std::ifstream input(path, std::ios::binary); + if (!input) throw std::runtime_error("Table does not exist: " + path); + input.seekg(0, std::ios::end); + const auto length = input.tellg(); + if (length < 0) throw std::runtime_error("Unable to determine file size: " + path); + input.seekg(0); + Bytes bytes(static_cast(length)); + if (!bytes.empty()) { + input.read(reinterpret_cast(bytes.data()), static_cast(bytes.size())); + if (input.gcount() != static_cast(bytes.size())) { + throw std::runtime_error("Unable to read complete file: " + path); + } + } + return bytes; +} + +Bytes read_chain( + const std::vector& pages, + std::uint32_t head, + PageType expected_type, + std::set& owned +) { + Bytes output; + std::set chain; + auto current = head; + while (current != NO_PAGE) { + if (current >= pages.size()) throw CorruptionError("page chain points outside the file"); + if (!chain.insert(current).second) throw CorruptionError("page chain contains a cycle"); + if (!owned.insert(current).second) throw CorruptionError("page belongs to multiple structures"); + const auto& page = pages[current]; + if (page.type != expected_type) throw CorruptionError("page chain contains the wrong page type"); + output.insert(output.end(), page.payload.begin(), page.payload.end()); + current = page.next; + } + return output; +} + +struct ValidatedNode { + std::string minimum_key; + std::string maximum_key; + std::uint16_t level = 0; + std::vector> entries; + std::vector leaves; +}; + +ValidatedNode validate_index_node( + const std::vector& pages, + std::uint32_t id, + std::set& owned, + std::set& active +) { + if (id >= pages.size()) throw CorruptionError("index child pointer is outside the file"); + if (!active.insert(id).second) throw CorruptionError("index contains a child cycle"); + if (!owned.insert(id).second) throw CorruptionError("index page is referenced more than once"); + const auto& page = pages[id]; + ValidatedNode result; + if (page.type == PageType::IndexLeaf) { + Reader reader(page.payload, "index leaf"); + const auto count = reader.u16(); + static_cast(reader.u32()); + if (reader.u16() != 0) throw CorruptionError("index leaf reserved value is not zero"); + if (count == 0) throw CorruptionError("index leaf is empty"); + for (std::uint16_t i = 0; i < count; ++i) { + result.entries.emplace_back(reader.string(), reader.u64()); + } + if (reader.remaining() != 0) throw CorruptionError("index leaf has trailing bytes"); + if (!std::is_sorted(result.entries.begin(), result.entries.end())) { + throw CorruptionError("index leaf entries are not ordered"); + } + result.minimum_key = result.entries.front().first; + result.maximum_key = result.entries.back().first; + result.leaves.push_back(id); + } else if (page.type == PageType::IndexInternal) { + if (page.next != NO_PAGE) throw CorruptionError("internal index page has a sibling link"); + Reader reader(page.payload, "internal index page"); + const auto level = reader.u16(); + const auto key_count = reader.u16(); + if (level == 0 || key_count == 0) throw CorruptionError("internal index key count is invalid"); + std::vector> separators; + std::vector children; + children.push_back(reader.u32()); + for (std::uint16_t i = 0; i < key_count; ++i) { + const auto key = reader.string(); + separators.emplace_back(key, reader.u64()); + children.push_back(reader.u32()); + } + if (reader.remaining() != 0) throw CorruptionError("internal index page has trailing bytes"); + if (!std::is_sorted(separators.begin(), separators.end())) { + throw CorruptionError("internal index separators are not ordered"); + } + result.level = level; + for (std::size_t child_index = 0; child_index < children.size(); ++child_index) { + auto child = validate_index_node(pages, children[child_index], owned, active); + if (child.level + 1 != level) throw CorruptionError("index child level is invalid"); + if (child_index > 0 && separators[child_index - 1] != child.entries.front()) { + throw CorruptionError("index separator does not match child minimum"); + } + if (!result.entries.empty() && result.entries.back() > child.entries.front()) { + throw CorruptionError("index child ranges overlap out of order"); + } + result.entries.insert(result.entries.end(), child.entries.begin(), child.entries.end()); + result.leaves.insert(result.leaves.end(), child.leaves.begin(), child.leaves.end()); + if (child_index == 0) result.minimum_key = child.minimum_key; + result.maximum_key = child.maximum_key; + } + } else { + throw CorruptionError("index root or child has the wrong page type"); + } + active.erase(id); + return result; +} + +ParsedImage parse_image(const Bytes& image, bool wal_present) { + if (image.empty() || image.size() % PAGE_SIZE != 0) { + throw CorruptionError("database file is not a complete page image"); + } + const auto page_count = image.size() / PAGE_SIZE; + if (page_count > std::numeric_limits::max()) { + throw CorruptionError("database contains too many pages"); + } + std::vector pages; + pages.reserve(page_count); + for (std::uint32_t id = 0; id < page_count; ++id) { + pages.push_back(decode_page(image.data() + static_cast(id) * PAGE_SIZE, id)); + } + if (pages[0].type != PageType::Superblock || pages[0].next != NO_PAGE) { + throw CorruptionError("page zero is not a standalone superblock"); + } + Reader super_reader(pages[0].payload, "superblock"); + Superblock super; + super.generation = super_reader.u64(); + if (super_reader.u32() != PAGE_SIZE) throw CorruptionError("superblock page size is invalid"); + super.page_count = super_reader.u32(); + super.schema_head = super_reader.u32(); + super.rows_head = super_reader.u32(); + super.index_directory_head = super_reader.u32(); + super.free_head = super_reader.u32(); + super.field_count = super_reader.u32(); + super.row_count = super_reader.u64(); + if (super_reader.remaining() != 0) throw CorruptionError("superblock has trailing bytes"); + if (super.page_count != pages.size()) throw CorruptionError("superblock page count is invalid"); + if (super.free_head != NO_PAGE) throw CorruptionError("unsupported non-empty free-page list"); + if (super.field_count == 0 || super.field_count > MAX_FIELDS) { + throw CorruptionError("superblock field count is invalid"); + } + + std::set owned{0}; + const auto schema_bytes = read_chain(pages, super.schema_head, PageType::Schema, owned); + Reader schema_reader(schema_bytes, "schema"); + if (schema_reader.u32() != super.field_count) throw CorruptionError("schema field count mismatch"); + std::vector fields; + for (std::uint32_t i = 0; i < super.field_count; ++i) fields.push_back(schema_reader.string()); + if (schema_reader.remaining() != 0) throw CorruptionError("schema has trailing bytes"); + std::set unique_fields(fields.begin(), fields.end()); + if (unique_fields.size() != fields.size() || unique_fields.count("") != 0) { + throw CorruptionError("schema contains empty or duplicate fields"); + } + + const auto row_bytes = read_chain(pages, super.rows_head, PageType::Rows, owned); + Reader row_reader(row_bytes, "row stream"); + if (row_reader.u64() != super.row_count) throw CorruptionError("row count mismatch"); + std::vector> rows; + if (super.row_count > std::numeric_limits::max()) { + throw CorruptionError("row count exceeds addressable memory"); + } + rows.reserve(static_cast(super.row_count)); + for (std::uint64_t row = 0; row < super.row_count; ++row) { + if (row_reader.u32() != super.field_count) throw CorruptionError("stored row field count mismatch"); + std::vector values; + values.reserve(super.field_count); + for (std::uint32_t field = 0; field < super.field_count; ++field) { + values.push_back(row_reader.string()); + } + rows.push_back(std::move(values)); + } + if (row_reader.remaining() != 0) throw CorruptionError("row stream has trailing bytes"); + + const auto directory_bytes = read_chain( + pages, + super.index_directory_head, + PageType::IndexDirectory, + owned + ); + Reader directory_reader(directory_bytes, "index directory"); + if (directory_reader.u32() != super.field_count) { + throw CorruptionError("index directory field count mismatch"); + } + std::vector roots; + for (std::uint32_t i = 0; i < super.field_count; ++i) roots.push_back(directory_reader.u32()); + if (directory_reader.remaining() != 0) throw CorruptionError("index directory has trailing bytes"); + + std::vector>> indexes; + indexes.reserve(fields.size()); + for (std::size_t field = 0; field < fields.size(); ++field) { + if (roots[field] == NO_PAGE) { + if (!rows.empty()) throw CorruptionError("non-empty table has an empty index root"); + indexes.emplace_back(); + continue; + } + std::set active; + auto node = validate_index_node(pages, roots[field], owned, active); + if (node.entries.size() != rows.size()) throw CorruptionError("index entry count mismatch"); + for (std::size_t leaf = 0; leaf < node.leaves.size(); ++leaf) { + const auto& page = pages[node.leaves[leaf]]; + Reader leaf_reader(page.payload, "index leaf"); + static_cast(leaf_reader.u16()); + const auto previous = leaf_reader.u32(); + const auto expected_previous = leaf == 0 ? NO_PAGE : node.leaves[leaf - 1]; + const auto expected_next = leaf + 1 < node.leaves.size() ? node.leaves[leaf + 1] : NO_PAGE; + if (previous != expected_previous || page.next != expected_next) { + throw CorruptionError("index leaf sibling chain is invalid"); + } + } + std::vector seen(rows.size(), false); + for (const auto& entry : node.entries) { + if (entry.second >= rows.size()) throw CorruptionError("index record identifier is out of range"); + const auto record = static_cast(entry.second); + if (seen[record]) throw CorruptionError("index record identifier is duplicated"); + seen[record] = true; + if (entry.first != rows[record][field]) throw CorruptionError("index key does not match stored row"); + } + indexes.push_back(std::move(node.entries)); + } + if (owned.size() != pages.size()) throw CorruptionError("database contains unreferenced pages"); + + Inspection inspection; + inspection.generation = super.generation; + inspection.page_count = super.page_count; + inspection.row_count = super.row_count; + inspection.field_count = super.field_count; + inspection.index_roots = roots; + inspection.wal_present = wal_present; + return ParsedImage{LoadedTable{fields, rows, indexes, inspection}, std::move(pages)}; +} + +void write_u32(Bytes& bytes, std::size_t offset, std::uint32_t value) { + if (offset + 4 > bytes.size()) throw std::logic_error("u32 write exceeds payload"); + for (int shift = 0; shift < 32; shift += 8) { + bytes[offset++] = static_cast((value >> shift) & 0xffU); + } +} + +void write_u64(Bytes& bytes, std::size_t offset, std::uint64_t value) { + if (offset + 8 > bytes.size()) throw std::logic_error("u64 write exceeds payload"); + for (int shift = 0; shift < 64; shift += 8) { + bytes[offset++] = static_cast((value >> shift) & 0xffU); + } +} + +Bytes encode_pages(const std::vector& pages) { + Bytes image; + image.reserve(pages.size() * PAGE_SIZE); + for (const auto& page : pages) { + const auto encoded = encode_page(page); + image.insert(image.end(), encoded.begin(), encoded.end()); + } + return image; +} + +struct LeafContents { + std::uint32_t previous; + std::vector> entries; +}; + +LeafContents decode_leaf_contents(const Page& page) { + if (page.type != PageType::IndexLeaf) throw std::logic_error("Expected an index leaf"); + Reader reader(page.payload, "index leaf"); + const auto count = reader.u16(); + const auto previous = reader.u32(); + if (reader.u16() != 0) throw CorruptionError("index leaf reserved value is not zero"); + std::vector> entries; + entries.reserve(count); + for (std::uint16_t index = 0; index < count; ++index) { + entries.emplace_back(reader.string(), reader.u64()); + } + if (reader.remaining() != 0) throw CorruptionError("index leaf has trailing bytes"); + return LeafContents{previous, std::move(entries)}; +} + +Bytes encode_leaf_payload( + std::uint32_t previous, + const std::vector>& entries +) { + if (entries.empty() || entries.size() > std::numeric_limits::max()) { + throw std::logic_error("Index leaf entry count is invalid"); + } + Bytes payload; + append_u16(payload, static_cast(entries.size())); + append_u32(payload, previous); + append_u16(payload, 0); + for (const auto& entry : entries) { + append_string(payload, entry.first); + append_u64(payload, entry.second); + } + return payload; +} + +struct InternalContents { + std::uint16_t level; + std::vector> separators; + std::vector children; +}; + +InternalContents decode_internal_contents(const Page& page) { + if (page.type != PageType::IndexInternal) throw std::logic_error("Expected an internal index page"); + Reader reader(page.payload, "internal index page"); + const auto level = reader.u16(); + const auto count = reader.u16(); + std::vector> separators; + std::vector children; + separators.reserve(count); + children.reserve(static_cast(count) + 1); + children.push_back(reader.u32()); + for (std::uint16_t index = 0; index < count; ++index) { + const auto key = reader.string(); + separators.emplace_back(key, reader.u64()); + children.push_back(reader.u32()); + } + if (reader.remaining() != 0) throw CorruptionError("internal index page has trailing bytes"); + return InternalContents{level, std::move(separators), std::move(children)}; +} + +Bytes encode_internal_payload(const InternalContents& contents) { + if (contents.level == 0 || contents.children.size() < 2 || + contents.separators.size() + 1 != contents.children.size() || + contents.separators.size() > std::numeric_limits::max()) { + throw std::logic_error("Internal index shape is invalid"); + } + Bytes payload; + append_u16(payload, contents.level); + append_u16(payload, static_cast(contents.separators.size())); + append_u32(payload, contents.children.front()); + for (std::size_t index = 0; index < contents.separators.size(); ++index) { + append_string(payload, contents.separators[index].first); + append_u64(payload, contents.separators[index].second); + append_u32(payload, contents.children[index + 1]); + } + return payload; +} + +struct NodeSplit { + std::pair separator; + std::uint32_t right_page; +}; + +std::uint16_t node_level(const Page& page) { + if (page.type == PageType::IndexLeaf) return 0; + return decode_internal_contents(page).level; +} + +std::optional insert_index_entry( + std::vector& pages, + std::uint32_t page_id, + const std::pair& entry, + std::set& dirty +) { + if (page_id >= pages.size()) throw CorruptionError("index insert points outside the file"); + auto& page = pages[page_id]; + if (page.type == PageType::IndexLeaf) { + auto leaf = decode_leaf_contents(page); + const auto position = std::lower_bound(leaf.entries.begin(), leaf.entries.end(), entry); + if (position != leaf.entries.end() && *position == entry) { + throw CorruptionError("index already contains the inserted record identifier"); + } + leaf.entries.insert(position, entry); + auto payload = encode_leaf_payload(leaf.previous, leaf.entries); + if (payload.size() <= PAGE_PAYLOAD_SIZE) { + page.payload = std::move(payload); + dirty.insert(page_id); + return std::nullopt; + } + + const auto split_at = leaf.entries.size() / 2; + std::vector> left( + leaf.entries.begin(), + leaf.entries.begin() + static_cast(split_at) + ); + std::vector> right( + leaf.entries.begin() + static_cast(split_at), + leaf.entries.end() + ); + const auto old_next = page.next; + const auto right_id = static_cast(pages.size()); + page.payload = encode_leaf_payload(leaf.previous, left); + page.next = right_id; + pages.push_back(Page{ + PageType::IndexLeaf, + right_id, + old_next, + encode_leaf_payload(page_id, right), + }); + dirty.insert(page_id); + dirty.insert(right_id); + if (old_next != NO_PAGE) { + if (old_next >= pages.size() || pages[old_next].type != PageType::IndexLeaf) { + throw CorruptionError("leaf sibling points outside its index"); + } + write_u32(pages[old_next].payload, 2, right_id); + dirty.insert(old_next); + } + return NodeSplit{right.front(), right_id}; + } + if (page.type != PageType::IndexInternal) { + throw CorruptionError("index insert encountered a non-index page"); + } + + auto internal = decode_internal_contents(page); + const auto separator_position = std::upper_bound( + internal.separators.begin(), + internal.separators.end(), + entry + ); + const auto child_index = static_cast( + std::distance(internal.separators.begin(), separator_position) + ); + const auto child_split = insert_index_entry( + pages, + internal.children[child_index], + entry, + dirty + ); + if (!child_split.has_value()) return std::nullopt; + + internal.separators.insert( + internal.separators.begin() + static_cast(child_index), + child_split->separator + ); + internal.children.insert( + internal.children.begin() + static_cast(child_index + 1), + child_split->right_page + ); + auto payload = encode_internal_payload(internal); + if (payload.size() <= PAGE_PAYLOAD_SIZE) { + pages[page_id].payload = std::move(payload); + dirty.insert(page_id); + return std::nullopt; + } + + const auto right_child_begin = internal.children.size() / 2; + if (right_child_begin < 2 || internal.children.size() - right_child_begin < 2) { + throw std::logic_error("Internal split cannot produce two valid nodes"); + } + const auto promoted = internal.separators[right_child_begin - 1]; + InternalContents left{ + internal.level, + std::vector>( + internal.separators.begin(), + internal.separators.begin() + static_cast(right_child_begin - 1) + ), + std::vector( + internal.children.begin(), + internal.children.begin() + static_cast(right_child_begin) + ), + }; + InternalContents right{ + internal.level, + std::vector>( + internal.separators.begin() + static_cast(right_child_begin), + internal.separators.end() + ), + std::vector( + internal.children.begin() + static_cast(right_child_begin), + internal.children.end() + ), + }; + pages[page_id].payload = encode_internal_payload(left); + const auto right_id = static_cast(pages.size()); + pages.push_back(Page{ + PageType::IndexInternal, + right_id, + NO_PAGE, + encode_internal_payload(right), + }); + dirty.insert(page_id); + dirty.insert(right_id); + return NodeSplit{promoted, right_id}; +} + +struct IncrementalImage { + Bytes image; + std::set dirty_pages; + std::uint64_t generation; +}; + +IncrementalImage build_incremental_insert( + const Bytes& current_image, + const std::vector& row +) { + auto parsed = parse_image(current_image, false); + if (row.size() != parsed.table.fields.size()) { + throw std::invalid_argument( + "Expected " + std::to_string(parsed.table.fields.size()) + + " values, received " + std::to_string(row.size()) + ); + } + for (const auto& value : row) { + if (value.size() > MAX_VALUE_LENGTH) throw std::length_error("Values must not exceed 100 bytes"); + } + if (parsed.table.inspection.generation == std::numeric_limits::max()) { + throw std::overflow_error("Database generation is exhausted"); + } + if (parsed.table.rows.size() == std::numeric_limits::max()) { + throw std::overflow_error("Record identifier space is exhausted"); + } + + auto& pages = parsed.pages; + std::set dirty; + Reader super_reader(pages[0].payload, "superblock"); + static_cast(super_reader.u64()); + static_cast(super_reader.u32()); + static_cast(super_reader.u32()); + static_cast(super_reader.u32()); + const auto rows_head = super_reader.u32(); + const auto directory_head = super_reader.u32(); + static_cast(super_reader.u32()); + const auto field_count = super_reader.u32(); + const auto old_row_count = super_reader.u64(); + + if (rows_head >= pages.size() || pages[rows_head].type != PageType::Rows) { + throw CorruptionError("row chain head is invalid"); + } + write_u64(pages[rows_head].payload, 0, old_row_count + 1); + dirty.insert(rows_head); + Bytes encoded_row; + append_u32(encoded_row, field_count); + for (const auto& value : row) append_string(encoded_row, value); + auto row_page = rows_head; + while (pages[row_page].next != NO_PAGE) row_page = pages[row_page].next; + std::size_t row_offset = 0; + while (row_offset < encoded_row.size()) { + auto& target = pages[row_page]; + const auto capacity = PAGE_PAYLOAD_SIZE - target.payload.size(); + const auto copied = std::min(capacity, encoded_row.size() - row_offset); + target.payload.insert( + target.payload.end(), + encoded_row.begin() + static_cast(row_offset), + encoded_row.begin() + static_cast(row_offset + copied) + ); + dirty.insert(row_page); + row_offset += copied; + if (row_offset < encoded_row.size()) { + const auto next = static_cast(pages.size()); + target.next = next; + pages.push_back(Page{PageType::Rows, next, NO_PAGE, {}}); + dirty.insert(next); + row_page = next; + } + } + + if (directory_head >= pages.size() || pages[directory_head].type != PageType::IndexDirectory) { + throw CorruptionError("index directory head is invalid"); + } + Reader directory_reader(pages[directory_head].payload, "index directory"); + if (directory_reader.u32() != field_count) throw CorruptionError("index field count mismatch"); + std::vector roots; + for (std::uint32_t field = 0; field < field_count; ++field) roots.push_back(directory_reader.u32()); + const auto record_id = old_row_count; + for (std::size_t field = 0; field < roots.size(); ++field) { + const auto entry = std::make_pair(row[field], record_id); + if (roots[field] == NO_PAGE) { + const auto root = static_cast(pages.size()); + pages.push_back(Page{ + PageType::IndexLeaf, + root, + NO_PAGE, + encode_leaf_payload(NO_PAGE, {entry}), + }); + roots[field] = root; + dirty.insert(root); + } else { + const auto old_root = roots[field]; + const auto split = insert_index_entry(pages, old_root, entry, dirty); + if (split.has_value()) { + const auto root = static_cast(pages.size()); + InternalContents new_root{ + static_cast(node_level(pages[old_root]) + 1), + {split->separator}, + {old_root, split->right_page}, + }; + pages.push_back(Page{ + PageType::IndexInternal, + root, + NO_PAGE, + encode_internal_payload(new_root), + }); + roots[field] = root; + dirty.insert(root); + } + } + write_u32(pages[directory_head].payload, 4 + field * 4, roots[field]); + } + dirty.insert(directory_head); + + const auto generation = parsed.table.inspection.generation + 1; + write_u64(pages[0].payload, 0, generation); + write_u32(pages[0].payload, 12, static_cast(pages.size())); + write_u64(pages[0].payload, 36, old_row_count + 1); + dirty.insert(0); + auto image = encode_pages(pages); + static_cast(parse_image(image, false)); + return IncrementalImage{std::move(image), std::move(dirty), generation}; +} + +void throw_system_error(const std::string& action, const std::string& path) { + throw std::runtime_error(action + " " + path + ": " + std::strerror(errno)); +} + +void write_all(int descriptor, const std::uint8_t* data, std::size_t size) { + while (size > 0) { + const auto chunk = std::min( + size, + static_cast(std::numeric_limits::max()) + ); + const auto written = ::write(descriptor, data, chunk); + if (written < 0) { + if (errno == EINTR) continue; + throw_system_error("Unable to write", "file descriptor"); + } + if (written == 0) throw std::runtime_error("File write made no progress"); + data += written; + size -= static_cast(written); + } +} + +void maybe_crash(const std::string& point) { + const char* trace_path = std::getenv("SQL_CRASH_TRACE_FILE"); + if (trace_path != nullptr) { + const int trace = ::open(trace_path, O_WRONLY | O_CREAT | O_APPEND, 0600); + if (trace >= 0) { + const auto line = point + "\n"; + write_all(trace, reinterpret_cast(line.data()), line.size()); + ::close(trace); + } + } + const char* configured = std::getenv("SQL_CRASH_AT"); + if (configured != nullptr && point == configured) { + ::_exit(86); + } +} + +void fsync_parent(const std::string& path) { + auto parent = std::filesystem::path(path).parent_path(); + if (parent.empty()) parent = "."; + const int descriptor = ::open(parent.c_str(), O_RDONLY); + if (descriptor < 0) throw_system_error("Unable to open directory", parent.string()); + if (::fsync(descriptor) != 0) { + const int saved = errno; + ::close(descriptor); + errno = saved; + throw_system_error("Unable to fsync directory", parent.string()); + } + ::close(descriptor); +} + +Bytes wal_record(const Bytes& image, std::uint64_t generation) { + Bytes record; + const std::array magic{{'S', 'Q', 'L', '2', 'W', 'A', 'L', 0}}; + record.insert(record.end(), magic.begin(), magic.end()); + append_u32(record, 1); + append_u32(record, 40); + append_u64(record, static_cast(image.size())); + append_u64(record, generation); + append_u32(record, crc32(image.data(), image.size())); + append_u32(record, 0); + record.insert(record.end(), image.begin(), image.end()); + const auto record_checksum = crc32(record.data(), record.size()); + const std::array commit{{'C', 'O', 'M', 'M', 'I', 'T', '2', 0}}; + record.insert(record.end(), commit.begin(), commit.end()); + append_u32(record, record_checksum); + append_u32(record, 0); + return record; +} + +struct WalImage { + Bytes image; + std::uint64_t generation; +}; + +WalImage parse_wal(const Bytes& wal) { + constexpr std::size_t header_size = 40; + constexpr std::size_t footer_size = 16; + if (wal.size() < header_size + footer_size) throw CorruptionError("WAL contains an incomplete record"); + const std::array magic{{'S', 'Q', 'L', '2', 'W', 'A', 'L', 0}}; + if (!std::equal(magic.begin(), magic.end(), wal.begin())) throw CorruptionError("WAL magic is invalid"); + Bytes header(wal.begin() + 8, wal.begin() + static_cast(header_size)); + Reader reader(header, "WAL header"); + if (reader.u32() != 1 || reader.u32() != header_size) throw CorruptionError("WAL version is invalid"); + const auto image_size = reader.u64(); + const auto generation = reader.u64(); + const auto image_checksum = reader.u32(); + if (reader.u32() != 0 || reader.remaining() != 0) throw CorruptionError("WAL header is invalid"); + if (image_size > std::numeric_limits::max() || + image_size != wal.size() - header_size - footer_size) { + throw CorruptionError("WAL image length is invalid"); + } + const auto footer_offset = header_size + static_cast(image_size); + const std::array commit{{'C', 'O', 'M', 'M', 'I', 'T', '2', 0}}; + if (!std::equal(commit.begin(), commit.end(), wal.begin() + static_cast(footer_offset))) { + throw CorruptionError("WAL commit footer is invalid"); + } + const auto stored_record_checksum = read_u32_at(wal.data() + footer_offset + 8); + if (read_u32_at(wal.data() + footer_offset + 12) != 0) throw CorruptionError("WAL footer is invalid"); + if (crc32(wal.data(), footer_offset) != stored_record_checksum) { + throw CorruptionError("WAL record checksum mismatch"); + } + Bytes image( + wal.begin() + static_cast(header_size), + wal.begin() + static_cast(footer_offset) + ); + if (crc32(image.data(), image.size()) != image_checksum) throw CorruptionError("WAL image checksum mismatch"); + const auto parsed = parse_image(image, true); + if (parsed.table.inspection.generation != generation) { + throw CorruptionError("WAL generation does not match its database image"); + } + return WalImage{std::move(image), generation}; +} + +void clear_wal(const std::string& path) { + const int descriptor = ::open(path.c_str(), O_WRONLY | O_CREAT, 0600); + if (descriptor < 0) throw_system_error("Unable to clear WAL", path); + if (::ftruncate(descriptor, 0) != 0) { + const int saved = errno; + ::close(descriptor); + errno = saved; + throw_system_error("Unable to truncate WAL", path); + } + maybe_crash("wal-truncate"); + if (::fsync(descriptor) != 0) { + const int saved = errno; + ::close(descriptor); + errno = saved; + throw_system_error("Unable to fsync WAL", path); + } + ::close(descriptor); +} + +void apply_image(const std::string& path, const PageBuffer& buffer) { + if (buffer.bytes().size() > static_cast(std::numeric_limits::max())) { + throw std::length_error("Database image exceeds the platform file-offset range"); + } + const bool first_creation = !std::filesystem::exists(path); + const int descriptor = ::open(path.c_str(), O_WRONLY | O_CREAT, 0600); + if (descriptor < 0) throw_system_error("Unable to open database", path); + try { + if (::ftruncate(descriptor, static_cast(buffer.bytes().size())) != 0) { + throw_system_error("Unable to size database", path); + } + buffer.visit_dirty_pages([&](std::uint32_t page, const std::uint8_t* page_bytes) { + const auto offset = static_cast(static_cast(page) * PAGE_SIZE); + if (::lseek(descriptor, offset, SEEK_SET) != offset) { + throw_system_error("Unable to seek database", path); + } + write_all(descriptor, page_bytes, PAGE_SIZE); + maybe_crash("page-" + std::to_string(page)); + }); + if (::fsync(descriptor) != 0) throw_system_error("Unable to fsync database", path); + maybe_crash("data-fsync"); + } catch (...) { + ::close(descriptor); + throw; + } + ::close(descriptor); + if (first_creation) { + fsync_parent(path); + maybe_crash("database-directory-fsync"); + } +} + +void commit_image( + const std::string& path, + const Bytes& image, + std::uint64_t generation, + std::set dirty_pages = {} +) { + const PageBuffer buffer(image, std::move(dirty_pages)); + const auto wal = wal_record(buffer.bytes(), generation); + const auto path_wal = wal_path(path); + const bool first_wal_creation = !std::filesystem::exists(path_wal); + const int descriptor = ::open(path_wal.c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0600); + if (descriptor < 0) throw_system_error("Unable to open WAL", path_wal); + if (const char* point = std::getenv("SQL_CRASH_AT"); point != nullptr && + std::string(point) == "wal-partial") { + write_all(descriptor, wal.data(), wal.size() / 2); + ::fsync(descriptor); + ::_exit(86); + } + try { + write_all(descriptor, wal.data(), wal.size()); + maybe_crash("wal-write"); + if (::fsync(descriptor) != 0) throw_system_error("Unable to fsync WAL", path_wal); + maybe_crash("wal-fsync"); + } catch (...) { + ::close(descriptor); + throw; + } + ::close(descriptor); + if (first_wal_creation) { + fsync_parent(path_wal); + maybe_crash("wal-directory-fsync"); + } + apply_image(path, buffer); + clear_wal(path_wal); + maybe_crash("wal-clear"); +} + +void recover(const std::string& path) { + const auto path_wal = wal_path(path); + if (!std::filesystem::exists(path_wal) || std::filesystem::file_size(path_wal) == 0) return; + const auto wal = parse_wal(read_file(path_wal)); + if (std::filesystem::exists(path)) { + bool database_is_valid = false; + std::uint64_t database_generation = 0; + try { + const auto database = parse_image(read_file(path), true); + database_generation = database.table.inspection.generation; + database_is_valid = true; + } catch (const CorruptionError&) { + // A committed WAL is allowed to replace a torn database image. + } + if (database_is_valid && database_generation > wal.generation) { + throw CorruptionError("stale WAL generation would roll back the database"); + } + } + apply_image(path, PageBuffer(wal.image)); + clear_wal(path_wal); +} + +} // namespace + +std::string database_path(const std::string& table_name) { + return table_name + ".sql2"; +} + +std::string wal_path(const std::string& path) { + return path + "-wal"; +} + +bool exists(const std::string& path) { + return std::filesystem::exists(path); +} + +void create_table(const std::string& path, const std::vector& fields) { + create_from_rows(path, fields, {}); +} + +void create_from_rows( + const std::string& path, + const std::vector& fields, + const std::vector>& rows +) { + if (std::filesystem::exists(path)) throw std::runtime_error("Table already exists: " + path); + if (std::filesystem::exists(wal_path(path)) && std::filesystem::file_size(wal_path(path)) != 0) { + throw std::runtime_error("Non-empty WAL already exists: " + wal_path(path)); + } + const auto image = build_image(fields, rows, 1); + commit_image(path, image, 1); +} + +void initialize_owned_empty_file( + const std::string& path, + const std::vector& fields, + const std::vector>& rows +) { + if (!std::filesystem::is_regular_file(path) || std::filesystem::file_size(path) != 0) { + throw std::runtime_error("Owned migration file is not an empty regular file: " + path); + } + if (std::filesystem::exists(wal_path(path))) { + throw std::runtime_error("Owned migration WAL already exists: " + wal_path(path)); + } + const auto image = build_image(fields, rows, 1); + commit_image(path, image, 1); +} + +LoadedTable open_table(const std::string& path) { + recover(path); + const bool wal_present = std::filesystem::exists(wal_path(path)) && + std::filesystem::file_size(wal_path(path)) != 0; + return parse_image(read_file(path), wal_present).table; +} + +void insert_row(const std::string& path, const std::vector& row) { + recover(path); + const auto next = build_incremental_insert(read_file(path), row); + commit_image(path, next.image, next.generation, next.dirty_pages); +} + +std::vector query_index( + const std::string& path, + std::size_t field_index, + const std::string& operation, + const std::string& value +) { + const auto table = open_table(path); + if (field_index >= table.indexes.size()) throw std::invalid_argument("Unknown field index"); + const auto& index = table.indexes[field_index]; + std::vector records; + for (const auto& entry : index) { + bool matches = false; + if (operation == "=") matches = entry.first == value; + else if (operation == "!=") matches = entry.first != value; + else if (operation == "<") matches = entry.first < value; + else if (operation == "<=") matches = entry.first <= value; + else if (operation == ">") matches = entry.first > value; + else if (operation == ">=") matches = entry.first >= value; + else throw std::invalid_argument("Unsupported comparison operator: " + operation); + if (matches) records.push_back(entry.second); + } + return records; +} + +Inspection inspect(const std::string& path) { + const auto path_wal = wal_path(path); + if (std::filesystem::exists(path_wal) && std::filesystem::file_size(path_wal) != 0) { + auto wal = parse_wal(read_file(path_wal)); + if (std::filesystem::exists(path)) { + bool database_is_valid = false; + std::uint64_t database_generation = 0; + try { + const auto database = parse_image(read_file(path), true); + database_generation = database.table.inspection.generation; + database_is_valid = true; + } catch (const CorruptionError&) { + // Inspection may report a valid WAL that can repair a torn database. + } + if (database_is_valid && database_generation > wal.generation) { + throw CorruptionError("stale WAL generation would roll back the database"); + } + } + auto inspection = parse_image(wal.image, true).table.inspection; + inspection.wal_present = true; + return inspection; + } + return parse_image(read_file(path), false).table.inspection; +} + +std::string describe(const std::string& path) { + const auto table = inspect(path); + std::ostringstream output; + output << "path: " << path << '\n'; + output << "storage-format: " << table.format_version << '\n'; + output << "generation: " << table.generation << '\n'; + output << "page-size: " << PAGE_SIZE << '\n'; + output << "page-count: " << table.page_count << '\n'; + output << "field-count: " << table.field_count << '\n'; + output << "row-count: " << table.row_count << '\n'; + output << "index-roots:"; + for (const auto root : table.index_roots) { + if (root == NO_PAGE) output << " none"; + else output << ' ' << root; + } + output << '\n'; + output << "wal: " << (table.wal_present ? "pending" : "clean") << '\n'; + return output.str(); +} + +} // namespace storage_v2 diff --git a/includes/storage/storage_v2.h b/includes/storage/storage_v2.h new file mode 100644 index 0000000..6d2a9d2 --- /dev/null +++ b/includes/storage/storage_v2.h @@ -0,0 +1,78 @@ +#ifndef STORAGE_V2_H +#define STORAGE_V2_H + +#include +#include +#include +#include +#include + +namespace storage_v2 { + +constexpr std::uint32_t PAGE_SIZE = 4096; +constexpr std::uint16_t FORMAT_VERSION = 2; + +class CorruptionError : public std::runtime_error { +public: + explicit CorruptionError(const std::string& message) + : std::runtime_error("Storage corruption: " + message) {} +}; + +struct Inspection { + std::uint16_t format_version = FORMAT_VERSION; + std::uint64_t generation = 0; + std::uint32_t page_count = 0; + std::uint64_t row_count = 0; + std::uint32_t field_count = 0; + std::vector index_roots; + bool wal_present = false; +}; + +struct LoadedTable { + std::vector fields; + std::vector> rows; + std::vector>> indexes; + Inspection inspection; +}; + +std::string database_path(const std::string& table_name); +std::string wal_path(const std::string& database_path); +bool exists(const std::string& path); + +void create_table( + const std::string& path, + const std::vector& fields +); + +void create_from_rows( + const std::string& path, + const std::vector& fields, + const std::vector>& rows +); + +void initialize_owned_empty_file( + const std::string& path, + const std::vector& fields, + const std::vector>& rows +); + +LoadedTable open_table(const std::string& path); + +void insert_row( + const std::string& path, + const std::vector& row +); + +std::vector query_index( + const std::string& path, + std::size_t field_index, + const std::string& operation, + const std::string& value +); + +Inspection inspect(const std::string& path); +std::string describe(const std::string& path); + +} // namespace storage_v2 + +#endif diff --git a/includes/table/table.cpp b/includes/table/table.cpp index b07dbe7..ed68ac8 100644 --- a/includes/table/table.cpp +++ b/includes/table/table.cpp @@ -21,7 +21,9 @@ string Table::create_table(const string& table_name, const vector& field Table::Table(const string& table_name, const vector& field_names) : Table() { string table_file = table_name + ".bin"; string field_file = table_name + "_fields.bin"; - if (file_exists(table_file.c_str()) || file_exists(field_file.c_str())) { + const string v2_file = storage_v2::database_path(table_name); + if (file_exists(table_file.c_str()) || file_exists(field_file.c_str()) || + storage_v2::exists(v2_file)) { throw std::runtime_error("Table already exists: " + table_name); } if (field_names.empty() || field_names.size() > MAX_FIELDS) { @@ -39,25 +41,8 @@ Table::Table(const string& table_name, const vector& field_names) : Tabl tableName = table_name; fieldNames = field_names; selectedFields = field_names; - - fstream f; - - // Record the number of fields - open_fileW(f, field_file.c_str()); - vector mark; - int size = fieldNames.size(); - string mark_size = to_string(size); - mark.push_back(mark_size); - for (const auto & fieldName : fieldNames) { - mark.push_back(fieldName); - } - FileRecord r(mark); - r.write(f); - f.close(); - - // Create record file - open_fileW(f, table_file.c_str()); - f.close(); + storage_v2::create_table(v2_file, fieldNames); + usesV2Storage = true; recordIndices.clear(); // Initialize fieldNameMap and cache @@ -71,16 +56,39 @@ Table::Table(const string& table_name, const vector& field_names) : Tabl Table::Table(const string& table_name) : Table() { string table_file = table_name + ".bin"; string field_file = table_name + "_fields.bin"; - if (!file_exists(table_file.c_str()) || !file_exists(field_file.c_str())) { + const string v2_file = storage_v2::database_path(table_name); + const bool has_v2 = storage_v2::exists(v2_file); + if (!has_v2 && (!file_exists(table_file.c_str()) || !file_exists(field_file.c_str()))) { throw std::runtime_error("Table does not exist: " + table_name); } tableName = table_name; - fstream f; + if (has_v2) { + const auto loaded = storage_v2::open_table(v2_file); + fieldNames = loaded.fields; + selectedFields = loaded.fields; + storedRows = loaded.rows; + usesV2Storage = true; + if (storedRows.size() > static_cast(std::numeric_limits::max())) { + throw std::runtime_error("Table contains too many records for this build"); + } + numRecords = static_cast(storedRows.size()); + for (std::size_t field = 0; field < fieldNames.size(); ++field) { + fieldNameMap[fieldNames[field]] = static_cast(field); + } + for (std::size_t row = 0; row < storedRows.size(); ++row) { + recordIndices.push_back(static_cast(row)); + printQueue += storedRows[row]; + } + return; + } + + ifstream f; FileRecord r; // Determine the fields - open_fileRW(f, field_file.c_str()); + f.open(field_file, ios::in | ios::binary); + if (!f.is_open()) throw std::runtime_error("File could not be opened: " + field_file); r.resize(1); if (r.read(f, 0) != static_cast(r.encoded_size())) { throw std::runtime_error("Field metadata is incomplete for table: " + table_name); @@ -133,7 +141,8 @@ Table::Table(const string& table_name) : Table() { if (file_bytes % r.encoded_size() != 0) { throw std::runtime_error("Record file contains a partial row for table: " + table_name); } - open_fileRW(f, table_file.c_str()); + f.open(table_file, ios::in | ios::binary); + if (!f.is_open()) throw std::runtime_error("File could not be opened: " + table_file); // Read every record for (long i = 0;; ++i) { @@ -144,6 +153,7 @@ Table::Table(const string& table_name) : Table() { // Put the ith_entry into cache vector ith_entry = r.get_records_string(); + storedRows.push_back(ith_entry); for (std::size_t ith_entry_walker = 0; ith_entry_walker < ith_entry.size(); ++ith_entry_walker) { string field_value = ith_entry[ith_entry_walker]; long index = i; @@ -170,19 +180,13 @@ string Table::insert_into(const vector& field_values) { throw std::length_error("Values must not exceed 100 bytes"); } } - // Insert data into binary file - fstream f; - string table_file = tableName + ".bin"; - - // Open the file in read/write mode - open_fileRW(f, table_file.c_str()); - // Create a new FileRecord with the given field_values - FileRecord r(field_values); - // Write the record to the binary file and obtain its index - long index = r.write(f); - - // Close the file - f.close(); + if (!usesV2Storage) { + throw std::runtime_error( + "Storage format v1 is read-only; run sql_migrate_v1 before inserting" + ); + } + storage_v2::insert_row(storage_v2::database_path(tableName), field_values); + const long index = numRecords; // Insert the record into the cache for (std::size_t i = 0; i < field_values.size(); ++i) { @@ -198,6 +202,7 @@ string Table::insert_into(const vector& field_values) { recordIndices.push_back(index); // Add field_values to the printQueue for printing printQueue += field_values; + storedRows.push_back(field_values); // Increment the number of records in the table numRecords++; return "insert success"; @@ -224,8 +229,25 @@ ostream& operator<<(ostream& outs, const Table& print_me){ // Helper function to select records based on a given condition vector Table::selectHelp(const string& field_name, const string& op, const string& field_value) { - if (!cache.contains(field_name)) { + if (!fieldNameMap.contains(field_name)) { throw std::invalid_argument("Unknown field: " + field_name); + } + if (usesV2Storage) { + const auto records = storage_v2::query_index( + storage_v2::database_path(tableName), + static_cast(fieldNameMap[field_name]), + op, + field_value + ); + vector converted; + converted.reserve(records.size()); + for (const auto record : records) { + if (record > static_cast(std::numeric_limits::max())) { + throw std::runtime_error("Record identifier exceeds the public API range"); + } + converted.push_back(static_cast(record)); + } + return converted; } // If the operation is "=", return the record indices for the matching field_value else if (op == "=" && cache[field_name].contains(field_value)) { @@ -290,25 +312,17 @@ Table Table::select(const vector& selected_fields, const string& field_n recordIndices = temp.recordIndices; temp.numRecords = numRecords; - // read the entries - fstream f; - string table_file = temp.tableName + ".bin"; - open_fileRW(f, table_file.c_str()); for (auto idx : temp.recordIndices){ - FileRecord r; - r.resize(fieldNames.size()); - long byte = r.read(f, idx); - - if (byte == 0) break; - vector entry = r.get_records_string(); + if (idx < 0 || static_cast(idx) >= storedRows.size()) { + throw std::runtime_error("Selected record identifier is out of range"); + } + const vector& entry = storedRows[static_cast(idx)]; for (std::size_t i = 0; i < entry.size(); ++i) if (contains(temp.selectedFields, fieldNames[i])) { temp.printQueue += entry[i]; } } - f.close(); - return temp; } @@ -372,20 +386,14 @@ Table Table::select(const vector& selected_fields, const Queue& } } - fstream f; - string table_file = temp.tableName + ".bin"; - open_fileRW(f, table_file.c_str()); for (auto idx : temp.recordIndices){ - FileRecord r; - r.resize(fieldNames.size()); - long byte = r.read(f, idx); - - if (byte == 0) break; - vector entry = r.get_records_string(); + if (idx < 0 || static_cast(idx) >= storedRows.size()) { + throw std::runtime_error("Selected record identifier is out of range"); + } + const vector& entry = storedRows[static_cast(idx)]; for (std::size_t i = 0; i < entry.size(); ++i) if (contains(temp.fieldNames, fieldNames[i])) temp.printQueue += entry[i]; } - f.close(); recordIndices = temp.recordIndices; return temp; } diff --git a/includes/table/table.h b/includes/table/table.h index f0ef980..85a902b 100644 --- a/includes/table/table.h +++ b/includes/table/table.h @@ -29,6 +29,7 @@ #include "../shunting_yard/rightparen.h" #include "../shunting_yard/token.h" #include "../shunting_yard/tokenstr.h" +#include "../storage/storage_v2.h" using namespace std; @@ -140,6 +141,8 @@ class Table{ Map fieldNameMap; // map field name to entry indices vector printQueue; + vector> storedRows; + bool usesV2Storage = false; static inline void allocate_tokens_memory(const vector& a, Queue& q){ for (auto toke : a){ diff --git a/includes/version.h.in b/includes/version.h.in new file mode 100644 index 0000000..c6d6879 --- /dev/null +++ b/includes/version.h.in @@ -0,0 +1,6 @@ +#ifndef SQL_IN_CPP_VERSION_H +#define SQL_IN_CPP_VERSION_H + +#define SQL_IN_CPP_VERSION "@PROJECT_VERSION@" + +#endif diff --git a/run_sql.cpp b/run_sql.cpp index b38906b..c3941b5 100644 --- a/run_sql.cpp +++ b/run_sql.cpp @@ -1,4 +1,8 @@ #include "includes/sql/sql.h" +#include "includes/storage/storage_v2.h" +#include "version.h" + +#include using namespace std; @@ -17,7 +21,25 @@ void clearScreen() { #endif } -int main() { +int main(int argc, char* argv[]) { + if (argc == 2 && string(argv[1]) == "--version") { + cout << "SQL in C++ " << SQL_IN_CPP_VERSION + << " (storage format " << storage_v2::FORMAT_VERSION << ")" << endl; + return 0; + } + if (argc == 3 && string(argv[1]) == "--inspect-storage") { + try { + cout << storage_v2::describe(storage_v2::database_path(argv[2])); + return 0; + } catch (const exception& error) { + cerr << "Error: " << error.what() << endl; + return 1; + } + } + if (argc != 1) { + cerr << "Usage: run_sql [--version | --inspect-storage
]" << endl; + return 2; + } SQL sql; displayInstructions(); while (true) { diff --git a/tests/cli_tests.cpp b/tests/cli_tests.cpp new file mode 100644 index 0000000..4531fcb --- /dev/null +++ b/tests/cli_tests.cpp @@ -0,0 +1,106 @@ +#include "includes/storage/storage_v2.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +void expect(bool condition, const std::string& message) { + if (!condition) throw std::runtime_error(message); +} + +std::vector read_bytes(const std::string& path) { + std::ifstream input(path, std::ios::binary); + input.seekg(0, std::ios::end); + const auto length = input.tellg(); + input.seekg(0); + std::vector bytes(static_cast(length)); + input.read(reinterpret_cast(bytes.data()), static_cast(bytes.size())); + return bytes; +} + +std::pair run_inspection(const std::string& table) { + std::array descriptors{}; + if (::pipe(descriptors.data()) != 0) throw std::runtime_error("pipe failed"); + const auto process = ::fork(); + if (process < 0) throw std::runtime_error("fork failed"); + if (process == 0) { + ::close(descriptors[0]); + ::dup2(descriptors[1], STDOUT_FILENO); + ::dup2(descriptors[1], STDERR_FILENO); + ::close(descriptors[1]); + ::execl( + RUN_SQL_PATH, + RUN_SQL_PATH, + "--inspect-storage", + table.c_str(), + nullptr + ); + ::_exit(127); + } + ::close(descriptors[1]); + std::string output; + std::array buffer{}; + while (true) { + const auto count = ::read(descriptors[0], buffer.data(), buffer.size()); + if (count <= 0) break; + output.append(buffer.data(), static_cast(count)); + } + ::close(descriptors[0]); + int status = 0; + if (::waitpid(process, &status, 0) != process || !WIFEXITED(status)) return {-1, output}; + return {WEXITSTATUS(status), output}; +} + +} // namespace + +int main() { + const std::string table = "cli_inspect"; + const auto path = storage_v2::database_path(table); + std::filesystem::remove(path); + std::filesystem::remove(storage_v2::wal_path(path)); + try { + storage_v2::create_from_rows(path, {"name", "group"}, {{"Ada", "math"}}); + const auto before = read_bytes(path); + const auto valid = run_inspection(table); + expect(valid.first == 0, "valid storage inspection should exit zero"); + expect(valid.second.find("storage-format: 2") != std::string::npos, + "inspection should report format version"); + expect(valid.second.find("row-count: 1") != std::string::npos, + "inspection should report row count"); + expect(valid.second.find("index-roots:") != std::string::npos, + "inspection should report persistent roots"); + expect(read_bytes(path) == before, "inspection CLI must not mutate the database"); + + auto corrupt = before; + corrupt[storage_v2::PAGE_SIZE + 40] ^= 0x4bU; + { + std::ofstream output(path, std::ios::binary | std::ios::trunc); + output.write( + reinterpret_cast(corrupt.data()), + static_cast(corrupt.size()) + ); + } + const auto invalid = run_inspection(table); + expect(invalid.first == 1, "corrupt storage inspection should exit nonzero"); + expect(invalid.second.find("checksum") != std::string::npos, + "corrupt storage inspection should explain checksum failure"); + std::filesystem::remove(path); + std::filesystem::remove(storage_v2::wal_path(path)); + return 0; + } catch (const std::exception& error) { + std::filesystem::remove(path); + std::filesystem::remove(storage_v2::wal_path(path)); + std::cerr << "FAIL: " << error.what() << '\n'; + return 1; + } +} diff --git a/tests/migration_tests.cpp b/tests/migration_tests.cpp new file mode 100644 index 0000000..73a7e9c --- /dev/null +++ b/tests/migration_tests.cpp @@ -0,0 +1,217 @@ +#include "includes/sql/sql.h" +#include "includes/storage/storage_v2.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using Bytes = std::vector; + +void expect(bool condition, const std::string& message) { + if (!condition) throw std::runtime_error(message); +} + +Bytes read_bytes(const std::string& path) { + std::ifstream input(path, std::ios::binary); + input.seekg(0, std::ios::end); + const auto length = input.tellg(); + input.seekg(0); + Bytes bytes(static_cast(length)); + input.read(reinterpret_cast(bytes.data()), static_cast(bytes.size())); + return bytes; +} + +void write_slot(std::ofstream& output, const std::string& value) { + if (value.size() > 100) throw std::runtime_error("fixture value exceeds v1 slot"); + std::array slot{}; + std::copy(value.begin(), value.end(), slot.begin()); + output.write(slot.data(), static_cast(slot.size())); +} + +void write_v1( + const std::string& table, + const std::vector& fields, + const std::vector>& rows +) { + std::ofstream schema(table + "_fields.bin", std::ios::binary | std::ios::trunc); + write_slot(schema, std::to_string(fields.size())); + for (const auto& field : fields) write_slot(schema, field); + schema.close(); + std::ofstream data(table + ".bin", std::ios::binary | std::ios::trunc); + for (const auto& row : rows) { + for (const auto& value : row) write_slot(data, value); + } +} + +int run_migration( + const std::string& source, + const std::string& destination, + bool dry_run, + const std::string& crash_point = "" +) { + const auto process = ::fork(); + if (process < 0) throw std::runtime_error("fork failed"); + if (process == 0) { + if (!crash_point.empty()) { + ::setenv("SQL_MIGRATION_CRASH_AT", crash_point.c_str(), 1); + } + if (dry_run) { + ::execl( + MIGRATION_TOOL_PATH, + MIGRATION_TOOL_PATH, + "--dry-run", + "--destination", + destination.c_str(), + source.c_str(), + nullptr + ); + } else { + ::execl( + MIGRATION_TOOL_PATH, + MIGRATION_TOOL_PATH, + "--destination", + destination.c_str(), + source.c_str(), + nullptr + ); + } + ::_exit(127); + } + int status = 0; + if (::waitpid(process, &status, 0) != process || !WIFEXITED(status)) return -1; + return WEXITSTATUS(status); +} + +void cleanup(const std::string& table, const std::string& destination) { + std::filesystem::remove(table + ".bin"); + std::filesystem::remove(table + "_fields.bin"); + std::filesystem::remove(destination); + std::filesystem::remove(storage_v2::wal_path(destination)); + const auto prefix = std::filesystem::path(destination).filename().string() + ".tmp."; + for (const auto& entry : std::filesystem::directory_iterator(".")) { + const auto filename = entry.path().filename().string(); + if (filename.rfind(prefix, 0) == 0) std::filesystem::remove(entry.path()); + } +} + +void test_copy_first_migration_and_queries() { + const std::string source = "legacy_values"; + const std::string destination = "migrated_values.sql2"; + cleanup(source, destination); + const std::vector fields{"value", "category"}; + const std::vector> rows{ + {"", "empty"}, + {std::string(100, 'x'), "maximum"}, + {"10", "lexical"}, + {"2", "lexical"}, + {"2", "duplicate"}, + }; + write_v1(source, fields, rows); + const auto source_data = read_bytes(source + ".bin"); + const auto source_schema = read_bytes(source + "_fields.bin"); + + expect(run_migration(source, destination, true) == 0, "migration dry run should succeed"); + expect(!std::filesystem::exists(destination), "dry run must not create a destination"); + expect(read_bytes(source + ".bin") == source_data, "dry run must preserve v1 rows"); + expect(read_bytes(source + "_fields.bin") == source_schema, "dry run must preserve v1 schema"); + + expect(run_migration(source, destination, false) == 0, "migration should succeed"); + const auto migrated = storage_v2::open_table(destination); + expect(migrated.fields == fields, "migrated fields should match v1"); + expect(migrated.rows == rows, "migrated rows should match v1 byte-for-byte"); + expect(read_bytes(source + ".bin") == source_data, "migration must preserve v1 rows"); + expect(read_bytes(source + "_fields.bin") == source_schema, "migration must preserve v1 schema"); + expect(run_migration(source, destination, false) != 0, "existing destination should be rejected"); + + std::filesystem::rename(destination, storage_v2::database_path(source)); + SQL sql; + const auto equal = sql.command("select * from legacy_values where value = 2"); + expect(!sql.is_error(), "migrated equality query should succeed"); + expect(equal.select_recnos() == std::vector({3, 4}), "duplicate equality query should match"); + const auto range = sql.command("select * from legacy_values where value < 2"); + expect(!sql.is_error(), "migrated range query should succeed"); + expect(range.select_recnos() == std::vector({0, 2}), "lexical range should match v1 semantics"); + std::filesystem::rename(storage_v2::database_path(source), destination); + cleanup(source, destination); +} + +void test_maximum_v1_row_and_read_only_runtime() { + const std::string source = "legacy_maximum"; + const std::string destination = "migrated_maximum.sql2"; + cleanup(source, destination); + std::vector fields; + std::vector row; + for (int field = 0; field < 64; ++field) { + fields.push_back("f" + std::to_string(field)); + row.push_back(std::string(100, static_cast('A' + field % 26))); + } + write_v1(source, fields, {row}); + expect(run_migration(source, destination, false) == 0, "maximum-width v1 row should migrate"); + expect(storage_v2::open_table(destination).rows == std::vector>({row}), + "maximum-width row should remain equivalent"); + + bool read_only_rejected = false; + try { + Table legacy(source); + legacy.insert_into(row); + } catch (const std::runtime_error& error) { + read_only_rejected = std::string(error.what()).find("read-only") != std::string::npos; + } + expect(read_only_rejected, "v1 runtime inserts should explain migration requirement"); + cleanup(source, destination); +} + +void test_atomic_promotion_crash_points() { + const std::string source = "legacy_promotion"; + const std::string destination = "migrated_promotion.sql2"; + cleanup(source, destination); + write_v1(source, {"value"}, {{"safe"}}); + const auto source_data = read_bytes(source + ".bin"); + const auto source_schema = read_bytes(source + "_fields.bin"); + + expect(run_migration(source, destination, false, "before-promotion") == 86, + "migration should stop before promotion"); + expect(!std::filesystem::exists(destination), "pre-rename crash must not expose destination"); + expect(read_bytes(source + ".bin") == source_data, "pre-rename crash must preserve source rows"); + expect(read_bytes(source + "_fields.bin") == source_schema, + "pre-rename crash must preserve source schema"); + cleanup("unused", destination); + + expect(run_migration(source, destination, false, "after-promotion") == 86, + "migration should stop after atomic promotion"); + expect(storage_v2::open_table(destination).rows == + std::vector>({{"safe"}}), + "post-rename crash must expose only a complete verified destination"); + expect(read_bytes(source + ".bin") == source_data, "post-rename crash must preserve source rows"); + expect(read_bytes(source + "_fields.bin") == source_schema, + "post-rename crash must preserve source schema"); + cleanup(source, destination); +} + +} // namespace + +int main() { + try { + test_copy_first_migration_and_queries(); + std::cout << "PASS: dry-run, copy-first migration, and query equivalence\n"; + test_maximum_v1_row_and_read_only_runtime(); + std::cout << "PASS: maximum-width v1 migration and read-only compatibility\n"; + test_atomic_promotion_crash_points(); + std::cout << "PASS: atomic migration promotion crash points\n"; + return 0; + } catch (const std::exception& error) { + std::cerr << "FAIL: " << error.what() << '\n'; + return 1; + } +} diff --git a/tests/sql_tests.cpp b/tests/sql_tests.cpp index 1c9ce93..7120091 100644 --- a/tests/sql_tests.cpp +++ b/tests/sql_tests.cpp @@ -1,4 +1,5 @@ #include "includes/sql/sql.h" +#include "includes/storage/storage_v2.h" #include #include @@ -47,6 +48,8 @@ void expect_error(SQL& sql, const std::string& message_fragment) { void remove_table(const std::string& name) { std::filesystem::remove(name + ".bin"); std::filesystem::remove(name + "_fields.bin"); + std::filesystem::remove(name + ".sql2"); + std::filesystem::remove(name + ".sql2-wal"); } void test_batch_executes_repository_fixture() { @@ -139,12 +142,13 @@ void test_command_failures_do_not_terminate_or_mutate_storage() { SQL sql; sql.command("create table integrity_test fields first, last"); expect_true(!sql.is_error(), "valid create should succeed"); - const auto empty_size = std::filesystem::file_size("integrity_test.bin"); + const auto storage_path = storage_v2::database_path("integrity_test"); + const auto empty_size = std::filesystem::file_size(storage_path); sql.command("insert into integrity_test values only_one"); expect_error(sql, "Expected 2 values, received 1"); expect_equal( - static_cast(std::filesystem::file_size("integrity_test.bin")), + static_cast(std::filesystem::file_size(storage_path)), static_cast(empty_size), "short insert must not change storage" ); @@ -152,30 +156,18 @@ void test_command_failures_do_not_terminate_or_mutate_storage() { sql.command("insert into integrity_test values one, two, three"); expect_error(sql, "Expected 2 values, received 3"); expect_equal( - static_cast(std::filesystem::file_size("integrity_test.bin")), + static_cast(std::filesystem::file_size(storage_path)), static_cast(empty_size), "long insert must not change storage" ); sql.command("insert into integrity_test values Ada, Lovelace"); expect_true(!sql.is_error(), "valid insert should succeed after errors"); - expect_equal( - static_cast(std::filesystem::file_size("integrity_test.bin")), - 200, - "two-field row should use two fixed-width slots" + const auto stored = storage_v2::open_table(storage_path); + expect_true( + stored.rows == std::vector>({{"Ada", "Lovelace"}}), + "v2 row should reopen with exact values" ); - { - std::ifstream stored("integrity_test.bin", std::ios::binary); - std::vector bytes(200); - stored.read(bytes.data(), static_cast(bytes.size())); - expect_true(stored.gcount() == 200, "stored row should be complete"); - expect_true( - std::all_of(bytes.begin() + 3, bytes.begin() + 100, [](char byte) { - return byte == '\0'; - }), - "short values should be deterministically zero padded" - ); - } sql.command("create table integrity_test fields replacement"); expect_error(sql, "already exists"); @@ -217,24 +209,25 @@ void test_schema_value_and_corrupt_file_boundaries() { const std::string exact_value(100, 'x'); sql.command("insert into boundary_test values \"" + exact_value + "\""); expect_true(!sql.is_error(), "100-byte value should fit the fixed slot"); - const auto valid_size = std::filesystem::file_size("boundary_test.bin"); + const auto storage_path = storage_v2::database_path("boundary_test"); + const auto valid_size = std::filesystem::file_size(storage_path); const std::string oversized_value(101, 'y'); sql.command("insert into boundary_test values \"" + oversized_value + "\""); expect_error(sql, "must not exceed 100 bytes"); expect_equal( - static_cast(std::filesystem::file_size("boundary_test.bin")), + static_cast(std::filesystem::file_size(storage_path)), static_cast(valid_size), "oversized value must not change storage" ); { - std::ofstream corrupt("boundary_test.bin", std::ios::binary | std::ios::app); + std::ofstream corrupt(storage_path, std::ios::binary | std::ios::app); corrupt.put('x'); } SQL corrupt_reader; corrupt_reader.command("select * from boundary_test"); - expect_error(corrupt_reader, "partial row"); + expect_error(corrupt_reader, "complete page image"); remove_table("boundary_test"); } @@ -273,25 +266,25 @@ void test_schema_metadata_size_must_match_exactly() { sql.command("create table metadata_extra_test fields value"); { std::ofstream extra( - "metadata_extra_test_fields.bin", + storage_v2::database_path("metadata_extra_test"), std::ios::binary | std::ios::app ); extra.put('x'); } SQL extra_reader; extra_reader.command("select * from metadata_extra_test"); - expect_error(extra_reader, "metadata size is invalid"); + expect_error(extra_reader, "complete page image"); sql.command("create table metadata_short_test fields value"); - const auto metadata_size = - std::filesystem::file_size("metadata_short_test_fields.bin"); + const auto metadata_path = storage_v2::database_path("metadata_short_test"); + const auto metadata_size = std::filesystem::file_size(metadata_path); std::filesystem::resize_file( - "metadata_short_test_fields.bin", + metadata_path, metadata_size - 1 ); SQL short_reader; short_reader.command("select * from metadata_short_test"); - expect_error(short_reader, "metadata size is invalid"); + expect_error(short_reader, "complete page image"); remove_table("metadata_extra_test"); remove_table("metadata_short_test"); diff --git a/tests/storage_crash_tests.cpp b/tests/storage_crash_tests.cpp new file mode 100644 index 0000000..3d77ea2 --- /dev/null +++ b/tests/storage_crash_tests.cpp @@ -0,0 +1,323 @@ +#include "includes/storage/storage_v2.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +void expect(bool condition, const std::string& message) { + if (!condition) throw std::runtime_error(message); +} + +std::vector read_bytes(const std::string& path) { + std::ifstream input(path, std::ios::binary); + input.seekg(0, std::ios::end); + const auto length = input.tellg(); + input.seekg(0); + std::vector bytes(static_cast(length)); + input.read(reinterpret_cast(bytes.data()), static_cast(bytes.size())); + return bytes; +} + +void write_bytes(const std::string& path, const std::vector& bytes) { + std::ofstream output(path, std::ios::binary | std::ios::trunc); + output.write(reinterpret_cast(bytes.data()), static_cast(bytes.size())); +} + +int crash_value( + const std::string& point, + const std::string& path, + const std::string& value +) { + const auto process = ::fork(); + if (process < 0) throw std::runtime_error("fork failed"); + if (process == 0) { + ::setenv("SQL_CRASH_AT", point.c_str(), 1); + ::execl( + STORAGE_CRASH_WORKER_PATH, + STORAGE_CRASH_WORKER_PATH, + path.c_str(), + value.c_str(), + nullptr + ); + ::_exit(127); + } + int status = 0; + if (::waitpid(process, &status, 0) != process) throw std::runtime_error("waitpid failed"); + if (!WIFEXITED(status)) return -1; + return WEXITSTATUS(status); +} + +int crash_insert(const std::string& point, const std::string& path) { + return crash_value(point, path, "new"); +} + +int crash_create(const std::string& point, const std::string& path) { + const auto process = ::fork(); + if (process < 0) throw std::runtime_error("fork failed"); + if (process == 0) { + ::setenv("SQL_CRASH_AT", point.c_str(), 1); + ::execl( + STORAGE_CRASH_WORKER_PATH, + STORAGE_CRASH_WORKER_PATH, + path.c_str(), + "__create__", + nullptr + ); + ::_exit(127); + } + int status = 0; + if (::waitpid(process, &status, 0) != process) throw std::runtime_error("waitpid failed"); + if (!WIFEXITED(status)) return -1; + return WEXITSTATUS(status); +} + +std::vector trace_points( + const std::string& path, + const std::string& value, + const std::string& trace_path +) { + std::filesystem::remove(trace_path); + const auto process = ::fork(); + if (process < 0) throw std::runtime_error("fork failed"); + if (process == 0) { + ::setenv("SQL_CRASH_TRACE_FILE", trace_path.c_str(), 1); + ::execl( + STORAGE_CRASH_WORKER_PATH, + STORAGE_CRASH_WORKER_PATH, + path.c_str(), + value.c_str(), + nullptr + ); + ::_exit(127); + } + int status = 0; + if (::waitpid(process, &status, 0) != process || !WIFEXITED(status) || + WEXITSTATUS(status) != 0) { + throw std::runtime_error("trace worker failed"); + } + std::ifstream input(trace_path); + std::vector points; + std::string line; + while (std::getline(input, line)) { + if (!line.empty()) points.push_back(line); + } + std::filesystem::remove(trace_path); + expect(!points.empty(), "trace should enumerate crash boundaries"); + expect(std::set(points.begin(), points.end()).size() == points.size(), + "one statement should expose each named boundary once"); + return points; +} + +void cleanup(const std::string& path) { + std::filesystem::remove(path); + std::filesystem::remove(storage_v2::wal_path(path)); +} + +void install_baseline( + const std::string& path, + const std::vector& image +) { + cleanup(path); + write_bytes(path, image); + std::ofstream wal(storage_v2::wal_path(path), std::ios::binary | std::ios::trunc); +} + +void test_crash_matrix() { + const std::string probe = "crash-trace-probe.sql2"; + cleanup(probe); + storage_v2::create_from_rows(probe, {"value"}, {{"old"}}); + const auto points = trace_points(probe, "new", "insert-crash-points.txt"); + cleanup(probe); + for (const auto& point : points) { + const auto path = "crash-" + point + ".sql2"; + cleanup(path); + storage_v2::create_from_rows(path, {"value"}, {{"old"}}); + const auto old_image = read_bytes(path); + expect(crash_insert(point, path) == 86, "worker should terminate at " + point); + + if (point == "wal-fsync") { + const auto wal = storage_v2::wal_path(path); + const auto wal_size = std::filesystem::file_size(wal); + const auto inspection = storage_v2::inspect(path); + expect(inspection.wal_present, "inspection should report pending committed WAL"); + expect(std::filesystem::file_size(wal) == wal_size, "inspection must not clear the WAL"); + expect(read_bytes(path) == old_image, "inspection must not apply pending pages"); + } + + const auto recovered = storage_v2::open_table(path); + const auto old_state = std::vector>({{"old"}}); + const auto new_state = std::vector>({{"old"}, {"new"}}); + expect(recovered.rows == old_state || recovered.rows == new_state, + "crash recovery should expose only the old or new complete state: " + point); + if (point != "wal-write") { + expect(recovered.rows == new_state, + "durably committed crash point should recover the new state: " + point); + } + expect( + std::filesystem::file_size(storage_v2::wal_path(path)) == 0, + "recovery should clear WAL: " + point + ); + const auto recovered_image = read_bytes(path); + const auto second = storage_v2::open_table(path); + expect(second.rows == recovered.rows, "second recovery should be idempotent: " + point); + expect(read_bytes(path) == recovered_image, "second recovery should not rewrite data: " + point); + cleanup(path); + } +} + +void test_partial_wal_rejection() { + const std::string path = "crash-partial.sql2"; + cleanup(path); + storage_v2::create_from_rows(path, {"value"}, {{"old"}}); + const auto old_image = read_bytes(path); + expect(crash_insert("wal-partial", path) == 86, "partial WAL worker should terminate"); + bool rejected = false; + try { + static_cast(storage_v2::open_table(path)); + } catch (const storage_v2::CorruptionError& error) { + rejected = std::string(error.what()).find("WAL") != std::string::npos; + } + expect(rejected, "partial WAL should fail closed explicitly"); + expect(read_bytes(path) == old_image, "partial WAL must leave committed database unchanged"); + cleanup(path); +} + +void test_internal_split_crash_matrix() { + const std::string baseline_path = "deep-crash-baseline.sql2"; + cleanup(baseline_path); + std::vector> rows; + for (int record = 0; record < 1'296; ++record) { + auto value = std::to_string(record); + value.insert(value.begin(), 100 - value.size(), '0'); + rows.push_back({value}); + } + storage_v2::create_from_rows(baseline_path, {"value"}, rows); + const auto baseline = read_bytes(baseline_path); + const auto inserted_value = std::string(100, 'z'); + + const std::string probe = "deep-crash-probe.sql2"; + install_baseline(probe, baseline); + const auto points = trace_points(probe, inserted_value, "deep-crash-points.txt"); + cleanup(probe); + expect(std::count_if(points.begin(), points.end(), [](const std::string& point) { + return point.rfind("page-", 0) == 0; + }) >= 7, "internal split trace should expose every dirty path and split page"); + + for (const auto& point : points) { + const auto path = "deep-crash-" + point + ".sql2"; + install_baseline(path, baseline); + expect(crash_value(point, path, inserted_value) == 86, + "deep split worker should terminate at " + point); + const auto recovered = storage_v2::open_table(path); + expect(recovered.rows.size() == rows.size() || recovered.rows.size() == rows.size() + 1, + "deep crash should expose only the old or new row count: " + point); + if (point != "wal-write") { + expect(recovered.rows.size() == rows.size() + 1, + "durable deep split should recover the inserted row: " + point); + } + expect(recovered.indexes[0].size() == recovered.rows.size(), + "deep split recovery should keep index and row counts equal: " + point); + cleanup(path); + } + cleanup(baseline_path); +} + +void test_create_crash_matrix() { + const std::string probe = "create-crash-trace-probe.sql2"; + cleanup(probe); + const auto points = trace_points(probe, "__create__", "create-crash-points.txt"); + cleanup(probe); + for (const auto& point : points) { + const auto path = "create-crash-" + point + ".sql2"; + cleanup(path); + expect(crash_create(point, path) == 86, "create worker should terminate at " + point); + const auto recovered = storage_v2::open_table(path); + expect(recovered.fields == std::vector({"value"}), + "recovered create should contain complete schema: " + point); + expect(recovered.rows.empty(), "recovered create should contain no partial row: " + point); + cleanup(path); + } +} + +void test_wal_checksum_rejection() { + const std::string path = "crash-wal-checksum.sql2"; + cleanup(path); + storage_v2::create_from_rows(path, {"value"}, {{"old"}}); + const auto old_image = read_bytes(path); + expect(crash_insert("wal-fsync", path) == 86, "WAL fixture worker should terminate"); + const auto wal_path = storage_v2::wal_path(path); + auto wal = read_bytes(wal_path); + expect(wal.size() > 64, "committed WAL fixture should contain an image"); + wal[48] ^= 0x7fU; + { + std::ofstream output(wal_path, std::ios::binary | std::ios::trunc); + output.write(reinterpret_cast(wal.data()), static_cast(wal.size())); + } + bool rejected = false; + try { + static_cast(storage_v2::open_table(path)); + } catch (const storage_v2::CorruptionError& error) { + rejected = std::string(error.what()).find("checksum") != std::string::npos; + } + expect(rejected, "corrupt committed WAL checksum should fail closed"); + expect(read_bytes(path) == old_image, "corrupt WAL must not modify the database"); + cleanup(path); +} + +void test_stale_wal_rejection() { + const std::string path = "crash-stale-wal.sql2"; + cleanup(path); + storage_v2::create_from_rows(path, {"value"}, {{"old"}}); + expect(crash_insert("wal-fsync", path) == 86, "stale WAL fixture should terminate"); + const auto stale_wal = read_bytes(storage_v2::wal_path(path)); + static_cast(storage_v2::open_table(path)); + storage_v2::insert_row(path, {"newer"}); + write_bytes(storage_v2::wal_path(path), stale_wal); + bool inspect_rejected = false; + try { + static_cast(storage_v2::inspect(path)); + } catch (const storage_v2::CorruptionError& error) { + inspect_rejected = std::string(error.what()).find("stale WAL") != std::string::npos; + } + expect(inspect_rejected, "inspection should reject a rollback WAL"); + bool recovery_rejected = false; + try { + static_cast(storage_v2::open_table(path)); + } catch (const storage_v2::CorruptionError& error) { + recovery_rejected = std::string(error.what()).find("stale WAL") != std::string::npos; + } + expect(recovery_rejected, "normal recovery should reject the same rollback WAL"); + cleanup(path); +} + +} // namespace + +int main() { + try { + test_crash_matrix(); + std::cout << "PASS: deterministic committed crash matrix and idempotent recovery\n"; + test_partial_wal_rejection(); + std::cout << "PASS: partial WAL rejection\n"; + test_internal_split_crash_matrix(); + std::cout << "PASS: dynamic internal-split crash matrix\n"; + test_create_crash_matrix(); + std::cout << "PASS: atomic create crash matrix\n"; + test_wal_checksum_rejection(); + std::cout << "PASS: WAL checksum rejection\n"; + test_stale_wal_rejection(); + std::cout << "PASS: stale WAL rejection in inspection and recovery\n"; + return 0; + } catch (const std::exception& error) { + std::cerr << "FAIL: " << error.what() << '\n'; + return 1; + } +} diff --git a/tests/storage_crash_worker.cpp b/tests/storage_crash_worker.cpp new file mode 100644 index 0000000..8a7bc4d --- /dev/null +++ b/tests/storage_crash_worker.cpp @@ -0,0 +1,20 @@ +#include "includes/storage/storage_v2.h" + +#include +#include +#include + +int main(int argc, char* argv[]) { + if (argc != 3) return 2; + try { + if (std::string(argv[2]) == "__create__") { + storage_v2::create_table(argv[1], {"value"}); + } else { + storage_v2::insert_row(argv[1], {argv[2]}); + } + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/tests/storage_v2_tests.cpp b/tests/storage_v2_tests.cpp new file mode 100644 index 0000000..40c5518 --- /dev/null +++ b/tests/storage_v2_tests.cpp @@ -0,0 +1,321 @@ +#include "includes/storage/storage_v2.h" +#include "includes/storage/page_buffer.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using Bytes = std::vector; + +void expect(bool condition, const std::string& message) { + if (!condition) throw std::runtime_error(message); +} + +template +void expect_corruption(Function function, const std::string& fragment) { + try { + function(); + } catch (const storage_v2::CorruptionError& error) { + expect( + std::string(error.what()).find(fragment) != std::string::npos, + "corruption error should contain: " + fragment + ", received: " + error.what() + ); + return; + } + throw std::runtime_error("operation should reject corrupt storage"); +} + +Bytes read_bytes(const std::string& path) { + std::ifstream input(path, std::ios::binary); + input.seekg(0, std::ios::end); + const auto length = input.tellg(); + input.seekg(0); + Bytes bytes(static_cast(length)); + input.read(reinterpret_cast(bytes.data()), static_cast(bytes.size())); + return bytes; +} + +void write_bytes(const std::string& path, const Bytes& bytes) { + std::ofstream output(path, std::ios::binary | std::ios::trunc); + output.write(reinterpret_cast(bytes.data()), static_cast(bytes.size())); +} + +std::uint32_t crc32(const std::uint8_t* data, std::size_t size) { + std::uint32_t crc = 0xffffffffU; + for (std::size_t i = 0; i < size; ++i) { + crc ^= data[i]; + for (int bit = 0; bit < 8; ++bit) { + const std::uint32_t mask = 0U - (crc & 1U); + crc = (crc >> 1U) ^ (0xedb88320U & mask); + } + } + return ~crc; +} + +void put_u32(std::uint8_t* target, std::uint32_t value) { + for (int shift = 0; shift < 32; shift += 8) { + *target++ = static_cast((value >> shift) & 0xffU); + } +} + +void refresh_page_checksum(Bytes& bytes, std::size_t page) { + auto* page_bytes = bytes.data() + page * storage_v2::PAGE_SIZE; + put_u32(page_bytes + 20, 0); + put_u32(page_bytes + 20, crc32(page_bytes, storage_v2::PAGE_SIZE)); +} + +void remove_database(const std::string& path) { + std::filesystem::remove(path); + std::filesystem::remove(storage_v2::wal_path(path)); +} + +void test_page_format_and_large_rows() { + const std::string path = "format.sql2"; + remove_database(path); + std::vector fields; + std::vector maximum_row; + std::vector empty_row(64); + for (int field = 0; field < 64; ++field) { + fields.push_back("field_" + std::to_string(field)); + maximum_row.push_back(std::string(100, static_cast('a' + field % 26))); + } + storage_v2::create_from_rows(path, fields, {empty_row, maximum_row}); + const auto opened = storage_v2::open_table(path); + expect(opened.fields == fields, "64-field schema should round-trip"); + expect(opened.rows.size() == 2, "large-row table should contain both rows"); + expect(opened.rows[0].size() == fields.size(), "empty-value row should retain every field"); + expect( + std::all_of(opened.rows[0].begin(), opened.rows[0].end(), [](const std::string& value) { + return value.empty(); + }), + "empty values should round-trip" + ); + expect(opened.rows[1] == maximum_row, "6,400-byte logical row should cross pages safely"); + expect(opened.inspection.format_version == 2, "format version should be 2"); + expect(opened.inspection.page_count > 4, "large row should allocate multiple pages"); + expect(read_bytes(path).size() == opened.inspection.page_count * storage_v2::PAGE_SIZE, + "physical file size should exactly match page count"); + storage_v2::PageBuffer buffer(read_bytes(path)); + std::vector visited; + buffer.visit_pages([&visited](std::uint32_t page, const std::uint8_t*) { + visited.push_back(page); + }); + expect(visited.size() == opened.inspection.page_count, + "buffer flush visitor should receive every owned page"); + for (std::size_t page = 0; page < visited.size(); ++page) { + expect(visited[page] == page, "buffer flush order should be deterministic by page id"); + } + bool bounded = false; + try { + static_cast(buffer.page_data(static_cast(buffer.page_count()))); + } catch (const std::out_of_range&) { + bounded = true; + } + expect(bounded, "buffer reads should reject page_count as out of bounds"); + storage_v2::PageBuffer dirty_buffer(read_bytes(path), {0, 2}); + std::vector dirty_visited; + dirty_buffer.visit_dirty_pages( + [&dirty_visited](std::uint32_t page, const std::uint8_t*) { + dirty_visited.push_back(page); + } + ); + expect(dirty_visited == std::vector({0, 2}), + "dirty-page flush order should be explicit and deterministic"); + remove_database(path); +} + +void test_persistent_index_property_and_reopen() { + const std::string path = "property.sql2"; + remove_database(path); + std::vector> rows; + std::mt19937 generator(0x51A2B3C4U); + std::uniform_int_distribution value_distribution(0, 150); + for (std::uint64_t record = 0; record < 2'000; ++record) { + rows.push_back({ + "key_" + std::to_string(value_distribution(generator)), + "record_" + std::to_string(record), + }); + } + storage_v2::create_from_rows(path, {"group", "record"}, rows); + const auto first = storage_v2::open_table(path); + const auto first_roots = first.inspection.index_roots; + expect(first.rows == rows, "randomized rows should reopen exactly"); + for (std::size_t field = 0; field < 2; ++field) { + std::vector> expected; + for (std::uint64_t record = 0; record < rows.size(); ++record) { + expected.emplace_back(rows[record][field], record); + } + std::sort(expected.begin(), expected.end()); + expect(first.indexes[field] == expected, "persistent index should match sorted reference"); + } + const auto before = read_bytes(path); + const auto second = storage_v2::open_table(path); + expect(second.inspection.index_roots == first_roots, "reopen should retain persistent roots"); + expect(read_bytes(path) == before, "clean reopen should not rewrite or rebuild storage"); + expect(second.indexes == first.indexes, "reopen should load identical persisted index entries"); + remove_database(path); +} + +void test_leaf_split_and_root_change() { + const std::string path = "root-change.sql2"; + remove_database(path); + std::vector> rows; + for (int record = 0; record < 36; ++record) { + auto value = "key_" + std::to_string(record); + value.resize(100, 'x'); + rows.push_back({value}); + } + storage_v2::create_from_rows(path, {"value"}, rows); + const auto before = read_bytes(path); + const auto leaf_root = storage_v2::inspect(path).index_roots[0]; + auto split_value = std::string("key_36"); + split_value.resize(100, 'x'); + storage_v2::insert_row(path, {split_value}); + const auto reopened = storage_v2::open_table(path); + expect(reopened.inspection.index_roots[0] != leaf_root, + "leaf split should replace the field index root"); + expect(reopened.indexes[0].size() == 37, "root split should preserve every index entry"); + const auto after = read_bytes(path); + expect(std::equal( + before.begin() + storage_v2::PAGE_SIZE, + before.begin() + 2 * storage_v2::PAGE_SIZE, + after.begin() + storage_v2::PAGE_SIZE + ), + "incremental insert should leave the schema page byte-identical"); + remove_database(path); +} + +void test_incremental_randomized_insert_and_internal_split() { + const std::string path = "incremental-property.sql2"; + remove_database(path); + storage_v2::create_table(path, {"value"}); + std::vector> rows; + std::mt19937 generator(0x1A2B3C4DU); + std::uniform_int_distribution distribution(0, 40); + for (int record = 0; record < 120; ++record) { + const auto value = "value_" + std::to_string(distribution(generator)); + storage_v2::insert_row(path, {value}); + rows.push_back({value}); + if (record % 20 == 19) { + const auto reopened = storage_v2::open_table(path); + expect(reopened.rows == rows, "incremental reopen should preserve randomized rows"); + std::vector> expected; + for (std::uint64_t index = 0; index < rows.size(); ++index) { + expected.emplace_back(rows[index][0], index); + } + std::sort(expected.begin(), expected.end()); + expect(reopened.indexes[0] == expected, + "incremental persistent tree should match the sorted reference"); + } + } + const auto matches = storage_v2::query_index(path, 0, ">=", "value_30"); + std::vector expected_matches; + std::vector> ordered; + for (std::uint64_t index = 0; index < rows.size(); ++index) { + ordered.emplace_back(rows[index][0], index); + } + std::sort(ordered.begin(), ordered.end()); + for (const auto& entry : ordered) { + if (entry.first >= "value_30") expected_matches.push_back(entry.second); + } + expect(matches == expected_matches, "page-backed range query should match the reference"); + remove_database(path); + + const std::string deep_path = "internal-root-change.sql2"; + remove_database(deep_path); + std::vector> deep_rows; + for (int record = 0; record < 1'296; ++record) { + auto value = std::to_string(record); + value.insert(value.begin(), 100 - value.size(), '0'); + deep_rows.push_back({value}); + } + storage_v2::create_from_rows(deep_path, {"value"}, deep_rows); + const auto old_root = storage_v2::inspect(deep_path).index_roots[0]; + const auto old_page_count = storage_v2::inspect(deep_path).page_count; + storage_v2::insert_row(deep_path, {std::string(100, 'z')}); + const auto deep = storage_v2::open_table(deep_path); + expect(deep.inspection.index_roots[0] != old_root, + "incremental internal split should replace the root"); + expect(deep.inspection.page_count >= old_page_count + 3, + "internal split should allocate a leaf, sibling internal node, and root"); + expect(deep.indexes[0].size() == 1'297, + "internal root split should preserve every persistent entry"); + remove_database(deep_path); +} + +void test_corruption_fails_closed() { + const std::string path = "corruption.sql2"; + remove_database(path); + std::vector> rows; + for (int record = 0; record < 2'000; ++record) { + rows.push_back({std::string(100, static_cast('a' + record % 26)) + std::to_string(record)}); + rows.back()[0].resize(100); + } + storage_v2::create_from_rows(path, {"value"}, rows); + const auto valid = read_bytes(path); + + auto checksum_corrupt = valid; + checksum_corrupt[storage_v2::PAGE_SIZE + 40] ^= 0x5aU; + write_bytes(path, checksum_corrupt); + expect_corruption([&]() { static_cast(storage_v2::open_table(path)); }, "checksum"); + + auto count_corrupt = valid; + put_u32(count_corrupt.data() + 32 + 12, 1); + refresh_page_checksum(count_corrupt, 0); + write_bytes(path, count_corrupt); + expect_corruption([&]() { static_cast(storage_v2::open_table(path)); }, "page count"); + + auto child_corrupt = valid; + std::size_t internal_page = 0; + for (std::size_t page = 0; page < child_corrupt.size() / storage_v2::PAGE_SIZE; ++page) { + if (child_corrupt[page * storage_v2::PAGE_SIZE + 4] == 6) { + internal_page = page; + break; + } + } + expect(internal_page != 0, "property fixture should contain an internal index page"); + put_u32(child_corrupt.data() + internal_page * storage_v2::PAGE_SIZE + 32 + 4, 0); + refresh_page_checksum(child_corrupt, internal_page); + write_bytes(path, child_corrupt); + expect_corruption([&]() { static_cast(storage_v2::open_table(path)); }, "index"); + + write_bytes(path, valid); + { + std::ofstream wal(storage_v2::wal_path(path), std::ios::binary | std::ios::trunc); + wal << "partial"; + } + const auto database_before = read_bytes(path); + expect_corruption([&]() { static_cast(storage_v2::open_table(path)); }, "incomplete"); + expect(read_bytes(path) == database_before, "invalid WAL must not modify the database"); + remove_database(path); +} + +} // namespace + +int main() { + try { + test_page_format_and_large_rows(); + std::cout << "PASS: page format and large rows\n"; + test_persistent_index_property_and_reopen(); + std::cout << "PASS: persistent randomized indexes and reopen\n"; + test_leaf_split_and_root_change(); + std::cout << "PASS: persistent leaf split and root change\n"; + test_incremental_randomized_insert_and_internal_split(); + std::cout << "PASS: incremental randomized inserts and internal split\n"; + test_corruption_fails_closed(); + std::cout << "PASS: page, pointer, and WAL corruption fail closed\n"; + return 0; + } catch (const std::exception& error) { + std::cerr << "FAIL: " << error.what() << '\n'; + return 1; + } +} diff --git a/tools/sql_migrate_v1.cpp b/tools/sql_migrate_v1.cpp new file mode 100644 index 0000000..f6fbdae --- /dev/null +++ b/tools/sql_migrate_v1.cpp @@ -0,0 +1,246 @@ +#include "includes/storage/storage_v2.h" +#include "version.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +constexpr std::size_t SLOT_SIZE = 100; +constexpr std::size_t MAX_FIELDS = 64; + +struct LegacyTable { + std::vector fields; + std::vector> rows; +}; + +std::vector read_bytes(const std::string& path) { + std::ifstream input(path, std::ios::binary); + if (!input) throw std::runtime_error("Unable to open v1 file read-only: " + path); + input.seekg(0, std::ios::end); + const auto length = input.tellg(); + if (length < 0) throw std::runtime_error("Unable to determine v1 file size: " + path); + input.seekg(0); + std::vector bytes(static_cast(length)); + if (!bytes.empty()) { + input.read(reinterpret_cast(bytes.data()), static_cast(bytes.size())); + if (input.gcount() != static_cast(bytes.size())) { + throw std::runtime_error("Unable to read complete v1 file: " + path); + } + } + return bytes; +} + +std::string decode_slot(const std::uint8_t* slot, const std::string& context) { + const auto end = std::find(slot, slot + SLOT_SIZE, 0); + if (end != slot + SLOT_SIZE) { + for (auto cursor = end; cursor != slot + SLOT_SIZE; ++cursor) { + if (*cursor != 0) throw std::runtime_error(context + " has non-zero bytes after its terminator"); + } + } + return std::string(reinterpret_cast(slot), reinterpret_cast(end)); +} + +LegacyTable read_legacy(const std::string& source) { + const auto schema_path = source + "_fields.bin"; + const auto rows_path = source + ".bin"; + const auto schema = read_bytes(schema_path); + if (schema.size() < SLOT_SIZE || schema.size() % SLOT_SIZE != 0) { + throw std::runtime_error("V1 field metadata size is invalid"); + } + const auto count_text = decode_slot(schema.data(), "V1 field count"); + if (count_text.empty() || !std::all_of(count_text.begin(), count_text.end(), [](unsigned char value) { + return std::isdigit(value) != 0; + })) { + throw std::runtime_error("V1 field count is invalid"); + } + std::size_t parsed = 0; + const auto raw_count = std::stoull(count_text, &parsed); + if (parsed != count_text.size() || raw_count == 0 || raw_count > MAX_FIELDS) { + throw std::runtime_error("V1 field count is invalid"); + } + const auto field_count = static_cast(raw_count); + if (schema.size() != (field_count + 1) * SLOT_SIZE) { + throw std::runtime_error("V1 field metadata size is invalid"); + } + std::vector fields; + fields.reserve(field_count); + std::set unique; + for (std::size_t field = 0; field < field_count; ++field) { + auto name = decode_slot(schema.data() + (field + 1) * SLOT_SIZE, "V1 field name"); + if (name.empty() || !unique.insert(name).second) { + throw std::runtime_error("V1 schema contains empty or duplicate fields"); + } + fields.push_back(std::move(name)); + } + + const auto row_bytes = read_bytes(rows_path); + if (field_count > std::numeric_limits::max() / SLOT_SIZE) { + throw std::runtime_error("V1 row width overflows this build"); + } + const auto row_width = field_count * SLOT_SIZE; + if (row_bytes.size() % row_width != 0) { + throw std::runtime_error("V1 record file contains a partial row"); + } + std::vector> rows; + rows.reserve(row_bytes.size() / row_width); + for (std::size_t offset = 0; offset < row_bytes.size(); offset += row_width) { + std::vector row; + row.reserve(field_count); + for (std::size_t field = 0; field < field_count; ++field) { + row.push_back(decode_slot( + row_bytes.data() + offset + field * SLOT_SIZE, + "V1 row value" + )); + } + rows.push_back(std::move(row)); + } + return LegacyTable{std::move(fields), std::move(rows)}; +} + +void fsync_directory(const std::filesystem::path& path) { + auto directory = path.parent_path(); + if (directory.empty()) directory = "."; + const int descriptor = ::open(directory.c_str(), O_RDONLY); + if (descriptor < 0) { + throw std::runtime_error( + "Unable to open destination directory: " + std::string(std::strerror(errno)) + ); + } + if (::fsync(descriptor) != 0) { + const auto message = std::string(std::strerror(errno)); + ::close(descriptor); + throw std::runtime_error("Unable to fsync destination directory: " + message); + } + ::close(descriptor); +} + +void print_usage() { + std::cerr + << "Usage: sql_migrate_v1 [--dry-run] [--destination ] \n"; +} + +void maybe_migration_crash(const std::string& point) { + const char* configured = std::getenv("SQL_MIGRATION_CRASH_AT"); + if (configured != nullptr && point == configured) ::_exit(86); +} + +} // namespace + +int main(int argc, char* argv[]) { + try { + bool dry_run = false; + std::string destination; + std::string source; + for (int index = 1; index < argc; ++index) { + const std::string argument = argv[index]; + if (argument == "--version") { + std::cout << "sql_migrate_v1 " << SQL_IN_CPP_VERSION + << " (storage format " << storage_v2::FORMAT_VERSION << ")\n"; + return 0; + } + if (argument == "--dry-run") { + dry_run = true; + } else if (argument == "--destination") { + if (++index >= argc) throw std::invalid_argument("--destination requires a path"); + destination = argv[index]; + } else if (!argument.empty() && argument.front() == '-') { + throw std::invalid_argument("Unknown option: " + argument); + } else if (source.empty()) { + source = argument; + } else { + throw std::invalid_argument("Only one v1 table can be migrated per invocation"); + } + } + if (source.empty()) { + print_usage(); + return 2; + } + if (destination.empty()) destination = storage_v2::database_path(source); + + const auto source_schema_path = source + "_fields.bin"; + const auto source_rows_path = source + ".bin"; + const auto source_schema_before = read_bytes(source_schema_path); + const auto source_rows_before = read_bytes(source_rows_path); + const auto legacy = read_legacy(source); + std::cout << "source: " << source << '\n'; + std::cout << "destination: " << destination << '\n'; + std::cout << "fields: " << legacy.fields.size() << '\n'; + std::cout << "rows: " << legacy.rows.size() << '\n'; + if (dry_run) { + std::cout << "result: dry-run verified, no files written\n"; + return 0; + } + if (std::filesystem::exists(destination) || + std::filesystem::exists(storage_v2::wal_path(destination))) { + throw std::runtime_error("Destination already exists: " + destination); + } + + std::string temporary_template = destination + ".tmp.XXXXXX"; + std::vector temporary_buffer( + temporary_template.begin(), + temporary_template.end() + ); + temporary_buffer.push_back('\0'); + const int temporary_descriptor = ::mkstemp(temporary_buffer.data()); + if (temporary_descriptor < 0) { + throw std::runtime_error( + "Unable to create an exclusive migration file: " + + std::string(std::strerror(errno)) + ); + } + ::close(temporary_descriptor); + const std::string temporary = temporary_buffer.data(); + const auto temporary_wal = storage_v2::wal_path(temporary); + bool temporary_owned = true; + try { + storage_v2::initialize_owned_empty_file(temporary, legacy.fields, legacy.rows); + const auto migrated = storage_v2::open_table(temporary); + if (migrated.fields != legacy.fields || migrated.rows != legacy.rows) { + throw std::runtime_error("V2 verification does not match the v1 source"); + } + std::filesystem::remove(temporary_wal); + if (read_bytes(source_schema_path) != source_schema_before || + read_bytes(source_rows_path) != source_rows_before) { + throw std::runtime_error("V1 source changed during migration; promotion aborted"); + } + maybe_migration_crash("before-promotion"); + if (::link(temporary.c_str(), destination.c_str()) != 0) { + throw std::runtime_error( + "Unable to promote without replacing the destination: " + + std::string(std::strerror(errno)) + ); + } + maybe_migration_crash("after-promotion"); + fsync_directory(destination); + maybe_migration_crash("after-directory-fsync"); + std::filesystem::remove(temporary); + temporary_owned = false; + fsync_directory(destination); + } catch (...) { + if (temporary_owned) { + std::filesystem::remove(temporary); + std::filesystem::remove(temporary_wal); + } + throw; + } + std::cout << "result: migrated and verified\n"; + return 0; + } catch (const std::exception& error) { + std::cerr << "Error: " << error.what() << '\n'; + return 1; + } +} From 9d7b3c8f57c3169eb6252556c3977ed378afefe2 Mon Sep 17 00:00:00 2001 From: hskl18 Date: Tue, 14 Jul 2026 00:49:59 -0700 Subject: [PATCH 2/4] fix: include algorithm in crash tests --- tests/storage_crash_tests.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/storage_crash_tests.cpp b/tests/storage_crash_tests.cpp index 3d77ea2..348cc21 100644 --- a/tests/storage_crash_tests.cpp +++ b/tests/storage_crash_tests.cpp @@ -1,5 +1,6 @@ #include "includes/storage/storage_v2.h" +#include #include #include #include From 94f1eab59b3ceb80c62337764573c75dfa855e87 Mon Sep 17 00:00:00 2001 From: hskl18 Date: Tue, 14 Jul 2026 00:52:05 -0700 Subject: [PATCH 3/4] fix: make index decoding order explicit --- includes/storage/storage_v2.cpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/includes/storage/storage_v2.cpp b/includes/storage/storage_v2.cpp index b960701..dff512a 100644 --- a/includes/storage/storage_v2.cpp +++ b/includes/storage/storage_v2.cpp @@ -527,7 +527,9 @@ ValidatedNode validate_index_node( if (reader.u16() != 0) throw CorruptionError("index leaf reserved value is not zero"); if (count == 0) throw CorruptionError("index leaf is empty"); for (std::uint16_t i = 0; i < count; ++i) { - result.entries.emplace_back(reader.string(), reader.u64()); + const auto key = reader.string(); + const auto record_id = reader.u64(); + result.entries.emplace_back(key, record_id); } if (reader.remaining() != 0) throw CorruptionError("index leaf has trailing bytes"); if (!std::is_sorted(result.entries.begin(), result.entries.end())) { @@ -737,7 +739,9 @@ LeafContents decode_leaf_contents(const Page& page) { std::vector> entries; entries.reserve(count); for (std::uint16_t index = 0; index < count; ++index) { - entries.emplace_back(reader.string(), reader.u64()); + const auto key = reader.string(); + const auto record_id = reader.u64(); + entries.emplace_back(key, record_id); } if (reader.remaining() != 0) throw CorruptionError("index leaf has trailing bytes"); return LeafContents{previous, std::move(entries)}; From 68eec3cc9aa1e075306e56321fcc35242a18a071 Mon Sep 17 00:00:00 2001 From: hskl18 Date: Tue, 14 Jul 2026 01:35:55 -0700 Subject: [PATCH 4/4] fix: recover orphan create WALs through SQL --- CMakeLists.txt | 3 +- includes/table/table.cpp | 11 +- tests/storage_crash_tests.cpp | 249 ++++++++++++++++++++++++++++++++++ 3 files changed, 260 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 0de66d7..ede533c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -104,8 +104,9 @@ if(BUILD_TESTING) target_link_libraries(storage_crash_tests PRIVATE sql_core) target_compile_definitions(storage_crash_tests PRIVATE STORAGE_CRASH_WORKER_PATH="$" + RUN_SQL_PATH="$" ) - add_dependencies(storage_crash_tests storage_crash_worker) + add_dependencies(storage_crash_tests storage_crash_worker run_sql) sql_enable_warnings(storage_crash_tests) add_test(NAME storage_crash_recovery COMMAND storage_crash_tests) set_tests_properties(storage_crash_recovery PROPERTIES TIMEOUT 60 LABELS crash) diff --git a/includes/table/table.cpp b/includes/table/table.cpp index ed68ac8..08c23b8 100644 --- a/includes/table/table.cpp +++ b/includes/table/table.cpp @@ -58,12 +58,19 @@ Table::Table(const string& table_name) : Table() { string field_file = table_name + "_fields.bin"; const string v2_file = storage_v2::database_path(table_name); const bool has_v2 = storage_v2::exists(v2_file); - if (!has_v2 && (!file_exists(table_file.c_str()) || !file_exists(field_file.c_str()))) { + const bool has_v1_table = file_exists(table_file.c_str()); + const bool has_v1_fields = file_exists(field_file.c_str()); + const auto v2_wal = storage_v2::wal_path(v2_file); + const bool has_orphan_v2_wal = !has_v2 && !has_v1_table && !has_v1_fields && + storage_v2::exists(v2_wal) && std::filesystem::is_regular_file(v2_wal) && + std::filesystem::file_size(v2_wal) != 0; + const bool opens_v2 = has_v2 || has_orphan_v2_wal; + if (!opens_v2 && (!has_v1_table || !has_v1_fields)) { throw std::runtime_error("Table does not exist: " + table_name); } tableName = table_name; - if (has_v2) { + if (opens_v2) { const auto loaded = storage_v2::open_table(v2_file); fieldNames = loaded.fields; selectedFields = loaded.fields; diff --git a/tests/storage_crash_tests.cpp b/tests/storage_crash_tests.cpp index 348cc21..03e2353 100644 --- a/tests/storage_crash_tests.cpp +++ b/tests/storage_crash_tests.cpp @@ -1,6 +1,7 @@ #include "includes/storage/storage_v2.h" #include +#include #include #include #include @@ -18,6 +19,61 @@ void expect(bool condition, const std::string& message) { if (!condition) throw std::runtime_error(message); } +struct ProcessResult { + int status; + std::string output; +}; + +ProcessResult run_sql_cli( + const std::string& input, + const std::string& crash_point = "", + const std::string& trace_path = "" +) { + std::array input_pipe{}; + std::array output_pipe{}; + if (::pipe(input_pipe.data()) != 0 || ::pipe(output_pipe.data()) != 0) { + throw std::runtime_error("pipe failed"); + } + const auto process = ::fork(); + if (process < 0) throw std::runtime_error("fork failed"); + if (process == 0) { + ::close(input_pipe[1]); + ::close(output_pipe[0]); + ::dup2(input_pipe[0], STDIN_FILENO); + ::dup2(output_pipe[1], STDOUT_FILENO); + ::dup2(output_pipe[1], STDERR_FILENO); + ::close(input_pipe[0]); + ::close(output_pipe[1]); + if (!crash_point.empty()) ::setenv("SQL_CRASH_AT", crash_point.c_str(), 1); + if (!trace_path.empty()) ::setenv("SQL_CRASH_TRACE_FILE", trace_path.c_str(), 1); + ::execl(RUN_SQL_PATH, RUN_SQL_PATH, nullptr); + ::_exit(127); + } + + ::close(input_pipe[0]); + ::close(output_pipe[1]); + const auto written = ::write(input_pipe[1], input.data(), input.size()); + ::close(input_pipe[1]); + if (written != static_cast(input.size())) { + ::close(output_pipe[0]); + throw std::runtime_error("unable to send SQL input"); + } + + std::string output; + std::array buffer{}; + while (true) { + const auto count = ::read(output_pipe[0], buffer.data(), buffer.size()); + if (count <= 0) break; + output.append(buffer.data(), static_cast(count)); + } + ::close(output_pipe[0]); + int status = 0; + if (::waitpid(process, &status, 0) != process || !WIFEXITED(status)) { + return {-1, output}; + } + return {WEXITSTATUS(status), output}; +} + std::vector read_bytes(const std::string& path) { std::ifstream input(path, std::ios::binary); input.seekg(0, std::ios::end); @@ -33,6 +89,27 @@ void write_bytes(const std::string& path, const std::vector& bytes output.write(reinterpret_cast(bytes.data()), static_cast(bytes.size())); } +void write_v1_slot(std::ofstream& output, const std::string& value) { + std::array slot{}; + std::copy(value.begin(), value.end(), slot.begin()); + output.write(slot.data(), static_cast(slot.size())); +} + +void write_v1_table( + const std::string& table, + const std::vector& fields, + const std::vector>& rows +) { + std::ofstream schema(table + "_fields.bin", std::ios::binary | std::ios::trunc); + write_v1_slot(schema, std::to_string(fields.size())); + for (const auto& field : fields) write_v1_slot(schema, field); + schema.close(); + std::ofstream data(table + ".bin", std::ios::binary | std::ios::trunc); + for (const auto& row : rows) { + for (const auto& value : row) write_v1_slot(data, value); + } +} + int crash_value( const std::string& point, const std::string& path, @@ -118,11 +195,43 @@ std::vector trace_points( return points; } +std::vector trace_public_create_points( + const std::string& table, + const std::string& trace_path +) { + std::filesystem::remove(trace_path); + const auto traced = run_sql_cli( + "create table " + table + " fields value\nend\n", + "", + trace_path + ); + expect(traced.status == 0, "public CREATE trace process should exit normally"); + expect(traced.output.find("Error:") == std::string::npos, + "public CREATE trace should succeed: " + traced.output); + std::ifstream input(trace_path); + std::vector points; + std::string line; + while (std::getline(input, line)) { + if (!line.empty()) points.push_back(line); + } + std::filesystem::remove(trace_path); + expect(!points.empty(), "public CREATE trace should enumerate crash boundaries"); + expect(std::set(points.begin(), points.end()).size() == points.size(), + "public CREATE should expose each crash boundary once"); + return points; +} + void cleanup(const std::string& path) { std::filesystem::remove(path); std::filesystem::remove(storage_v2::wal_path(path)); } +void cleanup_public_table(const std::string& table) { + cleanup(storage_v2::database_path(table)); + std::filesystem::remove(table + ".bin"); + std::filesystem::remove(table + "_fields.bin"); +} + void install_baseline( const std::string& path, const std::vector& image @@ -249,6 +358,140 @@ void test_create_crash_matrix() { } } +void test_public_orphan_create_wal_recovers_on_select() { + const std::string table = "public_create_wal_fsync"; + const auto path = storage_v2::database_path(table); + cleanup(path); + const auto crashed = run_sql_cli( + "create table " + table + " fields value\nend\n", + "wal-fsync" + ); + expect(crashed.status == 86, "public CREATE should terminate at wal-fsync"); + expect(!std::filesystem::exists(path), "wal-fsync CREATE should not yet have a data file"); + expect(std::filesystem::file_size(storage_v2::wal_path(path)) > 0, + "wal-fsync CREATE should leave a committed WAL"); + + const auto selected = run_sql_cli("select * from " + table + "\nend\n"); + expect(selected.status == 0, "public SELECT process should exit normally"); + expect(selected.output.find("Error:") == std::string::npos, + "public SELECT should recover the orphan CREATE WAL: " + selected.output); + expect(selected.output.find("Selected fields:") != std::string::npos, + "public SELECT should expose the recovered schema"); + expect(selected.output.find("Record count:") != std::string::npos, + "public SELECT should print its record result"); + expect(std::filesystem::exists(path), "public SELECT should materialize the database image"); + expect(std::filesystem::file_size(storage_v2::wal_path(path)) == 0, + "public SELECT recovery should clear the WAL"); + expect(storage_v2::open_table(path).rows.empty(), + "public SELECT should expose the complete empty table"); + cleanup(path); +} + +void test_public_create_crash_matrix() { + const std::string probe = "public_create_trace_probe"; + cleanup_public_table(probe); + const auto points = trace_public_create_points(probe, "public-create-crash-points.txt"); + cleanup_public_table(probe); + expect(std::find(points.begin(), points.end(), "wal-write") != points.end(), + "public CREATE matrix should cover WAL write"); + expect(std::find(points.begin(), points.end(), "wal-fsync") != points.end(), + "public CREATE matrix should cover WAL fsync"); + expect(std::find_if(points.begin(), points.end(), [](const std::string& point) { + return point.rfind("page-", 0) == 0; + }) != points.end(), "public CREATE matrix should cover page writes"); + expect(std::find(points.begin(), points.end(), "data-fsync") != points.end(), + "public CREATE matrix should cover data fsync"); + expect(std::find(points.begin(), points.end(), "wal-clear") != points.end(), + "public CREATE matrix should cover the committed WAL-clear boundary"); + + for (const auto& point : points) { + auto suffix = point; + std::replace(suffix.begin(), suffix.end(), '-', '_'); + const auto table = "public_create_" + suffix; + const auto path = storage_v2::database_path(table); + cleanup_public_table(table); + const auto crashed = run_sql_cli( + "create table " + table + " fields value\nend\n", + point + ); + expect(crashed.status == 86, "public CREATE should terminate at " + point); + + auto selected = run_sql_cli("select * from " + table + "\nend\n"); + if (selected.output.find("Table does not exist") != std::string::npos) { + const auto retried = run_sql_cli( + "create table " + table + " fields value\nend\n" + ); + expect(retried.output.find("Error:") == std::string::npos, + "old-state CREATE retry should complete recovery at " + point + + ": " + retried.output); + selected = run_sql_cli("select * from " + table + "\nend\n"); + } + expect(selected.status == 0, "public SELECT process should exit normally at " + point); + expect(selected.output.find("Error:") == std::string::npos, + "public path should expose an old-or-new recoverable state at " + point + + ": " + selected.output); + expect(selected.output.find("Selected fields: value") != std::string::npos, + "public SELECT should expose the complete schema at " + point); + expect(std::filesystem::exists(path), + "public recovery should materialize the database at " + point); + expect(std::filesystem::file_size(storage_v2::wal_path(path)) == 0, + "public recovery should clear the WAL at " + point); + expect(storage_v2::open_table(path).rows.empty(), + "public recovery should expose no partial rows at " + point); + cleanup_public_table(table); + } +} + +void test_public_orphan_wal_ownership_boundaries() { + const std::string owner = "public_wal_owner"; + const std::string other = "public_wal_other"; + cleanup_public_table(owner); + cleanup_public_table(other); + const auto crashed = run_sql_cli( + "create table " + owner + " fields value\nend\n", + "wal-fsync" + ); + expect(crashed.status == 86, "owner CREATE should leave an orphan WAL"); + const auto unrelated = run_sql_cli("select * from " + other + "\nend\n"); + expect(unrelated.output.find("Table does not exist: " + other) != std::string::npos, + "an orphan WAL must not identify a different table"); + expect(!std::filesystem::exists(storage_v2::database_path(owner)), + "opening another table must not recover the owner's WAL"); + expect(!std::filesystem::exists(storage_v2::database_path(other)), + "opening another table must not create a database"); + const auto owner_selected = run_sql_cli("select * from " + owner + "\nend\n"); + expect(owner_selected.output.find("Error:") == std::string::npos, + "the exact owner path should recover its WAL"); + cleanup_public_table(owner); + cleanup_public_table(other); + + const std::string legacy = "public_wal_legacy"; + cleanup_public_table(legacy); + const auto legacy_crash = run_sql_cli( + "create table " + legacy + " fields value\nend\n", + "wal-fsync" + ); + expect(legacy_crash.status == 86, "legacy boundary should leave an orphan WAL"); + const auto wal_before = read_bytes(storage_v2::wal_path(storage_v2::database_path(legacy))); + write_v1_table(legacy, {"value"}, {{"legacy"}}); + const auto v1_data_before = read_bytes(legacy + ".bin"); + const auto v1_schema_before = read_bytes(legacy + "_fields.bin"); + const auto legacy_selected = run_sql_cli("select * from " + legacy + "\nend\n"); + expect(legacy_selected.output.find("Error:") == std::string::npos, + "valid v1 should remain publicly readable beside an orphan WAL"); + expect(legacy_selected.output.find("legacy") != std::string::npos, + "public SELECT should return the v1 row"); + expect(!std::filesystem::exists(storage_v2::database_path(legacy)), + "orphan WAL recovery must not supersede v1"); + expect(read_bytes(legacy + ".bin") == v1_data_before, + "orphan WAL recovery must not modify v1 rows"); + expect(read_bytes(legacy + "_fields.bin") == v1_schema_before, + "orphan WAL recovery must not modify v1 schema"); + expect(read_bytes(storage_v2::wal_path(storage_v2::database_path(legacy))) == wal_before, + "opening v1 must leave the conflicting orphan WAL untouched"); + cleanup_public_table(legacy); +} + void test_wal_checksum_rejection() { const std::string path = "crash-wal-checksum.sql2"; cleanup(path); @@ -312,6 +555,12 @@ int main() { std::cout << "PASS: dynamic internal-split crash matrix\n"; test_create_crash_matrix(); std::cout << "PASS: atomic create crash matrix\n"; + test_public_orphan_create_wal_recovers_on_select(); + std::cout << "PASS: public orphan CREATE WAL recovery\n"; + test_public_create_crash_matrix(); + std::cout << "PASS: public CREATE crash matrix\n"; + test_public_orphan_wal_ownership_boundaries(); + std::cout << "PASS: public orphan WAL ownership boundaries\n"; test_wal_checksum_rejection(); std::cout << "PASS: WAL checksum rejection\n"; test_stale_wal_rejection();