From c0d0e292b7a9fc64580dbdd4dbd5480b4826e3e4 Mon Sep 17 00:00:00 2001 From: Darick Tong Date: Thu, 8 Oct 2026 15:50:36 -0700 Subject: [PATCH 1/2] feat: add Database#persistWal() to keep the WAL on close Add a persistWal([toggleState]) method that sets SQLITE_FCNTL_PERSIST_WAL on the connection's main database. While it is on, the last connection to close still checkpoints the WAL into the database, but keeps the -wal and -shm files instead of deleting them. The next connection then appends to the same WAL with the same salt. This lets zero-cache keep a replica's WAL intact across closes, so a litestream backup can continue from it instead of taking a new snapshot. Today zero-cache does that by holding an extra connection open (WalKeeper). Persistence is off by default, so behavior is unchanged for existing users. It is a no-op for in-memory and temporary databases, and throws if the connection is closed. Turn it off before switching the database out of WAL mode. Otherwise SQLite leaves the -wal file in place and reopens it on the next transaction, so the database effectively stays in WAL mode. --- docs/api.md | 17 +++++++ lib/database.js | 1 + lib/index.d.ts | 1 + lib/methods/wrappers.js | 5 +++ src/objects/database.cpp | 15 +++++++ src/objects/database.hpp | 1 + test/53.persist-wal.js | 97 ++++++++++++++++++++++++++++++++++++++++ 7 files changed, 137 insertions(+) create mode 100644 test/53.persist-wal.js diff --git a/docs/api.md b/docs/api.md index c935478..25de1a0 100644 --- a/docs/api.md +++ b/docs/api.md @@ -19,6 +19,7 @@ - [Database#loadExtension()](#loadextensionpath-entrypoint---this) - [Database#exec()](#execstring---this) - [Database#close()](#close---this) +- [Database#persistWal()](#persistwaltogglestate---this) - [Properties](#properties) ### new Database(*path*, [*options*]) @@ -394,6 +395,22 @@ process.on('SIGINT', () => process.exit(128 + 2)); process.on('SIGTERM', () => process.exit(128 + 15)); ``` +### .persistWal([*toggleState*]) -> *this* + +Toggles whether the connection keeps the [write-ahead log](https://www.sqlite.org/wal.html) when it is the last connection to close the database (see [`SQLITE_FCNTL_PERSIST_WAL`](https://www.sqlite.org/c3ref/c_fcntl_begin_atomic_write.html#sqlitefcntlpersistwal)). The last connection still checkpoints the WAL into the database, but leaves the `-wal` and `-shm` files in place instead of deleting them, so the next connection continues the same WAL. It is off by default. Invoking it without an argument turns it on. + +The setting applies to the `main` database of this connection only, and has no effect on in-memory or temporary databases. + +Turn it off before switching the database out of WAL mode (e.g. `PRAGMA journal_mode = DELETE` or `OFF`). Otherwise the `-wal` file is left in place and SQLite reopens it on the next transaction, so the database effectively stays in WAL mode. + +```js +db.pragma('journal_mode = WAL'); +db.persistWal(); // Keep the WAL when this connection closes. + +db.persistWal(false); +db.pragma('journal_mode = DELETE'); // Folds in and deletes the WAL. +``` + ## Properties **.open -> _boolean_** - Whether the database connection is currently open. diff --git a/lib/database.js b/lib/database.js index c46fef8..150fff8 100644 --- a/lib/database.js +++ b/lib/database.js @@ -84,6 +84,7 @@ Database.prototype.exec = wrappers.exec; Database.prototype.close = wrappers.close; Database.prototype.defaultSafeIntegers = wrappers.defaultSafeIntegers; Database.prototype.unsafeMode = wrappers.unsafeMode; +Database.prototype.persistWal = wrappers.persistWal; Database.prototype.loadExtension = wrappers.loadExtension; Database.prototype[util.inspect] = require('./methods/inspect'); diff --git a/lib/index.d.ts b/lib/index.d.ts index e57c14c..d48eba8 100644 --- a/lib/index.d.ts +++ b/lib/index.d.ts @@ -102,6 +102,7 @@ declare namespace BetterSqlite3 { backup(destinationFile: string, options?: Database.BackupOptions): Promise; table(name: string, options: VirtualTableOptions): this; unsafeMode(unsafe?: boolean): this; + persistWal(persist?: boolean): this; serialize(options?: Database.SerializeOptions): Buffer; } diff --git a/lib/methods/wrappers.js b/lib/methods/wrappers.js index 67d80a4..49bd235 100644 --- a/lib/methods/wrappers.js +++ b/lib/methods/wrappers.js @@ -25,6 +25,11 @@ exports.unsafeMode = function unsafeMode(...args) { return this; }; +exports.persistWal = function persistWal(...args) { + this[cppdb].persistWal(...args); + return this; +}; + exports.loadExtension = function loadExtension(...args) { this[cppdb].loadExtension(...args); return this; diff --git a/src/objects/database.cpp b/src/objects/database.cpp index 19e615f..4910f50 100644 --- a/src/objects/database.cpp +++ b/src/objects/database.cpp @@ -136,6 +136,7 @@ INIT(Database::Init) { SetPrototypeMethod(isolate, data, t, "close", JS_close); SetPrototypeMethod(isolate, data, t, "defaultSafeIntegers", JS_defaultSafeIntegers); SetPrototypeMethod(isolate, data, t, "unsafeMode", JS_unsafeMode); + SetPrototypeMethod(isolate, data, t, "persistWal", JS_persistWal); SetPrototypeGetter(isolate, data, t, "open", JS_open); SetPrototypeGetter(isolate, data, t, "inTransaction", JS_inTransaction); return t->GetFunction(OnlyContext).ToLocalChecked(); @@ -407,6 +408,20 @@ NODE_METHOD(Database::JS_unsafeMode) { sqlite3_db_config(db->db_handle, SQLITE_DBCONFIG_DEFENSIVE, static_cast(!db->unsafe_mode), NULL); } +NODE_METHOD(Database::JS_persistWal) { + Database* db = Unwrap(info.This()); + REQUIRE_DATABASE_OPEN(db); + bool persist_wal = true; + if (info.Length() != 0) { REQUIRE_ARGUMENT_BOOLEAN(first, persist_wal); } + int arg = persist_wal ? 1 : 0; + int status = sqlite3_file_control(db->db_handle, "main", SQLITE_FCNTL_PERSIST_WAL, &arg); + // The VFS of an in-memory or temporary database has no WAL to persist and + // reports SQLITE_NOTFOUND, which is not an error. + if (status != SQLITE_OK && status != SQLITE_NOTFOUND) { + ThrowSqliteError(db->addon, sqlite3_errstr(status), status); + } +} + NODE_GETTER(Database::JS_open) { info.GetReturnValue().Set(Unwrap(PROPERTY_HOLDER(info))->open); } diff --git a/src/objects/database.hpp b/src/objects/database.hpp index bd8889d..7f59722 100644 --- a/src/objects/database.hpp +++ b/src/objects/database.hpp @@ -79,6 +79,7 @@ class Database : public node::ObjectWrap { static NODE_METHOD(JS_close); static NODE_METHOD(JS_defaultSafeIntegers); static NODE_METHOD(JS_unsafeMode); + static NODE_METHOD(JS_persistWal); static NODE_GETTER(JS_open); static NODE_GETTER(JS_inTransaction); diff --git a/test/53.persist-wal.js b/test/53.persist-wal.js new file mode 100644 index 0000000..592613e --- /dev/null +++ b/test/53.persist-wal.js @@ -0,0 +1,97 @@ +'use strict'; +const fs = require('fs'); +const Database = require('../.'); + +describe('Database#persistWal()', function () { + beforeEach(function () { + this.db = new Database(util.next()); + this.wal = `${this.db.name}-wal`; + this.db.pragma('journal_mode = WAL'); + this.db.exec('CREATE TABLE t (x INTEGER)'); + this.db.prepare('INSERT INTO t VALUES (?)').run(1); + }); + afterEach(function () { + if (this.db.open) this.db.close(); + }); + + // The checkpoint sequence number and the salts of the WAL header. + const walHeader = wal => fs.readFileSync(wal).subarray(12, 24).toString('hex'); + + it('should return the database object', function () { + expect(this.db.persistWal()).to.equal(this.db); + expect(this.db.persistWal(false)).to.equal(this.db); + expect(this.db.persistWal(true)).to.equal(this.db); + }); + it('should throw if the argument is not a boolean', function () { + expect(() => this.db.persistWal(1)).to.throw(TypeError); + expect(() => this.db.persistWal('true')).to.throw(TypeError); + expect(() => this.db.persistWal(null)).to.throw(TypeError); + }); + it('should throw if the database is closed', function () { + this.db.close(); + expect(() => this.db.persistWal()).to.throw(TypeError); + }); + it('should be a no-op for in-memory databases', function () { + const db = new Database(':memory:'); + expect(db.persistWal()).to.equal(db); + expect(db.persistWal(false)).to.equal(db); + db.close(); + }); + it('should delete the WAL on the last close by default', function () { + expect(fs.existsSync(this.wal)).to.be.true; + this.db.close(); + expect(fs.existsSync(this.wal)).to.be.false; + }); + it('should keep the WAL on the last close when toggled on', function () { + this.db.persistWal(); + this.db.close(); + expect(fs.statSync(this.wal).size).to.be.above(32); + const header = walHeader(this.wal); + + // Reopening and writing appends to the same WAL, rather than restarting it. + const size = fs.statSync(this.wal).size; + const db = new Database(this.db.name); + db.persistWal(); + db.prepare('INSERT INTO t VALUES (?)').run(2); + expect(fs.statSync(this.wal).size).to.be.above(size); + expect(walHeader(this.wal)).to.equal(header); + db.close(); + expect(walHeader(this.wal)).to.equal(header); + + const reader = new Database(this.db.name, { readonly: true }); + expect(reader.prepare('SELECT x FROM t ORDER BY x').pluck().all()).to.deep.equal([1, 2]); + reader.close(); + }); + it('should delete the WAL on the last close when toggled off', function () { + this.db.persistWal(); + this.db.persistWal(false); + this.db.close(); + expect(fs.existsSync(this.wal)).to.be.false; + }); + it('should keep the WAL when leaving WAL mode while toggled on', function () { + // This is why persistence must be toggled off before leaving WAL mode. + this.db.persistWal(); + expect(this.db.pragma('journal_mode = DELETE', { simple: true })).to.equal('delete'); + expect(fs.existsSync(this.wal)).to.be.true; + }); + it('should leave WAL mode when toggled off first', function () { + this.db.persistWal(); + this.db.unsafeMode(); + this.db.persistWal(false); + expect(this.db.pragma('journal_mode = OFF', { simple: true })).to.equal('off'); + expect(fs.existsSync(this.wal)).to.be.false; + this.db.exec('VACUUM'); + this.db.prepare('INSERT INTO t VALUES (?)').run(2); + expect(fs.existsSync(this.wal)).to.be.false; + this.db.unsafeMode(false); + this.db.persistWal(); + this.db.close(); + + const db = new Database(this.db.name); + expect(db.pragma('journal_mode', { simple: true })).to.equal('delete'); + expect(db.prepare('SELECT x FROM t ORDER BY x').pluck().all()).to.deep.equal([1, 2]); + expect(db.pragma('integrity_check', { simple: true })).to.equal('ok'); + db.close(); + expect(fs.existsSync(this.wal)).to.be.false; + }); +}); From 27cd337ff950bcf3d6c9c5b225a02fd4c046fd4e Mon Sep 17 00:00:00 2001 From: Darick Tong Date: Thu, 8 Oct 2026 16:28:44 -0700 Subject: [PATCH 2/2] workaround bug in Node 24.19.0+ --- .github/workflows/build.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 69e4ab3..a0b30f6 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -59,7 +59,14 @@ jobs: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6 with: - node-version: ${{ matrix.node }} + # Node 24.19.0+ (through at least 24.21.0) aborts when it garbage + # collects a node::ObjectWrap compiled against its own headers: the + # header's destructor calls RemoveEnvironmentCleanupHook, which the + # 24.x runtime can't handle without a current Environment + # (https://github.com/nodejs/node/issues/65446). The release + # prebuilds compile against 24.0.0 headers and are unaffected. + # Unpin once a 24.x release ships the fix. + node-version: ${{ matrix.node == 24 && '24.18.1' || matrix.node }} - if: ${{ startsWith(matrix.os, 'windows') }} run: pip.exe install setuptools - if: ${{ startsWith(matrix.os, 'macos') }}