From a2bf4dd76fceff003cca4d782747a19d66d36a89 Mon Sep 17 00:00:00 2001
From: venkat1701 <thejeastdev@gmail.com>
Date: Sun, 11 Oct 2026 05:44:30 +0530
Subject: [PATCH 01/15] refactor(engine): move the mapped long array out of
 PageDirectory

The reference counts file in the next commit is laid out the same way:
a 64-byte header and one 8-byte entry per page number. MappedLongs holds
the mapping, the header check and the chunked growth for both.
---
 .../io/hstore/engine/page/MappedLongs.java    | 116 ++++++++++++++++++
 .../io/hstore/engine/page/PageDirectory.java  |  91 +++-----------
 2 files changed, 131 insertions(+), 76 deletions(-)
 create mode 100644 engine/src/main/java/io/hstore/engine/page/MappedLongs.java

diff --git a/engine/src/main/java/io/hstore/engine/page/MappedLongs.java b/engine/src/main/java/io/hstore/engine/page/MappedLongs.java
new file mode 100644
index 0000000..a211349
--- /dev/null
+++ b/engine/src/main/java/io/hstore/engine/page/MappedLongs.java
@@ -0,0 +1,116 @@
+package io.hstore.engine.page;
+
+import io.hstore.engine.HStoreException;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.lang.foreign.Arena;
+import java.lang.foreign.MemorySegment;
+import java.lang.foreign.ValueLayout;
+import java.lang.invoke.VarHandle;
+import java.nio.ByteOrder;
+import java.nio.channels.FileChannel;
+import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
+import java.util.Arrays;
+
+final class MappedLongs implements AutoCloseable {
+
+    static final long HEADER = 64;
+
+    private static final int CHUNK_SHIFT = 23;
+    private static final long CHUNK_BYTES = 1L << CHUNK_SHIFT;
+    private static final ValueLayout.OfInt INT = ValueLayout.JAVA_INT.withOrder(ByteOrder.LITTLE_ENDIAN);
+    private static final ValueLayout.OfLong LONG = ValueLayout.JAVA_LONG.withOrder(ByteOrder.LITTLE_ENDIAN);
+    private static final VarHandle ENTRY = LONG.varHandle();
+
+    private final Path path;
+    private final FileChannel channel;
+    private final boolean fresh;
+    private volatile MemorySegment[] chunks = new MemorySegment[0];
+
+    private MappedLongs(Path path, FileChannel channel, boolean fresh) {
+        this.path = path;
+        this.channel = channel;
+        this.fresh = fresh;
+    }
+
+    static MappedLongs open(Path path, int magic, int format, String what) {
+        try {
+            FileChannel channel = FileChannel.open(path, StandardOpenOption.CREATE, StandardOpenOption.READ, StandardOpenOption.WRITE);
+            boolean fresh = channel.size() == 0;
+            MappedLongs longs = new MappedLongs(path, channel, fresh);
+            longs.mapThrough(fresh ? 0 : (int) ((channel.size() - 1) >>> CHUNK_SHIFT));
+            MemorySegment header = longs.chunks[0];
+            if (fresh) {
+                header.set(INT, 0, magic);
+                header.set(INT, 4, format);
+            } else if (header.get(INT, 0) != magic || header.get(INT, 4) != format) {
+                channel.close();
+                throw HStoreException.invalid(what + " " + path + " has an unknown format");
+            }
+            return longs;
+        } catch (IOException e) {
+            throw new UncheckedIOException("cannot open " + what + " " + path, e);
+        }
+    }
+
+    boolean fresh() {
+        return fresh;
+    }
+
+    long header(int offset) {
+        return chunks[0].get(LONG, offset);
+    }
+
+    void header(int offset, long value) {
+        chunks[0].set(LONG, offset, value);
+    }
+
+    long get(long index) {
+        long position = HEADER + index * Long.BYTES;
+        MemorySegment[] mapped = chunks;
+        int chunk = (int) (position >>> CHUNK_SHIFT);
+        return chunk < mapped.length ? (long) ENTRY.getAcquire(mapped[chunk], position & (CHUNK_BYTES - 1)) : 0;
+    }
+
+    void put(long index, long value) {
+        long position = HEADER + index * Long.BYTES;
+        int chunk = (int) (position >>> CHUNK_SHIFT);
+        if (chunk >= chunks.length) {
+            mapThrough(chunk);
+        }
+        ENTRY.setRelease(chunks[chunk], position & (CHUNK_BYTES - 1), value);
+    }
+
+    void force() {
+        for (MemorySegment chunk : chunks) {
+            chunk.force();
+        }
+    }
+
+    private synchronized void mapThrough(int last) {
+        MemorySegment[] mapped = chunks;
+        if (last < mapped.length) {
+            return;
+        }
+        MemorySegment[] grown = Arrays.copyOf(mapped, last + 1);
+        try {
+            for (int chunk = mapped.length; chunk <= last; chunk++) {
+                grown[chunk] = channel.map(FileChannel.MapMode.READ_WRITE, chunk * CHUNK_BYTES, CHUNK_BYTES, Arena.ofAuto());
+            }
+        } catch (IOException e) {
+            throw new UncheckedIOException("cannot grow " + path, e);
+        }
+        chunks = grown;
+    }
+
+    @Override
+    public void close() {
+        try {
+            channel.close();
+        } catch (IOException e) {
+            throw new UncheckedIOException("cannot close " + path, e);
+        }
+    }
+}
diff --git a/engine/src/main/java/io/hstore/engine/page/PageDirectory.java b/engine/src/main/java/io/hstore/engine/page/PageDirectory.java
index c7e2c50..1e2dbdb 100644
--- a/engine/src/main/java/io/hstore/engine/page/PageDirectory.java
+++ b/engine/src/main/java/io/hstore/engine/page/PageDirectory.java
@@ -2,17 +2,7 @@
 
 import io.hstore.engine.HStoreException;
 
-import java.io.IOException;
-import java.io.UncheckedIOException;
-import java.lang.foreign.Arena;
-import java.lang.foreign.MemorySegment;
-import java.lang.foreign.ValueLayout;
-import java.lang.invoke.VarHandle;
-import java.nio.ByteOrder;
-import java.nio.channels.FileChannel;
 import java.nio.file.Path;
-import java.nio.file.StandardOpenOption;
-import java.util.Arrays;
 
 final class PageDirectory implements AutoCloseable {
 
@@ -20,60 +10,31 @@ final class PageDirectory implements AutoCloseable {
 
     private static final int MAGIC = 0x52494450;
     private static final int FORMAT = 1;
-    private static final long HEADER = 64;
-    private static final long NEXT_OFFSET = 8;
-    private static final int CHUNK_SHIFT = 23;
-    private static final long CHUNK_BYTES = 1L << CHUNK_SHIFT;
-    private static final ValueLayout.OfInt INT = ValueLayout.JAVA_INT.withOrder(ByteOrder.LITTLE_ENDIAN);
-    private static final ValueLayout.OfLong LONG = ValueLayout.JAVA_LONG.withOrder(ByteOrder.LITTLE_ENDIAN);
-    private static final VarHandle ENTRY = LONG.varHandle();
+    private static final int NEXT_OFFSET = 8;
 
-    private final Path path;
-    private final FileChannel channel;
-    private volatile MemorySegment[] chunks = new MemorySegment[0];
+    private final MappedLongs entries;
     private long next;
 
-    private PageDirectory(Path path, FileChannel channel) {
-        this.path = path;
-        this.channel = channel;
+    private PageDirectory(MappedLongs entries) {
+        this.entries = entries;
     }
 
     static PageDirectory open(Path path) {
-        try {
-            FileChannel channel = FileChannel.open(path, StandardOpenOption.CREATE, StandardOpenOption.READ, StandardOpenOption.WRITE);
-            PageDirectory directory = new PageDirectory(path, channel);
-            boolean fresh = channel.size() == 0;
-            directory.mapThrough(fresh ? 0 : (int) ((channel.size() - 1) >>> CHUNK_SHIFT));
-            MemorySegment header = directory.chunks[0];
-            if (fresh) {
-                header.set(INT, 0, MAGIC);
-                header.set(INT, 4, FORMAT);
-                header.set(LONG, NEXT_OFFSET, 1L);
-            } else if (header.get(INT, 0) != MAGIC || header.get(INT, 4) != FORMAT) {
-                channel.close();
-                throw HStoreException.invalid("page directory " + path + " has an unknown format");
-            }
-            directory.next = header.get(LONG, NEXT_OFFSET);
-            return directory;
-        } catch (IOException e) {
-            throw new UncheckedIOException("cannot open page directory " + path, e);
+        MappedLongs entries = MappedLongs.open(path, MAGIC, FORMAT, "page directory");
+        if (entries.fresh()) {
+            entries.header(NEXT_OFFSET, 1L);
         }
+        PageDirectory directory = new PageDirectory(entries);
+        directory.next = entries.header(NEXT_OFFSET);
+        return directory;
     }
 
     long get(long index) {
-        long position = HEADER + index * Long.BYTES;
-        MemorySegment[] mapped = chunks;
-        int chunk = (int) (position >>> CHUNK_SHIFT);
-        return chunk < mapped.length ? (long) ENTRY.getAcquire(mapped[chunk], position & (CHUNK_BYTES - 1)) : PageAddress.NONE;
+        return entries.get(index);
     }
 
     void put(long index, long address) {
-        long position = HEADER + index * Long.BYTES;
-        int chunk = (int) (position >>> CHUNK_SHIFT);
-        if (chunk >= chunks.length) {
-            mapThrough(chunk);
-        }
-        ENTRY.setRelease(chunks[chunk], position & (CHUNK_BYTES - 1), address);
+        entries.put(index, address);
     }
 
     long allocate() {
@@ -88,7 +49,7 @@ long allocate() {
     void reserve(long index) {
         if (index >= next) {
             next = index + 1;
-            chunks[0].set(LONG, NEXT_OFFSET, next);
+            entries.header(NEXT_OFFSET, next);
         }
     }
 
@@ -97,33 +58,11 @@ long next() {
     }
 
     void force() {
-        for (MemorySegment chunk : chunks) {
-            chunk.force();
-        }
-    }
-
-    private void mapThrough(int last) {
-        MemorySegment[] mapped = chunks;
-        if (last < mapped.length) {
-            return;
-        }
-        MemorySegment[] grown = Arrays.copyOf(mapped, last + 1);
-        try {
-            for (int chunk = mapped.length; chunk <= last; chunk++) {
-                grown[chunk] = channel.map(FileChannel.MapMode.READ_WRITE, chunk * CHUNK_BYTES, CHUNK_BYTES, Arena.ofAuto());
-            }
-        } catch (IOException e) {
-            throw new UncheckedIOException("cannot grow page directory " + path, e);
-        }
-        chunks = grown;
+        entries.force();
     }
 
     @Override
     public void close() {
-        try {
-            channel.close();
-        } catch (IOException e) {
-            throw new UncheckedIOException("cannot close page directory " + path, e);
-        }
+        entries.close();
     }
 }

From aaa6b567c2d0bfe9cda182285bd543a323bdc42a Mon Sep 17 00:00:00 2001
From: venkat1701 <thejeastdev@gmail.com>
Date: Sun, 11 Oct 2026 05:55:22 +0530
Subject: [PATCH 02/15] fix(engine): replay a new branch before the roots
 logged for it

createBranch logs the new branch's Root records ahead of its BranchMeta
record, and redo applied them in log order. A branch created after the
last checkpoint therefore made recovery fail with a NullPointerException
on the next open. Redo now applies BranchMeta records first.
---
 .../hstore/engine/maintenance/Recovery.java   |  5 +++-
 .../io/hstore/engine/CrashRecoveryTest.java   | 28 +++++++++++++++++++
 2 files changed, 32 insertions(+), 1 deletion(-)

diff --git a/engine/src/main/java/io/hstore/engine/maintenance/Recovery.java b/engine/src/main/java/io/hstore/engine/maintenance/Recovery.java
index 298da98..a20bc55 100644
--- a/engine/src/main/java/io/hstore/engine/maintenance/Recovery.java
+++ b/engine/src/main/java/io/hstore/engine/maintenance/Recovery.java
@@ -19,6 +19,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.TreeMap;
+import java.util.stream.Stream;
 
 public final class Recovery {
 
@@ -107,7 +108,9 @@ private static boolean intact(List<WalRecord> records, PageStore pages) {
     private static Generation redo(Generation base, WalRecord.Commit commit, List<WalRecord> records,
                                    PageStore pages, ChangeFeed feed, FeedCodec codec) {
         Map<Integer, Branch> branches = new TreeMap<>(base.branches());
-        for (WalRecord record : records) {
+        List<WalRecord> metaFirst = Stream.concat(records.stream().filter(WalRecord.BranchMeta.class::isInstance),
+                records.stream().filter(record -> !(record instanceof WalRecord.BranchMeta))).toList();
+        for (WalRecord record : metaFirst) {
             switch (record) {
                 case WalRecord.Page(long _, long pageId, long _, byte[] image) -> pages.write(pageId, MemorySegment.ofArray(image));
                 case WalRecord.BranchMeta(long _, int id, String name, int parent, long baseGeneration, long created, Branch.State state) -> {
diff --git a/engine/src/test/java/io/hstore/engine/CrashRecoveryTest.java b/engine/src/test/java/io/hstore/engine/CrashRecoveryTest.java
index 61cd5bf..780296c 100644
--- a/engine/src/test/java/io/hstore/engine/CrashRecoveryTest.java
+++ b/engine/src/test/java/io/hstore/engine/CrashRecoveryTest.java
@@ -1,9 +1,12 @@
 package io.hstore.engine;
 
+import io.hstore.engine.catalog.Branch;
 import io.hstore.engine.topology.EdgeKind;
 import io.hstore.engine.topology.Incidence;
 import io.hstore.engine.txn.CrashPoint;
+import io.hstore.engine.txn.Snapshot;
 import io.hstore.engine.txn.Transaction;
+import io.hstore.engine.txn.TxnOptions;
 import io.hstore.engine.txn.WalMode;
 
 import org.junit.jupiter.api.Test;
@@ -113,6 +116,31 @@ void recoveryStopsAtTheFirstCommitWhosePagesAreMissing() throws Exception {
         }
     }
 
+    @ParameterizedTest
+    @EnumSource(WalMode.class)
+    void aBranchCreatedSinceTheLastCheckpointComesBackFromTheLog(WalMode mode) {
+        EngineOptions options = EngineTest.small().withWalMode(mode).withCheckpointWalBytes(Long.MAX_VALUE);
+        StorageEngine engine = StorageEngine.open(directory, options);
+        long edge = engine.write(txn -> {
+            long e = txn.createEdge(EngineTest.EDGE, EdgeKind.SET);
+            txn.insert(e, 1);
+            return e;
+        });
+        Branch branch = engine.createBranch("draft", Branch.MAIN);
+        engine.write(TxnOptions.defaults().onBranch(branch.id()), txn -> {
+            txn.insert(edge, 2);
+            return null;
+        });
+        engine.halt();
+        try (StorageEngine recovered = StorageEngine.open(directory, options)) {
+            assertEquals(List.of("draft"), recovered.branches().stream().map(Branch::name).filter("draft"::equals).toList());
+            assertEquals(1L, (long) recovered.read(snapshot -> snapshot.requireEdge(edge).size()));
+            try (Snapshot draft = recovered.snapshot(branch.id())) {
+                assertEquals(2L, draft.requireEdge(edge).size());
+            }
+        }
+    }
+
     @ParameterizedTest
     @EnumSource(WalMode.class)
     void directoryEntriesLostInACrashAreRebuiltFromTheLog(WalMode mode) throws Exception {

From d509cbcd361eca9fdfa7fff15c6a950d1c3f2eb1 Mon Sep 17 00:00:00 2001
From: venkat1701 <thejeastdev@gmail.com>
Date: Sun, 11 Oct 2026 05:59:44 +0530
Subject: [PATCH 03/15] feat(engine): count references to every page and
 release unused ones

Each page number now has a count of the pages and branch roots that point
to it, kept in data/segments/pages.ref next to the page directory. A
background thread updates the counts after every published commit, so the
commit path only hands it the generation before and after. Pages that
nothing points to any more are marked with the generation they died in,
and a checkpoint releases them once no retained generation or open reader
is older than that. Releasing clears the directory entry and lowers the
live byte count of the segment the page sits in.

The counts file is only written at checkpoints. The checkpoint writes the
changed counts and the released directory entries to pages.ref.journal
first, publishes the catalog, and then applies the journal, so a crash at
any point leaves the counts as of one checkpoint or the next. Applying the
journal happens after the commit lock is let go, with only the counter
paused, and the segments are synced once before the lock is taken, so a
checkpoint holds commits up for less time than before. On open the
journal is applied if it matches the published checkpoint, and the counts
are brought forward by replaying the generations recovered from the log.
If the counter ever fails it marks the file for rebuild, and the next
open recounts everything from the retained generations.

A full scan of the directory on open also releases pages that no
committed generation refers to, such as pages spilled by a transaction
that was cut short by a crash. Pages spilled by a transaction that aborts
or conflicts are handed back straight away.

Reader pins are now checked against the retained history after they are
taken, and trimming puts a generation back if a reader pinned it at the
same time, so a reader can't hold a generation whose pages were released.

Leaves now set a header flag when any of their values points at another
page, and decoding checks it. The counter reads only the header of an
uncached leaf without the flag, which is most of them, so counting the
commits replayed after a crash costs little more than replaying them.

liveness() reports the per-segment counters instead of walking the trees,
and hstore check compares the counts against a full walk.

Storage format goes to 5.
---
 .../io/hstore/db/property/PropertySlots.java  |   5 +
 .../java/io/hstore/engine/StorageEngine.java  |  28 +-
 .../io/hstore/engine/catalog/AtomCodec.java   |   5 +
 .../io/hstore/engine/index/PostingsCodec.java |   5 +
 .../engine/maintenance/Checkpointer.java      |  62 +-
 .../hstore/engine/maintenance/Recovery.java   |  14 +-
 .../engine/maintenance/ReferenceCounter.java  | 570 ++++++++++++++++++
 .../io/hstore/engine/page/LongLongMap.java    | 126 ++++
 .../io/hstore/engine/page/PageHeader.java     |   5 +
 .../java/io/hstore/engine/page/PageStore.java | 139 ++++-
 .../hstore/engine/page/ReferenceCounts.java   | 101 ++++
 .../hstore/engine/page/ReferenceJournal.java  | 150 +++++
 .../java/io/hstore/engine/tree/NodeCodec.java |  25 +-
 .../io/hstore/engine/tree/NodeSource.java     |   4 +
 .../hstore/engine/tree/PagedNodeSource.java   |  30 +
 .../io/hstore/engine/tree/TreeWalker.java     |  51 +-
 .../io/hstore/engine/tree/ValueCodec.java     |   9 +
 .../java/io/hstore/engine/txn/CrashPoint.java |   3 +-
 .../hstore/engine/txn/ReferenceTracker.java   |  20 +
 .../io/hstore/engine/txn/Transaction.java     |  25 +-
 .../hstore/engine/txn/TransactionManager.java |  60 +-
 .../src/main/java/io/hstore/server/Check.java |   6 +-
 22 files changed, 1369 insertions(+), 74 deletions(-)
 create mode 100644 engine/src/main/java/io/hstore/engine/maintenance/ReferenceCounter.java
 create mode 100644 engine/src/main/java/io/hstore/engine/page/LongLongMap.java
 create mode 100644 engine/src/main/java/io/hstore/engine/page/ReferenceCounts.java
 create mode 100644 engine/src/main/java/io/hstore/engine/page/ReferenceJournal.java
 create mode 100644 engine/src/main/java/io/hstore/engine/txn/ReferenceTracker.java

diff --git a/database/src/main/java/io/hstore/db/property/PropertySlots.java b/database/src/main/java/io/hstore/db/property/PropertySlots.java
index 8759a31..af87369 100644
--- a/database/src/main/java/io/hstore/db/property/PropertySlots.java
+++ b/database/src/main/java/io/hstore/db/property/PropertySlots.java
@@ -52,6 +52,11 @@ public boolean holdsRefs() {
                     return true;
                 }
 
+                @Override
+                public boolean refersToPages(IndexRoot value) {
+                    return value.root() instanceof Ref.Stored;
+                }
+
                 @Override
                 public IndexRoot mapRefs(IndexRoot value, RefMapper mapper) {
                     return new IndexRoot(mapper.map(value.root(), VALUE_POSTINGS.schema()));
diff --git a/engine/src/main/java/io/hstore/engine/StorageEngine.java b/engine/src/main/java/io/hstore/engine/StorageEngine.java
index 7eab8a8..4e185ee 100644
--- a/engine/src/main/java/io/hstore/engine/StorageEngine.java
+++ b/engine/src/main/java/io/hstore/engine/StorageEngine.java
@@ -9,6 +9,7 @@
 import io.hstore.engine.maintenance.Checkpointer;
 import io.hstore.engine.maintenance.Compactor;
 import io.hstore.engine.maintenance.Recovery;
+import io.hstore.engine.maintenance.ReferenceCounter;
 import io.hstore.engine.page.PageStore;
 import io.hstore.engine.tree.NodeCache;
 import io.hstore.engine.tree.NodeSource;
@@ -38,7 +39,8 @@
 public final class StorageEngine implements AutoCloseable {
 
     private static final System.Logger LOG = System.getLogger("hstore.engine");
-    private static final String FORMAT_VERSION = "4";
+    private static final String FORMAT_VERSION = "5";
+    private static final int CHANGED_COUNTS_LIMIT = 1 << 20;
 
     private static final int MAX_ATTEMPTS = 8;
     private static final Duration MAINTENANCE_INTERVAL = Duration.ofMillis(500);
@@ -55,6 +57,7 @@ public final class StorageEngine implements AutoCloseable {
     private final TransactionManager transactions;
     private final Checkpointer checkpointer;
     private final Compactor compactor;
+    private final ReferenceCounter references;
     private final Dictionary dictionary;
     private final Recovery.Outcome recovery;
     private final ReentrantLock maintenance = new ReentrantLock();
@@ -70,7 +73,8 @@ private StorageEngine(Path directory, EngineOptions options, FileChannel lockCha
         this.lock = lock;
         CatalogStore catalog = CatalogStore.open(directory.resolve("catalog"));
         CatalogImage checkpoint = Recovery.discover(catalog);
-        this.pages = PageStore.open(directory.resolve("data").resolve("segments"), options.pageSize(), options.pagesPerSegment(), checkpoint.segments());
+        this.pages = PageStore.open(directory.resolve("data").resolve("segments"), options.pageSize(), options.pagesPerSegment(),
+                checkpoint.segments(), checkpoint.checkpointLsn());
         this.wal = WriteAheadLog.open(directory.resolve("wal").resolve("segments"), options.walSegmentBytes());
         this.slots = new SlotRegistry(options.slots());
         FeedCodec codec = new FeedCodec(slots);
@@ -78,14 +82,18 @@ private StorageEngine(Path directory, EngineOptions options, FileChannel lockCha
         this.recovery = Recovery.recover(checkpoint, wal, pages, feed, codec);
         this.cache = new NodeCache(options.cachedNodes());
         PagedNodeSource source = new PagedNodeSource(pages, cache);
+        this.references = ReferenceCounter.recover(pages, source, slots, checkpoint.current(), recovery.replayed(),
+                recovery.image().history(), recovery.image().current());
         TransactionManager.Storage storage = new TransactionManager.Storage(pages, source, wal, feed, codec, slots,
-                options.derivations(), options.durability(), options.walMode(), options.faults(), options.historyLimit());
+                options.derivations(), options.durability(), options.walMode(), options.faults(), options.historyLimit(), references);
         this.transactions = new TransactionManager(storage, recovery.image().current(), recovery.image().history());
-        this.checkpointer = new Checkpointer(transactions, pages, wal, feed, catalog, options.historyLimit(), checkpoint.checkpointLsn());
+        this.checkpointer = new Checkpointer(transactions, pages, wal, feed, catalog, references, options.faults(), options.historyLimit(),
+                checkpoint.checkpointLsn());
         this.compactor = new Compactor(transactions, pages, slots, checkpointer, options.faults());
         this.dictionary = new Dictionary(transactions);
         checkpointer.checkpoint();
         compactor.reclaim();
+        references.start();
         LOG.log(System.Logger.Level.INFO, "database at {0} is ready: generation {1}, {2} data segments, page size {3}, {4} durability, {5} wal",
                 directory, transactions.current().id(), pages.segments().size(), options.pageSize(), options.durability(), options.walMode());
         this.maintainer = Thread.ofVirtual().name("hstore-maintenance").start(this::maintain);
@@ -116,7 +124,7 @@ private static void verifyFormat(Path directory, EngineOptions options) throws I
             }
             if (!format.getProperty("format", "1").equals(FORMAT_VERSION)) {
                 throw HStoreException.invalid("database " + directory + " uses storage format " + format.getProperty("format", "1")
-                        + "; this build reads format " + FORMAT_VERSION + " (page directory); export and reload it");
+                        + "; this build reads format " + FORMAT_VERSION + " (reference counts); export and reload it");
             }
             int stored = Integer.parseInt(format.getProperty("page-size"));
             if (stored != options.pageSize()) {
@@ -282,7 +290,12 @@ public Compactor.Report compact() {
     }
 
     public Map<Integer, Long> liveness() {
-        return compactor.liveness();
+        return pages.liveBytes();
+    }
+
+    public ReferenceCounter.Verification verifyReferences() {
+        requireOpen();
+        return transactions.exclusive(() -> references.verify(transactions.current(), transactions.history()));
     }
 
     public EngineStats stats() {
@@ -300,7 +313,7 @@ private void maintain() {
                     continue;
                 }
                 try {
-                    if (checkpointer.walSinceCheckpoint() > options.checkpointWalBytes()) {
+                    if (checkpointer.walSinceCheckpoint() > options.checkpointWalBytes() || references.pending() > CHANGED_COUNTS_LIMIT) {
                         timedCheckpoint();
                     }
                     if (pages.bytesWritten() - writtenAtCompaction > options.checkpointWalBytes()) {
@@ -358,6 +371,7 @@ public void halt() {
     private void shutdown() {
         closed = true;
         transactions.shutdown();
+        references.close();
         maintainer.interrupt();
         try {
             maintainer.join(MAINTENANCE_INTERVAL.multipliedBy(4));
diff --git a/engine/src/main/java/io/hstore/engine/catalog/AtomCodec.java b/engine/src/main/java/io/hstore/engine/catalog/AtomCodec.java
index f2706c5..a94a81d 100644
--- a/engine/src/main/java/io/hstore/engine/catalog/AtomCodec.java
+++ b/engine/src/main/java/io/hstore/engine/catalog/AtomCodec.java
@@ -76,6 +76,11 @@ public boolean holdsRefs() {
         return true;
     }
 
+    @Override
+    public boolean refersToPages(AtomRecord record) {
+        return record instanceof EdgeRecord edge && (edge.members() instanceof Ref.Stored || edge.order() instanceof Ref.Stored);
+    }
+
     @Override
     public AtomRecord mapRefs(AtomRecord record, RefMapper mapper) {
         return switch (record) {
diff --git a/engine/src/main/java/io/hstore/engine/index/PostingsCodec.java b/engine/src/main/java/io/hstore/engine/index/PostingsCodec.java
index 3d2a60a..9ba10ee 100644
--- a/engine/src/main/java/io/hstore/engine/index/PostingsCodec.java
+++ b/engine/src/main/java/io/hstore/engine/index/PostingsCodec.java
@@ -87,6 +87,11 @@ public boolean holdsRefs() {
         return true;
     }
 
+    @Override
+    public boolean refersToPages(Postings<P> value) {
+        return value instanceof Postings.Promoted<P>(Ref root) && root instanceof Ref.Stored;
+    }
+
     @Override
     public Postings<P> mapRefs(Postings<P> value, RefMapper mapper) {
         return value instanceof Postings.Promoted<P>(Ref root) ? new Postings.Promoted<>(mapper.map(root, postings)) : value;
diff --git a/engine/src/main/java/io/hstore/engine/maintenance/Checkpointer.java b/engine/src/main/java/io/hstore/engine/maintenance/Checkpointer.java
index f4b627b..3be0b9d 100644
--- a/engine/src/main/java/io/hstore/engine/maintenance/Checkpointer.java
+++ b/engine/src/main/java/io/hstore/engine/maintenance/Checkpointer.java
@@ -5,6 +5,7 @@
 import io.hstore.engine.catalog.Generation;
 import io.hstore.engine.feed.ChangeFeed;
 import io.hstore.engine.page.PageStore;
+import io.hstore.engine.txn.CrashPoint;
 import io.hstore.engine.txn.TransactionManager;
 import io.hstore.engine.wal.WalRecord;
 import io.hstore.engine.wal.WriteAheadLog;
@@ -21,17 +22,22 @@ public record Result(long generation, long lsn, int walSegments) {
     private final WriteAheadLog wal;
     private final ChangeFeed feed;
     private final CatalogStore catalog;
+    private final ReferenceCounter references;
+    private final CrashPoint.Injector faults;
     private final int historyLimit;
     private volatile long lastLsn;
     private volatile long completed;
 
     public Checkpointer(TransactionManager transactions, PageStore pages, WriteAheadLog wal, ChangeFeed feed,
-                        CatalogStore catalog, int historyLimit, long recoveredLsn) {
+                        CatalogStore catalog, ReferenceCounter references, CrashPoint.Injector faults, int historyLimit,
+                        long recoveredLsn) {
         this.transactions = transactions;
         this.pages = pages;
         this.wal = wal;
         this.feed = feed;
         this.catalog = catalog;
+        this.references = references;
+        this.faults = faults;
         this.historyLimit = historyLimit;
         this.lastLsn = recoveredLsn;
     }
@@ -45,23 +51,43 @@ public long completed() {
     }
 
     public Result checkpoint() {
-        return transactions.exclusive(() -> {
-            long lsn = wal.end();
-            pages.checkpoint();
-            feed.sync();
-            Generation current = transactions.current();
-            List<Generation> history = transactions.history().stream()
-                    .filter(generation -> generation.id() != current.id())
-                    .sorted((a, b) -> Long.compare(b.id(), a.id()))
-                    .limit(historyLimit)
-                    .toList();
-            catalog.publish(new CatalogImage(current, history, pages.segments(), lsn, feed.size()));
-            wal.append(new WalRecord.Checkpoint(0, current.id(), lsn));
-            wal.sync();
-            wal.truncateBefore(lsn);
-            lastLsn = lsn;
-            completed++;
-            return new Result(current.id(), lsn, wal.segmentCount());
+        pages.checkpoint();
+        Result result = transactions.exclusive(() -> {
+            references.pause();
+            try {
+                return publish();
+            } catch (RuntimeException | Error failure) {
+                references.resume();
+                throw failure;
+            }
         });
+        try {
+            faults.reach(CrashPoint.CHECKPOINT_JOURNAL);
+            pages.applyJournal();
+            lastLsn = result.lsn();
+            completed++;
+            return result;
+        } finally {
+            references.resume();
+        }
+    }
+
+    private Result publish() {
+        long lsn = wal.end();
+        references.release(transactions.oldestReadable());
+        pages.checkpoint();
+        pages.journal(lsn);
+        feed.sync();
+        Generation current = transactions.current();
+        List<Generation> history = transactions.history().stream()
+                .filter(generation -> generation.id() != current.id())
+                .sorted((a, b) -> Long.compare(b.id(), a.id()))
+                .limit(historyLimit)
+                .toList();
+        catalog.publish(new CatalogImage(current, history, pages.segments(), lsn, feed.size()));
+        wal.append(new WalRecord.Checkpoint(0, current.id(), lsn));
+        wal.sync();
+        wal.truncateBefore(lsn);
+        return new Result(current.id(), lsn, wal.segmentCount());
     }
 }
diff --git a/engine/src/main/java/io/hstore/engine/maintenance/Recovery.java b/engine/src/main/java/io/hstore/engine/maintenance/Recovery.java
index a20bc55..88ab45b 100644
--- a/engine/src/main/java/io/hstore/engine/maintenance/Recovery.java
+++ b/engine/src/main/java/io/hstore/engine/maintenance/Recovery.java
@@ -23,7 +23,11 @@
 
 public final class Recovery {
 
-    public record Outcome(CatalogImage image, int replayedCommits, int discardedTransactions, int discardedCommits) {
+    public record Outcome(CatalogImage image, List<Generation> replayed, int discardedTransactions, int discardedCommits) {
+
+        public int replayedCommits() {
+            return replayed.size();
+        }
     }
 
     private static final System.Logger LOG = System.getLogger("hstore.recovery");
@@ -49,7 +53,7 @@ public static Outcome recover(CatalogImage checkpoint, WriteAheadLog wal, PageSt
         });
         Generation generation = checkpoint.current();
         List<Generation> history = new ArrayList<>(checkpoint.history());
-        int replayed = 0;
+        List<Generation> replayed = new ArrayList<>();
         int discarded = 0;
         for (WalRecord.Commit commit : commits) {
             List<WalRecord> records = pending.remove(commit.txnId());
@@ -68,15 +72,15 @@ public static Outcome recover(CatalogImage checkpoint, WriteAheadLog wal, PageSt
             }
             generation = redo(generation, commit, body, pages, feed, codec);
             history.add(generation);
-            replayed++;
+            replayed.add(generation);
         }
         pages.sync();
         feed.truncateAfter(generation.id());
         feed.sync();
         CatalogImage recovered = new CatalogImage(generation, history, pages.segments(), wal.end(), feed.size());
         LOG.log(System.Logger.Level.INFO, "recovered generation {0}: replayed {1} commits from lsn {2}, discarded {3} unfinished transactions and {4} non-durable commits",
-                generation.id(), replayed, checkpoint.checkpointLsn(), pending.size(), discarded);
-        return new Outcome(recovered, replayed, pending.size(), discarded);
+                generation.id(), replayed.size(), checkpoint.checkpointLsn(), pending.size(), discarded);
+        return new Outcome(recovered, List.copyOf(replayed), pending.size(), discarded);
     }
 
     private static void place(WalRecord record, PageStore pages) {
diff --git a/engine/src/main/java/io/hstore/engine/maintenance/ReferenceCounter.java b/engine/src/main/java/io/hstore/engine/maintenance/ReferenceCounter.java
new file mode 100644
index 0000000..9996497
--- /dev/null
+++ b/engine/src/main/java/io/hstore/engine/maintenance/ReferenceCounter.java
@@ -0,0 +1,570 @@
+package io.hstore.engine.maintenance;
+
+import io.hstore.engine.HStoreException;
+import io.hstore.engine.catalog.Branch;
+import io.hstore.engine.catalog.Generation;
+import io.hstore.engine.catalog.RootVector;
+import io.hstore.engine.catalog.SlotRegistry;
+import io.hstore.engine.page.LongLongMap;
+import io.hstore.engine.page.PageAddress;
+import io.hstore.engine.page.PageId;
+import io.hstore.engine.page.PageStore;
+import io.hstore.engine.page.ReferenceCounts;
+import io.hstore.engine.tree.PagedNodeSource;
+import io.hstore.engine.tree.Ref;
+import io.hstore.engine.tree.Summary;
+import io.hstore.engine.tree.TreeSchema;
+import io.hstore.engine.tree.TreeWalker;
+import io.hstore.engine.txn.ReferenceTracker;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Comparator;
+import java.util.List;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.Semaphore;
+import java.util.concurrent.locks.ReentrantLock;
+import java.util.function.Supplier;
+import java.util.stream.IntStream;
+
+import static io.hstore.engine.tree.TreeWalker.UNKNOWN_HEIGHT;
+
+public final class ReferenceCounter implements ReferenceTracker, AutoCloseable {
+
+    public record Verification(long livePages, long waitingPages) {
+    }
+
+    private sealed interface Change {
+    }
+
+    private record Committed(Generation before, Generation after, long[] spilled) implements Change {
+    }
+
+    private record Discarded(long[] spilled) implements Change {
+    }
+
+    private static final System.Logger LOG = System.getLogger("hstore.references");
+    private static final int QUEUE_LIMIT = 4096;
+    private static final long[] NO_PAGES = new long[0];
+
+    private final PageStore pages;
+    private final ReferenceCounts counts;
+    private final TreeWalker walker;
+    private final SlotRegistry slots;
+    private final LinkedBlockingQueue<Change> queue = new LinkedBlockingQueue<>(QUEUE_LIMIT);
+    private final Semaphore ready = new Semaphore(0);
+    private final ReentrantLock processing = new ReentrantLock();
+    private final LongLongMap deltas = new LongLongMap(1 << 10);
+    private final Work work = new Work();
+    private final Work lost = new Work();
+    private final Limbo limbo = new Limbo();
+    private final TreeWalker.Reference gaining = this::gain;
+    private final TreeWalker.Reference losing = this::lose;
+    private long generation;
+    private long accounting;
+    private volatile Throwable broken;
+    private volatile boolean closed;
+    private Thread worker;
+
+    private ReferenceCounter(PageStore pages, PagedNodeSource source, SlotRegistry slots, long generation) {
+        this.pages = pages;
+        this.counts = pages.references();
+        this.walker = new TreeWalker(source.uncached());
+        this.slots = slots;
+        this.generation = generation;
+    }
+
+    public static ReferenceCounter recover(PageStore pages, PagedNodeSource source, SlotRegistry slots, Generation checkpointed,
+                                           List<Generation> replayed, Collection<Generation> retained, Generation current) {
+        ReferenceCounter counter = new ReferenceCounter(pages, source, slots, current.id());
+        counter.catchUp(checkpointed, replayed, retained, current);
+        return counter;
+    }
+
+    private void catchUp(Generation checkpointed, List<Generation> replayed, Collection<Generation> retained, Generation current) {
+        if (counts.trusted()) {
+            try {
+                Generation before = checkpointed;
+                for (Generation after : replayed) {
+                    account(before, after, NO_PAGES);
+                    before = after;
+                }
+            } catch (RuntimeException failure) {
+                LOG.log(System.Logger.Level.WARNING, "reference counts could not follow the log, rebuilding them", failure);
+                counts.distrust();
+            }
+        }
+        if (!counts.trusted()) {
+            long started = System.nanoTime();
+            rebuild(retained, current);
+            LOG.log(System.Logger.Level.INFO, "rebuilt reference counts in {0} ms", (System.nanoTime() - started) / 1_000_000);
+        }
+        PageStore.Scan scan = pages.scan(true);
+        limbo.restore(scan.dead());
+        if (scan.orphans() > 0) {
+            LOG.log(System.Logger.Level.INFO, "released {0} pages that no committed generation refers to", scan.orphans());
+        }
+        if (scan.unplaced() > 0) {
+            LOG.log(System.Logger.Level.WARNING, "{0} referenced pages have no address", scan.unplaced());
+        }
+    }
+
+    public void start() {
+        worker = Thread.ofPlatform().name("hstore-references").daemon(true).start(this::run);
+    }
+
+    @Override
+    public void committed(Generation before, Generation after, long[] spilled) {
+        enqueue(new Committed(before, after, spilled));
+    }
+
+    @Override
+    public void discarded(long[] spilled) {
+        enqueue(new Discarded(spilled));
+    }
+
+    private void enqueue(Change change) {
+        if (closed) {
+            return;
+        }
+        boolean interrupted = false;
+        while (true) {
+            try {
+                queue.put(change);
+                break;
+            } catch (InterruptedException e) {
+                interrupted = true;
+            }
+        }
+        ready.release();
+        if (interrupted) {
+            Thread.currentThread().interrupt();
+        }
+    }
+
+    private void run() {
+        while (!closed) {
+            try {
+                ready.acquire();
+            } catch (InterruptedException e) {
+                return;
+            }
+            ready.drainPermits();
+            processing.lock();
+            try {
+                drain();
+            } finally {
+                processing.unlock();
+            }
+        }
+    }
+
+    private void drain() {
+        for (Change change = queue.poll(); change != null; change = queue.poll()) {
+            if (broken != null) {
+                continue;
+            }
+            try {
+                switch (change) {
+                    case Committed(Generation before, Generation after, long[] spilled) -> account(before, after, spilled);
+                    case Discarded(long[] spilled) -> orphaned(spilled, generation);
+                }
+            } catch (RuntimeException failure) {
+                broken = failure;
+                LOG.log(System.Logger.Level.ERROR, "reference counting stopped; the counts will be rebuilt when the database next opens", failure);
+                counts.distrust();
+            }
+        }
+    }
+
+    public <T> T quiesce(Supplier<T> action) {
+        pause();
+        try {
+            return action.get();
+        } finally {
+            resume();
+        }
+    }
+
+    public void pause() {
+        processing.lock();
+        try {
+            drain();
+        } catch (RuntimeException | Error failure) {
+            processing.unlock();
+            throw failure;
+        }
+    }
+
+    public void resume() {
+        processing.unlock();
+    }
+
+    public int pending() {
+        return counts.changed();
+    }
+
+    public long waiting() {
+        return limbo.size();
+    }
+
+    public long release(long oldestReadable) {
+        if (broken != null || !counts.trusted()) {
+            return 0;
+        }
+        long released = 0;
+        while (!limbo.isEmpty() && limbo.headGeneration() <= oldestReadable) {
+            long since = limbo.headGeneration();
+            long index = limbo.headIndex();
+            limbo.pop();
+            if (counts.entry(index) == ReferenceCounts.deadSince(since)) {
+                counts.set(index, 0);
+                pages.release(PageId.pack(index, 0));
+                released++;
+            }
+        }
+        return released;
+    }
+
+    private void account(Generation before, Generation after, long[] spilled) {
+        accounting = after.id();
+        deltas.clear();
+        work.clear();
+        lost.clear();
+        after.branches().forEach((branch, now) -> compare(roots(before, branch), now.roots()));
+        before.branches().forEach((branch, was) -> {
+            if (!after.branches().containsKey(branch)) {
+                compare(was.roots(), RootVector.EMPTY);
+            }
+        });
+        while (!work.isEmpty()) {
+            work.pop();
+            walker.references(work.page, work.schema, work.height, gaining);
+        }
+        while (!lost.isEmpty()) {
+            lost.pop();
+            lose(lost.page, lost.schema, lost.height);
+        }
+        while (!work.isEmpty()) {
+            work.pop();
+            walker.references(work.page, work.schema, work.height, losing);
+        }
+        deltas.forEach((index, delta) -> {
+            long entry = counts.entry(index);
+            long base = ReferenceCounts.count(entry);
+            long total = base + delta;
+            if (total > 0) {
+                if (total != entry) {
+                    counts.set(index, total);
+                }
+            } else if (base > 0) {
+                die(index);
+            }
+        });
+        orphaned(spilled, accounting);
+        generation = accounting;
+    }
+
+    private void compare(RootVector was, RootVector now) {
+        if (was == now) {
+            return;
+        }
+        now.roots().forEach((slot, next) -> {
+            Ref previous = was.get(slot);
+            if (!Ref.same(previous, next)) {
+                changed(slot, previous, next);
+            }
+        });
+        was.roots().forEach((slot, previous) -> {
+            if (!now.roots().containsKey(slot)) {
+                changed(slot, previous, Ref.EMPTY);
+            }
+        });
+    }
+
+    private void changed(int slot, Ref previous, Ref next) {
+        TreeSchema<?> schema = slots.slot(slot).schema();
+        if (next instanceof Ref.Stored(long pageId, int _, Summary _)) {
+            gain(pageId, schema, UNKNOWN_HEIGHT);
+        }
+        if (previous instanceof Ref.Stored(long pageId, int _, Summary _)) {
+            lost.push(pageId, schema, UNKNOWN_HEIGHT);
+        }
+    }
+
+    private static RootVector roots(Generation generation, int branch) {
+        Branch found = generation.branches().get(branch);
+        return found == null ? RootVector.EMPTY : found.roots();
+    }
+
+    private void gain(long pageId, TreeSchema<?> schema, int height) {
+        long index = PageId.indexOf(pageId);
+        if (deltas.add(index, 1) == 1 && ReferenceCounts.count(counts.entry(index)) == 0) {
+            work.push(pageId, schema, height);
+        }
+    }
+
+    private void lose(long pageId, TreeSchema<?> schema, int height) {
+        long index = PageId.indexOf(pageId);
+        long base = ReferenceCounts.count(counts.entry(index));
+        if (base == 0) {
+            throw inconsistent(pageId, "lost a reference it never had");
+        }
+        if (deltas.containsKey(index)) {
+            long remaining = base + deltas.add(index, -1);
+            if (remaining < 0) {
+                throw inconsistent(pageId, "lost more references than it had");
+            }
+            if (remaining == 0) {
+                work.push(pageId, schema, height);
+            }
+        } else if (base == 1) {
+            die(index);
+            work.push(pageId, schema, height);
+        } else {
+            counts.set(index, base - 1);
+        }
+    }
+
+    private void die(long index) {
+        counts.set(index, ReferenceCounts.deadSince(accounting));
+        limbo.add(accounting, index);
+    }
+
+    private void orphaned(long[] spilled, long since) {
+        for (long pageId : spilled) {
+            long index = PageId.indexOf(pageId);
+            if (counts.entry(index) == 0) {
+                counts.set(index, ReferenceCounts.deadSince(since));
+                limbo.add(since, index);
+            }
+        }
+    }
+
+    private static HStoreException inconsistent(long pageId, String problem) {
+        return HStoreException.corrupt(pageId, "reference count of page " + PageId.unpack(pageId) + " " + problem);
+    }
+
+    private void rebuild(Collection<Generation> retained, Generation current) {
+        counts.reset(pages.pageNumbers());
+        work.clear();
+        forEachRoot(current, (pageId, schema, height) -> counted(pageId, schema, height));
+        while (!work.isEmpty()) {
+            work.pop();
+            walker.references(work.page, work.schema, work.height, this::counted);
+        }
+        long since = current.id();
+        TreeWalker.Reference retainedOnly = (pageId, schema, height) -> {
+            long index = PageId.indexOf(pageId);
+            if (counts.entry(index) == 0) {
+                counts.store(index, ReferenceCounts.deadSince(since));
+                work.push(pageId, schema, height);
+            }
+        };
+        for (Generation generation : retained) {
+            forEachRoot(generation, retainedOnly);
+            while (!work.isEmpty()) {
+                work.pop();
+                walker.references(work.page, work.schema, work.height, retainedOnly);
+            }
+        }
+        counts.trust();
+    }
+
+    private void counted(long pageId, TreeSchema<?> schema, int height) {
+        long index = PageId.indexOf(pageId);
+        long count = counts.entry(index);
+        counts.store(index, count + 1);
+        if (count == 0) {
+            work.push(pageId, schema, height);
+        }
+    }
+
+    private void forEachRoot(Generation generation, TreeWalker.Reference into) {
+        generation.branches().values().forEach(branch -> branch.roots().roots().forEach((slot, ref) -> {
+            if (ref instanceof Ref.Stored(long pageId, int _, Summary _)) {
+                into.accept(pageId, slots.slot(slot).schema(), UNKNOWN_HEIGHT);
+            }
+        }));
+    }
+
+    public Verification verify(Generation current, Collection<Generation> retained) {
+        return quiesce(() -> {
+            LongLongMap expected = new LongLongMap(1 << 12);
+            Work pending = new Work();
+            TreeWalker.Reference count = (pageId, schema, height) -> {
+                if (expected.add(PageId.indexOf(pageId), 1) == 1) {
+                    pending.push(pageId, schema, height);
+                }
+            };
+            forEachRoot(current, count);
+            while (!pending.isEmpty()) {
+                pending.pop();
+                walker.references(pending.page, pending.schema, pending.height, count);
+            }
+            List<String> problems = new ArrayList<>();
+            long live = 0;
+            long waiting = 0;
+            long end = pages.pageNumbers();
+            for (long index = 1; index < end; index++) {
+                long entry = counts.entry(index);
+                long references = expected.get(index, 0);
+                if (references > 0) {
+                    live++;
+                    if (entry != references) {
+                        problems.add("page #" + index + " is counted " + describe(entry) + " but has " + references + " references");
+                    }
+                } else if (entry > 0) {
+                    problems.add("page #" + index + " is counted " + entry + " but nothing refers to it");
+                } else if (entry < 0) {
+                    waiting++;
+                }
+            }
+            LongLongMap seen = new LongLongMap(1 << 12);
+            long[] reader = new long[1];
+            TreeWalker.Reference readable = (pageId, schema, height) -> {
+                long index = PageId.indexOf(pageId);
+                if (seen.add(index, 1) != 1) {
+                    return;
+                }
+                long entry = counts.entry(index);
+                if (entry == 0) {
+                    problems.add("page #" + index + " is still readable from generation " + reader[0] + " but is not counted");
+                } else if (ReferenceCounts.dead(entry) && ReferenceCounts.deadAt(entry) <= reader[0]) {
+                    problems.add("page #" + index + " is readable from generation " + reader[0] + " but counted " + describe(entry));
+                } else if (pages.addressOrNone(pageId) == PageAddress.NONE) {
+                    problems.add("page #" + index + " is still readable from generation " + reader[0] + " but has no address");
+                } else {
+                    pending.push(pageId, schema, height);
+                }
+            };
+            for (Generation generation : retained.stream().sorted(Comparator.comparingLong(Generation::id)).toList()) {
+                reader[0] = generation.id();
+                forEachRoot(generation, readable);
+                while (!pending.isEmpty()) {
+                    pending.pop();
+                    walker.references(pending.page, pending.schema, pending.height, readable);
+                }
+            }
+            if (!problems.isEmpty()) {
+                throw HStoreException.invalid(problems.size() + " reference count problems, first: " + String.join("; ", problems.subList(0, Math.min(5, problems.size()))));
+            }
+            return new Verification(live, waiting);
+        });
+    }
+
+    private static String describe(long entry) {
+        return ReferenceCounts.dead(entry) ? "dead since generation " + ReferenceCounts.deadAt(entry) : Long.toString(entry);
+    }
+
+    @Override
+    public void close() {
+        closed = true;
+        ready.release();
+        if (worker != null) {
+            try {
+                worker.join();
+            } catch (InterruptedException e) {
+                Thread.currentThread().interrupt();
+            }
+        }
+    }
+
+    private static final class Work {
+        private long[] pages = new long[256];
+        private TreeSchema<?>[] schemas = new TreeSchema<?>[256];
+        private int[] heights = new int[256];
+        private int size;
+        long page;
+        TreeSchema<?> schema;
+        int height;
+
+        void push(long pageId, TreeSchema<?> pageSchema, int pageHeight) {
+            if (size == pages.length) {
+                pages = Arrays.copyOf(pages, size * 2);
+                schemas = Arrays.copyOf(schemas, size * 2);
+                heights = Arrays.copyOf(heights, size * 2);
+            }
+            pages[size] = pageId;
+            schemas[size] = pageSchema;
+            heights[size] = pageHeight;
+            size++;
+        }
+
+        boolean isEmpty() {
+            return size == 0;
+        }
+
+        void pop() {
+            size--;
+            page = pages[size];
+            schema = schemas[size];
+            height = heights[size];
+            schemas[size] = null;
+        }
+
+        void clear() {
+            Arrays.fill(schemas, 0, size, null);
+            size = 0;
+        }
+    }
+
+    private static final class Limbo {
+        private long[] generations = new long[256];
+        private long[] indexes = new long[256];
+        private int head;
+        private int tail;
+
+        void add(long since, long index) {
+            if (tail == generations.length) {
+                int size = tail - head;
+                boolean grow = size * 2 >= generations.length;
+                long[] movedGenerations = grow ? new long[generations.length * 2] : generations;
+                long[] movedIndexes = grow ? new long[indexes.length * 2] : indexes;
+                System.arraycopy(generations, head, movedGenerations, 0, size);
+                System.arraycopy(indexes, head, movedIndexes, 0, size);
+                generations = movedGenerations;
+                indexes = movedIndexes;
+                head = 0;
+                tail = size;
+            }
+            generations[tail] = since;
+            indexes[tail] = index;
+            tail++;
+        }
+
+        boolean isEmpty() {
+            return head == tail;
+        }
+
+        int size() {
+            return tail - head;
+        }
+
+        long headGeneration() {
+            return generations[head];
+        }
+
+        long headIndex() {
+            return indexes[head];
+        }
+
+        void pop() {
+            head++;
+            if (head == tail) {
+                head = 0;
+                tail = 0;
+            }
+        }
+
+        void restore(long[] pairs) {
+            head = 0;
+            tail = 0;
+            int count = pairs.length / 2;
+            IntStream.range(0, count).boxed()
+                    .sorted(Comparator.comparingLong(i -> pairs[2 * i]))
+                    .forEach(i -> add(pairs[2 * i], pairs[2 * i + 1]));
+        }
+    }
+}
diff --git a/engine/src/main/java/io/hstore/engine/page/LongLongMap.java b/engine/src/main/java/io/hstore/engine/page/LongLongMap.java
new file mode 100644
index 0000000..748645e
--- /dev/null
+++ b/engine/src/main/java/io/hstore/engine/page/LongLongMap.java
@@ -0,0 +1,126 @@
+package io.hstore.engine.page;
+
+public final class LongLongMap {
+
+    @FunctionalInterface
+    public interface Visitor {
+        void accept(long key, long value);
+    }
+
+    private static final int MINIMUM = 16;
+    private static final int RETAINED = 1 << 20;
+
+    private final int initial;
+    private long[] keys;
+    private long[] values;
+    private int[] used;
+    private int size;
+
+    public LongLongMap(int expected) {
+        this.initial = capacityFor(expected);
+        this.keys = new long[initial];
+        this.values = new long[initial];
+        this.used = new int[initial];
+    }
+
+    private static int capacityFor(int entries) {
+        return Math.max(MINIMUM, Integer.highestOneBit(Math.max(1, entries) * 2 - 1) << 1);
+    }
+
+    private int slot(long key) {
+        long mixed = key * 0x9E3779B97F4A7C15L;
+        return (int) (mixed ^ (mixed >>> 32)) & (keys.length - 1);
+    }
+
+    private int find(long key) {
+        if (key == 0) {
+            throw new IllegalArgumentException("zero is not a valid key");
+        }
+        int mask = keys.length - 1;
+        int slot = slot(key);
+        while (keys[slot] != 0 && keys[slot] != key) {
+            slot = (slot + 1) & mask;
+        }
+        return slot;
+    }
+
+    public long get(long key, long missing) {
+        int slot = find(key);
+        return keys[slot] == 0 ? missing : values[slot];
+    }
+
+    public boolean containsKey(long key) {
+        return keys[find(key)] != 0;
+    }
+
+    public void put(long key, long value) {
+        int slot = find(key);
+        if (keys[slot] == 0) {
+            insert(slot, key, value);
+        } else {
+            values[slot] = value;
+        }
+    }
+
+    public long add(long key, long delta) {
+        int slot = find(key);
+        if (keys[slot] == 0) {
+            insert(slot, key, delta);
+            return delta;
+        }
+        values[slot] += delta;
+        return values[slot];
+    }
+
+    private void insert(int slot, long key, long value) {
+        keys[slot] = key;
+        values[slot] = value;
+        used[size++] = slot;
+        if (size * 3L >= keys.length * 2L) {
+            grow();
+        }
+    }
+
+    public int size() {
+        return size;
+    }
+
+    public void forEach(Visitor visitor) {
+        for (int i = 0; i < size; i++) {
+            visitor.accept(keys[used[i]], values[used[i]]);
+        }
+    }
+
+    public void clear() {
+        if (keys.length > RETAINED) {
+            keys = new long[initial];
+            values = new long[initial];
+            used = new int[initial];
+        } else {
+            for (int i = 0; i < size; i++) {
+                keys[used[i]] = 0;
+            }
+        }
+        size = 0;
+    }
+
+    private void grow() {
+        long[] oldKeys = keys;
+        long[] oldValues = values;
+        int[] oldUsed = used;
+        keys = new long[oldKeys.length * 2];
+        values = new long[oldValues.length * 2];
+        used = new int[keys.length];
+        int mask = keys.length - 1;
+        for (int i = 0; i < size; i++) {
+            long key = oldKeys[oldUsed[i]];
+            int slot = slot(key);
+            while (keys[slot] != 0) {
+                slot = (slot + 1) & mask;
+            }
+            keys[slot] = key;
+            values[slot] = oldValues[oldUsed[i]];
+            used[i] = slot;
+        }
+    }
+}
diff --git a/engine/src/main/java/io/hstore/engine/page/PageHeader.java b/engine/src/main/java/io/hstore/engine/page/PageHeader.java
index 0617e81..6a32da3 100644
--- a/engine/src/main/java/io/hstore/engine/page/PageHeader.java
+++ b/engine/src/main/java/io/hstore/engine/page/PageHeader.java
@@ -21,6 +21,7 @@ public record PageHeader(
         long fingerprint) {
 
     public static final int MAGIC = 0x47505348;
+    public static final int REFERS_TO_PAGES = 1;
     public static final int FORMAT = 1;
     public static final int SIZE = 80;
     private static final int PAGE_ID_OFFSET = 8;
@@ -28,6 +29,10 @@ public record PageHeader(
     private static final int CHECKSUM_OFFSET = 72;
     private static final ValueLayout.OfInt INT = ValueLayout.JAVA_INT_UNALIGNED.withOrder(ByteOrder.LITTLE_ENDIAN);
 
+    public boolean refersToPages() {
+        return (flags & REFERS_TO_PAGES) != 0;
+    }
+
     public void writeTo(ByteCursor page) {
         page.position(0)
                 .putInt(MAGIC)
diff --git a/engine/src/main/java/io/hstore/engine/page/PageStore.java b/engine/src/main/java/io/hstore/engine/page/PageStore.java
index ebf4103..41c0e1e 100644
--- a/engine/src/main/java/io/hstore/engine/page/PageStore.java
+++ b/engine/src/main/java/io/hstore/engine/page/PageStore.java
@@ -7,9 +7,11 @@
 import java.lang.foreign.MemorySegment;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.util.Arrays;
 import java.util.Collection;
 import java.util.List;
 import java.util.Map;
+import java.util.TreeMap;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentSkipListMap;
 import java.util.concurrent.atomic.AtomicLong;
@@ -18,16 +20,23 @@
 
 public final class PageStore implements AutoCloseable {
 
+    public record Scan(long[] dead, long orphans, long unplaced) {
+    }
+
     private final Path directory;
     private final int pageSize;
     private final int segmentUnits;
     private final Map<Integer, SegmentFile> files = new ConcurrentHashMap<>();
     private final ConcurrentSkipListMap<Integer, SegmentInfo> segments = new ConcurrentSkipListMap<>();
+    private final Map<Integer, AtomicLong> live = new ConcurrentHashMap<>();
     private final ReentrantLock allocation = new ReentrantLock();
     private final AtomicLong pagesWritten = new AtomicLong();
     private final AtomicLong bytesWritten = new AtomicLong();
     private final AtomicLong pagesRead = new AtomicLong();
     private PageDirectory addresses;
+    private ReferenceCounts references;
+    private long[] released = new long[64];
+    private int releasedCount;
     private int activeSegment;
     private int nextUnit;
 
@@ -38,15 +47,23 @@ private PageStore(Path directory, int pageSize, int pagesPerSegment) {
     }
 
     public static PageStore open(Path directory, int pageSize, int pagesPerSegment, Collection<SegmentInfo> known) {
+        return open(directory, pageSize, pagesPerSegment, known, -1);
+    }
+
+    public static PageStore open(Path directory, int pageSize, int pagesPerSegment, Collection<SegmentInfo> known, long checkpointLsn) {
         PageStore store = new PageStore(directory, pageSize, pagesPerSegment);
-        store.restore(known);
+        store.restore(known, checkpointLsn);
         return store;
     }
 
-    private void restore(Collection<SegmentInfo> known) {
+    private void restore(Collection<SegmentInfo> known, long checkpointLsn) {
         try {
             Files.createDirectories(directory);
             addresses = PageDirectory.open(directory.resolve(PageDirectory.FILE));
+            references = ReferenceCounts.open(directory.resolve(ReferenceCounts.FILE));
+            Path journal = directory.resolve(ReferenceJournal.FILE);
+            ReferenceJournal.read(journal, checkpointLsn).ifPresent(this::replay);
+            ReferenceJournal.clear(journal);
             Map<Integer, SegmentInfo> byId = new ConcurrentHashMap<>();
             known.forEach(info -> byId.put(info.id(), info));
             try (Stream<Path> listing = Files.list(directory)) {
@@ -79,10 +96,35 @@ private void restore(Collection<SegmentInfo> known) {
         });
     }
 
+    private void replay(ReferenceJournal.Contents journal) {
+        long[] counts = journal.counts();
+        for (int i = 0; i < counts.length; i += 2) {
+            references.store(counts[i], counts[i + 1]);
+        }
+        for (long index : journal.cleared()) {
+            addresses.put(index, PageAddress.NONE);
+        }
+        references.force();
+        addresses.force();
+    }
+
     public int pageSize() {
         return pageSize;
     }
 
+    public ReferenceCounts references() {
+        return references;
+    }
+
+    public long pageNumbers() {
+        allocation.lock();
+        try {
+            return addresses.next();
+        } finally {
+            allocation.unlock();
+        }
+    }
+
     public long allocate(int length) {
         if (length > pageSize) {
             throw new IllegalArgumentException("page image exceeds page size: " + length);
@@ -91,7 +133,9 @@ public long allocate(int length) {
         allocation.lock();
         try {
             long index = addresses.allocate();
-            addresses.put(index, claim(units));
+            long address = claim(units);
+            addresses.put(index, address);
+            adjustLive(address, 1);
             return PageId.pack(index, 0);
         } finally {
             allocation.unlock();
@@ -140,6 +184,11 @@ private void rollSegment() {
         nextUnit = 0;
     }
 
+    private void adjustLive(long address, int sign) {
+        live.computeIfAbsent(PageAddress.segmentOf(address), _ -> new AtomicLong())
+                .addAndGet(sign * (long) PageAddress.unitsOf(address) * PageId.UNIT_BYTES);
+    }
+
     public void write(long pageId, MemorySegment image) {
         if (image.byteSize() > pageSize) {
             throw new IllegalArgumentException("page image exceeds page size: " + image.byteSize());
@@ -180,6 +229,10 @@ public long addressOf(long pageId) {
         return locate(pageId);
     }
 
+    public long addressOrNone(long pageId) {
+        return addresses.get(PageId.indexOf(pageId));
+    }
+
     public int segmentOf(long pageId) {
         return PageAddress.segmentOf(locate(pageId));
     }
@@ -188,7 +241,12 @@ public void place(long pageId, long address) {
         allocation.lock();
         try {
             long index = PageId.indexOf(pageId);
+            long previous = addresses.get(index);
+            if (previous != PageAddress.NONE) {
+                adjustLive(previous, -1);
+            }
             addresses.put(index, address);
+            adjustLive(address, 1);
             addresses.reserve(index);
             if (PageAddress.segmentOf(address) == activeSegment) {
                 nextUnit = Math.max(nextUnit, PageAddress.offsetOf(address) + PageAddress.unitsOf(address));
@@ -198,6 +256,53 @@ public void place(long pageId, long address) {
         }
     }
 
+    public void release(long pageId) {
+        long index = PageId.indexOf(pageId);
+        long address = addresses.get(index);
+        if (address == PageAddress.NONE) {
+            return;
+        }
+        adjustLive(address, -1);
+        if (releasedCount == released.length) {
+            released = Arrays.copyOf(released, released.length * 2);
+        }
+        released[releasedCount++] = index;
+    }
+
+    public Scan scan(boolean clearOrphans) {
+        long[] dead = new long[64];
+        int deadCount = 0;
+        long orphans = 0;
+        long unplaced = 0;
+        allocation.lock();
+        try {
+            live.clear();
+            long end = addresses.next();
+            for (long index = 1; index < end; index++) {
+                long address = addresses.get(index);
+                long entry = references.entry(index);
+                if (address == PageAddress.NONE) {
+                    unplaced += entry > 0 ? 1 : 0;
+                } else if (entry == 0 && clearOrphans) {
+                    addresses.put(index, PageAddress.NONE);
+                    orphans++;
+                } else {
+                    adjustLive(address, 1);
+                }
+                if (ReferenceCounts.dead(entry)) {
+                    if (deadCount + 2 > dead.length) {
+                        dead = Arrays.copyOf(dead, dead.length * 2);
+                    }
+                    dead[deadCount++] = ReferenceCounts.deadAt(entry);
+                    dead[deadCount++] = index;
+                }
+            }
+        } finally {
+            allocation.unlock();
+        }
+        return new Scan(Arrays.copyOf(dead, deadCount), orphans, unplaced);
+    }
+
     private long locate(long pageId) {
         long address = addresses.get(PageId.indexOf(pageId));
         if (address == PageAddress.NONE) {
@@ -206,6 +311,16 @@ private long locate(long pageId) {
         return address;
     }
 
+    public Map<Integer, Long> liveBytes() {
+        Map<Integer, Long> bytes = new TreeMap<>();
+        live.forEach((segment, total) -> {
+            if (total.get() > 0) {
+                bytes.put(segment, total.get());
+            }
+        });
+        return bytes;
+    }
+
     public void sync() {
         files.values().forEach(SegmentFile::force);
     }
@@ -215,6 +330,22 @@ public void checkpoint() {
         addresses.force();
     }
 
+    public void journal(long lsn) {
+        ReferenceJournal.write(directory.resolve(ReferenceJournal.FILE), lsn, references, released, releasedCount);
+    }
+
+    public void applyJournal() {
+        references.forEachChange(references::store);
+        for (int i = 0; i < releasedCount; i++) {
+            addresses.put(released[i], PageAddress.NONE);
+        }
+        references.force();
+        addresses.force();
+        ReferenceJournal.clear(directory.resolve(ReferenceJournal.FILE));
+        references.discardChanges();
+        releasedCount = 0;
+    }
+
     public List<SegmentInfo> segments() {
         return List.copyOf(segments.values());
     }
@@ -238,6 +369,7 @@ public void delete(int segment) {
             file.delete();
         }
         segments.remove(segment);
+        live.remove(segment);
     }
 
     public long pagesWritten() {
@@ -257,5 +389,6 @@ public void close() {
         files.values().forEach(SegmentFile::close);
         files.clear();
         addresses.close();
+        references.close();
     }
 }
diff --git a/engine/src/main/java/io/hstore/engine/page/ReferenceCounts.java b/engine/src/main/java/io/hstore/engine/page/ReferenceCounts.java
new file mode 100644
index 0000000..10c9446
--- /dev/null
+++ b/engine/src/main/java/io/hstore/engine/page/ReferenceCounts.java
@@ -0,0 +1,101 @@
+package io.hstore.engine.page;
+
+import java.nio.file.Path;
+
+public final class ReferenceCounts implements AutoCloseable {
+
+    static final String FILE = "pages.ref";
+
+    private static final int MAGIC = 0x53464552;
+    private static final int FORMAT = 1;
+    private static final int STATE_OFFSET = 8;
+    private static final long TRUSTED = 0;
+    private static final long REBUILD = 1;
+    private static final long DEAD = Long.MIN_VALUE;
+    private static final long UNCHANGED = Long.MAX_VALUE;
+
+    private final MappedLongs file;
+    private final LongLongMap changed = new LongLongMap(1 << 12);
+
+    private ReferenceCounts(MappedLongs file) {
+        this.file = file;
+    }
+
+    static ReferenceCounts open(Path path) {
+        return new ReferenceCounts(MappedLongs.open(path, MAGIC, FORMAT, "reference counts"));
+    }
+
+    public static long count(long entry) {
+        return Math.max(entry, 0);
+    }
+
+    public static boolean dead(long entry) {
+        return entry < 0;
+    }
+
+    public static long deadAt(long entry) {
+        return entry & Long.MAX_VALUE;
+    }
+
+    public static long deadSince(long generation) {
+        return DEAD | generation;
+    }
+
+    public long entry(long index) {
+        long overlay = changed.get(index, UNCHANGED);
+        return overlay == UNCHANGED ? file.get(index) : overlay;
+    }
+
+    public void set(long index, long entry) {
+        changed.put(index, entry);
+    }
+
+    public int changed() {
+        return changed.size();
+    }
+
+    public boolean trusted() {
+        return file.header(STATE_OFFSET) == TRUSTED;
+    }
+
+    public void distrust() {
+        file.header(STATE_OFFSET, REBUILD);
+        file.force();
+    }
+
+    public void trust() {
+        file.force();
+        file.header(STATE_OFFSET, TRUSTED);
+        file.force();
+    }
+
+    void forEachChange(LongLongMap.Visitor visitor) {
+        changed.forEach(visitor);
+    }
+
+    public void store(long index, long entry) {
+        file.put(index, entry);
+    }
+
+    public void reset(long end) {
+        changed.clear();
+        for (long index = 1; index < end; index++) {
+            if (file.get(index) != 0) {
+                file.put(index, 0);
+            }
+        }
+    }
+
+    void discardChanges() {
+        changed.clear();
+    }
+
+    void force() {
+        file.force();
+    }
+
+    @Override
+    public void close() {
+        file.close();
+    }
+}
diff --git a/engine/src/main/java/io/hstore/engine/page/ReferenceJournal.java b/engine/src/main/java/io/hstore/engine/page/ReferenceJournal.java
new file mode 100644
index 0000000..9e32c43
--- /dev/null
+++ b/engine/src/main/java/io/hstore/engine/page/ReferenceJournal.java
@@ -0,0 +1,150 @@
+package io.hstore.engine.page;
+
+import io.hstore.engine.HStoreException;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.nio.channels.FileChannel;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
+import java.util.Optional;
+import java.util.zip.CRC32C;
+
+final class ReferenceJournal {
+
+    static final String FILE = "pages.ref.journal";
+
+    record Contents(long[] counts, long[] cleared) {
+    }
+
+    private static final int MAGIC = 0x4C4E524A;
+    private static final int FORMAT = 1;
+    private static final int HEADER = 24;
+    private static final int BUFFER = 1 << 20;
+
+    private ReferenceJournal() {
+    }
+
+    static void write(Path path, long lsn, ReferenceCounts counts, long[] cleared, int clearedCount) {
+        boolean created = !Files.exists(path);
+        try (FileChannel channel = FileChannel.open(path, StandardOpenOption.CREATE, StandardOpenOption.WRITE,
+                StandardOpenOption.TRUNCATE_EXISTING)) {
+            Writer out = new Writer(channel);
+            out.putInt(MAGIC).putInt(FORMAT).putLong(lsn).putInt(counts.changed()).putInt(clearedCount);
+            counts.forEachChange((index, entry) -> out.putLong(index).putLong(entry));
+            for (int i = 0; i < clearedCount; i++) {
+                out.putLong(cleared[i]);
+            }
+            out.finish();
+            channel.force(true);
+        } catch (IOException e) {
+            throw HStoreException.io("cannot write reference journal " + path, e);
+        }
+        if (created) {
+            syncDirectory(path.getParent());
+        }
+    }
+
+    static Optional<Contents> read(Path path, long expectedLsn) {
+        try {
+            if (!Files.exists(path) || Files.size(path) < HEADER + Integer.BYTES) {
+                return Optional.empty();
+            }
+            byte[] bytes = Files.readAllBytes(path);
+            ByteBuffer in = ByteBuffer.wrap(bytes).order(ByteOrder.LITTLE_ENDIAN);
+            CRC32C crc = new CRC32C();
+            crc.update(bytes, 0, bytes.length - Integer.BYTES);
+            if (in.getInt(bytes.length - Integer.BYTES) != (int) crc.getValue()
+                    || in.getInt() != MAGIC || in.getInt() != FORMAT || in.getLong() != expectedLsn) {
+                return Optional.empty();
+            }
+            int pairs = in.getInt();
+            int clearedCount = in.getInt();
+            if ((long) HEADER + pairs * 16L + clearedCount * 8L + Integer.BYTES != bytes.length) {
+                return Optional.empty();
+            }
+            long[] counts = new long[pairs * 2];
+            for (int i = 0; i < counts.length; i++) {
+                counts[i] = in.getLong();
+            }
+            long[] cleared = new long[clearedCount];
+            for (int i = 0; i < clearedCount; i++) {
+                cleared[i] = in.getLong();
+            }
+            return Optional.of(new Contents(counts, cleared));
+        } catch (IOException e) {
+            throw HStoreException.io("cannot read reference journal " + path, e);
+        }
+    }
+
+    static void clear(Path path) {
+        if (!Files.exists(path)) {
+            return;
+        }
+        try (FileChannel channel = FileChannel.open(path, StandardOpenOption.WRITE)) {
+            channel.truncate(0);
+            channel.force(true);
+        } catch (IOException e) {
+            throw HStoreException.io("cannot clear reference journal " + path, e);
+        }
+    }
+
+    private static void syncDirectory(Path directory) {
+        try (FileChannel channel = FileChannel.open(directory, StandardOpenOption.READ)) {
+            channel.force(true);
+        } catch (IOException _) {
+        }
+    }
+
+    private static final class Writer {
+        private final FileChannel channel;
+        private final ByteBuffer buffer = ByteBuffer.allocate(BUFFER).order(ByteOrder.LITTLE_ENDIAN);
+        private final CRC32C crc = new CRC32C();
+
+        Writer(FileChannel channel) {
+            this.channel = channel;
+        }
+
+        Writer putInt(int value) {
+            room(Integer.BYTES);
+            buffer.putInt(value);
+            return this;
+        }
+
+        Writer putLong(long value) {
+            room(Long.BYTES);
+            buffer.putLong(value);
+            return this;
+        }
+
+        void finish() throws IOException {
+            flush();
+            buffer.putInt((int) crc.getValue());
+            buffer.flip();
+            while (buffer.hasRemaining()) {
+                channel.write(buffer);
+            }
+        }
+
+        private void room(int bytes) {
+            if (buffer.remaining() < bytes) {
+                try {
+                    flush();
+                } catch (IOException e) {
+                    throw HStoreException.io("cannot write reference journal", e);
+                }
+            }
+        }
+
+        private void flush() throws IOException {
+            buffer.flip();
+            crc.update(buffer.duplicate());
+            while (buffer.hasRemaining()) {
+                channel.write(buffer);
+            }
+            buffer.clear();
+        }
+    }
+}
diff --git a/engine/src/main/java/io/hstore/engine/tree/NodeCodec.java b/engine/src/main/java/io/hstore/engine/tree/NodeCodec.java
index 39579bc..0fb2ef8 100644
--- a/engine/src/main/java/io/hstore/engine/tree/NodeCodec.java
+++ b/engine/src/main/java/io/hstore/engine/tree/NodeCodec.java
@@ -6,6 +6,7 @@
 import io.hstore.engine.page.PageType;
 
 import java.lang.foreign.MemorySegment;
+import java.util.Arrays;
 import java.util.List;
 
 public final class NodeCodec {
@@ -33,7 +34,8 @@ public static MemorySegment encode(Node node, long pageId, long epoch, int pageS
         }
         int payloadLength = Math.toIntExact(out.position() - PageHeader.SIZE);
         PageType type = node instanceof Leaf ? PageType.LEAF : PageType.INTERNAL;
-        new PageHeader(type, node.schema.id(), 0, pageId, epoch, payloadLength, node.size(), summary.count(),
+        int flags = node instanceof Leaf leaf && refersToPages(leaf) ? PageHeader.REFERS_TO_PAGES : 0;
+        new PageHeader(type, node.schema.id(), flags, pageId, epoch, payloadLength, node.size(), summary.count(),
                 summary.min(), summary.max(), node.height(), summary.fingerprint()).writeTo(out);
         MemorySegment image = out.segment().asSlice(0, PageHeader.SIZE + payloadLength);
         PageHeader.seal(image, payloadLength);
@@ -55,6 +57,19 @@ private static <V> void encodeValues(ByteCursor out, TreeSchema<V> schema, long[
         schema.codec().encode(out, keys, values);
     }
 
+    private static boolean refersToPages(Leaf leaf) {
+        return leaf.schema.codec().holdsRefs() && refersToPages(leaf.schema, leaf.valueList());
+    }
+
+    private static <V> boolean refersToPages(TreeSchema<V> schema, List<?> values) {
+        for (Object value : values) {
+            if (schema.codec().refersToPages(schema.cast(value))) {
+                return true;
+            }
+        }
+        return false;
+    }
+
     private static void encodeBranch(ByteCursor out, Branch branch) {
         long[] separators = branch.separatorArray();
         Ref[] children = branch.childArray();
@@ -68,7 +83,10 @@ private static void encodeBranch(ByteCursor out, Branch branch) {
     }
 
     public static Node decode(MemorySegment page, long pageId, TreeSchema<?> schema) {
-        PageHeader header = PageHeader.verify(page, pageId);
+        return decode(page, PageHeader.verify(page, pageId), pageId, schema);
+    }
+
+    public static Node decode(MemorySegment page, PageHeader header, long pageId, TreeSchema<?> schema) {
         if (header.schema() != schema.id()) {
             throw HStoreException.corrupt(pageId, "page belongs to schema " + header.schema() + ", expected " + schema);
         }
@@ -86,6 +104,9 @@ public static Node decode(MemorySegment page, long pageId, TreeSchema<?> schema)
             case LEAF -> {
                 Object[] values = new Object[count];
                 schema.codec().decode(in, keys, values);
+                if (header.refersToPages() != (schema.codec().holdsRefs() && refersToPages(schema, Arrays.asList(values)))) {
+                    throw HStoreException.corrupt(pageId, "page header and contents disagree on whether it refers to other pages");
+                }
                 yield Leaf.decoded(schema, keys, values, summary);
             }
             case INTERNAL -> {
diff --git a/engine/src/main/java/io/hstore/engine/tree/NodeSource.java b/engine/src/main/java/io/hstore/engine/tree/NodeSource.java
index 312ea1e..339cbd1 100644
--- a/engine/src/main/java/io/hstore/engine/tree/NodeSource.java
+++ b/engine/src/main/java/io/hstore/engine/tree/NodeSource.java
@@ -4,6 +4,10 @@ public interface NodeSource {
 
     Node load(long pageId, TreeSchema<?> schema);
 
+    default Node loadIfReferencing(long pageId, TreeSchema<?> schema) {
+        return load(pageId, schema);
+    }
+
     Layout layout();
 
     static NodeSource ephemeral(int pageSize) {
diff --git a/engine/src/main/java/io/hstore/engine/tree/PagedNodeSource.java b/engine/src/main/java/io/hstore/engine/tree/PagedNodeSource.java
index 7f45720..f841852 100644
--- a/engine/src/main/java/io/hstore/engine/tree/PagedNodeSource.java
+++ b/engine/src/main/java/io/hstore/engine/tree/PagedNodeSource.java
@@ -2,7 +2,11 @@
 
 import io.hstore.engine.HStoreException;
 import io.hstore.engine.page.IoTrace;
+import io.hstore.engine.page.PageHeader;
 import io.hstore.engine.page.PageStore;
+import io.hstore.engine.page.PageType;
+
+import java.lang.foreign.MemorySegment;
 
 public final class PagedNodeSource implements NodeSource {
 
@@ -32,6 +36,32 @@ public Node load(long pageId, TreeSchema<?> schema) {
         return node;
     }
 
+    public NodeSource uncached() {
+        return new NodeSource() {
+            @Override
+            public Node load(long pageId, TreeSchema<?> schema) {
+                Node cached = cache.get(pageId);
+                return cached != null ? cached : NodeCodec.decode(store.read(pageId), pageId, schema);
+            }
+
+            @Override
+            public Node loadIfReferencing(long pageId, TreeSchema<?> schema) {
+                Node cached = cache.get(pageId);
+                if (cached != null) {
+                    return cached;
+                }
+                MemorySegment page = store.read(pageId);
+                PageHeader header = PageHeader.verify(page, pageId);
+                return header.type() == PageType.LEAF && !header.refersToPages() ? null : NodeCodec.decode(page, header, pageId, schema);
+            }
+
+            @Override
+            public Layout layout() {
+                return layout;
+            }
+        };
+    }
+
     public void admit(long pageId, Node frozen) {
         cache.put(pageId, frozen);
     }
diff --git a/engine/src/main/java/io/hstore/engine/tree/TreeWalker.java b/engine/src/main/java/io/hstore/engine/tree/TreeWalker.java
index 8e2ac4a..404cf03 100644
--- a/engine/src/main/java/io/hstore/engine/tree/TreeWalker.java
+++ b/engine/src/main/java/io/hstore/engine/tree/TreeWalker.java
@@ -4,7 +4,12 @@
 
 public final class TreeWalker {
 
-    private static final int UNKNOWN_HEIGHT = -1;
+    public static final int UNKNOWN_HEIGHT = -1;
+
+    @FunctionalInterface
+    public interface Reference {
+        void accept(long pageId, TreeSchema<?> schema, int height);
+    }
 
     private final NodeSource source;
 
@@ -13,36 +18,50 @@ public TreeWalker(NodeSource source) {
     }
 
     public void visit(Ref root, TreeSchema<?> schema, LongPredicate firstVisit) {
-        visit(root, schema, UNKNOWN_HEIGHT, firstVisit);
+        if (root instanceof Ref.Stored(long pageId, int _, Summary _)) {
+            visit(pageId, schema, UNKNOWN_HEIGHT, firstVisit);
+        }
     }
 
-    private void visit(Ref ref, TreeSchema<?> schema, int height, LongPredicate firstVisit) {
-        if (!(ref instanceof Ref.Stored(long pageId, int _, Summary _)) || !firstVisit.test(pageId)) {
-            return;
+    private void visit(long pageId, TreeSchema<?> schema, int height, LongPredicate firstVisit) {
+        if (firstVisit.test(pageId)) {
+            references(pageId, schema, height, (child, childSchema, childHeight) -> visit(child, childSchema, childHeight, firstVisit));
         }
+    }
+
+    public void references(long pageId, TreeSchema<?> schema, int height, Reference into) {
         if (height == 0 && !schema.codec().holdsRefs()) {
             return;
         }
-        switch (source.load(pageId, schema)) {
+        Node node = source.loadIfReferencing(pageId, schema);
+        if (node == null) {
+            return;
+        }
+        switch (node) {
             case Leaf leaf -> {
-                for (int i = 0; i < leaf.size(); i++) {
-                    visitNested(schema, leaf.value(i), firstVisit);
+                if (schema.codec().holdsRefs()) {
+                    ValueCodec.RefMapper mapper = (nested, nestedSchema) -> {
+                        if (nested instanceof Ref.Stored(long child, int _, Summary _)) {
+                            into.accept(child, nestedSchema, UNKNOWN_HEIGHT);
+                        }
+                        return nested;
+                    };
+                    for (int i = 0; i < leaf.size(); i++) {
+                        nested(schema, leaf.value(i), mapper);
+                    }
                 }
             }
             case Branch branch -> {
                 for (int i = 0; i < branch.size(); i++) {
-                    visit(branch.child(i), schema, branch.height() - 1, firstVisit);
+                    if (branch.child(i) instanceof Ref.Stored(long child, int _, Summary _)) {
+                        into.accept(child, schema, branch.height() - 1);
+                    }
                 }
             }
         }
     }
 
-    private <V> void visitNested(TreeSchema<V> schema, Object value, LongPredicate firstVisit) {
-        if (schema.codec().holdsRefs()) {
-            schema.codec().mapRefs(schema.cast(value), (nested, nestedSchema) -> {
-                visit(nested, nestedSchema, UNKNOWN_HEIGHT, firstVisit);
-                return nested;
-            });
-        }
+    private static <V> void nested(TreeSchema<V> schema, Object value, ValueCodec.RefMapper mapper) {
+        schema.codec().mapRefs(schema.cast(value), mapper);
     }
 }
diff --git a/engine/src/main/java/io/hstore/engine/tree/ValueCodec.java b/engine/src/main/java/io/hstore/engine/tree/ValueCodec.java
index 2b31733..fc86ab2 100644
--- a/engine/src/main/java/io/hstore/engine/tree/ValueCodec.java
+++ b/engine/src/main/java/io/hstore/engine/tree/ValueCodec.java
@@ -27,6 +27,15 @@ default V mapRefs(V value, RefMapper mapper) {
         return value;
     }
 
+    default boolean refersToPages(V value) {
+        boolean[] found = new boolean[1];
+        mapRefs(value, (ref, _) -> {
+            found[0] |= ref instanceof Ref.Stored;
+            return ref;
+        });
+        return found[0];
+    }
+
     static <V> ValueCodec<V> rows(ToIntFunction<V> size, BiConsumer<ByteCursor, V> writer, Function<ByteCursor, V> reader) {
         return new ValueCodec<>() {
             @Override
diff --git a/engine/src/main/java/io/hstore/engine/txn/CrashPoint.java b/engine/src/main/java/io/hstore/engine/txn/CrashPoint.java
index 56da57a..e1c53ec 100644
--- a/engine/src/main/java/io/hstore/engine/txn/CrashPoint.java
+++ b/engine/src/main/java/io/hstore/engine/txn/CrashPoint.java
@@ -9,7 +9,8 @@ public enum CrashPoint {
     WAL_SYNC,
     CATALOG_PUBLISH,
     COMPACTION_MOVE,
-    COMPACTION_RETIRE;
+    COMPACTION_RETIRE,
+    CHECKPOINT_JOURNAL;
 
     @FunctionalInterface
     public interface Injector {
diff --git a/engine/src/main/java/io/hstore/engine/txn/ReferenceTracker.java b/engine/src/main/java/io/hstore/engine/txn/ReferenceTracker.java
new file mode 100644
index 0000000..3184780
--- /dev/null
+++ b/engine/src/main/java/io/hstore/engine/txn/ReferenceTracker.java
@@ -0,0 +1,20 @@
+package io.hstore.engine.txn;
+
+import io.hstore.engine.catalog.Generation;
+
+public interface ReferenceTracker {
+
+    ReferenceTracker NONE = new ReferenceTracker() {
+        @Override
+        public void committed(Generation before, Generation after, long[] spilled) {
+        }
+
+        @Override
+        public void discarded(long[] spilled) {
+        }
+    };
+
+    void committed(Generation before, Generation after, long[] spilled);
+
+    void discarded(long[] spilled);
+}
diff --git a/engine/src/main/java/io/hstore/engine/txn/Transaction.java b/engine/src/main/java/io/hstore/engine/txn/Transaction.java
index 1ea0b30..edb8137 100644
--- a/engine/src/main/java/io/hstore/engine/txn/Transaction.java
+++ b/engine/src/main/java/io/hstore/engine/txn/Transaction.java
@@ -20,6 +20,7 @@
 import io.hstore.engine.tree.WriteScope;
 
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
@@ -58,7 +59,8 @@ private record KeyRead(Slot<?> slot, long key) {
     private final Map<Long, Optional<AtomRecord>> atomReads = new HashMap<>();
     private final Map<KeyRead, Optional<?>> keyReads = new HashMap<>();
     private State state = State.ACTIVE;
-    private long spilledPages;
+    private long[] spilled = new long[0];
+    private int spilledPages;
 
     Transaction(TransactionManager manager, long id, Generation snapshot, TxnOptions options) {
         this.manager = manager;
@@ -67,11 +69,7 @@ private record KeyRead(Slot<?> slot, long key) {
         this.snapshot = snapshot;
         this.branch = snapshot.branch(options.branch());
         this.options = options;
-        this.scope = new WriteScope((ref, schema) -> {
-            Ref stored = manager.spill(this.id, ref, schema);
-            spilledPages++;
-            return stored;
-        });
+        this.scope = new WriteScope((ref, schema) -> manager.spill(this.id, ref, schema, this::spilled));
         this.workspace = manager.workspace(scope, branch.roots(), Workspace.EdgeGuard.NONE, false);
     }
 
@@ -281,6 +279,9 @@ public CommitResult commit() {
             return result;
         } catch (RuntimeException | Error failure) {
             state = State.ABORTED;
+            if (failure instanceof HStoreException.Conflict || failure instanceof HStoreException.ResourceLimit) {
+                manager.discard(spilled());
+            }
             throw failure;
         } finally {
             manager.unpin(snapshot.id());
@@ -292,6 +293,7 @@ public void abort() {
             state = State.ABORTED;
             if (spilledPages > 0) {
                 manager.logAbort(id);
+                manager.discard(spilled());
             }
             manager.unpin(snapshot.id());
         }
@@ -314,6 +316,17 @@ long spilledPages() {
         return spilledPages;
     }
 
+    long[] spilled() {
+        return Arrays.copyOf(spilled, spilledPages);
+    }
+
+    private void spilled(long pageId) {
+        if (spilledPages == spilled.length) {
+            spilled = Arrays.copyOf(spilled, Math.max(16, spilled.length * 2));
+        }
+        spilled[spilledPages++] = pageId;
+    }
+
     Workspace replay(RootVector latest) {
         Set<Long> verified = new HashSet<>();
         Workspace replayed = manager.workspace(scope, latest, (edge, target) -> {
diff --git a/engine/src/main/java/io/hstore/engine/txn/TransactionManager.java b/engine/src/main/java/io/hstore/engine/txn/TransactionManager.java
index 8e9dae6..1ccf014 100644
--- a/engine/src/main/java/io/hstore/engine/txn/TransactionManager.java
+++ b/engine/src/main/java/io/hstore/engine/txn/TransactionManager.java
@@ -41,6 +41,7 @@
 import java.util.concurrent.atomic.LongAdder;
 import java.util.concurrent.locks.ReentrantLock;
 import java.util.function.Function;
+import java.util.function.LongConsumer;
 import java.util.function.Supplier;
 import java.util.stream.Stream;
 
@@ -50,7 +51,7 @@ public final class TransactionManager {
 
     public record Storage(PageStore pages, PagedNodeSource source, WriteAheadLog wal, ChangeFeed feed, FeedCodec feedCodec,
                           SlotRegistry slots, List<Derivation> derivations, Durability durability, WalMode walMode,
-                          CrashPoint.Injector faults, int historyLimit) {
+                          CrashPoint.Injector faults, int historyLimit, ReferenceTracker references) {
     }
 
     public record Statistics(long commits, long rebases, long conflicts, long readOnly, long pagesWritten, long walBytes,
@@ -63,6 +64,8 @@ private record Appended(GroupCommitter.Pending pending, long pages, long walByte
     private record PageWrite(long pageId, MemorySegment image) {
     }
 
+    private static final long[] NO_PAGES = new long[0];
+
     private final Storage storage;
     private final List<Derivation> derivations;
     private final AtomicReference<Generation> current;
@@ -157,9 +160,14 @@ public Transaction begin(TxnOptions options) {
     }
 
     private Generation pinCurrent() {
-        Generation generation = current.get();
-        pin(generation.id());
-        return generation;
+        while (true) {
+            Generation generation = current.get();
+            pin(generation.id());
+            if (history.containsKey(generation.id())) {
+                return generation;
+            }
+            unpin(generation.id());
+        }
     }
 
     void pin(long generation) {
@@ -175,6 +183,12 @@ public OptionalLong oldestPinned() {
         return first == null ? OptionalLong.empty() : OptionalLong.of(first.getKey());
     }
 
+    public long oldestReadable() {
+        long oldest = history.firstKey();
+        OptionalLong pinned = oldestPinned();
+        return pinned.isPresent() ? Math.min(oldest, pinned.getAsLong()) : oldest;
+    }
+
     CommitResult commit(Transaction txn) {
         if (txn.readOnly()) {
             readOnly.increment();
@@ -190,6 +204,7 @@ CommitResult commit(Transaction txn) {
             Optional<String> request = txn.options().requestId();
             Optional<RequestRecord> prior = request.flatMap(id -> priorRequest(branch, id));
             if (prior.isPresent()) {
+                discard(txn.spilled());
                 return new CommitResult(prior.get().txnId(), prior.get().generation(), CommitResult.Outcome.DUPLICATE, 0, 0, 0, 0);
             }
             boolean requestsStable = request.isEmpty()
@@ -214,8 +229,8 @@ CommitResult commit(Transaction txn) {
             request.ifPresent(id -> workspace.write(EngineSlots.REQUESTS, Hashing.of(id),
                     _ -> Optional.of(new RequestRecord(id, txn.id(), latest.id() + 1))));
             RootVector merged = fast ? workspace.rebasedOnto(branch.roots()) : workspace.roots();
-            appended = append(latest, branch.withRoots(merged), txn.id(),
-                    workspace.memberChanges(), workspace.slotChanges(), List.of(), txn.options().maxPages() - txn.spilledPages());
+            appended = append(latest, branch.withRoots(merged), txn.id(), workspace.memberChanges(), workspace.slotChanges(),
+                    List.of(), txn.options().maxPages() - txn.spilledPages(), txn.spilled());
         } finally {
             commitLock.unlock();
         }
@@ -250,7 +265,7 @@ public <T> T system(int branchId, Function<Workspace, T> work) {
             RootVector roots = workspace.roots();
             if (!roots.sameAs(branch.roots())) {
                 appended = Optional.of(append(latest, branch.withRoots(roots), txns.getAndIncrement(), workspace.memberChanges(),
-                        workspace.slotChanges(), List.of(), Long.MAX_VALUE));
+                        workspace.slotChanges(), List.of(), Long.MAX_VALUE, NO_PAGES));
             }
         } finally {
             commitLock.unlock();
@@ -273,7 +288,7 @@ public Branch createBranch(String name, int parentId) {
             int id = new TreeSet<>(latest.branches().keySet()).last() + 1;
             long txn = txns.getAndIncrement();
             created = new Branch(id, name, parentId, latest.id(), System.currentTimeMillis(), Branch.State.ACTIVE, parent.roots());
-            appended = append(latest, created, txn, List.of(), List.of(), List.of(meta(txn, created)), Long.MAX_VALUE);
+            appended = append(latest, created, txn, List.of(), List.of(), List.of(meta(txn, created)), Long.MAX_VALUE, NO_PAGES);
         } finally {
             commitLock.unlock();
         }
@@ -292,7 +307,7 @@ public void closeBranch(int id, Branch.State state) {
             Generation latest = head.get();
             long txn = txns.getAndIncrement();
             Branch closed = latest.branch(id).withState(state).withRoots(RootVector.EMPTY);
-            appended = append(latest, closed, txn, List.of(), List.of(), List.of(meta(txn, closed)), Long.MAX_VALUE);
+            appended = append(latest, closed, txn, List.of(), List.of(), List.of(meta(txn, closed)), Long.MAX_VALUE, NO_PAGES);
         } finally {
             commitLock.unlock();
         }
@@ -335,12 +350,21 @@ void logAbort(long txnId) {
         storage.wal().append(List.of(new WalRecord.Abort(txnId)));
     }
 
-    Ref spill(long txnId, Ref ref, TreeSchema<?> schema) {
+    void discard(long[] spilled) {
+        if (spilled.length > 0) {
+            storage.references().discarded(spilled);
+        }
+    }
+
+    Ref spill(long txnId, Ref ref, TreeSchema<?> schema, LongConsumer written) {
         List<WalRecord> records = new ArrayList<>();
         List<PageWrite> writes = new ArrayList<>();
         Materializer materializer = materializer(head.get().id() + 1, txnId, records, writes);
         Ref stored = materializer.materialize(ref, schema);
-        writes.forEach(write -> storage.pages().write(write.pageId(), write.image()));
+        writes.forEach(write -> {
+            storage.pages().write(write.pageId(), write.image());
+            written.accept(write.pageId());
+        });
         storage.wal().append(records);
         pagesWritten.add(materializer.pagesWritten());
         return stored;
@@ -365,7 +389,7 @@ public void accept(long pageId, MemorySegment image, Node frozen) {
     }
 
     private Appended append(Generation latest, Branch changed, long txnId, List<MemberChange> members,
-                            List<SlotChange<?>> slotChanges, List<WalRecord> meta, long pageBudget) {
+                            List<SlotChange<?>> slotChanges, List<WalRecord> meta, long pageBudget, long[] spilled) {
         long pendingPages = changed.roots().roots().entrySet().stream()
                 .mapToLong(entry -> Materializer.pendingNodes(entry.getValue(), storage.slots().slot(entry.getKey()).schema()))
                 .sum();
@@ -405,8 +429,9 @@ private Appended append(Generation latest, Branch changed, long txnId, List<Memb
         pagesWritten.add(materializer.pagesWritten());
         walBytes.add(written);
         GroupCommitter.Pending pending = new GroupCommitter.Pending(next, () -> {
-            current.set(next);
             history.put(next.id(), next);
+            current.set(next);
+            storage.references().committed(latest, next, spilled);
             if (!event.isEmpty()) {
                 storage.feed().append(event);
             }
@@ -434,8 +459,13 @@ private void trimHistory() {
         long latest = current.get().id();
         for (Iterator<Long> generations = history.keySet().iterator(); excess > 0 && generations.hasNext(); ) {
             long generation = generations.next();
-            if (generation != latest && !pins.containsKey(generation)) {
-                generations.remove();
+            if (generation == latest || pins.containsKey(generation)) {
+                continue;
+            }
+            Generation trimmed = history.remove(generation);
+            if (pins.containsKey(generation)) {
+                history.put(generation, trimmed);
+            } else {
                 excess--;
             }
         }
diff --git a/server/src/main/java/io/hstore/server/Check.java b/server/src/main/java/io/hstore/server/Check.java
index 3e39d57..def3974 100644
--- a/server/src/main/java/io/hstore/server/Check.java
+++ b/server/src/main/java/io/hstore/server/Check.java
@@ -4,6 +4,7 @@
 import io.hstore.engine.catalog.Branch;
 import io.hstore.engine.catalog.Generation;
 import io.hstore.engine.catalog.Slot;
+import io.hstore.engine.maintenance.ReferenceCounter;
 import io.hstore.engine.tree.TreeVerifier;
 
 import java.util.Map;
@@ -33,7 +34,10 @@ static String run(StorageEngine engine) {
                         slot.name(), verified.entries(), verified.pages(), verified.nestedTrees(), verified.height()));
             }
         }
-        report.append("verified ").append(pages).append(" pages: checksums, ordering, balance and summaries are consistent");
+        report.append("verified ").append(pages).append(" pages: checksums, ordering, balance and summaries are consistent\n");
+        ReferenceCounter.Verification references = engine.verifyReferences();
+        report.append("reference counts match the trees: ").append(references.livePages()).append(" pages in use, ")
+                .append(references.waitingPages()).append(" waiting for older readers");
         return report.toString();
     }
 }

From b7aa78e478eab696dc9466cce32087754b147515 Mon Sep 17 00:00:00 2001
From: venkat1701 <thejeastdev@gmail.com>
Date: Sun, 11 Oct 2026 06:04:04 +0530
Subject: [PATCH 04/15] feat(engine): compact segments from their own pages and
 live counts

Compaction no longer walks every retained tree to find live pages. It
picks victims from the per-segment live byte counts that the reference
counts keep up to date, and moves pages by reading through the victim
itself: every page header whose number still maps to that spot in the
directory is copied to the active segment, and the directory entries are
repointed every 4 MiB after a sync, as before. The work is proportional
to the size of the victim instead of the size of the database, and the
thread allocates nothing beyond the moved pages.

Pages that a transaction has spilled but not yet committed are moved like
any other page, since they are in the directory too. That makes the
bulk-load fence unnecessary, so a transaction that loaded data before a
compaction now commits instead of retrying. A segment is only chosen once
every page allocated in it has been written.

compact() takes a checkpoint first so that pages which became free since
the last one count as free, and background maintenance now compacts one
segment at a time whenever one falls below compaction_live_ratio rather
than after a fixed amount of writing. It also checkpoints after every
checkpoint_wal_mb of pages written, since that is what releases dead pages
and a bulk load writes a lot of pages for very little WAL.
---
 .../java/io/hstore/engine/StorageEngine.java  |  13 +-
 .../hstore/engine/maintenance/Compactor.java  | 151 +++++-------------
 .../io/hstore/engine/page/PageHeader.java     |   9 ++
 .../java/io/hstore/engine/page/PageStore.java |  86 ++++++++--
 .../io/hstore/engine/page/SegmentFile.java    |   9 ++
 .../io/hstore/engine/tree/TreeWalker.java     |  14 --
 .../io/hstore/engine/txn/Transaction.java     |  10 --
 .../hstore/engine/txn/TransactionManager.java |  13 --
 .../java/io/hstore/engine/CompactionTest.java |  23 +--
 9 files changed, 151 insertions(+), 177 deletions(-)

diff --git a/engine/src/main/java/io/hstore/engine/StorageEngine.java b/engine/src/main/java/io/hstore/engine/StorageEngine.java
index 4e185ee..00f68f3 100644
--- a/engine/src/main/java/io/hstore/engine/StorageEngine.java
+++ b/engine/src/main/java/io/hstore/engine/StorageEngine.java
@@ -63,7 +63,7 @@ public final class StorageEngine implements AutoCloseable {
     private final ReentrantLock maintenance = new ReentrantLock();
     private final Thread maintainer;
     private final List<Runnable> checkpointListeners = new CopyOnWriteArrayList<>();
-    private long writtenAtCompaction;
+    private long writtenAtCheckpoint;
     private volatile boolean closed;
 
     private StorageEngine(Path directory, EngineOptions options, FileChannel lockChannel, FileLock lock) {
@@ -89,7 +89,7 @@ private StorageEngine(Path directory, EngineOptions options, FileChannel lockCha
         this.transactions = new TransactionManager(storage, recovery.image().current(), recovery.image().history());
         this.checkpointer = new Checkpointer(transactions, pages, wal, feed, catalog, references, options.faults(), options.historyLimit(),
                 checkpoint.checkpointLsn());
-        this.compactor = new Compactor(transactions, pages, slots, checkpointer, options.faults());
+        this.compactor = new Compactor(transactions, pages, checkpointer, options.faults());
         this.dictionary = new Dictionary(transactions);
         checkpointer.checkpoint();
         compactor.reclaim();
@@ -262,6 +262,7 @@ public Checkpointer.Result checkpoint() {
     private Checkpointer.Result timedCheckpoint() {
         long started = System.nanoTime();
         Checkpointer.Result result = checkpointer.checkpoint();
+        writtenAtCheckpoint = pages.bytesWritten();
         LOG.log(System.Logger.Level.INFO, "checkpoint complete: generation {0}, lsn {1}, {2} wal segments retained, {3} ms",
                 result.generation(), result.lsn(), result.walSegments(), (System.nanoTime() - started) / 1_000_000);
         for (Runnable listener : checkpointListeners) {
@@ -279,6 +280,7 @@ public Compactor.Report compact() {
         requireOpen();
         maintenance.lock();
         try {
+            timedCheckpoint();
             Compactor.Report report = compactor.compact(options.compactionLiveRatio());
             timedCheckpoint();
             List<Integer> reclaimed = compactor.reclaim();
@@ -313,11 +315,12 @@ private void maintain() {
                     continue;
                 }
                 try {
-                    if (checkpointer.walSinceCheckpoint() > options.checkpointWalBytes() || references.pending() > CHANGED_COUNTS_LIMIT) {
+                    if (checkpointer.walSinceCheckpoint() > options.checkpointWalBytes()
+                            || pages.bytesWritten() - writtenAtCheckpoint > options.checkpointWalBytes()
+                            || references.pending() > CHANGED_COUNTS_LIMIT) {
                         timedCheckpoint();
                     }
-                    if (pages.bytesWritten() - writtenAtCompaction > options.checkpointWalBytes()) {
-                        writtenAtCompaction = pages.bytesWritten();
+                    if (compactor.worthwhile(options.compactionLiveRatio())) {
                         Compactor.Report report = compactor.compact(options.compactionLiveRatio(), 1);
                         if (!report.compacted().isEmpty()) {
                             timedCheckpoint();
diff --git a/engine/src/main/java/io/hstore/engine/maintenance/Compactor.java b/engine/src/main/java/io/hstore/engine/maintenance/Compactor.java
index b39f87b..1c568c5 100644
--- a/engine/src/main/java/io/hstore/engine/maintenance/Compactor.java
+++ b/engine/src/main/java/io/hstore/engine/maintenance/Compactor.java
@@ -1,99 +1,49 @@
 package io.hstore.engine.maintenance;
 
-import io.hstore.engine.catalog.Generation;
-import io.hstore.engine.catalog.SlotRegistry;
-import io.hstore.engine.page.PageAddress;
-import io.hstore.engine.page.PageId;
 import io.hstore.engine.page.PageStore;
 import io.hstore.engine.page.SegmentInfo;
 import io.hstore.engine.page.SegmentState;
-import io.hstore.engine.tree.TreeWalker;
 import io.hstore.engine.txn.CrashPoint;
 import io.hstore.engine.txn.TransactionManager;
 
-import java.util.Arrays;
-import java.util.BitSet;
+import java.util.ArrayList;
 import java.util.Comparator;
 import java.util.List;
 import java.util.Map;
 import java.util.OptionalLong;
-import java.util.Set;
-import java.util.TreeMap;
 import java.util.concurrent.ConcurrentHashMap;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
 
 public final class Compactor {
 
     public record Report(Map<Integer, Long> liveBytes, List<Integer> compacted, long generation) {
     }
 
-    private record Cut(List<Generation> generations, Set<Integer> sealed) {
-    }
-
-    private static final long PASS_BYTES = 4L << 20;
-
     private final TransactionManager transactions;
     private final PageStore pages;
-    private final SlotRegistry slots;
     private final Checkpointer checkpointer;
     private final CrashPoint.Injector faults;
     private final Map<Integer, Long> retiredBeforeCheckpoint = new ConcurrentHashMap<>();
 
-    public Compactor(TransactionManager transactions, PageStore pages, SlotRegistry slots, Checkpointer checkpointer,
-                     CrashPoint.Injector faults) {
+    public Compactor(TransactionManager transactions, PageStore pages, Checkpointer checkpointer, CrashPoint.Injector faults) {
         this.transactions = transactions;
         this.pages = pages;
-        this.slots = slots;
         this.checkpointer = checkpointer;
         this.faults = faults;
     }
 
-    public Map<Integer, Long> liveness() {
-        return bytesBySegment(reachable(retained()));
-    }
-
-    private List<Generation> retained() {
-        return Stream.concat(transactions.history().stream(), Stream.of(transactions.current())).toList();
+    public boolean worthwhile(double liveThreshold) {
+        return !victims(pages.liveBytes(), liveThreshold, 1).isEmpty();
     }
 
-    private Cut cut() {
-        return transactions.exclusive(() -> {
-            int active = pages.activeSegment();
-            Set<Integer> sealed = pages.segments().stream()
-                    .filter(segment -> segment.state() == SegmentState.SEALED && segment.id() != active)
-                    .map(SegmentInfo::id)
-                    .collect(Collectors.toUnmodifiableSet());
-            return new Cut(retained(), sealed);
-        });
-    }
-
-    private BitSet reachable(List<Generation> generations) {
-        BitSet live = new BitSet();
-        TreeWalker walker = new TreeWalker(transactions.source());
-        generations.stream()
-                .flatMap(generation -> generation.branches().values().stream())
-                .forEach(branch -> branch.roots().roots().forEach((slot, ref) ->
-                        walker.visit(ref, slots.slot(slot).schema(), pageId -> mark(live, pageId))));
-        return live;
-    }
-
-    private static boolean mark(BitSet live, long pageId) {
-        int index = Math.toIntExact(PageId.indexOf(pageId));
-        if (live.get(index)) {
-            return false;
-        }
-        live.set(index);
-        return true;
-    }
-
-    private Map<Integer, Long> bytesBySegment(BitSet live) {
-        Map<Integer, Long> bytes = new TreeMap<>();
-        live.stream().forEach(index -> {
-            long address = pages.addressOf(PageId.pack(index, 0));
-            bytes.merge(PageAddress.segmentOf(address), (long) PageAddress.unitsOf(address) * PageId.UNIT_BYTES, Long::sum);
-        });
-        return bytes;
+    private List<Integer> victims(Map<Integer, Long> liveBytes, double liveThreshold, int maxVictims) {
+        int active = pages.activeSegment();
+        return pages.segments().stream()
+                .filter(segment -> segment.state() == SegmentState.SEALED && segment.id() != active && pages.settled(segment.id()))
+                .filter(segment -> liveBytes.getOrDefault(segment.id(), 0L) < segment.bytes() * liveThreshold)
+                .sorted(Comparator.comparingDouble(segment -> (double) liveBytes.getOrDefault(segment.id(), 0L) / Math.max(1, segment.bytes())))
+                .limit(maxVictims)
+                .map(SegmentInfo::id)
+                .toList();
     }
 
     public Report compact(double liveThreshold) {
@@ -101,66 +51,37 @@ public Report compact(double liveThreshold) {
     }
 
     public Report compact(double liveThreshold, int maxVictims) {
-        Cut cut = cut();
-        BitSet live = reachable(cut.generations());
-        Map<Integer, Long> liveBytes = bytesBySegment(live);
-        List<Integer> victims = pages.segments().stream()
-                .filter(segment -> segment.state() == SegmentState.SEALED && cut.sealed().contains(segment.id()))
-                .filter(segment -> liveBytes.getOrDefault(segment.id(), 0L) < segment.bytes() * liveThreshold)
-                .sorted(Comparator.comparingDouble(segment -> (double) liveBytes.getOrDefault(segment.id(), 0L) / Math.max(1, segment.bytes())))
-                .limit(maxVictims)
-                .map(SegmentInfo::id)
-                .toList();
-        if (victims.isEmpty()) {
-            return new Report(liveBytes, victims, transactions.current().id());
+        Map<Integer, Long> liveBytes = pages.liveBytes();
+        List<Integer> victims = victims(liveBytes, liveThreshold, maxVictims);
+        List<Integer> emptied = new ArrayList<>();
+        RuntimeException failure = null;
+        for (int victim : victims) {
+            pages.transition(victim, SegmentState.COMPACTING, 0);
+            try {
+                pages.evacuate(victim);
+                emptied.add(victim);
+            } catch (RuntimeException e) {
+                pages.transition(victim, SegmentState.SEALED, 0);
+                failure = e;
+                break;
+            }
+        }
+        if (!emptied.isEmpty()) {
+            faults.reach(CrashPoint.COMPACTION_MOVE);
         }
-        victims.forEach(id -> pages.transition(id, SegmentState.COMPACTING, 0));
-        move(live, Set.copyOf(victims));
-        faults.reach(CrashPoint.COMPACTION_MOVE);
         long retiredAt = transactions.current().id();
-        transactions.compacted();
         long checkpoints = checkpointer.completed();
-        victims.forEach(id -> {
+        emptied.forEach(id -> {
             pages.transition(id, SegmentState.RETIRED, retiredAt);
             retiredBeforeCheckpoint.put(id, checkpoints);
         });
-        faults.reach(CrashPoint.COMPACTION_RETIRE);
-        return new Report(liveBytes, victims, retiredAt);
-    }
-
-    private void move(BitSet live, Set<Integer> victims) {
-        long[] moved = new long[64];
-        int count = 0;
-        long bytes = 0;
-        for (int index = live.nextSetBit(0); index >= 0; index = live.nextSetBit(index + 1)) {
-            long pageId = PageId.pack(index, 0);
-            long from = pages.addressOf(pageId);
-            if (!victims.contains(PageAddress.segmentOf(from))) {
-                continue;
-            }
-            if (count + 2 > moved.length) {
-                moved = Arrays.copyOf(moved, moved.length * 2);
-            }
-            moved[count++] = pageId;
-            moved[count++] = pages.copy(pageId);
-            bytes += (long) PageAddress.unitsOf(from) * PageId.UNIT_BYTES;
-            if (bytes >= PASS_BYTES) {
-                repoint(moved, count);
-                count = 0;
-                bytes = 0;
-            }
-        }
-        repoint(moved, count);
-    }
-
-    private void repoint(long[] moved, int count) {
-        if (count == 0) {
-            return;
+        if (!emptied.isEmpty()) {
+            faults.reach(CrashPoint.COMPACTION_RETIRE);
         }
-        pages.sync();
-        for (int i = 0; i < count; i += 2) {
-            pages.place(moved[i], moved[i + 1]);
+        if (failure != null) {
+            throw failure;
         }
+        return new Report(liveBytes, emptied, retiredAt);
     }
 
     public List<Integer> reclaim() {
diff --git a/engine/src/main/java/io/hstore/engine/page/PageHeader.java b/engine/src/main/java/io/hstore/engine/page/PageHeader.java
index 6a32da3..5cf32ac 100644
--- a/engine/src/main/java/io/hstore/engine/page/PageHeader.java
+++ b/engine/src/main/java/io/hstore/engine/page/PageHeader.java
@@ -28,6 +28,7 @@ public record PageHeader(
     private static final int PAYLOAD_LENGTH_OFFSET = 24;
     private static final int CHECKSUM_OFFSET = 72;
     private static final ValueLayout.OfInt INT = ValueLayout.JAVA_INT_UNALIGNED.withOrder(ByteOrder.LITTLE_ENDIAN);
+    private static final ValueLayout.OfLong LONG = ValueLayout.JAVA_LONG_UNALIGNED.withOrder(ByteOrder.LITTLE_ENDIAN);
 
     public boolean refersToPages() {
         return (flags & REFERS_TO_PAGES) != 0;
@@ -67,6 +68,14 @@ static int payloadLengthAt(MemorySegment file, long position) {
         return file.get(INT, position + PAYLOAD_LENGTH_OFFSET);
     }
 
+    static boolean startsAt(MemorySegment file, long position) {
+        return position + SIZE <= file.byteSize() && file.get(INT, position) == MAGIC;
+    }
+
+    static long pageIdAt(MemorySegment file, long position) {
+        return file.get(LONG, position + PAGE_ID_OFFSET);
+    }
+
     public static void seal(MemorySegment page, int payloadLength) {
         int checksum = Checksums.crc32c(page.asSlice(0, CHECKSUM_OFFSET), page.asSlice(SIZE, payloadLength));
         ByteCursor.over(page).position(CHECKSUM_OFFSET).putInt(checksum);
diff --git a/engine/src/main/java/io/hstore/engine/page/PageStore.java b/engine/src/main/java/io/hstore/engine/page/PageStore.java
index 41c0e1e..671bdc1 100644
--- a/engine/src/main/java/io/hstore/engine/page/PageStore.java
+++ b/engine/src/main/java/io/hstore/engine/page/PageStore.java
@@ -14,6 +14,7 @@
 import java.util.TreeMap;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentSkipListMap;
+import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.locks.ReentrantLock;
 import java.util.stream.Stream;
@@ -23,12 +24,15 @@ public final class PageStore implements AutoCloseable {
     public record Scan(long[] dead, long orphans, long unplaced) {
     }
 
+    private static final long PASS_BYTES = 4L << 20;
+
     private final Path directory;
     private final int pageSize;
     private final int segmentUnits;
     private final Map<Integer, SegmentFile> files = new ConcurrentHashMap<>();
     private final ConcurrentSkipListMap<Integer, SegmentInfo> segments = new ConcurrentSkipListMap<>();
     private final Map<Integer, AtomicLong> live = new ConcurrentHashMap<>();
+    private final Map<Integer, AtomicInteger> unwritten = new ConcurrentHashMap<>();
     private final ReentrantLock allocation = new ReentrantLock();
     private final AtomicLong pagesWritten = new AtomicLong();
     private final AtomicLong bytesWritten = new AtomicLong();
@@ -136,27 +140,84 @@ public long allocate(int length) {
             long address = claim(units);
             addresses.put(index, address);
             adjustLive(address, 1);
+            unwritten.computeIfAbsent(PageAddress.segmentOf(address), _ -> new AtomicInteger()).incrementAndGet();
             return PageId.pack(index, 0);
         } finally {
             allocation.unlock();
         }
     }
 
-    public long copy(long pageId) {
-        long from = locate(pageId);
-        MemorySegment image = read(pageId);
-        PageHeader.verify(image, pageId);
-        long to;
+    public boolean settled(int segment) {
+        AtomicInteger pending = unwritten.get(segment);
+        return pending == null || pending.get() == 0;
+    }
+
+    public long evacuate(int segment) {
+        SegmentFile file = files.get(segment);
+        MemorySegment view = file == null ? MemorySegment.NULL : file.view();
+        long units = view.byteSize() / PageId.UNIT_BYTES;
+        long[] moved = new long[96];
+        int count = 0;
+        long pass = 0;
+        long total = 0;
+        for (long unit = 0; unit < units; ) {
+            long position = unit * PageId.UNIT_BYTES;
+            long address = PageHeader.startsAt(view, position) ? addresses.get(PageId.indexOf(PageHeader.pageIdAt(view, position))) : PageAddress.NONE;
+            if (address == PageAddress.NONE || PageAddress.segmentOf(address) != segment || PageAddress.offsetOf(address) != unit) {
+                unit++;
+                continue;
+            }
+            long pageId = PageHeader.pageIdAt(view, position);
+            int pageUnits = PageAddress.unitsOf(address);
+            MemorySegment image = view.asSlice(position, Math.min((long) pageUnits * PageId.UNIT_BYTES, view.byteSize() - position));
+            MemorySegment bytes = image.asSlice(0, PageHeader.SIZE + PageHeader.verify(image, pageId).payloadLength());
+            long to;
+            allocation.lock();
+            try {
+                to = claim(pageUnits);
+            } finally {
+                allocation.unlock();
+            }
+            files.get(PageAddress.segmentOf(to)).write((long) PageAddress.offsetOf(to) * PageId.UNIT_BYTES, bytes);
+            pagesWritten.incrementAndGet();
+            bytesWritten.addAndGet(bytes.byteSize());
+            if (count + 3 > moved.length) {
+                moved = Arrays.copyOf(moved, moved.length * 2);
+            }
+            moved[count++] = pageId;
+            moved[count++] = address;
+            moved[count++] = to;
+            pass += bytes.byteSize();
+            total += bytes.byteSize();
+            if (pass >= PASS_BYTES) {
+                repoint(moved, count);
+                count = 0;
+                pass = 0;
+            }
+            unit += pageUnits;
+        }
+        repoint(moved, count);
+        return total;
+    }
+
+    private void repoint(long[] moved, int count) {
+        if (count == 0) {
+            return;
+        }
+        sync();
         allocation.lock();
         try {
-            to = claim(PageAddress.unitsOf(from));
+            for (int i = 0; i < count; i += 3) {
+                long index = PageId.indexOf(moved[i]);
+                if (addresses.get(index) == moved[i + 1]) {
+                    addresses.put(index, moved[i + 2]);
+                    adjustLive(moved[i + 1], -1);
+                    adjustLive(moved[i + 2], 1);
+                }
+            }
         } finally {
             allocation.unlock();
         }
-        files.get(PageAddress.segmentOf(to)).write((long) PageAddress.offsetOf(to) * PageId.UNIT_BYTES, image);
-        pagesWritten.incrementAndGet();
-        bytesWritten.addAndGet(image.byteSize());
-        return to;
     }
 
     private long claim(int units) {
@@ -204,6 +265,10 @@ public void write(long pageId, MemorySegment image) {
             return SegmentFile.open(id, SegmentFile.pathFor(directory, id));
         });
         file.write((long) offset * PageId.UNIT_BYTES, image);
+        AtomicInteger pending = unwritten.get(segment);
+        if (pending != null) {
+            pending.updateAndGet(value -> Math.max(0, value - 1));
+        }
         long end = offset + (long) PageId.unitsFor(image.byteSize());
         segments.computeIfPresent(segment, (_, info) -> info.units() >= end ? info : info.withAllocation(info.pages() + 1, end));
         pagesWritten.incrementAndGet();
@@ -370,6 +435,7 @@ public void delete(int segment) {
         }
         segments.remove(segment);
         live.remove(segment);
+        unwritten.remove(segment);
     }
 
     public long pagesWritten() {
diff --git a/engine/src/main/java/io/hstore/engine/page/SegmentFile.java b/engine/src/main/java/io/hstore/engine/page/SegmentFile.java
index 8d41843..c7056dc 100644
--- a/engine/src/main/java/io/hstore/engine/page/SegmentFile.java
+++ b/engine/src/main/java/io/hstore/engine/page/SegmentFile.java
@@ -86,6 +86,15 @@ MemorySegment read(long position, int maximum) {
         return onHeap(view, position, length);
     }
 
+    MemorySegment view() {
+        try {
+            MemorySegment whole = covering(channel.size());
+            return whole == null ? MemorySegment.NULL : whole;
+        } catch (IOException e) {
+            throw HStoreException.io("cannot size segment " + path, e);
+        }
+    }
+
     long mappedBytes() {
         return mapped.byteSize();
     }
diff --git a/engine/src/main/java/io/hstore/engine/tree/TreeWalker.java b/engine/src/main/java/io/hstore/engine/tree/TreeWalker.java
index 404cf03..13bca0c 100644
--- a/engine/src/main/java/io/hstore/engine/tree/TreeWalker.java
+++ b/engine/src/main/java/io/hstore/engine/tree/TreeWalker.java
@@ -1,7 +1,5 @@
 package io.hstore.engine.tree;
 
-import java.util.function.LongPredicate;
-
 public final class TreeWalker {
 
     public static final int UNKNOWN_HEIGHT = -1;
@@ -17,18 +15,6 @@ public TreeWalker(NodeSource source) {
         this.source = source;
     }
 
-    public void visit(Ref root, TreeSchema<?> schema, LongPredicate firstVisit) {
-        if (root instanceof Ref.Stored(long pageId, int _, Summary _)) {
-            visit(pageId, schema, UNKNOWN_HEIGHT, firstVisit);
-        }
-    }
-
-    private void visit(long pageId, TreeSchema<?> schema, int height, LongPredicate firstVisit) {
-        if (firstVisit.test(pageId)) {
-            references(pageId, schema, height, (child, childSchema, childHeight) -> visit(child, childSchema, childHeight, firstVisit));
-        }
-    }
-
     public void references(long pageId, TreeSchema<?> schema, int height, Reference into) {
         if (height == 0 && !schema.codec().holdsRefs()) {
             return;
diff --git a/engine/src/main/java/io/hstore/engine/txn/Transaction.java b/engine/src/main/java/io/hstore/engine/txn/Transaction.java
index edb8137..3649c57 100644
--- a/engine/src/main/java/io/hstore/engine/txn/Transaction.java
+++ b/engine/src/main/java/io/hstore/engine/txn/Transaction.java
@@ -48,7 +48,6 @@ private record KeyRead(Slot<?> slot, long key) {
 
     private final TransactionManager manager;
     private final long id;
-    private final long compactionsAtStart;
     private final Generation snapshot;
     private final Branch branch;
     private final TxnOptions options;
@@ -65,7 +64,6 @@ private record KeyRead(Slot<?> slot, long key) {
     Transaction(TransactionManager manager, long id, Generation snapshot, TxnOptions options) {
         this.manager = manager;
         this.id = id;
-        this.compactionsAtStart = manager.compactions();
         this.snapshot = snapshot;
         this.branch = snapshot.branch(options.branch());
         this.options = options;
@@ -259,14 +257,6 @@ private void requireActive() {
         }
     }
 
-    long compactionsAtStart() {
-        return compactionsAtStart;
-    }
-
-    boolean carriesStoredRoots() {
-        return ops.stream().anyMatch(op -> op instanceof Op.EdgeOp(long _, EdgeAction.Load _));
-    }
-
     public boolean readOnly() {
         return ops.isEmpty();
     }
diff --git a/engine/src/main/java/io/hstore/engine/txn/TransactionManager.java b/engine/src/main/java/io/hstore/engine/txn/TransactionManager.java
index 1ccf014..51bcdf6 100644
--- a/engine/src/main/java/io/hstore/engine/txn/TransactionManager.java
+++ b/engine/src/main/java/io/hstore/engine/txn/TransactionManager.java
@@ -82,7 +82,6 @@ private record PageWrite(long pageId, MemorySegment image) {
     private final LongAdder readOnly = new LongAdder();
     private final LongAdder pagesWritten = new LongAdder();
     private final LongAdder walBytes = new LongAdder();
-    private final AtomicLong compactions = new AtomicLong();
 
     public TransactionManager(Storage storage, Generation recovered, Collection<Generation> retained) {
         this.storage = storage;
@@ -211,10 +210,6 @@ CommitResult commit(Transaction txn) {
                     || Ref.same(txn.baseRoots().get(EngineSlots.REQUESTS), branch.roots().get(EngineSlots.REQUESTS));
             boolean serializable = txn.options().isolation() == Isolation.SERIALIZABLE;
             fast = branch.roots().sameAs(txn.baseRoots()) || (requestsStable && txn.workspace().untouchedSince(branch.roots(), serializable));
-            if (txn.compactionsAtStart() < compactions.get() && txn.carriesStoredRoots()) {
-                conflicts.increment();
-                throw HStoreException.conflict("bulk-loaded roots of transaction " + txn.id() + " predate a compaction");
-            }
             try {
                 if (fast) {
                     workspace = txn.workspace();
@@ -319,14 +314,6 @@ private static WalRecord meta(long txn, Branch branch) {
         return new WalRecord.BranchMeta(txn, branch.id(), branch.name(), branch.parent(), branch.baseGeneration(), branch.createdAt(), branch.state());
     }
 
-    public void compacted() {
-        compactions.incrementAndGet();
-    }
-
-    long compactions() {
-        return compactions.get();
-    }
-
     public <T> T exclusive(Supplier<T> action) {
         commitLock.lock();
         try {
diff --git a/engine/src/test/java/io/hstore/engine/CompactionTest.java b/engine/src/test/java/io/hstore/engine/CompactionTest.java
index 4e16727..98a5659 100644
--- a/engine/src/test/java/io/hstore/engine/CompactionTest.java
+++ b/engine/src/test/java/io/hstore/engine/CompactionTest.java
@@ -152,22 +152,25 @@ void commitsCarryOnWhilePagesMove() throws Exception {
     }
 
     @Test
-    void aBulkLoadStartedBeforeCompactionHasToRetry() {
-        try (StorageEngine engine = StorageEngine.open(directory, options())) {
+    void aBulkLoadStartedBeforeCompactionStillCommits() {
+        EngineOptions options = options().withCompactionLiveRatio(0.99);
+        List<Incidence> members = Stream.iterate(2_000_000L, i -> i + 7_919).limit(20_000).map(Incidence::of).toList();
+        long target;
+        try (StorageEngine engine = StorageEngine.open(directory, options)) {
             long edge = churn(engine);
-            long target = engine.write(txn -> txn.createEdge(EngineTest.EDGE, EdgeKind.SET));
-            List<Incidence> members = Stream.iterate(2_000_000L, i -> i + 1).limit(5_000).map(Incidence::of).toList();
+            target = engine.write(txn -> txn.createEdge(EngineTest.EDGE, EdgeKind.SET));
             Transaction early = engine.begin();
             early.load(target, members);
+            churn(engine);
             assertFalse(engine.compact().compacted().isEmpty());
-            assertThrows(HStoreException.Conflict.class, early::commit);
-            engine.write(txn -> {
-                txn.load(target, members);
-                return null;
-            });
-            assertEquals(5_000L, (long) engine.read(snapshot -> snapshot.requireEdge(target).size()));
+            early.commit();
+            assertEquals(members, engine.read(snapshot -> snapshot.requireEdge(target).stream().toList()));
             assertEquals(expected(), members(engine, edge));
         }
+        try (StorageEngine engine = StorageEngine.open(directory, options)) {
+            assertEquals(members, engine.read(snapshot -> snapshot.requireEdge(target).stream().toList()));
+            engine.verifyReferences();
+        }
     }
 
     static Stream<Arguments> crashes() {

From e85e6b374ef6074ae483c73dada04ef9bfe042ba Mon Sep 17 00:00:00 2001
From: venkat1701 <thejeastdev@gmail.com>
Date: Sun, 11 Oct 2026 06:06:25 +0530
Subject: [PATCH 05/15] test(engine): cover reference counting and segment
 evacuation

ReferenceCountTest runs random work over several seeds: inserts and
removals, edge deletes, bulk loads, aborted loads, and branches that are
written to and sometimes dropped. It checks the counts against a full
walk of the trees after checkpoints and after reopening. Other cases:
pages a pinned reader can still reach wait until it closes, aborted and
replaced loads hand their pages back, pages spilled by a transaction cut
short by a crash are released on open, the counts survive a crash with or
without a half-finished checkpoint, and counts marked for rebuild are
rebuilt.

PageStoreTest covers a segment with an unwritten page not being settled,
and evacuation moving only the pages the directory still points at.
TreeTest checks that a leaf's header flag says whether it points at other
pages.
Compaction and crash tests now also check the counts after recovery.
---
 .../java/io/hstore/engine/CompactionTest.java |   5 +
 .../io/hstore/engine/CrashRecoveryTest.java   |   1 +
 .../io/hstore/engine/ReferenceCountTest.java  | 318 ++++++++++++++++++
 .../hstore/engine/page/LongLongMapTest.java   |  42 +++
 .../io/hstore/engine/page/PageStoreTest.java  |  57 ++++
 .../java/io/hstore/engine/tree/TreeTest.java  |  20 ++
 6 files changed, 443 insertions(+)
 create mode 100644 engine/src/test/java/io/hstore/engine/ReferenceCountTest.java
 create mode 100644 engine/src/test/java/io/hstore/engine/page/LongLongMapTest.java

diff --git a/engine/src/test/java/io/hstore/engine/CompactionTest.java b/engine/src/test/java/io/hstore/engine/CompactionTest.java
index 98a5659..6ec85ad 100644
--- a/engine/src/test/java/io/hstore/engine/CompactionTest.java
+++ b/engine/src/test/java/io/hstore/engine/CompactionTest.java
@@ -61,6 +61,7 @@ void compactionMovesPagesWithoutWritingAGeneration() {
         try (StorageEngine engine = StorageEngine.open(directory, options())) {
             assertTrue(engine.liveness().keySet().stream().allMatch(segmentIds(engine)::contains));
             assertEquals(expected(), members(engine, onlyEdge(engine)));
+            engine.verifyReferences();
         }
     }
 
@@ -110,6 +111,7 @@ void aReaderFromBeforeTheMoveKeepsTheOldSegmentAround() {
         }
         try (StorageEngine engine = StorageEngine.open(directory, options())) {
             assertEquals(expected(), members(engine, onlyEdge(engine)));
+            engine.verifyReferences();
         }
     }
 
@@ -148,6 +150,7 @@ void commitsCarryOnWhilePagesMove() throws Exception {
         }
         try (StorageEngine engine = StorageEngine.open(directory, options())) {
             assertEquals(all, members(engine, onlyEdge(engine)));
+            engine.verifyReferences();
         }
     }
 
@@ -202,11 +205,13 @@ void aCrashDuringCompactionLosesNothing(CrashPoint point, boolean directoryWrite
         }
         try (StorageEngine recovered = StorageEngine.open(directory, options())) {
             assertEquals(expected(), members(recovered, edge));
+            recovered.verifyReferences();
             recovered.compact();
             assertEquals(expected(), members(recovered, edge));
         }
         try (StorageEngine reopened = StorageEngine.open(directory, options())) {
             assertEquals(expected(), members(reopened, edge));
+            reopened.verifyReferences();
         }
     }
 
diff --git a/engine/src/test/java/io/hstore/engine/CrashRecoveryTest.java b/engine/src/test/java/io/hstore/engine/CrashRecoveryTest.java
index 780296c..8c568c3 100644
--- a/engine/src/test/java/io/hstore/engine/CrashRecoveryTest.java
+++ b/engine/src/test/java/io/hstore/engine/CrashRecoveryTest.java
@@ -68,6 +68,7 @@ void recoveryYieldsPriorOrNewStateNeverAMixture(WalMode mode, CrashPoint point)
         engine.halt();
 
         try (StorageEngine recovered = StorageEngine.open(directory, EngineTest.small().withWalMode(mode))) {
+            recovered.verifyReferences();
             recovered.read(snapshot -> {
                 long size = snapshot.requireEdge(edge).size();
                 boolean committed = point.ordinal() >= CrashPoint.COMMIT_APPEND.ordinal();
diff --git a/engine/src/test/java/io/hstore/engine/ReferenceCountTest.java b/engine/src/test/java/io/hstore/engine/ReferenceCountTest.java
new file mode 100644
index 0000000..c98a11c
--- /dev/null
+++ b/engine/src/test/java/io/hstore/engine/ReferenceCountTest.java
@@ -0,0 +1,318 @@
+package io.hstore.engine;
+
+import io.hstore.engine.catalog.Branch;
+import io.hstore.engine.maintenance.ReferenceCounter;
+import io.hstore.engine.topology.EdgeKind;
+import io.hstore.engine.topology.Incidence;
+import io.hstore.engine.txn.CommitResult;
+import io.hstore.engine.txn.CrashPoint;
+import io.hstore.engine.txn.Snapshot;
+import io.hstore.engine.txn.Transaction;
+import io.hstore.engine.txn.TxnOptions;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.nio.channels.FileChannel;
+import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.TreeSet;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.stream.LongStream;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+class ReferenceCountTest {
+
+    @TempDir
+    Path directory;
+
+    private static EngineOptions options() {
+        return EngineTest.small().withPagesPerSegment(64).withHistoryLimit(3).withCheckpointWalBytes(Long.MAX_VALUE);
+    }
+
+    @ParameterizedTest
+    @ValueSource(longs = {1, 2, 3})
+    void countsMatchAFullWalkThroughRandomWork(long seed) {
+        Random random = new Random(seed);
+        Map<Long, TreeSet<Long>> model = new HashMap<>();
+        try (StorageEngine engine = StorageEngine.open(directory, options())) {
+            for (int round = 0; round < 120; round++) {
+                step(engine, random, model);
+                if (round % 15 == 14) {
+                    engine.checkpoint();
+                    engine.verifyReferences();
+                }
+            }
+            engine.verifyReferences();
+            assertModel(engine, model);
+        }
+        try (StorageEngine engine = StorageEngine.open(directory, options())) {
+            engine.verifyReferences();
+            assertModel(engine, model);
+        }
+    }
+
+    private static void step(StorageEngine engine, Random random, Map<Long, TreeSet<Long>> model) {
+        int choice = random.nextInt(10);
+        List<Long> edges = new ArrayList<>(model.keySet());
+        if (choice == 0 || edges.isEmpty()) {
+            long edge = engine.write(txn -> txn.createEdge(EngineTest.EDGE, EdgeKind.SET));
+            model.put(edge, new TreeSet<>());
+        } else if (choice == 1 && edges.size() > 2) {
+            long edge = edges.get(random.nextInt(edges.size()));
+            engine.write(txn -> {
+                txn.delete(edge);
+                return null;
+            });
+            model.remove(edge);
+        } else if (choice == 2) {
+            long edge = edges.get(random.nextInt(edges.size()));
+            long start = 1_000_000L + random.nextInt(1_000_000);
+            List<Incidence> members = LongStream.range(start, start + 2_000).mapToObj(Incidence::of).toList();
+            long target = engine.write(txn -> txn.createEdge(EngineTest.EDGE, EdgeKind.SET));
+            engine.write(txn -> {
+                txn.load(target, members);
+                return null;
+            });
+            model.put(target, new TreeSet<>(members.stream().map(Incidence::member).toList()));
+        } else if (choice == 3) {
+            long target = engine.write(txn -> txn.createEdge(EngineTest.EDGE, EdgeKind.SET));
+            model.put(target, new TreeSet<>());
+            try (Transaction abandoned = engine.begin()) {
+                abandoned.load(target, LongStream.range(5_000_000, 5_002_000).mapToObj(Incidence::of).toList());
+            }
+        } else if (choice == 4) {
+            Branch branch = engine.createBranch("b" + engine.stats().generation(), Branch.MAIN);
+            long edge = edges.get(random.nextInt(edges.size()));
+            for (int i = 0; i < 3; i++) {
+                long member = 9_000_000L + random.nextInt(1_000_000);
+                engine.write(TxnOptions.defaults().onBranch(branch.id()), txn -> {
+                    txn.insert(edge, member);
+                    return null;
+                });
+            }
+            if (random.nextBoolean()) {
+                engine.dropBranch(branch.id());
+            }
+        } else {
+            long edge = edges.get(random.nextInt(edges.size()));
+            TreeSet<Long> members = model.get(edge);
+            List<Long> added = new ArrayList<>();
+            List<Long> removed = new ArrayList<>();
+            for (int i = 0; i < 1 + random.nextInt(60); i++) {
+                long member = random.nextInt(20_000);
+                if (members.contains(member) || added.contains(member)) {
+                    if (!added.contains(member) && !removed.contains(member)) {
+                        removed.add(member);
+                    }
+                } else if (!removed.contains(member)) {
+                    added.add(member);
+                }
+            }
+            engine.write(txn -> {
+                added.forEach(member -> txn.insert(edge, member));
+                removed.forEach(member -> txn.remove(edge, member));
+                return null;
+            });
+            members.addAll(added);
+            removed.forEach(members::remove);
+        }
+    }
+
+    private static void assertModel(StorageEngine engine, Map<Long, TreeSet<Long>> model) {
+        engine.read(snapshot -> {
+            model.forEach((edge, members) ->
+                    assertEquals(List.copyOf(members), snapshot.requireEdge(edge).stream().map(Incidence::member).toList(), "edge " + edge));
+            return null;
+        });
+    }
+
+    @Test
+    void pagesAreReleasedOnlyOnceNoReaderCanReachThem() {
+        try (StorageEngine engine = StorageEngine.open(directory, options())) {
+            long edge = engine.write(txn -> txn.createEdge(EngineTest.EDGE, EdgeKind.SET));
+            engine.write(txn -> {
+                LongStream.range(0, 3_000).forEach(member -> txn.insert(edge, member));
+                return null;
+            });
+            long full;
+            try (Snapshot reader = engine.snapshot()) {
+                for (int round = 0; round < 30; round++) {
+                    long start = round * 100L;
+                    engine.write(txn -> {
+                        LongStream.range(start, start + 100).forEach(member -> txn.remove(edge, member));
+                        return null;
+                    });
+                }
+                engine.checkpoint();
+                ReferenceCounter.Verification held = engine.verifyReferences();
+                assertTrue(held.waitingPages() > 0, "pages the reader can reach must wait");
+                assertEquals(3_000L, reader.requireEdge(edge).size());
+                full = live(engine);
+            }
+            for (int i = 0; i < 4; i++) {
+                engine.write(txn -> txn.createNode(EngineTest.NODE, "after-" + txn.id()));
+            }
+            engine.checkpoint();
+            ReferenceCounter.Verification released = engine.verifyReferences();
+            assertTrue(live(engine) < full, "live bytes " + live(engine) + " should fall below " + full);
+            assertEquals(0L, (long) engine.read(snapshot -> snapshot.requireEdge(edge).size()));
+            assertTrue(released.waitingPages() < 10, "waiting " + released.waitingPages());
+        }
+        try (StorageEngine engine = StorageEngine.open(directory, options())) {
+            engine.verifyReferences();
+        }
+    }
+
+    @Test
+    void anAbortedBulkLoadGivesItsPagesBack() {
+        try (StorageEngine engine = StorageEngine.open(directory, options())) {
+            long edge = engine.write(txn -> txn.createEdge(EngineTest.EDGE, EdgeKind.SET));
+            long before = live(engine);
+            try (Transaction abandoned = engine.begin()) {
+                abandoned.load(edge, spread(20_000));
+                assertTrue(live(engine) > before + 10_000, "the load should have spilled pages");
+            }
+            for (int i = 0; i < 4; i++) {
+                engine.write(txn -> txn.createNode(EngineTest.NODE, "after-" + txn.id()));
+            }
+            engine.checkpoint();
+            assertTrue(live(engine) < before + 10_000, "live bytes " + live(engine) + " from " + before);
+            engine.verifyReferences();
+        }
+    }
+
+    @Test
+    void pagesALoadSpillsAndTheSameTransactionReplacesAreGivenBack() {
+        try (StorageEngine engine = StorageEngine.open(directory, options())) {
+            long edge = engine.write(txn -> txn.createEdge(EngineTest.EDGE, EdgeKind.SET));
+            long before = live(engine);
+            engine.write(txn -> {
+                txn.load(edge, spread(20_000));
+                txn.delete(edge);
+                return null;
+            });
+            for (int i = 0; i < 4; i++) {
+                engine.write(txn -> txn.createNode(EngineTest.NODE, "after-" + txn.id()));
+            }
+            engine.checkpoint();
+            assertTrue(live(engine) < before + 10_000, "live bytes " + live(engine) + " from " + before);
+            engine.verifyReferences();
+        }
+    }
+
+    @Test
+    void aDuplicateRequestGivesBackThePagesItsLoadSpilled() {
+        try (StorageEngine engine = StorageEngine.open(directory, options())) {
+            long edge = engine.write(txn -> txn.createEdge(EngineTest.EDGE, EdgeKind.SET));
+            TxnOptions request = TxnOptions.defaults().withRequestId("load-once");
+            engine.write(request, txn -> {
+                txn.insert(edge, 1);
+                return null;
+            });
+            long before = live(engine);
+            long other = engine.write(txn -> txn.createEdge(EngineTest.EDGE, EdgeKind.SET));
+            try (Transaction repeated = engine.begin(request)) {
+                repeated.load(other, spread(20_000));
+                assertEquals(CommitResult.Outcome.DUPLICATE, repeated.commit().outcome());
+            }
+            for (int i = 0; i < 4; i++) {
+                engine.write(txn -> txn.createNode(EngineTest.NODE, "after-" + txn.id()));
+            }
+            engine.checkpoint();
+            assertTrue(live(engine) < before + 10_000, "live bytes " + live(engine) + " from " + before);
+            engine.verifyReferences();
+        }
+    }
+
+    @Test
+    void pagesSpilledByATransactionCutShortByACrashAreReleasedOnOpen() {
+        long before;
+        StorageEngine engine = StorageEngine.open(directory, options());
+        long edge = engine.write(txn -> txn.createEdge(EngineTest.EDGE, EdgeKind.SET));
+        engine.checkpoint();
+        before = live(engine);
+        Transaction inFlight = engine.begin();
+        inFlight.load(edge, spread(20_000));
+        engine.halt();
+        try (StorageEngine reopened = StorageEngine.open(directory, options())) {
+            assertTrue(live(reopened) < before + 10_000, "live bytes " + live(reopened) + " from " + before);
+            reopened.verifyReferences();
+        }
+    }
+
+    @ParameterizedTest
+    @ValueSource(booleans = {false, true})
+    void countsSurviveACrash(boolean duringCheckpoint) {
+        AtomicBoolean armed = new AtomicBoolean();
+        EngineOptions options = options().withFaults(point -> {
+            if (point == CrashPoint.CHECKPOINT_JOURNAL && armed.compareAndSet(true, false)) {
+                throw new CrashPoint.SimulatedCrash(point);
+            }
+        });
+        Random random = new Random(11);
+        Map<Long, TreeSet<Long>> model = new HashMap<>();
+        StorageEngine engine = StorageEngine.open(directory, options);
+        for (int round = 0; round < 40; round++) {
+            step(engine, random, model);
+        }
+        engine.checkpoint();
+        for (int round = 0; round < 40; round++) {
+            step(engine, random, model);
+        }
+        if (duringCheckpoint) {
+            armed.set(true);
+            assertThrows(CrashPoint.SimulatedCrash.class, engine::checkpoint);
+        }
+        engine.halt();
+        try (StorageEngine recovered = StorageEngine.open(directory, options())) {
+            recovered.verifyReferences();
+            assertModel(recovered, model);
+        }
+    }
+
+    @Test
+    void countsMarkedForRebuildAreRebuiltOnOpen() throws Exception {
+        Random random = new Random(5);
+        Map<Long, TreeSet<Long>> model = new HashMap<>();
+        try (StorageEngine engine = StorageEngine.open(directory, options())) {
+            for (int round = 0; round < 60; round++) {
+                step(engine, random, model);
+            }
+        }
+        Path counts = directory.resolve("data").resolve("segments").resolve("pages.ref");
+        try (FileChannel file = FileChannel.open(counts, StandardOpenOption.WRITE)) {
+            file.write(ByteBuffer.allocate(Long.BYTES).order(ByteOrder.LITTLE_ENDIAN).putLong(0, 1), 8);
+            file.write(ByteBuffer.allocate(Long.BYTES * 64), 64);
+        }
+        try (StorageEngine engine = StorageEngine.open(directory, options())) {
+            engine.verifyReferences();
+            assertModel(engine, model);
+            for (int round = 0; round < 20; round++) {
+                step(engine, random, model);
+            }
+            engine.checkpoint();
+            engine.verifyReferences();
+        }
+    }
+
+    private static List<Incidence> spread(int count) {
+        return LongStream.range(0, count).map(i -> i * 7_919).mapToObj(Incidence::of).toList();
+    }
+
+    private static long live(StorageEngine engine) {
+        return engine.liveness().values().stream().mapToLong(Long::longValue).sum();
+    }
+}
diff --git a/engine/src/test/java/io/hstore/engine/page/LongLongMapTest.java b/engine/src/test/java/io/hstore/engine/page/LongLongMapTest.java
new file mode 100644
index 0000000..8511524
--- /dev/null
+++ b/engine/src/test/java/io/hstore/engine/page/LongLongMapTest.java
@@ -0,0 +1,42 @@
+package io.hstore.engine.page;
+
+import org.junit.jupiter.api.Test;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Random;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+class LongLongMapTest {
+
+    @Test
+    void behavesLikeAHashMapThroughGrowthAndClears() {
+        Random random = new Random(17);
+        LongLongMap map = new LongLongMap(4);
+        Map<Long, Long> model = new HashMap<>();
+        for (int round = 0; round < 4; round++) {
+            for (int i = 0; i < 50_000; i++) {
+                long key = 1 + random.nextInt(round % 2 == 0 ? 20_000 : 2_000_000);
+                long value = random.nextLong();
+                if (random.nextBoolean()) {
+                    map.put(key, value);
+                    model.put(key, value);
+                } else {
+                    assertEquals(model.merge(key, value, Long::sum), map.add(key, value));
+                }
+                long probe = 1 + random.nextInt(2_000_000);
+                assertEquals(model.getOrDefault(probe, -1L), map.get(probe, -1L));
+                assertEquals(model.containsKey(probe), map.containsKey(probe));
+            }
+            assertEquals(model.size(), map.size());
+            Map<Long, Long> seen = new HashMap<>();
+            map.forEach(seen::put);
+            assertEquals(model, seen);
+            map.clear();
+            model.clear();
+            assertEquals(0, map.size());
+            assertEquals(-1L, map.get(1, -1L));
+        }
+    }
+}
diff --git a/engine/src/test/java/io/hstore/engine/page/PageStoreTest.java b/engine/src/test/java/io/hstore/engine/page/PageStoreTest.java
index 8844c38..e9528ad 100644
--- a/engine/src/test/java/io/hstore/engine/page/PageStoreTest.java
+++ b/engine/src/test/java/io/hstore/engine/page/PageStoreTest.java
@@ -16,6 +16,7 @@
 
 import static org.junit.jupiter.api.Assertions.assertArrayEquals;
 import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
 import static org.junit.jupiter.api.Assertions.assertNotEquals;
 import static org.junit.jupiter.api.Assertions.assertThrows;
 import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -121,6 +122,62 @@ void aPageCannotBeWrittenPastWhatWasAllocatedForIt() {
         }
     }
 
+    @Test
+    void aSegmentIsNotSettledWhileAPageAllocatedInItIsUnwritten() {
+        try (PageStore store = PageStore.open(directory, PAGE_SIZE, PAGES_PER_SEGMENT, List.of())) {
+            long pending = store.allocate(PAGE_SIZE);
+            int segment = store.segmentOf(pending);
+            fillPastOneSegment(store);
+            assertFalse(store.settled(segment));
+            store.write(pending, MemorySegment.ofArray(image(new Random(3), PAGE_SIZE)));
+            assertTrue(store.settled(segment));
+        }
+    }
+
+    @Test
+    void evacuatingASegmentMovesOnlyThePagesTheDirectoryStillPointsAt() {
+        try (PageStore store = PageStore.open(directory, PAGE_SIZE, PAGES_PER_SEGMENT, List.of())) {
+            Map<Long, byte[]> kept = new LinkedHashMap<>();
+            List<Long> released = new ArrayList<>();
+            int segment = store.activeSegment() == 0 ? 1 : store.activeSegment();
+            Random random = new Random(9);
+            while (store.activeSegment() <= segment) {
+                int length = PageHeader.SIZE + random.nextInt(PAGE_SIZE - PageHeader.SIZE + 1);
+                long pageId = store.allocate(length);
+                byte[] image = page(random, pageId, length);
+                store.write(pageId, MemorySegment.ofArray(image));
+                if (store.segmentOf(pageId) != segment) {
+                    kept.put(pageId, image);
+                } else if (random.nextInt(3) == 0) {
+                    store.release(pageId);
+                    released.add(pageId);
+                } else {
+                    kept.put(pageId, image);
+                }
+            }
+            store.journal(1);
+            store.applyJournal();
+            long before = store.liveBytes().get(segment);
+            long moved = store.evacuate(segment);
+            assertTrue(moved > 0 && moved <= before, "moved " + moved + " of " + before);
+            assertEquals(null, store.liveBytes().get(segment));
+            kept.forEach((pageId, image) -> {
+                assertNotEquals(segment, store.segmentOf(pageId));
+                assertArrayEquals(image, bytes(store.read(pageId)));
+            });
+            released.forEach(pageId -> assertEquals(PageAddress.NONE, store.addressOrNone(pageId)));
+            assertFalse(released.isEmpty());
+        }
+    }
+
+    private static byte[] page(Random random, long pageId, int length) {
+        byte[] image = new byte[length];
+        random.nextBytes(image);
+        new PageHeader(PageType.LEAF, 1, 0, pageId, 1, length - PageHeader.SIZE, 0, 0, 0, 0, 0, 0).writeTo(ByteCursor.wrap(image));
+        PageHeader.assign(MemorySegment.ofArray(image), pageId);
+        return image;
+    }
+
     private static long write(PageStore store, int length) {
         byte[] image = image(new Random(length), length);
         long pageId = store.allocate(image.length);
diff --git a/engine/src/test/java/io/hstore/engine/tree/TreeTest.java b/engine/src/test/java/io/hstore/engine/tree/TreeTest.java
index af7788f..2ebdda6 100644
--- a/engine/src/test/java/io/hstore/engine/tree/TreeTest.java
+++ b/engine/src/test/java/io/hstore/engine/tree/TreeTest.java
@@ -1,11 +1,17 @@
 package io.hstore.engine.tree;
 
+import io.hstore.engine.catalog.AtomRecord;
+import io.hstore.engine.catalog.EngineSlots;
 import io.hstore.engine.page.ByteCursor;
+import io.hstore.engine.page.PageHeader;
+import io.hstore.engine.page.PageId;
+import io.hstore.engine.topology.EdgeKind;
 
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.ValueSource;
 
+import java.lang.foreign.MemorySegment;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
@@ -229,6 +235,20 @@ static void checkInvariants(Tree<?> tree) {
         assertEquals(recomputed, tree.summary());
     }
 
+    @Test
+    void aLeafSaysInItsHeaderWhetherItRefersToOtherPages() {
+        AtomRecord node = AtomRecord.NodeRecord.of(1, "n");
+        AtomRecord edge = new AtomRecord.EdgeRecord(2, EdgeKind.SET, new Ref.Stored(PageId.pack(7, 0), 1, Summary.EMPTY), Ref.EMPTY, 0, 0, 0, 0);
+        long pageId = PageId.pack(1, 0);
+        for (List<AtomRecord> values : List.of(List.of(node), List.of(node, edge))) {
+            Leaf leaf = Leaf.owned(EngineSlots.ATOM_SCHEMA, null, LongStream.range(0, values.size()).toArray(), values.toArray(), values.size());
+            MemorySegment image = NodeCodec.encode(leaf, pageId, 1, 4096);
+            assertEquals(values.contains(edge), PageHeader.verify(image, pageId).refersToPages());
+            Leaf decoded = (Leaf) NodeCodec.decode(image, pageId, EngineSlots.ATOM_SCHEMA);
+            assertEquals(values, decoded.valueList());
+        }
+    }
+
     private static Summary verify(Tree<?> tree, Ref ref) {
         Node node = tree.node(ref);
         Summary.Builder builder = Summary.builder(tree.schema().fingerprint());

From fbfb3f449b79871be98f705eaa8bb80c5fdbe2a8 Mon Sep 17 00:00:00 2001
From: venkat1701 <thejeastdev@gmail.com>
Date: Sun, 11 Oct 2026 06:13:15 +0530
Subject: [PATCH 06/15] docs: describe reference counts, releasing pages and
 format 5

maintenance.md replaces the liveness walk with the reference counts: how
the counter follows commits, when a dead page is released, how the counts
survive a crash and how they are rebuilt, and how compaction now reads
through a victim instead of walking the trees. The bulk-load fence is
gone from the compaction and transactions docs.

pages.md documents pages.ref and its journal byte for byte, and the WAL
and recovery doc adds the journal to the checkpoint steps, the counting
of replayed generations to recovery, and the order in which redo applies
a new branch. The overview, logging, docker and development pages pick up
the new files, thread, log lines and format 5. The directory listing and
the hstore check example are fresh output from the native build.
---
 docs/architecture/overview.md         |  23 ++--
 docs/development.md                   |   2 +-
 docs/operations/configuration.md      |   7 +-
 docs/operations/docker.md             |   4 +-
 docs/operations/logging.md            |  10 +-
 docs/storage/maintenance.md           | 174 ++++++++++++++++++++------
 docs/storage/pages.md                 |  43 ++++++-
 docs/storage/persistent-tree.md       |   2 +-
 docs/storage/topology.md              |   2 +-
 docs/transactions/transactions.md     |  14 ++-
 docs/transactions/wal-and-recovery.md |  40 +++---
 11 files changed, 238 insertions(+), 83 deletions(-)

diff --git a/docs/architecture/overview.md b/docs/architecture/overview.md
index bbe0f5d..a10cef0 100644
--- a/docs/architecture/overview.md
+++ b/docs/architecture/overview.md
@@ -83,7 +83,7 @@ The engine has no dependencies outside `java.base`. All three modules build into
 | Package | Purpose |
 |---|---|
 | `io.hstore.engine` | `StorageEngine` (open/close, maintenance thread, checkpoint/compact entry points), `EngineOptions`, `EngineStats`, `Dictionary` (interned role names and role sets), `HStoreException` (error codes, retryable conflicts). |
-| `engine.page` | Byte-level storage. `PageStore` and `SegmentFile` hold segment files and allocation. `PageHeader` is the 80-byte header and crc32c seal. `PageId` is a page's permanent number, `PageAddress` its place in a segment, and `PageDirectory` the memory-mapped `pages.dir` that maps one to the other. `ByteCursor` provides little-endian and varint codecs over `MemorySegment`. `IoTrace` counts page visits per query through a `ScopedValue`. |
+| `engine.page` | Byte-level storage. `PageStore` and `SegmentFile` hold segment files and allocation. `PageHeader` is the 80-byte header and crc32c seal. `PageId` is a page's permanent number, `PageAddress` its place in a segment, and `PageDirectory` the memory-mapped `pages.dir` that maps one to the other. `ReferenceCounts` is `pages.ref`, the count of references to each page, with its checkpoint journal. `ByteCursor` provides little-endian and varint codecs over `MemorySegment`. `IoTrace` counts page visits per query through a `ScopedValue`. |
 | `engine.tree` | The copy-on-write counted B+tree. `Tree` is the immutable handle and `TreeWriter` does path-copy updates. Nodes are `Leaf` and `Branch`, referenced by `Ref` (`Empty`, `Pending` or `Stored`). Also here: `Summary` (monoid annotations), `TreeCursor`, `TreeAlgebra` (leapfrog, set algebra), `TreeDiff`, `BulkBuilder`, `NodeCodec`, `NodeCache`, `Materializer`, `TreeWalker` and `TreeVerifier`. |
 | `engine.topology` | Hyperedge representation: `Hyperedge` (members tree plus order index), `Incidence` (one member's role set, weight, validity, payload and qualifier), `IncidenceCodec` (columnar leaf encoding), `MemberChange`, `Weight` (fixed point) and `Validity`. |
 | `engine.index` | `PostingIndex`, a two-level index from a key to a postings set. Small sets are inlined and large ones are promoted to their own tree. Used by the reverse incidence, type, canonical-key and symbol indexes. |
@@ -91,7 +91,7 @@ The engine has no dependencies outside `java.base`. All three modules build into
 | `engine.txn` | `TransactionManager` (commit lock, append, publication, pins, history), `Transaction` (op log, read sets, rebase), `Snapshot`, `View` (read API shared by both), `Workspace` (mutable root vector plus derivations), `GroupCommitter`, `Op`/`EdgeAction`, `Derivation`/`EngineDerivations`, `CrashPoint`, `Durability`, `WalMode`, `Isolation` and `TxnOptions`. |
 | `engine.wal` | `WriteAheadLog` (segmented, checksummed records) and `WalRecord` (`Begin`, `Page`, `PageRef`, `Root`, `BranchMeta`, `Feed`, `Commit`, `Abort`, `Checkpoint`). |
 | `engine.feed` | `ChangeFeed` (segmented append-only `feed/*.feed` files, sparse generation index, retention, tailing subscriptions), `CommitEvent` and `FeedCodec`; see [change-feed.md](../storage/change-feed.md). |
-| `engine.maintenance` | `Checkpointer`, `Compactor` (liveness, moving live pages out of victim segments, reclamation) and `Recovery` (catalog discovery, WAL redo, durable-prefix rule). |
+| `engine.maintenance` | `Checkpointer`, `ReferenceCounter` (counts references after each commit, releases pages no reader can reach), `Compactor` (moving live pages out of victim segments, reclamation) and `Recovery` (catalog discovery, WAL redo, durable-prefix rule). |
 
 ### `io.hstore.db`
 
@@ -209,7 +209,8 @@ The ordering above is what makes recovery safe. Under `PAGE_REFERENCES` the WAL
 |---|---|---|---|
 | Caller threads | any (the server uses virtual threads) | — | Run reads without locks. Run a transaction's operations without locks, then take `commitLock` only for append (encoding, WAL append, page writes). They then block in `GroupCommitter.await` outside the lock. |
 | `hstore-group-commit` | platform, daemon | `GroupCommitter` constructor | Takes every queued `Pending` as one batch and calls `makeDurable(batchSize)` once: data sync in `PAGE_REFERENCES` mode, then WAL sync, both only under `Durability.SYNC`. It then runs each publication in generation order and wakes the waiters. One fsync pair covers the whole batch (`averageGroupCommit` in the statistics). A failure poisons the pipeline, and every later commit fails until restart. |
-| `hstore-maintenance` | virtual | `StorageEngine` constructor | Every 500 ms, under the `maintenance` lock (`tryLock`): checkpoint once the WAL written since the last checkpoint exceeds `checkpoint_wal_mb`, then reclaim retired segments that no pinned reader can still see. |
+| `hstore-maintenance` | virtual | `StorageEngine` constructor | Every 500 ms, under the `maintenance` lock (`tryLock`): checkpoint once the WAL or the pages written since the last checkpoint exceed `checkpoint_wal_mb`, compact one segment if any has fallen below `compaction_live_ratio`, then reclaim retired segments that no pinned reader can still see. |
+| `hstore-references` | platform, daemon | `StorageEngine` constructor | Takes the generations that publications queue, in order, and updates the reference counts of the pages each commit added or dropped. The queue holds up to 4 096 commits; past that, publication waits for it. |
 | `feed-subscriber` | virtual, one per subscription | `ChangeFeed.subscribe` | Tails the feed segments, waiting on a condition signalled by `append` (with a 250 ms timeout). Subscribers are `SemanticPlane`, `MaterializedViews` and `Statistics`. |
 | Server connections | virtual, one per socket | `Server` (`Executors.newVirtualThreadPerTaskExecutor`) | Each connection runs a `Session`. |
 | Studio requests | virtual | `Studio` (`HttpServer.setExecutor`) | HTTP API and static assets. |
@@ -219,6 +220,7 @@ Locks:
 * `TransactionManager.commitLock` serializes generation assignment and the WAL append order.
 * `StorageEngine.maintenance` serializes checkpoint and compaction.
 * `TransactionManager.exclusive` holds `commitLock` and drains the group committer, so a checkpoint sees exactly the published state.
+* `ReferenceCounter.pause` takes the counter's lock after counting every queued commit, so a checkpoint journals counts that match the generation it publishes. The checkpoint keeps it until the journal is applied, after it has let go of `commitLock`.
 
 Readers take no locks at all.
 
@@ -227,15 +229,17 @@ Readers take no locks at all.
 The following is real output from `hstore init` followed by loading `examples/clinical-claims.hql` and running `CHECKPOINT;` (default 16 KiB pages):
 
 ```text
-FORMAT                                   55 B  format=4, page-size=16384 (java.util.Properties)
+FORMAT                                   55 B  format=5, page-size=16384 (java.util.Properties)
 LOCK                                      0 B  FileChannel.tryLock() held while open
-hstore.conf                            2397 B  server configuration template written by `hstore init`
+hstore.conf                            2863 B  server configuration template written by `hstore init`
 catalog/manifest                         20 B  name of the latest catalog file
 catalog/generations/0000000000000000.cat 37 B  checkpoint of generation 0
-catalog/generations/000000000000009d.cat 49147 B  checkpoint of generation 157 (current + retained history)
-data/segments/00000001.seg           1210439 B  segment 1: packed node images
+catalog/generations/000000000000009d.cat 49851 B  checkpoint of generation 157 (current + retained history)
+data/segments/00000001.seg           1213651 B  segment 1: packed node images
 data/segments/pages.dir              8388608 B  page directory, one 8 MiB chunk mapped so far (sparse)
-wal/segments/0000000000000000.wal      156180 B  WAL segment starting at LSN 0
+data/segments/pages.ref              8388608 B  reference counts, same layout (sparse)
+data/segments/pages.ref.journal            0 B  emptied after every checkpoint
+wal/segments/0000000000000000.wal      165595 B  WAL segment starting at LSN 0
 feed/0000000000000000.feed             29846 B  change feed segment starting at byte 0
 payload/000001.pay                        0 B  payload segment (database layer)
 semantic/state                           53 B  persisted HNSW index state (database layer)
@@ -243,12 +247,13 @@ semantic/state                           53 B  persisted HNSW index state (datab
 
 | Path | Owner | Format |
 |---|---|---|
-| `FORMAT` | `StorageEngine.verifyFormat` | `Properties` with `format=4` (page numbers resolved through a page directory) and `page-size` (the maximum node size). Opening a directory written in an older format, or opening with a different page size, fails. |
+| `FORMAT` | `StorageEngine.verifyFormat` | `Properties` with `format=5` (page numbers resolved through a page directory, reference counts per page) and `page-size` (the maximum node size). Opening a directory written in an older format, or opening with a different page size, fails. |
 | `LOCK` | `StorageEngine.open` | An exclusive OS file lock. A second process gets `database ... is opened by another process`. |
 | `catalog/generations/%016x.cat` | `CatalogStore.publish` | `int magic 0x54414348`, `int crc32c(payload)`, `long length`, then the `CatalogImage` payload. Written to `*.tmp`, fsynced, atomically renamed, and the directory fsynced. The three newest files are kept. |
 | `catalog/manifest` | `CatalogStore` | The file name of the latest `.cat`, written durably the same way. |
 | `data/segments/%08x.seg` | `PageStore` / `SegmentFile` | Append-only extents of node images packed at 64-byte alignment, up to 256 MiB per segment by default (1 GiB maximum); see [pages.md](../storage/pages.md). |
 | `data/segments/pages.dir` | `PageDirectory` | 64-byte header, then one `i64` address (segment, offset, units) per page number, memory-mapped in 8 MiB chunks; see [pages.md](../storage/pages.md#page-numbers-and-the-page-directory). |
+| `data/segments/pages.ref`, `pages.ref.journal` | `ReferenceCounts`, `ReferenceJournal` | One `i64` reference count per page number in the same layout, changed only at checkpoints through the journal; see [pages.md](../storage/pages.md#reference-counts). |
 | `wal/segments/%016x.wal` | `WriteAheadLog` | Each segment is named by its starting LSN; see [wal-and-recovery.md](../transactions/wal-and-recovery.md). |
 | `feed/%016x.feed` | `ChangeFeed` | Segments (64 MiB, named by starting byte offset) of frames `int length`, `int crc32c`, `long generation`, `payload`, trimmed by retention after checkpoints; see [change-feed.md](../storage/change-feed.md). |
 | `payload/%06x.pay` | `db.payload.PayloadStore` | Append-only payload extents. |
diff --git a/docs/development.md b/docs/development.md
index e18eb87..b2afa0f 100644
--- a/docs/development.md
+++ b/docs/development.md
@@ -24,7 +24,7 @@ flowchart LR
 
 | Package | Contents |
 |---|---|
-| `engine.page` | Segment files, 80-byte page header with CRC32C, `PageId` page numbers, `PageAddress` (segment 24 / offset 24 / units 16 bits), the `pages.dir` page directory, page store and cache |
+| `engine.page` | Segment files, 80-byte page header with CRC32C, `PageId` page numbers, `PageAddress` (segment 24 / offset 24 / units 16 bits), the `pages.dir` page directory, the `pages.ref` reference counts, page store and cache |
 | `engine.tree` | Copy-on-write counted B+tree, monoid summaries, write scopes, cursors, `TreeAlgebra` (leapfrog intersection, synchronized traversal), `TreeDiff`, `TreeVerifier` |
 | `engine.topology` | Hyperedge membership and incidence encodings, `Incidence`, `Validity` |
 | `engine.catalog` | Slots, root vectors, branches, generations, dictionary |
diff --git a/docs/operations/configuration.md b/docs/operations/configuration.md
index 72bc767..8f45b9d 100644
--- a/docs/operations/configuration.md
+++ b/docs/operations/configuration.md
@@ -92,9 +92,9 @@ max_connections          = 200
 | `durability` | `HSTORE_DURABILITY` | `sync` | `sync`: a commit is acknowledged after the group-commit barrier has `fsync`ed it. `async`: acknowledged once appended, so a crash can lose a suffix of acknowledged commits but never corrupts the database. |
 | `wal_mode` | `HSTORE_WAL_MODE` | `references` | `references`: the WAL logs `PageRef(pageId, address, length, crc32c)` records and data pages are synced before the WAL. `images`: the WAL logs full page images. See [WAL and recovery](../transactions/wal-and-recovery.md). |
 | `cache_nodes` | `HSTORE_CACHE_NODES` | `65536` | Decoded tree nodes kept in the node cache. |
-| `checkpoint_wal_mb` | `HSTORE_CHECKPOINT_WAL_MB` | `256` | WAL volume since the last checkpoint that triggers a background checkpoint. |
+| `checkpoint_wal_mb` | `HSTORE_CHECKPOINT_WAL_MB` | `256` | WAL volume, or volume of pages written, since the last checkpoint that triggers a background checkpoint. |
 | `history_limit` | `HSTORE_HISTORY_LIMIT` | `64` | Committed generations kept addressable for `AT GENERATION`, `AS OF`, `HISTORY` and the Studio time slider. |
-| `compaction_live_ratio` | `HSTORE_COMPACTION_LIVE_RATIO` | `0.5` | Sealed segments whose live bytes fall below this fraction of their allocated bytes are emptied by compaction, which moves their live pages to the active segment: one segment (the emptiest) per background pass after every `checkpoint_wal_mb` of written node images, or all of them with `COMPACT`. Time-travel history is preserved. Emptied segments are deleted after the next checkpoint, once no reader that started before the move is still open. Must satisfy 0 < r < 1; other values fail at startup. |
+| `compaction_live_ratio` | `HSTORE_COMPACTION_LIVE_RATIO` | `0.5` | Sealed segments whose live bytes fall below this fraction of their allocated bytes are emptied by compaction, which moves their live pages to the active segment: one segment (the emptiest) per background pass whenever one is below the ratio, or all of them with `COMPACT`. Pages count as free once nothing refers to them and no retained generation or open reader can reach them; a checkpoint releases them. Time-travel history is preserved. Emptied segments are deleted after the next checkpoint, once no reader that started before the move is still open. Must satisfy 0 < r < 1; other values fail at startup. |
 | `feed_retention_generations` | `HSTORE_FEED_RETENTION_GENERATIONS` | `100000` | Committed generations kept in the [change feed](../storage/change-feed.md) for `HISTORY`, `DIFF` and subscribers. After every checkpoint, whole feed segments that lie entirely below `current − N` are deleted. *Holds* lower that floor: the semantic index's last persisted generation and the generation of each continuous materialised view. Feed segments roll at the WAL segment size (`EngineOptions.walSegmentBytes`, 64 MiB). Must be ≥ 1. |
 
 ### Resource budgets
@@ -146,7 +146,8 @@ working set of hot leaves. The Dashboard *Cache hit rate* tile and `STATS;` show
 **`checkpoint_wal_mb`.** A checkpoint writes the current generation's catalog image and truncates WAL segments
 before the checkpoint LSN. Lower values shorten recovery, since recovery replays WAL after the last checkpoint,
 at the cost of more frequent catalog writes. Recovery time scales roughly linearly with WAL volume since the
-last checkpoint.
+last checkpoint. The same volume of page writes also triggers a checkpoint, which is what releases dead pages
+during a bulk load.
 
 **`durability` × `wal_mode`.**
 
diff --git a/docs/operations/docker.md b/docs/operations/docker.md
index 160f648..09df130 100644
--- a/docs/operations/docker.md
+++ b/docs/operations/docker.md
@@ -258,8 +258,8 @@ skipped because `FORMAT` exists. To verify a backup, run
 `docker run --rm -v restored:/var/lib/hstore/data hstore check /var/lib/hstore/data`.
 
 To upgrade, stop the container, back up the volume, and start the new image on the same volume. `FORMAT`
-records the on-disk format version (`format=4`, page directory) and the page size. Both are checked at
+records the on-disk format version (`format=5`, reference counts) and the page size. Both are checked at
 open. A directory written in another format is refused with
-`database … uses storage format N; this build reads format 2 (packed segment extents); export and reload it`,
+`database … uses storage format N; this build reads format 5 (reference counts); export and reload it`,
 so the server never misreads it. Export the data with HQL against the old version and reload it into a fresh
 volume.
diff --git a/docs/operations/logging.md b/docs/operations/logging.md
index 3c01209..abd0351 100644
--- a/docs/operations/logging.md
+++ b/docs/operations/logging.md
@@ -51,6 +51,7 @@ clean. `init` uses the configured level.
 | `studio` | `hstore.studio` | Studio HTTP server (`Studio`) |
 | `engine` | `hstore.engine` | open, checkpoint, compaction, reclamation, shutdown (`StorageEngine`) |
 | `recovery` | `hstore.recovery` | crash recovery (`Recovery`) |
+| `references` | `hstore.references` | page reference counts (`ReferenceCounter`) |
 | `txn` | `hstore.txn` | branch lifecycle (`TransactionManager`) |
 | `commit` | `hstore.commit` | group-commit pipeline (`GroupCommitter`) |
 | `feed` | `hstore.feed` | change-feed retention and subscribers (`ChangeFeed`) |
@@ -71,6 +72,10 @@ This is every message the server emits. Placeholders are in `{braces}`.
 | LOG | main | `configuration: {key}={value} ({source}), …` | Settings that differ from their default, with source `file`, `environment` or `command line`. |
 | LOG | recovery | `recovered generation {g}: replayed {n} commits from lsn {lsn}, discarded {t} unfinished transactions and {c} non-durable commits` | Always logged at open. `{lsn}` is the checkpoint LSN replay started from. |
 | WARNING | recovery | `durable prefix ends before generation {g}: transaction {txn} references pages that never reached storage` | In `references` WAL mode, a committed transaction whose `PageRef` pages fail verification ends the recovered prefix. Later commits are discarded. Seen only after a crash with `durability = async` or a storage fault. |
+| LOG | references | `released {n} pages that no committed generation refers to` | Logged at open when pages written by transactions that never committed, for example ones cut short by a crash, were found and released. |
+| WARNING | references | `reference counts could not follow the log, rebuilding them` | Counting the commits replayed at open failed. The counts are rebuilt from the retained generations. |
+| LOG | references | `rebuilt reference counts in {ms} ms` | The counts were marked for rebuilding, or could not follow the log, and were recounted at open. |
+| WARNING | references | `{n} referenced pages have no address` | Pages that are counted as in use have no directory entry. Run `hstore check`. |
 | LOG | engine | `database at {dir} is ready: generation {g}, {n} data segments, page size {bytes}, {durability} durability, {walMode} wal` | Engine open complete. |
 | LOG | semantic | `loaded semantic index at generation {g} ({n} models)` | HNSW indexes restored from `<data>/semantic`. |
 | WARNING | semantic | `semantic index at generation {g} is ahead of the database; rebuilding` | The index state file is newer than the recovered generation, for example after an `async` crash. |
@@ -90,10 +95,11 @@ This is every message the server emits. Placeholders are in `{braces}`.
 
 | Severity | Component | Message |
 |---|---|---|
-| LOG | engine | `background compaction moved the live pages out of segments {list}` (write-volume-triggered pass) |
+| LOG | engine | `background compaction moved the live pages out of segments {list}` (a segment fell below `compaction_live_ratio`) |
 | LOG | engine | `compaction moved the live pages out of segments {list} and reclaimed {list}` |
 | LOG | engine | `reclaimed retired segments {list}` |
 | WARNING | hstore | `background maintenance failed` (with DETAIL) |
+| ERROR | references | `reference counting stopped; the counts will be rebuilt when the database next opens` (with DETAIL). No more pages are released until a restart, so space stops being reclaimed but nothing is lost. |
 | LOG | txn | `created branch {name} (#{id}) from generation {g}` |
 | LOG | txn | `branch #{id} is now {MERGED or DROPPED}` |
 | ERROR | commit | `commit pipeline failed; the engine must be restarted` (with DETAIL). The flusher thread could not make a batch durable. All waiting and future committers fail. |
@@ -109,7 +115,7 @@ and in Docker it is the last line of `docker logs`. The storage-integrity failur
 
 | Message | Meaning |
 |---|---|
-| `database <dir> uses storage format N; this build reads format 4 (page directory); export and reload it` | `FORMAT` was written by an incompatible build. |
+| `database <dir> uses storage format N; this build reads format 5 (reference counts); export and reload it` | `FORMAT` was written by an incompatible build. |
 | `database uses N byte pages, options request M` | `page_size` differs from the value fixed at `init`. |
 | `write-ahead log segment <file> is damaged (<reason>) but later segments exist; replay would silently drop committed transactions` | `CORRUPT_LOG`: damage in a non-tail WAL segment. A torn record at the end of the *last* segment is normal after a crash. Replay simply ends there. Damage before later segments means storage corruption, so recovery refuses to guess. Restore from backup. |
 
diff --git a/docs/storage/maintenance.md b/docs/storage/maintenance.md
index acedc26..2715133 100644
--- a/docs/storage/maintenance.md
+++ b/docs/storage/maintenance.md
@@ -3,22 +3,24 @@
 Copy-on-write storage never overwrites data, so three background concerns replace in-place update:
 
 * **Checkpoints** bound recovery time and WAL size.
-* **Compaction** returns the space of unreachable node images to the file system.
+* **Reference counting and compaction** find the node images nothing can reach any more and return their space to
+  the file system.
 * **Verification** proves that what is on disk is internally consistent.
 
-Sources are under `engine/src/main/java/io/hstore/engine/maintenance/`, plus `StorageEngine.java`, `tree/TreeWalker.java` and `tree/TreeVerifier.java`.
+Sources are under `engine/src/main/java/io/hstore/engine/maintenance/`, plus `StorageEngine.java`, `page/PageStore.java`, `page/ReferenceCounts.java`, `tree/TreeWalker.java` and `tree/TreeVerifier.java`.
 
 ## Who runs what
 
 | Trigger | Action | Code |
 |---|---|---|
 | Engine open, after recovery | checkpoint, then reclaim | `StorageEngine` constructor |
-| `hstore-maintenance` virtual thread, every 500 ms (skipped when the `maintenance` lock is busy) | checkpoint if `walSinceCheckpoint() > checkpointWalBytes`; a compaction pass when more than `checkpointWalBytes` of node images were written since the previous pass (followed by a checkpoint if anything was moved); then reclaim | `StorageEngine.maintain` |
+| `hstore-maintenance` virtual thread, every 500 ms (skipped when the `maintenance` lock is busy) | checkpoint if more than `checkpointWalBytes` of WAL or of pages were written since the last one, or more than 1 048 576 counts changed; compact one segment if any is below `compaction_live_ratio` (followed by a checkpoint); then reclaim | `StorageEngine.maintain` |
 | `CHECKPOINT;` (HQL), `StorageEngine.checkpoint()` | checkpoint | `timedCheckpoint` |
-| `COMPACT;` (HQL), `StorageEngine.compact()` | compact, checkpoint, reclaim | `StorageEngine.compact` |
+| `COMPACT;` (HQL), `StorageEngine.compact()` | checkpoint, compact, checkpoint, reclaim | `StorageEngine.compact` |
+| `hstore-references` platform thread, after every published commit | update reference counts | `ReferenceCounter` |
 | `StorageEngine.close()` | final checkpoint, then shutdown | `close` |
 
-Checkpoint and compaction are serialized by `StorageEngine.maintenance`, a `ReentrantLock`. Background compaction plays the role of PostgreSQL's autovacuum. Its trigger is write volume, `pages.bytesWritten()` advancing by more than `checkpoint_wal_mb` since the last pass, so an idle database never pays for a liveness walk. The background pass empties at most one segment, the emptiest one whose live bytes are below `compaction_live_ratio` of its allocated bytes. `COMPACT;` empties every such segment on demand.
+Checkpoint and compaction are serialized by `StorageEngine.maintenance`, a `ReentrantLock`. Background compaction plays the role of PostgreSQL's autovacuum. It runs whenever a sealed segment's live bytes fall below `compaction_live_ratio` of its allocated bytes, which the reference counts keep up to date, and it empties one segment per pass, the emptiest. `COMPACT;` empties every such segment on demand.
 
 Every checkpoint is logged, and then the checkpoint listeners run. The database registers `SemanticPlane.persist`, which saves the HNSW index alongside the catalog. A sample log line:
 
@@ -28,15 +30,19 @@ Every checkpoint is logged, and then the checkpoint listeners run. The database
 
 ## Checkpoints
 
-`Checkpointer.checkpoint()` (`maintenance/Checkpointer.java`) runs inside `TransactionManager.exclusive(...)`. That holds the commit lock **and drains the group committer**, so every appended commit is also published and nothing is in flight. Then:
+`Checkpointer.checkpoint()` (`maintenance/Checkpointer.java`) first syncs the segment files and forces the page directory without holding anything, so that most of the data written since the last checkpoint is on disk before commits have to wait. The rest runs inside `TransactionManager.exclusive(...)`. That holds the commit lock **and drains the group committer**, so every appended commit is also published and nothing is in flight. It then lets the reference counter catch up with every published commit and pauses it (`ReferenceCounter.pause`). Then:
 
-1. `pages.sync()` fsyncs every dirty segment file.
-2. `feed.sync()` fsyncs the active feed segment.
-3. It captures `current`, `lsn = wal.end()`, and the retained history: all retained generations except current, newest first, at most `historyLimit`.
-4. `catalog.publish(new CatalogImage(current, history, pages.segments(), lsn, feed.size()))` writes the image durably and atomically ([catalog-and-generations.md](catalog-and-generations.md#catalog-images-checkpoints)). The image includes every segment's `(id, state, pages, units, retiredAt)`.
-5. It appends a `WalRecord.Checkpoint(0, generation, lsn)` and calls `wal.sync()`.
-6. `wal.truncateBefore(lsn)` deletes WAL segment files that end before the checkpoint LSN. They are no longer needed, because the catalog image now covers them.
-7. It remembers `lastLsn = lsn`.
+1. It reads `lsn = wal.end()`.
+2. It releases the pages that died before the oldest generation anyone can still read (see [Releasing pages](#releasing-pages)).
+3. `pages.checkpoint()` fsyncs every dirty segment file and forces the page directory.
+4. `pages.journal(lsn)` writes every count changed since the last checkpoint, and every directory entry released, to `pages.ref.journal`, and fsyncs it.
+5. `feed.sync()` fsyncs the active feed segment.
+6. It captures `current` and the retained history: all retained generations except current, newest first, at most `historyLimit`.
+7. `catalog.publish(new CatalogImage(current, history, pages.segments(), lsn, feed.size()))` writes the image durably and atomically ([catalog-and-generations.md](catalog-and-generations.md#catalog-images-checkpoints)). The image includes every segment's `(id, state, pages, units, retiredAt)`.
+8. It appends a `WalRecord.Checkpoint(0, generation, lsn)` and calls `wal.sync()`.
+9. `wal.truncateBefore(lsn)` deletes WAL segment files that end before the checkpoint LSN. They are no longer needed, because the catalog image now covers them.
+10. The commit lock is released here, and commits carry on. `pages.applyJournal()` writes the journaled counts into `pages.ref` and clears the released directory entries, forces both files, and empties the journal. The counter stays paused until this is done, so the counts it writes next start from the applied state.
+11. It remembers `lastLsn = lsn` and resumes the counter.
 
 **Trigger threshold.** `walSinceCheckpoint() = wal.end() − lastLsn` is compared with `checkpoint_wal_mb` (default 256 MiB, `EngineOptions.checkpointWalBytes`). In the default `PAGE_REFERENCES` WAL mode a commit logs about 47 bytes per new node image instead of the image itself. The threshold therefore corresponds to far more committed work than in `PAGE_IMAGES` mode, and replay after a crash stays short.
 
@@ -50,9 +56,10 @@ Each commit writes fresh node images for every path it changes and references al
 
 Segments are append-only extents ([pages.md](pages.md#segment-files)), so garbage is never reused piecemeal. It is reclaimed one whole segment at a time:
 
-1. measure how many images in each segment are still reachable;
-2. copy the reachable ones out of mostly-dead segments, byte for byte;
-3. point their entries in the page directory at the copies;
+1. count, for every page, how many pages and branch roots point at it, and note when that number reaches zero;
+2. once no reader can reach such a page any more, release it and take its bytes off its segment's live total;
+3. copy the remaining pages out of mostly-dead segments, byte for byte, and point their entries in the page
+   directory at the copies;
 4. delete the files once a checkpoint has made the new entries durable and no reader can still be using an old
    address.
 
@@ -74,34 +81,110 @@ stateDiagram-v2
 
 `SegmentState.canTransitionTo` allows only the next state, plus `COMPACTING → SEALED`. `SegmentInfo.withState` throws on anything else. `PageStore.open` turns a `COMPACTING` segment back into `SEALED`, so a compaction cut short by a crash leaves the segment a candidate for the next pass. Only one segment is `ACTIVE` at a time: the one `PageStore.allocate` appends to. It is never chosen as a victim.
 
-### Liveness
-
-`Compactor.liveness()` returns, per segment, the bytes of distinct node images reachable from:
-
-* **every retained generation** (`transactions.history()`) plus `current`;
-* **every branch** in each of those generations;
-* **every slot root** in each branch.
-
-`TreeWalker.visit(root, schema, firstVisit)` calls `firstVisit` with each `Ref.Stored` page id it meets and stops descending when that returns false. The compactor sets one bit per page number in a `BitSet`, so an image shared by several generations, branches or trees is visited once, and the whole walk costs one bit per page number. Sizes come from the page directory afterwards: each set bit is looked up and its length added to its segment. The walker descends into branches and, for codecs that hold references (catalog edge records, promoted posting lists), into nested trees.
-
-A detail that keeps this cheap: at height 0 of a tree whose codec has no nested references, the walker records the leaf's id **without loading it**. The summaries in the parent already prove that the leaf exists. Liveness is therefore proportional to the number of internal nodes plus the reference-holding leaves, not to total data size.
-
-Liveness is exposed as `StorageEngine.liveness()` (live bytes per segment). The Studio dashboard shows it per segment next to `SegmentInfo.pages` and `bytes()`.
+### Reference counts
+
+Every page number has a count in `data/segments/pages.ref` ([pages.md](pages.md#reference-counts)): the number of
+places that point at it in the current state of the database. A place is a branch's root for a slot, a child
+reference in a branch node, or a nested tree's root inside a leaf value, such as an edge's member tree in the
+catalog. A page that two branches share through one parent is counted once for that parent, not once per branch.
+
+**After each commit.** When the group committer publishes a generation, it hands `ReferenceCounter` the generation
+before and the one after. The counter works on its own thread (`hstore-references`), in commit order, so the commit
+itself only pays for putting that pair on a queue. For each branch whose roots changed it compares the two root
+vectors and then:
+
+1. Adds one to every new root. A page whose count goes up from zero is new, or was written by a bulk load, so the
+   pages it points at get one more as well, and so on down.
+2. Takes one from every old root. A page whose count reaches zero is dead, so the pages it pointed at lose one too.
+
+Step 1 runs to the end before step 2 starts, and its additions are collected in a small map first. When a commit
+replaces a node, the new copy adds one to each child and the old one takes it away again, so those children cancel
+out and their counts are never touched. What the counter really writes is the few pages that a commit added or
+removed. Reading a node's references needs the node, which almost always comes from the node cache since the commit
+just wrote it or read it. A leaf below a branch node is never read when its tree's values hold no references,
+since the branch already says its children are leaves. Other leaves are read only as far as the header when they
+aren't cached: a header flag (`PageHeader.REFERS_TO_PAGES`) says whether any value in the leaf points at another
+page, and most leaves, such as catalog leaves holding nodes and index leaves with inline postings, don't. That keeps
+counting the commits replayed after a crash about as cheap as replaying them.
+
+A page that dies is marked with the generation it died in. Older generations can still reach it, so its bytes stay
+where they are until no one can read those generations (see below).
+
+**Pages nothing commits.** `Transaction.load` can write pages before the transaction commits. If the transaction
+aborts, fails with a conflict, or turns out to repeat a request that already committed, those pages are marked dead
+straight away. If it commits but replaced some of them
+before committing, the ones the new trees don't reach are marked dead after the commit is counted. Pages written by
+a transaction that a crash cut short are found on the next open (see below).
+
+**Between checkpoints** the changes live in a map in memory (`ReferenceCounts.changed`) and `pages.ref` itself is
+not touched. A checkpoint journals them and then applies them (steps 4 and 10 above), so a crash at any point leaves
+the file as of one checkpoint or the next, never a mix. The background thread checkpoints early when more than
+1 048 576 counts have changed, which keeps that map to a few tens of megabytes. It also checkpoints after every
+`checkpoint_wal_mb` of pages written, because a bulk load writes many pages for little WAL and its dead pages would
+otherwise wait a long time to be released.
+
+**On open**, `PageStore.open` applies `pages.ref.journal` if it was written by the checkpoint the catalog points at
+and ignores it otherwise. `ReferenceCounter.recover` then counts the generations recovery replayed from the WAL, in
+order, exactly as it would have after each commit. Last, one pass over `pages.dir` and `pages.ref`:
+
+* releases every page that has an address but a count of zero and isn't marked dead. No committed generation
+  points at it; it was written by a transaction that never committed;
+* adds up the live bytes of each segment;
+* collects the dead pages that are still waiting, in order of the generation they died in.
+
+**If counting fails**, for example on a page that can't be read, the counter stops, logs an error and marks
+`pages.ref` for rebuilding. It releases nothing more until the engine restarts, so the worst case is space that
+isn't reclaimed. The next open sees the mark and recounts from the retained generations: every page reachable from
+the current state gets its exact count, and every page only older generations reach is marked dead as of the current
+generation. That walk reads every node of every retained tree, so it takes as long as `hstore check`.
+
+### Releasing pages
+
+A page that died in generation `g` can only be reached from generations older than `g`. The checkpoint computes the
+oldest generation anyone can still read, `TransactionManager.oldestReadable()`: the oldest retained generation or the
+oldest pinned one, whichever is smaller. Every waiting page that died at or before that generation is released: its
+count goes to zero, its directory entry is cleared, and its bytes come off the live total of the segment it sits in.
+Those live totals are what compaction chooses victims by.
+
+Readers pin their generation and then check that it is still retained, and `trimHistory` puts a generation back if
+it finds a pin on it after removing it. Between the two, a reader can't end up holding a generation whose pages were
+released.
+
+Releases only happen inside a checkpoint, and the checkpoint journals the cleared directory entries with the counts.
+A release therefore never reaches the disk before the catalog that justifies it, and a crash can't bring back an old
+generation whose pages are gone.
+
+`StorageEngine.liveness()` returns the live bytes of each segment, and the Studio dashboard shows them next to
+`SegmentInfo.pages` and `bytes()`. Dead pages that are still waiting count as live, since they still have to be
+moved if their segment is compacted.
 
 ### Compaction
 
 `Compactor.compact(liveThreshold)` (default threshold `compaction_live_ratio = 0.5`):
 
-1. **Take a cut.** Inside `TransactionManager.exclusive`, which holds the commit lock and drains the group committer, record the retained generations (history plus `current`) and the set of segments that are already `SEALED`. Only those segments can become victims. A commit writes its pages under the commit lock but publishes its generation a moment later, and the active segment can fill up and be sealed while the walk runs. Taking the cut with nothing in flight means every page the walk doesn't see sits in a segment that wasn't sealed yet.
-2. **Walk** the recorded generations as described under *Liveness*.
-3. **Choose victims.** A victim is a segment from the cut, still `SEALED`, with `live bytes < allocated bytes × threshold`, so the ratio is *live bytes / bytes written*. Candidates are ordered from emptiest to fullest. The background pass takes only the first one (`compact(threshold, 1)`); `COMPACT` takes all of them. If there are no victims, it reports and returns.
-4. Mark the victims `COMPACTING`.
-5. **Move.** For every set bit whose directory entry points into a victim, `PageStore.copy` reads the image, checks it with `PageHeader.verify`, and writes the same bytes to the active segment. The header holds the page number, not the address, so neither the header nor the checksum changes. Every 4 MiB of copies, `PageStore.sync()` forces the segments and only then are the directory entries pointed at the copies. A directory entry therefore never reaches the disk pointing at bytes that are not already there.
-6. Count the compaction in `TransactionManager.compacted()` (see below) and mark the victims `RETIRED` with `retiredAt` set to the current generation.
-
-`StorageEngine.compact()` then checkpoints, which forces the page directory and persists the segment states, and calls `reclaim()`.
-
-Nothing in this takes the commit lock except the short cut in step 1, and nothing commits. Commits carry on while pages move, readers keep using the node cache, whose entries are keyed by page number and stay valid, and retained history keeps working because old generations reach the moved pages through the same directory.
+1. **Choose victims.** A victim is a `SEALED` segment, not the active one, with `live bytes < allocated bytes ×
+   threshold`, so the ratio is *live bytes / bytes written*. Every page allocated in it must also have been written
+   (`PageStore.settled`); a commit allocates its pages before it logs them and writes them right after, so a
+   segment can be sealed while a few of its pages are still on their way. Candidates are ordered from emptiest to
+   fullest. The background pass takes only the first one (`compact(threshold, 1)`); `COMPACT` takes all of them.
+2. Mark the victims `COMPACTING`.
+3. **Move.** `PageStore.evacuate` reads through the victim one 64-byte unit at a time. Where a page header starts and
+   the directory entry for that header's page number points at exactly this spot, the page is live: it is checked
+   with `PageHeader.verify` and its bytes are written to the active segment. Anything else, such as a released page
+   or the old copy of a page that has already moved, is skipped. The header holds the page number, not
+   the address, so neither the header nor the checksum changes. Every 4 MiB of copies, the segments are synced and
+   only then are the directory entries pointed at the copies, so an entry never reaches the disk pointing at bytes
+   that aren't there yet.
+4. Mark the victims `RETIRED` with `retiredAt` set to the current generation.
+
+The work is proportional to the size of the victims, not of the database, and the only memory it needs is a short
+list of the pages moved in the current 4 MiB. `StorageEngine.compact()` checkpoints before compacting, so pages
+that died since the last checkpoint are released and counted as free, and again afterwards, which forces the page
+directory and persists the segment states. Then it calls `reclaim()`.
+
+Nothing here takes the commit lock and nothing commits. Commits carry on while pages move, readers keep using the
+node cache, whose entries are keyed by page number and stay valid, and retained history keeps working because old
+generations reach the moved pages through the same directory.
 
 Moves are not written to the WAL. If the engine crashes before the checkpoint that follows a compaction, recovery replays the commits since the previous checkpoint and writes their original addresses back into the directory. Those addresses are in the victims, which are only deleted after that checkpoint, so they still hold the same bytes. The copies made before the crash become unreferenced space in the active segment, and the victims are picked again by a later pass. `CompactionTest` crashes at `COMPACTION_MOVE` and `COMPACTION_RETIRE`, with and without losing the directory writes made since the last checkpoint, and checks the data after reopening and after another compaction.
 
@@ -109,7 +192,8 @@ Moves are not written to the WAL. If the engine crashes before the checkpoint th
 
 Compaction doesn't change any root, so transactions that started before it commit exactly as they would have without it, on the fast path or by rebasing.
 
-The one exception is a bulk load. `Transaction.load` can spill finished subtrees to disk before commit (`Transaction.carriesStoredRoots`), and those pages are not reachable from any generation until the transaction commits, so the walk can't see them. If one of them sits in a victim, it isn't moved. Each transaction therefore records `TransactionManager.compactions()` when it begins, and a transaction that carries stored roots and saw fewer compactions than have since finished fails with a retryable conflict (`bulk-loaded roots of transaction N predate a compaction`). `HypergraphDatabase.write` retries it. The check applies on the fast path too, since compaction no longer changes the roots that would otherwise push such a transaction off it.
+That includes bulk loads. Pages that `Transaction.load` writes before commit already have directory entries, so a
+victim's scan finds and moves them like any other page, and the transaction commits normally afterwards.
 
 ### Reclaim
 
@@ -126,6 +210,8 @@ Retained history no longer holds a segment back. Old generations reach moved pag
 
 `TransactionManager.trimHistory` keeps at most `history_limit` generations (default 64). The current generation and every pinned generation are always kept. An unpinned old generation is dropped even when a newer pinned one exists, so one long-running snapshot keeps its own generation alive without holding back trimming of the generations before it.
 
+History and pins are also what hold dead pages back. A page waits until every generation older than the one it died in has been trimmed and unpinned, so a long-running snapshot delays the release of everything that died after it began, and a larger `history_limit` keeps more dead pages on disk.
+
 ## The change feed on open
 
 `ChangeFeed.open` validates the feed segments frame by frame and truncates the file at the first incomplete or corrupt frame ([catalog-and-generations.md](catalog-and-generations.md#file-format)). Frame headers and bodies are read with loops that continue until the requested bytes have arrived or end of file is reached. A short read in the middle of the file is therefore never mistaken for a torn tail. Recovery then rewrites any missing frames from the WAL's `Feed` records and truncates frames past the recovered generation. After each checkpoint, `feed.retain(current − feedRetention)` deletes whole segments that are no longer needed; see [change-feed.md](change-feed.md).
@@ -145,6 +231,11 @@ Retained history no longer holds a segment back. Old generations reach moved pag
 
 Nested trees are verified recursively and shared images only once.
 
+It then calls `StorageEngine.verifyReferences()`, which counts the references in the current state again from
+scratch and compares every entry of `pages.ref` with the result. It also walks the retained history and checks that
+every page an old generation can reach still has an address and isn't due to be released while that generation is
+kept.
+
 Output from a database loaded with `examples/clinical-claims.hql` (16 KiB maximum node size, packed extents):
 
 ```text
@@ -155,6 +246,7 @@ branch main
   ...
   catalog                 458 entries       86 pages     85 nested trees  height 1
 verified 105 pages: checksums, ordering, balance and summaries are consistent
+reference counts match the trees: 105 pages in use, 423 waiting for older readers
 ```
 
 A failure names the page, for example `HStoreException.corrupt(pageId, "checksum mismatch")` or `"separator does not route child 3 in atoms#1"`, and the command exits non-zero.
@@ -163,7 +255,7 @@ A failure names the page, for example `HStoreException.corrupt(pageId, "checksum
 
 | Setting | Default | Effect |
 |---|---|---|
-| `checkpoint_wal_mb` | 256 | WAL volume since the last checkpoint that triggers a background checkpoint |
+| `checkpoint_wal_mb` | 256 | WAL or page volume since the last checkpoint that triggers a background checkpoint |
 | `history_limit` | 64 | Generations kept for time travel (more while pinned) |
 | `compaction_live_ratio` | 0.5 | A sealed segment is compacted when fewer than this fraction of its images are live |
 | `page_size` | 16384 | Maximum node image size; fixed at `init` |
diff --git a/docs/storage/pages.md b/docs/storage/pages.md
index 908e3b5..2021251 100644
--- a/docs/storage/pages.md
+++ b/docs/storage/pages.md
@@ -36,7 +36,7 @@ Sizes are computed exactly before writing (`varLongSize`, `signedVarLongSize`, `
 | 4 | 1 | `format` | Page format version, `1` (`PageHeader.FORMAT`). Distinct from the data-directory format (see [Format versioning](#format-versioning)). |
 | 5 | 1 | `type` | `PageType`: `1` = `INTERNAL` (branch), `2` = `LEAF` |
 | 6 | 1 | `schema` | `TreeSchema.id` (1–255) of the tree the node belongs to |
-| 7 | 1 | `flags` | Currently always `0` |
+| 7 | 1 | `flags` | Bit 0 (`REFERS_TO_PAGES`): a leaf with at least one value that points at another page, such as a catalog edge with a member tree. `NodeCodec.decode` checks it against the values. The other bits are `0`. |
 | 8 | 8 | `pageId` | The image's own page number (below). It is written last, after allocation (`PageHeader.assign`). |
 | 16 | 8 | `creationEpoch` | Id of the generation whose commit wrote the image. Spilled pages use `head + 1`. |
 | 24 | 4 | `payloadLength` | Payload bytes after the header. Readers use it to size the second read. |
@@ -175,6 +175,43 @@ writes each logged address back into the directory before it checks or replays t
 lost comes back from the log. The header's next index is updated on every allocation, and recovery raises it past
 every number it restores, so a restored number is never handed out a second time.
 
+### Reference counts
+
+`<data>/data/segments/pages.ref` (`page/ReferenceCounts.java`) has the same layout as the directory: a 64-byte
+header (`i32 magic 0x53464552`, `i32 format 1`, `i64` state), then one little-endian `i64` per index at byte
+`64 + 8 × i`, memory-mapped in 8 MiB chunks. An entry is one of:
+
+| Entry | Meaning |
+|---|---|
+| `0` | No count: the number was never used, was released, or belongs to a page nothing has committed yet. |
+| `n > 0` | The page is in use and `n` places point at it: branch roots, branch nodes, or leaf values holding a nested tree. |
+| `0x8000_0000_0000_0000 \| g` (negative) | The page died in generation `g`. Generations before `g` can still reach it, so it keeps its address until it is released. |
+
+The header's state is `0` when the counts can be trusted and `1` when they have to be rebuilt on the next open.
+How the counts change and when pages are released is in [maintenance.md](maintenance.md#reference-counts).
+
+The file is written only at checkpoints, through `<data>/data/segments/pages.ref.journal`
+(`page/ReferenceJournal.java`):
+
+```text
+i32 magic 0x4C4E524A   i32 format 1   i64 checkpoint lsn   i32 count entries (n)   i32 released entries (m)
+n × (i64 index, i64 entry)       the new value of every count changed since the last checkpoint
+m × i64 index                    directory entries to clear
+i32 crc32c of everything above
+```
+
+The journal is written and forced before the catalog is published and applied afterwards, then truncated to zero
+bytes. On open it is applied again only if its LSN is the published checkpoint's, so a half-applied journal is
+finished and one from a checkpoint that never published is ignored.
+
+Page `#1234` in use by two parents, then dead since generation 31:
+
+```text
+xxd -s $((64 + 1234 * 8)) -l 8 -g 8 -e data/segments/pages.ref
+# 000026d0: 0000000000000002   in use, two references
+# 000026d0: 800000000000001f   the same entry after the page died in generation 31
+```
+
 ## Segment files
 
 | Item | Value |
@@ -309,10 +346,12 @@ Because images are never overwritten in place, a torn write can only damage an i
 
 | Version | Where | Meaning |
 |---|---|---|
-| `format=4` in `<data>/FORMAT` | `StorageEngine.verifyFormat` | Page numbers resolved through `pages.dir`, images packed in 64-byte units, image sizes in every stored reference. Directories written by older builds (`format=1` fixed page slots, `format=2` references without sizes, `format=3` references by address) fail to open: `uses storage format 3; this build reads format 4 (page directory); export and reload it`. |
+| `format=5` in `<data>/FORMAT` | `StorageEngine.verifyFormat` | Reference counts in `pages.ref`, page numbers resolved through `pages.dir`, images packed in 64-byte units, image sizes in every stored reference. Directories written by older builds (`format=1` fixed page slots, `format=2` references without sizes, `format=3` references by address, `format=4` no reference counts) fail to open: `uses storage format 4; this build reads format 5 (reference counts); export and reload it`. |
 | `page-size=N` in `<data>/FORMAT` | same | Fixed at creation. A mismatching `--page_size` is rejected. |
 | header byte 4 = `1` | `PageHeader.FORMAT` | Node image layout; unchanged by packing and by page numbers. |
 | `pages.dir` header | `PageDirectory` | `i32 magic 0x52494450`, `i32 format 1`. |
+| `pages.ref` header | `ReferenceCounts` | `i32 magic 0x53464552`, `i32 format 1`. |
+| `pages.ref.journal` header | `ReferenceJournal` | `i32 magic 0x4C4E524A`, `i32 format 1`. |
 | `SegmentInfo` in catalog images | `CatalogImage` | Now `(id, u8 state, pages, units, retiredAt)` |
 
 ## Inspecting images by hand
diff --git a/docs/storage/persistent-tree.md b/docs/storage/persistent-tree.md
index fe74b5a..8bc3c89 100644
--- a/docs/storage/persistent-tree.md
+++ b/docs/storage/persistent-tree.md
@@ -20,7 +20,7 @@ This document describes the node structure, the write path, the summaries and th
 | `TreeCursor` | `TreeCursor.java` | Bidirectional cursor with seek and summary-based pruning. |
 | `TreeAlgebra`, `TreeDiff` | `TreeAlgebra.java`, `TreeDiff.java` | Intersection (leapfrog, synchronized descent, probing), union, difference, subset, equality and structural diff. |
 | `Materializer` | `Materializer.java` | Freezes and encodes pending nodes into pages, post-order. |
-| `TreeWalker`, `TreeVerifier` | | Reachability (liveness for compaction) and invariant checking. |
+| `TreeWalker`, `TreeVerifier` | | The pages a node points at (for reference counting) and invariant checking. |
 
 ## Node structure
 
diff --git a/docs/storage/topology.md b/docs/storage/topology.md
index 6de2322..eff2e45 100644
--- a/docs/storage/topology.md
+++ b/docs/storage/topology.md
@@ -36,7 +36,7 @@ EdgeRecord: u8 tag=1 | varint type | u8 kind (0=SET, 1=ORDERED) | Ref members |
 Ref:        i64le pageId (0 = empty) | [Summary encoding, only when pageId != 0]
 ```
 
-Because `AtomCodec.holdsRefs()` is true, the materializer writes each edge's member and order trees *before* the catalog leaf that references them. `TreeWalker` and `TreeVerifier` descend into them, and compaction moves their pages like any other. The catalog therefore forms a two-level tree of trees: catalog leaves hold edge records, and each edge record roots its own membership tree.
+Because `AtomCodec.holdsRefs()` is true, the materializer writes each edge's member and order trees *before* the catalog leaf that references them. `TreeWalker` and `TreeVerifier` descend into them, so the reference counts and `hstore check` cover them, and compaction moves their pages like any other. The catalog therefore forms a two-level tree of trees: catalog leaves hold edge records, and each edge record roots its own membership tree.
 
 A record's `contentHash()` feeds the catalog's SET fingerprint. The hash of an edge record is computed from `(1, type, kind, members.count, members.fingerprint, version, dataRef, tenant, isolation)`. It is a function of the edge's member *contents*, never of page ids, and compaction never changes page ids anyway, so it leaves catalog fingerprints alone.
 
diff --git a/docs/transactions/transactions.md b/docs/transactions/transactions.md
index fa66a8e..4beaaa3 100644
--- a/docs/transactions/transactions.md
+++ b/docs/transactions/transactions.md
@@ -97,10 +97,10 @@ flowchart LR
     PUB[publication of g+1] -->|history.put, trimHistory| H
 ```
 
-* `snapshot(branch)` and `begin(options)` call `pinCurrent()`: read `current`, increment `pins[id]`.
+* `snapshot(branch)` and `begin(options)` call `pinCurrent()`: read `current`, increment `pins[id]`, then check that `history` still holds that generation. If a trim removed it in between, it unpins and tries again with the new `current`. Publication puts a generation into `history` before it makes it `current`, so this only loops when a reader races a trim.
 * `snapshotAt(g, branch)` pins `g` first and then looks it up in `history`, so a concurrent trim cannot remove it between lookup and pin. If `g` is gone it unpins and throws `INVALID_SCHEMA` (`"generation g is no longer retained"`).
-* `trimHistory()` (run during publication) computes `excess = history.size() - historyLimit` and walks the history oldest first, removing every generation that is neither pinned nor `current` until `excess` reaches 0. Pinned generations are **skipped**, not barriers: a long-running snapshot on an old generation keeps exactly that generation, while newer unpinned ones are still trimmed, so the history size stays bounded by `historyLimit + number of distinct pinned generations`.
-* `oldestPinned()` exposes the oldest pinned generation to maintenance so that compaction never reclaims pages still reachable from a pinned snapshot.
+* `trimHistory()` (run during publication) computes `excess = history.size() - historyLimit` and walks the history oldest first, removing every generation that is neither pinned nor `current` until `excess` reaches 0. After removing one it checks the pins again and puts the generation back if a reader pinned it in the meantime. Pinned generations are **skipped**, not barriers: a long-running snapshot on an old generation keeps exactly that generation, while newer unpinned ones are still trimmed, so the history size stays bounded by `historyLimit + number of distinct pinned generations`.
+* `oldestPinned()` exposes the oldest pinned generation to maintenance so that compaction never deletes a segment a pinned snapshot may still read. `oldestReadable()`, the smaller of that and the oldest retained generation, tells a checkpoint which dead pages it may release ([maintenance](../storage/maintenance.md#releasing-pages)).
 * `generationAsOf(wallTime)` returns the newest retained generation with `wallTime <= t` (used by `AS OF` time travel).
 
 A `Snapshot` is a `View` whose trees are rooted at the pinned branch's root vector; `Snapshot.close()` unpins once (`AtomicBoolean` guard). Retention defaults: `EngineOptions.historyLimit = 64`.
@@ -155,9 +155,11 @@ A footprint is *commutative* iff the edge was a `SET` edge at the base and every
 
 `TreeDiff.diff` uses the frontier algorithm over shared subtrees ([persistent tree](../storage/persistent-tree.md)), so the cost of the overlap check is proportional to the changed region, not the edge cardinality.
 
-### 5.3 Compaction fence
+### 5.3 Compaction and spilled pages
 
-Compaction moves pages without changing any root, so it doesn't affect other commits. A bulk load is different: a transaction that bulk-loaded stored subtrees (`carriesStoredRoots()`: any `EdgeOp` with `EdgeAction.Load`) may have spilled pages that no generation reaches yet, so compaction can't see them and won't move them. Each transaction records `TransactionManager.compactions()` when it begins, and one that carries stored roots is rejected with a conflict if a compaction has finished since, on the fast path as well as the slow one. Retrying rebuilds the subtree.
+Compaction moves pages without changing any root, so it doesn't affect commits. That holds for bulk loads too. The pages a load spills before commit already have page numbers and directory entries, and compaction finds pages by reading through the victim segment and checking each one against the directory, so a spilled page is moved like any other and the transaction commits normally ([maintenance](../storage/maintenance.md#interaction-with-running-transactions)).
+
+The transaction keeps the numbers of the pages it spilled. If it aborts, its commit fails with a conflict or a page budget error, or its request id turns out to have committed already, it hands them to the reference counter, which marks them dead so the next checkpoint releases them. If it commits, any spilled page the committed trees don't reach is marked dead once the commit is counted.
 
 ### 5.4 Page budget
 
@@ -206,7 +208,7 @@ Under `SNAPSHOT` both commits succeed (R replays `Insert(p)` into B, which L nev
 
 | Code | Class | Retryable | Raised by |
 |---|---|---|---|
-| `RETRYABLE_CONFLICT` | `Conflict` | yes | replay preconditions, `verifyEdge`, `validateReads`, compaction fence, concurrent atom id/canonical key |
+| `RETRYABLE_CONFLICT` | `Conflict` | yes | replay preconditions, `verifyEdge`, `validateReads`, concurrent atom id/canonical key |
 | `RETRYABLE_IO` | `TransientIo` | yes | WAL/page/feed I/O failures; a broken commit pipeline (`"the commit pipeline stopped after a durability failure; restart the engine"`) |
 | `ABORTED_RESOURCE_LIMIT` | `ResourceLimit` | no | page budget, quotas |
 | `INVALID_SCHEMA` | `InvalidSchema` | no | caller errors |
diff --git a/docs/transactions/wal-and-recovery.md b/docs/transactions/wal-and-recovery.md
index 6e1fdb8..7c70e48 100644
--- a/docs/transactions/wal-and-recovery.md
+++ b/docs/transactions/wal-and-recovery.md
@@ -12,6 +12,7 @@ Source of truth:
 | Crash points | `engine/src/main/java/io/hstore/engine/txn/CrashPoint.java` |
 | Recovery | `engine/src/main/java/io/hstore/engine/maintenance/Recovery.java` |
 | Checkpoints | `engine/src/main/java/io/hstore/engine/maintenance/Checkpointer.java`, `catalog/CatalogStore.java` |
+| Reference counts | `engine/src/main/java/io/hstore/engine/maintenance/ReferenceCounter.java`, `page/ReferenceCounts.java`, `page/ReferenceJournal.java` |
 | Byte encodings | `engine/src/main/java/io/hstore/engine/page/ByteCursor.java`, `tree/Ref.java`, `tree/Summary.java` |
 | Crash test matrix | `engine/src/test/java/io/hstore/engine/CrashRecoveryTest.java` |
 
@@ -23,12 +24,14 @@ Related: [transactions](transactions.md), [change feed](../storage/change-feed.m
 
 ```text
 <db>/
-  FORMAT                         properties: format=4, page-size=<bytes>
+  FORMAT                         properties: format=5, page-size=<bytes>
   LOCK                           exclusive FileLock held while open
   catalog/manifest               name of the current catalog image file
   catalog/generations/%016x.cat  checkpointed catalog images (3 newest retained)
   data/segments/*.seg            page segments (see pages.md)
   data/segments/pages.dir        page directory: the address of every page number (see pages.md)
+  data/segments/pages.ref        reference count of every page number, as of the last checkpoint
+  data/segments/pages.ref.journal  counts and released entries of a checkpoint, empty once applied
   wal/segments/%016x.wal         WAL segments, named by their starting LSN in hex
   feed/%016x.feed                change feed segments, named by global byte position (see change-feed.md)
 ```
@@ -162,7 +165,7 @@ PAGE < WAL_APPEND < DATA_WRITE < COMMIT_APPEND < DATA_SYNC < WAL_SYNC < CATALOG_
 
 The `Commit` record is appended in a separate call **after** the data pages were written. It is the atomic commit point: a transaction is part of the recovered state if and only if its `Commit` frame is valid in the recovered log and the durability-prefix rule (section 8) admits it. Consequently a crash at any point `>= COMMIT_APPEND` that preserves the operating system's page cache recovers the commit, and a crash at any point `< COMMIT_APPEND` recovers the prior state.
 
-Spilled subtrees (`TransactionManager.spill`, only for bulk loads) are handled earlier, outside the commit lock: the page images are written to their segments **first**, then the `Page`/`PageRef` records are appended, carrying the transaction id but no `Begin`. Recovery attaches them to the transaction's body by id. The data-before-log order matters for checkpoints, which can run while a spill is in progress. A checkpoint reads its LSN *before* it syncs pages and the page directory (section 9), so every spill record below that LSN was appended after its page was written and its directory entry set, and both are made durable before `truncateBefore` drops the record. If the transaction is aborted, `WalRecord.Abort` follows the spill records.
+Spilled subtrees (`TransactionManager.spill`, only for bulk loads) are handled earlier, outside the commit lock: the page images are written to their segments **first**, then the `Page`/`PageRef` records are appended, carrying the transaction id but no `Begin`. Recovery attaches them to the transaction's body by id. The data-before-log order matters for checkpoints, which can run while a spill is in progress. A checkpoint reads its LSN *before* it syncs pages and the page directory (section 9), so every spill record below that LSN was appended after its page was written and its directory entry set, and both are made durable before `truncateBefore` drops the record. If the transaction is aborted, `WalRecord.Abort` follows the spill records and its pages are marked dead for the next checkpoint to release ([maintenance](../storage/maintenance.md#reference-counts)).
 
 ## 6. WAL modes
 
@@ -176,7 +179,7 @@ Every materialized page is logged in full (`Page`). At commit (`SYNC`) only the
 
 Every materialized page is logged as a 47 or 48 byte `PageRef` holding its id, address, length and CRC-32C. The image itself lives only in the data segment. This is safe because of two invariants:
 
-1. **Pages are never overwritten in place while reachable.** The tree is copy-on-write: a commit writes new versions of changed nodes to *freshly allocated* `PageId`s and never modifies a page that any generation, pinned snapshot or retained catalog image can reach. Compaction copies live pages to new ids and only reclaims a segment after a checkpoint has moved the WAL past every record that could reference it ([maintenance](../storage/maintenance.md)). A page number is never handed out twice, and `PageHeader.verify` rejects an image whose stored number differs from the requested one.
+1. **Pages are never overwritten in place while reachable.** The tree is copy-on-write: a commit writes new versions of changed nodes to *freshly allocated* `PageId`s and never modifies a page that any generation, pinned snapshot or retained catalog image can reach. Compaction copies live pages to new addresses under the same numbers and only reclaims a segment after a checkpoint has moved the WAL past every record that could reference it ([maintenance](../storage/maintenance.md)). A page number is never handed out twice, and `PageHeader.verify` rejects an image whose stored number differs from the requested one.
 2. **Data is forced before the log.** `makeDurable` calls `pages.sync()` (every dirty segment file, `force(false)`) and only then `wal.sync()`. When the commit's `Commit` frame is durable, so is every page it references.
 
 Invariant 2 alone is not enough: the OS may write the WAL segment back to disk on its own *before* `pages.sync()` runs, and a power failure at that moment leaves a durable `Commit` frame whose pages are missing. That commit was never acknowledged (acknowledgement happens after `WAL_SYNC` and publication). Recovery detects the situation by verifying every `PageRef` (section 8) and discards the commit together with everything after it.
@@ -231,14 +234,15 @@ sequenceDiagram
     participant R as Recovery
     participant CK as Checkpointer
     SE->>CS: Recovery.discover = loadLatest() or CatalogImage.initial()
-    SE->>PS: open(segments from the image)
+    SE->>PS: open(segments and checkpoint lsn from the image): apply pages.ref.journal if it matches
     SE->>WAL: open: truncate the last segment's torn tail
     SE->>CF: open: truncate the feed's torn tail
     SE->>R: recover(image, wal, pages, feed, codec)
-    R-->>SE: Outcome(image, replayedCommits, discardedTransactions, discardedCommits)
+    R-->>SE: Outcome(image, replayed, discardedTransactions, discardedCommits)
+    SE->>SE: ReferenceCounter.recover: count the replayed generations, scan for unreferenced pages
     SE->>SE: new TransactionManager(recovered current, history)
-    SE->>CK: checkpoint() (publish image, truncate WAL)
-    SE->>SE: compactor.reclaim(), start maintenance thread
+    SE->>CK: checkpoint() (release dead pages, publish image, truncate WAL)
+    SE->>SE: compactor.reclaim(), start the reference counter and maintenance threads
 ```
 
 `Recovery.recover(checkpoint, wal, pages, feed, codec)`:
@@ -246,9 +250,11 @@ sequenceDiagram
 1. **Scan.** Read the WAL from `checkpoint.checkpointLsn()` (may throw `CORRUPT_LOG`, section 3). `Commit` frames are collected in log order; `Abort` drops the transaction's buffered spill records; `Checkpoint` is ignored; every other record is buffered in `pending[txnId]`.
 2. **Filter.** For each commit in log order, remove its body from `pending`. Skip it if `commit.generation <= checkpoint.current().id()` (already in the image). Otherwise write the address of every `Page` and `PageRef` in the body back into the page directory (`PageStore.place`), which also moves the directory's next index past that number. Entries written after the last checkpoint may not have reached the disk; this restores them.
 3. **Durability prefix.** If any earlier commit was discarded, discard this one too. Otherwise check `intact(body, pages)`: for every `PageRef` in the body, read the page, run `PageHeader.verify(page, pageId)` (magic, format, identity, header checksum), and require `page.byteSize() >= length` and `crc32c(page[0, length)) == checksum`. Any exception or mismatch means the page never reached storage; log a warning once (`durable prefix ends before generation g`) and discard this and every later commit. `Page` records need no verification because they carry their own bytes, protected by the frame CRC.
-4. **Redo.** Apply the body in order: `Page` writes the image at the address it carries; `BranchMeta` inserts or updates the branch (keeping its current roots); `Root` sets one slot root of one branch; `Feed` appends the decoded event to the change feed (a no-op if the feed already holds that generation). Build `Generation(commit.generation, commit.wallTime, commit.txnId, max(nextAtom), max(nextTxn), branches)` and append it to the history.
+4. **Redo.** Apply the body's `BranchMeta` records first, then the rest in order. `createBranch` logs a new branch's `Root` records ahead of its `BranchMeta`, so the branch has to exist before its roots are set. `Page` writes the image at the address it carries; `BranchMeta` inserts or updates the branch (keeping its current roots); `Root` sets one slot root of one branch; `Feed` appends the decoded event to the change feed (a no-op if the feed already holds that generation). Build `Generation(commit.generation, commit.wallTime, commit.txnId, max(nextAtom), max(nextTxn), branches)` and append it to the history.
 5. **Seal.** `pages.sync()`; `feed.truncateAfter(recovered generation)` drops feed frames for commits that did not survive (possible under `ASYNC`, or after the prefix rule fired); `feed.sync()`.
-6. **Result.** `CatalogImage(recovered generation, history, segments, wal.end(), feed.size())` and `Outcome(image, replayedCommits, discardedTransactions = pending.size(), discardedCommits)`. `discardedTransactions` counts transactions that wrote records but neither a `Commit` nor an `Abort` (crashed mid-commit, or spilled and then lost to a crash before committing or aborting).
+6. **Result.** `CatalogImage(recovered generation, history, segments, wal.end(), feed.size())` and `Outcome(image, replayed, discardedTransactions = pending.size(), discardedCommits)`, where `replayed` lists the generations redone, in order. `discardedTransactions` counts transactions that wrote records but neither a `Commit` nor an `Abort` (crashed mid-commit, or spilled and then lost to a crash before committing or aborting).
+
+The reference counts in `pages.ref` describe the checkpointed generation. `ReferenceCounter.recover` brings them forward by counting each replayed generation against the one before it, as the counter does after a commit, and then scans the page directory. A page with an address but no count and no death mark was written by a transaction that never committed, or by a commit the prefix rule discarded, so its entry is cleared. If `pages.ref` is marked for rebuilding, the counts are recomputed from the retained generations instead ([maintenance](../storage/maintenance.md#reference-counts)).
 
 The prefix rule is what makes recovery produce *the state after some prefix of the commit order*. Without it, a durable commit `k+1` whose pages survived could be replayed on top of a lost commit `k`, and its root records (which point at pages built on `k`'s trees) would expose a mixture.
 
@@ -262,20 +268,24 @@ LOG:  [recovery] recovered generation 158: replayed 0 commits from lsn 156,490,
 
 ## 9. Checkpoints
 
-`Checkpointer.checkpoint()` runs inside `TransactionManager.exclusive(...)`, which takes the commit lock **and** drains the group committer, so `current == head` and nothing is in flight:
+`Checkpointer.checkpoint()` first calls `pages.checkpoint()` without any lock, so the data written since the last checkpoint is mostly on disk before commits are held up. Steps 1 to 5 then run inside `TransactionManager.exclusive(...)`, which takes the commit lock **and** drains the group committer, so `current == head` and nothing is in flight. Inside that, `ReferenceCounter.pause` counts every published commit that is still queued and keeps the counter paused until step 6 is done.
 
 1. `lsn = wal.end()`. It is read first so that everything logged below it was written before the syncs that follow.
-2. `pages.checkpoint()` (every dirty segment, then the page directory), `feed.sync()` (forces the active feed segment; earlier feed segments were forced when the feed rolled). `history` = retained generations other than `current`, newest first, at most `historyLimit`.
-3. `catalog.publish(CatalogImage(current, history, segments, lsn, feed.size()))`. The image is framed as `i32 magic 0x54414348`, `i32 crc32c(payload)`, `i64 length`, payload; written to `generations/%016x.cat.tmp`, forced, atomically renamed, the directory forced; then `manifest` is rewritten the same way. The three newest `.cat` files are kept; `loadLatest()` falls back to the newest valid file if the manifest is missing or names a corrupt file.
-4. `wal.append(Checkpoint(0, current.id, lsn))`, `wal.sync()`, `wal.truncateBefore(lsn)`.
+2. Dead pages that no retained or pinned generation can reach are released ([maintenance](../storage/maintenance.md#releasing-pages)).
+3. `pages.checkpoint()` (every dirty segment, then the page directory), then `pages.journal(lsn)`: every count changed since the last checkpoint and every released directory entry, as absolute values, go to `pages.ref.journal` behind a header holding `lsn` and with a CRC-32C at the end, and the file is forced. `feed.sync()` (forces the active feed segment; earlier feed segments were forced when the feed rolled). `history` = retained generations other than `current`, newest first, at most `historyLimit`.
+4. `catalog.publish(CatalogImage(current, history, segments, lsn, feed.size()))`. The image is framed as `i32 magic 0x54414348`, `i32 crc32c(payload)`, `i64 length`, payload; written to `generations/%016x.cat.tmp`, forced, atomically renamed, the directory forced; then `manifest` is rewritten the same way. The three newest `.cat` files are kept; `loadLatest()` falls back to the newest valid file if the manifest is missing or names a corrupt file.
+5. `wal.append(Checkpoint(0, current.id, lsn))`, `wal.sync()`, `wal.truncateBefore(lsn)`.
+6. With the commit lock released, `pages.applyJournal()` writes the journaled values into `pages.ref` and `pages.dir`, forces both and truncates the journal to zero bytes. Commits run meanwhile; the counter queues their changes and resumes once the journal is applied.
+
+Because the journal holds absolute values, applying it twice gives the same files. If the process dies after step 4 but before step 6 is finished, the next open finds a journal whose LSN matches the published catalog and applies it again. If it dies before step 4, the journal's LSN doesn't match and it is ignored: `pages.ref` and `pages.dir` were not touched, so they still match the previous catalog. `ReferenceCountTest` crashes at `CHECKPOINT_JOURNAL`, between steps 5 and 6, and checks the counts after reopening.
 
 The checkpoint LSN is the position of the `Checkpoint` record itself, so the next recovery starts there and skips it. Triggers:
 
 | Trigger | Where |
 |---|---|
 | Every open, right after recovery | `StorageEngine` constructor |
-| WAL grew by more than `checkpointWalBytes` (default 256 MiB) since the last checkpoint; checked every 500 ms | `StorageEngine.maintain` |
-| Compaction, after moving pages and before deleting segments | `StorageEngine.compact` |
+| WAL or page writes grew by more than `checkpointWalBytes` (default 256 MiB) since the last checkpoint, or more than 1 048 576 reference counts changed; checked every 500 ms | `StorageEngine.maintain` |
+| Compaction, before choosing victims and again after moving pages, before deleting segments | `StorageEngine.compact` |
 | Clean shutdown | `StorageEngine.close` |
 | On demand | `StorageEngine.checkpoint()`, HQL `CHECKPOINT` |
 

From f53a638fbc1d0a0177220d7bb265ce8c9d957999 Mon Sep 17 00:00:00 2001
From: venkat1701 <thejeastdev@gmail.com>
Date: Sun, 11 Oct 2026 13:14:38 +0530
Subject: [PATCH 07/15] fix(engine): release the database lock when opening
 fails

The lock was only released by close(). If the constructor threw, for example
in the checkpoint at the end of recovery, the same process could not open the
database again.
---
 engine/src/main/java/io/hstore/engine/StorageEngine.java | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/engine/src/main/java/io/hstore/engine/StorageEngine.java b/engine/src/main/java/io/hstore/engine/StorageEngine.java
index 00f68f3..6beef67 100644
--- a/engine/src/main/java/io/hstore/engine/StorageEngine.java
+++ b/engine/src/main/java/io/hstore/engine/StorageEngine.java
@@ -109,7 +109,12 @@ public static StorageEngine open(Path directory, EngineOptions options) {
                 channel.close();
                 throw HStoreException.invalid("database " + directory + " is opened by another process");
             }
-            return new StorageEngine(directory, options, channel, lock);
+            try {
+                return new StorageEngine(directory, options, channel, lock);
+            } catch (RuntimeException | Error failure) {
+                channel.close();
+                throw failure;
+            }
         } catch (IOException e) {
             throw new UncheckedIOException("cannot open database at " + directory, e);
         }

From ac75bec8b43922526e5c5930d65a9f838c74dd37 Mon Sep 17 00:00:00 2001
From: venkat1701 <thejeastdev@gmail.com>
Date: Sun, 11 Oct 2026 13:14:54 +0530
Subject: [PATCH 08/15] fix(engine): trust rebuilt reference counts only after
 the open checkpoints

A rebuild writes the counts of the recovered generation straight into
pages.ref, but the catalog still names the older checkpoint until the open
publishes its own. If the open crashed in between, the next one trusted the
file and counted the replayed commits a second time. A branch dropped since
the checkpoint was then dropped twice, the pages it shared with main were
marked dead, and they were released. The file now stays marked for rebuilding
until that checkpoint is published. A new crash point, CHECKPOINT_CATALOG,
sits just before a checkpoint publishes its catalog.
---
 .../engine/maintenance/Checkpointer.java      |  1 +
 .../engine/maintenance/ReferenceCounter.java  |  2 +-
 .../java/io/hstore/engine/page/PageStore.java |  1 +
 .../hstore/engine/page/ReferenceCounts.java   | 15 ++++++--
 .../java/io/hstore/engine/txn/CrashPoint.java |  1 +
 .../io/hstore/engine/ReferenceCountTest.java  | 36 +++++++++++++++++++
 6 files changed, 52 insertions(+), 4 deletions(-)

diff --git a/engine/src/main/java/io/hstore/engine/maintenance/Checkpointer.java b/engine/src/main/java/io/hstore/engine/maintenance/Checkpointer.java
index 3be0b9d..ec2640c 100644
--- a/engine/src/main/java/io/hstore/engine/maintenance/Checkpointer.java
+++ b/engine/src/main/java/io/hstore/engine/maintenance/Checkpointer.java
@@ -84,6 +84,7 @@ private Result publish() {
                 .sorted((a, b) -> Long.compare(b.id(), a.id()))
                 .limit(historyLimit)
                 .toList();
+        faults.reach(CrashPoint.CHECKPOINT_CATALOG);
         catalog.publish(new CatalogImage(current, history, pages.segments(), lsn, feed.size()));
         wal.append(new WalRecord.Checkpoint(0, current.id(), lsn));
         wal.sync();
diff --git a/engine/src/main/java/io/hstore/engine/maintenance/ReferenceCounter.java b/engine/src/main/java/io/hstore/engine/maintenance/ReferenceCounter.java
index 9996497..8546d1f 100644
--- a/engine/src/main/java/io/hstore/engine/maintenance/ReferenceCounter.java
+++ b/engine/src/main/java/io/hstore/engine/maintenance/ReferenceCounter.java
@@ -369,7 +369,7 @@ private void rebuild(Collection<Generation> retained, Generation current) {
                 walker.references(work.page, work.schema, work.height, retainedOnly);
             }
         }
-        counts.trust();
+        counts.rebuilt();
     }
 
     private void counted(long pageId, TreeSchema<?> schema, int height) {
diff --git a/engine/src/main/java/io/hstore/engine/page/PageStore.java b/engine/src/main/java/io/hstore/engine/page/PageStore.java
index 671bdc1..67e5753 100644
--- a/engine/src/main/java/io/hstore/engine/page/PageStore.java
+++ b/engine/src/main/java/io/hstore/engine/page/PageStore.java
@@ -409,6 +409,7 @@ public void applyJournal() {
         ReferenceJournal.clear(directory.resolve(ReferenceJournal.FILE));
         references.discardChanges();
         releasedCount = 0;
+        references.checkpointed();
     }
 
     public List<SegmentInfo> segments() {
diff --git a/engine/src/main/java/io/hstore/engine/page/ReferenceCounts.java b/engine/src/main/java/io/hstore/engine/page/ReferenceCounts.java
index 10c9446..43b25f5 100644
--- a/engine/src/main/java/io/hstore/engine/page/ReferenceCounts.java
+++ b/engine/src/main/java/io/hstore/engine/page/ReferenceCounts.java
@@ -16,6 +16,7 @@ public final class ReferenceCounts implements AutoCloseable {
 
     private final MappedLongs file;
     private final LongLongMap changed = new LongLongMap(1 << 12);
+    private volatile boolean rebuilt;
 
     private ReferenceCounts(MappedLongs file) {
         this.file = file;
@@ -59,14 +60,22 @@ public boolean trusted() {
     }
 
     public void distrust() {
+        rebuilt = false;
         file.header(STATE_OFFSET, REBUILD);
         file.force();
     }
 
-    public void trust() {
-        file.force();
-        file.header(STATE_OFFSET, TRUSTED);
+    public void rebuilt() {
         file.force();
+        rebuilt = true;
+    }
+
+    void checkpointed() {
+        if (rebuilt) {
+            file.header(STATE_OFFSET, TRUSTED);
+            file.force();
+            rebuilt = false;
+        }
     }
 
     void forEachChange(LongLongMap.Visitor visitor) {
diff --git a/engine/src/main/java/io/hstore/engine/txn/CrashPoint.java b/engine/src/main/java/io/hstore/engine/txn/CrashPoint.java
index e1c53ec..9237401 100644
--- a/engine/src/main/java/io/hstore/engine/txn/CrashPoint.java
+++ b/engine/src/main/java/io/hstore/engine/txn/CrashPoint.java
@@ -10,6 +10,7 @@ public enum CrashPoint {
     CATALOG_PUBLISH,
     COMPACTION_MOVE,
     COMPACTION_RETIRE,
+    CHECKPOINT_CATALOG,
     CHECKPOINT_JOURNAL;
 
     @FunctionalInterface
diff --git a/engine/src/test/java/io/hstore/engine/ReferenceCountTest.java b/engine/src/test/java/io/hstore/engine/ReferenceCountTest.java
index c98a11c..cfa103d 100644
--- a/engine/src/test/java/io/hstore/engine/ReferenceCountTest.java
+++ b/engine/src/test/java/io/hstore/engine/ReferenceCountTest.java
@@ -308,6 +308,42 @@ void countsMarkedForRebuildAreRebuiltOnOpen() throws Exception {
         }
     }
 
+    @Test
+    void anOpenThatCrashesAfterRebuildingTheCountsRebuildsThemAgain() throws Exception {
+        Random random = new Random(13);
+        Map<Long, TreeSet<Long>> model = new HashMap<>();
+        StorageEngine engine = StorageEngine.open(directory, options());
+        for (int round = 0; round < 40; round++) {
+            step(engine, random, model);
+        }
+        Branch side = engine.createBranch("side", Branch.MAIN);
+        engine.checkpoint();
+        engine.dropBranch(side.id());
+        engine.halt();
+        markForRebuild();
+        AtomicBoolean armed = new AtomicBoolean(true);
+        EngineOptions crashing = options().withFaults(point -> {
+            if (point == CrashPoint.CHECKPOINT_CATALOG && armed.compareAndSet(true, false)) {
+                throw new CrashPoint.SimulatedCrash(point);
+            }
+        });
+        assertThrows(CrashPoint.SimulatedCrash.class, () -> StorageEngine.open(directory, crashing));
+        try (StorageEngine recovered = StorageEngine.open(directory, options())) {
+            recovered.verifyReferences();
+            assertModel(recovered, model);
+        }
+    }
+
+    private Path counts() {
+        return directory.resolve("data").resolve("segments").resolve("pages.ref");
+    }
+
+    private void markForRebuild() throws Exception {
+        try (FileChannel file = FileChannel.open(counts(), StandardOpenOption.WRITE)) {
+            file.write(ByteBuffer.allocate(Long.BYTES).order(ByteOrder.LITTLE_ENDIAN).putLong(0, 1), 8);
+        }
+    }
+
     private static List<Incidence> spread(int count) {
         return LongStream.range(0, count).map(i -> i * 7_919).mapToObj(Incidence::of).toList();
     }

From 41dff9aeb5c06f3e7a420d151d457b227934e92b Mon Sep 17 00:00:00 2001
From: venkat1701 <thejeastdev@gmail.com>
Date: Sun, 11 Oct 2026 13:16:14 +0530
Subject: [PATCH 09/15] fix(engine): apply a published journal before writing
 the next one

If a checkpoint failed after publishing its catalog, for example while syncing
the WAL or applying the journal, the engine kept running with that journal not
yet applied. The next checkpoint overwrote it, and a crash before that
checkpoint published left pages.ref a checkpoint behind the catalog. The open
then released every page written in between as an orphan. The next checkpoint
now applies the published journal first. If publishing the catalog itself
fails, it isn't known whether the catalog reached the disk, so the counts are
marked for rebuilding. The crash tests only inject faults on the test thread,
so a background checkpoint can't take the fault meant for the test.
---
 .../engine/maintenance/Checkpointer.java      | 14 ++++++-
 .../java/io/hstore/engine/page/PageStore.java |  7 ++++
 .../io/hstore/engine/ReferenceCountTest.java  | 37 ++++++++++++++++++-
 3 files changed, 56 insertions(+), 2 deletions(-)

diff --git a/engine/src/main/java/io/hstore/engine/maintenance/Checkpointer.java b/engine/src/main/java/io/hstore/engine/maintenance/Checkpointer.java
index ec2640c..0c9cb4d 100644
--- a/engine/src/main/java/io/hstore/engine/maintenance/Checkpointer.java
+++ b/engine/src/main/java/io/hstore/engine/maintenance/Checkpointer.java
@@ -27,6 +27,7 @@ public record Result(long generation, long lsn, int walSegments) {
     private final int historyLimit;
     private volatile long lastLsn;
     private volatile long completed;
+    private long unapplied = -1;
 
     public Checkpointer(TransactionManager transactions, PageStore pages, WriteAheadLog wal, ChangeFeed feed,
                         CatalogStore catalog, ReferenceCounter references, CrashPoint.Injector faults, int historyLimit,
@@ -64,6 +65,7 @@ public Result checkpoint() {
         try {
             faults.reach(CrashPoint.CHECKPOINT_JOURNAL);
             pages.applyJournal();
+            unapplied = -1;
             lastLsn = result.lsn();
             completed++;
             return result;
@@ -73,6 +75,10 @@ public Result checkpoint() {
     }
 
     private Result publish() {
+        if (unapplied >= 0) {
+            pages.settleJournal(unapplied);
+            unapplied = -1;
+        }
         long lsn = wal.end();
         references.release(transactions.oldestReadable());
         pages.checkpoint();
@@ -85,7 +91,13 @@ private Result publish() {
                 .limit(historyLimit)
                 .toList();
         faults.reach(CrashPoint.CHECKPOINT_CATALOG);
-        catalog.publish(new CatalogImage(current, history, pages.segments(), lsn, feed.size()));
+        try {
+            catalog.publish(new CatalogImage(current, history, pages.segments(), lsn, feed.size()));
+        } catch (RuntimeException | Error failure) {
+            pages.references().distrust();
+            throw failure;
+        }
+        unapplied = lsn;
         wal.append(new WalRecord.Checkpoint(0, current.id(), lsn));
         wal.sync();
         wal.truncateBefore(lsn);
diff --git a/engine/src/main/java/io/hstore/engine/page/PageStore.java b/engine/src/main/java/io/hstore/engine/page/PageStore.java
index 67e5753..abcf73f 100644
--- a/engine/src/main/java/io/hstore/engine/page/PageStore.java
+++ b/engine/src/main/java/io/hstore/engine/page/PageStore.java
@@ -412,6 +412,13 @@ public void applyJournal() {
         references.checkpointed();
     }
 
+    public void settleJournal(long lsn) {
+        Path journal = directory.resolve(ReferenceJournal.FILE);
+        ReferenceJournal.read(journal, lsn).ifPresent(this::replay);
+        ReferenceJournal.clear(journal);
+        references.checkpointed();
+    }
+
     public List<SegmentInfo> segments() {
         return List.copyOf(segments.values());
     }
diff --git a/engine/src/test/java/io/hstore/engine/ReferenceCountTest.java b/engine/src/test/java/io/hstore/engine/ReferenceCountTest.java
index cfa103d..95a149b 100644
--- a/engine/src/test/java/io/hstore/engine/ReferenceCountTest.java
+++ b/engine/src/test/java/io/hstore/engine/ReferenceCountTest.java
@@ -27,6 +27,7 @@
 import java.util.Random;
 import java.util.TreeSet;
 import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicReference;
 import java.util.stream.LongStream;
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
@@ -257,8 +258,9 @@ void pagesSpilledByATransactionCutShortByACrashAreReleasedOnOpen() {
     @ValueSource(booleans = {false, true})
     void countsSurviveACrash(boolean duringCheckpoint) {
         AtomicBoolean armed = new AtomicBoolean();
+        Thread test = Thread.currentThread();
         EngineOptions options = options().withFaults(point -> {
-            if (point == CrashPoint.CHECKPOINT_JOURNAL && armed.compareAndSet(true, false)) {
+            if (point == CrashPoint.CHECKPOINT_JOURNAL && Thread.currentThread() == test && armed.compareAndSet(true, false)) {
                 throw new CrashPoint.SimulatedCrash(point);
             }
         });
@@ -334,6 +336,39 @@ void anOpenThatCrashesAfterRebuildingTheCountsRebuildsThemAgain() throws Excepti
         }
     }
 
+    @Test
+    void aCheckpointThatFailsAfterPublishingIsFinishedByTheNextOne() {
+        AtomicReference<CrashPoint> failAt = new AtomicReference<>();
+        Thread test = Thread.currentThread();
+        EngineOptions options = options().withFaults(point -> {
+            if (Thread.currentThread() == test && failAt.compareAndSet(point, null)) {
+                if (point == CrashPoint.CHECKPOINT_JOURNAL) {
+                    throw new IllegalStateException("journal could not be applied");
+                }
+                throw new CrashPoint.SimulatedCrash(point);
+            }
+        });
+        Random random = new Random(17);
+        Map<Long, TreeSet<Long>> model = new HashMap<>();
+        StorageEngine engine = StorageEngine.open(directory, options);
+        for (int round = 0; round < 40; round++) {
+            step(engine, random, model);
+        }
+        engine.checkpoint();
+        for (int round = 0; round < 40; round++) {
+            step(engine, random, model);
+        }
+        failAt.set(CrashPoint.CHECKPOINT_JOURNAL);
+        assertThrows(IllegalStateException.class, engine::checkpoint);
+        failAt.set(CrashPoint.CHECKPOINT_CATALOG);
+        assertThrows(CrashPoint.SimulatedCrash.class, engine::checkpoint);
+        engine.halt();
+        try (StorageEngine recovered = StorageEngine.open(directory, options())) {
+            recovered.verifyReferences();
+            assertModel(recovered, model);
+        }
+    }
+
     private Path counts() {
         return directory.resolve("data").resolve("segments").resolve("pages.ref");
     }

From 05e0bc22e74073e9c4523f8048e5d2192cb26457 Mon Sep 17 00:00:00 2001
From: venkat1701 <thejeastdev@gmail.com>
Date: Sun, 11 Oct 2026 13:16:32 +0530
Subject: [PATCH 10/15] fix(engine): rebuild missing reference counts and start
 over on a blank header

A missing pages.ref was created empty and trusted, so the open released every
page in the directory. It is now marked for rebuilding when the directory
already has pages. The header of a new pages.ref or pages.dir is forced as soon
as it is written, and a header that is still all zeros, left by a crash before
that, is treated as a new file instead of an unknown format.
---
 .../java/io/hstore/engine/page/MappedLongs.java | 15 ++++++++-------
 .../java/io/hstore/engine/page/PageStore.java   |  3 +++
 .../io/hstore/engine/page/ReferenceCounts.java  |  4 ++++
 .../io/hstore/engine/ReferenceCountTest.java    | 17 +++++++++++++++++
 .../hstore/engine/page/PageDirectoryTest.java   | 16 +++++++++++++++-
 5 files changed, 47 insertions(+), 8 deletions(-)

diff --git a/engine/src/main/java/io/hstore/engine/page/MappedLongs.java b/engine/src/main/java/io/hstore/engine/page/MappedLongs.java
index a211349..4786db0 100644
--- a/engine/src/main/java/io/hstore/engine/page/MappedLongs.java
+++ b/engine/src/main/java/io/hstore/engine/page/MappedLongs.java
@@ -26,25 +26,26 @@ final class MappedLongs implements AutoCloseable {
 
     private final Path path;
     private final FileChannel channel;
-    private final boolean fresh;
+    private boolean fresh;
     private volatile MemorySegment[] chunks = new MemorySegment[0];
 
-    private MappedLongs(Path path, FileChannel channel, boolean fresh) {
+    private MappedLongs(Path path, FileChannel channel) {
         this.path = path;
         this.channel = channel;
-        this.fresh = fresh;
     }
 
     static MappedLongs open(Path path, int magic, int format, String what) {
         try {
             FileChannel channel = FileChannel.open(path, StandardOpenOption.CREATE, StandardOpenOption.READ, StandardOpenOption.WRITE);
-            boolean fresh = channel.size() == 0;
-            MappedLongs longs = new MappedLongs(path, channel, fresh);
-            longs.mapThrough(fresh ? 0 : (int) ((channel.size() - 1) >>> CHUNK_SHIFT));
+            long size = channel.size();
+            MappedLongs longs = new MappedLongs(path, channel);
+            longs.mapThrough(size == 0 ? 0 : (int) ((size - 1) >>> CHUNK_SHIFT));
             MemorySegment header = longs.chunks[0];
-            if (fresh) {
+            longs.fresh = header.get(INT, 0) == 0 && header.get(INT, 4) == 0 && header.get(LONG, 8) == 0;
+            if (longs.fresh) {
                 header.set(INT, 0, magic);
                 header.set(INT, 4, format);
+                header.force();
             } else if (header.get(INT, 0) != magic || header.get(INT, 4) != format) {
                 channel.close();
                 throw HStoreException.invalid(what + " " + path + " has an unknown format");
diff --git a/engine/src/main/java/io/hstore/engine/page/PageStore.java b/engine/src/main/java/io/hstore/engine/page/PageStore.java
index abcf73f..8daaf15 100644
--- a/engine/src/main/java/io/hstore/engine/page/PageStore.java
+++ b/engine/src/main/java/io/hstore/engine/page/PageStore.java
@@ -68,6 +68,9 @@ private void restore(Collection<SegmentInfo> known, long checkpointLsn) {
             Path journal = directory.resolve(ReferenceJournal.FILE);
             ReferenceJournal.read(journal, checkpointLsn).ifPresent(this::replay);
             ReferenceJournal.clear(journal);
+            if (references.fresh() && addresses.next() > 1) {
+                references.distrust();
+            }
             Map<Integer, SegmentInfo> byId = new ConcurrentHashMap<>();
             known.forEach(info -> byId.put(info.id(), info));
             try (Stream<Path> listing = Files.list(directory)) {
diff --git a/engine/src/main/java/io/hstore/engine/page/ReferenceCounts.java b/engine/src/main/java/io/hstore/engine/page/ReferenceCounts.java
index 43b25f5..6160726 100644
--- a/engine/src/main/java/io/hstore/engine/page/ReferenceCounts.java
+++ b/engine/src/main/java/io/hstore/engine/page/ReferenceCounts.java
@@ -59,6 +59,10 @@ public boolean trusted() {
         return file.header(STATE_OFFSET) == TRUSTED;
     }
 
+    boolean fresh() {
+        return file.fresh();
+    }
+
     public void distrust() {
         rebuilt = false;
         file.header(STATE_OFFSET, REBUILD);
diff --git a/engine/src/test/java/io/hstore/engine/ReferenceCountTest.java b/engine/src/test/java/io/hstore/engine/ReferenceCountTest.java
index 95a149b..5610100 100644
--- a/engine/src/test/java/io/hstore/engine/ReferenceCountTest.java
+++ b/engine/src/test/java/io/hstore/engine/ReferenceCountTest.java
@@ -18,6 +18,7 @@
 import java.nio.ByteBuffer;
 import java.nio.ByteOrder;
 import java.nio.channels.FileChannel;
+import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.StandardOpenOption;
 import java.util.ArrayList;
@@ -369,6 +370,22 @@ void aCheckpointThatFailsAfterPublishingIsFinishedByTheNextOne() {
         }
     }
 
+    @Test
+    void aMissingCountsFileIsRebuilt() throws Exception {
+        Random random = new Random(19);
+        Map<Long, TreeSet<Long>> model = new HashMap<>();
+        try (StorageEngine engine = StorageEngine.open(directory, options())) {
+            for (int round = 0; round < 60; round++) {
+                step(engine, random, model);
+            }
+        }
+        Files.delete(counts());
+        try (StorageEngine engine = StorageEngine.open(directory, options())) {
+            engine.verifyReferences();
+            assertModel(engine, model);
+        }
+    }
+
     private Path counts() {
         return directory.resolve("data").resolve("segments").resolve("pages.ref");
     }
diff --git a/engine/src/test/java/io/hstore/engine/page/PageDirectoryTest.java b/engine/src/test/java/io/hstore/engine/page/PageDirectoryTest.java
index 13e9b6c..8344c54 100644
--- a/engine/src/test/java/io/hstore/engine/page/PageDirectoryTest.java
+++ b/engine/src/test/java/io/hstore/engine/page/PageDirectoryTest.java
@@ -51,10 +51,24 @@ void aReservedNumberIsNeverHandedOut() {
     @Test
     void aFileInAnotherFormatIsRefused() throws Exception {
         Path file = directory.resolve(PageDirectory.FILE);
-        Files.write(file, new byte[64]);
+        byte[] header = new byte[64];
+        header[0] = 0x2A;
+        Files.write(file, header);
         assertThrows(HStoreException.InvalidSchema.class, () -> PageDirectory.open(file));
     }
 
+    @Test
+    void aFileWhoseHeaderNeverReachedTheDiskStartsOver() throws Exception {
+        Path file = directory.resolve(PageDirectory.FILE);
+        Files.write(file, new byte[1 << 20]);
+        try (PageDirectory pages = PageDirectory.open(file)) {
+            assertEquals(1, pages.allocate());
+        }
+        try (PageDirectory pages = PageDirectory.open(file)) {
+            assertEquals(2, pages.allocate());
+        }
+    }
+
     private static long address(long index) {
         return new PageAddress(1 + (int) (index % 1000), (int) (index % PageAddress.MAX_OFFSET), 1 + (int) (index % 200)).pack();
     }

From 2c13abd84eba607b3176231c893106109c7b9206 Mon Sep 17 00:00:00 2001
From: venkat1701 <thejeastdev@gmail.com>
Date: Sun, 11 Oct 2026 13:16:46 +0530
Subject: [PATCH 11/15] fix(engine): keep spilled pages out of compaction until
 they are logged

A spill writes its pages before it logs them, so for a moment a segment could
look settled while the PageRef records of its newest pages weren't in the WAL
yet. Compaction could then move those pages and retire the segment, and a
recovery that replayed the late records would point the pages back at the
deleted segment. Spilled pages now count as unwritten until their records are
appended.
---
 .../java/io/hstore/engine/page/PageStore.java    | 16 ++++++++++++----
 .../io/hstore/engine/txn/TransactionManager.java | 14 +++++++++-----
 2 files changed, 21 insertions(+), 9 deletions(-)

diff --git a/engine/src/main/java/io/hstore/engine/page/PageStore.java b/engine/src/main/java/io/hstore/engine/page/PageStore.java
index 8daaf15..dd6b394 100644
--- a/engine/src/main/java/io/hstore/engine/page/PageStore.java
+++ b/engine/src/main/java/io/hstore/engine/page/PageStore.java
@@ -254,6 +254,18 @@ private void adjustLive(long address, int sign) {
     }
 
     public void write(long pageId, MemorySegment image) {
+        writeUnsettled(pageId, image);
+        settle(pageId);
+    }
+
+    public void settle(long pageId) {
+        AtomicInteger pending = unwritten.get(PageAddress.segmentOf(locate(pageId)));
+        if (pending != null) {
+            pending.updateAndGet(value -> Math.max(0, value - 1));
+        }
+    }
+
+    public void writeUnsettled(long pageId, MemorySegment image) {
         if (image.byteSize() > pageSize) {
             throw new IllegalArgumentException("page image exceeds page size: " + image.byteSize());
         }
@@ -268,10 +280,6 @@ public void write(long pageId, MemorySegment image) {
             return SegmentFile.open(id, SegmentFile.pathFor(directory, id));
         });
         file.write((long) offset * PageId.UNIT_BYTES, image);
-        AtomicInteger pending = unwritten.get(segment);
-        if (pending != null) {
-            pending.updateAndGet(value -> Math.max(0, value - 1));
-        }
         long end = offset + (long) PageId.unitsFor(image.byteSize());
         segments.computeIfPresent(segment, (_, info) -> info.units() >= end ? info : info.withAllocation(info.pages() + 1, end));
         pagesWritten.incrementAndGet();
diff --git a/engine/src/main/java/io/hstore/engine/txn/TransactionManager.java b/engine/src/main/java/io/hstore/engine/txn/TransactionManager.java
index 51bcdf6..aff30b9 100644
--- a/engine/src/main/java/io/hstore/engine/txn/TransactionManager.java
+++ b/engine/src/main/java/io/hstore/engine/txn/TransactionManager.java
@@ -348,11 +348,15 @@ Ref spill(long txnId, Ref ref, TreeSchema<?> schema, LongConsumer written) {
         List<PageWrite> writes = new ArrayList<>();
         Materializer materializer = materializer(head.get().id() + 1, txnId, records, writes);
         Ref stored = materializer.materialize(ref, schema);
-        writes.forEach(write -> {
-            storage.pages().write(write.pageId(), write.image());
-            written.accept(write.pageId());
-        });
-        storage.wal().append(records);
+        try {
+            writes.forEach(write -> {
+                storage.pages().writeUnsettled(write.pageId(), write.image());
+                written.accept(write.pageId());
+            });
+            storage.wal().append(records);
+        } finally {
+            writes.forEach(write -> storage.pages().settle(write.pageId()));
+        }
         pagesWritten.add(materializer.pagesWritten());
         return stored;
     }

From 848226bca5272c027b48f9761cac21909463d39e Mon Sep 17 00:00:00 2001
From: venkat1701 <thejeastdev@gmail.com>
Date: Sun, 11 Oct 2026 13:16:50 +0530
Subject: [PATCH 12/15] fix(engine): keep a compaction victim whose pages could
 not all be moved

evacuate skipped a directory entry that pointed into the victim without a
valid page header there, and the victim was retired and deleted anyway, so the
page was lost for good. After moving, the victim's live total now has to be
zero. If it isn't, the pass fails and the victim goes back to SEALED.
---
 engine/src/main/java/io/hstore/engine/page/PageStore.java | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/engine/src/main/java/io/hstore/engine/page/PageStore.java b/engine/src/main/java/io/hstore/engine/page/PageStore.java
index dd6b394..4d7df65 100644
--- a/engine/src/main/java/io/hstore/engine/page/PageStore.java
+++ b/engine/src/main/java/io/hstore/engine/page/PageStore.java
@@ -200,6 +200,11 @@ public long evacuate(int segment) {
             unit += pageUnits;
         }
         repoint(moved, count);
+        AtomicLong left = live.get(segment);
+        if (left != null && left.get() > 0) {
+            throw HStoreException.corrupt(PageId.NONE, "segment " + segment + " still holds " + left.get()
+                    + " bytes of pages that could not be read and moved; it is kept");
+        }
         return total;
     }
 

From 611de1b981581a96d4a460cb3f2f3edb39fe1ed8 Mon Sep 17 00:00:00 2001
From: venkat1701 <thejeastdev@gmail.com>
Date: Sun, 11 Oct 2026 13:16:53 +0530
Subject: [PATCH 13/15] fix(engine): keep the reference counter draining after
 an Error

Only runtime exceptions were caught, so an OutOfMemoryError while walking a
tree ended the counter thread. Its queue then filled up and every commit
blocked on it. Errors are now handled like other failures: the counts are
marked for rebuilding and the queue keeps draining.
---
 .../java/io/hstore/engine/maintenance/ReferenceCounter.java     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/engine/src/main/java/io/hstore/engine/maintenance/ReferenceCounter.java b/engine/src/main/java/io/hstore/engine/maintenance/ReferenceCounter.java
index 8546d1f..e563dd8 100644
--- a/engine/src/main/java/io/hstore/engine/maintenance/ReferenceCounter.java
+++ b/engine/src/main/java/io/hstore/engine/maintenance/ReferenceCounter.java
@@ -170,7 +170,7 @@ private void drain() {
                     case Committed(Generation before, Generation after, long[] spilled) -> account(before, after, spilled);
                     case Discarded(long[] spilled) -> orphaned(spilled, generation);
                 }
-            } catch (RuntimeException failure) {
+            } catch (RuntimeException | Error failure) {
                 broken = failure;
                 LOG.log(System.Logger.Level.ERROR, "reference counting stopped; the counts will be rebuilt when the database next opens", failure);
                 counts.distrust();

From 989c6e571a525036496d3d4b44662b036f0eb1aa Mon Sep 17 00:00:00 2001
From: venkat1701 <thejeastdev@gmail.com>
Date: Sun, 11 Oct 2026 13:16:57 +0530
Subject: [PATCH 14/15] fix(server): pin the generation that hstore check walks

Pages are really released now, so a long check could walk a generation whose
pages were freed under it and report corruption that isn't there.
---
 server/src/main/java/io/hstore/server/Check.java | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/server/src/main/java/io/hstore/server/Check.java b/server/src/main/java/io/hstore/server/Check.java
index def3974..65b0cd4 100644
--- a/server/src/main/java/io/hstore/server/Check.java
+++ b/server/src/main/java/io/hstore/server/Check.java
@@ -6,6 +6,7 @@
 import io.hstore.engine.catalog.Slot;
 import io.hstore.engine.maintenance.ReferenceCounter;
 import io.hstore.engine.tree.TreeVerifier;
+import io.hstore.engine.txn.Snapshot;
 
 import java.util.Map;
 
@@ -15,8 +16,13 @@ private Check() {
     }
 
     static String run(StorageEngine engine) {
+        try (Snapshot pinned = engine.snapshot()) {
+            return run(engine, pinned.catalog());
+        }
+    }
+
+    private static String run(StorageEngine engine, Generation generation) {
         StringBuilder report = new StringBuilder();
-        Generation generation = engine.transactions().current();
         TreeVerifier verifier = new TreeVerifier(engine.source());
         report.append("generation ").append(generation.id()).append(", recovered ")
                 .append(engine.recovery().replayedCommits()).append(" commits from the log\n");

From d4135db22ee3e056ad7ff162b82267a204dbfed3 Mon Sep 17 00:00:00 2001
From: venkat1701 <thejeastdev@gmail.com>
Date: Sun, 11 Oct 2026 13:17:00 +0530
Subject: [PATCH 15/15] docs: describe failed checkpoints, count rebuilds and
 kept victims

---
 docs/storage/maintenance.md           | 14 +++++++++++---
 docs/transactions/transactions.md     |  2 +-
 docs/transactions/wal-and-recovery.md |  4 ++--
 3 files changed, 14 insertions(+), 6 deletions(-)

diff --git a/docs/storage/maintenance.md b/docs/storage/maintenance.md
index 2715133..094248a 100644
--- a/docs/storage/maintenance.md
+++ b/docs/storage/maintenance.md
@@ -42,6 +42,8 @@ Every checkpoint is logged, and then the checkpoint listeners run. The database
 8. It appends a `WalRecord.Checkpoint(0, generation, lsn)` and calls `wal.sync()`.
 9. `wal.truncateBefore(lsn)` deletes WAL segment files that end before the checkpoint LSN. They are no longer needed, because the catalog image now covers them.
 10. The commit lock is released here, and commits carry on. `pages.applyJournal()` writes the journaled counts into `pages.ref` and clears the released directory entries, forces both files, and empties the journal. The counter stays paused until this is done, so the counts it writes next start from the applied state.
+
+If a step after the catalog is published fails and the engine keeps running, the journal on disk still belongs to the published catalog. The next checkpoint applies that journal before it writes its own, so the file never falls more than one checkpoint behind the catalog. If publishing the catalog itself fails, it isn't known whether the new catalog reached the disk, so `pages.ref` is marked for rebuilding and the next open recounts.
 11. It remembers `lastLsn = lsn` and resumes the counter.
 
 **Trigger threshold.** `walSinceCheckpoint() = wal.end() − lastLsn` is compared with `checkpoint_wal_mb` (default 256 MiB, `EngineOptions.checkpointWalBytes`). In the default `PAGE_REFERENCES` WAL mode a commit logs about 47 bytes per new node image instead of the image itself. The threshold therefore corresponds to far more committed work than in `PAGE_IMAGES` mode, and replay after a crash stays short.
@@ -124,7 +126,8 @@ the file as of one checkpoint or the next, never a mix. The background thread ch
 otherwise wait a long time to be released.
 
 **On open**, `PageStore.open` applies `pages.ref.journal` if it was written by the checkpoint the catalog points at
-and ignores it otherwise. `ReferenceCounter.recover` then counts the generations recovery replayed from the WAL, in
+and ignores it otherwise. A `pages.ref` that is missing while the directory already has pages is marked for
+rebuilding. `ReferenceCounter.recover` then counts the generations recovery replayed from the WAL, in
 order, exactly as it would have after each commit. Last, one pass over `pages.dir` and `pages.ref`:
 
 * releases every page that has an address but a count of zero and isn't marked dead. No committed generation
@@ -136,7 +139,10 @@ order, exactly as it would have after each commit. Last, one pass over `pages.di
 `pages.ref` for rebuilding. It releases nothing more until the engine restarts, so the worst case is space that
 isn't reclaimed. The next open sees the mark and recounts from the retained generations: every page reachable from
 the current state gets its exact count, and every page only older generations reach is marked dead as of the current
-generation. That walk reads every node of every retained tree, so it takes as long as `hstore check`.
+generation. That walk reads every node of every retained tree, so it takes as long as `hstore check`. The rebuild
+writes `pages.ref` directly but leaves the mark in place until the checkpoint at the end of the open has published
+its catalog, because until then the catalog still describes an older generation. A crash in between rebuilds again.
+A thread error such as running out of memory is handled the same way as a failed count.
 
 ### Releasing pages
 
@@ -174,7 +180,9 @@ moved if their segment is compacted.
    or the old copy of a page that has already moved, is skipped. The header holds the page number, not
    the address, so neither the header nor the checksum changes. Every 4 MiB of copies, the segments are synced and
    only then are the directory entries pointed at the copies, so an entry never reaches the disk pointing at bytes
-   that aren't there yet.
+   that aren't there yet. Afterwards the victim's live total has to be zero. If it isn't, some page the directory
+   points at couldn't be read where it should be, and the victim goes back to `SEALED` instead of being retired, so
+   its file is kept.
 4. Mark the victims `RETIRED` with `retiredAt` set to the current generation.
 
 The work is proportional to the size of the victims, not of the database, and the only memory it needs is a short
diff --git a/docs/transactions/transactions.md b/docs/transactions/transactions.md
index 4beaaa3..56914c7 100644
--- a/docs/transactions/transactions.md
+++ b/docs/transactions/transactions.md
@@ -78,7 +78,7 @@ During replay (`Workspace.replaying() == true`) a failed precondition, a duplica
 
 ### Spilling large workspaces
 
-`Transaction` constructs its `WriteScope` with a `PageSink` that calls `TransactionManager.spill`. Only `BulkBuilder` spills (every `SPILL_BATCH = 32` leaves during `Hyperedge.build`, used by `Transaction.load`), so very large bulk loads do not have to hold every new leaf in memory. `spill` materializes the subtree, **writes the page images to their data segments first**, then appends the matching `Page`/`PageRef` WAL records under the transaction id (before the transaction commits), and returns a `Ref.Stored`. Writing data before logging means that by the time a checkpoint can observe an LSN past the spill records, the pages are already in the page cache, so the checkpoint's `pages.sync()` makes them durable before the records are truncated. The spilled pages count against the transaction's page budget (`spilledPages`).
+`Transaction` constructs its `WriteScope` with a `PageSink` that calls `TransactionManager.spill`. Only `BulkBuilder` spills (every `SPILL_BATCH = 32` leaves during `Hyperedge.build`, used by `Transaction.load`), so very large bulk loads do not have to hold every new leaf in memory. `spill` materializes the subtree, **writes the page images to their data segments first**, then appends the matching `Page`/`PageRef` WAL records under the transaction id (before the transaction commits), and returns a `Ref.Stored`. Writing data before logging means that by the time a checkpoint can observe an LSN past the spill records, the pages are already in the page cache, so the checkpoint's `pages.sync()` makes them durable before the records are truncated. Until the records are appended, the pages still count as unwritten for their segment, so compaction can't move them before the log says where they were written. The spilled pages count against the transaction's page budget (`spilledPages`).
 
 If a transaction that spilled is aborted (explicitly, or by `close()` without commit), `abort()` appends `WalRecord.Abort(txnId)`; recovery then drops its buffered records instead of reporting them as an unfinished transaction. A transaction that spilled and then crashed before `Commit` or `Abort` still appears in `Outcome.discardedTransactions`. In both cases the spilled pages are unreachable garbage reclaimed by compaction (see [maintenance](../storage/maintenance.md)). A commit that throws marks the transaction `ABORTED` directly and does not log `Abort`.
 
diff --git a/docs/transactions/wal-and-recovery.md b/docs/transactions/wal-and-recovery.md
index 7c70e48..05771df 100644
--- a/docs/transactions/wal-and-recovery.md
+++ b/docs/transactions/wal-and-recovery.md
@@ -277,7 +277,7 @@ LOG:  [recovery] recovered generation 158: replayed 0 commits from lsn 156,490,
 5. `wal.append(Checkpoint(0, current.id, lsn))`, `wal.sync()`, `wal.truncateBefore(lsn)`.
 6. With the commit lock released, `pages.applyJournal()` writes the journaled values into `pages.ref` and `pages.dir`, forces both and truncates the journal to zero bytes. Commits run meanwhile; the counter queues their changes and resumes once the journal is applied.
 
-Because the journal holds absolute values, applying it twice gives the same files. If the process dies after step 4 but before step 6 is finished, the next open finds a journal whose LSN matches the published catalog and applies it again. If it dies before step 4, the journal's LSN doesn't match and it is ignored: `pages.ref` and `pages.dir` were not touched, so they still match the previous catalog. `ReferenceCountTest` crashes at `CHECKPOINT_JOURNAL`, between steps 5 and 6, and checks the counts after reopening.
+Because the journal holds absolute values, applying it twice gives the same files. If the process dies after step 4 but before step 6 is finished, the next open finds a journal whose LSN matches the published catalog and applies it again. If it dies before step 4, the journal's LSN doesn't match and it is ignored: `pages.ref` and `pages.dir` were not touched, so they still match the previous catalog. If step 5 or 6 fails and the engine keeps running, the next checkpoint applies the old journal before writing a new one, and a failed step 4 marks `pages.ref` for rebuilding. `ReferenceCountTest` crashes at `CHECKPOINT_CATALOG`, just before step 4, and at `CHECKPOINT_JOURNAL`, between steps 5 and 6, and checks the counts after reopening, including after a checkpoint that failed at step 6 and a next one that crashed before step 4.
 
 The checkpoint LSN is the position of the `Checkpoint` record itself, so the next recovery starts there and skips it. Triggers:
 
@@ -293,7 +293,7 @@ After each completed checkpoint, `StorageEngine.timedCheckpoint` logs `checkpoin
 
 ## 10. What the crash tests prove
 
-`CrashRecoveryTest.recoveryYieldsPriorOrNewStateNeverAMixture` is parameterised over every `WalMode` x every `CrashPoint` (2 x 7 = 14 cases). Each case:
+`CrashRecoveryTest.recoveryYieldsPriorOrNewStateNeverAMixture` is parameterised over every `WalMode` x every commit-path `CrashPoint`, `PAGE` to `CATALOG_PUBLISH` (2 x 7 = 14 cases). Each case:
 
 1. Commits an ordered edge with 200 members (each a fresh node).
 2. Arms a fault injector that throws `CrashPoint.SimulatedCrash` (an `Error`) the first time the chosen point is reached.