Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 20 additions & 2 deletions apps/server/src/persistence/DatabaseStorage.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import * as Layer from "effect/Layer";
import * as Path from "effect/Path";
import * as Option from "effect/Option";
import * as Stream from "effect/Stream";
import * as Tracer from "effect/Tracer";
import * as TestClock from "effect/testing/TestClock";
import * as SqlClient from "effect/sql/SqlClient";

Expand Down Expand Up @@ -44,9 +45,26 @@ it.effect("a new database returns freed pages online without losing data", () =>
yield* sql`CREATE TABLE retained (value TEXT)`;
yield* sql`INSERT INTO retained VALUES ('keep')`;
yield* sql`CREATE TABLE discarded (value BLOB)`;
yield* sql`INSERT INTO discarded VALUES (zeroblob(1048576))`;
yield* sql`INSERT INTO discarded VALUES (zeroblob(6291456))`;
yield* sql`DROP TABLE discarded`;
assert.isAbove(yield* freePages, 0);
const before = yield* freePages;
assert.isAbove(before, 1024);
const spans: Array<Tracer.NativeSpan> = [];
const tracer = Tracer.make({
span(options) {
const span = new Tracer.NativeSpan(options);
spans.push(span);
return span;
},
});
yield* storage.reclaim.pipe(Effect.withTracer(tracer), Effect.withTracerEnabled(true));
assert.equal(before - (yield* freePages), 1024);
assert.lengthOf(spans, 1);
assert.deepEqual(
spans.map((span) => span.name),
["DatabaseStorage.reclaim"],
);
assert.equal(spans[0]?.attributes.get("db.reclaimed_pages"), 1024);
yield* storage.reclaim;
assert.equal(yield* freePages, 0);
assert.deepEqual(yield* sql`SELECT value FROM retained`, [{ value: "keep" }]);
Expand Down
31 changes: 19 additions & 12 deletions apps/server/src/persistence/DatabaseStorage.ts
Original file line number Diff line number Diff line change
Expand Up @@ -117,18 +117,25 @@ const make = Effect.gen(function* () {
Effect.catch((cause) => Effect.logWarning("Database storage inspection failed", { cause })),
);

const reclaim = Effect.gen(function* () {
if (!(yield* readLayout).incrementalVacuum) return;
// The driver steps once per statement; incremental_vacuum frees one page per step. Yield between pages.
for (let page = 0; page < RECLAIM_PAGES_PER_PASS; page++) {
const free = yield* sql<{ readonly freelist_count: number }>`PRAGMA freelist_count`;
if ((free[0]?.freelist_count ?? 0) === 0) break;
yield* sql`PRAGMA incremental_vacuum(1)`;
yield* Effect.yieldNow;
}
}).pipe(
Effect.catch((cause) => Effect.logWarning("Database space reclamation failed", { cause })),
);
const reclaim = Effect.suspend(() => {
let page = 0;
return Effect.gen(function* () {
if (!(yield* readLayout).incrementalVacuum) return;
// The driver steps once per statement; incremental_vacuum frees one page per step. Yield between pages.
for (; page < RECLAIM_PAGES_PER_PASS; page++) {
const free = yield* sql<{ readonly freelist_count: number }>`PRAGMA freelist_count`;
if ((free[0]?.freelist_count ?? 0) === 0) break;
yield* sql`PRAGMA incremental_vacuum(1)`;
yield* Effect.yieldNow;
}
}).pipe(
Effect.ensuring(Effect.suspend(() => Effect.annotateCurrentSpan("db.reclaimed_pages", page))),
Effect.catch((cause) => Effect.logWarning("Database space reclamation failed", { cause })),
// The pass owns the span; tracing each page's two PRAGMAs floods the server trace.
Effect.withTracerEnabled(false),
Effect.withSpan("DatabaseStorage.reclaim"),
);
});

// Even a pass that deletes nothing scans the event store. Persist completion
// so restarting does not trigger another scan before the six-hour interval.
Expand Down
Loading