diff --git a/package.json b/package.json index 3b9de28..2e00886 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "tuple-database", - "version": "2.2.3", + "version": "2.3.0", "description": "", "repository": "github:ccorcos/tuple-database", "main": "./main.js", diff --git a/release-git.sh b/release-git.sh new file mode 100755 index 0000000..7b5b90a --- /dev/null +++ b/release-git.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +# Build the package and commit the output to the `release` branch, whose root is +# installable directly from GitHub: +# "tuple-database": "github:tanishqkancharla/tuple-database#" +set -euo pipefail + +root="$(git rev-parse --show-toplevel)" +cd "$root" + +if [ -n "$(git status --porcelain)" ]; then + echo "Working tree must be clean so the build matches a commit." >&2 + exit 1 +fi +source_commit="$(git rev-parse HEAD)" + +rm -rf build +npm run build +cp README.md build/ +node -e ' +const pkg = require("./package.json") +delete pkg.scripts +delete pkg.devDependencies +require("fs").writeFileSync("build/package.json", JSON.stringify(pkg, null, 2) + "\n") +' +find build -name "*.test.*" -delete +rm -rf build/test build/tools + +worktree="$(mktemp -d)" +trap 'git worktree remove --force "$worktree"' EXIT +if git show-ref --verify --quiet refs/heads/release; then + git worktree add "$worktree" release +else + git worktree add --detach "$worktree" + git -C "$worktree" checkout --orphan release +fi +git -C "$worktree" rm -rf --quiet . 2>/dev/null || true +cp -R build/. "$worktree"/ +git -C "$worktree" add -A +if git -C "$worktree" diff --cached --quiet; then + echo "release branch unchanged: $(git rev-parse refs/heads/release)" + exit 0 +fi +git -C "$worktree" commit --quiet -m "Build $source_commit" +echo "release branch: $(git rev-parse refs/heads/release)" diff --git a/src/database/async/AsyncReactivityTracker.ts b/src/database/async/AsyncReactivityTracker.ts index 7633857..ac0bc77 100644 --- a/src/database/async/AsyncReactivityTracker.ts +++ b/src/database/async/AsyncReactivityTracker.ts @@ -1,5 +1,5 @@ import { maybePromiseAll } from "../../helpers/maybeWaitForPromises" -import { randomId } from "../../helpers/randomId" +import { defaultRng } from "../../helpers/randomId" import { Bounds, getPrefixContainingBounds, @@ -8,14 +8,16 @@ import { import { InMemoryTupleStorage } from "../../storage/InMemoryTupleStorage" import { MIN, ScanStorageArgs, Tuple, WriteOps } from "../../storage/types" import { TupleStorageApi } from "../sync/types" -import { TxId } from "../types" +import { RngApi, TxId } from "../types" import { AsyncCallback } from "./asyncTypes" export class AsyncReactivityTracker { private listenersDb = new InMemoryTupleStorage() + constructor(private rng: RngApi = defaultRng) {} + subscribe(args: ScanStorageArgs, callback: AsyncCallback) { - return subscribe(this.listenersDb, args, callback) + return subscribe(this.listenersDb, this.rng.randomId(), args, callback) } computeReactivityEmits(writes: WriteOps) { @@ -119,6 +121,7 @@ function getReactivityEmits(listenersDb: TupleStorageApi, writes: WriteOps) { function subscribe( listenersDb: TupleStorageApi, + id: string, args: ScanStorageArgs, callback: AsyncCallback ) { @@ -126,7 +129,6 @@ function subscribe( const prefix = getPrefixContainingBounds(args) - const id = randomId() const value: AsyncListener = { callback, bounds: args } listenersDb.commit({ set: [{ key: [prefix, id], value }] }) diff --git a/src/database/async/AsyncTupleDatabase.ts b/src/database/async/AsyncTupleDatabase.ts index ad950c2..1569308 100644 --- a/src/database/async/AsyncTupleDatabase.ts +++ b/src/database/async/AsyncTupleDatabase.ts @@ -1,9 +1,9 @@ import { iterateWrittenTuples } from "../../helpers/iterateTuples" -import { randomId } from "../../helpers/randomId" +import { defaultRng } from "../../helpers/randomId" import { KeyValuePair, ScanStorageArgs, WriteOps } from "../../storage/types" import { ConcurrencyLog } from "../ConcurrencyLog" import { TupleStorageApi } from "../sync/types" -import { TxId, Unsubscribe } from "../types" +import { RngApi, TupleDatabaseOptions, TxId, Unsubscribe } from "../types" import { AsyncReactivityTracker } from "./AsyncReactivityTracker" import { AsyncCallback, @@ -12,10 +12,17 @@ import { } from "./asyncTypes" export class AsyncTupleDatabase implements AsyncTupleDatabaseApi { - constructor(private storage: TupleStorageApi | AsyncTupleStorageApi) {} - + private rng: RngApi log = new ConcurrencyLog() - reactivity = new AsyncReactivityTracker() + reactivity: AsyncReactivityTracker + + constructor( + private storage: TupleStorageApi | AsyncTupleStorageApi, + options: TupleDatabaseOptions = {} + ) { + this.rng = options.rng ?? defaultRng + this.reactivity = new AsyncReactivityTracker(this.rng) + } async scan(args: ScanStorageArgs = {}, txId?: TxId): Promise { const { reverse, limit, ...bounds } = args @@ -40,7 +47,7 @@ export class AsyncTupleDatabase implements AsyncTupleDatabaseApi { } await this.storage.commit(writes) - return this.reactivity.emit(emits, txId || randomId()) + return this.reactivity.emit(emits, txId || this.rng.randomId()) } async cancel(txId: string) { diff --git a/src/database/rng.test.ts b/src/database/rng.test.ts new file mode 100644 index 0000000..e3daf97 --- /dev/null +++ b/src/database/rng.test.ts @@ -0,0 +1,64 @@ +import { strict as assert } from "assert" +import { describe, it } from "mocha" +import { InMemoryTupleStorage } from "../storage/InMemoryTupleStorage" +import { MAX, MIN, ScanStorageArgs } from "../storage/types" +import { AsyncTupleDatabase } from "./async/AsyncTupleDatabase" +import { TupleDatabase } from "./sync/TupleDatabase" +import { RngApi } from "./types" + +const todos: ScanStorageArgs = { gt: ["todos", MIN], lt: ["todos", MAX] } + +function sequenceRng(ids: string[]): RngApi { + let index = 0 + return { randomId: () => ids[index++] } +} + +describe("TupleDatabase rng option", () => { + it("orders listeners on the same prefix by the ids the rng returns", () => { + const db = new TupleDatabase(new InMemoryTupleStorage(), { + rng: sequenceRng(["b", "a", "tx"]), + }) + const calls: string[] = [] + db.subscribe(todos, () => { + calls.push("first") + }) + db.subscribe(todos, () => { + calls.push("second") + }) + + db.commit({ set: [{ key: ["todos", 1], value: null }] }) + + assert.deepEqual(calls, ["second", "first"]) + }) + + it("uses the rng for a commit made without a transaction id", () => { + const db = new TupleDatabase(new InMemoryTupleStorage(), { + rng: sequenceRng(["listener", "tx-1"]), + }) + const txIds: string[] = [] + db.subscribe(todos, (_writes, txId) => { + txIds.push(txId) + }) + + db.commit({ set: [{ key: ["todos", 1], value: null }] }) + + assert.deepEqual(txIds, ["tx-1"]) + }) + + it("threads the rng through the async database", async () => { + const db = new AsyncTupleDatabase(new InMemoryTupleStorage(), { + rng: sequenceRng(["b", "a", "tx-1"]), + }) + const calls: string[] = [] + await db.subscribe(todos, (_writes, txId) => { + calls.push(`first:${txId}`) + }) + await db.subscribe(todos, (_writes, txId) => { + calls.push(`second:${txId}`) + }) + + await db.commit({ set: [{ key: ["todos", 1], value: null }] }) + + assert.deepEqual(calls, ["second:tx-1", "first:tx-1"]) + }) +}) diff --git a/src/database/sync/ReactivityTracker.ts b/src/database/sync/ReactivityTracker.ts index cc54cef..badc88b 100644 --- a/src/database/sync/ReactivityTracker.ts +++ b/src/database/sync/ReactivityTracker.ts @@ -7,7 +7,7 @@ This file is generated from async/AsyncReactivityTracker.ts type Identity = T import { maybePromiseAll } from "../../helpers/maybeWaitForPromises" -import { randomId } from "../../helpers/randomId" +import { defaultRng } from "../../helpers/randomId" import { Bounds, getPrefixContainingBounds, @@ -16,14 +16,16 @@ import { import { InMemoryTupleStorage } from "../../storage/InMemoryTupleStorage" import { MIN, ScanStorageArgs, Tuple, WriteOps } from "../../storage/types" import { TupleStorageApi } from "../sync/types" -import { TxId } from "../types" +import { RngApi, TxId } from "../types" import { Callback } from "./types" export class ReactivityTracker { private listenersDb = new InMemoryTupleStorage() + constructor(private rng: RngApi = defaultRng) {} + subscribe(args: ScanStorageArgs, callback: Callback) { - return subscribe(this.listenersDb, args, callback) + return subscribe(this.listenersDb, this.rng.randomId(), args, callback) } computeReactivityEmits(writes: WriteOps) { @@ -127,6 +129,7 @@ function getReactivityEmits(listenersDb: TupleStorageApi, writes: WriteOps) { function subscribe( listenersDb: TupleStorageApi, + id: string, args: ScanStorageArgs, callback: Callback ) { @@ -134,7 +137,6 @@ function subscribe( const prefix = getPrefixContainingBounds(args) - const id = randomId() const value: Listener = { callback, bounds: args } listenersDb.commit({ set: [{ key: [prefix, id], value }] }) diff --git a/src/database/sync/TupleDatabase.ts b/src/database/sync/TupleDatabase.ts index 5d7a156..1b7438e 100644 --- a/src/database/sync/TupleDatabase.ts +++ b/src/database/sync/TupleDatabase.ts @@ -7,19 +7,26 @@ This file is generated from async/AsyncTupleDatabase.ts type Identity = T import { iterateWrittenTuples } from "../../helpers/iterateTuples" -import { randomId } from "../../helpers/randomId" +import { defaultRng } from "../../helpers/randomId" import { KeyValuePair, ScanStorageArgs, WriteOps } from "../../storage/types" import { ConcurrencyLog } from "../ConcurrencyLog" import { TupleStorageApi } from "../sync/types" -import { TxId, Unsubscribe } from "../types" +import { RngApi, TupleDatabaseOptions, TxId, Unsubscribe } from "../types" import { ReactivityTracker } from "./ReactivityTracker" import { Callback, TupleDatabaseApi } from "./types" export class TupleDatabase implements TupleDatabaseApi { - constructor(private storage: TupleStorageApi) {} - + private rng: RngApi log = new ConcurrencyLog() - reactivity = new ReactivityTracker() + reactivity: ReactivityTracker + + constructor( + private storage: TupleStorageApi, + options: TupleDatabaseOptions = {} + ) { + this.rng = options.rng ?? defaultRng + this.reactivity = new ReactivityTracker(this.rng) + } scan(args: ScanStorageArgs = {}, txId?: TxId): Identity { const { reverse, limit, ...bounds } = args @@ -41,7 +48,7 @@ export class TupleDatabase implements TupleDatabaseApi { } this.storage.commit(writes) - return this.reactivity.emit(emits, txId || randomId()) + return this.reactivity.emit(emits, txId || this.rng.randomId()) } cancel(txId: string) { diff --git a/src/database/types.ts b/src/database/types.ts index 4202521..330d449 100644 --- a/src/database/types.ts +++ b/src/database/types.ts @@ -23,3 +23,12 @@ type AllowMinMax = { export type TxId = string export type Unsubscribe = () => void + +export type RngApi = { + randomId: () => string +} + +export type TupleDatabaseOptions = { + /** Source of listener and fallback transaction ids. Defaults to random UUIDs. */ + rng?: RngApi +} diff --git a/src/helpers/codec.test.ts b/src/helpers/codec.test.ts index a7b10c4..ae7aa71 100644 --- a/src/helpers/codec.test.ts +++ b/src/helpers/codec.test.ts @@ -1,11 +1,11 @@ import { strict as assert } from "assert" -import * as _ from "lodash" import { describe, it } from "mocha" import { Tuple } from "../storage/types" import { sortedValues } from "../test/fixtures" import { decodeTuple, decodeValue, encodeTuple, encodeValue } from "./codec" import { compare } from "./compare" import { TupleToString, ValueToString } from "./compareTuple" +import { randomInt } from "./randomInt" describe("codec", () => { describe("encodeValue", () => { @@ -89,9 +89,11 @@ describe("codec", () => { const test = (aTuple: Tuple, bTuple: Tuple, result: number) => { const a = encodeTuple(aTuple) const b = encodeTuple(bTuple) + const actual = compare(a, b) + const expected = result assert.deepStrictEqual( - compare(a, b), - result, + actual, + expected, `compareTuple(${[TupleToString(aTuple), TupleToString(bTuple)].join( ", " )}) === compare(${[JSON.stringify(a), JSON.stringify(b)].join(", ")})` @@ -120,9 +122,9 @@ describe("codec", () => { const sample = () => { const x = sortedValues.length - const i = _.random(x - 1) - const j = _.random(x - 1) - const k = _.random(x - 1) + const i = randomInt(x - 1) + const j = randomInt(x - 1) + const k = randomInt(x - 1) const tuple: Tuple = [sortedValues[i], sortedValues[j], sortedValues[k]] const rank = i * x * x + j * x + k return { tuple, rank } @@ -136,8 +138,85 @@ describe("codec", () => { } }) }) -}) -function not(x: number) { - return x === 0 ? x : -1 * x -} + describe("codec options", () => { + it("Throws error if a value cannot be encoded", () => { + assert.throws(() => encodeValue("a\x00b", { disallow: ["\x00"] })) + }) + + it("Encodes and decodes with custom delimiter and escape characters", () => { + const options = { delimiter: ":", escape: "\\", disallow: [] } + const testCases = [ + "simple", + "with:delimiter", + "with\\escape", + "with\\:both", + "multiple::delimiters", + "multiple\\\\escapes", + "mixed\\::\\\\:cases", + ] + + for (const value of testCases) { + const encoded = encodeValue(value, options) + const decoded = decodeValue(encoded, options) + + assert.deepStrictEqual( + decoded, + value, + `Failed with custom options ${JSON.stringify(options, undefined, 2)}\n` + + [ + ValueToString(value), + ValueToString(encoded), + ValueToString(decoded), + ].join(" -> ") + ) + } + }) + + it("Handles all encoding options configurations", () => { + const options = { delimiter: "\x01", escape: "\x02", disallow: ["\x00"] } + + const testCases = [ + "normal string", + "with spaces", + "with,punctuation!", + "with\nnewline", + "with\ttab", + ] + + for (const value of testCases) { + const encoded = encodeValue(value, options) + const decoded = decodeValue(encoded, options) + + assert.deepStrictEqual( + decoded, + value, + `Failed with custom options ${JSON.stringify(options, undefined, 2)}\n` + + [ + ValueToString(value), + ValueToString(encoded), + ValueToString(decoded), + ].join(" -> ") + ) + } + }) + + it("Maintains proper escaping with nested delimiters", () => { + const options = { delimiter: ":", escape: "\\", disallow: [] } + const complexValue = "a:b\\:c\\\\:d" + const encoded = encodeValue(complexValue, options) + const decoded = decodeValue(encoded, options) + + assert.deepStrictEqual( + decoded, + complexValue, + `Failed with custom options ${JSON.stringify(options, undefined, 2)}\n` + + [ + ValueToString(complexValue), + ValueToString(encoded), + ValueToString(decoded), + ].join(" -> ") + ) + }) + }) +}) diff --git a/src/helpers/codec.ts b/src/helpers/codec.ts index 431d6c2..e915de1 100644 --- a/src/helpers/codec.ts +++ b/src/helpers/codec.ts @@ -6,6 +6,29 @@ import { Tuple, Value } from "../storage/types" import { compare } from "./compare" import { UnreachableError } from "./Unreachable" +const defaultOptions: Required = { + delimiter: "\x00", + escape: "\x01", + disallow: [], +} + +export type CodecOptions = { + /** + * The delimiter between values in a tuple. + * @default "\x00" + */ + delimiter?: string + /** + * The escape byte. + * @default "\x01" + */ + escape?: string + /** + * Disallowed characters in strings. Throws an error if any are found while encoding. + */ + disallow?: string[] +} + // null < object < array < number < string < boolean export const encodingByte = { null: "b", @@ -20,10 +43,10 @@ export type EncodingType = keyof typeof encodingByte export const encodingRank = sortBy( Object.entries(encodingByte), - ([key, value]) => value + ([_key, value]) => value ).map(([key]) => key as EncodingType) -export function encodeValue(value: Value): string { +export function encodeValue(value: Value, options?: CodecOptions): string { if (value === null) { return encodingByte.null } @@ -31,16 +54,21 @@ export function encodeValue(value: Value): string { return encodingByte.boolean + value } if (typeof value === "string") { + for (const disallowed of options?.disallow ?? defaultOptions.disallow) { + if (value.includes(disallowed)) { + throw new Error(`Disallowed character found: ${disallowed}.`) + } + } return encodingByte.string + value } if (typeof value === "number") { return encodingByte.number + elen.encode(value) } if (Array.isArray(value)) { - return encodingByte.array + encodeTuple(value) + return encodingByte.array + encodeTuple(value, options) } if (typeof value === "object") { - return encodingByte.object + encodeObjectValue(value) + return encodingByte.object + encodeObjectValue(value, options) } throw new UnreachableError(value, "Unknown value type") } @@ -71,7 +99,7 @@ const decodeType = invert(encodingByte) as { [key: string]: keyof typeof encodingByte } -export function decodeValue(str: string): Value { +export function decodeValue(str: string, options?: CodecOptions): Value { const encoding: EncodingType = decodeType[str[0]] const rest = str.slice(1) @@ -88,62 +116,75 @@ export function decodeValue(str: string): Value { return elen.decode(rest) } if (encoding === "array") { - return decodeTuple(rest) + return decodeTuple(rest, options) } if (encoding === "object") { - return decodeObjectValue(rest) + return decodeObjectValue(rest, options) } throw new UnreachableError(encoding, "Invalid encoding byte") } -export function encodeTuple(tuple: Tuple) { +export function encodeTuple(tuple: Tuple, options?: CodecOptions) { + const delimiter = options?.delimiter ?? defaultOptions.delimiter + const escape = options?.escape ?? defaultOptions.escape + const reEscapeByte = new RegExp(escape, "g") + const reDelimiterByte = new RegExp(delimiter, "g") return tuple .map((value, i) => { - const encoded = encodeValue(value) + const encoded = encodeValue(value, options) return ( encoded // B -> BB or \ -> \\ - .replace(/\x01/g, "\x01\x01") + .replace(reEscapeByte, escape + escape) // A -> BA or x -> \x - .replace(/\x00/g, "\x01\x00") + "\x00" + .replace(reDelimiterByte, escape + delimiter) + delimiter ) }) .join("") } -export function decodeTuple(str: string) { +export function decodeTuple(str: string, options?: CodecOptions) { if (str === "") { return [] } + + const delimiter = options?.delimiter ?? defaultOptions.delimiter + const escape = options?.escape ?? defaultOptions.escape + // Capture all of the escaped BB and BA pairs and wait // til we find an exposed A. - const re = /(\x01(\x01|\x00)|\x00)/g + const matcher = new RegExp( + `(${escape}(${escape}|${delimiter})|${delimiter})`, + "g" + ) + const reEncodedEscape = new RegExp(escape + escape, "g") + const reEncodedDelimiter = new RegExp(escape + delimiter, "g") const tuple: Tuple = [] let start = 0 while (true) { - const match = re.exec(str) + const match = matcher.exec(str) if (match === null) { return tuple } - if (match[0][0] === "\x01") { - // If we match a \x01\x01 or \x01\x00 then keep going. + if (match[0][0] === escape) { + // If we match a escape+escape or escape+delimiter then keep going. continue } const end = match.index const escaped = str.slice(start, end) const unescaped = escaped // BB -> B - .replace(/\x01\x01/g, "\x01") + .replace(reEncodedEscape, escape) // BA -> A - .replace(/\x01\x00/g, "\x00") - const decoded = decodeValue(unescaped) + .replace(reEncodedDelimiter, delimiter) + const decoded = decodeValue(unescaped, options) tuple.push(decoded) // Skip over the \x00. start = end + 1 } } -function encodeObjectValue(obj: object) { +function encodeObjectValue(obj: object, options?: CodecOptions) { if (!isPlainObject(obj)) { throw new Error("Cannot serialize this object.") } @@ -152,11 +193,11 @@ function encodeObjectValue(obj: object) { // We allow undefined values in objects, but we want to strip them out before // serializing. .filter(([key, value]) => value !== undefined) - return encodeTuple(entries as Tuple) + return encodeTuple(entries as Tuple, options) } -function decodeObjectValue(str: string) { - const entries = decodeTuple(str) as Array<[string, Value]> +function decodeObjectValue(str: string, options?: CodecOptions) { + const entries = decodeTuple(str, options) as Array<[string, Value]> const obj = {} for (const [key, value] of entries) { obj[key] = value diff --git a/src/helpers/randomId.ts b/src/helpers/randomId.ts index e71de8b..65f5454 100644 --- a/src/helpers/randomId.ts +++ b/src/helpers/randomId.ts @@ -1,6 +1,7 @@ import { chunk } from "lodash" import * as uuid from "uuid" import md5 from "md5" +import { RngApi } from "../database/types" export function randomId(seed?: string): string { if (seed) { @@ -11,3 +12,5 @@ export function randomId(seed?: string): string { return uuid.v4() } } + +export const defaultRng: RngApi = { randomId: () => randomId() } diff --git a/src/helpers/randomInt.ts b/src/helpers/randomInt.ts new file mode 100644 index 0000000..cc6851a --- /dev/null +++ b/src/helpers/randomInt.ts @@ -0,0 +1,3 @@ +export function randomInt(ceil: number): number { + return Math.floor(Math.random() * ceil) +}