diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7c772178..1b341a9b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,6 +31,8 @@ jobs: - name: Checkout code uses: actions/checkout@v6 + with: + fetch-depth: 0 - name: Install Go uses: actions/setup-go@v6 diff --git a/cmd/fraction/README.md b/cmd/fraction/README.md index fe98d8fa..3015f7a5 100644 --- a/cmd/fraction/README.md +++ b/cmd/fraction/README.md @@ -49,7 +49,7 @@ requested alone, token records are written without the `lids` arrays | `kind` | Content | |------------|-----------------------------------------------------------------------------| | `info` | fraction metadata (`common.Info` fields, flattened) | -| `doc` | document text with its LID; the system document (lid 0) is not included | +| `doc` | document text | | `id` | every fraction ID: LID, MID, RID and the document position (DocPos) | | `token` | inverted index: TID, field, token, frequency and the LID list (postings) | | `offsets` | doc block offsets in the documents file, single record | @@ -60,8 +60,8 @@ Records look like this: ```json {"kind":"info","name":"frac_000001","docs_total":3,"index_on_disk":888,...} -{"kind":"doc","lid":1,"doc":"{\"level\":\"error\",\"service\":\"api\",\"message\":\"db timeout\"}"} -{"kind":"id","lid":1,"mid":1704103202000000000,"rid":6530305922915827712,"pos":5} +{"kind":"doc","doc":"{\"level\":\"error\",\"service\":\"api\",\"message\":\"db timeout\"}"} +{"kind":"id","mid":1704103202000000000,"rid":6530305922915827712,"pos":5} {"kind":"token","tid":5,"field":"level","token":"error","freq":1,"lids":[1]} {"kind":"offsets","values":[0,132,264]} ``` diff --git a/cmd/fraction/decoding_suites_test.go b/cmd/fraction/decoding_suites_test.go new file mode 100644 index 00000000..cf456395 --- /dev/null +++ b/cmd/fraction/decoding_suites_test.go @@ -0,0 +1,187 @@ +package main + +// The suites run the same decode checks against fractions from different +// sources: sealed on the fly by the current code (default) or sealed by +// the version's own code for every legacy format version +// (see testdata/legacy/seal-fraction.sh). + +import ( + "fmt" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + + "github.com/ozontech/seq-db/config" +) + +// FractionDecoderTestSuite holds the decode checks. The source of the +// fraction is decided by the embedded suite: the current-version suite +// seals it with the local code, the legacy one seals it per version with +// the version's code via testdata/legacy/seal-fraction.sh. +type FractionDecoderTestSuite struct { + suite.Suite + + fracName string + expectedVer config.BinaryDataVersion +} + +func (s *FractionDecoderTestSuite) sealDocs(docsPerDocBlock int) { + s.fracName = filepath.Join(s.T().TempDir(), "frac_000001") + err := sealFraction( + s.fracName, + "", + strings.NewReader(strings.Join(docLines(s.T(), testDocs), "\n")), + docsPerDocBlock, + ) + s.Require().NoError(err) + s.expectedVer = config.CurrentFracVersion +} + +func (s *FractionDecoderTestSuite) sealDocsVersion(v config.BinaryDataVersion) { + s.fracName = filepath.Join(s.T().TempDir(), fmt.Sprintf("frac_v%d", v)) + + cmd := exec.Command("bash", "testdata/legacy/seal-fraction.sh", fmt.Sprintf("v%d", v), s.fracName) + cmd.Stdin = strings.NewReader(strings.Join(docLines(s.T(), testDocs), "\n")) + out, err := cmd.CombinedOutput() + s.Require().NoError(err, "seal-fraction.sh failed:\n%s", out) + + s.expectedVer = v +} + +func (s *FractionDecoderTestSuite) decodeAll() *collectedContent { + content := &collectedContent{t: s.T()} + err := decodeFraction(s.fracName, "", content) + s.Require().NoError(err) + return content +} + +func (s *FractionDecoderTestSuite) TestDecode() { + s.Run("info", s.checkInfo) + s.Run("docs keep raw json and skip the system doc", s.checkDocs) + s.Run("ids have contiguous lids", s.checkIDs) + s.Run("tokens are sequential and sorted with correct freq and postings", s.checkTokens) + s.Run("tokens without postings when requested alone", s.checkTokensWithoutPostings) + s.Run("only selects sections", s.checkOnlyFiltersSections) +} + +func (s *FractionDecoderTestSuite) checkInfo() { + content := s.decodeAll() + info := content.Info + s.Require().NotNil(info) + + // the decoder must recognize the format version, not just happen to + // read the files + s.Equal(s.expectedVer, info.BinaryDataVer) + + s.Equal(uint32(3), info.DocsTotal) + s.Positive(info.DocsOnDisk) + s.Less(uint64(info.From), uint64(info.To)) +} + +func (s *FractionDecoderTestSuite) checkDocs() { + content := s.decodeAll() + + s.Require().Len(content.Docs, 3) + s.ElementsMatch(docLines(s.T(), testDocs), docTexts(content.Docs)) +} + +func (s *FractionDecoderTestSuite) checkIDs() { + content := s.decodeAll() + + s.Require().Len(content.IDs, 4) + for i, id := range content.IDs { + s.Equal(uint32(i), id.LID) + } +} + +func (s *FractionDecoderTestSuite) checkTokens() { + content := s.decodeAll() + + s.Require().Len(content.Tokens, len(testTokens)) + for i, tok := range content.Tokens { + expected := testTokens[i] + expected.TID = uint32(i + 1) + expected.Kind = kindToken + expected.LIDs = tok.LIDs + s.Equal(expected, tok) + + s.Len(tok.LIDs, int(tok.Freq)) + for _, lid := range tok.LIDs { + s.NotZero(lid) + s.LessOrEqual(lid, uint32(3)) + } + } +} + +func (s *FractionDecoderTestSuite) checkTokensWithoutPostings() { + content := &collectedContent{t: s.T()} + err := decodeFraction(s.fracName, kindToken, content) + s.Require().NoError(err) + + s.Require().Len(content.Tokens, len(testTokens)) + for i, tok := range content.Tokens { + s.Equal(testTokens[i].Token, tok.Token) + s.Equal(testTokens[i].Freq, tok.Freq) + s.Empty(tok.LIDs) + } +} + +func (s *FractionDecoderTestSuite) checkOnlyFiltersSections() { + content := &collectedContent{t: s.T()} + err := decodeFraction(s.fracName, kindInfo+","+kindOffsets, content) + s.Require().NoError(err) + + s.NotNil(content.Info) + s.NotNil(content.Offsets) + s.Empty(content.Docs) + s.Empty(content.Tokens) + s.Empty(content.IDs) +} + +type CurrentVersionSuite struct { + FractionDecoderTestSuite +} + +func (s *CurrentVersionSuite) SetupTest() { + s.sealDocs(defaultDocsPerDocBlock) +} + +// LegacyVersionsSuite runs the same checks against a fraction sealed by +// the version's own code. The suite is instantiated per version by +// TestFractionDecoderLegacy; a future v7 in config/frac_version.go +// automatically adds v6 there. Versions older than v2 are not +// discoverable and are skipped. +type LegacyVersionsSuite struct { + FractionDecoderTestSuite + + Version config.BinaryDataVersion +} + +func (s *LegacyVersionsSuite) SetupTest() { + s.sealDocsVersion(s.Version) +} + +func TestFractionDecoderCurrent(t *testing.T) { + suite.Run(t, new(CurrentVersionSuite)) +} + +func TestFractionDecoderLegacy(t *testing.T) { + for ver := config.BinaryDataV2; ver < config.CurrentFracVersion; ver++ { + t.Run(fmt.Sprintf("v%d", ver), func(t *testing.T) { + suite.Run(t, &LegacyVersionsSuite{Version: config.BinaryDataVersion(ver)}) + }) + } +} + +func TestUnknownOnlySection(t *testing.T) { + fracName := filepath.Join(t.TempDir(), "frac_000001") + + err := decodeFraction(fracName, "amogus", &collectedContent{}) + require.Error(t, err) + assert.Contains(t, err.Error(), "unknown section to decode: amogus") +} diff --git a/cmd/fraction/decoding_test.go b/cmd/fraction/sealing_test.go similarity index 51% rename from cmd/fraction/decoding_test.go rename to cmd/fraction/sealing_test.go index 6dc62206..ae33f2ed 100644 --- a/cmd/fraction/decoding_test.go +++ b/cmd/fraction/sealing_test.go @@ -15,10 +15,11 @@ import ( "github.com/ozontech/seq-db/frac/common" ) -// sealAndDecode seals docs (one JSON document per element, empty elements are -// skipped just like on stdin) into a fresh fraction in a temp dir using the -// mapping at mappingPath (empty string means the built-in default) and decodes -// the requested sections back into a collected form for assertions. +// sealAndDecode seals docs (one JSON document per element, empty elements +// are skipped just like on stdin) into a fresh fraction in a temp dir +// using the mapping at mappingPath (empty string means the built-in +// default) and decodes the requested sections back into a collected form +// for assertions. func sealAndDecode( t *testing.T, mappingPath string, @@ -29,8 +30,9 @@ func sealAndDecode( t.Helper() fracName := filepath.Join(t.TempDir(), "frac_000001") + r := strings.NewReader(strings.Join(docs, "\n")) - err := sealFraction(fracName, mappingPath, strings.NewReader(strings.Join(docs, "\n")), docsPerDocBlock) + err := sealFraction(fracName, mappingPath, r, docsPerDocBlock) require.NoError(t, err) for _, suffix := range []string{ @@ -135,153 +137,52 @@ var testTokens = []tokenRecord{ {Field: "service", Token: "billing", Freq: 1}, } -func TestSealAndDecode(t *testing.T) { - tests := []struct { - name string - docs []string - docsPerDocBlock int - only string - check func(t *testing.T, content *collectedContent) - }{ - { - name: "info", - docsPerDocBlock: defaultDocsPerDocBlock, - only: "info", - check: func(t *testing.T, content *collectedContent) { - require.NotNil(t, content.Info) - assert.Equal(t, uint32(3), content.Info.DocsTotal) - assert.Positive(t, content.Info.DocsOnDisk) - assert.Less(t, uint64(content.Info.From), uint64(content.Info.To)) - }, - }, - { - name: "docs keep raw json and skip the system doc", - docsPerDocBlock: defaultDocsPerDocBlock, - only: kindDoc, - check: func(t *testing.T, content *collectedContent) { - require.Len(t, content.Docs, 3) - assert.ElementsMatch(t, docLines(t, testDocs), docTexts(content.Docs)) - }, - }, - { - name: "tokens are sequential and sorted with correct freq and postings", - docsPerDocBlock: defaultDocsPerDocBlock, - only: kindID + "," + kindToken, - check: func(t *testing.T, content *collectedContent) { - require.Len(t, content.Tokens, len(testTokens)) - for i, tok := range content.Tokens { - expected := testTokens[i] - expected.TID = uint32(i + 1) - expected.Kind = kindToken - expected.LIDs = tok.LIDs - assert.Equal(t, expected, tok) - } - for _, tok := range content.Tokens { - assert.Len(t, tok.LIDs, int(tok.Freq)) - for _, lid := range tok.LIDs { - assert.NotZero(t, lid) - assert.LessOrEqual(t, lid, uint32(3)) - } - } - }, - }, - { - name: "tokens without postings when requested alone", - docsPerDocBlock: defaultDocsPerDocBlock, - only: kindToken, - check: func(t *testing.T, content *collectedContent) { - require.Len(t, content.Tokens, len(testTokens)) - for i, tok := range content.Tokens { - assert.Equal(t, testTokens[i].Token, tok.Token) - assert.Equal(t, testTokens[i].Freq, tok.Freq) - assert.Empty(t, tok.LIDs) - } - }, - }, - { - name: "offsets of a single doc block", - docsPerDocBlock: defaultDocsPerDocBlock, - only: kindOffsets, - check: func(t *testing.T, content *collectedContent) { - require.NotNil(t, content.Offsets) - assert.Equal(t, []uint64{0}, content.Offsets.Values) - }, - }, - { - name: "only selects sections", - docsPerDocBlock: defaultDocsPerDocBlock, - only: kindInfo + "," + kindOffsets, - check: func(t *testing.T, content *collectedContent) { - require.NotNil(t, content.Info) - require.NotNil(t, content.Offsets) - assert.Empty(t, content.Docs) - assert.Empty(t, content.Tokens) - assert.Empty(t, content.IDs) - }, - }, - { - name: "empty lines are skipped", - docsPerDocBlock: defaultDocsPerDocBlock, - docs: []string{ - `{"timestamp":"2024-01-01T10:00:00.000Z","level":"info","message":"first"}`, - "", - `{"timestamp":"2024-01-01T10:00:01.000Z","level":"info","message":"second"}`, - }, - only: kindInfo + "," + kindDoc, - check: func(t *testing.T, content *collectedContent) { - require.NotNil(t, content.Info) - assert.Equal(t, uint32(2), content.Info.DocsTotal) - assert.Len(t, content.Docs, 2) - }, - }, - { - name: "docs are split into several blocks", - docs: generateDocs(5), - docsPerDocBlock: 2, - only: kindInfo + "," + kindDoc + "," + kindID + "," + kindToken + "," + kindOffsets, - check: func(t *testing.T, content *collectedContent) { - // 5 docs with 2 docs per block: [2][2][1] - require.Len(t, content.Offsets.Values, 3) - assert.Equal(t, uint32(5), content.Info.DocsTotal) - - // all docs are decoded, ids have contiguous lids across block boundaries - require.Len(t, content.Docs, 5) - require.Len(t, content.IDs, 6) // +1 for the system id - for i, id := range content.IDs { - assert.Equal(t, uint32(i), id.LID) - } - - // every posting lid points to an existing doc - for _, tok := range content.Tokens { - assert.Len(t, tok.LIDs, int(tok.Freq)) - for _, lid := range tok.LIDs { - assert.NotZero(t, lid) - assert.LessOrEqual(t, lid, uint32(5)) - } - } - - // every doc position points into a real doc block, and every - // block is used: doc indexers assign block indexes concurrently, - // so only the counts per block are guaranteed, not the order - blocksPerLID := map[uint32]int{} - for _, id := range content.IDs[1:] { // system id is not positioned - assert.Less(t, id.BlockIndex, uint32(len(content.Offsets.Values)), "block index out of range") - blocksPerLID[id.BlockIndex]++ - } - assert.Equal(t, map[uint32]int{0: 2, 1: 2, 2: 1}, blocksPerLID) - }, - }, +func TestSealSkipsEmptyLines(t *testing.T) { + content := sealAndDecode(t, "", defaultDocsPerDocBlock, kindInfo+","+kindDoc, + `{"timestamp":"2024-01-01T10:00:00.000Z","level":"info","message":"first"}`, + "", + `{"timestamp":"2024-01-01T10:00:01.000Z","level":"info","message":"second"}`, + ) + + require.NotNil(t, content.Info) + assert.Equal(t, uint32(2), content.Info.DocsTotal) + assert.Len(t, content.Docs, 2) +} + +func TestSealSplitsIntoDocBlocks(t *testing.T) { + content := sealAndDecode(t, "", 2, kindInfo+","+kindDoc+","+kindID+","+kindToken+","+kindOffsets, + generateDocs(5)..., + ) + + // 5 docs with 2 docs per block: [2][2][1] + require.Len(t, content.Offsets.Values, 3) + assert.Equal(t, uint32(5), content.Info.DocsTotal) + + // all docs are decoded, ids have contiguous lids across block boundaries + require.Len(t, content.Docs, 5) + require.Len(t, content.IDs, 6) // +1 for the system id + for i, id := range content.IDs { + assert.Equal(t, uint32(i), id.LID) } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - docs := tt.docs - if docs == nil { - docs = docLines(t, testDocs) - } - tt.check(t, sealAndDecode(t, "", tt.docsPerDocBlock, tt.only, docs...)) - }) + // every posting lid points to an existing doc + for _, tok := range content.Tokens { + assert.Len(t, tok.LIDs, int(tok.Freq)) + for _, lid := range tok.LIDs { + assert.NotZero(t, lid) + assert.LessOrEqual(t, lid, uint32(5)) + } } + + // every doc position points into a real doc block, and every block is + // used: doc indexers assign block indexes concurrently, so only the + // counts per block are guaranteed, not the order + blocksPerLID := map[uint32]int{} + for _, id := range content.IDs[1:] { // system id is not positioned + assert.Less(t, id.BlockIndex, uint32(len(content.Offsets.Values)), "block index out of range") + blocksPerLID[id.BlockIndex]++ + } + assert.Equal(t, map[uint32]int{0: 2, 1: 2, 2: 1}, blocksPerLID) } func TestSealFailures(t *testing.T) { @@ -308,15 +209,6 @@ func TestSealFailures(t *testing.T) { }) } } - -func TestUnknownOnlySection(t *testing.T) { - fracName := filepath.Join(t.TempDir(), "frac_000001") - - err := decodeFraction(fracName, "amogus", &collectedContent{}) - require.Error(t, err) - assert.Contains(t, err.Error(), "unknown section to decode: amogus") -} - func TestSealCustomMapping(t *testing.T) { mapping := filepath.Join(t.TempDir(), "mapping.yaml") err := os.WriteFile(mapping, []byte("mapping-list:\n - type: \"path\"\n name: \"request_uri\"\n"), 0o600) diff --git a/cmd/fraction/testdata/legacy/seal-fraction.sh b/cmd/fraction/testdata/legacy/seal-fraction.sh new file mode 100644 index 00000000..99817dd6 --- /dev/null +++ b/cmd/fraction/testdata/legacy/seal-fraction.sh @@ -0,0 +1,160 @@ +#!/usr/bin/env bash +# Seals a fraction in a given format version. Used by tests and CI to run +# the cmd/fraction tests against fractions produced by older seq-db code. +# +# Usage: +# seal-fraction.sh < docs.jsonl +# +# Optional flag: +# --mapping= mapping YAML for the indexed fields; the file must +# be readable by the version's seq.ReadMapping. Without +# the flag a built-in default mapping is used. +# +# is a fraction version: v2, v3, v4, v5, ... or "current". +# v2..v5 — a git worktree is created at that version's sealer commit +# (a version commit with cmd/sealer added on top, see the +# sealer_commit table below) and the committed standalone +# sealer is run there; old code has no `fraction seal`. +# v6+ — a git worktree is created at the version commit found +# dynamically (the parent of the first commit introducing +# the next version in config/frac_version.go) and the version's +# own `go run ./cmd/fraction seal` is used (it exists since +# v6 and seals in the code's current format). Needs no +# per-version support: a new vN+1 works with zero script +# changes. +# current — the local code's own `go run ./cmd/fraction seal` is used. +# +# is the output fraction base name (without suffixes). +# Documents (one JSON per line) are read from stdin. +# +# Prints the fraction base name on stdout. + +set -euo pipefail + +VERSION="" +FRAC="" +for arg in "$@"; do + case "$arg" in + --mapping=*) MAPPING="${arg#--mapping=}" ;; + *) if [[ -z "$VERSION" ]]; then VERSION="$arg"; else FRAC="$arg"; fi ;; + esac +done +VERSION="${VERSION:?usage: seal-fraction.sh [--mapping=file] < docs.jsonl}" +FRAC="${FRAC:?usage: seal-fraction.sh [--mapping=file] < docs.jsonl}" +MAPPING="${MAPPING:-}" + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO="$(git -C "$SCRIPT_DIR" rev-parse --show-toplevel)" + +# Version commit discovery needs history; a PR checkout has no local "main" +# branch (detached HEAD), so prefer origin/main, then main, then HEAD. +history_ref() { + for ref in origin/main main HEAD; do + if git -C "$REPO" rev-parse -q --verify "$ref" >/dev/null; then + echo "$ref" + return + fi + done + echo "no git history ref found (tried origin/main, main, HEAD)" >&2 + return 1 +} + +# the mapping file is opened by sealers running from other cwd's +[[ -z "$MAPPING" || "$MAPPING" == /* ]] || MAPPING="$(pwd)/$MAPPING" + +# Last commit of version: the parent of the first commit that +# introduced the next version (BinaryDataV) in config/frac_version.go. +# When the next version is not committed yet (e.g. a locally added V7), +# fall back to the history ref tip. Requires real +# history: in a shallow CI clone the caller must fetch it first +# (fetch-depth: 0 or git fetch --unshallow). +version_commit() { + local next=$((VERSION_NUM + 1)) + local history + history="$(history_ref)" + for commit in $(git -C "$REPO" rev-list --reverse "$history" -- config/frac_version.go); do + if git -C "$REPO" show "${commit}:config/frac_version.go" 2>/dev/null | grep -q "BinaryDataV${next}\b"; then + git -C "$REPO" rev-parse --short "${commit}^" + return + fi + done + if [[ "$VERSION_NUM" -ge "$(max_known_version_num)" ]]; then + git -C "$REPO" rev-parse --short "$history" + return + fi + echo "cannot find the commit introducing BinaryDataV${next}: version $VERSION is too old (supported: v6+)" >&2 + return 1 +} + +# The highest BinaryDataVN declared in the current config/frac_version.go. +max_known_version_num() { + local history + history="$(history_ref)" + git -C "$REPO" show "${history}:config/frac_version.go" | grep -oE 'BinaryDataV[0-9]+' | grep -oE '[0-9]+' | sort -n | tail -1 +} + +# Sealer commits for the pre-v6 formats: version commit with the +# standalone sealer (cmd/sealer) committed on top. +sealer_commit() { + case "$VERSION" in + v2) echo "98d29f2af902ee37ac24da18255a0a69a0da8266" ;; + v3) echo "1e214d0a282f29f577f38dcc920a8b8c6d95eab1" ;; + v4) echo "113f8663b80ce44149c12f1333237313b535e1d1" ;; + v5) echo "02e864f6a26b193fbfdcc973a5de3b8b9dd15f60" ;; + esac +} + +if [[ "$VERSION" == "current" ]]; then + if [[ -n "$MAPPING" ]]; then + (cd "$REPO" && go run ./cmd/fraction seal --mapping="$MAPPING" "$FRAC") + else + (cd "$REPO" && go run ./cmd/fraction seal "$FRAC") + fi +else + [[ "$VERSION" =~ ^v[0-9]+$ ]] || { + echo "unknown version: $VERSION (expected vN or current)" >&2 + exit 2 + } + VERSION_NUM="${VERSION#v}" + [[ "$VERSION_NUM" -ge 2 ]] || { + echo "version $VERSION is too old (supported: v2+)" >&2 + exit 2 + } + + if [[ "$VERSION_NUM" -ge 6 ]]; then + # v6+ seal with their own cmd/fraction + commit="$(version_commit)" + else + commit="$(sealer_commit)" + [[ -n "$commit" ]] || { + echo "no sealer commit for version $VERSION" >&2 + exit 2 + } + fi + + wt="${TMPDIR:-/tmp}/fraction-seal-$VERSION" + + if [[ -d "$wt/.git" || -f "$wt/.git" ]]; then + git -C "$wt" checkout -q --detach "$(git -C "$REPO" rev-parse "$commit")" + else + git -C "$REPO" worktree add --detach "$wt" "$commit" >/dev/null + fi + + if [[ "$VERSION_NUM" -ge 6 ]]; then + # v6+ have their own cmd/fraction seal + if [[ -n "$MAPPING" ]]; then + (cd "$wt" && go run ./cmd/fraction seal --mapping="$MAPPING" "$FRAC") + else + (cd "$wt" && go run ./cmd/fraction seal "$FRAC") + fi + else + # the sealer is committed at (cmd/sealer) + if [[ -n "$MAPPING" ]]; then + (cd "$wt" && CGO_ENABLED=0 go run ./cmd/sealer "$FRAC" --mapping="$MAPPING") + else + (cd "$wt" && CGO_ENABLED=0 go run ./cmd/sealer "$FRAC") + fi + fi +fi + +echo "$FRAC"