From af00f0e8fd0ed8d25ea15d8252863a91fa46e0f4 Mon Sep 17 00:00:00 2001 From: Andre Staats Date: Thu, 29 Oct 2026 01:55:32 +0100 Subject: [PATCH 1/5] pinnedimport, validate: one byte budget for the served-form build A strict org's served form is now built within one running budget of validate.MaxContentBytes (validate.ServedBudget), shared by the pinned module inliner and the __tenant_id__ drop pass. Each module text, nested expansion, re-quoted import.string content (its size counted with QuotedLen before it is built) and inserted drop is charged before it is allocated, and the bytes kept are charged when a pass joins its pieces. The charge never exceeds the finished length, so every served form that fits is built byte for byte as before; one that would not is refused (ErrNotInlinable wrapping ErrServedFormTooLarge) when it reaches the cap. Also: - at most MaxImportInstances (256) imports expanded per pipeline, nested ones and repeats included; - splices are applied in one pass (ApplySplices) instead of one copy of the text per splice; - the inliner reads and inlines each module once per build, References walks it once, and the tenant lint parses each text once per call. Differential tests hold the new code to the previous implementation (kept verbatim in _test.go files) over 60k random texts and 400 near-cap ones in each package; allocation specs cover the expansion shapes. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01TAAVzyeV4HSSNjU8xVWWtE --- CHANGELOG.md | 7 + docs/spec.md | 9 +- internal/pinnedimport/inline.go | 217 ++++++++-- internal/pinnedimport/inline_budget_test.go | 410 +++++++++++++++++++ internal/pinnedimport/oldinline_ref_test.go | 238 +++++++++++ internal/validate/labelmapdrop.go | 105 +++-- internal/validate/olddrop_ref_test.go | 228 +++++++++++ internal/validate/served_budget_diff_test.go | 392 ++++++++++++++++++ internal/validate/servedbudget.go | 143 +++++++ internal/validate/tenant.go | 27 +- 10 files changed, 1696 insertions(+), 80 deletions(-) create mode 100644 internal/pinnedimport/inline_budget_test.go create mode 100644 internal/pinnedimport/oldinline_ref_test.go create mode 100644 internal/validate/olddrop_ref_test.go create mode 100644 internal/validate/served_budget_diff_test.go create mode 100644 internal/validate/servedbudget.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 46f81fe5..9cb9ae6a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,13 @@ Categories used here: ### Changed +- **Shipped — a strict org's served form stops at the 1 MiB limit while it is built.** Inlining + pinned modules and inserting the `__tenant_id__` drops now count every byte against the pipeline + limit as they go, so a pipeline that would expand past 1 MiB (one module imported many times, + nested imports, re-quoted module contents, many drops) is refused as too large when it reaches the + limit instead of after being built in full. A pipeline that expands more than 256 imports + (`import.git` and `import.string`, nested ones and repeats included) is refused as well. Every + pipeline whose served form fits is served byte for byte as before. - **Shipped — group access is bound to the org that granted it.** A group assigned to a collector on its Access tab gives its members viewer access only while the collector's cluster stays in the org that made the assignment. Claiming a cluster into another org removes the diff --git a/docs/spec.md b/docs/spec.md index 7f8b63db..7dd243f1 100644 --- a/docs/spec.md +++ b/docs/spec.md @@ -1009,7 +1009,14 @@ every pipeline without an `import.git` block, are served byte for byte as before change: only a strict org's pipeline that uses a pinned import is served differently). The validation gate validates that served form too (`TenantPolicy.Inline`): Stages 1-2 and the tenant lint on the inlined text, refused when it cannot be built or is over the 1 MiB pipeline cap -(`validate.MaxContentBytes`); role/signal enforcement and the editor's derived signals see it as +(`validate.MaxContentBytes`). The cap holds while the form is built, not only after: the inliner +and the `__tenant_id__` drop pass charge one byte budget (`validate.ServedBudget`) for every module +text, nested expansion, re-quoted content and inserted drop before building it, and stop the moment +the result would pass 1 MiB. What is charged never exceeds the finished length, so a pipeline whose +served form fits is built byte for byte as before. A pipeline that expands more than 256 imports +(`pinnedimport.MaxImportInstances`: `import.git` and `import.string` blocks, nested ones and repeats +included) is refused too. A module imported several times is read and inlined once per build, and +the tenant lint parses each module text once per run; role/signal enforcement and the editor's derived signals see it as well, so a pinned module's declare body's signals count (an `import.git` instance would be Unknown). At serve time an `import.git` that cannot be inlined fails the recompute closed: the collector keeps its last good config and the failure is recorded (`last_failed_seq`, audited), never the raw import. diff --git a/internal/pinnedimport/inline.go b/internal/pinnedimport/inline.go index b6dbcc58..1f80e458 100644 --- a/internal/pinnedimport/inline.go +++ b/internal/pinnedimport/inline.go @@ -20,14 +20,27 @@ import ( // repository, revision or path is not a string literal, the revision is not a // full commit SHA, the repository is not on the allowlist, the module is not // cached, or it nests deeper than validate.MaxGitImportDepth. The serve path -// fails closed on it: a strict org is never served an import.git. +// fails closed on it: a strict org is never served an import.git. It is also +// a served form that would be more than validate.MaxContentBytes, or that +// expands more than MaxImportInstances imports. var ErrNotInlinable = errors.New("pinned import.git cannot be inlined") +// MaxImportInstances is how many import blocks (import.git and +// import.string) a strict org's pipeline may expand as served, nested ones +// included and a module imported twice counted twice. The pipelines in use +// have a handful; each instance is a module Alloy loads and runs. +const MaxImportInstances = 256 + // Ref is one pinned module, as the cache keys it. type Ref struct { Key, Revision, Path string } +// moduleQuerier is how the walker reads the cache (*sqlc.Queries). +type moduleQuerier interface { + GetPinnedImportModule(ctx context.Context, arg sqlc.GetPinnedImportModuleParams) (sqlc.PinnedImportModule, error) +} + // Inline is what a strict org is served for contents (#390): every // import.git block — at the top level or inside a declare — rewritten into // `import.string "" { content = "" }`, where module is @@ -47,14 +60,27 @@ type Ref struct { // is returned unchanged. // // Only the cache is read, through q. The result is at most -// validate.MaxContentBytes (the pipeline contents cap). +// validate.MaxContentBytes (the pipeline contents cap), and the whole build +// — every module, every nested expansion, every drop — is charged to one +// validate.ServedBudget of that size as it goes: a pipeline that would +// expand past it is refused when it gets there, never built in full. At +// most MaxImportInstances imports are expanded. func Inline(ctx context.Context, q *sqlc.Queries, orgID pgtype.UUID, repos []sqlc.OrgImportRepository, contents string) (string, error) { + return inline(ctx, q, orgID, repos, contents) +} + +func inline(ctx context.Context, q moduleQuerier, orgID pgtype.UUID, repos []sqlc.OrgImportRepository, contents string) (string, error) { w := newWalker(ctx, q, orgID, repos) out, err := w.inline(contents, 0, 0) if err != nil { return "", err } - if out, err = validate.InsertTenantDrops(out); err != nil { + // A changed text is charged; the drop pass charges what it builds from + // it (validate.ServedBudget). + if out != contents { + w.budget.Give(len(out)) + } + if out, err = validate.InsertTenantDropsWithin(out, w.budget); err != nil { return "", fmt.Errorf("%w: %w", ErrNotInlinable, err) } if len(out) > validate.MaxContentBytes { @@ -78,6 +104,10 @@ func InlineOrg(ctx context.Context, q *sqlc.Queries, orgID pgtype.UUID, contents // modules — what must stay cached while contents is enabled. A module that // is not cached is listed, and what it would import is not. func References(ctx context.Context, q *sqlc.Queries, orgID pgtype.UUID, repos []sqlc.OrgImportRepository, contents string) ([]Ref, error) { + return references(ctx, q, orgID, repos, contents) +} + +func references(ctx context.Context, q moduleQuerier, orgID pgtype.UUID, repos []sqlc.OrgImportRepository, contents string) ([]Ref, error) { w := newWalker(ctx, q, orgID, repos) w.collect = true if _, err := w.inline(contents, 0, 0); err != nil { @@ -88,15 +118,41 @@ func References(ctx context.Context, q *sqlc.Queries, orgID pgtype.UUID, repos [ type walker struct { ctx context.Context - q *sqlc.Queries + q moduleQuerier orgID pgtype.UUID repos map[string]bool collect bool refs []Ref + // budget is the served form's (Inline); instances counts the imports + // expanded (MaxImportInstances). + budget *validate.ServedBudget + instances int + // modules are the modules inlined so far, by where they were inlined: + // one module imported many times is read and inlined once. visited is + // References' (collect) record of the same. + modules map[moduleAt]inlinedModule + visited map[moduleAt]bool } -func newWalker(ctx context.Context, q *sqlc.Queries, orgID pgtype.UUID, repos []sqlc.OrgImportRepository) *walker { - w := &walker{ctx: ctx, q: q, orgID: orgID, repos: map[string]bool{}} +// moduleAt is a module at the depths it is inlined at: what inlining it +// depends on, besides the cache. +type moduleAt struct { + ref Ref + gitDepth, depth int +} + +// inlinedModule is a module's inlined text, quoted, and how many imports +// inlining it expanded. +type inlinedModule struct { + quoted string + instances int +} + +func newWalker(ctx context.Context, q moduleQuerier, orgID pgtype.UUID, repos []sqlc.OrgImportRepository) *walker { + w := &walker{ + ctx: ctx, q: q, orgID: orgID, repos: map[string]bool{}, budget: validate.NewServedBudget(), + modules: map[moduleAt]inlinedModule{}, visited: map[moduleAt]bool{}, + } for i := range repos { w.repos[repos[i].RepositoryKey] = true } @@ -107,19 +163,15 @@ func newWalker(ctx context.Context, q *sqlc.Queries, orgID pgtype.UUID, repos [] // follows into: the tenant lint's module depth. const maxModuleDepth = 8 -// splice is one rewrite of a byte range of a text. -type splice struct { - start, end int // [start, end) - text string -} - // inline rewrites text's import.git blocks — found in the parse tree, so // whatever spelling of the name the parser accepts — and descends into each // import.string whose content is a plain literal, rewriting the module text // it holds the same way. gitDepth is how many import.git modules enclose // text, depth how many modules of either kind. Text with nothing to rewrite // is returned unchanged; otherwise only the rewritten byte ranges change, -// located by the parsed nodes' positions. +// located by the parsed nodes' positions, and the result is charged to the +// budget (validate.ServedBudget). In collect mode (References) nothing is +// rewritten: the walk only lists the modules. func (w *walker) inline(text string, gitDepth, depth int) (string, error) { if depth > maxModuleDepth { return "", fmt.Errorf("%w: modules nest more than %d deep", ErrNotInlinable, maxModuleDepth) @@ -133,9 +185,14 @@ func (w *walker) inline(text string, gitDepth, depth int) (string, error) { } var blocks []*ast.BlockStmt collectImports(file.Body, &blocks, 0) - var splices []splice + var splices []validate.Splice for _, b := range blocks { - var sp *splice + if !w.collect { + if err := w.expand(1); err != nil { + return "", err + } + } + var sp *validate.Splice var err error if strings.Join(b.Name, ".") == "import.git" { sp, err = w.gitBlock(b, gitDepth, depth) @@ -155,18 +212,35 @@ func (w *walker) inline(text string, gitDepth, depth int) (string, error) { if len(splices) == 0 { return text, nil } - // Apply from the last range back, so earlier offsets stay valid. - slices.SortFunc(splices, func(a, b splice) int { return b.start - a.start }) - out := text - for _, sp := range splices { - out = out[:sp.start] + sp.text + out[sp.end:] + out, err := validate.ApplySplices(text, splices, w.budget) + if err != nil { + return "", fmt.Errorf("%w: %w", ErrNotInlinable, err) } return out, nil } +// expand counts n more import instances, refusing more than +// MaxImportInstances. +func (w *walker) expand(n int) error { + w.instances += n + if w.instances > MaxImportInstances { + return fmt.Errorf("%w: as served the pipeline expands more than %d imports (import.git and "+ + "import.string blocks, nested ones and repeats included)", ErrNotInlinable, MaxImportInstances) + } + return nil +} + +// take charges n bytes of served form to the budget. +func (w *walker) take(n int) error { + if err := w.budget.Take(n); err != nil { + return fmt.Errorf("%w: %w", ErrNotInlinable, err) + } + return nil +} + // gitBlock rewrites one import.git block into an import.string of its // cached module, itself inlined. -func (w *walker) gitBlock(b *ast.BlockStmt, gitDepth, depth int) (*splice, error) { +func (w *walker) gitBlock(b *ast.BlockStmt, gitDepth, depth int) (*validate.Splice, error) { if gitDepth >= validate.MaxGitImportDepth { return nil, fmt.Errorf("%w: import.git nests more than %d deep", ErrNotInlinable, validate.MaxGitImportDepth) } @@ -174,34 +248,90 @@ func (w *walker) gitBlock(b *ast.BlockStmt, gitDepth, depth int) (*splice, error if err != nil { return nil, err } - if w.collect && !slices.Contains(w.refs, ref) { - w.refs = append(w.refs, ref) + at := moduleAt{ref: ref, gitDepth: gitDepth, depth: depth} + if w.collect { + if !slices.Contains(w.refs, ref) { + w.refs = append(w.refs, ref) + } + // A second visit at the same depths lists nothing new. + if w.visited[at] { + return nil, nil //nolint:nilnil // nothing to rewrite in collect mode + } + w.visited[at] = true + content, err := w.module(ref) + if err != nil { + return nil, err + } + _, err = w.inline(content, gitDepth+1, depth+1) + return nil, err + } + m, ok := w.modules[at] + if ok { + if err := w.expand(m.instances); err != nil { + return nil, err + } + } else { + content, err := w.module(ref) + if err != nil { + return nil, err + } + before := w.instances + module, err := w.inline(content, gitDepth+1, depth+1) + if err != nil { + return nil, err + } + if module != content { + // Charged as inlined; what is served is the import.string below. + w.budget.Give(len(module)) + } + // The quoted module is built once, here, and is part of the first + // instance's import.string: it must fit the budget before it is + // built (the instance charges it below). + quoted := validate.QuotedLen(module) + if err := w.take(quoted); err != nil { + return nil, err + } + w.budget.Give(quoted) + m = inlinedModule{quoted: alloysyntax.QuoteString(module), instances: w.instances - before} + w.modules[at] = m } + start := ast.StartPos(b).Position() + columns := max(start.Column-1, 0) + const head, open, content, closing = "import.string ", " {\n", " content = ", "\n" + n := len(head) + validate.QuotedLen(b.Label) + len(open) + columns + len(content) + len(m.quoted) + + len(closing) + columns + 1 + if err := w.take(n); err != nil { + return nil, err + } + indent := strings.Repeat(" ", columns) + var sb strings.Builder + sb.Grow(n) + for _, s := range []string{head, alloysyntax.QuoteString(b.Label), open, indent, content, m.quoted, closing, indent, "}"} { + sb.WriteString(s) + } + return &validate.Splice{ + Start: start.Offset, + End: ast.EndPos(b).Position().Offset + 1, + Text: sb.String(), + }, nil +} + +// module is ref's cached text. +func (w *walker) module(ref Ref) (string, error) { row, err := w.q.GetPinnedImportModule(w.ctx, sqlc.GetPinnedImportModuleParams{ OrgID: w.orgID, RepositoryKey: ref.Key, Revision: ref.Revision, Path: ref.Path, }) if err != nil { - return nil, fmt.Errorf("%w: module %s at %s (%s) is not cached", ErrNotInlinable, ref.Path, ref.Revision, ref.Key) + return "", fmt.Errorf("%w: module %s at %s (%s) is not cached", ErrNotInlinable, ref.Path, ref.Revision, ref.Key) } - module, err := w.inline(row.Content, gitDepth+1, depth+1) - if err != nil { - return nil, err - } - start := ast.StartPos(b).Position() - indent := strings.Repeat(" ", max(start.Column-1, 0)) - return &splice{ - start: start.Offset, - end: ast.EndPos(b).Position().Offset + 1, - text: "import.string " + alloysyntax.QuoteString(b.Label) + " {\n" + - indent + " content = " + alloysyntax.QuoteString(module) + "\n" + indent + "}", - }, nil + return row.Content, nil } // stringBlock rewrites the content of one import.string whose module text // holds an import.git (at any depth): only its content literal is replaced, // re-quoted. Content that is not a plain literal is left alone (the served // form's bound, validate.ServedFormProblem, refuses it in a strict org). -func (w *walker) stringBlock(b *ast.BlockStmt, gitDepth, depth int) (*splice, error) { +func (w *walker) stringBlock(b *ast.BlockStmt, gitDepth, depth int) (*validate.Splice, error) { for _, st := range b.Body { a, ok := st.(*ast.AttributeStmt) if !ok || a.Name.Name != "content" { @@ -218,10 +348,15 @@ func (w *walker) stringBlock(b *ast.BlockStmt, gitDepth, depth int) (*splice, er if inner == content { return nil, nil //nolint:nilnil // unchanged: the bytes stay as written } - return &splice{ - start: ast.StartPos(a.Value).Position().Offset, - end: ast.EndPos(a.Value).Position().Offset + 1, - text: alloysyntax.QuoteString(inner), + // Charged as inlined; what is served is inner quoted. + w.budget.Give(len(inner)) + if err := w.take(validate.QuotedLen(inner)); err != nil { + return nil, err + } + return &validate.Splice{ + Start: ast.StartPos(a.Value).Position().Offset, + End: ast.EndPos(a.Value).Position().Offset + 1, + Text: alloysyntax.QuoteString(inner), }, nil } return nil, nil //nolint:nilnil // no content attribute diff --git a/internal/pinnedimport/inline_budget_test.go b/internal/pinnedimport/inline_budget_test.go new file mode 100644 index 00000000..68fe3230 --- /dev/null +++ b/internal/pinnedimport/inline_budget_test.go @@ -0,0 +1,410 @@ +package pinnedimport + +import ( + "context" + "errors" + "fmt" + "math/rand" + "os" + "reflect" + "runtime" + "strconv" + "strings" + "testing" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgtype" + + "shepherd/internal/alloysyntax" + "shepherd/internal/store/sqlc" + "shepherd/internal/validate" +) + +// The served-form budget must not change what a strict org is served: +// Inline returns what the pre-budget one (refInline, oldinline_ref_test.go) +// did whenever that fits, refuses whenever it did, and References lists the +// same modules in the same order. + +const ( + budgetRepo = "https://git.example.com/team/modules.git" + budgetSHA = "0123456789abcdef0123456789abcdef01234567" +) + +// fakeModules is a module cache: path -> content, under budgetRepo at +// budgetSHA. +type fakeModules map[string]string + +func (f fakeModules) GetPinnedImportModule(_ context.Context, a sqlc.GetPinnedImportModuleParams) (sqlc.PinnedImportModule, error) { + key, err := NormalizeRepository(budgetRepo) + if err != nil { + return sqlc.PinnedImportModule{}, err + } + c, ok := f[a.Path] + if !ok || a.RepositoryKey != key || a.Revision != budgetSHA { + return sqlc.PinnedImportModule{}, pgx.ErrNoRows + } + return sqlc.PinnedImportModule{Content: c}, nil +} + +func budgetRepos(t *testing.T) []sqlc.OrgImportRepository { + t.Helper() + key, err := NormalizeRepository(budgetRepo) + if err != nil { + t.Fatal(err) + } + return []sqlc.OrgImportRepository{{RepositoryKey: key}} +} + +func importGit(ind, label, path string) string { + return ind + "import.git " + strconv.Quote(label) + " {\n" + ind + " repository = \"" + budgetRepo + "\"\n" + ind + + " revision = \"" + budgetSHA + "\"\n" + ind + " path = " + strconv.Quote(path) + "\n" + ind + "}\n" +} + +// inlineGen writes random pipelines and modules: pinned import.git blocks +// (cached, not cached, refused for their revision, repository or a +// computed path), import.string contents holding them, declares, labelmaps +// and exporters the drop pass rewrites, odd columns and bytes. +type inlineGen struct { + r *rand.Rand +} + +func (g *inlineGen) pick(xs ...string) string { return xs[g.r.Intn(len(xs))] } + +func (g *inlineGen) stmt(depth, imports int, ind string) string { + label := g.pick("a", "b", "m", "é", "with space") + switch g.r.Intn(11) { + case 0, 1, 2: + if imports <= 0 { + return "" + } + path := g.pick("m0.alloy", "m1.alloy", "m2.alloy", "m3.alloy", "dir/m4.alloy", "missing.alloy") + switch g.r.Intn(12) { + case 0: + return strings.Replace(importGit(ind, label, path), budgetSHA, "main", 1) + case 1: + return strings.Replace(importGit(ind, label, path), budgetRepo, "https://elsewhere.example.com/x.git", 1) + case 2: + return strings.Replace(importGit(ind, label, path), strconv.Quote(path), `"m" + "0.alloy"`, 1) + case 3: // not at column 1 + return ind + "/* " + g.pick("x", "\t", "é") + " */ " + importGit("", label, path) + case 4: + return strings.Replace(importGit(ind, label, path), ind+"}\n", + ind+" basic_auth {\n"+ind+" username = \"u\"\n"+ind+" }\n"+ind+"}\n", 1) + default: + return importGit(ind, label, path) + } + case 3: + if depth > 2 { + return "" + } + inner := g.text(depth+1, imports-1) + return ind + "import.string " + strconv.Quote(label) + " {\n" + ind + " content = " + + g.pick(alloysyntax.QuoteString(inner), alloysyntax.QuoteString(inner), "local.file.f.content") + "\n" + ind + "}\n" + case 4: + if depth > 2 { + return "" + } + return ind + "declare " + strconv.Quote(label) + " {\n" + g.stmt(depth+1, imports, ind+" ") + ind + "}\n" + case 5: + return ind + "prometheus.relabel " + strconv.Quote(label) + " {\n" + ind + " forward_to = []\n" + ind + + " rule {\n" + ind + " action = \"labelmap\"\n" + ind + " }\n" + ind + "}\n" + case 6: + return ind + "otelcol.exporter.loki " + strconv.Quote(label) + " {\n" + ind + + " forward_to = [loki.write.a.receiver]\n" + ind + "}" + g.pick("", " // c") + "\n" + case 7: + return ind + "// " + g.pick("\t\x01", "é", "import.git \"x\" {}", "") + "\n" + case 8: + return ind + "x = " + strconv.Quote(g.pick("\t", "\x01", "a\"b", "😀")) + "\n" + case 9: + if g.r.Intn(10) == 0 { + return ind + g.pick("{", "import.git {", "\"") + "\n" + } + return "" + default: + return "" + } +} + +func (g *inlineGen) text(depth, imports int) string { + var sb strings.Builder + for range g.r.Intn(5) { + sb.WriteString(g.stmt(depth, imports, g.pick("", "", " ", "\t"))) + } + return sb.String() +} + +func (g *inlineGen) modules() fakeModules { + m := fakeModules{} + for _, p := range []string{"m0.alloy", "m1.alloy", "m2.alloy", "m3.alloy", "dir/m4.alloy"} { + if g.r.Intn(8) != 0 { + m[p] = g.text(1, 2) + } + } + return m +} + +func errText(err error) string { + if err == nil { + return "" + } + return err.Error() +} + +// checkInlineMatches fails t unless Inline and References agree with the +// reference on contents. +func checkInlineMatches(t *testing.T, mods fakeModules, repos []sqlc.OrgImportRepository, contents string, sameErrors, withRefs bool) bool { + t.Helper() + ctx := context.Background() + org := pgtype.UUID{} + want, wantErr := refInline(ctx, mods, org, repos, contents) + got, gotErr := inline(ctx, mods, org, repos, contents) + if withRefs { + wantRefs, wantRefsErr := refReferences(ctx, mods, org, repos, contents) + gotRefs, gotRefsErr := references(ctx, mods, org, repos, contents) + if !reflect.DeepEqual(gotRefs, wantRefs) || errText(gotRefsErr) != errText(wantRefsErr) { + t.Fatalf("References differ:\n got %v %v\n want %v %v\nfor %q", gotRefs, gotRefsErr, wantRefs, wantRefsErr, contents) + } + } + if wantErr != nil { + if gotErr == nil { + t.Fatalf("the reference refuses (%v), the budgeted Inline serves %d bytes for %q", wantErr, len(got), contents) + } + if sameErrors && gotErr.Error() != wantErr.Error() { + t.Fatalf("refusals differ:\n got %v\n want %v\nfor %q", gotErr, wantErr, contents) + } + return false + } + if gotErr != nil { + t.Fatalf("the reference serves %d bytes, the budgeted Inline refuses: %v\nfor %q", len(want), gotErr, contents) + } + if got != want { + t.Fatalf("served bytes differ for %q:\n got %q\n want %q", contents, got, want) + } + return true +} + +func TestInlineMatchesReference(t *testing.T) { + g := &inlineGen{r: rand.New(rand.NewSource(390))} + repos := budgetRepos(t) + const n = 60000 + var mods fakeModules + served, inlined := 0, 0 + for i := range n { + if i%50 == 0 { + mods = g.modules() + } + contents := g.text(0, 3) + if checkInlineMatches(t, mods, repos, contents, true, true) { + served++ + if out, err := refInline(context.Background(), mods, pgtype.UUID{}, repos, contents); err == nil && out != contents { + inlined++ + } + } + } + t.Logf("%d random pipelines: %d served (%d changed as served), %d refused alike", n, served, inlined, n-served) + if served < n/4 || inlined < n/10 { + t.Fatalf("the corpus is too thin: %d served, %d changed", served, inlined) + } +} + +// Near the cap: a module imported one to four times (directly, through an +// import.string, or from another module) whose served form is within a few +// hundred bytes of the cap either side. +func TestInlineMatchesReferenceNearCap(t *testing.T) { + g := &inlineGen{r: rand.New(rand.NewSource(3900))} + repos := budgetRepos(t) + const n = 400 + under, over := 0, 0 + // The served size is linear in the padding; its two coefficients + // depend only on the shape. + type shape struct { + copies, mix int + tabs bool + } + linear := map[shape][2]float64{} + for i := range n { + copies := 1 + g.r.Intn(4) + tabs := i%2 == 1 + build := func(pad int) (fakeModules, string) { + filler := strings.Repeat("a", pad) + if tabs { + filler = strings.Repeat("\t", pad) + } + mods := fakeModules{ + "m0.alloy": "prometheus.relabel \"r\" {\n forward_to = []\n rule {\n action = \"labelmap\"\n }\n}\n" + + "x = \"" + filler + "\"\n", + "m1.alloy": importGit("", "inner", "m0.alloy"), + } + var sb strings.Builder + for c := range copies { + label := "m" + strconv.Itoa(c) + switch (i + c) % 3 { + case 0: + sb.WriteString(importGit("", label, "m0.alloy")) + case 1: + sb.WriteString(importGit(" ", label, "m1.alloy")) + default: + sb.WriteString("import.string " + strconv.Quote(label) + " {\n content = " + + alloysyntax.QuoteString(importGit("", "x", "m0.alloy")) + "\n}\n") + } + } + return mods, sb.String() + } + size := func(pad int) int { + mods, contents := build(pad) + out, err := refInline(context.Background(), mods, pgtype.UUID{}, repos, contents) + if err != nil { + t.Fatalf("near-cap base refused: %v", err) + } + return len(out) + } + sh := shape{copies: copies, mix: i % 3, tabs: tabs} + if _, ok := linear[sh]; !ok { + l0, l1 := size(0), size(1000) + linear[sh] = [2]float64{float64(l0), float64(l1-l0) / 1000} + } + l0, perPad := linear[sh][0], linear[sh][1] + target := validate.MaxContentBytes + g.r.Intn(601) - 300 + pad := max(int((float64(target)-l0)/perPad), 0) + mods, contents := build(pad) + if checkInlineMatches(t, mods, repos, contents, false, false) { + under++ + } else { + over++ + } + } + t.Logf("%d near-cap pipelines: %d served identically, %d refused alike", n, under, over) + if under < n/5 || over < n/5 { + t.Fatalf("the near-cap corpus does not straddle the cap: %d under, %d over", under, over) + } +} + +// The import-instance cap counts every import expanded, repeats and nested +// ones included. +func TestInlineCapsImportInstances(t *testing.T) { + repos := budgetRepos(t) + ctx := context.Background() + mods := fakeModules{"leaf.alloy": "declare \"d\" {\n}\n"} + repeated := func(n int, path string) string { + var sb strings.Builder + for i := range n { + sb.WriteString(importGit("", "m"+strconv.Itoa(i), path)) + } + return sb.String() + } + if !checkInlineMatches(t, mods, repos, repeated(MaxImportInstances, "leaf.alloy"), true, true) { + t.Fatalf("%d imports of one module are refused", MaxImportInstances) + } + _, err := inline(ctx, mods, pgtype.UUID{}, repos, repeated(MaxImportInstances+1, "leaf.alloy")) + if !errors.Is(err, ErrNotInlinable) || !strings.Contains(err.Error(), fmt.Sprintf("more than %d imports", MaxImportInstances)) { + t.Fatalf("%d imports: %v", MaxImportInstances+1, err) + } + // Fan-out 4, four levels deep: 4+16+64+256 instances of tiny modules. + mods = fakeModules{"l3.alloy": "declare \"d\" {\n}\n"} + for _, l := range []string{"l2", "l1", "l0"} { + next := map[string]string{"l2": "l3", "l1": "l2", "l0": "l1"}[l] + mods[l+".alloy"] = repeated(4, next+".alloy") + } + _, err = inline(ctx, mods, pgtype.UUID{}, repos, repeated(4, "l0.alloy")) + if !errors.Is(err, ErrNotInlinable) || !strings.Contains(err.Error(), "imports") { + t.Fatalf("nested fan-out: %v", err) + } +} + +// The shapes that made Inline build far more than the cap from a pipeline +// within it. Each must be refused having allocated, beyond what parsing its +// texts costs, at most twice the cap. SERVED_BUDGET_REFERENCE=1 runs the +// same checks on the pre-budget Inline, which fails them. +func TestInlineStaysWithinTheBudget(t *testing.T) { + fn := inline + if os.Getenv("SERVED_BUDGET_REFERENCE") == "1" { + fn = refInline + } + repos := budgetRepos(t) + labelmap := "prometheus.relabel \"r\" {\n forward_to = []\n rule {\n action = \"labelmap\"\n }\n}\n" + repeated := func(n int, path string) string { + var sb strings.Builder + for i := range n { + sb.WriteString(importGit("", "m"+strconv.Itoa(i), path)) + } + return sb.String() + } + fanOut := fakeModules{"d.alloy": labelmap + "x = \"" + strings.Repeat("\t", 20<<10) + "\"\n"} + for _, l := range []string{"c", "b", "a"} { + next := map[string]string{"c": "d", "b": "c", "a": "b"}[l] + fanOut[l+".alloy"] = repeated(3, next+".alloy") + } + for _, tc := range []struct { + name string + mods fakeModules + contents string + }{ + { + // One 50 KiB module imported 200 times. + name: "one module many times", + mods: fakeModules{"big.alloy": labelmap + "// " + strings.Repeat("a", 50<<10) + "\n"}, + contents: repeated(200, "big.alloy"), + }, + { + // Fan-out 3 through four levels of modules: 81 copies of a + // 20 KiB leaf of tabs, re-escaped at every level. + name: "nested fan-out", + mods: fanOut, + contents: repeated(3, "a.alloy"), + }, + } { + t.Run(tc.name, func(t *testing.T) { + ctx := context.Background() + var out string + var err error + start := time.Now() + build := allocated(func() { out, err = fn(ctx, tc.mods, pgtype.UUID{}, repos, tc.contents) }) + elapsed := time.Since(start) + // Parsing: the pipeline, each module once and the served form + // once more per level (the drop pass's check), generously. + parse := allocated(func() { + errSink = validate.ServedFormProblem(tc.contents) + for _, m := range tc.mods { + errSink = validate.ServedFormProblem(m) + } + }) * 3 + // The Alloy encoder (alloysyntax.QuoteString) works in about a + // dozen times the memory it returns. Each level quotes its module + // once (memoized), and the levels' quotes sum to under twice the + // largest, which fits the cap: two cap-sized quotes' worth. + encoder := 2 * allocated(func() { _ = alloysyntax.QuoteString(strings.Repeat("\t", validate.MaxContentBytes/2)) }) + beyond := int64(build) - int64(parse) - int64(encoder) + t.Logf("%s: Inline allocated %.1f MiB, parsing %.1f MiB and two cap-sized quotes %.1f MiB of it: "+ + "%.1f MiB beyond, in %v; served %d bytes, err %v", tc.name, mib(build), mib(parse), mib(encoder), + float64(beyond)/(1<<20), elapsed.Round(time.Millisecond), len(out), err) + if err == nil { + t.Fatalf("served %d bytes, want a refusal", len(out)) + } + if !errors.Is(err, ErrNotInlinable) { + t.Fatalf("refused with %v, want ErrNotInlinable", err) + } + if beyond > 2*validate.MaxContentBytes { + t.Fatalf("allocated %.1f MiB beyond parsing, more than twice the cap", float64(beyond)/(1<<20)) + } + if elapsed > 5*time.Second { + t.Fatalf("took %v", elapsed) + } + }) + } +} + +func mib(n uint64) float64 { return float64(n) / (1 << 20) } + +// allocated is how many bytes fn allocates. +func allocated(fn func()) uint64 { + var before, after runtime.MemStats + runtime.GC() + runtime.ReadMemStats(&before) + fn() + runtime.ReadMemStats(&after) + return after.TotalAlloc - before.TotalAlloc +} + +// errSink keeps a result only measured for its allocations. +var errSink error diff --git a/internal/pinnedimport/oldinline_ref_test.go b/internal/pinnedimport/oldinline_ref_test.go new file mode 100644 index 00000000..72d5b7f3 --- /dev/null +++ b/internal/pinnedimport/oldinline_ref_test.go @@ -0,0 +1,238 @@ +package pinnedimport + +// The Inline and References of 3ec5c9c (before the served-form budget), +// verbatim but for the names (ref...) and q, which is the interface the +// walker reads the cache through: what inline_budget_test.go compares against. + +import ( + "context" + "fmt" + "slices" + "strings" + + "github.com/grafana/alloy/syntax/ast" + "github.com/jackc/pgx/v5/pgtype" + + "shepherd/internal/alloysyntax" + "shepherd/internal/gitrepo" + "shepherd/internal/store/sqlc" + "shepherd/internal/validate" +) + +// refInline is the old Inline. Its drop pass is today's +// validate.InsertTenantDrops, which validate's own differential test +// (served_budget_diff_test.go) holds to the old one. +func refInline(ctx context.Context, q moduleQuerier, orgID pgtype.UUID, repos []sqlc.OrgImportRepository, contents string) (string, error) { + w := newRefWalker(ctx, q, orgID, repos) + out, err := w.inline(contents, 0, 0) + if err != nil { + return "", err + } + if out, err = validate.InsertTenantDrops(out); err != nil { + return "", fmt.Errorf("%w: %w", ErrNotInlinable, err) + } + if len(out) > validate.MaxContentBytes { + return "", fmt.Errorf("%w: as served (pinned modules inlined, __tenant_id__ dropped after "+ + "each labelmap) the pipeline is %d bytes, more than the %d a "+ + "pipeline may be", ErrNotInlinable, len(out), validate.MaxContentBytes) + } + return out, nil +} + +// References lists every pinned module contents reaches, through cached +// modules — what must stay cached while contents is enabled. A module that +// is not cached is listed, and what it would import is not. +func refReferences(ctx context.Context, q moduleQuerier, orgID pgtype.UUID, repos []sqlc.OrgImportRepository, contents string) ([]Ref, error) { + w := newRefWalker(ctx, q, orgID, repos) + w.collect = true + if _, err := w.inline(contents, 0, 0); err != nil { + return nil, err + } + return w.refs, nil +} + +type refWalker struct { + ctx context.Context + q moduleQuerier + orgID pgtype.UUID + repos map[string]bool + collect bool + refs []Ref +} + +func newRefWalker(ctx context.Context, q moduleQuerier, orgID pgtype.UUID, repos []sqlc.OrgImportRepository) *refWalker { + w := &refWalker{ctx: ctx, q: q, orgID: orgID, repos: map[string]bool{}} + for i := range repos { + w.repos[repos[i].RepositoryKey] = true + } + return w +} + +// refMaxModuleDepth is how many modules (import.git or import.string) Inline +// follows into: the tenant lint's module depth. +const refMaxModuleDepth = 8 + +// refSplice is one rewrite of a byte range of a text. +type refSplice struct { + start, end int // [start, end) + text string +} + +// inline rewrites text's import.git blocks — found in the parse tree, so +// whatever spelling of the name the parser accepts — and descends into each +// import.string whose content is a plain literal, rewriting the module text +// it holds the same way. gitDepth is how many import.git modules enclose +// text, depth how many modules of either kind. Text with nothing to rewrite +// is returned unchanged; otherwise only the rewritten byte ranges change, +// located by the parsed nodes' positions. +func (w *refWalker) inline(text string, gitDepth, depth int) (string, error) { + if depth > refMaxModuleDepth { + return "", fmt.Errorf("%w: modules nest more than %d deep", ErrNotInlinable, refMaxModuleDepth) + } + file, err := alloysyntax.ParseFile("", []byte(text)) + if err != nil { + if w.collect { + return text, nil + } + return "", fmt.Errorf("%w: the text does not parse: %w", ErrNotInlinable, err) + } + var blocks []*ast.BlockStmt + refCollectImports(file.Body, &blocks, 0) + var splices []refSplice + for _, b := range blocks { + var sp *refSplice + var err error + if strings.Join(b.Name, ".") == "import.git" { + sp, err = w.gitBlock(b, gitDepth, depth) + } else { + sp, err = w.stringBlock(b, gitDepth, depth) + } + if err != nil { + if w.collect { + continue + } + return "", err + } + if sp != nil { + splices = append(splices, *sp) + } + } + if len(splices) == 0 { + return text, nil + } + // Apply from the last range back, so earlier offsets stay valid. + slices.SortFunc(splices, func(a, b refSplice) int { return b.start - a.start }) + out := text + for _, sp := range splices { + out = out[:sp.start] + sp.text + out[sp.end:] + } + return out, nil +} + +// gitBlock rewrites one import.git block into an import.string of its +// cached module, itself inlined. +func (w *refWalker) gitBlock(b *ast.BlockStmt, gitDepth, depth int) (*refSplice, error) { + if gitDepth >= validate.MaxGitImportDepth { + return nil, fmt.Errorf("%w: import.git nests more than %d deep", ErrNotInlinable, validate.MaxGitImportDepth) + } + ref, err := w.ref(b) + if err != nil { + return nil, err + } + if w.collect && !slices.Contains(w.refs, ref) { + w.refs = append(w.refs, ref) + } + row, err := w.q.GetPinnedImportModule(w.ctx, sqlc.GetPinnedImportModuleParams{ + OrgID: w.orgID, RepositoryKey: ref.Key, Revision: ref.Revision, Path: ref.Path, + }) + if err != nil { + return nil, fmt.Errorf("%w: module %s at %s (%s) is not cached", ErrNotInlinable, ref.Path, ref.Revision, ref.Key) + } + module, err := w.inline(row.Content, gitDepth+1, depth+1) + if err != nil { + return nil, err + } + start := ast.StartPos(b).Position() + indent := strings.Repeat(" ", max(start.Column-1, 0)) + return &refSplice{ + start: start.Offset, + end: ast.EndPos(b).Position().Offset + 1, + text: "import.string " + alloysyntax.QuoteString(b.Label) + " {\n" + + indent + " content = " + alloysyntax.QuoteString(module) + "\n" + indent + "}", + }, nil +} + +// stringBlock rewrites the content of one import.string whose module text +// holds an import.git (at any depth): only its content literal is replaced, +// re-quoted. Content that is not a plain literal is left alone (the served +// form's bound, validate.ServedFormProblem, refuses it in a strict org). +func (w *refWalker) stringBlock(b *ast.BlockStmt, gitDepth, depth int) (*refSplice, error) { + for _, st := range b.Body { + a, ok := st.(*ast.AttributeStmt) + if !ok || a.Name.Name != "content" { + continue + } + content, ok := validate.PlainStringLiteral(a.Value) + if !ok { + return nil, nil //nolint:nilnil // nothing this refWalker can rewrite + } + inner, err := w.inline(content, gitDepth, depth+1) + if err != nil { + return nil, err + } + if inner == content { + return nil, nil //nolint:nilnil // unchanged: the bytes stay as written + } + return &refSplice{ + start: ast.StartPos(a.Value).Position().Offset, + end: ast.EndPos(a.Value).Position().Offset + 1, + text: alloysyntax.QuoteString(inner), + }, nil + } + return nil, nil //nolint:nilnil // no content attribute +} + +// refCollectImports finds the import blocks Alloy loads — import.git and +// import.string — at the top level and in a declare body, at any declare +// depth, by their parsed names. +func refCollectImports(body ast.Body, out *[]*ast.BlockStmt, nesting int) { + if nesting > alloysyntax.MaxNesting { + return + } + for _, st := range body { + b, ok := st.(*ast.BlockStmt) + if !ok { + continue + } + switch strings.Join(b.Name, ".") { + case "import.git", "import.string": + *out = append(*out, b) + case "declare": + refCollectImports(b.Body, out, nesting+1) + } + } +} + +// ref reads a block's module reference under the lint's rules. +func (w *refWalker) ref(b *ast.BlockStmt) (Ref, error) { + attrs := map[string]string{} + for _, st := range b.Body { + a, ok := st.(*ast.AttributeStmt) + if !ok { + continue + } + if s, ok := validate.PlainStringLiteral(a.Value); ok { + attrs[a.Name.Name] = s + } + } + repo, rev, path := attrs["repository"], attrs["revision"], attrs["path"] + if repo == "" || path == "" || !gitrepo.IsFullCommitSHA(rev) || !validate.CleanModulePath(path) { + return Ref{}, fmt.Errorf("%w: import.git %q needs literal repository, a full commit SHA revision and a "+ + "clean path", ErrNotInlinable, b.Label) + } + key, err := NormalizeRepository(repo) + if err != nil || !w.repos[key] { + return Ref{}, fmt.Errorf("%w: import.git %q's repository is not on the allowlist", ErrNotInlinable, b.Label) + } + return Ref{Key: key, Revision: rev, Path: path}, nil +} diff --git a/internal/validate/labelmapdrop.go b/internal/validate/labelmapdrop.go index c0de1f83..d3be9d32 100644 --- a/internal/validate/labelmapdrop.go +++ b/internal/validate/labelmapdrop.go @@ -200,12 +200,20 @@ func isRelabelRuleSchema(b schemaBlock) bool { // already followed by that rule gets no second one. Text without such a // labelmap is returned unchanged, byte for byte; otherwise only the // inserted ranges change. The result must parse. It is part of the served -// form of a strict org's pipeline (pinnedimport.Inline). +// form of a strict org's pipeline (pinnedimport.Inline), so it is built +// within a ServedBudget of its own: a result that would be more than +// MaxContentBytes is ErrServedFormTooLarge. func InsertTenantDrops(text string) (string, error) { - return insertTenantDrops(text, 0) + return InsertTenantDropsWithin(text, NewServedBudget()) } -func insertTenantDrops(text string, depth int) (string, error) { +// InsertTenantDropsWithin is InsertTenantDrops charging b (see +// ServedBudget): when the text changes, b ends charged its length. +func InsertTenantDropsWithin(text string, b *ServedBudget) (string, error) { + return insertTenantDrops(text, 0, b) +} + +func insertTenantDrops(text string, depth int, budget *ServedBudget) (string, error) { if depth > maxDepth { return "", fmt.Errorf("%w: modules nest more than %d deep", ErrTenantDrop, maxDepth) } @@ -213,7 +221,7 @@ func insertTenantDrops(text string, depth int) (string, error) { if err != nil { return "", fmt.Errorf("%w: the text does not parse: %w", ErrTenantDrop, err) } - d := &dropper{text: text, depth: depth, newline: "\n"} + d := &dropper{text: text, depth: depth, newline: "\n", budget: budget} if strings.Contains(text, "\r\n") { d.newline = "\r\n" } @@ -223,11 +231,9 @@ func insertTenantDrops(text string, depth int) (string, error) { if len(d.splices) == 0 { return text, nil } - // From the last offset back, so earlier offsets stay valid. - slices.SortStableFunc(d.splices, func(a, b dropSplice) int { return b.start - a.start }) - out := text - for _, sp := range d.splices { - out = out[:sp.start] + sp.text + out[sp.end:] + out, err := ApplySplices(text, d.splices, budget) + if err != nil { + return "", err } if _, err := alloysyntax.ParseFile("", []byte(out)); err != nil { return "", fmt.Errorf("%w: the text with the drops inserted does not parse: %w", ErrTenantDrop, err) @@ -235,16 +241,23 @@ func insertTenantDrops(text string, depth int) (string, error) { return out, nil } -type dropSplice struct { - start, end int - text string -} - type dropper struct { text string depth int newline string - splices []dropSplice + splices []Splice + budget *ServedBudget +} + +// splice records the rewrite of [start, end) by the text build makes, +// charging the budget n bytes — the length that text will have — before it +// is built. +func (d *dropper) splice(start, end, n int, build func() string) error { + if err := d.budget.Take(n); err != nil { + return err + } + d.splices = append(d.splices, Splice{Start: start, End: end, Text: build()}) + return nil } // body walks stmts. component is the relabel-rule component enclosing them @@ -296,7 +309,9 @@ func (d *dropper) body(stmts ast.Body, component, path string, nesting int) erro } if name == "rule" && rules[component][p] && isLiteralLabelmap(b) && needsTenantDrop(b) && !isTenantDrop(nextRule(stmts, i)) { - d.insertAfter(b) + if err := d.insertAfter(b); err != nil { + return err + } } if err := d.body(b.Body, component, p, nesting+1); err != nil { return err @@ -309,19 +324,26 @@ func (d *dropper) body(stmts ast.Body, component, path string, nesting int) erro // lines of its own, indented with the leading whitespace (spaces and tabs, // byte for byte) of the line b starts on — never a column count, which a tab // or a multi-byte character before b would throw off. -func (d *dropper) insertAfter(b *ast.BlockStmt) { +func (d *dropper) insertAfter(b *ast.BlockStmt) error { start := ast.StartPos(b).Position().Offset end := ast.EndPos(b).Position().Offset + 1 lineStart := strings.LastIndexByte(d.text[:start], '\n') + 1 line := d.text[lineStart:start] indent := line[:len(line)-len(strings.TrimLeft(line, " \t"))] - var sb strings.Builder + n := 0 for _, line := range tenantDropRule { - sb.WriteString(d.newline) - sb.WriteString(indent) - sb.WriteString(line) + n += len(d.newline) + len(indent) + len(line) } - d.splices = append(d.splices, dropSplice{start: end, end: end, text: sb.String()}) + return d.splice(end, end, n, func() string { + var sb strings.Builder + sb.Grow(n) + for _, line := range tenantDropRule { + sb.WriteString(d.newline) + sb.WriteString(indent) + sb.WriteString(line) + } + return sb.String() + }) } // exporterLoki is the component the strict served form routes through a @@ -365,7 +387,10 @@ func (d *dropper) exporterLoki(b *ast.BlockStmt, stmts ast.Body) error { vStart := ast.StartPos(fwd.Value).Position().Offset vEnd := ast.EndPos(fwd.Value).Position().Offset + 1 original := d.text[vStart:vEnd] - d.splices = append(d.splices, dropSplice{start: vStart, end: vEnd, text: "[loki.process." + label + ".receiver]"}) + receiver := "[loki.process." + label + ".receiver]" + if err := d.splice(vStart, vEnd, len(receiver), func() string { return receiver }); err != nil { + return err + } start := ast.StartPos(b).Position().Offset end := ast.EndPos(b).Position().Offset + 1 @@ -382,17 +407,26 @@ func (d *dropper) exporterLoki(b *ast.BlockStmt, stmts ast.Body) error { " }", "}", } - var sb strings.Builder + n := 0 for _, l := range lines { - sb.WriteString(d.newline) + n += len(d.newline) if l != "" { - sb.WriteString(indent) - sb.WriteString(l) + n += len(indent) + len(l) } } at := afterTrailingComment(d.text, end) - d.splices = append(d.splices, dropSplice{start: at, end: at, text: sb.String()}) - return nil + return d.splice(at, at, n, func() string { + var sb strings.Builder + sb.Grow(n) + for _, l := range lines { + sb.WriteString(d.newline) + if l != "" { + sb.WriteString(indent) + sb.WriteString(l) + } + } + return sb.String() + }) } // afterTrailingComment is the offset to insert after a block that closes at @@ -454,16 +488,17 @@ func (d *dropper) importString(b *ast.BlockStmt) error { if !ok { return nil } - inner, err := insertTenantDrops(content, d.depth+1) + inner, err := insertTenantDrops(content, d.depth+1, d.budget) if err != nil { return fmt.Errorf("import.string %q: %w", b.Label, err) } if inner != content { - d.splices = append(d.splices, dropSplice{ - start: ast.StartPos(a.Value).Position().Offset, - end: ast.EndPos(a.Value).Position().Offset + 1, - text: alloysyntax.QuoteString(inner), - }) + // inner is charged; what is served is inner quoted. + d.budget.Give(len(inner)) + if err := d.splice(ast.StartPos(a.Value).Position().Offset, ast.EndPos(a.Value).Position().Offset+1, + QuotedLen(inner), func() string { return alloysyntax.QuoteString(inner) }); err != nil { + return fmt.Errorf("import.string %q: %w", b.Label, err) + } } } return nil diff --git a/internal/validate/olddrop_ref_test.go b/internal/validate/olddrop_ref_test.go new file mode 100644 index 00000000..74c0c527 --- /dev/null +++ b/internal/validate/olddrop_ref_test.go @@ -0,0 +1,228 @@ +package validate + +// The InsertTenantDrops of 3ec5c9c (before the served-form budget), verbatim but +// for the names (ref...): what served_budget_diff_test.go compares against. + +import ( + "fmt" + "slices" + "strconv" + "strings" + + "github.com/grafana/alloy/syntax/ast" + + "shepherd/internal/alloysyntax" +) + +func refInsertTenantDrops(text string) (string, error) { + return refInsertTenantDropsAt(text, 0) +} + +func refInsertTenantDropsAt(text string, depth int) (string, error) { + if depth > maxDepth { + return "", fmt.Errorf("%w: modules nest more than %d deep", ErrTenantDrop, maxDepth) + } + file, err := alloysyntax.ParseFile("", []byte(text)) + if err != nil { + return "", fmt.Errorf("%w: the text does not parse: %w", ErrTenantDrop, err) + } + d := &refDropper{text: text, depth: depth, newline: "\n"} + if strings.Contains(text, "\r\n") { + d.newline = "\r\n" + } + if err := d.body(file.Body, "", "", 0); err != nil { + return "", err + } + if len(d.splices) == 0 { + return text, nil + } + // From the last offset back, so earlier offsets stay valid. + slices.SortStableFunc(d.splices, func(a, b refDropSplice) int { return b.start - a.start }) + out := text + for _, sp := range d.splices { + out = out[:sp.start] + sp.text + out[sp.end:] + } + if _, err := alloysyntax.ParseFile("", []byte(out)); err != nil { + return "", fmt.Errorf("%w: the text with the drops inserted does not parse: %w", ErrTenantDrop, err) + } + return out, nil +} + +type refDropSplice struct { + start, end int + text string +} + +type refDropper struct { + text string + depth int + newline string + splices []refDropSplice +} + +// body walks stmts. component is the relabel-rule component enclosing them +// ("" outside one), path their block path inside it. +func (d *refDropper) body(stmts ast.Body, component, path string, nesting int) error { + if nesting > alloysyntax.MaxNesting { + return fmt.Errorf("%w: blocks nest too deeply", ErrTenantDrop) + } + for i, st := range stmts { + b, ok := st.(*ast.BlockStmt) + if !ok { + continue + } + name := strings.Join(b.Name, ".") + switch { + case component == "" && name == "import.string": + if err := d.importString(b); err != nil { + return err + } + continue + case component == "" && name == exporterLoki: + if err := d.exporterLoki(b, stmts); err != nil { + return err + } + continue + case component == "": + rules, err := RelabelRuleBlocks() + if err != nil { + return fmt.Errorf("%w: %w", ErrTenantDrop, err) + } + if _, ok := rules[name]; ok { + if err := d.body(b.Body, name, "", nesting+1); err != nil { + return err + } + continue + } + if err := d.body(b.Body, "", "", nesting+1); err != nil { + return err + } + continue + } + p := name + if path != "" { + p = path + "/" + name + } + rules, err := RelabelRuleBlocks() + if err != nil { + return fmt.Errorf("%w: %w", ErrTenantDrop, err) + } + if name == "rule" && rules[component][p] && isLiteralLabelmap(b) && needsTenantDrop(b) && + !isTenantDrop(nextRule(stmts, i)) { + d.insertAfter(b) + } + if err := d.body(b.Body, component, p, nesting+1); err != nil { + return err + } + } + return nil +} + +// insertAfter splices tenantDropRule in right after b's closing brace, on +// lines of its own, indented with the leading whitespace (spaces and tabs, +// byte for byte) of the line b starts on — never a column count, which a tab +// or a multi-byte character before b would throw off. +func (d *refDropper) insertAfter(b *ast.BlockStmt) { + start := ast.StartPos(b).Position().Offset + end := ast.EndPos(b).Position().Offset + 1 + lineStart := strings.LastIndexByte(d.text[:start], '\n') + 1 + line := d.text[lineStart:start] + indent := line[:len(line)-len(strings.TrimLeft(line, " \t"))] + var sb strings.Builder + for _, line := range tenantDropRule { + sb.WriteString(d.newline) + sb.WriteString(indent) + sb.WriteString(line) + } + d.splices = append(d.splices, refDropSplice{start: end, end: end, text: sb.String()}) +} + +// exporterLoki routes otelcol.exporter.loki b through a __tenant_id__ drop: +// its forward_to becomes [loki.process.