diff --git a/CHANGELOG.md b/CHANGELOG.md index a9add735..bec0216c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,15 @@ Categories used here: `pgx5://` included). A URL whose user name, password or database name holds an unencoded `/` or `@` that would split it in the wrong place is refused by both, with a message that names no part of it; percent-encode those characters (`%2F`, `%40`). +- **Shipped — a strict org's served form stops at the 1 MiB limit while it is built.** Inlining + pinned modules and inserting the `__tenant_id__` drops now count every byte against the pipeline + limit as they go, so a pipeline that would expand past 1 MiB (one module imported many times, + nested imports, re-quoted module contents, many drops) is refused as too large when it reaches the + limit instead of after being built in full. A pipeline that expands more than 256 imports + (`import.git` and `import.string`, nested ones and repeats included) is refused as well. Every + pipeline whose served form fits is served byte for byte as before. A pipeline that is too large + and also has another problem (a module not fetched yet, an unpinned revision, nesting too deep) + may now be refused for its size first; fixing that shows the other problem. - **Shipped — group access is bound to the org that granted it.** A group assigned to a collector on its Access tab gives its members viewer access only while the collector's cluster stays in the org that made the assignment. Claiming a cluster into another org removes the diff --git a/docs/spec.md b/docs/spec.md index 7f8b63db..72000cf9 100644 --- a/docs/spec.md +++ b/docs/spec.md @@ -1009,7 +1009,17 @@ every pipeline without an `import.git` block, are served byte for byte as before change: only a strict org's pipeline that uses a pinned import is served differently). The validation gate validates that served form too (`TenantPolicy.Inline`): Stages 1-2 and the tenant lint on the inlined text, refused when it cannot be built or is over the 1 MiB pipeline cap -(`validate.MaxContentBytes`); role/signal enforcement and the editor's derived signals see it as +(`validate.MaxContentBytes`). The cap holds while the form is built, not only after: each level (the +pipeline, each module, each literal `import.string` content) is inlined and given its drops in one +pass (`validate.BuildServedLevel`), so no intermediate text is built, and one byte budget +(`validate.ServedBudget`) is charged for every module text, nested expansion, re-quoted content and +inserted drop before it is built; the build stops the moment the result would pass 1 MiB. What is +charged never exceeds the finished length (a content literal written with escapes the encoder does +not use can re-quote shorter, which is why no intermediate is charged), so a pipeline whose served +form fits is built byte for byte as before. A pipeline that expands more than 256 imports +(`pinnedimport.MaxImportInstances`: `import.git` and `import.string` blocks, nested ones and repeats +included) is refused too. A module imported several times is read and inlined once per build, and +the tenant lint parses each module text once per run; role/signal enforcement and the editor's derived signals see it as well, so a pinned module's declare body's signals count (an `import.git` instance would be Unknown). At serve time an `import.git` that cannot be inlined fails the recompute closed: the collector keeps its last good config and the failure is recorded (`last_failed_seq`, audited), never the raw import. diff --git a/internal/pinnedimport/inline.go b/internal/pinnedimport/inline.go index b6dbcc58..ded64662 100644 --- a/internal/pinnedimport/inline.go +++ b/internal/pinnedimport/inline.go @@ -2,6 +2,7 @@ package pinnedimport import ( "context" + "crypto/sha256" "errors" "fmt" "slices" @@ -20,14 +21,27 @@ import ( // repository, revision or path is not a string literal, the revision is not a // full commit SHA, the repository is not on the allowlist, the module is not // cached, or it nests deeper than validate.MaxGitImportDepth. The serve path -// fails closed on it: a strict org is never served an import.git. +// fails closed on it: a strict org is never served an import.git. It is also +// a served form that would be more than validate.MaxContentBytes, or that +// expands more than MaxImportInstances imports. var ErrNotInlinable = errors.New("pinned import.git cannot be inlined") +// MaxImportInstances is how many import blocks (import.git and +// import.string) a strict org's pipeline may expand as served, nested ones +// included and a module imported twice counted twice. The pipelines in use +// have a handful; each instance is a module Alloy loads and runs. +const MaxImportInstances = 256 + // Ref is one pinned module, as the cache keys it. type Ref struct { Key, Revision, Path string } +// moduleQuerier is how the walker reads the cache (*sqlc.Queries). +type moduleQuerier interface { + GetPinnedImportModule(ctx context.Context, arg sqlc.GetPinnedImportModuleParams) (sqlc.PinnedImportModule, error) +} + // Inline is what a strict org is served for contents (#390): every // import.git block — at the top level or inside a declare — rewritten into // `import.string "" { content = "" }`, where module is @@ -47,16 +61,30 @@ type Ref struct { // is returned unchanged. // // Only the cache is read, through q. The result is at most -// validate.MaxContentBytes (the pipeline contents cap). +// validate.MaxContentBytes (the pipeline contents cap). It is built in two +// walks: the first checks every import (what can be inlined, at most +// MaxImportInstances of them) and builds nothing; the second builds the +// served form directly — each level's modules inlined and drops inserted +// in one pass (validate.BuildServedLevel) — charging one +// validate.ServedBudget of the cap as it goes, so a pipeline whose served +// form would pass the cap is refused when it gets there, never built in +// full, and one whose served form fits is built exactly as before. func Inline(ctx context.Context, q *sqlc.Queries, orgID pgtype.UUID, repos []sqlc.OrgImportRepository, contents string) (string, error) { + return inline(ctx, q, orgID, repos, contents) +} + +func inline(ctx context.Context, q moduleQuerier, orgID pgtype.UUID, repos []sqlc.OrgImportRepository, contents string) (string, error) { w := newWalker(ctx, q, orgID, repos) - out, err := w.inline(contents, 0, 0) - if err != nil { + if err := w.check(contents, 0, 0, false); err != nil { return "", err } - if out, err = validate.InsertTenantDrops(out); err != nil { + out, err := w.build(contents, 0, 0) + if err != nil { return "", fmt.Errorf("%w: %w", ErrNotInlinable, err) } + // A text the build leaves unchanged charges nothing (it builds + // nothing), so this is where one over the cap is refused; a changed + // one over it has already failed the budget. if len(out) > validate.MaxContentBytes { return "", fmt.Errorf("%w: as served (pinned modules inlined, __tenant_id__ dropped after "+ "each labelmap) the pipeline is %d bytes, more than the %d a "+ @@ -78,25 +106,64 @@ func InlineOrg(ctx context.Context, q *sqlc.Queries, orgID pgtype.UUID, contents // modules — what must stay cached while contents is enabled. A module that // is not cached is listed, and what it would import is not. func References(ctx context.Context, q *sqlc.Queries, orgID pgtype.UUID, repos []sqlc.OrgImportRepository, contents string) ([]Ref, error) { + return references(ctx, q, orgID, repos, contents) +} + +func references(ctx context.Context, q moduleQuerier, orgID pgtype.UUID, repos []sqlc.OrgImportRepository, contents string) ([]Ref, error) { w := newWalker(ctx, q, orgID, repos) - w.collect = true - if _, err := w.inline(contents, 0, 0); err != nil { - return nil, err - } + w.collect(contents, 0, 0) return w.refs, nil } type walker struct { - ctx context.Context - q *sqlc.Queries - orgID pgtype.UUID - repos map[string]bool - collect bool - refs []Ref + ctx context.Context + q moduleQuerier + orgID pgtype.UUID + repos map[string]bool + refs []Ref + // budget is the served form's (Inline); instances counts the imports + // the check expands (MaxImportInstances). + budget *validate.ServedBudget + instances int + // checked are the modules the check has walked, by where: how many + // imports walking them expanded and how many bytes of served form they + // make at least. built are the same modules' served form, quoted (part + // of what the budget is charged). One module imported many times is + // walked and built once per place; its text is held only while its + // level is walked or built, never between the walks. visited is + // References' record of the same. + checked map[moduleAt]checkedModule + built map[moduleAt]string + visited map[moduleAt]bool + // inlines records, for each text the check walked (by its SHA-256, so + // no text is held), whether inlining rewrites it: it holds an + // import.git, directly or in the content of an import.string it + // follows. Exactly then the old inliner re-quoted an import.string + // content (build reads the line ending accordingly). + inlines map[[sha256.Size]byte]bool + // atLeast is a lower bound on the served form's length, summed as the + // check walks (see check): past the cap, the pipeline is refused + // before anything more is read. + atLeast int +} + +// moduleAt is a module at the depths it is inlined at: what inlining it +// depends on, besides the cache. +type moduleAt struct { + ref Ref + gitDepth, depth int +} + +type checkedModule struct { + instances, atLeast int } -func newWalker(ctx context.Context, q *sqlc.Queries, orgID pgtype.UUID, repos []sqlc.OrgImportRepository) *walker { - w := &walker{ctx: ctx, q: q, orgID: orgID, repos: map[string]bool{}} +func newWalker(ctx context.Context, q moduleQuerier, orgID pgtype.UUID, repos []sqlc.OrgImportRepository) *walker { + w := &walker{ + ctx: ctx, q: q, orgID: orgID, repos: map[string]bool{}, budget: validate.NewServedBudget(), + checked: map[moduleAt]checkedModule{}, built: map[moduleAt]string{}, visited: map[moduleAt]bool{}, + inlines: map[[sha256.Size]byte]bool{}, + } for i := range repos { w.repos[repos[i].RepositoryKey] = true } @@ -107,101 +174,271 @@ func newWalker(ctx context.Context, q *sqlc.Queries, orgID pgtype.UUID, repos [] // follows into: the tenant lint's module depth. const maxModuleDepth = 8 -// splice is one rewrite of a byte range of a text. -type splice struct { - start, end int // [start, end) - text string -} - -// inline rewrites text's import.git blocks — found in the parse tree, so -// whatever spelling of the name the parser accepts — and descends into each -// import.string whose content is a plain literal, rewriting the module text -// it holds the same way. gitDepth is how many import.git modules enclose -// text, depth how many modules of either kind. Text with nothing to rewrite -// is returned unchanged; otherwise only the rewritten byte ranges change, -// located by the parsed nodes' positions. -func (w *walker) inline(text string, gitDepth, depth int) (string, error) { +// check walks text's import blocks — found in the parse tree, so whatever +// spelling of the name the parser accepts: each import.git must be +// inlinable (a literal, allowlisted, cached module, within the depths) and +// each import.string with plain literal content is followed into, the same +// way. gitDepth is how many import.git modules enclose text, depth how many +// modules of either kind. It builds nothing; its first refusal is Inline's, +// in the order Inline has always reported them. +// +// It also sums atLeast, a lower bound on the served form's length: every +// level that is served as a text of its own — the pipeline and each +// content it rewrites, and each module instance — adds its bytes outside +// every import block (importBytes). Those bytes appear in the served form, +// once per instance, re-quoted (never shorter); the drops only add to them +// (a forward_to they replace is served again in the loki.process they +// insert); and different levels' counted bytes are disjoint. A pipeline +// whose bound passes the cap is refused at once. +func (w *walker) check(text string, gitDepth, depth int, module bool) error { if depth > maxModuleDepth { - return "", fmt.Errorf("%w: modules nest more than %d deep", ErrNotInlinable, maxModuleDepth) + return fmt.Errorf("%w: modules nest more than %d deep", ErrNotInlinable, maxModuleDepth) } file, err := alloysyntax.ParseFile("", []byte(text)) if err != nil { - if w.collect { - return text, nil - } - return "", fmt.Errorf("%w: the text does not parse: %w", ErrNotInlinable, err) + return fmt.Errorf("%w: the text does not parse: %w", ErrNotInlinable, err) } var blocks []*ast.BlockStmt collectImports(file.Body, &blocks, 0) - var splices []splice + inlines := false for _, b := range blocks { - var sp *splice - var err error - if strings.Join(b.Name, ".") == "import.git" { - sp, err = w.gitBlock(b, gitDepth, depth) - } else { - sp, err = w.stringBlock(b, gitDepth, depth) + if err := w.expand(1); err != nil { + return err } + if strings.Join(b.Name, ".") != "import.git" { + if a := firstContentAttr(b); a != nil { + if content, ok := validate.PlainStringLiteral(a.Value); ok { + if err := w.check(content, gitDepth, depth+1, false); err != nil { + return err + } + inlines = inlines || w.inlines[sha256.Sum256([]byte(content))] + } + } + continue + } + inlines = true + if gitDepth >= validate.MaxGitImportDepth { + return fmt.Errorf("%w: import.git nests more than %d deep", ErrNotInlinable, validate.MaxGitImportDepth) + } + ref, err := w.ref(b) if err != nil { - if w.collect { - continue + return err + } + at := moduleAt{ref: ref, gitDepth: gitDepth, depth: depth} + if m, ok := w.checked[at]; ok { + if err := w.expand(m.instances); err != nil { + return err } - return "", err + if err := w.atLeastMore(m.atLeast); err != nil { + return err + } + continue + } + content, err := w.module(ref) + if err != nil { + return err } - if sp != nil { - splices = append(splices, *sp) + before, beforeBytes := w.instances, w.atLeast + if err := w.check(content, gitDepth+1, depth+1, true); err != nil { + return err } + w.checked[at] = checkedModule{instances: w.instances - before, atLeast: w.atLeast - beforeBytes} } - if len(splices) == 0 { - return text, nil + w.inlines[sha256.Sum256([]byte(text))] = inlines + // A content the inliner leaves alone is part of its parent's kept + // bytes; the pipeline left alone is held to the cap by Inline. + if module || inlines { + return w.atLeastMore(len(text) - importBytes(file.Body, 0)) } - // Apply from the last range back, so earlier offsets stay valid. - slices.SortFunc(splices, func(a, b splice) int { return b.start - a.start }) - out := text - for _, sp := range splices { - out = out[:sp.start] + sp.text + out[sp.end:] + return nil +} + +// importBytes is how many bytes of a text its import blocks span, at any +// block depth: what the served form may rewrite there (a content the drops +// re-quote may come out shorter than written). The rest is served as it +// is, or with only drops added. +func importBytes(body ast.Body, nesting int) int { + if nesting > alloysyntax.MaxNesting { + return 0 } - return out, nil + n := 0 + for _, st := range body { + b, ok := st.(*ast.BlockStmt) + if !ok { + continue + } + if len(b.Name) > 0 && b.Name[0] == "import" { + n += ast.EndPos(b).Position().Offset + 1 - ast.StartPos(b).Position().Offset + continue + } + n += importBytes(b.Body, nesting+1) + } + return n } -// gitBlock rewrites one import.git block into an import.string of its -// cached module, itself inlined. -func (w *walker) gitBlock(b *ast.BlockStmt, gitDepth, depth int) (*splice, error) { - if gitDepth >= validate.MaxGitImportDepth { - return nil, fmt.Errorf("%w: import.git nests more than %d deep", ErrNotInlinable, validate.MaxGitImportDepth) +// atLeastMore adds n bytes to the served form's lower bound, refusing it +// past the cap. +func (w *walker) atLeastMore(n int) error { + w.atLeast += n + if w.atLeast > validate.MaxContentBytes { + return fmt.Errorf("%w: %w", ErrNotInlinable, validate.ErrServedFormTooLarge) } - ref, err := w.ref(b) - if err != nil { - return nil, err + return nil +} + +// firstContent is the value of an import.string's first content attribute, +// when that is a plain literal: what Inline follows into. +func firstContent(b *ast.BlockStmt) (string, bool) { + if a := firstContentAttr(b); a != nil { + return validate.PlainStringLiteral(a.Value) } - if w.collect && !slices.Contains(w.refs, ref) { - w.refs = append(w.refs, ref) + return "", false +} + +// firstContentAttr is an import.string's first content attribute, or nil. +func firstContentAttr(b *ast.BlockStmt) *ast.AttributeStmt { + for _, st := range b.Body { + if a, ok := st.(*ast.AttributeStmt); ok && a.Name.Name == "content" { + return a + } } - row, err := w.q.GetPinnedImportModule(w.ctx, sqlc.GetPinnedImportModuleParams{ - OrgID: w.orgID, RepositoryKey: ref.Key, Revision: ref.Revision, Path: ref.Path, + return nil +} + +// expand counts n more import instances, refusing more than +// MaxImportInstances. +func (w *walker) expand(n int) error { + w.instances += n + if w.instances > MaxImportInstances { + return fmt.Errorf("%w: as served the pipeline expands more than %d imports (import.git and "+ + "import.string blocks, nested ones and repeats included)", ErrNotInlinable, MaxImportInstances) + } + return nil +} + +// build is the served form of one checked level, text: the drop pass +// (validate.BuildServedLevel) with the level's import.git blocks served as +// import.string of their module's served form, and its literal +// import.string contents as their own served form, re-quoted when that +// differs — byte for byte what inlining text and then inserting the drops +// made, without building the inlined text. Every piece is charged to the +// budget before it is built. +func (w *walker) build(text string, gitDepth, depth int) (string, error) { + return validate.BuildServedLevel(text, depth, w.budget, func(file *ast.File) validate.ServedImports { + var blocks []*ast.BlockStmt + collectImports(file.Body, &blocks, 0) + owned := make(map[*ast.BlockStmt]bool, len(blocks)) + var replaced [][2]int + // The drop pass reads the level's line ending from the text the + // old inliner left: without each import.git block, and without + // each import.string content it re-quoted (QuoteString writes no + // raw CR), which is the first content, a plain literal, exactly + // when inlining rewrites it (inlines). + for _, b := range blocks { + owned[b] = true + if strings.Join(b.Name, ".") == "import.git" { + replaced = append(replaced, [2]int{ast.StartPos(b).Position().Offset, ast.EndPos(b).Position().Offset + 1}) + continue + } + if a := firstContentAttr(b); a != nil { + if content, ok := validate.PlainStringLiteral(a.Value); ok && w.inlines[sha256.Sum256([]byte(content))] { + replaced = append(replaced, [2]int{ + ast.StartPos(a.Value).Position().Offset, ast.EndPos(a.Value).Position().Offset + 1, + }) + } + } + } + return validate.ServedImports{ + Replaced: replaced, + Serve: func(b *ast.BlockStmt) ([]validate.Splice, bool, error) { + if !owned[b] { + return nil, false, nil + } + var splices []validate.Splice + var err error + if strings.Join(b.Name, ".") == "import.git" { + splices, err = w.gitSplice(b, gitDepth, depth) + } else { + splices, err = w.stringSplices(b, gitDepth, depth) + } + return splices, true, err + }, + } }) - if err != nil { - return nil, fmt.Errorf("%w: module %s at %s (%s) is not cached", ErrNotInlinable, ref.Path, ref.Revision, ref.Key) +} + +// quoted is the splice of a nested level's served form, served, in place +// of [start, end): quoted. served is charged; what is served is it quoted, +// at least as long, charged instead before it is built. +func (w *walker) quoted(served string, changed bool) (string, error) { + if changed { + w.budget.Give(len(served)) + } + if err := w.budget.Take(validate.QuotedLen(served)); err != nil { + return "", err } - module, err := w.inline(row.Content, gitDepth+1, depth+1) + return alloysyntax.QuoteString(served), nil +} + +// gitSplice serves one checked import.git block as an import.string of its +// cached module's served form. +func (w *walker) gitSplice(b *ast.BlockStmt, gitDepth, depth int) ([]validate.Splice, error) { + ref, err := w.ref(b) if err != nil { return nil, err } + at := moduleAt{ref: ref, gitDepth: gitDepth, depth: depth} + // A module built for an earlier instance is reused; each instance + // charges its copy below. + quoted, ok := w.built[at] + if !ok { + content, err := w.module(ref) + if err != nil { + return nil, err + } + served, err := w.build(content, gitDepth+1, depth+1) + if err != nil { + return nil, fmt.Errorf("import.string %q: %w", b.Label, err) + } + // The quoted module is built once, here, and is part of the first + // instance's import.string: it must fit before it is built (the + // instance charges it with the rest of the block below). + if quoted, err = w.quoted(served, served != content); err != nil { + return nil, fmt.Errorf("import.string %q: %w", b.Label, err) + } + w.budget.Give(len(quoted)) + w.built[at] = quoted + } start := ast.StartPos(b).Position() - indent := strings.Repeat(" ", max(start.Column-1, 0)) - return &splice{ - start: start.Offset, - end: ast.EndPos(b).Position().Offset + 1, - text: "import.string " + alloysyntax.QuoteString(b.Label) + " {\n" + - indent + " content = " + alloysyntax.QuoteString(module) + "\n" + indent + "}", - }, nil + columns := max(start.Column-1, 0) + const head, open, content, closing = "import.string ", " {\n", " content = ", "\n" + n := len(head) + validate.QuotedLen(b.Label) + len(open) + columns + len(content) + len(quoted) + + len(closing) + columns + 1 + if err := w.budget.Take(n); err != nil { + return nil, err + } + indent := strings.Repeat(" ", columns) + var sb strings.Builder + sb.Grow(n) + for _, s := range []string{head, alloysyntax.QuoteString(b.Label), open, indent, content, quoted, closing, indent, "}"} { + sb.WriteString(s) + } + return []validate.Splice{{ + Start: start.Offset, + End: ast.EndPos(b).Position().Offset + 1, + Text: sb.String(), + }}, nil } -// stringBlock rewrites the content of one import.string whose module text -// holds an import.git (at any depth): only its content literal is replaced, -// re-quoted. Content that is not a plain literal is left alone (the served -// form's bound, validate.ServedFormProblem, refuses it in a strict org). -func (w *walker) stringBlock(b *ast.BlockStmt, gitDepth, depth int) (*splice, error) { +// stringSplices serves one import.string block: each content attribute +// that is a plain literal, up to the first that is not, is replaced by its +// served form, re-quoted, when that differs — the first as a level of its +// own (inlined and dropped), any later one with the drops only, as the drop +// pass always treated them. Other content is left alone (the served form's +// bound, validate.ServedFormProblem, refuses it in a strict org). +func (w *walker) stringSplices(b *ast.BlockStmt, gitDepth, depth int) ([]validate.Splice, error) { + var splices []validate.Splice + first := true for _, st := range b.Body { a, ok := st.(*ast.AttributeStmt) if !ok || a.Name.Name != "content" { @@ -209,22 +446,88 @@ func (w *walker) stringBlock(b *ast.BlockStmt, gitDepth, depth int) (*splice, er } content, ok := validate.PlainStringLiteral(a.Value) if !ok { - return nil, nil //nolint:nilnil // nothing this walker can rewrite + break + } + var served string + var err error + if first { + served, err = w.build(content, gitDepth, depth+1) + } else { + served, err = validate.BuildServedLevel(content, depth+1, w.budget, nil) } - inner, err := w.inline(content, gitDepth, depth+1) + first = false if err != nil { - return nil, err + return nil, fmt.Errorf("import.string %q: %w", b.Label, err) + } + if served == content { + continue + } + quoted, err := w.quoted(served, true) + if err != nil { + return nil, fmt.Errorf("import.string %q: %w", b.Label, err) + } + splices = append(splices, validate.Splice{ + Start: ast.StartPos(a.Value).Position().Offset, + End: ast.EndPos(a.Value).Position().Offset + 1, + Text: quoted, + }) + } + return splices, nil +} + +// collect is References' walk: every import.git reached through cached +// modules and literal import.string contents, recorded in order; what +// cannot be followed is skipped. +func (w *walker) collect(text string, gitDepth, depth int) { + if depth > maxModuleDepth { + return + } + file, err := alloysyntax.ParseFile("", []byte(text)) + if err != nil { + return + } + var blocks []*ast.BlockStmt + collectImports(file.Body, &blocks, 0) + for _, b := range blocks { + if strings.Join(b.Name, ".") != "import.git" { + if content, ok := firstContent(b); ok { + w.collect(content, gitDepth, depth+1) + } + continue } - if inner == content { - return nil, nil //nolint:nilnil // unchanged: the bytes stay as written + if gitDepth >= validate.MaxGitImportDepth { + continue } - return &splice{ - start: ast.StartPos(a.Value).Position().Offset, - end: ast.EndPos(a.Value).Position().Offset + 1, - text: alloysyntax.QuoteString(inner), - }, nil + ref, err := w.ref(b) + if err != nil { + continue + } + if !slices.Contains(w.refs, ref) { + w.refs = append(w.refs, ref) + } + // A second visit at the same depths lists nothing new. + at := moduleAt{ref: ref, gitDepth: gitDepth, depth: depth} + if w.visited[at] { + continue + } + w.visited[at] = true + content, err := w.module(ref) + if err != nil { + continue + } + w.collect(content, gitDepth+1, depth+1) + } +} + +// module is ref's cached text. +func (w *walker) module(ref Ref) (string, error) { + row, err := w.q.GetPinnedImportModule(w.ctx, sqlc.GetPinnedImportModuleParams{ + OrgID: w.orgID, RepositoryKey: ref.Key, Revision: ref.Revision, Path: ref.Path, + }) + if err != nil { + return "", fmt.Errorf("%w: module %s at %s (%s) is not cached", ErrNotInlinable, ref.Path, ref.Revision, ref.Key) } - return nil, nil //nolint:nilnil // no content attribute + return row.Content, nil } // collectImports finds the import blocks Alloy loads — import.git and diff --git a/internal/pinnedimport/inline_budget_test.go b/internal/pinnedimport/inline_budget_test.go new file mode 100644 index 00000000..7a2acfbb --- /dev/null +++ b/internal/pinnedimport/inline_budget_test.go @@ -0,0 +1,558 @@ +package pinnedimport + +import ( + "context" + "errors" + "fmt" + "math/rand" + "os" + "reflect" + "runtime" + "strconv" + "strings" + "testing" + "time" + "unicode/utf8" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgtype" + + "shepherd/internal/alloysyntax" + "shepherd/internal/store/sqlc" + "shepherd/internal/validate" +) + +// The served-form budget must not change what a strict org is served: +// Inline returns what the pre-budget one (refInline, oldinline_ref_test.go) +// did whenever that fits, refuses whenever it did, and References lists the +// same modules in the same order. + +const ( + budgetRepo = "https://git.example.com/team/modules.git" + budgetSHA = "0123456789abcdef0123456789abcdef01234567" +) + +// fakeModules is a module cache: path -> content, under budgetRepo at +// budgetSHA. +type fakeModules map[string]string + +func (f fakeModules) GetPinnedImportModule(_ context.Context, a sqlc.GetPinnedImportModuleParams) (sqlc.PinnedImportModule, error) { + key, err := NormalizeRepository(budgetRepo) + if err != nil { + return sqlc.PinnedImportModule{}, err + } + c, ok := f[a.Path] + if !ok || a.RepositoryKey != key || a.Revision != budgetSHA { + return sqlc.PinnedImportModule{}, pgx.ErrNoRows + } + return sqlc.PinnedImportModule{Content: c}, nil +} + +func budgetRepos(t *testing.T) []sqlc.OrgImportRepository { + t.Helper() + key, err := NormalizeRepository(budgetRepo) + if err != nil { + t.Fatal(err) + } + return []sqlc.OrgImportRepository{{RepositoryKey: key}} +} + +func importGit(ind, label, path string) string { + return ind + "import.git " + strconv.Quote(label) + " {\n" + ind + " repository = \"" + budgetRepo + "\"\n" + ind + + " revision = \"" + budgetSHA + "\"\n" + ind + " path = " + strconv.Quote(path) + "\n" + ind + "}\n" +} + +// inlineGen writes random pipelines and modules: pinned import.git blocks +// (cached, not cached, refused for their revision, repository or a +// computed path), import.string contents holding them, declares, labelmaps +// and exporters the drop pass rewrites, odd columns and bytes. +type inlineGen struct { + r *rand.Rand +} + +func (g *inlineGen) pick(xs ...string) string { return xs[g.r.Intn(len(xs))] } + +func (g *inlineGen) stmt(depth, imports int, ind string) string { + label := g.pick("a", "b", "m", "é", "with space") + switch g.r.Intn(11) { + case 0, 1, 2: + if imports <= 0 { + return "" + } + path := g.pick("m0.alloy", "m1.alloy", "m2.alloy", "m3.alloy", "dir/m4.alloy", "missing.alloy") + switch g.r.Intn(12) { + case 0: + return strings.Replace(importGit(ind, label, path), budgetSHA, "main", 1) + case 1: + return strings.Replace(importGit(ind, label, path), budgetRepo, "https://elsewhere.example.com/x.git", 1) + case 2: + return strings.Replace(importGit(ind, label, path), strconv.Quote(path), `"m" + "0.alloy"`, 1) + case 3: // not at column 1 + return ind + "/* " + g.pick("x", "\t", "é") + " */ " + importGit("", label, path) + case 4: + return strings.Replace(importGit(ind, label, path), ind+"}\n", + ind+" basic_auth {\n"+ind+" username = \"u\"\n"+ind+" }\n"+ind+"}\n", 1) + default: + return importGit(ind, label, path) + } + case 3: + if depth > 2 { + return "" + } + inner := g.text(depth+1, imports-1) + return ind + "import.string " + strconv.Quote(label) + " {\n" + ind + " content = " + + g.pick(alloysyntax.QuoteString(inner), noncanonicalQuote(g.r, inner), noncanonicalQuote(g.r, inner), rawLiteral(inner), + "local.file.f.content") + "\n" + ind + "}\n" + case 4: + if depth > 2 { + return "" + } + return ind + "declare " + strconv.Quote(label) + " {\n" + g.stmt(depth+1, imports, ind+" ") + ind + "}\n" + case 5: + return ind + "prometheus.relabel " + strconv.Quote(label) + " {\n" + ind + " forward_to = []\n" + ind + + " rule {\n" + ind + " action = \"labelmap\"\n" + ind + " }\n" + ind + "}\n" + case 6: + return ind + "otelcol.exporter.loki " + strconv.Quote(label) + " {\n" + ind + + " forward_to = [loki.write.a.receiver]\n" + ind + "}" + g.pick("", " // c") + "\n" + case 7: + return ind + "// " + g.pick("\t\x01", "é", "import.git \"x\" {}", "") + "\n" + case 8: + return ind + "x = " + strconv.Quote(g.pick("\t", "\x01", "a\"b", "😀")) + "\n" + case 9: + if g.r.Intn(10) == 0 { + return ind + g.pick("{", "import.git {", "\"") + "\n" + } + return "" + default: + return "" + } +} + +func (g *inlineGen) text(depth, imports int) string { + var sb strings.Builder + for range g.r.Intn(5) { + sb.WriteString(g.stmt(depth, imports, g.pick("", "", " ", "\t"))) + } + return lineEndings(g.r, sb.String()) +} + +func (g *inlineGen) modules() fakeModules { + m := fakeModules{} + for _, p := range []string{"m0.alloy", "m1.alloy", "m2.alloy", "m3.alloy", "dir/m4.alloy"} { + if g.r.Intn(8) != 0 { + m[p] = g.text(1, 2) + } + } + return m +} + +func errText(err error) string { + if err == nil { + return "" + } + return err.Error() +} + +// checkInlineMatches fails t unless Inline and References agree with the +// reference on contents. +func checkInlineMatches(t *testing.T, mods fakeModules, repos []sqlc.OrgImportRepository, contents string, sameErrors, withRefs bool) bool { + t.Helper() + ctx := context.Background() + org := pgtype.UUID{} + want, wantErr := refInline(ctx, mods, org, repos, contents) + got, gotErr := inline(ctx, mods, org, repos, contents) + if withRefs { + wantRefs, wantRefsErr := refReferences(ctx, mods, org, repos, contents) + gotRefs, gotRefsErr := references(ctx, mods, org, repos, contents) + if !reflect.DeepEqual(gotRefs, wantRefs) || errText(gotRefsErr) != errText(wantRefsErr) { + t.Fatalf("References differ:\n got %v %v\n want %v %v\nfor %q", gotRefs, gotRefsErr, wantRefs, wantRefsErr, contents) + } + } + if wantErr != nil { + if gotErr == nil { + t.Fatalf("the reference refuses (%v), the budgeted Inline serves %d bytes for %q", wantErr, len(got), contents) + } + if sameErrors && gotErr.Error() != wantErr.Error() { + t.Fatalf("refusals differ:\n got %v\n want %v\nfor %q", gotErr, wantErr, contents) + } + return false + } + if gotErr != nil { + t.Fatalf("the reference serves %d bytes, the budgeted Inline refuses: %v\nfor %q", len(want), gotErr, contents) + } + if got != want { + t.Fatalf("served bytes differ for %q:\n got %q\n want %q", contents, got, want) + } + return true +} + +func TestInlineMatchesReference(t *testing.T) { + g := &inlineGen{r: rand.New(rand.NewSource(390))} + repos := budgetRepos(t) + const n = 60000 + var mods fakeModules + served, inlined := 0, 0 + for i := range n { + if i%50 == 0 { + mods = g.modules() + } + contents := g.text(0, 3) + if checkInlineMatches(t, mods, repos, contents, true, true) { + served++ + if out, err := refInline(context.Background(), mods, pgtype.UUID{}, repos, contents); err == nil && out != contents { + inlined++ + } + } + } + t.Logf("%d random pipelines: %d served (%d changed as served), %d refused alike", n, served, inlined, n-served) + if served < n/4 || inlined < n/10 { + t.Fatalf("the corpus is too thin: %d served, %d changed", served, inlined) + } +} + +// Near the cap: a module imported one to four times (directly, through an +// import.string, or from another module) whose served form is within a few +// hundred bytes of the cap either side; or, beside it, an import.string +// whose content is written with a for "a" and holds a labelmap, so +// the drops make it re-quote shorter: the text inlined before the drops are +// inserted passes the cap while the served form may fit. 36 pipelines by +// default; SERVED_BUDGET_FULL=1 runs 400 (the same seed: the first 36 are +// the same). +func TestInlineMatchesReferenceNearCap(t *testing.T) { + g := &inlineGen{r: rand.New(rand.NewSource(3900))} + repos := budgetRepos(t) + n := 36 + if os.Getenv("SERVED_BUDGET_FULL") == "1" { + n = 400 + } + under, over, shrunk := 0, 0, 0 + // The served size is linear in the padding; its two coefficients + // depend only on the shape. + type shape struct { + copies, mix, kind int + } + linear := map[shape][2]float64{} + for i := range n { + copies := 1 + g.r.Intn(4) + kind := i % 3 // 0: "a" in the module, 1: tabs in it, 2: a literal that shrinks + build := func(pad int) (fakeModules, string) { + filler := strings.Repeat("a", pad) + switch kind { + case 1: + filler = strings.Repeat("\t", pad) + case 2: + filler = strings.Repeat("a", 600000/copies) + } + mods := fakeModules{ + "m0.alloy": "prometheus.relabel \"r\" {\n forward_to = []\n rule {\n action = \"labelmap\"\n }\n}\n" + + "x = \"" + filler + "\"\n", + "m1.alloy": importGit("", "inner", "m0.alloy"), + } + var sb strings.Builder + for c := range copies { + label := "m" + strconv.Itoa(c) + switch (i + c) % 3 { + case 0: + sb.WriteString(importGit("", label, "m0.alloy")) + case 1: + sb.WriteString(importGit(" ", label, "m1.alloy")) + default: + sb.WriteString("import.string " + strconv.Quote(label) + " {\n content = " + + alloysyntax.QuoteString(importGit("", "x", "m0.alloy")) + "\n}\n") + } + } + if kind == 2 { + sb.WriteString(shrinkingImport(pad)) + } + return mods, sb.String() + } + size := func(pad int) int { + mods, contents := build(pad) + out, err := refInline(context.Background(), mods, pgtype.UUID{}, repos, contents) + if err != nil { + t.Fatalf("near-cap base refused: %v", err) + } + return len(out) + } + sh := shape{copies: copies, mix: i % 3, kind: kind} + if _, ok := linear[sh]; !ok { + l0, l1 := size(0), size(1000) + linear[sh] = [2]float64{float64(l0), float64(l1-l0) / 1000} + } + l0, perPad := linear[sh][0], linear[sh][1] + target := validate.MaxContentBytes + g.r.Intn(601) - 300 + pad := max(int((float64(target)-l0)/perPad), 0) + mods, contents := build(pad) + served := checkInlineMatches(t, mods, repos, contents, false, false) + if served && kind == 2 && inlinedLen(t, mods, repos, contents) > validate.MaxContentBytes { + shrunk++ + } + if served { + under++ + } else { + over++ + } + } + t.Logf("%d near-cap pipelines: %d served identically (%d inlined past the cap before the drops shrank them), "+ + "%d refused alike", n, under, shrunk, over) + if under < n/5 || over < n/5 || shrunk == 0 { + t.Fatalf("the near-cap corpus does not straddle the cap: %d under (%d shrunk), %d over", under, shrunk, over) + } +} + +// shrinkingImport is an import.string holding a labelmap and a pad-long +// string of "a", each written a: the drops make the content re-quote +// five bytes shorter per "a". +func shrinkingImport(pad int) string { + return "import.string \"s\" {\n content = \"" + + `prometheus.relabel \"r\" {\n forward_to = []\n rule {\n action = \"labelmap\"\n }\n}\n` + + `y = \"` + strings.Repeat(`\u0061`, pad) + `\"\n` + "\"\n}\n" +} + +// inlinedLen is how long the text the old Inline built before inserting +// the drops was. +func inlinedLen(t *testing.T, mods fakeModules, repos []sqlc.OrgImportRepository, contents string) int { + t.Helper() + out, err := newRefWalker(context.Background(), mods, pgtype.UUID{}, repos).inline(contents, 0, 0) + if err != nil { + t.Fatalf("inlining: %v", err) + } + return len(out) +} + +// A pipeline the drops shrink back under the cap is served as before +// (#450 review): the old Inline inlined it past the cap (the module plus +// the import.string as written) and inserted the drops, which re-quote the +// content's a escapes as "a"; the served form fits. +func TestInlineServesWhatTheDropsShrinkUnderTheCap(t *testing.T) { + repos := budgetRepos(t) + mods := fakeModules{"m.alloy": "x = \"" + strings.Repeat("a", 600_000) + "\"\n"} + contents := importGit("", "a", "m.alloy") + shrinkingImport(90_000) + if inlinedLen(t, mods, repos, contents) <= validate.MaxContentBytes { + t.Fatal("the inlined text fits: not the shape under test") + } + if !checkInlineMatches(t, mods, repos, contents, true, true) { + t.Fatal("the old Inline refuses it: not the shape under test") + } +} + +// noncanonicalQuote writes s as a string literal with escapes chosen at +// random — \xHH, octal, \uXXXX, \UXXXXXXXX or the encoder's own — so the +// literal decodes to s but is not what alloysyntax.QuoteString writes. +func noncanonicalQuote(r *rand.Rand, s string) string { + var sb strings.Builder + sb.WriteByte('"') + for i := 0; i < len(s); { + c, w := utf8.DecodeRuneInString(s[i:]) + canonical := strconv.Quote(s[i : i+w]) + canonical = canonical[1 : len(canonical)-1] + switch k := r.Intn(6); { + case c == utf8.RuneError && w == 1: + fmt.Fprintf(&sb, `\x%02x`, s[i]) + case k == 0 && w == 1: + fmt.Fprintf(&sb, `\x%02X`, s[i]) + case k == 1 && w == 1: + fmt.Fprintf(&sb, `\%03o`, s[i]) + case k == 2 && c < 0x10000: + fmt.Fprintf(&sb, `\u%04x`, c) + case k == 3: + fmt.Fprintf(&sb, `\U%08x`, c) + default: + sb.WriteString(canonical) + } + i += w + } + sb.WriteByte('"') + return sb.String() +} + +// The import-instance cap counts every import expanded, repeats and nested +// ones included. +func TestInlineCapsImportInstances(t *testing.T) { + repos := budgetRepos(t) + ctx := context.Background() + mods := fakeModules{"leaf.alloy": "declare \"d\" {\n}\n"} + repeated := func(n int, path string) string { + var sb strings.Builder + for i := range n { + sb.WriteString(importGit("", "m"+strconv.Itoa(i), path)) + } + return sb.String() + } + if !checkInlineMatches(t, mods, repos, repeated(MaxImportInstances, "leaf.alloy"), true, true) { + t.Fatalf("%d imports of one module are refused", MaxImportInstances) + } + _, err := inline(ctx, mods, pgtype.UUID{}, repos, repeated(MaxImportInstances+1, "leaf.alloy")) + if !errors.Is(err, ErrNotInlinable) || !strings.Contains(err.Error(), fmt.Sprintf("more than %d imports", MaxImportInstances)) { + t.Fatalf("%d imports: %v", MaxImportInstances+1, err) + } + // Fan-out 4, four levels deep: 4+16+64+256 instances of tiny modules. + mods = fakeModules{"l3.alloy": "declare \"d\" {\n}\n"} + for _, l := range []string{"l2", "l1", "l0"} { + next := map[string]string{"l2": "l3", "l1": "l2", "l0": "l1"}[l] + mods[l+".alloy"] = repeated(4, next+".alloy") + } + _, err = inline(ctx, mods, pgtype.UUID{}, repos, repeated(4, "l0.alloy")) + if !errors.Is(err, ErrNotInlinable) || !strings.Contains(err.Error(), "imports") { + t.Fatalf("nested fan-out: %v", err) + } +} + +// The shapes that made Inline build far more than the cap from a pipeline +// within it. Each must be refused having allocated, beyond what parsing its +// texts costs, at most twice the cap. SERVED_BUDGET_REFERENCE=1 runs the +// same checks on the pre-budget Inline, which fails them. +func TestInlineStaysWithinTheBudget(t *testing.T) { + fn := inline + if os.Getenv("SERVED_BUDGET_REFERENCE") == "1" { + fn = refInline + } + repos := budgetRepos(t) + labelmap := "prometheus.relabel \"r\" {\n forward_to = []\n rule {\n action = \"labelmap\"\n }\n}\n" + repeated := func(n int, path string) string { + var sb strings.Builder + for i := range n { + sb.WriteString(importGit("", "m"+strconv.Itoa(i), path)) + } + return sb.String() + } + fanOut := fakeModules{"d.alloy": labelmap + "x = \"" + strings.Repeat("\t", 20<<10) + "\"\n"} + for _, l := range []string{"c", "b", "a"} { + next := map[string]string{"c": "d", "b": "c", "a": "b"}[l] + fanOut[l+".alloy"] = repeated(3, next+".alloy") + } + for _, tc := range []struct { + name string + mods fakeModules + contents string + }{ + { + // One 50 KiB module imported 200 times. + name: "one module many times", + mods: fakeModules{"big.alloy": labelmap + "// " + strings.Repeat("a", 50<<10) + "\n"}, + contents: repeated(200, "big.alloy"), + }, + { + // Fan-out 3 through four levels of modules: 81 copies of a + // 20 KiB leaf of tabs, re-escaped at every level. + name: "nested fan-out", + mods: fanOut, + contents: repeated(3, "a.alloy"), + }, + } { + t.Run(tc.name, func(t *testing.T) { + ctx := context.Background() + var out string + var err error + start := time.Now() + build := allocated(func() { out, err = fn(ctx, tc.mods, pgtype.UUID{}, repos, tc.contents) }) + elapsed := time.Since(start) + // Parsing: the pipeline, each module once and the served form + // once more per level (the drop pass's check), generously. + parse := allocated(func() { + errSink = validate.ServedFormProblem(tc.contents) + for _, m := range tc.mods { + errSink = validate.ServedFormProblem(m) + } + }) * 3 + // The Alloy encoder (alloysyntax.QuoteString) works in about a + // dozen times the memory it returns. Each level quotes its module + // once (memoized), and the levels' quotes sum to under twice the + // largest, which fits the cap: two cap-sized quotes' worth. + encoder := 2 * allocated(func() { _ = alloysyntax.QuoteString(strings.Repeat("\t", validate.MaxContentBytes/2)) }) + beyond := int64(build) - int64(parse) - int64(encoder) + t.Logf("%s: Inline allocated %.1f MiB, parsing %.1f MiB and two cap-sized quotes %.1f MiB of it: "+ + "%.1f MiB beyond, in %v; served %d bytes, err %v", tc.name, mib(build), mib(parse), mib(encoder), + float64(beyond)/(1<<20), elapsed.Round(time.Millisecond), len(out), err) + if err == nil { + t.Fatalf("served %d bytes, want a refusal", len(out)) + } + if !errors.Is(err, ErrNotInlinable) { + t.Fatalf("refused with %v, want ErrNotInlinable", err) + } + if beyond > 2*validate.MaxContentBytes { + t.Fatalf("allocated %.1f MiB beyond parsing, more than twice the cap", float64(beyond)/(1<<20)) + } + if elapsed > 5*time.Second { + t.Fatalf("took %v", elapsed) + } + }) + } +} + +func mib(n uint64) float64 { return float64(n) / (1 << 20) } + +// allocated is how many bytes fn allocates. +func allocated(fn func()) uint64 { + var before, after runtime.MemStats + runtime.GC() + runtime.ReadMemStats(&before) + fn() + runtime.ReadMemStats(&after) + return after.TotalAlloc - before.TotalAlloc +} + +// errSink keeps a result only measured for its allocations. +var errSink error + +// lineEndings is s with its line endings as written: LF (most), all CR LF, +// or a mix. +func lineEndings(r *rand.Rand, s string) string { + switch r.Intn(6) { + case 0: + return strings.ReplaceAll(s, "\n", "\r\n") + case 1: + var sb strings.Builder + for _, line := range strings.SplitAfter(s, "\n") { + if strings.HasSuffix(line, "\n") && r.Intn(2) == 0 { + line = strings.TrimSuffix(line, "\n") + "\r\n" + } + sb.WriteString(line) + } + return sb.String() + default: + return s + } +} + +// rawLiteral writes s as a raw (backtick) string, its CR LF kept raw, or +// quoted when it holds a backtick. +func rawLiteral(s string) string { + if strings.Contains(s, "`") { + return alloysyntax.QuoteString(s) + } + return "`" + s + "`" +} + +// The drop pass reads a level's line ending from the text the old inliner +// left (#450 review): without its import.git blocks and the import.string +// contents it re-quoted. A raw (backtick) content may hold raw CR LF; when +// it holds an import.git, directly or in a module or content it follows, +// the old inliner re-quoted it and its CR LF no longer counted. +func TestInlineReadsTheLineEndingTheOldInlinerLeft(t *testing.T) { + repos := budgetRepos(t) + relabel := "prometheus.relabel \"r\" {\n forward_to = []\n rule {\n action = \"labelmap\"\n }\n}\n" + crlf := func(s string) string { return strings.ReplaceAll(s, "\n", "\r\n") } + g := strings.TrimSuffix(importGit("", "g", "m0.alloy"), "\n") + rawMod := "x = 1\n" + relabel + "import.string \"s\" {\n content = `" + crlf(importGit("", "g", "m0.alloy")) + "`\n}\n" + mods := fakeModules{ + "m0.alloy": "x = 1\n", "crlf.alloy": "x = 1\r\n" + relabel, "raw.alloy": rawMod, "lm.alloy": relabel, + } + for name, contents := range map[string]string{ + "raw content with CR LF holding an import.git": relabel + "import.string \"s\" {\n content = `" + + crlf(importGit("", "g", "m0.alloy")) + "`\n}\n", + "the same in a cached module": importGit("", "r", "raw.alloy"), + "raw content importing a labelmap, LF outside": "import.string \"s\" {\n content = `" + + crlf(importGit("", "g", "lm.alloy")) + "`\n}\n" + relabel, + "raw content holding an import.git one level down": relabel + "import.string \"s\" {\n content = `" + + "import.string \"t\" {\r\n content = " + alloysyntax.QuoteString(importGit("", "g", "m0.alloy")) + "\r\n}\r\n`\n}\n", + "raw content with CR LF and no import.git": relabel + "import.string \"s\" {\n content = `y = 1\r\n`\n}\n", + "CR LF inside the import.git block only": relabel + crlf(importGit("", "g", "m0.alloy")), + "a module written with CR LF": relabel + importGit("", "g", "crlf.alloy"), + "a raw second content": relabel + "import.string \"s\" {\n content = \"x = 1\"\n content = `" + crlf(importGit("", "g", "m0.alloy")) + "`\n}\n", + "CR LF in a declare": "declare \"d\"\r\n{\r\n" + crlf(importGit(" ", "g", "lm.alloy")) + "}\n" + relabel, + "an import.git after an exporter's brace": "otelcol.exporter.loki \"a\" {\n forward_to = [loki.write.a.receiver]\n}" + g + "\n", + } { + t.Run(name, func(t *testing.T) { checkInlineMatches(t, mods, repos, contents, true, true) }) + } +} diff --git a/internal/pinnedimport/inline_memory_test.go b/internal/pinnedimport/inline_memory_test.go new file mode 100644 index 00000000..95413160 --- /dev/null +++ b/internal/pinnedimport/inline_memory_test.go @@ -0,0 +1,106 @@ +package pinnedimport + +import ( + "context" + "runtime" + "strconv" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/jackc/pgx/v5/pgtype" + + "shepherd/internal/alloysyntax" + "shepherd/internal/store/sqlc" + "shepherd/internal/validate" +) + +// cloningModules hands out a fresh copy of a module's text on every read, +// as the database does, so a text the walker keeps shows in the heap. +type cloningModules struct{ fakeModules } + +func (c cloningModules) GetPinnedImportModule(ctx context.Context, a sqlc.GetPinnedImportModuleParams) (sqlc.PinnedImportModule, error) { + r, err := c.fakeModules.GetPinnedImportModule(ctx, a) + r.Content = strings.Clone(r.Content) + return r, err +} + +// peakHeap is how far the live heap rose above where it was while fn ran, +// sampled every millisecond. +func peakHeap(fn func()) uint64 { + runtime.GC() + var base runtime.MemStats + runtime.ReadMemStats(&base) + var peak atomic.Uint64 + stop, done := make(chan struct{}), make(chan struct{}) + go func() { + defer close(done) + var m runtime.MemStats + for { + runtime.ReadMemStats(&m) + if m.HeapAlloc > base.HeapAlloc && m.HeapAlloc-base.HeapAlloc > peak.Load() { + peak.Store(m.HeapAlloc - base.HeapAlloc) + } + select { + case <-stop: + return + case <-time.After(time.Millisecond): + } + } + }() + fn() + close(stop) + <-done + return peak.Load() +} + +// One module reached at many depths is read once per place, and held only +// while its level is walked or built (#450 review): 16 cached 1 MiB modules, +// each imported at every level of a 7-deep import.string chain — 136 +// instances in a 62 KiB pipeline — once held a copy per place (about +// 200 MiB) before the build refused it. Also with the modules' bulk inside +// their own import.git blocks, which the served form drops, so no lower +// bound on the served form stops the walk early. +func TestInlineHoldsEachModuleBriefly(t *testing.T) { + repos := budgetRepos(t) + for _, tc := range []struct { + name string + chain int + module func(i int) string + serve bool + }{ + {"1 MiB modules", 7, func(i int) string { + return "// " + strconv.Itoa(i) + strings.Repeat("a", 1<<20-20) + "\n" + }, false}, + {"1 MiB inside each module's import.git", 6, func(i int) string { + return strings.Replace(importGit("", "leaf", "leaf.alloy"), "}\n", + " // "+strconv.Itoa(i)+strings.Repeat("a", 1<<20-300)+"\n}\n", 1) + }, true}, + } { + t.Run(tc.name, func(t *testing.T) { + mods := fakeModules{"leaf.alloy": "x = 1\n"} + var level strings.Builder + for i := range 16 { + p := "m" + strconv.Itoa(i) + ".alloy" + mods[p] = tc.module(i) + level.WriteString(importGit("", "g"+strconv.Itoa(i), p)) + } + text := level.String() + for range tc.chain { + text = level.String() + "import.string \"s\" {\n content = " + alloysyntax.QuoteString(text) + "\n}\n" + } + var err error + peak := peakHeap(func() { + _, err = inline(context.Background(), cloningModules{mods}, pgtype.UUID{}, repos, text) + }) + t.Logf("%d-byte pipeline: peak heap +%.1f MiB, %v", len(text), mib(peak), err) + if (err == nil) != tc.serve { + t.Fatalf("served: %v, want %v", err == nil, tc.serve) + } + if peak > 32*validate.MaxContentBytes { + t.Fatalf("peak heap +%.1f MiB, more than 32 times the cap", mib(peak)) + } + }) + } +} diff --git a/internal/pinnedimport/oldinline_ref_test.go b/internal/pinnedimport/oldinline_ref_test.go new file mode 100644 index 00000000..72d5b7f3 --- /dev/null +++ b/internal/pinnedimport/oldinline_ref_test.go @@ -0,0 +1,238 @@ +package pinnedimport + +// The Inline and References of 3ec5c9c (before the served-form budget), +// verbatim but for the names (ref...) and q, which is the interface the +// walker reads the cache through: what inline_budget_test.go compares against. + +import ( + "context" + "fmt" + "slices" + "strings" + + "github.com/grafana/alloy/syntax/ast" + "github.com/jackc/pgx/v5/pgtype" + + "shepherd/internal/alloysyntax" + "shepherd/internal/gitrepo" + "shepherd/internal/store/sqlc" + "shepherd/internal/validate" +) + +// refInline is the old Inline. Its drop pass is today's +// validate.InsertTenantDrops, which validate's own differential test +// (served_budget_diff_test.go) holds to the old one. +func refInline(ctx context.Context, q moduleQuerier, orgID pgtype.UUID, repos []sqlc.OrgImportRepository, contents string) (string, error) { + w := newRefWalker(ctx, q, orgID, repos) + out, err := w.inline(contents, 0, 0) + if err != nil { + return "", err + } + if out, err = validate.InsertTenantDrops(out); err != nil { + return "", fmt.Errorf("%w: %w", ErrNotInlinable, err) + } + if len(out) > validate.MaxContentBytes { + return "", fmt.Errorf("%w: as served (pinned modules inlined, __tenant_id__ dropped after "+ + "each labelmap) the pipeline is %d bytes, more than the %d a "+ + "pipeline may be", ErrNotInlinable, len(out), validate.MaxContentBytes) + } + return out, nil +} + +// References lists every pinned module contents reaches, through cached +// modules — what must stay cached while contents is enabled. A module that +// is not cached is listed, and what it would import is not. +func refReferences(ctx context.Context, q moduleQuerier, orgID pgtype.UUID, repos []sqlc.OrgImportRepository, contents string) ([]Ref, error) { + w := newRefWalker(ctx, q, orgID, repos) + w.collect = true + if _, err := w.inline(contents, 0, 0); err != nil { + return nil, err + } + return w.refs, nil +} + +type refWalker struct { + ctx context.Context + q moduleQuerier + orgID pgtype.UUID + repos map[string]bool + collect bool + refs []Ref +} + +func newRefWalker(ctx context.Context, q moduleQuerier, orgID pgtype.UUID, repos []sqlc.OrgImportRepository) *refWalker { + w := &refWalker{ctx: ctx, q: q, orgID: orgID, repos: map[string]bool{}} + for i := range repos { + w.repos[repos[i].RepositoryKey] = true + } + return w +} + +// refMaxModuleDepth is how many modules (import.git or import.string) Inline +// follows into: the tenant lint's module depth. +const refMaxModuleDepth = 8 + +// refSplice is one rewrite of a byte range of a text. +type refSplice struct { + start, end int // [start, end) + text string +} + +// inline rewrites text's import.git blocks — found in the parse tree, so +// whatever spelling of the name the parser accepts — and descends into each +// import.string whose content is a plain literal, rewriting the module text +// it holds the same way. gitDepth is how many import.git modules enclose +// text, depth how many modules of either kind. Text with nothing to rewrite +// is returned unchanged; otherwise only the rewritten byte ranges change, +// located by the parsed nodes' positions. +func (w *refWalker) inline(text string, gitDepth, depth int) (string, error) { + if depth > refMaxModuleDepth { + return "", fmt.Errorf("%w: modules nest more than %d deep", ErrNotInlinable, refMaxModuleDepth) + } + file, err := alloysyntax.ParseFile("", []byte(text)) + if err != nil { + if w.collect { + return text, nil + } + return "", fmt.Errorf("%w: the text does not parse: %w", ErrNotInlinable, err) + } + var blocks []*ast.BlockStmt + refCollectImports(file.Body, &blocks, 0) + var splices []refSplice + for _, b := range blocks { + var sp *refSplice + var err error + if strings.Join(b.Name, ".") == "import.git" { + sp, err = w.gitBlock(b, gitDepth, depth) + } else { + sp, err = w.stringBlock(b, gitDepth, depth) + } + if err != nil { + if w.collect { + continue + } + return "", err + } + if sp != nil { + splices = append(splices, *sp) + } + } + if len(splices) == 0 { + return text, nil + } + // Apply from the last range back, so earlier offsets stay valid. + slices.SortFunc(splices, func(a, b refSplice) int { return b.start - a.start }) + out := text + for _, sp := range splices { + out = out[:sp.start] + sp.text + out[sp.end:] + } + return out, nil +} + +// gitBlock rewrites one import.git block into an import.string of its +// cached module, itself inlined. +func (w *refWalker) gitBlock(b *ast.BlockStmt, gitDepth, depth int) (*refSplice, error) { + if gitDepth >= validate.MaxGitImportDepth { + return nil, fmt.Errorf("%w: import.git nests more than %d deep", ErrNotInlinable, validate.MaxGitImportDepth) + } + ref, err := w.ref(b) + if err != nil { + return nil, err + } + if w.collect && !slices.Contains(w.refs, ref) { + w.refs = append(w.refs, ref) + } + row, err := w.q.GetPinnedImportModule(w.ctx, sqlc.GetPinnedImportModuleParams{ + OrgID: w.orgID, RepositoryKey: ref.Key, Revision: ref.Revision, Path: ref.Path, + }) + if err != nil { + return nil, fmt.Errorf("%w: module %s at %s (%s) is not cached", ErrNotInlinable, ref.Path, ref.Revision, ref.Key) + } + module, err := w.inline(row.Content, gitDepth+1, depth+1) + if err != nil { + return nil, err + } + start := ast.StartPos(b).Position() + indent := strings.Repeat(" ", max(start.Column-1, 0)) + return &refSplice{ + start: start.Offset, + end: ast.EndPos(b).Position().Offset + 1, + text: "import.string " + alloysyntax.QuoteString(b.Label) + " {\n" + + indent + " content = " + alloysyntax.QuoteString(module) + "\n" + indent + "}", + }, nil +} + +// stringBlock rewrites the content of one import.string whose module text +// holds an import.git (at any depth): only its content literal is replaced, +// re-quoted. Content that is not a plain literal is left alone (the served +// form's bound, validate.ServedFormProblem, refuses it in a strict org). +func (w *refWalker) stringBlock(b *ast.BlockStmt, gitDepth, depth int) (*refSplice, error) { + for _, st := range b.Body { + a, ok := st.(*ast.AttributeStmt) + if !ok || a.Name.Name != "content" { + continue + } + content, ok := validate.PlainStringLiteral(a.Value) + if !ok { + return nil, nil //nolint:nilnil // nothing this refWalker can rewrite + } + inner, err := w.inline(content, gitDepth, depth+1) + if err != nil { + return nil, err + } + if inner == content { + return nil, nil //nolint:nilnil // unchanged: the bytes stay as written + } + return &refSplice{ + start: ast.StartPos(a.Value).Position().Offset, + end: ast.EndPos(a.Value).Position().Offset + 1, + text: alloysyntax.QuoteString(inner), + }, nil + } + return nil, nil //nolint:nilnil // no content attribute +} + +// refCollectImports finds the import blocks Alloy loads — import.git and +// import.string — at the top level and in a declare body, at any declare +// depth, by their parsed names. +func refCollectImports(body ast.Body, out *[]*ast.BlockStmt, nesting int) { + if nesting > alloysyntax.MaxNesting { + return + } + for _, st := range body { + b, ok := st.(*ast.BlockStmt) + if !ok { + continue + } + switch strings.Join(b.Name, ".") { + case "import.git", "import.string": + *out = append(*out, b) + case "declare": + refCollectImports(b.Body, out, nesting+1) + } + } +} + +// ref reads a block's module reference under the lint's rules. +func (w *refWalker) ref(b *ast.BlockStmt) (Ref, error) { + attrs := map[string]string{} + for _, st := range b.Body { + a, ok := st.(*ast.AttributeStmt) + if !ok { + continue + } + if s, ok := validate.PlainStringLiteral(a.Value); ok { + attrs[a.Name.Name] = s + } + } + repo, rev, path := attrs["repository"], attrs["revision"], attrs["path"] + if repo == "" || path == "" || !gitrepo.IsFullCommitSHA(rev) || !validate.CleanModulePath(path) { + return Ref{}, fmt.Errorf("%w: import.git %q needs literal repository, a full commit SHA revision and a "+ + "clean path", ErrNotInlinable, b.Label) + } + key, err := NormalizeRepository(repo) + if err != nil || !w.repos[key] { + return Ref{}, fmt.Errorf("%w: import.git %q's repository is not on the allowlist", ErrNotInlinable, b.Label) + } + return Ref{Key: key, Revision: rev, Path: path}, nil +} diff --git a/internal/validate/labelmapdrop.go b/internal/validate/labelmapdrop.go index c0de1f83..01843190 100644 --- a/internal/validate/labelmapdrop.go +++ b/internal/validate/labelmapdrop.go @@ -200,12 +200,59 @@ func isRelabelRuleSchema(b schemaBlock) bool { // already followed by that rule gets no second one. Text without such a // labelmap is returned unchanged, byte for byte; otherwise only the // inserted ranges change. The result must parse. It is part of the served -// form of a strict org's pipeline (pinnedimport.Inline). +// form of a strict org's pipeline (pinnedimport.Inline), so it is built +// within a ServedBudget of its own: a result that would be more than +// MaxContentBytes is ErrServedFormTooLarge. func InsertTenantDrops(text string) (string, error) { - return insertTenantDrops(text, 0) + return InsertTenantDropsWithin(text, NewServedBudget()) } -func insertTenantDrops(text string, depth int) (string, error) { +// InsertTenantDropsWithin is InsertTenantDrops charging b (see +// ServedBudget): when the text changes, b ends charged its length. +func InsertTenantDropsWithin(text string, b *ServedBudget) (string, error) { + return insertTenantDrops(text, 0, b, nil) +} + +// ServedImports is what the caller of BuildServedLevel serves in place of +// some of a level's import blocks (pinnedimport.Inline: the import.git and +// literal import.string blocks it inlines). +type ServedImports struct { + // Serve is called, in the drop pass's walk order, for every import.* + // block outside a relabel component. owned is whether the caller + // serves b: its splices (charged to the budget by the caller) replace + // what the drop pass would do with b. + Serve func(b *ast.BlockStmt) (splices []Splice, owned bool, err error) + // Replaced are the byte ranges [start, end) the caller replaces whole + // (an import.git block): the text's line ending is read outside them, + // as it is read in the text with them replaced. + Replaced [][2]int +} + +// BuildServedLevel is the drop pass over one level of a served form at +// module depth depth, with imports (made from the level's parse) serving +// the import blocks it owns: the inliner's and the drop pass's rewrites of +// one text applied at once, charged to b, so the served form is built +// without first building the text the inliner alone would make (see +// ServedBudget). imports nil is InsertTenantDrops at that depth. +func BuildServedLevel(text string, depth int, b *ServedBudget, imports func(*ast.File) ServedImports) (string, error) { + return insertTenantDrops(text, depth, b, imports) +} + +// crlfOutside reports whether text has a CR LF outside the ranges. +func crlfOutside(text string, ranges [][2]int) bool { + sorted := slices.Clone(ranges) + slices.SortFunc(sorted, func(a, b [2]int) int { return a[0] - b[0] }) + at := 0 + for _, r := range sorted { + if r[0] > at && strings.Contains(text[at:r[0]], "\r\n") { + return true + } + at = max(at, r[1]) + } + return strings.Contains(text[at:], "\r\n") +} + +func insertTenantDrops(text string, depth int, budget *ServedBudget, imports func(*ast.File) ServedImports) (string, error) { if depth > maxDepth { return "", fmt.Errorf("%w: modules nest more than %d deep", ErrTenantDrop, maxDepth) } @@ -213,8 +260,13 @@ func insertTenantDrops(text string, depth int) (string, error) { if err != nil { return "", fmt.Errorf("%w: the text does not parse: %w", ErrTenantDrop, err) } - d := &dropper{text: text, depth: depth, newline: "\n"} - if strings.Contains(text, "\r\n") { + d := &dropper{text: text, depth: depth, newline: "\n", budget: budget} + var replaced [][2]int + if imports != nil { + si := imports(file) + d.serve, replaced = si.Serve, si.Replaced + } + if crlfOutside(text, replaced) { d.newline = "\r\n" } if err := d.body(file.Body, "", "", 0); err != nil { @@ -223,11 +275,9 @@ func insertTenantDrops(text string, depth int) (string, error) { if len(d.splices) == 0 { return text, nil } - // From the last offset back, so earlier offsets stay valid. - slices.SortStableFunc(d.splices, func(a, b dropSplice) int { return b.start - a.start }) - out := text - for _, sp := range d.splices { - out = out[:sp.start] + sp.text + out[sp.end:] + out, err := ApplySplices(text, d.splices, budget) + if err != nil { + return "", err } if _, err := alloysyntax.ParseFile("", []byte(out)); err != nil { return "", fmt.Errorf("%w: the text with the drops inserted does not parse: %w", ErrTenantDrop, err) @@ -235,16 +285,24 @@ func insertTenantDrops(text string, depth int) (string, error) { return out, nil } -type dropSplice struct { - start, end int - text string -} - type dropper struct { text string depth int newline string - splices []dropSplice + splices []Splice + budget *ServedBudget + serve func(b *ast.BlockStmt) ([]Splice, bool, error) +} + +// splice records the rewrite of [start, end) by the text build makes, +// charging the budget n bytes — the length that text will have — before it +// is built. +func (d *dropper) splice(start, end, n int, build func() string) error { + if err := d.budget.Take(n); err != nil { + return err + } + d.splices = append(d.splices, Splice{Start: start, End: end, Text: build()}) + return nil } // body walks stmts. component is the relabel-rule component enclosing them @@ -259,6 +317,16 @@ func (d *dropper) body(stmts ast.Body, component, path string, nesting int) erro continue } name := strings.Join(b.Name, ".") + if component == "" && d.serve != nil && strings.HasPrefix(name, "import.") { + splices, owned, err := d.serve(b) + if err != nil { + return err + } + if owned { + d.splices = append(d.splices, splices...) + continue + } + } switch { case component == "" && name == "import.string": if err := d.importString(b); err != nil { @@ -296,7 +364,9 @@ func (d *dropper) body(stmts ast.Body, component, path string, nesting int) erro } if name == "rule" && rules[component][p] && isLiteralLabelmap(b) && needsTenantDrop(b) && !isTenantDrop(nextRule(stmts, i)) { - d.insertAfter(b) + if err := d.insertAfter(b); err != nil { + return err + } } if err := d.body(b.Body, component, p, nesting+1); err != nil { return err @@ -309,19 +379,26 @@ func (d *dropper) body(stmts ast.Body, component, path string, nesting int) erro // lines of its own, indented with the leading whitespace (spaces and tabs, // byte for byte) of the line b starts on — never a column count, which a tab // or a multi-byte character before b would throw off. -func (d *dropper) insertAfter(b *ast.BlockStmt) { +func (d *dropper) insertAfter(b *ast.BlockStmt) error { start := ast.StartPos(b).Position().Offset end := ast.EndPos(b).Position().Offset + 1 lineStart := strings.LastIndexByte(d.text[:start], '\n') + 1 line := d.text[lineStart:start] indent := line[:len(line)-len(strings.TrimLeft(line, " \t"))] - var sb strings.Builder + n := 0 for _, line := range tenantDropRule { - sb.WriteString(d.newline) - sb.WriteString(indent) - sb.WriteString(line) + n += len(d.newline) + len(indent) + len(line) } - d.splices = append(d.splices, dropSplice{start: end, end: end, text: sb.String()}) + return d.splice(end, end, n, func() string { + var sb strings.Builder + sb.Grow(n) + for _, line := range tenantDropRule { + sb.WriteString(d.newline) + sb.WriteString(indent) + sb.WriteString(line) + } + return sb.String() + }) } // exporterLoki is the component the strict served form routes through a @@ -365,7 +442,10 @@ func (d *dropper) exporterLoki(b *ast.BlockStmt, stmts ast.Body) error { vStart := ast.StartPos(fwd.Value).Position().Offset vEnd := ast.EndPos(fwd.Value).Position().Offset + 1 original := d.text[vStart:vEnd] - d.splices = append(d.splices, dropSplice{start: vStart, end: vEnd, text: "[loki.process." + label + ".receiver]"}) + receiver := "[loki.process." + label + ".receiver]" + if err := d.splice(vStart, vEnd, len(receiver), func() string { return receiver }); err != nil { + return err + } start := ast.StartPos(b).Position().Offset end := ast.EndPos(b).Position().Offset + 1 @@ -382,17 +462,26 @@ func (d *dropper) exporterLoki(b *ast.BlockStmt, stmts ast.Body) error { " }", "}", } - var sb strings.Builder + n := 0 for _, l := range lines { - sb.WriteString(d.newline) + n += len(d.newline) if l != "" { - sb.WriteString(indent) - sb.WriteString(l) + n += len(indent) + len(l) } } at := afterTrailingComment(d.text, end) - d.splices = append(d.splices, dropSplice{start: at, end: at, text: sb.String()}) - return nil + return d.splice(at, at, n, func() string { + var sb strings.Builder + sb.Grow(n) + for _, l := range lines { + sb.WriteString(d.newline) + if l != "" { + sb.WriteString(indent) + sb.WriteString(l) + } + } + return sb.String() + }) } // afterTrailingComment is the offset to insert after a block that closes at @@ -454,16 +543,17 @@ func (d *dropper) importString(b *ast.BlockStmt) error { if !ok { return nil } - inner, err := insertTenantDrops(content, d.depth+1) + inner, err := insertTenantDrops(content, d.depth+1, d.budget, nil) if err != nil { return fmt.Errorf("import.string %q: %w", b.Label, err) } if inner != content { - d.splices = append(d.splices, dropSplice{ - start: ast.StartPos(a.Value).Position().Offset, - end: ast.EndPos(a.Value).Position().Offset + 1, - text: alloysyntax.QuoteString(inner), - }) + // inner is charged; what is served is inner quoted. + d.budget.Give(len(inner)) + if err := d.splice(ast.StartPos(a.Value).Position().Offset, ast.EndPos(a.Value).Position().Offset+1, + QuotedLen(inner), func() string { return alloysyntax.QuoteString(inner) }); err != nil { + return fmt.Errorf("import.string %q: %w", b.Label, err) + } } } return nil diff --git a/internal/validate/lint_memory_test.go b/internal/validate/lint_memory_test.go new file mode 100644 index 00000000..92415338 --- /dev/null +++ b/internal/validate/lint_memory_test.go @@ -0,0 +1,39 @@ +package validate + +import ( + "strconv" + "strings" + "testing" +) + +// The tenant lint parses a module imported many times once, but keeps at +// most maxParsedBytes of module text parsed (#450 review): a tree is several +// times its text, and keeping every module's for the whole call let 16 +// large modules hold well over 100 MiB. The pipeline's own text is never +// kept (it is parsed once per run, as before). +func TestTenantLintKeepsFewParseTrees(t *testing.T) { + g := &gitState{} + module := func(i int) string { + return "// " + strconv.Itoa(i) + "\n" + strings.Repeat("x = 1\n", (MaxContentBytes/2)/6) + } + for i := range maxGitModules { + for range 3 { + if g.parse(module(i), 1) == nil { + t.Fatalf("module %d does not parse", i) + } + } + } + if g.parsedBytes > maxParsedBytes { + t.Fatalf("keeps %d bytes of text parsed, more than %d", g.parsedBytes, maxParsedBytes) + } + if len(g.parsed) != 1 { + t.Fatalf("keeps %d trees, want the one that fits", len(g.parsed)) + } + first := g.parse(module(0), 1) + if again := g.parse(module(0), 1); again != first { + t.Fatal("a kept module is parsed again") + } + if g.parse(strings.Repeat("y = 2\n", 10), 0); len(g.parsed) != 1 { + t.Fatal("the pipeline's own text is kept") + } +} diff --git a/internal/validate/olddrop_ref_test.go b/internal/validate/olddrop_ref_test.go new file mode 100644 index 00000000..74c0c527 --- /dev/null +++ b/internal/validate/olddrop_ref_test.go @@ -0,0 +1,228 @@ +package validate + +// The InsertTenantDrops of 3ec5c9c (before the served-form budget), verbatim but +// for the names (ref...): what served_budget_diff_test.go compares against. + +import ( + "fmt" + "slices" + "strconv" + "strings" + + "github.com/grafana/alloy/syntax/ast" + + "shepherd/internal/alloysyntax" +) + +func refInsertTenantDrops(text string) (string, error) { + return refInsertTenantDropsAt(text, 0) +} + +func refInsertTenantDropsAt(text string, depth int) (string, error) { + if depth > maxDepth { + return "", fmt.Errorf("%w: modules nest more than %d deep", ErrTenantDrop, maxDepth) + } + file, err := alloysyntax.ParseFile("", []byte(text)) + if err != nil { + return "", fmt.Errorf("%w: the text does not parse: %w", ErrTenantDrop, err) + } + d := &refDropper{text: text, depth: depth, newline: "\n"} + if strings.Contains(text, "\r\n") { + d.newline = "\r\n" + } + if err := d.body(file.Body, "", "", 0); err != nil { + return "", err + } + if len(d.splices) == 0 { + return text, nil + } + // From the last offset back, so earlier offsets stay valid. + slices.SortStableFunc(d.splices, func(a, b refDropSplice) int { return b.start - a.start }) + out := text + for _, sp := range d.splices { + out = out[:sp.start] + sp.text + out[sp.end:] + } + if _, err := alloysyntax.ParseFile("", []byte(out)); err != nil { + return "", fmt.Errorf("%w: the text with the drops inserted does not parse: %w", ErrTenantDrop, err) + } + return out, nil +} + +type refDropSplice struct { + start, end int + text string +} + +type refDropper struct { + text string + depth int + newline string + splices []refDropSplice +} + +// body walks stmts. component is the relabel-rule component enclosing them +// ("" outside one), path their block path inside it. +func (d *refDropper) body(stmts ast.Body, component, path string, nesting int) error { + if nesting > alloysyntax.MaxNesting { + return fmt.Errorf("%w: blocks nest too deeply", ErrTenantDrop) + } + for i, st := range stmts { + b, ok := st.(*ast.BlockStmt) + if !ok { + continue + } + name := strings.Join(b.Name, ".") + switch { + case component == "" && name == "import.string": + if err := d.importString(b); err != nil { + return err + } + continue + case component == "" && name == exporterLoki: + if err := d.exporterLoki(b, stmts); err != nil { + return err + } + continue + case component == "": + rules, err := RelabelRuleBlocks() + if err != nil { + return fmt.Errorf("%w: %w", ErrTenantDrop, err) + } + if _, ok := rules[name]; ok { + if err := d.body(b.Body, name, "", nesting+1); err != nil { + return err + } + continue + } + if err := d.body(b.Body, "", "", nesting+1); err != nil { + return err + } + continue + } + p := name + if path != "" { + p = path + "/" + name + } + rules, err := RelabelRuleBlocks() + if err != nil { + return fmt.Errorf("%w: %w", ErrTenantDrop, err) + } + if name == "rule" && rules[component][p] && isLiteralLabelmap(b) && needsTenantDrop(b) && + !isTenantDrop(nextRule(stmts, i)) { + d.insertAfter(b) + } + if err := d.body(b.Body, component, p, nesting+1); err != nil { + return err + } + } + return nil +} + +// insertAfter splices tenantDropRule in right after b's closing brace, on +// lines of its own, indented with the leading whitespace (spaces and tabs, +// byte for byte) of the line b starts on — never a column count, which a tab +// or a multi-byte character before b would throw off. +func (d *refDropper) insertAfter(b *ast.BlockStmt) { + start := ast.StartPos(b).Position().Offset + end := ast.EndPos(b).Position().Offset + 1 + lineStart := strings.LastIndexByte(d.text[:start], '\n') + 1 + line := d.text[lineStart:start] + indent := line[:len(line)-len(strings.TrimLeft(line, " \t"))] + var sb strings.Builder + for _, line := range tenantDropRule { + sb.WriteString(d.newline) + sb.WriteString(indent) + sb.WriteString(line) + } + d.splices = append(d.splices, refDropSplice{start: end, end: end, text: sb.String()}) +} + +// exporterLoki routes otelcol.exporter.loki b through a __tenant_id__ drop: +// its forward_to becomes [loki.process.