From 3ec73790591f54eed4225e4efa1476ac31470188 Mon Sep 17 00:00:00 2001 From: Kim Romero Date: Tue, 22 Sep 2026 16:42:55 +0300 Subject: [PATCH 1/2] images: describe builds in fields, emit images.json MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A container that pins a source repo described its build as a shell line. That string is opaque to anything but a local docker/podman run: an external builder cannot tell the repo from the Dockerfile from the secrets, so it ends up keeping its own copy of the pins, which then drift. ImageBuildSpec now carries the build in fields (repo, ref, name, variant, version, dockerfile/assetDockerfile, assets, target, buildArgs, secrets) and derives the command from them; `cmd` stays as an escape hatch. `decker build` writes every spec to manifests//images.json, keyed by the tag the manifests reference, so a builder in CI or in a cluster builds exactly what the run will pull. Build inputs that are not in the source repo — a Dockerfile that patches it, the patches — can sit with the recipe that needs them: assetsDir points a spec at the directory that owns them, so a recipe in its own repo ships its own. Those bytes hash into the image tag (a patched build cannot collide with a pristine one) and are copied next to images.json for whoever builds. Also here, all in support of running a mainnet-shaped L1: - MAINNET_SYSTEM_CONTRACTS predeploys the EIP-4788/2935/7002/7251 contracts the EL system-calls every block; without code there the state diff takes a shape mainnet never produces. - l1 artifacts take withdrawals: "none" for BLS (0x00) credentials, so a devnet can run without per-block withdrawal sweeps. - scripts/trie-padding.ts creates N fresh accounts before load: decker's genesis is 12 accounts, and a builder whose root hasher is only exercised on mainnet-deep tries can compute wrong roots against a dozen leaves. - lighthouse takes config.feeRecipient: a builder that proves the proposer payment against state needs a recipient that exists in genesis. --- Makefile | 2 +- README.md | 19 ++++ containers/helix-simulator.ts | 2 +- containers/lighthouse-beacon.ts | 4 +- containers/lighthouse-validator.ts | 4 +- containers/mev-boost-relay.ts | 2 +- e2e/images_test.ts | 34 +++++++ generators/l1/genesis-ssz.ts | 6 +- generators/l1/index.ts | 3 +- generators/l1/system-contracts.ts | 39 ++++++++ generators/l1/system-contracts_test.ts | 25 +++++ scripts/trie-padding.ts | 131 ++++++++++++++++++++++++ utils/emit.ts | 14 ++- utils/image-build.ts | 125 +++++++++++++++++++++-- utils/image-build_test.ts | 133 +++++++++++++++++++++++++ utils/resolve_test.ts | 28 ++++++ utils/types.ts | 43 +++++++- 17 files changed, 599 insertions(+), 15 deletions(-) create mode 100644 e2e/images_test.ts create mode 100644 generators/l1/system-contracts.ts create mode 100644 generators/l1/system-contracts_test.ts create mode 100644 scripts/trie-padding.ts create mode 100644 utils/image-build_test.ts create mode 100644 utils/resolve_test.ts diff --git a/Makefile b/Makefile index 2028b8f..d022ac6 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -INCLUDES := --include commands --include containers --include generators --include recipes --include renderers --include decker.example.ts +INCLUDES := --include commands --include containers --include generators --include recipes --include renderers --include _assets --include decker.example.ts OUTPUT ?= decker TARGET ?= diff --git a/README.md b/README.md index 3706ac5..ed32a27 100644 --- a/README.md +++ b/README.md @@ -95,6 +95,25 @@ You can evolve `decker` in multiple layers and use in dev or CI setups of your p - **Renderers:** Run your recipe on any target (podman, docker, process-compose and anything you want) - **CLI:** Hack on the clone, run immediately with preinstalled binary +## Images + +A container can name a published image, or pin a source repo and let decker +build it. A pinned build is described in fields — repo, ref, Dockerfile, +target, build args, secrets — and `decker build` writes them all to +`manifests//images.json` beside the rendered manifests, keyed by the +exact tag those manifests reference. Run the images locally and decker builds +what is missing; set `DECKER_IMAGE_MODE=pull` and it builds nothing, so CI or +an in-cluster build system can supply the images from that one file instead of +keeping its own copy of your pins. + +Builds that need files the source repo does not have — a Dockerfile that +patches it, the patches — name them in `assets` and `assetDockerfile`. They +live in `_assets/` here, or in your own repo: set `assetsDir` to +`new URL("../_assets/", import.meta.url).href` and a recipe outside decker +ships its own build inputs. Those bytes are hashed into the image tag, so a +patched image never collides with a pristine one, and they are copied next to +`images.json` for whoever does the building. + ## Why? Sophisticated tools and their abstraction layers speed up humans but slow down LLM problem solving capabilities and reduce success. In addition, developers often try to fix upstream the tools they depend on, to satisfy their own use-case specific necessities. diff --git a/containers/helix-simulator.ts b/containers/helix-simulator.ts index 1bf415e..45c2bb4 100644 --- a/containers/helix-simulator.ts +++ b/containers/helix-simulator.ts @@ -26,7 +26,7 @@ export const ports: Ports = { function image(def: ContainerDef): string | ImageBuildSpec { const build = def.config?.build as { repo: string; ref: string } | undefined; if (!build) return (def.config?.image as string | undefined) ?? DEFAULT_IMAGE; - return { repo: build.repo, ref: build.ref, cmd: "$ENGINE build -t $IMAGE -f simulator.Dockerfile ." }; + return { repo: build.repo, ref: build.ref, dockerfile: "simulator.Dockerfile" }; } export function buildContainer(def: ContainerDef, ctx: Ctx): ContainerResult { diff --git a/containers/lighthouse-beacon.ts b/containers/lighthouse-beacon.ts index a7a2b75..011ef6b 100644 --- a/containers/lighthouse-beacon.ts +++ b/containers/lighthouse-beacon.ts @@ -58,7 +58,9 @@ export function buildContainer(def: ContainerDef, ctx: Ctx): ContainerResult { "--execution-jwt", "/artifacts/jwtsecret", "--always-prepare-payload", "--prepare-payload-lookahead", "8000", - "--suggested-fee-recipient", "0x690B9A9E9aa1C9dB991C7721a92d351Db4FaC990", + // config.feeRecipient overrides the default. A builder that proves the + // proposer payment against state needs a recipient that EXISTS in genesis. + "--suggested-fee-recipient", (def.config?.feeRecipient as string | undefined) ?? "0x690B9A9E9aa1C9dB991C7721a92d351Db4FaC990", ...(supernode ? ["--supernode"] : []), ...(peerMultiaddrs.length > 0 ? ["--libp2p-addresses", peerMultiaddrs.join(",")] : []), ...(builder ? [ diff --git a/containers/lighthouse-validator.ts b/containers/lighthouse-validator.ts index 2904873..302269b 100644 --- a/containers/lighthouse-validator.ts +++ b/containers/lighthouse-validator.ts @@ -15,7 +15,9 @@ export function buildContainer(def: ContainerDef, ctx: Ctx): ContainerResult { "--testnet-dir", "/artifacts/testnet", "--init-slashing-protection", "--beacon-nodes", ctx.url(beacon, "http"), - "--suggested-fee-recipient", "0x690B9A9E9aa1C9dB991C7721a92d351Db4FaC990", + // config.feeRecipient overrides the default. A builder that proves the + // proposer payment against state needs a recipient that EXISTS in genesis. + "--suggested-fee-recipient", (def.config?.feeRecipient as string | undefined) ?? "0x690B9A9E9aa1C9dB991C7721a92d351Db4FaC990", "--builder-proposals", "--prefer-builder-proposals", ], diff --git a/containers/mev-boost-relay.ts b/containers/mev-boost-relay.ts index ba93dcf..b4e2f44 100644 --- a/containers/mev-boost-relay.ts +++ b/containers/mev-boost-relay.ts @@ -12,7 +12,7 @@ function imageSpec(def: ContainerDef): ImageBuildSpec { return { repo: (def.config?.repo as string | undefined) ?? DEFAULT_REPO, ref: (def.config?.ref as string | undefined) ?? DEFAULT_REF, - cmd: "$ENGINE build -t $IMAGE .", + dockerfile: "Dockerfile", }; } const BLS_KEYS_FIXTURE = new URL("../generators/l1/bls_keys.json", import.meta.url); diff --git a/e2e/images_test.ts b/e2e/images_test.ts new file mode 100644 index 0000000..dd99f2c --- /dev/null +++ b/e2e/images_test.ts @@ -0,0 +1,34 @@ +// images.json is the contract with external builders: every image a rendered +// recipe references, described in structured fields, next to the manifests. +import { assert, assertEquals } from "jsr:@std/assert@^1.0.0"; +import { join } from "jsr:@std/path@^1.0.0"; +import { runDecker, withTmp } from "./helpers.ts"; + +Deno.test("build: images.json describes every image the manifests reference", async () => { + await withTmp(async (root) => { + const r = await runDecker(["build", "rbuilder"], { cwd: root, env: { DECKER_ROOT: root } }); + assertEquals(r.code, 0, r.out); + + const images = JSON.parse(await Deno.readTextFile(join(root, "manifests", "rbuilder", "images.json"))); + const tags = Object.keys(images); + assert(tags.length > 0, "recipe builds images, so images.json must not be empty"); + + const rendered = await Deno.readTextFile(join(root, "manifests", "rbuilder", "podman.yaml")); + for (const tag of tags) { + assert(rendered.includes(tag), `${tag} is in images.json but nothing references it`); + const spec = images[tag]; + assert(spec.repo && spec.ref, `${tag} must pin a repo and a ref`); + assert(spec.cmd.includes("$ENGINE") && spec.cmd.includes("$IMAGE"), `${tag} cmd is not substitutable`); + assert(spec.dockerfile || spec.assetDockerfile, `${tag} names no Dockerfile`); + } + }); +}); + +Deno.test("build: a recipe with no source-built images still writes images.json", async () => { + await withTmp(async (root) => { + const r = await runDecker(["build", "contender-bench"], { cwd: root, env: { DECKER_ROOT: root } }); + assertEquals(r.code, 0, r.out); + const path = join(root, "manifests", "contender-bench", "images.json"); + assertEquals(JSON.parse(await Deno.readTextFile(path)), {}); + }); +}); diff --git a/generators/l1/genesis-ssz.ts b/generators/l1/genesis-ssz.ts index 555f130..58961fc 100644 --- a/generators/l1/genesis-ssz.ts +++ b/generators/l1/genesis-ssz.ts @@ -42,6 +42,8 @@ export type GenesisSszOpts = { // own. Must be the same root fed to the EL client, or CL/EL disagree on the // genesis execution block hash and the chain never advances. elStateRoot?: Uint8Array; + // see L1ArtifactsSpec.withdrawals + withdrawals?: "eth1" | "none"; }; export async function renderGenesisSsz(opts: GenesisSszOpts): Promise { @@ -68,7 +70,9 @@ export async function renderGenesisSsz(opts: GenesisSszOpts): Promise { await Deno.writeTextFile(`${outDir}/genesis.json`, el.json); await Deno.writeFile( `${testnetDir}/genesis.ssz`, - await renderGenesisSsz({ genesisTimeSeconds, fork, elStateRoot }), + await renderGenesisSsz({ genesisTimeSeconds, fork, elStateRoot, withdrawals: opts.withdrawals }), ); const keys = await loadBlsKeys(); diff --git a/generators/l1/system-contracts.ts b/generators/l1/system-contracts.ts new file mode 100644 index 0000000..3dd022e --- /dev/null +++ b/generators/l1/system-contracts.ts @@ -0,0 +1,39 @@ +// The execution-layer system contracts mainnet runs, as `genesisAccounts` for +// an L1 artifacts spec (see el-genesis.ts / state-root.ts). +// +// EIP-4788 / 2935 / 7002 / 7251 (bytecodes from alloy-eips 2.0.5, the constants +// reth itself uses): the EL makes their pre/post-block system calls EVERY +// block. With no code at those addresses the calls touch non-existent accounts +// and the block's state diff takes a shape mainnet never produces - which is +// exactly the kind of difference that makes a builder or a prover behave +// differently here than in production. +// +// A recipe spreads these into its own `genesisAccounts` and adds whatever else +// it needs: +// +// genesisAccounts: { ...MAINNET_SYSTEM_CONTRACTS, [myAddr]: myPredeploy } +import type { GenesisAlloc } from "./state-root.ts"; + +const CODE = { + eip4788: "0x3373fffffffffffffffffffffffffffffffffffffffe14604d57602036146024575f5ffd5b5f35801560495762001fff810690815414603c575f5ffd5b62001fff01545f5260205ff35b5f5ffd5b62001fff42064281555f359062001fff015500", + eip2935: "0x3373fffffffffffffffffffffffffffffffffffffffe14604657602036036042575f35600143038111604257611fff81430311604257611fff9006545f5260205ff35b5f5ffd5b5f35611fff60014303065500", + eip7002: "0x3373fffffffffffffffffffffffffffffffffffffffe1460cb5760115f54807fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff146101f457600182026001905f5b5f82111560685781019083028483029004916001019190604d565b909390049250505036603814608857366101f457346101f4575f5260205ff35b34106101f457600154600101600155600354806003026004013381556001015f35815560010160203590553360601b5f5260385f601437604c5fa0600101600355005b6003546002548082038060101160df575060105b5f5b8181146101835782810160030260040181604c02815460601b8152601401816001015481526020019060020154807fffffffffffffffffffffffffffffffff00000000000000000000000000000000168252906010019060401c908160381c81600701538160301c81600601538160281c81600501538160201c81600401538160181c81600301538160101c81600201538160081c81600101535360010160e1565b910180921461019557906002556101a0565b90505f6002555f6003555b5f54807fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff14156101cd57505f5b6001546002828201116101e25750505f6101e8565b01600290035b5f555f600155604c025ff35b5f5ffd", + eip7251: "0x3373fffffffffffffffffffffffffffffffffffffffe1460d35760115f54807fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff1461019a57600182026001905f5b5f82111560685781019083028483029004916001019190604d565b9093900492505050366060146088573661019a573461019a575f5260205ff35b341061019a57600154600101600155600354806004026004013381556001015f358155600101602035815560010160403590553360601b5f5260605f60143760745fa0600101600355005b6003546002548082038060021160e7575060025b5f5b8181146101295782810160040260040181607402815460601b815260140181600101548152602001816002015481526020019060030154905260010160e9565b910180921461013b5790600255610146565b90505f6002555f6003555b5f54807fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff141561017357505f5b6001546001828201116101885750505f61018e565b01600190035b5f555f6001556074025ff35b5f5ffd0000", +}; + +// nonce 1 and no balance: what a deployed contract looks like on mainnet. +export const predeploy = (code: string) => ({ nonce: "0x1", balance: "0x0", code }); + +export const SYSTEM_CONTRACT_ADDRESSES = { + eip4788: "0x000F3df6D732807Ef1319fB7B8bB8522d0Beac02", // beacon block roots + eip2935: "0x0000F90827F1C53a10cb7A02335B175320002935", // historical block hashes + eip7002: "0x00000961Ef480Eb55e80D19ad83579A64c007002", // withdrawal requests + eip7251: "0x0000BBdDc7CE488642fb579F8B00f3a590007251", // consolidation requests +} as const; + +export const MAINNET_SYSTEM_CONTRACTS: GenesisAlloc = { + [SYSTEM_CONTRACT_ADDRESSES.eip4788]: predeploy(CODE.eip4788), + [SYSTEM_CONTRACT_ADDRESSES.eip2935]: predeploy(CODE.eip2935), + [SYSTEM_CONTRACT_ADDRESSES.eip7002]: predeploy(CODE.eip7002), + [SYSTEM_CONTRACT_ADDRESSES.eip7251]: predeploy(CODE.eip7251), +}; diff --git a/generators/l1/system-contracts_test.ts b/generators/l1/system-contracts_test.ts new file mode 100644 index 0000000..93039e9 --- /dev/null +++ b/generators/l1/system-contracts_test.ts @@ -0,0 +1,25 @@ +// The EL calls these contracts every block; missing code silently produces a +// state-diff shape mainnet never has. Pin the addresses and the code. +import { assert, assertEquals } from "jsr:@std/assert@^1.0.0"; +import { MAINNET_SYSTEM_CONTRACTS, predeploy, SYSTEM_CONTRACT_ADDRESSES } from "./system-contracts.ts"; + +Deno.test("the four system contracts are present, at their mainnet addresses", () => { + assertEquals(Object.keys(MAINNET_SYSTEM_CONTRACTS).length, 4); + for (const addr of Object.values(SYSTEM_CONTRACT_ADDRESSES)) { + assert(addr in MAINNET_SYSTEM_CONTRACTS, `${addr} is not predeployed`); + } +}); + +Deno.test("every predeploy is a contract: nonce 1, zero balance, real code", () => { + for (const [addr, a] of Object.entries(MAINNET_SYSTEM_CONTRACTS)) { + assertEquals(a.nonce, "0x1", `${addr} must look like a deployed contract`); + assertEquals(a.balance, "0x0", `${addr} must hold no ether`); + const code = a.code ?? ""; + assert(code.startsWith("0x") && code.length > 10, `${addr} has no code`); + assertEquals(code.length % 2, 0, `${addr} code is not whole bytes`); + } +}); + +Deno.test("predeploy() shapes a recipe's own contract the same way", () => { + assertEquals(predeploy("0xfeed"), { nonce: "0x1", balance: "0x0", code: "0xfeed" }); +}); diff --git a/scripts/trie-padding.ts b/scripts/trie-padding.ts new file mode 100644 index 0000000..762a862 --- /dev/null +++ b/scripts/trie-padding.ts @@ -0,0 +1,131 @@ +// trie-padding: create N fresh accounts on the EL BEFORE any block building +// (a warmup script), by sending 1-gwei transfers through the EL's own RPC so +// they land in the client's local payloads. decker's L1 genesis alloc is 12 +// accounts: the state trie is a root with a dozen leaves, a shape mainnet +// never has. A builder whose root hasher is only exercised on mainnet-deep +// tries can compute wrong roots here - live-hit 2026-09-16 with an in-process +// builder; ~1000 leaves give the trie branch nodes two to three levels deep. +// Bisection lever and mainnet-shape knob, not a fix. +// +// reth caps pending txs per sender at 16, so each round sends 16 per sender +// and waits for inclusion. 4 senders -> 64 accounts per block. +import { + JsonRpcProvider, + keccak256, + toBeHex, + Wallet, +} from "npm:ethers@^6.13.0"; +import { STATIC_PREFUNDED_PRIVKEYS } from "../generators/l1/constants.ts"; +import { findComponent, lookup } from "../utils/resolve.ts"; +import { portNum } from "../utils/types.ts"; +import type { Recipe, Script } from "../utils/types.ts"; + +export type PaddingSpec = { + el: string; // EL container name (its "rpc" port) + accounts: number; // fresh accounts to create; 0 = no-op + senders?: readonly string[]; // private keys; default anvil #5..#8 (unused by contender/signproxy) +}; + +const DEFAULT_SENDERS = STATIC_PREFUNDED_PRIVKEYS.slice(5, 9); +const PER_SENDER_PER_ROUND = 16; // reth txpool max_account_slots +const RPC_PORT = "rpc"; + +function resolveRpcUrl(recipe: Recipe, name: string): string { + const loc = findComponent(recipe, name); + if (loc.kind !== "container") { + throw new Error(`trie-padding: ${name} is not a container`); + } + const proto = lookup(loc.def.prototype); + const portSpec = + (loc.def.config?.ports as Record | undefined) + ?.[RPC_PORT] ?? + proto.ports[RPC_PORT]; + if (portSpec === undefined) { + throw new Error(`trie-padding: ${name} has no port ${RPC_PORT}`); + } + // DECKER_SERVICE_HOSTS=1: containers are reachable by name (k8s Services, + // as when this runs inside the cluster); default is the local port-forward flow. + const host = Deno.env.get("DECKER_SERVICE_HOSTS") === "1" + ? name + : "localhost"; + return `http://${host}:${portNum(portSpec as Parameters[0])}`; +} + +async function waitForChain( + provider: JsonRpcProvider, + deadlineMs: number, +): Promise { + while (true) { + try { + if ((await provider.getBlockNumber()) >= 1) return; + } catch { /* not up yet */ } + if (Date.now() > deadlineMs) { + throw new Error("trie-padding: EL never produced block 1"); + } + await new Promise((r) => setTimeout(r, 2000)); + } +} + +export function triePadding(spec: PaddingSpec): Script { + const run: Script = async (recipe: Recipe) => { + if (spec.accounts <= 0) return; + const url = resolveRpcUrl(recipe, spec.el); + const provider = new JsonRpcProvider(url, undefined, { + staticNetwork: true, + polling: true, + }); + await waitForChain(provider, Date.now() + 10 * 60_000); + const chainId = (await provider.getNetwork()).chainId; + const senders = (spec.senders ?? DEFAULT_SENDERS).map((k) => + new Wallet(k, provider) + ); + let created = 0; + let round = 0; + while (created < spec.accounts) { + const expect = new Map(); + for (const w of senders) { + if (created >= spec.accounts) break; + const nonce = await provider.getTransactionCount(w.address, "latest"); + let sent = 0; + for ( + ; + sent < PER_SENDER_PER_ROUND && created < spec.accounts; + sent++, created++ + ) { + // deterministic fresh address: keccak(index) -> 20 bytes; nobody holds its key + const to = "0x" + keccak256(toBeHex(created, 32)).slice(26); + const raw = await w.signTransaction({ + type: 2, + chainId, + to, + value: 1_000_000_000n, + nonce: nonce + sent, + gasLimit: 21_000n, + maxFeePerGas: 10_000_000_000n, + maxPriorityFeePerGas: 1_000_000_000n, + }); + await provider.send("eth_sendRawTransaction", [raw]); + } + expect.set(w.address, nonce + sent); + } + // wait for this round to land (one block normally) + const deadline = Date.now() + 120_000; + for (const [addr, want] of expect) { + while ((await provider.getTransactionCount(addr, "latest")) < want) { + if (Date.now() > deadline) { + throw new Error( + `trie-padding: round ${round} not included in 120s`, + ); + } + await new Promise((r) => setTimeout(r, 1500)); + } + } + round++; + } + console.log( + `trie-padding: created ${created} accounts in ${round} rounds via ${url}`, + ); + }; + Object.defineProperty(run, "name", { value: "trie-padding" }); + return run; +} diff --git a/utils/emit.ts b/utils/emit.ts index 5ffa0b0..b346143 100644 --- a/utils/emit.ts +++ b/utils/emit.ts @@ -1,4 +1,5 @@ import { dirname, isAbsolute } from "jsr:@std/path@^1.0.0"; +import { assetFile, imagesAssets, imagesManifest } from "./image-build.ts"; import { rendererFor } from "./renderers.ts"; import type { BinaryBuildSpec, ImageBuildSpec, Recipe, Renderer, RendererPaths } from "./types.ts"; @@ -72,7 +73,7 @@ export async function emit( for (const [tag, spec] of out.imageBuilds) { const existing = imageBuilds.get(tag); if (existing) { - if (existing.repo !== spec.repo || existing.ref !== spec.ref || existing.cmd !== spec.cmd) { + if (JSON.stringify(existing) !== JSON.stringify(spec)) { throw new Error(`image tag ${tag} produced by conflicting ImageBuildSpec`); } } else { @@ -99,6 +100,17 @@ export async function emit( if (out.binaries) binaries.push(...out.binaries); } + // images.json (+ the _assets it names) sits next to the manifests so an + // external builder can build exactly the tags the manifests reference. This + // is the single source of truth for image pins: nothing downstream should + // carry its own copy of a repo/ref. + await Deno.writeTextFile(`${manifestDir}/images.json`, imagesManifest(imageBuilds)); + const assets = imagesAssets(imageBuilds); + if (assets.length > 0) { + await Deno.mkdir(`${manifestDir}/images-assets`, { recursive: true }); + for (const a of assets) await Deno.writeFile(`${manifestDir}/images-assets/${a.name}`, assetFile(a.name, a.dir)); + } + await materializeRuntime(manifestDir, runtimeDir); return { binaries, imageBuilds, binaryBuilds, selected, paths: { runtimeDir, manifestDir } }; } diff --git a/utils/image-build.ts b/utils/image-build.ts index cfde9ca..d85de5b 100644 --- a/utils/image-build.ts +++ b/utils/image-build.ts @@ -1,5 +1,6 @@ import type { ImageBuildSpec, ImageEngine } from "./types.ts"; +import { fromFileUrl, toFileUrl } from "jsr:@std/path@^1.0.0"; import { DECKER_ROOT } from "./root.ts"; const CACHE_DIR = `${DECKER_ROOT}/cache/images`; @@ -20,11 +21,122 @@ export function imagePullMode(): boolean { } export function imageTag(spec: ImageBuildSpec): string { - const base = `decker-${repoBasename(spec.repo)}:${slug(spec.ref)}`; + const tag = slug(spec.ref) + (spec.variant ? `-${slug(spec.variant)}` : ""); + const base = `decker-${spec.name ? slug(spec.name) : repoBasename(spec.repo)}:${tag}`; const reg = imageRegistry(); return reg ? `${reg}/${base}` : base; } +// Build assets are the files a build needs that do not come from the source +// repo: a Dockerfile that patches it, the patches themselves. decker's own live +// in _assets/, read relative to this module (bundled into the compiled binary +// via `make compile`) rather than under DECKER_ROOT, which is the runtime +// directory and need not hold the repo. +// +// A recipe outside decker owns its assets the same way: every function here +// takes the directory as an argument, and ImageBuildSpec.assetsDir carries it, +// so a recipe in its own repo can ship a Dockerfile without adding it to +// decker. Pass `new URL("../_assets/", import.meta.url).href` from the module +// that declares the spec; a plain absolute path works too. +const DECKER_ASSETS = new URL("../_assets/", import.meta.url); + +export function assetsRoot(dir?: string): URL { + if (!dir) return DECKER_ASSETS; + if (dir.includes("://")) return new URL(dir.endsWith("/") ? dir : `${dir}/`); + return toFileUrl(dir.endsWith("/") ? dir : `${dir}/`); +} + +// assetsVariant hashes those files into a short tag suffix: FNV-1a 64 over the +// bytes in the given order, first 12 hex chars. The scheme is deliberately +// trivial so an external builder can reproduce a tag without running decker. +export function assetsVariant(files: string[], dir?: string): string { + const root = assetsRoot(dir); + let h = 0xcbf29ce484222325n; + for (const f of files) { + const bytes = Deno.readFileSync(new URL(f, root)); + for (const b of bytes) { + h ^= BigInt(b); + h = (h * 0x100000001b3n) & 0xffffffffffffffffn; + } + } + return h.toString(16).padStart(16, "0").slice(0, 12); +} + +// buildCommand derives the docker/podman build line from the spec's structured +// fields. Two shapes: +// - repo context: the cloned repo is the build context, Dockerfile inside it. +// - asset Dockerfile: the assets dir is the context and the repo comes in as +// the named context `src`, so a Dockerfile that lives outside the repo can +// patch its sources. +// $ENGINE/$IMAGE/$ASSETS are substituted by the caller's environment ($ASSETS +// is the spec's assets dir, decker's own _assets unless assetsDir says else). +export function buildCommand(spec: ImageBuildSpec): string { + if (spec.cmd) return spec.cmd; + const args = ["$ENGINE", "build"]; + if (spec.assetDockerfile) { + args.push("--build-context", "src=.", "-f", `"$ASSETS/${spec.assetDockerfile}"`); + } else { + args.push("-f", spec.dockerfile ?? "Dockerfile"); + } + if (spec.target) args.push("--target", spec.target); + for (const [k, v] of Object.entries(spec.buildArgs ?? {})) args.push("--build-arg", `${k}=${v}`); + for (const id of spec.secrets ?? []) args.push("--secret", `id=${id},env=${id.toUpperCase()}`); + args.push("-t", "$IMAGE", spec.assetDockerfile ? '"$ASSETS"' : "."); + return args.join(" "); +} + +// imagesManifest is what `build` writes next to the manifests as images.json: +// every image the recipe references, keyed by the tag the manifests use, with +// the build described in full. An external builder (CI, an in-cluster build +// system) reads this instead of keeping its own copy of the pins. +export function imagesManifest(specs: Map): string { + const out: Record = {}; + for (const [tag, spec] of [...specs].sort(([a], [b]) => a < b ? -1 : 1)) { + out[tag] = { + repo: spec.repo, + ref: spec.ref, + name: spec.name ?? repoBasename(spec.repo), + ...(spec.variant ? { variant: spec.variant } : {}), + ...(spec.version ? { version: spec.version } : {}), + ...(spec.assetDockerfile + ? { assetDockerfile: spec.assetDockerfile } + : { dockerfile: spec.dockerfile ?? "Dockerfile" }), + ...(spec.assets?.length ? { assets: spec.assets } : {}), + ...(spec.target ? { target: spec.target } : {}), + ...(spec.buildArgs ? { buildArgs: spec.buildArgs } : {}), + ...(spec.secrets?.length ? { secrets: spec.secrets } : {}), + cmd: buildCommand(spec), + }; + } + return JSON.stringify(out, null, 2) + "\n"; +} + +// Every asset file a set of specs depends on: what an external builder must +// receive alongside images.json. Assets are addressed by bare filename there, +// so two specs contributing the same name from different directories is an +// error rather than a silent last-one-wins. +export type AssetFile = { name: string; dir?: string }; + +export function imagesAssets(specs: Map): AssetFile[] { + const files = new Map(); + const add = (name: string, dir?: string) => { + const seen = files.get(name); + if (seen && seen.dir !== dir) { + throw new Error(`asset ${name} comes from two directories: ${seen.dir ?? "_assets"} and ${dir ?? "_assets"}`); + } + files.set(name, { name, dir }); + }; + for (const spec of specs.values()) { + if (spec.assetDockerfile) add(spec.assetDockerfile, spec.assetsDir); + for (const a of spec.assets ?? []) add(a, spec.assetsDir); + } + return [...files.values()].sort((a, b) => a.name < b.name ? -1 : 1); +} + +export function assetFile(name: string, dir?: string): Uint8Array { + return Deno.readFileSync(new URL(name, assetsRoot(dir))); +} + function repoBasename(repo: string): string { const last = repo.replace(/\.git$/, "").replace(/\/$/, "").split("/").pop() ?? repo; return slug(last); @@ -72,17 +184,18 @@ async function ensureClone(spec: ImageBuildSpec): Promise { } catch { /* fine */ } await run(["git", "clone", spec.repo, cloneDir]); } - await run(["git", "fetch", "origin", spec.ref], { cwd: cloneDir }); - await run(["git", "checkout", spec.ref], { cwd: cloneDir }); - await run(["git", "reset", "--hard", `origin/${spec.ref}`], { cwd: cloneDir }); + // Branches, tags AND commit SHAs: fetch the ref itself and check out what + // arrived. `reset --hard origin/` only ever existed for branches. + await run(["git", "fetch", "--force", "origin", spec.ref], { cwd: cloneDir }); + await run(["git", "checkout", "--force", "--detach", "FETCH_HEAD"], { cwd: cloneDir }); return cloneDir; } async function buildOne(tag: string, spec: ImageBuildSpec, engine: ImageEngine): Promise { const cloneDir = await ensureClone(spec); - await run(["sh", "-c", spec.cmd], { + await run(["sh", "-c", buildCommand(spec)], { cwd: cloneDir, - env: { ...Deno.env.toObject(), IMAGE: tag, ENGINE: engine }, + env: { ...Deno.env.toObject(), IMAGE: tag, ENGINE: engine, DECKER_ROOT, ASSETS: fromFileUrl(assetsRoot(spec.assetsDir)) }, }); if (!(await imageExists(tag, engine))) { throw new Error(`build for ${tag} ran but image is not present afterward`); diff --git a/utils/image-build_test.ts b/utils/image-build_test.ts new file mode 100644 index 0000000..d5089a5 --- /dev/null +++ b/utils/image-build_test.ts @@ -0,0 +1,133 @@ +// Unit tests for the derived build command and the images.json payload: the +// two pieces external builders depend on byte-for-byte. +import { assertEquals, assertThrows } from "jsr:@std/assert@^1.0.0"; +import { assetsRoot, assetsVariant, buildCommand, imagesAssets, imagesManifest, imageTag } from "./image-build.ts"; +import type { ImageBuildSpec } from "./types.ts"; + +const REPO = "https://github.com/flashbots/reth.git"; + +Deno.test("buildCommand: repo context with target and build args", () => { + assertEquals( + buildCommand({ + repo: REPO, + ref: "abc", + dockerfile: "docker/Dockerfile.node", + target: "runtime", + buildArgs: { FEATURES: "jemalloc" }, + }), + "$ENGINE build -f docker/Dockerfile.node --target runtime " + + "--build-arg FEATURES=jemalloc -t $IMAGE .", + ); +}); + +Deno.test("buildCommand: asset Dockerfile makes the repo the named context src", () => { + assertEquals( + buildCommand({ repo: REPO, ref: "abc", assetDockerfile: "node.Dockerfile", secrets: ["gh_token"] }), + '$ENGINE build --build-context src=. -f "$ASSETS/node.Dockerfile" ' + + '--secret id=gh_token,env=GH_TOKEN -t $IMAGE "$ASSETS"', + ); +}); + +Deno.test("buildCommand: default Dockerfile, and cmd overrides everything", () => { + assertEquals(buildCommand({ repo: REPO, ref: "abc" }), "$ENGINE build -f Dockerfile -t $IMAGE ."); + assertEquals(buildCommand({ repo: REPO, ref: "abc", cmd: "custom" }), "custom"); +}); + +Deno.test("imagesManifest: the human version travels beside the pinned ref", () => { + const specs = new Map([ + ["decker-reth:abc", { + repo: REPO, + ref: "abc", + name: "reth", + version: "v1.16.0", + dockerfile: "docker/Dockerfile.node", + }], + ]); + const got = JSON.parse(imagesManifest(specs)); + assertEquals(got["decker-reth:abc"].version, "v1.16.0"); + assertEquals(imageTag(specs.get("decker-reth:abc")!), "decker-reth:abc"); +}); + +Deno.test("imagesManifest: one entry per tag, name defaulted, cmd derived", () => { + const specs = new Map([ + ["decker-reth:abc-v1", { + repo: REPO, + ref: "abc", + name: "reth", + variant: "v1", + assetDockerfile: "node.Dockerfile", + assets: ["node-0001.patch"], + secrets: ["gh_token"], + }], + ["decker-mev-boost-relay:main", { + repo: "https://github.com/flashbots/mev-boost-relay.git", + ref: "main", + dockerfile: "Dockerfile", + }], + ]); + const got = JSON.parse(imagesManifest(specs)); + assertEquals(Object.keys(got).sort(), ["decker-mev-boost-relay:main", "decker-reth:abc-v1"]); + assertEquals(got["decker-mev-boost-relay:main"].name, "mev-boost-relay"); + assertEquals(got["decker-reth:abc-v1"].secrets, ["gh_token"]); + assertEquals(got["decker-reth:abc-v1"].cmd, buildCommand(specs.get("decker-reth:abc-v1")!)); + assertEquals(imagesAssets(specs), [ + { name: "node-0001.patch", dir: undefined }, + { name: "node.Dockerfile", dir: undefined }, + ]); +}); + +Deno.test("imageTag: name and variant", () => { + const spec: ImageBuildSpec = { repo: REPO, ref: "abc", name: "reth", variant: "v1" }; + assertEquals(imageTag(spec), "decker-reth:abc-v1"); +}); + +// A recipe living outside decker (its own repo, private or not) declares specs +// whose Dockerfile and patches sit in ITS tree. assetsDir is how those files are +// found, hashed into the tag, and shipped next to images.json. +Deno.test("assetsDir: an out-of-tree recipe owns its build assets", async () => { + const dir = await Deno.makeTempDir(); + await Deno.writeTextFile(`${dir}/my.Dockerfile`, "FROM scratch\n"); + await Deno.writeTextFile(`${dir}/0001.patch`, "diff\n"); + + const spec: ImageBuildSpec = { + repo: REPO, + ref: "abc", + name: "mine", + assetsDir: dir, + assetDockerfile: "my.Dockerfile", + assets: ["0001.patch"], + }; + spec.variant = assetsVariant(["my.Dockerfile", "0001.patch"], dir); + + assertEquals(spec.variant.length, 12); + assertEquals(imageTag(spec), `decker-mine:abc-${spec.variant}`); + assertEquals(imagesAssets(new Map([["t", spec]])), [ + { name: "0001.patch", dir }, + { name: "my.Dockerfile", dir }, + ]); + // The build line stays dir-agnostic: $ASSETS is resolved when the build runs. + assertEquals( + buildCommand(spec), + '$ENGINE build --build-context src=. -f "$ASSETS/my.Dockerfile" -t $IMAGE "$ASSETS"', + ); + assertEquals(assetsRoot(dir).href, assetsRoot(`${dir}/`).href); + await Deno.remove(dir, { recursive: true }); +}); + +Deno.test("assetsVariant: same bytes, same suffix; changed bytes, new suffix", async () => { + const dir = await Deno.makeTempDir(); + await Deno.writeTextFile(`${dir}/a`, "one"); + const first = assetsVariant(["a"], dir); + assertEquals(assetsVariant(["a"], dir), first); + await Deno.writeTextFile(`${dir}/a`, "two"); + assertEquals(assetsVariant(["a"], dir) === first, false); + await Deno.remove(dir, { recursive: true }); +}); + +Deno.test("imagesAssets: one filename cannot mean two files", () => { + const specs = new Map([ + ["a", { repo: REPO, ref: "1", assetDockerfile: "x.Dockerfile" }], + ["b", { repo: REPO, ref: "2", assetDockerfile: "x.Dockerfile", assetsDir: "/tmp/elsewhere" }], + ]); + assertThrows(() => imagesAssets(specs), Error, "two directories"); +}); diff --git a/utils/resolve_test.ts b/utils/resolve_test.ts new file mode 100644 index 0000000..4dbd629 --- /dev/null +++ b/utils/resolve_test.ts @@ -0,0 +1,28 @@ +// buildContainerFor is the seam every renderer goes through, so the generic +// `config.image` override belongs here: it is how an external builder (or +// anyone) points a container at an already-built image without touching the +// prototype. +import { assertEquals } from "jsr:@std/assert@^1.0.0"; +import { buildContainerFor } from "./resolve.ts"; +import type { ContainerDef, Ctx, Prototype } from "./types.ts"; + +const ctx: Ctx = { url: (n, p) => `http://${n}:${p}`, artifactsHostPath: "/artifacts" }; + +const proto: Prototype = { + ports: { rpc: 8545 }, + buildContainer: () => ({ + container: { image: "upstream/image:pinned", ports: { rpc: 8545 } }, + }), +}; + +Deno.test("config.image replaces the prototype's image", () => { + const def: ContainerDef = { name: "el-1", prototype: proto, config: { image: "reg/mine:v0" } }; + assertEquals(buildContainerFor(def, ctx).container.image, "reg/mine:v0"); +}); + +Deno.test("no override, an empty one, or a non-string leaves the prototype's image", () => { + for (const config of [undefined, {}, { image: "" }, { image: 7 }]) { + const def: ContainerDef = { name: "el-1", prototype: proto, config } as ContainerDef; + assertEquals(buildContainerFor(def, ctx).container.image, "upstream/image:pinned"); + } +}); diff --git a/utils/types.ts b/utils/types.ts index 82cb631..27f0127 100644 --- a/utils/types.ts +++ b/utils/types.ts @@ -34,10 +34,47 @@ export function portInService(p: PortSpec): boolean { return typeof p === "number" ? true : p.service !== false; } +// An image built from a pinned source repo. The build is described in +// STRUCTURED fields, not as a shell line: the docker/podman command is derived +// from them (utils/image-build.ts buildCommand), and the same fields are +// emitted to manifests//images.json so an external builder - CI, or an +// in-cluster buildkit - can reproduce the build without parsing a shell string. +// `cmd` remains as an escape hatch for a build no field combination expresses. export type ImageBuildSpec = { repo: string; ref: string; - cmd: string; + // Image name when one repo yields several images (default: repo basename). + name?: string; + // Extra tag suffix for a build whose inputs are more than repo+ref (local + // Dockerfile/patches); imageTag appends "-" so a patched image + // never shares the pristine tag. + variant?: string; + // The human name of `ref` (a release tag like "v1.16.0", or a branch name). + // Never used to address the image - refs stay commit SHAs so a tag cannot + // move under us - but it travels in images.json so a UI can show "v1.16.0" + // beside the hash instead of the hash alone. + version?: string; + // Dockerfile path inside the repo (default "Dockerfile"). + dockerfile?: string; + // Dockerfile that lives OUTSIDE the source repo, in the assets dir. Wins over + // `dockerfile`: the repo is then a named build context (`src`) instead of the + // build context, which is how a patched build reaches the repo's sources. + assetDockerfile?: string; + // Other asset files the build reads (patches the Dockerfile applies). Both + // kinds of asset feed `variant` via assetsVariant. + assets?: string[]; + // Where those files live. Default: decker's own _assets/. A recipe outside + // decker points at its own directory - `new URL("../_assets/", + // import.meta.url).href`, or an absolute path - and owns its build inputs + // without adding them to decker. + assetsDir?: string; + target?: string; + buildArgs?: Record; + // Build secret ids; each is read from the equally named env var upper-cased + // ("gh_token" -> $GH_TOKEN) and passed as --secret, never baked in. + secrets?: string[]; + // Escape hatch: a full shell build line, overriding everything derived. + cmd?: string; }; // A host binary built from source, the process-side analogue of ImageBuildSpec. @@ -139,6 +176,10 @@ export type L1ArtifactsSpec = { // rather than an overwrite. The genesis state root is computed from the result, // so the CL's genesis state and the EL agree on the genesis block either way. genesisAccounts?: GenesisAlloc; + // Validator withdrawal credentials. "eth1" (default): 0x01, so every block + // sweeps partial withdrawals (mainnet-like). "none": 0x00 (BLS), no + // automatic withdrawals - a bisection lever, not a production shape. + withdrawals?: "eth1" | "none"; }; // OP-stack: an L1 (with the OP system contracts predeployed) plus the L2 genesis From 4f6f585c6ed27800325775f20920a525d0324167 Mon Sep 17 00:00:00 2001 From: Kim Romero Date: Wed, 23 Sep 2026 10:13:46 +0300 Subject: [PATCH 2/2] docs: move the images notes out of the README MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per review: the README keeps a one-line pointer, and notes/images.md has room to show the shapes — a spec, an images.json entry, an out-of-tree asset dir — instead of describing them in prose. --- README.md | 21 ++------------ notes/images.md | 74 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 77 insertions(+), 18 deletions(-) create mode 100644 notes/images.md diff --git a/README.md b/README.md index ed32a27..9535ad5 100644 --- a/README.md +++ b/README.md @@ -95,24 +95,9 @@ You can evolve `decker` in multiple layers and use in dev or CI setups of your p - **Renderers:** Run your recipe on any target (podman, docker, process-compose and anything you want) - **CLI:** Hack on the clone, run immediately with preinstalled binary -## Images - -A container can name a published image, or pin a source repo and let decker -build it. A pinned build is described in fields — repo, ref, Dockerfile, -target, build args, secrets — and `decker build` writes them all to -`manifests//images.json` beside the rendered manifests, keyed by the -exact tag those manifests reference. Run the images locally and decker builds -what is missing; set `DECKER_IMAGE_MODE=pull` and it builds nothing, so CI or -an in-cluster build system can supply the images from that one file instead of -keeping its own copy of your pins. - -Builds that need files the source repo does not have — a Dockerfile that -patches it, the patches — name them in `assets` and `assetDockerfile`. They -live in `_assets/` here, or in your own repo: set `assetsDir` to -`new URL("../_assets/", import.meta.url).href` and a recipe outside decker -ships its own build inputs. Those bytes are hashed into the image tag, so a -patched image never collides with a pristine one, and they are copied next to -`images.json` for whoever does the building. +Images can be pinned to a source repo and built by decker, or supplied by an +external builder from the `images.json` a build emits — see +[notes/images.md](notes/images.md). ## Why? diff --git a/notes/images.md b/notes/images.md new file mode 100644 index 0000000..abec66a --- /dev/null +++ b/notes/images.md @@ -0,0 +1,74 @@ +# Images + +A container can name a published image, or pin a source repo and let decker +build it. + +A pinned build is described in fields — repo, ref, Dockerfile, target, build +args, secrets — and the build command is derived from them. `cmd` remains as an +escape hatch for a build no combination of fields expresses. + +```ts +function image(def: ContainerDef): ImageBuildSpec { + return { + repo: "https://github.com/flashbots/mev-boost-relay.git", + ref: def.config?.ref as string ?? "main", + dockerfile: "Dockerfile", + }; +} +``` + +## images.json + +`decker build` writes every spec to `manifests//images.json`, beside +the rendered manifests and keyed by the exact tag those manifests reference: + +```json +{ + "decker-mev-boost-relay:main": { + "repo": "https://github.com/flashbots/mev-boost-relay.git", + "ref": "main", + "name": "mev-boost-relay", + "dockerfile": "Dockerfile", + "cmd": "$ENGINE build -f Dockerfile -t $IMAGE ." + } +} +``` + +Run the images locally and decker builds whatever is missing. Set +`DECKER_IMAGE_MODE=pull` and it builds nothing at all, so CI or an in-cluster +build system can supply them — reading this one file instead of keeping a +second copy of your pins, which is the copy that drifts. +`DECKER_IMAGE_REGISTRY` prefixes the tags so the rendered manifests point at +the registry those images were pushed to. + +## Build assets + +Some builds need files the source repo does not have: a Dockerfile that patches +it, the patches it applies. Name them in `assetDockerfile` and `assets`. The +repo then arrives as the named build context `src` instead of being the build +context, which is how a Dockerfile outside the repo reaches its sources. + +They live in decker's `_assets/`, or in your own repo — set `assetsDir` and a +recipe that lives outside decker ships its own build inputs: + +```ts +const ASSETS_DIR = new URL("../_assets/", import.meta.url).href; + +export const IMAGE: ImageBuildSpec = { + repo: "https://github.com/example/thing.git", + ref: "…", + assetsDir: ASSETS_DIR, + assetDockerfile: "thing.Dockerfile", + assets: ["0001-some.patch"], + variant: assetsVariant(["thing.Dockerfile", "0001-some.patch"], ASSETS_DIR), + secrets: ["gh_token"], +}; +``` + +Asset bytes hash into the image tag through `variant`, so a patched image can +never collide with a pristine build of the same ref, and moving the files +between repos does not change the tag. `decker build` copies them to +`manifests//images-assets/` for whoever does the building. + +Secrets are passed as `--secret id=,env=` and read from the environment +at build time; nothing is baked into the image.