Structured image builds and images.json

Makefile

@@ -1,4 +1,4 @@
-INCLUDES := --include commands --include containers --include generators --include recipes --include renderers --include decker.example.ts
+INCLUDES := --include commands --include containers --include generators --include recipes --include renderers --include _assets --include decker.example.ts
 OUTPUT   ?= decker
 TARGET   ?=
 

README.md

@@ -95,6 +95,10 @@ You can evolve `decker` in multiple layers and use in dev or CI setups of your p
 - **Renderers:** Run your recipe on any target (podman, docker, process-compose and anything you want)
 - **CLI:** Hack on the clone, run immediately with preinstalled binary
 
+Images can be pinned to a source repo and built by decker, or supplied by an
+external builder from the `images.json` a build emits — see
+[notes/images.md](notes/images.md).
+
 ## Why?
 
 Sophisticated tools and their abstraction layers speed up humans but slow down LLM problem solving capabilities and reduce success. In addition, developers often try to fix upstream the tools they depend on, to satisfy their own use-case specific necessities.

containers/helix-simulator.ts

@@ -26,7 +26,7 @@ export const ports: Ports = {
 function image(def: ContainerDef): string | ImageBuildSpec {
   const build = def.config?.build as { repo: string; ref: string } | undefined;
   if (!build) return (def.config?.image as string | undefined) ?? DEFAULT_IMAGE;
-  return { repo: build.repo, ref: build.ref, cmd: "$ENGINE build -t $IMAGE -f simulator.Dockerfile ." };
+  return { repo: build.repo, ref: build.ref, dockerfile: "simulator.Dockerfile" };
 }
 
 export function buildContainer(def: ContainerDef, ctx: Ctx): ContainerResult {

containers/lighthouse-beacon.ts

@@ -58,7 +58,9 @@ export function buildContainer(def: ContainerDef, ctx: Ctx): ContainerResult {
         "--execution-jwt", "/artifacts/jwtsecret",
         "--always-prepare-payload",
         "--prepare-payload-lookahead", "8000",
-        "--suggested-fee-recipient", "0x690B9A9E9aa1C9dB991C7721a92d351Db4FaC990",
+        // config.feeRecipient overrides the default. A builder that proves the
+        // proposer payment against state needs a recipient that EXISTS in genesis.
+        "--suggested-fee-recipient", (def.config?.feeRecipient as string | undefined) ?? "0x690B9A9E9aa1C9dB991C7721a92d351Db4FaC990",
         ...(supernode ? ["--supernode"] : []),
         ...(peerMultiaddrs.length > 0 ? ["--libp2p-addresses", peerMultiaddrs.join(",")] : []),
         ...(builder ? [

containers/lighthouse-validator.ts

@@ -15,7 +15,9 @@ export function buildContainer(def: ContainerDef, ctx: Ctx): ContainerResult {
         "--testnet-dir", "/artifacts/testnet",
         "--init-slashing-protection",
         "--beacon-nodes", ctx.url(beacon, "http"),
-        "--suggested-fee-recipient", "0x690B9A9E9aa1C9dB991C7721a92d351Db4FaC990",
+        // config.feeRecipient overrides the default. A builder that proves the
+        // proposer payment against state needs a recipient that EXISTS in genesis.
+        "--suggested-fee-recipient", (def.config?.feeRecipient as string | undefined) ?? "0x690B9A9E9aa1C9dB991C7721a92d351Db4FaC990",
         "--builder-proposals",
         "--prefer-builder-proposals",
       ],

containers/mev-boost-relay.ts

@@ -12,7 +12,7 @@ function imageSpec(def: ContainerDef): ImageBuildSpec {
   return {
     repo: (def.config?.repo as string | undefined) ?? DEFAULT_REPO,
     ref: (def.config?.ref as string | undefined) ?? DEFAULT_REF,
-    cmd: "$ENGINE build -t $IMAGE .",
+    dockerfile: "Dockerfile",
   };
 }
 const BLS_KEYS_FIXTURE = new URL("../generators/l1/bls_keys.json", import.meta.url);

e2e/images_test.ts

@@ -0,0 +1,34 @@
+// images.json is the contract with external builders: every image a rendered
+// recipe references, described in structured fields, next to the manifests.
+import { assert, assertEquals } from "jsr:@std/assert@^1.0.0";
+import { join } from "jsr:@std/path@^1.0.0";
+import { runDecker, withTmp } from "./helpers.ts";
+
+Deno.test("build: images.json describes every image the manifests reference", async () => {
+  await withTmp(async (root) => {
+    const r = await runDecker(["build", "rbuilder"], { cwd: root, env: { DECKER_ROOT: root } });
+    assertEquals(r.code, 0, r.out);
+
+    const images = JSON.parse(await Deno.readTextFile(join(root, "manifests", "rbuilder", "images.json")));
+    const tags = Object.keys(images);
+    assert(tags.length > 0, "recipe builds images, so images.json must not be empty");
+
+    const rendered = await Deno.readTextFile(join(root, "manifests", "rbuilder", "podman.yaml"));
+    for (const tag of tags) {
+      assert(rendered.includes(tag), `${tag} is in images.json but nothing references it`);
+      const spec = images[tag];
+      assert(spec.repo && spec.ref, `${tag} must pin a repo and a ref`);
+      assert(spec.cmd.includes("$ENGINE") && spec.cmd.includes("$IMAGE"), `${tag} cmd is not substitutable`);
+      assert(spec.dockerfile || spec.assetDockerfile, `${tag} names no Dockerfile`);
+    }
+  });
+});
+
+Deno.test("build: a recipe with no source-built images still writes images.json", async () => {
+  await withTmp(async (root) => {
+    const r = await runDecker(["build", "contender-bench"], { cwd: root, env: { DECKER_ROOT: root } });
+    assertEquals(r.code, 0, r.out);
+    const path = join(root, "manifests", "contender-bench", "images.json");
+    assertEquals(JSON.parse(await Deno.readTextFile(path)), {});
+  });
+});

generators/l1/genesis-ssz.ts

@@ -42,6 +42,8 @@ export type GenesisSszOpts = {
   // own. Must be the same root fed to the EL client, or CL/EL disagree on the
   // genesis execution block hash and the chain never advances.
   elStateRoot?: Uint8Array;
+  // see L1ArtifactsSpec.withdrawals
+  withdrawals?: "eth1" | "none";
 };
 
 export async function renderGenesisSsz(opts: GenesisSszOpts): Promise<Uint8Array> {
@@ -68,7 +70,9 @@ export async function renderGenesisSsz(opts: GenesisSszOpts): Promise<Uint8Array
   for (const k of blsKeys) {
     const pubkey = fromHex(k.pub);
     const wd = new Uint8Array(32);
-    wd[0] = 0x01;
+    // 0x01 = eth1 address creds (partial withdrawals swept every block);
+    // 0x00 = BLS creds: nothing is ever withdrawable automatically.
+    wd[0] = opts.withdrawals === "none" ? 0x00 : 0x01;
     wd.set(pubkey.slice(0, 20), 12);
     state.validators.push({
       pubkey,

generators/l1/index.ts

@@ -31,6 +31,7 @@ export type GenerateOpts = {
   // from the result and handed to both the EL genesis and the CL's genesis state,
   // so the two agree either way.
   genesisAccounts?: GenesisAlloc;
+  withdrawals?: "eth1" | "none";
 };
 
 export type GenerateResult = {
@@ -70,7 +71,7 @@ export async function generate(opts: GenerateOpts): Promise<GenerateResult> {
   await Deno.writeTextFile(`${outDir}/genesis.json`, el.json);
   await Deno.writeFile(
     `${testnetDir}/genesis.ssz`,
-    await renderGenesisSsz({ genesisTimeSeconds, fork, elStateRoot }),
+    await renderGenesisSsz({ genesisTimeSeconds, fork, elStateRoot, withdrawals: opts.withdrawals }),
   );
 
   const keys = await loadBlsKeys();

generators/l1/system-contracts.ts

@@ -0,0 +1,39 @@
+// The execution-layer system contracts mainnet runs, as `genesisAccounts` for
+// an L1 artifacts spec (see el-genesis.ts / state-root.ts).
+//
+// EIP-4788 / 2935 / 7002 / 7251 (bytecodes from alloy-eips 2.0.5, the constants
+// reth itself uses): the EL makes their pre/post-block system calls EVERY
+// block. With no code at those addresses the calls touch non-existent accounts
+// and the block's state diff takes a shape mainnet never produces - which is
+// exactly the kind of difference that makes a builder or a prover behave
+// differently here than in production.
+//
+// A recipe spreads these into its own `genesisAccounts` and adds whatever else
+// it needs:
+//
+//   genesisAccounts: { ...MAINNET_SYSTEM_CONTRACTS, [myAddr]: myPredeploy }
+import type { GenesisAlloc } from "./state-root.ts";
+
+const CODE = {
+  eip4788: "0x3373fffffffffffffffffffffffffffffffffffffffe14604d57602036146024575f5ffd5b5f35801560495762001fff810690815414603c575f5ffd5b62001fff01545f5260205ff35b5f5ffd5b62001fff42064281555f359062001fff015500",
+  eip2935: "0x3373fffffffffffffffffffffffffffffffffffffffe14604657602036036042575f35600143038111604257611fff81430311604257611fff9006545f5260205ff35b5f5ffd5b5f35611fff60014303065500",
+  eip7002: "0x3373fffffffffffffffffffffffffffffffffffffffe1460cb5760115f54807fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff146101f457600182026001905f5b5f82111560685781019083028483029004916001019190604d565b909390049250505036603814608857366101f457346101f4575f5260205ff35b34106101f457600154600101600155600354806003026004013381556001015f35815560010160203590553360601b5f5260385f601437604c5fa0600101600355005b6003546002548082038060101160df575060105b5f5b8181146101835782810160030260040181604c02815460601b8152601401816001015481526020019060020154807fffffffffffffffffffffffffffffffff00000000000000000000000000000000168252906010019060401c908160381c81600701538160301c81600601538160281c81600501538160201c81600401538160181c81600301538160101c81600201538160081c81600101535360010160e1565b910180921461019557906002556101a0565b90505f6002555f6003555b5f54807fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff14156101cd57505f5b6001546002828201116101e25750505f6101e8565b01600290035b5f555f600155604c025ff35b5f5ffd",
+  eip7251: "0x3373fffffffffffffffffffffffffffffffffffffffe1460d35760115f54807fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff1461019a57600182026001905f5b5f82111560685781019083028483029004916001019190604d565b9093900492505050366060146088573661019a573461019a575f5260205ff35b341061019a57600154600101600155600354806004026004013381556001015f358155600101602035815560010160403590553360601b5f5260605f60143760745fa0600101600355005b6003546002548082038060021160e7575060025b5f5b8181146101295782810160040260040181607402815460601b815260140181600101548152602001816002015481526020019060030154905260010160e9565b910180921461013b5790600255610146565b90505f6002555f6003555b5f54807fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff141561017357505f5b6001546001828201116101885750505f61018e565b01600190035b5f555f6001556074025ff35b5f5ffd0000",
+};
+
+// nonce 1 and no balance: what a deployed contract looks like on mainnet.
+export const predeploy = (code: string) => ({ nonce: "0x1", balance: "0x0", code });
+
+export const SYSTEM_CONTRACT_ADDRESSES = {
+  eip4788: "0x000F3df6D732807Ef1319fB7B8bB8522d0Beac02", // beacon block roots
+  eip2935: "0x0000F90827F1C53a10cb7A02335B175320002935", // historical block hashes
+  eip7002: "0x00000961Ef480Eb55e80D19ad83579A64c007002", // withdrawal requests
+  eip7251: "0x0000BBdDc7CE488642fb579F8B00f3a590007251", // consolidation requests
+} as const;
+
+export const MAINNET_SYSTEM_CONTRACTS: GenesisAlloc = {
+  [SYSTEM_CONTRACT_ADDRESSES.eip4788]: predeploy(CODE.eip4788),
+  [SYSTEM_CONTRACT_ADDRESSES.eip2935]: predeploy(CODE.eip2935),
+  [SYSTEM_CONTRACT_ADDRESSES.eip7002]: predeploy(CODE.eip7002),
+  [SYSTEM_CONTRACT_ADDRESSES.eip7251]: predeploy(CODE.eip7251),
+};

generators/l1/system-contracts_test.ts

@@ -0,0 +1,25 @@
+// The EL calls these contracts every block; missing code silently produces a
+// state-diff shape mainnet never has. Pin the addresses and the code.
+import { assert, assertEquals } from "jsr:@std/assert@^1.0.0";
+import { MAINNET_SYSTEM_CONTRACTS, predeploy, SYSTEM_CONTRACT_ADDRESSES } from "./system-contracts.ts";
+
+Deno.test("the four system contracts are present, at their mainnet addresses", () => {
+  assertEquals(Object.keys(MAINNET_SYSTEM_CONTRACTS).length, 4);
+  for (const addr of Object.values(SYSTEM_CONTRACT_ADDRESSES)) {
+    assert(addr in MAINNET_SYSTEM_CONTRACTS, `${addr} is not predeployed`);
+  }
+});
+
+Deno.test("every predeploy is a contract: nonce 1, zero balance, real code", () => {
+  for (const [addr, a] of Object.entries(MAINNET_SYSTEM_CONTRACTS)) {
+    assertEquals(a.nonce, "0x1", `${addr} must look like a deployed contract`);
+    assertEquals(a.balance, "0x0", `${addr} must hold no ether`);
+    const code = a.code ?? "";
+    assert(code.startsWith("0x") && code.length > 10, `${addr} has no code`);
+    assertEquals(code.length % 2, 0, `${addr} code is not whole bytes`);
+  }
+});
+
+Deno.test("predeploy() shapes a recipe's own contract the same way", () => {
+  assertEquals(predeploy("0xfeed"), { nonce: "0x1", balance: "0x0", code: "0xfeed" });
+});

notes/images.md

@@ -0,0 +1,74 @@
+# Images
+
+A container can name a published image, or pin a source repo and let decker
+build it.
+
+A pinned build is described in fields — repo, ref, Dockerfile, target, build
+args, secrets — and the build command is derived from them. `cmd` remains as an
+escape hatch for a build no combination of fields expresses.
+
+```ts
+function image(def: ContainerDef): ImageBuildSpec {
+  return {
+    repo: "https://github.com/flashbots/mev-boost-relay.git",
+    ref: def.config?.ref as string ?? "main",
+    dockerfile: "Dockerfile",
+  };
+}
+```
+
+## images.json
+
+`decker build` writes every spec to `manifests/<recipe>/images.json`, beside
+the rendered manifests and keyed by the exact tag those manifests reference:
+
+```json
+{
+  "decker-mev-boost-relay:main": {
+    "repo": "https://github.com/flashbots/mev-boost-relay.git",
+    "ref": "main",
+    "name": "mev-boost-relay",
+    "dockerfile": "Dockerfile",
+    "cmd": "$ENGINE build -f Dockerfile -t $IMAGE ."
+  }
+}
+```
+
+Run the images locally and decker builds whatever is missing. Set
+`DECKER_IMAGE_MODE=pull` and it builds nothing at all, so CI or an in-cluster
+build system can supply them — reading this one file instead of keeping a
+second copy of your pins, which is the copy that drifts.
+`DECKER_IMAGE_REGISTRY` prefixes the tags so the rendered manifests point at
+the registry those images were pushed to.
+
+## Build assets
+
+Some builds need files the source repo does not have: a Dockerfile that patches
+it, the patches it applies. Name them in `assetDockerfile` and `assets`. The
+repo then arrives as the named build context `src` instead of being the build
+context, which is how a Dockerfile outside the repo reaches its sources.
+
+They live in decker's `_assets/`, or in your own repo — set `assetsDir` and a
+recipe that lives outside decker ships its own build inputs:
+
+```ts
+const ASSETS_DIR = new URL("../_assets/", import.meta.url).href;
+
+export const IMAGE: ImageBuildSpec = {
+  repo: "https://github.com/example/thing.git",
+  ref: "…",
+  assetsDir: ASSETS_DIR,
+  assetDockerfile: "thing.Dockerfile",
+  assets: ["0001-some.patch"],
+  variant: assetsVariant(["thing.Dockerfile", "0001-some.patch"], ASSETS_DIR),
+  secrets: ["gh_token"],
+};
+```
+
+Asset bytes hash into the image tag through `variant`, so a patched image can
+never collide with a pristine build of the same ref, and moving the files
+between repos does not change the tag. `decker build` copies them to
+`manifests/<recipe>/images-assets/` for whoever does the building.
+
+Secrets are passed as `--secret id=<id>,env=<ID>` and read from the environment
+at build time; nothing is baked into the image.