From 949b8dce7e719527bb176e74693c3cc1af1b4fc6 Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Tue, 20 Oct 2026 12:10:33 +0000 Subject: [PATCH 1/6] fix(ci): stop the main-push twin run that cancels ci on every merge ci.yml ran on push to main and was also called by images.yml on the same push. Both runs shared the group ci-refs/heads/main with cancel-in-progress, so one always cancelled the other: 8 of the last 10 main-push ci runs ended cancelled while images succeeded. Drop the push trigger: images.yml already runs ci before publishing, so each merge now gets exactly one ci run, the one that gates publication. Key the group on github.workflow too (in a called run it is the caller's name, images), and only cancel in progress for pull_request, so a PR run and the images run never share a group and the images-called run is never cancelled. images.yml keeps serializing main pushes with its own images-main group. Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/ci.yml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4aaddb6..4aaed86 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,18 +1,20 @@ name: ci +# main is covered by images.yml, which calls this workflow before publishing. A +# push trigger here would start a twin run on the same ref (see concurrency). on: - push: - branches: [main] pull_request: workflow_call: permissions: contents: read -# Cancel superseded runs on the same ref instead of queueing them. +# Cancel superseded PR runs on the same ref. In a called run github.workflow is +# the caller's name, so the images run gets its own group and is never cancelled; +# images.yml serializes pushes to main itself. concurrency: - group: ci-${{ github.ref }} - cancel-in-progress: true + group: ci-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} # Bound CPU kernel dispatch to AVX2, the dev-host golden baseline. env: From c7789f4b9b4e0ffd4727d9e8d58ccea3aa7221b4 Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Tue, 20 Oct 2026 12:12:31 +0000 Subject: [PATCH 2/6] fix(ci): run heavy full-suite on a 16 vCPU Blacksmith runner full-suite failed on every nightly run. The last three were killed with "The runner has received a shutdown signal" after exactly 504 tests, each time while running tests/auditor/test_od_bisect.py. Run locally, two tests in that file peak at 18.4 and 18.9 GB RSS, above the 16 GB of the hosted ubuntu-24.04 runner, so the runner is evicted. blacksmith-16vcpu-ubuntu-2404 has 62 GB. native and canary pass on hosted runners and stay there. timeout-minutes is set from a measured dispatch on this branch. Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/heavy.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/heavy.yml b/.github/workflows/heavy.yml index 6073c70..dfd2a0d 100644 --- a/.github/workflows/heavy.yml +++ b/.github/workflows/heavy.yml @@ -125,8 +125,10 @@ jobs: - run: docker build -f docker/Dockerfile.server -t hypertrain:local . - run: ENGINE=docker bash scripts/canary.sh + # The serial suite peaks near 19 GB RSS (tests/auditor/test_od_bisect.py), more + # than a 16 GB hosted runner holds; 16 vCPU / 62 GB Blacksmith runner. full-suite: - runs-on: ubuntu-24.04 + runs-on: blacksmith-16vcpu-ubuntu-2404 timeout-minutes: 150 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 From d90a2d749227cb9f11ccc119c7b110ff786fa399 Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Tue, 20 Oct 2026 12:12:31 +0000 Subject: [PATCH 3/6] perf(ci): run lint, test shards and test-slow on 8 vCPU Blacksmith runners PR CI took ~45 min of wall clock on 2 vCPU hosted runners (run 39352445497): the slowest shard ran 21 min and test-slow (e2e) waited in the queue for 28 min before it started. blacksmith-8vcpu-ubuntu-2404 gives 8 vCPU / 30 GB and starts in seconds (canary run 39420280084). The shard count and file split stay the same. The astral-sh/setup-uv cache stays: it uses the actions cache, which Blacksmith serves. Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4aaed86..919c1e7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,7 +25,7 @@ env: jobs: # Format, lint and types only. Fast, high signal, no test execution. lint: - runs-on: ubuntu-24.04 + runs-on: blacksmith-8vcpu-ubuntu-2404 timeout-minutes: 15 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -51,7 +51,7 @@ jobs: # is a fraction of the serial run. The native-systemd-only cases are excluded here # and run in the nightly heavy workflow (heavy.yml). test: - runs-on: ubuntu-24.04 + runs-on: blacksmith-8vcpu-ubuntu-2404 timeout-minutes: 25 strategy: fail-fast: false @@ -110,7 +110,7 @@ jobs: # The slow subprocess suites, in parallel with test. test-slow: - runs-on: ubuntu-24.04 + runs-on: blacksmith-8vcpu-ubuntu-2404 timeout-minutes: 25 strategy: fail-fast: false From b9097b4de816b07f15350af1200389ba6035f5d1 Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Tue, 20 Oct 2026 12:12:31 +0000 Subject: [PATCH 4/6] perf(ci): build images on a 16 vCPU Blacksmith runner Only runs-on changes. docker/setup-buildx-action and docker/build-push-action stay: the attested subject is the build-push-action digest output and the provenance comes from actions/attest-build-provenance, so keeping the same actions keeps the exact-digest attestation path the Cortex supervisor verifies. A branch cannot publish, so a Blacksmith-specific builder could not be proven before merge. Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/images.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/images.yml b/.github/workflows/images.yml index 4cff028..937e40e 100644 --- a/.github/workflows/images.yml +++ b/.github/workflows/images.yml @@ -23,7 +23,7 @@ jobs: publish: if: github.ref == 'refs/heads/main' needs: ci - runs-on: ubuntu-24.04 + runs-on: blacksmith-16vcpu-ubuntu-2404 timeout-minutes: 120 strategy: fail-fast: false From ecd1d7f70bc11a8c4540ee3809d088303f79a44b Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Tue, 20 Oct 2026 13:10:41 +0000 Subject: [PATCH 5/6] fix(ci): set heavy full-suite timeout from a measured Blacksmith run Dispatch 39420692371 on this branch ran full-suite to completion on blacksmith-16vcpu-ubuntu-2404 in 55 min 27 s (pytest 3293 s). 90 min leaves about 60% margin; the old 150 min was sized for 2 vCPU. Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/heavy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/heavy.yml b/.github/workflows/heavy.yml index dfd2a0d..b0c8290 100644 --- a/.github/workflows/heavy.yml +++ b/.github/workflows/heavy.yml @@ -129,7 +129,7 @@ jobs: # than a 16 GB hosted runner holds; 16 vCPU / 62 GB Blacksmith runner. full-suite: runs-on: blacksmith-16vcpu-ubuntu-2404 - timeout-minutes: 150 + timeout-minutes: 90 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: From 14068ddd64603bee7fe2a37711d7c4dba680f460 Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Tue, 20 Oct 2026 14:36:34 +0000 Subject: [PATCH 6/6] fix(ci): keep the provenance-producing publish job on a GitHub-hosted runner Blacksmith runners are recorded in the build provenance as runnerEnvironment=self-hosted. Cortex verifies its own image attestations with `gh attestation verify --deny-self-hosted-runners`, and the existing hypertrain attestations are runnerEnvironment=github-hosted. A digest built on a Blacksmith runner could be rejected by a verifier with that flag, so the publish job (build, push and attest) returns to ubuntu-24.04. The ci called by images keeps the Blacksmith runners: tests do not produce provenance. Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/images.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/images.yml b/.github/workflows/images.yml index 937e40e..4a9aed0 100644 --- a/.github/workflows/images.yml +++ b/.github/workflows/images.yml @@ -23,7 +23,12 @@ jobs: publish: if: github.ref == 'refs/heads/main' needs: ci - runs-on: blacksmith-16vcpu-ubuntu-2404 + # GitHub-hosted on purpose: this job creates the build provenance, and a Blacksmith runner is + # recorded as runnerEnvironment=self-hosted, which `gh attestation verify + # --deny-self-hosted-runners` rejects (Cortex verifies its own images that way). The images + # published so far are runnerEnvironment=github-hosted; keep that property for a digest that + # may be pinned in a registry. The `ci` job above (its first line) stays on Blacksmith. + runs-on: ubuntu-24.04 timeout-minutes: 120 strategy: fail-fast: false