From 6d74c83eaff10f3bc014c06510bff7dcb51c7f84 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Thu, 22 Oct 2026 01:13:30 +0400 Subject: [PATCH] Stop trusting the compromised eMudhra CA certificates The private keys of eight eMudhra (emSign) intermediate CA certificates were disclosed on 2026-10-20, so anyone can issue a certificate that software trusting emSign ECC Root CA - G3 accepts (https://emudhra.we-have-a-problem.com). - uv.lock: certifi 2026.5.20 -> 2026.10.19, the first certifi release without that root. Nothing else in the lock moves. - worker_identity: re-pin APPROVED_UV_LOCK_SHA256 for the new lock. - Modal stage image: uv 0.11.7 -> 0.13.4 (uv verifies HTTPS against a CA list compiled into it), a certifi 2026.10.19 floor beside the runner's huggingface_hub, and ca-certificates named in apt_install so the image upgrades Debian's system CA store. - Runbook and changelog fragment. Co-Authored-By: Claude Opus 5.5 --- changelog.d/emudhra-ca-trust.fixed.md | 1 + docs/us-modal-stage-runbook.md | 6 +++++- .../src/microcosm/build/us_runtime/worker_identity.py | 2 +- tools/modal_us_stage.py | 7 ++++++- tools/modal_us_stage_plan.py | 11 ++++++++++- uv.lock | 6 +++--- 6 files changed, 26 insertions(+), 7 deletions(-) create mode 100644 changelog.d/emudhra-ca-trust.fixed.md diff --git a/changelog.d/emudhra-ca-trust.fixed.md b/changelog.d/emudhra-ca-trust.fixed.md new file mode 100644 index 000000000..9e118b9e3 --- /dev/null +++ b/changelog.d/emudhra-ca-trust.fixed.md @@ -0,0 +1 @@ +Stop trusting the compromised eMudhra (emSign) CA certificates, whose intermediate private keys were disclosed on 2026-10-20 (https://emudhra.we-have-a-problem.com). The lock moves certifi from 2026.5.20 to 2026.10.19, the first certifi release without emSign ECC Root CA - G3; nothing else in the lock moves, and `APPROVED_UV_LOCK_SHA256` is re-pinned for the new lock. The Modal stage image (`tools/modal_us_stage.py`) installs uv 0.13.4 instead of 0.11.7, because uv verifies HTTPS against a CA list compiled into it; requires certifi 2026.10.19 or later beside the runner's `huggingface_hub`; and upgrades `ca-certificates` to Debian's current release, which git and the stdlib's ssl module read. diff --git a/docs/us-modal-stage-runbook.md b/docs/us-modal-stage-runbook.md index 1d33ed5d5..dc2a309ad 100644 --- a/docs/us-modal-stage-runbook.md +++ b/docs/us-modal-stage-runbook.md @@ -62,7 +62,11 @@ release directory onto the runs volume. Publication stays the human step in three times the list price (see "Preemption and restarts"). 2. **The image is the commit.** It starts from `debian_slim` with Python 3.14, the minor version the local US builds record (`runtime.python` - 3.14.4 in the #974 build manifest; Modal served 3.14.2). It adds git and uv 0.11.7, makes a shallow + 3.14.4 in the #974 build manifest; Modal served 3.14.2). It adds git, + upgrades `ca-certificates` to Debian's current release, and installs + uv 0.13.4 and, for the runner, `huggingface_hub` with certifi 2026.10.19 + or later: the first releases whose CA lists drop the compromised eMudhra + roots (https://emudhra.we-have-a-problem.com). It then makes a shallow clone of the plan's commit from GitHub, checks out the plan's branch name, and asserts `HEAD` equals the commit. It then runs `uv sync --all-packages --extra us --frozen` against that tree's own diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/worker_identity.py b/packages/microcosm-build/src/microcosm/build/us_runtime/worker_identity.py index f4f9606e3..e10bc26f2 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/worker_identity.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/worker_identity.py @@ -28,7 +28,7 @@ PRIMARY_QRF_WORKER_MODULE = "microcosm.build.us_runtime.puf_qrf_worker" PRIMARY_QRF_INTERPRETER_PLACEHOLDER = "{python_interpreter}" APPROVED_UV_LOCK_SHA256 = ( - "26d2e9e4aab5efe60357c7b06affbf75c4129566dc3754cfa155501294feea7b" + "8aa3f9fe1b711bd83d9d04545715ce71bcc7a5a2e7e29e556e52b00dd96c1e5c" ) LEGACY_CAMPAIGN_UV_LOCK_SHA256 = ( "27f47e385cfa35e2644a37410d1804b361ad9aee123577551c8421547bda65ee" diff --git a/tools/modal_us_stage.py b/tools/modal_us_stage.py index cab09994c..90d9ea3f7 100644 --- a/tools/modal_us_stage.py +++ b/tools/modal_us_stage.py @@ -85,13 +85,18 @@ def _local_plan() -> tuple[dict, plan_lib.Plan] | None: def _image() -> modal.Image: image = ( modal.Image.debian_slim(python_version=plan_lib.IMAGE_PYTHON_VERSION) - .apt_install("git", "build-essential") + # Naming ca-certificates upgrades the base image's copy to the + # distribution's current release, which drops the compromised + # eMudhra roots; git's GitHub fetch and the stdlib's ssl module + # verify against that system CA store. + .apt_install("git", "build-essential", "ca-certificates") # The runner itself (not the build) runs on the image's interpreter; # it needs the Hub client to stage hf:// inputs. The build runs in # IMAGE_VENV, synced from the pinned tree's own lock. .pip_install( f"uv=={plan_lib.IMAGE_UV_VERSION}", f"huggingface_hub=={plan_lib.RUNNER_HF_HUB_VERSION}", + f"certifi>={plan_lib.RUNNER_CERTIFI_MINIMUM_VERSION}", ) .env( { diff --git a/tools/modal_us_stage_plan.py b/tools/modal_us_stage_plan.py index 0cbace7ff..fcaf74136 100644 --- a/tools/modal_us_stage_plan.py +++ b/tools/modal_us_stage_plan.py @@ -70,8 +70,17 @@ # too. compare-lineage checks the outputs the local run left (its first two # outer stages), not the stages after them. IMAGE_PYTHON_VERSION = "3.14" -IMAGE_UV_VERSION = "0.11.7" +# uv verifies HTTPS against a CA list compiled into it. 0.13.4 is the first +# release whose list drops the compromised eMudhra (emSign) roots; earlier +# releases trust certificates anyone can issue with the disclosed keys +# (https://emudhra.we-have-a-problem.com). +IMAGE_UV_VERSION = "0.13.4" RUNNER_HF_HUB_VERSION = "1.18.0" +# The runner's Hub client verifies HTTPS against certifi's CA list, and +# 2026.10.19 is the first certifi release without the eMudhra roots. Modal +# reuses a cached layer while its definition is unchanged, so the floor is +# stated rather than left to whatever certifi PyPI served at the first build. +RUNNER_CERTIFI_MINIMUM_VERSION = "2026.10.19" WORK_ROOT = "/work" # Modal list prices for standard (non-sandbox) compute, read from diff --git a/uv.lock b/uv.lock index 0a6a2b339..26a66cfb5 100644 --- a/uv.lock +++ b/uv.lock @@ -158,11 +158,11 @@ wheels = [ [[package]] name = "certifi" -version = "2026.5.20" +version = "2026.10.19" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/f3/ce/ee2ecad540810a79593028e88299baeae54d346cc7a0d94b6199988b89b1/certifi-2026.5.20.tar.gz", hash = "sha256:69dea482ab64caa7b9f6aba1c6bf48bb6a5448d1c0f1b17ab42ad8c763a5344d", size = 135422, upload-time = "2026-05-20T11:46:50.073Z" } +sdist = { url = "https://files.pythonhosted.org/packages/a6/bd/1436c50cc46699de0230fbaacc1d1de7bfec413176099949e9d5112e88f2/certifi-2026.10.19.tar.gz", hash = "sha256:035c1e85572b141bf1c3a3eff0e33a8eb74917522f2559075ff8d2fea780a1ae", size = 137598, upload-time = "2026-10-20T02:24:16.604Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/59/8c/57e832b7af6d7c5abe66eb3fbe3a3a32f4d11ea23a1aa7131371035be991/certifi-2026.5.20-py3-none-any.whl", hash = "sha256:3c52e209ba0a4ad7aebe60436a4ab349c39e1e602e8c134221e546902ad25897", size = 134134, upload-time = "2026-05-20T11:46:48.578Z" }, + { url = "https://files.pythonhosted.org/packages/c4/51/29046e48f693c54e5b7412216d447a1d8362c9653148d98c02cdf65fa733/certifi-2026.10.19-py3-none-any.whl", hash = "sha256:42d35340cc5b9ba78e818ddbff0c5b1c2b779de448a340fd1b989f5fa8670e1b", size = 136464, upload-time = "2026-10-20T02:24:15.072Z" }, ] [[package]]