diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 42af7331..d90c536f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,20 +1,24 @@ -# Per-push smoke: cold Zend seed + the whole AOT suite, on both Linux arches. +# Per-push: build the compiler and run the whole AOT suite, on both Linux +# arches and on macOS. # # The steps are NOT written here. They live in tools/docker/gate.sh, which is # also what `bash tools/docker/run_tests.sh` runs locally — one definition, so a # CI green and a local green mean the same thing. This file only supplies a -# machine, the image, and the environment. +# machine, the image, the cache and the environment. +# +# WARM, NOT COLD. Every job restores the compiler it built last time and +# self-hosts from it (`bin/build`), the way a developer does; the Zend cold seed +# is the fallback, not the loop. gate.sh validates the cache against +# arch + clang + php before trusting it, and a bootstrap gap (a MemoryAbi +# VERSION bump, new syntax) fails `bin/build` and falls through to the seed on +# its own — so a stale cache costs one slow run, never a wrong answer. name: ci -# PARKED: manual only. The gate is built and lint-clean, but CI is not the priority -# right now — pushing it with `push`/`pull_request` triggers live would spend runner -# minutes on every commit for a signal nobody is reading yet. Re-arm by restoring the -# two triggers below; nothing else in this file changes. on: + push: + branches: [main] + pull_request: workflow_dispatch: - # push: - # branches: ['**'] - # pull_request: concurrency: group: ci-${{ github.ref }} @@ -51,22 +55,59 @@ jobs: cache-from: type=gha,scope=toolchain-${{ matrix.arch }} cache-to: type=gha,mode=max,scope=toolchain-${{ matrix.arch }} - # /logs is a mount the unprivileged container user must be able to write; - # /build stays inside the container (the gate copies the tree into it, and - # that copy has no business crossing a bind mount). - - name: Seed + suite + # ~15 MB: bin/manticore + lib/*.o + lib/prelude. The key is unique per run + # (a cache entry is immutable), so the restore-keys prefix is what actually + # hits — the newest entry for this arch and this image definition. + - name: Restore the compiler cache + uses: actions/cache/restore@v4 + with: + path: ci-cache + key: mc-compiler-${{ matrix.arch }}-${{ hashFiles('Dockerfile') }}-${{ github.run_id }} + restore-keys: | + mc-compiler-${{ matrix.arch }}-${{ hashFiles('Dockerfile') }}- + + # /logs and /compiler-cache are mounts the unprivileged container user + # (uid 1000) must be able to write; /build stays inside the container (the + # gate copies the tree into it, and that copy has no business crossing a + # bind mount). + - name: Build + suite run: | - mkdir -p ci-logs && chmod 777 ci-logs + # -R, not just the top directory: actions/cache restores the tree as the + # RUNNER, and the container is uid 1000 — unlinking an entry needs write + # permission on the directory that holds it, so a restored lib/ is + # unremovable from inside without this and the cache never updates. + mkdir -p ci-logs ci-cache && chmod -R 777 ci-logs ci-cache docker run --rm \ -v "$PWD":/repo:ro \ -v "$PWD/ci-logs":/logs \ + -v "$PWD/ci-cache":/compiler-cache \ -e MC_GATE=0 \ -e MC_JOBS=0 \ -e MC_LOGDIR=/logs \ + -e MC_COMPILER_CACHE=/compiler-cache \ -e MC_COMMIT="${GITHUB_SHA::12} ${GITHUB_REF_NAME}" \ manticore-toolchain:${{ matrix.arch }} \ /bin/bash /repo/tools/docker/gate.sh + # A red suite on top of a good build still leaves a compiler worth keeping — + # save before the job's result is decided, not after. + - name: Save the compiler cache + if: always() && hashFiles('ci-cache/bin/manticore') != '' + uses: actions/cache/save@v4 + with: + path: ci-cache + key: mc-compiler-${{ matrix.arch }}-${{ hashFiles('Dockerfile') }}-${{ github.run_id }} + + - name: Summary + if: always() + run: | + { + echo "## linux-${{ matrix.arch }} — \`${GITHUB_SHA::12}\`" + echo '```' + tail -15 ci-logs/suite.log 2>/dev/null || echo 'no suite log' + echo '```' + } >> "$GITHUB_STEP_SUMMARY" + - name: Logs if: always() uses: actions/upload-artifact@v4 @@ -74,3 +115,67 @@ jobs: name: logs-${{ matrix.arch }} path: ci-logs/ if-no-files-found: warn + + macos: + name: macos-arm64 + runs-on: macos-15 + timeout-minutes: 120 + steps: + - uses: actions/checkout@v4 + + # Homebrew's `php` is the SEED interpreter and the difftest oracle. With a + # warm cache neither is touched — it is installed for the run where the + # cache misses, which is the only one that needs Zend. + - name: Toolchain + run: | + brew install php pcre2 openssl@3 sqlite + php -v + clang --version | head -1 + + - name: Restore the compiler cache + uses: actions/cache/restore@v4 + with: + path: ci-cache + key: mc-compiler-macos-arm64-${{ github.run_id }} + restore-keys: | + mc-compiler-macos-arm64- + + # The same gate.sh, run bare: it copies the checkout to a scratch tree + # (RUNNER_TEMP, never the checkout itself — the build writes bin/ and lib/ + # into whatever it is pointed at) and does the same steps the container does. + - name: Build + suite + run: | + mkdir -p ci-logs ci-cache + MC_GATE=0 \ + MC_JOBS=0 \ + MC_REPO="$PWD" \ + MC_WORK="$RUNNER_TEMP/build" \ + MC_LOGDIR="$PWD/ci-logs" \ + MC_COMPILER_CACHE="$PWD/ci-cache" \ + MC_COMMIT="${GITHUB_SHA::12} ${GITHUB_REF_NAME}" \ + bash tools/docker/gate.sh + + - name: Save the compiler cache + if: always() && hashFiles('ci-cache/bin/manticore') != '' + uses: actions/cache/save@v4 + with: + path: ci-cache + key: mc-compiler-macos-arm64-${{ github.run_id }} + + - name: Summary + if: always() + run: | + { + echo "## macos-arm64 — \`${GITHUB_SHA::12}\`" + echo '```' + tail -15 ci-logs/suite.log 2>/dev/null || echo 'no suite log' + echo '```' + } >> "$GITHUB_STEP_SUMMARY" + + - name: Logs + if: always() + uses: actions/upload-artifact@v4 + with: + name: logs-macos-arm64 + path: ci-logs/ + if-no-files-found: warn diff --git a/.github/workflows/gate.yml b/.github/workflows/gate.yml new file mode 100644 index 00000000..84a169d1 --- /dev/null +++ b/.github/workflows/gate.yml @@ -0,0 +1,158 @@ +# The heavy answer: the AOT suite PLUS difftest (byte parity against the `php` +# interpreter), on both Linux arches. Weekly, and on demand. +# +# Why not nightly, and why no fixpoint by default: ci.yml already answers "is +# the suite green" on every push, and the self-host fixpoint answers a question +# only a bootstrap or a MemoryAbi change can re-open — it is hours, and the tree +# has been at a fixpoint for long enough that running it nightly buys nothing. +# Ask for it explicitly (the `fixpoint` input) after a bootstrap, an ABI VERSION +# bump, or a codegen change big enough to doubt gen2 == gen3. +# +# The steps live in tools/docker/gate.sh — never inline them here. +name: gate + +on: + schedule: + - cron: '0 2 * * 0' + workflow_dispatch: + inputs: + parity: + description: 'run the glibc parity gate (suite + difftest) — off to test musl alone' + type: boolean + default: true + fixpoint: + description: 'also run tools/selfhost_fixpoint.sh (hours)' + type: boolean + default: false + alpine: + description: 'also run the musl (Alpine) build — prepared, not yet green' + type: boolean + default: false + +concurrency: + group: gate + cancel-in-progress: false + +jobs: + linux: + name: gate-${{ matrix.arch }} + # A schedule carries no inputs, so the weekly parity run must not depend on one. + if: github.event_name == 'schedule' || inputs.parity + runs-on: ${{ matrix.runner }} + timeout-minutes: 360 + strategy: + fail-fast: false + matrix: + include: + - arch: arm64 + runner: ubuntu-24.04-arm + - arch: amd64 + runner: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: docker/setup-buildx-action@v3 + + - name: Build the toolchain image + uses: docker/build-push-action@v6 + with: + context: . + target: toolchain + tags: manticore-toolchain:${{ matrix.arch }} + load: true + cache-from: type=gha,scope=toolchain-${{ matrix.arch }} + cache-to: type=gha,mode=max,scope=toolchain-${{ matrix.arch }} + + # Read-only: this job is a verdict on the tree, not a producer of + # compilers. ci.yml owns the cache, and a gate run writing it back would + # hand every later run a compiler built by a path nobody iterates on. + - name: Restore the compiler cache + uses: actions/cache/restore@v4 + with: + path: ci-cache + key: mc-compiler-${{ matrix.arch }}-${{ hashFiles('Dockerfile') }}- + + - name: Suite + difftest + run: | + # -R, not just the top directory: actions/cache restores the tree as the + # RUNNER, and the container is uid 1000 — unlinking an entry needs write + # permission on the directory that holds it, so a restored lib/ is + # unremovable from inside without this and the cache never updates. + mkdir -p ci-logs ci-cache && chmod -R 777 ci-logs ci-cache + docker run --rm \ + -v "$PWD":/repo:ro \ + -v "$PWD/ci-logs":/logs \ + -v "$PWD/ci-cache":/compiler-cache \ + -e MC_DIFFTEST=1 \ + -e MC_FIXPOINT=${{ inputs.fixpoint && '1' || '0' }} \ + -e MC_JOBS=0 \ + -e MC_STABILITY_N=2 \ + -e MC_LOGDIR=/logs \ + -e MC_COMPILER_CACHE=/compiler-cache \ + -e MC_COMMIT="${GITHUB_SHA::12} ${GITHUB_REF_NAME}" \ + manticore-toolchain:${{ matrix.arch }} \ + /bin/bash /repo/tools/docker/gate.sh + + - name: Summary + if: always() + run: | + { + echo "## linux-${{ matrix.arch }} — \`${GITHUB_SHA::12}\`" + echo '```' + tail -15 ci-logs/suite.log 2>/dev/null || echo 'no suite log' + tail -8 ci-logs/difftest.log 2>/dev/null || true + tail -12 ci-logs/fixpoint.log 2>/dev/null || true + echo '```' + } >> "$GITHUB_STEP_SUMMARY" + + - name: Logs + if: always() + uses: actions/upload-artifact@v4 + with: + name: gate-logs-${{ matrix.arch }} + path: ci-logs/ + if-no-files-found: warn + + # musl, opt-in. It is here so that "does Alpine still build" is a button rather + # than an afternoon, and `continue-on-error` because docs/install.md has claimed + # musl works for longer than anything has checked it — the first runs are + # evidence-gathering, and they must not turn the parity gate red while they are. + alpine: + name: alpine-arm64 (musl, experimental) + if: inputs.alpine + runs-on: ubuntu-24.04-arm + continue-on-error: true + timeout-minutes: 120 + steps: + - uses: actions/checkout@v4 + - uses: docker/setup-buildx-action@v3 + + - name: Build the musl toolchain image + uses: docker/build-push-action@v6 + with: + context: . + file: Dockerfile.alpine + target: toolchain + tags: manticore-toolchain:alpine + load: true + cache-from: type=gha,scope=toolchain-alpine + cache-to: type=gha,mode=max,scope=toolchain-alpine + + - name: Seed + suite + run: | + mkdir -p ci-logs && chmod 777 ci-logs + docker run --rm \ + -v "$PWD":/repo:ro \ + -v "$PWD/ci-logs":/logs \ + -e MC_JOBS=0 \ + -e MC_LOGDIR=/logs \ + -e MC_COMMIT="${GITHUB_SHA::12} ${GITHUB_REF_NAME}" \ + manticore-toolchain:alpine \ + /bin/bash /repo/tools/docker/gate.sh + + - name: Logs + if: always() + uses: actions/upload-artifact@v4 + with: + name: gate-logs-alpine + path: ci-logs/ + if-no-files-found: warn diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml deleted file mode 100644 index 3d2526cb..00000000 --- a/.github/workflows/nightly.yml +++ /dev/null @@ -1,114 +0,0 @@ -# The HEAVY gate, nightly: cold seed + suite + difftest (php parity) + -# selfhost_fixpoint (fixpoint, MIR golden, rebuild stability), on both Linux -# arches, plus a macOS suite run. -# -# Why nightly and not per push: the Linux gate is hours, and its value is -# answering "is main green, and at WHICH commit" without anyone remembering to -# ask. Every job writes that commit into the run summary for exactly that reason. -name: nightly - -# PARKED: manual only — see the note in ci.yml. Re-arm by uncommenting the schedule. -on: - workflow_dispatch: - # schedule: - # - cron: '0 2 * * *' - -concurrency: - group: nightly - cancel-in-progress: false - -jobs: - linux-gate: - name: gate-${{ matrix.arch }} - runs-on: ${{ matrix.runner }} - timeout-minutes: 360 - strategy: - fail-fast: false - matrix: - include: - - arch: arm64 - runner: ubuntu-24.04-arm - - arch: amd64 - runner: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: docker/setup-buildx-action@v3 - - - name: Build the toolchain image - uses: docker/build-push-action@v6 - with: - context: . - target: toolchain - tags: manticore-toolchain:${{ matrix.arch }} - load: true - cache-from: type=gha,scope=toolchain-${{ matrix.arch }} - cache-to: type=gha,mode=max,scope=toolchain-${{ matrix.arch }} - - - name: Full gate - run: | - mkdir -p ci-logs && chmod 777 ci-logs - docker run --rm \ - -v "$PWD":/repo:ro \ - -v "$PWD/ci-logs":/logs \ - -e MC_GATE=1 \ - -e MC_JOBS=0 \ - -e MC_STABILITY_N=2 \ - -e MC_LOGDIR=/logs \ - -e MC_COMMIT="${GITHUB_SHA::12} ${GITHUB_REF_NAME}" \ - manticore-toolchain:${{ matrix.arch }} \ - /bin/bash /repo/tools/docker/gate.sh - - - name: Summary - if: always() - run: | - { - echo "## linux-${{ matrix.arch }} — \`${GITHUB_SHA::12}\`" - echo '```' - tail -20 ci-logs/suite.log 2>/dev/null || echo 'no suite log' - tail -8 ci-logs/difftest.log 2>/dev/null || true - echo '```' - } >> "$GITHUB_STEP_SUMMARY" - - - name: Logs - if: always() - uses: actions/upload-artifact@v4 - with: - name: nightly-logs-${{ matrix.arch }} - path: ci-logs/ - if-no-files-found: warn - - macos: - name: macos-arm64 - runs-on: macos-14 - timeout-minutes: 180 - steps: - - uses: actions/checkout@v4 - - # Homebrew's `php` is the oracle AND the seed interpreter. difftest is - # only honest against 8.5 — the target language version — so it is guarded - # rather than run against whatever the formula currently ships. - - name: Toolchain - run: | - brew update - brew install php pcre2 openssl@3 sqlite - php -v - clang --version | head -1 - - - name: Cold seed - run: bin/compile - - - name: Suite - run: bash tests/aot/run.sh -j 0 - - - name: Difftest (php 8.5 only) - run: | - v=$(php -r 'echo PHP_MAJOR_VERSION . "." . PHP_MINOR_VERSION;') - if [ "$v" = "8.5" ]; then - bash tools/difftest.sh - else - echo "php $v is not 8.5 — skipping difftest, it would grade against the wrong oracle" - fi - - - name: Summary - if: always() - run: echo "## macos-arm64 — \`${GITHUB_SHA::12}\`" >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 00000000..c7ba8856 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,294 @@ +# What a release IS: a container image on GitHub's registry (ghcr.io), and +# per-platform tarballs on the GitHub release. Nothing else is published, and +# nothing is published from a cache — every artifact here is cold-seeded from the +# tagged source, so a release owes nothing to a compiler already lying around. +# +# The image is the primary artifact and the tarball is the secondary one, for a +# reason worth writing down: the compiler shells out to `clang` to assemble its +# IR and to `cc` to link, and it links against the host pcre2/openssl/sqlite/ +# curl. A tarball therefore cannot be self-contained — it needs a toolchain on +# the far side — while the image carries exactly the one it was built with. +# +# git tag v0.11.0 && git push origin v0.11.0 +# gh workflow run release.yml # dry run: builds and smokes, publishes nothing +name: release + +on: + push: + tags: ['v*'] + workflow_dispatch: + inputs: + publish: + description: 'push the image and create the GitHub release' + type: boolean + default: false + +concurrency: + group: release-${{ github.ref }} + cancel-in-progress: false + +env: + # ghcr, not docker.io: the manticorephp namespace already exists here, a + # public package costs nothing, and the push authenticates with the workflow's + # own token — no registry secret to hand out, rotate or leak. + IMAGE: ghcr.io/${{ github.repository }} + +jobs: + linux: + name: linux-${{ matrix.arch }} + runs-on: ${{ matrix.runner }} + permissions: + contents: read + packages: write + timeout-minutes: 120 + strategy: + fail-fast: false + matrix: + include: + - arch: arm64 + runner: ubuntu-24.04-arm + - arch: amd64 + runner: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: docker/setup-buildx-action@v3 + + # A tag names the version; a dispatch has no tag and must not be able to + # masquerade as one, so it is stamped with the commit instead. + - name: Version + id: v + run: | + if [ "${GITHUB_REF_TYPE}" = "tag" ]; then + echo "version=${GITHUB_REF_NAME#v}" >> "$GITHUB_OUTPUT" + echo "publish=${{ github.event_name == 'push' }}" >> "$GITHUB_OUTPUT" + else + echo "version=0.0.0-dev.${GITHUB_SHA::12}" >> "$GITHUB_OUTPUT" + echo "publish=${{ inputs.publish }}" >> "$GITHUB_OUTPUT" + fi + + # DEBIAN_TAG=12, while ci.yml and the development image track 13. glibc is + # backwards compatible and not forwards: a binary built against 2.41 (trixie) + # refuses to start on 2.36 (bookworm), so a release built on the newest base + # would run only on the newest distributions. The two axes are separate — + # develop on the fresh toolchain, ship against the older floor — and this is + # the whole of the difference, one build arg. + - name: Build the runtime image + uses: docker/build-push-action@v6 + with: + context: . + target: runtime + build-args: | + DEBIAN_TAG=12 + tags: manticore-runtime:${{ matrix.arch }} + load: true + cache-from: type=gha,scope=release-${{ matrix.arch }} + cache-to: type=gha,mode=max,scope=release-${{ matrix.arch }} + + # The version lives in one `puts()` in Main.php, so a tag can disagree with + # the binary it ships. Catch that here rather than in a user's bug report. + - name: The binary agrees with the tag + if: github.ref_type == 'tag' + run: | + got="$(docker run --rm manticore-runtime:${{ matrix.arch }} manticore version)" + want="manticore ${{ steps.v.outputs.version }}" + echo "binary: $got" + [ "$got" = "$want" ] || { echo "tag says '$want' — see Main.php cmd_version" >&2; exit 1; } + + # Compile something THROUGH the image, the way a user would: bind mount, + # bare `manticore` off $PATH, and a stdlib function that only links when + # the prelude and manticore_stdlib.o were both found. + - name: Smoke the image + run: | + mkdir -p smoke && printf ' smoke/app.php + docker run --rm -v "$PWD/smoke":/work -u "$(id -u):$(id -g)" \ + manticore-runtime:${{ matrix.arch }} manticore compile app.php -o app + got="$(./smoke/app)" + [ "$got" = "ok!!" ] || { echo "smoke printed '$got'" >&2; exit 1; } + + # bin/ + lib/ exactly as the image lays them out, so the tarball and the + # image ship one layout and Main.php's /../lib lookup holds in both. + - name: Tarball + run: | + name="manticore-${{ steps.v.outputs.version }}-linux-${{ matrix.arch }}" + cid="$(docker create manticore-runtime:${{ matrix.arch }})" + mkdir -p "dist/$name" + docker cp "$cid:/opt/manticore/bin" "dist/$name/bin" + docker cp "$cid:/opt/manticore/lib" "dist/$name/lib" + docker rm "$cid" > /dev/null + cp README.md LICENSE "dist/$name/" 2>/dev/null || true + tar -C dist -czf "dist/$name.tar.gz" "$name" + rm -rf "dist/$name" + ls -la dist/ + + - name: Push the arch image + if: steps.v.outputs.publish == 'true' + run: | + echo "${{ github.token }}" | docker login ghcr.io -u "${{ github.actor }}" --password-stdin + docker tag manticore-runtime:${{ matrix.arch }} \ + "$IMAGE:${{ steps.v.outputs.version }}-${{ matrix.arch }}" + docker push "$IMAGE:${{ steps.v.outputs.version }}-${{ matrix.arch }}" + + - uses: actions/upload-artifact@v4 + with: + name: tarball-linux-${{ matrix.arch }} + path: dist/*.tar.gz + + macos: + name: macos-arm64 + runs-on: macos-15 + timeout-minutes: 120 + steps: + - uses: actions/checkout@v4 + + - name: Toolchain + run: | + brew install php pcre2 openssl@3 sqlite + php -v + + - name: Version + id: v + run: | + if [ "${GITHUB_REF_TYPE}" = "tag" ]; then + echo "version=${GITHUB_REF_NAME#v}" >> "$GITHUB_OUTPUT" + else + echo "version=0.0.0-dev.${GITHUB_SHA::12}" >> "$GITHUB_OUTPUT" + fi + + # Cold, like the image: a release is built from the tagged source by the + # interpreter, never from a cached compiler. + - name: Cold seed + run: bin/compile + + - name: Installed layout holds + run: bash tools/install_smoke.sh + + - name: Tarball + run: | + name="manticore-${{ steps.v.outputs.version }}-macos-arm64" + mkdir -p "dist/$name/bin" + cp bin/manticore "dist/$name/bin/manticore" + cp -R lib "dist/$name/lib" + cp README.md LICENSE "dist/$name/" 2>/dev/null || true + tar -C dist -czf "dist/$name.tar.gz" "$name" + rm -rf "dist/$name" + ls -la dist/ + + - uses: actions/upload-artifact@v4 + with: + name: tarball-macos-arm64 + path: dist/*.tar.gz + + # One multi-arch tag out of the two single-arch ones, by digest — no rebuild, + # no emulation, and one `docker pull` resolves to the right arch per host. + manifest: + needs: linux + permissions: + packages: write + if: github.ref_type == 'tag' || inputs.publish + runs-on: ubuntu-latest + steps: + - name: Version + id: v + run: | + if [ "${GITHUB_REF_TYPE}" = "tag" ]; then + echo "version=${GITHUB_REF_NAME#v}" >> "$GITHUB_OUTPUT" + else + echo "version=0.0.0-dev.${GITHUB_SHA::12}" >> "$GITHUB_OUTPUT" + fi + + - name: Merge + run: | + echo "${{ github.token }}" | docker login ghcr.io -u "${{ github.actor }}" --password-stdin + v="${{ steps.v.outputs.version }}" + docker buildx imagetools create -t "$IMAGE:$v" "$IMAGE:$v-arm64" "$IMAGE:$v-amd64" + # `latest` follows tags only: a dispatch build must never become it. + if [ "${GITHUB_REF_TYPE}" = "tag" ]; then + docker buildx imagetools create -t "$IMAGE:latest" "$IMAGE:$v-arm64" "$IMAGE:$v-amd64" + fi + docker buildx imagetools inspect "$IMAGE:$v" + + release: + needs: [linux, macos] + if: github.ref_type == 'tag' + runs-on: ubuntu-latest + permissions: + contents: write + steps: + # Full history and tags: the changelog below is the range between the + # previous tag and this one, and a depth-1 checkout has neither end of it. + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - uses: actions/download-artifact@v4 + with: + path: dl + pattern: tarball-* + merge-multiple: true + + - name: Checksums + run: | + cp install.sh dl/install.sh + cd dl && sha256sum ./*.tar.gz install.sh > SHA256SUMS && cat SHA256SUMS + + - name: Release + env: + GH_TOKEN: ${{ github.token }} + run: | + v="${GITHUB_REF_NAME#v}" + + # NOT `gh release create --generate-notes`: that builds the list from + # merged PRs, and this tree is pushed to directly — 144 commits since + # v0.10.0 and one of them arrived through a PR, so the generated + # section would be empty and look like a release that changed nothing. + # The commits are the record instead. Their subjects are written as + # ": ", so the part before the first colon is the + # summary line and the rationale stays in the history where it belongs. + prev="$(git describe --tags --abbrev=0 "${GITHUB_REF_NAME}^" 2>/dev/null || true)" + if [ -n "$prev" ]; then + range="$prev..$GITHUB_REF_NAME" + since="since \`$prev\`" + else + range="$GITHUB_REF_NAME" + since="all of it" + fi + git log --no-merges --pretty='%s' "$range" \ + | sed 's/^\(.\{0,110\}\):.*/\1/' \ + | sed 's/^\(.\{110\}\).*/\1…/' \ + | sed 's/^/- /' > /tmp/changes.md + count="$(wc -l < /tmp/changes.md | tr -d ' ')" + changes="$(head -80 /tmp/changes.md)" + [ "$count" -le 80 ] || changes="$changes + - … and $((count - 80)) more, see the full log" + + notes="$(cat < /dev/null 2>&1; then + echo "release $GITHUB_REF_NAME exists — uploading the artifacts into it" + gh release upload "$GITHUB_REF_NAME" dl/* --clobber + else + gh release create "$GITHUB_REF_NAME" dl/* --title "$GITHUB_REF_NAME" --notes "$notes" + fi diff --git a/AGENTS.md b/AGENTS.md index e7f7691c..25cbf1b5 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -43,6 +43,7 @@ bash tests/aot/run.sh -k # filter cases (do this first — the full bash tests/aot/run.sh -j 0 # all cores (note: `-j 0` is two args) bash tools/difftest.sh # byte parity vs the `php` interpreter over the corpus bash tools/selfhost_fixpoint.sh # gen2 IR == gen3 IR, self-host suite, rebuild stability +bash tools/install_smoke.sh # an INSTALLED compiler (on $PATH, behind a symlink) finds its own lib/ bash tools/docker/run_tests.sh --gate [--amd64] # the same on Linux (~2 h arm64) ``` diff --git a/Dockerfile b/Dockerfile index a7c1f59f..0445efea 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,23 +1,27 @@ -# Manticore in a container. Two targets: +# Manticore in a container. Four stages, three of them worth building: # # docker build --target toolchain -t manticore-toolchain . -# docker build --target build -t manticore . # see the WARNING below +# docker build --target runtime -t manticore . +# docker build --target build -t manticore-build . # -# `toolchain` is a ready host environment (php 8.5 + clang + pcre2 + openssl); -# mount a checkout into it and build by hand. `build` bakes the compiler in. -# tools/docker/run_tests.sh builds `toolchain` too — one image definition, two -# consumers. +# `base` — clang + the -dev libraries the compiler links against. No php. +# `toolchain` — base + php 8.5: the seed interpreter and the difftest oracle. +# Mount a checkout into it and build by hand; this is what +# tools/docker/run_tests.sh and the workflows use. +# `build` — toolchain + the compiler cold-seeded from this source tree. +# `runtime` — base + that compiler. What gets published; no php in it. # -# Carries PHP 8.5 and the latest stable clang ON BOARD, deliberately -- Debian -# bookworm's stock php (8.2) and clang (14) are both wrong for this compiler: +# Carries PHP 8.5 and the latest stable clang ON BOARD, deliberately -- Debian's +# stock php and clang are both wrong for this compiler: # * PHP 8.5 is manticore's target language version, so the Zend seed must be # 8.5 or the seed disagrees with what it is compiling. # * clang 14 predates LLVM 15's opaque pointers and REJECTS the IR manticore # emits ("ptr type is only supported in -opaque-pointers mode"). Verified, -# not assumed -- stock bookworm clang-14 fails the seed assemble step. +# not assumed -- bookworm's stock clang-14 failed the seed assemble step. # So: php from sury.org, clang from apt.llvm.org. -FROM debian:12 AS toolchain +ARG DEBIAN_TAG=13 +FROM debian:${DEBIAN_TAG} AS base ENV DEBIAN_FRONTEND=noninteractive @@ -36,32 +40,31 @@ ENV DEBIAN_FRONTEND=noninteractive # `curl-config` and the `libcurl.so` symlink, and Main.php's # generic_link_flags() needs one of them — `pkg-config --libs # curl` fails everywhere, since the module is called libcurl. +# wget + gnupg + lsb-release are llvm.sh's own dependencies, and `wget` is not a +# stand-in for the `curl` next to it: llvm.sh calls wget by name. RUN apt-get update && apt-get install -y --no-install-recommends \ - ca-certificates curl gnupg lsb-release software-properties-common \ + ca-certificates curl wget gnupg lsb-release \ gcc libc6-dev libpcre2-dev libssl-dev libcurl4-openssl-dev libsqlite3-dev pkg-config \ binutils bash file make \ netbase \ && rm -rf /var/lib/apt/lists/* -# ---- PHP 8.5 (sury.org) ---- -# The `php8.5-*` extension packages are here for the ORACLE, not for linking: -# difftest grades our output against this php, so a case that calls curl_* or -# PDO can only be graded where the interpreter has that extension too. They are -# a different axis from the `lib*-dev` packages above, which are what our own -# FFI bindings link against — `libsqlite3-dev` without `php8.5-sqlite3` links -# fine and leaves the oracle unable to run a single pdo_* case. -RUN curl -sSLo /usr/share/keyrings/deb.sury.org-php.gpg https://packages.sury.org/php/apt.gpg \ - && echo "deb [signed-by=/usr/share/keyrings/deb.sury.org-php.gpg] https://packages.sury.org/php/ bookworm main" \ - > /etc/apt/sources.list.d/php.list \ - && apt-get update \ - && apt-get install -y --no-install-recommends \ - php8.5-cli php8.5-mbstring php8.5-curl php8.5-sqlite3 \ - && rm -rf /var/lib/apt/lists/* \ - && update-alternatives --set php /usr/bin/php8.5 +# `software-properties-common` exists on bookworm and NOT on trixie — Debian +# dropped it — and this image is built on both: 13 for development, 12 for the +# release tarballs, because glibc is backwards compatible and not forwards. +# llvm.sh needs it on exactly the base where it exists: on bookworm it calls +# add-apt-repository, and on a newer Debian it writes the deb822 source itself. +# Hence a probe rather than a package in the list above — naming it unconditionally +# fails trixie, and dropping it unconditionally fails bookworm. Both happened. +RUN apt-get update \ + && if apt-cache show software-properties-common > /dev/null 2>&1; then \ + apt-get install -y --no-install-recommends software-properties-common; \ + fi \ + && rm -rf /var/lib/apt/lists/* # ---- latest stable clang/LLVM (apt.llvm.org) ---- # NOT `llvm.sh` with no argument: that targets the development version (23 at time of -# writing), which publishes no bookworm packages and hard-fails the build. Walk +# writing), which publishes no packages for this suite and hard-fails the build. Walk # candidate versions newest-first and keep the first that actually installs, so # this tracks "latest that exists" without pinning to a version that will rot. ARG LLVM_VERSIONS="22 21 20" @@ -83,10 +86,9 @@ RUN CLANG_BIN="$(ls -1 /usr/bin/clang-[0-9]* | grep -E 'clang-[0-9]+$' | sort -V && ln -sf "$CLANG_BIN" /usr/local/bin/clang \ && ln -sf "$CLANG_BIN" /usr/local/bin/cc -RUN php --version && clang --version | head -1 && cc --version | head -1 \ +RUN clang --version | head -1 && cc --version | head -1 \ && pcre2-config --libs8 && pkg-config --libs openssl \ - && curl-config --libs && php -r 'exit(function_exists("curl_init") ? 0 : 1);' \ - && pkg-config --libs sqlite3 && php -r 'exit(extension_loaded("pdo_sqlite") ? 0 : 1);' + && curl-config --libs && pkg-config --libs sqlite3 # Run as a normal, unprivileged user. Under root every file is writable/executable # regardless of mode, so a suite that checks permissions diverges from a real @@ -102,6 +104,37 @@ USER manticore CMD ["/bin/bash"] +# ---- + PHP 8.5, the seed interpreter and the difftest oracle ---- +# +# A STAGE of its own, because the shipped compiler does not need it: php is what +# cold-seeds the build and what difftest grades against, and neither happens in +# the image a user runs. `runtime` therefore branches off `base`, not off this. +# +# The `php8.5-*` extension packages are here for the ORACLE, not for linking: +# difftest grades our output against this php, so a case that calls curl_* or +# PDO can only be graded where the interpreter has that extension too. They are +# a different axis from the `lib*-dev` packages in `base`, which are what our own +# FFI bindings link against — `libsqlite3-dev` without `php8.5-sqlite3` links +# fine and leaves the oracle unable to run a single pdo_* case. +FROM base AS toolchain + +USER root +RUN curl -sSLo /usr/share/keyrings/deb.sury.org-php.gpg https://packages.sury.org/php/apt.gpg \ + && echo "deb [signed-by=/usr/share/keyrings/deb.sury.org-php.gpg] https://packages.sury.org/php/ $(. /etc/os-release; echo "$VERSION_CODENAME") main" \ + > /etc/apt/sources.list.d/php.list \ + && apt-get update \ + && apt-get install -y --no-install-recommends \ + php8.5-cli php8.5-mbstring php8.5-curl php8.5-sqlite3 \ + && rm -rf /var/lib/apt/lists/* \ + && update-alternatives --set php /usr/bin/php8.5 + +RUN php --version \ + && php -r 'exit(function_exists("curl_init") ? 0 : 1);' \ + && php -r 'exit(extension_loaded("pdo_sqlite") ? 0 : 1);' + +USER manticore + + # ---- build the compiler from source ---- # # Bakes the compiler in: `bin/compile` cold-seeds src/ -> bin/manticore + lib/. @@ -121,3 +154,33 @@ RUN rm -rf bin/manticore lib tests/aot/tmp \ ENV PATH="/build/manticore/bin:${PATH}" CMD ["/bin/bash"] + + +# ---- what a USER runs: the compiler, and the toolchain it shells out to ---- +# +# docker run --rm -v "$PWD":/work -u "$(id -u):$(id -g)" \ +# manticorephp/manticore manticore compile app.php -o app +# +# Off `base`, so no php and no oracle extensions ride along: the compiler is a +# native binary and never asks for an interpreter. clang, cc, pkg-config and the +# -dev libraries DO stay — the compiler shells out to clang to assemble its IR +# and to cc to link, so an image without them could not compile anything. That +# is also the honest answer to "why not ship a tarball instead": the tarball is +# these three directories, and the toolchain around them is what the image adds. +# +# `-u $(id -u)` above is not decoration: the image runs as uid 1000, and without +# it a binary compiled into a bind mount comes back owned by the wrong user. +FROM base AS runtime + +COPY --from=build /build/manticore/bin/manticore /opt/manticore/bin/manticore +COPY --from=build /build/manticore/lib /opt/manticore/lib + +ENV PATH="/opt/manticore/bin:${PATH}" +WORKDIR /work + +# Reached by the bare name through $PATH — the shape that used to lose the +# prelude and the stdlib before self_dir() resolved the real executable. +RUN manticore version + +USER manticore +CMD ["manticore", "--help"] diff --git a/Dockerfile.alpine b/Dockerfile.alpine new file mode 100644 index 00000000..e3af704c --- /dev/null +++ b/Dockerfile.alpine @@ -0,0 +1,101 @@ +# Manticore on Alpine (musl). The same four stages as the Debian `Dockerfile`, +# and deliberately a SEPARATE file: apk and apt do not parametrise into one +# sensible build, and the interesting differences here are not package names. +# +# docker build -f Dockerfile.alpine --target toolchain -t manticore-toolchain:alpine . +# bash tools/docker/run_tests.sh --alpine +# +# Status: PREPARED, NOT GATED. docs/install.md has always said musl builds and +# loses only a few `glob` constants (`GLOB_BRACE`, `GLOB_ONLYDIR`) and the LFS64 +# aliases, but nothing in CI has ever checked that claim. This file is what makes +# checking it a matrix row instead of an afternoon. +# +# Why musl is worth preparing at all: it is the shape a statically linked, truly +# portable output binary would take. Until the compiler grows `-static` / `-rpath` +# that is a future, not a feature — so this stays out of the required checks. +ARG ALPINE_TAG=3.23 +FROM alpine:${ALPINE_TAG} AS base + +# clang assembles the IR; gcc is here for `cc`, the link driver, and it brings +# musl-dev's crt files with it. pkgconf provides pkg-config, pcre2-dev provides +# pcre2-config, curl-dev provides curl-config — the three probes Main.php runs. +# /etc/services ships in alpine-baselayout, so there is no netbase to add. +# The last line is what a first musl run costs, and none of it is guesswork — +# each package is one cluster of suite failures: +# tzdata -> /usr/share/zoneinfo, which TzInfo.php reads directly. Alpine +# ships no tz database at all, so every date_* case failed. +# libxml2-dev -> prelude/xml.php binds libxml2 through FFI (`#[Library('xml2')]`), +# and the dev package is what provides the linkable .so. +# gnu-libiconv -> musl keeps iconv INSIDE libc and ships no libiconv.so, while +# src/Runtime/Iconv.php binds `#[Library('iconv')]` by name. +RUN apk add --no-cache \ + clang lld gcc musl-dev binutils \ + pcre2-dev openssl-dev curl-dev sqlite-dev pkgconf \ + bash file make curl ca-certificates \ + tzdata libxml2-dev gnu-libiconv + +RUN clang --version | head -1 && cc --version | head -1 \ + && pcre2-config --libs8 && pkg-config --libs openssl \ + && curl-config --libs && pkg-config --libs sqlite3 + +# uid 1000, matching the Debian image: running as root would make is_writable() +# of a 0400 file answer true and quietly disagree with the recorded expectations. +RUN adduser -D -u 1000 -s /bin/bash manticore \ + && mkdir -p /build \ + && chown -R manticore:manticore /build + +WORKDIR /build +USER manticore +CMD ["/bin/bash"] + + +# ---- + PHP 8.5, the seed interpreter and the difftest oracle ---- +# Alpine 3.23 carries php85 in the main repository, so this needs no third-party +# source — the one place musl is EASIER than Debian, where php comes from sury. +FROM base AS toolchain + +# Alpine splits php far finer than Debian does, and the split is not cosmetic: +# `php85` alone has no ctype, and the Zend seed dies on line one of the bootstrap +# with `Call to undefined function ctype_digit()`. Everything below `mbstring` is +# what Debian's php8.5-cli bundles and Alpine does not. +USER root +RUN apk add --no-cache \ + php85 php85-mbstring php85-curl php85-pdo_sqlite \ + php85-ctype php85-tokenizer php85-openssl php85-phar \ + php85-session php85-posix php85-iconv php85-fileinfo \ + && ln -sf /usr/bin/php85 /usr/local/bin/php + +RUN php --version \ + && php -r 'exit(function_exists("ctype_digit") ? 0 : 1);' \ + && php -r 'exit(function_exists("curl_init") ? 0 : 1);' \ + && php -r 'exit(extension_loaded("pdo_sqlite") ? 0 : 1);' + +USER manticore + + +# ---- build the compiler from source ---- +FROM toolchain AS build + +COPY --chown=manticore:manticore . /build/manticore +WORKDIR /build/manticore + +RUN rm -rf bin/manticore lib tests/aot/tmp \ + && bin/compile + +ENV PATH="/build/manticore/bin:${PATH}" +CMD ["/bin/bash"] + + +# ---- what a user runs ---- +FROM base AS runtime + +COPY --from=build /build/manticore/bin/manticore /opt/manticore/bin/manticore +COPY --from=build /build/manticore/lib /opt/manticore/lib + +ENV PATH="/opt/manticore/bin:${PATH}" +WORKDIR /work + +RUN manticore version + +USER manticore +CMD ["manticore", "--help"] diff --git a/README.md b/README.md index 741e3d75..21f2d853 100644 --- a/README.md +++ b/README.md @@ -53,21 +53,37 @@ Per-OS package lists, Docker images and troubleshooting: ## Install -There is no prebuilt binary to download — the compiler compiles itself. The installer -checks the toolchain above, tells you what is missing, then installs under -`$MANTICORE_HOME` (default `~/.manticore`): +The installer takes a published build when there is one for your platform (linux and +macOS, arm64 and amd64), verifies it against the release checksums, and otherwise +falls back to what it has always done: compile the compiler with itself. It checks +the toolchain above, tells you what is missing, then installs under `$MANTICORE_HOME` +(default `~/.manticore`): ```bash curl -fsSL https://raw.githubusercontent.com/manticorephp/compiler/main/install.sh | bash export PATH="$HOME/.manticore/bin:$PATH" -manticore version # manticore 0.10.0 +manticore version # manticore 0.11.0 ``` -Re-running the installer **upgrades in place**: an existing `manticore` rebuilds the -new version *with itself* (self-host, fast) — the Zend seed is only the cold first -boot. Knobs: `MANTICORE_HOME`, `MANTICORE_REF` (branch/tag), `MANTICORE_REPO`, -`MANTICORE_SRC` (build a local checkout instead of cloning). +Re-running the installer **upgrades in place**. Knobs: `MANTICORE_HOME`, +`MANTICORE_VERSION` (a specific release), `MANTICORE_FROM_SOURCE=1` (skip the +download), `MANTICORE_REF` (branch/tag), `MANTICORE_REPO`, `MANTICORE_SRC` (build a +local checkout instead of cloning). Building from source is the same self-hosting +loop it always was: an existing `manticore` rebuilds the new version *with itself*, +and the Zend seed is only the cold first boot. + +In a container, with the toolchain already in it: + +```bash +docker run --rm -v "$PWD":/work -u "$(id -u):$(id -g)" \ + ghcr.io/manticorephp/compiler manticore compile app.php -o app +``` + +A tarball carries the compiler and its stdlib, not a toolchain: `manticore` +assembles its IR with `clang`, links with `cc`, and the binaries it emits link the +host's pcre2, openssl, sqlite3 and curl. The image is the one artifact that brings +all of that with it. Via Composer, which here is a delivery + build trigger rather than a runtime: @@ -356,8 +372,14 @@ bash tests/aot/run.sh -k hello # filter by substring bash tools/difftest.sh # parity vs `php` bash tools/selfhost_fixpoint.sh # fixpoint + self-host suite + rebuild stability bash tools/docker/run_tests.sh --gate # the same, on Linux +bash tools/install_smoke.sh # an installed compiler ($PATH, symlink) finds its own lib/ ``` +CI runs the suite on every push to `main` and every PR — Linux arm64 and amd64 in +the container, macOS bare — self-hosting from the compiler the previous run cached, +with the Zend seed as the fallback rather than the loop. `gate.yml` adds difftest +weekly, and the fixpoint only when asked for. + `selfhost_fixpoint.sh` asserts gen2 IR == gen3 IR, runs the suite through the self-built compiler, and rebuilds repeatedly to catch build-to-build layout roulette. The Linux gate is not optional for anything touching `src/Runtime/`, syscalls or errno. diff --git a/bin/build b/bin/build index 08112c5b..80c11383 100755 --- a/bin/build +++ b/bin/build @@ -278,6 +278,15 @@ else exit 1 fi echo "ok: lib/manticore_stdlib.o (built by $OUT)" + + # `lib/prelude/` is part of the SHIPPED layout, not of the dev tree: the + # compiler in a checkout reads `/../prelude` (the source), so a stale + # copy under lib/ is invisible here and only surfaces once something takes + # lib/ away with it — a CI compiler cache, tools/install_smoke.sh, a release + # tarball built warm. Only bin/compile used to publish it, which means every + # self-hosted build left it at whatever the last COLD seed wrote. + mkdir -p lib/prelude + cp prelude/*.php lib/prelude/ fi if [[ $VERIFY -eq 1 ]]; then diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index 5ecf0ac1..3efbc92f 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -229,7 +229,7 @@ dispatch for `__get`/`__set`/`__isset`/`__unset`/`__call` are **done**. What is [`design/module-system.md`](design/module-system.md) but nowhere in `src/`. Manifest targets and Composer source discovery work; transitive dependency fetch does not. - **The ABI version is not surfaced.** `MemoryAbi::VERSION` is 8 and `manticore version` - prints only `manticore 0.10.0`, so a vendored `.o` cannot detect a mismatch. + prints only `manticore 0.11.0`, so a vendored `.o` cannot detect a mismatch. - **`dump-mir --after=`** is described in [`design/mir.md`](design/mir.md) but not implemented. - **Cycle collector: manual trigger only**, and it does not scan static or global roots. A diff --git a/docs/design/memory-abi.md b/docs/design/memory-abi.md index 742c9a4c..f658678c 100644 --- a/docs/design/memory-abi.md +++ b/docs/design/memory-abi.md @@ -340,5 +340,5 @@ patch also: rewrite the drop-body / walker emission against the new shape, updat direct offset GEP, and bump the affected `*_HEADER_SIZE`. The constant is **not** currently surfaced by any command — `manticore version` prints the -release version (`manticore 0.10.0`) and nothing else. Exposing the ABI version so vendored +release version (`manticore 0.11.0`) and nothing else. Exposing the ABI version so vendored `.o` artefacts can detect a mismatch is open work. diff --git a/docs/install.md b/docs/install.md index 0a92c7de..9820f331 100644 --- a/docs/install.md +++ b/docs/install.md @@ -1,9 +1,11 @@ # Installing Manticore — host dependencies and platform support -Manticore compiles PHP to a native binary. The *output* binaries are fully -static and have no runtime dependencies — they call libc and nothing else. The -*compiler*, however, shells out to a real toolchain and links against a few -system libraries, so the host needs those present at build time. +Manticore compiles PHP to a native binary with no PHP runtime in it — no +interpreter, no `php.ini`, no extension loader. It is not statically linked, +though: an output binary links libc, PCRE2 and OpenSSL dynamically, plus whatever +a program's FFI bindings name (libcurl, libsqlite3, …), so the machine that RUNS +it needs those libraries too. The *compiler* additionally shells out to a real +toolchain, so the host needs clang and `cc` present at build time. This is an end-user guide. Quick version lives in the README's `Requirements`. @@ -11,23 +13,30 @@ This is an end-user guide. Quick version lives in the README's `Requirements`. ## Quick install -Manticore builds **from source** — there is no prebuilt binary to download; the -compiler compiles itself. The installer needs the [host toolchain](#what-the-host-needs-and-why) -present (it checks and tells you what is missing), then puts everything under -`$MANTICORE_HOME` (default `~/.manticore`). +The installer takes a published build when one exists for your platform (linux +and macOS, arm64 and amd64), verified against the release's `SHA256SUMS`, and +otherwise builds **from source** — the compiler compiles itself. Either way it +needs the [host toolchain](#what-the-host-needs-and-why) present (it checks and +tells you what is missing), then puts everything under `$MANTICORE_HOME` +(default `~/.manticore`). ```bash curl -fsSL https://raw.githubusercontent.com/manticorephp/compiler/main/install.sh | bash # then, as the script prints: export PATH="$HOME/.manticore/bin:$PATH" -manticore version # -> manticore 0.10.0 +manticore version # -> manticore 0.11.0 ``` -Re-running the installer **upgrades in place**: once a working `manticore` is -installed it rebuilds the new version *with itself* (self-host, fast); the Zend -seed is only the cold first boot. Knobs: `MANTICORE_HOME`, `MANTICORE_REF` -(branch/tag), `MANTICORE_REPO`, `MANTICORE_SRC` (build a local checkout instead -of cloning). +Re-running the installer **upgrades in place**. When it builds from source, a +working `manticore` rebuilds the new version *with itself* (self-host, fast) and +the Zend seed is only the cold first boot. Knobs: `MANTICORE_HOME`, +`MANTICORE_VERSION` (a specific release), `MANTICORE_FROM_SOURCE=1` (skip the +download), `MANTICORE_REF` (branch/tag), `MANTICORE_REPO`, `MANTICORE_SRC` +(build a local checkout instead of cloning). + +A published tarball is built on **Debian 12** (glibc 2.36) even though the +development image tracks Debian 13: a release has to run on the distribution +someone already has, and glibc is backwards compatible, not forwards. ### Via Composer @@ -148,9 +157,25 @@ a specific toolchain; otherwise follow the Dockerfile's approach. ### Alpine (musl) ```bash -apk add clang lld musl-dev pcre2-dev openssl-dev curl-dev pkgconf bash php85-cli +apk add clang lld gcc musl-dev binutils pcre2-dev openssl-dev curl-dev sqlite-dev \ + pkgconf bash file make \ + php85 php85-ctype php85-mbstring php85-tokenizer php85-openssl \ + php85-phar php85-session php85-posix php85-iconv php85-fileinfo \ + php85-curl php85-pdo_sqlite ``` +Alpine splits php far finer than Debian does, and the split is not cosmetic: `php85` +alone has no **ctype**, and the Zend seed dies on the first line of the bootstrap with +`Call to undefined function ctype_digit()`. Everything after `php85-mbstring` above is +what Debian's `php8.5-cli` bundles and Alpine does not. + +`Dockerfile.alpine` is this list, as the same four stages as the Debian image, and +`bash tools/docker/run_tests.sh --alpine` runs the usual gate against it (its own image +tag and its own compiler-cache volume — a glibc binary does not run in a musl +container). It is **prepared, not gated**: `gate.yml` carries it as an opt-in +`workflow_dispatch` input marked `continue-on-error`, because nothing had ever checked +the claim below until that button existed. + musl exports plain `stat`/`lstat`/`fstat` and has `glob`/`globfree`, but lacks `GLOB_BRACE`, `GLOB_ONLYDIR` and the LFS64 aliases (`stat64`) — a few filesystem functions degrade accordingly. @@ -165,7 +190,7 @@ functions degrade accordingly. | macOS x86_64 | **supported** | | Linux glibc ≥ 2.33 (arm64 / x86_64) | **supported** — full build + self-host fixpoint pass | | Linux glibc < 2.33 (e.g. Ubuntu 20.04) | **unsupported** — cannot link `stat` | -| Linux musl / Alpine | same as glibc, minus some `glob` constants | +| Linux musl / Alpine | **prepared, not gated** — builds, minus some `glob` constants; `Dockerfile.alpine` + `run_tests.sh --alpine`, and an opt-in `gate.yml` job | Both macOS and Linux build the compiler from the cold Zend seed, self-host (`bin/build` rebuilds the compiler byte-for-byte), and pass the full AOT suite @@ -178,23 +203,46 @@ see [`tools/docker/README.md`](../tools/docker/README.md). ## Docker -The root `Dockerfile` has two targets. +The published image carries the compiler **and** the toolchain it shells out to, +which is the one form of delivery that needs nothing installed on the far side: + +```bash +docker run --rm -v "$PWD":/work -u "$(id -u):$(id -g)" \ + ghcr.io/manticorephp/compiler manticore compile app.php -o app +``` + +`-u "$(id -u):$(id -g)"` is not decoration: the image runs as uid 1000, and +without it the binary it writes into your bind mount comes back owned by someone +else. + +The root `Dockerfile` builds that image, and three stages lead to it: -**`toolchain`** — a ready host environment, nothing baked in. Mount a checkout -and work in it: +| Target | What it is | +|---|---| +| `base` | clang + the `-dev` libraries the compiler links against. **No php.** | +| `toolchain` | `base` + PHP 8.5 — the cold-seed interpreter and the difftest oracle | +| `build` | `toolchain` + the compiler, cold-seeded from the source tree | +| `runtime` | `base` + that compiler — what gets published | ```bash +# a ready host environment; mount a checkout and work in it docker build --target toolchain -t manticore-toolchain . docker run --rm -it -v "$PWD":/build/manticore -w /build/manticore \ manticore-toolchain bash + +# the published shape, built locally +docker build --target runtime -t manticore . ``` -**`build`** — copies the repo in and runs `bin/compile`, baking the compiler into -the image. +php lives in `toolchain` and not in `base` on purpose: the shipped compiler is a +native binary and never asks for an interpreter, so `runtime` branches off +`base` and carries none. -```bash -docker build --target build -t manticore . -``` +The base is `ARG DEBIAN_TAG=13`. Release tarballs are built with +`--build-arg DEBIAN_TAG=12` (glibc 2.36) — glibc is backwards compatible and not +forwards, so shipping from the newest base would mean running only on the newest +distributions. `Dockerfile.alpine` is the musl counterpart, with the same four +stages. To run the libc probes and the AOT suite in a container, see [`tools/docker/README.md`](../tools/docker/README.md). @@ -220,5 +268,11 @@ than 2.33. See the hard floors above. install the PCRE2 *development* package (`libpcre2-dev`, `pcre2-dev`, or `brew install pcre2`), not just the runtime library. Same shape for OpenSSL. -**Seed build runs out of memory** — `bin/compile` invokes Zend with -`-d memory_limit=2048M`. A container with a lower hard limit will be OOM-killed. +**Seed build runs out of memory** — two different ceilings, and the second one +is the one people hit. `bin/compile` invokes Zend with `-d memory_limit=2048M`, +so a container with a lower hard limit is OOM-killed during the bootstrap. Past +that, the seed builds the whole compiler as one LLVM module, and **that peaks at +just under 7 GiB** (measured: 6.83 GiB on glibc, 6.94 GiB on musl — the libc is +not the variable). A machine or a Docker VM with 8 GB is therefore right at the +edge: glibc squeaks under and musl does not. Give the VM 12 GB, or use a warm +`bin/build`, which does not pay this at all. diff --git a/install.sh b/install.sh index 8fb53183..e8e516a1 100755 --- a/install.sh +++ b/install.sh @@ -1,12 +1,16 @@ #!/usr/bin/env bash # -# Manticore installer — builds the compiler FROM SOURCE into $MANTICORE_HOME -# (default ~/.manticore) and tells you how to put `manticore` on your PATH. +# Manticore installer — puts the compiler into $MANTICORE_HOME (default +# ~/.manticore) and tells you how to put `manticore` on your PATH. # # curl -fsSL https://raw.githubusercontent.com/manticorephp/compiler/main/install.sh | bash # ./install.sh # from a checkout # -# No prebuilt binary is downloaded — Manticore compiles ITSELF from PHP source: +# Two paths, in this order: +# * a PUBLISHED build for this platform (linux/macos × arm64/amd64), verified +# against the release's SHA256SUMS. Needs no php: the compiler is native and +# CI already paid the bootstrap. MANTICORE_FROM_SOURCE=1 skips it. +# * otherwise Manticore compiles ITSELF from PHP source: # * first install (no $MANTICORE_HOME/bin/manticore yet): cold bootstrap via # the Zend seed (bin/compile). # * upgrade (a working binary already installed): the installed compiler @@ -20,7 +24,8 @@ # $MANTICORE_HOME/lib/manticore_stdlib.o(.sig) # $MANTICORE_HOME/lib/prelude/*.php # -# Env knobs: MANTICORE_HOME, MANTICORE_REPO, MANTICORE_REF, MANTICORE_SRC. +# Env knobs: MANTICORE_HOME, MANTICORE_REPO, MANTICORE_REF, MANTICORE_SRC, +# MANTICORE_VERSION (a specific release), MANTICORE_FROM_SOURCE=1. set -euo pipefail @@ -41,6 +46,90 @@ case "$OS" in esac log "platform $OS/$ARCH -> prefix $PREFIX" +# ---- 1b. a published build, if there is one for this platform ------------- +# The fast path, and since 0.11 the usual one: a release tarball is steps 3-5 +# below, already done on CI and cold-seeded from the tag. It costs a download +# instead of a bootstrap, and it needs no php at all — the compiler is native. +# Skipped for an explicitly source-flavoured install (MANTICORE_SRC, a +# non-default MANTICORE_REF, MANTICORE_FROM_SOURCE=1), and ANY miss falls +# through to the build rather than failing: a platform with no published build, +# an unreachable github, a checksum that does not match. +REL_OS=""; REL_ARCH="" +case "$OS" in Darwin) REL_OS=macos;; Linux) REL_OS=linux;; esac +case "$ARCH" in arm64|aarch64) REL_ARCH=arm64;; x86_64|amd64) REL_ARCH=amd64;; esac + +fetch() { + if have curl; then curl -fsSL "$1" -o "$2" + elif have wget; then wget -qO "$2" "$1" + else return 1 + fi +} + +try_prebuilt() { + [ "${MANTICORE_FROM_SOURCE:-0}" = 0 ] || return 1 + [ -z "${MANTICORE_SRC:-}" ] || return 1 + [ "$REF" = main ] || return 1 + [ -n "$REL_OS" ] && [ -n "$REL_ARCH" ] || return 1 + have curl || have wget || return 1 + + local api="https://api.github.com/repos/manticorephp/compiler/releases" + local dl="https://github.com/manticorephp/compiler/releases" + local tmp ver base name + tmp="$(mktemp -d)" + + ver="${MANTICORE_VERSION:-}" + if [ -z "$ver" ]; then + fetch "$api/latest" "$tmp/latest.json" || { rm -rf "$tmp"; return 1; } + ver="$(sed -n 's/.*"tag_name"[^"]*"v\{0,1\}\([^"]*\)".*/\1/p' "$tmp/latest.json" | head -1)" + fi + [ -n "$ver" ] || { rm -rf "$tmp"; return 1; } + + base="$dl/download/v$ver" + name="manticore-$ver-$REL_OS-$REL_ARCH" + log "published build $ver ($REL_OS/$REL_ARCH) — downloading (MANTICORE_FROM_SOURCE=1 to build instead)" + fetch "$base/$name.tar.gz" "$tmp/$name.tar.gz" || { rm -rf "$tmp"; return 1; } + + # An unverified download is still better than no install, but say which it was. + if fetch "$base/SHA256SUMS" "$tmp/SHA256SUMS" 2>/dev/null; then + local want got sum + sum="" + have sha256sum && sum="sha256sum" + [ -n "$sum" ] || { have shasum && sum="shasum -a 256"; } + if [ -n "$sum" ]; then + want="$(sed -n "s|^\([0-9a-f]\{64\}\)[ *]*\./\{0,1\}$name\.tar\.gz\$|\1|p" "$tmp/SHA256SUMS" | head -1)" + got="$($sum "$tmp/$name.tar.gz" | cut -d' ' -f1)" + if [ -n "$want" ] && [ "$want" != "$got" ]; then + warn "checksum mismatch for $name.tar.gz — building from source instead" + rm -rf "$tmp"; return 1 + fi + else + warn "no sha256sum/shasum here — the download is unverified" + fi + fi + + tar -xzf "$tmp/$name.tar.gz" -C "$tmp" || { rm -rf "$tmp"; return 1; } + [ -x "$tmp/$name/bin/manticore" ] || { rm -rf "$tmp"; return 1; } + + log "installing into $PREFIX" + mkdir -p "$PREFIX/bin" "$PREFIX/lib" + rm -rf "$PREFIX/lib/prelude" + # macOS refuses to overwrite a RUNNING or signed binary in place (SIGKILL); + # removing first is the difference between an upgrade and a dead install. + rm -f "$PREFIX/bin/manticore" + cp "$tmp/$name/bin/manticore" "$PREFIX/bin/manticore" + cp -R "$tmp/$name/lib/." "$PREFIX/lib/" + if [ "$OS" = Darwin ]; then + # Downloaded and unsigned: without this Gatekeeper answers with a dialog + # about an unverified developer, which names nothing that would fix it. + xattr -d com.apple.quarantine "$PREFIX/bin/manticore" 2>/dev/null || true + fi + rm -rf "$tmp" + return 0 +} + +PREBUILT=0 +if try_prebuilt; then PREBUILT=1; fi + # ---- 2. toolchain --------------------------------------------------------- # Hard requirements to BUILD the compiler: php (seed), clang>=15, cc. The # stdlib's preg/TLS/hash bindings are declare-only in the object, resolved at @@ -48,7 +137,9 @@ log "platform $OS/$ARCH -> prefix $PREFIX" # program actually calls preg_*/https/hash. Missing them is a warning, not a # blocker. hard=() -have php || hard+=("php 8.5 (the cold-bootstrap seed)") +# php seeds the bootstrap and nothing else — a published build has already been +# through it, so an install that took the fast path does not want php at all. +[ "$PREBUILT" = 1 ] || have php || hard+=("php 8.5 (the cold-bootstrap seed)") have clang || hard+=("clang/LLVM>=15 (opaque-pointer IR)") have cc || hard+=("cc (final link driver)") soft=() @@ -76,6 +167,12 @@ cmajor="$(clang --version | sed -n 's/.*version \([0-9][0-9]*\).*/\1/p' | head - [ -n "$cmajor" ] && [ "$cmajor" -ge 15 ] 2>/dev/null \ || die "clang ${cmajor:-?} is too old — Manticore emits opaque-pointer IR (needs LLVM >= 15)." +# ---- 3-5. source, build, install ------------------------------------------ +# Everything below the guard is the FROM-SOURCE path; a published build has +# already landed in $PREFIX. Left unindented on purpose — the diff that added +# the guard should not be a diff that rewrote the build. +if [ "$PREBUILT" = 0 ]; then + # ---- 3. source ------------------------------------------------------------ CLEAN_SRC=0 _script_dir="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" 2>/dev/null && pwd || true)" @@ -121,6 +218,8 @@ cp "$SRC"/lib/manticore_stdlib.o.sig "$PREFIX/lib/" 2>/dev/null || true # so publish it from source unconditionally (idempotent, covers both paths). cp "$SRC"/prelude/*.php "$PREFIX/lib/prelude/" +fi # end of the from-source path + # ---- 6. verify ------------------------------------------------------------ ver="$("$PREFIX/bin/manticore" version 2>/dev/null || true)" printf ' "$PREFIX/.smoke.php" diff --git a/src/Manticore/Main.php b/src/Manticore/Main.php index ffd9d910..211000aa 100644 --- a/src/Manticore/Main.php +++ b/src/Manticore/Main.php @@ -1012,7 +1012,72 @@ function collect_stdlib_extern_decls(): array } /** - * Locate the prebuilt `stdlib.o` relative to argv[0] (one file → robust): + * Directory of the RUNNING compiler, symlinks resolved — the anchor every + * bundled asset (stdlib.o, its .sig, the prelude) is found relative to. + * + * Not argv[0] as it arrives: argv[0] is what the caller TYPED. Reached through + * $PATH it is a bare "manticore" with no slash in it at all, and the finders + * below used to give up on that — which is why the dev tree's `bin/manticore` + * worked while an INSTALLED compiler (the container image, a release tarball on + * $PATH) lost its prelude and its stdlib and died with "prelude not found". + * 1. argv[0] with a slash → realpath, so a symlinked install + * (/usr/local/bin/manticore → …/lib/manticore/bin/manticore) anchors at + * the REAL directory rather than at the symlink's. + * 2. /proc/self/exe — the kernel's own answer, immune to argv games. + * 3. a $PATH walk for a bare name — macOS has no /proc. + * "" when nothing resolves; every caller keeps its env override. + */ +function self_dir(): string +{ + // GUARDED: the libc `argv` binding and `cstr_to_str` are throwing stubs + // under the Zend cold-seed, where the env overrides are the resolution path. + $self = ""; + try { + $self = \cstr_to_str(argv(0)); + } catch (\Throwable $e) { + $self = ""; + } + + if ($self !== "" && \strpos($self, "/") !== false) { + $real = \realpath($self); + if (\is_string($real) && $real !== "") { $self = $real; } + } elseif (file_exists("/proc/self/exe")) { + $exe = \readlink("/proc/self/exe"); + if (\is_string($exe) && $exe !== "") { $self = $exe; } + } elseif ($self !== "") { + $self = path_lookup($self); + } + + if ($self === "") { return ""; } + $slashAt = \strrpos($self, "/"); + if ($slashAt === false || $slashAt < 0) { return ""; } + return \substr($self, 0, $slashAt); +} + +/** + * First executable named `$name` on $PATH, symlinks resolved; the name itself + * when $PATH holds no such file. `is_executable` and not `file_exists`: an + * earlier directory holding a same-named data file must not win over the real + * program. + */ +function path_lookup(string $name): string +{ + $path = \getenv("PATH"); + if (!\is_string($path) || $path === "") { return $name; } + foreach (\explode(":", $path) as $dir) { + if ($dir === "") { continue; } + $cand = $dir . "/" . $name; + if (\is_executable($cand)) { + $real = \realpath($cand); + if (\is_string($real) && $real !== "") { return $real; } + return $cand; + } + } + return $name; +} + +/** + * Locate the prebuilt `stdlib.o` relative to the compiler (one file → robust): * - MANTICORE_STDLIB_O env override * - /../lib/manticore_stdlib.o (dev tree: bin/manticore) * - /lib/manticore_stdlib.o (installed) @@ -1025,11 +1090,8 @@ function find_stdlib_object(): string if (\is_string($envPath) && $envPath !== "" && file_exists($envPath)) { return $envPath; } - $rawSelf = argv(0); - $self = \cstr_to_str($rawSelf); - $slashAt = \strrpos($self, "/"); - if ($slashAt === false || $slashAt < 0) { return ""; } - $selfDir = \substr($self, 0, $slashAt); + $selfDir = self_dir(); + if ($selfDir === "") { return ""; } $c1 = $selfDir . "/../lib/manticore_stdlib.o"; if (file_exists($c1)) { return $c1; } $c2 = $selfDir . "/lib/manticore_stdlib.o"; @@ -1053,11 +1115,8 @@ function find_stdlib_sig(): string if (\is_string($envPath) && $envPath !== "" && file_exists($envPath)) { return $envPath; } - $rawSelf = argv(0); - $self = \cstr_to_str($rawSelf); - $slashAt = \strrpos($self, "/"); - if ($slashAt === false || $slashAt < 0) { return ""; } - $selfDir = \substr($self, 0, $slashAt); + $selfDir = self_dir(); + if ($selfDir === "") { return ""; } // Preferred: the manifest's `.sig` (manticore_stdlib.o.sig). $p1 = $selfDir . "/../lib/manticore_stdlib.o.sig"; if (file_exists($p1)) { return $p1; } @@ -1109,24 +1168,15 @@ function find_prelude_src(string $file): string if (\is_string($envDir) && $envDir !== "") { $cands[] = $envDir . "/" . $file; } - // argv0-relative candidates. GUARDED: the libc `argv` binding + `cstr_to_str` - // are absent under the Zend cold-seed (Call-to-undefined Error) — without the - // catch the throw escapes before the env candidate is ever read, so a prelude - // fn the compiler itself uses (explode) never injects into the seed. Under - // Zend MANTICORE_PRELUDE (added above) is the resolution path. - try { - $rawSelf = argv(0); - $self = \cstr_to_str($rawSelf); - $slashAt = \strrpos($self, "/"); - if ($slashAt !== false && $slashAt >= 0) { - $selfDir = \substr($self, 0, $slashAt); - $cands[] = $selfDir . "/../prelude/" . $file; - $cands[] = $selfDir . "/prelude/" . $file; - $cands[] = $selfDir . "/../lib/prelude/" . $file; - $cands[] = $selfDir . "/lib/prelude/" . $file; - } - } catch (\Throwable $e) { - // Zend cold-seed — rely on MANTICORE_PRELUDE. + // Compiler-relative candidates. self_dir() swallows the cold-seed throw (the + // libc `argv` binding + `cstr_to_str` are absent under Zend) and answers "" + // there, so MANTICORE_PRELUDE above stays the seed's resolution path. + $selfDir = self_dir(); + if ($selfDir !== "") { + $cands[] = $selfDir . "/../prelude/" . $file; + $cands[] = $selfDir . "/prelude/" . $file; + $cands[] = $selfDir . "/../lib/prelude/" . $file; + $cands[] = $selfDir . "/lib/prelude/" . $file; } foreach ($cands as $path) { // `\file_get_contents` (global) works in BOTH worlds: PHP's builtin @@ -3093,7 +3143,7 @@ function cmd_dump_llvm(array $args): int { } function cmd_version(array $args): int { - puts("manticore 0.10.0"); + puts("manticore 0.11.0"); return 0; } diff --git a/src/Manticore/README.md b/src/Manticore/README.md index 4386abf9..73b847f8 100644 --- a/src/Manticore/README.md +++ b/src/Manticore/README.md @@ -18,7 +18,7 @@ The compiler self-builds via `manticore build manticore.json` (a self-contained | `dump-llvm` | Front-end + EmitLlvm, print LLVM IR (no link). Same as `dump-llvm-mir`. | | `dump-llvm-mir` | Same: parse → MIR pipeline → EmitLlvm → LLVM IR on stdout. | | `dump-sig` | Parse + lower, print the module-interface `.sig` (exported symbol table). | -| `version` | `manticore 0.10.0`. | +| `version` | `manticore 0.11.0`. | | `help` | Usage block. | ## `manticore build manticore.json` diff --git a/tools/docker/README.md b/tools/docker/README.md index 56d02b27..988e144e 100644 --- a/tools/docker/README.md +++ b/tools/docker/README.md @@ -33,8 +33,10 @@ re-measure, e.g. before changing one of those ABI tables. bash tools/docker/run_tests.sh # arm64: cached self-host build + full suite bash tools/docker/run_tests.sh --amd64 # amd64 (emulated, slow) bash tools/docker/run_tests.sh --both +bash tools/docker/run_tests.sh --alpine # musl instead of glibc (prepared, not gated) bash tools/docker/run_tests.sh --shell # interactive container bash tools/docker/run_tests.sh --gate # the HEAVY gate, on Linux +bash tools/docker/run_tests.sh -k http_ # one case (or a substring) bash tools/docker/run_tests.sh --cold # force a Zend cold seed ``` @@ -47,10 +49,13 @@ socket/errno constants or a glibc `free()` — macOS green proves nothing about in a container because each cold seed is minutes; `MC_STABILITY_N=5` for the full sweep. -The image is the **root `Dockerfile`'s `toolchain` target** -- the same one an -end user builds (see `docs/install.md`). It carries **PHP 8.5** (sury.org) and +The image is the **root `Dockerfile`'s `toolchain` target** (or +`Dockerfile.alpine`'s, with `--alpine`) -- the same one an end user builds (see +`docs/install.md`). Each libc gets its own image tag and its own compiler-cache +volume: a glibc binary does not run in a musl container, and a shared cache would +hand the gate a compiler the loader refuses. It carries **PHP 8.5** (sury.org) and the **latest stable clang** (apt.llvm.org, currently 22) on board, deliberately --- Debian bookworm's stock php 8.2 and clang 14 are both unusable here: +-- Debian's stock php and clang are both unusable here: - PHP 8.5 is manticore's target language, so the Zend seed must be 8.5. - clang 14 predates LLVM 15's opaque pointers and **rejects the IR manticore diff --git a/tools/docker/gate.sh b/tools/docker/gate.sh index 7456e7ac..1d4ba190 100755 --- a/tools/docker/gate.sh +++ b/tools/docker/gate.sh @@ -16,6 +16,12 @@ # AOT suite (bin/build from the cache, cold seed otherwise). # 1 = + difftest (php parity) + selfhost_fixpoint # (fixpoint, MIR golden, rebuild stability). +# It is a shorthand for MC_DIFFTEST=1 MC_FIXPOINT=1; either +# one can be asked for on its own instead. The fixpoint is +# hours and answers a question that only a bootstrap or an +# ABI change can re-open, so CI asks for difftest alone. +# MC_DIFFTEST=0|1 run tools/difftest.sh (default: MC_GATE) +# MC_FIXPOINT=0|1 run tools/selfhost_fixpoint.sh (default: MC_GATE) # MC_JOBS= forwarded to tests/aot/run.sh (0 = one case per core). # Default 0 here: a gate machine is idle otherwise. # MC_FILTER= narrow the suite step to matching case names (`-k`), for @@ -33,6 +39,8 @@ set -uo pipefail MC_GATE="${MC_GATE:-0}" +MC_DIFFTEST="${MC_DIFFTEST:-$MC_GATE}" +MC_FIXPOINT="${MC_FIXPOINT:-$MC_GATE}" MC_JOBS="${MC_JOBS:-0}" MC_STABILITY_N="${MC_STABILITY_N:-2}" MC_REPO="${MC_REPO:-/repo}" @@ -43,13 +51,20 @@ MC_COLD="${MC_COLD:-0}" mkdir -p "$MC_WORK" "$MC_LOGDIR" -echo "=== host: $(uname -m) / $(. /etc/os-release; echo "$PRETTY_NAME")" +# /etc/os-release is Linux-only, and this script now also runs bare on a macOS +# CI runner, where the same steps need the same definition. +if [ -r /etc/os-release ]; then + MC_OS="$(. /etc/os-release; echo "$PRETTY_NAME")" +else + MC_OS="$(sw_vers -productName 2>/dev/null) $(sw_vers -productVersion 2>/dev/null)" +fi +echo "=== host: $(uname -m) / $MC_OS" echo "=== php: $(php -r 'echo PHP_VERSION;')" echo "=== clang: $(clang --version | head -1)" # MC_COMMIT is what CI passes in: the image carries no git, and a bind-mounted # checkout is a different owner than the container user, which `git` refuses. echo "=== commit:${MC_COMMIT:-$(git -C "$MC_REPO" rev-parse --short HEAD 2>/dev/null || echo unknown)}" -echo "=== gate: MC_GATE=$MC_GATE MC_JOBS=$MC_JOBS MC_STABILITY_N=$MC_STABILITY_N opt=-O2 (default)" +echo "=== gate: difftest=$MC_DIFFTEST fixpoint=$MC_FIXPOINT MC_JOBS=$MC_JOBS MC_STABILITY_N=$MC_STABILITY_N opt=-O2 (default)" TREE="$MC_WORK/src-tree" rm -rf "$TREE" @@ -78,20 +93,38 @@ restore_compiler_cache() { bin/manticore version >/dev/null 2>&1 } +# SAYS whether it worked, and that is the point. The cache directory is a host +# mount shared by two different users: on CI the files restored by actions/cache +# belong to the runner, while this container is uid 1000 — and unlinking an entry +# needs write permission on its DIRECTORY, not on the file — so a restored tree +# is unremovable from in here unless the workflow chmods it recursively. When +# that step is missing the copy fails, every message is an `rm: Permission +# denied` nobody reads, the job still passes, and the cache silently never +# updates: every run goes back to a cold seed for ever. A cache that cannot be +# written is not fatal, but it must not be quiet. save_compiler_cache() { [ -n "$MC_COMPILER_CACHE" ] || return 0 [ -x bin/manticore ] || return 0 [ -f lib/manticore_stdlib.o ] || return 0 tmp="$MC_COMPILER_CACHE/.next.$$" - rm -rf "$tmp" - mkdir -p "$tmp/bin" "$tmp/lib" + if ! { rm -rf "$tmp" && mkdir -p "$tmp/bin" "$tmp/lib"; } 2>/dev/null; then + echo "cache: $MC_COMPILER_CACHE is not writable by uid $(id -u) — NOT saved" + return 0 + fi cp bin/manticore "$tmp/bin/manticore" cp -a lib/. "$tmp/lib/" cache_id > "$tmp/id" - rm -rf "$MC_COMPILER_CACHE/bin" "$MC_COMPILER_CACHE/lib" "$MC_COMPILER_CACHE/id" - mv "$tmp/bin" "$tmp/lib" "$tmp/id" "$MC_COMPILER_CACHE/" - rmdir "$tmp" + + if rm -rf "$MC_COMPILER_CACHE/bin" "$MC_COMPILER_CACHE/lib" "$MC_COMPILER_CACHE/id" 2>/dev/null \ + && mv "$tmp/bin" "$tmp/lib" "$tmp/id" "$MC_COMPILER_CACHE/" 2>/dev/null; then + rmdir "$tmp" 2>/dev/null + echo "cache: saved ($(du -sh "$MC_COMPILER_CACHE" 2>/dev/null | cut -f1))" + else + rm -rf "$tmp" 2>/dev/null + echo "cache: the existing entry belongs to another user and cannot be replaced" \ + "from uid $(id -u) — NOT saved, the next run will cold-seed again" + fi } echo @@ -132,6 +165,15 @@ fi save_compiler_cache +# Before the suite, because it is seconds and it covers what the suite cannot: +# the suite always calls `bin/manticore` by path, so it never notices a compiler +# that cannot find its own prelude when it is reached the way an installed one is. +echo +echo "=== tools/install_smoke.sh (an installed layout finds its own lib/) ===" +bash tools/install_smoke.sh > "$MC_LOGDIR/install_smoke.log" 2>&1 +install_rc=$? +tail -5 "$MC_LOGDIR/install_smoke.log" + echo if [ -n "${MC_FILTER:-}" ]; then echo "=== tests/aot/run.sh (-k $MC_FILTER, -j $MC_JOBS) — NOT the gate ===" @@ -143,25 +185,34 @@ fi suite_rc=$? tail -15 "$MC_LOGDIR/suite.log" -if [ "$MC_GATE" != "1" ]; then +if [ "$MC_DIFFTEST" != "1" ] && [ "$MC_FIXPOINT" != "1" ]; then echo - echo "=== RESULT: suite=$suite_rc ===" - exit $suite_rc + echo "=== RESULT: suite=$suite_rc install_smoke=$install_rc ===" + [ "$suite_rc" = "0" ] && [ "$install_rc" = "0" ] || exit 1 + exit 0 fi -echo -echo "=== tools/difftest.sh (php parity, Linux) ===" -bash tools/difftest.sh > "$MC_LOGDIR/difftest.log" 2>&1 -diff_rc=$? -tail -8 "$MC_LOGDIR/difftest.log" +diff_rc=0 +if [ "$MC_DIFFTEST" = "1" ]; then + echo + echo "=== tools/difftest.sh (php parity) ===" + bash tools/difftest.sh > "$MC_LOGDIR/difftest.log" 2>&1 + diff_rc=$? + tail -8 "$MC_LOGDIR/difftest.log" +fi -echo -echo "=== tools/selfhost_fixpoint.sh (fixpoint + MIR golden + stability) ===" -MC_STABILITY_N="$MC_STABILITY_N" bash tools/selfhost_fixpoint.sh > "$MC_LOGDIR/fixpoint.log" 2>&1 -fix_rc=$? -tail -12 "$MC_LOGDIR/fixpoint.log" +# ⚠ This one REPLACES bin/manticore with a stage binary while it runs. Harmless +# here — the tree is a scratch copy — but never point it at a working checkout. +fix_rc=0 +if [ "$MC_FIXPOINT" = "1" ]; then + echo + echo "=== tools/selfhost_fixpoint.sh (fixpoint + MIR golden + stability) ===" + MC_STABILITY_N="$MC_STABILITY_N" bash tools/selfhost_fixpoint.sh > "$MC_LOGDIR/fixpoint.log" 2>&1 + fix_rc=$? + tail -12 "$MC_LOGDIR/fixpoint.log" +fi echo -echo "=== RESULT (Linux gate): suite=$suite_rc difftest=$diff_rc fixpoint=$fix_rc ===" -[ "$suite_rc" = "0" ] && [ "$diff_rc" = "0" ] && [ "$fix_rc" = "0" ] || exit 1 +echo "=== RESULT (gate): suite=$suite_rc install_smoke=$install_rc difftest=$diff_rc fixpoint=$fix_rc ===" +[ "$suite_rc" = "0" ] && [ "$install_rc" = "0" ] && [ "$diff_rc" = "0" ] && [ "$fix_rc" = "0" ] || exit 1 exit 0 diff --git a/tools/docker/run_tests.sh b/tools/docker/run_tests.sh index e14e5811..13bc49cb 100755 --- a/tools/docker/run_tests.sh +++ b/tools/docker/run_tests.sh @@ -9,6 +9,7 @@ # bash tools/docker/run_tests.sh # arm64 # bash tools/docker/run_tests.sh --amd64 # amd64 (emulated) # bash tools/docker/run_tests.sh --both +# bash tools/docker/run_tests.sh --alpine # musl (PREPARED, not gated) # bash tools/docker/run_tests.sh --shell # drop into the container # bash tools/docker/run_tests.sh --cold # ignore the self-host cache # bash tools/docker/run_tests.sh -k http_workers # ONE case (or a substring) @@ -29,6 +30,8 @@ HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" ROOT="$(cd "$HERE/../.." && pwd)" PLATFORMS=(linux/arm64) +DOCKERFILE="" +LIBC=glibc SHELL_MODE=0 GATE_MODE=0 COLD_MODE=0 @@ -39,6 +42,11 @@ for arg in "$@"; do case "$arg" in --amd64) PLATFORMS=(linux/amd64) ;; --both) PLATFORMS=(linux/arm64 linux/amd64) ;; + # musl instead of glibc, via Dockerfile.alpine. Its own image tag and its + # own compiler-cache volume: a glibc binary in a musl container does not + # run, and a cache that mixed them would hand the gate a compiler the + # loader refuses. + --alpine) DOCKERFILE="$ROOT/Dockerfile.alpine"; LIBC=alpine ;; --shell) SHELL_MODE=1 ;; --cold) COLD_MODE=1 ;; # Narrow the SUITE step to the cases whose name contains this substring @@ -53,7 +61,7 @@ for arg in "$@"; do # Linux socket/errno constants or a glibc free(), so this is the only honest # gate for anything touching them. --gate) GATE_MODE=1 ;; - *) echo "usage: $0 [--amd64|--both|--shell|--gate|--cold] [-k ]" >&2; exit 2 ;; + *) echo "usage: $0 [--amd64|--both|--alpine|--shell|--gate|--cold] [-k ]" >&2; exit 2 ;; esac done if [ "$want_filter" = "1" ]; then echo "usage: $0 -k " >&2; exit 2; fi @@ -64,17 +72,18 @@ fi IMAGE_BASE=manticore-toolchain +[ -n "$DOCKERFILE" ] || DOCKERFILE="$ROOT/Dockerfile" for platform in "${PLATFORMS[@]}"; do arch="${platform#linux/}" - image="$IMAGE_BASE:$arch" - cache_volume="manticore-compiler-cache-$arch" + image="$IMAGE_BASE:$arch-$LIBC" + cache_volume="manticore-compiler-cache-$arch-$LIBC" echo "############ $platform ############" >&2 # The root Dockerfile's `toolchain` target — the same image an end user # builds. Its `build` target is deliberately NOT used here: this harness runs # bin/compile against a bind-mounted working tree, not a baked-in copy. docker build --platform "$platform" --target toolchain -t "$image" \ - -f "$ROOT/Dockerfile" "$ROOT" >&2 + -f "$DOCKERFILE" "$ROOT" >&2 # Keep Linux ELF artifacts outside the host checkout. A warmed compiler can # self-host the current source tree, so an edit need not pay the Zend cold diff --git a/tools/install_smoke.sh b/tools/install_smoke.sh new file mode 100755 index 00000000..a7600efb --- /dev/null +++ b/tools/install_smoke.sh @@ -0,0 +1,70 @@ +#!/usr/bin/env bash +# Installed-layout smoke gate: a compiler reached the way a USER reaches it +# still finds its prelude and its stdlib. +# +# The dev tree always calls `bin/manticore`, so argv[0] carries a slash and the +# bundled assets resolve relative to it. An install does not: on $PATH argv[0] +# is the bare name "manticore", and through /usr/local/bin it is a symlink into +# somewhere else entirely. Both used to end in `compile failed: prelude not +# found` — the container image and every release tarball, while the suite stayed +# green, because the suite never invokes the compiler the way a user does. +# +# bash tools/install_smoke.sh [path/to/manticore] +# +# Builds a throwaway install (bin/ + lib/ copied to a temp dir, nothing shared +# with the checkout) and compiles one program through three entry points: +# bare name on $PATH, a symlink from another directory, and a relative path. +# str_pad is the probe on purpose — it is a PHP-level stdlib function, so it +# only links when BOTH the prelude and manticore_stdlib.o were found. + +set -euo pipefail +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +cd "$ROOT" + +MC="${1:-$ROOT/bin/manticore}" +[ -x "$MC" ] || { echo "install_smoke: no compiler at $MC" >&2; exit 1; } +[ -f "$ROOT/lib/manticore_stdlib.o" ] || { echo "install_smoke: no lib/manticore_stdlib.o" >&2; exit 1; } + +WORK="$(mktemp -d)" +trap 'rm -rf "$WORK"' EXIT + +mkdir -p "$WORK/opt/bin" "$WORK/opt/lib" "$WORK/sym" +cp "$MC" "$WORK/opt/bin/manticore" +cp -a "$ROOT/lib/." "$WORK/opt/lib/" +ln -s "$WORK/opt/bin/manticore" "$WORK/sym/manticore" + +cat > "$WORK/hello.php" <<'PHP' + "$WORK/compile.log" 2>&1; then + echo "FAIL $what — compile:" + tail -3 "$WORK/compile.log" + fail=1 + return + fi + local got + got="$("$WORK/hello" 2>&1 || true)" + if [ "$got" = "$EXPECTED" ]; then + echo "PASS $what" + else + echo "FAIL $what — expected '$EXPECTED', got '$got'" + fail=1 + fi +} + +# A bare name: $PATH is searched, and argv[0] has no directory in it at all. +PATH="$WORK/opt/bin:$PATH" check "bare name on \$PATH" manticore +# A symlink from a directory that holds no lib/ of its own. +check "symlink from another dir" "$WORK/sym/manticore" +# The dev-tree shape, which is the one that already worked. +check "explicit path" "$WORK/opt/bin/manticore" + +[ "$fail" = "0" ] || { echo "=== install_smoke: FAILED ==="; exit 1; } +echo "=== install_smoke: ok ==="