Lockstep release tooling for ably-pubsub-core and ably-pubsub-server

.claude/skills/release/SKILL.md

@@ -4,8 +4,11 @@ allowed-tools: Bash, Read, Edit, Write
 ---
 
 Read the current version from `core/pyproject.toml` (the `version` property). Both
-distributions built from this repo are released in lockstep, so there is one
-version for all of the sites below and they must never diverge.
+distributions built from this repo — `ably-pubsub-core` and `ably-pubsub-server` —
+are released in lockstep, so there is one version for all of the sites below and
+they must never diverge. One `vX.Y.Z` tag releases both distributions: the release
+workflow builds them together, refuses to upload anything unless every site agrees,
+and publishes the core before the server.
 
 The bump type is: $ARGUMENTS
 
@@ -25,10 +28,14 @@ Then perform these steps in order:
     - `core/src/ably_pubsub/core/__init__.py` — `lib_version` value
     - `server/src/ably_pubsub/server/__init__.py` — `__version__` value
 3. Run `uv sync` to update the `uv.lock` file
-4. Run `uv run pytest test/unit/pubsub_packaging_test.py` — it asserts that
-   every one of those sites agrees
-5. Commit all files together with message: `chore: bump version to NEW_VERSION`
-6. Fetch merged PRs since the last release tag using:
+4. Run `uv run python scripts/release_preflight.py --version NEW_VERSION` — the
+   same check the release workflow runs before it uploads anything, in its
+   build-free mode (no `dist/` argument), so a missed site fails here rather
+   than mid-release
+5. Run `uv run pytest test/unit/pubsub_packaging_test.py` — the packaging
+   invariants that do not depend on a build
+6. Commit all files together with message: `chore: bump version to NEW_VERSION`
+7. Fetch merged PRs since the last release tag using:
    ```
    gh pr list --state merged --base main --json number,title,mergedAt --limit 200
    ```
@@ -42,7 +49,7 @@ Then perform these steps in order:
    ```
    If the tag doesn't exist or there are no merged PRs, use a single `-` placeholder bullet instead.
 
-7. In `CHANGELOG.md`, insert the following block immediately after the `# Change Log` heading (and its trailing blank line), before the first existing `## [` version entry:
+8. In `CHANGELOG.md`, insert the following block immediately after the `# Change Log` heading (and its trailing blank line), before the first existing `## [` version entry:
 
 ```
 ## [NEW_VERSION](https://github.com/ably/ably-python/tree/vNEW_VERSION)
@@ -51,10 +58,10 @@ Then perform these steps in order:
 
 ### What's Changed
 
-BULLETS_FROM_STEP_6
+BULLETS_FROM_STEP_7
 
 ```
 
-8. Commit `CHANGELOG.md` with message: `docs: update CHANGELOG for NEW_VERSION release`
+9. Commit `CHANGELOG.md` with message: `docs: update CHANGELOG for NEW_VERSION release`
 
 After completing all steps, show the user a summary of what was done. If PRs were found, list them. If the placeholder `-` was used instead, remind them to fill in the `### What's Changed` bullet points in `CHANGELOG.md` before merging.

.github/workflows/check.yml

@@ -55,29 +55,38 @@ jobs:
       - name: Test with pytest
         run: uv run pytest --verbose --tb=short --capture=no
 
-      # A namespace mistake would otherwise surface only after publish: an
-      # ably_pubsub/__init__.py in either wheel makes the two distributions
-      # fight over the same directory, and a core wheel without the generated
-      # sync flavour is a build that skipped unasync.
-      - name: Check the built distributions
+  # The release pre-flight, run on every pull request against the versions in
+  # the tree. A drifted version site, a broken namespace, or a core built
+  # without its generated sync flavour would otherwise surface only at release
+  # time, when the first upload is already irreversible. This runs exactly the
+  # script release.yml runs, minus the release version it has no source for.
+  release-dry-run:
+    permissions:
+      contents: read
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+        with:
+          submodules: 'recursive'
+          persist-credentials: false
+      - name: Set up Python 3.12
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
+        with:
+          python-version: '3.12'
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7
+        with:
+          enable-cache: true
+
+      - name: Install dependencies
+        run: uv sync
+      - name: Generate the sync flavour
+        run: uv run unasync
+      - name: Build both distributions
         run: |
-          uv build --package ably-pubsub-core --out-dir dist-check
-          uv build --package ably-pubsub-server --out-dir dist-check
-          CORE_WHEEL=$(ls dist-check/ably_pubsub_core-*.whl)
-          if ! unzip -l "$CORE_WHEEL" | grep -q "ably_pubsub/core/sync/"; then
-            unzip -l "$CORE_WHEEL"
-            echo "::error::ably_pubsub/core/sync/ not found in the core wheel"
-            exit 1
-          fi
-          for WHEEL in dist-check/*.whl; do
-            if unzip -l "$WHEEL" | grep -q "ably_pubsub/__init__.py"; then
-              echo "::error::$WHEEL ships ably_pubsub/__init__.py; the namespace must stay PEP 420"
-              exit 1
-            fi
-          done
-          CORE_TARBALL=$(ls dist-check/ably_pubsub_core-*.tar.gz)
-          if ! tar -tzf "$CORE_TARBALL" | grep -q "ably_pubsub/core/sync/"; then
-            tar -tzf "$CORE_TARBALL"
-            echo "::error::ably_pubsub/core/sync/ not found in the core sdist"
-            exit 1
-          fi
+          set -euo pipefail
+          uv build --package ably-pubsub-core --out-dir dist
+          uv build --package ably-pubsub-server --out-dir dist
+      - name: 'Release pre-flight (same script as release.yml)'
+        run: uv run python scripts/release_preflight.py dist/

.github/workflows/release.yml

@@ -1,12 +1,57 @@
-# TODO: this workflow still builds and publishes the single `ably` distribution
-# and does not work against the two-distribution workspace. Reworking it into a
-# lockstep release of ably-pubsub-core and ably-pubsub-server is the next PR on
-# the integration branch (plan step 15). The artifact checks it used to carry
-# have moved to check.yml, where they run on every PR.
-name: Publish Python distribution to PyPI
+# Releases ably-pubsub-core and ably-pubsub-server to PyPI in lockstep.
+#
+# LOCKSTEP. This repository builds two distributions and they are always
+# released together at one version: ably-pubsub-server pins
+# `ably-pubsub-core==<that version>` exactly, so a server release without its
+# core is uninstallable and a core release without its server is invisible.
+#
+# PRE-FLIGHT. Everything that can be checked is checked before the first
+# upload, by scripts/release_preflight.py — the same script check.yml's
+# release-dry-run job runs on every pull request. It verifies the release
+# version against every version site (both pyproject `version` fields, the
+# core's `lib_version`, the server's `__version__`) and the server's exact core
+# pins, that dist/ holds exactly one wheel and one sdist per distribution, that
+# the core artifacts carry the generated `ably_pubsub/core/sync/` flavour, that
+# neither wheel ships `ably_pubsub/__init__.py` (the namespace must stay PEP
+# 420) or any file the other wheel also ships, and that `twine check` passes.
+#
+# CORE BEFORE SERVER. The two publish steps are ordered, so the server is never
+# visible on the index before the core version it pins.
+#
+# PARTIAL RELEASES ARE RE-RUNNABLE, NOT IMPOSSIBLE. PyPI has no cross-project
+# transaction: two projects means two uploads, and the second can fail after
+# the first succeeded. Both steps therefore set `skip-existing: true`, so
+# re-running this workflow at the same version skips whatever already landed
+# and completes the release. Never bump the version to work around a partial
+# release — re-run it.
+#
+# TRUSTED PUBLISHING. Both projects must have a trusted publisher configured on
+# pypi.org (and on test.pypi.org) bound to this repository, this workflow file
+# (`release.yml`) and the `pypi` / `testpypi` environment respectively — plan
+# step 16, done immediately after the repo rename so the binding is made once
+# against the new name. PyPI's OIDC token covers every project that trusts the
+# requesting configuration, so one job's `id-token: write` publishes both.
+# The `pypi` environment's required-reviewer rule is the human approval gate.
+#
+# TRIGGERS. A `v<version>` tag push releases to PyPI. A manual dispatch always
+# goes to TestPyPI and only reaches PyPI when `publish` is set — which is how
+# prereleases are cut from a branch (`gh workflow run release.yml --ref
+# <branch> -f version=4.0.0rc1 -f publish=true`). `workflow_dispatch` only
+# works for workflow files present on the default branch.
+name: Publish Python distributions to PyPI
 
 on:
   workflow_dispatch:
+    inputs:
+      version:
+        description: 'Version to release, e.g. 4.0.0 or 4.0.0rc1 — must equal every version site and the server''s core pin'
+        required: true
+        type: string
+      publish:
+        description: 'Also publish to PyPI (not just TestPyPI)'
+        required: false
+        default: false
+        type: boolean
   push:
     tags:
       - 'v[0-9]+.[0-9]+.[0-9]+*'
@@ -15,16 +60,33 @@ permissions: {}
 
 jobs:
   build:
-    name: Build distribution 📦
+    name: Build distributions 📦
     runs-on: ubuntu-latest
     permissions:
       contents: read
+    outputs:
+      version: ${{ steps.release-version.outputs.version }}
 
     steps:
       - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
         with:
           submodules: 'recursive'
           persist-credentials: false
+
+      - name: Determine the release version
+        id: release-version
+        env:
+          VERSION_INPUT: ${{ inputs.version }}
+        run: |
+          set -euo pipefail
+          if [ -n "${VERSION_INPUT}" ]; then
+            RELEASE_VERSION="${VERSION_INPUT}"
+          else
+            RELEASE_VERSION="${GITHUB_REF#refs/tags/v}"
+          fi
+          echo "Releasing ${RELEASE_VERSION}"
+          echo "version=${RELEASE_VERSION}" >> "$GITHUB_OUTPUT"
+
       - name: Set up Python 3.12
         uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
         id: setup-python
@@ -37,26 +99,49 @@ jobs:
           enable-cache: false
 
       - name: Install dependencies
-        run: uv sync --extra crypto --extra dev
-      - name: Generate rest sync code and tests
+        run: uv sync
+
+      # Cheap checks that need no build: the version sites, and — first of all
+      # — that this ref actually has the split layout. This workflow file also
+      # lives on `main` (workflow_dispatch only offers workflows present on the
+      # default branch), where it is inert: dispatched there, the pre-flight
+      # stops here with an explanation instead of a confusing build error.
+      - name: 'Pre-flight: layout and version sites (before anything is built)'
+        env:
+          RELEASE_VERSION: ${{ steps.release-version.outputs.version }}
+        run: uv run python scripts/release_preflight.py --version "$RELEASE_VERSION"
+
+      - name: Generate the sync flavour
         run: uv run unasync
-      - name: Build a binary wheel and a source tarball
-        run: uv build
+      - name: Build both distributions into one dist/
+        run: |
+          set -euo pipefail
+          uv build --package ably-pubsub-core --out-dir dist
+          uv build --package ably-pubsub-server --out-dir dist
+
+      # Nothing below this point is reversible, so this is the last chance to
+      # refuse the release. Same script as check.yml's release-dry-run job.
+      - name: 'Pre-flight: nothing is uploaded unless everything agrees'
+        env:
+          RELEASE_VERSION: ${{ steps.release-version.outputs.version }}
+        run: uv run python scripts/release_preflight.py --version "$RELEASE_VERSION" dist/
+
       - name: Store the distribution packages
         uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
         with:
           name: python-package-distributions
           path: dist/
 
-  publish-to-pypi:
-    name: Publish Python distribution to PyPI
-    if: startsWith(github.ref, 'refs/tags/v')  # only publish to PyPI on tag pushes
+  publish-to-testpypi:
+    name: Publish distributions 📦 to TestPyPI
     needs:
       - build
     runs-on: ubuntu-latest
+
     environment:
-      name: pypi
-      url: https://pypi.org/p/ably
+      name: testpypi
+      url: https://test.pypi.org/p/ably-pubsub-server
+
     permissions:
       id-token: write  # IMPORTANT: mandatory for trusted publishing
 
@@ -67,41 +152,41 @@ jobs:
           name: python-package-distributions
           path: dist/
 
-      - name: Extract tag
-        id: tag
+      # pypa/gh-action-pypi-publish uploads a whole directory, so the two
+      # projects are split into two directories to be uploaded in order.
+      - name: Split the dists by project
         run: |
-          TAG=${GITHUB_REF#refs/tags/v}
-          echo "tag=$TAG" >> $GITHUB_OUTPUT
+          set -euo pipefail
+          mkdir -p dist-core dist-server
+          mv dist/ably_pubsub_core-* dist-core/
+          mv dist/ably_pubsub_server-* dist-server/
 
-      - name: Read VERSION_NAME from dist/
-        id: version
-        run: |
-          VERSION_NAME=$(basename dist/ably-*.tar.gz | sed -E 's/^ably-([^-]+)\.tar\.gz$/\1/')
-          echo "version=$VERSION_NAME" >> $GITHUB_OUTPUT
-
-      - name: Compare version with tag
-        run: |
-          if [ "$VERSION" != "$TAG" ]; then
-            echo "VERSION ($VERSION) does not match tag ($TAG)."
-            exit 1
-          fi
-        env:
-          VERSION: ${{ steps.version.outputs.version }}
-          TAG: ${{ steps.tag.outputs.tag }}
+      - name: Publish ably-pubsub-core 📦 to TestPyPI
+        uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b # release/v1
+        with:
+          packages-dir: dist-core/
+          repository-url: https://test.pypi.org/legacy/
+          skip-existing: true
 
-      - name: Publish distribution 📦 to PyPI
+      - name: Publish ably-pubsub-server 📦 to TestPyPI
         uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b # release/v1
+        with:
+          packages-dir: dist-server/
+          repository-url: https://test.pypi.org/legacy/
+          skip-existing: true
 
-  publish-to-testpypi:
-    name: Publish Python distribution to TestPyPI
+  publish-to-pypi:
+    name: Publish distributions 📦 to PyPI
+    # Tag pushes release; a manual dispatch has to opt in explicitly.
+    if: startsWith(github.ref, 'refs/tags/v') || inputs.publish
+    # Deliberately not `needs: publish-to-testpypi`: TestPyPI is a staging
+    # signal, not a gate — an outage there must not block a real release.
     needs:
       - build
     runs-on: ubuntu-latest
-
     environment:
-      name: testpypi
-      url: https://test.pypi.org/p/ably
-
+      name: pypi
+      url: https://pypi.org/p/ably-pubsub-server
     permissions:
       id-token: write  # IMPORTANT: mandatory for trusted publishing
 
@@ -111,7 +196,24 @@ jobs:
         with:
           name: python-package-distributions
           path: dist/
-      - name: Publish distribution 📦 to TestPyPI
+
+      - name: Split the dists by project
+        run: |
+          set -euo pipefail
+          mkdir -p dist-core dist-server
+          mv dist/ably_pubsub_core-* dist-core/
+          mv dist/ably_pubsub_server-* dist-server/
+
+      # Core first: the server pins this exact version, so it must never be the
+      # one that is visible on the index alone.
+      - name: Publish ably-pubsub-core 📦 to PyPI
         uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b # release/v1
         with:
-          repository-url: https://test.pypi.org/legacy/
+          packages-dir: dist-core/
+          skip-existing: true
+
+      - name: Publish ably-pubsub-server 📦 to PyPI
+        uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b # release/v1
+        with:
+          packages-dir: dist-server/
+          skip-existing: true