ci: notify Slack when release jobs fail, are cancelled, or never run

.github/workflows/release-failure-notification.yml

@@ -0,0 +1,59 @@
+name: Release failure notification
+
+on:
+  workflow_run:
+    workflows: [Release]
+    types: [completed]
+    branches: [main]
+
+permissions:
+  actions: read
+  contents: read
+
+jobs:
+  notify-failed:
+    name: Notify Slack - Failed
+    runs-on: ubuntu-latest
+    if: >-
+      github.event.workflow_run.head_repository.full_name == github.repository &&
+      (github.event.workflow_run.event == 'push' || github.event.workflow_run.event == 'workflow_dispatch')
+    timeout-minutes: 10
+    steps:
+      # This workflow must survive cancellation or runner startup failure in Release.
+      # Never execute code from the triggering run's head in this privileged workflow.
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          ref: ${{ github.event.repository.default_branch }}
+          persist-credentials: false
+
+      - name: Download approval thread timestamps
+        continue-on-error: true
+        uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8
+        with:
+          pattern: release-slack-thread-*
+          path: ${{ runner.temp }}/release-slack-threads
+          github-token: ${{ github.token }}
+          run-id: ${{ github.event.workflow_run.id }}
+
+      - name: Check release jobs
+        id: release-status
+        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
+        env:
+          SLACK_CHANNEL_ID: ${{ vars.SLACK_APPROVALS_CLIENT_LIBRARIES_CHANNEL_ID }}
+          THREAD_DIRECTORY: ${{ runner.temp }}/release-slack-threads
+        with:
+          retries: 3
+          script: |
+            const notification = require('./scripts/release-failure-notification.cjs');
+            const payload = await notification({ github, context, channel: process.env.SLACK_CHANNEL_ID, threadDirectory: process.env.THREAD_DIRECTORY });
+            if (payload) core.setOutput('payload', JSON.stringify(payload));
+
+      - name: Notify Slack - Failed
+        if: steps.release-status.outputs.payload != ''
+        uses: slackapi/slack-github-action@dcb1066f776dd043e64d0e8ba94ca15cc7e1875d # v4.0.0
+        with:
+          method: chat.postMessage
+          token: ${{ secrets.SLACK_CLIENT_LIBRARIES_BOT_TOKEN }}
+          errors: true
+          retries: 5
+          payload: ${{ steps.release-status.outputs.payload }}

.github/workflows/release.yml

@@ -53,6 +53,26 @@ jobs:
       slack_bot_token: ${{ secrets.SLACK_CLIENT_LIBRARIES_BOT_TOKEN }}
       posthog_project_api_key: ${{ secrets.POSTHOG_PROJECT_API_KEY }}
 
+  preserve-slack-thread:
+    name: Preserve Slack thread
+    needs: notify-approval-needed
+    if: needs.notify-approval-needed.outputs.slack_ts != ''
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    continue-on-error: true
+    steps:
+      - name: Save approval thread timestamp
+        env:
+          THREAD_TS: ${{ needs.notify-approval-needed.outputs.slack_ts }}
+        run: printf '%s' "$THREAD_TS" > thread-ts.txt
+
+      - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
+        with:
+          name: release-slack-thread-${{ github.run_attempt }}
+          path: thread-ts.txt
+          retention-days: 30
+          if-no-files-found: error
+
   version-bump:
     name: Bump versions and commit to main
     needs: [check-changesets, notify-approval-needed]
@@ -204,16 +224,6 @@ jobs:
               "alertContext": "The version bump/changelog step failed before packages were published."
             }
 
-      - name: Notify Slack - Failed
-        if: ${{ failure() && needs.notify-approval-needed.outputs.slack_ts != '' }}
-        uses: posthog/.github/.github/actions/slack-thread-reply@cb0979b67dcd585828b61ac45927eef4da8f6287 # main
-        with:
-          slack_bot_token: ${{ secrets.SLACK_CLIENT_LIBRARIES_BOT_TOKEN }}
-          slack_channel_id: ${{ vars.SLACK_APPROVALS_CLIENT_LIBRARIES_CHANNEL_ID }}
-          thread_ts: ${{ needs.notify-approval-needed.outputs.slack_ts }}
-          message: "❌ Failed to bump package versions! <https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}|View logs>"
-          emoji_reaction: "x"
-
   notify-rejected:
     name: Notify Slack - Rejected
     needs: [version-bump, notify-approval-needed]
@@ -254,7 +264,7 @@ jobs:
           emoji_reaction: "no_entry_sign"
 
   publish:
-    name: Release and publish
+    name: Release and publish (${{ matrix.package.name }})
     needs: [version-bump, notify-approval-needed]
     runs-on: ubuntu-latest
     if: always() && needs.version-bump.outputs.committed == 'true'
@@ -410,16 +420,6 @@ jobs:
               "alertContext": "The package publish step failed."
             }
 
-      - name: Notify Slack - Failed
-        if: ${{ failure() && needs.notify-approval-needed.outputs.slack_ts != '' }}
-        uses: posthog/.github/.github/actions/slack-thread-reply@cb0979b67dcd585828b61ac45927eef4da8f6287 # main
-        with:
-          slack_bot_token: ${{ secrets.SLACK_CLIENT_LIBRARIES_BOT_TOKEN }}
-          slack_channel_id: ${{ vars.SLACK_APPROVALS_CLIENT_LIBRARIES_CHANNEL_ID }}
-          thread_ts: ${{ needs.notify-approval-needed.outputs.slack_ts }}
-          message: "❌ Failed to release `${{ matrix.package.name }}@${{ steps.detect-version.outputs.version }}`! <https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}|View logs>"
-          emoji_reaction: "x"
-
   notify-released:
     name: Notify Slack - Released
     needs: [version-bump, publish, notify-approval-needed]

RELEASING.md

@@ -51,3 +51,15 @@ When both packages are released, the workflow publishes `posthog-ruby` first, th
 ## Manual trigger
 
 You can also manually trigger the release workflow from the Actions tab, as long as there are pending changesets.
+
+## Troubleshooting
+
+### Failed or cancelled release
+
+The independent [release failure notification workflow](.github/workflows/release-failure-notification.yml) checks every completed `main` release attempt and alerts if a job failed, timed out, or was cancelled. This includes runner startup failures and cancellations with job records. It also alerts when the run did not succeed, no job failed, and the `Release and publish` job was never created, for example when GitHub Actions stops a run before the publish job starts. Once every `Release and publish` job has published its gem or skipped it for lack of a version bump, alerts describe follow-up problems rather than claiming the release failed, and the workflow alerts if the released Slack notification was skipped. It only acts on runs that a push or a manual dispatch started in this repository.
+
+Cancelled runs with no jobs are ignored, including superseded queued runs and manual cancellations before any job is created. A version-bump failure before any steps ran is also ignored when the rejection-reply step confirms an explicit rejection in the same attempt and no other job failed.
+
+Job inspection retries transient GitHub API failures up to three times. If inspection still fails, it logs the HTTP status or error type and message, then posts a fallback alert with the overall run status and logs link so the release can be checked manually. Error diagnostics stay in the run log, not in the Slack message.
+
+Alerts reply to the approval thread in the release approvals Slack channel. The release workflow preserves the thread timestamp in a run artifact for 30 days, independently of release approval. Failed-job reruns reuse the latest available thread from the same or an earlier attempt. If the timestamp is missing, invalid, or unavailable, the alert falls back to a new channel message, so it does not depend on the approval notification succeeding. The failure message surfaces Slack API errors rather than silently ignoring them. The notification workflow must be present on the default branch to run.

scripts/release-failure-notification.cjs

@@ -0,0 +1,114 @@
+const { readdirSync, readFileSync } = require('node:fs');
+const { join } = require('node:path');
+
+const unsuccessful = new Set(['failure', 'cancelled', 'timed_out', 'action_required', 'startup_failure', 'stale']);
+const publishJob = 'Release and publish';
+const publishStep = 'Publish';
+const gems = ['posthog-ruby', 'posthog-rails'];
+
+const isJob = (job, name) => job.name === name || job.name.startsWith(`${name} (`);
+const gemOf = job => /\((.*)\)$/.exec(job.name)?.[1];
+const publishOutcome = job => job.steps?.find(step => step.name === `${publishStep} ${gemOf(job)}`)?.conclusion;
+const didPublish = job => publishOutcome(job) === 'success';
+// A gem without a version bump skips its publish step; that is not a failure.
+const publishSettled = job => ['success', 'skipped'].includes(publishOutcome(job));
+
+function readThreadTimestamp(directory, attempt) {
+    if (!directory) return undefined;
+    try {
+        const entries = readdirSync(directory);
+        // A single matching artifact is extracted straight into the directory.
+        // Failed-job reruns reuse the approval thread from the latest earlier attempt.
+        const files = entries.includes('thread-ts.txt')
+            ? [join(directory, 'thread-ts.txt')]
+            : entries
+                .map(name => /^release-slack-thread-(\d+)$/.exec(name)?.[1])
+                .filter(value => value && Number(value) <= attempt)
+                .sort((a, b) => Number(b) - Number(a))
+                .map(value => join(directory, `release-slack-thread-${value}`, 'thread-ts.txt'));
+        if (files.length === 0) return undefined;
+        const timestamp = readFileSync(files[0], 'utf8').trim();
+        return /^\d+\.\d+$/.test(timestamp) ? timestamp : undefined;
+    } catch (error) {
+        if (error.code !== 'ENOENT') console.error('Could not read Slack thread:', error.message);
+        return undefined;
+    }
+}
+
+module.exports = async function releaseFailureNotification({ github, context, channel, threadDirectory }) {
+    const run = context.payload.workflow_run;
+    const threadTs = readThreadTimestamp(threadDirectory, run.run_attempt);
+    const destination = { channel, ...(threadTs ? { thread_ts: threadTs } : {}) };
+    let jobs;
+    try {
+        jobs = await github.paginate(github.rest.actions.listJobsForWorkflowRunAttempt, {
+            ...context.repo,
+            run_id: run.id,
+            attempt_number: run.run_attempt,
+            per_page: 100,
+        });
+    } catch (error) {
+        console.error('Failed to inspect release jobs:', error.status ?? error.name, error.message);
+        return {
+            ...destination,
+            text: [
+                `⚠️ Could not inspect release jobs for \`${context.repo.owner}/${context.repo.repo}\` (attempt ${run.run_attempt}).`,
+                'Please check this release manually; individual job results are unavailable.',
+                `<${run.html_url}|View release logs> (run: ${run.conclusion})`,
+            ].join('\n'),
+            unfurl_links: false,
+            unfurl_media: false,
+        };
+    }
+
+    if (jobs.length === 0 && run.conclusion === 'cancelled') {
+        return null;
+    }
+
+    const failedJobs = jobs.filter(job => unsuccessful.has(job.conclusion));
+    const versionBump = jobs.find(job => job.name === 'Bump versions and commit to main');
+    if (run.conclusion === 'failure' && failedJobs.length === 1
+        && versionBump?.conclusion === 'failure' && versionBump.steps?.length === 0
+        && jobs.some(job => job.name === 'Notify Slack - Rejected'
+            && job.steps?.some(step => step.name === 'Notify Slack - Rejected' && step.conclusion === 'success'))) {
+        return null;
+    }
+
+    const publishJobs = jobs.filter(job => isJob(job, publishJob));
+    const gemsPublished = publishJobs.some(didPublish) && publishJobs.every(publishSettled);
+    const releasedNotification = jobs.find(job => job.name === 'Notify Slack - Released');
+    const problems = failedJobs.map(job => `• <${job.html_url}|${job.name}>: ${job.conclusion}`);
+
+    const gemsIn = predicate => publishJobs.filter(predicate).map(gemOf).filter(gem => gems.includes(gem));
+    const publishedGems = gemsIn(didPublish);
+    const unpublishedGems = gemsIn(job => !publishSettled(job));
+    if (!gemsPublished && publishedGems.length > 0) {
+        problems.push(`• Published: ${publishedGems.join(', ')}`);
+        problems.push(`• Not published: ${unpublishedGems.join(', ')}`);
+    }
+
+    if (run.conclusion !== 'success' && failedJobs.length === 0 && publishJobs.length === 0) {
+        problems.push(`• ${publishJob}: never ran`);
+    }
+
+    if (gemsPublished && (!releasedNotification || releasedNotification.conclusion === 'skipped')) {
+        problems.push('• The released Slack notification did not run.');
+    }
+
+    if (problems.length === 0 && run.conclusion === 'success') {
+        return null;
+    }
+
+    return {
+        ...destination,
+        text: [
+            gemsPublished
+                ? `⚠️ Gems published; follow-up problems for \`${context.repo.owner}/${context.repo.repo}\` (attempt ${run.run_attempt}).`
+                : `❌ Release process failed or was cancelled for \`${context.repo.owner}/${context.repo.repo}\` (attempt ${run.run_attempt}).`,
+            ...problems,
+            `<${run.html_url}|View release logs> (run: ${run.conclusion})`,
+        ].join('\n'),
+        unfurl_links: false,
+        unfurl_media: false,
+    };
+};