From ffd5ac175a9484877fb2ce95899e77985d1dec5b Mon Sep 17 00:00:00 2001 From: Ioannis J Date: Tue, 10 Nov 2026 12:36:38 +0200 Subject: [PATCH 1/5] ci: notify Slack when release jobs fail, are cancelled, or never run --- .../release-failure-notification.yml | 56 +++++++++++ .github/workflows/release.yml | 20 ++++ RELEASING.md | 12 +++ scripts/release-failure-notification.cjs | 94 +++++++++++++++++++ 4 files changed, 182 insertions(+) create mode 100644 .github/workflows/release-failure-notification.yml create mode 100644 scripts/release-failure-notification.cjs diff --git a/.github/workflows/release-failure-notification.yml b/.github/workflows/release-failure-notification.yml new file mode 100644 index 00000000..856efac0 --- /dev/null +++ b/.github/workflows/release-failure-notification.yml @@ -0,0 +1,56 @@ +name: Release failure notification + +on: + workflow_run: + workflows: [Release] + types: [completed] + branches: [main] + +permissions: + actions: read + contents: read + +jobs: + notify-failed: + name: Notify Slack - Failed + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + # This workflow must survive cancellation or runner startup failure in Release. + # Never execute code from the triggering run's head in this privileged workflow. + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.event.repository.default_branch }} + persist-credentials: false + + - name: Download approval thread timestamps + continue-on-error: true + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8 + with: + pattern: release-slack-thread-* + path: ${{ runner.temp }}/release-slack-threads + github-token: ${{ github.token }} + run-id: ${{ github.event.workflow_run.id }} + + - name: Check release jobs + id: release-status + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + SLACK_CHANNEL_ID: ${{ vars.SLACK_APPROVALS_CLIENT_LIBRARIES_CHANNEL_ID }} + THREAD_DIRECTORY: ${{ runner.temp }}/release-slack-threads + with: + retries: 3 + script: | + const notification = require('./scripts/release-failure-notification.cjs'); + const payload = await notification({ github, context, channel: process.env.SLACK_CHANNEL_ID, threadDirectory: process.env.THREAD_DIRECTORY }); + if (payload) core.setOutput('payload', JSON.stringify(payload)); + + - name: Notify Slack - Failed + if: steps.release-status.outputs.payload != '' + uses: slackapi/slack-github-action@dcb1066f776dd043e64d0e8ba94ca15cc7e1875d # v4.0.0 + with: + method: chat.postMessage + token: ${{ secrets.SLACK_CLIENT_LIBRARIES_BOT_TOKEN }} + errors: true + retries: 5 + payload: ${{ steps.release-status.outputs.payload }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f1425ed2..788038fd 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -53,6 +53,26 @@ jobs: slack_bot_token: ${{ secrets.SLACK_CLIENT_LIBRARIES_BOT_TOKEN }} posthog_project_api_key: ${{ secrets.POSTHOG_PROJECT_API_KEY }} + preserve-slack-thread: + name: Preserve Slack thread + needs: notify-approval-needed + if: needs.notify-approval-needed.outputs.slack_ts != '' + runs-on: ubuntu-latest + timeout-minutes: 5 + continue-on-error: true + steps: + - name: Save approval thread timestamp + env: + THREAD_TS: ${{ needs.notify-approval-needed.outputs.slack_ts }} + run: printf '%s' "$THREAD_TS" > thread-ts.txt + + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: release-slack-thread-${{ github.run_attempt }} + path: thread-ts.txt + retention-days: 30 + if-no-files-found: error + version-bump: name: Bump versions and commit to main needs: [check-changesets, notify-approval-needed] diff --git a/RELEASING.md b/RELEASING.md index 757d3dba..3bce01fe 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -51,3 +51,15 @@ When both packages are released, the workflow publishes `posthog-ruby` first, th ## Manual trigger You can also manually trigger the release workflow from the Actions tab, as long as there are pending changesets. + +## Troubleshooting + +### Failed or cancelled release + +The independent [release failure notification workflow](.github/workflows/release-failure-notification.yml) checks every completed `main` release attempt and alerts if a job failed, timed out, or was cancelled. This includes runner startup failures and cancellations with job records. It also alerts when the run did not succeed, no job failed, and the `Release and publish` job was never created, for example when GitHub Actions stops a run before the publish job starts. Once every `Release and publish` job succeeds, alerts describe follow-up problems rather than claiming the release failed, and the workflow alerts if the released Slack notification was skipped. + +Cancelled runs with no jobs are ignored, including superseded queued runs and manual cancellations before any job is created. A version-bump failure before any steps ran is also ignored when the rejection-reply step confirms an explicit rejection in the same attempt and no other job failed. + +Job inspection retries transient GitHub API failures up to three times. If inspection still fails, it logs the HTTP status or error type and message, then posts a fallback alert with the overall run status and logs link so the release can be checked manually. Error diagnostics stay in the run log, not in the Slack message. + +Alerts reply to the approval thread in the release approvals Slack channel. The release workflow preserves the thread timestamp in a run artifact for 30 days, independently of release approval. Failed-job reruns reuse the latest available thread from the same or an earlier attempt. If the timestamp is missing, invalid, or unavailable, the alert falls back to a new channel message, so it does not depend on the approval notification succeeding. The failure message surfaces Slack API errors rather than silently ignoring them. The notification workflow must be present on the default branch to run; local tests do not send Slack messages. diff --git a/scripts/release-failure-notification.cjs b/scripts/release-failure-notification.cjs new file mode 100644 index 00000000..41bac248 --- /dev/null +++ b/scripts/release-failure-notification.cjs @@ -0,0 +1,94 @@ +const { readdirSync, readFileSync } = require('node:fs'); +const { join } = require('node:path'); + +const unsuccessful = new Set(['failure', 'cancelled', 'timed_out', 'action_required', 'startup_failure', 'stale']); +const publishJob = 'Release and publish'; + +const isJob = (job, name) => job.name === name || job.name.startsWith(`${name} (`); + +function readThreadTimestamp(directory, attempt) { + if (!directory) return undefined; + try { + // Failed-job reruns reuse the approval thread from the latest earlier attempt. + const attempts = readdirSync(directory) + .map(name => /^release-slack-thread-(\d+)$/.exec(name)?.[1]) + .filter(value => value && Number(value) <= attempt) + .sort((a, b) => Number(b) - Number(a)); + if (attempts.length === 0) return undefined; + const timestamp = readFileSync(join(directory, `release-slack-thread-${attempts[0]}`, 'thread-ts.txt'), 'utf8').trim(); + return /^\d+\.\d+$/.test(timestamp) ? timestamp : undefined; + } catch (error) { + if (error.code !== 'ENOENT') console.error('Could not read Slack thread:', error.message); + return undefined; + } +} + +module.exports = async function releaseFailureNotification({ github, context, channel, threadDirectory }) { + const run = context.payload.workflow_run; + const threadTs = readThreadTimestamp(threadDirectory, run.run_attempt); + const destination = { channel, ...(threadTs ? { thread_ts: threadTs } : {}) }; + let jobs; + try { + jobs = await github.paginate(github.rest.actions.listJobsForWorkflowRunAttempt, { + ...context.repo, + run_id: run.id, + attempt_number: run.run_attempt, + per_page: 100, + }); + } catch (error) { + console.error('Failed to inspect release jobs:', error.status ?? error.name, error.message); + return { + ...destination, + text: [ + `⚠️ Could not inspect release jobs for \`${context.repo.owner}/${context.repo.repo}\` (attempt ${run.run_attempt}).`, + 'Please check this release manually; individual job results are unavailable.', + `<${run.html_url}|View release logs> (run: ${run.conclusion})`, + ].join('\n'), + unfurl_links: false, + unfurl_media: false, + }; + } + + if (jobs.length === 0 && run.conclusion === 'cancelled') { + return null; + } + + const failedJobs = jobs.filter(job => unsuccessful.has(job.conclusion)); + const versionBump = jobs.find(job => job.name === 'Bump versions and commit to main'); + if (run.conclusion === 'failure' && failedJobs.length === 1 + && versionBump?.conclusion === 'failure' && versionBump.steps?.length === 0 + && jobs.some(job => job.name === 'Notify Slack - Rejected' + && job.steps?.some(step => step.name === 'Notify Slack - Rejected' && step.conclusion === 'success'))) { + return null; + } + + const publishJobs = jobs.filter(job => isJob(job, publishJob)); + const gemsPublished = publishJobs.length > 0 && publishJobs.every(job => job.conclusion === 'success'); + const releasedNotification = jobs.find(job => job.name === 'Notify Slack - Released'); + const problems = failedJobs.map(job => `• <${job.html_url}|${job.name}>: ${job.conclusion}`); + + if (run.conclusion !== 'success' && failedJobs.length === 0 && publishJobs.length === 0) { + problems.push(`• ${publishJob}: never ran`); + } + + if (gemsPublished && (!releasedNotification || releasedNotification.conclusion === 'skipped')) { + problems.push('• The released Slack notification did not run.'); + } + + if (problems.length === 0 && run.conclusion === 'success') { + return null; + } + + return { + ...destination, + text: [ + gemsPublished + ? `⚠️ Gems published; follow-up problems for \`${context.repo.owner}/${context.repo.repo}\` (attempt ${run.run_attempt}).` + : `❌ Release process failed or was cancelled for \`${context.repo.owner}/${context.repo.repo}\` (attempt ${run.run_attempt}).`, + ...problems, + `<${run.html_url}|View release logs> (run: ${run.conclusion})`, + ].join('\n'), + unfurl_links: false, + unfurl_media: false, + }; +}; From a18b145330f9cb82f74b41b8962a06fa4ff762fa Mon Sep 17 00:00:00 2001 From: Ioannis J Date: Tue, 10 Nov 2026 15:21:55 +0200 Subject: [PATCH 2/5] ci: drop in-job Slack failure replies now covered by the notifier --- .github/workflows/release.yml | 22 +--------------------- RELEASING.md | 2 +- scripts/release-failure-notification.cjs | 10 ++++++++++ 3 files changed, 12 insertions(+), 22 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 788038fd..de18ebf7 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -224,16 +224,6 @@ jobs: "alertContext": "The version bump/changelog step failed before packages were published." } - - name: Notify Slack - Failed - if: ${{ failure() && needs.notify-approval-needed.outputs.slack_ts != '' }} - uses: posthog/.github/.github/actions/slack-thread-reply@cb0979b67dcd585828b61ac45927eef4da8f6287 # main - with: - slack_bot_token: ${{ secrets.SLACK_CLIENT_LIBRARIES_BOT_TOKEN }} - slack_channel_id: ${{ vars.SLACK_APPROVALS_CLIENT_LIBRARIES_CHANNEL_ID }} - thread_ts: ${{ needs.notify-approval-needed.outputs.slack_ts }} - message: "❌ Failed to bump package versions! " - emoji_reaction: "x" - notify-rejected: name: Notify Slack - Rejected needs: [version-bump, notify-approval-needed] @@ -274,7 +264,7 @@ jobs: emoji_reaction: "no_entry_sign" publish: - name: Release and publish + name: Release and publish (${{ matrix.package.name }}) needs: [version-bump, notify-approval-needed] runs-on: ubuntu-latest if: always() && needs.version-bump.outputs.committed == 'true' @@ -430,16 +420,6 @@ jobs: "alertContext": "The package publish step failed." } - - name: Notify Slack - Failed - if: ${{ failure() && needs.notify-approval-needed.outputs.slack_ts != '' }} - uses: posthog/.github/.github/actions/slack-thread-reply@cb0979b67dcd585828b61ac45927eef4da8f6287 # main - with: - slack_bot_token: ${{ secrets.SLACK_CLIENT_LIBRARIES_BOT_TOKEN }} - slack_channel_id: ${{ vars.SLACK_APPROVALS_CLIENT_LIBRARIES_CHANNEL_ID }} - thread_ts: ${{ needs.notify-approval-needed.outputs.slack_ts }} - message: "❌ Failed to release `${{ matrix.package.name }}@${{ steps.detect-version.outputs.version }}`! " - emoji_reaction: "x" - notify-released: name: Notify Slack - Released needs: [version-bump, publish, notify-approval-needed] diff --git a/RELEASING.md b/RELEASING.md index 3bce01fe..6da1cb88 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -62,4 +62,4 @@ Cancelled runs with no jobs are ignored, including superseded queued runs and ma Job inspection retries transient GitHub API failures up to three times. If inspection still fails, it logs the HTTP status or error type and message, then posts a fallback alert with the overall run status and logs link so the release can be checked manually. Error diagnostics stay in the run log, not in the Slack message. -Alerts reply to the approval thread in the release approvals Slack channel. The release workflow preserves the thread timestamp in a run artifact for 30 days, independently of release approval. Failed-job reruns reuse the latest available thread from the same or an earlier attempt. If the timestamp is missing, invalid, or unavailable, the alert falls back to a new channel message, so it does not depend on the approval notification succeeding. The failure message surfaces Slack API errors rather than silently ignoring them. The notification workflow must be present on the default branch to run; local tests do not send Slack messages. +Alerts reply to the approval thread in the release approvals Slack channel. The release workflow preserves the thread timestamp in a run artifact for 30 days, independently of release approval. Failed-job reruns reuse the latest available thread from the same or an earlier attempt. If the timestamp is missing, invalid, or unavailable, the alert falls back to a new channel message, so it does not depend on the approval notification succeeding. The failure message surfaces Slack API errors rather than silently ignoring them. The notification workflow must be present on the default branch to run. diff --git a/scripts/release-failure-notification.cjs b/scripts/release-failure-notification.cjs index 41bac248..33f56312 100644 --- a/scripts/release-failure-notification.cjs +++ b/scripts/release-failure-notification.cjs @@ -3,6 +3,7 @@ const { join } = require('node:path'); const unsuccessful = new Set(['failure', 'cancelled', 'timed_out', 'action_required', 'startup_failure', 'stale']); const publishJob = 'Release and publish'; +const gems = ['posthog-ruby', 'posthog-rails']; const isJob = (job, name) => job.name === name || job.name.startsWith(`${name} (`); @@ -67,6 +68,15 @@ module.exports = async function releaseFailureNotification({ github, context, ch const releasedNotification = jobs.find(job => job.name === 'Notify Slack - Released'); const problems = failedJobs.map(job => `• <${job.html_url}|${job.name}>: ${job.conclusion}`); + const publishedGems = publishJobs + .filter(job => job.conclusion === 'success') + .map(job => /\((.*)\)$/.exec(job.name)?.[1]) + .filter(gem => gems.includes(gem)); + if (!gemsPublished && publishedGems.length > 0) { + problems.push(`• Published: ${publishedGems.join(', ')}`); + problems.push(`• Not published: ${gems.filter(gem => !publishedGems.includes(gem)).join(', ')}`); + } + if (run.conclusion !== 'success' && failedJobs.length === 0 && publishJobs.length === 0) { problems.push(`• ${publishJob}: never ran`); } From 884bee8fec04c827ca400ef7efc2c3ae61178198 Mon Sep 17 00:00:00 2001 From: Ioannis J Date: Tue, 10 Nov 2026 16:28:12 +0200 Subject: [PATCH 3/5] ci: accept a lone thread artifact and gate the notifier on trusted runs --- .../workflows/release-failure-notification.yml | 3 +++ RELEASING.md | 2 +- scripts/release-failure-notification.cjs | 17 +++++++++++------ 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/.github/workflows/release-failure-notification.yml b/.github/workflows/release-failure-notification.yml index 856efac0..caec2e75 100644 --- a/.github/workflows/release-failure-notification.yml +++ b/.github/workflows/release-failure-notification.yml @@ -14,6 +14,9 @@ jobs: notify-failed: name: Notify Slack - Failed runs-on: ubuntu-latest + if: >- + github.event.workflow_run.head_repository.full_name == github.repository && + (github.event.workflow_run.event == 'push' || github.event.workflow_run.event == 'workflow_dispatch') timeout-minutes: 10 steps: # This workflow must survive cancellation or runner startup failure in Release. diff --git a/RELEASING.md b/RELEASING.md index 6da1cb88..4ab24d94 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -56,7 +56,7 @@ You can also manually trigger the release workflow from the Actions tab, as long ### Failed or cancelled release -The independent [release failure notification workflow](.github/workflows/release-failure-notification.yml) checks every completed `main` release attempt and alerts if a job failed, timed out, or was cancelled. This includes runner startup failures and cancellations with job records. It also alerts when the run did not succeed, no job failed, and the `Release and publish` job was never created, for example when GitHub Actions stops a run before the publish job starts. Once every `Release and publish` job succeeds, alerts describe follow-up problems rather than claiming the release failed, and the workflow alerts if the released Slack notification was skipped. +The independent [release failure notification workflow](.github/workflows/release-failure-notification.yml) checks every completed `main` release attempt and alerts if a job failed, timed out, or was cancelled. This includes runner startup failures and cancellations with job records. It also alerts when the run did not succeed, no job failed, and the `Release and publish` job was never created, for example when GitHub Actions stops a run before the publish job starts. Once every `Release and publish` job succeeds, alerts describe follow-up problems rather than claiming the release failed, and the workflow alerts if the released Slack notification was skipped. It only acts on runs that a push or a manual dispatch started in this repository. Cancelled runs with no jobs are ignored, including superseded queued runs and manual cancellations before any job is created. A version-bump failure before any steps ran is also ignored when the rejection-reply step confirms an explicit rejection in the same attempt and no other job failed. diff --git a/scripts/release-failure-notification.cjs b/scripts/release-failure-notification.cjs index 33f56312..dac4d6c6 100644 --- a/scripts/release-failure-notification.cjs +++ b/scripts/release-failure-notification.cjs @@ -10,13 +10,18 @@ const isJob = (job, name) => job.name === name || job.name.startsWith(`${name} ( function readThreadTimestamp(directory, attempt) { if (!directory) return undefined; try { + const entries = readdirSync(directory); + // A single matching artifact is extracted straight into the directory. // Failed-job reruns reuse the approval thread from the latest earlier attempt. - const attempts = readdirSync(directory) - .map(name => /^release-slack-thread-(\d+)$/.exec(name)?.[1]) - .filter(value => value && Number(value) <= attempt) - .sort((a, b) => Number(b) - Number(a)); - if (attempts.length === 0) return undefined; - const timestamp = readFileSync(join(directory, `release-slack-thread-${attempts[0]}`, 'thread-ts.txt'), 'utf8').trim(); + const files = entries.includes('thread-ts.txt') + ? [join(directory, 'thread-ts.txt')] + : entries + .map(name => /^release-slack-thread-(\d+)$/.exec(name)?.[1]) + .filter(value => value && Number(value) <= attempt) + .sort((a, b) => Number(b) - Number(a)) + .map(value => join(directory, `release-slack-thread-${value}`, 'thread-ts.txt')); + if (files.length === 0) return undefined; + const timestamp = readFileSync(files[0], 'utf8').trim(); return /^\d+\.\d+$/.test(timestamp) ? timestamp : undefined; } catch (error) { if (error.code !== 'ENOENT') console.error('Could not read Slack thread:', error.message); From 8854532358798c2a92c8a17cd8aec3c992a94814 Mon Sep 17 00:00:00 2001 From: Ioannis J Date: Tue, 10 Nov 2026 16:28:13 +0200 Subject: [PATCH 4/5] ci: judge gem publication by the publish step, not the whole job --- RELEASING.md | 2 +- scripts/release-failure-notification.cjs | 9 ++++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/RELEASING.md b/RELEASING.md index 4ab24d94..f836d16c 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -56,7 +56,7 @@ You can also manually trigger the release workflow from the Actions tab, as long ### Failed or cancelled release -The independent [release failure notification workflow](.github/workflows/release-failure-notification.yml) checks every completed `main` release attempt and alerts if a job failed, timed out, or was cancelled. This includes runner startup failures and cancellations with job records. It also alerts when the run did not succeed, no job failed, and the `Release and publish` job was never created, for example when GitHub Actions stops a run before the publish job starts. Once every `Release and publish` job succeeds, alerts describe follow-up problems rather than claiming the release failed, and the workflow alerts if the released Slack notification was skipped. It only acts on runs that a push or a manual dispatch started in this repository. +The independent [release failure notification workflow](.github/workflows/release-failure-notification.yml) checks every completed `main` release attempt and alerts if a job failed, timed out, or was cancelled. This includes runner startup failures and cancellations with job records. It also alerts when the run did not succeed, no job failed, and the `Release and publish` job was never created, for example when GitHub Actions stops a run before the publish job starts. Once the publish step of every `Release and publish` job succeeds, alerts describe follow-up problems rather than claiming the release failed, and the workflow alerts if the released Slack notification was skipped. It only acts on runs that a push or a manual dispatch started in this repository. Cancelled runs with no jobs are ignored, including superseded queued runs and manual cancellations before any job is created. A version-bump failure before any steps ran is also ignored when the rejection-reply step confirms an explicit rejection in the same attempt and no other job failed. diff --git a/scripts/release-failure-notification.cjs b/scripts/release-failure-notification.cjs index dac4d6c6..7ae59ffb 100644 --- a/scripts/release-failure-notification.cjs +++ b/scripts/release-failure-notification.cjs @@ -3,9 +3,12 @@ const { join } = require('node:path'); const unsuccessful = new Set(['failure', 'cancelled', 'timed_out', 'action_required', 'startup_failure', 'stale']); const publishJob = 'Release and publish'; +const publishStep = 'Publish'; const gems = ['posthog-ruby', 'posthog-rails']; const isJob = (job, name) => job.name === name || job.name.startsWith(`${name} (`); +const gemOf = job => /\((.*)\)$/.exec(job.name)?.[1]; +const didPublish = job => job.steps?.some(step => step.name === `${publishStep} ${gemOf(job)}` && step.conclusion === 'success') ?? false; function readThreadTimestamp(directory, attempt) { if (!directory) return undefined; @@ -69,13 +72,13 @@ module.exports = async function releaseFailureNotification({ github, context, ch } const publishJobs = jobs.filter(job => isJob(job, publishJob)); - const gemsPublished = publishJobs.length > 0 && publishJobs.every(job => job.conclusion === 'success'); + const gemsPublished = publishJobs.length > 0 && publishJobs.every(didPublish); const releasedNotification = jobs.find(job => job.name === 'Notify Slack - Released'); const problems = failedJobs.map(job => `• <${job.html_url}|${job.name}>: ${job.conclusion}`); const publishedGems = publishJobs - .filter(job => job.conclusion === 'success') - .map(job => /\((.*)\)$/.exec(job.name)?.[1]) + .filter(didPublish) + .map(gemOf) .filter(gem => gems.includes(gem)); if (!gemsPublished && publishedGems.length > 0) { problems.push(`• Published: ${publishedGems.join(', ')}`); From 8b520ff8d8ab90118357c7cf56c1fe45c773d27d Mon Sep 17 00:00:00 2001 From: Ioannis J Date: Tue, 10 Nov 2026 16:30:08 +0200 Subject: [PATCH 5/5] ci: treat a skipped gem publish step as not released, not failed --- RELEASING.md | 2 +- scripts/release-failure-notification.cjs | 16 +++++++++------- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/RELEASING.md b/RELEASING.md index f836d16c..8649bfb6 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -56,7 +56,7 @@ You can also manually trigger the release workflow from the Actions tab, as long ### Failed or cancelled release -The independent [release failure notification workflow](.github/workflows/release-failure-notification.yml) checks every completed `main` release attempt and alerts if a job failed, timed out, or was cancelled. This includes runner startup failures and cancellations with job records. It also alerts when the run did not succeed, no job failed, and the `Release and publish` job was never created, for example when GitHub Actions stops a run before the publish job starts. Once the publish step of every `Release and publish` job succeeds, alerts describe follow-up problems rather than claiming the release failed, and the workflow alerts if the released Slack notification was skipped. It only acts on runs that a push or a manual dispatch started in this repository. +The independent [release failure notification workflow](.github/workflows/release-failure-notification.yml) checks every completed `main` release attempt and alerts if a job failed, timed out, or was cancelled. This includes runner startup failures and cancellations with job records. It also alerts when the run did not succeed, no job failed, and the `Release and publish` job was never created, for example when GitHub Actions stops a run before the publish job starts. Once every `Release and publish` job has published its gem or skipped it for lack of a version bump, alerts describe follow-up problems rather than claiming the release failed, and the workflow alerts if the released Slack notification was skipped. It only acts on runs that a push or a manual dispatch started in this repository. Cancelled runs with no jobs are ignored, including superseded queued runs and manual cancellations before any job is created. A version-bump failure before any steps ran is also ignored when the rejection-reply step confirms an explicit rejection in the same attempt and no other job failed. diff --git a/scripts/release-failure-notification.cjs b/scripts/release-failure-notification.cjs index 7ae59ffb..9cc43e54 100644 --- a/scripts/release-failure-notification.cjs +++ b/scripts/release-failure-notification.cjs @@ -8,7 +8,10 @@ const gems = ['posthog-ruby', 'posthog-rails']; const isJob = (job, name) => job.name === name || job.name.startsWith(`${name} (`); const gemOf = job => /\((.*)\)$/.exec(job.name)?.[1]; -const didPublish = job => job.steps?.some(step => step.name === `${publishStep} ${gemOf(job)}` && step.conclusion === 'success') ?? false; +const publishOutcome = job => job.steps?.find(step => step.name === `${publishStep} ${gemOf(job)}`)?.conclusion; +const didPublish = job => publishOutcome(job) === 'success'; +// A gem without a version bump skips its publish step; that is not a failure. +const publishSettled = job => ['success', 'skipped'].includes(publishOutcome(job)); function readThreadTimestamp(directory, attempt) { if (!directory) return undefined; @@ -72,17 +75,16 @@ module.exports = async function releaseFailureNotification({ github, context, ch } const publishJobs = jobs.filter(job => isJob(job, publishJob)); - const gemsPublished = publishJobs.length > 0 && publishJobs.every(didPublish); + const gemsPublished = publishJobs.some(didPublish) && publishJobs.every(publishSettled); const releasedNotification = jobs.find(job => job.name === 'Notify Slack - Released'); const problems = failedJobs.map(job => `• <${job.html_url}|${job.name}>: ${job.conclusion}`); - const publishedGems = publishJobs - .filter(didPublish) - .map(gemOf) - .filter(gem => gems.includes(gem)); + const gemsIn = predicate => publishJobs.filter(predicate).map(gemOf).filter(gem => gems.includes(gem)); + const publishedGems = gemsIn(didPublish); + const unpublishedGems = gemsIn(job => !publishSettled(job)); if (!gemsPublished && publishedGems.length > 0) { problems.push(`• Published: ${publishedGems.join(', ')}`); - problems.push(`• Not published: ${gems.filter(gem => !publishedGems.includes(gem)).join(', ')}`); + problems.push(`• Not published: ${unpublishedGems.join(', ')}`); } if (run.conclusion !== 'success' && failedJobs.length === 0 && publishJobs.length === 0) {