From 715ee08617ff8f384afd897a26e15986a27ccd8b Mon Sep 17 00:00:00 2001 From: vmelikyan Date: Sun, 11 Oct 2026 12:54:31 -0700 Subject: [PATCH 1/2] fix(reconciliation): deploy each service of an environment independently An environment was reconciled by one job that owned one generation counter and one run token for every service. A merge to service B raised the counter, so the run for service A lost its authority: its build, its after-build step and its rollout were discarded and the next pass redid them. In a static environment with hundreds of services, a steady flow of merges kept the slowest services from ever landing. A configuration error in one service marked the whole generation observed, so services that had not run were recorded as done. Every deploy row now carries its own desired and observed generation (migration 037). Each accepted change is one intent and one queue job, and a job claims only the rows its intent selects. A run owns a row only while the row carries the run token at the run generation, is not yet observed, and the environment is not blocked. A newer accept re-stamps the rows of its own scope, so the older run of that scope stops without publishing, while runs for other services continue. Accepted push SHAs are floors for full passes. Prerequisites order services inside one run only; a service never waits for, or fails because of, a run of another service. The environment status is derived from the rows. A root configuration error is recorded on the environment by any intent and cleared by the next import that succeeds. A service with a broken lifecycle.yaml fails alone and blocks only its dependents. Intents finish and move the environment watermark in one transaction under the Build row lock, and the sweep repairs a stale watermark instead of replaying a finished generation. A teardown closes the work accepted before it and keeps a deploy request accepted during it. Every renewable deployment lock uses a 60-second lease renewed every 20 seconds, so a worker that dies frees its locks within a minute. Before a service is applied, the run waits for an earlier deploy job of that service to be terminal with its pods stopped, with a 45-minute limit. A lost promotion lease lets launched applies settle before the queue retries the run. An Aurora restore keeps an endpoint it cannot verify and restores again only when the database is confirmed gone. Migration 037 marks intents at or below the old watermark as finished, so nothing replays after the upgrade. --- ...7_add_deploy_reconciliation_generations.ts | 63 + .../__tests__/apiEnvironmentJobWiring.test.ts | 4 +- src/server/jobs/index.ts | 2 +- .../deploymentManager/deploymentManager.ts | 365 +++- .../__tests__/authority.test.ts | 177 ++ .../__tests__/mailbox.test.ts | 218 +- .../lib/deploymentReconciliation/authority.ts | 111 + .../lib/deploymentReconciliation/mailbox.ts | 104 +- .../lib/helm/__tests__/helm.behavior.test.ts | 32 +- src/server/lib/helm/helm.ts | 11 +- .../lib/kubernetesApply/applyManifest.test.ts | 90 +- .../lib/kubernetesApply/applyManifest.ts | 32 + .../__tests__/helm.behavior.test.ts | 39 +- src/server/lib/nativeHelm/helm.ts | 23 +- .../lib/tests/deploymentManager.test.ts | 528 ++++- .../mcp/__tests__/deployRequestState.test.ts | 85 + .../mcp/tools/core/waitForEnvironment.ts | 22 +- src/server/models/Deploy.ts | 2 + .../services/__tests__/apiEnvironment.test.ts | 18 +- src/server/services/__tests__/build.test.ts | 1783 +++++------------ src/server/services/__tests__/deploy.test.ts | 106 +- .../__tests__/deployableSourceSeam.test.ts | 73 + .../__tests__/github.behavior.test.ts | 31 +- src/server/services/__tests__/github.test.ts | 4 +- .../__tests__/githubAutoTrack.test.ts | 11 +- src/server/services/__tests__/ingress.test.ts | 52 +- .../__tests__/serviceReconciliation.test.ts | 1663 +++++++++++++++ src/server/services/build.ts | 1333 ++++++++---- src/server/services/deploy.ts | 154 +- src/server/services/deployable.ts | 35 +- src/server/services/github.ts | 54 +- src/server/services/ingress.ts | 82 +- src/shared/config.ts | 1 + 33 files changed, 5259 insertions(+), 2049 deletions(-) create mode 100644 src/server/db/migrations/037_add_deploy_reconciliation_generations.ts create mode 100644 src/server/lib/deploymentReconciliation/__tests__/authority.test.ts create mode 100644 src/server/lib/deploymentReconciliation/authority.ts create mode 100644 src/server/mcp/__tests__/deployRequestState.test.ts create mode 100644 src/server/services/__tests__/serviceReconciliation.test.ts diff --git a/src/server/db/migrations/037_add_deploy_reconciliation_generations.ts b/src/server/db/migrations/037_add_deploy_reconciliation_generations.ts new file mode 100644 index 00000000..20c4ea12 --- /dev/null +++ b/src/server/db/migrations/037_add_deploy_reconciliation_generations.ts @@ -0,0 +1,63 @@ +/** + * Copyright 2026 GoodRx, Inc. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import type { Knex } from 'knex'; + +const PENDING_INDEX = 'deploys_reconcile_pending'; + +// PostgreSQL concurrent index operations cannot run inside Knex's migration transaction. +export const config = { transaction: false }; + +// Runs outside a transaction, so every step is guarded to make a retry after a failed concurrent index build safe. +export async function up(knex: Knex): Promise { + for (const column of ['desiredGeneration', 'observedGeneration']) { + if (await knex.schema.hasColumn('deploys', column)) continue; + await knex.schema.alterTable('deploys', (table) => { + table.bigInteger(column).notNullable().defaultTo(0); + }); + } + + // Intents at or below the old watermark are finished; marking them keeps the sweep from replaying them. + await knex.raw(` + update builds + set "acceptedRefs" = ( + select coalesce(jsonb_object_agg( + entry.key, + case + when jsonb_typeof(entry.value -> 'gen') <> 'number' then entry.value + when (entry.value ->> 'gen')::numeric <= builds."observedGeneration" + then entry.value || jsonb_build_object('observedGen', entry.value -> 'gen') + else entry.value + end + ), '{}'::jsonb) + from jsonb_each("acceptedRefs") as entry + ) + where "acceptedRefs" <> '{}'::jsonb + and "deletedAt" is null + `); + + // A failed concurrent build leaves an invalid index with this name behind. + await knex.raw('drop index concurrently if exists ??', [PENDING_INDEX]); + await knex.raw( + `create index concurrently ?? on deploys ("buildId") where "desiredGeneration" > "observedGeneration"`, + [PENDING_INDEX] + ); +} + +export async function down(knex: Knex): Promise { + await knex.raw('drop index concurrently if exists ??', [PENDING_INDEX]); + + await knex.schema.alterTable('deploys', (table) => { + table.dropColumn('observedGeneration'); + table.dropColumn('desiredGeneration'); + }); +} diff --git a/src/server/jobs/__tests__/apiEnvironmentJobWiring.test.ts b/src/server/jobs/__tests__/apiEnvironmentJobWiring.test.ts index 73fa7c29..c1d4bb73 100644 --- a/src/server/jobs/__tests__/apiEnvironmentJobWiring.test.ts +++ b/src/server/jobs/__tests__/apiEnvironmentJobWiring.test.ts @@ -173,7 +173,7 @@ describe('bootstrapJobs', () => { const createWorker = registeredWorkers.find((worker) => worker.queue === QUEUE_NAMES.API_ENV_CREATE); const expiryWorker = registeredWorkers.find((worker) => worker.queue === QUEUE_NAMES.API_ENV_EXPIRY); const reconciliationWorker = registeredWorkers.find( - (worker) => worker.queue === QUEUE_NAMES.DEPLOYMENT_RECONCILIATION + (worker) => worker.queue === QUEUE_NAMES.SERVICE_RECONCILIATION ); expect(createWorker).toMatchObject({ handler: services.BuildService.processApiEnvironmentCreateQueue, @@ -181,6 +181,8 @@ describe('bootstrapJobs', () => { }); expect(expiryWorker?.handler).toBe(services.BuildService.processApiEnvironmentExpiryQueue); expect(reconciliationWorker?.handler).toBe(services.BuildService.processDeploymentReconciliationQueue); + // Old-queue jobs are drained by old workers only; leftovers are recovered by the sweep, never by a second model. + expect(registeredWorkers.some((worker) => worker.queue === QUEUE_NAMES.DEPLOYMENT_RECONCILIATION)).toBe(false); expect(workerOnHandlers[QUEUE_NAMES.API_ENV_CREATE]?.failed).toBe( services.BuildService.handleApiEnvironmentCreateFailure ); diff --git a/src/server/jobs/index.ts b/src/server/jobs/index.ts index 34e27997..41fc686e 100644 --- a/src/server/jobs/index.ts +++ b/src/server/jobs/index.ts @@ -166,7 +166,7 @@ export default function bootstrapJobs(services: IServices) { }); queueManager.registerWorker( - QUEUE_NAMES.DEPLOYMENT_RECONCILIATION, + QUEUE_NAMES.SERVICE_RECONCILIATION, services.BuildService.processDeploymentReconciliationQueue, { connection: redisClient.getConnection(), diff --git a/src/server/lib/deploymentManager/deploymentManager.ts b/src/server/lib/deploymentManager/deploymentManager.ts index da3d53da..5786b71b 100644 --- a/src/server/lib/deploymentManager/deploymentManager.ts +++ b/src/server/lib/deploymentManager/deploymentManager.ts @@ -18,7 +18,11 @@ import { Deploy } from 'server/models'; import { deployHelm } from '../helm'; import { shouldUseNativeHelm } from '../nativeHelm'; import { DeployStatus, DeployTypes, CLIDeployTypes } from 'shared/constants'; -import { createKubernetesApplyJob, monitorKubernetesJob } from '../kubernetesApply/applyManifest'; +import { + listUnfinishedDeployJobs, + createKubernetesApplyJob, + monitorKubernetesJob, +} from '../kubernetesApply/applyManifest'; import { nanoid, customAlphabet } from 'nanoid'; import DeployService from 'server/services/deploy'; import { getLogger, withLogContext } from 'server/lib/logger'; @@ -28,15 +32,20 @@ import { buildDeployJobName } from '../kubernetes/jobNames'; import GlobalConfigService from 'server/services/globalConfig'; import { getLogArchivalService } from 'server/services/logArchival'; import { DeploymentSupersededError } from 'server/lib/deploymentReconciliation/errors'; +import { AuthorityLockLostError } from 'server/lib/authorityLock'; import type { HelmSecretMutationGate } from 'server/lib/nativeHelm/helm'; export { DeploymentSupersededError } from 'server/lib/deploymentReconciliation/errors'; const generateJobId = customAlphabet('abcdefghijklmnopqrstuvwxyz0123456789', 6); +const EARLIER_DEPLOY_POLL_MS = 5_000; +// A native Helm Job lives at most its 30-minute deadline plus a 5-minute grace period. +const EARLIER_DEPLOY_WAIT_MS = 45 * 60_000; +const UNSTARTED_DEPLOY_MESSAGE = 'Deployment failed. Redeploy the service to try again.'; export type NativeMutationGateResult = { admitted: true; value: T } | { admitted: false }; -export type NativeMutationGate = (action: () => Promise) => Promise>; +export type NativeMutationGate = (deploy: Deploy, action: () => Promise) => Promise>; export interface DeploymentManagerOptions { /** Returns false once this deployment generation is no longer authoritative. */ @@ -48,8 +57,14 @@ export interface DeploymentManagerOptions { nativeMutationGate?: NativeMutationGate; /** Serializes only writers of one Deploy's ExternalSecret resources. */ nativeSecretMutationGate?: HelmSecretMutationGate; + /** Services that already failed before rollout; they stay in the plan so their dependents are blocked. */ + failedServices?: string[]; + /** Rows re-stamped to a newer generation are no longer this run's, even before the successor claims them. */ + expectedGeneration?: number; } +type ServiceOutcome = 'ready' | 'failed' | 'superseded'; + interface KubernetesDeploymentContext { deploy: Deploy; deployService: DeployService; @@ -59,16 +74,24 @@ interface KubernetesDeploymentContext { export class DeploymentManager { private deploys: Map = new Map(); + // Declared dependencies as configured; leveling consumes the lists on the deployables themselves. + private dependencies: Map = new Map(); private deploymentLevels: Map = new Map(); // Deploys never placed in a level: members of a dependency cycle, or dependents of one. private unresolvedDeploys: Deploy[] = []; private dependencyCycleDescription = ''; private readonly options: DeploymentManagerOptions; + // The first error a launch threw: nothing launches after it, and deploy() rethrows it once every launch has settled. + private abort?: { error: unknown }; constructor(deploys: Deploy[], options: DeploymentManagerOptions = {}) { this.options = options; deploys.forEach((deploy) => { this.deploys.set(deploy.deployable.name, deploy); + this.dependencies.set( + deploy.deployable.name, + (deploy.deployable.deploymentDependsOn ?? []).filter((dependency) => dependency !== deploy.deployable.name) + ); }); this.calculateDeploymentOrder(); @@ -165,13 +188,21 @@ export class DeploymentManager { }); } - public async deploy(): Promise { + public async deploy(): Promise<{ failed: Deploy[] }> { await this.assertCurrent(); const unresolved = new Set(this.unresolvedDeploys); + const superseded = new Set(); + // Keyed by name so a failed prerequisite that never reaches this manager (a Codefresh service) still blocks dependents. + const failed = new Map(); + for (const name of this.options.failedServices ?? []) { + failed.set(name, this.deploys.get(name) ?? null); + } for (const value of this.deploys.values()) { - if (!unresolved.has(value)) { - await this.patchDeployIfCurrent(value, { status: DeployStatus.QUEUED }); + if (unresolved.has(value) || failed.has(value.deployable.name)) continue; + if (!(await this.tryPatchDeploy(value, { status: DeployStatus.QUEUED }))) { + superseded.add(value.deployable.name); + getLogger().info(`Deploy: ${value.deployable.name} skipped reason=superseded`); } } @@ -181,91 +212,193 @@ export class DeploymentManager { const statusMessage = `Dependency cycle detected: ${this.dependencyCycleDescription}; deploy order cannot be resolved`; for (const deploy of this.unresolvedDeploys) { getLogger().error(`Deploy: ${deploy.deployable.name} failed — ${statusMessage}`); - await this.patchDeployIfCurrent(deploy, { status: DeployStatus.DEPLOY_FAILED, statusMessage }); + await this.tryPatchDeploy(deploy, { status: DeployStatus.DEPLOY_FAILED, statusMessage }); + failed.set(deploy.deployable.name, deploy); } } - for (let level = 0; level < this.deploymentLevels.size; level++) { - await this.assertCurrent(); + // Each service launches as soon as its own prerequisites are done, so a slow service never holds back unrelated work. + const levelOf = new Map(); + this.deploymentLevels.forEach((deploys, level) => deploys.forEach((d) => levelOf.set(d.deployable.name, level))); + const outcomes = new Map>(); + const schedule = (deploy: Deploy): Promise => { + const name = deploy.deployable.name; + let pending = outcomes.get(name); + if (!pending) { + pending = this.launchWhenPrerequisitesDone(deploy, levelOf.get(name) ?? 0, failed, superseded, schedule).catch( + (error) => { + if (!this.abort) this.abort = { error }; + throw error; + } + ); + outcomes.set(name, pending); + } + return pending; + }; + // Work still in flight settles before the error surfaces, so a queue retry never overlaps it. + await Promise.allSettled(Array.from(this.deploys.values()).map((deploy) => schedule(deploy))); + this.assertNotAborted(); - const deploysAtLevel = this.deploymentLevels.get(level); - if (deploysAtLevel) { - const helmDeploys = deploysAtLevel.filter((d) => this.shouldDeployWithHelm(d)); - const githubDeploys = deploysAtLevel.filter((d) => this.shouldDeployWithKubernetes(d)); + return { failed: Array.from(failed.values()).filter((deploy): deploy is Deploy => deploy != null) }; + } - const helmMethods = await Promise.all( - helmDeploys.map(async (deploy) => ({ deploy, native: await shouldUseNativeHelm(deploy) })) - ); - await this.assertCurrent(); + private async launchWhenPrerequisitesDone( + deploy: Deploy, + level: number, + failed: Map, + superseded: Set, + outcomeOf: (prerequisite: Deploy) => Promise + ): Promise { + const name = deploy.deployable.name; + if (superseded.has(name)) return 'superseded'; + if (failed.has(name)) return 'failed'; + + for (const dependency of this.dependencies.get(name) ?? []) { + const inPlan = this.deploys.get(dependency); + // A prerequisite outside this run, or one a newer run took, is never waited on: its running version serves. + let outcome: ServiceOutcome = 'ready'; + if (failed.has(dependency)) outcome = 'failed'; + else if (inPlan && !superseded.has(dependency)) outcome = await outcomeOf(inPlan); + this.assertNotAborted(); + if (outcome !== 'failed') continue; + if (!(await this.claimForLaunch(deploy, superseded))) return 'superseded'; + const statusMessage = `Not deployed: ${dependency} failed.`; + getLogger().warn(`Deploy: ${name} skipped — ${statusMessage}`); + await this.tryPatchDeploy(deploy, { status: DeployStatus.DEPLOY_FAILED, statusMessage }); + failed.set(name, deploy); + return 'failed'; + } - const nativeHelmDeploys = helmMethods.filter(({ native }) => native).map(({ deploy }) => deploy); - const codefreshHelmDeploys = helmMethods.filter(({ native }) => !native).map(({ deploy }) => deploy); + this.assertNotAborted(); + if (!(await this.claimForLaunch(deploy, superseded))) return 'superseded'; + await this.launchLevel(level, [deploy], failed, superseded); + if (failed.has(name)) return 'failed'; + if (superseded.has(name)) return 'superseded'; + return 'ready'; + } - const nativeHelmServices = nativeHelmDeploys.map((d) => d.deployable.name).join(','); - const codefreshHelmServices = codefreshHelmDeploys.map((d) => d.deployable.name).join(','); - const k8sServices = githubDeploys.map((d) => d.deployable.name).join(','); - getLogger().info( - `Deploy: level ${level} nativeHelm=[${nativeHelmServices}] codefreshHelm=[${codefreshHelmServices}] k8s=[${k8sServices}]` - ); + /** A newer intent may have taken this row while earlier levels ran; nothing launches for a row we no longer own. */ + private async claimForLaunch(deploy: Deploy, superseded: Set): Promise { + if (await this.tryPatchDeploy(deploy, { status: DeployStatus.QUEUED })) return true; + superseded.add(deploy.deployable.name); + getLogger().info(`Deploy: ${deploy.deployable.name} skipped reason=superseded`); + return false; + } - // Codefresh is intentionally not part of native mutation admission. It - // may continue in the provider while a newer generation starts. - const codefreshDeploy = codefreshHelmDeploys.length > 0 ? deployHelm(codefreshHelmDeploys) : Promise.resolve(); - // Attach a handler immediately because supersession may deliberately - // detach this provider wait before we await it below. - void codefreshDeploy.catch(() => undefined); - - let kubernetesDeployments: KubernetesDeploymentContext[] = []; - let nativeFailure: unknown; - try { - if (nativeHelmDeploys.length > 0 || githubDeploys.length > 0) { - const nativeResult = await this.runNativeMutation(async () => { - // Run same-generation siblings in parallel, but do not release the - // promotion gate until every admitted native mutation is terminal. - const settled = await Promise.allSettled([ - ...nativeHelmDeploys.map((deploy) => - deployHelm([deploy], { secretMutationGate: this.options.nativeSecretMutationGate }) - ), - ...githubDeploys.map((deploy) => this.applyManifests(deploy)), - ]); - const failure = settled.find((result): result is PromiseRejectedResult => result.status === 'rejected'); - const successfulKubernetesDeployments = settled - .slice(nativeHelmDeploys.length) - .filter( - (result): result is PromiseFulfilledResult => - result.status === 'fulfilled' - ) - .map((result) => result.value); - return { kubernetesDeployments: successfulKubernetesDeployments, failure: failure?.reason }; - }); - kubernetesDeployments = nativeResult.kubernetesDeployments; - nativeFailure = nativeResult.failure; - } + private assertNotAborted(): void { + if (this.abort) throw this.abort.error; + } - await this.assertCurrent(); + private async launchLevel( + level: number, + runnable: Deploy[], + failed: Map, + superseded: Set + ): Promise { + if (runnable.length === 0) return; - // Pod readiness is observational and must not hold the native mutation - // gate. Settle every successful sibling even when another native - // mutation failed so no Deploy is left indefinitely at DEPLOYING. - const readiness = await Promise.allSettled( - kubernetesDeployments.map((deployment) => this.waitForManifestReadiness(deployment)) - ); - if (nativeFailure) throw nativeFailure; - const readinessFailure = readiness.find( - (result): result is PromiseRejectedResult => result.status === 'rejected' - ); - if (readinessFailure) throw readinessFailure.reason; - await codefreshDeploy; - } catch (error) { - if (error instanceof DeploymentSupersededError) { - getLogger().info(`Deploy: level ${level} stopped reason=superseded`); - } + const helmDeploys = runnable.filter((d) => this.shouldDeployWithHelm(d)); + const githubDeploys = runnable.filter((d) => this.shouldDeployWithKubernetes(d)); + + const helmMethods = await Promise.all( + helmDeploys.map(async (deploy) => ({ deploy, native: await shouldUseNativeHelm(deploy) })) + ); + await this.assertCurrent(); + + const nativeHelmDeploys = helmMethods.filter(({ native }) => native).map(({ deploy }) => deploy); + const codefreshHelmDeploys = helmMethods.filter(({ native }) => !native).map(({ deploy }) => deploy); + + const nativeHelmServices = nativeHelmDeploys.map((d) => d.deployable.name).join(','); + const codefreshHelmServices = codefreshHelmDeploys.map((d) => d.deployable.name).join(','); + const k8sServices = githubDeploys.map((d) => d.deployable.name).join(','); + getLogger().info( + `Deploy: level ${level} nativeHelm=[${nativeHelmServices}] codefreshHelm=[${codefreshHelmServices}] k8s=[${k8sServices}]` + ); + + // Codefresh is intentionally not part of native mutation admission. It + // may continue in the provider while a newer generation starts. Each + // service is launched on its own so one failure is attributed to one row. + const codefreshOutcomes = Promise.allSettled( + codefreshHelmDeploys.map(async (deploy) => { + // Provider launches sit outside the promotion gate; ownership is re-checked right before handing off. + const owned = await this.tryPatchDeploy(deploy, { status: DeployStatus.QUEUED }).catch(async (error) => { + await this.recordOwnershipCheckFailure(deploy, error); throw error; - } + }); + if (!owned) throw new DeploymentSupersededError(); + return deployHelm([deploy], { + providerSubmissionGate: (candidate) => this.tryPatchDeploy(candidate, { status: DeployStatus.DEPLOYING }), + }); + }) + ); + + const nativeDeploys = [...nativeHelmDeploys, ...githubDeploys]; + const nativeOutcomes = await Promise.allSettled([ + ...nativeHelmDeploys.map((deploy) => + this.runNativeMutation(deploy, async () => { + await this.waitForEarlierDeployJobs(deploy); + return deployHelm([deploy], { secretMutationGate: this.options.nativeSecretMutationGate }); + }) + ), + ...githubDeploys.map((deploy) => + this.runNativeMutation(deploy, async () => { + await this.waitForEarlierDeployJobs(deploy); + return this.applyManifests(deploy); + }) + ), + ]); + await this.assertCurrent(); + + // A lost promotion lease is not a service failure: the apply may have landed, so the run must retry. + const lostLease = nativeOutcomes.find( + (outcome): outcome is PromiseRejectedResult => + outcome.status === 'rejected' && outcome.reason instanceof AuthorityLockLostError + ); + if (lostLease) throw lostLease.reason; + + const kubernetesDeployments: KubernetesDeploymentContext[] = []; + nativeOutcomes.forEach((outcome, index) => { + const deploy = nativeDeploys[index]; + if (outcome.status === 'fulfilled') { + if (index >= nativeHelmDeploys.length) kubernetesDeployments.push(outcome.value as KubernetesDeploymentContext); + return; + } + if (outcome.reason instanceof DeploymentSupersededError) { + superseded.add(deploy.deployable.name); + getLogger().info(`Deploy: ${deploy.deployable.name} stopped reason=superseded`); + return; } + failed.set(deploy.deployable.name, deploy); + }); - await this.assertCurrent(); - } + // Pod readiness is observational and must not hold a promotion gate. + const readiness = await Promise.allSettled( + kubernetesDeployments.map((deployment) => this.waitForManifestReadiness(deployment)) + ); + readiness.forEach((outcome, index) => { + if (outcome.status === 'fulfilled') return; + const { deploy } = kubernetesDeployments[index]; + if (outcome.reason instanceof DeploymentSupersededError) { + superseded.add(deploy.deployable.name); + getLogger().info(`Deploy: ${deploy.deployable.name} stopped reason=superseded`); + return; + } + failed.set(deploy.deployable.name, deploy); + }); + + (await codefreshOutcomes).forEach((outcome, index) => { + if (outcome.status === 'fulfilled') return; + const deploy = codefreshHelmDeploys[index]; + if (outcome.reason instanceof DeploymentSupersededError) { + superseded.add(deploy.deployable.name); + getLogger().info(`Deploy: ${deploy.deployable.name} codefresh stopped reason=superseded`); + return; + } + getLogger().error({ error: outcome.reason }, `Deploy: ${deploy.deployable.name} codefresh helm failed`); + failed.set(deploy.deployable.name, deploy); + }); + + await this.assertCurrent(); } private async assertCurrent(): Promise { @@ -274,24 +407,90 @@ export class DeploymentManager { } } - private async runNativeMutation(action: () => Promise): Promise { + private async runNativeMutation(deploy: Deploy, action: () => Promise): Promise { if (!this.options.nativeMutationGate) return action(); - const result = await this.options.nativeMutationGate(action); + let actionFailed = false; + const result = await this.options + .nativeMutationGate(deploy, () => + action().catch((error) => { + actionFailed = true; + throw error; + }) + ) + .catch(async (error) => { + if (!actionFailed) await this.recordOwnershipCheckFailure(deploy, error); + throw error; + }); if (!result.admitted) throw new DeploymentSupersededError(); return result.value; } - private async patchDeployIfCurrent(deploy: Deploy, patch: Partial): Promise { + /** Launch paths record their own failures; a failed ownership check around one must not leave the row looking deployed. */ + private async recordOwnershipCheckFailure(deploy: Deploy, error: unknown): Promise { + if (error instanceof DeploymentSupersededError || error instanceof AuthorityLockLostError) return; + getLogger({ error }).warn(`Deploy: ${deploy.deployable.name} ownership check failed`); + await new DeployService().recordDeployFailure(deploy, deploy.runUUID, { + status: DeployStatus.DEPLOY_FAILED, + fallbackMessage: UNSTARTED_DEPLOY_MESSAGE, + }); + } + + /** A deploy Job outlives the worker that started it; killing it cannot prove it stopped, so the next one waits. */ + private async waitForEarlierDeployJobs(deploy: Deploy): Promise { + try { + if (!deploy.build) await deploy.$fetchGraph('build'); + const deadline = Date.now() + EARLIER_DEPLOY_WAIT_MS; + let logged = false; + for (;;) { + const unfinished = await listUnfinishedDeployJobs(deploy, deploy.build.namespace); + if (unfinished.length === 0) return; + if (Date.now() >= deadline) { + throw new Error( + 'The previous deploy of this service is still running after 45 minutes. Redeploy the service after it stops.' + ); + } + if (!logged) { + getLogger().info(`Deploy: ${deploy.deployable.name} waiting for earlier jobs=${unfinished.join(',')}`); + logged = true; + } + const statusMessage = 'Waiting for the previous deploy to finish'; + if (!(await this.tryPatchDeploy(deploy, { status: DeployStatus.DEPLOYING, statusMessage }))) { + throw new DeploymentSupersededError(); + } + await new Promise((resolve) => setTimeout(resolve, EARLIER_DEPLOY_POLL_MS)); + } + } catch (error) { + if (!(error instanceof DeploymentSupersededError)) { + await new DeployService().recordDeployFailure(deploy, deploy.runUUID, { + status: DeployStatus.DEPLOY_FAILED, + error, + fallbackMessage: `Deployment failed for ${deploy.uuid}. Check deploy logs in Console > Deploy tab for details.`, + }); + } + throw error; + } + } + + /** False means another run owns the row now; the caller skips it instead of failing its siblings. */ + private async tryPatchDeploy(deploy: Deploy, patch: Partial): Promise { if (deploy.id == null || !deploy.runUUID) { getLogger().info( `Deploy: status patch skipped reason=missing_run_identity deployUuid=${deploy.uuid || 'unknown'}` ); - return; + return true; } - const patched = await deploy.$query().patch(patch).where({ id: deploy.id, runUUID: deploy.runUUID }); - if (!patched) throw new DeploymentSupersededError(); + const generation = this.options.expectedGeneration; + const patched = await deploy + .$query() + .patch(patch) + .where({ + id: deploy.id, + runUUID: deploy.runUUID, + ...(generation != null ? { desiredGeneration: generation } : {}), + }); + return patched > 0; } private shouldDeployWithHelm(deploy: Deploy): boolean { diff --git a/src/server/lib/deploymentReconciliation/__tests__/authority.test.ts b/src/server/lib/deploymentReconciliation/__tests__/authority.test.ts new file mode 100644 index 00000000..243411a6 --- /dev/null +++ b/src/server/lib/deploymentReconciliation/__tests__/authority.test.ts @@ -0,0 +1,177 @@ +/** + * Copyright 2026 GoodRx, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { BuildStatus, PullRequestStatus } from 'shared/constants'; +import { + environmentBlockReason, + isDeployAuthorityCurrent, + isDeployOwnedByRun, + isEnvironmentUnblocked, + isIntentRunCurrent, + loadEnvironmentAuthority, +} from '../authority'; + +function buildQuery(row: any) { + const query: any = { + findById: jest.fn(() => query), + select: jest.fn().mockResolvedValue(row), + }; + return query; +} + +function deployQuery(row: any) { + const query: any = { + findOne: jest.fn(() => query), + select: jest.fn().mockResolvedValue(row), + }; + return query; +} + +function models(build: any, deploy?: any) { + const buildRead = buildQuery(build); + const deployRead = deployQuery(deploy); + return { + models: { Build: { query: jest.fn(() => buildRead) }, Deploy: { query: jest.fn(() => deployRead) } } as any, + buildRead, + deployRead, + }; +} + +const openPullRequest = { status: PullRequestStatus.OPEN, deployOnUpdate: true }; + +describe('environmentBlockReason', () => { + it('blocks a missing or deleted build', () => { + expect(environmentBlockReason(null)).toBe('build_missing'); + expect(environmentBlockReason({ deletedAt: new Date() } as any)).toBe('build_deleted'); + }); + + it('follows the pull request for a PR environment', () => { + expect(environmentBlockReason({ pullRequestId: 1, status: BuildStatus.TEARING_DOWN } as any)).toBe('tearing_down'); + expect(environmentBlockReason({ pullRequestId: 1, status: BuildStatus.DEPLOYED } as any)).toBe( + 'pull_request_missing' + ); + expect(environmentBlockReason({ pullRequestId: 1, pullRequest: { status: PullRequestStatus.CLOSED } } as any)).toBe( + 'pull_request_closed' + ); + expect( + environmentBlockReason({ + pullRequestId: 1, + pullRequest: { status: PullRequestStatus.OPEN, deployOnUpdate: false }, + } as any) + ).toBe('deploy_disabled'); + expect(environmentBlockReason({ pullRequestId: 1, pullRequest: openPullRequest } as any)).toBeNull(); + expect( + environmentBlockReason({ pullRequestId: 1, pullRequest: openPullRequest, status: BuildStatus.TORN_DOWN } as any) + ).toBeNull(); + }); + + it('follows the kill switch and teardown states for a PR-less environment', () => { + expect(environmentBlockReason({ status: BuildStatus.TEARING_DOWN, deployEnabled: true } as any)).toBe('torn_down'); + expect(environmentBlockReason({ status: BuildStatus.TORN_DOWN, deployEnabled: true } as any)).toBe('torn_down'); + expect(environmentBlockReason({ status: BuildStatus.DEPLOYED, deployEnabled: false } as any)).toBe( + 'deploy_disabled' + ); + expect(environmentBlockReason({ status: BuildStatus.DEPLOYED, deployEnabled: true } as any)).toBeNull(); + }); +}); + +describe('loadEnvironmentAuthority and isEnvironmentUnblocked', () => { + it('loads the pull request graph only for PR environments', async () => { + const fetchGraph = jest.fn().mockResolvedValue(undefined); + const build = { id: 4, pullRequestId: 9, $fetchGraph: fetchGraph }; + const prLess = { id: 5, pullRequestId: null, deployEnabled: true, $fetchGraph: jest.fn() }; + + await expect(loadEnvironmentAuthority(models(build).models, 4)).resolves.toBe(build); + expect(fetchGraph).toHaveBeenCalledWith('pullRequest'); + await expect(loadEnvironmentAuthority(models(prLess).models, 5)).resolves.toBe(prLess); + expect(prLess.$fetchGraph).not.toHaveBeenCalled(); + await expect(loadEnvironmentAuthority(models(undefined).models, 6)).resolves.toBeNull(); + }); + + it('reports whether the environment may run deployment work', async () => { + await expect( + isEnvironmentUnblocked(models({ id: 4, status: BuildStatus.DEPLOYED, deployEnabled: true }).models, 4) + ).resolves.toBe(true); + await expect( + isEnvironmentUnblocked(models({ id: 4, status: BuildStatus.TORN_DOWN, deployEnabled: true }).models, 4) + ).resolves.toBe(false); + await expect(isEnvironmentUnblocked(models(undefined).models, 4)).resolves.toBe(false); + }); +}); + +describe('isDeployOwnedByRun', () => { + it('requires the row to carry the run token and, when fenced, the generation', async () => { + const { models: m, deployRead } = models(undefined, { id: 7, observedGeneration: 1 }); + + await expect(isDeployOwnedByRun(m, 7, 'run-a', 3)).resolves.toBe(true); + expect(deployRead.findOne).toHaveBeenCalledWith({ id: 7, runUUID: 'run-a', desiredGeneration: 3 }); + expect(deployRead.select).toHaveBeenCalledWith('id', 'observedGeneration'); + + await expect(isDeployOwnedByRun(m, 7, 'run-a')).resolves.toBe(true); + expect(deployRead.findOne).toHaveBeenLastCalledWith({ id: 7, runUUID: 'run-a' }); + }); + + it('treats an observed row as finished work and a missing row as not owned', async () => { + await expect( + isDeployOwnedByRun(models(undefined, { id: 7, observedGeneration: 3 }).models, 7, 'run-a', 3) + ).resolves.toBe(false); + await expect( + isDeployOwnedByRun(models(undefined, { id: 7, observedGeneration: '3' }).models, 7, 'run-a', 3) + ).resolves.toBe(false); + await expect( + isDeployOwnedByRun(models(undefined, { id: 7, observedGeneration: 3 }).models, 7, 'run-a') + ).resolves.toBe(true); + await expect(isDeployOwnedByRun(models(undefined, { id: 7 }).models, 7, 'run-a', 3)).resolves.toBe(true); + await expect(isDeployOwnedByRun(models(undefined, undefined).models, 7, 'run-a', 3)).resolves.toBe(false); + }); +}); + +describe('isDeployAuthorityCurrent', () => { + const deploy = { id: 7, buildId: 4 } as any; + + it('is false when the run no longer owns the row', async () => { + await expect(isDeployAuthorityCurrent(models(undefined, undefined).models, deploy, 'run-a', 3)).resolves.toBe( + false + ); + }); + + it('checks the environment only for fenced runs', async () => { + const blocked = models( + { id: 4, status: BuildStatus.TORN_DOWN, deployEnabled: true }, + { id: 7, observedGeneration: 1 } + ); + await expect(isDeployAuthorityCurrent(blocked.models, deploy, 'run-a')).resolves.toBe(true); + expect(blocked.models.Build.query).not.toHaveBeenCalled(); + await expect(isDeployAuthorityCurrent(blocked.models, deploy, 'run-a', 3)).resolves.toBe(false); + + const open = models({ id: 4, status: BuildStatus.DEPLOYED, deployEnabled: true }, { id: 7, observedGeneration: 1 }); + await expect(isDeployAuthorityCurrent(open.models, deploy, 'run-a', 3)).resolves.toBe(true); + }); +}); + +describe('isIntentRunCurrent', () => { + it('needs an owned row at the generation and an unblocked environment', async () => { + const none = models({ id: 4, status: BuildStatus.DEPLOYED, deployEnabled: true }, undefined); + await expect(isIntentRunCurrent(none.models, 4, 'run-a', 3)).resolves.toBe(false); + expect(none.deployRead.findOne).toHaveBeenCalledWith({ buildId: 4, runUUID: 'run-a', desiredGeneration: 3 }); + + const owned = models({ id: 4, status: BuildStatus.DEPLOYED, deployEnabled: true }, { id: 7 }); + await expect(isIntentRunCurrent(owned.models, 4, 'run-a', 3)).resolves.toBe(true); + + const blocked = models({ id: 4, status: BuildStatus.TORN_DOWN, deployEnabled: true }, { id: 7 }); + await expect(isIntentRunCurrent(blocked.models, 4, 'run-a', 3)).resolves.toBe(false); + }); +}); diff --git a/src/server/lib/deploymentReconciliation/__tests__/mailbox.test.ts b/src/server/lib/deploymentReconciliation/__tests__/mailbox.test.ts index 14af85fa..26fbd88f 100644 --- a/src/server/lib/deploymentReconciliation/__tests__/mailbox.test.ts +++ b/src/server/lib/deploymentReconciliation/__tests__/mailbox.test.ts @@ -22,12 +22,29 @@ jest.mock('server/models/Build', () => ({ }, })); +const deployStamp: any = { + patch: jest.fn(() => deployStamp), + where: jest.fn(() => deployStamp), + then: (resolve: (value: number) => void, reject: (reason: unknown) => void) => + Promise.resolve(2).then(resolve, reject), +}; + +jest.mock('server/models/Deploy', () => ({ + __esModule: true, + default: { + query: jest.fn(() => deployStamp), + }, +})); + import Build from 'server/models/Build'; +import Deploy from 'server/models/Deploy'; +import { DeployStatus } from 'shared/constants'; import { acceptDeploymentIntent, AcceptedDeploymentRefs, + deploymentIntentDeployFilter, deploymentIntentScopeKey, - dirtyDeploymentIntents, + recordIntentConfigFailures, } from '../mailbox'; const TRX = { transaction: true } as any; @@ -67,6 +84,16 @@ describe('deployment intent scope keys', () => { sha: 'abc', }) ).toBe('source:123:Feature%2FA%20B'); + expect( + deploymentIntentScopeKey({ + type: 'source', + requestId: 'request-config', + target: 'all', + githubRepositoryId: 123, + branch: 'Feature/A B', + sha: 'abc', + }) + ).toBe('source:123:Feature%2FA%20B:all'); expect( deploymentIntentScopeKey({ type: 'repository', requestId: 'request-repository', githubRepositoryId: 123 }) ).toBe('repository:123'); @@ -141,6 +168,20 @@ describe('acceptDeploymentIntent', () => { expect(read.findById).toHaveBeenCalledWith(42); expect(read.whereNull).toHaveBeenCalledWith('deletedAt'); expect(read.forUpdate).toHaveBeenCalledTimes(1); + expect(Deploy.query).toHaveBeenCalledWith(TRX); + expect(deployStamp.patch).toHaveBeenCalledWith({ + desiredGeneration: 3, + runUUID: null, + status: DeployStatus.QUEUED, + statusMessage: null, + }); + expect(deployStamp.where).toHaveBeenCalledWith({ + buildId: 42, + active: true, + githubRepositoryId: 123, + branchName: 'main', + }); + expect(deployStamp.where).toHaveBeenCalledWith('desiredGeneration', '<', 3); expect(write.patch).toHaveBeenCalledWith({ desiredGeneration: 3, acceptedRefs: { @@ -321,6 +362,64 @@ describe('acceptDeploymentIntent', () => { }); }); + it('marks older environment-wide entries observed when a newer environment-wide intent is accepted', async () => { + const existing: AcceptedDeploymentRefs = { + 'source:900:static:all': { + type: 'source', + requestId: 'request-5', + target: 'all', + githubRepositoryId: 900, + branch: 'static', + sha: 'config-sha', + gen: 5, + }, + 'source:100:main': { + type: 'source', + requestId: 'request-6', + target: 'repository', + githubRepositoryId: 100, + branch: 'main', + sha: 'service-sha', + gen: 6, + }, + }; + const read = readQuery({ desiredGeneration: 6, acceptedRefs: existing }); + const write = writeQuery(); + (Build.query as jest.Mock).mockReturnValueOnce(read).mockReturnValueOnce(write); + + await expect(acceptDeploymentIntent(42, { type: 'all', requestId: 'request-7' })).resolves.toEqual({ + accepted: true, + generation: 7, + scopeKey: 'all', + }); + + expect(write.patch).toHaveBeenCalledWith({ + desiredGeneration: 7, + acceptedRefs: { + 'source:900:static:all': { ...existing['source:900:static:all'], observedGen: 5 }, + 'source:100:main': existing['source:100:main'], + all: { type: 'all', requestId: 'request-7', gen: 7 }, + }, + }); + }); + + it('leaves an older environment-wide entry pending when a service-scoped intent is accepted', async () => { + const existing: AcceptedDeploymentRefs = { all: { type: 'all', requestId: 'request-5', gen: 5 } }; + const read = readQuery({ desiredGeneration: 5, acceptedRefs: existing }); + const write = writeQuery(); + (Build.query as jest.Mock).mockReturnValueOnce(read).mockReturnValueOnce(write); + + await acceptDeploymentIntent(42, { type: 'repository', requestId: 'request-6', githubRepositoryId: 100 }); + + expect(write.patch).toHaveBeenCalledWith({ + desiredGeneration: 6, + acceptedRefs: { + ...existing, + 'repository:100': { type: 'repository', requestId: 'request-6', githubRepositoryId: 100, gen: 6 }, + }, + }); + }); + it('returns null without writing when the Build does not exist', async () => { (Build.query as jest.Mock).mockReturnValueOnce(readQuery(undefined)); @@ -365,52 +464,107 @@ describe('acceptDeploymentIntent', () => { }); }); -describe('dirtyDeploymentIntents', () => { - it('extracts only unobserved latest intents in generation order', () => { - const acceptedRefs: AcceptedDeploymentRefs = { - all: { type: 'all', requestId: 'request-5', gen: 5 }, +describe('deploymentIntentDeployFilter', () => { + it('selects one repository and branch for a tracked push', () => { + expect( + deploymentIntentDeployFilter({ + type: 'source', + requestId: 'r', + target: 'repository', + githubRepositoryId: 123, + branch: 'main', + sha: 'c', + }) + ).toEqual({ githubRepositoryId: 123, branchName: 'main' }); + }); + + it('selects every active row for a config-repo push and for a full redeploy', () => { + expect( + deploymentIntentDeployFilter({ + type: 'source', + requestId: 'r', + target: 'all', + githubRepositoryId: 900, + branch: 'static', + sha: 'z', + }) + ).toEqual({}); + expect(deploymentIntentDeployFilter({ type: 'all', requestId: 'r' })).toEqual({}); + }); + + it('selects every branch of one repository for a repository redeploy', () => { + expect(deploymentIntentDeployFilter({ type: 'repository', requestId: 'r', githubRepositoryId: 9 })).toEqual({ + githubRepositoryId: 9, + }); + }); +}); + +describe('acceptDeploymentIntent row stamping', () => { + it('does not touch Deploy rows when a redelivery is deduplicated', async () => { + const existing: AcceptedDeploymentRefs = { 'source:123:main': { type: 'source', - requestId: 'request-2', + requestId: 'request-1', target: 'repository', githubRepositoryId: 123, branch: 'main', - sha: 'old', - gen: 2, + sha: 'commit-a', + gen: 1, }, - 'repository:456': { type: 'repository', requestId: 'request-3', githubRepositoryId: 456, gen: 3 }, }; + (Build.query as jest.Mock).mockReturnValueOnce(readQuery({ desiredGeneration: '1', acceptedRefs: existing })); + + await expect( + acceptDeploymentIntent(42, { + type: 'source', + requestId: 'request-redelivery', + target: 'repository', + githubRepositoryId: 123, + branch: 'main', + sha: 'commit-a', + }) + ).resolves.toEqual({ accepted: false, generation: 1, scopeKey: 'source:123:main' }); - expect(dirtyDeploymentIntents(acceptedRefs, 2)).toEqual([ - { scopeKey: 'repository:456', intent: acceptedRefs['repository:456'] }, - { scopeKey: 'all', intent: acceptedRefs.all }, - ]); + expect(Deploy.query).not.toHaveBeenCalled(); }); +}); + +describe('recordIntentConfigFailures', () => { + it('stores the failures on the entry accepted at that generation', async () => { + const existing: AcceptedDeploymentRefs = { + all: { type: 'all', requestId: 'request-3', gen: 3 }, + 'repository:9': { type: 'repository', requestId: 'request-2', githubRepositoryId: 9, gen: 2 }, + }; + const read = readQuery({ id: 42, acceptedRefs: existing }); + const write = writeQuery(); + (Build.query as jest.Mock).mockReturnValueOnce(read).mockReturnValueOnce(write); - it('treats an absent mailbox as having no dirty intents', () => { - expect(dirtyDeploymentIntents(null, 0)).toEqual([]); - expect(dirtyDeploymentIntents(undefined, 0)).toEqual([]); + await expect(recordIntentConfigFailures(42, 3, { billing: 'billing lifecycle.yaml is invalid' })).resolves.toBe( + true + ); + + expect(write.patch).toHaveBeenCalledWith({ + acceptedRefs: { + ...existing, + all: { ...existing.all, configFailures: { billing: 'billing lifecycle.yaml is invalid' } }, + }, + }); }); - it('ignores malformed persisted entries', () => { - const validIntent = { type: 'all' as const, requestId: 'request-2', gen: 2 }; - const acceptedRefs = { - malformed: null, - all: validIntent, - } as unknown as AcceptedDeploymentRefs; + it('writes nothing when no entry carries that generation', async () => { + const read = readQuery({ id: 42, acceptedRefs: { all: { type: 'all', requestId: 'request-3', gen: 3 } } }); + (Build.query as jest.Mock).mockReturnValueOnce(read); - expect(dirtyDeploymentIntents(acceptedRefs, 0)).toEqual([{ scopeKey: 'all', intent: validIntent }]); + await expect(recordIntentConfigFailures(42, 9, {})).resolves.toBe(false); + expect(Build.query).toHaveBeenCalledTimes(1); }); - it('uses the scope key as a deterministic tie-breaker for equal generations', () => { - const acceptedRefs: AcceptedDeploymentRefs = { - 'repository:456': { type: 'repository', requestId: 'request-repository', githubRepositoryId: 456, gen: 3 }, - all: { type: 'all', requestId: 'request-all', gen: 3 }, - }; + it('returns false for a missing build or a malformed mailbox', async () => { + (Build.query as jest.Mock).mockReturnValueOnce(readQuery(undefined)); + await expect(recordIntentConfigFailures(42, 3, {})).resolves.toBe(false); - expect(dirtyDeploymentIntents(acceptedRefs, 0)).toEqual([ - { scopeKey: 'all', intent: acceptedRefs.all }, - { scopeKey: 'repository:456', intent: acceptedRefs['repository:456'] }, - ]); + (Build.query as jest.Mock).mockReturnValueOnce(readQuery({ id: 42, acceptedRefs: 'broken' })); + await expect(recordIntentConfigFailures(42, 3, {})).resolves.toBe(false); + expect(Build.query).toHaveBeenCalledTimes(2); }); }); diff --git a/src/server/lib/deploymentReconciliation/authority.ts b/src/server/lib/deploymentReconciliation/authority.ts new file mode 100644 index 00000000..a5f30c16 --- /dev/null +++ b/src/server/lib/deploymentReconciliation/authority.ts @@ -0,0 +1,111 @@ +/** + * Copyright 2026 GoodRx, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { isDeployEnabled } from 'server/lib/buildSource'; +import type Build from 'server/models/Build'; +import type Deploy from 'server/models/Deploy'; +import { BuildStatus, PullRequestStatus } from 'shared/constants'; + +export type EnvironmentBlockReason = + | 'build_missing' + | 'build_deleted' + | 'tearing_down' + | 'pull_request_missing' + | 'pull_request_closed' + | 'deploy_disabled' + | 'torn_down'; + +export interface AuthorityModels { + Build: typeof Build; + Deploy: typeof Deploy; +} + +/** Why no deployment work may run for this environment right now, or null. */ +export function environmentBlockReason(build: Build | null | undefined): EnvironmentBlockReason | null { + if (!build) return 'build_missing'; + if (build.deletedAt != null) return 'build_deleted'; + + if (build.pullRequest || build.pullRequestId != null) { + if (build.status === BuildStatus.TEARING_DOWN) return 'tearing_down'; + if (!build.pullRequest) return 'pull_request_missing'; + if (build.pullRequest.status !== PullRequestStatus.OPEN) return 'pull_request_closed'; + if (!isDeployEnabled(build)) return 'deploy_disabled'; + // Re-adding the deploy label to an open PR may reclaim the row after teardown completes. + return null; + } + + if (build.status === BuildStatus.TEARING_DOWN || build.status === BuildStatus.TORN_DOWN) return 'torn_down'; + return build.deployEnabled === true ? null : 'deploy_disabled'; +} + +export async function loadEnvironmentAuthority(models: AuthorityModels, buildId: number): Promise { + const build = await models.Build.query() + .findById(buildId) + .select('id', 'runUUID', 'status', 'deployEnabled', 'deletedAt', 'pullRequestId', 'desiredGeneration'); + if (build?.pullRequestId != null) await build.$fetchGraph('pullRequest'); + return build ?? null; +} + +export async function isEnvironmentUnblocked(models: AuthorityModels, buildId: number): Promise { + return environmentBlockReason(await loadEnvironmentAuthority(models, buildId)) == null; +} + +/** The row is still owned by this run, still desired at this generation, and not yet marked finished. */ +export async function isDeployOwnedByRun( + models: AuthorityModels, + deployId: number, + runUUID: string, + expectedGeneration?: number +): Promise { + const owned = await models.Deploy.query() + .findOne({ + id: deployId, + runUUID, + ...(expectedGeneration != null ? { desiredGeneration: expectedGeneration } : {}), + }) + .select('id', 'observedGeneration'); + if (!owned) return false; + if (expectedGeneration == null || owned.observedGeneration == null) return true; + return Number(owned.observedGeneration) < expectedGeneration; +} + +/** + * The one authority predicate for service work. Nothing about other services + * appears in it: a newer intent that selects this row changes the row itself. + */ +export async function isDeployAuthorityCurrent( + models: AuthorityModels, + deploy: Pick, + runUUID: string, + expectedGeneration?: number +): Promise { + if (!(await isDeployOwnedByRun(models, deploy.id, runUUID, expectedGeneration))) return false; + return expectedGeneration == null || isEnvironmentUnblocked(models, deploy.buildId); +} + +/** An intent stays current while it still owns at least one row at its generation. */ +export async function isIntentRunCurrent( + models: AuthorityModels, + buildId: number, + runUUID: string, + expectedGeneration: number +): Promise { + const owned = await models.Deploy.query() + .findOne({ buildId, runUUID, desiredGeneration: expectedGeneration }) + .select('id'); + if (!owned) return false; + return isEnvironmentUnblocked(models, buildId); +} diff --git a/src/server/lib/deploymentReconciliation/mailbox.ts b/src/server/lib/deploymentReconciliation/mailbox.ts index e608ea7e..6fa82c05 100644 --- a/src/server/lib/deploymentReconciliation/mailbox.ts +++ b/src/server/lib/deploymentReconciliation/mailbox.ts @@ -14,8 +14,11 @@ * limitations under the License. */ +import type { PartialModelObject, Transaction } from 'objection'; import type { JobDataWithContext } from 'server/lib/logger'; import Build from 'server/models/Build'; +import Deploy from 'server/models/Deploy'; +import { DeployStatus } from 'shared/constants'; export interface SourcePushIntent { type: 'source'; @@ -41,7 +44,12 @@ export interface EnvironmentRedeployIntent { } export type DeploymentIntent = SourcePushIntent | RepositoryRedeployIntent | EnvironmentRedeployIntent; -export type AcceptedDeploymentIntent = DeploymentIntent & { gen: number }; +export type AcceptedDeploymentIntent = DeploymentIntent & { + gen: number; + observedGen?: number; + /** Services whose YAML failed in this intent's import and had no row to carry it; absent until the import ran. */ + configFailures?: Record; +}; export type AcceptedDeploymentRefs = Record; export interface DeploymentReconciliationJobData extends JobDataWithContext { @@ -50,11 +58,6 @@ export interface DeploymentReconciliationJobData extends JobDataWithContext { generation: number; } -export interface DirtyDeploymentIntent { - scopeKey: string; - intent: AcceptedDeploymentIntent; -} - export interface AcceptDeploymentIntentResult { accepted: boolean; generation: number; @@ -65,7 +68,10 @@ export interface AcceptDeploymentIntentResult { export function deploymentIntentScopeKey(intent: DeploymentIntent): string { switch (intent.type) { case 'source': - return `source:${intent.githubRepositoryId}:${encodeURIComponent(intent.branch)}`; + // A config-repo push selects every row; it must never replace the repo-scoped entry of the same source. + return `source:${intent.githubRepositoryId}:${encodeURIComponent(intent.branch)}${ + intent.target === 'all' ? ':all' : '' + }`; case 'repository': return `repository:${intent.githubRepositoryId}`; case 'all': @@ -73,17 +79,40 @@ export function deploymentIntentScopeKey(intent: DeploymentIntent): string { } } -/** Returns the latest intent for every scope that has not yet been observed. */ -export function dirtyDeploymentIntents( - acceptedRefs: AcceptedDeploymentRefs | null | undefined, - observedGeneration: number -): DirtyDeploymentIntent[] { - if (!acceptedRefs || typeof acceptedRefs !== 'object' || Array.isArray(acceptedRefs)) return []; - - return Object.entries(acceptedRefs) - .filter(([, intent]) => Number.isSafeInteger(intent?.gen) && intent.gen > observedGeneration) - .map(([scopeKey, intent]) => ({ scopeKey, intent })) - .sort((left, right) => left.intent.gen - right.intent.gen || left.scopeKey.localeCompare(right.scopeKey)); +export function deploymentIntentSelectsAllDeploys(intent: DeploymentIntent): boolean { + return intent.type === 'all' || (intent.type === 'source' && intent.target === 'all'); +} + +/** The Deploy rows an intent selects; an empty filter means every active row. */ +export function deploymentIntentDeployFilter(intent: DeploymentIntent): { + githubRepositoryId?: number; + branchName?: string; +} { + if (intent.type === 'all') return {}; + if (intent.type === 'source') { + return intent.target === 'all' ? {} : { githubRepositoryId: intent.githubRepositoryId, branchName: intent.branch }; + } + return { githubRepositoryId: intent.githubRepositoryId }; +} + +/** A stamped row is unowned and queued: the older run can no longer write to it, and its old status is not current. */ +export function deployStampForGeneration(generation: number): PartialModelObject { + const stamp = { desiredGeneration: generation, runUUID: null, status: DeployStatus.QUEUED, statusMessage: null }; + // The token column is nullable; the model types it as a string. + return stamp as unknown as PartialModelObject; +} + +/** Raises the selected rows to `generation`; rows already desired at a newer generation are left alone. */ +export async function stampDeploysForIntent( + trx: Transaction | undefined, + buildId: number, + intent: DeploymentIntent, + generation: number +): Promise { + return Deploy.query(trx) + .patch(deployStampForGeneration(generation)) + .where({ buildId, active: true, ...deploymentIntentDeployFilter(intent) }) + .where('desiredGeneration', '<', generation); } /** @@ -147,12 +176,51 @@ export async function acceptDeploymentIntent( ...acceptedRefs, [scopeKey]: { ...intent, gen: generation }, }; + // Every row moves to this generation, so an older environment-wide entry has nothing left to execute. + if (deploymentIntentSelectsAllDeploys(intent)) { + for (const [key, accepted] of Object.entries(acceptedRefs)) { + if (key === scopeKey || !deploymentIntentSelectsAllDeploys(accepted) || accepted.observedGen === accepted.gen) { + continue; + } + nextRefs[key] = { ...accepted, observedGen: accepted.gen }; + } + } await Build.query(trx).findById(buildId).patch({ desiredGeneration: generation, acceptedRefs: nextRefs, }); + await stampDeploysForIntent(trx, buildId, intent, generation); return { accepted: true, generation, scopeKey }; }); } + +/** Records which row-less services the import at `generation` could not read; an empty map means the import was clean. */ +export async function recordIntentConfigFailures( + buildId: number, + generation: number, + configFailures: Record +): Promise { + return Build.transact(async (trx) => { + const build = await Build.query(trx) + .select('id', 'acceptedRefs') + .findById(buildId) + .whereNull('deletedAt') + .forUpdate(); + if (!build) return false; + + const acceptedRefs = + build.acceptedRefs && typeof build.acceptedRefs === 'object' && !Array.isArray(build.acceptedRefs) + ? build.acceptedRefs + : {}; + const entry = Object.entries(acceptedRefs).find(([, intent]) => intent?.gen === generation); + if (!entry) return false; + const [scopeKey, intent] = entry; + + await Build.query(trx) + .findById(buildId) + .patch({ acceptedRefs: { ...acceptedRefs, [scopeKey]: { ...intent, configFailures } } }); + return true; + }); +} diff --git a/src/server/lib/helm/__tests__/helm.behavior.test.ts b/src/server/lib/helm/__tests__/helm.behavior.test.ts index 242114ff..0da22ef9 100644 --- a/src/server/lib/helm/__tests__/helm.behavior.test.ts +++ b/src/server/lib/helm/__tests__/helm.behavior.test.ts @@ -614,21 +614,26 @@ describe('Codefresh Helm YAML generation', () => { const deploy = makePublicDeploy(); deploy.deployable.helm.chart.valueFiles = ['helm/preview.yaml']; - const command = await generateCodefreshRunCommand(deploy); + const { command, configPath } = await generateCodefreshRunCommand(deploy); expect(deploy.$fetchGraph).toHaveBeenCalledWith('build'); expect(mockMkdir).toHaveBeenCalledWith('/tmp/lifecycle/codefresh', { recursive: true }); - expect(mockWriteFile).toHaveBeenCalledWith( - '/tmp/lifecycle/codefresh/helm-deploy-Preview-ABC.yaml', - expect.any(String), - 'utf8' - ); + expect(configPath).toMatch(/^\/tmp\/lifecycle\/codefresh\/helm-deploy-Preview-ABC-[\w-]{10}\.yaml$/); + expect(mockWriteFile).toHaveBeenCalledWith(configPath, expect.any(String), 'utf8'); const writtenYaml = yaml.load(mockWriteFile.mock.calls[0][1] as string) as any; expect(writtenYaml.steps.clone.type).toBe('parallel'); expect(writtenYaml.steps.deploy.arguments.custom_value_files).toEqual(['helm/preview.yaml']); - expect(command).toBe( - 'codefresh run goodrx/helm-deploy -b "feature/helm-preview" -y /tmp/lifecycle/codefresh/helm-deploy-Preview-ABC.yaml -v ENV=lfc -d' - ); + expect(command).toBe(`codefresh run goodrx/helm-deploy -b "feature/helm-preview" -y ${configPath} -v ENV=lfc -d`); + }); + + test('writes a separate file for every submission of the same service', async () => { + const deploy = makePublicDeploy(); + + const first = await generateCodefreshRunCommand(deploy); + const second = await generateCodefreshRunCommand(deploy); + + expect(first.configPath).not.toBe(second.configPath); + expect(mockWriteFile).toHaveBeenCalledTimes(2); }); test('logs and preserves a YAML write failure without constructing a command', async () => { @@ -653,8 +658,9 @@ describe('image metadata and teardown', () => { expect(mockRandomAlphanumeric).toHaveBeenCalledWith(4); }); - test('uninstalls only active, non-queued Helm releases and records successful cleanup', async () => { + test('uninstalls every active Helm release, also one queued for a new deploy, and records the cleanup', async () => { const successfulPatch = jest.fn().mockResolvedValue(undefined); + const queuedPatch = jest.fn().mockResolvedValue(undefined); const skippedPatch = jest.fn().mockResolvedValue(undefined); const activeDeploy = { uuid: 'active-release', @@ -675,7 +681,7 @@ describe('image metadata and teardown', () => { active: true, status: 'queued', deployable: { helm: { chart: { name: 'public-chart' } } }, - $query: jest.fn(() => ({ patch: skippedPatch })), + $query: jest.fn(() => ({ patch: queuedPatch })), }; const nonHelmDeploy = { uuid: 'not-a-helm-release', @@ -690,9 +696,11 @@ describe('image metadata and teardown', () => { await uninstallHelmReleases({ id: 42, namespace: 'env-build-123' } as any); - expect(mockShellPromise).toHaveBeenCalledTimes(1); + expect(mockShellPromise).toHaveBeenCalledTimes(2); expect(mockShellPromise).toHaveBeenCalledWith('helm uninstall active-release --namespace env-build-123'); + expect(mockShellPromise).toHaveBeenCalledWith('helm uninstall queued-release --namespace env-build-123'); expect(successfulPatch).toHaveBeenCalledWith({ statusMessage: 'Uninstalled via Helm' }); + expect(queuedPatch).toHaveBeenCalledWith({ statusMessage: 'Uninstalled via Helm' }); expect(skippedPatch).not.toHaveBeenCalled(); expect(mockLoggerDebug).toHaveBeenCalledWith('Helm: releases uninstalled'); }); diff --git a/src/server/lib/helm/helm.ts b/src/server/lib/helm/helm.ts index 58f3a35d..788a65c3 100644 --- a/src/server/lib/helm/helm.ts +++ b/src/server/lib/helm/helm.ts @@ -16,10 +16,10 @@ import yaml from 'js-yaml'; import fs from 'fs'; +import { nanoid } from 'nanoid'; import Deploy from 'server/models/Deploy'; import GlobalConfigService from 'server/services/globalConfig'; import { TMP_PATH } from 'shared/config'; -import { DeployStatus } from 'shared/constants'; import { getLogger } from 'server/lib/logger'; import { getBuildSource, resolveBuildSourceRepository } from 'server/lib/buildSource'; import { shellPromise } from 'server/lib/shell'; @@ -270,7 +270,7 @@ export async function deployHelm( * @param {Deploy} deploy - The deploy object containing deployment details. * @returns {Promise} A promise that resolves to the generated Codefresh command. */ -export async function generateCodefreshRunCommand(deploy: Deploy): Promise { +export async function generateCodefreshRunCommand(deploy: Deploy): Promise<{ command: string; configPath: string }> { const hasValueFiles = deploy?.deployable?.helm?.chart?.valueFiles?.length > 0; await deploy.$fetchGraph('build'); const yamlContent = hasValueFiles @@ -278,7 +278,8 @@ export async function generateCodefreshRunCommand(deploy: Deploy): Promise d.deployable.helm && Object.keys(d.deployable.helm).length > 0); for (const deploy of helmDeploys) { - if (!deploy.active || deploy.status === DeployStatus.QUEUED) continue; + if (!deploy.active) continue; try { await shellPromise(`helm uninstall ${deploy.uuid} --namespace ${build.namespace}`); diff --git a/src/server/lib/kubernetesApply/applyManifest.test.ts b/src/server/lib/kubernetesApply/applyManifest.test.ts index 30f5ad19..b873243e 100644 --- a/src/server/lib/kubernetesApply/applyManifest.test.ts +++ b/src/server/lib/kubernetesApply/applyManifest.test.ts @@ -16,6 +16,10 @@ var mockCreateNamespacedConfigMap: jest.Mock; var mockCreateNamespacedJob: jest.Mock; +var mockListNamespacedJob: jest.Mock; +var mockDeleteNamespacedJob: jest.Mock; +var mockListNamespacedPod: jest.Mock; +var mockDeleteNamespacedPod: jest.Mock; var mockGetAllConfigs: jest.Mock; var mockWaitForJobAndGetLogs: jest.Mock; var mockGetLogger: jest.Mock; @@ -26,11 +30,19 @@ jest.mock('@kubernetes/client-node', () => { const actual = jest.requireActual('@kubernetes/client-node'); mockCreateNamespacedConfigMap = jest.fn(); mockCreateNamespacedJob = jest.fn(); + mockListNamespacedJob = jest.fn(); + mockDeleteNamespacedJob = jest.fn(); + mockListNamespacedPod = jest.fn(); + mockDeleteNamespacedPod = jest.fn(); const coreApi = { createNamespacedConfigMap: (...args: unknown[]) => mockCreateNamespacedConfigMap(...args), + listNamespacedPod: (...args: unknown[]) => mockListNamespacedPod(...args), + deleteNamespacedPod: (...args: unknown[]) => mockDeleteNamespacedPod(...args), }; const batchApi = { createNamespacedJob: (...args: unknown[]) => mockCreateNamespacedJob(...args), + listNamespacedJob: (...args: unknown[]) => mockListNamespacedJob(...args), + deleteNamespacedJob: (...args: unknown[]) => mockDeleteNamespacedJob(...args), }; return { @@ -71,7 +83,12 @@ jest.mock('server/lib/logger', () => { }); import { HttpError } from '@kubernetes/client-node'; -import { createKubernetesApplyJob, getKubernetesApplyJobName, monitorKubernetesJob } from './applyManifest'; +import { + createKubernetesApplyJob, + getKubernetesApplyJobName, + listUnfinishedDeployJobs, + monitorKubernetesJob, +} from './applyManifest'; function deploy(overrides: Record = {}) { return { @@ -283,3 +300,74 @@ describe('Kubernetes apply jobs', () => { expect(mockError).toHaveBeenCalledWith('Job: monitor failed name=apply-job'); }); }); + +describe('listUnfinishedDeployJobs', () => { + const applySelector = 'app=lifecycle-deploy,type=kubernetes-apply,deploy_uuid=catalog-deploy-uuid'; + const helmSelector = 'app.kubernetes.io/name=native-helm,deploy-id=37'; + const job = (name: string, condition?: 'Complete' | 'Failed') => ({ + metadata: { name }, + status: condition ? { conditions: [{ type: condition, status: 'True' }] } : { active: 1 }, + }); + const podsOf = (phase: string, extra: Record = {}) => ({ + body: { items: [{ metadata: { name: 'pod', ...extra }, status: { phase } }] }, + }); + + beforeEach(() => { + jest.clearAllMocks(); + mockListNamespacedJob.mockResolvedValue({ body: { items: [] } }); + mockListNamespacedPod.mockResolvedValue({ body: { items: [] } }); + }); + + it('reports apply and native Helm jobs of the service that can still write, and never deletes anything', async () => { + mockListNamespacedJob.mockImplementation(async (_namespace: string, ...args: unknown[]) => + args[4] === applySelector + ? { + body: { + items: [ + job('apply-running'), + job('apply-done', 'Complete'), + job('apply-failed', 'Failed'), + job('apply-deadline', 'Failed'), + ], + }, + } + : { body: { items: [job('helm-running')] } } + ); + mockListNamespacedPod.mockImplementation(async (_namespace: string, ...args: unknown[]) => + args[4] === 'job-name=apply-deadline' + ? podsOf('Running', { deletionTimestamp: '2026-10-11T00:00:00Z' }) + : podsOf('Failed') + ); + + await expect(listUnfinishedDeployJobs(deploy(), 'env-build')).resolves.toEqual([ + 'apply-running', + 'apply-deadline', + 'helm-running', + ]); + expect(mockListNamespacedJob).toHaveBeenCalledWith( + 'env-build', + undefined, + undefined, + undefined, + undefined, + applySelector + ); + expect(mockListNamespacedJob).toHaveBeenCalledWith( + 'env-build', + undefined, + undefined, + undefined, + undefined, + helmSelector + ); + expect(mockDeleteNamespacedJob).not.toHaveBeenCalled(); + expect(mockDeleteNamespacedPod).not.toHaveBeenCalled(); + }); + + it('reports nothing once every earlier job is terminal and its pods have stopped', async () => { + mockListNamespacedJob.mockResolvedValue({ body: { items: [job('old', 'Failed'), job('older', 'Complete')] } }); + mockListNamespacedPod.mockResolvedValue(podsOf('Succeeded')); + + await expect(listUnfinishedDeployJobs(deploy(), 'env-build')).resolves.toEqual([]); + }); +}); diff --git a/src/server/lib/kubernetesApply/applyManifest.ts b/src/server/lib/kubernetesApply/applyManifest.ts index 097fa3d5..fc1c5211 100644 --- a/src/server/lib/kubernetesApply/applyManifest.ts +++ b/src/server/lib/kubernetesApply/applyManifest.ts @@ -152,6 +152,38 @@ export async function createKubernetesApplyJob({ return createdJob.body; } +/** A deploy Job is done only once it is terminal and the kubelet reports each of its pods stopped. */ +export async function listUnfinishedDeployJobs(deploy: Deploy, namespace: string): Promise { + const kc = new k8s.KubeConfig(); + kc.loadFromDefault(); + const batchApi = kc.makeApiClient(k8s.BatchV1Api); + const coreApi = kc.makeApiClient(k8s.CoreV1Api); + const selectors = [ + `app=lifecycle-deploy,type=kubernetes-apply,deploy_uuid=${deploy.uuid}`, + `app.kubernetes.io/name=native-helm,deploy-id=${deploy.id}`, + ]; + const unfinished: string[] = []; + for (const selector of selectors) { + const { body } = await batchApi.listNamespacedJob(namespace, undefined, undefined, undefined, undefined, selector); + for (const job of body.items ?? []) { + const name = job.metadata?.name; + if (!name || hasCondition(job, 'Complete')) continue; + if (!hasCondition(job, 'Failed') || (await hasRunningPod(coreApi, namespace, name))) unfinished.push(name); + } + } + return unfinished; +} + +function hasCondition(job: k8s.V1Job, type: 'Complete' | 'Failed'): boolean { + return (job.status?.conditions ?? []).some((condition) => condition.type === type && condition.status === 'True'); +} + +async function hasRunningPod(coreApi: k8s.CoreV1Api, namespace: string, jobName: string): Promise { + const selector = `job-name=${jobName}`; + const { body } = await coreApi.listNamespacedPod(namespace, undefined, undefined, undefined, undefined, selector); + return (body.items ?? []).some((pod) => pod.status?.phase !== 'Succeeded' && pod.status?.phase !== 'Failed'); +} + async function createManifestConfigMap(deploy: Deploy, configMapName: string, namespace: string): Promise { const kc = new k8s.KubeConfig(); kc.loadFromDefault(); diff --git a/src/server/lib/nativeHelm/__tests__/helm.behavior.test.ts b/src/server/lib/nativeHelm/__tests__/helm.behavior.test.ts index 73a474fa..4dd4974d 100644 --- a/src/server/lib/nativeHelm/__tests__/helm.behavior.test.ts +++ b/src/server/lib/nativeHelm/__tests__/helm.behavior.test.ts @@ -24,6 +24,7 @@ const mockWithLogContext = jest.fn(); const mockMkdir = jest.fn(); const mockWriteFile = jest.fn(); const mockShellPromise = jest.fn(); +const mockUnlink = jest.fn(); const mockNanoid = jest.fn(); const mockMetricsConstructor = jest.fn(); const mockMetricIncrement = jest.fn(); @@ -73,6 +74,7 @@ jest.mock('fs', () => ({ promises: { mkdir: (...args: unknown[]) => mockMkdir(...args), writeFile: (...args: unknown[]) => mockWriteFile(...args), + unlink: (...args: unknown[]) => mockUnlink(...args), }, }, })); @@ -99,7 +101,10 @@ jest.mock('server/lib/helm/utils', () => ({ jest.mock('server/lib/helm/helm', () => ({ constructHelmDeploysBuildMetaData: (...args: unknown[]) => mockConstructBuildMetadata(...args), - generateCodefreshRunCommand: (...args: unknown[]) => mockGenerateCodefreshRunCommand(...args), + generateCodefreshRunCommand: async (...args: unknown[]) => { + const plan = await mockGenerateCodefreshRunCommand(...args); + return typeof plan === 'string' ? { command: plan, configPath: null } : plan; + }, })); jest.mock('server/lib/codefresh/utils', () => ({ @@ -683,6 +688,38 @@ describe('native Helm orchestration behavior', () => { expect(mockPatchIngress).not.toHaveBeenCalled(); }); + it('removes the submission file once the provider has the command', async () => { + const { deploy } = createDeploy({ + deployable: { name: 'sample-service', repository: { fullName: 'example/repository' } }, + }); + mockGenerateCodefreshRunCommand.mockResolvedValueOnce({ + command: 'codefresh run pipeline', + configPath: '/tmp/lifecycle/codefresh/helm-deploy-sample-abc.yaml', + }); + mockUnlink.mockResolvedValue(undefined); + + await expect(deployHelm([deploy])).resolves.toBeUndefined(); + + expect(mockShellPromise).toHaveBeenCalledWith('codefresh run pipeline'); + expect(mockUnlink).toHaveBeenCalledWith('/tmp/lifecycle/codefresh/helm-deploy-sample-abc.yaml'); + }); + + it('removes the submission file when the provider gate refuses the row', async () => { + const { deploy } = createDeploy({ + deployable: { name: 'sample-service', repository: { fullName: 'example/repository' } }, + }); + mockGenerateCodefreshRunCommand.mockResolvedValueOnce({ + command: 'codefresh run pipeline', + configPath: '/tmp/x.yaml', + }); + mockUnlink.mockResolvedValue(undefined); + + await expect(deployHelm([deploy], { providerSubmissionGate: async () => false })).rejects.toThrow(); + + expect(mockShellPromise).not.toHaveBeenCalled(); + expect(mockUnlink).toHaveBeenCalledWith('/tmp/x.yaml'); + }); + it('records a normal deployment failure after tracking failure metrics', async () => { const deployError = new Error('codefresh unavailable'); const { deploy } = createDeploy({ diff --git a/src/server/lib/nativeHelm/helm.ts b/src/server/lib/nativeHelm/helm.ts index 78e0f843..0c49723b 100644 --- a/src/server/lib/nativeHelm/helm.ts +++ b/src/server/lib/nativeHelm/helm.ts @@ -66,6 +66,8 @@ export type HelmSecretMutationGate = (deploy: Deploy, action: () => Promise Promise; } export interface JobResult { @@ -466,7 +468,12 @@ export async function deployNativeHelm(deploy: Deploy, options: HelmDeploymentEx } } -async function deployCodefreshHelm(deploy: Deploy, deployService: DeployService, runUUID: string): Promise { +async function deployCodefreshHelm( + deploy: Deploy, + deployService: DeployService, + runUUID: string, + options: HelmDeploymentExecutionOptions = {} +): Promise { const deployable = requireDeployable(deploy); const { build } = deploy; @@ -474,8 +481,16 @@ async function deployCodefreshHelm(deploy: Deploy, deployService: DeployService, const { getCodefreshPipelineIdFromOutput } = await import('server/lib/codefresh/utils'); const { checkPipelineStatus } = await import('server/lib/codefresh'); - const codefreshRunCommand = await generateCodefreshRunCommand(deploy); - const output = await shellPromise(codefreshRunCommand); + const { command: codefreshRunCommand, configPath } = await generateCodefreshRunCommand(deploy); + let output = ''; + try { + if (options.providerSubmissionGate && !(await options.providerSubmissionGate(deploy))) { + throw new DeploymentSupersededError(); + } + output = await shellPromise(codefreshRunCommand); + } finally { + if (configPath) await fs.promises.unlink(configPath).catch(() => undefined); + } const deployPipelineId = getCodefreshPipelineIdFromOutput(output); const statusMessage = 'Starting deployment via Helm'; @@ -545,7 +560,7 @@ export async function deployHelm(deploys: Deploy[], options: HelmDeploymentExecu if (useNative) { await deployNativeHelm(deploy, options); } else { - await deployCodefreshHelm(deploy, deployService, runUUID); + await deployCodefreshHelm(deploy, deployService, runUUID, options); } await deployService.patchAndUpdateActivityFeed( diff --git a/src/server/lib/tests/deploymentManager.test.ts b/src/server/lib/tests/deploymentManager.test.ts index 269077ba..2eb12953 100644 --- a/src/server/lib/tests/deploymentManager.test.ts +++ b/src/server/lib/tests/deploymentManager.test.ts @@ -15,6 +15,7 @@ */ import { DeploymentManager, DeploymentSupersededError } from '../deploymentManager/deploymentManager'; +import { AuthorityLockLostError } from 'server/lib/authorityLock'; import { Deploy } from 'server/models'; import { buildDeployJobName } from '../kubernetes/jobNames'; import { deployHelm } from '../helm'; @@ -27,6 +28,7 @@ jest.mock('../nativeHelm', () => ({ shouldUseNativeHelm: jest.fn().mockResolvedValue(false), })); jest.mock('../kubernetesApply/applyManifest', () => ({ + listUnfinishedDeployJobs: jest.fn().mockResolvedValue([]), createKubernetesApplyJob: jest.fn().mockResolvedValue(void 0), monitorKubernetesJob: jest.fn().mockResolvedValue({ success: true, message: 'ok', logs: 'apply logs' }), })); @@ -54,7 +56,11 @@ jest.mock('server/services/deploy', () => { })); }); -import { createKubernetesApplyJob, monitorKubernetesJob } from '../kubernetesApply/applyManifest'; +import { + listUnfinishedDeployJobs, + createKubernetesApplyJob, + monitorKubernetesJob, +} from '../kubernetesApply/applyManifest'; import { waitForDeployPodReady } from '../kubernetes'; import GlobalConfigService from 'server/services/globalConfig'; import { getLogArchivalService } from 'server/services/logArchival'; @@ -78,6 +84,7 @@ describe('DeploymentManager', () => { mockRecordDeployFailure.mockReset().mockResolvedValue(false); (deployHelm as jest.Mock).mockReset().mockResolvedValue(undefined); (shouldUseNativeHelm as jest.Mock).mockReset().mockResolvedValue(false); + (listUnfinishedDeployJobs as jest.Mock).mockReset().mockResolvedValue([]); (createKubernetesApplyJob as jest.Mock).mockReset().mockResolvedValue(undefined); (monitorKubernetesJob as jest.Mock) .mockReset() @@ -311,12 +318,13 @@ describe('DeploymentManager', () => { } as unknown as Deploy; } - it('admits all native siblings for a level together while Codefresh and pod readiness stay outside', async () => { + it('admits each native service through its own gate while Codefresh and pod readiness stay outside', async () => { const nativeHelm = runnableDeploy('native-chart', 'helm'); const codefreshHelm = runnableDeploy('codefresh-chart', 'helm'); const kubernetes = runnableDeploy('web', 'github'); const events: string[] = []; let insideGate = false; + let gateDepth = 0; (shouldUseNativeHelm as jest.Mock).mockImplementation(async (deploy: Deploy) => { return deploy.deployable.name === 'native-chart'; @@ -336,12 +344,14 @@ describe('DeploymentManager', () => { return { ready: true }; }); - const nativeMutationGate = jest.fn(async (action: () => Promise) => { + const nativeMutationGate = jest.fn(async (_deploy: Deploy, action: () => Promise) => { + gateDepth += 1; insideGate = true; try { return { admitted: true as const, value: await action() }; } finally { - insideGate = false; + gateDepth -= 1; + insideGate = gateDepth > 0; } }); const nativeSecretMutationGate = jest.fn(); @@ -354,25 +364,27 @@ describe('DeploymentManager', () => { await manager.deploy(); - expect(nativeMutationGate).toHaveBeenCalledTimes(1); + expect(nativeMutationGate).toHaveBeenCalledTimes(2); + expect(nativeMutationGate).toHaveBeenCalledWith(nativeHelm, expect.any(Function)); + expect(nativeMutationGate).toHaveBeenCalledWith(kubernetes, expect.any(Function)); expect(deployHelm).toHaveBeenCalledWith([nativeHelm], { secretMutationGate: nativeSecretMutationGate, }); + expect(nativeMutationGate).not.toHaveBeenCalledWith(codefreshHelm, expect.any(Function)); + expect(deployHelm).toHaveBeenCalledWith( + [codefreshHelm], + expect.objectContaining({ providerSubmissionGate: expect.any(Function) }) + ); expect(events).toEqual( - expect.arrayContaining([ - 'codefresh-chart:false', - 'native-chart:true', - 'apply:true', - 'monitor:true', - 'ready:false', - ]) + expect.arrayContaining(['native-chart:true', 'apply:true', 'monitor:true', 'ready:false']) ); }); - it('keeps the promotion gate until every admitted native sibling is terminal', async () => { + it('holds a service gate until its native mutation is terminal, and a failed sibling does not block it', async () => { const nativeHelm = runnableDeploy('native-chart', 'helm'); const kubernetes = runnableDeploy('web', 'github'); let insideGate = false; + let gateDepth = 0; let releaseMonitor!: () => void; let markMonitorStarted!: () => void; const monitorStarted = new Promise((resolve) => { @@ -389,12 +401,14 @@ describe('DeploymentManager', () => { return { success: true, message: 'ok', logs: 'apply logs' }; }); - const nativeMutationGate = jest.fn(async (action: () => Promise) => { + const nativeMutationGate = jest.fn(async (_deploy: Deploy, action: () => Promise) => { + gateDepth += 1; insideGate = true; try { return { admitted: true as const, value: await action() }; } finally { - insideGate = false; + gateDepth -= 1; + insideGate = gateDepth > 0; } }); const manager = new DeploymentManager([nativeHelm, kubernetes], { @@ -406,31 +420,31 @@ describe('DeploymentManager', () => { expect(insideGate).toBe(true); releaseMonitor(); - await expect(deployment).rejects.toThrow('helm failed'); + await expect(deployment).resolves.toEqual({ failed: [nativeHelm] }); expect(insideGate).toBe(false); expect(waitForDeployPodReady).toHaveBeenCalledWith(kubernetes); }); - it('treats denied native admission as supersession without creating a job or recording failure', async () => { + it('skips a service whose native admission is denied without creating a job or recording failure', async () => { const deploy = runnableDeploy('web', 'github'); const manager = new DeploymentManager([deploy], { nativeMutationGate: (async () => ({ admitted: false as const })) as any, }); - await expect(manager.deploy()).rejects.toBeInstanceOf(DeploymentSupersededError); + await expect(manager.deploy()).resolves.toEqual({ failed: [] }); expect(createKubernetesApplyJob).not.toHaveBeenCalled(); expect(monitorKubernetesJob).not.toHaveBeenCalled(); expect(mockRecordDeployFailure).not.toHaveBeenCalled(); }); - it('stops as superseded when the run-fenced queued patch affects no row', async () => { + it('skips a service as superseded when the run-fenced queued patch affects no row', async () => { const deploy = runnableDeploy('web', 'github'); const where = jest.fn().mockResolvedValue(0); deploy.$query = () => ({ patch: jest.fn().mockReturnValue({ where }) } as any); const manager = new DeploymentManager([deploy]); - await expect(manager.deploy()).rejects.toBeInstanceOf(DeploymentSupersededError); + await expect(manager.deploy()).resolves.toEqual({ failed: [] }); expect(where).toHaveBeenCalledWith({ id: deploy.id, runUUID: deploy.runUUID }); expect(createKubernetesApplyJob).not.toHaveBeenCalled(); @@ -449,7 +463,7 @@ describe('DeploymentManager', () => { const manager = new DeploymentManager([first, second], { isCurrent: async () => current, - nativeMutationGate: (async (action: () => Promise) => ({ + nativeMutationGate: (async (_deploy: Deploy, action: () => Promise) => ({ admitted: true as const, value: await action(), })) as any, @@ -619,7 +633,7 @@ describe('DeploymentManager', () => { return { deploy, patch, where }; } - it('removes a self-dependency and surfaces a Codefresh provider failure', async () => { + it('removes a self-dependency and records a Codefresh provider failure on the service', async () => { const { deploy, patch } = managedDeploy({ name: 'self-dependent-chart', type: 'helm', @@ -630,13 +644,16 @@ describe('DeploymentManager', () => { const manager = new DeploymentManager([deploy]); - await expect(manager.deploy()).rejects.toBe(providerError); - expect(deployHelm).toHaveBeenCalledWith([deploy]); + await expect(manager.deploy()).resolves.toEqual({ failed: [deploy] }); + expect(deployHelm).toHaveBeenCalledWith( + [deploy], + expect.objectContaining({ providerSubmissionGate: expect.any(Function) }) + ); expect(patch).toHaveBeenCalledWith({ status: DeployStatus.QUEUED }); expect(createKubernetesApplyJob).not.toHaveBeenCalled(); }); - it('surfaces an aggregated readiness failure from deploy()', async () => { + it('records a readiness failure on the service and returns it from deploy()', async () => { const { deploy } = managedDeploy({ name: 'unready-service' }); (waitForDeployPodReady as jest.Mock).mockResolvedValueOnce({ ready: false, @@ -645,13 +662,16 @@ describe('DeploymentManager', () => { const manager = new DeploymentManager([deploy]); - await expect(manager.deploy()).rejects.toThrow( - 'Pods failed to become ready within timeout: container waiting=ImagePullBackOff' - ); + await expect(manager.deploy()).resolves.toEqual({ failed: [deploy] }); expect(mockRecordDeployFailure).toHaveBeenCalledWith( deploy, deploy.runUUID, - expect.objectContaining({ status: DeployStatus.DEPLOY_FAILED }) + expect.objectContaining({ + status: DeployStatus.DEPLOY_FAILED, + error: expect.objectContaining({ + message: 'Pods failed to become ready within timeout: container waiting=ImagePullBackOff', + }), + }) ); expect(deployHelm).not.toHaveBeenCalled(); }); @@ -693,11 +713,14 @@ describe('DeploymentManager', () => { (waitForDeployPodReady as jest.Mock).mockResolvedValueOnce({ ready: false }); const manager = new DeploymentManager([deploy]); - await expect(manager.deploy()).rejects.toThrow('Pods failed to become ready within timeout'); + await expect(manager.deploy()).resolves.toEqual({ failed: [deploy] }); expect(mockRecordDeployFailure).toHaveBeenCalledWith( deploy, deploy.runUUID, - expect.objectContaining({ status: DeployStatus.DEPLOY_FAILED }) + expect.objectContaining({ + status: DeployStatus.DEPLOY_FAILED, + error: expect.objectContaining({ message: 'Pods failed to become ready within timeout' }), + }) ); }); @@ -720,13 +743,16 @@ describe('DeploymentManager', () => { const { deploy } = managedDeploy({ name: 'missing-manifest', manifest: '' }); const manager = new DeploymentManager([deploy]); - await expect(manager.deploy()).rejects.toThrow( - `Deploy ${deploy.uuid} has no manifest. Ensure manifests are generated before deployment.` - ); + await expect(manager.deploy()).resolves.toEqual({ failed: [deploy] }); expect(mockRecordDeployFailure).toHaveBeenCalledWith( deploy, deploy.runUUID, - expect.objectContaining({ status: DeployStatus.DEPLOY_FAILED }) + expect.objectContaining({ + status: DeployStatus.DEPLOY_FAILED, + error: expect.objectContaining({ + message: `Deploy ${deploy.uuid} has no manifest. Ensure manifests are generated before deployment.`, + }), + }) ); expect(createKubernetesApplyJob).not.toHaveBeenCalled(); expect(monitorKubernetesJob).not.toHaveBeenCalled(); @@ -742,7 +768,7 @@ describe('DeploymentManager', () => { }); const manager = new DeploymentManager([deploy]); - await expect(manager.deploy()).rejects.toThrow('Kubernetes apply job failed'); + await expect(manager.deploy()).resolves.toEqual({ failed: [deploy] }); expect(mockArchiveLogs).toHaveBeenCalledWith( expect.objectContaining({ status: 'Failed', deployUuid: deploy.uuid }), 'kubectl error output' @@ -755,24 +781,24 @@ describe('DeploymentManager', () => { expect(waitForDeployPodReady).not.toHaveBeenCalled(); }); - it('propagates supersession during manifest application without recording a provider failure', async () => { + it('skips a service superseded during manifest application without recording a provider failure', async () => { const { deploy } = managedDeploy({ name: 'superseded-apply' }); const superseded = new DeploymentSupersededError(); mockPatchAndUpdateActivityFeed.mockRejectedValueOnce(superseded); const manager = new DeploymentManager([deploy]); - await expect(manager.deploy()).rejects.toBe(superseded); + await expect(manager.deploy()).resolves.toEqual({ failed: [] }); expect(mockRecordDeployFailure).not.toHaveBeenCalled(); expect(createKubernetesApplyJob).not.toHaveBeenCalled(); }); - it('propagates supersession during readiness without recording a provider failure', async () => { + it('skips a service superseded during readiness without recording a provider failure', async () => { const { deploy } = managedDeploy({ name: 'superseded-readiness' }); const superseded = new DeploymentSupersededError(); (waitForDeployPodReady as jest.Mock).mockRejectedValueOnce(superseded); const manager = new DeploymentManager([deploy]); - await expect(manager.deploy()).rejects.toBe(superseded); + await expect(manager.deploy()).resolves.toEqual({ failed: [] }); expect(mockRecordDeployFailure).not.toHaveBeenCalled(); expect(mockPatchAndUpdateActivityFeed).not.toHaveBeenCalledWith( deploy, @@ -781,6 +807,428 @@ describe('DeploymentManager', () => { ); }); + it('blocks dependents of a service that failed before rollout and still deploys unrelated services', async () => { + const { deploy: db, patch: dbPatch } = managedDeploy({ name: 'db' }); + const { deploy: api, patch: apiPatch } = managedDeploy({ name: 'api', deploymentDependsOn: ['db'] }); + const { deploy: web } = managedDeploy({ name: 'web' }); + const manager = new DeploymentManager([db, api, web], { failedServices: ['db'] }); + + const result = await manager.deploy(); + + expect(result.failed.map((failed) => failed.deployable.name).sort()).toEqual(['api', 'db']); + expect(apiPatch).toHaveBeenCalledWith({ + status: DeployStatus.DEPLOY_FAILED, + statusMessage: 'Not deployed: db failed.', + }); + expect(dbPatch).not.toHaveBeenCalled(); + expect(createKubernetesApplyJob).toHaveBeenCalledTimes(1); + expect(createKubernetesApplyJob).toHaveBeenCalledWith(expect.objectContaining({ deploy: web })); + }); + + it('attributes a Codefresh failure to its own service and lets a healthy sibling finish', async () => { + const { deploy: good } = managedDeploy({ name: 'good-chart', type: 'helm' }); + const { deploy: bad } = managedDeploy({ name: 'bad-chart', type: 'helm' }); + (shouldUseNativeHelm as jest.Mock).mockResolvedValue(false); + (deployHelm as jest.Mock).mockImplementation(async (deploys: Deploy[]) => { + if (deploys[0].deployable.name === 'bad-chart') throw new Error('Codefresh deploy failed'); + }); + const manager = new DeploymentManager([good, bad]); + + await expect(manager.deploy()).resolves.toEqual({ failed: [bad] }); + expect(deployHelm).toHaveBeenCalledWith( + [good], + expect.objectContaining({ providerSubmissionGate: expect.any(Function) }) + ); + expect(deployHelm).toHaveBeenCalledWith( + [bad], + expect.objectContaining({ providerSubmissionGate: expect.any(Function) }) + ); + }); + + it('rethrows a lost promotion lease instead of recording it as a service failure', async () => { + const { deploy } = managedDeploy({ name: 'leased' }); + const lost = new AuthorityLockLostError('deploy-promotion.1'); + const manager = new DeploymentManager([deploy], { + nativeMutationGate: (async () => { + throw lost; + }) as any, + }); + + await expect(manager.deploy()).rejects.toBe(lost); + expect(mockRecordDeployFailure).not.toHaveBeenCalled(); + expect(waitForDeployPodReady).not.toHaveBeenCalled(); + }); + + it('records a service failed when its ownership check fails before the launch', async () => { + const { deploy } = managedDeploy({ name: 'checked' }); + const manager = new DeploymentManager([deploy], { + nativeMutationGate: (async () => { + throw new Error('connection reset'); + }) as any, + }); + + await expect(manager.deploy()).resolves.toEqual({ failed: [deploy] }); + expect(createKubernetesApplyJob).not.toHaveBeenCalled(); + expect(mockRecordDeployFailure).toHaveBeenCalledTimes(1); + expect(mockRecordDeployFailure).toHaveBeenCalledWith(deploy, 'run-checked', { + status: DeployStatus.DEPLOY_FAILED, + fallbackMessage: 'Deployment failed. Redeploy the service to try again.', + }); + }); + + it('records a Codefresh service failed when its ownership check fails before the launch', async () => { + const { deploy, patch } = managedDeploy({ name: 'chart', type: 'helm' }); + let queuedPatches = 0; + patch.mockImplementation(() => ({ + where: jest.fn(async () => { + queuedPatches += 1; + if (queuedPatches === 3) throw new Error('connection reset'); + return 1; + }), + })); + (shouldUseNativeHelm as jest.Mock).mockResolvedValue(false); + const manager = new DeploymentManager([deploy]); + + await expect(manager.deploy()).resolves.toEqual({ failed: [deploy] }); + expect(deployHelm).not.toHaveBeenCalled(); + expect(mockRecordDeployFailure).toHaveBeenCalledWith(deploy, 'run-chart', { + status: DeployStatus.DEPLOY_FAILED, + fallbackMessage: 'Deployment failed. Redeploy the service to try again.', + }); + }); + + it('records a failed apply once, with its own message', async () => { + const { deploy } = managedDeploy({ name: 'broken' }); + (createKubernetesApplyJob as jest.Mock).mockRejectedValueOnce(new Error('apply failed')); + const manager = new DeploymentManager([deploy], { + nativeMutationGate: (async (_deploy: Deploy, action: () => Promise) => ({ + admitted: true, + value: await action(), + })) as any, + }); + + await expect(manager.deploy()).resolves.toEqual({ failed: [deploy] }); + expect(mockRecordDeployFailure).toHaveBeenCalledTimes(1); + expect(mockRecordDeployFailure).toHaveBeenCalledWith( + deploy, + 'run-broken', + expect.objectContaining({ status: DeployStatus.DEPLOY_FAILED, error: expect.any(Error) }) + ); + }); + + it('lets launched siblings finish and launches nothing more once a promotion lease is lost', async () => { + const { deploy: leased } = managedDeploy({ name: 'leased' }); + const { deploy: slow } = managedDeploy({ name: 'slow' }); + const { deploy: dependent, patch: dependentPatch } = managedDeploy({ + name: 'dependent', + deploymentDependsOn: ['slow'], + }); + const lost = new AuthorityLockLostError('deploy-promotion.1'); + const events: string[] = []; + let finishSlow!: () => void; + const slowApply = new Promise((resolve) => { + finishSlow = resolve; + }); + (createKubernetesApplyJob as jest.Mock).mockImplementation(async ({ deploy }: { deploy: Deploy }) => { + if (deploy !== slow) return; + await slowApply; + events.push('slow applied'); + }); + const manager = new DeploymentManager([leased, slow, dependent], { + nativeMutationGate: (async (deploy: Deploy, action: () => Promise) => { + if (deploy === leased) throw lost; + return { admitted: true, value: await action() }; + }) as any, + }); + + try { + const outcome = manager.deploy().catch((error) => { + events.push('rejected'); + throw error; + }); + for (let i = 0; i < 20; i += 1) await new Promise((resolve) => setImmediate(resolve)); + expect(events).toEqual([]); + + finishSlow(); + await expect(outcome).rejects.toBe(lost); + expect(events).toEqual(['slow applied', 'rejected']); + expect(createKubernetesApplyJob).toHaveBeenCalledTimes(1); + expect(createKubernetesApplyJob).toHaveBeenCalledWith(expect.objectContaining({ deploy: slow })); + expect(dependentPatch).not.toHaveBeenCalledWith( + expect.objectContaining({ status: DeployStatus.DEPLOY_FAILED }) + ); + expect(mockRecordDeployFailure).not.toHaveBeenCalled(); + } finally { + (createKubernetesApplyJob as jest.Mock).mockResolvedValue(undefined); + } + }); + + it('waits for an earlier deploy job of the service to finish before creating a new one', async () => { + jest.useFakeTimers(); + try { + const { deploy, patch } = managedDeploy({ name: 'svc' }); + (listUnfinishedDeployJobs as jest.Mock).mockResolvedValueOnce(['svc-deploy-old']).mockResolvedValue([]); + const manager = new DeploymentManager([deploy]); + + const deployment = manager.deploy(); + await jest.advanceTimersByTimeAsync(5_000); + + await expect(deployment).resolves.toEqual({ failed: [] }); + expect(listUnfinishedDeployJobs).toHaveBeenCalledWith(deploy, 'testns'); + expect(patch).toHaveBeenCalledWith({ + status: DeployStatus.DEPLOYING, + statusMessage: 'Waiting for the previous deploy to finish', + }); + expect((listUnfinishedDeployJobs as jest.Mock).mock.invocationCallOrder[1]).toBeLessThan( + (createKubernetesApplyJob as jest.Mock).mock.invocationCallOrder[0] + ); + } finally { + jest.useRealTimers(); + } + }); + + it('waits for an earlier deploy job of the service before a native Helm deploy', async () => { + jest.useFakeTimers(); + try { + const { deploy } = managedDeploy({ name: 'chart', type: 'helm' }); + (shouldUseNativeHelm as jest.Mock).mockResolvedValue(true); + (listUnfinishedDeployJobs as jest.Mock).mockResolvedValueOnce(['chart-deploy-old']).mockResolvedValue([]); + const manager = new DeploymentManager([deploy]); + + const deployment = manager.deploy(); + await jest.advanceTimersByTimeAsync(5_000); + + await expect(deployment).resolves.toEqual({ failed: [] }); + expect((listUnfinishedDeployJobs as jest.Mock).mock.invocationCallOrder[1]).toBeLessThan( + (deployHelm as jest.Mock).mock.invocationCallOrder[0] + ); + } finally { + jest.useRealTimers(); + } + }); + + it('fails the service, without applying, when an earlier deploy job is still running after 45 minutes', async () => { + jest.useFakeTimers(); + try { + const { deploy } = managedDeploy({ name: 'svc' }); + (listUnfinishedDeployJobs as jest.Mock).mockResolvedValue(['svc-deploy-old']); + const manager = new DeploymentManager([deploy]); + + const deployment = manager.deploy(); + await jest.advanceTimersByTimeAsync(45 * 60_000 + 5_000); + + await expect(deployment).resolves.toEqual({ failed: [deploy] }); + expect(createKubernetesApplyJob).not.toHaveBeenCalled(); + expect(mockRecordDeployFailure).toHaveBeenCalledWith( + deploy, + deploy.runUUID, + expect.objectContaining({ + status: DeployStatus.DEPLOY_FAILED, + error: new Error( + 'The previous deploy of this service is still running after 45 minutes. Redeploy the service after it stops.' + ), + }) + ); + } finally { + jest.useRealTimers(); + } + }); + + it('stops waiting, without failing, once a newer run takes the service from this run', async () => { + const { deploy, where } = managedDeploy({ name: 'svc' }); + (listUnfinishedDeployJobs as jest.Mock).mockResolvedValue(['svc-deploy-old']); + where.mockResolvedValueOnce(1).mockResolvedValueOnce(1).mockResolvedValue(0); + const manager = new DeploymentManager([deploy]); + + await expect(manager.deploy()).resolves.toEqual({ failed: [] }); + expect(createKubernetesApplyJob).not.toHaveBeenCalled(); + expect(mockRecordDeployFailure).not.toHaveBeenCalled(); + }); + + it('does not apply and records a failure when earlier deploy jobs cannot be checked', async () => { + const { deploy } = managedDeploy({ name: 'svc' }); + (listUnfinishedDeployJobs as jest.Mock).mockRejectedValueOnce(new Error('jobs.batch is forbidden')); + const manager = new DeploymentManager([deploy]); + + await expect(manager.deploy()).resolves.toEqual({ failed: [deploy] }); + expect(createKubernetesApplyJob).not.toHaveBeenCalled(); + expect(mockRecordDeployFailure).toHaveBeenCalledWith( + deploy, + deploy.runUUID, + expect.objectContaining({ status: DeployStatus.DEPLOY_FAILED }) + ); + }); + + it('deploys a service whose prerequisite is outside this run without waiting, and its dependents after it', async () => { + const { deploy: api } = managedDeploy({ name: 'api', deploymentDependsOn: ['db'] }); + const { deploy: web } = managedDeploy({ name: 'web', deploymentDependsOn: ['api'] }); + const { deploy: worker } = managedDeploy({ name: 'worker' }); + const manager = new DeploymentManager([api, web, worker]); + + await expect(manager.deploy()).resolves.toEqual({ failed: [] }); + + const applied = (createKubernetesApplyJob as jest.Mock).mock.calls.map(([{ deploy }]) => deploy); + expect(applied).toHaveLength(3); + expect(applied.indexOf(api)).toBeLessThan(applied.indexOf(web)); + expect(mockRecordDeployFailure).not.toHaveBeenCalled(); + }); + + it('skips a service another run took while an earlier level was still deploying', async () => { + const { deploy: first } = managedDeploy({ name: 'first' }); + const { deploy: second, where } = managedDeploy({ name: 'second', deploymentDependsOn: ['first'] }); + where.mockResolvedValueOnce(1).mockResolvedValueOnce(0); + const manager = new DeploymentManager([first, second]); + + await expect(manager.deploy()).resolves.toEqual({ failed: [] }); + expect(createKubernetesApplyJob).toHaveBeenCalledTimes(1); + expect(createKubernetesApplyJob).toHaveBeenCalledWith(expect.objectContaining({ deploy: first })); + }); + + it('blocks a dependent of a failed prerequisite that never reaches the manager, such as a Codefresh service', async () => { + const { deploy: api, patch: apiPatch } = managedDeploy({ name: 'api', deploymentDependsOn: ['migrations'] }); + const { deploy: web } = managedDeploy({ name: 'web' }); + const manager = new DeploymentManager([api, web], { failedServices: ['migrations'] }); + + const result = await manager.deploy(); + + expect(result.failed).toEqual([api]); + expect(apiPatch).toHaveBeenCalledWith({ + status: DeployStatus.DEPLOY_FAILED, + statusMessage: 'Not deployed: migrations failed.', + }); + expect(createKubernetesApplyJob).toHaveBeenCalledTimes(1); + expect(createKubernetesApplyJob).toHaveBeenCalledWith(expect.objectContaining({ deploy: web })); + }); + + it('does not hand a Codefresh service to the provider once another run owns its row', async () => { + const { deploy, where } = managedDeploy({ name: 'chart', type: 'helm' }); + (shouldUseNativeHelm as jest.Mock).mockResolvedValue(false); + where.mockResolvedValueOnce(1).mockResolvedValueOnce(1).mockResolvedValueOnce(0); + const manager = new DeploymentManager([deploy]); + + await expect(manager.deploy()).resolves.toEqual({ failed: [] }); + expect(deployHelm).not.toHaveBeenCalled(); + }); + + it('hands the provider a submission gate that denies a row another run took during command generation', async () => { + const { deploy, where } = managedDeploy({ name: 'chart', type: 'helm' }); + (shouldUseNativeHelm as jest.Mock).mockResolvedValue(false); + where.mockResolvedValueOnce(1).mockResolvedValueOnce(1).mockResolvedValueOnce(1).mockResolvedValueOnce(0); + (deployHelm as jest.Mock).mockImplementation(async (_deploys: Deploy[], options: any) => { + if (!(await options.providerSubmissionGate(deploy))) throw new DeploymentSupersededError(); + }); + const manager = new DeploymentManager([deploy]); + + await expect(manager.deploy()).resolves.toEqual({ failed: [] }); + expect(mockRecordDeployFailure).not.toHaveBeenCalled(); + }); + + it('deploys a dependent without waiting when a newer run took its prerequisite from this run', async () => { + const { deploy: a, where: whereA } = managedDeploy({ name: 'a' }); + const { deploy: b } = managedDeploy({ name: 'b', deploymentDependsOn: ['a'] }); + whereA.mockResolvedValue(0); + const manager = new DeploymentManager([a, b]); + + await expect(manager.deploy()).resolves.toEqual({ failed: [] }); + + expect(createKubernetesApplyJob).toHaveBeenCalledTimes(1); + expect(createKubernetesApplyJob).toHaveBeenCalledWith(expect.objectContaining({ deploy: b })); + }); + + it('deploys a dependent without waiting when its prerequisite loses admission to a newer run', async () => { + const { deploy: a } = managedDeploy({ name: 'a' }); + const { deploy: b } = managedDeploy({ name: 'b', deploymentDependsOn: ['a'] }); + const manager = new DeploymentManager([a, b], { + nativeMutationGate: (async (deploy: Deploy, action: () => Promise) => + deploy === a ? { admitted: false as const } : { admitted: true as const, value: await action() }) as any, + }); + + await expect(manager.deploy()).resolves.toEqual({ failed: [] }); + + expect(createKubernetesApplyJob).toHaveBeenCalledTimes(1); + expect(createKubernetesApplyJob).toHaveBeenCalledWith(expect.objectContaining({ deploy: b })); + }); + + it('deploys a service whose declared prerequisite is outside this run without consulting it', async () => { + const { deploy: b, patch } = managedDeploy({ name: 'b', deploymentDependsOn: ['a'] }); + const manager = new DeploymentManager([b]); + + await expect(manager.deploy()).resolves.toEqual({ failed: [] }); + + expect(createKubernetesApplyJob).toHaveBeenCalledWith(expect.objectContaining({ deploy: b })); + expect(patch).not.toHaveBeenCalledWith(expect.objectContaining({ status: DeployStatus.DEPLOY_FAILED })); + }); + + it('deploys a prerequisite this run owns before its dependent', async () => { + const { deploy: a } = managedDeploy({ name: 'a' }); + const { deploy: b } = managedDeploy({ name: 'b', deploymentDependsOn: ['a'] }); + const manager = new DeploymentManager([a, b]); + + await expect(manager.deploy()).resolves.toEqual({ failed: [] }); + + const applied = (createKubernetesApplyJob as jest.Mock).mock.calls.map(([{ deploy }]) => deploy); + expect(applied).toEqual([a, b]); + }); + + it('launches a service whose prerequisite is absent without waiting for its level mates to finish', async () => { + const { deploy: a } = managedDeploy({ name: 'a' }); + const { deploy: b } = managedDeploy({ name: 'b', deploymentDependsOn: ['absent'] }); + let releaseA!: () => void; + const dependentLaunched = new Promise((resolve) => { + releaseA = resolve; + }); + (createKubernetesApplyJob as jest.Mock).mockImplementation(async ({ deploy }: { deploy: Deploy }) => { + if (deploy === b) releaseA(); + if (deploy === a) await dependentLaunched; + }); + const manager = new DeploymentManager([a, b]); + + try { + await expect(manager.deploy()).resolves.toEqual({ failed: [] }); + expect(createKubernetesApplyJob).toHaveBeenCalledTimes(2); + } finally { + (createKubernetesApplyJob as jest.Mock).mockResolvedValue(undefined); + } + }); + + it('starts a service as soon as its own prerequisite is done while an unrelated slow service is still applying', async () => { + const { deploy: slow } = managedDeploy({ name: 'slow' }); + const { deploy: x } = managedDeploy({ name: 'x' }); + const { deploy: z } = managedDeploy({ name: 'z', deploymentDependsOn: ['x'] }); + let releaseSlow!: () => void; + const zApplied = new Promise((resolve) => { + releaseSlow = resolve; + }); + (createKubernetesApplyJob as jest.Mock).mockImplementation(async ({ deploy }: { deploy: Deploy }) => { + if (deploy === z) releaseSlow(); + if (deploy === slow) await zApplied; + }); + const manager = new DeploymentManager([slow, x, z]); + + try { + await expect(manager.deploy()).resolves.toEqual({ failed: [] }); + expect(createKubernetesApplyJob).toHaveBeenCalledTimes(3); + } finally { + (createKubernetesApplyJob as jest.Mock).mockResolvedValue(undefined); + } + }); + + it('fences every ownership patch on the generation it was handed', async () => { + const { deploy, where } = managedDeploy({ name: 'chart', type: 'helm' }); + (shouldUseNativeHelm as jest.Mock).mockResolvedValue(false); + (deployHelm as jest.Mock).mockImplementation(async (_deploys: Deploy[], options: any) => { + await options.providerSubmissionGate(deploy); + }); + const manager = new DeploymentManager([deploy], { expectedGeneration: 4 }); + + await expect(manager.deploy()).resolves.toEqual({ failed: [] }); + + expect(where).toHaveBeenCalled(); + for (const [criteria] of where.mock.calls) { + expect(criteria).toEqual({ id: deploy.id, runUUID: deploy.runUUID, desiredGeneration: 4 }); + } + }); + it('claims a persisted deploy whose nullable run identity has not been assigned yet', async () => { // The deploys.runUUID database column is nullable even though the model field is typed as string. const { deploy, patch, where } = managedDeploy({ name: 'unclaimed-run', runUUID: null }); diff --git a/src/server/mcp/__tests__/deployRequestState.test.ts b/src/server/mcp/__tests__/deployRequestState.test.ts new file mode 100644 index 00000000..e331b8f9 --- /dev/null +++ b/src/server/mcp/__tests__/deployRequestState.test.ts @@ -0,0 +1,85 @@ +/** + * Copyright 2026 GoodRx, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +jest.mock('server/services/build', () => ({ + __esModule: true, + default: jest.fn().mockImplementation(() => ({ getBuildByUUID: jest.fn() })), +})); + +jest.mock('server/models/Repository', () => ({ + __esModule: true, + default: { query: jest.fn() }, +})); + +jest.mock('server/lib/environments/readiness', () => ({ + getEnvironmentPhase: jest.fn(), + isEnvironmentTerminal: jest.fn(), +})); + +jest.mock('../config', () => ({ + DEFAULT_MCP_WAIT_SECONDS: 10, + MAX_MCP_WAIT_SECONDS: 15, + loadMcpRuntimeConfig: jest.fn(), +})); + +jest.mock('../tools/core/getEnvironment', () => ({ + isEnvironmentBuild: jest.fn(), + serializeEnvironmentState: jest.fn(), +})); + +jest.mock('../tools/core/listRepositories', () => ({ + mapCoreToolError: jest.fn((error: unknown) => error), +})); + +import { deployRequestState } from '../tools/core/waitForEnvironment'; + +const row = (runUUID: string, desiredGeneration: number, observedGeneration: number, active = true) => ({ + runUUID, + desiredGeneration, + observedGeneration, + active, +}); + +describe('deployRequestState', () => { + it('settles a whole-environment deploy id once every active row has caught up', () => { + const build: any = { runUUID: 'env-run', deploys: [row('a', 3, 3), row('b', 3, 2, false)] }; + expect(deployRequestState(build, 'env-run')).toBe('settled'); + + build.deploys = [row('a', 3, 3), row('b', 3, 2)]; + expect(deployRequestState(build, 'env-run')).toBe('pending'); + }); + + it('follows the rows a service-scoped run owns', () => { + const build: any = { runUUID: 'env-run', deploys: [row('run-b', 4, 3), row('run-c', 2, 2)] }; + expect(deployRequestState(build, 'run-b')).toBe('pending'); + expect(deployRequestState(build, 'run-c')).toBe('settled'); + }); + + it('falls back to the accepted entry when the rows were taken by a newer intent', () => { + const build: any = { + runUUID: 'env-run', + deploys: [row('run-newer', 5, 4)], + acceptedRefs: { + 'source:1:main': { requestId: 'run-old', gen: 4, observedGen: 4 }, + 'source:2:main': { requestId: 'run-waiting', gen: 5 }, + }, + }; + expect(deployRequestState(build, 'run-old')).toBe('settled'); + expect(deployRequestState(build, 'run-waiting')).toBe('pending'); + expect(deployRequestState(build, 'run-unknown')).toBe('unknown'); + expect(deployRequestState({ runUUID: 'env-run', deploys: [], acceptedRefs: null } as any, 'x')).toBe('unknown'); + }); +}); diff --git a/src/server/mcp/tools/core/waitForEnvironment.ts b/src/server/mcp/tools/core/waitForEnvironment.ts index 2434c5fc..98b26e98 100644 --- a/src/server/mcp/tools/core/waitForEnvironment.ts +++ b/src/server/mcp/tools/core/waitForEnvironment.ts @@ -257,6 +257,24 @@ interface EvaluatedWait { note?: string; } +type DeployRequestState = 'settled' | 'pending' | 'unknown'; + +/** A deploy id is settled once every service row it selected has caught up with its desired generation. */ +export function deployRequestState(build: LoadedEnvironment['build'], deployId: string): DeployRequestState { + const rows = (build.deploys ?? []).filter((deploy) => deploy.active !== false); + const pendingRow = (deploy: { desiredGeneration?: number; observedGeneration?: number }) => + Number(deploy.desiredGeneration) > Number(deploy.observedGeneration); + if (build.runUUID === deployId) return rows.some(pendingRow) ? 'pending' : 'settled'; + const owned = rows.filter((deploy) => deploy.runUUID === deployId); + if (owned.length > 0) return owned.some(pendingRow) ? 'pending' : 'settled'; + const refs = build.acceptedRefs; + const entry = + refs && typeof refs === 'object' ? Object.values(refs).find((intent) => intent?.requestId === deployId) : undefined; + // An accepted entry whose job finished, including one whose rows a newer intent took over, is settled. + if (entry) return entry.observedGen === entry.gen ? 'settled' : 'pending'; + return 'unknown'; +} + function evaluateWait( target: EnvironmentWaitLoadedTarget, goal: EnvironmentWaitGoal, @@ -281,7 +299,7 @@ function evaluateWait( note: 'This exact environment is being or has been destroyed, so the deploy cannot finish.', }; } - if (build.runUUID !== deployId) { + if (deployRequestState(build, deployId) !== 'settled') { return {}; } } @@ -408,7 +426,7 @@ export function createWaitForEnvironmentToolDefinition( const liveCurrent = current as Extract; const environment = serializeEnvironmentState(liveCurrent.loaded, { format: 'concise' }); const phase = getEnvironmentPhase(liveCurrent.loaded.build); - if (deployId && liveCurrent.loaded.build.runUUID !== deployId) { + if (deployId && deployRequestState(liveCurrent.loaded.build, deployId) === 'unknown') { return { target: { uuid, environmentId }, result: { diff --git a/src/server/models/Deploy.ts b/src/server/models/Deploy.ts index df729d4e..2f6facca 100644 --- a/src/server/models/Deploy.ts +++ b/src/server/models/Deploy.ts @@ -47,6 +47,8 @@ export default class Deploy extends Model { deployable?: Deployable; runUUID: string; + desiredGeneration!: number; + observedGeneration!: number; replicaCount: number; diff --git a/src/server/services/__tests__/apiEnvironment.test.ts b/src/server/services/__tests__/apiEnvironment.test.ts index 53aec922..d953106b 100644 --- a/src/server/services/__tests__/apiEnvironment.test.ts +++ b/src/server/services/__tests__/apiEnvironment.test.ts @@ -152,6 +152,7 @@ function makeService({ const deployQueryChain: any = { patch: jest.fn(() => deployQueryChain), where: jest.fn(() => deployQueryChain), + select: jest.fn(() => ({ where: jest.fn().mockResolvedValue([]) })), }; const models = { Build: { @@ -160,6 +161,7 @@ function makeService({ query: jest.fn(() => ({ findOne: jest.fn(() => buildFindOneChain), findById: jest.fn(), + select: jest.fn(() => ({ findById: jest.fn(() => ({ forUpdate: jest.fn().mockResolvedValue(undefined) })) })), patchAndFetchById: jest.fn(async (_id: number, patch: any) => ({ uuid: 'x', ...patch })), patch: jest.fn().mockReturnThis(), where: jest.fn().mockReturnThis(), @@ -2090,7 +2092,7 @@ describe('applyApiEnvironmentPatch', () => { grantLock({ unlock, extend: jest.fn() }); await pause; - expect((service as any).redlock.lock).toHaveBeenCalledWith('build-deployment.1', 15 * 60 * 1000); + expect((service as any).redlock.lock).toHaveBeenCalledWith('build-deployment.1', 60_000); expect(patchFn).toHaveBeenCalledWith( expect.objectContaining({ deployEnabled: false, runUUID: expect.any(String) }) ); @@ -3187,13 +3189,13 @@ describe('processDeleteQueue', () => { expect(lockWithOptions).toHaveBeenNthCalledWith( 1, 'build-deployment.3', - 15 * 60 * 1000, + 60_000, expect.objectContaining({ retryCount: 4 }) ); expect(lockWithOptions).toHaveBeenNthCalledWith( 2, 'build-promotion.3', - 15 * 60 * 1000, + 60_000, expect.objectContaining({ retryCount: 4 }) ); }); @@ -3868,15 +3870,7 @@ describe('recordBuildFailure teardown ownership', () => { await (service as any).recordBuildFailure(build, BuildStatus.ERROR, 'new-run', new Error('boom'), 'fallback'); - expect(updateStatus).toHaveBeenCalledWith( - build, - BuildStatus.ERROR, - 'new-run', - true, - true, - expect.any(Error), - undefined - ); + expect(updateStatus).toHaveBeenCalledWith(build, BuildStatus.ERROR, 'new-run', true, true, expect.any(Error)); }); it('re-stamps PR builds unconditionally', async () => { diff --git a/src/server/services/__tests__/build.test.ts b/src/server/services/__tests__/build.test.ts index 10925488..7f62269c 100644 --- a/src/server/services/__tests__/build.test.ts +++ b/src/server/services/__tests__/build.test.ts @@ -221,8 +221,8 @@ import { ParsingError, YamlConfigParser } from 'server/lib/yamlConfigParser'; import { ValidationError, YamlConfigValidator } from 'server/lib/yamlConfigValidator'; import { DeploymentManager, DeploymentSupersededError } from 'server/lib/deploymentManager/deploymentManager'; import { UniqueViolationError } from 'objection'; -import AgentPrewarmService from 'server/services/agentPrewarm'; import { AuthorityLockLostError } from 'server/lib/authorityLock'; +import AgentPrewarmService from 'server/services/agentPrewarm'; import { LifecycleError } from 'server/lib/errors'; function createThenableQuery(result: any[] = []) { @@ -556,7 +556,8 @@ describe('BuildService status updates', () => { status: BuildStatus.DEPLOYED, statusMessage: '', }); - expect(query.where).toHaveBeenCalledWith({ id: 1, runUUID: 'run-1' }); + expect(query.where).toHaveBeenCalledWith({ id: 1 }); + expect(query.where).toHaveBeenCalledWith('runUUID', 'run-1'); }); test('does not abort teardown status progress when webhook notification enqueue fails', async () => { @@ -600,44 +601,6 @@ describe('BuildService status updates', () => { }); expect(webhookAdd).toHaveBeenCalledTimes(1); }); - - test('does not publish after a newer desired generation takes ownership', async () => { - const query = statusQuery(0); - const webhookAdd = jest.fn(); - const activityUpdate = jest.fn(); - const buildService = new BuildService( - { - models: { Build: { query: jest.fn(() => query) } }, - services: { - ActivityStream: { updatePullRequestActivityStream: activityUpdate }, - Webhook: { webhookQueue: { add: webhookAdd } }, - }, - } as any, - {} as any, - {} as any, - { - registerQueue: jest.fn(() => ({ add: mockQueueAdd, process: jest.fn(), on: jest.fn() })), - } as any - ); - const build = { - id: 1, - uuid: 'sample-build', - runUUID: 'run-a', - status: BuildStatus.DEPLOYING, - kind: BuildKind.ENVIRONMENT, - deploys: [], - pullRequest: { repository: {} }, - reload: jest.fn().mockResolvedValue(undefined), - $fetchGraph: jest.fn().mockResolvedValue(undefined), - }; - - await buildService.updateStatusAndComment(build as any, BuildStatus.DEPLOYED, 'run-a', true, true, null, 2); - - expect(query.where).toHaveBeenCalledWith('desiredGeneration', 2); - expect(build.status).toBe(BuildStatus.DEPLOYING); - expect(activityUpdate).not.toHaveBeenCalled(); - expect(webhookAdd).not.toHaveBeenCalled(); - }); }); describe('BuildService destroyBuildEnvironment', () => { @@ -1268,8 +1231,8 @@ describe('BuildService stale deploy reconciliation', () => { return true; }); const promotion = jest - .spyOn(buildService, 'withCurrentBuildPromotionLock') - .mockImplementation(async (_buildId, isCurrent, action) => { + .spyOn(buildService, 'withCurrentDeployPromotionLock') + .mockImplementation(async (_deployId, isCurrent, action) => { expect(await isCurrent()).toBe(true); return { admitted: true, value: await action() }; }); @@ -1284,7 +1247,8 @@ describe('BuildService stale deploy reconciliation', () => { 3 ); - expect(promotion).toHaveBeenCalledWith(10, expect.any(Function), nativeAction); + // Removal serializes with the service's own rollout lock, not the environment-wide one. + expect(promotion).toHaveBeenCalledWith(77, expect.any(Function), nativeAction); expect(mockDeleteServiceRows).toHaveBeenCalledWith({ buildId: 10, deployableIds: [1] }); }); @@ -1294,7 +1258,7 @@ describe('BuildService stale deploy reconciliation', () => { await options.nativeMutationGate(async () => true); return true; }); - jest.spyOn(buildService, 'withCurrentBuildPromotionLock').mockResolvedValue({ admitted: false }); + jest.spyOn(buildService, 'withCurrentDeployPromotionLock').mockResolvedValue({ admitted: false }); await expect( (buildService as any).reconcileDeletedDeployables( @@ -1380,7 +1344,8 @@ describe('BuildService stale deploy reconciliation', () => { targetRepoId, undefined, undefined, - targetRepoId + targetRepoId, + undefined ); expect(reconcileDeletedDeployables).not.toHaveBeenCalled(); expect(upsertWebhooksWithYaml).toHaveBeenCalledWith(build, build.pullRequest, null); @@ -1424,102 +1389,11 @@ describe('BuildService deployment reconciliation', () => { return { service, buildQuery, add }; }; - const reconciliationWorkerHarness = () => { - const { service } = serviceHarness(); - const failure = new Error('reconciliation infrastructure failed'); - const claim = { - generation: 7, - token: 'run-current', - dirty: [{ scopeKey: 'all', intent: { type: 'all', requestId: 'run-current', gen: 7 } }], - }; - const build = createBuild({ id: 1, runUUID: claim.token }); - - jest - .spyOn(service as any, 'tryWithDeploymentGenerationLock') - .mockImplementation(async (...args: any[]) => (args[2] as () => Promise)()); - const claimReconciliation = jest.spyOn(service as any, 'claimDeploymentReconciliation').mockResolvedValue(claim); - const withDeploymentLock = jest - .spyOn(service as any, 'withCurrentBuildDeploymentLock') - .mockImplementation(async (...args: any[]) => { - const isCurrent = args[1] as () => Promise; - const action = args[2] as () => Promise; - expect(await isCurrent()).toBe(true); - return { admitted: true, value: await action() }; - }); - const loadBuild = jest.spyOn(service as any, 'loadBuildDeploymentAuthority').mockResolvedValue(build); - jest.spyOn(service as any, 'claimDeploymentRun').mockResolvedValue(claim.token); - jest.spyOn(service as any, 'deploymentReconciliationScopes').mockImplementation(() => { - throw failure; - }); - const isCurrent = jest.spyOn(service as any, 'isDeploymentRunCurrent').mockResolvedValue(true); - const recordFailure = jest.spyOn(service as any, 'recordBuildFailure').mockResolvedValue(undefined); - const markObserved = jest.spyOn(service as any, 'markDeploymentReconciliationObserved').mockResolvedValue(true); - - const job = (attemptsMade: number) => ({ - data: { buildId: 1, generation: claim.generation }, - attemptsMade, - opts: { attempts: 10 }, - }); - - return { - service, - failure, - claim, - build, - claimReconciliation, - withDeploymentLock, - loadBuild, - isCurrent, - recordFailure, - markObserved, - job, - }; - }; - beforeEach(() => { jest.clearAllMocks(); mockAcceptDeploymentIntent.mockResolvedValue({ accepted: true, generation: 1, scopeKey: 'all' }); }); - const deploymentScopeHarness = (buildImagesResult: boolean) => { - const { service } = serviceHarness(); - const build = createBuild({ id: 1, runUUID: 'run-current', namespace: 'env-sample' }); - jest.spyOn(service as any, 'isDeploymentRunCurrent').mockResolvedValue(true); - const buildImages = jest.spyOn(service as any, 'buildImages').mockResolvedValue(buildImagesResult); - jest.spyOn(service as any, 'deployCLIServices').mockResolvedValue(true); - const updateStatus = jest.spyOn(service as any, 'updateStatusAndComment').mockResolvedValue(undefined); - const applyManifests = jest.spyOn(service as any, 'generateAndApplyManifests').mockResolvedValue(true); - const preparation = { - build, - runUUID: 'run-current', - githubRepositoryId: 100, - sourceGithubRepositoryId: 100, - sourceRef: 'commit-a', - sourceBranch: 'main', - }; - return { service, preparation, buildImages, updateStatus, applyManifests }; - }; - - test('a failed image phase reports ERROR and never reaches manifest apply', async () => { - const { service, preparation, buildImages, updateStatus, applyManifests } = deploymentScopeHarness(false); - - const result = await (service as any).executeDeploymentScope(preparation, 7); - - expect(result).toEqual({ status: BuildStatus.ERROR }); - expect(buildImages).toHaveBeenCalledTimes(1); - expect(updateStatus).not.toHaveBeenCalled(); - expect(applyManifests).not.toHaveBeenCalled(); - }); - - test('a successful image phase proceeds to manifest apply and reports DEPLOYED', async () => { - const { service, preparation, applyManifests } = deploymentScopeHarness(true); - - const result = await (service as any).executeDeploymentScope(preparation, 7); - - expect(result).toEqual({ status: BuildStatus.DEPLOYED }); - expect(applyManifests).toHaveBeenCalledTimes(1); - }); - test('signals only the exact durable generation accepted by the mailbox', async () => { const { service, add } = serviceHarness(); @@ -1668,15 +1542,6 @@ describe('BuildService deployment reconciliation', () => { expect(add).not.toHaveBeenCalled(); }); - test('the recovery sweep re-signals durable pending work', async () => { - const { service, buildQuery, add } = serviceHarness(); - buildQuery.limit.mockResolvedValueOnce([{ id: 7, desiredGeneration: '3' }]); - - await service.enqueuePendingDeploymentReconciliations(); - - expect(add).toHaveBeenCalledWith('reconcile', { buildId: 7, generation: 3 }, { jobId: 'reconcile-7-3' }); - }); - test('service redeploy uses the effective source repository and leaves row ownership to the worker', async () => { const { service, buildQuery, add } = serviceHarness(); const directPatch = jest.fn(); @@ -1737,211 +1602,6 @@ describe('BuildService deployment reconciliation', () => { expect(add).not.toHaveBeenCalled(); }); - test('repository redeploy bulk-claims every Deploy from that repository without a service-id predicate', async () => { - const deployClaim: any = { - patch: jest.fn(() => deployClaim), - where: jest.fn(() => deployClaim), - then: (resolve: (value: number) => void, reject: (reason: unknown) => void) => - Promise.resolve(2).then(resolve, reject), - }; - const deploys = [ - { id: 11, githubRepositoryId: 42, branchName: 'main' }, - { id: 12, githubRepositoryId: 42, branchName: 'release' }, - { id: 13, githubRepositoryId: 99, branchName: 'main' }, - ]; - const findOrCreateDeploys = jest.fn().mockResolvedValue(deploys); - const service = new BuildService( - { - models: { Deploy: { query: jest.fn(() => deployClaim) } }, - services: { Deploy: { findOrCreateDeploys } }, - } as any, - {} as any, - {} as any, - queueManager() as any - ); - jest.spyOn(service as any, 'isDeploymentRunCurrent').mockResolvedValue(true); - jest.spyOn(service, 'markConfigurationsAsBuilt').mockResolvedValue(undefined); - jest.spyOn(service, 'updateStatusAndComment').mockResolvedValue(undefined); - mockGenerateGraph.mockRejectedValueOnce(new Error('graph omitted from scope assertion')); - const build: any = { - id: 4, - uuid: 'large-static', - runUUID: 'run-c', - environment: { id: 7 }, - pullRequest: { - fullName: 'org/root', - branchName: 'main', - latestCommit: 'root-sha', - repository: { githubRepositoryId: 1 }, - $fetchGraph: jest.fn().mockResolvedValue(undefined), - }, - $fetchGraph: jest.fn().mockResolvedValue(undefined), - $setRelated: jest.fn(), - }; - - await (service as any).prepareDeploymentScope(build, 42, undefined, { - runUUID: 'run-c', - }); - - expect(deployClaim.patch).toHaveBeenCalledWith({ runUUID: 'run-c' }); - expect(deployClaim.where).toHaveBeenCalledWith({ buildId: 4 }); - expect(deployClaim.where).toHaveBeenCalledWith('githubRepositoryId', 42); - expect(deployClaim.where).not.toHaveBeenCalledWith('branchName', expect.anything()); - expect(deployClaim.where).not.toHaveBeenCalledWith('id', expect.anything()); - expect(deploys.map((deploy: any) => deploy.runUUID)).toEqual(['run-c', 'run-c', undefined]); - }); - - test('a stale B signal cannot claim or execute C mailbox contents', async () => { - const row = { - desiredGeneration: 3, - observedGeneration: 0, - acceptedRefs: { - 'source:100:main': { - type: 'source', - requestId: 'request-c', - target: 'repository', - githubRepositoryId: 100, - branch: 'main', - sha: 'commit-c', - gen: 3, - }, - }, - }; - const query: any = { - findById: jest.fn(() => query), - whereNull: jest.fn().mockResolvedValue(row), - }; - const service = new BuildService( - { models: { Build: { query: jest.fn(() => query) } } } as any, - {} as any, - {} as any, - queueManager() as any - ); - - await expect((service as any).claimDeploymentReconciliation(1, 2)).resolves.toBeNull(); - await expect((service as any).claimDeploymentReconciliation(1, 3)).resolves.toEqual({ - generation: 3, - token: 'request-c', - dirty: [ - { - scopeKey: 'source:100:main', - intent: row.acceptedRefs['source:100:main'], - }, - ], - }); - }); - - test('repository-scoped work stays selective while a root-source intent remains full-scope', () => { - const { service } = serviceHarness(); - const scopes = (service as any).deploymentReconciliationScopes([ - { - scopeKey: 'source:100:main', - intent: { - type: 'source', - requestId: 'repo-c', - target: 'repository', - githubRepositoryId: 100, - branch: 'main', - sha: 'commit-c', - gen: 1, - }, - }, - { - scopeKey: 'source:200:main', - intent: { - type: 'source', - requestId: 'root-d', - target: 'all', - githubRepositoryId: 200, - branch: 'main', - sha: 'commit-d', - gen: 2, - }, - }, - ]); - - // The full pass runs first, then the delivered repo SHA is re-applied as a - // selective floor so a lagging live-head read cannot roll C back to B. - expect(scopes).toEqual([ - { - githubRepositoryId: null, - sourceGithubRepositoryId: 200, - sourceRef: 'commit-d', - sourceBeforeRef: undefined, - sourceBranch: 'main', - }, - { - githubRepositoryId: 100, - sourceGithubRepositoryId: 100, - sourceRef: 'commit-c', - sourceBeforeRef: undefined, - sourceBranch: 'main', - }, - ]); - }); - - test('manual full redeploy cannot erase an older delivered repository SHA', () => { - const { service } = serviceHarness(); - const scopes = (service as any).deploymentReconciliationScopes([ - { - scopeKey: 'source:100:main', - intent: { - type: 'source', - requestId: 'repo-c', - target: 'repository', - githubRepositoryId: 100, - branch: 'main', - sha: 'commit-c', - gen: 1, - }, - }, - { scopeKey: 'all', intent: { type: 'all', requestId: 'manual-all', gen: 2 } }, - ]); - - expect(scopes).toEqual([ - { githubRepositoryId: null }, - { - githubRepositoryId: 100, - sourceGithubRepositoryId: 100, - sourceRef: 'commit-c', - sourceBeforeRef: undefined, - sourceBranch: 'main', - }, - ]); - }); - - test('starts the newest repository source before retained work for another repository', () => { - const { service } = serviceHarness(); - const scopes = (service as any).deploymentReconciliationScopes([ - { - scopeKey: 'source:100:main', - intent: { - type: 'source', - requestId: 'repo-a', - target: 'repository', - githubRepositoryId: 100, - branch: 'main', - sha: 'commit-a', - gen: 1, - }, - }, - { - scopeKey: 'source:200:main', - intent: { - type: 'source', - requestId: 'repo-b', - target: 'repository', - githubRepositoryId: 200, - branch: 'main', - sha: 'commit-b', - gen: 2, - }, - }, - ]); - - expect(scopes.map((scope) => scope.githubRepositoryId)).toEqual([200, 100]); - }); - test('different generations use different execution locks so C does not wait for A', async () => { let releaseA!: () => void; let signalAStarted!: () => void; @@ -2004,116 +1664,6 @@ describe('BuildService deployment reconciliation', () => { expect(applyManifests).not.toHaveBeenCalled(); }); - test('leaves the latest generation pending while PR teardown is in progress', async () => { - const { service, build, markObserved, job } = reconciliationWorkerHarness(); - build.status = BuildStatus.TEARING_DOWN; - - await expect(service.processDeploymentReconciliationQueue(job(0))).resolves.toBeUndefined(); - - expect((service as any).claimDeploymentRun).not.toHaveBeenCalled(); - expect((service as any).deploymentReconciliationScopes).not.toHaveBeenCalled(); - expect(markObserved).not.toHaveBeenCalled(); - }); - - test('retries an intermediate generic reconciliation failure without publishing a terminal error', async () => { - const { service, failure, recordFailure, markObserved, job } = reconciliationWorkerHarness(); - - await expect(service.processDeploymentReconciliationQueue(job(0))).rejects.toBe(failure); - - expect(recordFailure).not.toHaveBeenCalled(); - expect(markObserved).not.toHaveBeenCalled(); - }); - - test('retries a still-current authority-lock loss without publishing terminal failure', async () => { - const { service, recordFailure, markObserved, job } = reconciliationWorkerHarness(); - const lockError = new AuthorityLockLostError('build-deployment.1'); - jest.spyOn(service as any, 'deploymentReconciliationScopes').mockImplementation(() => { - throw lockError; - }); - - await expect(service.processDeploymentReconciliationQueue(job(0))).rejects.toBe(lockError); - - expect(recordFailure).not.toHaveBeenCalled(); - expect(markObserved).not.toHaveBeenCalled(); - }); - - test('ignores a reconciliation failure after generation authority has moved on', async () => { - const { service, recordFailure, markObserved, isCurrent, job } = reconciliationWorkerHarness(); - isCurrent.mockResolvedValue(false); - - await expect(service.processDeploymentReconciliationQueue(job(0))).resolves.toBeUndefined(); - - expect(recordFailure).not.toHaveBeenCalled(); - expect(markObserved).not.toHaveBeenCalled(); - }); - - test('publishes and observes one fenced generic failure on the final queue attempt', async () => { - const { service, failure, build, claim, recordFailure, markObserved, job } = reconciliationWorkerHarness(); - - await expect(service.processDeploymentReconciliationQueue(job(9))).resolves.toBeUndefined(); - - expect(recordFailure).toHaveBeenCalledTimes(1); - expect(recordFailure).toHaveBeenCalledWith( - build, - BuildStatus.ERROR, - claim.token, - failure, - 'Build queue processing failed.', - claim.generation - ); - expect(markObserved).toHaveBeenCalledTimes(1); - expect(markObserved).toHaveBeenCalledWith(1, claim.generation, claim.token); - }); - - test('rethrows a still-current final failure when its terminal write cannot be confirmed', async () => { - const { service, failure, job } = reconciliationWorkerHarness(); - jest.spyOn(service as any, 'recordFinalDeploymentReconciliationFailure').mockResolvedValue(false); - - await expect(service.processDeploymentReconciliationQueue(job(9))).rejects.toBe(failure); - }); - - test('rethrows a still-current generic failure that happens before the run becomes active', async () => { - const { service, failure, loadBuild, recordFailure, markObserved, job } = reconciliationWorkerHarness(); - loadBuild.mockRejectedValueOnce(failure); - - await expect(service.processDeploymentReconciliationQueue(job(0))).rejects.toBe(failure); - - expect(recordFailure).not.toHaveBeenCalled(); - expect(markObserved).not.toHaveBeenCalled(); - }); - - test('claims the token to publish and observe a pre-active failure only on its final attempt', async () => { - const { service, failure, build, claim, withDeploymentLock, loadBuild, recordFailure, markObserved, job } = - reconciliationWorkerHarness(); - loadBuild.mockRejectedValueOnce(failure); - - await expect(service.processDeploymentReconciliationQueue(job(9))).resolves.toBeUndefined(); - - expect(withDeploymentLock).toHaveBeenCalledTimes(2); - expect(recordFailure).toHaveBeenCalledTimes(1); - expect(recordFailure).toHaveBeenCalledWith( - build, - BuildStatus.ERROR, - claim.token, - failure, - 'Build queue processing failed.', - claim.generation - ); - expect(markObserved).toHaveBeenCalledWith(1, claim.generation, claim.token); - }); - - test('does not mistake an authority-read failure for proof that a failed pass is stale', async () => { - const { service, failure, isCurrent, recordFailure, markObserved, job } = reconciliationWorkerHarness(); - const authorityError = new Error('authority read unavailable'); - isCurrent.mockRejectedValueOnce(authorityError); - - await expect(service.processDeploymentReconciliationQueue(job(0))).rejects.toBe(authorityError); - - expect(recordFailure).not.toHaveBeenCalled(); - expect(markObserved).not.toHaveBeenCalled(); - expect(failure).not.toBe(authorityError); - }); - test('deployment scope fails closed when an execution error cannot be fenced by an authority read', async () => { const executionError = new Error('image builder unavailable'); const authorityError = new Error('build authority unavailable'); @@ -2145,7 +1695,7 @@ describe('BuildService deployment reconciliation', () => { }, 7 ) - ).resolves.toBeNull(); + ).rejects.toBe(authorityError); expect(buildQuery.select).toHaveBeenCalledWith( 'id', @@ -2360,6 +1910,7 @@ describe('BuildService focused changed-line coverage', () => { const deployQuery = (deploys: any[]) => { const query: any = { where: jest.fn(() => query), + whereNotIn: jest.fn(() => query), withGraphFetched: jest.fn().mockResolvedValue(deploys), }; return query; @@ -2436,6 +1987,8 @@ describe('BuildService focused changed-line coverage', () => { 'deployPipelineId', 'githubRepositoryId', 'runUUID', + 'desiredGeneration', + 'observedGeneration', 'publicUrl', 'dockerImage', 'buildLogs', @@ -2503,7 +2056,7 @@ describe('BuildService focused changed-line coverage', () => { expect(buildImage.mock.calls.map(([deploy]) => deploy.uuid)).toEqual(['docker', 'github', 'helm']); }); - test('keeps every static execution lane scoped to the changed repository and branch', async () => { + test('keeps legacy lanes scoped to the changed repository and branch while a fenced lane takes exactly the rows it claimed', async () => { const imageQuery = deployQuery([]); const cliQuery = deployQuery([]); const manifestQuery = deployQuery([]); @@ -2532,7 +2085,7 @@ describe('BuildService focused changed-line coverage', () => { namespace: build.namespace, }); - for (const query of [imageQuery, cliQuery, manifestQuery]) { + for (const query of [imageQuery, cliQuery]) { expect(query.where).toHaveBeenCalledWith({ buildId: 4, runUUID: 'run-c', @@ -2540,6 +2093,8 @@ describe('BuildService focused changed-line coverage', () => { branchName: 'main', }); } + expect(manifestQuery.where).toHaveBeenCalledWith({ buildId: 4, runUUID: 'run-c' }); + expect(manifestQuery.where).toHaveBeenCalledWith('desiredGeneration', 3); }); test('treats a retained targeted scope with no remaining Deploy rows as a successful no-op', async () => { @@ -2550,10 +2105,88 @@ describe('BuildService focused changed-line coverage', () => { await expect(service.deployCLIServices(build as any, 'run-c', 42, 'sha-c', 'main')).resolves.toBe(true); }); + test('an aborted rollout does not fail a service whose prerequisite is outside the run', async () => { + const managed = (name: string, deploymentDependsOn: string[]) => { + const where = jest.fn().mockResolvedValue(1); + const patch = jest.fn().mockReturnValue({ where }); + const deploy = { + id: name.length, + uuid: name, + runUUID: 'run-g', + manifest: 'kind: ConfigMap', + build: { namespace: 'ns' }, + deployable: { name, type: 'github', deploymentDependsOn }, + service: { type: 'github' }, + $query: () => ({ patch }), + $fetchGraph: jest.fn().mockResolvedValue(undefined), + } as any; + return { deploy, patch }; + }; + const a = managed('a', []); + const b = managed('b', ['x']); + const lost = new AuthorityLockLostError('deploy-promotion.1'); + const manager = new DeploymentManager([a.deploy, b.deploy], { + nativeMutationGate: (async (deploy: any, action: () => Promise) => { + if (deploy === a.deploy) throw lost; + return { admitted: true, value: await action() }; + }) as any, + }); + + await expect(manager.deploy()).rejects.toBe(lost); + expect(b.patch).not.toHaveBeenCalledWith(expect.objectContaining({ status: DeployStatus.DEPLOY_FAILED })); + }); + + test('acceptedSourcePins keeps the newest accepted SHA per repository and branch, including full-environment pushes', () => { + const service = serviceWith({}); + + const pins = service['acceptedSourcePins']({ + 'source:100:main': { + type: 'source', + target: 'repository', + requestId: 'r3', + githubRepositoryId: 100, + branch: 'main', + sha: 'a1', + gen: 3, + }, + 'source:100:main:all': { + type: 'source', + target: 'all', + requestId: 'r5', + githubRepositoryId: 100, + branch: 'main', + sha: 'a2', + gen: 5, + observedGen: 5, + }, + 'source:200:main:all': { + type: 'source', + target: 'all', + requestId: 'r6', + githubRepositoryId: 200, + branch: 'main', + sha: 'b1', + gen: 6, + }, + 'repository:300': { type: 'repository', requestId: 'r7', githubRepositoryId: 300, gen: 7 }, + all: { type: 'all', requestId: 'r8', gen: 8 }, + }); + + expect(pins).toEqual({ '100:main': 'a2', '200:main': 'b1' }); + }); + test('fails image and CLI processing cleanly for a loaded deploy missing its deployable', async () => { const missing = { uuid: 'missing', active: true }; mockDeployQuery.mockReturnValue(deployQuery([{ uuid: 'inactive', active: false }, missing])); - const service = serviceWith({ services: { Deploy: { buildImage: jest.fn(), deployCLI: jest.fn() } } }); + const failureWrite: any = { + patch: jest.fn(() => failureWrite), + where: jest.fn(() => failureWrite), + whereNotIn: jest.fn().mockResolvedValue(1), + }; + const service = serviceWith({ + models: { Deploy: { query: jest.fn(() => failureWrite) } }, + services: { Deploy: { buildImage: jest.fn(), deployCLI: jest.fn() } }, + }); await expect(service.buildImages({ id: 4 } as any, 'build-run')).resolves.toBe(false); await expect( @@ -2561,7 +2194,44 @@ describe('BuildService focused changed-line coverage', () => { ).resolves.toBe(false); }); - test('deploys only active CLI services and records an individual CLI failure', async () => { + test('a fenced image phase skips rows this run already finished and fails an orphaned row on its own', async () => { + const orphan = { id: 31, uuid: 'orphan', active: true, runUUID: 'run-g', deployable: null }; + const docker = { id: 32, uuid: 'docker', active: true, runUUID: 'run-g', deployable: { type: DeployTypes.DOCKER } }; + const read = deployQuery([orphan, docker]); + mockDeployQuery.mockReturnValue(read); + const failureWrite: any = { + patch: jest.fn(() => failureWrite), + where: jest.fn(() => failureWrite), + whereNotIn: jest.fn().mockResolvedValue(1), + }; + const buildImage = jest.fn().mockResolvedValue(true); + const service = serviceWith({ + models: { Deploy: { query: jest.fn(() => failureWrite) } }, + services: { Deploy: { buildImage } }, + }); + + await expect( + service.buildImages({ id: 4, $fetchGraph: jest.fn() } as any, 'run-g', 100, null, 'main', 3) + ).resolves.toBe(true); + + expect(read.where).toHaveBeenCalledWith({ buildId: 4, runUUID: 'run-g' }); + expect(read.whereNotIn).toHaveBeenCalledWith('status', [ + DeployStatus.READY, + DeployStatus.DEPLOYED, + DeployStatus.ERROR, + DeployStatus.BUILD_FAILED, + DeployStatus.DEPLOY_FAILED, + ]); + expect(buildImage).toHaveBeenCalledTimes(1); + expect(buildImage).toHaveBeenCalledWith(docker, 0, 'run-g', null, 100, 'main', 3, undefined, undefined); + expect(failureWrite.patch).toHaveBeenCalledWith({ + status: DeployStatus.ERROR, + statusMessage: 'Service definition missing.', + }); + expect(failureWrite.where).toHaveBeenCalledWith({ id: 31, runUUID: 'run-g', desiredGeneration: 3 }); + }); + + test('deploys only active CLI services and records an individual CLI failure', async () => { const cliSuccess = { uuid: 'cli-success', active: true, @@ -2596,6 +2266,102 @@ describe('BuildService focused changed-line coverage', () => { }); }); + test('a legacy rollout without a generation still throws when a service fails', async () => { + const deploy = { + id: 12, + uuid: 'legacy-web', + active: true, + deployable: { name: 'web', type: DeployTypes.GITHUB, deploymentDependsOn: [] }, + }; + mockDeployQuery.mockReturnValue(deployQuery([deploy])); + const service = serviceWith({}); + const build = { + id: 4, + uuid: 'legacy', + namespace: 'env-legacy', + kind: BuildKind.SANDBOX, + $query: jest.fn(), + }; + jest.spyOn(DeploymentManager.prototype, 'deploy').mockResolvedValueOnce({ failed: [deploy as any] }); + + await expect( + service.generateAndApplyManifests({ build: build as any, githubRepositoryId: null, namespace: build.namespace }) + ).rejects.toThrow('Deployment failed for web'); + }); + + test('a fenced rollout keeps failed rows in the plan by name and gives the manager no way to wait on other runs', async () => { + const failed = { + id: 13, + uuid: 'migrations-run', + active: true, + status: DeployStatus.BUILD_FAILED, + runUUID: 'run-g', + deployable: { name: 'migrations', type: DeployTypes.GITHUB, deploymentDependsOn: [] }, + }; + const api = { + id: 14, + uuid: 'api-run', + active: true, + status: DeployStatus.BUILT, + runUUID: 'run-g', + deployable: { name: 'api', type: DeployTypes.GITHUB, deploymentDependsOn: ['cache'] }, + }; + mockDeployQuery.mockReturnValue(deployQuery([failed, api])); + const service = serviceWith({ + models: { + Build: { + query: jest.fn(() => ({ + findById: jest.fn(() => ({ + select: jest + .fn() + .mockResolvedValue({ id: 4, status: BuildStatus.DEPLOYING, deployEnabled: true, pullRequestId: null }), + })), + })), + }, + Deploy: { + query: jest.fn(() => ({ + findOne: jest.fn(() => ({ select: jest.fn().mockResolvedValue({ id: 14, observedGeneration: 1 }) })), + })), + }, + }, + }); + jest.spyOn(service as any, 'updateDeploysImageDetails').mockResolvedValue(undefined); + const build = { + id: 4, + uuid: 'fenced', + namespace: 'env-fenced', + kind: BuildKind.SANDBOX, + pullRequest: null, + $query: jest.fn(), + }; + let managerOptions: Record = {}; + const deployments = jest + .spyOn(DeploymentManager.prototype, 'deploy') + .mockImplementationOnce(async function (this: any) { + managerOptions = this.options; + return { failed: [] }; + }); + + try { + await expect( + service.generateAndApplyManifests({ + build: build as any, + runUUID: 'run-g', + expectedGeneration: 2, + githubRepositoryId: null, + namespace: build.namespace, + enqueueIngress: false, + unresolvedServices: ['billing'], + }) + ).resolves.toBe(true); + } finally { + deployments.mockRestore(); + } + + expect(managerOptions.failedServices).toEqual(['billing', 'migrations']); + expect(managerOptions).not.toHaveProperty('prerequisiteOutcome'); + }); + test('filters inactive manifests and rejects a loaded active deploy missing its deployable', async () => { mockDeployQuery.mockReturnValue( deployQuery([ @@ -2919,182 +2685,6 @@ describe('BuildService uncovered public behavior', () => { }; }; - const publicReconciliationHarness = ( - options: { - intent?: Record; - repository?: { fullName: string } | null; - mailboxResult?: (read: number, build: any) => any; - loadResult?: (read: number, build: any) => any; - authorityResult?: (read: number, build: any) => any; - onYamlImported?: (build: any) => void; - onDeploysAssociated?: (build: any) => void; - onGraphGenerated?: (build: any) => void; - yamlFailure?: unknown; - onLockAcquired?: (resource: string, build: any) => void; - deploys?: any[]; - } = {} - ) => { - const intent = - options.intent ?? ({ type: 'all', requestId: 'run-generation-3', gen: 3 } as Record); - const scopeKey = intent.type === 'source' ? 'source:42:main' : 'all'; - const build: any = { - id: 7, - uuid: 'stateful-reconciliation', - namespace: 'env-stateful-reconciliation', - kind: BuildKind.SANDBOX, - status: BuildStatus.DEPLOYED, - statusMessage: '', - runUUID: 'prior-run', - deployEnabled: true, - deletedAt: null, - pullRequestId: null, - pullRequest: null, - environment: { id: 5 }, - githubRepositoryId: 42, - branchName: 'main', - desiredGeneration: 3, - observedGeneration: 2, - acceptedRefs: { [scopeKey]: intent }, - deploys: [], - reload: jest.fn().mockResolvedValue(undefined), - $fetchGraph: jest.fn().mockResolvedValue(undefined), - $setRelated: jest.fn((_relation: string, related: any[]) => { - build.deploys = related; - }), - }; - let mailboxReads = 0; - let loadReads = 0; - let authorityReads = 0; - const buildPatch = jest.fn(); - const mutationFor = (value: Record) => { - const mutation: any = { - where: jest.fn(() => mutation), - whereNull: jest.fn(() => mutation), - whereNotIn: jest.fn(() => mutation), - then: (resolve: (value: number) => unknown, reject?: (reason: unknown) => unknown) => { - buildPatch(value); - Object.assign(build, value); - return Promise.resolve(1).then(resolve, reject); - }, - }; - return mutation; - }; - const BuildModel = { - query: jest.fn(() => { - let lookup: 'id' | 'one' | null = null; - let projectedAuthority = false; - const query: any = { - findById: jest.fn(() => { - lookup = 'id'; - return query; - }), - findOne: jest.fn(() => { - lookup = 'one'; - return query; - }), - select: jest.fn(() => { - projectedAuthority = true; - return query; - }), - where: jest.fn(() => query), - whereNull: jest.fn(() => query), - patch: jest.fn((value: Record) => mutationFor(value)), - then: (resolve: (value: any) => unknown, reject?: (reason: unknown) => unknown) => { - let result: any = build; - if (projectedAuthority) { - authorityReads += 1; - result = options.authorityResult ? options.authorityResult(authorityReads, build) : build; - } else if (lookup === 'one') { - loadReads += 1; - result = options.loadResult ? options.loadResult(loadReads, build) : build; - } else if (lookup === 'id') { - mailboxReads += 1; - result = options.mailboxResult ? options.mailboxResult(mailboxReads, build) : build; - } - return Promise.resolve(result).then(resolve, reject); - }, - }; - return query; - }), - }; - const deployMutation: any = { - patch: jest.fn(() => deployMutation), - where: jest.fn(() => deployMutation), - then: (resolve: (value: number) => unknown, reject?: (reason: unknown) => unknown) => - Promise.resolve(1).then(resolve, reject), - }; - const importedDeployQuery: any = { - where: jest.fn(() => importedDeployQuery), - withGraphFetched: jest.fn().mockResolvedValue([]), - }; - mockDeployQuery.mockReturnValue(importedDeployQuery); - const upsertDeployables = jest.fn(async () => { - if ('yamlFailure' in options) throw options.yamlFailure; - return { canReconcile: false }; - }); - const upsertWebhooksWithYaml = jest.fn(async () => { - options.onYamlImported?.(build); - }); - const findOrCreateDeploys = jest.fn(async () => { - options.onDeploysAssociated?.(build); - return options.deploys ?? []; - }); - if (options.onGraphGenerated) { - mockGenerateGraph.mockImplementation(async () => { - options.onGraphGenerated?.(build); - return {}; - }); - } - const repositoryQuery: any = { - findOne: jest.fn(() => repositoryQuery), - whereNull: jest.fn().mockResolvedValue(options.repository ?? null), - }; - const redlock = options.onLockAcquired - ? { - lock: jest.fn(async (resource: string) => { - options.onLockAcquired?.(resource, build); - return { extend: jest.fn(), unlock: jest.fn().mockResolvedValue(undefined) }; - }), - } - : {}; - const service = serviceWith( - { - models: { - Build: BuildModel, - Deploy: { query: jest.fn(() => deployMutation) }, - Repository: { query: jest.fn(() => repositoryQuery) }, - }, - services: { - Deployable: { upsertDeployables }, - Deploy: { findOrCreateDeploys }, - Webhook: { - upsertWebhooksWithYaml, - webhookQueue: { add: jest.fn().mockResolvedValue(undefined) }, - }, - }, - }, - redlock - ); - const job = { - data: { buildId: 7, generation: 3 }, - attemptsMade: 0, - opts: { attempts: 3 }, - } as any; - return { - service, - job, - build, - buildPatch, - upsertDeployables, - upsertWebhooksWithYaml, - findOrCreateDeploys, - repositoryQuery, - deployMutation, - redlock, - reads: () => ({ mailboxReads, loadReads, authorityReads }), - }; - }; - const passOneContentionRetry = async (lockWithOptions: jest.Mock) => { for (let iteration = 0; iteration < 25 && lockWithOptions.mock.calls.length === 0; iteration += 1) { await Promise.resolve(); @@ -4154,10 +3744,79 @@ describe('BuildService uncovered public behavior', () => { const lock = jest.fn().mockResolvedValue({ unlock, extend: jest.fn() }); const withLock = serviceWith({}, { lock }); await expect(withLock.withBuildDeploymentLock(2, action)).resolves.toBe('done'); - expect(lock).toHaveBeenCalledWith('build-deployment.2', 15 * 60 * 1000); + expect(lock).toHaveBeenCalledWith('build-deployment.2', 60_000); expect(unlock).toHaveBeenCalledTimes(1); }); + test('withCurrentDeployPromotionLock keeps waiting on a contended service lock and logs once a minute', async () => { + jest.useFakeTimers(); + try { + const unlock = jest.fn().mockResolvedValue(undefined); + const lock = jest.fn(async () => { + if (Date.now() < 60_500) throw new Error('resource is locked'); + return { unlock, extend: jest.fn() }; + }); + const service = serviceWith({}, { lock }); + + const admitted = service.withCurrentDeployPromotionLock( + 3, + async () => true, + async () => 'applied' + ); + await jest.advanceTimersByTimeAsync(61_000); + + await expect(admitted).resolves.toEqual({ admitted: true, value: 'applied' }); + expect(lock).toHaveBeenCalledWith('deploy-promotion.3', 60_000); + expect(unlock).toHaveBeenCalledTimes(1); + } finally { + jest.useRealTimers(); + } + }); + + test('barrierDeployPromotionLocks takes and releases every service lock of the environment', async () => { + const unlock = jest.fn().mockResolvedValue(undefined); + const lock = jest.fn().mockResolvedValue({ unlock, extend: jest.fn() }); + const service = serviceWith( + { + models: { + Deploy: { + query: jest.fn(() => ({ + select: jest.fn(() => ({ where: jest.fn().mockResolvedValue([{ id: 1 }, { id: 2 }]) })), + })), + }, + }, + }, + { lock } + ); + + await (service as any).barrierDeployPromotionLocks(4); + + expect(lock.mock.calls.map(([resource]) => resource)).toEqual(['deploy-promotion.1', 'deploy-promotion.2']); + expect(unlock).toHaveBeenCalledTimes(2); + }); + + test('the generation try-lock is short-lived and renewed often, so a dead worker frees its generation quickly', async () => { + jest.useFakeTimers(); + try { + const extend = jest.fn(async () => ({ unlock, extend })); + const unlock = jest.fn().mockResolvedValue(undefined); + const lock = jest.fn().mockResolvedValue({ extend, unlock }); + const service = serviceWith({}, { lock }); + const action = jest.fn(async () => { + await jest.advanceTimersByTimeAsync(45_000); + }); + + await (service as any).tryWithDeploymentGenerationLock(1, 7, action); + + expect(lock).toHaveBeenCalledWith('build-reconcile.1.7', 60_000); + expect(extend).toHaveBeenCalledTimes(2); + expect(extend).toHaveBeenCalledWith(60_000); + expect(unlock).toHaveBeenCalledTimes(1); + } finally { + jest.useRealTimers(); + } + }); + test('withBuildDeploymentLock surfaces renewal loss while containing lock-release failure', async () => { jest.useFakeTimers(); const renewalFailure = new Error('lock renewal failed'); @@ -4166,7 +3825,7 @@ describe('BuildService uncovered public behavior', () => { const lock = jest.fn().mockResolvedValue({ extend, unlock }); const service = serviceWith({}, { lock }); const action = jest.fn(async () => { - jest.advanceTimersByTime(5 * 60 * 1000); + jest.advanceTimersByTime(20_000); await Promise.resolve(); await Promise.resolve(); return 'done'; @@ -4174,18 +3833,22 @@ describe('BuildService uncovered public behavior', () => { await expect(service.withBuildDeploymentLock(7, action)).rejects.toBe(renewalFailure); - expect(extend).toHaveBeenCalledWith(15 * 60 * 1000); + expect(extend).toHaveBeenCalledWith(60_000); expect(unlock).toHaveBeenCalledTimes(1); }); test('enqueuePendingDeploymentReconciliations wraps the cursor and contains an individual signal failure', async () => { - const first = lookupQuery([{ id: 5, desiredGeneration: 2 }]); + const first = lookupQuery([ + { id: 5, desiredGeneration: 2, acceptedRefs: { all: { type: 'all', requestId: 'request-2', gen: 2 } } }, + ]); const afterCursor = lookupQuery([]); - const wrapped = lookupQuery([{ id: 2, desiredGeneration: 3 }]); + const wrapped = lookupQuery([ + { id: 2, desiredGeneration: 3, acceptedRefs: { all: { type: 'all', requestId: 'request-3', gen: 3 } } }, + ]); const BuildModel = { query: jest.fn().mockReturnValueOnce(first).mockReturnValueOnce(afterCursor).mockReturnValueOnce(wrapped), }; - const service = serviceWith({ models: { Build: BuildModel } }); + const service = serviceWith({ models: { Build: BuildModel, Deploy: { query: jest.fn(() => lookupQuery([])) } } }); const add = jest.fn().mockResolvedValueOnce(undefined).mockRejectedValueOnce(new Error('queue unavailable')); (service as any).deploymentReconciliationQueue = { add }; @@ -4236,27 +3899,6 @@ describe('BuildService uncovered public behavior', () => { clearInterval(timer); }); - test('deployment reconciliation rejects malformed jobs and both legacy queue adapters accept durable work', async () => { - const service = serviceWith({}); - const add = jest.fn().mockResolvedValue(undefined); - (service as any).deploymentReconciliationQueue = { add }; - - await expect(service.processDeploymentReconciliationQueue({ data: { buildId: 0 } } as any)).rejects.toThrow( - 'buildId and generation are required' - ); - - await service.processBuildQueue({ data: { buildId: 7, runUUID: 'legacy-build' } } as any); - await service.processResolveAndDeployBuildQueue({ data: { buildId: 8, githubRepositoryId: 42 } } as any); - - expect(mockAcceptDeploymentIntent).toHaveBeenCalledTimes(2); - expect(add).toHaveBeenCalledWith('reconcile', expect.objectContaining({ buildId: 7, generation: 1 }), { - jobId: 'reconcile-7-1', - }); - expect(add).toHaveBeenCalledWith('reconcile', expect.objectContaining({ buildId: 8, generation: 1 }), { - jobId: 'reconcile-8-1', - }); - }); - test('legacy queue adapters stop without signaling when the build disappeared during mailbox acceptance', async () => { mockAcceptDeploymentIntent.mockResolvedValueOnce(null); const service = serviceWith({}); @@ -4359,6 +4001,7 @@ describe('BuildService uncovered public behavior', () => { undefined, undefined, undefined, + undefined, undefined ); expect(findOrCreateDeploys).toHaveBeenCalledWith({ id: 5 }, build); @@ -4552,6 +4195,7 @@ describe('BuildService uncovered public behavior', () => { }; const deployRead: any = { where: jest.fn(() => deployRead), + whereNotIn: jest.fn(() => deployRead), withGraphFetched: jest.fn().mockResolvedValue([deploy]), }; mockDeployQuery.mockReturnValue(deployRead); @@ -4575,7 +4219,12 @@ describe('BuildService uncovered public behavior', () => { const service = serviceWith({ models: { Build: { query: jest.fn(() => currentQuery) }, - Deploy: { query: jest.fn(() => mutation) }, + Deploy: { + query: jest.fn(() => ({ + ...mutation, + findOne: jest.fn(() => ({ select: jest.fn().mockResolvedValue({ id: 11 }) })), + })), + }, }, services: { Deploy: { buildImage } }, }); @@ -4594,7 +4243,7 @@ describe('BuildService uncovered public behavior', () => { expect(kubernetes.createOrUpdateNamespace).toHaveBeenCalledWith( expect.objectContaining({ name: 'env-native-build', waitForReady: true }) ); - expect(buildImage).toHaveBeenCalledWith(deploy, 0, 'run-native', 'sha-a', null, 'main', 4, 'default'); + expect(buildImage).toHaveBeenCalledWith(deploy, 0, 'run-native', 'sha-a', null, 'main', 4, 'default', undefined); }); test('generateAndApplyManifests persists current deploy and legacy manifests behind native mutation gates', async () => { @@ -4613,12 +4262,14 @@ describe('BuildService uncovered public behavior', () => { }; const deployRead: any = { where: jest.fn(() => deployRead), + whereNotIn: jest.fn(() => deployRead), withGraphFetched: jest.fn().mockResolvedValue([deploy]), }; mockDeployQuery.mockReturnValue(deployRead); const mutation: any = { patch: jest.fn(() => mutation), where: jest.fn(() => mutation), + whereNull: jest.fn(() => mutation), then: (resolve: (value: number) => unknown, reject?: (reason: unknown) => unknown) => Promise.resolve(1).then(resolve, reject), }; @@ -4648,7 +4299,12 @@ describe('BuildService uncovered public behavior', () => { const service = serviceWith({ models: { Build: { query: jest.fn(() => buildQuery) }, - Deploy: { query: jest.fn(() => mutation) }, + Deploy: { + query: jest.fn(() => ({ + ...mutation, + findOne: jest.fn(() => ({ select: jest.fn().mockResolvedValue({ id: 11 }) })), + })), + }, }, }); mockGenerateDeployManifest.mockReturnValue('kind: Deployment\n'); @@ -4657,8 +4313,9 @@ describe('BuildService uncovered public behavior', () => { .spyOn(DeploymentManager.prototype, 'deploy') .mockImplementation(async function (this: any) { expect(await this.options.isCurrent()).toBe(true); - await this.options.nativeMutationGate(async () => undefined); + await this.options.nativeMutationGate(deploy, async () => undefined); await this.options.nativeSecretMutationGate(deploy, async () => undefined); + return { failed: [] }; }); await expect( @@ -4699,20 +4356,21 @@ describe('BuildService uncovered public behavior', () => { deploys: [], $fetchGraph: jest.fn().mockResolvedValue(undefined), }; - let authorityReads = 0; - const BuildModel = { + let ownershipReads = 0; + const BuildModel = { query: jest.fn(() => lookupQuery({ ...build, acceptedRefs: {} })) }; + const DeployModel = { query: jest.fn(() => { - authorityReads += 1; - const authority = authorityReads <= 3 ? build : { ...build, runUUID: 'newer-run' }; - return lookupQuery(authority); + ownershipReads += 1; + return lookupQuery(ownershipReads <= 3 ? { id: 11 } : undefined); }), }; const noDeploys: any = { where: jest.fn(() => noDeploys), + whereNotIn: jest.fn(() => noDeploys), withGraphFetched: jest.fn().mockResolvedValue([]), }; mockDeployQuery.mockReturnValue(noDeploys); - const service = serviceWith({ models: { Build: BuildModel } }); + const service = serviceWith({ models: { Build: BuildModel, Deploy: DeployModel } }); await expect( service.generateAndApplyManifests({ @@ -4761,11 +4419,125 @@ describe('BuildService uncovered public behavior', () => { queueBuildPrewarm.mockRestore(); }); + test('a teardown closes the work accepted before it and keeps a deploy request accepted during it', async () => { + const refs: any = { + 'source:1:main': { + type: 'source', + requestId: 'old', + target: 'repository', + githubRepositoryId: 1, + branch: 'main', + sha: 'a', + gen: 4, + }, + all: { type: 'all', requestId: 'full', gen: 5 }, + 'repository:2': { type: 'repository', requestId: 'relabel', githubRepositoryId: 2, gen: 6 }, + }; + const buildPatch = jest.fn().mockResolvedValue(1); + const buildQuery: any = { + select: jest.fn(() => buildQuery), + findById: jest.fn(() => buildQuery), + forUpdate: jest.fn().mockResolvedValue({ id: 7, desiredGeneration: 6, acceptedRefs: refs }), + patch: buildPatch, + }; + const rowPatch = jest.fn(() => ({ where: jest.fn().mockResolvedValue(1) })); + const deployQuery: any = { + select: jest.fn(() => deployQuery), + where: jest.fn().mockResolvedValue([ + { id: 1, desiredGeneration: 5, observedGeneration: 3 }, + { id: 2, desiredGeneration: 6, observedGeneration: 3 }, + { id: 3, desiredGeneration: 4, observedGeneration: 4 }, + ]), + patch: rowPatch, + }; + const service = serviceWith({ + models: { + Build: { query: jest.fn(() => buildQuery), transact: jest.fn(async (action: any) => action({})) }, + Deploy: { query: jest.fn(() => deployQuery) }, + }, + }); + + await service['observeAcceptedWorkForTeardown'](7, 5); + + expect(rowPatch).toHaveBeenCalledTimes(1); + expect(rowPatch).toHaveBeenCalledWith({ observedGeneration: 5 }); + expect(buildPatch).toHaveBeenCalledWith({ + acceptedRefs: { + 'source:1:main': { ...refs['source:1:main'], observedGen: 4 }, + all: { ...refs.all, observedGen: 5 }, + 'repository:2': refs['repository:2'], + }, + observedGeneration: 5, + }); + }); + + test('deleteBuild takes the teardown cutoff before it waits for in-flight mutations', async () => { + const mutation: any = { + patch: jest.fn(() => mutation), + where: jest.fn(() => mutation), + whereNull: jest.fn(() => mutation), + findById: jest.fn(() => mutation), + select: jest.fn(() => ({ + where: jest.fn().mockResolvedValue([]), + findById: jest.fn(() => ({ forUpdate: jest.fn().mockResolvedValue({ id: 7, desiredGeneration: 0 }) })), + })), + then: (resolve: (value: number) => unknown, reject?: (reason: unknown) => unknown) => + Promise.resolve(1).then(resolve, reject), + }; + const build: any = { + id: 7, + uuid: 'delete-build', + namespace: 'env-delete-build', + kind: BuildKind.SANDBOX, + status: BuildStatus.DEPLOYED, + runUUID: 'old-run', + desiredGeneration: 5, + pullRequestId: null, + pullRequest: null, + deployEnabled: true, + githubDeployments: false, + deploys: [], + idempotencyKey: null, + reload: jest.fn().mockResolvedValue(undefined), + $fetchGraph: jest.fn().mockResolvedValue(undefined), + $query: jest.fn(() => ({ patch: jest.fn().mockResolvedValue(1) })), + }; + const service = serviceWith({ + models: { + Build: { query: jest.fn(() => mutation), transact: jest.fn(async (action: any) => action({})) }, + Deploy: { query: jest.fn(() => mutation) }, + }, + services: { + GithubService: { githubDeploymentQueue: { add: jest.fn() } }, + Ingress: { ingressCleanupQueue: {} }, + Webhook: { webhookQueue: { add: jest.fn().mockResolvedValue(undefined) } }, + }, + }); + const order: string[] = []; + jest.spyOn(service as any, 'barrierDeployPromotionLocks').mockImplementation(async () => { + order.push('barrier'); + build.desiredGeneration = 6; + }); + const observe = jest.spyOn(service as any, 'observeAcceptedWorkForTeardown').mockImplementation(async () => { + order.push('observe'); + }); + + await expect(service.deleteBuild(build)).resolves.toBeUndefined(); + + expect(order).toEqual(['barrier', 'observe']); + expect(observe).toHaveBeenCalledWith(7, 5); + }); + test('deleteBuild queues provider and ingress cleanup for every torn-down GitHub deployment', async () => { const mutation: any = { patch: jest.fn(() => mutation), where: jest.fn(() => mutation), whereNull: jest.fn(() => mutation), + findById: jest.fn(() => mutation), + select: jest.fn(() => ({ + where: jest.fn().mockResolvedValue([]), + findById: jest.fn(() => ({ forUpdate: jest.fn().mockResolvedValue({ id: 7, desiredGeneration: 0 }) })), + })), then: (resolve: (value: number) => unknown, reject?: (reason: unknown) => unknown) => Promise.resolve(1).then(resolve, reject), }; @@ -4791,7 +4563,7 @@ describe('BuildService uncovered public behavior', () => { }; const service = serviceWith({ models: { - Build: { query: jest.fn(() => mutation) }, + Build: { query: jest.fn(() => mutation), transact: jest.fn(async (action: any) => action({})) }, Deploy: { query: jest.fn(() => mutation) }, }, services: { @@ -5437,6 +5209,11 @@ describe('BuildService uncovered public behavior', () => { patch: jest.fn(() => mutation), where: jest.fn(() => mutation), whereNull: jest.fn(() => mutation), + findById: jest.fn(() => mutation), + select: jest.fn(() => ({ + where: jest.fn().mockResolvedValue([]), + findById: jest.fn(() => ({ forUpdate: jest.fn().mockResolvedValue({ id: 7, desiredGeneration: 0 }) })), + })), then: (resolve: (value: number) => unknown, reject?: (reason: unknown) => unknown) => Promise.resolve(1).then(resolve, reject), }; @@ -5462,7 +5239,7 @@ describe('BuildService uncovered public behavior', () => { }; const service = serviceWith({ models: { - Build: { query: jest.fn(() => mutation) }, + Build: { query: jest.fn(() => mutation), transact: jest.fn(async (action: any) => action({})) }, Deploy: { query: jest.fn(() => mutation) }, }, services: { Webhook: { webhookQueue: { add: jest.fn().mockResolvedValue(undefined) } } }, @@ -5481,620 +5258,6 @@ describe('BuildService uncovered public behavior', () => { ]); }); - test('deployment reconciliation rejects corrupt persisted generations before mutation', async () => { - const corrupt = { desiredGeneration: 'not-a-number', observedGeneration: 0, acceptedRefs: {} }; - const query = lookupQuery(corrupt); - const service = serviceWith({ models: { Build: { query: jest.fn(() => query) } } }); - - await expect( - service.processDeploymentReconciliationQueue({ data: { buildId: 7, generation: 3 } } as any) - ).rejects.toThrow('Build 7 has invalid deployment generations'); - }); - - test.each([ - ['the claim patch loses its row', 0, false], - ['authority changes immediately after the claim', 1, true], - ])('deployment reconciliation stops before configuration when %s', async (_case, claimedRows, staleAfterClaim) => { - const build: any = { - id: 7, - uuid: 'claim-race', - status: BuildStatus.DEPLOYED, - deployEnabled: true, - deletedAt: null, - pullRequestId: null, - pullRequest: null, - desiredGeneration: 3, - observedGeneration: 2, - acceptedRefs: { all: { type: 'all', requestId: 'run-generation-3', gen: 3 } }, - $fetchGraph: jest.fn().mockResolvedValue(undefined), - }; - const stale = { ...build, runUUID: 'newer-run' }; - const mutation: any = { - patch: jest.fn(() => mutation), - where: jest.fn(() => mutation), - whereNull: jest.fn(() => mutation), - whereNotIn: jest.fn(() => mutation), - then: (resolve: (value: number) => unknown, reject?: (reason: unknown) => unknown) => - Promise.resolve(claimedRows).then(resolve, reject), - }; - const BuildModel = { - query: jest.fn(() => { - let authorityProjection = false; - const query: any = { - findById: jest.fn(() => query), - findOne: jest.fn(() => query), - select: jest.fn(() => { - authorityProjection = true; - return query; - }), - whereNull: jest.fn(() => query), - patch: jest.fn(() => mutation), - then: (resolve: (value: any) => unknown, reject?: (reason: unknown) => unknown) => - Promise.resolve(authorityProjection && staleAfterClaim ? stale : build).then(resolve, reject), - }; - return query; - }), - }; - const upsertDeployables = jest.fn(); - const service = serviceWith({ - models: { Build: BuildModel }, - services: { Deployable: { upsertDeployables } }, - }); - - await expect( - service.processDeploymentReconciliationQueue({ data: { buildId: 7, generation: 3 } } as any) - ).resolves.toBeUndefined(); - - expect(upsertDeployables).not.toHaveBeenCalled(); - }); - - test('deployment reconciliation observes disabled builds without importing or deploying them', async () => { - const build: any = { - id: 7, - uuid: 'disabled-build', - status: BuildStatus.DEPLOYED, - deployEnabled: false, - deletedAt: null, - pullRequestId: null, - desiredGeneration: 3, - observedGeneration: 2, - acceptedRefs: { all: { type: 'all', requestId: 'run-generation-3', gen: 3 } }, - $fetchGraph: jest.fn().mockResolvedValue(undefined), - }; - const mutation: any = { - patch: jest.fn(() => mutation), - where: jest.fn(() => mutation), - whereNull: jest.fn(() => mutation), - then: (resolve: (value: number) => unknown, reject?: (reason: unknown) => unknown) => - Promise.resolve(1).then(resolve, reject), - }; - const query: any = { - findById: jest.fn(() => query), - findOne: jest.fn(() => query), - whereNull: jest.fn(() => query), - patch: jest.fn(() => mutation), - then: (resolve: (value: any) => unknown, reject?: (reason: unknown) => unknown) => - Promise.resolve(build).then(resolve, reject), - }; - const upsertDeployables = jest.fn(); - const service = serviceWith({ - models: { Build: { query: jest.fn(() => query) } }, - services: { Deployable: { upsertDeployables } }, - }); - - await service.processDeploymentReconciliationQueue({ data: { buildId: 7, generation: 3 } } as any); - - expect(query.patch).toHaveBeenCalledWith({ observedGeneration: 3 }); - expect(upsertDeployables).not.toHaveBeenCalled(); - }); - - test('deployment reconciliation records and observes a claimed configuration parse failure', async () => { - const parseError = new ParsingError('invalid lifecycle yaml'); - const build: any = { - id: 7, - uuid: 'invalid-config-build', - namespace: 'env-invalid-config-build', - kind: BuildKind.SANDBOX, - status: BuildStatus.DEPLOYED, - statusMessage: '', - runUUID: 'prior-run', - deployEnabled: true, - deletedAt: null, - pullRequestId: null, - pullRequest: null, - environment: { id: 5 }, - desiredGeneration: 3, - observedGeneration: 2, - acceptedRefs: { all: { type: 'all', requestId: 'run-generation-3', gen: 3 } }, - deploys: [], - reload: jest.fn().mockResolvedValue(undefined), - $fetchGraph: jest.fn().mockResolvedValue(undefined), - }; - const buildMutation: any = { - where: jest.fn(() => buildMutation), - whereNull: jest.fn(() => buildMutation), - whereNotIn: jest.fn(() => buildMutation), - patch: jest.fn(() => buildMutation), - then: (resolve: (value: number) => unknown, reject?: (reason: unknown) => unknown) => - Promise.resolve(1).then(resolve, reject), - }; - const BuildModel = { - query: jest.fn(() => { - const query: any = { - findById: jest.fn(() => query), - findOne: jest.fn(() => query), - select: jest.fn(() => query), - whereNull: jest.fn(() => query), - patch: jest.fn((value: Record) => { - buildMutation.patch(value); - return buildMutation; - }), - then: (resolve: (value: any) => unknown, reject?: (reason: unknown) => unknown) => - Promise.resolve(build).then(resolve, reject), - }; - return query; - }), - }; - const upsertDeployables = jest.fn().mockRejectedValue(parseError); - const findOrCreateDeploys = jest.fn(); - const service = serviceWith({ - models: { Build: BuildModel }, - services: { - Deployable: { upsertDeployables }, - Deploy: { findOrCreateDeploys }, - Webhook: { webhookQueue: { add: jest.fn() } }, - }, - }); - - await expect( - service.processDeploymentReconciliationQueue({ - data: { buildId: 7, generation: 3 }, - attemptsMade: 0, - opts: { attempts: 3 }, - } as any) - ).resolves.toBeUndefined(); - - expect(upsertDeployables).toHaveBeenCalledTimes(1); - expect(findOrCreateDeploys).not.toHaveBeenCalled(); - expect(buildMutation.patch).toHaveBeenCalledWith({ - status: BuildStatus.CONFIG_ERROR, - statusMessage: 'invalid lifecycle yaml', - }); - expect(buildMutation.patch).toHaveBeenCalledWith({ observedGeneration: 3 }); - expect(mockQueueAdd).not.toHaveBeenCalled(); - }); - - test('processDeploymentReconciliationQueue carries one durable generation through rollout and observation', async () => { - const build: any = { - id: 7, - uuid: 'reconciled-build', - namespace: 'env-reconciled-build', - kind: BuildKind.SANDBOX, - status: BuildStatus.DEPLOYED, - statusMessage: '', - runUUID: 'prior-run', - deployEnabled: true, - deletedAt: null, - pullRequestId: null, - pullRequest: null, - environment: { id: 5 }, - desiredGeneration: 3, - observedGeneration: 2, - acceptedRefs: { - 'repository:42': { - type: 'repository', - requestId: 'run-generation-3', - githubRepositoryId: 42, - gen: 3, - }, - }, - deploys: [], - reload: jest.fn().mockResolvedValue(undefined), - $fetchGraph: jest.fn().mockResolvedValue(undefined), - $setRelated: jest.fn((_relation: string, related: any[]) => { - build.deploys = related; - }), - }; - const buildMutation: any = { - where: jest.fn(() => buildMutation), - whereNull: jest.fn(() => buildMutation), - whereNotIn: jest.fn(() => buildMutation), - patch: jest.fn(() => buildMutation), - then: (resolve: (value: number) => unknown, reject?: (reason: unknown) => unknown) => - Promise.resolve(1).then(resolve, reject), - }; - const buildQuery: any = { - findById: jest.fn(() => buildQuery), - findOne: jest.fn(() => buildQuery), - select: jest.fn(() => buildQuery), - whereNull: jest.fn(() => buildQuery), - patch: jest.fn((value: Record) => { - buildMutation.patch(value); - return buildMutation; - }), - then: (resolve: (value: any) => unknown, reject?: (reason: unknown) => unknown) => - Promise.resolve(build).then(resolve, reject), - }; - const deployMutation: any = { - patch: jest.fn(() => deployMutation), - where: jest.fn(() => deployMutation), - then: (resolve: (value: number) => unknown, reject?: (reason: unknown) => unknown) => - Promise.resolve(1).then(resolve, reject), - }; - const noDeploys: any = { - where: jest.fn(() => noDeploys), - withGraphFetched: jest.fn().mockResolvedValue([]), - }; - mockDeployQuery.mockReturnValue(noDeploys); - const upsertDeployables = jest.fn().mockResolvedValue({ canReconcile: false }); - const findOrCreateDeploys = jest.fn().mockResolvedValue([]); - const upsertWebhooksWithYaml = jest.fn().mockResolvedValue(undefined); - const service = serviceWith({ - models: { - Build: { query: jest.fn(() => buildQuery) }, - Deploy: { query: jest.fn(() => deployMutation) }, - }, - services: { - Deployable: { upsertDeployables }, - Deploy: { findOrCreateDeploys }, - Webhook: { - upsertWebhooksWithYaml, - webhookQueue: { add: jest.fn().mockResolvedValue(undefined) }, - }, - }, - }); - - await expect( - service.processDeploymentReconciliationQueue({ - data: { buildId: 7, generation: 3 }, - attemptsMade: 0, - opts: { attempts: 3 }, - } as any) - ).resolves.toBeUndefined(); - - expect(upsertDeployables).toHaveBeenCalledWith( - 7, - 'reconciled-build', - null, - build.environment, - build, - 42, - undefined, - undefined, - 42 - ); - expect(findOrCreateDeploys).toHaveBeenCalledWith(build.environment, build, 42, undefined, undefined, 42); - expect(buildMutation.patch.mock.calls.map(([value]) => value.status).filter(Boolean)).toEqual([ - BuildStatus.PENDING, - BuildStatus.BUILDING, - BuildStatus.DEPLOYING, - BuildStatus.DEPLOYED, - ]); - expect(buildMutation.patch).toHaveBeenCalledWith({ observedGeneration: 3 }); - expect(mockQueueAdd).toHaveBeenCalledWith( - 'manifest', - expect.objectContaining({ buildId: 7, runUUID: 'run-generation-3', expectedGeneration: 3 }) - ); - expect(build).toMatchObject({ runUUID: 'run-generation-3', status: BuildStatus.DEPLOYED }); - }); - - test('deployment reconciliation propagates an accepted root-source SHA through YAML and rollout', async () => { - const intent = { - type: 'source', - requestId: 'run-generation-3', - target: 'all', - githubRepositoryId: 42, - branch: 'main', - sha: 'commit-c', - beforeSha: 'commit-b', - gen: 3, - }; - (github.getSHAForBranch as jest.Mock).mockResolvedValue('commit-c'); - const harness = publicReconciliationHarness({ intent, repository: { fullName: 'org/repo' } }); - - await expect(harness.service.processDeploymentReconciliationQueue(harness.job)).resolves.toBeUndefined(); - - expect(github.getSHAForBranch).toHaveBeenCalledWith('main', 'org', 'repo'); - expect(github.compareCommits).not.toHaveBeenCalled(); - expect(harness.upsertDeployables).toHaveBeenCalledWith( - 7, - 'stateful-reconciliation', - null, - harness.build.environment, - harness.build, - undefined, - 'commit-c', - 'main', - 42 - ); - expect(harness.upsertWebhooksWithYaml).toHaveBeenCalledWith(harness.build, null, 'commit-c'); - expect(harness.findOrCreateDeploys).toHaveBeenCalledWith( - harness.build.environment, - harness.build, - undefined, - 'commit-c', - 'main', - 42 - ); - expect(harness.buildPatch).toHaveBeenCalledWith({ observedGeneration: 3 }); - expect(mockQueueAdd).toHaveBeenCalledWith( - 'manifest', - expect.objectContaining({ buildId: 7, runUUID: 'run-generation-3', expectedGeneration: 3 }) - ); - }); - - test('deployment reconciliation keeps the accepted source when its repository was removed', async () => { - const intent = { - type: 'source', - requestId: 'run-generation-3', - target: 'all', - githubRepositoryId: 42, - branch: 'main', - sha: 'commit-c', - gen: 3, - }; - const harness = publicReconciliationHarness({ intent, repository: null }); - - await expect(harness.service.processDeploymentReconciliationQueue(harness.job)).resolves.toBeUndefined(); - - expect(harness.repositoryQuery.findOne).toHaveBeenCalledWith({ githubRepositoryId: 42 }); - expect(github.getSHAForBranch).not.toHaveBeenCalled(); - expect(harness.upsertDeployables).toHaveBeenCalledWith( - 7, - 'stateful-reconciliation', - null, - harness.build.environment, - harness.build, - undefined, - 'commit-c', - 'main', - 42 - ); - expect(harness.buildPatch).toHaveBeenCalledWith({ observedGeneration: 3 }); - }); - - test('deployment reconciliation scopes a source update to the matching repository and branch', async () => { - const matching = { id: 11, githubRepositoryId: 42, branchName: 'main' }; - const otherBranch = { id: 12, githubRepositoryId: 42, branchName: 'release' }; - const otherRepository = { id: 13, githubRepositoryId: 99, branchName: 'main' }; - const harness = publicReconciliationHarness({ - intent: { - type: 'source', - requestId: 'run-generation-3', - target: 'repository', - githubRepositoryId: 42, - branch: 'main', - sha: 'commit-c', - gen: 3, - }, - repository: null, - deploys: [matching, otherBranch, otherRepository], - }); - - await expect(harness.service.processDeploymentReconciliationQueue(harness.job)).resolves.toBeUndefined(); - - expect(harness.findOrCreateDeploys).toHaveBeenCalledWith( - harness.build.environment, - harness.build, - 42, - 'commit-c', - 'main', - 42 - ); - expect(harness.deployMutation.where).toHaveBeenCalledWith('githubRepositoryId', 42); - expect(harness.deployMutation.where).toHaveBeenCalledWith('branchName', 'main'); - expect(matching).toHaveProperty('runUUID', 'run-generation-3'); - expect(otherBranch).not.toHaveProperty('runUUID'); - expect(otherRepository).not.toHaveProperty('runUUID'); - expect(harness.buildPatch).toHaveBeenCalledWith({ observedGeneration: 3 }); - }); - - test('deployment reconciliation stops when its mailbox generation is observed during lock acquisition', async () => { - const acquiredResources: string[] = []; - const harness = publicReconciliationHarness({ - onLockAcquired: (resource, build) => { - acquiredResources.push(resource); - if (resource === 'build-deployment.7') build.observedGeneration = 3; - }, - }); - - await expect(harness.service.processDeploymentReconciliationQueue(harness.job)).resolves.toBeUndefined(); - - expect(harness.buildPatch).not.toHaveBeenCalled(); - expect(harness.upsertDeployables).not.toHaveBeenCalled(); - expect(harness.build.$fetchGraph).not.toHaveBeenCalled(); - expect(acquiredResources).toEqual(['build-reconcile.7.3', 'build-deployment.7']); - }); - - test('deployment reconciliation stops when authority moves after configuration-lock admission', async () => { - const harness = publicReconciliationHarness({ - authorityResult: (read, build) => (read >= 4 ? { ...build, runUUID: 'newer-run' } : build), - }); - - await expect(harness.service.processDeploymentReconciliationQueue(harness.job)).resolves.toBeUndefined(); - - expect(harness.upsertDeployables).not.toHaveBeenCalled(); - expect(harness.findOrCreateDeploys).not.toHaveBeenCalled(); - expect(harness.buildPatch.mock.calls.map(([value]) => value.status).filter(Boolean)).toEqual([BuildStatus.PENDING]); - expect(harness.buildPatch).not.toHaveBeenCalledWith({ observedGeneration: 3 }); - }); - - test('deployment reconciliation stops when the build disappears after its mailbox claim', async () => { - const harness = publicReconciliationHarness({ loadResult: () => null }); - - await expect(harness.service.processDeploymentReconciliationQueue(harness.job)).resolves.toBeUndefined(); - - expect(harness.buildPatch).not.toHaveBeenCalled(); - expect(harness.upsertDeployables).not.toHaveBeenCalled(); - expect(harness.build.$fetchGraph).not.toHaveBeenCalled(); - }); - - test('deployment reconciliation stops after YAML import when another run takes authority', async () => { - let current = true; - const harness = publicReconciliationHarness({ - authorityResult: (_read, build) => (current ? build : { ...build, runUUID: 'newer-run' }), - onYamlImported: () => { - current = false; - }, - }); - - await expect(harness.service.processDeploymentReconciliationQueue(harness.job)).resolves.toBeUndefined(); - - expect(harness.upsertDeployables).toHaveBeenCalledTimes(1); - expect(harness.upsertWebhooksWithYaml).toHaveBeenCalledTimes(1); - expect(harness.findOrCreateDeploys).not.toHaveBeenCalled(); - expect(harness.buildPatch.mock.calls.map(([value]) => value.status).filter(Boolean)).toEqual([BuildStatus.PENDING]); - expect(harness.buildPatch).not.toHaveBeenCalledWith({ observedGeneration: 3 }); - expect(mockQueueAdd).not.toHaveBeenCalled(); - }); - - test('deployment reconciliation stops after deploy association when another run takes authority', async () => { - let current = true; - const harness = publicReconciliationHarness({ - authorityResult: (_read, build) => (current ? build : { ...build, runUUID: 'newer-run' }), - onDeploysAssociated: () => { - current = false; - }, - }); - - await expect(harness.service.processDeploymentReconciliationQueue(harness.job)).resolves.toBeUndefined(); - - expect(harness.upsertDeployables).toHaveBeenCalledTimes(1); - expect(harness.findOrCreateDeploys).toHaveBeenCalledTimes(1); - expect(harness.build.$setRelated).toHaveBeenCalledWith('deploys', []); - expect(harness.buildPatch.mock.calls.map(([value]) => value.status).filter(Boolean)).toEqual([BuildStatus.PENDING]); - expect(harness.buildPatch).not.toHaveBeenCalledWith({ observedGeneration: 3 }); - expect(mockQueueAdd).not.toHaveBeenCalled(); - }); - - test('deployment reconciliation stops before execution when authority moves after graph generation', async () => { - let current = true; - const harness = publicReconciliationHarness({ - authorityResult: (_read, build) => (current ? build : { ...build, runUUID: 'newer-run' }), - onGraphGenerated: () => { - current = false; - }, - }); - - await expect(harness.service.processDeploymentReconciliationQueue(harness.job)).resolves.toBeUndefined(); - - expect(harness.findOrCreateDeploys).toHaveBeenCalledTimes(1); - expect(harness.buildPatch.mock.calls.map(([value]) => value.status).filter(Boolean)).toEqual([ - BuildStatus.PENDING, - BuildStatus.BUILDING, - ]); - expect(harness.buildPatch).not.toHaveBeenCalledWith({ observedGeneration: 3 }); - expect(mockQueueAdd).not.toHaveBeenCalled(); - }); - - test('deployment reconciliation stops before execution when authority moves after the configuration lock releases', async () => { - const harness = publicReconciliationHarness({ - authorityResult: (read, build) => (read >= 8 ? { ...build, runUUID: 'newer-run' } : build), - }); - - await expect(harness.service.processDeploymentReconciliationQueue(harness.job)).resolves.toBeUndefined(); - - expect(harness.upsertDeployables).toHaveBeenCalledTimes(1); - expect(harness.findOrCreateDeploys).toHaveBeenCalledTimes(1); - expect(harness.buildPatch.mock.calls.map(([value]) => value.status).filter(Boolean)).toEqual([ - BuildStatus.PENDING, - BuildStatus.BUILDING, - ]); - expect(harness.buildPatch).not.toHaveBeenCalledWith({ observedGeneration: 3 }); - expect(mockQueueAdd).not.toHaveBeenCalled(); - }); - - test('deployment reconciliation stops after scope execution when authority moves before publication', async () => { - const harness = publicReconciliationHarness({ - authorityResult: (read, build) => (read >= 16 ? { ...build, runUUID: 'newer-run' } : build), - }); - - await expect(harness.service.processDeploymentReconciliationQueue(harness.job)).resolves.toBeUndefined(); - - expect(harness.upsertDeployables).toHaveBeenCalledTimes(1); - expect(harness.findOrCreateDeploys).toHaveBeenCalledTimes(1); - expect(harness.buildPatch.mock.calls.map(([value]) => value.status).filter(Boolean)).toEqual([ - BuildStatus.PENDING, - BuildStatus.BUILDING, - BuildStatus.DEPLOYING, - ]); - expect(harness.buildPatch).not.toHaveBeenCalledWith({ observedGeneration: 3 }); - expect(mockQueueAdd).not.toHaveBeenCalled(); - }); - - test('deployment reconciliation treats omitted queue attempt metadata as one final attempt', async () => { - const failure = new Error('deployable import unavailable'); - const harness = publicReconciliationHarness({ yamlFailure: failure }); - - await expect( - harness.service.processDeploymentReconciliationQueue({ data: harness.job.data } as any) - ).resolves.toBeUndefined(); - - expect(harness.upsertDeployables).toHaveBeenCalledTimes(1); - expect(harness.findOrCreateDeploys).not.toHaveBeenCalled(); - expect(harness.buildPatch).toHaveBeenCalledWith({ - status: BuildStatus.ERROR, - statusMessage: 'deployable import unavailable', - }); - expect(harness.buildPatch).toHaveBeenCalledWith({ observedGeneration: 3 }); - }); - - test('deployment reconciliation reacquires authority to publish a final pre-active read failure', async () => { - const failure = new Error('build authority read unavailable'); - let databaseRecovered = false; - const harness = publicReconciliationHarness({ - loadResult: (_read, build) => { - if (!databaseRecovered) { - databaseRecovered = true; - throw failure; - } - return build; - }, - }); - - await expect( - harness.service.processDeploymentReconciliationQueue({ data: harness.job.data } as any) - ).resolves.toBeUndefined(); - - expect(harness.upsertDeployables).not.toHaveBeenCalled(); - expect(harness.buildPatch).toHaveBeenCalledWith({ - status: BuildStatus.ERROR, - statusMessage: 'build authority read unavailable', - }); - expect(harness.buildPatch).toHaveBeenCalledWith({ observedGeneration: 3 }); - }); - - test('deployment reconciliation contains a pre-active failure when the build is deleted before reclaim', async () => { - const failure = new Error('build authority read unavailable'); - let readFailed = false; - let deleted = false; - const harness = publicReconciliationHarness({ - mailboxResult: (_read, build) => (deleted ? null : build), - loadResult: (_read, build) => { - if (!readFailed) { - readFailed = true; - throw failure; - } - return deleted ? null : build; - }, - onLockAcquired: (resource, build) => { - if (resource === 'build-deployment.7' && readFailed) { - deleted = true; - build.deletedAt = new Date().toISOString(); - } - }, - }); - - await expect( - harness.service.processDeploymentReconciliationQueue({ data: harness.job.data } as any) - ).resolves.toBeUndefined(); - - expect(deleted).toBe(true); - expect(harness.upsertDeployables).not.toHaveBeenCalled(); - expect(harness.buildPatch).not.toHaveBeenCalledWith(expect.objectContaining({ status: BuildStatus.ERROR })); - expect(harness.buildPatch).not.toHaveBeenCalledWith({ observedGeneration: 3 }); - }); - test.each([ ['before YAML import', 4, 0, 0], ['after YAML import', 5, 1, 0], @@ -6239,6 +5402,7 @@ describe('BuildService uncovered public behavior', () => { }; const deployRead: any = { where: jest.fn(() => deployRead), + whereNotIn: jest.fn(() => deployRead), withGraphFetched: jest.fn().mockResolvedValue([deploy]), }; mockDeployQuery.mockReturnValue(deployRead); @@ -6250,11 +5414,15 @@ describe('BuildService uncovered public behavior', () => { deletedAt: null, pullRequestId: null, desiredGeneration: 5, + acceptedRefs: {}, }; const currentQuery = lookupQuery(supersedingBuild); const buildImage = jest.fn(); const service = serviceWith({ - models: { Build: { query: jest.fn(() => currentQuery) } }, + models: { + Build: { query: jest.fn(() => currentQuery) }, + Deploy: { query: jest.fn(() => lookupQuery(undefined)) }, + }, services: { Deploy: { buildImage } }, }); const build: any = { @@ -6286,9 +5454,15 @@ describe('BuildService uncovered public behavior', () => { deletedAt: null, pullRequestId: null, desiredGeneration: 5, + acceptedRefs: {}, }; const currentQuery = lookupQuery(supersedingBuild); - const service = serviceWith({ models: { Build: { query: jest.fn(() => currentQuery) } } }); + const service = serviceWith({ + models: { + Build: { query: jest.fn(() => currentQuery) }, + Deploy: { query: jest.fn(() => lookupQuery(undefined)) }, + }, + }); const build: any = { id: 7, uuid: 'superseded-manifest-build', @@ -6447,7 +5621,7 @@ describe('BuildService uncovered public behavior', () => { { name: 'build promotion', resource: 'build-promotion.41', - ttlMs: 15 * 60 * 1000, + ttlMs: 60_000, warning: 'Build promotion: waiting for admitted native mutation', context: (error: Error) => ({ error, buildId: 41 }), run: (service: BuildService, isCurrent: () => Promise, action: () => Promise) => @@ -6500,6 +5674,7 @@ describe('BuildService uncovered public behavior', () => { const { acquiredLock, lockWithOptions, redlock } = contendedAuthorityRedlock(waitError); const noDeploys: any = { where: jest.fn(() => noDeploys), + whereNotIn: jest.fn(() => noDeploys), withGraphFetched: jest.fn().mockResolvedValue([]), }; mockDeployQuery.mockReturnValue(noDeploys); @@ -6520,7 +5695,15 @@ describe('BuildService uncovered public behavior', () => { $fetchGraph: jest.fn().mockResolvedValue(undefined), }; const currentBuildQuery = lookupQuery(build); - const service = serviceWith({ models: { Build: { query: jest.fn(() => currentBuildQuery) } } }, redlock); + const service = serviceWith( + { + models: { + Build: { query: jest.fn(() => currentBuildQuery) }, + Deploy: { query: jest.fn(() => lookupQuery({ id: 11 })) }, + }, + }, + redlock + ); const result = service.generateAndApplyManifests({ build, @@ -6534,7 +5717,7 @@ describe('BuildService uncovered public behavior', () => { await expect(result).resolves.toBe(true); expect(lockWithOptions).toHaveBeenCalledTimes(2); - expect(lockWithOptions).toHaveBeenNthCalledWith(1, 'build-deployment.7', 15 * 60 * 1000, { + expect(lockWithOptions).toHaveBeenNthCalledWith(1, 'build-deployment.7', 60_000, { retryCount: 4, retryDelay: 1000, retryJitter: 200, @@ -6610,7 +5793,7 @@ describe('BuildService uncovered public behavior', () => { await expect(result).resolves.toBeUndefined(); expect(lockWithOptions).toHaveBeenCalledTimes(2); - expect(lockWithOptions).toHaveBeenNthCalledWith(1, 'build-deployment.7', 15 * 60 * 1000, { + expect(lockWithOptions).toHaveBeenNthCalledWith(1, 'build-deployment.7', 60_000, { retryCount: 4, retryDelay: 1000, retryJitter: 200, diff --git a/src/server/services/__tests__/deploy.test.ts b/src/server/services/__tests__/deploy.test.ts index bf4ee7d6..7305ba95 100644 --- a/src/server/services/__tests__/deploy.test.ts +++ b/src/server/services/__tests__/deploy.test.ts @@ -732,15 +732,10 @@ describe('DeployService - shouldTriggerGithubDeployment', () => { expect(mockBuildWithNative).not.toHaveBeenCalled(); }); - test('generation check uses buildId before the Build relation is loaded', async () => { - const currentBuildWhere = jest.fn().mockResolvedValue({ id: 91 }); - mockDb.models.Build = { - query: jest.fn(() => ({ - findOne: jest.fn(() => ({ - whereNull: jest.fn(() => ({ where: currentBuildWhere })), - })), - })), - }; + test('generation check fences on the Deploy row and the environment block state before the Build relation is loaded', async () => { + const authoritySelect = jest.fn().mockResolvedValue({ id: 91, deployEnabled: true }); + const authorityFindById = jest.fn(() => ({ select: authoritySelect })); + mockDb.models.Build = { query: jest.fn(() => ({ findById: authorityFindById })) }; const patchSpy = jest.spyOn(deployService, 'patchAndUpdateActivityFeed').mockResolvedValue(undefined); const deploy = { id: 17, @@ -755,7 +750,8 @@ describe('DeployService - shouldTriggerGithubDeployment', () => { deployService.buildImage(deploy as any, 0, 'run-c', undefined, undefined, undefined, 7) ).resolves.toBe(true); - expect(currentBuildWhere).toHaveBeenCalledWith('desiredGeneration', 7); + expect(currentDeployFindOne).toHaveBeenCalledWith({ id: 17, runUUID: 'run-c', desiredGeneration: 7 }); + expect(authorityFindById).toHaveBeenCalledWith(91); expect(deploy.$fetchGraph).toHaveBeenCalled(); expect(patchSpy).toHaveBeenCalledWith( deploy, @@ -789,6 +785,15 @@ describe('DeployService - shouldTriggerGithubDeployment', () => { expect(deploy.$fetchGraph).not.toHaveBeenCalled(); }); + test('patchDeployForRun fences on the generation it was handed', async () => { + const deploy = { id: 17, uuid: 'sample-service-build', runUUID: 'run-1' }; + + await deployService['patchDeployForRun'](deploy as any, 'run-1', { status: DeployStatus.DEPLOYING }, 6); + + expect(conditionalDeployWhere).toHaveBeenCalledWith({ id: 17, runUUID: 'run-1', desiredGeneration: 6 }); + expect(conditionalDeployPatch).toHaveBeenCalledWith({ status: DeployStatus.DEPLOYING }); + }); + test('recordDeployFailure writes a terminal status with the original error message', async () => { const patchSpy = jest.spyOn(deployService, 'patchAndUpdateActivityFeed').mockResolvedValue(undefined); const deploy = { @@ -1785,10 +1790,12 @@ describe('DeployService uncovered public behavior', () => { }; const buildQuery: any = { findOne: jest.fn(() => buildQuery), + findById: jest.fn(() => buildQuery), + select: jest.fn(() => buildQuery), whereNull: jest.fn(() => buildQuery), where: jest.fn(() => buildQuery), then: (resolve: (value: unknown) => unknown, reject?: (reason: unknown) => unknown) => - Promise.resolve({ id: 91 }).then(resolve, reject), + Promise.resolve({ id: 91, deployEnabled: true }).then(resolve, reject), }; const githubDeploymentAdd = jest.fn().mockResolvedValue(undefined); const updatePullRequestActivityStream = jest.fn().mockResolvedValue(undefined); @@ -2205,6 +2212,80 @@ describe('DeployService uncovered public behavior', () => { expect(mockCliDeploy).not.toHaveBeenCalled(); }); + test.each([DeployStatus.QUEUED, DeployStatus.ERROR])( + 'deployAurora keeps a restored endpoint from %s once the check finds its database', + async (status) => { + const { service, deployPatch } = serviceHarness(); + mockTaggingGetResources.mockResolvedValue({ + ResourceTagMappingList: [{ ResourceARN: 'arn:aws:rds:us-west-2:123:db:instance-1' }], + }); + mockRdsDescribeDBInstances.mockResolvedValue({ + DBInstances: [{ Endpoint: { Address: 'database-env.cluster-abc.us-west-2.rds.amazonaws.com' } }], + }); + const deploy: any = { + id: 1, + uuid: 'database-env', + status, + cname: 'database-env.cluster-abc.us-west-2.rds.amazonaws.com', + build: { uuid: 'env' }, + deployable: { name: 'database', type: DeployTypes.AURORA_RESTORE }, + reload: jest.fn().mockResolvedValue(undefined), + $fetchGraph: jest.fn().mockResolvedValue(undefined), + }; + + await expect(service.deployAurora(deploy, 'run-1')).resolves.toBe(true); + + expect(mockCliDeploy).not.toHaveBeenCalled(); + expect(deployPatch).toHaveBeenCalledTimes(1); + expect(deployPatch).toHaveBeenCalledWith({ + cname: 'database-env.cluster-abc.us-west-2.rds.amazonaws.com', + status: DeployStatus.BUILT, + }); + } + ); + + test('deployAurora keeps a restored endpoint when the database check fails', async () => { + const { service, deployPatch } = serviceHarness(); + mockTaggingGetResources.mockRejectedValue(new Error('throttled')); + const deploy: any = { + id: 1, + uuid: 'database-env', + status: DeployStatus.QUEUED, + cname: 'database-env.cluster-abc.us-west-2.rds.amazonaws.com', + build: { uuid: 'env' }, + deployable: { name: 'database', type: DeployTypes.AURORA_RESTORE }, + reload: jest.fn().mockResolvedValue(undefined), + $fetchGraph: jest.fn().mockResolvedValue(undefined), + }; + + await expect(service.deployAurora(deploy, 'run-1')).resolves.toBe(true); + + expect(mockCliDeploy).not.toHaveBeenCalled(); + expect(deployPatch).toHaveBeenCalledTimes(1); + expect(deployPatch).toHaveBeenCalledWith({ status: DeployStatus.BUILT }); + }); + + test('deployAurora restores again when its database is gone, as after a teardown that kept the row', async () => { + const { service, deployPatch } = serviceHarness(); + mockTaggingGetResources.mockResolvedValue({ ResourceTagMappingList: [] }); + const deploy: any = { + id: 1, + uuid: 'database-env', + status: DeployStatus.QUEUED, + cname: 'database-env.cluster-abc.us-west-2.rds.amazonaws.com', + build: { uuid: 'env' }, + deployable: { name: 'database', type: DeployTypes.AURORA_RESTORE }, + reload: jest.fn().mockResolvedValue(undefined), + $fetchGraph: jest.fn().mockResolvedValue(undefined), + }; + + await expect(service.deployAurora(deploy, 'run-1')).resolves.toBe(true); + + expect(mockCliDeploy).toHaveBeenCalledWith(deploy); + expect(deployPatch).toHaveBeenCalledWith(expect.objectContaining({ status: DeployStatus.BUILDING })); + expect(deployPatch).toHaveBeenLastCalledWith({ status: DeployStatus.BUILT }); + }); + test.each([ DeployStatus.QUEUED, DeployStatus.BUILDING, @@ -2366,7 +2447,7 @@ describe('DeployService uncovered public behavior', () => { await expect(service.deployCLI({ deployable: null } as any, 'run-4')).resolves.toBeUndefined(); expect(aurora).toHaveBeenCalledWith(auroraDeploy, 'run-1'); - expect(codefresh).toHaveBeenCalledWith(codefreshDeployable, 'run-2', 'sha', 42, 'main'); + expect(codefresh).toHaveBeenCalledWith(codefreshDeployable, 'run-2', 'sha', 42, 'main', undefined, undefined); }); test('hostForDeployableDeploy and acmARNForDeploy expose the stable routing fallbacks', () => { @@ -2576,6 +2657,7 @@ describe('DeployService uncovered public behavior', () => { undefined, undefined, undefined, + undefined, undefined ); }); diff --git a/src/server/services/__tests__/deployableSourceSeam.test.ts b/src/server/services/__tests__/deployableSourceSeam.test.ts index 39c906f7..c60e4f89 100644 --- a/src/server/services/__tests__/deployableSourceSeam.test.ts +++ b/src/server/services/__tests__/deployableSourceSeam.test.ts @@ -214,6 +214,78 @@ describe('deployable source seam (PR vs API build)', () => { expect(mockFetchLifecycleConfigByRepository).toHaveBeenCalledWith(repository, 'root-config-sha'); }); + it('reads the root config at the newest accepted push to the root branch when the delivered source is a dependency', async () => { + const service = makeService(); + (service as any).db.models.Repository = { + query: jest.fn(() => ({ + findOne: jest.fn(() => ({ + whereNull: jest.fn().mockResolvedValue({ githubRepositoryId: 99, fullName: 'org/dependency' }), + })), + })), + }; + const repository = { githubRepositoryId: 42, fullName: 'org/repo' }; + const pullRequest: any = { + branchName: 'feature-1', + repository, + build: { deploys: [], environment: { id: 5 } }, + $fetchGraph: jest.fn().mockResolvedValue(undefined), + }; + mockFetchLifecycleConfigByRepository.mockResolvedValue(null); + + await (service as any).updateOrCreateDeployableUsingYamlConfig( + new Map(), + 1, + 'uuid-1', + pullRequest, + { id: 1 }, + 99, + 'dependency-push-sha', + 'main', + 99, + new Set(), + new Set(), + new Map(), + { '42:feature-1': 'root-accepted-sha', '99:main': 'dependency-push-sha' } + ); + + expect(mockFetchLifecycleConfigByRepository).toHaveBeenCalledWith(repository, 'root-accepted-sha'); + }); + + it('keeps an explicitly pinned config SHA ahead of any accepted root floor', async () => { + const service = makeService(); + const repository = { githubRepositoryId: 42, fullName: 'org/repo' }; + mockRepositoryWhereNull.mockResolvedValue(repository); + mockFetchLifecycleConfigByRepository.mockResolvedValue(null); + const build: any = { + id: 9, + triggerType: 'api', + githubRepositoryId: 42, + branchName: 'main', + configSha: 'root-config-sha', + deploys: [], + environment: { id: 5 }, + $fetchGraph: jest.fn().mockResolvedValue(undefined), + }; + + await (service as any).updateOrCreateDeployableUsingYamlConfig( + new Map(), + 9, + 'uuid-9', + null, + build, + undefined, + undefined, + undefined, + undefined, + new Set(), + new Set(), + new Map(), + { '42:main': 'accepted-later' } + ); + + expect(mockFetchLifecycleConfigByRepository).toHaveBeenCalledWith(repository, 'root-config-sha'); + }); + it('fetches a targeted dependency config at the same pushed SHA used for its code', async () => { const service = makeService(); const rootRepository = { githubRepositoryId: 42, fullName: 'org/root' }; @@ -1113,6 +1185,7 @@ describe('deployable source seam (PR vs API build)', () => { expect(result).toEqual({ deployables: [persistedDeployable], canReconcile: true, + configFailures: {}, filterGithubRepositoryId: 42, unresolvedServiceNames: ['unresolved-api'], unresolvedRepositoryIds: [99], diff --git a/src/server/services/__tests__/github.behavior.test.ts b/src/server/services/__tests__/github.behavior.test.ts index b464f390..9772e544 100644 --- a/src/server/services/__tests__/github.behavior.test.ts +++ b/src/server/services/__tests__/github.behavior.test.ts @@ -111,7 +111,7 @@ function buildQueryResult(result: unknown) { const query = { whereIn: jest.fn(), andWhere: jest.fn(), - first: jest.fn().mockResolvedValue(result), + whereNull: jest.fn().mockResolvedValue(result == null ? [] : [result]), }; query.whereIn.mockImplementation((_column, callback) => { const nested = { @@ -616,9 +616,9 @@ describe('GithubService push fallback behavior', () => { throw new Error('query unavailable'); }); - await expect( - service.handlePushForStaticEnv({ githubRepositoryId: 101, branchName: 'main' }) - ).resolves.toBeUndefined(); + await expect(service.handlePushForStaticEnv({ githubRepositoryId: 101, branchName: 'main' })).resolves.toEqual( + new Set() + ); expect(mockLoggerError).toHaveBeenCalledWith( expect.objectContaining({ error: expect.any(Error) }), @@ -642,6 +642,29 @@ describe('GithubService push fallback behavior', () => { ); }); + it('claims every root static build even when one enqueue fails', async () => { + const { service, db } = createHarness(); + const query = buildQueryResult({ id: 44 }); + query.whereNull.mockResolvedValue([{ id: 44 }, { id: 45 }]); + db.models.Build.query.mockReturnValue(query); + db.services.BuildService.enqueueResolveAndDeployBuild + .mockRejectedValueOnce(new Error('redis unavailable')) + .mockResolvedValue(undefined); + + await expect( + service.handlePushForStaticEnv({ githubRepositoryId: 101, branchName: 'main', headCommit: 'head-sha' }) + ).resolves.toEqual(new Set([44, 45])); + + expect(db.services.BuildService.enqueueResolveAndDeployBuild).toHaveBeenCalledTimes(2); + expect(db.services.BuildService.enqueueResolveAndDeployBuild).toHaveBeenCalledWith( + expect.objectContaining({ buildId: 45 }) + ); + expect(mockLoggerError).toHaveBeenCalledWith( + expect.objectContaining({ error: expect.any(Error) }), + 'Push: static env redeploy enqueue failed' + ); + }); + it('builds the nested static-environment lookup and forwards optional commit refs', async () => { const { service, db } = createHarness(); db.models.Build.query.mockReturnValue(buildQueryResult({ id: 44 })); diff --git a/src/server/services/__tests__/github.test.ts b/src/server/services/__tests__/github.test.ts index 668370ec..c4b9418d 100644 --- a/src/server/services/__tests__/github.test.ts +++ b/src/server/services/__tests__/github.test.ts @@ -1103,7 +1103,7 @@ describe('Github Service - handlePushWebhook', () => { return buildQuery; }), andWhere: jest.fn().mockReturnThis(), - first: jest.fn().mockResolvedValue({ id: buildId }), + whereNull: jest.fn().mockResolvedValue([{ id: buildId }]), }; mockDb.models.Build = { query: jest.fn().mockReturnValue(buildQuery) }; @@ -1215,7 +1215,7 @@ describe('Github Service - handlePushWebhook', () => { return buildQuery; }), andWhere: jest.fn().mockReturnThis(), - first: jest.fn().mockResolvedValue({ id: buildId }), + whereNull: jest.fn().mockResolvedValue([{ id: buildId }]), }; mockDb.models.Build = { query: jest.fn().mockReturnValue(buildQuery) }; diff --git a/src/server/services/__tests__/githubAutoTrack.test.ts b/src/server/services/__tests__/githubAutoTrack.test.ts index 4dbf5c0d..44a9ff4d 100644 --- a/src/server/services/__tests__/githubAutoTrack.test.ts +++ b/src/server/services/__tests__/githubAutoTrack.test.ts @@ -119,7 +119,7 @@ describe('handlePushWebhook auto-track wiring', () => { { registerQueue: jest.fn(() => ({ add: jest.fn(), on: jest.fn() })) } as any ); const autoTrack = jest.spyOn(service as any, 'enqueueAutoTrackedApiBuilds').mockResolvedValue(undefined); - jest.spyOn(service, 'handlePushForStaticEnv').mockResolvedValue(undefined); + jest.spyOn(service, 'handlePushForStaticEnv').mockResolvedValue(new Set()); await service.handlePushWebhook({ ref: 'refs/heads/main', @@ -156,7 +156,7 @@ describe('handlePushWebhook auto-track wiring', () => { { registerQueue: jest.fn(() => ({ add: jest.fn(), on: jest.fn() })) } as any ); jest.spyOn(service as any, 'enqueueAutoTrackedApiBuilds').mockResolvedValue(undefined); - const staticFallback = jest.spyOn(service, 'handlePushForStaticEnv').mockResolvedValue(undefined); + const staticFallback = jest.spyOn(service, 'handlePushForStaticEnv').mockResolvedValue(new Set()); await service.handlePushWebhook({ ref: 'refs/heads/main', @@ -207,7 +207,7 @@ describe('handlePushWebhook auto-track wiring', () => { { registerQueue: jest.fn(() => ({ add: jest.fn(), on: jest.fn() })) } as any ); jest.spyOn(service as any, 'enqueueAutoTrackedApiBuilds').mockResolvedValue(undefined); - const staticFallback = jest.spyOn(service, 'handlePushForStaticEnv').mockResolvedValue(undefined); + const staticFallback = jest.spyOn(service, 'handlePushForStaticEnv').mockResolvedValue(new Set()); await service.handlePushWebhook({ ref: 'refs/heads/main', @@ -217,7 +217,10 @@ describe('handlePushWebhook auto-track wiring', () => { repository: { id: 42, full_name: 'org/repo' }, } as any); - expect(staticFallback).not.toHaveBeenCalled(); + // Static environments rooted at the pushed repo are always looked up, even when the repo also owns services. + expect(staticFallback).toHaveBeenCalledWith( + expect.objectContaining({ githubRepositoryId: 42, branchName: 'main', headCommit: 'sha123' }) + ); expect(enqueueResolveAndDeployBuild).toHaveBeenCalledWith( expect.objectContaining({ buildId: 7, sourceGithubRepositoryId: 42, sourceBranch: 'main' }) ); diff --git a/src/server/services/__tests__/ingress.test.ts b/src/server/services/__tests__/ingress.test.ts index c4d6fd11..5e8f3b09 100644 --- a/src/server/services/__tests__/ingress.test.ts +++ b/src/server/services/__tests__/ingress.test.ts @@ -83,6 +83,7 @@ function authorityQuery(result: any) { const query: any = { findOne: jest.fn(() => query), findById: jest.fn(() => query), + select: jest.fn(() => query), whereNull: jest.fn(() => query), where: jest.fn(() => query), then: (resolve: (value: any) => void, reject: (reason: unknown) => void) => @@ -287,9 +288,8 @@ describe('IngressService', () => { data: { buildId: 7, runUUID: 'old-run', expectedGeneration: 4 }, }); - expect(authority.findOne).toHaveBeenCalledWith({ id: 7, runUUID: 'old-run' }); + expect(authority.findById).toHaveBeenCalledWith(7); expect(authority.whereNull).toHaveBeenCalledWith('deletedAt'); - expect(authority.where).toHaveBeenCalledWith('desiredGeneration', 4); expect(configurationsForBuildId).not.toHaveBeenCalled(); expect(mockGetAllConfigs).not.toHaveBeenCalled(); expect(mockLoggerInfo).toHaveBeenCalledWith('Ingress: skipped reason=superseded'); @@ -379,15 +379,21 @@ describe('IngressService', () => { ingressAnnotations: {}, ipWhitelist: [], }; + const configurationsForBuildId = jest.fn().mockResolvedValue([configuration]); const gate = jest.fn(async (_buildId, isCurrent, action) => { expect(await isCurrent()).toBe(true); + // Routes are read only once the lock is held, so a sibling's newer route is never overwritten by a stale snapshot. + expect(configurationsForBuildId).not.toHaveBeenCalled(); return { admitted: true, value: await action() }; }); const db = { - models: { Build: { query: jest.fn(() => authorityQuery({ id: 7 })) } }, + models: { + Build: { query: jest.fn(() => authorityQuery({ id: 7 })) }, + Deploy: { query: jest.fn(() => authorityQuery({ id: 11 })) }, + }, services: { BuildService: { - configurationsForBuildId: jest.fn().mockResolvedValue([configuration]), + configurationsForBuildId, getNamespace: jest.fn().mockResolvedValue('env-test'), withCurrentBuildPromotionLock: gate, }, @@ -407,7 +413,10 @@ describe('IngressService', () => { test('does not apply when authority is lost before promotion admission', async () => { const gate = jest.fn().mockResolvedValue({ admitted: false }); const db = { - models: { Build: { query: jest.fn(() => authorityQuery({ id: 7 })) } }, + models: { + Build: { query: jest.fn(() => authorityQuery({ id: 7 })) }, + Deploy: { query: jest.fn(() => authorityQuery({ id: 11 })) }, + }, services: { BuildService: { configurationsForBuildId: jest.fn().mockResolvedValue([ @@ -435,28 +444,49 @@ describe('IngressService', () => { expect(apply).not.toHaveBeenCalled(); }); - test('fences a late ingress failure note by run token and generation', async () => { + test('records a late ingress failure note for a generation-fenced run that still owns a service', async () => { const read = authorityQuery({ id: 7, statusMessage: 'deployed' }); const patch: any = { patch: jest.fn(() => patch), where: jest.fn(() => patch), + whereNull: jest.fn(() => patch), then: (resolve: (value: number) => void, reject: (reason: unknown) => void) => Promise.resolve(1).then(resolve, reject), }; const db = { - models: { Build: { query: jest.fn().mockReturnValueOnce(read).mockReturnValueOnce(patch) } }, + models: { + Build: { query: jest.fn().mockReturnValueOnce(read).mockReturnValueOnce(patch) }, + Deploy: { query: jest.fn(() => authorityQuery({ id: 11 })) }, + }, services: {}, }; const service = new IngressService(db as any, {} as any, {} as any, queueManager as any); await (service as any).recordIngressFailureOnBuild(7, new Error('bad route'), 'run-c', 3); - expect(read.where).toHaveBeenCalledWith('runUUID', 'run-c'); - expect(read.where).toHaveBeenCalledWith('desiredGeneration', 3); + expect(read.whereNull).toHaveBeenCalledWith('deletedAt'); + expect(read.where).not.toHaveBeenCalledWith('runUUID', 'run-c'); expect(patch.patch).toHaveBeenCalledWith({ statusMessage: 'deployed | Ingress apply failed: bad route' }); expect(patch.where).toHaveBeenCalledWith({ id: 7 }); - expect(patch.where).toHaveBeenCalledWith('runUUID', 'run-c'); - expect(patch.where).toHaveBeenCalledWith('desiredGeneration', 3); + expect(patch.whereNull).toHaveBeenCalledWith('deletedAt'); + expect(patch.where).not.toHaveBeenCalledWith('runUUID', 'run-c'); + expect(patch.where).not.toHaveBeenCalledWith('desiredGeneration', 3); + }); + + test('drops a late ingress failure note once the run owns no service any more', async () => { + const buildQuery = jest.fn(); + const db = { + models: { + Build: { query: buildQuery }, + Deploy: { query: jest.fn(() => authorityQuery(undefined)) }, + }, + services: {}, + }; + const service = new IngressService(db as any, {} as any, {} as any, queueManager as any); + + await (service as any).recordIngressFailureOnBuild(7, new Error('bad route'), 'run-old', 3); + + expect(buildQuery).not.toHaveBeenCalled(); }); test.each([ diff --git a/src/server/services/__tests__/serviceReconciliation.test.ts b/src/server/services/__tests__/serviceReconciliation.test.ts new file mode 100644 index 00000000..3740806b --- /dev/null +++ b/src/server/services/__tests__/serviceReconciliation.test.ts @@ -0,0 +1,1663 @@ +/** + * Copyright 2026 GoodRx, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Drives the real reconciliation path (webhook intent -> mailbox -> queue -> + * claim -> config -> image -> rollout -> observe -> status) on a simulated + * clock. Only the leaves are simulated: the image build, the Kubernetes + * apply, the YAML import and the queue transport. + */ + +const logs: string[] = []; +const sim: any = {}; + +jest.mock('server/lib/dependencies', () => ({ + defaultDb: {}, + defaultRedis: {}, + defaultRedlock: {}, + defaultQueueManager: {}, + redisClient: { getConnection: jest.fn() }, +})); + +jest.mock('server/lib/tracer', () => ({ + Tracer: { getInstance: jest.fn(() => ({ initialize: jest.fn() })) }, +})); + +jest.mock('server/lib/logger', () => { + const record = (level: string) => (a: any, b?: any) => { + const msg = typeof a === 'string' ? a : b; + if (typeof msg === 'string') logs.push(`${level} ${msg}`); + }; + return { + getLogger: jest.fn(() => ({ + error: record('error'), + fatal: record('fatal'), + info: record('info'), + warn: record('warn'), + debug: () => undefined, + })), + withLogContext: jest.fn((_ctx, fn) => fn()), + extractContextForQueue: jest.fn(() => ({})), + updateLogContext: jest.fn(), + LogStage: {}, + }; +}); + +jest.mock('shared/config', () => ({ + TMP_PATH: '/tmp', + QUEUE_NAMES: { + DELETE_QUEUE: 'delete_queue_test', + BUILD_QUEUE: 'build_queue_test', + DEPLOYMENT_RECONCILIATION: 'deployment_reconciliation_test', + SERVICE_RECONCILIATION: 'service_reconciliation_test', + API_ENV_CREATE: 'api_env_create_test', + API_ENV_EXPIRY: 'api_env_expiry_test', + RESOLVE_AND_DEPLOY: 'resolve_and_deploy_test', + BUILD_CLEANUP_QUEUE: 'build_cleanup_test', + BUILD_REQUEST_QUEUE: 'build_request_test', + DEPLOY_CLEANUP: 'deploy_cleanup_test', + GLOBAL_CONFIG_CACHE_REFRESH: 'global-config-refresh', + GITHUB_CLIENT_TOKEN_CACHE_REFRESH: 'github-client-token-refresh', + INGRESS_MANIFEST_QUEUE: 'ingress-manifest', + INGRESS_MANIFEST: 'ingress-manifest', + INGRESS_CLEANUP: 'ingress-cleanup', + AGENT_PREWARM_QUEUE: 'agent-prewarm', + }, +})); + +jest.mock('server/models/Build', () => ({ + __esModule: true, + default: { + query: () => sim.buildQuery(), + transact: (action: any) => sim.transact(action), + }, +})); + +jest.mock('server/models/Deploy', () => ({ + __esModule: true, + default: { query: () => sim.deployQuery() }, +})); + +jest.mock('server/models', () => ({ + Build: class {}, + Deploy: { query: () => sim.deployQuery() }, + Deployable: class {}, + Environment: class {}, + Repository: { query: () => sim.emptyQuery() }, +})); + +jest.mock('server/lib/kubernetes', () => ({ + generateManifest: jest.fn(), + generateDeployManifest: jest.fn(), + applyManifests: jest.fn(), + waitForPodReady: jest.fn(), + createOrUpdateNamespace: jest.fn(), + deleteBuild: jest.fn(), + deleteNamespace: jest.fn(), +})); +jest.mock('server/lib/cli', () => ({ deleteBuild: jest.fn() })); +jest.mock('server/lib/kubernetes/common/serviceAccount', () => ({ + ensureServiceAccountForJob: jest.fn().mockResolvedValue('default'), +})); +jest.mock('server/lib/github', () => ({ + createGitDeployment: jest.fn(), + updateGitDeploymentStatus: jest.fn(), + getPullRequest: jest.fn(), + getSHAForBranch: jest.fn(), + compareCommits: jest.fn(), + getYamlFileContent: jest.fn(), + getYamlFileContentFromBranch: jest.fn(), +})); +jest.mock('server/lib/helm', () => ({ uninstallHelmReleases: jest.fn() })); +jest.mock('server/lib/helm/utils', () => ({ ingressBannerSnippet: jest.fn(() => '') })); +jest.mock('server/lib/buildEnvVariables', () => ({ + BuildEnvironmentVariables: jest.fn().mockImplementation(() => ({ resolve: jest.fn().mockResolvedValue({}) })), +})); +jest.mock('server/lib/dependencyGraph', () => ({ generateGraph: jest.fn().mockResolvedValue({}) })); +jest.mock('server/services/globalConfig', () => ({ + __esModule: true, + default: { + getInstance: jest.fn(() => ({ + getAllConfigs: jest.fn().mockResolvedValue({ serviceAccount: { name: 'default' } }), + isFeatureEnabled: jest.fn().mockResolvedValue(false), + })), + }, +})); +jest.mock('server/services/deployCleanup', () => jest.fn().mockImplementation(() => ({}))); +jest.mock('server/services/deploy', () => ({ + __esModule: true, + default: jest.fn().mockImplementation(() => ({})), +})); +jest.mock('server/services/webhook', () => ({ + __esModule: true, + default: jest.fn().mockImplementation(() => ({})), +})); +jest.mock('server/services/override', () => ({ + __esModule: true, + isBranchOrExternalUrlEditable: () => true, + default: jest.fn().mockImplementation(() => ({})), +})); +jest.mock('server/lib/fastly', () => jest.fn().mockImplementation(() => ({}))); + +import BuildService from '../build'; +import { DeploymentSupersededError } from 'server/lib/deploymentReconciliation/errors'; +import { AuthorityLockLostError } from 'server/lib/authorityLock'; +import { BuildKind, BuildStatus, DeployStatus, DeployTypes, PullRequestStatus } from 'shared/constants'; +import { ParsingError } from 'server/lib/yamlConfigParser'; + +const MIN = 60_000; +const BUILD_ID = 7; +const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + +interface ServiceSpec { + name: string; + repo: number; + buildMinutes: number; +} + +function createWorld( + specs: ServiceSpec[], + options: { applyMinutes?: number; readyMinutes?: number; concurrency?: number } = {} +) { + const applyMs = (options.applyMinutes ?? 0.5) * MIN; + const readyMs = (options.readyMinutes ?? 2) * MIN; + const events: string[] = []; + const now = () => Number((Date.now() / MIN).toFixed(1)); + const log = (line: string) => events.push(`t=${now()}m ${line}`); + + const pullRequest = { + id: 11, + status: PullRequestStatus.OPEN, + deployOnUpdate: true, + branchName: 'static', + repository: { githubRepositoryId: 900, fullName: 'org/z' }, + $fetchGraph: async () => undefined, + }; + const buildRow: any = { + id: BUILD_ID, + uuid: 'static-z', + namespace: 'env-static-z', + kind: BuildKind.ENVIRONMENT, + isStatic: true, + status: BuildStatus.DEPLOYED, + statusMessage: '', + runUUID: 'initial', + deletedAt: null, + pullRequestId: 11, + deployEnabled: true, + desiredGeneration: 0, + observedGeneration: 0, + acceptedRefs: {}, + }; + const deployRows: any[] = specs.map((spec, index) => ({ + id: index + 1, + deployableId: index + 1, + uuid: `${spec.name}-static-z`, + buildId: BUILD_ID, + githubRepositoryId: spec.repo, + branchName: 'main', + active: true, + runUUID: 'initial', + sha: `${spec.name}-v0`, + dockerImage: `${spec.name}@${spec.name}-v0`, + status: DeployStatus.READY, + statusMessage: '', + desiredGeneration: 0, + observedGeneration: 0, + deployableName: spec.name, + })); + const specByName = new Map(specs.map((spec) => [spec.name, spec])); + const registry = new Set(deployRows.map((row) => row.dockerImage)); + const live = new Map(deployRows.map((row) => [row.deployableName, row.dockerImage])); + const liveAt = new Map(); + const failImport = new Map Error | null>(); + const failBuild = new Set(); + const applyFailures: Error[] = []; + const importCounts = new Map(); + const serviceConfigFailures: Record = {}; + const servicesAddedByConfig: ServiceSpec[] = []; + const orphanRows = new Set(); + const statusWrites: string[] = []; + const dependsOn = new Map(); + const serviceMoves: Array<{ name: string; repo: number; sha: string }> = []; + const importRefs: Array<{ + filter: number | null; + repo: number | null; + ref: string | null; + pins: Record | null; + }> = []; + const counters = { builds: new Map(), applies: new Map() }; + + const hydrateDeploy = (row: any) => { + const deploy: any = { + ...row, + deployable: orphanRows.has(row.deployableName) + ? null + : { + name: row.deployableName, + type: DeployTypes.GITHUB, + deploymentDependsOn: [...(dependsOn.get(row.deployableName) ?? [])], + }, + }; + deploy.$fetchGraph = async () => undefined; + deploy.reload = async () => Object.assign(deploy, row); + deploy.$query = () => query(() => [row], hydrateDeploy, 'deploy'); + return deploy; + }; + const hydrateBuild = (row: any) => { + const build: any = { ...row }; + const attach = async () => { + build.pullRequest = row.pullRequestId != null ? pullRequest : null; + build.environment = { id: 5 }; + build.deploys = deployRows.map(hydrateDeploy); + build.deployables = []; + }; + void attach(); + build.$fetchGraph = async () => { + if (sim.fetchGraphGate) { + const gate = sim.fetchGraphGate; + sim.fetchGraphGate = null; + await gate; + } + return attach(); + }; + build.$setRelated = (relation: string, value: any) => { + build[relation] = value; + }; + build.reload = async () => Object.assign(build, row); + build.$query = () => query(() => [row], hydrateBuild, 'build'); + return build; + }; + + function query(rows: () => any[], hydrate: (row: any) => any, table = 'deploy') { + const conditions: Array<(row: any) => boolean> = []; + let patch: any; + let single = false; + const evaluate = () => { + const found = rows().filter((row) => conditions.every((condition) => condition(row))); + if (patch) { + if (sim.failPatchOnce?.(patch, table)) { + sim.failPatchOnce = null; + throw new Error('database connection reset'); + } + found.forEach((row) => { + if (row === buildRow && patch.status) statusWrites.push(`${patch.status}:${patch.statusMessage ?? ''}`); + Object.assign(row, patch); + }); + return found.length; + } + if (single) return found[0] ? hydrate(found[0]) : undefined; + return found.map(hydrate); + }; + const compare = (operator: string, left: any, right: any) => { + switch (operator) { + case '<': + return Number(left) < Number(right); + case '>': + return Number(left) > Number(right); + case '<=': + return Number(left) <= Number(right); + case '>=': + return Number(left) >= Number(right); + default: + return left === right; + } + }; + const q: any = { + select: () => q, + distinct: () => q, + withGraphFetched: () => q, + forUpdate: () => q, + orderBy: () => q, + limit: () => q, + findById: (id: number) => { + single = true; + conditions.push((row) => row.id === id); + return q; + }, + findOne: (criteria: Record) => { + single = true; + Object.entries(criteria).forEach(([key, value]) => conditions.push((row) => row[key] === value)); + return q; + }, + where: (key: any, operatorOrValue?: any, value?: any) => { + if (typeof key === 'object') { + Object.entries(key).forEach(([field, expected]) => conditions.push((row) => row[field] === expected)); + } else if (value === undefined) { + conditions.push((row) => row[key] === operatorOrValue); + } else { + conditions.push((row) => compare(operatorOrValue, row[key], value)); + } + return q; + }, + whereRaw: (sql: string, bindings: string[]) => { + const operator = sql.includes('>') ? '>' : '<'; + conditions.push((row) => compare(operator, row[bindings[0]], row[bindings[1]])); + return q; + }, + whereNull: (key: string) => { + conditions.push((row) => row[key] == null); + return q; + }, + whereNot: (key: string, value: any) => { + conditions.push((row) => row[key] !== value); + return q; + }, + whereNotIn: (key: string, values: any[]) => { + conditions.push((row) => !values.includes(row[key])); + return q; + }, + whereIn: (key: string, values: any[]) => { + conditions.push((row) => values.includes(row[key])); + return q; + }, + patch: (value: any) => { + patch = value; + return q; + }, + then: (resolve: any, reject: any) => Promise.resolve().then(evaluate).then(resolve, reject), + }; + return q; + } + + sim.buildQuery = () => query(() => [buildRow], hydrateBuild, 'build'); + sim.deployQuery = () => query(() => deployRows, hydrateDeploy, 'deploy'); + sim.deployableQuery = () => + query( + () => + deployRows + .filter((row) => !orphanRows.has(row.deployableName)) + .map((row) => ({ + id: row.deployableId, + buildId: BUILD_ID, + name: row.deployableName, + type: DeployTypes.GITHUB, + })), + (row) => row, + 'deployable' + ); + sim.emptyQuery = () => + query( + () => [], + (row) => row, + 'none' + ); + // Every transaction locks the one Build row first, so transactions run one at a time. + let lastTransaction: Promise = Promise.resolve(); + sim.transact = (action: (trx: object) => Promise) => { + const run = lastTransaction.then(() => action({})); + lastTransaction = run.catch(() => undefined); + return run; + }; + sim.failPatchOnce = null; + sim.fetchGraphGate = null; + sim.importGate = null; + sim.importGateRepo = null; + + // One mutex per resource, so the simulator exercises the same serialization the Redis locks provide. + const held = new Map>(); + const redlock = { + lock: async (resource: string) => { + const previous = held.get(resource) ?? Promise.resolve(); + let release!: () => void; + const current = new Promise((resolve) => { + release = resolve; + }); + held.set( + resource, + previous.then(() => current) + ); + await previous; + const lock: any = { unlock: async () => release(), extend: async () => lock }; + return lock; + }, + }; + + const isRowCurrent = (deployId: number, runUUID: string, generation?: number) => { + const deploy = deployRows.find((row) => row.id === deployId); + return Boolean( + deploy?.runUUID === runUUID && + (generation == null || (deploy.desiredGeneration === generation && deploy.observedGeneration < generation)) + ); + }; + + const fencedDeployPatch = (deployId: number, runUUID: string, patch: any) => { + const deploy = deployRows.find((row) => row.id === deployId); + if (deploy?.runUUID === runUUID) Object.assign(deploy, patch); + }; + + const DeployLeaf = { + findOrCreateDeploys: async ( + _env: any, + _build: any, + githubRepositoryId?: number, + sourceRef?: string | null, + sourceBranch?: string | null, + _sourceGithubRepositoryId?: number | null, + acceptedSourcePins?: Record + ) => { + for (const row of deployRows) { + const target = + !githubRepositoryId || + (row.githubRepositoryId === githubRepositoryId && (!sourceBranch || row.branchName === sourceBranch)); + if (!target) continue; + if (sourceRef && row.githubRepositoryId === githubRepositoryId) row.sha = sourceRef; + else row.sha = acceptedSourcePins?.[`${row.githubRepositoryId}:${row.branchName}`] ?? row.sha; + } + return deployRows.map(hydrateDeploy); + }, + buildImage: async ( + deploy: any, + _index: number, + runUUID: string, + _sourceRef: string | null, + _sourceRepo: number | null, + _sourceBranch: string | null, + generation?: number + ) => { + const name = deploy.deployableName; + if (!isRowCurrent(deploy.id, runUUID, generation)) return true; + const row = deployRows.find((candidate) => candidate.id === deploy.id); + const tag = `${name}@${row.sha}`; + if (failBuild.has(name)) { + fencedDeployPatch(deploy.id, runUUID, { + status: DeployStatus.BUILD_FAILED, + statusMessage: 'Image build failed.', + }); + log(`gen${generation} ${name}: build FAILED`); + return false; + } + if (registry.has(tag)) { + fencedDeployPatch(deploy.id, runUUID, { status: DeployStatus.BUILT, dockerImage: tag }); + log(`gen${generation} ${name}: image exists ${tag}`); + return true; + } + counters.builds.set(name, (counters.builds.get(name) ?? 0) + 1); + fencedDeployPatch(deploy.id, runUUID, { status: DeployStatus.BUILDING }); + log(`gen${generation} ${name}: build start ${tag}`); + await sleep(specByName.get(name)!.buildMinutes * MIN); + registry.add(tag); + if (!isRowCurrent(deploy.id, runUUID, generation)) { + log(`gen${generation} ${name}: build done ${tag} (superseded, result not published)`); + return true; + } + log(`gen${generation} ${name}: build done ${tag}`); + fencedDeployPatch(deploy.id, runUUID, { status: DeployStatus.BUILT, dockerImage: tag }); + return true; + }, + }; + + const queue = { + active: new Set(), + adds: [] as Array<{ data: any; opts: any }>, + finalFailures: [] as string[], + attempts: [] as string[], + dropNext: 0, + // Worker slots, so a scenario can show what happens when every slot is busy. + concurrency: options.concurrency ?? Infinity, + running: 0, + waiting: [] as Array<() => void>, + async acquire() { + if (queue.running < queue.concurrency) { + queue.running += 1; + return; + } + await new Promise((resolve) => queue.waiting.push(resolve)); + }, + release() { + const next = queue.waiting.shift(); + if (next) next(); + else queue.running -= 1; + }, + add(data: any, opts: any) { + queue.adds.push({ data, opts }); + if (queue.dropNext > 0) { + queue.dropNext -= 1; + log(`SIGNAL DROPPED ${opts?.jobId}`); + return Promise.resolve(); + } + const jobId = opts?.jobId; + if (queue.active.has(jobId)) return Promise.resolve(); + queue.active.add(jobId); + const attempts = 10; + const job: any = { id: jobId, data, attemptsMade: 0, opts: { attempts } }; + const run = async (attemptsMade: number): Promise => { + await queue.acquire(); + job.attemptsMade = attemptsMade; + queue.attempts.push(`${jobId}#${attemptsMade}`); + let retryIn: number | null = null; + try { + await service.processDeploymentReconciliationQueue(job); + queue.active.delete(jobId); + } catch (error: any) { + if (attemptsMade + 1 >= attempts) { + queue.active.delete(jobId); + queue.finalFailures.push(`${jobId}: ${error?.message}`); + } else { + retryIn = 5000 * 2 ** attemptsMade; + } + } finally { + queue.release(); + } + if (retryIn != null) { + await sleep(retryIn); + return run(attemptsMade + 1); + } + }; + void run(0); + return Promise.resolve(); + }, + }; + + const service: any = new BuildService( + { + models: { + Build: { query: sim.buildQuery, transact: (action: any) => sim.transact(action) }, + Deploy: { query: sim.deployQuery }, + Deployable: { query: sim.deployableQuery }, + Repository: { query: sim.emptyQuery }, + }, + services: { + Deployable: { + upsertDeployables: async (...args: any[]) => { + const repo = args[5]; + const scope = repo == null ? 'root' : String(repo); + importCounts.set(scope, (importCounts.get(scope) ?? 0) + 1); + importRefs.push({ + filter: repo ?? null, + repo: args[8] ?? null, + ref: args[6] ?? null, + pins: args[9] ?? null, + }); + if ((repo ?? null) === sim.importGateRepo && sim.importGate) { + const gate = sim.importGate; + sim.importGate = null; + await gate; + } + const failure = failImport.get(repo ?? 900)?.(); + if (failure) { + log(`import repo=${scope} FAILED: ${failure.message}`); + throw failure; + } + if (repo == null) { + for (const move of serviceMoves.splice(0)) { + const moved = deployRows.find((candidate) => candidate.deployableName === move.name)!; + Object.assign(moved, { githubRepositoryId: move.repo, sha: move.sha }); + log(`import moved ${move.name} -> repo ${move.repo} @ ${move.sha}`); + } + for (const spec of servicesAddedByConfig.splice(0)) { + deployRows.push({ + id: deployRows.length + 1, + deployableId: deployRows.length + 1, + uuid: `${spec.name}-static-z`, + buildId: BUILD_ID, + githubRepositoryId: spec.repo, + branchName: 'main', + active: true, + runUUID: null, + sha: `${spec.name}-v0`, + dockerImage: `${spec.name}@${spec.name}-v0`, + status: DeployStatus.PENDING, + statusMessage: '', + desiredGeneration: 0, + observedGeneration: 0, + deployableName: spec.name, + }); + specByName.set(spec.name, spec); + registry.add(`${spec.name}@${spec.name}-v0`); + live.set(spec.name, `${spec.name}@${spec.name}-v0`); + log(`import created ${spec.name}`); + } + } + const configFailures = Object.fromEntries( + Object.entries(serviceConfigFailures).filter( + ([name]) => repo == null || specByName.get(name)?.repo === repo + ) + ); + return { canReconcile: false, configFailures }; + }, + }, + Deploy: DeployLeaf, + Webhook: { + upsertWebhooksWithYaml: async () => undefined, + webhookQueue: { add: async () => undefined }, + }, + ActivityStream: { updatePullRequestActivityStream: async () => undefined }, + }, + } as any, + {} as any, + redlock as any, + { + registerQueue: (name: string) => ({ + add: (_jobName: string, data: any, opts: any) => + name === 'service_reconciliation_test' ? queue.add(data, opts) : Promise.resolve(), + process: jest.fn(), + on: jest.fn(), + }), + } as any + ); + + jest.spyOn(service, 'generateAndApplyManifests').mockImplementation(async (params: any) => { + const { build, runUUID, expectedGeneration, unresolvedServices } = params; + const isCurrent = () => service.isDeploymentRunCurrent(build.id, runUUID, expectedGeneration); + if (!(await isCurrent())) throw new DeploymentSupersededError(); + const injected = applyFailures.shift(); + if (injected) throw injected; + const failed = new Set([DeployStatus.ERROR, DeployStatus.BUILD_FAILED, DeployStatus.DEPLOY_FAILED]); + // A fenced rollout takes exactly the rows this run claimed; rows already READY were rolled out by an earlier attempt. + const owned = deployRows.filter((row) => row.runUUID === runUUID && row.desiredGeneration === expectedGeneration); + const targets = owned.filter((row) => !failed.has(row.status) && row.status !== DeployStatus.READY); + // As in the manager: a prerequisite this run failed blocks its dependents; one outside the run is never consulted. + const blocked = new Set([ + ...(unresolvedServices ?? []), + ...owned.filter((row) => failed.has(row.status)).map((row) => row.deployableName), + ]); + const applied: any[] = []; + await sleep(applyMs); + for (const row of targets) { + // The per-service promotion gate re-checks the row inside the lock. + if (!isRowCurrent(row.id, runUUID, expectedGeneration)) continue; + const blockedBy = (dependsOn.get(row.deployableName) ?? []).find((name) => blocked.has(name)); + if (blockedBy) { + Object.assign(row, { + status: DeployStatus.DEPLOY_FAILED, + statusMessage: `Not deployed: ${blockedBy} failed.`, + }); + log(`gen${expectedGeneration} ${row.deployableName}: BLOCKED by ${blockedBy}`); + continue; + } + applied.push(row); + // The real apply path writes progress fenced on the run token alone. + fencedDeployPatch(row.id, runUUID, { + status: DeployStatus.DEPLOYING, + statusMessage: 'Waiting for pods to be ready', + }); + const before = live.get(row.deployableName); + live.set(row.deployableName, row.dockerImage); + if (before !== row.dockerImage) liveAt.set(row.deployableName, now()); + counters.applies.set(row.deployableName, (counters.applies.get(row.deployableName) ?? 0) + 1); + log( + `gen${expectedGeneration} ${row.deployableName}: APPLIED ${row.dockerImage}${ + before === row.dockerImage ? ' (re-rollout)' : '' + }` + ); + } + if (applied.length > 0) await sleep(readyMs); + for (const row of applied) { + if (isRowCurrent(row.id, runUUID, expectedGeneration)) Object.assign(row, { status: DeployStatus.READY }); + } + if (!(await isCurrent())) throw new DeploymentSupersededError(); + return true; + }); + + let shaCounter = 0; + const merge = async (name: string) => { + const spec = specByName.get(name)!; + shaCounter += 1; + const sha = `${name}-v${shaCounter}`; + log(`MERGE ${name} -> ${sha}`); + await service.enqueueResolveAndDeployBuild({ + buildId: BUILD_ID, + githubRepositoryId: spec.repo, + sourceRef: sha, + sourceGithubRepositoryId: spec.repo, + sourceBranch: 'main', + }); + return sha; + }; + + // A push that asks for a full pass, as a pull request with failed services does. + const mergeFull = async (name: string) => { + const spec = specByName.get(name)!; + shaCounter += 1; + const sha = `${name}-v${shaCounter}`; + log(`MERGE(full) ${name} -> ${sha}`); + await service.enqueueResolveAndDeployBuild({ + buildId: BUILD_ID, + sourceRef: sha, + sourceGithubRepositoryId: spec.repo, + sourceBranch: 'main', + }); + return sha; + }; + + const pushConfigRepo = async () => { + shaCounter += 1; + const sha = `z-v${shaCounter}`; + log(`PUSH Z -> ${sha}`); + await service.enqueueResolveAndDeployBuild({ + buildId: BUILD_ID, + sourceRef: sha, + sourceGithubRepositoryId: 900, + sourceBranch: 'static', + }); + return sha; + }; + + const redeployEnvironment = async (runUUID?: string) => { + log(`ENV REDEPLOY ${runUUID ?? 'auto'}`); + await service.enqueueResolveAndDeployBuild({ buildId: BUILD_ID, ...(runUUID ? { runUUID } : {}) }); + }; + + const row = (name: string) => deployRows.find((candidate) => candidate.deployableName === name)!; + const settled = () => deployRows.every((candidate) => candidate.observedGeneration === candidate.desiredGeneration); + + return { + service, + buildRow, + deployRows, + row, + live, + liveAt, + registry, + events, + counters, + failImport, + failBuild, + applyFailures, + importCounts, + serviceConfigFailures, + servicesAddedByConfig, + orphanRows, + statusWrites, + dependsOn, + serviceMoves, + importRefs, + queue, + merge, + mergeFull, + pushConfigRepo, + redeployEnvironment, + settled, + log, + }; +} + +const flush = async () => { + for (let i = 0; i < 30; i += 1) await Promise.resolve(); +}; + +beforeEach(() => { + logs.length = 0; + jest.useFakeTimers({ now: 0 }); +}); + +afterEach(() => { + jest.useRealTimers(); +}); + +const SPECS: ServiceSpec[] = [ + { name: 'A', repo: 100, buildMinutes: 20 }, + { name: 'B', repo: 200, buildMinutes: 3 }, + { name: 'C', repo: 300, buildMinutes: 3 }, + { name: 'D', repo: 400, buildMinutes: 3 }, + { name: 'E', repo: 400, buildMinutes: 3 }, +]; + +describe('service-independent reconciliation', () => { + test('a merge to B while A builds leaves A untouched: A ships once, on its own schedule', async () => { + const world = createWorld(SPECS); + const aSha = await world.merge('A'); + await jest.advanceTimersByTimeAsync(5 * MIN); + const bSha = await world.merge('B'); + await jest.advanceTimersByTimeAsync(6 * MIN); + + expect(world.live.get('B')).toBe(`B@${bSha}`); + expect(world.buildRow.status).toBe(BuildStatus.BUILDING); + + await jest.advanceTimersByTimeAsync(30 * MIN); + + expect(world.live.get('A')).toBe(`A@${aSha}`); + expect(world.liveAt.get('A')).toBe(20.5); + expect(world.liveAt.get('B')).toBe(8.5); + expect(world.counters.builds.get('A')).toBe(1); + expect(world.counters.applies.get('B')).toBe(1); + expect(logs.filter((line) => line.includes('rollout skipped'))).toEqual([]); + expect(world.settled()).toBe(true); + expect(world.buildRow.observedGeneration).toBe(world.buildRow.desiredGeneration); + expect(world.buildRow.status).toBe(BuildStatus.DEPLOYED); + }); + + test('a newer merge to A supersedes its in-flight run and only the newest SHA rolls out', async () => { + const world = createWorld(SPECS); + const first = await world.merge('A'); + await jest.advanceTimersByTimeAsync(5 * MIN); + const second = await world.merge('A'); + await jest.advanceTimersByTimeAsync(40 * MIN); + + expect(world.live.get('A')).toBe(`A@${second}`); + expect(world.events.some((line) => line.includes(`A@${first} (superseded, result not published)`))).toBe(true); + expect(world.counters.builds.get('A')).toBe(2); + expect(world.counters.applies.get('A')).toBe(1); + expect(world.row('A').observedGeneration).toBe(2); + expect(world.settled()).toBe(true); + }); + + test('a config error in C fails only C; A and B still ship and C recovers on its next merge', async () => { + const world = createWorld(SPECS); + const aSha = await world.merge('A'); + await jest.advanceTimersByTimeAsync(5 * MIN); + const bSha = await world.merge('B'); + await jest.advanceTimersByTimeAsync(1 * MIN); + world.serviceConfigFailures.C = 'C lifecycle.yaml is invalid'; + await world.merge('C'); + await jest.advanceTimersByTimeAsync(30 * MIN); + + expect(world.row('C').status).toBe(DeployStatus.ERROR); + expect(world.row('C').statusMessage).toContain('C lifecycle.yaml is invalid'); + expect(world.row('C').observedGeneration).toBe(world.row('C').desiredGeneration); + expect(world.live.get('C')).toBe('C@C-v0'); + expect(world.live.get('A')).toBe(`A@${aSha}`); + expect(world.live.get('B')).toBe(`B@${bSha}`); + expect(world.buildRow.status).toBe(BuildStatus.ERROR); + expect(world.buildRow.statusMessage).toContain('C'); + + delete world.serviceConfigFailures.C; + const cSha = await world.merge('C'); + await jest.advanceTimersByTimeAsync(10 * MIN); + + expect(world.live.get('C')).toBe(`C@${cSha}`); + expect(world.buildRow.status).toBe(BuildStatus.DEPLOYED); + expect(world.settled()).toBe(true); + }); + + test('a config-repo push supersedes in-flight work, and a later A merge supersedes only the A part of that redeploy', async () => { + const world = createWorld(SPECS); + const first = await world.merge('A'); + await jest.advanceTimersByTimeAsync(5 * MIN); + await world.pushConfigRepo(); + await jest.advanceTimersByTimeAsync(5 * MIN); + const second = await world.merge('A'); + await jest.advanceTimersByTimeAsync(60 * MIN); + + expect(world.live.get('A')).toBe(`A@${second}`); + expect(world.events.some((line) => line.includes(`A@${first} (superseded, result not published)`))).toBe(true); + for (const name of ['B', 'C', 'D', 'E']) { + expect(world.row(name).desiredGeneration).toBe(2); + expect(world.row(name).observedGeneration).toBe(2); + expect(world.counters.applies.get(name)).toBe(1); + } + expect(world.row('A').desiredGeneration).toBe(3); + expect(world.row('A').observedGeneration).toBe(3); + expect(world.buildRow.status).toBe(BuildStatus.DEPLOYED); + expect(world.settled()).toBe(true); + }); + + test('D failing its image leaves E, from the same repository, deploying and reports only D', async () => { + const world = createWorld(SPECS); + world.failBuild.add('D'); + const sha = await world.merge('D'); + await jest.advanceTimersByTimeAsync(15 * MIN); + + expect(world.row('D').status).toBe(DeployStatus.BUILD_FAILED); + expect(world.row('D').observedGeneration).toBe(1); + expect(world.live.get('D')).toBe('D@D-v0'); + expect(world.live.get('E')).toBe(`E@${sha}`); + expect(world.row('E').status).toBe(DeployStatus.READY); + expect(world.buildRow.status).toBe(BuildStatus.ERROR); + expect(world.buildRow.statusMessage).toContain('D'); + expect(world.buildRow.statusMessage).not.toContain('E'); + expect(world.settled()).toBe(true); + }); + + test('the service-row redeploy selects only that service repository', async () => { + const world = createWorld(SPECS); + await expect(world.service.redeployServiceFromBuild('static-z', 'B', BUILD_ID)).resolves.toMatchObject({ + status: 'success', + }); + await jest.advanceTimersByTimeAsync(10 * MIN); + + expect(world.row('B').desiredGeneration).toBe(1); + expect(world.row('B').observedGeneration).toBe(1); + expect(world.counters.applies.get('B')).toBe(1); + for (const name of ['A', 'C', 'D', 'E']) { + expect(world.row(name).desiredGeneration).toBe(0); + expect(world.counters.applies.get(name)).toBeUndefined(); + } + }); + + test('a disabled environment observes new work without building or deploying', async () => { + const world = createWorld(SPECS); + world.buildRow.deployEnabled = false; + (world.buildRow as any).pullRequestId = null; + await world.merge('A'); + await jest.advanceTimersByTimeAsync(5 * MIN); + + expect(world.row('A').desiredGeneration).toBe(1); + expect(world.row('A').observedGeneration).toBe(1); + expect(world.counters.builds.get('A')).toBeUndefined(); + expect(logs.some((line) => line.includes('observed without execution reason=deploy_disabled'))).toBe(true); + }); + + test('an environment that is tearing down defers pending work', async () => { + const world = createWorld(SPECS); + world.buildRow.status = BuildStatus.TEARING_DOWN; + await world.merge('A'); + await jest.advanceTimersByTimeAsync(5 * MIN); + + expect(world.row('A').desiredGeneration).toBe(1); + expect(world.row('A').observedGeneration).toBe(0); + expect(world.counters.builds.get('A')).toBeUndefined(); + expect(logs.some((line) => line.includes('deferred reason=tearing_down'))).toBe(true); + }); + + test('a transient failure retries only the failing service while B ships, and the final attempt records it', async () => { + const world = createWorld(SPECS); + world.failImport.set(100, () => new Error('GitHub API rate limit exceeded')); + await world.merge('A'); + await jest.advanceTimersByTimeAsync(5 * MIN); + const bSha = await world.merge('B'); + await jest.advanceTimersByTimeAsync(60 * MIN); + + expect(world.live.get('B')).toBe(`B@${bSha}`); + expect(world.liveAt.get('B')).toBe(8.5); + expect(world.counters.applies.get('B')).toBe(1); + expect(world.row('A').status).toBe(DeployStatus.ERROR); + expect(world.row('A').statusMessage).toContain('rate limit'); + expect(world.row('A').observedGeneration).toBe(1); + expect(world.live.get('A')).toBe('A@A-v0'); + expect(world.buildRow.status).toBe(BuildStatus.ERROR); + expect(world.settled()).toBe(true); + }); + + test('a signal for a replaced generation executes nothing; only the newest intent owns the rows', async () => { + const world = createWorld(SPECS); + await world.merge('A'); + const second = await world.merge('A'); + await jest.advanceTimersByTimeAsync(30 * MIN); + + expect(world.row('A').desiredGeneration).toBe(2); + expect(world.row('A').observedGeneration).toBe(2); + expect(world.counters.builds.get('A')).toBe(1); + expect(world.live.get('A')).toBe(`A@${second}`); + expect(world.events.filter((line) => line.includes('gen1 A: APPLIED'))).toEqual([]); + expect(world.settled()).toBe(true); + }); + + test('legacy queue payloads are adopted as intents and malformed jobs are rejected', async () => { + const world = createWorld(SPECS); + await world.service.processBuildQueue({ + data: { + buildId: BUILD_ID, + githubRepositoryId: 200, + sourceRef: 'B-legacy', + sourceGithubRepositoryId: 200, + sourceBranch: 'main', + }, + }); + expect(world.row('B').desiredGeneration).toBe(1); + expect(world.queue.adds.filter(({ opts }) => opts.jobId === `reconcile-${BUILD_ID}-1`)).toHaveLength(1); + await world.service.processResolveAndDeployBuildQueue({ data: { buildId: BUILD_ID, runUUID: 'legacy-full' } }); + expect(world.buildRow.acceptedRefs.all.gen).toBe(2); + expect(world.queue.adds.filter(({ opts }) => opts.jobId === `reconcile-${BUILD_ID}-2`)).toHaveLength(1); + await expect(world.service.processDeploymentReconciliationQueue({ data: {} } as any)).rejects.toThrow( + 'buildId and generation are required' + ); + }); + + test('the sweep signals every pending generation found on the rows', async () => { + const world = createWorld(SPECS); + world.row('A').desiredGeneration = 3; + world.row('B').desiredGeneration = 4; + world.buildRow.desiredGeneration = 4; + world.buildRow.observedGeneration = 2; + const add = jest.fn().mockResolvedValue(undefined); + world.service.deploymentReconciliationQueue = { add }; + + await world.service.enqueuePendingDeploymentReconciliations(); + + const jobIds = add.mock.calls.map(([, , opts]) => opts.jobId).sort(); + expect(jobIds).toEqual([`reconcile-${BUILD_ID}-3`, `reconcile-${BUILD_ID}-4`]); + }); + + test('the sweep also signals a build whose watermark says settled while a row is still pending', async () => { + const world = createWorld(SPECS); + world.buildRow.desiredGeneration = 5; + world.buildRow.observedGeneration = 5; + world.row('A').desiredGeneration = 5; + world.row('A').observedGeneration = 4; + const add = jest.fn().mockResolvedValue(undefined); + world.service.deploymentReconciliationQueue = { add }; + + await world.service.enqueuePendingDeploymentReconciliations(); + + expect(add.mock.calls.map(([, , opts]) => opts.jobId)).toEqual([`reconcile-${BUILD_ID}-5`]); + }); + + test('two services finishing together leave the watermark at the newest generation', async () => { + const world = createWorld(SPECS); + const publish = world.service.publishDerivedBuildStatus.bind(world.service); + let release!: () => void; + const gate = new Promise((resolve) => { + release = resolve; + }); + let held = false; + jest.spyOn(world.service, 'publishDerivedBuildStatus').mockImplementation(async (...args: any[]) => { + if (!held && world.row('B').observedGeneration === 1) { + held = true; + await gate; + } + return publish(...args); + }); + await world.merge('B'); + await world.merge('C'); + await jest.advanceTimersByTimeAsync(10 * MIN); + release(); + await jest.advanceTimersByTimeAsync(1 * MIN); + + expect(world.settled()).toBe(true); + expect(world.buildRow.observedGeneration).toBe(2); + const signalsBefore = world.queue.adds.length; + await world.service.enqueuePendingDeploymentReconciliations(); + expect(world.queue.adds.length).toBe(signalsBefore); + }); + + test('the sweep repairs a stale watermark instead of signalling a finished generation', async () => { + const world = createWorld(SPECS); + await world.merge('B'); + await jest.advanceTimersByTimeAsync(10 * MIN); + world.buildRow.observedGeneration = 0; + const signalsBefore = world.queue.adds.length; + + await world.service.enqueuePendingDeploymentReconciliations(); + + expect(world.queue.adds.length).toBe(signalsBefore); + expect(world.buildRow.observedGeneration).toBe(1); + }); + + test('steady merges to other services never delay A', async () => { + const many: ServiceSpec[] = [ + { name: 'A', repo: 100, buildMinutes: 20 }, + ...Array.from({ length: 12 }, (_, index) => ({ name: `S${index + 1}`, repo: 1000 + index, buildMinutes: 3 })), + ]; + const world = createWorld(many); + const aSha = await world.merge('A'); + for (let minute = 0; minute < 60; minute += 0.5) { + await jest.advanceTimersByTimeAsync(0.5 * MIN); + const now = minute + 0.5; + if (now % 5 === 0) await world.merge(`S${(now / 5) % 12 || 12}`); + } + + expect(world.live.get('A')).toBe(`A@${aSha}`); + expect(world.liveAt.get('A')).toBe(20.5); + expect(world.counters.builds.get('A')).toBe(1); + expect(world.events.filter((line) => line.includes('(re-rollout)'))).toEqual([]); + }); +}); + +describe('service-independent reconciliation: review regressions', () => { + test('a service row that no run has ever owned is claimed and deployed', async () => { + const world = createWorld(SPECS); + world.row('C').runUUID = null; + const sha = await world.merge('C'); + await jest.advanceTimersByTimeAsync(10 * MIN); + + expect(world.live.get('C')).toBe(`C@${sha}`); + expect(world.row('C').observedGeneration).toBe(1); + expect(world.settled()).toBe(true); + }); + + test('a row that another run backfills is never stamped to an intent that already claimed its rows', async () => { + const world = createWorld(SPECS); + const aSha = await world.merge('A'); + await jest.advanceTimersByTimeAsync(1 * MIN); + world.deployRows.push({ + id: world.deployRows.length + 1, + deployableId: world.deployRows.length + 1, + uuid: 'A2-static-z', + buildId: BUILD_ID, + githubRepositoryId: 100, + branchName: 'main', + active: true, + runUUID: null, + sha: null, + dockerImage: null, + status: DeployStatus.PENDING, + statusMessage: '', + desiredGeneration: 0, + observedGeneration: 0, + deployableName: 'A2', + }); + await world.merge('B'); + await jest.advanceTimersByTimeAsync(30 * MIN); + for (let tick = 0; tick < 3; tick += 1) { + await world.service.enqueuePendingDeploymentReconciliations(); + await jest.advanceTimersByTimeAsync(5_000); + } + + expect(world.live.get('A')).toBe(`A@${aSha}`); + expect(world.row('A2')).toMatchObject({ desiredGeneration: 0, observedGeneration: 0, runUUID: null }); + expect(world.settled()).toBe(true); + expect(world.buildRow.observedGeneration).toBe(2); + }); + + test('a full redeploy accepted before a teardown never brings the torn-down environment back', async () => { + const world = createWorld(SPECS); + world.queue.dropNext = 1; + await world.redeployEnvironment(); + world.buildRow.runUUID = 'teardown-run'; + const build = await world.service.db.models.Build.query().findById(BUILD_ID); + await world.service.deleteBuild(build, { runUUID: 'teardown-run', reason: 'manual_destroy', rethrow: true }); + expect(world.buildRow.status).toBe(BuildStatus.TORN_DOWN); + + for (let tick = 0; tick < 3; tick += 1) { + await world.service.enqueuePendingDeploymentReconciliations(); + await jest.advanceTimersByTimeAsync(5 * MIN); + } + + expect(world.buildRow.status).toBe(BuildStatus.TORN_DOWN); + expect(world.importCounts.get('root')).toBeUndefined(); + expect(world.settled()).toBe(true); + expect(world.buildRow.observedGeneration).toBe(world.buildRow.desiredGeneration); + }); + + test('a lost promotion lease after the claim retries the job and resumes the rows it already owns', async () => { + const world = createWorld(SPECS); + world.applyFailures.push(new AuthorityLockLostError('deploy-promotion.2')); + const sha = await world.merge('B'); + await jest.advanceTimersByTimeAsync(15 * MIN); + + expect(world.live.get('B')).toBe(`B@${sha}`); + expect(world.row('B').observedGeneration).toBe(1); + expect(world.counters.builds.get('B')).toBe(1); + expect(world.counters.applies.get('B')).toBe(1); + expect(world.queue.finalFailures).toEqual([]); + expect(world.settled()).toBe(true); + }); + + test('a finished generation is never re-imported while an older service is still building', async () => { + const world = createWorld(SPECS); + await world.merge('A'); + await jest.advanceTimersByTimeAsync(5 * MIN); + await world.merge('B'); + await jest.advanceTimersByTimeAsync(6 * MIN); + expect(world.row('B').observedGeneration).toBe(2); + const importsAfterB = world.importCounts.get('200') ?? 0; + + const signalsBefore = world.queue.adds.filter(({ opts }) => opts.jobId === `reconcile-${BUILD_ID}-2`).length; + for (let tick = 0; tick < 12; tick += 1) { + await world.service.enqueuePendingDeploymentReconciliations(); + await jest.advanceTimersByTimeAsync(0.5 * MIN); + } + // The sweep signals only generations that rows still carry, so the finished one is never re-enqueued. + expect(world.queue.adds.filter(({ opts }) => opts.jobId === `reconcile-${BUILD_ID}-2`).length).toBe(signalsBefore); + + // A replayed delivery of the finished generation exits before touching configuration. + await world.service.processDeploymentReconciliationQueue({ + data: { buildId: BUILD_ID, generation: 2 }, + attemptsMade: 0, + opts: { attempts: 10 }, + }); + expect(logs.filter((line) => line.includes('signal ignored reason=already_observed'))).toHaveLength(1); + expect(world.importCounts.get('200')).toBe(importsAfterB); + expect(world.row('A').observedGeneration).toBe(0); + }); + + test('a service with a broken lifecycle.yaml fails alone during a config-repo push; the root config and every other service proceed', async () => { + const world = createWorld(SPECS); + world.serviceConfigFailures.C = 'C lifecycle.yaml is invalid'; + await world.pushConfigRepo(); + await jest.advanceTimersByTimeAsync(45 * MIN); + + expect(world.row('C').status).toBe(DeployStatus.ERROR); + expect(world.row('C').statusMessage).toContain('C lifecycle.yaml is invalid'); + expect(world.row('C').observedGeneration).toBe(1); + expect(world.counters.builds.get('C')).toBeUndefined(); + for (const name of ['A', 'B', 'D', 'E']) { + expect(world.row(name).status).toBe(DeployStatus.READY); + expect(world.row(name).observedGeneration).toBe(1); + } + expect(world.buildRow.status).toBe(BuildStatus.ERROR); + expect(world.buildRow.statusMessage).toContain('C'); + expect(world.settled()).toBe(true); + }); + + test('a config-repo push whose signal was lost is recovered by the sweep even after every row was taken by later merges', async () => { + const world = createWorld([{ name: 'A', repo: 100, buildMinutes: 3 }]); + world.servicesAddedByConfig.push({ name: 'N', repo: 500, buildMinutes: 3 }); + world.queue.dropNext = 1; + await world.pushConfigRepo(); + await jest.advanceTimersByTimeAsync(1 * MIN); + const aSha = await world.merge('A'); + await jest.advanceTimersByTimeAsync(10 * MIN); + expect(world.live.get('A')).toBe(`A@${aSha}`); + expect(world.deployRows.some((row) => row.deployableName === 'N')).toBe(false); + + for (let tick = 0; tick < 6; tick += 1) { + await world.service.enqueuePendingDeploymentReconciliations(); + await jest.advanceTimersByTimeAsync(2 * MIN); + } + + expect(world.deployRows.some((row) => row.deployableName === 'N')).toBe(true); + expect(world.row('N').observedGeneration).toBe(1); + expect(world.buildRow.acceptedRefs['source:900:static:all'].observedGen).toBe(1); + expect(world.settled()).toBe(true); + }); + + test('a service with a broken lifecycle.yaml blocks its dependents and is reported when it has no row yet', async () => { + const world = createWorld(SPECS); + world.serviceConfigFailures.N = 'N lifecycle.yaml is invalid'; + await world.pushConfigRepo(); + await jest.advanceTimersByTimeAsync(45 * MIN); + + expect(world.settled()).toBe(true); + expect(world.buildRow.status).toBe(BuildStatus.ERROR); + expect(world.buildRow.statusMessage).toContain('N'); + expect(world.buildRow.statusMessage).toContain('N lifecycle.yaml is invalid'); + }); + + test('a root config error is cleared by the next push whose root import succeeds, even a service-scoped one', async () => { + const world = createWorld(SPECS); + world.failImport.set(900, () => new ParsingError('root lifecycle.yaml is invalid')); + await world.redeployEnvironment('label-added'); + await jest.advanceTimersByTimeAsync(5 * MIN); + expect(world.buildRow.status).toBe(BuildStatus.CONFIG_ERROR); + expect(world.buildRow.statusMessage).toContain('root lifecycle.yaml is invalid'); + expect(world.settled()).toBe(true); + + world.failImport.delete(900); + const aSha = await world.merge('A'); + await jest.advanceTimersByTimeAsync(30 * MIN); + + expect(world.importCounts.get('100')).toBe(1); + expect(world.live.get('A')).toBe(`A@${aSha}`); + expect(world.row('A').status).toBe(DeployStatus.READY); + expect(world.settled()).toBe(true); + expect(world.buildRow.status).toBe(BuildStatus.DEPLOYED); + expect(world.buildRow.statusMessage).toBe(''); + expect(world.statusWrites).toContain('config_error:root lifecycle.yaml is invalid'); + }); + + test('a broken root config on a service push flags the environment, not the pushed service', async () => { + const world = createWorld(SPECS); + world.failImport.set(100, () => new ParsingError('root lifecycle.yaml is invalid')); + await world.merge('A'); + await jest.advanceTimersByTimeAsync(5 * MIN); + + expect(world.buildRow.status).toBe(BuildStatus.CONFIG_ERROR); + expect(world.buildRow.statusMessage).toContain('root lifecycle.yaml is invalid'); + expect(world.row('A').status).not.toBe(DeployStatus.ERROR); + expect(world.live.get('A')).toBe('A@A-v0'); + expect(world.settled()).toBe(true); + + world.failImport.delete(100); + const aSha = await world.merge('A'); + await jest.advanceTimersByTimeAsync(30 * MIN); + + expect(world.live.get('A')).toBe(`A@${aSha}`); + expect(world.buildRow.status).toBe(BuildStatus.DEPLOYED); + }); + + test('a root config error recorded while another service finishes is never overwritten by that finish', async () => { + const world = createWorld(SPECS); + world.buildRow.status = BuildStatus.BUILDING; + const configBuild = await sim.buildQuery().findById(BUILD_ID); + + let release!: () => void; + sim.fetchGraphGate = new Promise((resolve) => { + release = resolve; + }); + const derived = world.service.publishDerivedBuildStatus(BUILD_ID); + await flush(); + expect(sim.fetchGraphGate).toBeNull(); + + const rootError = world.service.recordRootConfigError( + configBuild, + new ParsingError('root lifecycle.yaml is invalid') + ); + await flush(); + expect(world.buildRow.status).toBe(BuildStatus.BUILDING); + + release(); + await derived; + await rootError; + expect(world.statusWrites).toEqual(['deployed:', 'config_error:root lifecycle.yaml is invalid']); + expect(world.buildRow.status).toBe(BuildStatus.CONFIG_ERROR); + + await world.service.publishDerivedBuildStatus(BUILD_ID); + expect(world.buildRow.status).toBe(BuildStatus.CONFIG_ERROR); + }); + + test('a watermark write that fails after rollout is repaired by the retry, which republishes the final status', async () => { + const world = createWorld(SPECS); + sim.failPatchOnce = (patch: any, table: string) => table === 'build' && 'observedGeneration' in patch; + + const bSha = await world.merge('B'); + await jest.advanceTimersByTimeAsync(10 * MIN); + + expect(world.live.get('B')).toBe(`B@${bSha}`); + expect(world.row('B').status).toBe(DeployStatus.READY); + expect(world.queue.attempts).toEqual([`reconcile-${BUILD_ID}-1#0`, `reconcile-${BUILD_ID}-1#1`]); + expect(world.queue.finalFailures).toEqual([]); + expect(world.buildRow.observedGeneration).toBe(1); + expect(world.buildRow.status).toBe(BuildStatus.DEPLOYED); + expect(world.buildRow.acceptedRefs['source:200:main'].observedGen).toBe(1); + + const signalsBefore = world.queue.adds.length; + for (let tick = 0; tick < 3; tick += 1) { + await world.service.enqueuePendingDeploymentReconciliations(); + await jest.advanceTimersByTimeAsync(5_000); + } + expect(world.queue.adds.length).toBe(signalsBefore); + expect(logs.filter((line) => line.includes('signal ignored reason=already_observed'))).toHaveLength(0); + }); + + test('a database error during image setup fails the rows it was building; the detached sibling build cannot land after the intent finished', async () => { + const world = createWorld(SPECS); + sim.failPatchOnce = (patch: any) => 'deployPipelineId' in patch; + + await world.merge('D'); + await jest.advanceTimersByTimeAsync(10 * MIN); + + expect(logs.some((line) => line.includes('Docker: build error'))).toBe(true); + expect(world.queue.finalFailures).toEqual([]); + expect(world.settled()).toBe(true); + expect(world.row('D').status).toBe(DeployStatus.BUILD_FAILED); + expect(world.row('E').status).toBe(DeployStatus.BUILD_FAILED); + expect(world.live.get('D')).toBe('D@D-v0'); + expect(world.live.get('E')).toBe('E@E-v0'); + expect(world.row('E').dockerImage).toBe('E@E-v0'); + expect(world.events.some((line) => line.includes('E: build done') && line.includes('result not published'))).toBe( + true + ); + expect(world.events.filter((line) => line.includes('APPLIED'))).toEqual([]); + expect(world.buildRow.status).toBe(BuildStatus.ERROR); + }); + + test('an ingress failure note is kept by a derived republish and cleared only by a clean ingress apply', async () => { + const world = createWorld(SPECS); + world.buildRow.status = BuildStatus.DEPLOYED; + world.buildRow.statusMessage = 'Ingress apply failed: kubectl apply exited 1'; + + await world.service.publishDerivedBuildStatus(BUILD_ID); + expect(world.buildRow.statusMessage).toBe('Ingress apply failed: kubectl apply exited 1'); + + await world.service.publishDerivedBuildStatus(BUILD_ID, { clearStatusMessage: true }); + expect(world.buildRow.status).toBe(BuildStatus.DEPLOYED); + expect(world.buildRow.statusMessage).toBe(''); + }); + + test('a retry after a successful rollout resumes the rows already live instead of rolling them out again', async () => { + const world = createWorld(SPECS); + world.service.ingressService.ingressManifestQueue.add = jest + .fn() + .mockRejectedValueOnce(new Error('redis unavailable')) + .mockResolvedValue(undefined); + + const bSha = await world.merge('B'); + await jest.advanceTimersByTimeAsync(20 * MIN); + + expect(world.live.get('B')).toBe(`B@${bSha}`); + expect(world.row('B').status).toBe(DeployStatus.READY); + expect(world.settled()).toBe(true); + expect(world.queue.attempts).toEqual([`reconcile-${BUILD_ID}-1#0`, `reconcile-${BUILD_ID}-1#1`]); + expect(world.counters.builds.get('B')).toBe(1); + expect(world.counters.applies.get('B')).toBe(1); + expect(world.events.filter((line) => line.includes('(re-rollout)'))).toEqual([]); + expect(world.service.ingressService.ingressManifestQueue.add).toHaveBeenCalledTimes(2); + expect(world.buildRow.status).toBe(BuildStatus.DEPLOYED); + }); + + test('a full redeploy accepted after a config push subsumes it: one root import and no second environment pass', async () => { + const world = createWorld(SPECS); + world.queue.dropNext = 1; + await world.pushConfigRepo(); + await world.redeployEnvironment('manual'); + await jest.advanceTimersByTimeAsync(30 * MIN); + + expect(world.settled()).toBe(true); + expect(world.importCounts.get('root')).toBe(1); + expect(world.buildRow.acceptedRefs['source:900:static:all'].observedGen).toBe(1); + expect(world.buildRow.acceptedRefs.all.observedGen).toBe(2); + expect(world.buildRow.observedGeneration).toBe(2); + + for (let tick = 0; tick < 3; tick += 1) { + await world.service.enqueuePendingDeploymentReconciliations(); + await jest.advanceTimersByTimeAsync(5_000); + } + expect(world.queue.adds.filter(({ opts }) => opts.jobId === `reconcile-${BUILD_ID}-1`)).toHaveLength(1); + expect(world.importCounts.get('root')).toBe(1); + }); + + test('a service whose definition disappeared fails alone; its sibling from the same repository still ships', async () => { + const world = createWorld(SPECS); + world.orphanRows.add('E'); + + const sha = await world.merge('D'); + await jest.advanceTimersByTimeAsync(10 * MIN); + + expect(world.live.get('D')).toBe(`D@${sha}`); + expect(world.row('E').status).toBe(DeployStatus.ERROR); + expect(world.row('E').statusMessage).toBe('Service definition missing.'); + expect(world.live.get('E')).toBe('E@E-v0'); + expect(world.settled()).toBe(true); + expect(world.queue.finalFailures).toEqual([]); + expect(world.buildRow.status).toBe(BuildStatus.ERROR); + expect(world.buildRow.statusMessage).toContain('Service definition missing.'); + }); + + test('a service that fails to import and has no row blocks its dependents and stays reported until an import clears it', async () => { + const world = createWorld(SPECS); + world.dependsOn.set('B', ['N']); + world.serviceConfigFailures.N = 'N lifecycle.yaml is invalid'; + await world.pushConfigRepo(); + await jest.advanceTimersByTimeAsync(45 * MIN); + + expect(world.settled()).toBe(true); + expect(world.row('B').status).toBe(DeployStatus.DEPLOY_FAILED); + expect(world.row('B').statusMessage).toBe('Not deployed: N failed.'); + expect(world.live.get('B')).toBe('B@B-v0'); + expect(world.buildRow.status).toBe(BuildStatus.ERROR); + expect(world.buildRow.statusMessage).toContain('Not imported: N (N lifecycle.yaml is invalid)'); + + const aSha = await world.merge('A'); + await jest.advanceTimersByTimeAsync(30 * MIN); + expect(world.live.get('A')).toBe(`A@${aSha}`); + expect(world.buildRow.status).toBe(BuildStatus.ERROR); + expect(world.buildRow.statusMessage).toContain('Not imported: N'); + + // A manual full redeploy re-evaluates N and re-reports it from the newest environment-wide import. + await world.redeployEnvironment('manual'); + await jest.advanceTimersByTimeAsync(45 * MIN); + expect(world.settled()).toBe(true); + expect(world.buildRow.status).toBe(BuildStatus.ERROR); + expect(world.buildRow.statusMessage).toContain('Not imported: N (N lifecycle.yaml is invalid)'); + + delete world.serviceConfigFailures.N; + await world.pushConfigRepo(); + await jest.advanceTimersByTimeAsync(45 * MIN); + expect(world.settled()).toBe(true); + expect(world.row('B').status).toBe(DeployStatus.READY); + expect(world.buildRow.status).toBe(BuildStatus.DEPLOYED); + expect(world.buildRow.statusMessage).toBe(''); + }); + + test("a full redeploy accepted after a config push imports at that push's revision, not a lagging branch head", async () => { + const world = createWorld(SPECS); + world.queue.dropNext = 1; + const zSha = await world.pushConfigRepo(); + await world.redeployEnvironment('manual'); + await jest.advanceTimersByTimeAsync(30 * MIN); + + expect(world.settled()).toBe(true); + expect(world.importRefs.filter((entry) => entry.filter == null)).toEqual([ + { filter: null, repo: null, ref: null, pins: { '900:static': zSha } }, + ]); + expect(world.buildRow.acceptedRefs['source:900:static:all'].observedGen).toBe(1); + }); + + test("a service push honours the root config floor but pays for no other repository's lookup", async () => { + const world = createWorld(SPECS); + const zSha = await world.pushConfigRepo(); + await jest.advanceTimersByTimeAsync(30 * MIN); + await world.merge('B'); + await world.merge('A'); + await jest.advanceTimersByTimeAsync(30 * MIN); + + expect(world.settled()).toBe(true); + expect(world.importRefs.find((entry) => entry.filter === 100)?.pins).toEqual({ '900:static': zSha }); + }); + + test("a full pass that replaces an earlier full-pass push still deploys that push's revision", async () => { + const world = createWorld(SPECS); + world.queue.dropNext = 1; + const a1 = await world.mergeFull('A'); + const b1 = await world.mergeFull('B'); + await jest.advanceTimersByTimeAsync(45 * MIN); + + expect(world.settled()).toBe(true); + expect(world.buildRow.acceptedRefs['source:100:main:all'].observedGen).toBe(1); + expect(world.live.get('A')).toBe(`A@${a1}`); + expect(world.live.get('B')).toBe(`B@${b1}`); + }); + + test('a service that moves to another repository is still deployed by the push that claimed it', async () => { + const world = createWorld(SPECS); + let releaseImport!: () => void; + sim.importGate = new Promise((resolve) => { + releaseImport = resolve; + }); + await world.redeployEnvironment('full'); + await jest.advanceTimersByTimeAsync(1_000); + world.serviceMoves.push({ name: 'A', repo: 101, sha: 'A-moved' }); + await world.merge('A'); + await jest.advanceTimersByTimeAsync(1_000); + releaseImport(); + await jest.advanceTimersByTimeAsync(45 * MIN); + + expect(world.settled()).toBe(true); + expect(world.row('A').githubRepositoryId).toBe(101); + expect(world.row('A').desiredGeneration).toBe(2); + expect(world.live.get('A')).toBe('A@A-moved'); + expect(world.buildRow.status).toBe(BuildStatus.DEPLOYED); + }); + + test("a config error on a push that lands mid-rollout keeps its row failed despite the superseded run's late writes", async () => { + const world = createWorld(SPECS); + await world.merge('B'); + await jest.advanceTimersByTimeAsync(3.2 * MIN); + expect(world.row('B').status).toBe(DeployStatus.BUILT); + + world.serviceConfigFailures.B = 'B lifecycle.yaml is invalid'; + await world.merge('B'); + await jest.advanceTimersByTimeAsync(5 * MIN); + + expect(world.settled()).toBe(true); + expect(world.row('B').status).toBe(DeployStatus.ERROR); + expect(world.row('B').statusMessage).toBe('B lifecycle.yaml is invalid'); + expect(world.buildRow.status).toBe(BuildStatus.ERROR); + expect(world.buildRow.statusMessage).toContain('B lifecycle.yaml is invalid'); + }); + + test('a config-repo push deploys every accepted service SHA instead of a laggy branch head', async () => { + const world = createWorld(SPECS); + const accepted = await world.merge('A'); + await world.pushConfigRepo(); + await jest.advanceTimersByTimeAsync(45 * MIN); + + expect(world.live.get('A')).toBe(`A@${accepted}`); + expect(world.row('A').desiredGeneration).toBe(2); + expect(world.row('A').observedGeneration).toBe(2); + expect(world.settled()).toBe(true); + }); + test('a dependent ships without waiting for, or failing on, a prerequisite another run is fixing', async () => { + const world = createWorld([ + { name: 'A', repo: 100, buildMinutes: 1 }, + { name: 'B', repo: 200, buildMinutes: 3 }, + ]); + world.dependsOn.set('B', ['A']); + world.failBuild.add('A'); + await world.merge('A'); + await jest.advanceTimersByTimeAsync(5 * MIN); + expect(world.row('A').status).toBe(DeployStatus.BUILD_FAILED); + expect(world.settled()).toBe(true); + + world.failBuild.delete('A'); + const bSha = await world.merge('B'); + await jest.advanceTimersByTimeAsync(1 * MIN); + // The fix for A lands while B builds; its job is held before the claim, so A's row still shows its old failure. + let releaseA!: () => void; + sim.importGateRepo = 100; + sim.importGate = new Promise((resolve) => { + releaseA = resolve; + }); + const aSha = await world.merge('A'); + await jest.advanceTimersByTimeAsync(6 * MIN); + + expect(world.row('A').runUUID).toBeNull(); + expect(world.row('A').status).toBe(DeployStatus.QUEUED); + expect(world.live.get('B')).toBe(`B@${bSha}`); + expect(world.row('B').status).toBe(DeployStatus.READY); + + releaseA(); + await jest.advanceTimersByTimeAsync(30 * MIN); + + expect(world.live.get('A')).toBe(`A@${aSha}`); + expect(world.counters.applies.get('B')).toBe(1); + expect(world.queue.finalFailures).toEqual([]); + expect(world.settled()).toBe(true); + expect(world.buildRow.status).toBe(BuildStatus.DEPLOYED); + }); + + test('dependents never hold a worker slot for a prerequisite in another run, even with every slot busy', async () => { + const world = createWorld( + [ + { name: 'A', repo: 100, buildMinutes: 1 }, + { name: 'B1', repo: 200, buildMinutes: 3 }, + { name: 'B2', repo: 300, buildMinutes: 3 }, + ], + { concurrency: 2 } + ); + world.dependsOn.set('B1', ['A']); + world.dependsOn.set('B2', ['A']); + const b1 = await world.merge('B1'); + const b2 = await world.merge('B2'); + await jest.advanceTimersByTimeAsync(1 * MIN); + // A is accepted while both slots build its dependents; its job runs once one of them finishes. + const a = await world.merge('A'); + await jest.advanceTimersByTimeAsync(20 * MIN); + + expect(world.live.get('A')).toBe(`A@${a}`); + expect(world.live.get('B1')).toBe(`B1@${b1}`); + expect(world.live.get('B2')).toBe(`B2@${b2}`); + expect(world.liveAt.get('B1')!).toBeLessThan(world.liveAt.get('A')!); + expect(world.liveAt.get('B2')!).toBeLessThan(world.liveAt.get('A')!); + expect(world.counters.applies.get('B1')).toBe(1); + expect(world.counters.applies.get('B2')).toBe(1); + expect(world.queue.finalFailures).toEqual([]); + expect(world.settled()).toBe(true); + expect(world.buildRow.status).toBe(BuildStatus.DEPLOYED); + }); + + test('a dependent ships even when its prerequisite never gets to run', async () => { + const world = createWorld([ + { name: 'A', repo: 100, buildMinutes: 1 }, + { name: 'B', repo: 200, buildMinutes: 1 }, + ]); + world.dependsOn.set('B', ['A']); + // The signal for A is lost and nothing sweeps, so A stays accepted but never claimed. + world.queue.dropNext = 1; + await world.merge('A'); + const bSha = await world.merge('B'); + await jest.advanceTimersByTimeAsync(10 * MIN); + + expect(world.live.get('B')).toBe(`B@${bSha}`); + expect(world.row('B').status).toBe(DeployStatus.READY); + expect(world.row('B').observedGeneration).toBe(2); + expect(world.row('A').observedGeneration).toBe(0); + expect(world.queue.finalFailures).toEqual([]); + }); +}); diff --git a/src/server/services/build.ts b/src/server/services/build.ts index c58d19d8..cf386fb7 100644 --- a/src/server/services/build.ts +++ b/src/server/services/build.ts @@ -68,21 +68,32 @@ import ApiAccessConfigService from './apiAccessConfig'; import { getBranchName, getDeployType, getRepositoryName, type Service } from 'server/models/yaml/YamlService'; import type { LifecycleConfig } from 'server/models/yaml/Config'; import * as YamlService from 'server/models/yaml'; -import { getEnvironmentPhaseFromState, isActiveServiceReady } from 'server/lib/environments/readiness'; +import { getEnvironmentPhaseFromState, isActiveServiceReady, isDeployFailure } from 'server/lib/environments/readiness'; import { acceptDeploymentIntent, - dirtyDeploymentIntents, + deploymentIntentDeployFilter, + deploymentIntentSelectsAllDeploys, + deployStampForGeneration, + recordIntentConfigFailures, + type AcceptedDeploymentIntent, + type AcceptedDeploymentRefs, type DeploymentIntent, type DeploymentReconciliationJobData, - type DirtyDeploymentIntent, } from 'server/lib/deploymentReconciliation/mailbox'; +import { + environmentBlockReason, + isDeployAuthorityCurrent, + loadEnvironmentAuthority, +} from 'server/lib/deploymentReconciliation/authority'; const tracer = Tracer.getInstance(); tracer.initialize('build-service'); const TEARDOWN_RETRY_GRACE_MS = 15 * 60 * 1000; -const BUILD_DEPLOYMENT_LOCK_TTL_MS = 15 * 60 * 1000; +// Short with frequent renewal, so a worker that dies mid-run frees its locks within a minute. +const BUILD_DEPLOYMENT_LOCK_TTL_MS = 60 * 1000; const DEPLOY_SECRET_MUTATION_LOCK_TTL_MS = 2 * 60 * 1000; const DEPLOYMENT_RECONCILIATION_SWEEP_MS = 5_000; +const BUILD_STATUS_LOCK_TTL_MS = 30_000; const HOUR_MS = 60 * 60 * 1000; const PR_AUTHORITY_REVALIDATED_DELETE_REASONS = new Set([ 'pull_request_closed', @@ -128,12 +139,17 @@ interface DeploymentScopeOptions { sourceGithubRepositoryId?: number | null; /** Exact mailbox generation owned by this pass. */ expectedGeneration?: number; + /** Latest accepted push SHA per `:`; a full pass must not read a laggy head below it. */ + acceptedSourcePins?: AcceptedSourcePins; } +type AcceptedSourcePins = Record; + interface DeploymentReconciliationClaim { generation: number; token: string; - dirty: DirtyDeploymentIntent[]; + intent: AcceptedDeploymentIntent; + acceptedSourcePins: AcceptedSourcePins; } interface DeploymentReconciliationScope { @@ -152,13 +168,21 @@ interface DeploymentScopePreparation { sourceGithubRepositoryId?: number | null; sourceRef?: string | null; sourceBranch?: string | null; + acceptedSourcePins?: AcceptedSourcePins; + /** Services the import could not read and that have no row; their dependents are blocked. */ + unresolvedServices?: string[]; } interface DeploymentScopeOutcome { - status: BuildStatus; - error?: unknown; + deployed: number; + failed: number; } +type IntentConfiguration = + | { kind: 'deferred' } + | { kind: 'observed' } + | { kind: 'ready'; preparation: DeploymentScopePreparation; owned: number }; + interface DeploymentReconciliationRequest { buildId: number; githubRepositoryId?: number | null; @@ -510,7 +534,8 @@ export default class BuildService extends BaseService { 'createdByUserId', 'createdByTokenId', 'createdByGithubLogin', - 'runUUID' + 'runUUID', + 'acceptedRefs' ) .withGraphFetched('[baseBuild, pullRequest, deploys.[deployable, repository]]') .modifyGraph('pullRequest', (b) => { @@ -545,6 +570,8 @@ export default class BuildService extends BaseService { 'deployPipelineId', 'githubRepositoryId', 'runUUID', + 'desiredGeneration', + 'observedGeneration', 'publicUrl', 'dockerImage', 'buildLogs', @@ -2314,6 +2341,7 @@ export default class BuildService extends BaseService { sourceGithubRepositoryId?: number | null; runUUID?: string; expectedGeneration?: number; + acceptedSourcePins?: AcceptedSourcePins; } = {} ) { const buildSource = getBuildSource(build); @@ -2325,11 +2353,14 @@ export default class BuildService extends BaseService { buildSource.githubRepositoryId != null && Number(sourceGithubRepositoryId) === Number(buildSource.githubRepositoryId) && options.sourceBranch === buildSource.branchName; - const rootSourceRef = sourceRefTargetsRoot ? options.sourceRef : null; + const rootSourceRef = sourceRefTargetsRoot + ? options.sourceRef + : options.acceptedSourcePins?.[`${buildSource.githubRepositoryId}:${buildSource.branchName}`] ?? null; + let reconciliationResult: DeployableReconciliationResult | undefined; // Write the deployables here for now and not going to use them yet. try { const buildId = build?.id; - const reconciliationResult = await this.db.services.Deployable.upsertDeployables( + reconciliationResult = await this.db.services.Deployable.upsertDeployables( buildId, build.uuid, build.pullRequest, @@ -2338,7 +2369,8 @@ export default class BuildService extends BaseService { filterGithubRepositoryId, options.sourceRef, options.sourceBranch, - sourceGithubRepositoryId + sourceGithubRepositoryId, + options.acceptedSourcePins ); if (options.skipDeletedServiceReconciliation) { @@ -2374,6 +2406,7 @@ export default class BuildService extends BaseService { await this.db.services.Webhook.upsertWebhooksWithYaml(build, build.pullRequest, rootSourceRef).catch((error) => { getLogger().warn({ error }, 'Config: webhook import warning'); }); + return reconciliationResult; } private async reconcileDeletedDeployables( @@ -2509,8 +2542,8 @@ export default class BuildService extends BaseService { ...(runUUID ? { nativeMutationGate: async (action: () => Promise) => { - const result = await this.withCurrentBuildPromotionLock( - buildId, + const result = await this.withCurrentDeployPromotionLock( + deploy.id, () => this.isDeploymentRunCurrent(buildId, runUUID, expectedGeneration), action ); @@ -2645,20 +2678,7 @@ export default class BuildService extends BaseService { * queued or in-flight PR job from recreating resources while deletion waits. */ private deploymentBlockReason(build: Build | null | undefined): string | null { - if (!build) return 'build_missing'; - if (build.deletedAt != null) return 'build_deleted'; - - if (build.pullRequest || build.pullRequestId != null) { - if (build.status === BuildStatus.TEARING_DOWN) return 'tearing_down'; - if (!build.pullRequest) return 'pull_request_missing'; - if (build.pullRequest.status !== PullRequestStatus.OPEN) return 'pull_request_closed'; - if (!isDeployEnabled(build)) return 'deploy_disabled'; - // Re-adding the deploy label to an open PR may reclaim the row after teardown completes. - return null; - } - - if (build.status === BuildStatus.TEARING_DOWN || build.status === BuildStatus.TORN_DOWN) return 'torn_down'; - return build.deployEnabled === true ? null : 'deploy_disabled'; + return environmentBlockReason(build); } /** PR setup is allowed without a deploy label, but not while teardown is active or the PR is closed. */ @@ -2708,10 +2728,6 @@ export default class BuildService extends BaseService { .where({ id: build.id }) .whereNull('deletedAt'); - if (expectedGeneration != null) { - claim = claim.where('desiredGeneration', expectedGeneration); - } - // API-created environments keep their kill switch on the Build row, so make // that half of the claim atomic. PR authority is re-read immediately below. if (!build.pullRequest && build.pullRequestId == null) { @@ -2756,7 +2772,6 @@ export default class BuildService extends BaseService { getLogger().info('Deploy: preparation skipped reason=ownership_lost'); return null; } - build.runUUID = runUUID; const deploys = await this.db.services.Deploy.findOrCreateDeploys( build.environment!, @@ -2764,7 +2779,8 @@ export default class BuildService extends BaseService { githubRepositoryId ?? undefined, sourceRef, options.sourceBranch, - options.sourceGithubRepositoryId ?? githubRepositoryId + options.sourceGithubRepositoryId ?? githubRepositoryId, + options.acceptedSourcePins ); build.$setRelated('deploys', deploys); await build.$fetchGraph('pullRequest'); @@ -2774,31 +2790,11 @@ export default class BuildService extends BaseService { getLogger().info('Deploy: preparation skipped after config reason=ownership_lost'); return null; } - - let deployClaim = this.db.models.Deploy.query().patch({ runUUID }).where({ buildId: build.id }); - if (githubRepositoryId != null) { - deployClaim = deployClaim.where('githubRepositoryId', githubRepositoryId); - if (options.sourceBranch) deployClaim = deployClaim.where('branchName', options.sourceBranch); - } - await deployClaim; - for (const deploy of deploys ?? []) { - if ( - (githubRepositoryId == null || deploy.githubRepositoryId === githubRepositoryId) && - (githubRepositoryId == null || !options.sourceBranch || deploy.branchName === options.sourceBranch) - ) { - deploy.runUUID = runUUID; - } - } - - await this.markConfigurationsAsBuilt(build, runUUID, githubRepositoryId, options.sourceBranch); - await this.updateStatusAndComment(build, BuildStatus.BUILDING, runUUID, true, true, null, expectedGeneration); await build.pullRequest?.$fetchGraph('repository'); try { const dependencyGraph = await generateGraph(build, 'TB'); - let graphPatch = this.db.models.Build.query().patch({ dependencyGraph }).where({ id: build.id, runUUID }); - if (expectedGeneration != null) graphPatch = graphPatch.where('desiredGeneration', expectedGeneration); - await graphPatch; + await this.db.models.Build.query().patch({ dependencyGraph }).where({ id: build.id }).whereNull('deletedAt'); } catch (error) { getLogger().warn({ error }, 'Graph: generation failed'); } @@ -2810,19 +2806,28 @@ export default class BuildService extends BaseService { sourceGithubRepositoryId: options.sourceGithubRepositoryId ?? githubRepositoryId, sourceRef, sourceBranch: options.sourceBranch, + acceptedSourcePins: options.acceptedSourcePins, }; } private async executeDeploymentScope( preparation: DeploymentScopePreparation, - expectedGeneration?: number + expectedGeneration: number ): Promise { - const { build, runUUID, githubRepositoryId, sourceGithubRepositoryId, sourceRef, sourceBranch } = preparation; + const { + build, + runUUID, + githubRepositoryId, + sourceGithubRepositoryId, + sourceRef, + sourceBranch, + acceptedSourcePins, + } = preparation; try { if (!(await this.isDeploymentRunCurrent(build.id, runUUID, expectedGeneration))) return null; - const results = await Promise.all([ + await Promise.all([ this.buildImages( build, runUUID, @@ -2830,28 +2835,29 @@ export default class BuildService extends BaseService { sourceRef, sourceBranch, expectedGeneration, - sourceGithubRepositoryId + sourceGithubRepositoryId, + acceptedSourcePins + ), + this.deployCLIServices( + build, + runUUID, + githubRepositoryId, + sourceRef, + sourceBranch, + sourceGithubRepositoryId, + expectedGeneration, + acceptedSourcePins ), - this.deployCLIServices(build, runUUID, githubRepositoryId, sourceRef, sourceBranch, sourceGithubRepositoryId), ]); - getLogger().debug(`Build results: buildImages=${results[0]} deployCLIServices=${results[1]}`); - // A/B may finish the expensive operation they already entered, but may not - // enter native promotion after C has become authoritative. + // A row that failed its image phase is recorded on the row; its siblings still roll out. if (!(await this.isDeploymentRunCurrent(build.id, runUUID, expectedGeneration))) { getLogger().info('Deploy: rollout skipped reason=ownership_lost'); return null; } + await this.publishDerivedBuildStatus(build.id); - if (!_.every(results)) { - getLogger({ buildId: build.id, githubRepositoryId, sourceBranch }).warn('Build: errored skipping=rollout'); - return { status: BuildStatus.ERROR }; - } - - await this.updateStatusAndComment(build, BuildStatus.DEPLOYING, runUUID, true, true, null, expectedGeneration); - if (!(await this.isDeploymentRunCurrent(build.id, runUUID, expectedGeneration))) return null; - - const applySuccess = await this.generateAndApplyManifests({ + await this.generateAndApplyManifests({ build, runUUID, expectedGeneration, @@ -2859,10 +2865,9 @@ export default class BuildService extends BaseService { sourceBranch, namespace: build.namespace, enqueueIngress: false, + unresolvedServices: preparation.unresolvedServices, }); - if (!(await this.isDeploymentRunCurrent(build.id, runUUID, expectedGeneration))) return null; - - return { status: applySuccess ? BuildStatus.DEPLOYED : BuildStatus.ERROR }; + return this.summarizeIntentDeploys(build.id, runUUID, expectedGeneration); } catch (error) { if (error instanceof DeploymentSupersededError) { getLogger().info('Build: deployment phase stopped reason=superseded'); @@ -2870,34 +2875,65 @@ export default class BuildService extends BaseService { } if (error instanceof AuthorityLockLostError) throw error; getLogger().error({ error }, 'Build: deploy failed'); - if (!(await this.isDeploymentRunCurrent(build.id, runUUID, expectedGeneration).catch(() => false))) { + if (!(await this.isDeploymentRunCurrent(build.id, runUUID, expectedGeneration))) { getLogger().info('Build: failure ignored reason=ownership_lost'); return null; } - - return { status: BuildStatus.ERROR, error }; + await this.recordFailureOnOwnedDeploys(build.id, runUUID, expectedGeneration, error, 'Deployment failed.'); + return this.summarizeIntentDeploys(build.id, runUUID, expectedGeneration); } } + private async summarizeIntentDeploys( + buildId: number, + runUUID: string, + generation: number + ): Promise { + const rows = await this.db.models.Deploy.query() + .select('status') + .where({ buildId, runUUID, desiredGeneration: generation }); + const failed = rows.filter((row) => isDeployFailure(row.status)).length; + return { deployed: rows.length - failed, failed }; + } + + /** + * With a generation this is intent authority: the run still owns a row at that + * generation, or its accepted intent is still the newest for its key before + * rows are claimed. Without a generation it is the legacy environment token. + */ private async isDeploymentRunCurrent( buildId: number, runUUID: string, expectedGeneration?: number ): Promise { - const current = await this.db.models.Build.query() - .findById(buildId) - .select('id', 'runUUID', 'status', 'deployEnabled', 'deletedAt', 'pullRequestId', 'desiredGeneration'); - if (current?.pullRequestId != null) { - await current.$fetchGraph('pullRequest'); - } - return Boolean( - current && - current.runUUID === runUUID && - (expectedGeneration == null || Number(current.desiredGeneration) === expectedGeneration) && - this.deploymentBlockReason(current) == null + const current = await loadEnvironmentAuthority(this.db.models, buildId); + if (!current || this.deploymentBlockReason(current) != null) return false; + if (expectedGeneration == null) return current.runUUID === runUUID; + const owned = await this.db.models.Deploy.query() + .findOne({ buildId, runUUID, desiredGeneration: expectedGeneration }) + .select('id'); + if (owned) return true; + return this.isIntentAccepted(buildId, runUUID, expectedGeneration); + } + + private async isIntentAccepted(buildId: number, runUUID: string, generation: number): Promise { + const build = await this.db.models.Build.query().findById(buildId).select('id', 'acceptedRefs'); + const refs = build?.acceptedRefs; + if (!refs || typeof refs !== 'object' || Array.isArray(refs)) return false; + return Object.values(refs).some( + (intent) => intent?.gen === generation && intent?.requestId === runUUID && intent?.observedGen !== generation ); } + /** Admission to the config lock ignores block reasons so a blocked environment can still observe or defer its work. */ + private async isIntentPending(buildId: number, runUUID: string, generation: number): Promise { + const owned = await this.db.models.Deploy.query() + .findOne({ buildId, runUUID, desiredGeneration: generation }) + .select('id'); + if (owned) return true; + return this.isIntentAccepted(buildId, runUUID, generation); + } + async withBuildDeploymentLock(buildId: number, action: () => Promise): Promise { if (!this.redlock?.lock) return action(); @@ -2942,6 +2978,72 @@ export default class BuildService extends BaseService { }); } + async withCurrentDeployPromotionLock( + deployId: number, + isCurrent: () => Promise, + action: () => Promise + ): Promise> { + return withAuthorityLock({ + redlock: this.redlock, + resource: `deploy-promotion.${deployId}`, + ttlMs: BUILD_DEPLOYMENT_LOCK_TTL_MS, + isCurrent, + action, + onWait: (error) => getLogger({ error, deployId }).warn('Deploy promotion: waiting for admitted native mutation'), + }); + } + + /** Teardown cancels the work accepted up to `cutoff`; the Build row lock orders it against every accept. */ + private async observeAcceptedWorkForTeardown(buildId: number, cutoff: number): Promise { + await this.db.models.Build.transact(async (trx) => { + const build = await this.db.models.Build.query(trx) + .select('id', 'desiredGeneration', 'acceptedRefs') + .findById(buildId) + .forUpdate(); + if (!build) return; + const refs: AcceptedDeploymentRefs = + build.acceptedRefs && typeof build.acceptedRefs === 'object' && !Array.isArray(build.acceptedRefs) + ? build.acceptedRefs + : {}; + const acceptedRefs = Object.fromEntries( + Object.entries(refs).map(([key, intent]) => [ + key, + Number.isSafeInteger(intent?.gen) && intent.gen <= cutoff ? { ...intent, observedGen: intent.gen } : intent, + ]) + ); + const rows = await this.db.models.Deploy.query(trx) + .select('id', 'desiredGeneration', 'observedGeneration') + .where({ buildId }); + for (const row of rows) { + const desired = Number(row.desiredGeneration); + if (desired <= Number(row.observedGeneration) || desired > cutoff) continue; + await this.db.models.Deploy.query(trx) + .patch({ observedGeneration: row.desiredGeneration }) + .where({ id: row.id }); + } + await this.db.models.Build.query(trx) + .findById(buildId) + .patch({ + acceptedRefs, + observedGeneration: Math.min(cutoff, Number(build.desiredGeneration)), + } as Partial); + }); + } + + /** Lets every admitted per-service mutation finish before cleanup touches the namespace. */ + private async barrierDeployPromotionLocks(buildId: number): Promise { + const deploys = await this.db.models.Deploy.query().select('id').where({ buildId }); + for (const deploy of deploys) { + await withAuthorityLock({ + redlock: this.redlock, + resource: `deploy-promotion.${deploy.id}`, + ttlMs: BUILD_DEPLOYMENT_LOCK_TTL_MS, + isCurrent: async () => true, + action: async () => undefined, + }); + } + } + private async withCurrentBuildDeploymentLock( buildId: number, isCurrent: () => Promise, @@ -3148,7 +3250,11 @@ export default class BuildService extends BaseService { // Revoke every in-flight Deploy writer before cleanup starts. This one // bulk fence prevents detached readiness/provider waits from restoring // READY or failure state after teardown records TORN_DOWN. + // Work accepted from here on, such as a re-added deploy label, belongs to a later deploy and survives the teardown. + const cutoff = Number(build.desiredGeneration ?? 0); await this.db.models.Deploy.query().where({ buildId: build.id }).patch({ runUUID: teardownRunUUID }); + await this.barrierDeployPromotionLocks(build.id); + await this.observeAcceptedWorkForTeardown(build.id, cutoff); } // A failing cleanup step must never block namespace deletion below; retries pick up the rest. const cleanupResults = await Promise.allSettled([ @@ -3222,8 +3328,91 @@ export default class BuildService extends BaseService { runUUID: string, updateMissionControl: boolean, updateStatus: boolean, - error: Error | null = null, - expectedGeneration?: number + error: Error | null = null + ) { + return this.publishBuildStatus(build, status, { runUUID, updateMissionControl, updateStatus, error }); + } + + /** Environment status is derived from its services; only teardown and a root config error override it. */ + async publishDerivedBuildStatus( + buildId: number, + options: { clearConfigError?: boolean; clearStatusMessage?: boolean } = {} + ): Promise { + // Concurrent jobs derive from different row snapshots; serializing the read-modify-write keeps the last write current. + await withAuthorityLock({ + redlock: this.redlock, + resource: `build-status.${buildId}`, + ttlMs: BUILD_STATUS_LOCK_TTL_MS, + isCurrent: async () => true, + action: () => this.publishDerivedBuildStatusUnlocked(buildId, options), + }); + } + + private async publishDerivedBuildStatusUnlocked( + buildId: number, + options: { clearConfigError?: boolean; clearStatusMessage?: boolean } = {} + ): Promise { + const build = await this.db.models.Build.query().findById(buildId).whereNull('deletedAt'); + if (!build) return; + if (build.status === BuildStatus.TEARING_DOWN || build.status === BuildStatus.TORN_DOWN) return; + if (build.status === BuildStatus.CONFIG_ERROR && !options.clearConfigError) return; + await build.$fetchGraph('[deploys.[deployable], pullRequest.[repository]]'); + + const active = (build.deploys ?? []).filter((deploy) => deploy.active !== false); + const pending = active.filter((deploy) => Number(deploy.desiredGeneration) > Number(deploy.observedGeneration)); + const failedRows = active.some( + (deploy) => isDeployFailure(deploy.status) && deploy.status !== DeployStatus.TORN_DOWN + ); + const unresolvedNote = Object.entries(unresolvedServiceFailures(build.acceptedRefs, active)) + .map(([name, reason]) => `${name} (${reason})`) + .join('; '); + let status: BuildStatus; + if (pending.length > 0) { + const rollingOut = pending.some((deploy) => + [DeployStatus.BUILT, DeployStatus.DEPLOYING].includes(deploy.status as DeployStatus) + ); + status = rollingOut ? BuildStatus.DEPLOYING : BuildStatus.BUILDING; + } else if (failedRows || unresolvedNote) { + status = BuildStatus.ERROR; + } else { + status = isDeployEnabled(build) ? BuildStatus.DEPLOYED : BuildStatus.BUILT; + } + + let error: Error | null = null; + if (status === BuildStatus.ERROR && unresolvedNote) { + const rowMessage = failedRows ? this.resolveBuildStatusMessage(status, active, null) : ''; + error = new Error([rowMessage, `Not imported: ${unresolvedNote}`].filter(Boolean).join(' | ')); + } + const statusMessage = this.resolveBuildStatusMessage(status, active, error); + if (build.status === status && (build.statusMessage ?? '') === statusMessage) return; + // An ingress failure note on an otherwise healthy environment is cleared only by the ingress job itself. + if (build.status === status && statusMessage === '' && !options.clearStatusMessage) return; + await this.publishBuildStatus(build, status, { updateMissionControl: true, updateStatus: true, error }); + } + + private async recordRootConfigError(build: Build, error: unknown): Promise { + const statusError = + error instanceof Error + ? error + : new Error(statusMessageFromError(error, 'Lifecycle configuration failed validation.')); + await withAuthorityLock({ + redlock: this.redlock, + resource: `build-status.${build.id}`, + ttlMs: BUILD_STATUS_LOCK_TTL_MS, + isCurrent: async () => true, + action: () => + this.publishBuildStatus(build, BuildStatus.CONFIG_ERROR, { + updateMissionControl: true, + updateStatus: true, + error: statusError, + }), + }); + } + + private async publishBuildStatus( + build: Build, + status: BuildStatus, + options: { runUUID?: string; updateMissionControl: boolean; updateStatus: boolean; error: Error | null } ) { return withLogContext({ buildUuid: build.uuid }, async () => { let published = false; @@ -3236,12 +3425,13 @@ export default class BuildService extends BaseService { const isSandboxBuild = build.kind === BuildKind.SANDBOX; const repository = pullRequest?.repository; - const statusMessage = this.resolveBuildStatusMessage(status, deploys || [], error); + const statusMessage = this.resolveBuildStatusMessage(status, deploys || [], options.error); let patch = this.db.models.Build.query() .patch({ status, statusMessage }) - .where({ id: build.id, runUUID }) + .where({ id: build.id }) .whereNull('deletedAt'); - if (expectedGeneration != null) patch = patch.where('desiredGeneration', expectedGeneration); + if (options.runUUID) patch = patch.where('runUUID', options.runUUID); + else patch = patch.whereNotIn('status', [BuildStatus.TEARING_DOWN, BuildStatus.TORN_DOWN]); if ((await patch) !== 1) return; published = true; @@ -3253,9 +3443,9 @@ export default class BuildService extends BaseService { deploys, pullRequest, repository, - updateMissionControl, - updateStatus, - error + options.updateMissionControl, + options.updateStatus, + options.error ).catch((e) => { getLogger().error({ error: e }, 'ActivityStream: update failed'); }); @@ -3305,7 +3495,7 @@ export default class BuildService extends BaseService { } const statusError = error instanceof Error ? error : new Error(statusMessageFromError(error, fallbackMessage)); - await this.updateStatusAndComment(build, status, activeRunUUID, true, true, statusError, expectedGeneration); + await this.updateStatusAndComment(build, status, activeRunUUID, true, true, statusError); } private resolveBuildStatusMessage(status: BuildStatus, deploys: Deploy[], error: Error | null): string { @@ -3375,13 +3565,59 @@ export default class BuildService extends BaseService { } } + /** + * A fenced run processes exactly the rows it claimed, whichever repository + * they belong to now; the intent's repository filter was applied when the + * rows were stamped. Rows already READY belong to an earlier attempt of the + * same run and are left alone; a row whose service definition is gone fails + * on its own instead of failing its siblings. + */ + private async selectRunDeploys( + buildId: number, + runUUID: string | undefined, + githubRepositoryId: number | null | undefined, + sourceBranch: string | null | undefined, + expectedGeneration?: number, + excludeFailed = false + ): Promise { + const fenced = expectedGeneration != null; + let query = Deploy.query().where({ + buildId, + ...(runUUID ? { runUUID } : {}), + ...(!fenced && githubRepositoryId ? { githubRepositoryId } : {}), + ...(!fenced && githubRepositoryId && sourceBranch ? { branchName: sourceBranch } : {}), + }); + if (fenced) query = query.where('desiredGeneration', expectedGeneration); + const skipped: DeployStatus[] = [ + ...(expectedGeneration != null ? [DeployStatus.READY, DeployStatus.DEPLOYED] : []), + ...(excludeFailed ? [DeployStatus.ERROR, DeployStatus.BUILD_FAILED, DeployStatus.DEPLOY_FAILED] : []), + ]; + if (skipped.length > 0) query = query.whereNotIn('status', skipped); + const deploys = await query.withGraphFetched({ deployable: true }); + if (expectedGeneration == null || !runUUID) return deploys; + const orphaned = deploys.filter((deploy) => deploy.active && !deploy.deployable); + for (const deploy of orphaned) { + getLogger().error(`Deploy: service definition missing uuid=${deploy.uuid}`); + await this.recordPhaseFailure( + deploy, + runUUID, + DeployStatus.ERROR, + 'Service definition missing.', + expectedGeneration + ); + } + return deploys.filter((deploy) => !orphaned.includes(deploy)); + } + async deployCLIServices( build: Build, runUUID: string, githubRepositoryId: number | null = null, sourceRef?: string | null, sourceBranch?: string | null, - sourceGithubRepositoryId: number | null | undefined = githubRepositoryId + sourceGithubRepositoryId: number | null | undefined = githubRepositoryId, + expectedGeneration?: number, + acceptedSourcePins?: AcceptedSourcePins ): Promise { await build?.$fetchGraph({ deploys: { @@ -3392,14 +3628,14 @@ export default class BuildService extends BaseService { if (!buildId) { getLogger().error('Build: id missing for=deployCLIServices'); } - const deploys = await Deploy.query() - .where({ - buildId, - runUUID, - ...(githubRepositoryId ? { githubRepositoryId } : {}), - ...(githubRepositoryId && sourceBranch ? { branchName: sourceBranch } : {}), - }) - .withGraphFetched({ deployable: true }); + const deploys = await this.selectRunDeploys( + buildId, + runUUID, + githubRepositoryId, + sourceBranch, + expectedGeneration, + expectedGeneration != null + ); try { return _.every( await Promise.all( @@ -3418,8 +3654,19 @@ export default class BuildService extends BaseService { runUUID, sourceRef, sourceGithubRepositoryId, - sourceBranch + sourceBranch, + acceptedSourcePins, + expectedGeneration ); + if (result === false) { + await this.recordPhaseFailure( + deploy, + runUUID, + DeployStatus.ERROR, + 'CLI deploy failed.', + expectedGeneration + ); + } return result; } catch (err) { getLogger().error({ error: err }, `CLI: deploy failed uuid=${deploy?.uuid}`); @@ -3434,6 +3681,7 @@ export default class BuildService extends BaseService { ); } catch (error) { getLogger().error({ error }, 'CLI: build failed'); + await this.recordPhaseFailures(deploys, runUUID, DeployStatus.ERROR, 'CLI deploy failed.', expectedGeneration); return false; } } @@ -3450,23 +3698,22 @@ export default class BuildService extends BaseService { sourceRef?: string | null, sourceBranch?: string | null, expectedGeneration?: number, - sourceGithubRepositoryId: number | null | undefined = githubRepositoryId + sourceGithubRepositoryId: number | null | undefined = githubRepositoryId, + acceptedSourcePins?: AcceptedSourcePins ): Promise { const buildId = build?.id; if (!buildId) { getLogger().error('Build: id missing for=buildImages'); } - const deploys = await Deploy.query() - .where({ - buildId, - runUUID, - ...(githubRepositoryId ? { githubRepositoryId } : {}), - ...(githubRepositoryId && sourceBranch ? { branchName: sourceBranch } : {}), - }) - .withGraphFetched({ - deployable: true, - }); + const deploys = await this.selectRunDeploys( + buildId, + runUUID, + githubRepositoryId, + sourceBranch, + expectedGeneration, + expectedGeneration != null + ); try { const deploysToBuild = deploys.filter((deploy): deploy is Deploy & { deployable: Deployable } => { @@ -3520,9 +3767,19 @@ export default class BuildService extends BaseService { sourceGithubRepositoryId, sourceBranch, expectedGeneration, - nativeServiceAccount + nativeServiceAccount, + acceptedSourcePins ); getLogger().debug(`buildImage completed: deployUuid=${deploy.uuid} result=${result}`); + if (result === false) { + await this.recordPhaseFailure( + deploy, + runUUID, + DeployStatus.BUILD_FAILED, + 'Image build failed.', + expectedGeneration + ); + } return result; }) ); @@ -3532,6 +3789,13 @@ export default class BuildService extends BaseService { } catch (error) { if (error instanceof AuthorityLockLostError || error instanceof DeploymentSupersededError) throw error; getLogger().error({ error }, 'Docker: build error'); + await this.recordPhaseFailures( + deploys, + runUUID, + DeployStatus.BUILD_FAILED, + 'Image build setup failed.', + expectedGeneration + ); return false; } } @@ -3548,6 +3812,7 @@ export default class BuildService extends BaseService { sourceBranch, namespace, enqueueIngress = true, + unresolvedServices = [], }: { build: Build; runUUID?: string; @@ -3556,6 +3821,7 @@ export default class BuildService extends BaseService { sourceBranch?: string | null; namespace: string; enqueueIngress?: boolean; + unresolvedServices?: string[]; }): Promise { try { const buildId = build?.id; @@ -3591,16 +3857,21 @@ export default class BuildService extends BaseService { }); } - const allDeploys = await Deploy.query() - .where({ - buildId, - ...(runUUID ? { runUUID } : {}), - ...(githubRepositoryId ? { githubRepositoryId } : {}), - ...(githubRepositoryId && sourceBranch ? { branchName: sourceBranch } : {}), - }) - .withGraphFetched({ - deployable: true, - }); + const allDeploys = await this.selectRunDeploys( + buildId, + runUUID, + githubRepositoryId, + sourceBranch, + expectedGeneration + ); + // A service whose image or import failed stays in the plan so its dependents are blocked, but it is never applied. + const failedServices = new Set([ + ...unresolvedServices, + ...allDeploys + .filter((deploy) => expectedGeneration != null && isDeployFailure(deploy.status)) + .map((deploy) => deploy.deployable?.name) + .filter((name): name is string => Boolean(name)), + ]); const activeDeploys = allDeploys.filter((deploy): deploy is Deploy & { deployable: Deployable } => { if (!deploy.active) return false; @@ -3612,6 +3883,7 @@ export default class BuildService extends BaseService { // Generate manifests for GitHub/Docker/CLI deploys for (const deploy of activeDeploys) { + if (failedServices.has(deploy.deployable.name)) continue; const deployType = deploy.deployable.type; if (deployType === DeployTypes.GITHUB || deployType === DeployTypes.DOCKER || CLIDeployTypes.has(deployType)) { // Generate individual manifest for this deploy @@ -3638,19 +3910,35 @@ export default class BuildService extends BaseService { const managedDeploys = activeDeploys.filter( (d) => d.deployable.type !== DeployTypes.CODEFRESH && d.deployable.type !== DeployTypes.CONFIGURATION ); + if (managedDeploys.length === 0 && expectedGeneration != null) { + getLogger().info('Deploy: nothing to roll out reason=already_deployed'); + } const isCurrent = runUUID ? () => this.isDeploymentRunCurrent(build.id, runUUID, expectedGeneration) : undefined; + const deployIsCurrent = (deploy: Deploy) => + expectedGeneration != null + ? () => isDeployAuthorityCurrent(this.db.models, deploy, runUUID!, expectedGeneration) + : isCurrent!; const deploymentManager = new DeploymentManager(managedDeploys, { isCurrent, + expectedGeneration, + failedServices: Array.from(failedServices), nativeMutationGate: runUUID - ? (action) => this.withCurrentBuildPromotionLock(build.id, isCurrent!, action) + ? (deploy, action) => + expectedGeneration != null + ? this.withCurrentDeployPromotionLock(deploy.id, deployIsCurrent(deploy), action) + : this.withCurrentBuildPromotionLock(build.id, isCurrent!, action) : undefined, nativeSecretMutationGate: runUUID - ? (deploy, action) => this.withCurrentDeploySecretMutationLock(deploy.id, isCurrent!, action) + ? (deploy, action) => this.withCurrentDeploySecretMutationLock(deploy.id, deployIsCurrent(deploy), action) : undefined, }); - await deploymentManager.deploy(); + const { failed } = await deploymentManager.deploy(); + // Fenced runs carry failures on their rows; legacy whole-environment callers still expect a throw. + if (failed.length > 0 && expectedGeneration == null) { + throw new Error(`Deployment failed for ${failed.map((deploy) => deploy.deployable.name).join(', ')}`); + } } if (runUUID && !(await this.isDeploymentRunCurrent(build.id, runUUID, expectedGeneration))) { @@ -3672,15 +3960,17 @@ export default class BuildService extends BaseService { if (githubTypeDeploys.length > 0) { const legacyManifest = k8s.generateManifest({ build, - deploys: githubTypeDeploys, + deploys: githubTypeDeploys.filter((d) => !failedServices.has(d.deployable.name)), uuid: build.uuid, namespace, serviceAccountName, }); if (legacyManifest && legacyManifest.replace(/---/g, '').trim().length > 0) { - let manifestPatch = this.db.models.Build.query().patch({ manifest: legacyManifest }).where({ id: build.id }); - if (runUUID) manifestPatch = manifestPatch.where('runUUID', runUUID); - if (expectedGeneration != null) manifestPatch = manifestPatch.where('desiredGeneration', expectedGeneration); + let manifestPatch = this.db.models.Build.query() + .patch({ manifest: legacyManifest }) + .where({ id: build.id }) + .whereNull('deletedAt'); + if (runUUID && expectedGeneration == null) manifestPatch = manifestPatch.where('runUUID', runUUID); await manifestPatch; } } @@ -3732,6 +4022,7 @@ export default class BuildService extends BaseService { await build?.$fetchGraph('deploys'); const deploys = (build.deploys ?? []).filter( (deploy) => + !isDeployFailure(deploy.status) && (!githubRepositoryId || deploy.githubRepositoryId === githubRepositoryId) && (!githubRepositoryId || !sourceBranch || deploy.branchName === sourceBranch) ); @@ -3759,7 +4050,7 @@ export default class BuildService extends BaseService { }, }); - deploymentReconciliationQueue = this.queueManager.registerQueue(QUEUE_NAMES.DEPLOYMENT_RECONCILIATION, { + deploymentReconciliationQueue = this.queueManager.registerQueue(QUEUE_NAMES.SERVICE_RECONCILIATION, { connection: redisClient.getConnection(), defaultJobOptions: { attempts: 10, @@ -3991,73 +4282,84 @@ export default class BuildService extends BaseService { }); }; - private async claimDeploymentReconciliation( + private async claimDeploymentIntent( buildId: number, generation: number ): Promise { const build = await this.db.models.Build.query().findById(buildId).whereNull('deletedAt'); if (!build) return null; - - const desiredGeneration = Number(build.desiredGeneration); - const observedGeneration = Number(build.observedGeneration); - if (!Number.isSafeInteger(desiredGeneration) || !Number.isSafeInteger(observedGeneration)) { - throw new Error(`Build ${buildId} has invalid deployment generations`); + const refs: AcceptedDeploymentRefs = + build.acceptedRefs && typeof build.acceptedRefs === 'object' && !Array.isArray(build.acceptedRefs) + ? build.acceptedRefs + : {}; + const intent = Object.values(refs).find((candidate) => candidate?.gen === generation); + if (!intent?.requestId) { + getLogger({ buildId, generation }).info('Build reconciliation: signal ignored reason=intent_replaced'); + return null; } - // One signal owns exactly one generation. B must never wake up later and - // read/execute C's mailbox contents. - if (desiredGeneration !== generation || generation <= observedGeneration) return null; - - const dirty = dirtyDeploymentIntents(build.acceptedRefs, observedGeneration).filter( - ({ intent }) => intent.gen <= generation - ); - const latest = dirty.find(({ intent }) => intent.gen === generation)?.intent; - if (!latest?.requestId) throw new Error(`Build ${buildId} generation ${generation} has no request token`); + if (intent.observedGen === generation) { + getLogger({ buildId, generation }).info('Build reconciliation: signal ignored reason=already_observed'); + return null; + } + // Intents accepted before rows carried generations never stamped them; adopting here is idempotent. + await this.adoptDeploysForIntent(buildId, intent, generation); + return { generation, token: intent.requestId, intent, acceptedSourcePins: this.acceptedSourcePins(refs) }; + } - return { generation, token: latest.requestId, dirty }; + /** + * Every accepted push SHA is a floor for its repository and branch, whatever + * pass deploys the row. A full-environment push (config repo, or PR failure + * recovery) is a floor too, since a later full pass replaces its entry. + */ + private acceptedSourcePins(refs: AcceptedDeploymentRefs): AcceptedSourcePins { + const newest = new Map(); + for (const intent of Object.values(refs)) { + if (intent?.type !== 'source' || !intent.sha) continue; + const key = `${intent.githubRepositoryId}:${intent.branch}`; + const current = newest.get(key); + if (!current || Number(intent.gen) > current.gen) newest.set(key, { gen: Number(intent.gen), sha: intent.sha }); + } + return Object.fromEntries(Array.from(newest.entries()).map(([key, { sha }]) => [key, sha])); } - private deploymentReconciliationScopes(dirty: DirtyDeploymentIntent[]): DeploymentReconciliationScope[] { - const newestBroadIndex = _.findLastIndex( - dirty, - ({ intent }) => intent.type === 'all' || (intent.type === 'source' && intent.target === 'all') - ); + private async adoptDeploysForIntent(buildId: number, intent: DeploymentIntent, generation: number): Promise { + await this.db.models.Deploy.query() + .patch(deployStampForGeneration(generation)) + .where({ buildId, active: true, ...deploymentIntentDeployFilter(intent) }) + .where('desiredGeneration', '<', generation) + .where('observedGeneration', '<', generation); + } - // A full pass subsumes older unpinned requests, but it must not erase a - // delivered source SHA. Run unpinned/live-head work first and immutable - // source floors last so a temporarily stale provider read cannot roll C - // back to B. A source-targeted full pass still precedes selective pins. - const retained = dirty.filter( - ({ intent }, index) => newestBroadIndex < 0 || index >= newestBroadIndex || intent.type === 'source' - ); - const ordered = _.orderBy( - retained, - [ - ({ intent }) => { - if (intent.type === 'all') return 0; - if (intent.type === 'repository') return 1; - return intent.target === 'all' ? 2 : 3; - }, - ({ intent }) => intent.gen, - ], - ['asc', 'desc'] - ); + /** + * Taking a row from another run resets it: whatever it showed came from an + * older generation. Rows this run already owns keep what they reached, so a + * retry resumes rather than repeats them. The count covers every pending row + * the run owns afterwards. + */ + private async claimIntentDeploys(buildId: number, generation: number, token: string): Promise { + const pending = () => + this.db.models.Deploy.query() + .where({ buildId, desiredGeneration: generation }) + .where('observedGeneration', '<', generation); + const reset = { runUUID: token, status: DeployStatus.QUEUED, statusMessage: null }; + await pending().patch(reset).whereNull('runUUID'); + await pending().patch(reset).whereNot('runUUID', token); + const owned = await pending().where('runUUID', token).select('id'); + return owned.length; + } - return ordered.map(({ intent }) => { - if (intent.type === 'all') return { githubRepositoryId: null }; - if (intent.type === 'source') { - return { - githubRepositoryId: intent.target === 'all' ? null : intent.githubRepositoryId, - sourceGithubRepositoryId: intent.githubRepositoryId, - sourceRef: intent.sha, - sourceBeforeRef: intent.beforeSha, - sourceBranch: intent.branch, - }; - } + private deploymentReconciliationScope(intent: AcceptedDeploymentIntent): DeploymentReconciliationScope { + if (intent.type === 'all') return { githubRepositoryId: null }; + if (intent.type === 'source') { return { - githubRepositoryId: intent.githubRepositoryId, - skipDeletedServiceReconciliation: true, + githubRepositoryId: intent.target === 'all' ? null : intent.githubRepositoryId, + sourceGithubRepositoryId: intent.githubRepositoryId, + sourceRef: intent.sha, + sourceBeforeRef: intent.beforeSha, + sourceBranch: intent.branch, }; - }); + } + return { githubRepositoryId: intent.githubRepositoryId, skipDeletedServiceReconciliation: true }; } /** @@ -4070,47 +4372,117 @@ export default class BuildService extends BaseService { const repository: Repository | undefined = await this.db.models.Repository.query() .findOne({ githubRepositoryId: scope.sourceGithubRepositoryId }) .whereNull('deletedAt'); - const repositoryFullName = repository?.fullName; + return this.advanceAcceptedSourceRef( + repository?.fullName, + scope.sourceBranch, + scope.sourceRef, + scope.sourceBeforeRef + ); + } + + /** An accepted push is a floor. The live head replaces it only when GitHub proves the head descends from it. */ + private async advanceAcceptedSourceRef( + repositoryFullName: string | undefined, + branch: string, + sha: string, + beforeSha?: string + ): Promise { const [owner, name] = repositoryFullName?.split('/') ?? []; - if (!repositoryFullName || !owner || !name) return scope.sourceRef; + if (!repositoryFullName || !owner || !name) return sha; let currentRef: string; try { - currentRef = await github.getSHAForBranch(scope.sourceBranch, owner, name); + currentRef = await github.getSHAForBranch(branch, owner, name); } catch (error) { - getLogger({ error, repositoryFullName, branch: scope.sourceBranch }).warn( + getLogger({ error, repositoryFullName, branch }).warn( 'Build reconciliation: branch head lookup failed; using accepted source' ); - return scope.sourceRef; + return sha; } - if (!currentRef || currentRef === scope.sourceRef || currentRef === scope.sourceBeforeRef) return scope.sourceRef; + if (!currentRef || currentRef === sha || currentRef === beforeSha) return sha; try { - const comparison = await github.compareCommits({ - fullName: repositoryFullName, - base: scope.sourceRef, - head: currentRef, - }); + const comparison = await github.compareCommits({ fullName: repositoryFullName, base: sha, head: currentRef }); if (comparison === 'ahead') return currentRef; } catch (error) { - getLogger({ error, repositoryFullName, branch: scope.sourceBranch }).warn( + getLogger({ error, repositoryFullName, branch }).warn( 'Build reconciliation: commit comparison failed; using accepted source' ); } - return scope.sourceRef; + return sha; } - private async markDeploymentReconciliationObserved( - buildId: number, - generation: number, - token?: string - ): Promise { - let update = this.db.models.Build.query() + /** Pins are resolved once per full pass so YAML, image and rollout all see one revision per service. */ + private async resolveAcceptedSourcePins(pins: AcceptedSourcePins): Promise { + const resolved: AcceptedSourcePins = {}; + for (const [key, sha] of Object.entries(pins)) { + const separator = key.indexOf(':'); + const repositoryId = Number(key.slice(0, separator)); + const branch = key.slice(separator + 1); + const repository: Repository | undefined = await this.db.models.Repository.query() + .findOne({ githubRepositoryId: repositoryId }) + .whereNull('deletedAt'); + resolved[key] = await this.advanceAcceptedSourceRef(repository?.fullName, branch, sha); + } + return resolved; + } + + /** Rows this run still owns at this generation are caught up, whatever their outcome. */ + private async observeIntentDeploys(buildId: number, generation: number, token?: string): Promise { + let update = this.db.models.Deploy.query() .patch({ observedGeneration: generation }) - .where({ id: buildId, desiredGeneration: generation }) - .whereNull('deletedAt'); + .where({ buildId, desiredGeneration: generation }) + .where('observedGeneration', '<', generation); if (token) update = update.where('runUUID', token); - return (await update) === 1; + return update; + } + + /** + * The Build watermark is the newest generation every active service has caught up with. It is + * recomputed under the Build row lock in the same write that marks an intent finished, so + * concurrent finishes cannot leave it behind. + */ + private async settleBuildObservedGeneration(buildId: number, finishedGeneration?: number): Promise { + await this.db.models.Build.transact(async (trx) => { + const build = await this.db.models.Build.query(trx) + .select('id', 'desiredGeneration', 'acceptedRefs') + .findById(buildId) + .forUpdate(); + if (!build) return; + let acceptedRefs: AcceptedDeploymentRefs = + build.acceptedRefs && typeof build.acceptedRefs === 'object' && !Array.isArray(build.acceptedRefs) + ? build.acceptedRefs + : {}; + const finished = + finishedGeneration != null + ? Object.entries(acceptedRefs).find(([, intent]) => intent?.gen === finishedGeneration) + : undefined; + if (finished) acceptedRefs = { ...acceptedRefs, [finished[0]]: { ...finished[1], observedGen: finished[1].gen } }; + const pending = await this.db.models.Deploy.query(trx) + .select('desiredGeneration') + .where({ buildId, active: true }) + .whereRaw('?? > ??', ['desiredGeneration', 'observedGeneration']); + // An accepted intent that never finished is pending work even when no row carries its generation, + // which is the shape of an environment-wide intent whose rows were all taken by later intents. + const minPending = [ + ...pending.map((row) => Number(row.desiredGeneration)), + ...unobservedAcceptedGenerations(acceptedRefs), + ].reduce((min, generation) => Math.min(min, generation), Infinity); + const observedGeneration = Number.isFinite(minPending) ? minPending - 1 : Number(build.desiredGeneration); + await this.db.models.Build.query(trx) + .findById(buildId) + .patch({ acceptedRefs, observedGeneration } as Partial); + }); + } + + /** The completion mark is written last: a retry after any earlier failure republishes instead of exiting early. */ + private async finishIntent( + buildId: number, + generation: number, + options: { clearConfigError?: boolean } = {} + ): Promise { + await this.publishDerivedBuildStatus(buildId, options); + await this.settleBuildObservedGeneration(buildId, generation); } private async signalPendingDeploymentReconciliation(buildId: number, generation: number): Promise { @@ -4134,7 +4506,7 @@ export default class BuildService extends BaseService { async enqueuePendingDeploymentReconciliations(): Promise { const pendingAfter = (id: number) => this.db.models.Build.query() - .select('id', 'desiredGeneration') + .select('id', 'desiredGeneration', 'acceptedRefs') .whereRaw('?? > ??', ['desiredGeneration', 'observedGeneration']) .whereNull('deletedAt') .where('id', '>', id) @@ -4148,12 +4520,57 @@ export default class BuildService extends BaseService { } if (builds.length > 0) this.deploymentReconciliationSweepCursor = Number(builds[builds.length - 1].id); + const pendingRows = await this.db.models.Deploy.query() + .select('buildId', 'desiredGeneration') + .where('active', true) + .whereRaw('?? > ??', ['desiredGeneration', 'observedGeneration']); + const pendingByBuild = new Map>(); + for (const row of pendingRows) { + const buildId = Number(row.buildId); + const generations = pendingByBuild.get(buildId) ?? new Set(); + generations.add(Number(row.desiredGeneration)); + pendingByBuild.set(buildId, generations); + } + + const signals = new Map>(); + const staleWatermarks: number[] = []; + for (const build of builds) { + const buildId = Number(build.id); + // Rows are the durable record of pending work, and an accepted entry that never finished still is work. + const generations = new Set([ + ...(pendingByBuild.get(buildId) ?? []), + ...unobservedAcceptedGenerations(build.acceptedRefs), + ]); + if (generations.size === 0) staleWatermarks.push(buildId); + else signals.set(buildId, generations); + } + for (const buildId of staleWatermarks) { + await this.settleBuildObservedGeneration(buildId).catch((error) => + getLogger({ error, buildId }).warn('Build reconciliation: watermark repair failed') + ); + } + const candidateIds = Array.from(pendingByBuild.keys()).filter((id) => !signals.has(id)); + const liveIds = new Set( + candidateIds.length > 0 + ? (await this.db.models.Build.query().select('id').whereNull('deletedAt').whereIn('id', candidateIds)).map( + (build) => Number(build.id) + ) + : [] + ); + for (const [buildId, generations] of pendingByBuild) { + if (!signals.has(buildId) && liveIds.has(buildId)) signals.set(buildId, generations); + } + await Promise.all( - builds.map((build) => - this.signalPendingDeploymentReconciliation(build.id, Number(build.desiredGeneration)).catch((error) => { - getLogger({ error, buildId: build.id }).warn('Build reconciliation: sweep signal failed'); - }) - ) + Array.from(signals.entries()).map(async ([buildId, generations]) => { + for (const generation of generations) { + try { + await this.signalPendingDeploymentReconciliation(buildId, generation); + } catch (error) { + getLogger({ error, buildId }).warn('Build reconciliation: sweep signal failed'); + } + } + }) ); } @@ -4175,50 +4592,132 @@ export default class BuildService extends BaseService { return Number(job.attemptsMade ?? 0) + 1 >= attempts; } - /** - * A failure can happen before the normal run claim finishes. On the final - * BullMQ attempt, reacquire the short mutation lock and establish the same - * generation token before publishing one terminal failure. - */ + /** On the final queue attempt, rows this run owns that never settled are recorded failed so nothing stays pending. */ private async recordFinalDeploymentReconciliationFailure( buildId: number, claim: DeploymentReconciliationClaim, - build: Build | null, - runUUID: string | null, error: unknown ): Promise { - let failureBuild = build; - let failureRunUUID = runUUID; + await this.claimIntentDeploys(buildId, claim.generation, claim.token); + await this.recordFailureOnOwnedDeploys( + buildId, + claim.token, + claim.generation, + error, + 'Build queue processing failed.' + ); + const observed = await this.observeIntentDeploys(buildId, claim.generation, claim.token); + await this.finishIntent(buildId, claim.generation); + return observed > 0; + } - if (!failureBuild || !failureRunUUID) { - const terminalClaim = await this.withCurrentBuildDeploymentLock( - buildId, - async () => Boolean(await this.claimDeploymentReconciliation(buildId, claim.generation)), - async () => { - const currentClaim = await this.claimDeploymentReconciliation(buildId, claim.generation); - if (!currentClaim || currentClaim.token !== claim.token) return null; - - const currentBuild = await this.loadBuildDeploymentAuthority(buildId); - if (!currentBuild) return null; - const claimedRunUUID = await this.claimDeploymentRun(currentBuild, claim.token, claim.generation); - return claimedRunUUID ? { build: currentBuild, runUUID: claimedRunUUID } : null; - } - ); - if (!terminalClaim.admitted || !terminalClaim.value) return false; - failureBuild = terminalClaim.value.build; - failureRunUUID = terminalClaim.value.runUUID; + private async recordFailureOnOwnedDeploys( + buildId: number, + token: string, + generation: number, + error: unknown, + fallbackMessage: string + ): Promise { + const statusMessage = compactStatusMessage(statusMessageFromError(error, fallbackMessage)); + await this.db.models.Deploy.query() + .patch({ status: DeployStatus.ERROR, statusMessage }) + .where({ buildId, runUUID: token, desiredGeneration: generation }) + .where('observedGeneration', '<', generation) + .whereNotIn('status', [ + DeployStatus.READY, + DeployStatus.DEPLOYED, + DeployStatus.ERROR, + DeployStatus.BUILD_FAILED, + DeployStatus.DEPLOY_FAILED, + DeployStatus.TORN_DOWN, + ]); + } + + private async recordPhaseFailure( + deploy: Deploy, + runUUID: string, + status: DeployStatus, + fallbackMessage: string, + expectedGeneration?: number + ): Promise { + await this.db.models.Deploy.query() + .patch({ status, statusMessage: fallbackMessage }) + .where({ + id: deploy.id, + runUUID, + ...(expectedGeneration != null ? { desiredGeneration: expectedGeneration } : {}), + }) + .whereNotIn('status', [ + DeployStatus.ERROR, + DeployStatus.BUILD_FAILED, + DeployStatus.DEPLOY_FAILED, + DeployStatus.TORN_DOWN, + ]); + } + + /** Rows still in flight when a whole phase fails must not roll out as if they had succeeded. */ + private async recordPhaseFailures( + deploys: Array>, + runUUID: string, + status: DeployStatus, + fallbackMessage: string, + expectedGeneration?: number + ): Promise { + for (const deploy of deploys) { + await this.db.models.Deploy.query() + .patch({ status, statusMessage: fallbackMessage }) + .where({ + id: deploy.id, + runUUID, + ...(expectedGeneration != null ? { desiredGeneration: expectedGeneration } : {}), + }) + .whereNotIn('status', [ + DeployStatus.READY, + DeployStatus.DEPLOYED, + DeployStatus.BUILT, + DeployStatus.ERROR, + DeployStatus.BUILD_FAILED, + DeployStatus.DEPLOY_FAILED, + DeployStatus.TORN_DOWN, + ]); + } + } + + /** A service whose own lifecycle.yaml is broken fails alone; the root config and every other service proceed. */ + private async recordServiceConfigFailures( + buildId: number, + generation: number, + token: string, + failures: Record + ): Promise { + const rows = await this.db.models.Deploy.query() + .where({ buildId, desiredGeneration: generation }) + .where('observedGeneration', '<', generation) + .withGraphFetched('deployable'); + const matched = new Set(); + for (const row of rows) { + const message = row.deployable?.name ? failures[row.deployable.name] : undefined; + if (!message) continue; + matched.add(row.deployable.name); + // The row keeps this run's token so rollout planning still sees it as a failed prerequisite. + await this.db.models.Deploy.query() + .patch({ + runUUID: token, + status: DeployStatus.ERROR, + statusMessage: compactStatusMessage(message), + observedGeneration: generation, + }) + .where({ id: row.id, desiredGeneration: generation }); } + return Object.keys(failures).filter((name) => !matched.has(name)); + } - if (!(await this.isDeploymentRunCurrent(buildId, failureRunUUID, claim.generation))) return false; - await this.recordBuildFailure( - failureBuild, - BuildStatus.ERROR, - failureRunUUID, - error, - 'Build queue processing failed.', - claim.generation - ); - return this.markDeploymentReconciliationObserved(buildId, claim.generation, failureRunUUID); + /** The failing intent takes its rows, so a superseded run's token-fenced writes cannot land after they are observed. */ + private async takeIntentDeploys(buildId: number, generation: number, token: string): Promise { + await this.db.models.Deploy.query() + .patch({ runUUID: token }) + .where({ buildId, desiredGeneration: generation }) + .where('observedGeneration', '<', generation); } processDeploymentReconciliationQueue = async (job: DeploymentReconciliationQueueJob) => { @@ -4229,175 +4728,169 @@ export default class BuildService extends BaseService { } await this.tryWithDeploymentGenerationLock(buildId, Number(generation), async () => { - const claim = await this.claimDeploymentReconciliation(buildId, Number(generation)); + const claim = await this.claimDeploymentIntent(buildId, Number(generation)); if (!claim) return; + const { token, intent } = claim; + const envWide = deploymentIntentSelectsAllDeploys(intent); + const scope = this.deploymentReconciliationScope(intent); + const logContext = { buildId, generation: claim.generation }; + const state: { build: Build | null } = { build: null }; - let build: Build | null = null; - let activeBuild: Build | null = null; - let activeRunUUID: string | null = null; try { - const initialClaim = await this.withCurrentBuildDeploymentLock( + const configuration = await this.withCurrentBuildDeploymentLock( buildId, - async () => Boolean(await this.claimDeploymentReconciliation(buildId, claim.generation)), - async () => { - const currentClaim = await this.claimDeploymentReconciliation(buildId, claim.generation); - if (!currentClaim) return; - build = await this.loadBuildDeploymentAuthority(buildId); - if (!build) return; + () => this.isIntentPending(buildId, token, claim.generation), + async (): Promise => { + const build = await this.loadBuildDeploymentAuthority(buildId); + if (!build) return null; if (build.uuid) updateLogContext({ buildUuid: build.uuid }); + state.build = build; const blocked = this.deploymentBlockReason(build); + if (blocked === 'tearing_down') { + getLogger(logContext).info('Build reconciliation: deferred reason=tearing_down'); + return { kind: 'deferred' }; + } if (blocked) { - if (blocked === 'tearing_down') { - getLogger({ buildId, generation: claim.generation }).info( - 'Build reconciliation: deferred reason=tearing_down' - ); - return; + getLogger(logContext).info(`Build reconciliation: observed without execution reason=${blocked}`); + await this.observeIntentDeploys(buildId, claim.generation); + return { kind: 'observed' }; + } + if (!envWide) { + const pendingRow = await this.db.models.Deploy.query() + .findOne({ buildId, desiredGeneration: claim.generation }) + .where('observedGeneration', '<', claim.generation) + .select('id'); + if (!pendingRow) { + getLogger(logContext).info('Build reconciliation: nothing pending reason=rows_settled_or_taken'); + return { kind: 'observed' }; } - getLogger({ buildId, generation: claim.generation }).info( - `Build reconciliation: observed without execution reason=${blocked}` - ); - await this.markDeploymentReconciliationObserved(buildId, claim.generation); - return; } - activeRunUUID = await this.claimDeploymentRun(build, claim.token, claim.generation); - } - ); + if (envWide && !(await this.claimDeploymentRun(build, token, claim.generation))) return null; - if (!initialClaim.admitted || !build || !activeRunUUID) return; - const reconciliationBuild = build; - activeBuild = reconciliationBuild; - const terminalOutcomes: Array<{ status: BuildStatus; error?: unknown }> = []; - const scopes = this.deploymentReconciliationScopes(claim.dirty); - getLogger({ stage: LogStage.BUILD_STARTING, buildId, generation: claim.generation }).info( - `Build reconciliation: started scopes=${scopes.length}` - ); - - for (const scope of scopes) { - const sourceRef = await this.resolveCurrentSourceRef(scope); - let preparation: DeploymentScopePreparation | null = null; - const configuration = await this.withCurrentBuildDeploymentLock( - buildId, - () => this.isDeploymentRunCurrent(buildId, claim.token, claim.generation), - async () => { - if (!(await this.isDeploymentRunCurrent(buildId, claim.token, claim.generation))) return; - - await this.importYamlConfigFile( - reconciliationBuild.environment!, - reconciliationBuild, + const sourceRef = await this.resolveCurrentSourceRef(scope); + const acceptedSourcePins = await this.resolveAcceptedSourcePins( + acceptedSourcePinsForIntent(claim.acceptedSourcePins, intent, rootSourcePinKey(build)) + ); + let imported: DeployableReconciliationResult | undefined; + let preparation: DeploymentScopePreparation | null; + try { + imported = await this.importYamlConfigFile( + build.environment!, + build, scope.githubRepositoryId ?? undefined, { skipDeletedServiceReconciliation: scope.skipDeletedServiceReconciliation, sourceRef, sourceBranch: scope.sourceBranch, sourceGithubRepositoryId: scope.sourceGithubRepositoryId, - runUUID: claim.token, + runUUID: token, expectedGeneration: claim.generation, + acceptedSourcePins, } ); - preparation = await this.prepareDeploymentScope( - reconciliationBuild, - scope.githubRepositoryId, - sourceRef, - { - runUUID: claim.token, - sourceBranch: scope.sourceBranch, - sourceGithubRepositoryId: scope.sourceGithubRepositoryId, - expectedGeneration: claim.generation, - } + // Every intent reads the root config, so any successful import proves it valid again. + if (build.status === BuildStatus.CONFIG_ERROR) { + await this.publishDerivedBuildStatus(buildId, { clearConfigError: true }); + } + preparation = await this.prepareDeploymentScope(build, scope.githubRepositoryId, sourceRef, { + runUUID: token, + sourceBranch: scope.sourceBranch, + sourceGithubRepositoryId: scope.sourceGithubRepositoryId, + expectedGeneration: claim.generation, + acceptedSourcePins, + }); + } catch (error) { + if (!(error instanceof ParsingError || error instanceof ValidationError)) throw error; + // A service's own broken YAML is reported per service, so an error that escapes the import is the root config's. + // Recorded while the configuration lock is held, so the status follows the order the config was read in. + await this.recordRootConfigError(build, error); + await this.takeIntentDeploys(buildId, claim.generation, token); + await this.observeIntentDeploys(buildId, claim.generation); + return { kind: 'observed' }; + } + if (!preparation) return null; + await this.adoptDeploysForIntent(buildId, intent, claim.generation); + let unresolvedServices: string[] = []; + if (imported?.configFailures && Object.keys(imported.configFailures).length > 0) { + unresolvedServices = await this.recordServiceConfigFailures( + buildId, + claim.generation, + token, + imported.configFailures ); } - ); - - if (!configuration.admitted || !preparation) return; - const outcome = await this.executeDeploymentScope(preparation, claim.generation); - if (!outcome) return; - terminalOutcomes.push(outcome); - if (!(await this.isDeploymentRunCurrent(buildId, claim.token, claim.generation))) return; + await recordIntentConfigFailures( + buildId, + claim.generation, + Object.fromEntries( + unresolvedServices.map((name) => [name, imported?.configFailures?.[name] ?? 'invalid lifecycle.yaml']) + ) + ); + preparation.unresolvedServices = unresolvedServices; + const owned = await this.claimIntentDeploys(buildId, claim.generation, token); + await this.markConfigurationsAsBuilt(build, token, scope.githubRepositoryId, scope.sourceBranch); + return { kind: 'ready', preparation, owned }; + } + ); + if (!configuration.admitted || !configuration.value) return; + const outcome = configuration.value; + if (outcome.kind === 'deferred') return; + if (outcome.kind === 'observed') { + await this.finishIntent(buildId, claim.generation); + return; } - - // Scope workers mutate one shared ingress snapshot. Publish it once, - // after every selected scope has settled, so an earlier scope cannot - // overwrite the final configuration for the same generation. - if (terminalOutcomes.some(({ status }) => status === BuildStatus.DEPLOYED)) { - await this.enqueueIngressManifest(buildId, claim.token, claim.generation); + if (outcome.owned === 0) { + getLogger(logContext).info('Build reconciliation: completed services=0'); + await this.finishIntent(buildId, claim.generation); + return; } - const failedOutcome = terminalOutcomes.find(({ status }) => status === BuildStatus.ERROR); - const finalStatus = failedOutcome - ? BuildStatus.ERROR - : isDeployEnabled(reconciliationBuild) - ? BuildStatus.DEPLOYED - : BuildStatus.BUILT; - await this.updateStatusAndComment( - reconciliationBuild, - finalStatus, - claim.token, - true, - true, - failedOutcome?.error instanceof Error ? failedOutcome.error : null, - claim.generation + getLogger({ stage: LogStage.BUILD_STARTING, ...logContext }).info( + `Build reconciliation: started services=${outcome.owned}` ); - - if (await this.markDeploymentReconciliationObserved(buildId, claim.generation, claim.token)) { - getLogger({ stage: LogStage.BUILD_COMPLETE, buildId, generation: claim.generation }).info( - 'Build reconciliation: completed' + await this.publishDerivedBuildStatus(buildId); + const summary = await this.executeDeploymentScope(outcome.preparation, claim.generation); + if (summary?.deployed) await this.enqueueIngressManifest(buildId, token, claim.generation); + const observed = await this.observeIntentDeploys(buildId, claim.generation, token); + await this.finishIntent(buildId, claim.generation); + if (observed > 0) { + getLogger({ stage: LogStage.BUILD_COMPLETE, ...logContext }).info( + `Build reconciliation: completed services=${observed}` ); } } catch (error) { - if (error instanceof AuthorityLockLostError) { - const stillCurrent = activeRunUUID - ? await this.isDeploymentRunCurrent(buildId, activeRunUUID, claim.generation) - : Boolean(await this.claimDeploymentReconciliation(buildId, claim.generation)); - if (stillCurrent) { - getLogger({ error, buildId, generation: claim.generation }).warn( - 'Build reconciliation: mutation lock lost; leaving generation pending for retry' - ); - throw error; - } + const stillCurrent = await this.isDeploymentRunCurrent(buildId, token, claim.generation).catch(() => false); + if (error instanceof AuthorityLockLostError && stillCurrent) { + getLogger({ error, ...logContext }).warn( + 'Build reconciliation: mutation lock lost; leaving generation pending for retry' + ); + throw error; + } + if (!stillCurrent) { + getLogger(logContext).info('Build reconciliation: stale failure ignored'); + return; } - const stillCurrent = activeRunUUID - ? await this.isDeploymentRunCurrent(buildId, activeRunUUID, claim.generation) - : Boolean(await this.claimDeploymentReconciliation(buildId, claim.generation)); - if (stillCurrent) { - const configError = error instanceof ParsingError || error instanceof ValidationError; - if (configError && activeBuild && activeRunUUID) { - await this.recordBuildFailure( - activeBuild, - BuildStatus.CONFIG_ERROR, - activeRunUUID, - error, - 'Lifecycle configuration failed validation.', - claim.generation - ); - await this.markDeploymentReconciliationObserved(buildId, claim.generation, activeRunUUID); - return; - } - - if (!this.isFinalDeploymentReconciliationAttempt(job)) { - getLogger({ error, buildId, generation: claim.generation }).warn( - 'Build reconciliation: transient failure; leaving generation pending for queue retry' - ); - throw error; - } - if ( - await this.recordFinalDeploymentReconciliationFailure(buildId, claim, activeBuild, activeRunUUID, error) - ) { - getLogger({ stage: LogStage.BUILD_FAILED, buildId, generation: claim.generation }).error( - { error }, - 'Build reconciliation: failed after final queue attempt' - ); - return; - } + if (error instanceof ParsingError || error instanceof ValidationError) { + if (state.build) await this.recordRootConfigError(state.build, error); + await this.takeIntentDeploys(buildId, claim.generation, token); + await this.observeIntentDeploys(buildId, claim.generation); + await this.finishIntent(buildId, claim.generation); + return; + } - // Authority may have changed while the final failure was being - // recorded. Stale failures are intentionally silent; a still-current - // failure must remain retryable if its terminal write could not land. - if (!(await this.claimDeploymentReconciliation(buildId, claim.generation))) return; + if (!this.isFinalDeploymentReconciliationAttempt(job)) { + getLogger({ error, ...logContext }).warn( + 'Build reconciliation: transient failure; leaving generation pending for queue retry' + ); throw error; } - getLogger({ buildId, generation: claim.generation }).info('Build reconciliation: stale failure ignored'); + + await this.recordFinalDeploymentReconciliationFailure(buildId, claim, error); + getLogger({ stage: LogStage.BUILD_FAILED, error, ...logContext }).error( + 'Build reconciliation: failed after final queue attempt' + ); } }); }); @@ -4413,6 +4906,66 @@ export default class BuildService extends BaseService { }; } +/** + * A full pass honours every floor. A narrower intent honours the root config + * floor plus, for a repository redeploy, that repository's own; a push carries + * its own revision. Floors are verified against GitHub, so a narrow intent + * must not pay for every repository's lookup. + */ +function acceptedSourcePinsForIntent( + pins: AcceptedSourcePins, + intent: DeploymentIntent, + rootKey: string | null +): AcceptedSourcePins { + if (deploymentIntentSelectsAllDeploys(intent)) return pins; + const ownPrefix = intent.type === 'repository' ? `${intent.githubRepositoryId}:` : null; + return Object.fromEntries( + Object.entries(pins).filter(([key]) => key === rootKey || (ownPrefix != null && key.startsWith(ownPrefix))) + ); +} + +function rootSourcePinKey(build: Build): string | null { + const source = getBuildSource(build); + if (source.githubRepositoryId == null || !source.branchName) return null; + return `${source.githubRepositoryId}:${source.branchName}`; +} + +/** + * Row-less services whose YAML failed in the newest import that evaluated them. + * An environment-wide import evaluates everything and replaces every older + * note; a newer service-scoped import adds its own. A service that has a row + * again carries its own state. + */ +function unresolvedServiceFailures( + refs: AcceptedDeploymentRefs | null | undefined, + active: Array> +): Record { + if (!refs || typeof refs !== 'object' || Array.isArray(refs)) return {}; + const imported = Object.values(refs).filter((entry) => entry?.configFailures && Number.isSafeInteger(entry.gen)); + const newestEnvWide = imported + .filter((entry) => deploymentIntentSelectsAllDeploys(entry)) + .sort((left, right) => right.gen - left.gen)[0]; + const carried = new Set(active.map((deploy) => deploy.deployable?.name).filter(Boolean)); + const failures: Record = {}; + for (const entry of imported) { + const counts = + entry === newestEnvWide || + (!deploymentIntentSelectsAllDeploys(entry) && (!newestEnvWide || entry.gen > newestEnvWide.gen)); + if (!counts) continue; + for (const [name, reason] of Object.entries(entry.configFailures ?? {})) { + if (!carried.has(name)) failures[name] = reason; + } + } + return failures; +} + +function unobservedAcceptedGenerations(refs: AcceptedDeploymentRefs | null | undefined): number[] { + if (!refs || typeof refs !== 'object' || Array.isArray(refs)) return []; + return Object.values(refs) + .filter((intent) => Number.isSafeInteger(intent?.gen) && intent.observedGen !== intent.gen) + .map((intent) => intent.gen); +} + function canonicalJson(value: unknown): unknown { if (Array.isArray(value)) return value.map(canonicalJson); if (value && typeof value === 'object') { diff --git a/src/server/services/deploy.ts b/src/server/services/deploy.ts index e14bf393..deb1bbb2 100644 --- a/src/server/services/deploy.ts +++ b/src/server/services/deploy.ts @@ -20,6 +20,7 @@ import * as codefresh from 'server/lib/codefresh'; import { getLogger, withLogContext, extractContextForQueue } from 'server/lib/logger'; import hash from 'object-hash'; import { BuildKind, DeployStatus, DeployTypes } from 'shared/constants'; +import { isDeployAuthorityCurrent } from 'server/lib/deploymentReconciliation/authority'; import * as cli from 'server/lib/cli'; import RDS from 'aws-sdk/clients/rds'; import resourceGroupsTagging from 'aws-sdk/clients/resourcegroupstaggingapi'; @@ -83,7 +84,8 @@ export default class DeployService extends BaseService { githubRepositoryId?: number, sourceRef?: string | null, sourceBranch?: string | null, - sourceGithubRepositoryId: number | null | undefined = githubRepositoryId + sourceGithubRepositoryId: number | null | undefined = githubRepositoryId, + acceptedSourcePins?: Record ): Promise { await build?.$fetchGraph('[deployables.[repository]]'); @@ -162,7 +164,9 @@ export default class DeployService extends BaseService { sourceRef, sourceGithubRepositoryId, sourceBranch - ) ?? (await getShaForDeploy(deploy)); + ) ?? + acceptedSourcePins?.[`${deployableRepositoryId}:${effectiveBranch}`] ?? + (await getShaForDeploy(deploy)); patchFields.sha = sha; } catch (error) { getLogger().debug({ error }, 'Deploy: SHA fetch failed continuing=true'); @@ -193,9 +197,10 @@ export default class DeployService extends BaseService { * Helper function to check if an Aurora database already exists in AWS * @param buildUuid The build UUID to search for * @param serviceName The service name to search for - * @returns The database cluster endpoint address if found (or instance endpoint if not clustered), null otherwise + * @returns The database cluster endpoint address if found (or instance endpoint if not clustered), null when + * none is found, undefined when the check itself failed */ - private async findExistingAuroraDatabase(buildUuid: string, serviceName: string): Promise { + private async findExistingAuroraDatabase(buildUuid: string, serviceName: string): Promise { try { const rds = new RDS(); const taggingApi = new resourceGroupsTagging(); @@ -250,7 +255,7 @@ export default class DeployService extends BaseService { return null; } catch (error) { getLogger().debug({ error }, 'Aurora: check failed'); - return null; + return undefined; } } @@ -281,6 +286,12 @@ export default class DeployService extends BaseService { } const existingDbEndpoint = await this.findExistingAuroraDatabase(deploy.build.uuid, deploy.deployable.name); + if (existingDbEndpoint === undefined && deploy.cname) { + // A failed check is no proof the database is gone; a new restore would replace a live endpoint. + getLogger().info('Aurora: skipped reason=checkFailed'); + await this.patchDeployForRun(deploy, runUUID, { status: DeployStatus.BUILT }); + return true; + } if (existingDbEndpoint) { getLogger().info('Aurora: skipped reason=exists'); await this.patchDeployForRun(deploy, runUUID, { @@ -330,7 +341,9 @@ export default class DeployService extends BaseService { runUUID: string, sourceRef?: string | null, sourceGithubRepositoryId?: number | null, - sourceBranch?: string | null + sourceBranch?: string | null, + acceptedSourcePins?: Record, + expectedGeneration?: number ): Promise { return withLogContext({ deployUuid: deploy.uuid, serviceName: deploy.deployable?.name }, async () => { let result: boolean = false; @@ -348,7 +361,8 @@ export default class DeployService extends BaseService { deploy.branchName, sourceRef, sourceGithubRepositoryId, - sourceBranch + sourceBranch, + acceptedSourcePins ); const shortSha = fullSha.substring(0, 7); const envSha = hash(merge(deploy.env || {}, build.commentRuntimeEnv)); @@ -366,26 +380,40 @@ export default class DeployService extends BaseService { let buildLogs: string; let codefreshBuildId: string; try { - const current = await this.patchDeployForRun(deploy, runUUID, { - buildLogs: null, - buildPipelineId: null, - buildOutput: null, - deployPipelineId: null, - deployOutput: null, - }); + const current = await this.patchDeployForRun( + deploy, + runUUID, + { + buildLogs: null, + buildPipelineId: null, + buildOutput: null, + deployPipelineId: null, + deployOutput: null, + }, + expectedGeneration + ); if (!current) { getLogger().info('Codefresh: skipped reason=superseded'); return true; } - const pinnedSourceRef = this.getPinnedBuildSourceRef( - build, - deploy.githubRepositoryId, - deploy.branchName, - sourceRef, - sourceGithubRepositoryId, - sourceBranch - ); + if ( + !(await this.patchDeployForRun(deploy, runUUID, { status: DeployStatus.DEPLOYING }, expectedGeneration)) + ) { + getLogger().info('Codefresh: deploy skipped reason=superseded'); + return true; + } + const pinnedSourceRef = + this.getPinnedBuildSourceRef( + build, + deploy.githubRepositoryId, + deploy.branchName, + sourceRef, + sourceGithubRepositoryId, + sourceBranch + ) ?? + acceptedSourcePins?.[`${deploy.githubRepositoryId}:${deploy.branchName}`] ?? + null; codefreshBuildId = await cli.codefreshDeploy(deploy, build, deployable, pinnedSourceRef).catch((error) => { getLogger().error({ error }, 'Codefresh: build id missing'); return null; @@ -455,13 +483,23 @@ export default class DeployService extends BaseService { runUUID: string, sourceRef?: string | null, sourceGithubRepositoryId?: number | null, - sourceBranch?: string | null + sourceBranch?: string | null, + acceptedSourcePins?: Record, + expectedGeneration?: number ): Promise { if (deploy.deployable != null) { if (deploy.deployable.type === DeployTypes.AURORA_RESTORE) { return this.deployAurora(deploy, runUUID); } else if (deploy.deployable.type === DeployTypes.CODEFRESH) { - return this.deployCodefresh(deploy, runUUID, sourceRef, sourceGithubRepositoryId, sourceBranch); + return this.deployCodefresh( + deploy, + runUUID, + sourceRef, + sourceGithubRepositoryId, + sourceBranch, + acceptedSourcePins, + expectedGeneration + ); } } } @@ -478,7 +516,8 @@ export default class DeployService extends BaseService { sourceGithubRepositoryId?: number | null, sourceBranch?: string | null, expectedGeneration?: number, - nativeServiceAccount?: string + nativeServiceAccount?: string, + acceptedSourcePins?: Record ): Promise { return withLogContext({ deployUuid: deploy.uuid, serviceName: deploy.deployable?.name }, async () => { try { @@ -499,7 +538,8 @@ export default class DeployService extends BaseService { sourceGithubRepositoryId, sourceBranch, expectedGeneration, - nativeServiceAccount + nativeServiceAccount, + acceptedSourcePins ); case DeployTypes.DOCKER: await this.patchAndUpdateActivityFeed( @@ -524,7 +564,8 @@ export default class DeployService extends BaseService { sourceGithubRepositoryId, sourceBranch, expectedGeneration, - nativeServiceAccount + nativeServiceAccount, + acceptedSourcePins ); } @@ -539,7 +580,8 @@ export default class DeployService extends BaseService { deploy.branchName, sourceRef, sourceGithubRepositoryId, - sourceBranch + sourceBranch, + acceptedSourcePins ); } catch (shaError) { getLogger().debug( @@ -654,16 +696,18 @@ export default class DeployService extends BaseService { branchName: string | null, sourceRef?: string | null, sourceGithubRepositoryId?: number | null, - sourceBranch?: string | null + sourceBranch?: string | null, + acceptedSourcePins?: Record ): Promise { - const pinned = this.getPinnedBuildSourceRef( - deploy.build, - deploy.githubRepositoryId, - deploy.branchName, - sourceRef, - sourceGithubRepositoryId, - sourceBranch - ); + const pinned = + this.getPinnedBuildSourceRef( + deploy.build, + deploy.githubRepositoryId, + deploy.branchName, + sourceRef, + sourceGithubRepositoryId, + sourceBranch + ) ?? acceptedSourcePins?.[`${deploy.githubRepositoryId}:${deploy.branchName}`]; if (pinned) return pinned; const [owner, name] = repo?.split('/') || []; @@ -693,9 +737,16 @@ export default class DeployService extends BaseService { private async patchDeployForRun( deploy: Deploy, runUUID: string, - params: Objection.PartialModelObject + params: Objection.PartialModelObject, + expectedGeneration?: number ): Promise { - const updated = await this.db.models.Deploy.query().where({ id: deploy.id, runUUID }).patch(params); + const updated = await this.db.models.Deploy.query() + .where({ + id: deploy.id, + runUUID, + ...(expectedGeneration != null ? { desiredGeneration: expectedGeneration } : {}), + }) + .patch(params); if (!updated) { getLogger().debug(`Deploy: stale write skipped deployId=${deploy.id} runUUID=${runUUID}`); return false; @@ -703,25 +754,8 @@ export default class DeployService extends BaseService { return true; } - private async isDeployRunCurrent(deploy: Deploy, runUUID: string): Promise { - const current = await this.db.models.Deploy.query().findOne({ id: deploy.id, runUUID }).select('id'); - return Boolean(current); - } - - private async isBuildRunCurrent( - buildId: number | null | undefined, - runUUID: string, - expectedGeneration?: number - ): Promise { - if (!buildId) return false; - let current = this.db.models.Build.query().findOne({ id: buildId, runUUID }).whereNull('deletedAt'); - if (expectedGeneration != null) current = current.where('desiredGeneration', expectedGeneration); - return Boolean(await current); - } - private async isDeploymentRunCurrent(deploy: Deploy, runUUID: string, expectedGeneration?: number): Promise { - if (!(await this.isDeployRunCurrent(deploy, runUUID))) return false; - return expectedGeneration == null || this.isBuildRunCurrent(deploy.buildId, runUUID, expectedGeneration); + return isDeployAuthorityCurrent(this.db.models, deploy, runUUID, expectedGeneration); } private async withDeploySecretMutation( @@ -1087,7 +1121,8 @@ export default class DeployService extends BaseService { sourceGithubRepositoryId?: number | null, sourceBranch?: string | null, expectedGeneration?: number, - nativeServiceAccount?: string + nativeServiceAccount?: string, + acceptedSourcePins?: Record ) { const { build, deployable } = deploy; const uuid = build?.uuid; @@ -1113,7 +1148,8 @@ export default class DeployService extends BaseService { deploy.branchName, sourceRef, sourceGithubRepositoryId, - sourceBranch + sourceBranch, + acceptedSourcePins ); const repositoryName: string = deployable.repository.fullName; diff --git a/src/server/services/deployable.ts b/src/server/services/deployable.ts index 2f2f8183..5705e42b 100644 --- a/src/server/services/deployable.ts +++ b/src/server/services/deployable.ts @@ -24,6 +24,8 @@ import { CAPACITY_TYPE, DeployTypes } from 'shared/constants'; import { Builder, Helm } from 'server/models/yaml'; import GlobalConfigService from './globalConfig'; +import { ParsingError } from 'server/lib/yamlConfigParser'; +import { ValidationError } from 'server/lib/yamlConfigValidator'; export type DeployableConfigSource = 'yaml'; @@ -44,6 +46,8 @@ export interface DeployableReconciliationResult { unresolvedServiceNames: string[]; /** Repositories whose YAML could not be read; their deployables are never reaped as stale. */ unresolvedRepositoryIds: number[]; + /** Services whose own lifecycle.yaml failed to parse or validate, by service name. */ + configFailures: Record; } export interface DeployableAttributes { @@ -444,7 +448,9 @@ export default class DeployableService extends BaseService { sourceBranch?: string | null, sourceGithubRepositoryId: number | null | undefined = filterGithubRepositoryId, unresolvedServiceNames: Set = new Set(), - unresolvedRepositoryIds: Set = new Set() + unresolvedRepositoryIds: Set = new Set(), + configFailures: Map = new Map(), + acceptedSourcePins?: Record ): Promise { try { let sourceRepository: Repository | null = null; @@ -571,7 +577,10 @@ export default class DeployableService extends BaseService { } if (filterGithubRepositoryId != null && sourceBranch != null) targetAttributionResolved = true; - const dependencyConfigRef = sourceRefTargetsDependency ? sourceRef : branchName; + // A full pass reads each service's YAML at the same accepted revision its image is built from. + const dependencyConfigRef = sourceRefTargetsDependency + ? sourceRef + : acceptedSourcePins?.[`${repository.githubRepositoryId}:${branchName}`] ?? branchName; dependencyYamlConfig = await YamlService.fetchLifecycleConfigByRepository( repository, dependencyConfigRef @@ -641,6 +650,12 @@ export default class DeployableService extends BaseService { service: yamlEnvService.name, error, }).error('Deployable: create/update from yaml failed'); + // A broken service lifecycle.yaml fails that service alone; only the root config fails the environment. + if (error instanceof ParsingError || error instanceof ValidationError) { + configFailures.set(yamlEnvService.name, (error as Error).message); + markUnresolved(yamlEnvService.name, undefined); + return; + } throw error; } }) @@ -671,7 +686,12 @@ export default class DeployableService extends BaseService { rootBaseConfigRef != null ) { const sourceRefTargetsRoot = sourceRef != null && matchesDeliveredSource(sourceRepository, rootBranch); - const rootConfigRef = sourceRefTargetsRoot ? sourceRef : rootBaseConfigRef; + // A branch-named root ref honours the newest accepted push to that branch; a pinned config SHA stays as is. + const rootPin = + rootBaseConfigRef === rootBranch + ? acceptedSourcePins?.[`${sourceRepository.githubRepositoryId}:${rootBranch}`] + : undefined; + const rootConfigRef = sourceRefTargetsRoot ? sourceRef : rootPin ?? rootBaseConfigRef; const yamlConfig: YamlService.LifecycleConfig = await YamlService.fetchLifecycleConfigByRepository( sourceRepository, rootConfigRef @@ -770,7 +790,8 @@ export default class DeployableService extends BaseService { filterGithubRepositoryId?: number, sourceRef?: string | null, sourceBranch?: string | null, - sourceGithubRepositoryId: number | null | undefined = filterGithubRepositoryId + sourceGithubRepositoryId: number | null | undefined = filterGithubRepositoryId, + acceptedSourcePins?: Record ): Promise { // We are going to ingest all the database and yaml configuration and process in the memory before writes into the database let deployables: Deployable[] = []; @@ -783,6 +804,7 @@ export default class DeployableService extends BaseService { const deployableServices: Map = new Map(); const unresolvedServiceNames = new Set(); const unresolvedRepositoryIds = new Set(); + const configFailures = new Map(); try { if (pullRequest != null || hasBuildSource) { if (pullRequest != null && pullRequest.branchName == null) { @@ -801,7 +823,9 @@ export default class DeployableService extends BaseService { sourceBranch, sourceGithubRepositoryId, unresolvedServiceNames, - unresolvedRepositoryIds + unresolvedRepositoryIds, + configFailures, + acceptedSourcePins ); // Finally, Upsert the deployables into the database @@ -828,6 +852,7 @@ export default class DeployableService extends BaseService { filterGithubRepositoryId: filterGithubRepositoryId ?? null, unresolvedServiceNames: Array.from(unresolvedServiceNames), unresolvedRepositoryIds: Array.from(unresolvedRepositoryIds), + configFailures: Object.fromEntries(configFailures), reconcileEligibleDeployables: Array.from(deployableServices.values()) .filter((deployable) => deployable.reconcileEligible) .map((deployable) => ({ diff --git a/src/server/services/github.ts b/src/server/services/github.ts index 1dc61e98..735ed8bd 100644 --- a/src/server/services/github.ts +++ b/src/server/services/github.ts @@ -563,17 +563,17 @@ export default class GithubService extends Service { getLogger({ error, githubRepositoryId, branchName }).error('Push: API auto-track processing failed'); }); - if (!allDeploys.length) { - // additional check for static env branch - await this.handlePushForStaticEnv({ - githubRepositoryId, - branchName, - headCommit: deployTriggerRef ?? null, - beforeCommit: !this.isVoidCommit(previousCommit) ? previousCommit : null, - }); - return; - } + // A config-repo push redeploys every static environment rooted at it, even when that repo also owns + // services; those environments are excluded from the repository-targeted path below. + const rootStaticBuildIds = await this.handlePushForStaticEnv({ + githubRepositoryId, + branchName, + headCommit: deployTriggerRef ?? null, + beforeCommit: !this.isVoidCommit(previousCommit) ? previousCommit : null, + }); + if (!allDeploys.length) return; const deploysToRebuild = allDeploys.filter((deploy) => { + if (rootStaticBuildIds.has(Number(deploy.buildId))) return false; if (!deploy?.build) return false; if (deploy.devMode) { getLogger().info(`Push: skipping dev mode service deployId=${deploy.id} service=${deploy.deployable?.name}`); @@ -756,9 +756,10 @@ export default class GithubService extends Service { branchName: string; headCommit?: string | null; beforeCommit?: string | null; - }): Promise => { + }): Promise> => { + const enqueued = new Set(); try { - const build = await this.db.models.Build.query() + const builds = await this.db.models.Build.query() .whereIn('pullRequestId', (prBuilder) => { prBuilder .from(this.db.models.PullRequest.tableName) @@ -773,24 +774,31 @@ export default class GithubService extends Service { }) .andWhere('isStatic', true) .andWhere('trackDefaultBranches', true) - .first(); + .whereNull('deletedAt'); - if (!build) return; - - getLogger().info(`Push: redeploying reason=staticEnv`); - await this.db.services.BuildService.enqueueResolveAndDeployBuild({ - buildId: build?.id, - ...(headCommit ? { sourceRef: headCommit } : {}), - ...(beforeCommit ? { sourceBeforeRef: beforeCommit } : {}), - sourceGithubRepositoryId: githubRepositoryId, - sourceBranch: branchName, - }); + for (const build of builds) { + // A root build is claimed here even when its enqueue fails: a repository-scoped intent is the wrong shape for it. + enqueued.add(Number(build.id)); + getLogger({ buildId: build.id }).info(`Push: redeploying reason=staticEnv`); + try { + await this.db.services.BuildService.enqueueResolveAndDeployBuild({ + buildId: build.id, + ...(headCommit ? { sourceRef: headCommit } : {}), + ...(beforeCommit ? { sourceBeforeRef: beforeCommit } : {}), + sourceGithubRepositoryId: githubRepositoryId, + sourceBranch: branchName, + }); + } catch (error) { + getLogger({ buildId: build.id }).error({ error }, 'Push: static env redeploy enqueue failed'); + } + } } catch (error) { getLogger({}).error( { error }, `Push: static env webhook failed branch=${branchName} repositoryId=${githubRepositoryId}` ); } + return enqueued; }; dispatchWebhook = async (req: NextApiRequest) => { diff --git a/src/server/services/ingress.ts b/src/server/services/ingress.ts index 02512298..6aeb5499 100644 --- a/src/server/services/ingress.ts +++ b/src/server/services/ingress.ts @@ -25,6 +25,7 @@ import yaml from 'js-yaml'; import { redisClient } from 'server/lib/dependencies'; import GlobalConfigService from './globalConfig'; import { buildLifecycleLabels } from 'server/lib/kubernetes/labels'; +import { BuildStatus } from 'shared/constants'; const MANIFEST_PATH = `${TMP_PATH}/ingress`; @@ -95,9 +96,17 @@ export default class IngressService extends BaseService { return withLogContext({ correlationId, buildUuid, sender, _ddTraceContext }, async () => { const isCurrent = async () => { if (!runUUID) return true; - let authority = this.db.models.Build.query().findOne({ id: buildId, runUUID }).whereNull('deletedAt'); - if (expectedGeneration != null) authority = authority.where('desiredGeneration', expectedGeneration); - return Boolean(await authority); + const build = await this.db.models.Build.query() + .findById(buildId) + .select('id', 'runUUID', 'status') + .whereNull('deletedAt'); + if (!build || build.status === BuildStatus.TEARING_DOWN || build.status === BuildStatus.TORN_DOWN) return false; + if (expectedGeneration == null) return build.runUUID === runUUID; + // Ingress is regenerated from every service's current row, so it stays valid while the run owns any of them. + const owned = await this.db.models.Deploy.query() + .findOne({ buildId, runUUID, desiredGeneration: expectedGeneration }) + .select('id'); + return Boolean(owned); }; if (!(await isCurrent())) { getLogger({ buildId }).info('Ingress: skipped reason=superseded'); @@ -105,39 +114,48 @@ export default class IngressService extends BaseService { } getLogger({ stage: LogStage.INGRESS_PROCESSING }).info('Ingress: creating'); - // We just want to create/update ingress for active services only - const configurations = await this.db.services.BuildService.configurationsForBuildId(buildId, false); - const namespace = await this.db.services.BuildService.getNamespace({ id: buildId }); - const { lifecycleDefaults, domainDefaults } = await GlobalConfigService.getInstance().getAllConfigs(); - const manifests = configurations.map((configuration) => { - return yaml.dump( - this.generateNginxManifestForConfiguration({ - configuration, - ingressClassName: lifecycleDefaults?.ingressClassName, - altHosts: domainDefaults?.altHttp || [], - }), - { - skipInvalid: true, - } - ); - }); - const apply = () => - Promise.all( + // Every service's current route is read under the promotion lock so a job that + // owns one service cannot publish a snapshot older than a sibling's rollout. + const apply = async () => { + const configurations = await this.db.services.BuildService.configurationsForBuildId(buildId, false); + const namespace = await this.db.services.BuildService.getNamespace({ id: buildId }); + const { lifecycleDefaults, domainDefaults } = await GlobalConfigService.getInstance().getAllConfigs(); + const manifests = configurations.map((configuration) => { + return yaml.dump( + this.generateNginxManifestForConfiguration({ + configuration, + ingressClassName: lifecycleDefaults?.ingressClassName, + altHosts: domainDefaults?.altHttp || [], + }), + { + skipInvalid: true, + } + ); + }); + const applied = await Promise.all( manifests.map((manifest, idx) => this.applyManifests(manifest, `${buildId}-${idx}-nginx`, namespace, buildId, runUUID, expectedGeneration) ) ); + return applied.every(Boolean); + }; + let applied = false; if (runUUID && this.db.services.BuildService.withCurrentBuildPromotionLock) { const result = await this.db.services.BuildService.withCurrentBuildPromotionLock(buildId, isCurrent, apply); if (!result.admitted) { getLogger({ buildId }).info('Ingress: skipped at promotion reason=superseded'); return; } + applied = result.value === true; } else { - await apply(); + applied = await apply(); } + // Only a run whose every manifest applied clears an earlier ingress failure note from the environment status. + if (applied && expectedGeneration != null) { + await this.db.services.BuildService.publishDerivedBuildStatus?.(buildId, { clearStatusMessage: true }); + } getLogger({ stage: LogStage.INGRESS_COMPLETE }).info('Ingress: created'); }); }; @@ -234,11 +252,13 @@ export default class IngressService extends BaseService { await shellPromise(`kubectl apply -f ${localPath} --namespace ${namespace}`, { timeout: 90_000, }); + return true; } catch (error) { getLogger({ stage: LogStage.INGRESS_FAILED }).warn({ error }, 'Ingress: manifest apply failed'); if (buildId !== undefined) { await this.recordIngressFailureOnBuild(buildId, error, runUUID, expectedGeneration).catch(() => undefined); } + return false; } }; @@ -252,17 +272,25 @@ export default class IngressService extends BaseService { const note = `Ingress apply failed: ${(error as Error)?.message || String(error)}` .replace(/\s+/g, ' ') .slice(0, 300); + if (runUUID && expectedGeneration != null) { + // A late failure from a run that no longer owns any service must not land on the current environment. + const owned = await this.db.models.Deploy.query() + .findOne({ buildId, runUUID, desiredGeneration: expectedGeneration }) + .select('id'); + if (!owned) return; + } let buildRead = this.db.models.Build.query().findById(buildId).whereNull('deletedAt'); - if (runUUID) buildRead = buildRead.where('runUUID', runUUID); - if (expectedGeneration != null) buildRead = buildRead.where('desiredGeneration', expectedGeneration); + if (runUUID && expectedGeneration == null) buildRead = buildRead.where('runUUID', runUUID); const build = await buildRead; if (!build || build.statusMessage?.includes(note)) { return; } const statusMessage = [build.statusMessage, note].filter(Boolean).join(' | ').slice(-500); - let buildPatch = this.db.models.Build.query().patch({ statusMessage }).where({ id: buildId }); - if (runUUID) buildPatch = buildPatch.where('runUUID', runUUID); - if (expectedGeneration != null) buildPatch = buildPatch.where('desiredGeneration', expectedGeneration); + let buildPatch = this.db.models.Build.query() + .patch({ statusMessage }) + .where({ id: buildId }) + .whereNull('deletedAt'); + if (runUUID && expectedGeneration == null) buildPatch = buildPatch.where('runUUID', runUUID); await buildPatch; }; } diff --git a/src/shared/config.ts b/src/shared/config.ts index 9fcf93b4..c3f06354 100644 --- a/src/shared/config.ts +++ b/src/shared/config.ts @@ -110,6 +110,7 @@ export const QUEUE_NAMES = { RESOLVE_AND_DEPLOY: `resolve_and_deploy_${JOB_VERSION}`, BUILD_QUEUE: `build_queue_${JOB_VERSION}`, DEPLOYMENT_RECONCILIATION: `deployment_reconciliation_${JOB_VERSION}`, + SERVICE_RECONCILIATION: `service_reconciliation_${JOB_VERSION}`, GITHUB_DEPLOYMENT: `github_deployment_${JOB_VERSION}`, DEPLOY_CLEANUP: `deploy_cleanup_${JOB_VERSION}`, LABEL: `label_${JOB_VERSION}`, From 75c3e36e9edf82ee3b5c3a13119990afe8746ec7 Mon Sep 17 00:00:00 2001 From: vmelikyan Date: Mon, 12 Oct 2026 11:14:41 -0700 Subject: [PATCH 2/2] test(reconciliation): lock in the per-service rollout error paths Cover the production paths the simulator stubs or that no unit test reached: - a broken service lifecycle.yaml in the real import fails that service alone; any other import error still fails the import - a rollout error of a current run is recorded on its rows in the same attempt; after the run lost its rows it is dropped - the environment-level authority predicate: blocked environments, owned rows, unfinished accepted intents, and the token-only legacy path - the queue failure policy: an escaping configuration error finishes the intent, an error after supersession is ignored, transient errors retry until the final attempt, a lost lock rethrows, and an environment-wide claim that fails leaves the generation pending - a replaced or finished generation at claim time - the accepted SHA stays the floor when GitHub lookups fail - the derived environment status: teardown and config-error guards, pending before failed, inactive rows ignored, unchanged status not rewritten - a CLI deploy that reports failure, the Codefresh deploying fence, and the ingress job clearing a stale failure note only after a clean apply --- src/server/services/__tests__/build.test.ts | 368 +++++++++++++++++- src/server/services/__tests__/deploy.test.ts | 12 + .../__tests__/deployableSourceSeam.test.ts | 93 +++++ src/server/services/__tests__/ingress.test.ts | 42 ++ .../__tests__/serviceReconciliation.test.ts | 17 + 5 files changed, 531 insertions(+), 1 deletion(-) diff --git a/src/server/services/__tests__/build.test.ts b/src/server/services/__tests__/build.test.ts index 7f62269c..f52495ee 100644 --- a/src/server/services/__tests__/build.test.ts +++ b/src/server/services/__tests__/build.test.ts @@ -214,7 +214,7 @@ import BuildService, { assertIdempotentReplayAllowed, assertEnvironmentDestroyAllowed, } from '../build'; -import { BuildKind, BuildStatus, DeployStatus, DeployTypes } from 'shared/constants'; +import { BuildKind, BuildStatus, DeployStatus, DeployTypes, PullRequestStatus } from 'shared/constants'; import * as github from 'server/lib/github'; import { ingressBannerSnippet } from 'server/lib/helm/utils'; import { ParsingError, YamlConfigParser } from 'server/lib/yamlConfigParser'; @@ -1764,6 +1764,102 @@ describe('BuildService deployment reconciliation', () => { head: 'commit-d', }); }); + + test.each([ + [ + 'the branch head lookup fails', + () => (github.getSHAForBranch as jest.Mock).mockRejectedValue(new Error('rate limited')), + ], + [ + 'the commit comparison fails', + () => { + (github.getSHAForBranch as jest.Mock).mockResolvedValue('commit-d'); + (github.compareCommits as jest.Mock).mockRejectedValue(new Error('rate limited')); + }, + ], + ])('keeps the accepted SHA as the floor when %s', async (_name, arrange) => { + const repositoryQuery: any = { + findOne: jest.fn(() => repositoryQuery), + whereNull: jest.fn().mockResolvedValue({ fullName: 'org/service' }), + }; + const service = new BuildService( + { models: { Repository: { query: jest.fn(() => repositoryQuery) } } } as any, + {} as any, + {} as any, + queueManager() as any + ); + arrange(); + + await expect( + (service as any).resolveCurrentSourceRef({ + githubRepositoryId: 42, + sourceGithubRepositoryId: 42, + sourceBranch: 'main', + sourceRef: 'commit-c', + }) + ).resolves.toBe('commit-c'); + }); + + test('a rollout error of a current run is recorded on its rows and the scope still returns its summary', async () => { + const executionError = new Error('kubectl apply exited 1'); + const service = new BuildService({ models: {} } as any, {} as any, {} as any, queueManager() as any); + const build = createBuild({ id: 1, runUUID: 'run-current', namespace: 'env-sample' }); + jest.spyOn(service, 'buildImages').mockResolvedValue(undefined as any); + jest.spyOn(service, 'deployCLIServices').mockResolvedValue(true); + jest.spyOn(service as any, 'isDeploymentRunCurrent').mockResolvedValue(true); + jest.spyOn(service as any, 'publishDerivedBuildStatus').mockResolvedValue(undefined); + jest.spyOn(service as any, 'generateAndApplyManifests').mockRejectedValue(executionError); + const recordFailure = jest.spyOn(service as any, 'recordFailureOnOwnedDeploys').mockResolvedValue(undefined); + const summary = { deployed: false }; + jest.spyOn(service as any, 'summarizeIntentDeploys').mockResolvedValue(summary); + + await expect( + (service as any).executeDeploymentScope( + { + build, + runUUID: 'run-current', + githubRepositoryId: 100, + sourceGithubRepositoryId: 100, + sourceRef: 'commit-a', + sourceBranch: 'main', + }, + 7 + ) + ).resolves.toBe(summary); + + expect(recordFailure).toHaveBeenCalledWith(1, 'run-current', 7, executionError, 'Deployment failed.'); + }); + + test('a rollout error of a run that lost its rows meanwhile is dropped without recording anything', async () => { + const service = new BuildService({ models: {} } as any, {} as any, {} as any, queueManager() as any); + const build = createBuild({ id: 1, runUUID: 'run-old', namespace: 'env-sample' }); + jest.spyOn(service, 'buildImages').mockResolvedValue(undefined as any); + jest.spyOn(service, 'deployCLIServices').mockResolvedValue(true); + jest + .spyOn(service as any, 'isDeploymentRunCurrent') + .mockResolvedValueOnce(true) + .mockResolvedValueOnce(true) + .mockResolvedValueOnce(false); + jest.spyOn(service as any, 'publishDerivedBuildStatus').mockResolvedValue(undefined); + jest.spyOn(service as any, 'generateAndApplyManifests').mockRejectedValue(new Error('kubectl apply exited 1')); + const recordFailure = jest.spyOn(service as any, 'recordFailureOnOwnedDeploys').mockResolvedValue(undefined); + + await expect( + (service as any).executeDeploymentScope( + { + build, + runUUID: 'run-old', + githubRepositoryId: 100, + sourceGithubRepositoryId: 100, + sourceRef: 'a', + sourceBranch: 'main', + }, + 7 + ) + ).resolves.toBeNull(); + + expect(recordFailure).not.toHaveBeenCalled(); + }); }); describe('idempotency digest + replay authorization (D12)', () => { @@ -2105,6 +2201,20 @@ describe('BuildService focused changed-line coverage', () => { await expect(service.deployCLIServices(build as any, 'run-c', 42, 'sha-c', 'main')).resolves.toBe(true); }); + test('a CLI deploy that reports failure is recorded on its row as a phase failure', async () => { + const deploy = { id: 5, uuid: 'aurora-1', active: true, deployable: { type: DeployTypes.AURORA_RESTORE } }; + mockDeployQuery.mockReturnValue(deployQuery([deploy])); + const deployCLI = jest.fn().mockResolvedValue(false); + const service = serviceWith({ services: { Deploy: { deployCLI } } }); + const recordPhaseFailure = jest.spyOn(service as any, 'recordPhaseFailure').mockResolvedValue(undefined); + const build = { id: 4, $fetchGraph: jest.fn().mockResolvedValue(undefined) }; + + await expect(service.deployCLIServices(build as any, 'run-c', 42, 'sha-c', 'main', 42, 3)).resolves.toBe(false); + + expect(deployCLI).toHaveBeenCalledWith(deploy, 'run-c', 'sha-c', 42, 'main', undefined, 3); + expect(recordPhaseFailure).toHaveBeenCalledWith(deploy, 'run-c', DeployStatus.ERROR, 'CLI deploy failed.', 3); + }); + test('an aborted rollout does not fail a service whose prerequisite is outside the run', async () => { const managed = (name: string, deploymentDependsOn: string[]) => { const where = jest.fn().mockResolvedValue(1); @@ -5517,6 +5627,262 @@ describe('BuildService uncovered public behavior', () => { expect(mockQueueAdd).not.toHaveBeenCalled(); }); + test('claimDeploymentIntent ignores a generation that was replaced or already finished, and adopts only a live one', async () => { + const refs = { + 'source:1:main': { + type: 'source', + requestId: 'newer', + target: 'repository', + githubRepositoryId: 1, + branch: 'main', + sha: 'b', + gen: 4, + }, + all: { type: 'all', requestId: 'done', gen: 2, observedGen: 2 }, + }; + const buildQuery = lookupQuery({ id: 7, acceptedRefs: refs }); + const service = serviceWith({ models: { Build: { query: jest.fn(() => buildQuery) } } }); + const adopt = jest.spyOn(service as any, 'adoptDeploysForIntent').mockResolvedValue(undefined); + + await expect((service as any).claimDeploymentIntent(7, 3)).resolves.toBeNull(); + await expect((service as any).claimDeploymentIntent(7, 2)).resolves.toBeNull(); + expect(adopt).not.toHaveBeenCalled(); + await expect((service as any).claimDeploymentIntent(7, 4)).resolves.toMatchObject({ + generation: 4, + token: 'newer', + }); + expect(adopt).toHaveBeenCalledTimes(1); + }); + + describe('processDeploymentReconciliationQueue failure policy', () => { + const run = (error: Error, options: { current: boolean; attemptsMade?: number }) => { + const service = serviceWith({ models: {} }); + jest + .spyOn(service as any, 'tryWithDeploymentGenerationLock') + .mockImplementation((_buildId: unknown, _generation: unknown, action: any) => action()); + jest.spyOn(service as any, 'claimDeploymentIntent').mockResolvedValue({ + generation: 3, + token: 'run-t', + intent: { type: 'all', requestId: 'run-t', gen: 3 }, + acceptedSourcePins: {}, + }); + jest.spyOn(service as any, 'withCurrentBuildDeploymentLock').mockRejectedValue(error); + jest.spyOn(service as any, 'isDeploymentRunCurrent').mockResolvedValue(options.current); + const spies = { + takeIntentDeploys: jest.spyOn(service as any, 'takeIntentDeploys').mockResolvedValue(undefined), + observeIntentDeploys: jest.spyOn(service as any, 'observeIntentDeploys').mockResolvedValue(0), + finishIntent: jest.spyOn(service as any, 'finishIntent').mockResolvedValue(undefined), + recordFinalFailure: jest + .spyOn(service as any, 'recordFinalDeploymentReconciliationFailure') + .mockResolvedValue(undefined), + }; + const job: any = { + data: { buildId: 7, generation: 3 }, + attemptsMade: options.attemptsMade ?? 0, + opts: { attempts: 10 }, + }; + return { outcome: service.processDeploymentReconciliationQueue(job), spies }; + }; + + test('a configuration error that escapes the configuration step takes and observes the rows and finishes the intent', async () => { + const { outcome, spies } = run(new ValidationError('services must be a list'), { current: true }); + + await expect(outcome).resolves.toBeUndefined(); + expect(spies.takeIntentDeploys).toHaveBeenCalledWith(7, 3, 'run-t'); + expect(spies.observeIntentDeploys).toHaveBeenCalledWith(7, 3); + expect(spies.finishIntent).toHaveBeenCalledWith(7, 3); + expect(spies.recordFinalFailure).not.toHaveBeenCalled(); + }); + + test('an error after the run lost its rows is ignored: no retry, no record, no finish', async () => { + const { outcome, spies } = run(new Error('database connection reset'), { current: false }); + + await expect(outcome).resolves.toBeUndefined(); + expect(spies.takeIntentDeploys).not.toHaveBeenCalled(); + expect(spies.finishIntent).not.toHaveBeenCalled(); + expect(spies.recordFinalFailure).not.toHaveBeenCalled(); + }); + + test('a transient error of a current run is rethrown for the queue retry until the final attempt records it', async () => { + const error = new Error('database connection reset'); + const first = run(error, { current: true, attemptsMade: 0 }); + await expect(first.outcome).rejects.toBe(error); + expect(first.spies.recordFinalFailure).not.toHaveBeenCalled(); + + const last = run(error, { current: true, attemptsMade: 9 }); + await expect(last.outcome).resolves.toBeUndefined(); + expect(last.spies.recordFinalFailure).toHaveBeenCalledWith(7, expect.objectContaining({ generation: 3 }), error); + expect(last.spies.finishIntent).not.toHaveBeenCalled(); + }); + + test('a lost mutation lock of a current run is rethrown so the retry resumes the rows it owns', async () => { + const error = new AuthorityLockLostError('build-reconcile.7.3'); + const { outcome, spies } = run(error, { current: true }); + + await expect(outcome).rejects.toBe(error); + expect(spies.recordFinalFailure).not.toHaveBeenCalled(); + }); + + test('an environment-wide intent whose run claim fails leaves its generation pending instead of consuming it', async () => { + const service = serviceWith({ models: {} }); + jest + .spyOn(service as any, 'tryWithDeploymentGenerationLock') + .mockImplementation((_buildId: unknown, _generation: unknown, action: any) => action()); + jest.spyOn(service as any, 'claimDeploymentIntent').mockResolvedValue({ + generation: 3, + token: 'run-t', + intent: { type: 'all', requestId: 'run-t', gen: 3 }, + acceptedSourcePins: {}, + }); + jest + .spyOn(service as any, 'withCurrentBuildDeploymentLock') + .mockImplementation(async (_buildId: unknown, _isPending: unknown, action: any) => ({ + admitted: true, + value: await action(), + })); + jest.spyOn(service as any, 'loadBuildDeploymentAuthority').mockResolvedValue({ + id: 7, + status: BuildStatus.DEPLOYED, + deployEnabled: true, + deletedAt: null, + pullRequestId: null, + }); + const claimRun = jest.spyOn(service as any, 'claimDeploymentRun').mockResolvedValue(null); + const observe = jest.spyOn(service as any, 'observeIntentDeploys').mockResolvedValue(0); + const finish = jest.spyOn(service as any, 'finishIntent').mockResolvedValue(undefined); + const importYaml = jest.spyOn(service as any, 'importYamlConfigFile').mockResolvedValue(undefined); + + await expect( + service.processDeploymentReconciliationQueue({ + data: { buildId: 7, generation: 3 }, + attemptsMade: 0, + opts: { attempts: 10 }, + } as any) + ).resolves.toBeUndefined(); + + expect(claimRun).toHaveBeenCalledWith(expect.objectContaining({ id: 7 }), 'run-t', 3); + expect(importYaml).not.toHaveBeenCalled(); + expect(observe).not.toHaveBeenCalled(); + expect(finish).not.toHaveBeenCalled(); + }); + }); + + describe('isDeploymentRunCurrent', () => { + const check = ( + authority: Record | null, + ownedRow: unknown, + refs: Record = {} + ) => { + const build = authority + ? { acceptedRefs: refs, $fetchGraph: jest.fn().mockResolvedValue(undefined), ...authority } + : null; + const service = serviceWith({ + models: { + Build: { query: jest.fn(() => lookupQuery(build)) }, + Deploy: { query: jest.fn(() => lookupQuery(ownedRow)) }, + }, + }); + return (service as any).isDeploymentRunCurrent(7, 'run-t', 3) as Promise; + }; + const open = { id: 7, status: BuildStatus.DEPLOYING, deployEnabled: true, deletedAt: null, pullRequestId: null }; + + test('is false while the environment is blocked, even for a run that still owns a row', async () => { + await expect(check({ ...open, status: BuildStatus.TEARING_DOWN }, { id: 11 })).resolves.toBe(false); + await expect(check({ ...open, deployEnabled: false }, { id: 11 })).resolves.toBe(false); + await expect( + check( + { ...open, pullRequestId: 5, pullRequest: { status: PullRequestStatus.CLOSED, deployOnUpdate: true } }, + { id: 11 } + ) + ).resolves.toBe(false); + await expect(check(null, { id: 11 })).resolves.toBe(false); + }); + + test('is true for an owned row, or for an accepted intent that is still unfinished, and false otherwise', async () => { + await expect(check(open, { id: 11 })).resolves.toBe(true); + await expect(check(open, undefined, { all: { type: 'all', requestId: 'run-t', gen: 3 } })).resolves.toBe(true); + await expect( + check(open, undefined, { all: { type: 'all', requestId: 'run-t', gen: 3, observedGen: 3 } }) + ).resolves.toBe(false); + await expect(check(open, undefined, { all: { type: 'all', requestId: 'other', gen: 3 } })).resolves.toBe(false); + }); + + test('a run without a generation is current while the environment carries its token', async () => { + const service = serviceWith({ + models: { Build: { query: jest.fn(() => lookupQuery({ ...open, runUUID: 'run-legacy' })) } }, + }); + await expect((service as any).isDeploymentRunCurrent(7, 'run-legacy')).resolves.toBe(true); + await expect((service as any).isDeploymentRunCurrent(7, 'run-other')).resolves.toBe(false); + }); + }); + + describe('publishDerivedBuildStatusUnlocked', () => { + const row = (overrides: Record) => ({ + active: true, + desiredGeneration: 3, + observedGeneration: 3, + status: DeployStatus.READY, + statusMessage: '', + deployable: { name: 'svc' }, + ...overrides, + }); + const derive = async (build: Record, options: Record = {}) => { + const loaded: any = { + id: 7, + statusMessage: '', + deployEnabled: true, + pullRequest: null, + acceptedRefs: {}, + deploys: [], + ...build, + $fetchGraph: jest.fn().mockResolvedValue(undefined), + }; + const service = serviceWith({ models: { Build: { query: jest.fn(() => lookupQuery(loaded)) } } }); + const publish = jest.spyOn(service as any, 'publishBuildStatus').mockResolvedValue(undefined); + await (service as any).publishDerivedBuildStatusUnlocked(7, options); + return publish; + }; + + test('keeps a tearing-down or torn-down environment and a standing config error', async () => { + expect(await derive({ status: BuildStatus.TEARING_DOWN })).not.toHaveBeenCalled(); + expect(await derive({ status: BuildStatus.TORN_DOWN })).not.toHaveBeenCalled(); + expect(await derive({ status: BuildStatus.CONFIG_ERROR })).not.toHaveBeenCalled(); + const cleared = await derive({ status: BuildStatus.CONFIG_ERROR }, { clearConfigError: true }); + expect(cleared).toHaveBeenCalledWith(expect.anything(), BuildStatus.DEPLOYED, expect.anything()); + }); + + test('a pending row outranks a failed one, a failed row outranks success, and an unchanged status is not rewritten', async () => { + const pendingAndFailed = await derive({ + status: BuildStatus.ERROR, + deploys: [ + row({ desiredGeneration: 4, status: DeployStatus.BUILDING }), + row({ status: DeployStatus.DEPLOY_FAILED }), + ], + }); + expect(pendingAndFailed).toHaveBeenCalledWith(expect.anything(), BuildStatus.BUILDING, expect.anything()); + + const rolling = await derive({ + status: BuildStatus.BUILDING, + deploys: [row({ desiredGeneration: 4, status: DeployStatus.DEPLOYING })], + }); + expect(rolling).toHaveBeenCalledWith(expect.anything(), BuildStatus.DEPLOYING, expect.anything()); + + const failed = await derive({ + status: BuildStatus.DEPLOYED, + deploys: [row({ status: DeployStatus.DEPLOY_FAILED, statusMessage: 'Not deployed: db failed.' })], + }); + expect(failed).toHaveBeenCalledWith(expect.anything(), BuildStatus.ERROR, expect.anything()); + + expect(await derive({ status: BuildStatus.DEPLOYED, deploys: [row({})] })).not.toHaveBeenCalled(); + + const inactiveFailure = await derive({ + status: BuildStatus.BUILDING, + deploys: [row({}), row({ active: false, status: DeployStatus.DEPLOY_FAILED })], + }); + expect(inactiveFailure).toHaveBeenCalledWith(expect.anything(), BuildStatus.DEPLOYED, expect.anything()); + }); + }); + test('createApiEnvironment does not retry a generated name after a non-unique insert failure', async () => { const apiConfig = { enabled: true, defaultTtlHours: 72, maxTtlHours: 336, extensionHours: 24 }; const repository = { id: 2, githubRepositoryId: 42, fullName: 'org/repo', defaultEnvId: 5 }; diff --git a/src/server/services/__tests__/deploy.test.ts b/src/server/services/__tests__/deploy.test.ts index 7305ba95..6dd65860 100644 --- a/src/server/services/__tests__/deploy.test.ts +++ b/src/server/services/__tests__/deploy.test.ts @@ -2853,6 +2853,18 @@ describe('DeployService uncovered public behavior', () => { expect(mockWaitForCodefresh).not.toHaveBeenCalled(); }); + test('deployCodefresh stops before pipeline invocation when the deploying fence finds the row taken', async () => { + const { service, deployPatch } = serviceHarness(); + deployPatch.mockResolvedValueOnce(1).mockResolvedValueOnce(0); + const deploy = codefreshDeploy(); + + await expect(service.deployCodefresh(deploy as any, 'run-1')).resolves.toBe(true); + + expect(deployPatch).toHaveBeenCalledTimes(2); + expect(mockCodefreshDeploy).not.toHaveBeenCalled(); + expect(mockWaitForCodefresh).not.toHaveBeenCalled(); + }); + test('deployCodefresh records a source-resolution failure before invoking the pipeline', async () => { const { service } = serviceHarness(); const deploy = codefreshDeploy({ branchName: null }); diff --git a/src/server/services/__tests__/deployableSourceSeam.test.ts b/src/server/services/__tests__/deployableSourceSeam.test.ts index c60e4f89..ec9341f8 100644 --- a/src/server/services/__tests__/deployableSourceSeam.test.ts +++ b/src/server/services/__tests__/deployableSourceSeam.test.ts @@ -68,6 +68,7 @@ jest.mock('server/services/globalConfig', () => ({ })); import DeployableService from '../deployable'; +import { ParsingError } from 'server/lib/yamlConfigParser'; const makeService = () => { const db = { models: {}, services: { PullRequest: { updatePullRequestBranchName: jest.fn() } } }; @@ -1258,4 +1259,96 @@ describe('deployable source seam (PR vs API build)', () => { await expect((service as any).upsertDeployablesWithDatabase('uuid-9', 9, [])).resolves.toEqual([]); expect(query).toHaveBeenCalledTimes(4); }); + + it('a service whose lifecycle.yaml does not parse fails alone, without protecting its repository or failing the import', async () => { + const service = makeService(); + const rootRepository = { githubRepositoryId: 42, fullName: 'org/root' }; + const brokenRepository = { githubRepositoryId: 99, fullName: 'org/broken' }; + mockRepositoryWhereNull.mockResolvedValue(rootRepository); + mockFetchLifecycleConfigByRepository + .mockResolvedValueOnce({ + environment: { + defaultServices: [{ name: 'broken-api', repository: 'org/broken', branch: 'main' }], + optionalServices: [], + }, + services: [], + }) + .mockRejectedValueOnce(new ParsingError('bad indentation of a mapping entry')); + mockResolveRepository.mockResolvedValueOnce(brokenRepository); + const build: any = { + id: 9, + githubRepositoryId: 42, + branchName: 'main', + configSha: 'root-sha', + deploys: [{ uuid: 'deploy-1', deployable: { name: 'broken-api' } }], + $fetchGraph: jest.fn().mockResolvedValue(undefined), + }; + const unresolvedServiceNames = new Set(); + const unresolvedRepositoryIds = new Set(); + const configFailures = new Map(); + + const result = await (service as any).updateOrCreateDeployableUsingYamlConfig( + new Map(), + 9, + 'uuid-9', + null, + build, + undefined, + null, + null, + undefined, + unresolvedServiceNames, + unresolvedRepositoryIds, + configFailures + ); + + expect(result).toBe(true); + expect(configFailures.get('broken-api')).toContain('bad indentation of a mapping entry'); + expect(Array.from(unresolvedServiceNames)).toEqual(['broken-api']); + expect(Array.from(unresolvedRepositoryIds)).toEqual([]); + expect(mockResolveExactEnvironmentService).not.toHaveBeenCalled(); + }); + + it('an error that is not a configuration error still fails the whole import', async () => { + const service = makeService(); + mockRepositoryWhereNull.mockResolvedValue({ githubRepositoryId: 42, fullName: 'org/root' }); + mockFetchLifecycleConfigByRepository + .mockResolvedValueOnce({ + environment: { + defaultServices: [{ name: 'broken-api', repository: 'org/broken', branch: 'main' }], + optionalServices: [], + }, + services: [], + }) + .mockRejectedValueOnce(new Error('GitHub unavailable')); + mockResolveRepository.mockResolvedValueOnce({ githubRepositoryId: 99, fullName: 'org/broken' }); + const build: any = { + id: 9, + githubRepositoryId: 42, + branchName: 'main', + configSha: 'root-sha', + deploys: [{ uuid: 'deploy-1', deployable: { name: 'broken-api' } }], + $fetchGraph: jest.fn().mockResolvedValue(undefined), + }; + const configFailures = new Map(); + + await expect( + (service as any).updateOrCreateDeployableUsingYamlConfig( + new Map(), + 9, + 'uuid-9', + null, + build, + undefined, + null, + null, + undefined, + new Set(), + new Set(), + configFailures + ) + ).rejects.toThrow('GitHub unavailable'); + + expect(configFailures.size).toBe(0); + }); }); diff --git a/src/server/services/__tests__/ingress.test.ts b/src/server/services/__tests__/ingress.test.ts index 5e8f3b09..a5146fe5 100644 --- a/src/server/services/__tests__/ingress.test.ts +++ b/src/server/services/__tests__/ingress.test.ts @@ -410,6 +410,48 @@ describe('IngressService', () => { expect(apply).toHaveBeenCalledWith(expect.any(String), '7-0-nginx', 'env-test', 7, 'run-c', 3); }); + test('a clean generation-fenced apply clears a stale ingress failure note; a failed apply keeps it', async () => { + const configuration = { + deployUUID: 'deploy-a', + host: 'a.example.test', + pathPortMapping: { '/': 8080 }, + serviceHost: 'service-a', + ingressAnnotations: {}, + ipWhitelist: [], + }; + const publishDerivedBuildStatus = jest.fn().mockResolvedValue(undefined); + const db = { + models: { + Build: { query: jest.fn(() => authorityQuery({ id: 7 })) }, + Deploy: { query: jest.fn(() => authorityQuery({ id: 11 })) }, + }, + services: { + BuildService: { + configurationsForBuildId: jest.fn().mockResolvedValue([configuration]), + getNamespace: jest.fn().mockResolvedValue('env-test'), + withCurrentBuildPromotionLock: jest.fn(async (_buildId, _isCurrent, action) => ({ + admitted: true, + value: await action(), + })), + publishDerivedBuildStatus, + }, + }, + }; + const service = new IngressService(db as any, {} as any, {} as any, queueManager as any); + jest + .spyOn(service as any, 'applyManifests') + .mockResolvedValueOnce(true) + .mockResolvedValueOnce(false); + const job = { data: { buildId: 7, runUUID: 'run-c', expectedGeneration: 3 } }; + + await service.createOrUpdateIngressForBuild(job); + expect(publishDerivedBuildStatus).toHaveBeenCalledWith(7, { clearStatusMessage: true }); + + publishDerivedBuildStatus.mockClear(); + await service.createOrUpdateIngressForBuild(job); + expect(publishDerivedBuildStatus).not.toHaveBeenCalled(); + }); + test('does not apply when authority is lost before promotion admission', async () => { const gate = jest.fn().mockResolvedValue({ admitted: false }); const db = { diff --git a/src/server/services/__tests__/serviceReconciliation.test.ts b/src/server/services/__tests__/serviceReconciliation.test.ts index 3740806b..c4e620f8 100644 --- a/src/server/services/__tests__/serviceReconciliation.test.ts +++ b/src/server/services/__tests__/serviceReconciliation.test.ts @@ -846,6 +846,23 @@ describe('service-independent reconciliation', () => { expect(world.settled()).toBe(true); }); + test('an error inside a superseded run is ignored, not retried, and the newer run still lands', async () => { + const world = createWorld(SPECS); + await world.merge('A'); + await jest.advanceTimersByTimeAsync(5 * MIN); + const second = await world.merge('A'); + sim.failPatchOnce = (patch: any, table: string) => table === 'deploy' && 'observedGeneration' in patch; + await jest.advanceTimersByTimeAsync(40 * MIN); + + expect(logs.some((line) => line.includes('stale failure ignored'))).toBe(true); + expect(world.queue.attempts.filter((attempt) => attempt.startsWith(`reconcile-${BUILD_ID}-1#`))).toEqual([ + `reconcile-${BUILD_ID}-1#0`, + ]); + expect(world.live.get('A')).toBe(`A@${second}`); + expect(world.row('A').observedGeneration).toBe(2); + expect(world.settled()).toBe(true); + }); + test('a config error in C fails only C; A and B still ship and C recovers on its next merge', async () => { const world = createWorld(SPECS); const aSha = await world.merge('A');