diff --git a/src/server/db/migrations/037_add_deploy_reconciliation_generations.ts b/src/server/db/migrations/037_add_deploy_reconciliation_generations.ts new file mode 100644 index 00000000..20c4ea12 --- /dev/null +++ b/src/server/db/migrations/037_add_deploy_reconciliation_generations.ts @@ -0,0 +1,63 @@ +/** + * Copyright 2026 GoodRx, Inc. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import type { Knex } from 'knex'; + +const PENDING_INDEX = 'deploys_reconcile_pending'; + +// PostgreSQL concurrent index operations cannot run inside Knex's migration transaction. +export const config = { transaction: false }; + +// Runs outside a transaction, so every step is guarded to make a retry after a failed concurrent index build safe. +export async function up(knex: Knex): Promise { + for (const column of ['desiredGeneration', 'observedGeneration']) { + if (await knex.schema.hasColumn('deploys', column)) continue; + await knex.schema.alterTable('deploys', (table) => { + table.bigInteger(column).notNullable().defaultTo(0); + }); + } + + // Intents at or below the old watermark are finished; marking them keeps the sweep from replaying them. + await knex.raw(` + update builds + set "acceptedRefs" = ( + select coalesce(jsonb_object_agg( + entry.key, + case + when jsonb_typeof(entry.value -> 'gen') <> 'number' then entry.value + when (entry.value ->> 'gen')::numeric <= builds."observedGeneration" + then entry.value || jsonb_build_object('observedGen', entry.value -> 'gen') + else entry.value + end + ), '{}'::jsonb) + from jsonb_each("acceptedRefs") as entry + ) + where "acceptedRefs" <> '{}'::jsonb + and "deletedAt" is null + `); + + // A failed concurrent build leaves an invalid index with this name behind. + await knex.raw('drop index concurrently if exists ??', [PENDING_INDEX]); + await knex.raw( + `create index concurrently ?? on deploys ("buildId") where "desiredGeneration" > "observedGeneration"`, + [PENDING_INDEX] + ); +} + +export async function down(knex: Knex): Promise { + await knex.raw('drop index concurrently if exists ??', [PENDING_INDEX]); + + await knex.schema.alterTable('deploys', (table) => { + table.dropColumn('observedGeneration'); + table.dropColumn('desiredGeneration'); + }); +} diff --git a/src/server/jobs/__tests__/apiEnvironmentJobWiring.test.ts b/src/server/jobs/__tests__/apiEnvironmentJobWiring.test.ts index 73fa7c29..c1d4bb73 100644 --- a/src/server/jobs/__tests__/apiEnvironmentJobWiring.test.ts +++ b/src/server/jobs/__tests__/apiEnvironmentJobWiring.test.ts @@ -173,7 +173,7 @@ describe('bootstrapJobs', () => { const createWorker = registeredWorkers.find((worker) => worker.queue === QUEUE_NAMES.API_ENV_CREATE); const expiryWorker = registeredWorkers.find((worker) => worker.queue === QUEUE_NAMES.API_ENV_EXPIRY); const reconciliationWorker = registeredWorkers.find( - (worker) => worker.queue === QUEUE_NAMES.DEPLOYMENT_RECONCILIATION + (worker) => worker.queue === QUEUE_NAMES.SERVICE_RECONCILIATION ); expect(createWorker).toMatchObject({ handler: services.BuildService.processApiEnvironmentCreateQueue, @@ -181,6 +181,8 @@ describe('bootstrapJobs', () => { }); expect(expiryWorker?.handler).toBe(services.BuildService.processApiEnvironmentExpiryQueue); expect(reconciliationWorker?.handler).toBe(services.BuildService.processDeploymentReconciliationQueue); + // Old-queue jobs are drained by old workers only; leftovers are recovered by the sweep, never by a second model. + expect(registeredWorkers.some((worker) => worker.queue === QUEUE_NAMES.DEPLOYMENT_RECONCILIATION)).toBe(false); expect(workerOnHandlers[QUEUE_NAMES.API_ENV_CREATE]?.failed).toBe( services.BuildService.handleApiEnvironmentCreateFailure ); diff --git a/src/server/jobs/index.ts b/src/server/jobs/index.ts index 34e27997..41fc686e 100644 --- a/src/server/jobs/index.ts +++ b/src/server/jobs/index.ts @@ -166,7 +166,7 @@ export default function bootstrapJobs(services: IServices) { }); queueManager.registerWorker( - QUEUE_NAMES.DEPLOYMENT_RECONCILIATION, + QUEUE_NAMES.SERVICE_RECONCILIATION, services.BuildService.processDeploymentReconciliationQueue, { connection: redisClient.getConnection(), diff --git a/src/server/lib/deploymentManager/deploymentManager.ts b/src/server/lib/deploymentManager/deploymentManager.ts index da3d53da..5786b71b 100644 --- a/src/server/lib/deploymentManager/deploymentManager.ts +++ b/src/server/lib/deploymentManager/deploymentManager.ts @@ -18,7 +18,11 @@ import { Deploy } from 'server/models'; import { deployHelm } from '../helm'; import { shouldUseNativeHelm } from '../nativeHelm'; import { DeployStatus, DeployTypes, CLIDeployTypes } from 'shared/constants'; -import { createKubernetesApplyJob, monitorKubernetesJob } from '../kubernetesApply/applyManifest'; +import { + listUnfinishedDeployJobs, + createKubernetesApplyJob, + monitorKubernetesJob, +} from '../kubernetesApply/applyManifest'; import { nanoid, customAlphabet } from 'nanoid'; import DeployService from 'server/services/deploy'; import { getLogger, withLogContext } from 'server/lib/logger'; @@ -28,15 +32,20 @@ import { buildDeployJobName } from '../kubernetes/jobNames'; import GlobalConfigService from 'server/services/globalConfig'; import { getLogArchivalService } from 'server/services/logArchival'; import { DeploymentSupersededError } from 'server/lib/deploymentReconciliation/errors'; +import { AuthorityLockLostError } from 'server/lib/authorityLock'; import type { HelmSecretMutationGate } from 'server/lib/nativeHelm/helm'; export { DeploymentSupersededError } from 'server/lib/deploymentReconciliation/errors'; const generateJobId = customAlphabet('abcdefghijklmnopqrstuvwxyz0123456789', 6); +const EARLIER_DEPLOY_POLL_MS = 5_000; +// A native Helm Job lives at most its 30-minute deadline plus a 5-minute grace period. +const EARLIER_DEPLOY_WAIT_MS = 45 * 60_000; +const UNSTARTED_DEPLOY_MESSAGE = 'Deployment failed. Redeploy the service to try again.'; export type NativeMutationGateResult = { admitted: true; value: T } | { admitted: false }; -export type NativeMutationGate = (action: () => Promise) => Promise>; +export type NativeMutationGate = (deploy: Deploy, action: () => Promise) => Promise>; export interface DeploymentManagerOptions { /** Returns false once this deployment generation is no longer authoritative. */ @@ -48,8 +57,14 @@ export interface DeploymentManagerOptions { nativeMutationGate?: NativeMutationGate; /** Serializes only writers of one Deploy's ExternalSecret resources. */ nativeSecretMutationGate?: HelmSecretMutationGate; + /** Services that already failed before rollout; they stay in the plan so their dependents are blocked. */ + failedServices?: string[]; + /** Rows re-stamped to a newer generation are no longer this run's, even before the successor claims them. */ + expectedGeneration?: number; } +type ServiceOutcome = 'ready' | 'failed' | 'superseded'; + interface KubernetesDeploymentContext { deploy: Deploy; deployService: DeployService; @@ -59,16 +74,24 @@ interface KubernetesDeploymentContext { export class DeploymentManager { private deploys: Map = new Map(); + // Declared dependencies as configured; leveling consumes the lists on the deployables themselves. + private dependencies: Map = new Map(); private deploymentLevels: Map = new Map(); // Deploys never placed in a level: members of a dependency cycle, or dependents of one. private unresolvedDeploys: Deploy[] = []; private dependencyCycleDescription = ''; private readonly options: DeploymentManagerOptions; + // The first error a launch threw: nothing launches after it, and deploy() rethrows it once every launch has settled. + private abort?: { error: unknown }; constructor(deploys: Deploy[], options: DeploymentManagerOptions = {}) { this.options = options; deploys.forEach((deploy) => { this.deploys.set(deploy.deployable.name, deploy); + this.dependencies.set( + deploy.deployable.name, + (deploy.deployable.deploymentDependsOn ?? []).filter((dependency) => dependency !== deploy.deployable.name) + ); }); this.calculateDeploymentOrder(); @@ -165,13 +188,21 @@ export class DeploymentManager { }); } - public async deploy(): Promise { + public async deploy(): Promise<{ failed: Deploy[] }> { await this.assertCurrent(); const unresolved = new Set(this.unresolvedDeploys); + const superseded = new Set(); + // Keyed by name so a failed prerequisite that never reaches this manager (a Codefresh service) still blocks dependents. + const failed = new Map(); + for (const name of this.options.failedServices ?? []) { + failed.set(name, this.deploys.get(name) ?? null); + } for (const value of this.deploys.values()) { - if (!unresolved.has(value)) { - await this.patchDeployIfCurrent(value, { status: DeployStatus.QUEUED }); + if (unresolved.has(value) || failed.has(value.deployable.name)) continue; + if (!(await this.tryPatchDeploy(value, { status: DeployStatus.QUEUED }))) { + superseded.add(value.deployable.name); + getLogger().info(`Deploy: ${value.deployable.name} skipped reason=superseded`); } } @@ -181,91 +212,193 @@ export class DeploymentManager { const statusMessage = `Dependency cycle detected: ${this.dependencyCycleDescription}; deploy order cannot be resolved`; for (const deploy of this.unresolvedDeploys) { getLogger().error(`Deploy: ${deploy.deployable.name} failed — ${statusMessage}`); - await this.patchDeployIfCurrent(deploy, { status: DeployStatus.DEPLOY_FAILED, statusMessage }); + await this.tryPatchDeploy(deploy, { status: DeployStatus.DEPLOY_FAILED, statusMessage }); + failed.set(deploy.deployable.name, deploy); } } - for (let level = 0; level < this.deploymentLevels.size; level++) { - await this.assertCurrent(); + // Each service launches as soon as its own prerequisites are done, so a slow service never holds back unrelated work. + const levelOf = new Map(); + this.deploymentLevels.forEach((deploys, level) => deploys.forEach((d) => levelOf.set(d.deployable.name, level))); + const outcomes = new Map>(); + const schedule = (deploy: Deploy): Promise => { + const name = deploy.deployable.name; + let pending = outcomes.get(name); + if (!pending) { + pending = this.launchWhenPrerequisitesDone(deploy, levelOf.get(name) ?? 0, failed, superseded, schedule).catch( + (error) => { + if (!this.abort) this.abort = { error }; + throw error; + } + ); + outcomes.set(name, pending); + } + return pending; + }; + // Work still in flight settles before the error surfaces, so a queue retry never overlaps it. + await Promise.allSettled(Array.from(this.deploys.values()).map((deploy) => schedule(deploy))); + this.assertNotAborted(); - const deploysAtLevel = this.deploymentLevels.get(level); - if (deploysAtLevel) { - const helmDeploys = deploysAtLevel.filter((d) => this.shouldDeployWithHelm(d)); - const githubDeploys = deploysAtLevel.filter((d) => this.shouldDeployWithKubernetes(d)); + return { failed: Array.from(failed.values()).filter((deploy): deploy is Deploy => deploy != null) }; + } - const helmMethods = await Promise.all( - helmDeploys.map(async (deploy) => ({ deploy, native: await shouldUseNativeHelm(deploy) })) - ); - await this.assertCurrent(); + private async launchWhenPrerequisitesDone( + deploy: Deploy, + level: number, + failed: Map, + superseded: Set, + outcomeOf: (prerequisite: Deploy) => Promise + ): Promise { + const name = deploy.deployable.name; + if (superseded.has(name)) return 'superseded'; + if (failed.has(name)) return 'failed'; + + for (const dependency of this.dependencies.get(name) ?? []) { + const inPlan = this.deploys.get(dependency); + // A prerequisite outside this run, or one a newer run took, is never waited on: its running version serves. + let outcome: ServiceOutcome = 'ready'; + if (failed.has(dependency)) outcome = 'failed'; + else if (inPlan && !superseded.has(dependency)) outcome = await outcomeOf(inPlan); + this.assertNotAborted(); + if (outcome !== 'failed') continue; + if (!(await this.claimForLaunch(deploy, superseded))) return 'superseded'; + const statusMessage = `Not deployed: ${dependency} failed.`; + getLogger().warn(`Deploy: ${name} skipped — ${statusMessage}`); + await this.tryPatchDeploy(deploy, { status: DeployStatus.DEPLOY_FAILED, statusMessage }); + failed.set(name, deploy); + return 'failed'; + } - const nativeHelmDeploys = helmMethods.filter(({ native }) => native).map(({ deploy }) => deploy); - const codefreshHelmDeploys = helmMethods.filter(({ native }) => !native).map(({ deploy }) => deploy); + this.assertNotAborted(); + if (!(await this.claimForLaunch(deploy, superseded))) return 'superseded'; + await this.launchLevel(level, [deploy], failed, superseded); + if (failed.has(name)) return 'failed'; + if (superseded.has(name)) return 'superseded'; + return 'ready'; + } - const nativeHelmServices = nativeHelmDeploys.map((d) => d.deployable.name).join(','); - const codefreshHelmServices = codefreshHelmDeploys.map((d) => d.deployable.name).join(','); - const k8sServices = githubDeploys.map((d) => d.deployable.name).join(','); - getLogger().info( - `Deploy: level ${level} nativeHelm=[${nativeHelmServices}] codefreshHelm=[${codefreshHelmServices}] k8s=[${k8sServices}]` - ); + /** A newer intent may have taken this row while earlier levels ran; nothing launches for a row we no longer own. */ + private async claimForLaunch(deploy: Deploy, superseded: Set): Promise { + if (await this.tryPatchDeploy(deploy, { status: DeployStatus.QUEUED })) return true; + superseded.add(deploy.deployable.name); + getLogger().info(`Deploy: ${deploy.deployable.name} skipped reason=superseded`); + return false; + } - // Codefresh is intentionally not part of native mutation admission. It - // may continue in the provider while a newer generation starts. - const codefreshDeploy = codefreshHelmDeploys.length > 0 ? deployHelm(codefreshHelmDeploys) : Promise.resolve(); - // Attach a handler immediately because supersession may deliberately - // detach this provider wait before we await it below. - void codefreshDeploy.catch(() => undefined); - - let kubernetesDeployments: KubernetesDeploymentContext[] = []; - let nativeFailure: unknown; - try { - if (nativeHelmDeploys.length > 0 || githubDeploys.length > 0) { - const nativeResult = await this.runNativeMutation(async () => { - // Run same-generation siblings in parallel, but do not release the - // promotion gate until every admitted native mutation is terminal. - const settled = await Promise.allSettled([ - ...nativeHelmDeploys.map((deploy) => - deployHelm([deploy], { secretMutationGate: this.options.nativeSecretMutationGate }) - ), - ...githubDeploys.map((deploy) => this.applyManifests(deploy)), - ]); - const failure = settled.find((result): result is PromiseRejectedResult => result.status === 'rejected'); - const successfulKubernetesDeployments = settled - .slice(nativeHelmDeploys.length) - .filter( - (result): result is PromiseFulfilledResult => - result.status === 'fulfilled' - ) - .map((result) => result.value); - return { kubernetesDeployments: successfulKubernetesDeployments, failure: failure?.reason }; - }); - kubernetesDeployments = nativeResult.kubernetesDeployments; - nativeFailure = nativeResult.failure; - } + private assertNotAborted(): void { + if (this.abort) throw this.abort.error; + } - await this.assertCurrent(); + private async launchLevel( + level: number, + runnable: Deploy[], + failed: Map, + superseded: Set + ): Promise { + if (runnable.length === 0) return; - // Pod readiness is observational and must not hold the native mutation - // gate. Settle every successful sibling even when another native - // mutation failed so no Deploy is left indefinitely at DEPLOYING. - const readiness = await Promise.allSettled( - kubernetesDeployments.map((deployment) => this.waitForManifestReadiness(deployment)) - ); - if (nativeFailure) throw nativeFailure; - const readinessFailure = readiness.find( - (result): result is PromiseRejectedResult => result.status === 'rejected' - ); - if (readinessFailure) throw readinessFailure.reason; - await codefreshDeploy; - } catch (error) { - if (error instanceof DeploymentSupersededError) { - getLogger().info(`Deploy: level ${level} stopped reason=superseded`); - } + const helmDeploys = runnable.filter((d) => this.shouldDeployWithHelm(d)); + const githubDeploys = runnable.filter((d) => this.shouldDeployWithKubernetes(d)); + + const helmMethods = await Promise.all( + helmDeploys.map(async (deploy) => ({ deploy, native: await shouldUseNativeHelm(deploy) })) + ); + await this.assertCurrent(); + + const nativeHelmDeploys = helmMethods.filter(({ native }) => native).map(({ deploy }) => deploy); + const codefreshHelmDeploys = helmMethods.filter(({ native }) => !native).map(({ deploy }) => deploy); + + const nativeHelmServices = nativeHelmDeploys.map((d) => d.deployable.name).join(','); + const codefreshHelmServices = codefreshHelmDeploys.map((d) => d.deployable.name).join(','); + const k8sServices = githubDeploys.map((d) => d.deployable.name).join(','); + getLogger().info( + `Deploy: level ${level} nativeHelm=[${nativeHelmServices}] codefreshHelm=[${codefreshHelmServices}] k8s=[${k8sServices}]` + ); + + // Codefresh is intentionally not part of native mutation admission. It + // may continue in the provider while a newer generation starts. Each + // service is launched on its own so one failure is attributed to one row. + const codefreshOutcomes = Promise.allSettled( + codefreshHelmDeploys.map(async (deploy) => { + // Provider launches sit outside the promotion gate; ownership is re-checked right before handing off. + const owned = await this.tryPatchDeploy(deploy, { status: DeployStatus.QUEUED }).catch(async (error) => { + await this.recordOwnershipCheckFailure(deploy, error); throw error; - } + }); + if (!owned) throw new DeploymentSupersededError(); + return deployHelm([deploy], { + providerSubmissionGate: (candidate) => this.tryPatchDeploy(candidate, { status: DeployStatus.DEPLOYING }), + }); + }) + ); + + const nativeDeploys = [...nativeHelmDeploys, ...githubDeploys]; + const nativeOutcomes = await Promise.allSettled([ + ...nativeHelmDeploys.map((deploy) => + this.runNativeMutation(deploy, async () => { + await this.waitForEarlierDeployJobs(deploy); + return deployHelm([deploy], { secretMutationGate: this.options.nativeSecretMutationGate }); + }) + ), + ...githubDeploys.map((deploy) => + this.runNativeMutation(deploy, async () => { + await this.waitForEarlierDeployJobs(deploy); + return this.applyManifests(deploy); + }) + ), + ]); + await this.assertCurrent(); + + // A lost promotion lease is not a service failure: the apply may have landed, so the run must retry. + const lostLease = nativeOutcomes.find( + (outcome): outcome is PromiseRejectedResult => + outcome.status === 'rejected' && outcome.reason instanceof AuthorityLockLostError + ); + if (lostLease) throw lostLease.reason; + + const kubernetesDeployments: KubernetesDeploymentContext[] = []; + nativeOutcomes.forEach((outcome, index) => { + const deploy = nativeDeploys[index]; + if (outcome.status === 'fulfilled') { + if (index >= nativeHelmDeploys.length) kubernetesDeployments.push(outcome.value as KubernetesDeploymentContext); + return; + } + if (outcome.reason instanceof DeploymentSupersededError) { + superseded.add(deploy.deployable.name); + getLogger().info(`Deploy: ${deploy.deployable.name} stopped reason=superseded`); + return; } + failed.set(deploy.deployable.name, deploy); + }); - await this.assertCurrent(); - } + // Pod readiness is observational and must not hold a promotion gate. + const readiness = await Promise.allSettled( + kubernetesDeployments.map((deployment) => this.waitForManifestReadiness(deployment)) + ); + readiness.forEach((outcome, index) => { + if (outcome.status === 'fulfilled') return; + const { deploy } = kubernetesDeployments[index]; + if (outcome.reason instanceof DeploymentSupersededError) { + superseded.add(deploy.deployable.name); + getLogger().info(`Deploy: ${deploy.deployable.name} stopped reason=superseded`); + return; + } + failed.set(deploy.deployable.name, deploy); + }); + + (await codefreshOutcomes).forEach((outcome, index) => { + if (outcome.status === 'fulfilled') return; + const deploy = codefreshHelmDeploys[index]; + if (outcome.reason instanceof DeploymentSupersededError) { + superseded.add(deploy.deployable.name); + getLogger().info(`Deploy: ${deploy.deployable.name} codefresh stopped reason=superseded`); + return; + } + getLogger().error({ error: outcome.reason }, `Deploy: ${deploy.deployable.name} codefresh helm failed`); + failed.set(deploy.deployable.name, deploy); + }); + + await this.assertCurrent(); } private async assertCurrent(): Promise { @@ -274,24 +407,90 @@ export class DeploymentManager { } } - private async runNativeMutation(action: () => Promise): Promise { + private async runNativeMutation(deploy: Deploy, action: () => Promise): Promise { if (!this.options.nativeMutationGate) return action(); - const result = await this.options.nativeMutationGate(action); + let actionFailed = false; + const result = await this.options + .nativeMutationGate(deploy, () => + action().catch((error) => { + actionFailed = true; + throw error; + }) + ) + .catch(async (error) => { + if (!actionFailed) await this.recordOwnershipCheckFailure(deploy, error); + throw error; + }); if (!result.admitted) throw new DeploymentSupersededError(); return result.value; } - private async patchDeployIfCurrent(deploy: Deploy, patch: Partial): Promise { + /** Launch paths record their own failures; a failed ownership check around one must not leave the row looking deployed. */ + private async recordOwnershipCheckFailure(deploy: Deploy, error: unknown): Promise { + if (error instanceof DeploymentSupersededError || error instanceof AuthorityLockLostError) return; + getLogger({ error }).warn(`Deploy: ${deploy.deployable.name} ownership check failed`); + await new DeployService().recordDeployFailure(deploy, deploy.runUUID, { + status: DeployStatus.DEPLOY_FAILED, + fallbackMessage: UNSTARTED_DEPLOY_MESSAGE, + }); + } + + /** A deploy Job outlives the worker that started it; killing it cannot prove it stopped, so the next one waits. */ + private async waitForEarlierDeployJobs(deploy: Deploy): Promise { + try { + if (!deploy.build) await deploy.$fetchGraph('build'); + const deadline = Date.now() + EARLIER_DEPLOY_WAIT_MS; + let logged = false; + for (;;) { + const unfinished = await listUnfinishedDeployJobs(deploy, deploy.build.namespace); + if (unfinished.length === 0) return; + if (Date.now() >= deadline) { + throw new Error( + 'The previous deploy of this service is still running after 45 minutes. Redeploy the service after it stops.' + ); + } + if (!logged) { + getLogger().info(`Deploy: ${deploy.deployable.name} waiting for earlier jobs=${unfinished.join(',')}`); + logged = true; + } + const statusMessage = 'Waiting for the previous deploy to finish'; + if (!(await this.tryPatchDeploy(deploy, { status: DeployStatus.DEPLOYING, statusMessage }))) { + throw new DeploymentSupersededError(); + } + await new Promise((resolve) => setTimeout(resolve, EARLIER_DEPLOY_POLL_MS)); + } + } catch (error) { + if (!(error instanceof DeploymentSupersededError)) { + await new DeployService().recordDeployFailure(deploy, deploy.runUUID, { + status: DeployStatus.DEPLOY_FAILED, + error, + fallbackMessage: `Deployment failed for ${deploy.uuid}. Check deploy logs in Console > Deploy tab for details.`, + }); + } + throw error; + } + } + + /** False means another run owns the row now; the caller skips it instead of failing its siblings. */ + private async tryPatchDeploy(deploy: Deploy, patch: Partial): Promise { if (deploy.id == null || !deploy.runUUID) { getLogger().info( `Deploy: status patch skipped reason=missing_run_identity deployUuid=${deploy.uuid || 'unknown'}` ); - return; + return true; } - const patched = await deploy.$query().patch(patch).where({ id: deploy.id, runUUID: deploy.runUUID }); - if (!patched) throw new DeploymentSupersededError(); + const generation = this.options.expectedGeneration; + const patched = await deploy + .$query() + .patch(patch) + .where({ + id: deploy.id, + runUUID: deploy.runUUID, + ...(generation != null ? { desiredGeneration: generation } : {}), + }); + return patched > 0; } private shouldDeployWithHelm(deploy: Deploy): boolean { diff --git a/src/server/lib/deploymentReconciliation/__tests__/authority.test.ts b/src/server/lib/deploymentReconciliation/__tests__/authority.test.ts new file mode 100644 index 00000000..243411a6 --- /dev/null +++ b/src/server/lib/deploymentReconciliation/__tests__/authority.test.ts @@ -0,0 +1,177 @@ +/** + * Copyright 2026 GoodRx, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { BuildStatus, PullRequestStatus } from 'shared/constants'; +import { + environmentBlockReason, + isDeployAuthorityCurrent, + isDeployOwnedByRun, + isEnvironmentUnblocked, + isIntentRunCurrent, + loadEnvironmentAuthority, +} from '../authority'; + +function buildQuery(row: any) { + const query: any = { + findById: jest.fn(() => query), + select: jest.fn().mockResolvedValue(row), + }; + return query; +} + +function deployQuery(row: any) { + const query: any = { + findOne: jest.fn(() => query), + select: jest.fn().mockResolvedValue(row), + }; + return query; +} + +function models(build: any, deploy?: any) { + const buildRead = buildQuery(build); + const deployRead = deployQuery(deploy); + return { + models: { Build: { query: jest.fn(() => buildRead) }, Deploy: { query: jest.fn(() => deployRead) } } as any, + buildRead, + deployRead, + }; +} + +const openPullRequest = { status: PullRequestStatus.OPEN, deployOnUpdate: true }; + +describe('environmentBlockReason', () => { + it('blocks a missing or deleted build', () => { + expect(environmentBlockReason(null)).toBe('build_missing'); + expect(environmentBlockReason({ deletedAt: new Date() } as any)).toBe('build_deleted'); + }); + + it('follows the pull request for a PR environment', () => { + expect(environmentBlockReason({ pullRequestId: 1, status: BuildStatus.TEARING_DOWN } as any)).toBe('tearing_down'); + expect(environmentBlockReason({ pullRequestId: 1, status: BuildStatus.DEPLOYED } as any)).toBe( + 'pull_request_missing' + ); + expect(environmentBlockReason({ pullRequestId: 1, pullRequest: { status: PullRequestStatus.CLOSED } } as any)).toBe( + 'pull_request_closed' + ); + expect( + environmentBlockReason({ + pullRequestId: 1, + pullRequest: { status: PullRequestStatus.OPEN, deployOnUpdate: false }, + } as any) + ).toBe('deploy_disabled'); + expect(environmentBlockReason({ pullRequestId: 1, pullRequest: openPullRequest } as any)).toBeNull(); + expect( + environmentBlockReason({ pullRequestId: 1, pullRequest: openPullRequest, status: BuildStatus.TORN_DOWN } as any) + ).toBeNull(); + }); + + it('follows the kill switch and teardown states for a PR-less environment', () => { + expect(environmentBlockReason({ status: BuildStatus.TEARING_DOWN, deployEnabled: true } as any)).toBe('torn_down'); + expect(environmentBlockReason({ status: BuildStatus.TORN_DOWN, deployEnabled: true } as any)).toBe('torn_down'); + expect(environmentBlockReason({ status: BuildStatus.DEPLOYED, deployEnabled: false } as any)).toBe( + 'deploy_disabled' + ); + expect(environmentBlockReason({ status: BuildStatus.DEPLOYED, deployEnabled: true } as any)).toBeNull(); + }); +}); + +describe('loadEnvironmentAuthority and isEnvironmentUnblocked', () => { + it('loads the pull request graph only for PR environments', async () => { + const fetchGraph = jest.fn().mockResolvedValue(undefined); + const build = { id: 4, pullRequestId: 9, $fetchGraph: fetchGraph }; + const prLess = { id: 5, pullRequestId: null, deployEnabled: true, $fetchGraph: jest.fn() }; + + await expect(loadEnvironmentAuthority(models(build).models, 4)).resolves.toBe(build); + expect(fetchGraph).toHaveBeenCalledWith('pullRequest'); + await expect(loadEnvironmentAuthority(models(prLess).models, 5)).resolves.toBe(prLess); + expect(prLess.$fetchGraph).not.toHaveBeenCalled(); + await expect(loadEnvironmentAuthority(models(undefined).models, 6)).resolves.toBeNull(); + }); + + it('reports whether the environment may run deployment work', async () => { + await expect( + isEnvironmentUnblocked(models({ id: 4, status: BuildStatus.DEPLOYED, deployEnabled: true }).models, 4) + ).resolves.toBe(true); + await expect( + isEnvironmentUnblocked(models({ id: 4, status: BuildStatus.TORN_DOWN, deployEnabled: true }).models, 4) + ).resolves.toBe(false); + await expect(isEnvironmentUnblocked(models(undefined).models, 4)).resolves.toBe(false); + }); +}); + +describe('isDeployOwnedByRun', () => { + it('requires the row to carry the run token and, when fenced, the generation', async () => { + const { models: m, deployRead } = models(undefined, { id: 7, observedGeneration: 1 }); + + await expect(isDeployOwnedByRun(m, 7, 'run-a', 3)).resolves.toBe(true); + expect(deployRead.findOne).toHaveBeenCalledWith({ id: 7, runUUID: 'run-a', desiredGeneration: 3 }); + expect(deployRead.select).toHaveBeenCalledWith('id', 'observedGeneration'); + + await expect(isDeployOwnedByRun(m, 7, 'run-a')).resolves.toBe(true); + expect(deployRead.findOne).toHaveBeenLastCalledWith({ id: 7, runUUID: 'run-a' }); + }); + + it('treats an observed row as finished work and a missing row as not owned', async () => { + await expect( + isDeployOwnedByRun(models(undefined, { id: 7, observedGeneration: 3 }).models, 7, 'run-a', 3) + ).resolves.toBe(false); + await expect( + isDeployOwnedByRun(models(undefined, { id: 7, observedGeneration: '3' }).models, 7, 'run-a', 3) + ).resolves.toBe(false); + await expect( + isDeployOwnedByRun(models(undefined, { id: 7, observedGeneration: 3 }).models, 7, 'run-a') + ).resolves.toBe(true); + await expect(isDeployOwnedByRun(models(undefined, { id: 7 }).models, 7, 'run-a', 3)).resolves.toBe(true); + await expect(isDeployOwnedByRun(models(undefined, undefined).models, 7, 'run-a', 3)).resolves.toBe(false); + }); +}); + +describe('isDeployAuthorityCurrent', () => { + const deploy = { id: 7, buildId: 4 } as any; + + it('is false when the run no longer owns the row', async () => { + await expect(isDeployAuthorityCurrent(models(undefined, undefined).models, deploy, 'run-a', 3)).resolves.toBe( + false + ); + }); + + it('checks the environment only for fenced runs', async () => { + const blocked = models( + { id: 4, status: BuildStatus.TORN_DOWN, deployEnabled: true }, + { id: 7, observedGeneration: 1 } + ); + await expect(isDeployAuthorityCurrent(blocked.models, deploy, 'run-a')).resolves.toBe(true); + expect(blocked.models.Build.query).not.toHaveBeenCalled(); + await expect(isDeployAuthorityCurrent(blocked.models, deploy, 'run-a', 3)).resolves.toBe(false); + + const open = models({ id: 4, status: BuildStatus.DEPLOYED, deployEnabled: true }, { id: 7, observedGeneration: 1 }); + await expect(isDeployAuthorityCurrent(open.models, deploy, 'run-a', 3)).resolves.toBe(true); + }); +}); + +describe('isIntentRunCurrent', () => { + it('needs an owned row at the generation and an unblocked environment', async () => { + const none = models({ id: 4, status: BuildStatus.DEPLOYED, deployEnabled: true }, undefined); + await expect(isIntentRunCurrent(none.models, 4, 'run-a', 3)).resolves.toBe(false); + expect(none.deployRead.findOne).toHaveBeenCalledWith({ buildId: 4, runUUID: 'run-a', desiredGeneration: 3 }); + + const owned = models({ id: 4, status: BuildStatus.DEPLOYED, deployEnabled: true }, { id: 7 }); + await expect(isIntentRunCurrent(owned.models, 4, 'run-a', 3)).resolves.toBe(true); + + const blocked = models({ id: 4, status: BuildStatus.TORN_DOWN, deployEnabled: true }, { id: 7 }); + await expect(isIntentRunCurrent(blocked.models, 4, 'run-a', 3)).resolves.toBe(false); + }); +}); diff --git a/src/server/lib/deploymentReconciliation/__tests__/mailbox.test.ts b/src/server/lib/deploymentReconciliation/__tests__/mailbox.test.ts index 14af85fa..26fbd88f 100644 --- a/src/server/lib/deploymentReconciliation/__tests__/mailbox.test.ts +++ b/src/server/lib/deploymentReconciliation/__tests__/mailbox.test.ts @@ -22,12 +22,29 @@ jest.mock('server/models/Build', () => ({ }, })); +const deployStamp: any = { + patch: jest.fn(() => deployStamp), + where: jest.fn(() => deployStamp), + then: (resolve: (value: number) => void, reject: (reason: unknown) => void) => + Promise.resolve(2).then(resolve, reject), +}; + +jest.mock('server/models/Deploy', () => ({ + __esModule: true, + default: { + query: jest.fn(() => deployStamp), + }, +})); + import Build from 'server/models/Build'; +import Deploy from 'server/models/Deploy'; +import { DeployStatus } from 'shared/constants'; import { acceptDeploymentIntent, AcceptedDeploymentRefs, + deploymentIntentDeployFilter, deploymentIntentScopeKey, - dirtyDeploymentIntents, + recordIntentConfigFailures, } from '../mailbox'; const TRX = { transaction: true } as any; @@ -67,6 +84,16 @@ describe('deployment intent scope keys', () => { sha: 'abc', }) ).toBe('source:123:Feature%2FA%20B'); + expect( + deploymentIntentScopeKey({ + type: 'source', + requestId: 'request-config', + target: 'all', + githubRepositoryId: 123, + branch: 'Feature/A B', + sha: 'abc', + }) + ).toBe('source:123:Feature%2FA%20B:all'); expect( deploymentIntentScopeKey({ type: 'repository', requestId: 'request-repository', githubRepositoryId: 123 }) ).toBe('repository:123'); @@ -141,6 +168,20 @@ describe('acceptDeploymentIntent', () => { expect(read.findById).toHaveBeenCalledWith(42); expect(read.whereNull).toHaveBeenCalledWith('deletedAt'); expect(read.forUpdate).toHaveBeenCalledTimes(1); + expect(Deploy.query).toHaveBeenCalledWith(TRX); + expect(deployStamp.patch).toHaveBeenCalledWith({ + desiredGeneration: 3, + runUUID: null, + status: DeployStatus.QUEUED, + statusMessage: null, + }); + expect(deployStamp.where).toHaveBeenCalledWith({ + buildId: 42, + active: true, + githubRepositoryId: 123, + branchName: 'main', + }); + expect(deployStamp.where).toHaveBeenCalledWith('desiredGeneration', '<', 3); expect(write.patch).toHaveBeenCalledWith({ desiredGeneration: 3, acceptedRefs: { @@ -321,6 +362,64 @@ describe('acceptDeploymentIntent', () => { }); }); + it('marks older environment-wide entries observed when a newer environment-wide intent is accepted', async () => { + const existing: AcceptedDeploymentRefs = { + 'source:900:static:all': { + type: 'source', + requestId: 'request-5', + target: 'all', + githubRepositoryId: 900, + branch: 'static', + sha: 'config-sha', + gen: 5, + }, + 'source:100:main': { + type: 'source', + requestId: 'request-6', + target: 'repository', + githubRepositoryId: 100, + branch: 'main', + sha: 'service-sha', + gen: 6, + }, + }; + const read = readQuery({ desiredGeneration: 6, acceptedRefs: existing }); + const write = writeQuery(); + (Build.query as jest.Mock).mockReturnValueOnce(read).mockReturnValueOnce(write); + + await expect(acceptDeploymentIntent(42, { type: 'all', requestId: 'request-7' })).resolves.toEqual({ + accepted: true, + generation: 7, + scopeKey: 'all', + }); + + expect(write.patch).toHaveBeenCalledWith({ + desiredGeneration: 7, + acceptedRefs: { + 'source:900:static:all': { ...existing['source:900:static:all'], observedGen: 5 }, + 'source:100:main': existing['source:100:main'], + all: { type: 'all', requestId: 'request-7', gen: 7 }, + }, + }); + }); + + it('leaves an older environment-wide entry pending when a service-scoped intent is accepted', async () => { + const existing: AcceptedDeploymentRefs = { all: { type: 'all', requestId: 'request-5', gen: 5 } }; + const read = readQuery({ desiredGeneration: 5, acceptedRefs: existing }); + const write = writeQuery(); + (Build.query as jest.Mock).mockReturnValueOnce(read).mockReturnValueOnce(write); + + await acceptDeploymentIntent(42, { type: 'repository', requestId: 'request-6', githubRepositoryId: 100 }); + + expect(write.patch).toHaveBeenCalledWith({ + desiredGeneration: 6, + acceptedRefs: { + ...existing, + 'repository:100': { type: 'repository', requestId: 'request-6', githubRepositoryId: 100, gen: 6 }, + }, + }); + }); + it('returns null without writing when the Build does not exist', async () => { (Build.query as jest.Mock).mockReturnValueOnce(readQuery(undefined)); @@ -365,52 +464,107 @@ describe('acceptDeploymentIntent', () => { }); }); -describe('dirtyDeploymentIntents', () => { - it('extracts only unobserved latest intents in generation order', () => { - const acceptedRefs: AcceptedDeploymentRefs = { - all: { type: 'all', requestId: 'request-5', gen: 5 }, +describe('deploymentIntentDeployFilter', () => { + it('selects one repository and branch for a tracked push', () => { + expect( + deploymentIntentDeployFilter({ + type: 'source', + requestId: 'r', + target: 'repository', + githubRepositoryId: 123, + branch: 'main', + sha: 'c', + }) + ).toEqual({ githubRepositoryId: 123, branchName: 'main' }); + }); + + it('selects every active row for a config-repo push and for a full redeploy', () => { + expect( + deploymentIntentDeployFilter({ + type: 'source', + requestId: 'r', + target: 'all', + githubRepositoryId: 900, + branch: 'static', + sha: 'z', + }) + ).toEqual({}); + expect(deploymentIntentDeployFilter({ type: 'all', requestId: 'r' })).toEqual({}); + }); + + it('selects every branch of one repository for a repository redeploy', () => { + expect(deploymentIntentDeployFilter({ type: 'repository', requestId: 'r', githubRepositoryId: 9 })).toEqual({ + githubRepositoryId: 9, + }); + }); +}); + +describe('acceptDeploymentIntent row stamping', () => { + it('does not touch Deploy rows when a redelivery is deduplicated', async () => { + const existing: AcceptedDeploymentRefs = { 'source:123:main': { type: 'source', - requestId: 'request-2', + requestId: 'request-1', target: 'repository', githubRepositoryId: 123, branch: 'main', - sha: 'old', - gen: 2, + sha: 'commit-a', + gen: 1, }, - 'repository:456': { type: 'repository', requestId: 'request-3', githubRepositoryId: 456, gen: 3 }, }; + (Build.query as jest.Mock).mockReturnValueOnce(readQuery({ desiredGeneration: '1', acceptedRefs: existing })); + + await expect( + acceptDeploymentIntent(42, { + type: 'source', + requestId: 'request-redelivery', + target: 'repository', + githubRepositoryId: 123, + branch: 'main', + sha: 'commit-a', + }) + ).resolves.toEqual({ accepted: false, generation: 1, scopeKey: 'source:123:main' }); - expect(dirtyDeploymentIntents(acceptedRefs, 2)).toEqual([ - { scopeKey: 'repository:456', intent: acceptedRefs['repository:456'] }, - { scopeKey: 'all', intent: acceptedRefs.all }, - ]); + expect(Deploy.query).not.toHaveBeenCalled(); }); +}); + +describe('recordIntentConfigFailures', () => { + it('stores the failures on the entry accepted at that generation', async () => { + const existing: AcceptedDeploymentRefs = { + all: { type: 'all', requestId: 'request-3', gen: 3 }, + 'repository:9': { type: 'repository', requestId: 'request-2', githubRepositoryId: 9, gen: 2 }, + }; + const read = readQuery({ id: 42, acceptedRefs: existing }); + const write = writeQuery(); + (Build.query as jest.Mock).mockReturnValueOnce(read).mockReturnValueOnce(write); - it('treats an absent mailbox as having no dirty intents', () => { - expect(dirtyDeploymentIntents(null, 0)).toEqual([]); - expect(dirtyDeploymentIntents(undefined, 0)).toEqual([]); + await expect(recordIntentConfigFailures(42, 3, { billing: 'billing lifecycle.yaml is invalid' })).resolves.toBe( + true + ); + + expect(write.patch).toHaveBeenCalledWith({ + acceptedRefs: { + ...existing, + all: { ...existing.all, configFailures: { billing: 'billing lifecycle.yaml is invalid' } }, + }, + }); }); - it('ignores malformed persisted entries', () => { - const validIntent = { type: 'all' as const, requestId: 'request-2', gen: 2 }; - const acceptedRefs = { - malformed: null, - all: validIntent, - } as unknown as AcceptedDeploymentRefs; + it('writes nothing when no entry carries that generation', async () => { + const read = readQuery({ id: 42, acceptedRefs: { all: { type: 'all', requestId: 'request-3', gen: 3 } } }); + (Build.query as jest.Mock).mockReturnValueOnce(read); - expect(dirtyDeploymentIntents(acceptedRefs, 0)).toEqual([{ scopeKey: 'all', intent: validIntent }]); + await expect(recordIntentConfigFailures(42, 9, {})).resolves.toBe(false); + expect(Build.query).toHaveBeenCalledTimes(1); }); - it('uses the scope key as a deterministic tie-breaker for equal generations', () => { - const acceptedRefs: AcceptedDeploymentRefs = { - 'repository:456': { type: 'repository', requestId: 'request-repository', githubRepositoryId: 456, gen: 3 }, - all: { type: 'all', requestId: 'request-all', gen: 3 }, - }; + it('returns false for a missing build or a malformed mailbox', async () => { + (Build.query as jest.Mock).mockReturnValueOnce(readQuery(undefined)); + await expect(recordIntentConfigFailures(42, 3, {})).resolves.toBe(false); - expect(dirtyDeploymentIntents(acceptedRefs, 0)).toEqual([ - { scopeKey: 'all', intent: acceptedRefs.all }, - { scopeKey: 'repository:456', intent: acceptedRefs['repository:456'] }, - ]); + (Build.query as jest.Mock).mockReturnValueOnce(readQuery({ id: 42, acceptedRefs: 'broken' })); + await expect(recordIntentConfigFailures(42, 3, {})).resolves.toBe(false); + expect(Build.query).toHaveBeenCalledTimes(2); }); }); diff --git a/src/server/lib/deploymentReconciliation/authority.ts b/src/server/lib/deploymentReconciliation/authority.ts new file mode 100644 index 00000000..a5f30c16 --- /dev/null +++ b/src/server/lib/deploymentReconciliation/authority.ts @@ -0,0 +1,111 @@ +/** + * Copyright 2026 GoodRx, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { isDeployEnabled } from 'server/lib/buildSource'; +import type Build from 'server/models/Build'; +import type Deploy from 'server/models/Deploy'; +import { BuildStatus, PullRequestStatus } from 'shared/constants'; + +export type EnvironmentBlockReason = + | 'build_missing' + | 'build_deleted' + | 'tearing_down' + | 'pull_request_missing' + | 'pull_request_closed' + | 'deploy_disabled' + | 'torn_down'; + +export interface AuthorityModels { + Build: typeof Build; + Deploy: typeof Deploy; +} + +/** Why no deployment work may run for this environment right now, or null. */ +export function environmentBlockReason(build: Build | null | undefined): EnvironmentBlockReason | null { + if (!build) return 'build_missing'; + if (build.deletedAt != null) return 'build_deleted'; + + if (build.pullRequest || build.pullRequestId != null) { + if (build.status === BuildStatus.TEARING_DOWN) return 'tearing_down'; + if (!build.pullRequest) return 'pull_request_missing'; + if (build.pullRequest.status !== PullRequestStatus.OPEN) return 'pull_request_closed'; + if (!isDeployEnabled(build)) return 'deploy_disabled'; + // Re-adding the deploy label to an open PR may reclaim the row after teardown completes. + return null; + } + + if (build.status === BuildStatus.TEARING_DOWN || build.status === BuildStatus.TORN_DOWN) return 'torn_down'; + return build.deployEnabled === true ? null : 'deploy_disabled'; +} + +export async function loadEnvironmentAuthority(models: AuthorityModels, buildId: number): Promise { + const build = await models.Build.query() + .findById(buildId) + .select('id', 'runUUID', 'status', 'deployEnabled', 'deletedAt', 'pullRequestId', 'desiredGeneration'); + if (build?.pullRequestId != null) await build.$fetchGraph('pullRequest'); + return build ?? null; +} + +export async function isEnvironmentUnblocked(models: AuthorityModels, buildId: number): Promise { + return environmentBlockReason(await loadEnvironmentAuthority(models, buildId)) == null; +} + +/** The row is still owned by this run, still desired at this generation, and not yet marked finished. */ +export async function isDeployOwnedByRun( + models: AuthorityModels, + deployId: number, + runUUID: string, + expectedGeneration?: number +): Promise { + const owned = await models.Deploy.query() + .findOne({ + id: deployId, + runUUID, + ...(expectedGeneration != null ? { desiredGeneration: expectedGeneration } : {}), + }) + .select('id', 'observedGeneration'); + if (!owned) return false; + if (expectedGeneration == null || owned.observedGeneration == null) return true; + return Number(owned.observedGeneration) < expectedGeneration; +} + +/** + * The one authority predicate for service work. Nothing about other services + * appears in it: a newer intent that selects this row changes the row itself. + */ +export async function isDeployAuthorityCurrent( + models: AuthorityModels, + deploy: Pick, + runUUID: string, + expectedGeneration?: number +): Promise { + if (!(await isDeployOwnedByRun(models, deploy.id, runUUID, expectedGeneration))) return false; + return expectedGeneration == null || isEnvironmentUnblocked(models, deploy.buildId); +} + +/** An intent stays current while it still owns at least one row at its generation. */ +export async function isIntentRunCurrent( + models: AuthorityModels, + buildId: number, + runUUID: string, + expectedGeneration: number +): Promise { + const owned = await models.Deploy.query() + .findOne({ buildId, runUUID, desiredGeneration: expectedGeneration }) + .select('id'); + if (!owned) return false; + return isEnvironmentUnblocked(models, buildId); +} diff --git a/src/server/lib/deploymentReconciliation/mailbox.ts b/src/server/lib/deploymentReconciliation/mailbox.ts index e608ea7e..6fa82c05 100644 --- a/src/server/lib/deploymentReconciliation/mailbox.ts +++ b/src/server/lib/deploymentReconciliation/mailbox.ts @@ -14,8 +14,11 @@ * limitations under the License. */ +import type { PartialModelObject, Transaction } from 'objection'; import type { JobDataWithContext } from 'server/lib/logger'; import Build from 'server/models/Build'; +import Deploy from 'server/models/Deploy'; +import { DeployStatus } from 'shared/constants'; export interface SourcePushIntent { type: 'source'; @@ -41,7 +44,12 @@ export interface EnvironmentRedeployIntent { } export type DeploymentIntent = SourcePushIntent | RepositoryRedeployIntent | EnvironmentRedeployIntent; -export type AcceptedDeploymentIntent = DeploymentIntent & { gen: number }; +export type AcceptedDeploymentIntent = DeploymentIntent & { + gen: number; + observedGen?: number; + /** Services whose YAML failed in this intent's import and had no row to carry it; absent until the import ran. */ + configFailures?: Record; +}; export type AcceptedDeploymentRefs = Record; export interface DeploymentReconciliationJobData extends JobDataWithContext { @@ -50,11 +58,6 @@ export interface DeploymentReconciliationJobData extends JobDataWithContext { generation: number; } -export interface DirtyDeploymentIntent { - scopeKey: string; - intent: AcceptedDeploymentIntent; -} - export interface AcceptDeploymentIntentResult { accepted: boolean; generation: number; @@ -65,7 +68,10 @@ export interface AcceptDeploymentIntentResult { export function deploymentIntentScopeKey(intent: DeploymentIntent): string { switch (intent.type) { case 'source': - return `source:${intent.githubRepositoryId}:${encodeURIComponent(intent.branch)}`; + // A config-repo push selects every row; it must never replace the repo-scoped entry of the same source. + return `source:${intent.githubRepositoryId}:${encodeURIComponent(intent.branch)}${ + intent.target === 'all' ? ':all' : '' + }`; case 'repository': return `repository:${intent.githubRepositoryId}`; case 'all': @@ -73,17 +79,40 @@ export function deploymentIntentScopeKey(intent: DeploymentIntent): string { } } -/** Returns the latest intent for every scope that has not yet been observed. */ -export function dirtyDeploymentIntents( - acceptedRefs: AcceptedDeploymentRefs | null | undefined, - observedGeneration: number -): DirtyDeploymentIntent[] { - if (!acceptedRefs || typeof acceptedRefs !== 'object' || Array.isArray(acceptedRefs)) return []; - - return Object.entries(acceptedRefs) - .filter(([, intent]) => Number.isSafeInteger(intent?.gen) && intent.gen > observedGeneration) - .map(([scopeKey, intent]) => ({ scopeKey, intent })) - .sort((left, right) => left.intent.gen - right.intent.gen || left.scopeKey.localeCompare(right.scopeKey)); +export function deploymentIntentSelectsAllDeploys(intent: DeploymentIntent): boolean { + return intent.type === 'all' || (intent.type === 'source' && intent.target === 'all'); +} + +/** The Deploy rows an intent selects; an empty filter means every active row. */ +export function deploymentIntentDeployFilter(intent: DeploymentIntent): { + githubRepositoryId?: number; + branchName?: string; +} { + if (intent.type === 'all') return {}; + if (intent.type === 'source') { + return intent.target === 'all' ? {} : { githubRepositoryId: intent.githubRepositoryId, branchName: intent.branch }; + } + return { githubRepositoryId: intent.githubRepositoryId }; +} + +/** A stamped row is unowned and queued: the older run can no longer write to it, and its old status is not current. */ +export function deployStampForGeneration(generation: number): PartialModelObject { + const stamp = { desiredGeneration: generation, runUUID: null, status: DeployStatus.QUEUED, statusMessage: null }; + // The token column is nullable; the model types it as a string. + return stamp as unknown as PartialModelObject; +} + +/** Raises the selected rows to `generation`; rows already desired at a newer generation are left alone. */ +export async function stampDeploysForIntent( + trx: Transaction | undefined, + buildId: number, + intent: DeploymentIntent, + generation: number +): Promise { + return Deploy.query(trx) + .patch(deployStampForGeneration(generation)) + .where({ buildId, active: true, ...deploymentIntentDeployFilter(intent) }) + .where('desiredGeneration', '<', generation); } /** @@ -147,12 +176,51 @@ export async function acceptDeploymentIntent( ...acceptedRefs, [scopeKey]: { ...intent, gen: generation }, }; + // Every row moves to this generation, so an older environment-wide entry has nothing left to execute. + if (deploymentIntentSelectsAllDeploys(intent)) { + for (const [key, accepted] of Object.entries(acceptedRefs)) { + if (key === scopeKey || !deploymentIntentSelectsAllDeploys(accepted) || accepted.observedGen === accepted.gen) { + continue; + } + nextRefs[key] = { ...accepted, observedGen: accepted.gen }; + } + } await Build.query(trx).findById(buildId).patch({ desiredGeneration: generation, acceptedRefs: nextRefs, }); + await stampDeploysForIntent(trx, buildId, intent, generation); return { accepted: true, generation, scopeKey }; }); } + +/** Records which row-less services the import at `generation` could not read; an empty map means the import was clean. */ +export async function recordIntentConfigFailures( + buildId: number, + generation: number, + configFailures: Record +): Promise { + return Build.transact(async (trx) => { + const build = await Build.query(trx) + .select('id', 'acceptedRefs') + .findById(buildId) + .whereNull('deletedAt') + .forUpdate(); + if (!build) return false; + + const acceptedRefs = + build.acceptedRefs && typeof build.acceptedRefs === 'object' && !Array.isArray(build.acceptedRefs) + ? build.acceptedRefs + : {}; + const entry = Object.entries(acceptedRefs).find(([, intent]) => intent?.gen === generation); + if (!entry) return false; + const [scopeKey, intent] = entry; + + await Build.query(trx) + .findById(buildId) + .patch({ acceptedRefs: { ...acceptedRefs, [scopeKey]: { ...intent, configFailures } } }); + return true; + }); +} diff --git a/src/server/lib/helm/__tests__/helm.behavior.test.ts b/src/server/lib/helm/__tests__/helm.behavior.test.ts index 242114ff..0da22ef9 100644 --- a/src/server/lib/helm/__tests__/helm.behavior.test.ts +++ b/src/server/lib/helm/__tests__/helm.behavior.test.ts @@ -614,21 +614,26 @@ describe('Codefresh Helm YAML generation', () => { const deploy = makePublicDeploy(); deploy.deployable.helm.chart.valueFiles = ['helm/preview.yaml']; - const command = await generateCodefreshRunCommand(deploy); + const { command, configPath } = await generateCodefreshRunCommand(deploy); expect(deploy.$fetchGraph).toHaveBeenCalledWith('build'); expect(mockMkdir).toHaveBeenCalledWith('/tmp/lifecycle/codefresh', { recursive: true }); - expect(mockWriteFile).toHaveBeenCalledWith( - '/tmp/lifecycle/codefresh/helm-deploy-Preview-ABC.yaml', - expect.any(String), - 'utf8' - ); + expect(configPath).toMatch(/^\/tmp\/lifecycle\/codefresh\/helm-deploy-Preview-ABC-[\w-]{10}\.yaml$/); + expect(mockWriteFile).toHaveBeenCalledWith(configPath, expect.any(String), 'utf8'); const writtenYaml = yaml.load(mockWriteFile.mock.calls[0][1] as string) as any; expect(writtenYaml.steps.clone.type).toBe('parallel'); expect(writtenYaml.steps.deploy.arguments.custom_value_files).toEqual(['helm/preview.yaml']); - expect(command).toBe( - 'codefresh run goodrx/helm-deploy -b "feature/helm-preview" -y /tmp/lifecycle/codefresh/helm-deploy-Preview-ABC.yaml -v ENV=lfc -d' - ); + expect(command).toBe(`codefresh run goodrx/helm-deploy -b "feature/helm-preview" -y ${configPath} -v ENV=lfc -d`); + }); + + test('writes a separate file for every submission of the same service', async () => { + const deploy = makePublicDeploy(); + + const first = await generateCodefreshRunCommand(deploy); + const second = await generateCodefreshRunCommand(deploy); + + expect(first.configPath).not.toBe(second.configPath); + expect(mockWriteFile).toHaveBeenCalledTimes(2); }); test('logs and preserves a YAML write failure without constructing a command', async () => { @@ -653,8 +658,9 @@ describe('image metadata and teardown', () => { expect(mockRandomAlphanumeric).toHaveBeenCalledWith(4); }); - test('uninstalls only active, non-queued Helm releases and records successful cleanup', async () => { + test('uninstalls every active Helm release, also one queued for a new deploy, and records the cleanup', async () => { const successfulPatch = jest.fn().mockResolvedValue(undefined); + const queuedPatch = jest.fn().mockResolvedValue(undefined); const skippedPatch = jest.fn().mockResolvedValue(undefined); const activeDeploy = { uuid: 'active-release', @@ -675,7 +681,7 @@ describe('image metadata and teardown', () => { active: true, status: 'queued', deployable: { helm: { chart: { name: 'public-chart' } } }, - $query: jest.fn(() => ({ patch: skippedPatch })), + $query: jest.fn(() => ({ patch: queuedPatch })), }; const nonHelmDeploy = { uuid: 'not-a-helm-release', @@ -690,9 +696,11 @@ describe('image metadata and teardown', () => { await uninstallHelmReleases({ id: 42, namespace: 'env-build-123' } as any); - expect(mockShellPromise).toHaveBeenCalledTimes(1); + expect(mockShellPromise).toHaveBeenCalledTimes(2); expect(mockShellPromise).toHaveBeenCalledWith('helm uninstall active-release --namespace env-build-123'); + expect(mockShellPromise).toHaveBeenCalledWith('helm uninstall queued-release --namespace env-build-123'); expect(successfulPatch).toHaveBeenCalledWith({ statusMessage: 'Uninstalled via Helm' }); + expect(queuedPatch).toHaveBeenCalledWith({ statusMessage: 'Uninstalled via Helm' }); expect(skippedPatch).not.toHaveBeenCalled(); expect(mockLoggerDebug).toHaveBeenCalledWith('Helm: releases uninstalled'); }); diff --git a/src/server/lib/helm/helm.ts b/src/server/lib/helm/helm.ts index 58f3a35d..788a65c3 100644 --- a/src/server/lib/helm/helm.ts +++ b/src/server/lib/helm/helm.ts @@ -16,10 +16,10 @@ import yaml from 'js-yaml'; import fs from 'fs'; +import { nanoid } from 'nanoid'; import Deploy from 'server/models/Deploy'; import GlobalConfigService from 'server/services/globalConfig'; import { TMP_PATH } from 'shared/config'; -import { DeployStatus } from 'shared/constants'; import { getLogger } from 'server/lib/logger'; import { getBuildSource, resolveBuildSourceRepository } from 'server/lib/buildSource'; import { shellPromise } from 'server/lib/shell'; @@ -270,7 +270,7 @@ export async function deployHelm( * @param {Deploy} deploy - The deploy object containing deployment details. * @returns {Promise} A promise that resolves to the generated Codefresh command. */ -export async function generateCodefreshRunCommand(deploy: Deploy): Promise { +export async function generateCodefreshRunCommand(deploy: Deploy): Promise<{ command: string; configPath: string }> { const hasValueFiles = deploy?.deployable?.helm?.chart?.valueFiles?.length > 0; await deploy.$fetchGraph('build'); const yamlContent = hasValueFiles @@ -278,7 +278,8 @@ export async function generateCodefreshRunCommand(deploy: Deploy): Promise d.deployable.helm && Object.keys(d.deployable.helm).length > 0); for (const deploy of helmDeploys) { - if (!deploy.active || deploy.status === DeployStatus.QUEUED) continue; + if (!deploy.active) continue; try { await shellPromise(`helm uninstall ${deploy.uuid} --namespace ${build.namespace}`); diff --git a/src/server/lib/kubernetesApply/applyManifest.test.ts b/src/server/lib/kubernetesApply/applyManifest.test.ts index 30f5ad19..b873243e 100644 --- a/src/server/lib/kubernetesApply/applyManifest.test.ts +++ b/src/server/lib/kubernetesApply/applyManifest.test.ts @@ -16,6 +16,10 @@ var mockCreateNamespacedConfigMap: jest.Mock; var mockCreateNamespacedJob: jest.Mock; +var mockListNamespacedJob: jest.Mock; +var mockDeleteNamespacedJob: jest.Mock; +var mockListNamespacedPod: jest.Mock; +var mockDeleteNamespacedPod: jest.Mock; var mockGetAllConfigs: jest.Mock; var mockWaitForJobAndGetLogs: jest.Mock; var mockGetLogger: jest.Mock; @@ -26,11 +30,19 @@ jest.mock('@kubernetes/client-node', () => { const actual = jest.requireActual('@kubernetes/client-node'); mockCreateNamespacedConfigMap = jest.fn(); mockCreateNamespacedJob = jest.fn(); + mockListNamespacedJob = jest.fn(); + mockDeleteNamespacedJob = jest.fn(); + mockListNamespacedPod = jest.fn(); + mockDeleteNamespacedPod = jest.fn(); const coreApi = { createNamespacedConfigMap: (...args: unknown[]) => mockCreateNamespacedConfigMap(...args), + listNamespacedPod: (...args: unknown[]) => mockListNamespacedPod(...args), + deleteNamespacedPod: (...args: unknown[]) => mockDeleteNamespacedPod(...args), }; const batchApi = { createNamespacedJob: (...args: unknown[]) => mockCreateNamespacedJob(...args), + listNamespacedJob: (...args: unknown[]) => mockListNamespacedJob(...args), + deleteNamespacedJob: (...args: unknown[]) => mockDeleteNamespacedJob(...args), }; return { @@ -71,7 +83,12 @@ jest.mock('server/lib/logger', () => { }); import { HttpError } from '@kubernetes/client-node'; -import { createKubernetesApplyJob, getKubernetesApplyJobName, monitorKubernetesJob } from './applyManifest'; +import { + createKubernetesApplyJob, + getKubernetesApplyJobName, + listUnfinishedDeployJobs, + monitorKubernetesJob, +} from './applyManifest'; function deploy(overrides: Record = {}) { return { @@ -283,3 +300,74 @@ describe('Kubernetes apply jobs', () => { expect(mockError).toHaveBeenCalledWith('Job: monitor failed name=apply-job'); }); }); + +describe('listUnfinishedDeployJobs', () => { + const applySelector = 'app=lifecycle-deploy,type=kubernetes-apply,deploy_uuid=catalog-deploy-uuid'; + const helmSelector = 'app.kubernetes.io/name=native-helm,deploy-id=37'; + const job = (name: string, condition?: 'Complete' | 'Failed') => ({ + metadata: { name }, + status: condition ? { conditions: [{ type: condition, status: 'True' }] } : { active: 1 }, + }); + const podsOf = (phase: string, extra: Record = {}) => ({ + body: { items: [{ metadata: { name: 'pod', ...extra }, status: { phase } }] }, + }); + + beforeEach(() => { + jest.clearAllMocks(); + mockListNamespacedJob.mockResolvedValue({ body: { items: [] } }); + mockListNamespacedPod.mockResolvedValue({ body: { items: [] } }); + }); + + it('reports apply and native Helm jobs of the service that can still write, and never deletes anything', async () => { + mockListNamespacedJob.mockImplementation(async (_namespace: string, ...args: unknown[]) => + args[4] === applySelector + ? { + body: { + items: [ + job('apply-running'), + job('apply-done', 'Complete'), + job('apply-failed', 'Failed'), + job('apply-deadline', 'Failed'), + ], + }, + } + : { body: { items: [job('helm-running')] } } + ); + mockListNamespacedPod.mockImplementation(async (_namespace: string, ...args: unknown[]) => + args[4] === 'job-name=apply-deadline' + ? podsOf('Running', { deletionTimestamp: '2026-10-11T00:00:00Z' }) + : podsOf('Failed') + ); + + await expect(listUnfinishedDeployJobs(deploy(), 'env-build')).resolves.toEqual([ + 'apply-running', + 'apply-deadline', + 'helm-running', + ]); + expect(mockListNamespacedJob).toHaveBeenCalledWith( + 'env-build', + undefined, + undefined, + undefined, + undefined, + applySelector + ); + expect(mockListNamespacedJob).toHaveBeenCalledWith( + 'env-build', + undefined, + undefined, + undefined, + undefined, + helmSelector + ); + expect(mockDeleteNamespacedJob).not.toHaveBeenCalled(); + expect(mockDeleteNamespacedPod).not.toHaveBeenCalled(); + }); + + it('reports nothing once every earlier job is terminal and its pods have stopped', async () => { + mockListNamespacedJob.mockResolvedValue({ body: { items: [job('old', 'Failed'), job('older', 'Complete')] } }); + mockListNamespacedPod.mockResolvedValue(podsOf('Succeeded')); + + await expect(listUnfinishedDeployJobs(deploy(), 'env-build')).resolves.toEqual([]); + }); +}); diff --git a/src/server/lib/kubernetesApply/applyManifest.ts b/src/server/lib/kubernetesApply/applyManifest.ts index 097fa3d5..fc1c5211 100644 --- a/src/server/lib/kubernetesApply/applyManifest.ts +++ b/src/server/lib/kubernetesApply/applyManifest.ts @@ -152,6 +152,38 @@ export async function createKubernetesApplyJob({ return createdJob.body; } +/** A deploy Job is done only once it is terminal and the kubelet reports each of its pods stopped. */ +export async function listUnfinishedDeployJobs(deploy: Deploy, namespace: string): Promise { + const kc = new k8s.KubeConfig(); + kc.loadFromDefault(); + const batchApi = kc.makeApiClient(k8s.BatchV1Api); + const coreApi = kc.makeApiClient(k8s.CoreV1Api); + const selectors = [ + `app=lifecycle-deploy,type=kubernetes-apply,deploy_uuid=${deploy.uuid}`, + `app.kubernetes.io/name=native-helm,deploy-id=${deploy.id}`, + ]; + const unfinished: string[] = []; + for (const selector of selectors) { + const { body } = await batchApi.listNamespacedJob(namespace, undefined, undefined, undefined, undefined, selector); + for (const job of body.items ?? []) { + const name = job.metadata?.name; + if (!name || hasCondition(job, 'Complete')) continue; + if (!hasCondition(job, 'Failed') || (await hasRunningPod(coreApi, namespace, name))) unfinished.push(name); + } + } + return unfinished; +} + +function hasCondition(job: k8s.V1Job, type: 'Complete' | 'Failed'): boolean { + return (job.status?.conditions ?? []).some((condition) => condition.type === type && condition.status === 'True'); +} + +async function hasRunningPod(coreApi: k8s.CoreV1Api, namespace: string, jobName: string): Promise { + const selector = `job-name=${jobName}`; + const { body } = await coreApi.listNamespacedPod(namespace, undefined, undefined, undefined, undefined, selector); + return (body.items ?? []).some((pod) => pod.status?.phase !== 'Succeeded' && pod.status?.phase !== 'Failed'); +} + async function createManifestConfigMap(deploy: Deploy, configMapName: string, namespace: string): Promise { const kc = new k8s.KubeConfig(); kc.loadFromDefault(); diff --git a/src/server/lib/nativeHelm/__tests__/helm.behavior.test.ts b/src/server/lib/nativeHelm/__tests__/helm.behavior.test.ts index 73a474fa..4dd4974d 100644 --- a/src/server/lib/nativeHelm/__tests__/helm.behavior.test.ts +++ b/src/server/lib/nativeHelm/__tests__/helm.behavior.test.ts @@ -24,6 +24,7 @@ const mockWithLogContext = jest.fn(); const mockMkdir = jest.fn(); const mockWriteFile = jest.fn(); const mockShellPromise = jest.fn(); +const mockUnlink = jest.fn(); const mockNanoid = jest.fn(); const mockMetricsConstructor = jest.fn(); const mockMetricIncrement = jest.fn(); @@ -73,6 +74,7 @@ jest.mock('fs', () => ({ promises: { mkdir: (...args: unknown[]) => mockMkdir(...args), writeFile: (...args: unknown[]) => mockWriteFile(...args), + unlink: (...args: unknown[]) => mockUnlink(...args), }, }, })); @@ -99,7 +101,10 @@ jest.mock('server/lib/helm/utils', () => ({ jest.mock('server/lib/helm/helm', () => ({ constructHelmDeploysBuildMetaData: (...args: unknown[]) => mockConstructBuildMetadata(...args), - generateCodefreshRunCommand: (...args: unknown[]) => mockGenerateCodefreshRunCommand(...args), + generateCodefreshRunCommand: async (...args: unknown[]) => { + const plan = await mockGenerateCodefreshRunCommand(...args); + return typeof plan === 'string' ? { command: plan, configPath: null } : plan; + }, })); jest.mock('server/lib/codefresh/utils', () => ({ @@ -683,6 +688,38 @@ describe('native Helm orchestration behavior', () => { expect(mockPatchIngress).not.toHaveBeenCalled(); }); + it('removes the submission file once the provider has the command', async () => { + const { deploy } = createDeploy({ + deployable: { name: 'sample-service', repository: { fullName: 'example/repository' } }, + }); + mockGenerateCodefreshRunCommand.mockResolvedValueOnce({ + command: 'codefresh run pipeline', + configPath: '/tmp/lifecycle/codefresh/helm-deploy-sample-abc.yaml', + }); + mockUnlink.mockResolvedValue(undefined); + + await expect(deployHelm([deploy])).resolves.toBeUndefined(); + + expect(mockShellPromise).toHaveBeenCalledWith('codefresh run pipeline'); + expect(mockUnlink).toHaveBeenCalledWith('/tmp/lifecycle/codefresh/helm-deploy-sample-abc.yaml'); + }); + + it('removes the submission file when the provider gate refuses the row', async () => { + const { deploy } = createDeploy({ + deployable: { name: 'sample-service', repository: { fullName: 'example/repository' } }, + }); + mockGenerateCodefreshRunCommand.mockResolvedValueOnce({ + command: 'codefresh run pipeline', + configPath: '/tmp/x.yaml', + }); + mockUnlink.mockResolvedValue(undefined); + + await expect(deployHelm([deploy], { providerSubmissionGate: async () => false })).rejects.toThrow(); + + expect(mockShellPromise).not.toHaveBeenCalled(); + expect(mockUnlink).toHaveBeenCalledWith('/tmp/x.yaml'); + }); + it('records a normal deployment failure after tracking failure metrics', async () => { const deployError = new Error('codefresh unavailable'); const { deploy } = createDeploy({ diff --git a/src/server/lib/nativeHelm/helm.ts b/src/server/lib/nativeHelm/helm.ts index 78e0f843..0c49723b 100644 --- a/src/server/lib/nativeHelm/helm.ts +++ b/src/server/lib/nativeHelm/helm.ts @@ -66,6 +66,8 @@ export type HelmSecretMutationGate = (deploy: Deploy, action: () => Promise Promise; } export interface JobResult { @@ -466,7 +468,12 @@ export async function deployNativeHelm(deploy: Deploy, options: HelmDeploymentEx } } -async function deployCodefreshHelm(deploy: Deploy, deployService: DeployService, runUUID: string): Promise { +async function deployCodefreshHelm( + deploy: Deploy, + deployService: DeployService, + runUUID: string, + options: HelmDeploymentExecutionOptions = {} +): Promise { const deployable = requireDeployable(deploy); const { build } = deploy; @@ -474,8 +481,16 @@ async function deployCodefreshHelm(deploy: Deploy, deployService: DeployService, const { getCodefreshPipelineIdFromOutput } = await import('server/lib/codefresh/utils'); const { checkPipelineStatus } = await import('server/lib/codefresh'); - const codefreshRunCommand = await generateCodefreshRunCommand(deploy); - const output = await shellPromise(codefreshRunCommand); + const { command: codefreshRunCommand, configPath } = await generateCodefreshRunCommand(deploy); + let output = ''; + try { + if (options.providerSubmissionGate && !(await options.providerSubmissionGate(deploy))) { + throw new DeploymentSupersededError(); + } + output = await shellPromise(codefreshRunCommand); + } finally { + if (configPath) await fs.promises.unlink(configPath).catch(() => undefined); + } const deployPipelineId = getCodefreshPipelineIdFromOutput(output); const statusMessage = 'Starting deployment via Helm'; @@ -545,7 +560,7 @@ export async function deployHelm(deploys: Deploy[], options: HelmDeploymentExecu if (useNative) { await deployNativeHelm(deploy, options); } else { - await deployCodefreshHelm(deploy, deployService, runUUID); + await deployCodefreshHelm(deploy, deployService, runUUID, options); } await deployService.patchAndUpdateActivityFeed( diff --git a/src/server/lib/tests/deploymentManager.test.ts b/src/server/lib/tests/deploymentManager.test.ts index 269077ba..2eb12953 100644 --- a/src/server/lib/tests/deploymentManager.test.ts +++ b/src/server/lib/tests/deploymentManager.test.ts @@ -15,6 +15,7 @@ */ import { DeploymentManager, DeploymentSupersededError } from '../deploymentManager/deploymentManager'; +import { AuthorityLockLostError } from 'server/lib/authorityLock'; import { Deploy } from 'server/models'; import { buildDeployJobName } from '../kubernetes/jobNames'; import { deployHelm } from '../helm'; @@ -27,6 +28,7 @@ jest.mock('../nativeHelm', () => ({ shouldUseNativeHelm: jest.fn().mockResolvedValue(false), })); jest.mock('../kubernetesApply/applyManifest', () => ({ + listUnfinishedDeployJobs: jest.fn().mockResolvedValue([]), createKubernetesApplyJob: jest.fn().mockResolvedValue(void 0), monitorKubernetesJob: jest.fn().mockResolvedValue({ success: true, message: 'ok', logs: 'apply logs' }), })); @@ -54,7 +56,11 @@ jest.mock('server/services/deploy', () => { })); }); -import { createKubernetesApplyJob, monitorKubernetesJob } from '../kubernetesApply/applyManifest'; +import { + listUnfinishedDeployJobs, + createKubernetesApplyJob, + monitorKubernetesJob, +} from '../kubernetesApply/applyManifest'; import { waitForDeployPodReady } from '../kubernetes'; import GlobalConfigService from 'server/services/globalConfig'; import { getLogArchivalService } from 'server/services/logArchival'; @@ -78,6 +84,7 @@ describe('DeploymentManager', () => { mockRecordDeployFailure.mockReset().mockResolvedValue(false); (deployHelm as jest.Mock).mockReset().mockResolvedValue(undefined); (shouldUseNativeHelm as jest.Mock).mockReset().mockResolvedValue(false); + (listUnfinishedDeployJobs as jest.Mock).mockReset().mockResolvedValue([]); (createKubernetesApplyJob as jest.Mock).mockReset().mockResolvedValue(undefined); (monitorKubernetesJob as jest.Mock) .mockReset() @@ -311,12 +318,13 @@ describe('DeploymentManager', () => { } as unknown as Deploy; } - it('admits all native siblings for a level together while Codefresh and pod readiness stay outside', async () => { + it('admits each native service through its own gate while Codefresh and pod readiness stay outside', async () => { const nativeHelm = runnableDeploy('native-chart', 'helm'); const codefreshHelm = runnableDeploy('codefresh-chart', 'helm'); const kubernetes = runnableDeploy('web', 'github'); const events: string[] = []; let insideGate = false; + let gateDepth = 0; (shouldUseNativeHelm as jest.Mock).mockImplementation(async (deploy: Deploy) => { return deploy.deployable.name === 'native-chart'; @@ -336,12 +344,14 @@ describe('DeploymentManager', () => { return { ready: true }; }); - const nativeMutationGate = jest.fn(async (action: () => Promise) => { + const nativeMutationGate = jest.fn(async (_deploy: Deploy, action: () => Promise) => { + gateDepth += 1; insideGate = true; try { return { admitted: true as const, value: await action() }; } finally { - insideGate = false; + gateDepth -= 1; + insideGate = gateDepth > 0; } }); const nativeSecretMutationGate = jest.fn(); @@ -354,25 +364,27 @@ describe('DeploymentManager', () => { await manager.deploy(); - expect(nativeMutationGate).toHaveBeenCalledTimes(1); + expect(nativeMutationGate).toHaveBeenCalledTimes(2); + expect(nativeMutationGate).toHaveBeenCalledWith(nativeHelm, expect.any(Function)); + expect(nativeMutationGate).toHaveBeenCalledWith(kubernetes, expect.any(Function)); expect(deployHelm).toHaveBeenCalledWith([nativeHelm], { secretMutationGate: nativeSecretMutationGate, }); + expect(nativeMutationGate).not.toHaveBeenCalledWith(codefreshHelm, expect.any(Function)); + expect(deployHelm).toHaveBeenCalledWith( + [codefreshHelm], + expect.objectContaining({ providerSubmissionGate: expect.any(Function) }) + ); expect(events).toEqual( - expect.arrayContaining([ - 'codefresh-chart:false', - 'native-chart:true', - 'apply:true', - 'monitor:true', - 'ready:false', - ]) + expect.arrayContaining(['native-chart:true', 'apply:true', 'monitor:true', 'ready:false']) ); }); - it('keeps the promotion gate until every admitted native sibling is terminal', async () => { + it('holds a service gate until its native mutation is terminal, and a failed sibling does not block it', async () => { const nativeHelm = runnableDeploy('native-chart', 'helm'); const kubernetes = runnableDeploy('web', 'github'); let insideGate = false; + let gateDepth = 0; let releaseMonitor!: () => void; let markMonitorStarted!: () => void; const monitorStarted = new Promise((resolve) => { @@ -389,12 +401,14 @@ describe('DeploymentManager', () => { return { success: true, message: 'ok', logs: 'apply logs' }; }); - const nativeMutationGate = jest.fn(async (action: () => Promise) => { + const nativeMutationGate = jest.fn(async (_deploy: Deploy, action: () => Promise) => { + gateDepth += 1; insideGate = true; try { return { admitted: true as const, value: await action() }; } finally { - insideGate = false; + gateDepth -= 1; + insideGate = gateDepth > 0; } }); const manager = new DeploymentManager([nativeHelm, kubernetes], { @@ -406,31 +420,31 @@ describe('DeploymentManager', () => { expect(insideGate).toBe(true); releaseMonitor(); - await expect(deployment).rejects.toThrow('helm failed'); + await expect(deployment).resolves.toEqual({ failed: [nativeHelm] }); expect(insideGate).toBe(false); expect(waitForDeployPodReady).toHaveBeenCalledWith(kubernetes); }); - it('treats denied native admission as supersession without creating a job or recording failure', async () => { + it('skips a service whose native admission is denied without creating a job or recording failure', async () => { const deploy = runnableDeploy('web', 'github'); const manager = new DeploymentManager([deploy], { nativeMutationGate: (async () => ({ admitted: false as const })) as any, }); - await expect(manager.deploy()).rejects.toBeInstanceOf(DeploymentSupersededError); + await expect(manager.deploy()).resolves.toEqual({ failed: [] }); expect(createKubernetesApplyJob).not.toHaveBeenCalled(); expect(monitorKubernetesJob).not.toHaveBeenCalled(); expect(mockRecordDeployFailure).not.toHaveBeenCalled(); }); - it('stops as superseded when the run-fenced queued patch affects no row', async () => { + it('skips a service as superseded when the run-fenced queued patch affects no row', async () => { const deploy = runnableDeploy('web', 'github'); const where = jest.fn().mockResolvedValue(0); deploy.$query = () => ({ patch: jest.fn().mockReturnValue({ where }) } as any); const manager = new DeploymentManager([deploy]); - await expect(manager.deploy()).rejects.toBeInstanceOf(DeploymentSupersededError); + await expect(manager.deploy()).resolves.toEqual({ failed: [] }); expect(where).toHaveBeenCalledWith({ id: deploy.id, runUUID: deploy.runUUID }); expect(createKubernetesApplyJob).not.toHaveBeenCalled(); @@ -449,7 +463,7 @@ describe('DeploymentManager', () => { const manager = new DeploymentManager([first, second], { isCurrent: async () => current, - nativeMutationGate: (async (action: () => Promise) => ({ + nativeMutationGate: (async (_deploy: Deploy, action: () => Promise) => ({ admitted: true as const, value: await action(), })) as any, @@ -619,7 +633,7 @@ describe('DeploymentManager', () => { return { deploy, patch, where }; } - it('removes a self-dependency and surfaces a Codefresh provider failure', async () => { + it('removes a self-dependency and records a Codefresh provider failure on the service', async () => { const { deploy, patch } = managedDeploy({ name: 'self-dependent-chart', type: 'helm', @@ -630,13 +644,16 @@ describe('DeploymentManager', () => { const manager = new DeploymentManager([deploy]); - await expect(manager.deploy()).rejects.toBe(providerError); - expect(deployHelm).toHaveBeenCalledWith([deploy]); + await expect(manager.deploy()).resolves.toEqual({ failed: [deploy] }); + expect(deployHelm).toHaveBeenCalledWith( + [deploy], + expect.objectContaining({ providerSubmissionGate: expect.any(Function) }) + ); expect(patch).toHaveBeenCalledWith({ status: DeployStatus.QUEUED }); expect(createKubernetesApplyJob).not.toHaveBeenCalled(); }); - it('surfaces an aggregated readiness failure from deploy()', async () => { + it('records a readiness failure on the service and returns it from deploy()', async () => { const { deploy } = managedDeploy({ name: 'unready-service' }); (waitForDeployPodReady as jest.Mock).mockResolvedValueOnce({ ready: false, @@ -645,13 +662,16 @@ describe('DeploymentManager', () => { const manager = new DeploymentManager([deploy]); - await expect(manager.deploy()).rejects.toThrow( - 'Pods failed to become ready within timeout: container waiting=ImagePullBackOff' - ); + await expect(manager.deploy()).resolves.toEqual({ failed: [deploy] }); expect(mockRecordDeployFailure).toHaveBeenCalledWith( deploy, deploy.runUUID, - expect.objectContaining({ status: DeployStatus.DEPLOY_FAILED }) + expect.objectContaining({ + status: DeployStatus.DEPLOY_FAILED, + error: expect.objectContaining({ + message: 'Pods failed to become ready within timeout: container waiting=ImagePullBackOff', + }), + }) ); expect(deployHelm).not.toHaveBeenCalled(); }); @@ -693,11 +713,14 @@ describe('DeploymentManager', () => { (waitForDeployPodReady as jest.Mock).mockResolvedValueOnce({ ready: false }); const manager = new DeploymentManager([deploy]); - await expect(manager.deploy()).rejects.toThrow('Pods failed to become ready within timeout'); + await expect(manager.deploy()).resolves.toEqual({ failed: [deploy] }); expect(mockRecordDeployFailure).toHaveBeenCalledWith( deploy, deploy.runUUID, - expect.objectContaining({ status: DeployStatus.DEPLOY_FAILED }) + expect.objectContaining({ + status: DeployStatus.DEPLOY_FAILED, + error: expect.objectContaining({ message: 'Pods failed to become ready within timeout' }), + }) ); }); @@ -720,13 +743,16 @@ describe('DeploymentManager', () => { const { deploy } = managedDeploy({ name: 'missing-manifest', manifest: '' }); const manager = new DeploymentManager([deploy]); - await expect(manager.deploy()).rejects.toThrow( - `Deploy ${deploy.uuid} has no manifest. Ensure manifests are generated before deployment.` - ); + await expect(manager.deploy()).resolves.toEqual({ failed: [deploy] }); expect(mockRecordDeployFailure).toHaveBeenCalledWith( deploy, deploy.runUUID, - expect.objectContaining({ status: DeployStatus.DEPLOY_FAILED }) + expect.objectContaining({ + status: DeployStatus.DEPLOY_FAILED, + error: expect.objectContaining({ + message: `Deploy ${deploy.uuid} has no manifest. Ensure manifests are generated before deployment.`, + }), + }) ); expect(createKubernetesApplyJob).not.toHaveBeenCalled(); expect(monitorKubernetesJob).not.toHaveBeenCalled(); @@ -742,7 +768,7 @@ describe('DeploymentManager', () => { }); const manager = new DeploymentManager([deploy]); - await expect(manager.deploy()).rejects.toThrow('Kubernetes apply job failed'); + await expect(manager.deploy()).resolves.toEqual({ failed: [deploy] }); expect(mockArchiveLogs).toHaveBeenCalledWith( expect.objectContaining({ status: 'Failed', deployUuid: deploy.uuid }), 'kubectl error output' @@ -755,24 +781,24 @@ describe('DeploymentManager', () => { expect(waitForDeployPodReady).not.toHaveBeenCalled(); }); - it('propagates supersession during manifest application without recording a provider failure', async () => { + it('skips a service superseded during manifest application without recording a provider failure', async () => { const { deploy } = managedDeploy({ name: 'superseded-apply' }); const superseded = new DeploymentSupersededError(); mockPatchAndUpdateActivityFeed.mockRejectedValueOnce(superseded); const manager = new DeploymentManager([deploy]); - await expect(manager.deploy()).rejects.toBe(superseded); + await expect(manager.deploy()).resolves.toEqual({ failed: [] }); expect(mockRecordDeployFailure).not.toHaveBeenCalled(); expect(createKubernetesApplyJob).not.toHaveBeenCalled(); }); - it('propagates supersession during readiness without recording a provider failure', async () => { + it('skips a service superseded during readiness without recording a provider failure', async () => { const { deploy } = managedDeploy({ name: 'superseded-readiness' }); const superseded = new DeploymentSupersededError(); (waitForDeployPodReady as jest.Mock).mockRejectedValueOnce(superseded); const manager = new DeploymentManager([deploy]); - await expect(manager.deploy()).rejects.toBe(superseded); + await expect(manager.deploy()).resolves.toEqual({ failed: [] }); expect(mockRecordDeployFailure).not.toHaveBeenCalled(); expect(mockPatchAndUpdateActivityFeed).not.toHaveBeenCalledWith( deploy, @@ -781,6 +807,428 @@ describe('DeploymentManager', () => { ); }); + it('blocks dependents of a service that failed before rollout and still deploys unrelated services', async () => { + const { deploy: db, patch: dbPatch } = managedDeploy({ name: 'db' }); + const { deploy: api, patch: apiPatch } = managedDeploy({ name: 'api', deploymentDependsOn: ['db'] }); + const { deploy: web } = managedDeploy({ name: 'web' }); + const manager = new DeploymentManager([db, api, web], { failedServices: ['db'] }); + + const result = await manager.deploy(); + + expect(result.failed.map((failed) => failed.deployable.name).sort()).toEqual(['api', 'db']); + expect(apiPatch).toHaveBeenCalledWith({ + status: DeployStatus.DEPLOY_FAILED, + statusMessage: 'Not deployed: db failed.', + }); + expect(dbPatch).not.toHaveBeenCalled(); + expect(createKubernetesApplyJob).toHaveBeenCalledTimes(1); + expect(createKubernetesApplyJob).toHaveBeenCalledWith(expect.objectContaining({ deploy: web })); + }); + + it('attributes a Codefresh failure to its own service and lets a healthy sibling finish', async () => { + const { deploy: good } = managedDeploy({ name: 'good-chart', type: 'helm' }); + const { deploy: bad } = managedDeploy({ name: 'bad-chart', type: 'helm' }); + (shouldUseNativeHelm as jest.Mock).mockResolvedValue(false); + (deployHelm as jest.Mock).mockImplementation(async (deploys: Deploy[]) => { + if (deploys[0].deployable.name === 'bad-chart') throw new Error('Codefresh deploy failed'); + }); + const manager = new DeploymentManager([good, bad]); + + await expect(manager.deploy()).resolves.toEqual({ failed: [bad] }); + expect(deployHelm).toHaveBeenCalledWith( + [good], + expect.objectContaining({ providerSubmissionGate: expect.any(Function) }) + ); + expect(deployHelm).toHaveBeenCalledWith( + [bad], + expect.objectContaining({ providerSubmissionGate: expect.any(Function) }) + ); + }); + + it('rethrows a lost promotion lease instead of recording it as a service failure', async () => { + const { deploy } = managedDeploy({ name: 'leased' }); + const lost = new AuthorityLockLostError('deploy-promotion.1'); + const manager = new DeploymentManager([deploy], { + nativeMutationGate: (async () => { + throw lost; + }) as any, + }); + + await expect(manager.deploy()).rejects.toBe(lost); + expect(mockRecordDeployFailure).not.toHaveBeenCalled(); + expect(waitForDeployPodReady).not.toHaveBeenCalled(); + }); + + it('records a service failed when its ownership check fails before the launch', async () => { + const { deploy } = managedDeploy({ name: 'checked' }); + const manager = new DeploymentManager([deploy], { + nativeMutationGate: (async () => { + throw new Error('connection reset'); + }) as any, + }); + + await expect(manager.deploy()).resolves.toEqual({ failed: [deploy] }); + expect(createKubernetesApplyJob).not.toHaveBeenCalled(); + expect(mockRecordDeployFailure).toHaveBeenCalledTimes(1); + expect(mockRecordDeployFailure).toHaveBeenCalledWith(deploy, 'run-checked', { + status: DeployStatus.DEPLOY_FAILED, + fallbackMessage: 'Deployment failed. Redeploy the service to try again.', + }); + }); + + it('records a Codefresh service failed when its ownership check fails before the launch', async () => { + const { deploy, patch } = managedDeploy({ name: 'chart', type: 'helm' }); + let queuedPatches = 0; + patch.mockImplementation(() => ({ + where: jest.fn(async () => { + queuedPatches += 1; + if (queuedPatches === 3) throw new Error('connection reset'); + return 1; + }), + })); + (shouldUseNativeHelm as jest.Mock).mockResolvedValue(false); + const manager = new DeploymentManager([deploy]); + + await expect(manager.deploy()).resolves.toEqual({ failed: [deploy] }); + expect(deployHelm).not.toHaveBeenCalled(); + expect(mockRecordDeployFailure).toHaveBeenCalledWith(deploy, 'run-chart', { + status: DeployStatus.DEPLOY_FAILED, + fallbackMessage: 'Deployment failed. Redeploy the service to try again.', + }); + }); + + it('records a failed apply once, with its own message', async () => { + const { deploy } = managedDeploy({ name: 'broken' }); + (createKubernetesApplyJob as jest.Mock).mockRejectedValueOnce(new Error('apply failed')); + const manager = new DeploymentManager([deploy], { + nativeMutationGate: (async (_deploy: Deploy, action: () => Promise) => ({ + admitted: true, + value: await action(), + })) as any, + }); + + await expect(manager.deploy()).resolves.toEqual({ failed: [deploy] }); + expect(mockRecordDeployFailure).toHaveBeenCalledTimes(1); + expect(mockRecordDeployFailure).toHaveBeenCalledWith( + deploy, + 'run-broken', + expect.objectContaining({ status: DeployStatus.DEPLOY_FAILED, error: expect.any(Error) }) + ); + }); + + it('lets launched siblings finish and launches nothing more once a promotion lease is lost', async () => { + const { deploy: leased } = managedDeploy({ name: 'leased' }); + const { deploy: slow } = managedDeploy({ name: 'slow' }); + const { deploy: dependent, patch: dependentPatch } = managedDeploy({ + name: 'dependent', + deploymentDependsOn: ['slow'], + }); + const lost = new AuthorityLockLostError('deploy-promotion.1'); + const events: string[] = []; + let finishSlow!: () => void; + const slowApply = new Promise((resolve) => { + finishSlow = resolve; + }); + (createKubernetesApplyJob as jest.Mock).mockImplementation(async ({ deploy }: { deploy: Deploy }) => { + if (deploy !== slow) return; + await slowApply; + events.push('slow applied'); + }); + const manager = new DeploymentManager([leased, slow, dependent], { + nativeMutationGate: (async (deploy: Deploy, action: () => Promise) => { + if (deploy === leased) throw lost; + return { admitted: true, value: await action() }; + }) as any, + }); + + try { + const outcome = manager.deploy().catch((error) => { + events.push('rejected'); + throw error; + }); + for (let i = 0; i < 20; i += 1) await new Promise((resolve) => setImmediate(resolve)); + expect(events).toEqual([]); + + finishSlow(); + await expect(outcome).rejects.toBe(lost); + expect(events).toEqual(['slow applied', 'rejected']); + expect(createKubernetesApplyJob).toHaveBeenCalledTimes(1); + expect(createKubernetesApplyJob).toHaveBeenCalledWith(expect.objectContaining({ deploy: slow })); + expect(dependentPatch).not.toHaveBeenCalledWith( + expect.objectContaining({ status: DeployStatus.DEPLOY_FAILED }) + ); + expect(mockRecordDeployFailure).not.toHaveBeenCalled(); + } finally { + (createKubernetesApplyJob as jest.Mock).mockResolvedValue(undefined); + } + }); + + it('waits for an earlier deploy job of the service to finish before creating a new one', async () => { + jest.useFakeTimers(); + try { + const { deploy, patch } = managedDeploy({ name: 'svc' }); + (listUnfinishedDeployJobs as jest.Mock).mockResolvedValueOnce(['svc-deploy-old']).mockResolvedValue([]); + const manager = new DeploymentManager([deploy]); + + const deployment = manager.deploy(); + await jest.advanceTimersByTimeAsync(5_000); + + await expect(deployment).resolves.toEqual({ failed: [] }); + expect(listUnfinishedDeployJobs).toHaveBeenCalledWith(deploy, 'testns'); + expect(patch).toHaveBeenCalledWith({ + status: DeployStatus.DEPLOYING, + statusMessage: 'Waiting for the previous deploy to finish', + }); + expect((listUnfinishedDeployJobs as jest.Mock).mock.invocationCallOrder[1]).toBeLessThan( + (createKubernetesApplyJob as jest.Mock).mock.invocationCallOrder[0] + ); + } finally { + jest.useRealTimers(); + } + }); + + it('waits for an earlier deploy job of the service before a native Helm deploy', async () => { + jest.useFakeTimers(); + try { + const { deploy } = managedDeploy({ name: 'chart', type: 'helm' }); + (shouldUseNativeHelm as jest.Mock).mockResolvedValue(true); + (listUnfinishedDeployJobs as jest.Mock).mockResolvedValueOnce(['chart-deploy-old']).mockResolvedValue([]); + const manager = new DeploymentManager([deploy]); + + const deployment = manager.deploy(); + await jest.advanceTimersByTimeAsync(5_000); + + await expect(deployment).resolves.toEqual({ failed: [] }); + expect((listUnfinishedDeployJobs as jest.Mock).mock.invocationCallOrder[1]).toBeLessThan( + (deployHelm as jest.Mock).mock.invocationCallOrder[0] + ); + } finally { + jest.useRealTimers(); + } + }); + + it('fails the service, without applying, when an earlier deploy job is still running after 45 minutes', async () => { + jest.useFakeTimers(); + try { + const { deploy } = managedDeploy({ name: 'svc' }); + (listUnfinishedDeployJobs as jest.Mock).mockResolvedValue(['svc-deploy-old']); + const manager = new DeploymentManager([deploy]); + + const deployment = manager.deploy(); + await jest.advanceTimersByTimeAsync(45 * 60_000 + 5_000); + + await expect(deployment).resolves.toEqual({ failed: [deploy] }); + expect(createKubernetesApplyJob).not.toHaveBeenCalled(); + expect(mockRecordDeployFailure).toHaveBeenCalledWith( + deploy, + deploy.runUUID, + expect.objectContaining({ + status: DeployStatus.DEPLOY_FAILED, + error: new Error( + 'The previous deploy of this service is still running after 45 minutes. Redeploy the service after it stops.' + ), + }) + ); + } finally { + jest.useRealTimers(); + } + }); + + it('stops waiting, without failing, once a newer run takes the service from this run', async () => { + const { deploy, where } = managedDeploy({ name: 'svc' }); + (listUnfinishedDeployJobs as jest.Mock).mockResolvedValue(['svc-deploy-old']); + where.mockResolvedValueOnce(1).mockResolvedValueOnce(1).mockResolvedValue(0); + const manager = new DeploymentManager([deploy]); + + await expect(manager.deploy()).resolves.toEqual({ failed: [] }); + expect(createKubernetesApplyJob).not.toHaveBeenCalled(); + expect(mockRecordDeployFailure).not.toHaveBeenCalled(); + }); + + it('does not apply and records a failure when earlier deploy jobs cannot be checked', async () => { + const { deploy } = managedDeploy({ name: 'svc' }); + (listUnfinishedDeployJobs as jest.Mock).mockRejectedValueOnce(new Error('jobs.batch is forbidden')); + const manager = new DeploymentManager([deploy]); + + await expect(manager.deploy()).resolves.toEqual({ failed: [deploy] }); + expect(createKubernetesApplyJob).not.toHaveBeenCalled(); + expect(mockRecordDeployFailure).toHaveBeenCalledWith( + deploy, + deploy.runUUID, + expect.objectContaining({ status: DeployStatus.DEPLOY_FAILED }) + ); + }); + + it('deploys a service whose prerequisite is outside this run without waiting, and its dependents after it', async () => { + const { deploy: api } = managedDeploy({ name: 'api', deploymentDependsOn: ['db'] }); + const { deploy: web } = managedDeploy({ name: 'web', deploymentDependsOn: ['api'] }); + const { deploy: worker } = managedDeploy({ name: 'worker' }); + const manager = new DeploymentManager([api, web, worker]); + + await expect(manager.deploy()).resolves.toEqual({ failed: [] }); + + const applied = (createKubernetesApplyJob as jest.Mock).mock.calls.map(([{ deploy }]) => deploy); + expect(applied).toHaveLength(3); + expect(applied.indexOf(api)).toBeLessThan(applied.indexOf(web)); + expect(mockRecordDeployFailure).not.toHaveBeenCalled(); + }); + + it('skips a service another run took while an earlier level was still deploying', async () => { + const { deploy: first } = managedDeploy({ name: 'first' }); + const { deploy: second, where } = managedDeploy({ name: 'second', deploymentDependsOn: ['first'] }); + where.mockResolvedValueOnce(1).mockResolvedValueOnce(0); + const manager = new DeploymentManager([first, second]); + + await expect(manager.deploy()).resolves.toEqual({ failed: [] }); + expect(createKubernetesApplyJob).toHaveBeenCalledTimes(1); + expect(createKubernetesApplyJob).toHaveBeenCalledWith(expect.objectContaining({ deploy: first })); + }); + + it('blocks a dependent of a failed prerequisite that never reaches the manager, such as a Codefresh service', async () => { + const { deploy: api, patch: apiPatch } = managedDeploy({ name: 'api', deploymentDependsOn: ['migrations'] }); + const { deploy: web } = managedDeploy({ name: 'web' }); + const manager = new DeploymentManager([api, web], { failedServices: ['migrations'] }); + + const result = await manager.deploy(); + + expect(result.failed).toEqual([api]); + expect(apiPatch).toHaveBeenCalledWith({ + status: DeployStatus.DEPLOY_FAILED, + statusMessage: 'Not deployed: migrations failed.', + }); + expect(createKubernetesApplyJob).toHaveBeenCalledTimes(1); + expect(createKubernetesApplyJob).toHaveBeenCalledWith(expect.objectContaining({ deploy: web })); + }); + + it('does not hand a Codefresh service to the provider once another run owns its row', async () => { + const { deploy, where } = managedDeploy({ name: 'chart', type: 'helm' }); + (shouldUseNativeHelm as jest.Mock).mockResolvedValue(false); + where.mockResolvedValueOnce(1).mockResolvedValueOnce(1).mockResolvedValueOnce(0); + const manager = new DeploymentManager([deploy]); + + await expect(manager.deploy()).resolves.toEqual({ failed: [] }); + expect(deployHelm).not.toHaveBeenCalled(); + }); + + it('hands the provider a submission gate that denies a row another run took during command generation', async () => { + const { deploy, where } = managedDeploy({ name: 'chart', type: 'helm' }); + (shouldUseNativeHelm as jest.Mock).mockResolvedValue(false); + where.mockResolvedValueOnce(1).mockResolvedValueOnce(1).mockResolvedValueOnce(1).mockResolvedValueOnce(0); + (deployHelm as jest.Mock).mockImplementation(async (_deploys: Deploy[], options: any) => { + if (!(await options.providerSubmissionGate(deploy))) throw new DeploymentSupersededError(); + }); + const manager = new DeploymentManager([deploy]); + + await expect(manager.deploy()).resolves.toEqual({ failed: [] }); + expect(mockRecordDeployFailure).not.toHaveBeenCalled(); + }); + + it('deploys a dependent without waiting when a newer run took its prerequisite from this run', async () => { + const { deploy: a, where: whereA } = managedDeploy({ name: 'a' }); + const { deploy: b } = managedDeploy({ name: 'b', deploymentDependsOn: ['a'] }); + whereA.mockResolvedValue(0); + const manager = new DeploymentManager([a, b]); + + await expect(manager.deploy()).resolves.toEqual({ failed: [] }); + + expect(createKubernetesApplyJob).toHaveBeenCalledTimes(1); + expect(createKubernetesApplyJob).toHaveBeenCalledWith(expect.objectContaining({ deploy: b })); + }); + + it('deploys a dependent without waiting when its prerequisite loses admission to a newer run', async () => { + const { deploy: a } = managedDeploy({ name: 'a' }); + const { deploy: b } = managedDeploy({ name: 'b', deploymentDependsOn: ['a'] }); + const manager = new DeploymentManager([a, b], { + nativeMutationGate: (async (deploy: Deploy, action: () => Promise) => + deploy === a ? { admitted: false as const } : { admitted: true as const, value: await action() }) as any, + }); + + await expect(manager.deploy()).resolves.toEqual({ failed: [] }); + + expect(createKubernetesApplyJob).toHaveBeenCalledTimes(1); + expect(createKubernetesApplyJob).toHaveBeenCalledWith(expect.objectContaining({ deploy: b })); + }); + + it('deploys a service whose declared prerequisite is outside this run without consulting it', async () => { + const { deploy: b, patch } = managedDeploy({ name: 'b', deploymentDependsOn: ['a'] }); + const manager = new DeploymentManager([b]); + + await expect(manager.deploy()).resolves.toEqual({ failed: [] }); + + expect(createKubernetesApplyJob).toHaveBeenCalledWith(expect.objectContaining({ deploy: b })); + expect(patch).not.toHaveBeenCalledWith(expect.objectContaining({ status: DeployStatus.DEPLOY_FAILED })); + }); + + it('deploys a prerequisite this run owns before its dependent', async () => { + const { deploy: a } = managedDeploy({ name: 'a' }); + const { deploy: b } = managedDeploy({ name: 'b', deploymentDependsOn: ['a'] }); + const manager = new DeploymentManager([a, b]); + + await expect(manager.deploy()).resolves.toEqual({ failed: [] }); + + const applied = (createKubernetesApplyJob as jest.Mock).mock.calls.map(([{ deploy }]) => deploy); + expect(applied).toEqual([a, b]); + }); + + it('launches a service whose prerequisite is absent without waiting for its level mates to finish', async () => { + const { deploy: a } = managedDeploy({ name: 'a' }); + const { deploy: b } = managedDeploy({ name: 'b', deploymentDependsOn: ['absent'] }); + let releaseA!: () => void; + const dependentLaunched = new Promise((resolve) => { + releaseA = resolve; + }); + (createKubernetesApplyJob as jest.Mock).mockImplementation(async ({ deploy }: { deploy: Deploy }) => { + if (deploy === b) releaseA(); + if (deploy === a) await dependentLaunched; + }); + const manager = new DeploymentManager([a, b]); + + try { + await expect(manager.deploy()).resolves.toEqual({ failed: [] }); + expect(createKubernetesApplyJob).toHaveBeenCalledTimes(2); + } finally { + (createKubernetesApplyJob as jest.Mock).mockResolvedValue(undefined); + } + }); + + it('starts a service as soon as its own prerequisite is done while an unrelated slow service is still applying', async () => { + const { deploy: slow } = managedDeploy({ name: 'slow' }); + const { deploy: x } = managedDeploy({ name: 'x' }); + const { deploy: z } = managedDeploy({ name: 'z', deploymentDependsOn: ['x'] }); + let releaseSlow!: () => void; + const zApplied = new Promise((resolve) => { + releaseSlow = resolve; + }); + (createKubernetesApplyJob as jest.Mock).mockImplementation(async ({ deploy }: { deploy: Deploy }) => { + if (deploy === z) releaseSlow(); + if (deploy === slow) await zApplied; + }); + const manager = new DeploymentManager([slow, x, z]); + + try { + await expect(manager.deploy()).resolves.toEqual({ failed: [] }); + expect(createKubernetesApplyJob).toHaveBeenCalledTimes(3); + } finally { + (createKubernetesApplyJob as jest.Mock).mockResolvedValue(undefined); + } + }); + + it('fences every ownership patch on the generation it was handed', async () => { + const { deploy, where } = managedDeploy({ name: 'chart', type: 'helm' }); + (shouldUseNativeHelm as jest.Mock).mockResolvedValue(false); + (deployHelm as jest.Mock).mockImplementation(async (_deploys: Deploy[], options: any) => { + await options.providerSubmissionGate(deploy); + }); + const manager = new DeploymentManager([deploy], { expectedGeneration: 4 }); + + await expect(manager.deploy()).resolves.toEqual({ failed: [] }); + + expect(where).toHaveBeenCalled(); + for (const [criteria] of where.mock.calls) { + expect(criteria).toEqual({ id: deploy.id, runUUID: deploy.runUUID, desiredGeneration: 4 }); + } + }); + it('claims a persisted deploy whose nullable run identity has not been assigned yet', async () => { // The deploys.runUUID database column is nullable even though the model field is typed as string. const { deploy, patch, where } = managedDeploy({ name: 'unclaimed-run', runUUID: null }); diff --git a/src/server/mcp/__tests__/deployRequestState.test.ts b/src/server/mcp/__tests__/deployRequestState.test.ts new file mode 100644 index 00000000..e331b8f9 --- /dev/null +++ b/src/server/mcp/__tests__/deployRequestState.test.ts @@ -0,0 +1,85 @@ +/** + * Copyright 2026 GoodRx, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +jest.mock('server/services/build', () => ({ + __esModule: true, + default: jest.fn().mockImplementation(() => ({ getBuildByUUID: jest.fn() })), +})); + +jest.mock('server/models/Repository', () => ({ + __esModule: true, + default: { query: jest.fn() }, +})); + +jest.mock('server/lib/environments/readiness', () => ({ + getEnvironmentPhase: jest.fn(), + isEnvironmentTerminal: jest.fn(), +})); + +jest.mock('../config', () => ({ + DEFAULT_MCP_WAIT_SECONDS: 10, + MAX_MCP_WAIT_SECONDS: 15, + loadMcpRuntimeConfig: jest.fn(), +})); + +jest.mock('../tools/core/getEnvironment', () => ({ + isEnvironmentBuild: jest.fn(), + serializeEnvironmentState: jest.fn(), +})); + +jest.mock('../tools/core/listRepositories', () => ({ + mapCoreToolError: jest.fn((error: unknown) => error), +})); + +import { deployRequestState } from '../tools/core/waitForEnvironment'; + +const row = (runUUID: string, desiredGeneration: number, observedGeneration: number, active = true) => ({ + runUUID, + desiredGeneration, + observedGeneration, + active, +}); + +describe('deployRequestState', () => { + it('settles a whole-environment deploy id once every active row has caught up', () => { + const build: any = { runUUID: 'env-run', deploys: [row('a', 3, 3), row('b', 3, 2, false)] }; + expect(deployRequestState(build, 'env-run')).toBe('settled'); + + build.deploys = [row('a', 3, 3), row('b', 3, 2)]; + expect(deployRequestState(build, 'env-run')).toBe('pending'); + }); + + it('follows the rows a service-scoped run owns', () => { + const build: any = { runUUID: 'env-run', deploys: [row('run-b', 4, 3), row('run-c', 2, 2)] }; + expect(deployRequestState(build, 'run-b')).toBe('pending'); + expect(deployRequestState(build, 'run-c')).toBe('settled'); + }); + + it('falls back to the accepted entry when the rows were taken by a newer intent', () => { + const build: any = { + runUUID: 'env-run', + deploys: [row('run-newer', 5, 4)], + acceptedRefs: { + 'source:1:main': { requestId: 'run-old', gen: 4, observedGen: 4 }, + 'source:2:main': { requestId: 'run-waiting', gen: 5 }, + }, + }; + expect(deployRequestState(build, 'run-old')).toBe('settled'); + expect(deployRequestState(build, 'run-waiting')).toBe('pending'); + expect(deployRequestState(build, 'run-unknown')).toBe('unknown'); + expect(deployRequestState({ runUUID: 'env-run', deploys: [], acceptedRefs: null } as any, 'x')).toBe('unknown'); + }); +}); diff --git a/src/server/mcp/tools/core/waitForEnvironment.ts b/src/server/mcp/tools/core/waitForEnvironment.ts index 2434c5fc..98b26e98 100644 --- a/src/server/mcp/tools/core/waitForEnvironment.ts +++ b/src/server/mcp/tools/core/waitForEnvironment.ts @@ -257,6 +257,24 @@ interface EvaluatedWait { note?: string; } +type DeployRequestState = 'settled' | 'pending' | 'unknown'; + +/** A deploy id is settled once every service row it selected has caught up with its desired generation. */ +export function deployRequestState(build: LoadedEnvironment['build'], deployId: string): DeployRequestState { + const rows = (build.deploys ?? []).filter((deploy) => deploy.active !== false); + const pendingRow = (deploy: { desiredGeneration?: number; observedGeneration?: number }) => + Number(deploy.desiredGeneration) > Number(deploy.observedGeneration); + if (build.runUUID === deployId) return rows.some(pendingRow) ? 'pending' : 'settled'; + const owned = rows.filter((deploy) => deploy.runUUID === deployId); + if (owned.length > 0) return owned.some(pendingRow) ? 'pending' : 'settled'; + const refs = build.acceptedRefs; + const entry = + refs && typeof refs === 'object' ? Object.values(refs).find((intent) => intent?.requestId === deployId) : undefined; + // An accepted entry whose job finished, including one whose rows a newer intent took over, is settled. + if (entry) return entry.observedGen === entry.gen ? 'settled' : 'pending'; + return 'unknown'; +} + function evaluateWait( target: EnvironmentWaitLoadedTarget, goal: EnvironmentWaitGoal, @@ -281,7 +299,7 @@ function evaluateWait( note: 'This exact environment is being or has been destroyed, so the deploy cannot finish.', }; } - if (build.runUUID !== deployId) { + if (deployRequestState(build, deployId) !== 'settled') { return {}; } } @@ -408,7 +426,7 @@ export function createWaitForEnvironmentToolDefinition( const liveCurrent = current as Extract; const environment = serializeEnvironmentState(liveCurrent.loaded, { format: 'concise' }); const phase = getEnvironmentPhase(liveCurrent.loaded.build); - if (deployId && liveCurrent.loaded.build.runUUID !== deployId) { + if (deployId && deployRequestState(liveCurrent.loaded.build, deployId) === 'unknown') { return { target: { uuid, environmentId }, result: { diff --git a/src/server/models/Deploy.ts b/src/server/models/Deploy.ts index df729d4e..2f6facca 100644 --- a/src/server/models/Deploy.ts +++ b/src/server/models/Deploy.ts @@ -47,6 +47,8 @@ export default class Deploy extends Model { deployable?: Deployable; runUUID: string; + desiredGeneration!: number; + observedGeneration!: number; replicaCount: number; diff --git a/src/server/services/__tests__/apiEnvironment.test.ts b/src/server/services/__tests__/apiEnvironment.test.ts index 53aec922..d953106b 100644 --- a/src/server/services/__tests__/apiEnvironment.test.ts +++ b/src/server/services/__tests__/apiEnvironment.test.ts @@ -152,6 +152,7 @@ function makeService({ const deployQueryChain: any = { patch: jest.fn(() => deployQueryChain), where: jest.fn(() => deployQueryChain), + select: jest.fn(() => ({ where: jest.fn().mockResolvedValue([]) })), }; const models = { Build: { @@ -160,6 +161,7 @@ function makeService({ query: jest.fn(() => ({ findOne: jest.fn(() => buildFindOneChain), findById: jest.fn(), + select: jest.fn(() => ({ findById: jest.fn(() => ({ forUpdate: jest.fn().mockResolvedValue(undefined) })) })), patchAndFetchById: jest.fn(async (_id: number, patch: any) => ({ uuid: 'x', ...patch })), patch: jest.fn().mockReturnThis(), where: jest.fn().mockReturnThis(), @@ -2090,7 +2092,7 @@ describe('applyApiEnvironmentPatch', () => { grantLock({ unlock, extend: jest.fn() }); await pause; - expect((service as any).redlock.lock).toHaveBeenCalledWith('build-deployment.1', 15 * 60 * 1000); + expect((service as any).redlock.lock).toHaveBeenCalledWith('build-deployment.1', 60_000); expect(patchFn).toHaveBeenCalledWith( expect.objectContaining({ deployEnabled: false, runUUID: expect.any(String) }) ); @@ -3187,13 +3189,13 @@ describe('processDeleteQueue', () => { expect(lockWithOptions).toHaveBeenNthCalledWith( 1, 'build-deployment.3', - 15 * 60 * 1000, + 60_000, expect.objectContaining({ retryCount: 4 }) ); expect(lockWithOptions).toHaveBeenNthCalledWith( 2, 'build-promotion.3', - 15 * 60 * 1000, + 60_000, expect.objectContaining({ retryCount: 4 }) ); }); @@ -3868,15 +3870,7 @@ describe('recordBuildFailure teardown ownership', () => { await (service as any).recordBuildFailure(build, BuildStatus.ERROR, 'new-run', new Error('boom'), 'fallback'); - expect(updateStatus).toHaveBeenCalledWith( - build, - BuildStatus.ERROR, - 'new-run', - true, - true, - expect.any(Error), - undefined - ); + expect(updateStatus).toHaveBeenCalledWith(build, BuildStatus.ERROR, 'new-run', true, true, expect.any(Error)); }); it('re-stamps PR builds unconditionally', async () => { diff --git a/src/server/services/__tests__/build.test.ts b/src/server/services/__tests__/build.test.ts index 10925488..f52495ee 100644 --- a/src/server/services/__tests__/build.test.ts +++ b/src/server/services/__tests__/build.test.ts @@ -214,15 +214,15 @@ import BuildService, { assertIdempotentReplayAllowed, assertEnvironmentDestroyAllowed, } from '../build'; -import { BuildKind, BuildStatus, DeployStatus, DeployTypes } from 'shared/constants'; +import { BuildKind, BuildStatus, DeployStatus, DeployTypes, PullRequestStatus } from 'shared/constants'; import * as github from 'server/lib/github'; import { ingressBannerSnippet } from 'server/lib/helm/utils'; import { ParsingError, YamlConfigParser } from 'server/lib/yamlConfigParser'; import { ValidationError, YamlConfigValidator } from 'server/lib/yamlConfigValidator'; import { DeploymentManager, DeploymentSupersededError } from 'server/lib/deploymentManager/deploymentManager'; import { UniqueViolationError } from 'objection'; -import AgentPrewarmService from 'server/services/agentPrewarm'; import { AuthorityLockLostError } from 'server/lib/authorityLock'; +import AgentPrewarmService from 'server/services/agentPrewarm'; import { LifecycleError } from 'server/lib/errors'; function createThenableQuery(result: any[] = []) { @@ -556,7 +556,8 @@ describe('BuildService status updates', () => { status: BuildStatus.DEPLOYED, statusMessage: '', }); - expect(query.where).toHaveBeenCalledWith({ id: 1, runUUID: 'run-1' }); + expect(query.where).toHaveBeenCalledWith({ id: 1 }); + expect(query.where).toHaveBeenCalledWith('runUUID', 'run-1'); }); test('does not abort teardown status progress when webhook notification enqueue fails', async () => { @@ -600,44 +601,6 @@ describe('BuildService status updates', () => { }); expect(webhookAdd).toHaveBeenCalledTimes(1); }); - - test('does not publish after a newer desired generation takes ownership', async () => { - const query = statusQuery(0); - const webhookAdd = jest.fn(); - const activityUpdate = jest.fn(); - const buildService = new BuildService( - { - models: { Build: { query: jest.fn(() => query) } }, - services: { - ActivityStream: { updatePullRequestActivityStream: activityUpdate }, - Webhook: { webhookQueue: { add: webhookAdd } }, - }, - } as any, - {} as any, - {} as any, - { - registerQueue: jest.fn(() => ({ add: mockQueueAdd, process: jest.fn(), on: jest.fn() })), - } as any - ); - const build = { - id: 1, - uuid: 'sample-build', - runUUID: 'run-a', - status: BuildStatus.DEPLOYING, - kind: BuildKind.ENVIRONMENT, - deploys: [], - pullRequest: { repository: {} }, - reload: jest.fn().mockResolvedValue(undefined), - $fetchGraph: jest.fn().mockResolvedValue(undefined), - }; - - await buildService.updateStatusAndComment(build as any, BuildStatus.DEPLOYED, 'run-a', true, true, null, 2); - - expect(query.where).toHaveBeenCalledWith('desiredGeneration', 2); - expect(build.status).toBe(BuildStatus.DEPLOYING); - expect(activityUpdate).not.toHaveBeenCalled(); - expect(webhookAdd).not.toHaveBeenCalled(); - }); }); describe('BuildService destroyBuildEnvironment', () => { @@ -1268,8 +1231,8 @@ describe('BuildService stale deploy reconciliation', () => { return true; }); const promotion = jest - .spyOn(buildService, 'withCurrentBuildPromotionLock') - .mockImplementation(async (_buildId, isCurrent, action) => { + .spyOn(buildService, 'withCurrentDeployPromotionLock') + .mockImplementation(async (_deployId, isCurrent, action) => { expect(await isCurrent()).toBe(true); return { admitted: true, value: await action() }; }); @@ -1284,7 +1247,8 @@ describe('BuildService stale deploy reconciliation', () => { 3 ); - expect(promotion).toHaveBeenCalledWith(10, expect.any(Function), nativeAction); + // Removal serializes with the service's own rollout lock, not the environment-wide one. + expect(promotion).toHaveBeenCalledWith(77, expect.any(Function), nativeAction); expect(mockDeleteServiceRows).toHaveBeenCalledWith({ buildId: 10, deployableIds: [1] }); }); @@ -1294,7 +1258,7 @@ describe('BuildService stale deploy reconciliation', () => { await options.nativeMutationGate(async () => true); return true; }); - jest.spyOn(buildService, 'withCurrentBuildPromotionLock').mockResolvedValue({ admitted: false }); + jest.spyOn(buildService, 'withCurrentDeployPromotionLock').mockResolvedValue({ admitted: false }); await expect( (buildService as any).reconcileDeletedDeployables( @@ -1380,7 +1344,8 @@ describe('BuildService stale deploy reconciliation', () => { targetRepoId, undefined, undefined, - targetRepoId + targetRepoId, + undefined ); expect(reconcileDeletedDeployables).not.toHaveBeenCalled(); expect(upsertWebhooksWithYaml).toHaveBeenCalledWith(build, build.pullRequest, null); @@ -1424,102 +1389,11 @@ describe('BuildService deployment reconciliation', () => { return { service, buildQuery, add }; }; - const reconciliationWorkerHarness = () => { - const { service } = serviceHarness(); - const failure = new Error('reconciliation infrastructure failed'); - const claim = { - generation: 7, - token: 'run-current', - dirty: [{ scopeKey: 'all', intent: { type: 'all', requestId: 'run-current', gen: 7 } }], - }; - const build = createBuild({ id: 1, runUUID: claim.token }); - - jest - .spyOn(service as any, 'tryWithDeploymentGenerationLock') - .mockImplementation(async (...args: any[]) => (args[2] as () => Promise)()); - const claimReconciliation = jest.spyOn(service as any, 'claimDeploymentReconciliation').mockResolvedValue(claim); - const withDeploymentLock = jest - .spyOn(service as any, 'withCurrentBuildDeploymentLock') - .mockImplementation(async (...args: any[]) => { - const isCurrent = args[1] as () => Promise; - const action = args[2] as () => Promise; - expect(await isCurrent()).toBe(true); - return { admitted: true, value: await action() }; - }); - const loadBuild = jest.spyOn(service as any, 'loadBuildDeploymentAuthority').mockResolvedValue(build); - jest.spyOn(service as any, 'claimDeploymentRun').mockResolvedValue(claim.token); - jest.spyOn(service as any, 'deploymentReconciliationScopes').mockImplementation(() => { - throw failure; - }); - const isCurrent = jest.spyOn(service as any, 'isDeploymentRunCurrent').mockResolvedValue(true); - const recordFailure = jest.spyOn(service as any, 'recordBuildFailure').mockResolvedValue(undefined); - const markObserved = jest.spyOn(service as any, 'markDeploymentReconciliationObserved').mockResolvedValue(true); - - const job = (attemptsMade: number) => ({ - data: { buildId: 1, generation: claim.generation }, - attemptsMade, - opts: { attempts: 10 }, - }); - - return { - service, - failure, - claim, - build, - claimReconciliation, - withDeploymentLock, - loadBuild, - isCurrent, - recordFailure, - markObserved, - job, - }; - }; - beforeEach(() => { jest.clearAllMocks(); mockAcceptDeploymentIntent.mockResolvedValue({ accepted: true, generation: 1, scopeKey: 'all' }); }); - const deploymentScopeHarness = (buildImagesResult: boolean) => { - const { service } = serviceHarness(); - const build = createBuild({ id: 1, runUUID: 'run-current', namespace: 'env-sample' }); - jest.spyOn(service as any, 'isDeploymentRunCurrent').mockResolvedValue(true); - const buildImages = jest.spyOn(service as any, 'buildImages').mockResolvedValue(buildImagesResult); - jest.spyOn(service as any, 'deployCLIServices').mockResolvedValue(true); - const updateStatus = jest.spyOn(service as any, 'updateStatusAndComment').mockResolvedValue(undefined); - const applyManifests = jest.spyOn(service as any, 'generateAndApplyManifests').mockResolvedValue(true); - const preparation = { - build, - runUUID: 'run-current', - githubRepositoryId: 100, - sourceGithubRepositoryId: 100, - sourceRef: 'commit-a', - sourceBranch: 'main', - }; - return { service, preparation, buildImages, updateStatus, applyManifests }; - }; - - test('a failed image phase reports ERROR and never reaches manifest apply', async () => { - const { service, preparation, buildImages, updateStatus, applyManifests } = deploymentScopeHarness(false); - - const result = await (service as any).executeDeploymentScope(preparation, 7); - - expect(result).toEqual({ status: BuildStatus.ERROR }); - expect(buildImages).toHaveBeenCalledTimes(1); - expect(updateStatus).not.toHaveBeenCalled(); - expect(applyManifests).not.toHaveBeenCalled(); - }); - - test('a successful image phase proceeds to manifest apply and reports DEPLOYED', async () => { - const { service, preparation, applyManifests } = deploymentScopeHarness(true); - - const result = await (service as any).executeDeploymentScope(preparation, 7); - - expect(result).toEqual({ status: BuildStatus.DEPLOYED }); - expect(applyManifests).toHaveBeenCalledTimes(1); - }); - test('signals only the exact durable generation accepted by the mailbox', async () => { const { service, add } = serviceHarness(); @@ -1668,15 +1542,6 @@ describe('BuildService deployment reconciliation', () => { expect(add).not.toHaveBeenCalled(); }); - test('the recovery sweep re-signals durable pending work', async () => { - const { service, buildQuery, add } = serviceHarness(); - buildQuery.limit.mockResolvedValueOnce([{ id: 7, desiredGeneration: '3' }]); - - await service.enqueuePendingDeploymentReconciliations(); - - expect(add).toHaveBeenCalledWith('reconcile', { buildId: 7, generation: 3 }, { jobId: 'reconcile-7-3' }); - }); - test('service redeploy uses the effective source repository and leaves row ownership to the worker', async () => { const { service, buildQuery, add } = serviceHarness(); const directPatch = jest.fn(); @@ -1737,211 +1602,6 @@ describe('BuildService deployment reconciliation', () => { expect(add).not.toHaveBeenCalled(); }); - test('repository redeploy bulk-claims every Deploy from that repository without a service-id predicate', async () => { - const deployClaim: any = { - patch: jest.fn(() => deployClaim), - where: jest.fn(() => deployClaim), - then: (resolve: (value: number) => void, reject: (reason: unknown) => void) => - Promise.resolve(2).then(resolve, reject), - }; - const deploys = [ - { id: 11, githubRepositoryId: 42, branchName: 'main' }, - { id: 12, githubRepositoryId: 42, branchName: 'release' }, - { id: 13, githubRepositoryId: 99, branchName: 'main' }, - ]; - const findOrCreateDeploys = jest.fn().mockResolvedValue(deploys); - const service = new BuildService( - { - models: { Deploy: { query: jest.fn(() => deployClaim) } }, - services: { Deploy: { findOrCreateDeploys } }, - } as any, - {} as any, - {} as any, - queueManager() as any - ); - jest.spyOn(service as any, 'isDeploymentRunCurrent').mockResolvedValue(true); - jest.spyOn(service, 'markConfigurationsAsBuilt').mockResolvedValue(undefined); - jest.spyOn(service, 'updateStatusAndComment').mockResolvedValue(undefined); - mockGenerateGraph.mockRejectedValueOnce(new Error('graph omitted from scope assertion')); - const build: any = { - id: 4, - uuid: 'large-static', - runUUID: 'run-c', - environment: { id: 7 }, - pullRequest: { - fullName: 'org/root', - branchName: 'main', - latestCommit: 'root-sha', - repository: { githubRepositoryId: 1 }, - $fetchGraph: jest.fn().mockResolvedValue(undefined), - }, - $fetchGraph: jest.fn().mockResolvedValue(undefined), - $setRelated: jest.fn(), - }; - - await (service as any).prepareDeploymentScope(build, 42, undefined, { - runUUID: 'run-c', - }); - - expect(deployClaim.patch).toHaveBeenCalledWith({ runUUID: 'run-c' }); - expect(deployClaim.where).toHaveBeenCalledWith({ buildId: 4 }); - expect(deployClaim.where).toHaveBeenCalledWith('githubRepositoryId', 42); - expect(deployClaim.where).not.toHaveBeenCalledWith('branchName', expect.anything()); - expect(deployClaim.where).not.toHaveBeenCalledWith('id', expect.anything()); - expect(deploys.map((deploy: any) => deploy.runUUID)).toEqual(['run-c', 'run-c', undefined]); - }); - - test('a stale B signal cannot claim or execute C mailbox contents', async () => { - const row = { - desiredGeneration: 3, - observedGeneration: 0, - acceptedRefs: { - 'source:100:main': { - type: 'source', - requestId: 'request-c', - target: 'repository', - githubRepositoryId: 100, - branch: 'main', - sha: 'commit-c', - gen: 3, - }, - }, - }; - const query: any = { - findById: jest.fn(() => query), - whereNull: jest.fn().mockResolvedValue(row), - }; - const service = new BuildService( - { models: { Build: { query: jest.fn(() => query) } } } as any, - {} as any, - {} as any, - queueManager() as any - ); - - await expect((service as any).claimDeploymentReconciliation(1, 2)).resolves.toBeNull(); - await expect((service as any).claimDeploymentReconciliation(1, 3)).resolves.toEqual({ - generation: 3, - token: 'request-c', - dirty: [ - { - scopeKey: 'source:100:main', - intent: row.acceptedRefs['source:100:main'], - }, - ], - }); - }); - - test('repository-scoped work stays selective while a root-source intent remains full-scope', () => { - const { service } = serviceHarness(); - const scopes = (service as any).deploymentReconciliationScopes([ - { - scopeKey: 'source:100:main', - intent: { - type: 'source', - requestId: 'repo-c', - target: 'repository', - githubRepositoryId: 100, - branch: 'main', - sha: 'commit-c', - gen: 1, - }, - }, - { - scopeKey: 'source:200:main', - intent: { - type: 'source', - requestId: 'root-d', - target: 'all', - githubRepositoryId: 200, - branch: 'main', - sha: 'commit-d', - gen: 2, - }, - }, - ]); - - // The full pass runs first, then the delivered repo SHA is re-applied as a - // selective floor so a lagging live-head read cannot roll C back to B. - expect(scopes).toEqual([ - { - githubRepositoryId: null, - sourceGithubRepositoryId: 200, - sourceRef: 'commit-d', - sourceBeforeRef: undefined, - sourceBranch: 'main', - }, - { - githubRepositoryId: 100, - sourceGithubRepositoryId: 100, - sourceRef: 'commit-c', - sourceBeforeRef: undefined, - sourceBranch: 'main', - }, - ]); - }); - - test('manual full redeploy cannot erase an older delivered repository SHA', () => { - const { service } = serviceHarness(); - const scopes = (service as any).deploymentReconciliationScopes([ - { - scopeKey: 'source:100:main', - intent: { - type: 'source', - requestId: 'repo-c', - target: 'repository', - githubRepositoryId: 100, - branch: 'main', - sha: 'commit-c', - gen: 1, - }, - }, - { scopeKey: 'all', intent: { type: 'all', requestId: 'manual-all', gen: 2 } }, - ]); - - expect(scopes).toEqual([ - { githubRepositoryId: null }, - { - githubRepositoryId: 100, - sourceGithubRepositoryId: 100, - sourceRef: 'commit-c', - sourceBeforeRef: undefined, - sourceBranch: 'main', - }, - ]); - }); - - test('starts the newest repository source before retained work for another repository', () => { - const { service } = serviceHarness(); - const scopes = (service as any).deploymentReconciliationScopes([ - { - scopeKey: 'source:100:main', - intent: { - type: 'source', - requestId: 'repo-a', - target: 'repository', - githubRepositoryId: 100, - branch: 'main', - sha: 'commit-a', - gen: 1, - }, - }, - { - scopeKey: 'source:200:main', - intent: { - type: 'source', - requestId: 'repo-b', - target: 'repository', - githubRepositoryId: 200, - branch: 'main', - sha: 'commit-b', - gen: 2, - }, - }, - ]); - - expect(scopes.map((scope) => scope.githubRepositoryId)).toEqual([200, 100]); - }); - test('different generations use different execution locks so C does not wait for A', async () => { let releaseA!: () => void; let signalAStarted!: () => void; @@ -2004,116 +1664,6 @@ describe('BuildService deployment reconciliation', () => { expect(applyManifests).not.toHaveBeenCalled(); }); - test('leaves the latest generation pending while PR teardown is in progress', async () => { - const { service, build, markObserved, job } = reconciliationWorkerHarness(); - build.status = BuildStatus.TEARING_DOWN; - - await expect(service.processDeploymentReconciliationQueue(job(0))).resolves.toBeUndefined(); - - expect((service as any).claimDeploymentRun).not.toHaveBeenCalled(); - expect((service as any).deploymentReconciliationScopes).not.toHaveBeenCalled(); - expect(markObserved).not.toHaveBeenCalled(); - }); - - test('retries an intermediate generic reconciliation failure without publishing a terminal error', async () => { - const { service, failure, recordFailure, markObserved, job } = reconciliationWorkerHarness(); - - await expect(service.processDeploymentReconciliationQueue(job(0))).rejects.toBe(failure); - - expect(recordFailure).not.toHaveBeenCalled(); - expect(markObserved).not.toHaveBeenCalled(); - }); - - test('retries a still-current authority-lock loss without publishing terminal failure', async () => { - const { service, recordFailure, markObserved, job } = reconciliationWorkerHarness(); - const lockError = new AuthorityLockLostError('build-deployment.1'); - jest.spyOn(service as any, 'deploymentReconciliationScopes').mockImplementation(() => { - throw lockError; - }); - - await expect(service.processDeploymentReconciliationQueue(job(0))).rejects.toBe(lockError); - - expect(recordFailure).not.toHaveBeenCalled(); - expect(markObserved).not.toHaveBeenCalled(); - }); - - test('ignores a reconciliation failure after generation authority has moved on', async () => { - const { service, recordFailure, markObserved, isCurrent, job } = reconciliationWorkerHarness(); - isCurrent.mockResolvedValue(false); - - await expect(service.processDeploymentReconciliationQueue(job(0))).resolves.toBeUndefined(); - - expect(recordFailure).not.toHaveBeenCalled(); - expect(markObserved).not.toHaveBeenCalled(); - }); - - test('publishes and observes one fenced generic failure on the final queue attempt', async () => { - const { service, failure, build, claim, recordFailure, markObserved, job } = reconciliationWorkerHarness(); - - await expect(service.processDeploymentReconciliationQueue(job(9))).resolves.toBeUndefined(); - - expect(recordFailure).toHaveBeenCalledTimes(1); - expect(recordFailure).toHaveBeenCalledWith( - build, - BuildStatus.ERROR, - claim.token, - failure, - 'Build queue processing failed.', - claim.generation - ); - expect(markObserved).toHaveBeenCalledTimes(1); - expect(markObserved).toHaveBeenCalledWith(1, claim.generation, claim.token); - }); - - test('rethrows a still-current final failure when its terminal write cannot be confirmed', async () => { - const { service, failure, job } = reconciliationWorkerHarness(); - jest.spyOn(service as any, 'recordFinalDeploymentReconciliationFailure').mockResolvedValue(false); - - await expect(service.processDeploymentReconciliationQueue(job(9))).rejects.toBe(failure); - }); - - test('rethrows a still-current generic failure that happens before the run becomes active', async () => { - const { service, failure, loadBuild, recordFailure, markObserved, job } = reconciliationWorkerHarness(); - loadBuild.mockRejectedValueOnce(failure); - - await expect(service.processDeploymentReconciliationQueue(job(0))).rejects.toBe(failure); - - expect(recordFailure).not.toHaveBeenCalled(); - expect(markObserved).not.toHaveBeenCalled(); - }); - - test('claims the token to publish and observe a pre-active failure only on its final attempt', async () => { - const { service, failure, build, claim, withDeploymentLock, loadBuild, recordFailure, markObserved, job } = - reconciliationWorkerHarness(); - loadBuild.mockRejectedValueOnce(failure); - - await expect(service.processDeploymentReconciliationQueue(job(9))).resolves.toBeUndefined(); - - expect(withDeploymentLock).toHaveBeenCalledTimes(2); - expect(recordFailure).toHaveBeenCalledTimes(1); - expect(recordFailure).toHaveBeenCalledWith( - build, - BuildStatus.ERROR, - claim.token, - failure, - 'Build queue processing failed.', - claim.generation - ); - expect(markObserved).toHaveBeenCalledWith(1, claim.generation, claim.token); - }); - - test('does not mistake an authority-read failure for proof that a failed pass is stale', async () => { - const { service, failure, isCurrent, recordFailure, markObserved, job } = reconciliationWorkerHarness(); - const authorityError = new Error('authority read unavailable'); - isCurrent.mockRejectedValueOnce(authorityError); - - await expect(service.processDeploymentReconciliationQueue(job(0))).rejects.toBe(authorityError); - - expect(recordFailure).not.toHaveBeenCalled(); - expect(markObserved).not.toHaveBeenCalled(); - expect(failure).not.toBe(authorityError); - }); - test('deployment scope fails closed when an execution error cannot be fenced by an authority read', async () => { const executionError = new Error('image builder unavailable'); const authorityError = new Error('build authority unavailable'); @@ -2145,7 +1695,7 @@ describe('BuildService deployment reconciliation', () => { }, 7 ) - ).resolves.toBeNull(); + ).rejects.toBe(authorityError); expect(buildQuery.select).toHaveBeenCalledWith( 'id', @@ -2214,31 +1764,127 @@ describe('BuildService deployment reconciliation', () => { head: 'commit-d', }); }); -}); - -describe('idempotency digest + replay authorization (D12)', () => { - const caught = (fn: () => void): any => { - try { - fn(); - } catch (error) { - return error; - } - return null; - }; - const baseInput = () => ({ - repositoryFullName: 'Org/Repo', - branch: 'main', - services: [ - { name: 'web', active: true }, - { name: 'api', active: false, branchOrExternalUrl: 'feature/x' }, + test.each([ + [ + 'the branch head lookup fails', + () => (github.getSHAForBranch as jest.Mock).mockRejectedValue(new Error('rate limited')), ], - env: { B: '2', A: '1' }, - }); - - it('is stable across service order and env key order', () => { - const a = computeIdempotencyRequestDigest(baseInput() as any); - const reordered = { + [ + 'the commit comparison fails', + () => { + (github.getSHAForBranch as jest.Mock).mockResolvedValue('commit-d'); + (github.compareCommits as jest.Mock).mockRejectedValue(new Error('rate limited')); + }, + ], + ])('keeps the accepted SHA as the floor when %s', async (_name, arrange) => { + const repositoryQuery: any = { + findOne: jest.fn(() => repositoryQuery), + whereNull: jest.fn().mockResolvedValue({ fullName: 'org/service' }), + }; + const service = new BuildService( + { models: { Repository: { query: jest.fn(() => repositoryQuery) } } } as any, + {} as any, + {} as any, + queueManager() as any + ); + arrange(); + + await expect( + (service as any).resolveCurrentSourceRef({ + githubRepositoryId: 42, + sourceGithubRepositoryId: 42, + sourceBranch: 'main', + sourceRef: 'commit-c', + }) + ).resolves.toBe('commit-c'); + }); + + test('a rollout error of a current run is recorded on its rows and the scope still returns its summary', async () => { + const executionError = new Error('kubectl apply exited 1'); + const service = new BuildService({ models: {} } as any, {} as any, {} as any, queueManager() as any); + const build = createBuild({ id: 1, runUUID: 'run-current', namespace: 'env-sample' }); + jest.spyOn(service, 'buildImages').mockResolvedValue(undefined as any); + jest.spyOn(service, 'deployCLIServices').mockResolvedValue(true); + jest.spyOn(service as any, 'isDeploymentRunCurrent').mockResolvedValue(true); + jest.spyOn(service as any, 'publishDerivedBuildStatus').mockResolvedValue(undefined); + jest.spyOn(service as any, 'generateAndApplyManifests').mockRejectedValue(executionError); + const recordFailure = jest.spyOn(service as any, 'recordFailureOnOwnedDeploys').mockResolvedValue(undefined); + const summary = { deployed: false }; + jest.spyOn(service as any, 'summarizeIntentDeploys').mockResolvedValue(summary); + + await expect( + (service as any).executeDeploymentScope( + { + build, + runUUID: 'run-current', + githubRepositoryId: 100, + sourceGithubRepositoryId: 100, + sourceRef: 'commit-a', + sourceBranch: 'main', + }, + 7 + ) + ).resolves.toBe(summary); + + expect(recordFailure).toHaveBeenCalledWith(1, 'run-current', 7, executionError, 'Deployment failed.'); + }); + + test('a rollout error of a run that lost its rows meanwhile is dropped without recording anything', async () => { + const service = new BuildService({ models: {} } as any, {} as any, {} as any, queueManager() as any); + const build = createBuild({ id: 1, runUUID: 'run-old', namespace: 'env-sample' }); + jest.spyOn(service, 'buildImages').mockResolvedValue(undefined as any); + jest.spyOn(service, 'deployCLIServices').mockResolvedValue(true); + jest + .spyOn(service as any, 'isDeploymentRunCurrent') + .mockResolvedValueOnce(true) + .mockResolvedValueOnce(true) + .mockResolvedValueOnce(false); + jest.spyOn(service as any, 'publishDerivedBuildStatus').mockResolvedValue(undefined); + jest.spyOn(service as any, 'generateAndApplyManifests').mockRejectedValue(new Error('kubectl apply exited 1')); + const recordFailure = jest.spyOn(service as any, 'recordFailureOnOwnedDeploys').mockResolvedValue(undefined); + + await expect( + (service as any).executeDeploymentScope( + { + build, + runUUID: 'run-old', + githubRepositoryId: 100, + sourceGithubRepositoryId: 100, + sourceRef: 'a', + sourceBranch: 'main', + }, + 7 + ) + ).resolves.toBeNull(); + + expect(recordFailure).not.toHaveBeenCalled(); + }); +}); + +describe('idempotency digest + replay authorization (D12)', () => { + const caught = (fn: () => void): any => { + try { + fn(); + } catch (error) { + return error; + } + return null; + }; + + const baseInput = () => ({ + repositoryFullName: 'Org/Repo', + branch: 'main', + services: [ + { name: 'web', active: true }, + { name: 'api', active: false, branchOrExternalUrl: 'feature/x' }, + ], + env: { B: '2', A: '1' }, + }); + + it('is stable across service order and env key order', () => { + const a = computeIdempotencyRequestDigest(baseInput() as any); + const reordered = { repositoryFullName: 'Org/Repo', branch: 'main', services: [ @@ -2360,6 +2006,7 @@ describe('BuildService focused changed-line coverage', () => { const deployQuery = (deploys: any[]) => { const query: any = { where: jest.fn(() => query), + whereNotIn: jest.fn(() => query), withGraphFetched: jest.fn().mockResolvedValue(deploys), }; return query; @@ -2436,6 +2083,8 @@ describe('BuildService focused changed-line coverage', () => { 'deployPipelineId', 'githubRepositoryId', 'runUUID', + 'desiredGeneration', + 'observedGeneration', 'publicUrl', 'dockerImage', 'buildLogs', @@ -2503,7 +2152,7 @@ describe('BuildService focused changed-line coverage', () => { expect(buildImage.mock.calls.map(([deploy]) => deploy.uuid)).toEqual(['docker', 'github', 'helm']); }); - test('keeps every static execution lane scoped to the changed repository and branch', async () => { + test('keeps legacy lanes scoped to the changed repository and branch while a fenced lane takes exactly the rows it claimed', async () => { const imageQuery = deployQuery([]); const cliQuery = deployQuery([]); const manifestQuery = deployQuery([]); @@ -2532,7 +2181,7 @@ describe('BuildService focused changed-line coverage', () => { namespace: build.namespace, }); - for (const query of [imageQuery, cliQuery, manifestQuery]) { + for (const query of [imageQuery, cliQuery]) { expect(query.where).toHaveBeenCalledWith({ buildId: 4, runUUID: 'run-c', @@ -2540,6 +2189,8 @@ describe('BuildService focused changed-line coverage', () => { branchName: 'main', }); } + expect(manifestQuery.where).toHaveBeenCalledWith({ buildId: 4, runUUID: 'run-c' }); + expect(manifestQuery.where).toHaveBeenCalledWith('desiredGeneration', 3); }); test('treats a retained targeted scope with no remaining Deploy rows as a successful no-op', async () => { @@ -2550,10 +2201,102 @@ describe('BuildService focused changed-line coverage', () => { await expect(service.deployCLIServices(build as any, 'run-c', 42, 'sha-c', 'main')).resolves.toBe(true); }); + test('a CLI deploy that reports failure is recorded on its row as a phase failure', async () => { + const deploy = { id: 5, uuid: 'aurora-1', active: true, deployable: { type: DeployTypes.AURORA_RESTORE } }; + mockDeployQuery.mockReturnValue(deployQuery([deploy])); + const deployCLI = jest.fn().mockResolvedValue(false); + const service = serviceWith({ services: { Deploy: { deployCLI } } }); + const recordPhaseFailure = jest.spyOn(service as any, 'recordPhaseFailure').mockResolvedValue(undefined); + const build = { id: 4, $fetchGraph: jest.fn().mockResolvedValue(undefined) }; + + await expect(service.deployCLIServices(build as any, 'run-c', 42, 'sha-c', 'main', 42, 3)).resolves.toBe(false); + + expect(deployCLI).toHaveBeenCalledWith(deploy, 'run-c', 'sha-c', 42, 'main', undefined, 3); + expect(recordPhaseFailure).toHaveBeenCalledWith(deploy, 'run-c', DeployStatus.ERROR, 'CLI deploy failed.', 3); + }); + + test('an aborted rollout does not fail a service whose prerequisite is outside the run', async () => { + const managed = (name: string, deploymentDependsOn: string[]) => { + const where = jest.fn().mockResolvedValue(1); + const patch = jest.fn().mockReturnValue({ where }); + const deploy = { + id: name.length, + uuid: name, + runUUID: 'run-g', + manifest: 'kind: ConfigMap', + build: { namespace: 'ns' }, + deployable: { name, type: 'github', deploymentDependsOn }, + service: { type: 'github' }, + $query: () => ({ patch }), + $fetchGraph: jest.fn().mockResolvedValue(undefined), + } as any; + return { deploy, patch }; + }; + const a = managed('a', []); + const b = managed('b', ['x']); + const lost = new AuthorityLockLostError('deploy-promotion.1'); + const manager = new DeploymentManager([a.deploy, b.deploy], { + nativeMutationGate: (async (deploy: any, action: () => Promise) => { + if (deploy === a.deploy) throw lost; + return { admitted: true, value: await action() }; + }) as any, + }); + + await expect(manager.deploy()).rejects.toBe(lost); + expect(b.patch).not.toHaveBeenCalledWith(expect.objectContaining({ status: DeployStatus.DEPLOY_FAILED })); + }); + + test('acceptedSourcePins keeps the newest accepted SHA per repository and branch, including full-environment pushes', () => { + const service = serviceWith({}); + + const pins = service['acceptedSourcePins']({ + 'source:100:main': { + type: 'source', + target: 'repository', + requestId: 'r3', + githubRepositoryId: 100, + branch: 'main', + sha: 'a1', + gen: 3, + }, + 'source:100:main:all': { + type: 'source', + target: 'all', + requestId: 'r5', + githubRepositoryId: 100, + branch: 'main', + sha: 'a2', + gen: 5, + observedGen: 5, + }, + 'source:200:main:all': { + type: 'source', + target: 'all', + requestId: 'r6', + githubRepositoryId: 200, + branch: 'main', + sha: 'b1', + gen: 6, + }, + 'repository:300': { type: 'repository', requestId: 'r7', githubRepositoryId: 300, gen: 7 }, + all: { type: 'all', requestId: 'r8', gen: 8 }, + }); + + expect(pins).toEqual({ '100:main': 'a2', '200:main': 'b1' }); + }); + test('fails image and CLI processing cleanly for a loaded deploy missing its deployable', async () => { const missing = { uuid: 'missing', active: true }; mockDeployQuery.mockReturnValue(deployQuery([{ uuid: 'inactive', active: false }, missing])); - const service = serviceWith({ services: { Deploy: { buildImage: jest.fn(), deployCLI: jest.fn() } } }); + const failureWrite: any = { + patch: jest.fn(() => failureWrite), + where: jest.fn(() => failureWrite), + whereNotIn: jest.fn().mockResolvedValue(1), + }; + const service = serviceWith({ + models: { Deploy: { query: jest.fn(() => failureWrite) } }, + services: { Deploy: { buildImage: jest.fn(), deployCLI: jest.fn() } }, + }); await expect(service.buildImages({ id: 4 } as any, 'build-run')).resolves.toBe(false); await expect( @@ -2561,6 +2304,43 @@ describe('BuildService focused changed-line coverage', () => { ).resolves.toBe(false); }); + test('a fenced image phase skips rows this run already finished and fails an orphaned row on its own', async () => { + const orphan = { id: 31, uuid: 'orphan', active: true, runUUID: 'run-g', deployable: null }; + const docker = { id: 32, uuid: 'docker', active: true, runUUID: 'run-g', deployable: { type: DeployTypes.DOCKER } }; + const read = deployQuery([orphan, docker]); + mockDeployQuery.mockReturnValue(read); + const failureWrite: any = { + patch: jest.fn(() => failureWrite), + where: jest.fn(() => failureWrite), + whereNotIn: jest.fn().mockResolvedValue(1), + }; + const buildImage = jest.fn().mockResolvedValue(true); + const service = serviceWith({ + models: { Deploy: { query: jest.fn(() => failureWrite) } }, + services: { Deploy: { buildImage } }, + }); + + await expect( + service.buildImages({ id: 4, $fetchGraph: jest.fn() } as any, 'run-g', 100, null, 'main', 3) + ).resolves.toBe(true); + + expect(read.where).toHaveBeenCalledWith({ buildId: 4, runUUID: 'run-g' }); + expect(read.whereNotIn).toHaveBeenCalledWith('status', [ + DeployStatus.READY, + DeployStatus.DEPLOYED, + DeployStatus.ERROR, + DeployStatus.BUILD_FAILED, + DeployStatus.DEPLOY_FAILED, + ]); + expect(buildImage).toHaveBeenCalledTimes(1); + expect(buildImage).toHaveBeenCalledWith(docker, 0, 'run-g', null, 100, 'main', 3, undefined, undefined); + expect(failureWrite.patch).toHaveBeenCalledWith({ + status: DeployStatus.ERROR, + statusMessage: 'Service definition missing.', + }); + expect(failureWrite.where).toHaveBeenCalledWith({ id: 31, runUUID: 'run-g', desiredGeneration: 3 }); + }); + test('deploys only active CLI services and records an individual CLI failure', async () => { const cliSuccess = { uuid: 'cli-success', @@ -2596,6 +2376,102 @@ describe('BuildService focused changed-line coverage', () => { }); }); + test('a legacy rollout without a generation still throws when a service fails', async () => { + const deploy = { + id: 12, + uuid: 'legacy-web', + active: true, + deployable: { name: 'web', type: DeployTypes.GITHUB, deploymentDependsOn: [] }, + }; + mockDeployQuery.mockReturnValue(deployQuery([deploy])); + const service = serviceWith({}); + const build = { + id: 4, + uuid: 'legacy', + namespace: 'env-legacy', + kind: BuildKind.SANDBOX, + $query: jest.fn(), + }; + jest.spyOn(DeploymentManager.prototype, 'deploy').mockResolvedValueOnce({ failed: [deploy as any] }); + + await expect( + service.generateAndApplyManifests({ build: build as any, githubRepositoryId: null, namespace: build.namespace }) + ).rejects.toThrow('Deployment failed for web'); + }); + + test('a fenced rollout keeps failed rows in the plan by name and gives the manager no way to wait on other runs', async () => { + const failed = { + id: 13, + uuid: 'migrations-run', + active: true, + status: DeployStatus.BUILD_FAILED, + runUUID: 'run-g', + deployable: { name: 'migrations', type: DeployTypes.GITHUB, deploymentDependsOn: [] }, + }; + const api = { + id: 14, + uuid: 'api-run', + active: true, + status: DeployStatus.BUILT, + runUUID: 'run-g', + deployable: { name: 'api', type: DeployTypes.GITHUB, deploymentDependsOn: ['cache'] }, + }; + mockDeployQuery.mockReturnValue(deployQuery([failed, api])); + const service = serviceWith({ + models: { + Build: { + query: jest.fn(() => ({ + findById: jest.fn(() => ({ + select: jest + .fn() + .mockResolvedValue({ id: 4, status: BuildStatus.DEPLOYING, deployEnabled: true, pullRequestId: null }), + })), + })), + }, + Deploy: { + query: jest.fn(() => ({ + findOne: jest.fn(() => ({ select: jest.fn().mockResolvedValue({ id: 14, observedGeneration: 1 }) })), + })), + }, + }, + }); + jest.spyOn(service as any, 'updateDeploysImageDetails').mockResolvedValue(undefined); + const build = { + id: 4, + uuid: 'fenced', + namespace: 'env-fenced', + kind: BuildKind.SANDBOX, + pullRequest: null, + $query: jest.fn(), + }; + let managerOptions: Record = {}; + const deployments = jest + .spyOn(DeploymentManager.prototype, 'deploy') + .mockImplementationOnce(async function (this: any) { + managerOptions = this.options; + return { failed: [] }; + }); + + try { + await expect( + service.generateAndApplyManifests({ + build: build as any, + runUUID: 'run-g', + expectedGeneration: 2, + githubRepositoryId: null, + namespace: build.namespace, + enqueueIngress: false, + unresolvedServices: ['billing'], + }) + ).resolves.toBe(true); + } finally { + deployments.mockRestore(); + } + + expect(managerOptions.failedServices).toEqual(['billing', 'migrations']); + expect(managerOptions).not.toHaveProperty('prerequisiteOutcome'); + }); + test('filters inactive manifests and rejects a loaded active deploy missing its deployable', async () => { mockDeployQuery.mockReturnValue( deployQuery([ @@ -2919,182 +2795,6 @@ describe('BuildService uncovered public behavior', () => { }; }; - const publicReconciliationHarness = ( - options: { - intent?: Record; - repository?: { fullName: string } | null; - mailboxResult?: (read: number, build: any) => any; - loadResult?: (read: number, build: any) => any; - authorityResult?: (read: number, build: any) => any; - onYamlImported?: (build: any) => void; - onDeploysAssociated?: (build: any) => void; - onGraphGenerated?: (build: any) => void; - yamlFailure?: unknown; - onLockAcquired?: (resource: string, build: any) => void; - deploys?: any[]; - } = {} - ) => { - const intent = - options.intent ?? ({ type: 'all', requestId: 'run-generation-3', gen: 3 } as Record); - const scopeKey = intent.type === 'source' ? 'source:42:main' : 'all'; - const build: any = { - id: 7, - uuid: 'stateful-reconciliation', - namespace: 'env-stateful-reconciliation', - kind: BuildKind.SANDBOX, - status: BuildStatus.DEPLOYED, - statusMessage: '', - runUUID: 'prior-run', - deployEnabled: true, - deletedAt: null, - pullRequestId: null, - pullRequest: null, - environment: { id: 5 }, - githubRepositoryId: 42, - branchName: 'main', - desiredGeneration: 3, - observedGeneration: 2, - acceptedRefs: { [scopeKey]: intent }, - deploys: [], - reload: jest.fn().mockResolvedValue(undefined), - $fetchGraph: jest.fn().mockResolvedValue(undefined), - $setRelated: jest.fn((_relation: string, related: any[]) => { - build.deploys = related; - }), - }; - let mailboxReads = 0; - let loadReads = 0; - let authorityReads = 0; - const buildPatch = jest.fn(); - const mutationFor = (value: Record) => { - const mutation: any = { - where: jest.fn(() => mutation), - whereNull: jest.fn(() => mutation), - whereNotIn: jest.fn(() => mutation), - then: (resolve: (value: number) => unknown, reject?: (reason: unknown) => unknown) => { - buildPatch(value); - Object.assign(build, value); - return Promise.resolve(1).then(resolve, reject); - }, - }; - return mutation; - }; - const BuildModel = { - query: jest.fn(() => { - let lookup: 'id' | 'one' | null = null; - let projectedAuthority = false; - const query: any = { - findById: jest.fn(() => { - lookup = 'id'; - return query; - }), - findOne: jest.fn(() => { - lookup = 'one'; - return query; - }), - select: jest.fn(() => { - projectedAuthority = true; - return query; - }), - where: jest.fn(() => query), - whereNull: jest.fn(() => query), - patch: jest.fn((value: Record) => mutationFor(value)), - then: (resolve: (value: any) => unknown, reject?: (reason: unknown) => unknown) => { - let result: any = build; - if (projectedAuthority) { - authorityReads += 1; - result = options.authorityResult ? options.authorityResult(authorityReads, build) : build; - } else if (lookup === 'one') { - loadReads += 1; - result = options.loadResult ? options.loadResult(loadReads, build) : build; - } else if (lookup === 'id') { - mailboxReads += 1; - result = options.mailboxResult ? options.mailboxResult(mailboxReads, build) : build; - } - return Promise.resolve(result).then(resolve, reject); - }, - }; - return query; - }), - }; - const deployMutation: any = { - patch: jest.fn(() => deployMutation), - where: jest.fn(() => deployMutation), - then: (resolve: (value: number) => unknown, reject?: (reason: unknown) => unknown) => - Promise.resolve(1).then(resolve, reject), - }; - const importedDeployQuery: any = { - where: jest.fn(() => importedDeployQuery), - withGraphFetched: jest.fn().mockResolvedValue([]), - }; - mockDeployQuery.mockReturnValue(importedDeployQuery); - const upsertDeployables = jest.fn(async () => { - if ('yamlFailure' in options) throw options.yamlFailure; - return { canReconcile: false }; - }); - const upsertWebhooksWithYaml = jest.fn(async () => { - options.onYamlImported?.(build); - }); - const findOrCreateDeploys = jest.fn(async () => { - options.onDeploysAssociated?.(build); - return options.deploys ?? []; - }); - if (options.onGraphGenerated) { - mockGenerateGraph.mockImplementation(async () => { - options.onGraphGenerated?.(build); - return {}; - }); - } - const repositoryQuery: any = { - findOne: jest.fn(() => repositoryQuery), - whereNull: jest.fn().mockResolvedValue(options.repository ?? null), - }; - const redlock = options.onLockAcquired - ? { - lock: jest.fn(async (resource: string) => { - options.onLockAcquired?.(resource, build); - return { extend: jest.fn(), unlock: jest.fn().mockResolvedValue(undefined) }; - }), - } - : {}; - const service = serviceWith( - { - models: { - Build: BuildModel, - Deploy: { query: jest.fn(() => deployMutation) }, - Repository: { query: jest.fn(() => repositoryQuery) }, - }, - services: { - Deployable: { upsertDeployables }, - Deploy: { findOrCreateDeploys }, - Webhook: { - upsertWebhooksWithYaml, - webhookQueue: { add: jest.fn().mockResolvedValue(undefined) }, - }, - }, - }, - redlock - ); - const job = { - data: { buildId: 7, generation: 3 }, - attemptsMade: 0, - opts: { attempts: 3 }, - } as any; - return { - service, - job, - build, - buildPatch, - upsertDeployables, - upsertWebhooksWithYaml, - findOrCreateDeploys, - repositoryQuery, - deployMutation, - redlock, - reads: () => ({ mailboxReads, loadReads, authorityReads }), - }; - }; - const passOneContentionRetry = async (lockWithOptions: jest.Mock) => { for (let iteration = 0; iteration < 25 && lockWithOptions.mock.calls.length === 0; iteration += 1) { await Promise.resolve(); @@ -4154,38 +3854,111 @@ describe('BuildService uncovered public behavior', () => { const lock = jest.fn().mockResolvedValue({ unlock, extend: jest.fn() }); const withLock = serviceWith({}, { lock }); await expect(withLock.withBuildDeploymentLock(2, action)).resolves.toBe('done'); - expect(lock).toHaveBeenCalledWith('build-deployment.2', 15 * 60 * 1000); + expect(lock).toHaveBeenCalledWith('build-deployment.2', 60_000); expect(unlock).toHaveBeenCalledTimes(1); }); - test('withBuildDeploymentLock surfaces renewal loss while containing lock-release failure', async () => { + test('withCurrentDeployPromotionLock keeps waiting on a contended service lock and logs once a minute', async () => { jest.useFakeTimers(); - const renewalFailure = new Error('lock renewal failed'); - const extend = jest.fn().mockRejectedValue(renewalFailure); - const unlock = jest.fn().mockRejectedValue(new Error('lock release failed')); - const lock = jest.fn().mockResolvedValue({ extend, unlock }); - const service = serviceWith({}, { lock }); - const action = jest.fn(async () => { - jest.advanceTimersByTime(5 * 60 * 1000); - await Promise.resolve(); - await Promise.resolve(); - return 'done'; - }); - + try { + const unlock = jest.fn().mockResolvedValue(undefined); + const lock = jest.fn(async () => { + if (Date.now() < 60_500) throw new Error('resource is locked'); + return { unlock, extend: jest.fn() }; + }); + const service = serviceWith({}, { lock }); + + const admitted = service.withCurrentDeployPromotionLock( + 3, + async () => true, + async () => 'applied' + ); + await jest.advanceTimersByTimeAsync(61_000); + + await expect(admitted).resolves.toEqual({ admitted: true, value: 'applied' }); + expect(lock).toHaveBeenCalledWith('deploy-promotion.3', 60_000); + expect(unlock).toHaveBeenCalledTimes(1); + } finally { + jest.useRealTimers(); + } + }); + + test('barrierDeployPromotionLocks takes and releases every service lock of the environment', async () => { + const unlock = jest.fn().mockResolvedValue(undefined); + const lock = jest.fn().mockResolvedValue({ unlock, extend: jest.fn() }); + const service = serviceWith( + { + models: { + Deploy: { + query: jest.fn(() => ({ + select: jest.fn(() => ({ where: jest.fn().mockResolvedValue([{ id: 1 }, { id: 2 }]) })), + })), + }, + }, + }, + { lock } + ); + + await (service as any).barrierDeployPromotionLocks(4); + + expect(lock.mock.calls.map(([resource]) => resource)).toEqual(['deploy-promotion.1', 'deploy-promotion.2']); + expect(unlock).toHaveBeenCalledTimes(2); + }); + + test('the generation try-lock is short-lived and renewed often, so a dead worker frees its generation quickly', async () => { + jest.useFakeTimers(); + try { + const extend = jest.fn(async () => ({ unlock, extend })); + const unlock = jest.fn().mockResolvedValue(undefined); + const lock = jest.fn().mockResolvedValue({ extend, unlock }); + const service = serviceWith({}, { lock }); + const action = jest.fn(async () => { + await jest.advanceTimersByTimeAsync(45_000); + }); + + await (service as any).tryWithDeploymentGenerationLock(1, 7, action); + + expect(lock).toHaveBeenCalledWith('build-reconcile.1.7', 60_000); + expect(extend).toHaveBeenCalledTimes(2); + expect(extend).toHaveBeenCalledWith(60_000); + expect(unlock).toHaveBeenCalledTimes(1); + } finally { + jest.useRealTimers(); + } + }); + + test('withBuildDeploymentLock surfaces renewal loss while containing lock-release failure', async () => { + jest.useFakeTimers(); + const renewalFailure = new Error('lock renewal failed'); + const extend = jest.fn().mockRejectedValue(renewalFailure); + const unlock = jest.fn().mockRejectedValue(new Error('lock release failed')); + const lock = jest.fn().mockResolvedValue({ extend, unlock }); + const service = serviceWith({}, { lock }); + const action = jest.fn(async () => { + jest.advanceTimersByTime(20_000); + await Promise.resolve(); + await Promise.resolve(); + return 'done'; + }); + await expect(service.withBuildDeploymentLock(7, action)).rejects.toBe(renewalFailure); - expect(extend).toHaveBeenCalledWith(15 * 60 * 1000); + expect(extend).toHaveBeenCalledWith(60_000); expect(unlock).toHaveBeenCalledTimes(1); }); test('enqueuePendingDeploymentReconciliations wraps the cursor and contains an individual signal failure', async () => { - const first = lookupQuery([{ id: 5, desiredGeneration: 2 }]); + const first = lookupQuery([ + { id: 5, desiredGeneration: 2, acceptedRefs: { all: { type: 'all', requestId: 'request-2', gen: 2 } } }, + ]); const afterCursor = lookupQuery([]); - const wrapped = lookupQuery([{ id: 2, desiredGeneration: 3 }]); + const wrapped = lookupQuery([ + { id: 2, desiredGeneration: 3, acceptedRefs: { all: { type: 'all', requestId: 'request-3', gen: 3 } } }, + ]); const BuildModel = { query: jest.fn().mockReturnValueOnce(first).mockReturnValueOnce(afterCursor).mockReturnValueOnce(wrapped), }; - const service = serviceWith({ models: { Build: BuildModel } }); + const service = serviceWith({ models: { Build: BuildModel, Deploy: { query: jest.fn(() => lookupQuery([])) } } }); const add = jest.fn().mockResolvedValueOnce(undefined).mockRejectedValueOnce(new Error('queue unavailable')); (service as any).deploymentReconciliationQueue = { add }; @@ -4236,27 +4009,6 @@ describe('BuildService uncovered public behavior', () => { clearInterval(timer); }); - test('deployment reconciliation rejects malformed jobs and both legacy queue adapters accept durable work', async () => { - const service = serviceWith({}); - const add = jest.fn().mockResolvedValue(undefined); - (service as any).deploymentReconciliationQueue = { add }; - - await expect(service.processDeploymentReconciliationQueue({ data: { buildId: 0 } } as any)).rejects.toThrow( - 'buildId and generation are required' - ); - - await service.processBuildQueue({ data: { buildId: 7, runUUID: 'legacy-build' } } as any); - await service.processResolveAndDeployBuildQueue({ data: { buildId: 8, githubRepositoryId: 42 } } as any); - - expect(mockAcceptDeploymentIntent).toHaveBeenCalledTimes(2); - expect(add).toHaveBeenCalledWith('reconcile', expect.objectContaining({ buildId: 7, generation: 1 }), { - jobId: 'reconcile-7-1', - }); - expect(add).toHaveBeenCalledWith('reconcile', expect.objectContaining({ buildId: 8, generation: 1 }), { - jobId: 'reconcile-8-1', - }); - }); - test('legacy queue adapters stop without signaling when the build disappeared during mailbox acceptance', async () => { mockAcceptDeploymentIntent.mockResolvedValueOnce(null); const service = serviceWith({}); @@ -4359,6 +4111,7 @@ describe('BuildService uncovered public behavior', () => { undefined, undefined, undefined, + undefined, undefined ); expect(findOrCreateDeploys).toHaveBeenCalledWith({ id: 5 }, build); @@ -4552,6 +4305,7 @@ describe('BuildService uncovered public behavior', () => { }; const deployRead: any = { where: jest.fn(() => deployRead), + whereNotIn: jest.fn(() => deployRead), withGraphFetched: jest.fn().mockResolvedValue([deploy]), }; mockDeployQuery.mockReturnValue(deployRead); @@ -4575,7 +4329,12 @@ describe('BuildService uncovered public behavior', () => { const service = serviceWith({ models: { Build: { query: jest.fn(() => currentQuery) }, - Deploy: { query: jest.fn(() => mutation) }, + Deploy: { + query: jest.fn(() => ({ + ...mutation, + findOne: jest.fn(() => ({ select: jest.fn().mockResolvedValue({ id: 11 }) })), + })), + }, }, services: { Deploy: { buildImage } }, }); @@ -4594,7 +4353,7 @@ describe('BuildService uncovered public behavior', () => { expect(kubernetes.createOrUpdateNamespace).toHaveBeenCalledWith( expect.objectContaining({ name: 'env-native-build', waitForReady: true }) ); - expect(buildImage).toHaveBeenCalledWith(deploy, 0, 'run-native', 'sha-a', null, 'main', 4, 'default'); + expect(buildImage).toHaveBeenCalledWith(deploy, 0, 'run-native', 'sha-a', null, 'main', 4, 'default', undefined); }); test('generateAndApplyManifests persists current deploy and legacy manifests behind native mutation gates', async () => { @@ -4613,12 +4372,14 @@ describe('BuildService uncovered public behavior', () => { }; const deployRead: any = { where: jest.fn(() => deployRead), + whereNotIn: jest.fn(() => deployRead), withGraphFetched: jest.fn().mockResolvedValue([deploy]), }; mockDeployQuery.mockReturnValue(deployRead); const mutation: any = { patch: jest.fn(() => mutation), where: jest.fn(() => mutation), + whereNull: jest.fn(() => mutation), then: (resolve: (value: number) => unknown, reject?: (reason: unknown) => unknown) => Promise.resolve(1).then(resolve, reject), }; @@ -4648,7 +4409,12 @@ describe('BuildService uncovered public behavior', () => { const service = serviceWith({ models: { Build: { query: jest.fn(() => buildQuery) }, - Deploy: { query: jest.fn(() => mutation) }, + Deploy: { + query: jest.fn(() => ({ + ...mutation, + findOne: jest.fn(() => ({ select: jest.fn().mockResolvedValue({ id: 11 }) })), + })), + }, }, }); mockGenerateDeployManifest.mockReturnValue('kind: Deployment\n'); @@ -4657,8 +4423,9 @@ describe('BuildService uncovered public behavior', () => { .spyOn(DeploymentManager.prototype, 'deploy') .mockImplementation(async function (this: any) { expect(await this.options.isCurrent()).toBe(true); - await this.options.nativeMutationGate(async () => undefined); + await this.options.nativeMutationGate(deploy, async () => undefined); await this.options.nativeSecretMutationGate(deploy, async () => undefined); + return { failed: [] }; }); await expect( @@ -4699,20 +4466,21 @@ describe('BuildService uncovered public behavior', () => { deploys: [], $fetchGraph: jest.fn().mockResolvedValue(undefined), }; - let authorityReads = 0; - const BuildModel = { + let ownershipReads = 0; + const BuildModel = { query: jest.fn(() => lookupQuery({ ...build, acceptedRefs: {} })) }; + const DeployModel = { query: jest.fn(() => { - authorityReads += 1; - const authority = authorityReads <= 3 ? build : { ...build, runUUID: 'newer-run' }; - return lookupQuery(authority); + ownershipReads += 1; + return lookupQuery(ownershipReads <= 3 ? { id: 11 } : undefined); }), }; const noDeploys: any = { where: jest.fn(() => noDeploys), + whereNotIn: jest.fn(() => noDeploys), withGraphFetched: jest.fn().mockResolvedValue([]), }; mockDeployQuery.mockReturnValue(noDeploys); - const service = serviceWith({ models: { Build: BuildModel } }); + const service = serviceWith({ models: { Build: BuildModel, Deploy: DeployModel } }); await expect( service.generateAndApplyManifests({ @@ -4761,11 +4529,125 @@ describe('BuildService uncovered public behavior', () => { queueBuildPrewarm.mockRestore(); }); + test('a teardown closes the work accepted before it and keeps a deploy request accepted during it', async () => { + const refs: any = { + 'source:1:main': { + type: 'source', + requestId: 'old', + target: 'repository', + githubRepositoryId: 1, + branch: 'main', + sha: 'a', + gen: 4, + }, + all: { type: 'all', requestId: 'full', gen: 5 }, + 'repository:2': { type: 'repository', requestId: 'relabel', githubRepositoryId: 2, gen: 6 }, + }; + const buildPatch = jest.fn().mockResolvedValue(1); + const buildQuery: any = { + select: jest.fn(() => buildQuery), + findById: jest.fn(() => buildQuery), + forUpdate: jest.fn().mockResolvedValue({ id: 7, desiredGeneration: 6, acceptedRefs: refs }), + patch: buildPatch, + }; + const rowPatch = jest.fn(() => ({ where: jest.fn().mockResolvedValue(1) })); + const deployQuery: any = { + select: jest.fn(() => deployQuery), + where: jest.fn().mockResolvedValue([ + { id: 1, desiredGeneration: 5, observedGeneration: 3 }, + { id: 2, desiredGeneration: 6, observedGeneration: 3 }, + { id: 3, desiredGeneration: 4, observedGeneration: 4 }, + ]), + patch: rowPatch, + }; + const service = serviceWith({ + models: { + Build: { query: jest.fn(() => buildQuery), transact: jest.fn(async (action: any) => action({})) }, + Deploy: { query: jest.fn(() => deployQuery) }, + }, + }); + + await service['observeAcceptedWorkForTeardown'](7, 5); + + expect(rowPatch).toHaveBeenCalledTimes(1); + expect(rowPatch).toHaveBeenCalledWith({ observedGeneration: 5 }); + expect(buildPatch).toHaveBeenCalledWith({ + acceptedRefs: { + 'source:1:main': { ...refs['source:1:main'], observedGen: 4 }, + all: { ...refs.all, observedGen: 5 }, + 'repository:2': refs['repository:2'], + }, + observedGeneration: 5, + }); + }); + + test('deleteBuild takes the teardown cutoff before it waits for in-flight mutations', async () => { + const mutation: any = { + patch: jest.fn(() => mutation), + where: jest.fn(() => mutation), + whereNull: jest.fn(() => mutation), + findById: jest.fn(() => mutation), + select: jest.fn(() => ({ + where: jest.fn().mockResolvedValue([]), + findById: jest.fn(() => ({ forUpdate: jest.fn().mockResolvedValue({ id: 7, desiredGeneration: 0 }) })), + })), + then: (resolve: (value: number) => unknown, reject?: (reason: unknown) => unknown) => + Promise.resolve(1).then(resolve, reject), + }; + const build: any = { + id: 7, + uuid: 'delete-build', + namespace: 'env-delete-build', + kind: BuildKind.SANDBOX, + status: BuildStatus.DEPLOYED, + runUUID: 'old-run', + desiredGeneration: 5, + pullRequestId: null, + pullRequest: null, + deployEnabled: true, + githubDeployments: false, + deploys: [], + idempotencyKey: null, + reload: jest.fn().mockResolvedValue(undefined), + $fetchGraph: jest.fn().mockResolvedValue(undefined), + $query: jest.fn(() => ({ patch: jest.fn().mockResolvedValue(1) })), + }; + const service = serviceWith({ + models: { + Build: { query: jest.fn(() => mutation), transact: jest.fn(async (action: any) => action({})) }, + Deploy: { query: jest.fn(() => mutation) }, + }, + services: { + GithubService: { githubDeploymentQueue: { add: jest.fn() } }, + Ingress: { ingressCleanupQueue: {} }, + Webhook: { webhookQueue: { add: jest.fn().mockResolvedValue(undefined) } }, + }, + }); + const order: string[] = []; + jest.spyOn(service as any, 'barrierDeployPromotionLocks').mockImplementation(async () => { + order.push('barrier'); + build.desiredGeneration = 6; + }); + const observe = jest.spyOn(service as any, 'observeAcceptedWorkForTeardown').mockImplementation(async () => { + order.push('observe'); + }); + + await expect(service.deleteBuild(build)).resolves.toBeUndefined(); + + expect(order).toEqual(['barrier', 'observe']); + expect(observe).toHaveBeenCalledWith(7, 5); + }); + test('deleteBuild queues provider and ingress cleanup for every torn-down GitHub deployment', async () => { const mutation: any = { patch: jest.fn(() => mutation), where: jest.fn(() => mutation), whereNull: jest.fn(() => mutation), + findById: jest.fn(() => mutation), + select: jest.fn(() => ({ + where: jest.fn().mockResolvedValue([]), + findById: jest.fn(() => ({ forUpdate: jest.fn().mockResolvedValue({ id: 7, desiredGeneration: 0 }) })), + })), then: (resolve: (value: number) => unknown, reject?: (reason: unknown) => unknown) => Promise.resolve(1).then(resolve, reject), }; @@ -4791,7 +4673,7 @@ describe('BuildService uncovered public behavior', () => { }; const service = serviceWith({ models: { - Build: { query: jest.fn(() => mutation) }, + Build: { query: jest.fn(() => mutation), transact: jest.fn(async (action: any) => action({})) }, Deploy: { query: jest.fn(() => mutation) }, }, services: { @@ -5437,6 +5319,11 @@ describe('BuildService uncovered public behavior', () => { patch: jest.fn(() => mutation), where: jest.fn(() => mutation), whereNull: jest.fn(() => mutation), + findById: jest.fn(() => mutation), + select: jest.fn(() => ({ + where: jest.fn().mockResolvedValue([]), + findById: jest.fn(() => ({ forUpdate: jest.fn().mockResolvedValue({ id: 7, desiredGeneration: 0 }) })), + })), then: (resolve: (value: number) => unknown, reject?: (reason: unknown) => unknown) => Promise.resolve(1).then(resolve, reject), }; @@ -5462,7 +5349,7 @@ describe('BuildService uncovered public behavior', () => { }; const service = serviceWith({ models: { - Build: { query: jest.fn(() => mutation) }, + Build: { query: jest.fn(() => mutation), transact: jest.fn(async (action: any) => action({})) }, Deploy: { query: jest.fn(() => mutation) }, }, services: { Webhook: { webhookQueue: { add: jest.fn().mockResolvedValue(undefined) } } }, @@ -5481,716 +5368,102 @@ describe('BuildService uncovered public behavior', () => { ]); }); - test('deployment reconciliation rejects corrupt persisted generations before mutation', async () => { - const corrupt = { desiredGeneration: 'not-a-number', observedGeneration: 0, acceptedRefs: {} }; - const query = lookupQuery(corrupt); - const service = serviceWith({ models: { Build: { query: jest.fn(() => query) } } }); + test.each([ + ['before YAML import', 4, 0, 0], + ['after YAML import', 5, 1, 0], + ['after deploy association', 6, 1, 1], + ])( + 'createBuild stops %s when post-claim setup authority moves to another run', + async (_stage, loseAuthorityOnRead, expectedImports, expectedDeployLookups) => { + const mutation: any = { + where: jest.fn(() => mutation), + whereNull: jest.fn(() => mutation), + then: (resolve: (value: number) => unknown, reject?: (reason: unknown) => unknown) => + Promise.resolve(1).then(resolve, reject), + }; + const deploys = [{ id: 11, uuid: 'app' }]; + const build: any = { + id: 7, + uuid: 'post-claim-race', + kind: BuildKind.SANDBOX, + status: BuildStatus.QUEUED, + deletedAt: null, + runUUID: null, + pullRequestId: 55, + pullRequest: { status: 'open', deployOnUpdate: true }, + deploys: [], + $fetchGraph: jest.fn().mockResolvedValue(undefined), + $setRelated: jest.fn((_relation: string, related: any[]) => { + build.deploys = related; + }), + }; + const supersedingBuild: any = { + ...build, + runUUID: 'new-run-owner', + $fetchGraph: jest.fn().mockResolvedValue(undefined), + }; + let authorityReads = 0; + const buildQuery: any = { + where: jest.fn(() => buildQuery), + whereNull: jest.fn(() => buildQuery), + first: jest.fn().mockResolvedValue(build), + findOne: jest.fn(async () => { + authorityReads += 1; + return authorityReads >= loseAuthorityOnRead ? supersedingBuild : build; + }), + patch: jest.fn(() => mutation), + }; + const upsertDeployables = jest.fn().mockResolvedValue({ canReconcile: false }); + const findOrCreateDeploys = jest.fn().mockResolvedValue(deploys); + const upsertWebhooksWithYaml = jest.fn().mockResolvedValue(undefined); + const service = serviceWith({ + models: { Build: { query: jest.fn(() => buildQuery) } }, + services: { + Deployable: { upsertDeployables }, + Deploy: { findOrCreateDeploys }, + Webhook: { upsertWebhooksWithYaml }, + }, + }); - await expect( - service.processDeploymentReconciliationQueue({ data: { buildId: 7, generation: 3 } } as any) - ).rejects.toThrow('Build 7 has invalid deployment generations'); - }); + await expect( + service.createBuild( + { id: 5 } as any, + { pullRequestId: 55, repositoryId: 2, repositoryBranchName: 'feature' }, + {} as any + ) + ).resolves.toBeUndefined(); - test.each([ - ['the claim patch loses its row', 0, false], - ['authority changes immediately after the claim', 1, true], - ])('deployment reconciliation stops before configuration when %s', async (_case, claimedRows, staleAfterClaim) => { - const build: any = { - id: 7, - uuid: 'claim-race', - status: BuildStatus.DEPLOYED, - deployEnabled: true, - deletedAt: null, - pullRequestId: null, - pullRequest: null, - desiredGeneration: 3, - observedGeneration: 2, - acceptedRefs: { all: { type: 'all', requestId: 'run-generation-3', gen: 3 } }, - $fetchGraph: jest.fn().mockResolvedValue(undefined), - }; - const stale = { ...build, runUUID: 'newer-run' }; + expect(upsertDeployables).toHaveBeenCalledTimes(expectedImports); + expect(upsertWebhooksWithYaml).toHaveBeenCalledTimes(expectedImports); + expect(findOrCreateDeploys).toHaveBeenCalledTimes(expectedDeployLookups); + expect(build.$setRelated).toHaveBeenCalledTimes(expectedDeployLookups); + expect(buildQuery.patch).toHaveBeenCalledTimes(1); + expect(buildQuery.patch).not.toHaveBeenCalledWith(expect.objectContaining({ status: expect.anything() })); + } + ); + + test('createBuild records a fenced failure when deploy discovery returns no result', async () => { const mutation: any = { patch: jest.fn(() => mutation), where: jest.fn(() => mutation), whereNull: jest.fn(() => mutation), whereNotIn: jest.fn(() => mutation), then: (resolve: (value: number) => unknown, reject?: (reason: unknown) => unknown) => - Promise.resolve(claimedRows).then(resolve, reject), - }; - const BuildModel = { - query: jest.fn(() => { - let authorityProjection = false; - const query: any = { - findById: jest.fn(() => query), - findOne: jest.fn(() => query), - select: jest.fn(() => { - authorityProjection = true; - return query; - }), - whereNull: jest.fn(() => query), - patch: jest.fn(() => mutation), - then: (resolve: (value: any) => unknown, reject?: (reason: unknown) => unknown) => - Promise.resolve(authorityProjection && staleAfterClaim ? stale : build).then(resolve, reject), - }; - return query; - }), + Promise.resolve(1).then(resolve, reject), }; - const upsertDeployables = jest.fn(); - const service = serviceWith({ - models: { Build: BuildModel }, - services: { Deployable: { upsertDeployables } }, - }); - - await expect( - service.processDeploymentReconciliationQueue({ data: { buildId: 7, generation: 3 } } as any) - ).resolves.toBeUndefined(); - - expect(upsertDeployables).not.toHaveBeenCalled(); - }); - - test('deployment reconciliation observes disabled builds without importing or deploying them', async () => { const build: any = { id: 7, - uuid: 'disabled-build', - status: BuildStatus.DEPLOYED, - deployEnabled: false, + uuid: 'missing-deploy-result', + kind: BuildKind.SANDBOX, + status: BuildStatus.QUEUED, deletedAt: null, - pullRequestId: null, - desiredGeneration: 3, - observedGeneration: 2, - acceptedRefs: { all: { type: 'all', requestId: 'run-generation-3', gen: 3 } }, + runUUID: null, + pullRequestId: 55, + pullRequest: { status: 'open', deployOnUpdate: true }, + deploys: [], + reload: jest.fn().mockResolvedValue(undefined), $fetchGraph: jest.fn().mockResolvedValue(undefined), - }; - const mutation: any = { - patch: jest.fn(() => mutation), - where: jest.fn(() => mutation), - whereNull: jest.fn(() => mutation), - then: (resolve: (value: number) => unknown, reject?: (reason: unknown) => unknown) => - Promise.resolve(1).then(resolve, reject), - }; - const query: any = { - findById: jest.fn(() => query), - findOne: jest.fn(() => query), - whereNull: jest.fn(() => query), - patch: jest.fn(() => mutation), - then: (resolve: (value: any) => unknown, reject?: (reason: unknown) => unknown) => - Promise.resolve(build).then(resolve, reject), - }; - const upsertDeployables = jest.fn(); - const service = serviceWith({ - models: { Build: { query: jest.fn(() => query) } }, - services: { Deployable: { upsertDeployables } }, - }); - - await service.processDeploymentReconciliationQueue({ data: { buildId: 7, generation: 3 } } as any); - - expect(query.patch).toHaveBeenCalledWith({ observedGeneration: 3 }); - expect(upsertDeployables).not.toHaveBeenCalled(); - }); - - test('deployment reconciliation records and observes a claimed configuration parse failure', async () => { - const parseError = new ParsingError('invalid lifecycle yaml'); - const build: any = { - id: 7, - uuid: 'invalid-config-build', - namespace: 'env-invalid-config-build', - kind: BuildKind.SANDBOX, - status: BuildStatus.DEPLOYED, - statusMessage: '', - runUUID: 'prior-run', - deployEnabled: true, - deletedAt: null, - pullRequestId: null, - pullRequest: null, - environment: { id: 5 }, - desiredGeneration: 3, - observedGeneration: 2, - acceptedRefs: { all: { type: 'all', requestId: 'run-generation-3', gen: 3 } }, - deploys: [], - reload: jest.fn().mockResolvedValue(undefined), - $fetchGraph: jest.fn().mockResolvedValue(undefined), - }; - const buildMutation: any = { - where: jest.fn(() => buildMutation), - whereNull: jest.fn(() => buildMutation), - whereNotIn: jest.fn(() => buildMutation), - patch: jest.fn(() => buildMutation), - then: (resolve: (value: number) => unknown, reject?: (reason: unknown) => unknown) => - Promise.resolve(1).then(resolve, reject), - }; - const BuildModel = { - query: jest.fn(() => { - const query: any = { - findById: jest.fn(() => query), - findOne: jest.fn(() => query), - select: jest.fn(() => query), - whereNull: jest.fn(() => query), - patch: jest.fn((value: Record) => { - buildMutation.patch(value); - return buildMutation; - }), - then: (resolve: (value: any) => unknown, reject?: (reason: unknown) => unknown) => - Promise.resolve(build).then(resolve, reject), - }; - return query; - }), - }; - const upsertDeployables = jest.fn().mockRejectedValue(parseError); - const findOrCreateDeploys = jest.fn(); - const service = serviceWith({ - models: { Build: BuildModel }, - services: { - Deployable: { upsertDeployables }, - Deploy: { findOrCreateDeploys }, - Webhook: { webhookQueue: { add: jest.fn() } }, - }, - }); - - await expect( - service.processDeploymentReconciliationQueue({ - data: { buildId: 7, generation: 3 }, - attemptsMade: 0, - opts: { attempts: 3 }, - } as any) - ).resolves.toBeUndefined(); - - expect(upsertDeployables).toHaveBeenCalledTimes(1); - expect(findOrCreateDeploys).not.toHaveBeenCalled(); - expect(buildMutation.patch).toHaveBeenCalledWith({ - status: BuildStatus.CONFIG_ERROR, - statusMessage: 'invalid lifecycle yaml', - }); - expect(buildMutation.patch).toHaveBeenCalledWith({ observedGeneration: 3 }); - expect(mockQueueAdd).not.toHaveBeenCalled(); - }); - - test('processDeploymentReconciliationQueue carries one durable generation through rollout and observation', async () => { - const build: any = { - id: 7, - uuid: 'reconciled-build', - namespace: 'env-reconciled-build', - kind: BuildKind.SANDBOX, - status: BuildStatus.DEPLOYED, - statusMessage: '', - runUUID: 'prior-run', - deployEnabled: true, - deletedAt: null, - pullRequestId: null, - pullRequest: null, - environment: { id: 5 }, - desiredGeneration: 3, - observedGeneration: 2, - acceptedRefs: { - 'repository:42': { - type: 'repository', - requestId: 'run-generation-3', - githubRepositoryId: 42, - gen: 3, - }, - }, - deploys: [], - reload: jest.fn().mockResolvedValue(undefined), - $fetchGraph: jest.fn().mockResolvedValue(undefined), - $setRelated: jest.fn((_relation: string, related: any[]) => { - build.deploys = related; - }), - }; - const buildMutation: any = { - where: jest.fn(() => buildMutation), - whereNull: jest.fn(() => buildMutation), - whereNotIn: jest.fn(() => buildMutation), - patch: jest.fn(() => buildMutation), - then: (resolve: (value: number) => unknown, reject?: (reason: unknown) => unknown) => - Promise.resolve(1).then(resolve, reject), - }; - const buildQuery: any = { - findById: jest.fn(() => buildQuery), - findOne: jest.fn(() => buildQuery), - select: jest.fn(() => buildQuery), - whereNull: jest.fn(() => buildQuery), - patch: jest.fn((value: Record) => { - buildMutation.patch(value); - return buildMutation; - }), - then: (resolve: (value: any) => unknown, reject?: (reason: unknown) => unknown) => - Promise.resolve(build).then(resolve, reject), - }; - const deployMutation: any = { - patch: jest.fn(() => deployMutation), - where: jest.fn(() => deployMutation), - then: (resolve: (value: number) => unknown, reject?: (reason: unknown) => unknown) => - Promise.resolve(1).then(resolve, reject), - }; - const noDeploys: any = { - where: jest.fn(() => noDeploys), - withGraphFetched: jest.fn().mockResolvedValue([]), - }; - mockDeployQuery.mockReturnValue(noDeploys); - const upsertDeployables = jest.fn().mockResolvedValue({ canReconcile: false }); - const findOrCreateDeploys = jest.fn().mockResolvedValue([]); - const upsertWebhooksWithYaml = jest.fn().mockResolvedValue(undefined); - const service = serviceWith({ - models: { - Build: { query: jest.fn(() => buildQuery) }, - Deploy: { query: jest.fn(() => deployMutation) }, - }, - services: { - Deployable: { upsertDeployables }, - Deploy: { findOrCreateDeploys }, - Webhook: { - upsertWebhooksWithYaml, - webhookQueue: { add: jest.fn().mockResolvedValue(undefined) }, - }, - }, - }); - - await expect( - service.processDeploymentReconciliationQueue({ - data: { buildId: 7, generation: 3 }, - attemptsMade: 0, - opts: { attempts: 3 }, - } as any) - ).resolves.toBeUndefined(); - - expect(upsertDeployables).toHaveBeenCalledWith( - 7, - 'reconciled-build', - null, - build.environment, - build, - 42, - undefined, - undefined, - 42 - ); - expect(findOrCreateDeploys).toHaveBeenCalledWith(build.environment, build, 42, undefined, undefined, 42); - expect(buildMutation.patch.mock.calls.map(([value]) => value.status).filter(Boolean)).toEqual([ - BuildStatus.PENDING, - BuildStatus.BUILDING, - BuildStatus.DEPLOYING, - BuildStatus.DEPLOYED, - ]); - expect(buildMutation.patch).toHaveBeenCalledWith({ observedGeneration: 3 }); - expect(mockQueueAdd).toHaveBeenCalledWith( - 'manifest', - expect.objectContaining({ buildId: 7, runUUID: 'run-generation-3', expectedGeneration: 3 }) - ); - expect(build).toMatchObject({ runUUID: 'run-generation-3', status: BuildStatus.DEPLOYED }); - }); - - test('deployment reconciliation propagates an accepted root-source SHA through YAML and rollout', async () => { - const intent = { - type: 'source', - requestId: 'run-generation-3', - target: 'all', - githubRepositoryId: 42, - branch: 'main', - sha: 'commit-c', - beforeSha: 'commit-b', - gen: 3, - }; - (github.getSHAForBranch as jest.Mock).mockResolvedValue('commit-c'); - const harness = publicReconciliationHarness({ intent, repository: { fullName: 'org/repo' } }); - - await expect(harness.service.processDeploymentReconciliationQueue(harness.job)).resolves.toBeUndefined(); - - expect(github.getSHAForBranch).toHaveBeenCalledWith('main', 'org', 'repo'); - expect(github.compareCommits).not.toHaveBeenCalled(); - expect(harness.upsertDeployables).toHaveBeenCalledWith( - 7, - 'stateful-reconciliation', - null, - harness.build.environment, - harness.build, - undefined, - 'commit-c', - 'main', - 42 - ); - expect(harness.upsertWebhooksWithYaml).toHaveBeenCalledWith(harness.build, null, 'commit-c'); - expect(harness.findOrCreateDeploys).toHaveBeenCalledWith( - harness.build.environment, - harness.build, - undefined, - 'commit-c', - 'main', - 42 - ); - expect(harness.buildPatch).toHaveBeenCalledWith({ observedGeneration: 3 }); - expect(mockQueueAdd).toHaveBeenCalledWith( - 'manifest', - expect.objectContaining({ buildId: 7, runUUID: 'run-generation-3', expectedGeneration: 3 }) - ); - }); - - test('deployment reconciliation keeps the accepted source when its repository was removed', async () => { - const intent = { - type: 'source', - requestId: 'run-generation-3', - target: 'all', - githubRepositoryId: 42, - branch: 'main', - sha: 'commit-c', - gen: 3, - }; - const harness = publicReconciliationHarness({ intent, repository: null }); - - await expect(harness.service.processDeploymentReconciliationQueue(harness.job)).resolves.toBeUndefined(); - - expect(harness.repositoryQuery.findOne).toHaveBeenCalledWith({ githubRepositoryId: 42 }); - expect(github.getSHAForBranch).not.toHaveBeenCalled(); - expect(harness.upsertDeployables).toHaveBeenCalledWith( - 7, - 'stateful-reconciliation', - null, - harness.build.environment, - harness.build, - undefined, - 'commit-c', - 'main', - 42 - ); - expect(harness.buildPatch).toHaveBeenCalledWith({ observedGeneration: 3 }); - }); - - test('deployment reconciliation scopes a source update to the matching repository and branch', async () => { - const matching = { id: 11, githubRepositoryId: 42, branchName: 'main' }; - const otherBranch = { id: 12, githubRepositoryId: 42, branchName: 'release' }; - const otherRepository = { id: 13, githubRepositoryId: 99, branchName: 'main' }; - const harness = publicReconciliationHarness({ - intent: { - type: 'source', - requestId: 'run-generation-3', - target: 'repository', - githubRepositoryId: 42, - branch: 'main', - sha: 'commit-c', - gen: 3, - }, - repository: null, - deploys: [matching, otherBranch, otherRepository], - }); - - await expect(harness.service.processDeploymentReconciliationQueue(harness.job)).resolves.toBeUndefined(); - - expect(harness.findOrCreateDeploys).toHaveBeenCalledWith( - harness.build.environment, - harness.build, - 42, - 'commit-c', - 'main', - 42 - ); - expect(harness.deployMutation.where).toHaveBeenCalledWith('githubRepositoryId', 42); - expect(harness.deployMutation.where).toHaveBeenCalledWith('branchName', 'main'); - expect(matching).toHaveProperty('runUUID', 'run-generation-3'); - expect(otherBranch).not.toHaveProperty('runUUID'); - expect(otherRepository).not.toHaveProperty('runUUID'); - expect(harness.buildPatch).toHaveBeenCalledWith({ observedGeneration: 3 }); - }); - - test('deployment reconciliation stops when its mailbox generation is observed during lock acquisition', async () => { - const acquiredResources: string[] = []; - const harness = publicReconciliationHarness({ - onLockAcquired: (resource, build) => { - acquiredResources.push(resource); - if (resource === 'build-deployment.7') build.observedGeneration = 3; - }, - }); - - await expect(harness.service.processDeploymentReconciliationQueue(harness.job)).resolves.toBeUndefined(); - - expect(harness.buildPatch).not.toHaveBeenCalled(); - expect(harness.upsertDeployables).not.toHaveBeenCalled(); - expect(harness.build.$fetchGraph).not.toHaveBeenCalled(); - expect(acquiredResources).toEqual(['build-reconcile.7.3', 'build-deployment.7']); - }); - - test('deployment reconciliation stops when authority moves after configuration-lock admission', async () => { - const harness = publicReconciliationHarness({ - authorityResult: (read, build) => (read >= 4 ? { ...build, runUUID: 'newer-run' } : build), - }); - - await expect(harness.service.processDeploymentReconciliationQueue(harness.job)).resolves.toBeUndefined(); - - expect(harness.upsertDeployables).not.toHaveBeenCalled(); - expect(harness.findOrCreateDeploys).not.toHaveBeenCalled(); - expect(harness.buildPatch.mock.calls.map(([value]) => value.status).filter(Boolean)).toEqual([BuildStatus.PENDING]); - expect(harness.buildPatch).not.toHaveBeenCalledWith({ observedGeneration: 3 }); - }); - - test('deployment reconciliation stops when the build disappears after its mailbox claim', async () => { - const harness = publicReconciliationHarness({ loadResult: () => null }); - - await expect(harness.service.processDeploymentReconciliationQueue(harness.job)).resolves.toBeUndefined(); - - expect(harness.buildPatch).not.toHaveBeenCalled(); - expect(harness.upsertDeployables).not.toHaveBeenCalled(); - expect(harness.build.$fetchGraph).not.toHaveBeenCalled(); - }); - - test('deployment reconciliation stops after YAML import when another run takes authority', async () => { - let current = true; - const harness = publicReconciliationHarness({ - authorityResult: (_read, build) => (current ? build : { ...build, runUUID: 'newer-run' }), - onYamlImported: () => { - current = false; - }, - }); - - await expect(harness.service.processDeploymentReconciliationQueue(harness.job)).resolves.toBeUndefined(); - - expect(harness.upsertDeployables).toHaveBeenCalledTimes(1); - expect(harness.upsertWebhooksWithYaml).toHaveBeenCalledTimes(1); - expect(harness.findOrCreateDeploys).not.toHaveBeenCalled(); - expect(harness.buildPatch.mock.calls.map(([value]) => value.status).filter(Boolean)).toEqual([BuildStatus.PENDING]); - expect(harness.buildPatch).not.toHaveBeenCalledWith({ observedGeneration: 3 }); - expect(mockQueueAdd).not.toHaveBeenCalled(); - }); - - test('deployment reconciliation stops after deploy association when another run takes authority', async () => { - let current = true; - const harness = publicReconciliationHarness({ - authorityResult: (_read, build) => (current ? build : { ...build, runUUID: 'newer-run' }), - onDeploysAssociated: () => { - current = false; - }, - }); - - await expect(harness.service.processDeploymentReconciliationQueue(harness.job)).resolves.toBeUndefined(); - - expect(harness.upsertDeployables).toHaveBeenCalledTimes(1); - expect(harness.findOrCreateDeploys).toHaveBeenCalledTimes(1); - expect(harness.build.$setRelated).toHaveBeenCalledWith('deploys', []); - expect(harness.buildPatch.mock.calls.map(([value]) => value.status).filter(Boolean)).toEqual([BuildStatus.PENDING]); - expect(harness.buildPatch).not.toHaveBeenCalledWith({ observedGeneration: 3 }); - expect(mockQueueAdd).not.toHaveBeenCalled(); - }); - - test('deployment reconciliation stops before execution when authority moves after graph generation', async () => { - let current = true; - const harness = publicReconciliationHarness({ - authorityResult: (_read, build) => (current ? build : { ...build, runUUID: 'newer-run' }), - onGraphGenerated: () => { - current = false; - }, - }); - - await expect(harness.service.processDeploymentReconciliationQueue(harness.job)).resolves.toBeUndefined(); - - expect(harness.findOrCreateDeploys).toHaveBeenCalledTimes(1); - expect(harness.buildPatch.mock.calls.map(([value]) => value.status).filter(Boolean)).toEqual([ - BuildStatus.PENDING, - BuildStatus.BUILDING, - ]); - expect(harness.buildPatch).not.toHaveBeenCalledWith({ observedGeneration: 3 }); - expect(mockQueueAdd).not.toHaveBeenCalled(); - }); - - test('deployment reconciliation stops before execution when authority moves after the configuration lock releases', async () => { - const harness = publicReconciliationHarness({ - authorityResult: (read, build) => (read >= 8 ? { ...build, runUUID: 'newer-run' } : build), - }); - - await expect(harness.service.processDeploymentReconciliationQueue(harness.job)).resolves.toBeUndefined(); - - expect(harness.upsertDeployables).toHaveBeenCalledTimes(1); - expect(harness.findOrCreateDeploys).toHaveBeenCalledTimes(1); - expect(harness.buildPatch.mock.calls.map(([value]) => value.status).filter(Boolean)).toEqual([ - BuildStatus.PENDING, - BuildStatus.BUILDING, - ]); - expect(harness.buildPatch).not.toHaveBeenCalledWith({ observedGeneration: 3 }); - expect(mockQueueAdd).not.toHaveBeenCalled(); - }); - - test('deployment reconciliation stops after scope execution when authority moves before publication', async () => { - const harness = publicReconciliationHarness({ - authorityResult: (read, build) => (read >= 16 ? { ...build, runUUID: 'newer-run' } : build), - }); - - await expect(harness.service.processDeploymentReconciliationQueue(harness.job)).resolves.toBeUndefined(); - - expect(harness.upsertDeployables).toHaveBeenCalledTimes(1); - expect(harness.findOrCreateDeploys).toHaveBeenCalledTimes(1); - expect(harness.buildPatch.mock.calls.map(([value]) => value.status).filter(Boolean)).toEqual([ - BuildStatus.PENDING, - BuildStatus.BUILDING, - BuildStatus.DEPLOYING, - ]); - expect(harness.buildPatch).not.toHaveBeenCalledWith({ observedGeneration: 3 }); - expect(mockQueueAdd).not.toHaveBeenCalled(); - }); - - test('deployment reconciliation treats omitted queue attempt metadata as one final attempt', async () => { - const failure = new Error('deployable import unavailable'); - const harness = publicReconciliationHarness({ yamlFailure: failure }); - - await expect( - harness.service.processDeploymentReconciliationQueue({ data: harness.job.data } as any) - ).resolves.toBeUndefined(); - - expect(harness.upsertDeployables).toHaveBeenCalledTimes(1); - expect(harness.findOrCreateDeploys).not.toHaveBeenCalled(); - expect(harness.buildPatch).toHaveBeenCalledWith({ - status: BuildStatus.ERROR, - statusMessage: 'deployable import unavailable', - }); - expect(harness.buildPatch).toHaveBeenCalledWith({ observedGeneration: 3 }); - }); - - test('deployment reconciliation reacquires authority to publish a final pre-active read failure', async () => { - const failure = new Error('build authority read unavailable'); - let databaseRecovered = false; - const harness = publicReconciliationHarness({ - loadResult: (_read, build) => { - if (!databaseRecovered) { - databaseRecovered = true; - throw failure; - } - return build; - }, - }); - - await expect( - harness.service.processDeploymentReconciliationQueue({ data: harness.job.data } as any) - ).resolves.toBeUndefined(); - - expect(harness.upsertDeployables).not.toHaveBeenCalled(); - expect(harness.buildPatch).toHaveBeenCalledWith({ - status: BuildStatus.ERROR, - statusMessage: 'build authority read unavailable', - }); - expect(harness.buildPatch).toHaveBeenCalledWith({ observedGeneration: 3 }); - }); - - test('deployment reconciliation contains a pre-active failure when the build is deleted before reclaim', async () => { - const failure = new Error('build authority read unavailable'); - let readFailed = false; - let deleted = false; - const harness = publicReconciliationHarness({ - mailboxResult: (_read, build) => (deleted ? null : build), - loadResult: (_read, build) => { - if (!readFailed) { - readFailed = true; - throw failure; - } - return deleted ? null : build; - }, - onLockAcquired: (resource, build) => { - if (resource === 'build-deployment.7' && readFailed) { - deleted = true; - build.deletedAt = new Date().toISOString(); - } - }, - }); - - await expect( - harness.service.processDeploymentReconciliationQueue({ data: harness.job.data } as any) - ).resolves.toBeUndefined(); - - expect(deleted).toBe(true); - expect(harness.upsertDeployables).not.toHaveBeenCalled(); - expect(harness.buildPatch).not.toHaveBeenCalledWith(expect.objectContaining({ status: BuildStatus.ERROR })); - expect(harness.buildPatch).not.toHaveBeenCalledWith({ observedGeneration: 3 }); - }); - - test.each([ - ['before YAML import', 4, 0, 0], - ['after YAML import', 5, 1, 0], - ['after deploy association', 6, 1, 1], - ])( - 'createBuild stops %s when post-claim setup authority moves to another run', - async (_stage, loseAuthorityOnRead, expectedImports, expectedDeployLookups) => { - const mutation: any = { - where: jest.fn(() => mutation), - whereNull: jest.fn(() => mutation), - then: (resolve: (value: number) => unknown, reject?: (reason: unknown) => unknown) => - Promise.resolve(1).then(resolve, reject), - }; - const deploys = [{ id: 11, uuid: 'app' }]; - const build: any = { - id: 7, - uuid: 'post-claim-race', - kind: BuildKind.SANDBOX, - status: BuildStatus.QUEUED, - deletedAt: null, - runUUID: null, - pullRequestId: 55, - pullRequest: { status: 'open', deployOnUpdate: true }, - deploys: [], - $fetchGraph: jest.fn().mockResolvedValue(undefined), - $setRelated: jest.fn((_relation: string, related: any[]) => { - build.deploys = related; - }), - }; - const supersedingBuild: any = { - ...build, - runUUID: 'new-run-owner', - $fetchGraph: jest.fn().mockResolvedValue(undefined), - }; - let authorityReads = 0; - const buildQuery: any = { - where: jest.fn(() => buildQuery), - whereNull: jest.fn(() => buildQuery), - first: jest.fn().mockResolvedValue(build), - findOne: jest.fn(async () => { - authorityReads += 1; - return authorityReads >= loseAuthorityOnRead ? supersedingBuild : build; - }), - patch: jest.fn(() => mutation), - }; - const upsertDeployables = jest.fn().mockResolvedValue({ canReconcile: false }); - const findOrCreateDeploys = jest.fn().mockResolvedValue(deploys); - const upsertWebhooksWithYaml = jest.fn().mockResolvedValue(undefined); - const service = serviceWith({ - models: { Build: { query: jest.fn(() => buildQuery) } }, - services: { - Deployable: { upsertDeployables }, - Deploy: { findOrCreateDeploys }, - Webhook: { upsertWebhooksWithYaml }, - }, - }); - - await expect( - service.createBuild( - { id: 5 } as any, - { pullRequestId: 55, repositoryId: 2, repositoryBranchName: 'feature' }, - {} as any - ) - ).resolves.toBeUndefined(); - - expect(upsertDeployables).toHaveBeenCalledTimes(expectedImports); - expect(upsertWebhooksWithYaml).toHaveBeenCalledTimes(expectedImports); - expect(findOrCreateDeploys).toHaveBeenCalledTimes(expectedDeployLookups); - expect(build.$setRelated).toHaveBeenCalledTimes(expectedDeployLookups); - expect(buildQuery.patch).toHaveBeenCalledTimes(1); - expect(buildQuery.patch).not.toHaveBeenCalledWith(expect.objectContaining({ status: expect.anything() })); - } - ); - - test('createBuild records a fenced failure when deploy discovery returns no result', async () => { - const mutation: any = { - patch: jest.fn(() => mutation), - where: jest.fn(() => mutation), - whereNull: jest.fn(() => mutation), - whereNotIn: jest.fn(() => mutation), - then: (resolve: (value: number) => unknown, reject?: (reason: unknown) => unknown) => - Promise.resolve(1).then(resolve, reject), - }; - const build: any = { - id: 7, - uuid: 'missing-deploy-result', - kind: BuildKind.SANDBOX, - status: BuildStatus.QUEUED, - deletedAt: null, - runUUID: null, - pullRequestId: 55, - pullRequest: { status: 'open', deployOnUpdate: true }, - deploys: [], - reload: jest.fn().mockResolvedValue(undefined), - $fetchGraph: jest.fn().mockResolvedValue(undefined), - $setRelated: jest.fn(), + $setRelated: jest.fn(), }; const buildQuery: any = { where: jest.fn(() => buildQuery), @@ -6239,6 +5512,7 @@ describe('BuildService uncovered public behavior', () => { }; const deployRead: any = { where: jest.fn(() => deployRead), + whereNotIn: jest.fn(() => deployRead), withGraphFetched: jest.fn().mockResolvedValue([deploy]), }; mockDeployQuery.mockReturnValue(deployRead); @@ -6250,11 +5524,15 @@ describe('BuildService uncovered public behavior', () => { deletedAt: null, pullRequestId: null, desiredGeneration: 5, + acceptedRefs: {}, }; const currentQuery = lookupQuery(supersedingBuild); const buildImage = jest.fn(); const service = serviceWith({ - models: { Build: { query: jest.fn(() => currentQuery) } }, + models: { + Build: { query: jest.fn(() => currentQuery) }, + Deploy: { query: jest.fn(() => lookupQuery(undefined)) }, + }, services: { Deploy: { buildImage } }, }); const build: any = { @@ -6286,9 +5564,15 @@ describe('BuildService uncovered public behavior', () => { deletedAt: null, pullRequestId: null, desiredGeneration: 5, + acceptedRefs: {}, }; const currentQuery = lookupQuery(supersedingBuild); - const service = serviceWith({ models: { Build: { query: jest.fn(() => currentQuery) } } }); + const service = serviceWith({ + models: { + Build: { query: jest.fn(() => currentQuery) }, + Deploy: { query: jest.fn(() => lookupQuery(undefined)) }, + }, + }); const build: any = { id: 7, uuid: 'superseded-manifest-build', @@ -6343,6 +5627,262 @@ describe('BuildService uncovered public behavior', () => { expect(mockQueueAdd).not.toHaveBeenCalled(); }); + test('claimDeploymentIntent ignores a generation that was replaced or already finished, and adopts only a live one', async () => { + const refs = { + 'source:1:main': { + type: 'source', + requestId: 'newer', + target: 'repository', + githubRepositoryId: 1, + branch: 'main', + sha: 'b', + gen: 4, + }, + all: { type: 'all', requestId: 'done', gen: 2, observedGen: 2 }, + }; + const buildQuery = lookupQuery({ id: 7, acceptedRefs: refs }); + const service = serviceWith({ models: { Build: { query: jest.fn(() => buildQuery) } } }); + const adopt = jest.spyOn(service as any, 'adoptDeploysForIntent').mockResolvedValue(undefined); + + await expect((service as any).claimDeploymentIntent(7, 3)).resolves.toBeNull(); + await expect((service as any).claimDeploymentIntent(7, 2)).resolves.toBeNull(); + expect(adopt).not.toHaveBeenCalled(); + await expect((service as any).claimDeploymentIntent(7, 4)).resolves.toMatchObject({ + generation: 4, + token: 'newer', + }); + expect(adopt).toHaveBeenCalledTimes(1); + }); + + describe('processDeploymentReconciliationQueue failure policy', () => { + const run = (error: Error, options: { current: boolean; attemptsMade?: number }) => { + const service = serviceWith({ models: {} }); + jest + .spyOn(service as any, 'tryWithDeploymentGenerationLock') + .mockImplementation((_buildId: unknown, _generation: unknown, action: any) => action()); + jest.spyOn(service as any, 'claimDeploymentIntent').mockResolvedValue({ + generation: 3, + token: 'run-t', + intent: { type: 'all', requestId: 'run-t', gen: 3 }, + acceptedSourcePins: {}, + }); + jest.spyOn(service as any, 'withCurrentBuildDeploymentLock').mockRejectedValue(error); + jest.spyOn(service as any, 'isDeploymentRunCurrent').mockResolvedValue(options.current); + const spies = { + takeIntentDeploys: jest.spyOn(service as any, 'takeIntentDeploys').mockResolvedValue(undefined), + observeIntentDeploys: jest.spyOn(service as any, 'observeIntentDeploys').mockResolvedValue(0), + finishIntent: jest.spyOn(service as any, 'finishIntent').mockResolvedValue(undefined), + recordFinalFailure: jest + .spyOn(service as any, 'recordFinalDeploymentReconciliationFailure') + .mockResolvedValue(undefined), + }; + const job: any = { + data: { buildId: 7, generation: 3 }, + attemptsMade: options.attemptsMade ?? 0, + opts: { attempts: 10 }, + }; + return { outcome: service.processDeploymentReconciliationQueue(job), spies }; + }; + + test('a configuration error that escapes the configuration step takes and observes the rows and finishes the intent', async () => { + const { outcome, spies } = run(new ValidationError('services must be a list'), { current: true }); + + await expect(outcome).resolves.toBeUndefined(); + expect(spies.takeIntentDeploys).toHaveBeenCalledWith(7, 3, 'run-t'); + expect(spies.observeIntentDeploys).toHaveBeenCalledWith(7, 3); + expect(spies.finishIntent).toHaveBeenCalledWith(7, 3); + expect(spies.recordFinalFailure).not.toHaveBeenCalled(); + }); + + test('an error after the run lost its rows is ignored: no retry, no record, no finish', async () => { + const { outcome, spies } = run(new Error('database connection reset'), { current: false }); + + await expect(outcome).resolves.toBeUndefined(); + expect(spies.takeIntentDeploys).not.toHaveBeenCalled(); + expect(spies.finishIntent).not.toHaveBeenCalled(); + expect(spies.recordFinalFailure).not.toHaveBeenCalled(); + }); + + test('a transient error of a current run is rethrown for the queue retry until the final attempt records it', async () => { + const error = new Error('database connection reset'); + const first = run(error, { current: true, attemptsMade: 0 }); + await expect(first.outcome).rejects.toBe(error); + expect(first.spies.recordFinalFailure).not.toHaveBeenCalled(); + + const last = run(error, { current: true, attemptsMade: 9 }); + await expect(last.outcome).resolves.toBeUndefined(); + expect(last.spies.recordFinalFailure).toHaveBeenCalledWith(7, expect.objectContaining({ generation: 3 }), error); + expect(last.spies.finishIntent).not.toHaveBeenCalled(); + }); + + test('a lost mutation lock of a current run is rethrown so the retry resumes the rows it owns', async () => { + const error = new AuthorityLockLostError('build-reconcile.7.3'); + const { outcome, spies } = run(error, { current: true }); + + await expect(outcome).rejects.toBe(error); + expect(spies.recordFinalFailure).not.toHaveBeenCalled(); + }); + + test('an environment-wide intent whose run claim fails leaves its generation pending instead of consuming it', async () => { + const service = serviceWith({ models: {} }); + jest + .spyOn(service as any, 'tryWithDeploymentGenerationLock') + .mockImplementation((_buildId: unknown, _generation: unknown, action: any) => action()); + jest.spyOn(service as any, 'claimDeploymentIntent').mockResolvedValue({ + generation: 3, + token: 'run-t', + intent: { type: 'all', requestId: 'run-t', gen: 3 }, + acceptedSourcePins: {}, + }); + jest + .spyOn(service as any, 'withCurrentBuildDeploymentLock') + .mockImplementation(async (_buildId: unknown, _isPending: unknown, action: any) => ({ + admitted: true, + value: await action(), + })); + jest.spyOn(service as any, 'loadBuildDeploymentAuthority').mockResolvedValue({ + id: 7, + status: BuildStatus.DEPLOYED, + deployEnabled: true, + deletedAt: null, + pullRequestId: null, + }); + const claimRun = jest.spyOn(service as any, 'claimDeploymentRun').mockResolvedValue(null); + const observe = jest.spyOn(service as any, 'observeIntentDeploys').mockResolvedValue(0); + const finish = jest.spyOn(service as any, 'finishIntent').mockResolvedValue(undefined); + const importYaml = jest.spyOn(service as any, 'importYamlConfigFile').mockResolvedValue(undefined); + + await expect( + service.processDeploymentReconciliationQueue({ + data: { buildId: 7, generation: 3 }, + attemptsMade: 0, + opts: { attempts: 10 }, + } as any) + ).resolves.toBeUndefined(); + + expect(claimRun).toHaveBeenCalledWith(expect.objectContaining({ id: 7 }), 'run-t', 3); + expect(importYaml).not.toHaveBeenCalled(); + expect(observe).not.toHaveBeenCalled(); + expect(finish).not.toHaveBeenCalled(); + }); + }); + + describe('isDeploymentRunCurrent', () => { + const check = ( + authority: Record | null, + ownedRow: unknown, + refs: Record = {} + ) => { + const build = authority + ? { acceptedRefs: refs, $fetchGraph: jest.fn().mockResolvedValue(undefined), ...authority } + : null; + const service = serviceWith({ + models: { + Build: { query: jest.fn(() => lookupQuery(build)) }, + Deploy: { query: jest.fn(() => lookupQuery(ownedRow)) }, + }, + }); + return (service as any).isDeploymentRunCurrent(7, 'run-t', 3) as Promise; + }; + const open = { id: 7, status: BuildStatus.DEPLOYING, deployEnabled: true, deletedAt: null, pullRequestId: null }; + + test('is false while the environment is blocked, even for a run that still owns a row', async () => { + await expect(check({ ...open, status: BuildStatus.TEARING_DOWN }, { id: 11 })).resolves.toBe(false); + await expect(check({ ...open, deployEnabled: false }, { id: 11 })).resolves.toBe(false); + await expect( + check( + { ...open, pullRequestId: 5, pullRequest: { status: PullRequestStatus.CLOSED, deployOnUpdate: true } }, + { id: 11 } + ) + ).resolves.toBe(false); + await expect(check(null, { id: 11 })).resolves.toBe(false); + }); + + test('is true for an owned row, or for an accepted intent that is still unfinished, and false otherwise', async () => { + await expect(check(open, { id: 11 })).resolves.toBe(true); + await expect(check(open, undefined, { all: { type: 'all', requestId: 'run-t', gen: 3 } })).resolves.toBe(true); + await expect( + check(open, undefined, { all: { type: 'all', requestId: 'run-t', gen: 3, observedGen: 3 } }) + ).resolves.toBe(false); + await expect(check(open, undefined, { all: { type: 'all', requestId: 'other', gen: 3 } })).resolves.toBe(false); + }); + + test('a run without a generation is current while the environment carries its token', async () => { + const service = serviceWith({ + models: { Build: { query: jest.fn(() => lookupQuery({ ...open, runUUID: 'run-legacy' })) } }, + }); + await expect((service as any).isDeploymentRunCurrent(7, 'run-legacy')).resolves.toBe(true); + await expect((service as any).isDeploymentRunCurrent(7, 'run-other')).resolves.toBe(false); + }); + }); + + describe('publishDerivedBuildStatusUnlocked', () => { + const row = (overrides: Record) => ({ + active: true, + desiredGeneration: 3, + observedGeneration: 3, + status: DeployStatus.READY, + statusMessage: '', + deployable: { name: 'svc' }, + ...overrides, + }); + const derive = async (build: Record, options: Record = {}) => { + const loaded: any = { + id: 7, + statusMessage: '', + deployEnabled: true, + pullRequest: null, + acceptedRefs: {}, + deploys: [], + ...build, + $fetchGraph: jest.fn().mockResolvedValue(undefined), + }; + const service = serviceWith({ models: { Build: { query: jest.fn(() => lookupQuery(loaded)) } } }); + const publish = jest.spyOn(service as any, 'publishBuildStatus').mockResolvedValue(undefined); + await (service as any).publishDerivedBuildStatusUnlocked(7, options); + return publish; + }; + + test('keeps a tearing-down or torn-down environment and a standing config error', async () => { + expect(await derive({ status: BuildStatus.TEARING_DOWN })).not.toHaveBeenCalled(); + expect(await derive({ status: BuildStatus.TORN_DOWN })).not.toHaveBeenCalled(); + expect(await derive({ status: BuildStatus.CONFIG_ERROR })).not.toHaveBeenCalled(); + const cleared = await derive({ status: BuildStatus.CONFIG_ERROR }, { clearConfigError: true }); + expect(cleared).toHaveBeenCalledWith(expect.anything(), BuildStatus.DEPLOYED, expect.anything()); + }); + + test('a pending row outranks a failed one, a failed row outranks success, and an unchanged status is not rewritten', async () => { + const pendingAndFailed = await derive({ + status: BuildStatus.ERROR, + deploys: [ + row({ desiredGeneration: 4, status: DeployStatus.BUILDING }), + row({ status: DeployStatus.DEPLOY_FAILED }), + ], + }); + expect(pendingAndFailed).toHaveBeenCalledWith(expect.anything(), BuildStatus.BUILDING, expect.anything()); + + const rolling = await derive({ + status: BuildStatus.BUILDING, + deploys: [row({ desiredGeneration: 4, status: DeployStatus.DEPLOYING })], + }); + expect(rolling).toHaveBeenCalledWith(expect.anything(), BuildStatus.DEPLOYING, expect.anything()); + + const failed = await derive({ + status: BuildStatus.DEPLOYED, + deploys: [row({ status: DeployStatus.DEPLOY_FAILED, statusMessage: 'Not deployed: db failed.' })], + }); + expect(failed).toHaveBeenCalledWith(expect.anything(), BuildStatus.ERROR, expect.anything()); + + expect(await derive({ status: BuildStatus.DEPLOYED, deploys: [row({})] })).not.toHaveBeenCalled(); + + const inactiveFailure = await derive({ + status: BuildStatus.BUILDING, + deploys: [row({}), row({ active: false, status: DeployStatus.DEPLOY_FAILED })], + }); + expect(inactiveFailure).toHaveBeenCalledWith(expect.anything(), BuildStatus.DEPLOYED, expect.anything()); + }); + }); + test('createApiEnvironment does not retry a generated name after a non-unique insert failure', async () => { const apiConfig = { enabled: true, defaultTtlHours: 72, maxTtlHours: 336, extensionHours: 24 }; const repository = { id: 2, githubRepositoryId: 42, fullName: 'org/repo', defaultEnvId: 5 }; @@ -6447,7 +5987,7 @@ describe('BuildService uncovered public behavior', () => { { name: 'build promotion', resource: 'build-promotion.41', - ttlMs: 15 * 60 * 1000, + ttlMs: 60_000, warning: 'Build promotion: waiting for admitted native mutation', context: (error: Error) => ({ error, buildId: 41 }), run: (service: BuildService, isCurrent: () => Promise, action: () => Promise) => @@ -6500,6 +6040,7 @@ describe('BuildService uncovered public behavior', () => { const { acquiredLock, lockWithOptions, redlock } = contendedAuthorityRedlock(waitError); const noDeploys: any = { where: jest.fn(() => noDeploys), + whereNotIn: jest.fn(() => noDeploys), withGraphFetched: jest.fn().mockResolvedValue([]), }; mockDeployQuery.mockReturnValue(noDeploys); @@ -6520,7 +6061,15 @@ describe('BuildService uncovered public behavior', () => { $fetchGraph: jest.fn().mockResolvedValue(undefined), }; const currentBuildQuery = lookupQuery(build); - const service = serviceWith({ models: { Build: { query: jest.fn(() => currentBuildQuery) } } }, redlock); + const service = serviceWith( + { + models: { + Build: { query: jest.fn(() => currentBuildQuery) }, + Deploy: { query: jest.fn(() => lookupQuery({ id: 11 })) }, + }, + }, + redlock + ); const result = service.generateAndApplyManifests({ build, @@ -6534,7 +6083,7 @@ describe('BuildService uncovered public behavior', () => { await expect(result).resolves.toBe(true); expect(lockWithOptions).toHaveBeenCalledTimes(2); - expect(lockWithOptions).toHaveBeenNthCalledWith(1, 'build-deployment.7', 15 * 60 * 1000, { + expect(lockWithOptions).toHaveBeenNthCalledWith(1, 'build-deployment.7', 60_000, { retryCount: 4, retryDelay: 1000, retryJitter: 200, @@ -6610,7 +6159,7 @@ describe('BuildService uncovered public behavior', () => { await expect(result).resolves.toBeUndefined(); expect(lockWithOptions).toHaveBeenCalledTimes(2); - expect(lockWithOptions).toHaveBeenNthCalledWith(1, 'build-deployment.7', 15 * 60 * 1000, { + expect(lockWithOptions).toHaveBeenNthCalledWith(1, 'build-deployment.7', 60_000, { retryCount: 4, retryDelay: 1000, retryJitter: 200, diff --git a/src/server/services/__tests__/deploy.test.ts b/src/server/services/__tests__/deploy.test.ts index bf4ee7d6..6dd65860 100644 --- a/src/server/services/__tests__/deploy.test.ts +++ b/src/server/services/__tests__/deploy.test.ts @@ -732,15 +732,10 @@ describe('DeployService - shouldTriggerGithubDeployment', () => { expect(mockBuildWithNative).not.toHaveBeenCalled(); }); - test('generation check uses buildId before the Build relation is loaded', async () => { - const currentBuildWhere = jest.fn().mockResolvedValue({ id: 91 }); - mockDb.models.Build = { - query: jest.fn(() => ({ - findOne: jest.fn(() => ({ - whereNull: jest.fn(() => ({ where: currentBuildWhere })), - })), - })), - }; + test('generation check fences on the Deploy row and the environment block state before the Build relation is loaded', async () => { + const authoritySelect = jest.fn().mockResolvedValue({ id: 91, deployEnabled: true }); + const authorityFindById = jest.fn(() => ({ select: authoritySelect })); + mockDb.models.Build = { query: jest.fn(() => ({ findById: authorityFindById })) }; const patchSpy = jest.spyOn(deployService, 'patchAndUpdateActivityFeed').mockResolvedValue(undefined); const deploy = { id: 17, @@ -755,7 +750,8 @@ describe('DeployService - shouldTriggerGithubDeployment', () => { deployService.buildImage(deploy as any, 0, 'run-c', undefined, undefined, undefined, 7) ).resolves.toBe(true); - expect(currentBuildWhere).toHaveBeenCalledWith('desiredGeneration', 7); + expect(currentDeployFindOne).toHaveBeenCalledWith({ id: 17, runUUID: 'run-c', desiredGeneration: 7 }); + expect(authorityFindById).toHaveBeenCalledWith(91); expect(deploy.$fetchGraph).toHaveBeenCalled(); expect(patchSpy).toHaveBeenCalledWith( deploy, @@ -789,6 +785,15 @@ describe('DeployService - shouldTriggerGithubDeployment', () => { expect(deploy.$fetchGraph).not.toHaveBeenCalled(); }); + test('patchDeployForRun fences on the generation it was handed', async () => { + const deploy = { id: 17, uuid: 'sample-service-build', runUUID: 'run-1' }; + + await deployService['patchDeployForRun'](deploy as any, 'run-1', { status: DeployStatus.DEPLOYING }, 6); + + expect(conditionalDeployWhere).toHaveBeenCalledWith({ id: 17, runUUID: 'run-1', desiredGeneration: 6 }); + expect(conditionalDeployPatch).toHaveBeenCalledWith({ status: DeployStatus.DEPLOYING }); + }); + test('recordDeployFailure writes a terminal status with the original error message', async () => { const patchSpy = jest.spyOn(deployService, 'patchAndUpdateActivityFeed').mockResolvedValue(undefined); const deploy = { @@ -1785,10 +1790,12 @@ describe('DeployService uncovered public behavior', () => { }; const buildQuery: any = { findOne: jest.fn(() => buildQuery), + findById: jest.fn(() => buildQuery), + select: jest.fn(() => buildQuery), whereNull: jest.fn(() => buildQuery), where: jest.fn(() => buildQuery), then: (resolve: (value: unknown) => unknown, reject?: (reason: unknown) => unknown) => - Promise.resolve({ id: 91 }).then(resolve, reject), + Promise.resolve({ id: 91, deployEnabled: true }).then(resolve, reject), }; const githubDeploymentAdd = jest.fn().mockResolvedValue(undefined); const updatePullRequestActivityStream = jest.fn().mockResolvedValue(undefined); @@ -2205,6 +2212,80 @@ describe('DeployService uncovered public behavior', () => { expect(mockCliDeploy).not.toHaveBeenCalled(); }); + test.each([DeployStatus.QUEUED, DeployStatus.ERROR])( + 'deployAurora keeps a restored endpoint from %s once the check finds its database', + async (status) => { + const { service, deployPatch } = serviceHarness(); + mockTaggingGetResources.mockResolvedValue({ + ResourceTagMappingList: [{ ResourceARN: 'arn:aws:rds:us-west-2:123:db:instance-1' }], + }); + mockRdsDescribeDBInstances.mockResolvedValue({ + DBInstances: [{ Endpoint: { Address: 'database-env.cluster-abc.us-west-2.rds.amazonaws.com' } }], + }); + const deploy: any = { + id: 1, + uuid: 'database-env', + status, + cname: 'database-env.cluster-abc.us-west-2.rds.amazonaws.com', + build: { uuid: 'env' }, + deployable: { name: 'database', type: DeployTypes.AURORA_RESTORE }, + reload: jest.fn().mockResolvedValue(undefined), + $fetchGraph: jest.fn().mockResolvedValue(undefined), + }; + + await expect(service.deployAurora(deploy, 'run-1')).resolves.toBe(true); + + expect(mockCliDeploy).not.toHaveBeenCalled(); + expect(deployPatch).toHaveBeenCalledTimes(1); + expect(deployPatch).toHaveBeenCalledWith({ + cname: 'database-env.cluster-abc.us-west-2.rds.amazonaws.com', + status: DeployStatus.BUILT, + }); + } + ); + + test('deployAurora keeps a restored endpoint when the database check fails', async () => { + const { service, deployPatch } = serviceHarness(); + mockTaggingGetResources.mockRejectedValue(new Error('throttled')); + const deploy: any = { + id: 1, + uuid: 'database-env', + status: DeployStatus.QUEUED, + cname: 'database-env.cluster-abc.us-west-2.rds.amazonaws.com', + build: { uuid: 'env' }, + deployable: { name: 'database', type: DeployTypes.AURORA_RESTORE }, + reload: jest.fn().mockResolvedValue(undefined), + $fetchGraph: jest.fn().mockResolvedValue(undefined), + }; + + await expect(service.deployAurora(deploy, 'run-1')).resolves.toBe(true); + + expect(mockCliDeploy).not.toHaveBeenCalled(); + expect(deployPatch).toHaveBeenCalledTimes(1); + expect(deployPatch).toHaveBeenCalledWith({ status: DeployStatus.BUILT }); + }); + + test('deployAurora restores again when its database is gone, as after a teardown that kept the row', async () => { + const { service, deployPatch } = serviceHarness(); + mockTaggingGetResources.mockResolvedValue({ ResourceTagMappingList: [] }); + const deploy: any = { + id: 1, + uuid: 'database-env', + status: DeployStatus.QUEUED, + cname: 'database-env.cluster-abc.us-west-2.rds.amazonaws.com', + build: { uuid: 'env' }, + deployable: { name: 'database', type: DeployTypes.AURORA_RESTORE }, + reload: jest.fn().mockResolvedValue(undefined), + $fetchGraph: jest.fn().mockResolvedValue(undefined), + }; + + await expect(service.deployAurora(deploy, 'run-1')).resolves.toBe(true); + + expect(mockCliDeploy).toHaveBeenCalledWith(deploy); + expect(deployPatch).toHaveBeenCalledWith(expect.objectContaining({ status: DeployStatus.BUILDING })); + expect(deployPatch).toHaveBeenLastCalledWith({ status: DeployStatus.BUILT }); + }); + test.each([ DeployStatus.QUEUED, DeployStatus.BUILDING, @@ -2366,7 +2447,7 @@ describe('DeployService uncovered public behavior', () => { await expect(service.deployCLI({ deployable: null } as any, 'run-4')).resolves.toBeUndefined(); expect(aurora).toHaveBeenCalledWith(auroraDeploy, 'run-1'); - expect(codefresh).toHaveBeenCalledWith(codefreshDeployable, 'run-2', 'sha', 42, 'main'); + expect(codefresh).toHaveBeenCalledWith(codefreshDeployable, 'run-2', 'sha', 42, 'main', undefined, undefined); }); test('hostForDeployableDeploy and acmARNForDeploy expose the stable routing fallbacks', () => { @@ -2576,6 +2657,7 @@ describe('DeployService uncovered public behavior', () => { undefined, undefined, undefined, + undefined, undefined ); }); @@ -2771,6 +2853,18 @@ describe('DeployService uncovered public behavior', () => { expect(mockWaitForCodefresh).not.toHaveBeenCalled(); }); + test('deployCodefresh stops before pipeline invocation when the deploying fence finds the row taken', async () => { + const { service, deployPatch } = serviceHarness(); + deployPatch.mockResolvedValueOnce(1).mockResolvedValueOnce(0); + const deploy = codefreshDeploy(); + + await expect(service.deployCodefresh(deploy as any, 'run-1')).resolves.toBe(true); + + expect(deployPatch).toHaveBeenCalledTimes(2); + expect(mockCodefreshDeploy).not.toHaveBeenCalled(); + expect(mockWaitForCodefresh).not.toHaveBeenCalled(); + }); + test('deployCodefresh records a source-resolution failure before invoking the pipeline', async () => { const { service } = serviceHarness(); const deploy = codefreshDeploy({ branchName: null }); diff --git a/src/server/services/__tests__/deployableSourceSeam.test.ts b/src/server/services/__tests__/deployableSourceSeam.test.ts index 39c906f7..ec9341f8 100644 --- a/src/server/services/__tests__/deployableSourceSeam.test.ts +++ b/src/server/services/__tests__/deployableSourceSeam.test.ts @@ -68,6 +68,7 @@ jest.mock('server/services/globalConfig', () => ({ })); import DeployableService from '../deployable'; +import { ParsingError } from 'server/lib/yamlConfigParser'; const makeService = () => { const db = { models: {}, services: { PullRequest: { updatePullRequestBranchName: jest.fn() } } }; @@ -214,6 +215,78 @@ describe('deployable source seam (PR vs API build)', () => { expect(mockFetchLifecycleConfigByRepository).toHaveBeenCalledWith(repository, 'root-config-sha'); }); + it('reads the root config at the newest accepted push to the root branch when the delivered source is a dependency', async () => { + const service = makeService(); + (service as any).db.models.Repository = { + query: jest.fn(() => ({ + findOne: jest.fn(() => ({ + whereNull: jest.fn().mockResolvedValue({ githubRepositoryId: 99, fullName: 'org/dependency' }), + })), + })), + }; + const repository = { githubRepositoryId: 42, fullName: 'org/repo' }; + const pullRequest: any = { + branchName: 'feature-1', + repository, + build: { deploys: [], environment: { id: 5 } }, + $fetchGraph: jest.fn().mockResolvedValue(undefined), + }; + mockFetchLifecycleConfigByRepository.mockResolvedValue(null); + + await (service as any).updateOrCreateDeployableUsingYamlConfig( + new Map(), + 1, + 'uuid-1', + pullRequest, + { id: 1 }, + 99, + 'dependency-push-sha', + 'main', + 99, + new Set(), + new Set(), + new Map(), + { '42:feature-1': 'root-accepted-sha', '99:main': 'dependency-push-sha' } + ); + + expect(mockFetchLifecycleConfigByRepository).toHaveBeenCalledWith(repository, 'root-accepted-sha'); + }); + + it('keeps an explicitly pinned config SHA ahead of any accepted root floor', async () => { + const service = makeService(); + const repository = { githubRepositoryId: 42, fullName: 'org/repo' }; + mockRepositoryWhereNull.mockResolvedValue(repository); + mockFetchLifecycleConfigByRepository.mockResolvedValue(null); + const build: any = { + id: 9, + triggerType: 'api', + githubRepositoryId: 42, + branchName: 'main', + configSha: 'root-config-sha', + deploys: [], + environment: { id: 5 }, + $fetchGraph: jest.fn().mockResolvedValue(undefined), + }; + + await (service as any).updateOrCreateDeployableUsingYamlConfig( + new Map(), + 9, + 'uuid-9', + null, + build, + undefined, + undefined, + undefined, + undefined, + new Set(), + new Set(), + new Map(), + { '42:main': 'accepted-later' } + ); + + expect(mockFetchLifecycleConfigByRepository).toHaveBeenCalledWith(repository, 'root-config-sha'); + }); + it('fetches a targeted dependency config at the same pushed SHA used for its code', async () => { const service = makeService(); const rootRepository = { githubRepositoryId: 42, fullName: 'org/root' }; @@ -1113,6 +1186,7 @@ describe('deployable source seam (PR vs API build)', () => { expect(result).toEqual({ deployables: [persistedDeployable], canReconcile: true, + configFailures: {}, filterGithubRepositoryId: 42, unresolvedServiceNames: ['unresolved-api'], unresolvedRepositoryIds: [99], @@ -1185,4 +1259,96 @@ describe('deployable source seam (PR vs API build)', () => { await expect((service as any).upsertDeployablesWithDatabase('uuid-9', 9, [])).resolves.toEqual([]); expect(query).toHaveBeenCalledTimes(4); }); + + it('a service whose lifecycle.yaml does not parse fails alone, without protecting its repository or failing the import', async () => { + const service = makeService(); + const rootRepository = { githubRepositoryId: 42, fullName: 'org/root' }; + const brokenRepository = { githubRepositoryId: 99, fullName: 'org/broken' }; + mockRepositoryWhereNull.mockResolvedValue(rootRepository); + mockFetchLifecycleConfigByRepository + .mockResolvedValueOnce({ + environment: { + defaultServices: [{ name: 'broken-api', repository: 'org/broken', branch: 'main' }], + optionalServices: [], + }, + services: [], + }) + .mockRejectedValueOnce(new ParsingError('bad indentation of a mapping entry')); + mockResolveRepository.mockResolvedValueOnce(brokenRepository); + const build: any = { + id: 9, + githubRepositoryId: 42, + branchName: 'main', + configSha: 'root-sha', + deploys: [{ uuid: 'deploy-1', deployable: { name: 'broken-api' } }], + $fetchGraph: jest.fn().mockResolvedValue(undefined), + }; + const unresolvedServiceNames = new Set(); + const unresolvedRepositoryIds = new Set(); + const configFailures = new Map(); + + const result = await (service as any).updateOrCreateDeployableUsingYamlConfig( + new Map(), + 9, + 'uuid-9', + null, + build, + undefined, + null, + null, + undefined, + unresolvedServiceNames, + unresolvedRepositoryIds, + configFailures + ); + + expect(result).toBe(true); + expect(configFailures.get('broken-api')).toContain('bad indentation of a mapping entry'); + expect(Array.from(unresolvedServiceNames)).toEqual(['broken-api']); + expect(Array.from(unresolvedRepositoryIds)).toEqual([]); + expect(mockResolveExactEnvironmentService).not.toHaveBeenCalled(); + }); + + it('an error that is not a configuration error still fails the whole import', async () => { + const service = makeService(); + mockRepositoryWhereNull.mockResolvedValue({ githubRepositoryId: 42, fullName: 'org/root' }); + mockFetchLifecycleConfigByRepository + .mockResolvedValueOnce({ + environment: { + defaultServices: [{ name: 'broken-api', repository: 'org/broken', branch: 'main' }], + optionalServices: [], + }, + services: [], + }) + .mockRejectedValueOnce(new Error('GitHub unavailable')); + mockResolveRepository.mockResolvedValueOnce({ githubRepositoryId: 99, fullName: 'org/broken' }); + const build: any = { + id: 9, + githubRepositoryId: 42, + branchName: 'main', + configSha: 'root-sha', + deploys: [{ uuid: 'deploy-1', deployable: { name: 'broken-api' } }], + $fetchGraph: jest.fn().mockResolvedValue(undefined), + }; + const configFailures = new Map(); + + await expect( + (service as any).updateOrCreateDeployableUsingYamlConfig( + new Map(), + 9, + 'uuid-9', + null, + build, + undefined, + null, + null, + undefined, + new Set(), + new Set(), + configFailures + ) + ).rejects.toThrow('GitHub unavailable'); + + expect(configFailures.size).toBe(0); + }); }); diff --git a/src/server/services/__tests__/github.behavior.test.ts b/src/server/services/__tests__/github.behavior.test.ts index b464f390..9772e544 100644 --- a/src/server/services/__tests__/github.behavior.test.ts +++ b/src/server/services/__tests__/github.behavior.test.ts @@ -111,7 +111,7 @@ function buildQueryResult(result: unknown) { const query = { whereIn: jest.fn(), andWhere: jest.fn(), - first: jest.fn().mockResolvedValue(result), + whereNull: jest.fn().mockResolvedValue(result == null ? [] : [result]), }; query.whereIn.mockImplementation((_column, callback) => { const nested = { @@ -616,9 +616,9 @@ describe('GithubService push fallback behavior', () => { throw new Error('query unavailable'); }); - await expect( - service.handlePushForStaticEnv({ githubRepositoryId: 101, branchName: 'main' }) - ).resolves.toBeUndefined(); + await expect(service.handlePushForStaticEnv({ githubRepositoryId: 101, branchName: 'main' })).resolves.toEqual( + new Set() + ); expect(mockLoggerError).toHaveBeenCalledWith( expect.objectContaining({ error: expect.any(Error) }), @@ -642,6 +642,29 @@ describe('GithubService push fallback behavior', () => { ); }); + it('claims every root static build even when one enqueue fails', async () => { + const { service, db } = createHarness(); + const query = buildQueryResult({ id: 44 }); + query.whereNull.mockResolvedValue([{ id: 44 }, { id: 45 }]); + db.models.Build.query.mockReturnValue(query); + db.services.BuildService.enqueueResolveAndDeployBuild + .mockRejectedValueOnce(new Error('redis unavailable')) + .mockResolvedValue(undefined); + + await expect( + service.handlePushForStaticEnv({ githubRepositoryId: 101, branchName: 'main', headCommit: 'head-sha' }) + ).resolves.toEqual(new Set([44, 45])); + + expect(db.services.BuildService.enqueueResolveAndDeployBuild).toHaveBeenCalledTimes(2); + expect(db.services.BuildService.enqueueResolveAndDeployBuild).toHaveBeenCalledWith( + expect.objectContaining({ buildId: 45 }) + ); + expect(mockLoggerError).toHaveBeenCalledWith( + expect.objectContaining({ error: expect.any(Error) }), + 'Push: static env redeploy enqueue failed' + ); + }); + it('builds the nested static-environment lookup and forwards optional commit refs', async () => { const { service, db } = createHarness(); db.models.Build.query.mockReturnValue(buildQueryResult({ id: 44 })); diff --git a/src/server/services/__tests__/github.test.ts b/src/server/services/__tests__/github.test.ts index 668370ec..c4b9418d 100644 --- a/src/server/services/__tests__/github.test.ts +++ b/src/server/services/__tests__/github.test.ts @@ -1103,7 +1103,7 @@ describe('Github Service - handlePushWebhook', () => { return buildQuery; }), andWhere: jest.fn().mockReturnThis(), - first: jest.fn().mockResolvedValue({ id: buildId }), + whereNull: jest.fn().mockResolvedValue([{ id: buildId }]), }; mockDb.models.Build = { query: jest.fn().mockReturnValue(buildQuery) }; @@ -1215,7 +1215,7 @@ describe('Github Service - handlePushWebhook', () => { return buildQuery; }), andWhere: jest.fn().mockReturnThis(), - first: jest.fn().mockResolvedValue({ id: buildId }), + whereNull: jest.fn().mockResolvedValue([{ id: buildId }]), }; mockDb.models.Build = { query: jest.fn().mockReturnValue(buildQuery) }; diff --git a/src/server/services/__tests__/githubAutoTrack.test.ts b/src/server/services/__tests__/githubAutoTrack.test.ts index 4dbf5c0d..44a9ff4d 100644 --- a/src/server/services/__tests__/githubAutoTrack.test.ts +++ b/src/server/services/__tests__/githubAutoTrack.test.ts @@ -119,7 +119,7 @@ describe('handlePushWebhook auto-track wiring', () => { { registerQueue: jest.fn(() => ({ add: jest.fn(), on: jest.fn() })) } as any ); const autoTrack = jest.spyOn(service as any, 'enqueueAutoTrackedApiBuilds').mockResolvedValue(undefined); - jest.spyOn(service, 'handlePushForStaticEnv').mockResolvedValue(undefined); + jest.spyOn(service, 'handlePushForStaticEnv').mockResolvedValue(new Set()); await service.handlePushWebhook({ ref: 'refs/heads/main', @@ -156,7 +156,7 @@ describe('handlePushWebhook auto-track wiring', () => { { registerQueue: jest.fn(() => ({ add: jest.fn(), on: jest.fn() })) } as any ); jest.spyOn(service as any, 'enqueueAutoTrackedApiBuilds').mockResolvedValue(undefined); - const staticFallback = jest.spyOn(service, 'handlePushForStaticEnv').mockResolvedValue(undefined); + const staticFallback = jest.spyOn(service, 'handlePushForStaticEnv').mockResolvedValue(new Set()); await service.handlePushWebhook({ ref: 'refs/heads/main', @@ -207,7 +207,7 @@ describe('handlePushWebhook auto-track wiring', () => { { registerQueue: jest.fn(() => ({ add: jest.fn(), on: jest.fn() })) } as any ); jest.spyOn(service as any, 'enqueueAutoTrackedApiBuilds').mockResolvedValue(undefined); - const staticFallback = jest.spyOn(service, 'handlePushForStaticEnv').mockResolvedValue(undefined); + const staticFallback = jest.spyOn(service, 'handlePushForStaticEnv').mockResolvedValue(new Set()); await service.handlePushWebhook({ ref: 'refs/heads/main', @@ -217,7 +217,10 @@ describe('handlePushWebhook auto-track wiring', () => { repository: { id: 42, full_name: 'org/repo' }, } as any); - expect(staticFallback).not.toHaveBeenCalled(); + // Static environments rooted at the pushed repo are always looked up, even when the repo also owns services. + expect(staticFallback).toHaveBeenCalledWith( + expect.objectContaining({ githubRepositoryId: 42, branchName: 'main', headCommit: 'sha123' }) + ); expect(enqueueResolveAndDeployBuild).toHaveBeenCalledWith( expect.objectContaining({ buildId: 7, sourceGithubRepositoryId: 42, sourceBranch: 'main' }) ); diff --git a/src/server/services/__tests__/ingress.test.ts b/src/server/services/__tests__/ingress.test.ts index c4d6fd11..a5146fe5 100644 --- a/src/server/services/__tests__/ingress.test.ts +++ b/src/server/services/__tests__/ingress.test.ts @@ -83,6 +83,7 @@ function authorityQuery(result: any) { const query: any = { findOne: jest.fn(() => query), findById: jest.fn(() => query), + select: jest.fn(() => query), whereNull: jest.fn(() => query), where: jest.fn(() => query), then: (resolve: (value: any) => void, reject: (reason: unknown) => void) => @@ -287,9 +288,8 @@ describe('IngressService', () => { data: { buildId: 7, runUUID: 'old-run', expectedGeneration: 4 }, }); - expect(authority.findOne).toHaveBeenCalledWith({ id: 7, runUUID: 'old-run' }); + expect(authority.findById).toHaveBeenCalledWith(7); expect(authority.whereNull).toHaveBeenCalledWith('deletedAt'); - expect(authority.where).toHaveBeenCalledWith('desiredGeneration', 4); expect(configurationsForBuildId).not.toHaveBeenCalled(); expect(mockGetAllConfigs).not.toHaveBeenCalled(); expect(mockLoggerInfo).toHaveBeenCalledWith('Ingress: skipped reason=superseded'); @@ -379,15 +379,21 @@ describe('IngressService', () => { ingressAnnotations: {}, ipWhitelist: [], }; + const configurationsForBuildId = jest.fn().mockResolvedValue([configuration]); const gate = jest.fn(async (_buildId, isCurrent, action) => { expect(await isCurrent()).toBe(true); + // Routes are read only once the lock is held, so a sibling's newer route is never overwritten by a stale snapshot. + expect(configurationsForBuildId).not.toHaveBeenCalled(); return { admitted: true, value: await action() }; }); const db = { - models: { Build: { query: jest.fn(() => authorityQuery({ id: 7 })) } }, + models: { + Build: { query: jest.fn(() => authorityQuery({ id: 7 })) }, + Deploy: { query: jest.fn(() => authorityQuery({ id: 11 })) }, + }, services: { BuildService: { - configurationsForBuildId: jest.fn().mockResolvedValue([configuration]), + configurationsForBuildId, getNamespace: jest.fn().mockResolvedValue('env-test'), withCurrentBuildPromotionLock: gate, }, @@ -404,10 +410,55 @@ describe('IngressService', () => { expect(apply).toHaveBeenCalledWith(expect.any(String), '7-0-nginx', 'env-test', 7, 'run-c', 3); }); + test('a clean generation-fenced apply clears a stale ingress failure note; a failed apply keeps it', async () => { + const configuration = { + deployUUID: 'deploy-a', + host: 'a.example.test', + pathPortMapping: { '/': 8080 }, + serviceHost: 'service-a', + ingressAnnotations: {}, + ipWhitelist: [], + }; + const publishDerivedBuildStatus = jest.fn().mockResolvedValue(undefined); + const db = { + models: { + Build: { query: jest.fn(() => authorityQuery({ id: 7 })) }, + Deploy: { query: jest.fn(() => authorityQuery({ id: 11 })) }, + }, + services: { + BuildService: { + configurationsForBuildId: jest.fn().mockResolvedValue([configuration]), + getNamespace: jest.fn().mockResolvedValue('env-test'), + withCurrentBuildPromotionLock: jest.fn(async (_buildId, _isCurrent, action) => ({ + admitted: true, + value: await action(), + })), + publishDerivedBuildStatus, + }, + }, + }; + const service = new IngressService(db as any, {} as any, {} as any, queueManager as any); + jest + .spyOn(service as any, 'applyManifests') + .mockResolvedValueOnce(true) + .mockResolvedValueOnce(false); + const job = { data: { buildId: 7, runUUID: 'run-c', expectedGeneration: 3 } }; + + await service.createOrUpdateIngressForBuild(job); + expect(publishDerivedBuildStatus).toHaveBeenCalledWith(7, { clearStatusMessage: true }); + + publishDerivedBuildStatus.mockClear(); + await service.createOrUpdateIngressForBuild(job); + expect(publishDerivedBuildStatus).not.toHaveBeenCalled(); + }); + test('does not apply when authority is lost before promotion admission', async () => { const gate = jest.fn().mockResolvedValue({ admitted: false }); const db = { - models: { Build: { query: jest.fn(() => authorityQuery({ id: 7 })) } }, + models: { + Build: { query: jest.fn(() => authorityQuery({ id: 7 })) }, + Deploy: { query: jest.fn(() => authorityQuery({ id: 11 })) }, + }, services: { BuildService: { configurationsForBuildId: jest.fn().mockResolvedValue([ @@ -435,28 +486,49 @@ describe('IngressService', () => { expect(apply).not.toHaveBeenCalled(); }); - test('fences a late ingress failure note by run token and generation', async () => { + test('records a late ingress failure note for a generation-fenced run that still owns a service', async () => { const read = authorityQuery({ id: 7, statusMessage: 'deployed' }); const patch: any = { patch: jest.fn(() => patch), where: jest.fn(() => patch), + whereNull: jest.fn(() => patch), then: (resolve: (value: number) => void, reject: (reason: unknown) => void) => Promise.resolve(1).then(resolve, reject), }; const db = { - models: { Build: { query: jest.fn().mockReturnValueOnce(read).mockReturnValueOnce(patch) } }, + models: { + Build: { query: jest.fn().mockReturnValueOnce(read).mockReturnValueOnce(patch) }, + Deploy: { query: jest.fn(() => authorityQuery({ id: 11 })) }, + }, services: {}, }; const service = new IngressService(db as any, {} as any, {} as any, queueManager as any); await (service as any).recordIngressFailureOnBuild(7, new Error('bad route'), 'run-c', 3); - expect(read.where).toHaveBeenCalledWith('runUUID', 'run-c'); - expect(read.where).toHaveBeenCalledWith('desiredGeneration', 3); + expect(read.whereNull).toHaveBeenCalledWith('deletedAt'); + expect(read.where).not.toHaveBeenCalledWith('runUUID', 'run-c'); expect(patch.patch).toHaveBeenCalledWith({ statusMessage: 'deployed | Ingress apply failed: bad route' }); expect(patch.where).toHaveBeenCalledWith({ id: 7 }); - expect(patch.where).toHaveBeenCalledWith('runUUID', 'run-c'); - expect(patch.where).toHaveBeenCalledWith('desiredGeneration', 3); + expect(patch.whereNull).toHaveBeenCalledWith('deletedAt'); + expect(patch.where).not.toHaveBeenCalledWith('runUUID', 'run-c'); + expect(patch.where).not.toHaveBeenCalledWith('desiredGeneration', 3); + }); + + test('drops a late ingress failure note once the run owns no service any more', async () => { + const buildQuery = jest.fn(); + const db = { + models: { + Build: { query: buildQuery }, + Deploy: { query: jest.fn(() => authorityQuery(undefined)) }, + }, + services: {}, + }; + const service = new IngressService(db as any, {} as any, {} as any, queueManager as any); + + await (service as any).recordIngressFailureOnBuild(7, new Error('bad route'), 'run-old', 3); + + expect(buildQuery).not.toHaveBeenCalled(); }); test.each([ diff --git a/src/server/services/__tests__/serviceReconciliation.test.ts b/src/server/services/__tests__/serviceReconciliation.test.ts new file mode 100644 index 00000000..c4e620f8 --- /dev/null +++ b/src/server/services/__tests__/serviceReconciliation.test.ts @@ -0,0 +1,1680 @@ +/** + * Copyright 2026 GoodRx, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Drives the real reconciliation path (webhook intent -> mailbox -> queue -> + * claim -> config -> image -> rollout -> observe -> status) on a simulated + * clock. Only the leaves are simulated: the image build, the Kubernetes + * apply, the YAML import and the queue transport. + */ + +const logs: string[] = []; +const sim: any = {}; + +jest.mock('server/lib/dependencies', () => ({ + defaultDb: {}, + defaultRedis: {}, + defaultRedlock: {}, + defaultQueueManager: {}, + redisClient: { getConnection: jest.fn() }, +})); + +jest.mock('server/lib/tracer', () => ({ + Tracer: { getInstance: jest.fn(() => ({ initialize: jest.fn() })) }, +})); + +jest.mock('server/lib/logger', () => { + const record = (level: string) => (a: any, b?: any) => { + const msg = typeof a === 'string' ? a : b; + if (typeof msg === 'string') logs.push(`${level} ${msg}`); + }; + return { + getLogger: jest.fn(() => ({ + error: record('error'), + fatal: record('fatal'), + info: record('info'), + warn: record('warn'), + debug: () => undefined, + })), + withLogContext: jest.fn((_ctx, fn) => fn()), + extractContextForQueue: jest.fn(() => ({})), + updateLogContext: jest.fn(), + LogStage: {}, + }; +}); + +jest.mock('shared/config', () => ({ + TMP_PATH: '/tmp', + QUEUE_NAMES: { + DELETE_QUEUE: 'delete_queue_test', + BUILD_QUEUE: 'build_queue_test', + DEPLOYMENT_RECONCILIATION: 'deployment_reconciliation_test', + SERVICE_RECONCILIATION: 'service_reconciliation_test', + API_ENV_CREATE: 'api_env_create_test', + API_ENV_EXPIRY: 'api_env_expiry_test', + RESOLVE_AND_DEPLOY: 'resolve_and_deploy_test', + BUILD_CLEANUP_QUEUE: 'build_cleanup_test', + BUILD_REQUEST_QUEUE: 'build_request_test', + DEPLOY_CLEANUP: 'deploy_cleanup_test', + GLOBAL_CONFIG_CACHE_REFRESH: 'global-config-refresh', + GITHUB_CLIENT_TOKEN_CACHE_REFRESH: 'github-client-token-refresh', + INGRESS_MANIFEST_QUEUE: 'ingress-manifest', + INGRESS_MANIFEST: 'ingress-manifest', + INGRESS_CLEANUP: 'ingress-cleanup', + AGENT_PREWARM_QUEUE: 'agent-prewarm', + }, +})); + +jest.mock('server/models/Build', () => ({ + __esModule: true, + default: { + query: () => sim.buildQuery(), + transact: (action: any) => sim.transact(action), + }, +})); + +jest.mock('server/models/Deploy', () => ({ + __esModule: true, + default: { query: () => sim.deployQuery() }, +})); + +jest.mock('server/models', () => ({ + Build: class {}, + Deploy: { query: () => sim.deployQuery() }, + Deployable: class {}, + Environment: class {}, + Repository: { query: () => sim.emptyQuery() }, +})); + +jest.mock('server/lib/kubernetes', () => ({ + generateManifest: jest.fn(), + generateDeployManifest: jest.fn(), + applyManifests: jest.fn(), + waitForPodReady: jest.fn(), + createOrUpdateNamespace: jest.fn(), + deleteBuild: jest.fn(), + deleteNamespace: jest.fn(), +})); +jest.mock('server/lib/cli', () => ({ deleteBuild: jest.fn() })); +jest.mock('server/lib/kubernetes/common/serviceAccount', () => ({ + ensureServiceAccountForJob: jest.fn().mockResolvedValue('default'), +})); +jest.mock('server/lib/github', () => ({ + createGitDeployment: jest.fn(), + updateGitDeploymentStatus: jest.fn(), + getPullRequest: jest.fn(), + getSHAForBranch: jest.fn(), + compareCommits: jest.fn(), + getYamlFileContent: jest.fn(), + getYamlFileContentFromBranch: jest.fn(), +})); +jest.mock('server/lib/helm', () => ({ uninstallHelmReleases: jest.fn() })); +jest.mock('server/lib/helm/utils', () => ({ ingressBannerSnippet: jest.fn(() => '') })); +jest.mock('server/lib/buildEnvVariables', () => ({ + BuildEnvironmentVariables: jest.fn().mockImplementation(() => ({ resolve: jest.fn().mockResolvedValue({}) })), +})); +jest.mock('server/lib/dependencyGraph', () => ({ generateGraph: jest.fn().mockResolvedValue({}) })); +jest.mock('server/services/globalConfig', () => ({ + __esModule: true, + default: { + getInstance: jest.fn(() => ({ + getAllConfigs: jest.fn().mockResolvedValue({ serviceAccount: { name: 'default' } }), + isFeatureEnabled: jest.fn().mockResolvedValue(false), + })), + }, +})); +jest.mock('server/services/deployCleanup', () => jest.fn().mockImplementation(() => ({}))); +jest.mock('server/services/deploy', () => ({ + __esModule: true, + default: jest.fn().mockImplementation(() => ({})), +})); +jest.mock('server/services/webhook', () => ({ + __esModule: true, + default: jest.fn().mockImplementation(() => ({})), +})); +jest.mock('server/services/override', () => ({ + __esModule: true, + isBranchOrExternalUrlEditable: () => true, + default: jest.fn().mockImplementation(() => ({})), +})); +jest.mock('server/lib/fastly', () => jest.fn().mockImplementation(() => ({}))); + +import BuildService from '../build'; +import { DeploymentSupersededError } from 'server/lib/deploymentReconciliation/errors'; +import { AuthorityLockLostError } from 'server/lib/authorityLock'; +import { BuildKind, BuildStatus, DeployStatus, DeployTypes, PullRequestStatus } from 'shared/constants'; +import { ParsingError } from 'server/lib/yamlConfigParser'; + +const MIN = 60_000; +const BUILD_ID = 7; +const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + +interface ServiceSpec { + name: string; + repo: number; + buildMinutes: number; +} + +function createWorld( + specs: ServiceSpec[], + options: { applyMinutes?: number; readyMinutes?: number; concurrency?: number } = {} +) { + const applyMs = (options.applyMinutes ?? 0.5) * MIN; + const readyMs = (options.readyMinutes ?? 2) * MIN; + const events: string[] = []; + const now = () => Number((Date.now() / MIN).toFixed(1)); + const log = (line: string) => events.push(`t=${now()}m ${line}`); + + const pullRequest = { + id: 11, + status: PullRequestStatus.OPEN, + deployOnUpdate: true, + branchName: 'static', + repository: { githubRepositoryId: 900, fullName: 'org/z' }, + $fetchGraph: async () => undefined, + }; + const buildRow: any = { + id: BUILD_ID, + uuid: 'static-z', + namespace: 'env-static-z', + kind: BuildKind.ENVIRONMENT, + isStatic: true, + status: BuildStatus.DEPLOYED, + statusMessage: '', + runUUID: 'initial', + deletedAt: null, + pullRequestId: 11, + deployEnabled: true, + desiredGeneration: 0, + observedGeneration: 0, + acceptedRefs: {}, + }; + const deployRows: any[] = specs.map((spec, index) => ({ + id: index + 1, + deployableId: index + 1, + uuid: `${spec.name}-static-z`, + buildId: BUILD_ID, + githubRepositoryId: spec.repo, + branchName: 'main', + active: true, + runUUID: 'initial', + sha: `${spec.name}-v0`, + dockerImage: `${spec.name}@${spec.name}-v0`, + status: DeployStatus.READY, + statusMessage: '', + desiredGeneration: 0, + observedGeneration: 0, + deployableName: spec.name, + })); + const specByName = new Map(specs.map((spec) => [spec.name, spec])); + const registry = new Set(deployRows.map((row) => row.dockerImage)); + const live = new Map(deployRows.map((row) => [row.deployableName, row.dockerImage])); + const liveAt = new Map(); + const failImport = new Map Error | null>(); + const failBuild = new Set(); + const applyFailures: Error[] = []; + const importCounts = new Map(); + const serviceConfigFailures: Record = {}; + const servicesAddedByConfig: ServiceSpec[] = []; + const orphanRows = new Set(); + const statusWrites: string[] = []; + const dependsOn = new Map(); + const serviceMoves: Array<{ name: string; repo: number; sha: string }> = []; + const importRefs: Array<{ + filter: number | null; + repo: number | null; + ref: string | null; + pins: Record | null; + }> = []; + const counters = { builds: new Map(), applies: new Map() }; + + const hydrateDeploy = (row: any) => { + const deploy: any = { + ...row, + deployable: orphanRows.has(row.deployableName) + ? null + : { + name: row.deployableName, + type: DeployTypes.GITHUB, + deploymentDependsOn: [...(dependsOn.get(row.deployableName) ?? [])], + }, + }; + deploy.$fetchGraph = async () => undefined; + deploy.reload = async () => Object.assign(deploy, row); + deploy.$query = () => query(() => [row], hydrateDeploy, 'deploy'); + return deploy; + }; + const hydrateBuild = (row: any) => { + const build: any = { ...row }; + const attach = async () => { + build.pullRequest = row.pullRequestId != null ? pullRequest : null; + build.environment = { id: 5 }; + build.deploys = deployRows.map(hydrateDeploy); + build.deployables = []; + }; + void attach(); + build.$fetchGraph = async () => { + if (sim.fetchGraphGate) { + const gate = sim.fetchGraphGate; + sim.fetchGraphGate = null; + await gate; + } + return attach(); + }; + build.$setRelated = (relation: string, value: any) => { + build[relation] = value; + }; + build.reload = async () => Object.assign(build, row); + build.$query = () => query(() => [row], hydrateBuild, 'build'); + return build; + }; + + function query(rows: () => any[], hydrate: (row: any) => any, table = 'deploy') { + const conditions: Array<(row: any) => boolean> = []; + let patch: any; + let single = false; + const evaluate = () => { + const found = rows().filter((row) => conditions.every((condition) => condition(row))); + if (patch) { + if (sim.failPatchOnce?.(patch, table)) { + sim.failPatchOnce = null; + throw new Error('database connection reset'); + } + found.forEach((row) => { + if (row === buildRow && patch.status) statusWrites.push(`${patch.status}:${patch.statusMessage ?? ''}`); + Object.assign(row, patch); + }); + return found.length; + } + if (single) return found[0] ? hydrate(found[0]) : undefined; + return found.map(hydrate); + }; + const compare = (operator: string, left: any, right: any) => { + switch (operator) { + case '<': + return Number(left) < Number(right); + case '>': + return Number(left) > Number(right); + case '<=': + return Number(left) <= Number(right); + case '>=': + return Number(left) >= Number(right); + default: + return left === right; + } + }; + const q: any = { + select: () => q, + distinct: () => q, + withGraphFetched: () => q, + forUpdate: () => q, + orderBy: () => q, + limit: () => q, + findById: (id: number) => { + single = true; + conditions.push((row) => row.id === id); + return q; + }, + findOne: (criteria: Record) => { + single = true; + Object.entries(criteria).forEach(([key, value]) => conditions.push((row) => row[key] === value)); + return q; + }, + where: (key: any, operatorOrValue?: any, value?: any) => { + if (typeof key === 'object') { + Object.entries(key).forEach(([field, expected]) => conditions.push((row) => row[field] === expected)); + } else if (value === undefined) { + conditions.push((row) => row[key] === operatorOrValue); + } else { + conditions.push((row) => compare(operatorOrValue, row[key], value)); + } + return q; + }, + whereRaw: (sql: string, bindings: string[]) => { + const operator = sql.includes('>') ? '>' : '<'; + conditions.push((row) => compare(operator, row[bindings[0]], row[bindings[1]])); + return q; + }, + whereNull: (key: string) => { + conditions.push((row) => row[key] == null); + return q; + }, + whereNot: (key: string, value: any) => { + conditions.push((row) => row[key] !== value); + return q; + }, + whereNotIn: (key: string, values: any[]) => { + conditions.push((row) => !values.includes(row[key])); + return q; + }, + whereIn: (key: string, values: any[]) => { + conditions.push((row) => values.includes(row[key])); + return q; + }, + patch: (value: any) => { + patch = value; + return q; + }, + then: (resolve: any, reject: any) => Promise.resolve().then(evaluate).then(resolve, reject), + }; + return q; + } + + sim.buildQuery = () => query(() => [buildRow], hydrateBuild, 'build'); + sim.deployQuery = () => query(() => deployRows, hydrateDeploy, 'deploy'); + sim.deployableQuery = () => + query( + () => + deployRows + .filter((row) => !orphanRows.has(row.deployableName)) + .map((row) => ({ + id: row.deployableId, + buildId: BUILD_ID, + name: row.deployableName, + type: DeployTypes.GITHUB, + })), + (row) => row, + 'deployable' + ); + sim.emptyQuery = () => + query( + () => [], + (row) => row, + 'none' + ); + // Every transaction locks the one Build row first, so transactions run one at a time. + let lastTransaction: Promise = Promise.resolve(); + sim.transact = (action: (trx: object) => Promise) => { + const run = lastTransaction.then(() => action({})); + lastTransaction = run.catch(() => undefined); + return run; + }; + sim.failPatchOnce = null; + sim.fetchGraphGate = null; + sim.importGate = null; + sim.importGateRepo = null; + + // One mutex per resource, so the simulator exercises the same serialization the Redis locks provide. + const held = new Map>(); + const redlock = { + lock: async (resource: string) => { + const previous = held.get(resource) ?? Promise.resolve(); + let release!: () => void; + const current = new Promise((resolve) => { + release = resolve; + }); + held.set( + resource, + previous.then(() => current) + ); + await previous; + const lock: any = { unlock: async () => release(), extend: async () => lock }; + return lock; + }, + }; + + const isRowCurrent = (deployId: number, runUUID: string, generation?: number) => { + const deploy = deployRows.find((row) => row.id === deployId); + return Boolean( + deploy?.runUUID === runUUID && + (generation == null || (deploy.desiredGeneration === generation && deploy.observedGeneration < generation)) + ); + }; + + const fencedDeployPatch = (deployId: number, runUUID: string, patch: any) => { + const deploy = deployRows.find((row) => row.id === deployId); + if (deploy?.runUUID === runUUID) Object.assign(deploy, patch); + }; + + const DeployLeaf = { + findOrCreateDeploys: async ( + _env: any, + _build: any, + githubRepositoryId?: number, + sourceRef?: string | null, + sourceBranch?: string | null, + _sourceGithubRepositoryId?: number | null, + acceptedSourcePins?: Record + ) => { + for (const row of deployRows) { + const target = + !githubRepositoryId || + (row.githubRepositoryId === githubRepositoryId && (!sourceBranch || row.branchName === sourceBranch)); + if (!target) continue; + if (sourceRef && row.githubRepositoryId === githubRepositoryId) row.sha = sourceRef; + else row.sha = acceptedSourcePins?.[`${row.githubRepositoryId}:${row.branchName}`] ?? row.sha; + } + return deployRows.map(hydrateDeploy); + }, + buildImage: async ( + deploy: any, + _index: number, + runUUID: string, + _sourceRef: string | null, + _sourceRepo: number | null, + _sourceBranch: string | null, + generation?: number + ) => { + const name = deploy.deployableName; + if (!isRowCurrent(deploy.id, runUUID, generation)) return true; + const row = deployRows.find((candidate) => candidate.id === deploy.id); + const tag = `${name}@${row.sha}`; + if (failBuild.has(name)) { + fencedDeployPatch(deploy.id, runUUID, { + status: DeployStatus.BUILD_FAILED, + statusMessage: 'Image build failed.', + }); + log(`gen${generation} ${name}: build FAILED`); + return false; + } + if (registry.has(tag)) { + fencedDeployPatch(deploy.id, runUUID, { status: DeployStatus.BUILT, dockerImage: tag }); + log(`gen${generation} ${name}: image exists ${tag}`); + return true; + } + counters.builds.set(name, (counters.builds.get(name) ?? 0) + 1); + fencedDeployPatch(deploy.id, runUUID, { status: DeployStatus.BUILDING }); + log(`gen${generation} ${name}: build start ${tag}`); + await sleep(specByName.get(name)!.buildMinutes * MIN); + registry.add(tag); + if (!isRowCurrent(deploy.id, runUUID, generation)) { + log(`gen${generation} ${name}: build done ${tag} (superseded, result not published)`); + return true; + } + log(`gen${generation} ${name}: build done ${tag}`); + fencedDeployPatch(deploy.id, runUUID, { status: DeployStatus.BUILT, dockerImage: tag }); + return true; + }, + }; + + const queue = { + active: new Set(), + adds: [] as Array<{ data: any; opts: any }>, + finalFailures: [] as string[], + attempts: [] as string[], + dropNext: 0, + // Worker slots, so a scenario can show what happens when every slot is busy. + concurrency: options.concurrency ?? Infinity, + running: 0, + waiting: [] as Array<() => void>, + async acquire() { + if (queue.running < queue.concurrency) { + queue.running += 1; + return; + } + await new Promise((resolve) => queue.waiting.push(resolve)); + }, + release() { + const next = queue.waiting.shift(); + if (next) next(); + else queue.running -= 1; + }, + add(data: any, opts: any) { + queue.adds.push({ data, opts }); + if (queue.dropNext > 0) { + queue.dropNext -= 1; + log(`SIGNAL DROPPED ${opts?.jobId}`); + return Promise.resolve(); + } + const jobId = opts?.jobId; + if (queue.active.has(jobId)) return Promise.resolve(); + queue.active.add(jobId); + const attempts = 10; + const job: any = { id: jobId, data, attemptsMade: 0, opts: { attempts } }; + const run = async (attemptsMade: number): Promise => { + await queue.acquire(); + job.attemptsMade = attemptsMade; + queue.attempts.push(`${jobId}#${attemptsMade}`); + let retryIn: number | null = null; + try { + await service.processDeploymentReconciliationQueue(job); + queue.active.delete(jobId); + } catch (error: any) { + if (attemptsMade + 1 >= attempts) { + queue.active.delete(jobId); + queue.finalFailures.push(`${jobId}: ${error?.message}`); + } else { + retryIn = 5000 * 2 ** attemptsMade; + } + } finally { + queue.release(); + } + if (retryIn != null) { + await sleep(retryIn); + return run(attemptsMade + 1); + } + }; + void run(0); + return Promise.resolve(); + }, + }; + + const service: any = new BuildService( + { + models: { + Build: { query: sim.buildQuery, transact: (action: any) => sim.transact(action) }, + Deploy: { query: sim.deployQuery }, + Deployable: { query: sim.deployableQuery }, + Repository: { query: sim.emptyQuery }, + }, + services: { + Deployable: { + upsertDeployables: async (...args: any[]) => { + const repo = args[5]; + const scope = repo == null ? 'root' : String(repo); + importCounts.set(scope, (importCounts.get(scope) ?? 0) + 1); + importRefs.push({ + filter: repo ?? null, + repo: args[8] ?? null, + ref: args[6] ?? null, + pins: args[9] ?? null, + }); + if ((repo ?? null) === sim.importGateRepo && sim.importGate) { + const gate = sim.importGate; + sim.importGate = null; + await gate; + } + const failure = failImport.get(repo ?? 900)?.(); + if (failure) { + log(`import repo=${scope} FAILED: ${failure.message}`); + throw failure; + } + if (repo == null) { + for (const move of serviceMoves.splice(0)) { + const moved = deployRows.find((candidate) => candidate.deployableName === move.name)!; + Object.assign(moved, { githubRepositoryId: move.repo, sha: move.sha }); + log(`import moved ${move.name} -> repo ${move.repo} @ ${move.sha}`); + } + for (const spec of servicesAddedByConfig.splice(0)) { + deployRows.push({ + id: deployRows.length + 1, + deployableId: deployRows.length + 1, + uuid: `${spec.name}-static-z`, + buildId: BUILD_ID, + githubRepositoryId: spec.repo, + branchName: 'main', + active: true, + runUUID: null, + sha: `${spec.name}-v0`, + dockerImage: `${spec.name}@${spec.name}-v0`, + status: DeployStatus.PENDING, + statusMessage: '', + desiredGeneration: 0, + observedGeneration: 0, + deployableName: spec.name, + }); + specByName.set(spec.name, spec); + registry.add(`${spec.name}@${spec.name}-v0`); + live.set(spec.name, `${spec.name}@${spec.name}-v0`); + log(`import created ${spec.name}`); + } + } + const configFailures = Object.fromEntries( + Object.entries(serviceConfigFailures).filter( + ([name]) => repo == null || specByName.get(name)?.repo === repo + ) + ); + return { canReconcile: false, configFailures }; + }, + }, + Deploy: DeployLeaf, + Webhook: { + upsertWebhooksWithYaml: async () => undefined, + webhookQueue: { add: async () => undefined }, + }, + ActivityStream: { updatePullRequestActivityStream: async () => undefined }, + }, + } as any, + {} as any, + redlock as any, + { + registerQueue: (name: string) => ({ + add: (_jobName: string, data: any, opts: any) => + name === 'service_reconciliation_test' ? queue.add(data, opts) : Promise.resolve(), + process: jest.fn(), + on: jest.fn(), + }), + } as any + ); + + jest.spyOn(service, 'generateAndApplyManifests').mockImplementation(async (params: any) => { + const { build, runUUID, expectedGeneration, unresolvedServices } = params; + const isCurrent = () => service.isDeploymentRunCurrent(build.id, runUUID, expectedGeneration); + if (!(await isCurrent())) throw new DeploymentSupersededError(); + const injected = applyFailures.shift(); + if (injected) throw injected; + const failed = new Set([DeployStatus.ERROR, DeployStatus.BUILD_FAILED, DeployStatus.DEPLOY_FAILED]); + // A fenced rollout takes exactly the rows this run claimed; rows already READY were rolled out by an earlier attempt. + const owned = deployRows.filter((row) => row.runUUID === runUUID && row.desiredGeneration === expectedGeneration); + const targets = owned.filter((row) => !failed.has(row.status) && row.status !== DeployStatus.READY); + // As in the manager: a prerequisite this run failed blocks its dependents; one outside the run is never consulted. + const blocked = new Set([ + ...(unresolvedServices ?? []), + ...owned.filter((row) => failed.has(row.status)).map((row) => row.deployableName), + ]); + const applied: any[] = []; + await sleep(applyMs); + for (const row of targets) { + // The per-service promotion gate re-checks the row inside the lock. + if (!isRowCurrent(row.id, runUUID, expectedGeneration)) continue; + const blockedBy = (dependsOn.get(row.deployableName) ?? []).find((name) => blocked.has(name)); + if (blockedBy) { + Object.assign(row, { + status: DeployStatus.DEPLOY_FAILED, + statusMessage: `Not deployed: ${blockedBy} failed.`, + }); + log(`gen${expectedGeneration} ${row.deployableName}: BLOCKED by ${blockedBy}`); + continue; + } + applied.push(row); + // The real apply path writes progress fenced on the run token alone. + fencedDeployPatch(row.id, runUUID, { + status: DeployStatus.DEPLOYING, + statusMessage: 'Waiting for pods to be ready', + }); + const before = live.get(row.deployableName); + live.set(row.deployableName, row.dockerImage); + if (before !== row.dockerImage) liveAt.set(row.deployableName, now()); + counters.applies.set(row.deployableName, (counters.applies.get(row.deployableName) ?? 0) + 1); + log( + `gen${expectedGeneration} ${row.deployableName}: APPLIED ${row.dockerImage}${ + before === row.dockerImage ? ' (re-rollout)' : '' + }` + ); + } + if (applied.length > 0) await sleep(readyMs); + for (const row of applied) { + if (isRowCurrent(row.id, runUUID, expectedGeneration)) Object.assign(row, { status: DeployStatus.READY }); + } + if (!(await isCurrent())) throw new DeploymentSupersededError(); + return true; + }); + + let shaCounter = 0; + const merge = async (name: string) => { + const spec = specByName.get(name)!; + shaCounter += 1; + const sha = `${name}-v${shaCounter}`; + log(`MERGE ${name} -> ${sha}`); + await service.enqueueResolveAndDeployBuild({ + buildId: BUILD_ID, + githubRepositoryId: spec.repo, + sourceRef: sha, + sourceGithubRepositoryId: spec.repo, + sourceBranch: 'main', + }); + return sha; + }; + + // A push that asks for a full pass, as a pull request with failed services does. + const mergeFull = async (name: string) => { + const spec = specByName.get(name)!; + shaCounter += 1; + const sha = `${name}-v${shaCounter}`; + log(`MERGE(full) ${name} -> ${sha}`); + await service.enqueueResolveAndDeployBuild({ + buildId: BUILD_ID, + sourceRef: sha, + sourceGithubRepositoryId: spec.repo, + sourceBranch: 'main', + }); + return sha; + }; + + const pushConfigRepo = async () => { + shaCounter += 1; + const sha = `z-v${shaCounter}`; + log(`PUSH Z -> ${sha}`); + await service.enqueueResolveAndDeployBuild({ + buildId: BUILD_ID, + sourceRef: sha, + sourceGithubRepositoryId: 900, + sourceBranch: 'static', + }); + return sha; + }; + + const redeployEnvironment = async (runUUID?: string) => { + log(`ENV REDEPLOY ${runUUID ?? 'auto'}`); + await service.enqueueResolveAndDeployBuild({ buildId: BUILD_ID, ...(runUUID ? { runUUID } : {}) }); + }; + + const row = (name: string) => deployRows.find((candidate) => candidate.deployableName === name)!; + const settled = () => deployRows.every((candidate) => candidate.observedGeneration === candidate.desiredGeneration); + + return { + service, + buildRow, + deployRows, + row, + live, + liveAt, + registry, + events, + counters, + failImport, + failBuild, + applyFailures, + importCounts, + serviceConfigFailures, + servicesAddedByConfig, + orphanRows, + statusWrites, + dependsOn, + serviceMoves, + importRefs, + queue, + merge, + mergeFull, + pushConfigRepo, + redeployEnvironment, + settled, + log, + }; +} + +const flush = async () => { + for (let i = 0; i < 30; i += 1) await Promise.resolve(); +}; + +beforeEach(() => { + logs.length = 0; + jest.useFakeTimers({ now: 0 }); +}); + +afterEach(() => { + jest.useRealTimers(); +}); + +const SPECS: ServiceSpec[] = [ + { name: 'A', repo: 100, buildMinutes: 20 }, + { name: 'B', repo: 200, buildMinutes: 3 }, + { name: 'C', repo: 300, buildMinutes: 3 }, + { name: 'D', repo: 400, buildMinutes: 3 }, + { name: 'E', repo: 400, buildMinutes: 3 }, +]; + +describe('service-independent reconciliation', () => { + test('a merge to B while A builds leaves A untouched: A ships once, on its own schedule', async () => { + const world = createWorld(SPECS); + const aSha = await world.merge('A'); + await jest.advanceTimersByTimeAsync(5 * MIN); + const bSha = await world.merge('B'); + await jest.advanceTimersByTimeAsync(6 * MIN); + + expect(world.live.get('B')).toBe(`B@${bSha}`); + expect(world.buildRow.status).toBe(BuildStatus.BUILDING); + + await jest.advanceTimersByTimeAsync(30 * MIN); + + expect(world.live.get('A')).toBe(`A@${aSha}`); + expect(world.liveAt.get('A')).toBe(20.5); + expect(world.liveAt.get('B')).toBe(8.5); + expect(world.counters.builds.get('A')).toBe(1); + expect(world.counters.applies.get('B')).toBe(1); + expect(logs.filter((line) => line.includes('rollout skipped'))).toEqual([]); + expect(world.settled()).toBe(true); + expect(world.buildRow.observedGeneration).toBe(world.buildRow.desiredGeneration); + expect(world.buildRow.status).toBe(BuildStatus.DEPLOYED); + }); + + test('a newer merge to A supersedes its in-flight run and only the newest SHA rolls out', async () => { + const world = createWorld(SPECS); + const first = await world.merge('A'); + await jest.advanceTimersByTimeAsync(5 * MIN); + const second = await world.merge('A'); + await jest.advanceTimersByTimeAsync(40 * MIN); + + expect(world.live.get('A')).toBe(`A@${second}`); + expect(world.events.some((line) => line.includes(`A@${first} (superseded, result not published)`))).toBe(true); + expect(world.counters.builds.get('A')).toBe(2); + expect(world.counters.applies.get('A')).toBe(1); + expect(world.row('A').observedGeneration).toBe(2); + expect(world.settled()).toBe(true); + }); + + test('an error inside a superseded run is ignored, not retried, and the newer run still lands', async () => { + const world = createWorld(SPECS); + await world.merge('A'); + await jest.advanceTimersByTimeAsync(5 * MIN); + const second = await world.merge('A'); + sim.failPatchOnce = (patch: any, table: string) => table === 'deploy' && 'observedGeneration' in patch; + await jest.advanceTimersByTimeAsync(40 * MIN); + + expect(logs.some((line) => line.includes('stale failure ignored'))).toBe(true); + expect(world.queue.attempts.filter((attempt) => attempt.startsWith(`reconcile-${BUILD_ID}-1#`))).toEqual([ + `reconcile-${BUILD_ID}-1#0`, + ]); + expect(world.live.get('A')).toBe(`A@${second}`); + expect(world.row('A').observedGeneration).toBe(2); + expect(world.settled()).toBe(true); + }); + + test('a config error in C fails only C; A and B still ship and C recovers on its next merge', async () => { + const world = createWorld(SPECS); + const aSha = await world.merge('A'); + await jest.advanceTimersByTimeAsync(5 * MIN); + const bSha = await world.merge('B'); + await jest.advanceTimersByTimeAsync(1 * MIN); + world.serviceConfigFailures.C = 'C lifecycle.yaml is invalid'; + await world.merge('C'); + await jest.advanceTimersByTimeAsync(30 * MIN); + + expect(world.row('C').status).toBe(DeployStatus.ERROR); + expect(world.row('C').statusMessage).toContain('C lifecycle.yaml is invalid'); + expect(world.row('C').observedGeneration).toBe(world.row('C').desiredGeneration); + expect(world.live.get('C')).toBe('C@C-v0'); + expect(world.live.get('A')).toBe(`A@${aSha}`); + expect(world.live.get('B')).toBe(`B@${bSha}`); + expect(world.buildRow.status).toBe(BuildStatus.ERROR); + expect(world.buildRow.statusMessage).toContain('C'); + + delete world.serviceConfigFailures.C; + const cSha = await world.merge('C'); + await jest.advanceTimersByTimeAsync(10 * MIN); + + expect(world.live.get('C')).toBe(`C@${cSha}`); + expect(world.buildRow.status).toBe(BuildStatus.DEPLOYED); + expect(world.settled()).toBe(true); + }); + + test('a config-repo push supersedes in-flight work, and a later A merge supersedes only the A part of that redeploy', async () => { + const world = createWorld(SPECS); + const first = await world.merge('A'); + await jest.advanceTimersByTimeAsync(5 * MIN); + await world.pushConfigRepo(); + await jest.advanceTimersByTimeAsync(5 * MIN); + const second = await world.merge('A'); + await jest.advanceTimersByTimeAsync(60 * MIN); + + expect(world.live.get('A')).toBe(`A@${second}`); + expect(world.events.some((line) => line.includes(`A@${first} (superseded, result not published)`))).toBe(true); + for (const name of ['B', 'C', 'D', 'E']) { + expect(world.row(name).desiredGeneration).toBe(2); + expect(world.row(name).observedGeneration).toBe(2); + expect(world.counters.applies.get(name)).toBe(1); + } + expect(world.row('A').desiredGeneration).toBe(3); + expect(world.row('A').observedGeneration).toBe(3); + expect(world.buildRow.status).toBe(BuildStatus.DEPLOYED); + expect(world.settled()).toBe(true); + }); + + test('D failing its image leaves E, from the same repository, deploying and reports only D', async () => { + const world = createWorld(SPECS); + world.failBuild.add('D'); + const sha = await world.merge('D'); + await jest.advanceTimersByTimeAsync(15 * MIN); + + expect(world.row('D').status).toBe(DeployStatus.BUILD_FAILED); + expect(world.row('D').observedGeneration).toBe(1); + expect(world.live.get('D')).toBe('D@D-v0'); + expect(world.live.get('E')).toBe(`E@${sha}`); + expect(world.row('E').status).toBe(DeployStatus.READY); + expect(world.buildRow.status).toBe(BuildStatus.ERROR); + expect(world.buildRow.statusMessage).toContain('D'); + expect(world.buildRow.statusMessage).not.toContain('E'); + expect(world.settled()).toBe(true); + }); + + test('the service-row redeploy selects only that service repository', async () => { + const world = createWorld(SPECS); + await expect(world.service.redeployServiceFromBuild('static-z', 'B', BUILD_ID)).resolves.toMatchObject({ + status: 'success', + }); + await jest.advanceTimersByTimeAsync(10 * MIN); + + expect(world.row('B').desiredGeneration).toBe(1); + expect(world.row('B').observedGeneration).toBe(1); + expect(world.counters.applies.get('B')).toBe(1); + for (const name of ['A', 'C', 'D', 'E']) { + expect(world.row(name).desiredGeneration).toBe(0); + expect(world.counters.applies.get(name)).toBeUndefined(); + } + }); + + test('a disabled environment observes new work without building or deploying', async () => { + const world = createWorld(SPECS); + world.buildRow.deployEnabled = false; + (world.buildRow as any).pullRequestId = null; + await world.merge('A'); + await jest.advanceTimersByTimeAsync(5 * MIN); + + expect(world.row('A').desiredGeneration).toBe(1); + expect(world.row('A').observedGeneration).toBe(1); + expect(world.counters.builds.get('A')).toBeUndefined(); + expect(logs.some((line) => line.includes('observed without execution reason=deploy_disabled'))).toBe(true); + }); + + test('an environment that is tearing down defers pending work', async () => { + const world = createWorld(SPECS); + world.buildRow.status = BuildStatus.TEARING_DOWN; + await world.merge('A'); + await jest.advanceTimersByTimeAsync(5 * MIN); + + expect(world.row('A').desiredGeneration).toBe(1); + expect(world.row('A').observedGeneration).toBe(0); + expect(world.counters.builds.get('A')).toBeUndefined(); + expect(logs.some((line) => line.includes('deferred reason=tearing_down'))).toBe(true); + }); + + test('a transient failure retries only the failing service while B ships, and the final attempt records it', async () => { + const world = createWorld(SPECS); + world.failImport.set(100, () => new Error('GitHub API rate limit exceeded')); + await world.merge('A'); + await jest.advanceTimersByTimeAsync(5 * MIN); + const bSha = await world.merge('B'); + await jest.advanceTimersByTimeAsync(60 * MIN); + + expect(world.live.get('B')).toBe(`B@${bSha}`); + expect(world.liveAt.get('B')).toBe(8.5); + expect(world.counters.applies.get('B')).toBe(1); + expect(world.row('A').status).toBe(DeployStatus.ERROR); + expect(world.row('A').statusMessage).toContain('rate limit'); + expect(world.row('A').observedGeneration).toBe(1); + expect(world.live.get('A')).toBe('A@A-v0'); + expect(world.buildRow.status).toBe(BuildStatus.ERROR); + expect(world.settled()).toBe(true); + }); + + test('a signal for a replaced generation executes nothing; only the newest intent owns the rows', async () => { + const world = createWorld(SPECS); + await world.merge('A'); + const second = await world.merge('A'); + await jest.advanceTimersByTimeAsync(30 * MIN); + + expect(world.row('A').desiredGeneration).toBe(2); + expect(world.row('A').observedGeneration).toBe(2); + expect(world.counters.builds.get('A')).toBe(1); + expect(world.live.get('A')).toBe(`A@${second}`); + expect(world.events.filter((line) => line.includes('gen1 A: APPLIED'))).toEqual([]); + expect(world.settled()).toBe(true); + }); + + test('legacy queue payloads are adopted as intents and malformed jobs are rejected', async () => { + const world = createWorld(SPECS); + await world.service.processBuildQueue({ + data: { + buildId: BUILD_ID, + githubRepositoryId: 200, + sourceRef: 'B-legacy', + sourceGithubRepositoryId: 200, + sourceBranch: 'main', + }, + }); + expect(world.row('B').desiredGeneration).toBe(1); + expect(world.queue.adds.filter(({ opts }) => opts.jobId === `reconcile-${BUILD_ID}-1`)).toHaveLength(1); + await world.service.processResolveAndDeployBuildQueue({ data: { buildId: BUILD_ID, runUUID: 'legacy-full' } }); + expect(world.buildRow.acceptedRefs.all.gen).toBe(2); + expect(world.queue.adds.filter(({ opts }) => opts.jobId === `reconcile-${BUILD_ID}-2`)).toHaveLength(1); + await expect(world.service.processDeploymentReconciliationQueue({ data: {} } as any)).rejects.toThrow( + 'buildId and generation are required' + ); + }); + + test('the sweep signals every pending generation found on the rows', async () => { + const world = createWorld(SPECS); + world.row('A').desiredGeneration = 3; + world.row('B').desiredGeneration = 4; + world.buildRow.desiredGeneration = 4; + world.buildRow.observedGeneration = 2; + const add = jest.fn().mockResolvedValue(undefined); + world.service.deploymentReconciliationQueue = { add }; + + await world.service.enqueuePendingDeploymentReconciliations(); + + const jobIds = add.mock.calls.map(([, , opts]) => opts.jobId).sort(); + expect(jobIds).toEqual([`reconcile-${BUILD_ID}-3`, `reconcile-${BUILD_ID}-4`]); + }); + + test('the sweep also signals a build whose watermark says settled while a row is still pending', async () => { + const world = createWorld(SPECS); + world.buildRow.desiredGeneration = 5; + world.buildRow.observedGeneration = 5; + world.row('A').desiredGeneration = 5; + world.row('A').observedGeneration = 4; + const add = jest.fn().mockResolvedValue(undefined); + world.service.deploymentReconciliationQueue = { add }; + + await world.service.enqueuePendingDeploymentReconciliations(); + + expect(add.mock.calls.map(([, , opts]) => opts.jobId)).toEqual([`reconcile-${BUILD_ID}-5`]); + }); + + test('two services finishing together leave the watermark at the newest generation', async () => { + const world = createWorld(SPECS); + const publish = world.service.publishDerivedBuildStatus.bind(world.service); + let release!: () => void; + const gate = new Promise((resolve) => { + release = resolve; + }); + let held = false; + jest.spyOn(world.service, 'publishDerivedBuildStatus').mockImplementation(async (...args: any[]) => { + if (!held && world.row('B').observedGeneration === 1) { + held = true; + await gate; + } + return publish(...args); + }); + await world.merge('B'); + await world.merge('C'); + await jest.advanceTimersByTimeAsync(10 * MIN); + release(); + await jest.advanceTimersByTimeAsync(1 * MIN); + + expect(world.settled()).toBe(true); + expect(world.buildRow.observedGeneration).toBe(2); + const signalsBefore = world.queue.adds.length; + await world.service.enqueuePendingDeploymentReconciliations(); + expect(world.queue.adds.length).toBe(signalsBefore); + }); + + test('the sweep repairs a stale watermark instead of signalling a finished generation', async () => { + const world = createWorld(SPECS); + await world.merge('B'); + await jest.advanceTimersByTimeAsync(10 * MIN); + world.buildRow.observedGeneration = 0; + const signalsBefore = world.queue.adds.length; + + await world.service.enqueuePendingDeploymentReconciliations(); + + expect(world.queue.adds.length).toBe(signalsBefore); + expect(world.buildRow.observedGeneration).toBe(1); + }); + + test('steady merges to other services never delay A', async () => { + const many: ServiceSpec[] = [ + { name: 'A', repo: 100, buildMinutes: 20 }, + ...Array.from({ length: 12 }, (_, index) => ({ name: `S${index + 1}`, repo: 1000 + index, buildMinutes: 3 })), + ]; + const world = createWorld(many); + const aSha = await world.merge('A'); + for (let minute = 0; minute < 60; minute += 0.5) { + await jest.advanceTimersByTimeAsync(0.5 * MIN); + const now = minute + 0.5; + if (now % 5 === 0) await world.merge(`S${(now / 5) % 12 || 12}`); + } + + expect(world.live.get('A')).toBe(`A@${aSha}`); + expect(world.liveAt.get('A')).toBe(20.5); + expect(world.counters.builds.get('A')).toBe(1); + expect(world.events.filter((line) => line.includes('(re-rollout)'))).toEqual([]); + }); +}); + +describe('service-independent reconciliation: review regressions', () => { + test('a service row that no run has ever owned is claimed and deployed', async () => { + const world = createWorld(SPECS); + world.row('C').runUUID = null; + const sha = await world.merge('C'); + await jest.advanceTimersByTimeAsync(10 * MIN); + + expect(world.live.get('C')).toBe(`C@${sha}`); + expect(world.row('C').observedGeneration).toBe(1); + expect(world.settled()).toBe(true); + }); + + test('a row that another run backfills is never stamped to an intent that already claimed its rows', async () => { + const world = createWorld(SPECS); + const aSha = await world.merge('A'); + await jest.advanceTimersByTimeAsync(1 * MIN); + world.deployRows.push({ + id: world.deployRows.length + 1, + deployableId: world.deployRows.length + 1, + uuid: 'A2-static-z', + buildId: BUILD_ID, + githubRepositoryId: 100, + branchName: 'main', + active: true, + runUUID: null, + sha: null, + dockerImage: null, + status: DeployStatus.PENDING, + statusMessage: '', + desiredGeneration: 0, + observedGeneration: 0, + deployableName: 'A2', + }); + await world.merge('B'); + await jest.advanceTimersByTimeAsync(30 * MIN); + for (let tick = 0; tick < 3; tick += 1) { + await world.service.enqueuePendingDeploymentReconciliations(); + await jest.advanceTimersByTimeAsync(5_000); + } + + expect(world.live.get('A')).toBe(`A@${aSha}`); + expect(world.row('A2')).toMatchObject({ desiredGeneration: 0, observedGeneration: 0, runUUID: null }); + expect(world.settled()).toBe(true); + expect(world.buildRow.observedGeneration).toBe(2); + }); + + test('a full redeploy accepted before a teardown never brings the torn-down environment back', async () => { + const world = createWorld(SPECS); + world.queue.dropNext = 1; + await world.redeployEnvironment(); + world.buildRow.runUUID = 'teardown-run'; + const build = await world.service.db.models.Build.query().findById(BUILD_ID); + await world.service.deleteBuild(build, { runUUID: 'teardown-run', reason: 'manual_destroy', rethrow: true }); + expect(world.buildRow.status).toBe(BuildStatus.TORN_DOWN); + + for (let tick = 0; tick < 3; tick += 1) { + await world.service.enqueuePendingDeploymentReconciliations(); + await jest.advanceTimersByTimeAsync(5 * MIN); + } + + expect(world.buildRow.status).toBe(BuildStatus.TORN_DOWN); + expect(world.importCounts.get('root')).toBeUndefined(); + expect(world.settled()).toBe(true); + expect(world.buildRow.observedGeneration).toBe(world.buildRow.desiredGeneration); + }); + + test('a lost promotion lease after the claim retries the job and resumes the rows it already owns', async () => { + const world = createWorld(SPECS); + world.applyFailures.push(new AuthorityLockLostError('deploy-promotion.2')); + const sha = await world.merge('B'); + await jest.advanceTimersByTimeAsync(15 * MIN); + + expect(world.live.get('B')).toBe(`B@${sha}`); + expect(world.row('B').observedGeneration).toBe(1); + expect(world.counters.builds.get('B')).toBe(1); + expect(world.counters.applies.get('B')).toBe(1); + expect(world.queue.finalFailures).toEqual([]); + expect(world.settled()).toBe(true); + }); + + test('a finished generation is never re-imported while an older service is still building', async () => { + const world = createWorld(SPECS); + await world.merge('A'); + await jest.advanceTimersByTimeAsync(5 * MIN); + await world.merge('B'); + await jest.advanceTimersByTimeAsync(6 * MIN); + expect(world.row('B').observedGeneration).toBe(2); + const importsAfterB = world.importCounts.get('200') ?? 0; + + const signalsBefore = world.queue.adds.filter(({ opts }) => opts.jobId === `reconcile-${BUILD_ID}-2`).length; + for (let tick = 0; tick < 12; tick += 1) { + await world.service.enqueuePendingDeploymentReconciliations(); + await jest.advanceTimersByTimeAsync(0.5 * MIN); + } + // The sweep signals only generations that rows still carry, so the finished one is never re-enqueued. + expect(world.queue.adds.filter(({ opts }) => opts.jobId === `reconcile-${BUILD_ID}-2`).length).toBe(signalsBefore); + + // A replayed delivery of the finished generation exits before touching configuration. + await world.service.processDeploymentReconciliationQueue({ + data: { buildId: BUILD_ID, generation: 2 }, + attemptsMade: 0, + opts: { attempts: 10 }, + }); + expect(logs.filter((line) => line.includes('signal ignored reason=already_observed'))).toHaveLength(1); + expect(world.importCounts.get('200')).toBe(importsAfterB); + expect(world.row('A').observedGeneration).toBe(0); + }); + + test('a service with a broken lifecycle.yaml fails alone during a config-repo push; the root config and every other service proceed', async () => { + const world = createWorld(SPECS); + world.serviceConfigFailures.C = 'C lifecycle.yaml is invalid'; + await world.pushConfigRepo(); + await jest.advanceTimersByTimeAsync(45 * MIN); + + expect(world.row('C').status).toBe(DeployStatus.ERROR); + expect(world.row('C').statusMessage).toContain('C lifecycle.yaml is invalid'); + expect(world.row('C').observedGeneration).toBe(1); + expect(world.counters.builds.get('C')).toBeUndefined(); + for (const name of ['A', 'B', 'D', 'E']) { + expect(world.row(name).status).toBe(DeployStatus.READY); + expect(world.row(name).observedGeneration).toBe(1); + } + expect(world.buildRow.status).toBe(BuildStatus.ERROR); + expect(world.buildRow.statusMessage).toContain('C'); + expect(world.settled()).toBe(true); + }); + + test('a config-repo push whose signal was lost is recovered by the sweep even after every row was taken by later merges', async () => { + const world = createWorld([{ name: 'A', repo: 100, buildMinutes: 3 }]); + world.servicesAddedByConfig.push({ name: 'N', repo: 500, buildMinutes: 3 }); + world.queue.dropNext = 1; + await world.pushConfigRepo(); + await jest.advanceTimersByTimeAsync(1 * MIN); + const aSha = await world.merge('A'); + await jest.advanceTimersByTimeAsync(10 * MIN); + expect(world.live.get('A')).toBe(`A@${aSha}`); + expect(world.deployRows.some((row) => row.deployableName === 'N')).toBe(false); + + for (let tick = 0; tick < 6; tick += 1) { + await world.service.enqueuePendingDeploymentReconciliations(); + await jest.advanceTimersByTimeAsync(2 * MIN); + } + + expect(world.deployRows.some((row) => row.deployableName === 'N')).toBe(true); + expect(world.row('N').observedGeneration).toBe(1); + expect(world.buildRow.acceptedRefs['source:900:static:all'].observedGen).toBe(1); + expect(world.settled()).toBe(true); + }); + + test('a service with a broken lifecycle.yaml blocks its dependents and is reported when it has no row yet', async () => { + const world = createWorld(SPECS); + world.serviceConfigFailures.N = 'N lifecycle.yaml is invalid'; + await world.pushConfigRepo(); + await jest.advanceTimersByTimeAsync(45 * MIN); + + expect(world.settled()).toBe(true); + expect(world.buildRow.status).toBe(BuildStatus.ERROR); + expect(world.buildRow.statusMessage).toContain('N'); + expect(world.buildRow.statusMessage).toContain('N lifecycle.yaml is invalid'); + }); + + test('a root config error is cleared by the next push whose root import succeeds, even a service-scoped one', async () => { + const world = createWorld(SPECS); + world.failImport.set(900, () => new ParsingError('root lifecycle.yaml is invalid')); + await world.redeployEnvironment('label-added'); + await jest.advanceTimersByTimeAsync(5 * MIN); + expect(world.buildRow.status).toBe(BuildStatus.CONFIG_ERROR); + expect(world.buildRow.statusMessage).toContain('root lifecycle.yaml is invalid'); + expect(world.settled()).toBe(true); + + world.failImport.delete(900); + const aSha = await world.merge('A'); + await jest.advanceTimersByTimeAsync(30 * MIN); + + expect(world.importCounts.get('100')).toBe(1); + expect(world.live.get('A')).toBe(`A@${aSha}`); + expect(world.row('A').status).toBe(DeployStatus.READY); + expect(world.settled()).toBe(true); + expect(world.buildRow.status).toBe(BuildStatus.DEPLOYED); + expect(world.buildRow.statusMessage).toBe(''); + expect(world.statusWrites).toContain('config_error:root lifecycle.yaml is invalid'); + }); + + test('a broken root config on a service push flags the environment, not the pushed service', async () => { + const world = createWorld(SPECS); + world.failImport.set(100, () => new ParsingError('root lifecycle.yaml is invalid')); + await world.merge('A'); + await jest.advanceTimersByTimeAsync(5 * MIN); + + expect(world.buildRow.status).toBe(BuildStatus.CONFIG_ERROR); + expect(world.buildRow.statusMessage).toContain('root lifecycle.yaml is invalid'); + expect(world.row('A').status).not.toBe(DeployStatus.ERROR); + expect(world.live.get('A')).toBe('A@A-v0'); + expect(world.settled()).toBe(true); + + world.failImport.delete(100); + const aSha = await world.merge('A'); + await jest.advanceTimersByTimeAsync(30 * MIN); + + expect(world.live.get('A')).toBe(`A@${aSha}`); + expect(world.buildRow.status).toBe(BuildStatus.DEPLOYED); + }); + + test('a root config error recorded while another service finishes is never overwritten by that finish', async () => { + const world = createWorld(SPECS); + world.buildRow.status = BuildStatus.BUILDING; + const configBuild = await sim.buildQuery().findById(BUILD_ID); + + let release!: () => void; + sim.fetchGraphGate = new Promise((resolve) => { + release = resolve; + }); + const derived = world.service.publishDerivedBuildStatus(BUILD_ID); + await flush(); + expect(sim.fetchGraphGate).toBeNull(); + + const rootError = world.service.recordRootConfigError( + configBuild, + new ParsingError('root lifecycle.yaml is invalid') + ); + await flush(); + expect(world.buildRow.status).toBe(BuildStatus.BUILDING); + + release(); + await derived; + await rootError; + expect(world.statusWrites).toEqual(['deployed:', 'config_error:root lifecycle.yaml is invalid']); + expect(world.buildRow.status).toBe(BuildStatus.CONFIG_ERROR); + + await world.service.publishDerivedBuildStatus(BUILD_ID); + expect(world.buildRow.status).toBe(BuildStatus.CONFIG_ERROR); + }); + + test('a watermark write that fails after rollout is repaired by the retry, which republishes the final status', async () => { + const world = createWorld(SPECS); + sim.failPatchOnce = (patch: any, table: string) => table === 'build' && 'observedGeneration' in patch; + + const bSha = await world.merge('B'); + await jest.advanceTimersByTimeAsync(10 * MIN); + + expect(world.live.get('B')).toBe(`B@${bSha}`); + expect(world.row('B').status).toBe(DeployStatus.READY); + expect(world.queue.attempts).toEqual([`reconcile-${BUILD_ID}-1#0`, `reconcile-${BUILD_ID}-1#1`]); + expect(world.queue.finalFailures).toEqual([]); + expect(world.buildRow.observedGeneration).toBe(1); + expect(world.buildRow.status).toBe(BuildStatus.DEPLOYED); + expect(world.buildRow.acceptedRefs['source:200:main'].observedGen).toBe(1); + + const signalsBefore = world.queue.adds.length; + for (let tick = 0; tick < 3; tick += 1) { + await world.service.enqueuePendingDeploymentReconciliations(); + await jest.advanceTimersByTimeAsync(5_000); + } + expect(world.queue.adds.length).toBe(signalsBefore); + expect(logs.filter((line) => line.includes('signal ignored reason=already_observed'))).toHaveLength(0); + }); + + test('a database error during image setup fails the rows it was building; the detached sibling build cannot land after the intent finished', async () => { + const world = createWorld(SPECS); + sim.failPatchOnce = (patch: any) => 'deployPipelineId' in patch; + + await world.merge('D'); + await jest.advanceTimersByTimeAsync(10 * MIN); + + expect(logs.some((line) => line.includes('Docker: build error'))).toBe(true); + expect(world.queue.finalFailures).toEqual([]); + expect(world.settled()).toBe(true); + expect(world.row('D').status).toBe(DeployStatus.BUILD_FAILED); + expect(world.row('E').status).toBe(DeployStatus.BUILD_FAILED); + expect(world.live.get('D')).toBe('D@D-v0'); + expect(world.live.get('E')).toBe('E@E-v0'); + expect(world.row('E').dockerImage).toBe('E@E-v0'); + expect(world.events.some((line) => line.includes('E: build done') && line.includes('result not published'))).toBe( + true + ); + expect(world.events.filter((line) => line.includes('APPLIED'))).toEqual([]); + expect(world.buildRow.status).toBe(BuildStatus.ERROR); + }); + + test('an ingress failure note is kept by a derived republish and cleared only by a clean ingress apply', async () => { + const world = createWorld(SPECS); + world.buildRow.status = BuildStatus.DEPLOYED; + world.buildRow.statusMessage = 'Ingress apply failed: kubectl apply exited 1'; + + await world.service.publishDerivedBuildStatus(BUILD_ID); + expect(world.buildRow.statusMessage).toBe('Ingress apply failed: kubectl apply exited 1'); + + await world.service.publishDerivedBuildStatus(BUILD_ID, { clearStatusMessage: true }); + expect(world.buildRow.status).toBe(BuildStatus.DEPLOYED); + expect(world.buildRow.statusMessage).toBe(''); + }); + + test('a retry after a successful rollout resumes the rows already live instead of rolling them out again', async () => { + const world = createWorld(SPECS); + world.service.ingressService.ingressManifestQueue.add = jest + .fn() + .mockRejectedValueOnce(new Error('redis unavailable')) + .mockResolvedValue(undefined); + + const bSha = await world.merge('B'); + await jest.advanceTimersByTimeAsync(20 * MIN); + + expect(world.live.get('B')).toBe(`B@${bSha}`); + expect(world.row('B').status).toBe(DeployStatus.READY); + expect(world.settled()).toBe(true); + expect(world.queue.attempts).toEqual([`reconcile-${BUILD_ID}-1#0`, `reconcile-${BUILD_ID}-1#1`]); + expect(world.counters.builds.get('B')).toBe(1); + expect(world.counters.applies.get('B')).toBe(1); + expect(world.events.filter((line) => line.includes('(re-rollout)'))).toEqual([]); + expect(world.service.ingressService.ingressManifestQueue.add).toHaveBeenCalledTimes(2); + expect(world.buildRow.status).toBe(BuildStatus.DEPLOYED); + }); + + test('a full redeploy accepted after a config push subsumes it: one root import and no second environment pass', async () => { + const world = createWorld(SPECS); + world.queue.dropNext = 1; + await world.pushConfigRepo(); + await world.redeployEnvironment('manual'); + await jest.advanceTimersByTimeAsync(30 * MIN); + + expect(world.settled()).toBe(true); + expect(world.importCounts.get('root')).toBe(1); + expect(world.buildRow.acceptedRefs['source:900:static:all'].observedGen).toBe(1); + expect(world.buildRow.acceptedRefs.all.observedGen).toBe(2); + expect(world.buildRow.observedGeneration).toBe(2); + + for (let tick = 0; tick < 3; tick += 1) { + await world.service.enqueuePendingDeploymentReconciliations(); + await jest.advanceTimersByTimeAsync(5_000); + } + expect(world.queue.adds.filter(({ opts }) => opts.jobId === `reconcile-${BUILD_ID}-1`)).toHaveLength(1); + expect(world.importCounts.get('root')).toBe(1); + }); + + test('a service whose definition disappeared fails alone; its sibling from the same repository still ships', async () => { + const world = createWorld(SPECS); + world.orphanRows.add('E'); + + const sha = await world.merge('D'); + await jest.advanceTimersByTimeAsync(10 * MIN); + + expect(world.live.get('D')).toBe(`D@${sha}`); + expect(world.row('E').status).toBe(DeployStatus.ERROR); + expect(world.row('E').statusMessage).toBe('Service definition missing.'); + expect(world.live.get('E')).toBe('E@E-v0'); + expect(world.settled()).toBe(true); + expect(world.queue.finalFailures).toEqual([]); + expect(world.buildRow.status).toBe(BuildStatus.ERROR); + expect(world.buildRow.statusMessage).toContain('Service definition missing.'); + }); + + test('a service that fails to import and has no row blocks its dependents and stays reported until an import clears it', async () => { + const world = createWorld(SPECS); + world.dependsOn.set('B', ['N']); + world.serviceConfigFailures.N = 'N lifecycle.yaml is invalid'; + await world.pushConfigRepo(); + await jest.advanceTimersByTimeAsync(45 * MIN); + + expect(world.settled()).toBe(true); + expect(world.row('B').status).toBe(DeployStatus.DEPLOY_FAILED); + expect(world.row('B').statusMessage).toBe('Not deployed: N failed.'); + expect(world.live.get('B')).toBe('B@B-v0'); + expect(world.buildRow.status).toBe(BuildStatus.ERROR); + expect(world.buildRow.statusMessage).toContain('Not imported: N (N lifecycle.yaml is invalid)'); + + const aSha = await world.merge('A'); + await jest.advanceTimersByTimeAsync(30 * MIN); + expect(world.live.get('A')).toBe(`A@${aSha}`); + expect(world.buildRow.status).toBe(BuildStatus.ERROR); + expect(world.buildRow.statusMessage).toContain('Not imported: N'); + + // A manual full redeploy re-evaluates N and re-reports it from the newest environment-wide import. + await world.redeployEnvironment('manual'); + await jest.advanceTimersByTimeAsync(45 * MIN); + expect(world.settled()).toBe(true); + expect(world.buildRow.status).toBe(BuildStatus.ERROR); + expect(world.buildRow.statusMessage).toContain('Not imported: N (N lifecycle.yaml is invalid)'); + + delete world.serviceConfigFailures.N; + await world.pushConfigRepo(); + await jest.advanceTimersByTimeAsync(45 * MIN); + expect(world.settled()).toBe(true); + expect(world.row('B').status).toBe(DeployStatus.READY); + expect(world.buildRow.status).toBe(BuildStatus.DEPLOYED); + expect(world.buildRow.statusMessage).toBe(''); + }); + + test("a full redeploy accepted after a config push imports at that push's revision, not a lagging branch head", async () => { + const world = createWorld(SPECS); + world.queue.dropNext = 1; + const zSha = await world.pushConfigRepo(); + await world.redeployEnvironment('manual'); + await jest.advanceTimersByTimeAsync(30 * MIN); + + expect(world.settled()).toBe(true); + expect(world.importRefs.filter((entry) => entry.filter == null)).toEqual([ + { filter: null, repo: null, ref: null, pins: { '900:static': zSha } }, + ]); + expect(world.buildRow.acceptedRefs['source:900:static:all'].observedGen).toBe(1); + }); + + test("a service push honours the root config floor but pays for no other repository's lookup", async () => { + const world = createWorld(SPECS); + const zSha = await world.pushConfigRepo(); + await jest.advanceTimersByTimeAsync(30 * MIN); + await world.merge('B'); + await world.merge('A'); + await jest.advanceTimersByTimeAsync(30 * MIN); + + expect(world.settled()).toBe(true); + expect(world.importRefs.find((entry) => entry.filter === 100)?.pins).toEqual({ '900:static': zSha }); + }); + + test("a full pass that replaces an earlier full-pass push still deploys that push's revision", async () => { + const world = createWorld(SPECS); + world.queue.dropNext = 1; + const a1 = await world.mergeFull('A'); + const b1 = await world.mergeFull('B'); + await jest.advanceTimersByTimeAsync(45 * MIN); + + expect(world.settled()).toBe(true); + expect(world.buildRow.acceptedRefs['source:100:main:all'].observedGen).toBe(1); + expect(world.live.get('A')).toBe(`A@${a1}`); + expect(world.live.get('B')).toBe(`B@${b1}`); + }); + + test('a service that moves to another repository is still deployed by the push that claimed it', async () => { + const world = createWorld(SPECS); + let releaseImport!: () => void; + sim.importGate = new Promise((resolve) => { + releaseImport = resolve; + }); + await world.redeployEnvironment('full'); + await jest.advanceTimersByTimeAsync(1_000); + world.serviceMoves.push({ name: 'A', repo: 101, sha: 'A-moved' }); + await world.merge('A'); + await jest.advanceTimersByTimeAsync(1_000); + releaseImport(); + await jest.advanceTimersByTimeAsync(45 * MIN); + + expect(world.settled()).toBe(true); + expect(world.row('A').githubRepositoryId).toBe(101); + expect(world.row('A').desiredGeneration).toBe(2); + expect(world.live.get('A')).toBe('A@A-moved'); + expect(world.buildRow.status).toBe(BuildStatus.DEPLOYED); + }); + + test("a config error on a push that lands mid-rollout keeps its row failed despite the superseded run's late writes", async () => { + const world = createWorld(SPECS); + await world.merge('B'); + await jest.advanceTimersByTimeAsync(3.2 * MIN); + expect(world.row('B').status).toBe(DeployStatus.BUILT); + + world.serviceConfigFailures.B = 'B lifecycle.yaml is invalid'; + await world.merge('B'); + await jest.advanceTimersByTimeAsync(5 * MIN); + + expect(world.settled()).toBe(true); + expect(world.row('B').status).toBe(DeployStatus.ERROR); + expect(world.row('B').statusMessage).toBe('B lifecycle.yaml is invalid'); + expect(world.buildRow.status).toBe(BuildStatus.ERROR); + expect(world.buildRow.statusMessage).toContain('B lifecycle.yaml is invalid'); + }); + + test('a config-repo push deploys every accepted service SHA instead of a laggy branch head', async () => { + const world = createWorld(SPECS); + const accepted = await world.merge('A'); + await world.pushConfigRepo(); + await jest.advanceTimersByTimeAsync(45 * MIN); + + expect(world.live.get('A')).toBe(`A@${accepted}`); + expect(world.row('A').desiredGeneration).toBe(2); + expect(world.row('A').observedGeneration).toBe(2); + expect(world.settled()).toBe(true); + }); + test('a dependent ships without waiting for, or failing on, a prerequisite another run is fixing', async () => { + const world = createWorld([ + { name: 'A', repo: 100, buildMinutes: 1 }, + { name: 'B', repo: 200, buildMinutes: 3 }, + ]); + world.dependsOn.set('B', ['A']); + world.failBuild.add('A'); + await world.merge('A'); + await jest.advanceTimersByTimeAsync(5 * MIN); + expect(world.row('A').status).toBe(DeployStatus.BUILD_FAILED); + expect(world.settled()).toBe(true); + + world.failBuild.delete('A'); + const bSha = await world.merge('B'); + await jest.advanceTimersByTimeAsync(1 * MIN); + // The fix for A lands while B builds; its job is held before the claim, so A's row still shows its old failure. + let releaseA!: () => void; + sim.importGateRepo = 100; + sim.importGate = new Promise((resolve) => { + releaseA = resolve; + }); + const aSha = await world.merge('A'); + await jest.advanceTimersByTimeAsync(6 * MIN); + + expect(world.row('A').runUUID).toBeNull(); + expect(world.row('A').status).toBe(DeployStatus.QUEUED); + expect(world.live.get('B')).toBe(`B@${bSha}`); + expect(world.row('B').status).toBe(DeployStatus.READY); + + releaseA(); + await jest.advanceTimersByTimeAsync(30 * MIN); + + expect(world.live.get('A')).toBe(`A@${aSha}`); + expect(world.counters.applies.get('B')).toBe(1); + expect(world.queue.finalFailures).toEqual([]); + expect(world.settled()).toBe(true); + expect(world.buildRow.status).toBe(BuildStatus.DEPLOYED); + }); + + test('dependents never hold a worker slot for a prerequisite in another run, even with every slot busy', async () => { + const world = createWorld( + [ + { name: 'A', repo: 100, buildMinutes: 1 }, + { name: 'B1', repo: 200, buildMinutes: 3 }, + { name: 'B2', repo: 300, buildMinutes: 3 }, + ], + { concurrency: 2 } + ); + world.dependsOn.set('B1', ['A']); + world.dependsOn.set('B2', ['A']); + const b1 = await world.merge('B1'); + const b2 = await world.merge('B2'); + await jest.advanceTimersByTimeAsync(1 * MIN); + // A is accepted while both slots build its dependents; its job runs once one of them finishes. + const a = await world.merge('A'); + await jest.advanceTimersByTimeAsync(20 * MIN); + + expect(world.live.get('A')).toBe(`A@${a}`); + expect(world.live.get('B1')).toBe(`B1@${b1}`); + expect(world.live.get('B2')).toBe(`B2@${b2}`); + expect(world.liveAt.get('B1')!).toBeLessThan(world.liveAt.get('A')!); + expect(world.liveAt.get('B2')!).toBeLessThan(world.liveAt.get('A')!); + expect(world.counters.applies.get('B1')).toBe(1); + expect(world.counters.applies.get('B2')).toBe(1); + expect(world.queue.finalFailures).toEqual([]); + expect(world.settled()).toBe(true); + expect(world.buildRow.status).toBe(BuildStatus.DEPLOYED); + }); + + test('a dependent ships even when its prerequisite never gets to run', async () => { + const world = createWorld([ + { name: 'A', repo: 100, buildMinutes: 1 }, + { name: 'B', repo: 200, buildMinutes: 1 }, + ]); + world.dependsOn.set('B', ['A']); + // The signal for A is lost and nothing sweeps, so A stays accepted but never claimed. + world.queue.dropNext = 1; + await world.merge('A'); + const bSha = await world.merge('B'); + await jest.advanceTimersByTimeAsync(10 * MIN); + + expect(world.live.get('B')).toBe(`B@${bSha}`); + expect(world.row('B').status).toBe(DeployStatus.READY); + expect(world.row('B').observedGeneration).toBe(2); + expect(world.row('A').observedGeneration).toBe(0); + expect(world.queue.finalFailures).toEqual([]); + }); +}); diff --git a/src/server/services/build.ts b/src/server/services/build.ts index c58d19d8..cf386fb7 100644 --- a/src/server/services/build.ts +++ b/src/server/services/build.ts @@ -68,21 +68,32 @@ import ApiAccessConfigService from './apiAccessConfig'; import { getBranchName, getDeployType, getRepositoryName, type Service } from 'server/models/yaml/YamlService'; import type { LifecycleConfig } from 'server/models/yaml/Config'; import * as YamlService from 'server/models/yaml'; -import { getEnvironmentPhaseFromState, isActiveServiceReady } from 'server/lib/environments/readiness'; +import { getEnvironmentPhaseFromState, isActiveServiceReady, isDeployFailure } from 'server/lib/environments/readiness'; import { acceptDeploymentIntent, - dirtyDeploymentIntents, + deploymentIntentDeployFilter, + deploymentIntentSelectsAllDeploys, + deployStampForGeneration, + recordIntentConfigFailures, + type AcceptedDeploymentIntent, + type AcceptedDeploymentRefs, type DeploymentIntent, type DeploymentReconciliationJobData, - type DirtyDeploymentIntent, } from 'server/lib/deploymentReconciliation/mailbox'; +import { + environmentBlockReason, + isDeployAuthorityCurrent, + loadEnvironmentAuthority, +} from 'server/lib/deploymentReconciliation/authority'; const tracer = Tracer.getInstance(); tracer.initialize('build-service'); const TEARDOWN_RETRY_GRACE_MS = 15 * 60 * 1000; -const BUILD_DEPLOYMENT_LOCK_TTL_MS = 15 * 60 * 1000; +// Short with frequent renewal, so a worker that dies mid-run frees its locks within a minute. +const BUILD_DEPLOYMENT_LOCK_TTL_MS = 60 * 1000; const DEPLOY_SECRET_MUTATION_LOCK_TTL_MS = 2 * 60 * 1000; const DEPLOYMENT_RECONCILIATION_SWEEP_MS = 5_000; +const BUILD_STATUS_LOCK_TTL_MS = 30_000; const HOUR_MS = 60 * 60 * 1000; const PR_AUTHORITY_REVALIDATED_DELETE_REASONS = new Set([ 'pull_request_closed', @@ -128,12 +139,17 @@ interface DeploymentScopeOptions { sourceGithubRepositoryId?: number | null; /** Exact mailbox generation owned by this pass. */ expectedGeneration?: number; + /** Latest accepted push SHA per `:`; a full pass must not read a laggy head below it. */ + acceptedSourcePins?: AcceptedSourcePins; } +type AcceptedSourcePins = Record; + interface DeploymentReconciliationClaim { generation: number; token: string; - dirty: DirtyDeploymentIntent[]; + intent: AcceptedDeploymentIntent; + acceptedSourcePins: AcceptedSourcePins; } interface DeploymentReconciliationScope { @@ -152,13 +168,21 @@ interface DeploymentScopePreparation { sourceGithubRepositoryId?: number | null; sourceRef?: string | null; sourceBranch?: string | null; + acceptedSourcePins?: AcceptedSourcePins; + /** Services the import could not read and that have no row; their dependents are blocked. */ + unresolvedServices?: string[]; } interface DeploymentScopeOutcome { - status: BuildStatus; - error?: unknown; + deployed: number; + failed: number; } +type IntentConfiguration = + | { kind: 'deferred' } + | { kind: 'observed' } + | { kind: 'ready'; preparation: DeploymentScopePreparation; owned: number }; + interface DeploymentReconciliationRequest { buildId: number; githubRepositoryId?: number | null; @@ -510,7 +534,8 @@ export default class BuildService extends BaseService { 'createdByUserId', 'createdByTokenId', 'createdByGithubLogin', - 'runUUID' + 'runUUID', + 'acceptedRefs' ) .withGraphFetched('[baseBuild, pullRequest, deploys.[deployable, repository]]') .modifyGraph('pullRequest', (b) => { @@ -545,6 +570,8 @@ export default class BuildService extends BaseService { 'deployPipelineId', 'githubRepositoryId', 'runUUID', + 'desiredGeneration', + 'observedGeneration', 'publicUrl', 'dockerImage', 'buildLogs', @@ -2314,6 +2341,7 @@ export default class BuildService extends BaseService { sourceGithubRepositoryId?: number | null; runUUID?: string; expectedGeneration?: number; + acceptedSourcePins?: AcceptedSourcePins; } = {} ) { const buildSource = getBuildSource(build); @@ -2325,11 +2353,14 @@ export default class BuildService extends BaseService { buildSource.githubRepositoryId != null && Number(sourceGithubRepositoryId) === Number(buildSource.githubRepositoryId) && options.sourceBranch === buildSource.branchName; - const rootSourceRef = sourceRefTargetsRoot ? options.sourceRef : null; + const rootSourceRef = sourceRefTargetsRoot + ? options.sourceRef + : options.acceptedSourcePins?.[`${buildSource.githubRepositoryId}:${buildSource.branchName}`] ?? null; + let reconciliationResult: DeployableReconciliationResult | undefined; // Write the deployables here for now and not going to use them yet. try { const buildId = build?.id; - const reconciliationResult = await this.db.services.Deployable.upsertDeployables( + reconciliationResult = await this.db.services.Deployable.upsertDeployables( buildId, build.uuid, build.pullRequest, @@ -2338,7 +2369,8 @@ export default class BuildService extends BaseService { filterGithubRepositoryId, options.sourceRef, options.sourceBranch, - sourceGithubRepositoryId + sourceGithubRepositoryId, + options.acceptedSourcePins ); if (options.skipDeletedServiceReconciliation) { @@ -2374,6 +2406,7 @@ export default class BuildService extends BaseService { await this.db.services.Webhook.upsertWebhooksWithYaml(build, build.pullRequest, rootSourceRef).catch((error) => { getLogger().warn({ error }, 'Config: webhook import warning'); }); + return reconciliationResult; } private async reconcileDeletedDeployables( @@ -2509,8 +2542,8 @@ export default class BuildService extends BaseService { ...(runUUID ? { nativeMutationGate: async (action: () => Promise) => { - const result = await this.withCurrentBuildPromotionLock( - buildId, + const result = await this.withCurrentDeployPromotionLock( + deploy.id, () => this.isDeploymentRunCurrent(buildId, runUUID, expectedGeneration), action ); @@ -2645,20 +2678,7 @@ export default class BuildService extends BaseService { * queued or in-flight PR job from recreating resources while deletion waits. */ private deploymentBlockReason(build: Build | null | undefined): string | null { - if (!build) return 'build_missing'; - if (build.deletedAt != null) return 'build_deleted'; - - if (build.pullRequest || build.pullRequestId != null) { - if (build.status === BuildStatus.TEARING_DOWN) return 'tearing_down'; - if (!build.pullRequest) return 'pull_request_missing'; - if (build.pullRequest.status !== PullRequestStatus.OPEN) return 'pull_request_closed'; - if (!isDeployEnabled(build)) return 'deploy_disabled'; - // Re-adding the deploy label to an open PR may reclaim the row after teardown completes. - return null; - } - - if (build.status === BuildStatus.TEARING_DOWN || build.status === BuildStatus.TORN_DOWN) return 'torn_down'; - return build.deployEnabled === true ? null : 'deploy_disabled'; + return environmentBlockReason(build); } /** PR setup is allowed without a deploy label, but not while teardown is active or the PR is closed. */ @@ -2708,10 +2728,6 @@ export default class BuildService extends BaseService { .where({ id: build.id }) .whereNull('deletedAt'); - if (expectedGeneration != null) { - claim = claim.where('desiredGeneration', expectedGeneration); - } - // API-created environments keep their kill switch on the Build row, so make // that half of the claim atomic. PR authority is re-read immediately below. if (!build.pullRequest && build.pullRequestId == null) { @@ -2756,7 +2772,6 @@ export default class BuildService extends BaseService { getLogger().info('Deploy: preparation skipped reason=ownership_lost'); return null; } - build.runUUID = runUUID; const deploys = await this.db.services.Deploy.findOrCreateDeploys( build.environment!, @@ -2764,7 +2779,8 @@ export default class BuildService extends BaseService { githubRepositoryId ?? undefined, sourceRef, options.sourceBranch, - options.sourceGithubRepositoryId ?? githubRepositoryId + options.sourceGithubRepositoryId ?? githubRepositoryId, + options.acceptedSourcePins ); build.$setRelated('deploys', deploys); await build.$fetchGraph('pullRequest'); @@ -2774,31 +2790,11 @@ export default class BuildService extends BaseService { getLogger().info('Deploy: preparation skipped after config reason=ownership_lost'); return null; } - - let deployClaim = this.db.models.Deploy.query().patch({ runUUID }).where({ buildId: build.id }); - if (githubRepositoryId != null) { - deployClaim = deployClaim.where('githubRepositoryId', githubRepositoryId); - if (options.sourceBranch) deployClaim = deployClaim.where('branchName', options.sourceBranch); - } - await deployClaim; - for (const deploy of deploys ?? []) { - if ( - (githubRepositoryId == null || deploy.githubRepositoryId === githubRepositoryId) && - (githubRepositoryId == null || !options.sourceBranch || deploy.branchName === options.sourceBranch) - ) { - deploy.runUUID = runUUID; - } - } - - await this.markConfigurationsAsBuilt(build, runUUID, githubRepositoryId, options.sourceBranch); - await this.updateStatusAndComment(build, BuildStatus.BUILDING, runUUID, true, true, null, expectedGeneration); await build.pullRequest?.$fetchGraph('repository'); try { const dependencyGraph = await generateGraph(build, 'TB'); - let graphPatch = this.db.models.Build.query().patch({ dependencyGraph }).where({ id: build.id, runUUID }); - if (expectedGeneration != null) graphPatch = graphPatch.where('desiredGeneration', expectedGeneration); - await graphPatch; + await this.db.models.Build.query().patch({ dependencyGraph }).where({ id: build.id }).whereNull('deletedAt'); } catch (error) { getLogger().warn({ error }, 'Graph: generation failed'); } @@ -2810,19 +2806,28 @@ export default class BuildService extends BaseService { sourceGithubRepositoryId: options.sourceGithubRepositoryId ?? githubRepositoryId, sourceRef, sourceBranch: options.sourceBranch, + acceptedSourcePins: options.acceptedSourcePins, }; } private async executeDeploymentScope( preparation: DeploymentScopePreparation, - expectedGeneration?: number + expectedGeneration: number ): Promise { - const { build, runUUID, githubRepositoryId, sourceGithubRepositoryId, sourceRef, sourceBranch } = preparation; + const { + build, + runUUID, + githubRepositoryId, + sourceGithubRepositoryId, + sourceRef, + sourceBranch, + acceptedSourcePins, + } = preparation; try { if (!(await this.isDeploymentRunCurrent(build.id, runUUID, expectedGeneration))) return null; - const results = await Promise.all([ + await Promise.all([ this.buildImages( build, runUUID, @@ -2830,28 +2835,29 @@ export default class BuildService extends BaseService { sourceRef, sourceBranch, expectedGeneration, - sourceGithubRepositoryId + sourceGithubRepositoryId, + acceptedSourcePins + ), + this.deployCLIServices( + build, + runUUID, + githubRepositoryId, + sourceRef, + sourceBranch, + sourceGithubRepositoryId, + expectedGeneration, + acceptedSourcePins ), - this.deployCLIServices(build, runUUID, githubRepositoryId, sourceRef, sourceBranch, sourceGithubRepositoryId), ]); - getLogger().debug(`Build results: buildImages=${results[0]} deployCLIServices=${results[1]}`); - // A/B may finish the expensive operation they already entered, but may not - // enter native promotion after C has become authoritative. + // A row that failed its image phase is recorded on the row; its siblings still roll out. if (!(await this.isDeploymentRunCurrent(build.id, runUUID, expectedGeneration))) { getLogger().info('Deploy: rollout skipped reason=ownership_lost'); return null; } + await this.publishDerivedBuildStatus(build.id); - if (!_.every(results)) { - getLogger({ buildId: build.id, githubRepositoryId, sourceBranch }).warn('Build: errored skipping=rollout'); - return { status: BuildStatus.ERROR }; - } - - await this.updateStatusAndComment(build, BuildStatus.DEPLOYING, runUUID, true, true, null, expectedGeneration); - if (!(await this.isDeploymentRunCurrent(build.id, runUUID, expectedGeneration))) return null; - - const applySuccess = await this.generateAndApplyManifests({ + await this.generateAndApplyManifests({ build, runUUID, expectedGeneration, @@ -2859,10 +2865,9 @@ export default class BuildService extends BaseService { sourceBranch, namespace: build.namespace, enqueueIngress: false, + unresolvedServices: preparation.unresolvedServices, }); - if (!(await this.isDeploymentRunCurrent(build.id, runUUID, expectedGeneration))) return null; - - return { status: applySuccess ? BuildStatus.DEPLOYED : BuildStatus.ERROR }; + return this.summarizeIntentDeploys(build.id, runUUID, expectedGeneration); } catch (error) { if (error instanceof DeploymentSupersededError) { getLogger().info('Build: deployment phase stopped reason=superseded'); @@ -2870,34 +2875,65 @@ export default class BuildService extends BaseService { } if (error instanceof AuthorityLockLostError) throw error; getLogger().error({ error }, 'Build: deploy failed'); - if (!(await this.isDeploymentRunCurrent(build.id, runUUID, expectedGeneration).catch(() => false))) { + if (!(await this.isDeploymentRunCurrent(build.id, runUUID, expectedGeneration))) { getLogger().info('Build: failure ignored reason=ownership_lost'); return null; } - - return { status: BuildStatus.ERROR, error }; + await this.recordFailureOnOwnedDeploys(build.id, runUUID, expectedGeneration, error, 'Deployment failed.'); + return this.summarizeIntentDeploys(build.id, runUUID, expectedGeneration); } } + private async summarizeIntentDeploys( + buildId: number, + runUUID: string, + generation: number + ): Promise { + const rows = await this.db.models.Deploy.query() + .select('status') + .where({ buildId, runUUID, desiredGeneration: generation }); + const failed = rows.filter((row) => isDeployFailure(row.status)).length; + return { deployed: rows.length - failed, failed }; + } + + /** + * With a generation this is intent authority: the run still owns a row at that + * generation, or its accepted intent is still the newest for its key before + * rows are claimed. Without a generation it is the legacy environment token. + */ private async isDeploymentRunCurrent( buildId: number, runUUID: string, expectedGeneration?: number ): Promise { - const current = await this.db.models.Build.query() - .findById(buildId) - .select('id', 'runUUID', 'status', 'deployEnabled', 'deletedAt', 'pullRequestId', 'desiredGeneration'); - if (current?.pullRequestId != null) { - await current.$fetchGraph('pullRequest'); - } - return Boolean( - current && - current.runUUID === runUUID && - (expectedGeneration == null || Number(current.desiredGeneration) === expectedGeneration) && - this.deploymentBlockReason(current) == null + const current = await loadEnvironmentAuthority(this.db.models, buildId); + if (!current || this.deploymentBlockReason(current) != null) return false; + if (expectedGeneration == null) return current.runUUID === runUUID; + const owned = await this.db.models.Deploy.query() + .findOne({ buildId, runUUID, desiredGeneration: expectedGeneration }) + .select('id'); + if (owned) return true; + return this.isIntentAccepted(buildId, runUUID, expectedGeneration); + } + + private async isIntentAccepted(buildId: number, runUUID: string, generation: number): Promise { + const build = await this.db.models.Build.query().findById(buildId).select('id', 'acceptedRefs'); + const refs = build?.acceptedRefs; + if (!refs || typeof refs !== 'object' || Array.isArray(refs)) return false; + return Object.values(refs).some( + (intent) => intent?.gen === generation && intent?.requestId === runUUID && intent?.observedGen !== generation ); } + /** Admission to the config lock ignores block reasons so a blocked environment can still observe or defer its work. */ + private async isIntentPending(buildId: number, runUUID: string, generation: number): Promise { + const owned = await this.db.models.Deploy.query() + .findOne({ buildId, runUUID, desiredGeneration: generation }) + .select('id'); + if (owned) return true; + return this.isIntentAccepted(buildId, runUUID, generation); + } + async withBuildDeploymentLock(buildId: number, action: () => Promise): Promise { if (!this.redlock?.lock) return action(); @@ -2942,6 +2978,72 @@ export default class BuildService extends BaseService { }); } + async withCurrentDeployPromotionLock( + deployId: number, + isCurrent: () => Promise, + action: () => Promise + ): Promise> { + return withAuthorityLock({ + redlock: this.redlock, + resource: `deploy-promotion.${deployId}`, + ttlMs: BUILD_DEPLOYMENT_LOCK_TTL_MS, + isCurrent, + action, + onWait: (error) => getLogger({ error, deployId }).warn('Deploy promotion: waiting for admitted native mutation'), + }); + } + + /** Teardown cancels the work accepted up to `cutoff`; the Build row lock orders it against every accept. */ + private async observeAcceptedWorkForTeardown(buildId: number, cutoff: number): Promise { + await this.db.models.Build.transact(async (trx) => { + const build = await this.db.models.Build.query(trx) + .select('id', 'desiredGeneration', 'acceptedRefs') + .findById(buildId) + .forUpdate(); + if (!build) return; + const refs: AcceptedDeploymentRefs = + build.acceptedRefs && typeof build.acceptedRefs === 'object' && !Array.isArray(build.acceptedRefs) + ? build.acceptedRefs + : {}; + const acceptedRefs = Object.fromEntries( + Object.entries(refs).map(([key, intent]) => [ + key, + Number.isSafeInteger(intent?.gen) && intent.gen <= cutoff ? { ...intent, observedGen: intent.gen } : intent, + ]) + ); + const rows = await this.db.models.Deploy.query(trx) + .select('id', 'desiredGeneration', 'observedGeneration') + .where({ buildId }); + for (const row of rows) { + const desired = Number(row.desiredGeneration); + if (desired <= Number(row.observedGeneration) || desired > cutoff) continue; + await this.db.models.Deploy.query(trx) + .patch({ observedGeneration: row.desiredGeneration }) + .where({ id: row.id }); + } + await this.db.models.Build.query(trx) + .findById(buildId) + .patch({ + acceptedRefs, + observedGeneration: Math.min(cutoff, Number(build.desiredGeneration)), + } as Partial); + }); + } + + /** Lets every admitted per-service mutation finish before cleanup touches the namespace. */ + private async barrierDeployPromotionLocks(buildId: number): Promise { + const deploys = await this.db.models.Deploy.query().select('id').where({ buildId }); + for (const deploy of deploys) { + await withAuthorityLock({ + redlock: this.redlock, + resource: `deploy-promotion.${deploy.id}`, + ttlMs: BUILD_DEPLOYMENT_LOCK_TTL_MS, + isCurrent: async () => true, + action: async () => undefined, + }); + } + } + private async withCurrentBuildDeploymentLock( buildId: number, isCurrent: () => Promise, @@ -3148,7 +3250,11 @@ export default class BuildService extends BaseService { // Revoke every in-flight Deploy writer before cleanup starts. This one // bulk fence prevents detached readiness/provider waits from restoring // READY or failure state after teardown records TORN_DOWN. + // Work accepted from here on, such as a re-added deploy label, belongs to a later deploy and survives the teardown. + const cutoff = Number(build.desiredGeneration ?? 0); await this.db.models.Deploy.query().where({ buildId: build.id }).patch({ runUUID: teardownRunUUID }); + await this.barrierDeployPromotionLocks(build.id); + await this.observeAcceptedWorkForTeardown(build.id, cutoff); } // A failing cleanup step must never block namespace deletion below; retries pick up the rest. const cleanupResults = await Promise.allSettled([ @@ -3222,8 +3328,91 @@ export default class BuildService extends BaseService { runUUID: string, updateMissionControl: boolean, updateStatus: boolean, - error: Error | null = null, - expectedGeneration?: number + error: Error | null = null + ) { + return this.publishBuildStatus(build, status, { runUUID, updateMissionControl, updateStatus, error }); + } + + /** Environment status is derived from its services; only teardown and a root config error override it. */ + async publishDerivedBuildStatus( + buildId: number, + options: { clearConfigError?: boolean; clearStatusMessage?: boolean } = {} + ): Promise { + // Concurrent jobs derive from different row snapshots; serializing the read-modify-write keeps the last write current. + await withAuthorityLock({ + redlock: this.redlock, + resource: `build-status.${buildId}`, + ttlMs: BUILD_STATUS_LOCK_TTL_MS, + isCurrent: async () => true, + action: () => this.publishDerivedBuildStatusUnlocked(buildId, options), + }); + } + + private async publishDerivedBuildStatusUnlocked( + buildId: number, + options: { clearConfigError?: boolean; clearStatusMessage?: boolean } = {} + ): Promise { + const build = await this.db.models.Build.query().findById(buildId).whereNull('deletedAt'); + if (!build) return; + if (build.status === BuildStatus.TEARING_DOWN || build.status === BuildStatus.TORN_DOWN) return; + if (build.status === BuildStatus.CONFIG_ERROR && !options.clearConfigError) return; + await build.$fetchGraph('[deploys.[deployable], pullRequest.[repository]]'); + + const active = (build.deploys ?? []).filter((deploy) => deploy.active !== false); + const pending = active.filter((deploy) => Number(deploy.desiredGeneration) > Number(deploy.observedGeneration)); + const failedRows = active.some( + (deploy) => isDeployFailure(deploy.status) && deploy.status !== DeployStatus.TORN_DOWN + ); + const unresolvedNote = Object.entries(unresolvedServiceFailures(build.acceptedRefs, active)) + .map(([name, reason]) => `${name} (${reason})`) + .join('; '); + let status: BuildStatus; + if (pending.length > 0) { + const rollingOut = pending.some((deploy) => + [DeployStatus.BUILT, DeployStatus.DEPLOYING].includes(deploy.status as DeployStatus) + ); + status = rollingOut ? BuildStatus.DEPLOYING : BuildStatus.BUILDING; + } else if (failedRows || unresolvedNote) { + status = BuildStatus.ERROR; + } else { + status = isDeployEnabled(build) ? BuildStatus.DEPLOYED : BuildStatus.BUILT; + } + + let error: Error | null = null; + if (status === BuildStatus.ERROR && unresolvedNote) { + const rowMessage = failedRows ? this.resolveBuildStatusMessage(status, active, null) : ''; + error = new Error([rowMessage, `Not imported: ${unresolvedNote}`].filter(Boolean).join(' | ')); + } + const statusMessage = this.resolveBuildStatusMessage(status, active, error); + if (build.status === status && (build.statusMessage ?? '') === statusMessage) return; + // An ingress failure note on an otherwise healthy environment is cleared only by the ingress job itself. + if (build.status === status && statusMessage === '' && !options.clearStatusMessage) return; + await this.publishBuildStatus(build, status, { updateMissionControl: true, updateStatus: true, error }); + } + + private async recordRootConfigError(build: Build, error: unknown): Promise { + const statusError = + error instanceof Error + ? error + : new Error(statusMessageFromError(error, 'Lifecycle configuration failed validation.')); + await withAuthorityLock({ + redlock: this.redlock, + resource: `build-status.${build.id}`, + ttlMs: BUILD_STATUS_LOCK_TTL_MS, + isCurrent: async () => true, + action: () => + this.publishBuildStatus(build, BuildStatus.CONFIG_ERROR, { + updateMissionControl: true, + updateStatus: true, + error: statusError, + }), + }); + } + + private async publishBuildStatus( + build: Build, + status: BuildStatus, + options: { runUUID?: string; updateMissionControl: boolean; updateStatus: boolean; error: Error | null } ) { return withLogContext({ buildUuid: build.uuid }, async () => { let published = false; @@ -3236,12 +3425,13 @@ export default class BuildService extends BaseService { const isSandboxBuild = build.kind === BuildKind.SANDBOX; const repository = pullRequest?.repository; - const statusMessage = this.resolveBuildStatusMessage(status, deploys || [], error); + const statusMessage = this.resolveBuildStatusMessage(status, deploys || [], options.error); let patch = this.db.models.Build.query() .patch({ status, statusMessage }) - .where({ id: build.id, runUUID }) + .where({ id: build.id }) .whereNull('deletedAt'); - if (expectedGeneration != null) patch = patch.where('desiredGeneration', expectedGeneration); + if (options.runUUID) patch = patch.where('runUUID', options.runUUID); + else patch = patch.whereNotIn('status', [BuildStatus.TEARING_DOWN, BuildStatus.TORN_DOWN]); if ((await patch) !== 1) return; published = true; @@ -3253,9 +3443,9 @@ export default class BuildService extends BaseService { deploys, pullRequest, repository, - updateMissionControl, - updateStatus, - error + options.updateMissionControl, + options.updateStatus, + options.error ).catch((e) => { getLogger().error({ error: e }, 'ActivityStream: update failed'); }); @@ -3305,7 +3495,7 @@ export default class BuildService extends BaseService { } const statusError = error instanceof Error ? error : new Error(statusMessageFromError(error, fallbackMessage)); - await this.updateStatusAndComment(build, status, activeRunUUID, true, true, statusError, expectedGeneration); + await this.updateStatusAndComment(build, status, activeRunUUID, true, true, statusError); } private resolveBuildStatusMessage(status: BuildStatus, deploys: Deploy[], error: Error | null): string { @@ -3375,13 +3565,59 @@ export default class BuildService extends BaseService { } } + /** + * A fenced run processes exactly the rows it claimed, whichever repository + * they belong to now; the intent's repository filter was applied when the + * rows were stamped. Rows already READY belong to an earlier attempt of the + * same run and are left alone; a row whose service definition is gone fails + * on its own instead of failing its siblings. + */ + private async selectRunDeploys( + buildId: number, + runUUID: string | undefined, + githubRepositoryId: number | null | undefined, + sourceBranch: string | null | undefined, + expectedGeneration?: number, + excludeFailed = false + ): Promise { + const fenced = expectedGeneration != null; + let query = Deploy.query().where({ + buildId, + ...(runUUID ? { runUUID } : {}), + ...(!fenced && githubRepositoryId ? { githubRepositoryId } : {}), + ...(!fenced && githubRepositoryId && sourceBranch ? { branchName: sourceBranch } : {}), + }); + if (fenced) query = query.where('desiredGeneration', expectedGeneration); + const skipped: DeployStatus[] = [ + ...(expectedGeneration != null ? [DeployStatus.READY, DeployStatus.DEPLOYED] : []), + ...(excludeFailed ? [DeployStatus.ERROR, DeployStatus.BUILD_FAILED, DeployStatus.DEPLOY_FAILED] : []), + ]; + if (skipped.length > 0) query = query.whereNotIn('status', skipped); + const deploys = await query.withGraphFetched({ deployable: true }); + if (expectedGeneration == null || !runUUID) return deploys; + const orphaned = deploys.filter((deploy) => deploy.active && !deploy.deployable); + for (const deploy of orphaned) { + getLogger().error(`Deploy: service definition missing uuid=${deploy.uuid}`); + await this.recordPhaseFailure( + deploy, + runUUID, + DeployStatus.ERROR, + 'Service definition missing.', + expectedGeneration + ); + } + return deploys.filter((deploy) => !orphaned.includes(deploy)); + } + async deployCLIServices( build: Build, runUUID: string, githubRepositoryId: number | null = null, sourceRef?: string | null, sourceBranch?: string | null, - sourceGithubRepositoryId: number | null | undefined = githubRepositoryId + sourceGithubRepositoryId: number | null | undefined = githubRepositoryId, + expectedGeneration?: number, + acceptedSourcePins?: AcceptedSourcePins ): Promise { await build?.$fetchGraph({ deploys: { @@ -3392,14 +3628,14 @@ export default class BuildService extends BaseService { if (!buildId) { getLogger().error('Build: id missing for=deployCLIServices'); } - const deploys = await Deploy.query() - .where({ - buildId, - runUUID, - ...(githubRepositoryId ? { githubRepositoryId } : {}), - ...(githubRepositoryId && sourceBranch ? { branchName: sourceBranch } : {}), - }) - .withGraphFetched({ deployable: true }); + const deploys = await this.selectRunDeploys( + buildId, + runUUID, + githubRepositoryId, + sourceBranch, + expectedGeneration, + expectedGeneration != null + ); try { return _.every( await Promise.all( @@ -3418,8 +3654,19 @@ export default class BuildService extends BaseService { runUUID, sourceRef, sourceGithubRepositoryId, - sourceBranch + sourceBranch, + acceptedSourcePins, + expectedGeneration ); + if (result === false) { + await this.recordPhaseFailure( + deploy, + runUUID, + DeployStatus.ERROR, + 'CLI deploy failed.', + expectedGeneration + ); + } return result; } catch (err) { getLogger().error({ error: err }, `CLI: deploy failed uuid=${deploy?.uuid}`); @@ -3434,6 +3681,7 @@ export default class BuildService extends BaseService { ); } catch (error) { getLogger().error({ error }, 'CLI: build failed'); + await this.recordPhaseFailures(deploys, runUUID, DeployStatus.ERROR, 'CLI deploy failed.', expectedGeneration); return false; } } @@ -3450,23 +3698,22 @@ export default class BuildService extends BaseService { sourceRef?: string | null, sourceBranch?: string | null, expectedGeneration?: number, - sourceGithubRepositoryId: number | null | undefined = githubRepositoryId + sourceGithubRepositoryId: number | null | undefined = githubRepositoryId, + acceptedSourcePins?: AcceptedSourcePins ): Promise { const buildId = build?.id; if (!buildId) { getLogger().error('Build: id missing for=buildImages'); } - const deploys = await Deploy.query() - .where({ - buildId, - runUUID, - ...(githubRepositoryId ? { githubRepositoryId } : {}), - ...(githubRepositoryId && sourceBranch ? { branchName: sourceBranch } : {}), - }) - .withGraphFetched({ - deployable: true, - }); + const deploys = await this.selectRunDeploys( + buildId, + runUUID, + githubRepositoryId, + sourceBranch, + expectedGeneration, + expectedGeneration != null + ); try { const deploysToBuild = deploys.filter((deploy): deploy is Deploy & { deployable: Deployable } => { @@ -3520,9 +3767,19 @@ export default class BuildService extends BaseService { sourceGithubRepositoryId, sourceBranch, expectedGeneration, - nativeServiceAccount + nativeServiceAccount, + acceptedSourcePins ); getLogger().debug(`buildImage completed: deployUuid=${deploy.uuid} result=${result}`); + if (result === false) { + await this.recordPhaseFailure( + deploy, + runUUID, + DeployStatus.BUILD_FAILED, + 'Image build failed.', + expectedGeneration + ); + } return result; }) ); @@ -3532,6 +3789,13 @@ export default class BuildService extends BaseService { } catch (error) { if (error instanceof AuthorityLockLostError || error instanceof DeploymentSupersededError) throw error; getLogger().error({ error }, 'Docker: build error'); + await this.recordPhaseFailures( + deploys, + runUUID, + DeployStatus.BUILD_FAILED, + 'Image build setup failed.', + expectedGeneration + ); return false; } } @@ -3548,6 +3812,7 @@ export default class BuildService extends BaseService { sourceBranch, namespace, enqueueIngress = true, + unresolvedServices = [], }: { build: Build; runUUID?: string; @@ -3556,6 +3821,7 @@ export default class BuildService extends BaseService { sourceBranch?: string | null; namespace: string; enqueueIngress?: boolean; + unresolvedServices?: string[]; }): Promise { try { const buildId = build?.id; @@ -3591,16 +3857,21 @@ export default class BuildService extends BaseService { }); } - const allDeploys = await Deploy.query() - .where({ - buildId, - ...(runUUID ? { runUUID } : {}), - ...(githubRepositoryId ? { githubRepositoryId } : {}), - ...(githubRepositoryId && sourceBranch ? { branchName: sourceBranch } : {}), - }) - .withGraphFetched({ - deployable: true, - }); + const allDeploys = await this.selectRunDeploys( + buildId, + runUUID, + githubRepositoryId, + sourceBranch, + expectedGeneration + ); + // A service whose image or import failed stays in the plan so its dependents are blocked, but it is never applied. + const failedServices = new Set([ + ...unresolvedServices, + ...allDeploys + .filter((deploy) => expectedGeneration != null && isDeployFailure(deploy.status)) + .map((deploy) => deploy.deployable?.name) + .filter((name): name is string => Boolean(name)), + ]); const activeDeploys = allDeploys.filter((deploy): deploy is Deploy & { deployable: Deployable } => { if (!deploy.active) return false; @@ -3612,6 +3883,7 @@ export default class BuildService extends BaseService { // Generate manifests for GitHub/Docker/CLI deploys for (const deploy of activeDeploys) { + if (failedServices.has(deploy.deployable.name)) continue; const deployType = deploy.deployable.type; if (deployType === DeployTypes.GITHUB || deployType === DeployTypes.DOCKER || CLIDeployTypes.has(deployType)) { // Generate individual manifest for this deploy @@ -3638,19 +3910,35 @@ export default class BuildService extends BaseService { const managedDeploys = activeDeploys.filter( (d) => d.deployable.type !== DeployTypes.CODEFRESH && d.deployable.type !== DeployTypes.CONFIGURATION ); + if (managedDeploys.length === 0 && expectedGeneration != null) { + getLogger().info('Deploy: nothing to roll out reason=already_deployed'); + } const isCurrent = runUUID ? () => this.isDeploymentRunCurrent(build.id, runUUID, expectedGeneration) : undefined; + const deployIsCurrent = (deploy: Deploy) => + expectedGeneration != null + ? () => isDeployAuthorityCurrent(this.db.models, deploy, runUUID!, expectedGeneration) + : isCurrent!; const deploymentManager = new DeploymentManager(managedDeploys, { isCurrent, + expectedGeneration, + failedServices: Array.from(failedServices), nativeMutationGate: runUUID - ? (action) => this.withCurrentBuildPromotionLock(build.id, isCurrent!, action) + ? (deploy, action) => + expectedGeneration != null + ? this.withCurrentDeployPromotionLock(deploy.id, deployIsCurrent(deploy), action) + : this.withCurrentBuildPromotionLock(build.id, isCurrent!, action) : undefined, nativeSecretMutationGate: runUUID - ? (deploy, action) => this.withCurrentDeploySecretMutationLock(deploy.id, isCurrent!, action) + ? (deploy, action) => this.withCurrentDeploySecretMutationLock(deploy.id, deployIsCurrent(deploy), action) : undefined, }); - await deploymentManager.deploy(); + const { failed } = await deploymentManager.deploy(); + // Fenced runs carry failures on their rows; legacy whole-environment callers still expect a throw. + if (failed.length > 0 && expectedGeneration == null) { + throw new Error(`Deployment failed for ${failed.map((deploy) => deploy.deployable.name).join(', ')}`); + } } if (runUUID && !(await this.isDeploymentRunCurrent(build.id, runUUID, expectedGeneration))) { @@ -3672,15 +3960,17 @@ export default class BuildService extends BaseService { if (githubTypeDeploys.length > 0) { const legacyManifest = k8s.generateManifest({ build, - deploys: githubTypeDeploys, + deploys: githubTypeDeploys.filter((d) => !failedServices.has(d.deployable.name)), uuid: build.uuid, namespace, serviceAccountName, }); if (legacyManifest && legacyManifest.replace(/---/g, '').trim().length > 0) { - let manifestPatch = this.db.models.Build.query().patch({ manifest: legacyManifest }).where({ id: build.id }); - if (runUUID) manifestPatch = manifestPatch.where('runUUID', runUUID); - if (expectedGeneration != null) manifestPatch = manifestPatch.where('desiredGeneration', expectedGeneration); + let manifestPatch = this.db.models.Build.query() + .patch({ manifest: legacyManifest }) + .where({ id: build.id }) + .whereNull('deletedAt'); + if (runUUID && expectedGeneration == null) manifestPatch = manifestPatch.where('runUUID', runUUID); await manifestPatch; } } @@ -3732,6 +4022,7 @@ export default class BuildService extends BaseService { await build?.$fetchGraph('deploys'); const deploys = (build.deploys ?? []).filter( (deploy) => + !isDeployFailure(deploy.status) && (!githubRepositoryId || deploy.githubRepositoryId === githubRepositoryId) && (!githubRepositoryId || !sourceBranch || deploy.branchName === sourceBranch) ); @@ -3759,7 +4050,7 @@ export default class BuildService extends BaseService { }, }); - deploymentReconciliationQueue = this.queueManager.registerQueue(QUEUE_NAMES.DEPLOYMENT_RECONCILIATION, { + deploymentReconciliationQueue = this.queueManager.registerQueue(QUEUE_NAMES.SERVICE_RECONCILIATION, { connection: redisClient.getConnection(), defaultJobOptions: { attempts: 10, @@ -3991,73 +4282,84 @@ export default class BuildService extends BaseService { }); }; - private async claimDeploymentReconciliation( + private async claimDeploymentIntent( buildId: number, generation: number ): Promise { const build = await this.db.models.Build.query().findById(buildId).whereNull('deletedAt'); if (!build) return null; - - const desiredGeneration = Number(build.desiredGeneration); - const observedGeneration = Number(build.observedGeneration); - if (!Number.isSafeInteger(desiredGeneration) || !Number.isSafeInteger(observedGeneration)) { - throw new Error(`Build ${buildId} has invalid deployment generations`); + const refs: AcceptedDeploymentRefs = + build.acceptedRefs && typeof build.acceptedRefs === 'object' && !Array.isArray(build.acceptedRefs) + ? build.acceptedRefs + : {}; + const intent = Object.values(refs).find((candidate) => candidate?.gen === generation); + if (!intent?.requestId) { + getLogger({ buildId, generation }).info('Build reconciliation: signal ignored reason=intent_replaced'); + return null; } - // One signal owns exactly one generation. B must never wake up later and - // read/execute C's mailbox contents. - if (desiredGeneration !== generation || generation <= observedGeneration) return null; - - const dirty = dirtyDeploymentIntents(build.acceptedRefs, observedGeneration).filter( - ({ intent }) => intent.gen <= generation - ); - const latest = dirty.find(({ intent }) => intent.gen === generation)?.intent; - if (!latest?.requestId) throw new Error(`Build ${buildId} generation ${generation} has no request token`); + if (intent.observedGen === generation) { + getLogger({ buildId, generation }).info('Build reconciliation: signal ignored reason=already_observed'); + return null; + } + // Intents accepted before rows carried generations never stamped them; adopting here is idempotent. + await this.adoptDeploysForIntent(buildId, intent, generation); + return { generation, token: intent.requestId, intent, acceptedSourcePins: this.acceptedSourcePins(refs) }; + } - return { generation, token: latest.requestId, dirty }; + /** + * Every accepted push SHA is a floor for its repository and branch, whatever + * pass deploys the row. A full-environment push (config repo, or PR failure + * recovery) is a floor too, since a later full pass replaces its entry. + */ + private acceptedSourcePins(refs: AcceptedDeploymentRefs): AcceptedSourcePins { + const newest = new Map(); + for (const intent of Object.values(refs)) { + if (intent?.type !== 'source' || !intent.sha) continue; + const key = `${intent.githubRepositoryId}:${intent.branch}`; + const current = newest.get(key); + if (!current || Number(intent.gen) > current.gen) newest.set(key, { gen: Number(intent.gen), sha: intent.sha }); + } + return Object.fromEntries(Array.from(newest.entries()).map(([key, { sha }]) => [key, sha])); } - private deploymentReconciliationScopes(dirty: DirtyDeploymentIntent[]): DeploymentReconciliationScope[] { - const newestBroadIndex = _.findLastIndex( - dirty, - ({ intent }) => intent.type === 'all' || (intent.type === 'source' && intent.target === 'all') - ); + private async adoptDeploysForIntent(buildId: number, intent: DeploymentIntent, generation: number): Promise { + await this.db.models.Deploy.query() + .patch(deployStampForGeneration(generation)) + .where({ buildId, active: true, ...deploymentIntentDeployFilter(intent) }) + .where('desiredGeneration', '<', generation) + .where('observedGeneration', '<', generation); + } - // A full pass subsumes older unpinned requests, but it must not erase a - // delivered source SHA. Run unpinned/live-head work first and immutable - // source floors last so a temporarily stale provider read cannot roll C - // back to B. A source-targeted full pass still precedes selective pins. - const retained = dirty.filter( - ({ intent }, index) => newestBroadIndex < 0 || index >= newestBroadIndex || intent.type === 'source' - ); - const ordered = _.orderBy( - retained, - [ - ({ intent }) => { - if (intent.type === 'all') return 0; - if (intent.type === 'repository') return 1; - return intent.target === 'all' ? 2 : 3; - }, - ({ intent }) => intent.gen, - ], - ['asc', 'desc'] - ); + /** + * Taking a row from another run resets it: whatever it showed came from an + * older generation. Rows this run already owns keep what they reached, so a + * retry resumes rather than repeats them. The count covers every pending row + * the run owns afterwards. + */ + private async claimIntentDeploys(buildId: number, generation: number, token: string): Promise { + const pending = () => + this.db.models.Deploy.query() + .where({ buildId, desiredGeneration: generation }) + .where('observedGeneration', '<', generation); + const reset = { runUUID: token, status: DeployStatus.QUEUED, statusMessage: null }; + await pending().patch(reset).whereNull('runUUID'); + await pending().patch(reset).whereNot('runUUID', token); + const owned = await pending().where('runUUID', token).select('id'); + return owned.length; + } - return ordered.map(({ intent }) => { - if (intent.type === 'all') return { githubRepositoryId: null }; - if (intent.type === 'source') { - return { - githubRepositoryId: intent.target === 'all' ? null : intent.githubRepositoryId, - sourceGithubRepositoryId: intent.githubRepositoryId, - sourceRef: intent.sha, - sourceBeforeRef: intent.beforeSha, - sourceBranch: intent.branch, - }; - } + private deploymentReconciliationScope(intent: AcceptedDeploymentIntent): DeploymentReconciliationScope { + if (intent.type === 'all') return { githubRepositoryId: null }; + if (intent.type === 'source') { return { - githubRepositoryId: intent.githubRepositoryId, - skipDeletedServiceReconciliation: true, + githubRepositoryId: intent.target === 'all' ? null : intent.githubRepositoryId, + sourceGithubRepositoryId: intent.githubRepositoryId, + sourceRef: intent.sha, + sourceBeforeRef: intent.beforeSha, + sourceBranch: intent.branch, }; - }); + } + return { githubRepositoryId: intent.githubRepositoryId, skipDeletedServiceReconciliation: true }; } /** @@ -4070,47 +4372,117 @@ export default class BuildService extends BaseService { const repository: Repository | undefined = await this.db.models.Repository.query() .findOne({ githubRepositoryId: scope.sourceGithubRepositoryId }) .whereNull('deletedAt'); - const repositoryFullName = repository?.fullName; + return this.advanceAcceptedSourceRef( + repository?.fullName, + scope.sourceBranch, + scope.sourceRef, + scope.sourceBeforeRef + ); + } + + /** An accepted push is a floor. The live head replaces it only when GitHub proves the head descends from it. */ + private async advanceAcceptedSourceRef( + repositoryFullName: string | undefined, + branch: string, + sha: string, + beforeSha?: string + ): Promise { const [owner, name] = repositoryFullName?.split('/') ?? []; - if (!repositoryFullName || !owner || !name) return scope.sourceRef; + if (!repositoryFullName || !owner || !name) return sha; let currentRef: string; try { - currentRef = await github.getSHAForBranch(scope.sourceBranch, owner, name); + currentRef = await github.getSHAForBranch(branch, owner, name); } catch (error) { - getLogger({ error, repositoryFullName, branch: scope.sourceBranch }).warn( + getLogger({ error, repositoryFullName, branch }).warn( 'Build reconciliation: branch head lookup failed; using accepted source' ); - return scope.sourceRef; + return sha; } - if (!currentRef || currentRef === scope.sourceRef || currentRef === scope.sourceBeforeRef) return scope.sourceRef; + if (!currentRef || currentRef === sha || currentRef === beforeSha) return sha; try { - const comparison = await github.compareCommits({ - fullName: repositoryFullName, - base: scope.sourceRef, - head: currentRef, - }); + const comparison = await github.compareCommits({ fullName: repositoryFullName, base: sha, head: currentRef }); if (comparison === 'ahead') return currentRef; } catch (error) { - getLogger({ error, repositoryFullName, branch: scope.sourceBranch }).warn( + getLogger({ error, repositoryFullName, branch }).warn( 'Build reconciliation: commit comparison failed; using accepted source' ); } - return scope.sourceRef; + return sha; } - private async markDeploymentReconciliationObserved( - buildId: number, - generation: number, - token?: string - ): Promise { - let update = this.db.models.Build.query() + /** Pins are resolved once per full pass so YAML, image and rollout all see one revision per service. */ + private async resolveAcceptedSourcePins(pins: AcceptedSourcePins): Promise { + const resolved: AcceptedSourcePins = {}; + for (const [key, sha] of Object.entries(pins)) { + const separator = key.indexOf(':'); + const repositoryId = Number(key.slice(0, separator)); + const branch = key.slice(separator + 1); + const repository: Repository | undefined = await this.db.models.Repository.query() + .findOne({ githubRepositoryId: repositoryId }) + .whereNull('deletedAt'); + resolved[key] = await this.advanceAcceptedSourceRef(repository?.fullName, branch, sha); + } + return resolved; + } + + /** Rows this run still owns at this generation are caught up, whatever their outcome. */ + private async observeIntentDeploys(buildId: number, generation: number, token?: string): Promise { + let update = this.db.models.Deploy.query() .patch({ observedGeneration: generation }) - .where({ id: buildId, desiredGeneration: generation }) - .whereNull('deletedAt'); + .where({ buildId, desiredGeneration: generation }) + .where('observedGeneration', '<', generation); if (token) update = update.where('runUUID', token); - return (await update) === 1; + return update; + } + + /** + * The Build watermark is the newest generation every active service has caught up with. It is + * recomputed under the Build row lock in the same write that marks an intent finished, so + * concurrent finishes cannot leave it behind. + */ + private async settleBuildObservedGeneration(buildId: number, finishedGeneration?: number): Promise { + await this.db.models.Build.transact(async (trx) => { + const build = await this.db.models.Build.query(trx) + .select('id', 'desiredGeneration', 'acceptedRefs') + .findById(buildId) + .forUpdate(); + if (!build) return; + let acceptedRefs: AcceptedDeploymentRefs = + build.acceptedRefs && typeof build.acceptedRefs === 'object' && !Array.isArray(build.acceptedRefs) + ? build.acceptedRefs + : {}; + const finished = + finishedGeneration != null + ? Object.entries(acceptedRefs).find(([, intent]) => intent?.gen === finishedGeneration) + : undefined; + if (finished) acceptedRefs = { ...acceptedRefs, [finished[0]]: { ...finished[1], observedGen: finished[1].gen } }; + const pending = await this.db.models.Deploy.query(trx) + .select('desiredGeneration') + .where({ buildId, active: true }) + .whereRaw('?? > ??', ['desiredGeneration', 'observedGeneration']); + // An accepted intent that never finished is pending work even when no row carries its generation, + // which is the shape of an environment-wide intent whose rows were all taken by later intents. + const minPending = [ + ...pending.map((row) => Number(row.desiredGeneration)), + ...unobservedAcceptedGenerations(acceptedRefs), + ].reduce((min, generation) => Math.min(min, generation), Infinity); + const observedGeneration = Number.isFinite(minPending) ? minPending - 1 : Number(build.desiredGeneration); + await this.db.models.Build.query(trx) + .findById(buildId) + .patch({ acceptedRefs, observedGeneration } as Partial); + }); + } + + /** The completion mark is written last: a retry after any earlier failure republishes instead of exiting early. */ + private async finishIntent( + buildId: number, + generation: number, + options: { clearConfigError?: boolean } = {} + ): Promise { + await this.publishDerivedBuildStatus(buildId, options); + await this.settleBuildObservedGeneration(buildId, generation); } private async signalPendingDeploymentReconciliation(buildId: number, generation: number): Promise { @@ -4134,7 +4506,7 @@ export default class BuildService extends BaseService { async enqueuePendingDeploymentReconciliations(): Promise { const pendingAfter = (id: number) => this.db.models.Build.query() - .select('id', 'desiredGeneration') + .select('id', 'desiredGeneration', 'acceptedRefs') .whereRaw('?? > ??', ['desiredGeneration', 'observedGeneration']) .whereNull('deletedAt') .where('id', '>', id) @@ -4148,12 +4520,57 @@ export default class BuildService extends BaseService { } if (builds.length > 0) this.deploymentReconciliationSweepCursor = Number(builds[builds.length - 1].id); + const pendingRows = await this.db.models.Deploy.query() + .select('buildId', 'desiredGeneration') + .where('active', true) + .whereRaw('?? > ??', ['desiredGeneration', 'observedGeneration']); + const pendingByBuild = new Map>(); + for (const row of pendingRows) { + const buildId = Number(row.buildId); + const generations = pendingByBuild.get(buildId) ?? new Set(); + generations.add(Number(row.desiredGeneration)); + pendingByBuild.set(buildId, generations); + } + + const signals = new Map>(); + const staleWatermarks: number[] = []; + for (const build of builds) { + const buildId = Number(build.id); + // Rows are the durable record of pending work, and an accepted entry that never finished still is work. + const generations = new Set([ + ...(pendingByBuild.get(buildId) ?? []), + ...unobservedAcceptedGenerations(build.acceptedRefs), + ]); + if (generations.size === 0) staleWatermarks.push(buildId); + else signals.set(buildId, generations); + } + for (const buildId of staleWatermarks) { + await this.settleBuildObservedGeneration(buildId).catch((error) => + getLogger({ error, buildId }).warn('Build reconciliation: watermark repair failed') + ); + } + const candidateIds = Array.from(pendingByBuild.keys()).filter((id) => !signals.has(id)); + const liveIds = new Set( + candidateIds.length > 0 + ? (await this.db.models.Build.query().select('id').whereNull('deletedAt').whereIn('id', candidateIds)).map( + (build) => Number(build.id) + ) + : [] + ); + for (const [buildId, generations] of pendingByBuild) { + if (!signals.has(buildId) && liveIds.has(buildId)) signals.set(buildId, generations); + } + await Promise.all( - builds.map((build) => - this.signalPendingDeploymentReconciliation(build.id, Number(build.desiredGeneration)).catch((error) => { - getLogger({ error, buildId: build.id }).warn('Build reconciliation: sweep signal failed'); - }) - ) + Array.from(signals.entries()).map(async ([buildId, generations]) => { + for (const generation of generations) { + try { + await this.signalPendingDeploymentReconciliation(buildId, generation); + } catch (error) { + getLogger({ error, buildId }).warn('Build reconciliation: sweep signal failed'); + } + } + }) ); } @@ -4175,50 +4592,132 @@ export default class BuildService extends BaseService { return Number(job.attemptsMade ?? 0) + 1 >= attempts; } - /** - * A failure can happen before the normal run claim finishes. On the final - * BullMQ attempt, reacquire the short mutation lock and establish the same - * generation token before publishing one terminal failure. - */ + /** On the final queue attempt, rows this run owns that never settled are recorded failed so nothing stays pending. */ private async recordFinalDeploymentReconciliationFailure( buildId: number, claim: DeploymentReconciliationClaim, - build: Build | null, - runUUID: string | null, error: unknown ): Promise { - let failureBuild = build; - let failureRunUUID = runUUID; + await this.claimIntentDeploys(buildId, claim.generation, claim.token); + await this.recordFailureOnOwnedDeploys( + buildId, + claim.token, + claim.generation, + error, + 'Build queue processing failed.' + ); + const observed = await this.observeIntentDeploys(buildId, claim.generation, claim.token); + await this.finishIntent(buildId, claim.generation); + return observed > 0; + } - if (!failureBuild || !failureRunUUID) { - const terminalClaim = await this.withCurrentBuildDeploymentLock( - buildId, - async () => Boolean(await this.claimDeploymentReconciliation(buildId, claim.generation)), - async () => { - const currentClaim = await this.claimDeploymentReconciliation(buildId, claim.generation); - if (!currentClaim || currentClaim.token !== claim.token) return null; - - const currentBuild = await this.loadBuildDeploymentAuthority(buildId); - if (!currentBuild) return null; - const claimedRunUUID = await this.claimDeploymentRun(currentBuild, claim.token, claim.generation); - return claimedRunUUID ? { build: currentBuild, runUUID: claimedRunUUID } : null; - } - ); - if (!terminalClaim.admitted || !terminalClaim.value) return false; - failureBuild = terminalClaim.value.build; - failureRunUUID = terminalClaim.value.runUUID; + private async recordFailureOnOwnedDeploys( + buildId: number, + token: string, + generation: number, + error: unknown, + fallbackMessage: string + ): Promise { + const statusMessage = compactStatusMessage(statusMessageFromError(error, fallbackMessage)); + await this.db.models.Deploy.query() + .patch({ status: DeployStatus.ERROR, statusMessage }) + .where({ buildId, runUUID: token, desiredGeneration: generation }) + .where('observedGeneration', '<', generation) + .whereNotIn('status', [ + DeployStatus.READY, + DeployStatus.DEPLOYED, + DeployStatus.ERROR, + DeployStatus.BUILD_FAILED, + DeployStatus.DEPLOY_FAILED, + DeployStatus.TORN_DOWN, + ]); + } + + private async recordPhaseFailure( + deploy: Deploy, + runUUID: string, + status: DeployStatus, + fallbackMessage: string, + expectedGeneration?: number + ): Promise { + await this.db.models.Deploy.query() + .patch({ status, statusMessage: fallbackMessage }) + .where({ + id: deploy.id, + runUUID, + ...(expectedGeneration != null ? { desiredGeneration: expectedGeneration } : {}), + }) + .whereNotIn('status', [ + DeployStatus.ERROR, + DeployStatus.BUILD_FAILED, + DeployStatus.DEPLOY_FAILED, + DeployStatus.TORN_DOWN, + ]); + } + + /** Rows still in flight when a whole phase fails must not roll out as if they had succeeded. */ + private async recordPhaseFailures( + deploys: Array>, + runUUID: string, + status: DeployStatus, + fallbackMessage: string, + expectedGeneration?: number + ): Promise { + for (const deploy of deploys) { + await this.db.models.Deploy.query() + .patch({ status, statusMessage: fallbackMessage }) + .where({ + id: deploy.id, + runUUID, + ...(expectedGeneration != null ? { desiredGeneration: expectedGeneration } : {}), + }) + .whereNotIn('status', [ + DeployStatus.READY, + DeployStatus.DEPLOYED, + DeployStatus.BUILT, + DeployStatus.ERROR, + DeployStatus.BUILD_FAILED, + DeployStatus.DEPLOY_FAILED, + DeployStatus.TORN_DOWN, + ]); + } + } + + /** A service whose own lifecycle.yaml is broken fails alone; the root config and every other service proceed. */ + private async recordServiceConfigFailures( + buildId: number, + generation: number, + token: string, + failures: Record + ): Promise { + const rows = await this.db.models.Deploy.query() + .where({ buildId, desiredGeneration: generation }) + .where('observedGeneration', '<', generation) + .withGraphFetched('deployable'); + const matched = new Set(); + for (const row of rows) { + const message = row.deployable?.name ? failures[row.deployable.name] : undefined; + if (!message) continue; + matched.add(row.deployable.name); + // The row keeps this run's token so rollout planning still sees it as a failed prerequisite. + await this.db.models.Deploy.query() + .patch({ + runUUID: token, + status: DeployStatus.ERROR, + statusMessage: compactStatusMessage(message), + observedGeneration: generation, + }) + .where({ id: row.id, desiredGeneration: generation }); } + return Object.keys(failures).filter((name) => !matched.has(name)); + } - if (!(await this.isDeploymentRunCurrent(buildId, failureRunUUID, claim.generation))) return false; - await this.recordBuildFailure( - failureBuild, - BuildStatus.ERROR, - failureRunUUID, - error, - 'Build queue processing failed.', - claim.generation - ); - return this.markDeploymentReconciliationObserved(buildId, claim.generation, failureRunUUID); + /** The failing intent takes its rows, so a superseded run's token-fenced writes cannot land after they are observed. */ + private async takeIntentDeploys(buildId: number, generation: number, token: string): Promise { + await this.db.models.Deploy.query() + .patch({ runUUID: token }) + .where({ buildId, desiredGeneration: generation }) + .where('observedGeneration', '<', generation); } processDeploymentReconciliationQueue = async (job: DeploymentReconciliationQueueJob) => { @@ -4229,175 +4728,169 @@ export default class BuildService extends BaseService { } await this.tryWithDeploymentGenerationLock(buildId, Number(generation), async () => { - const claim = await this.claimDeploymentReconciliation(buildId, Number(generation)); + const claim = await this.claimDeploymentIntent(buildId, Number(generation)); if (!claim) return; + const { token, intent } = claim; + const envWide = deploymentIntentSelectsAllDeploys(intent); + const scope = this.deploymentReconciliationScope(intent); + const logContext = { buildId, generation: claim.generation }; + const state: { build: Build | null } = { build: null }; - let build: Build | null = null; - let activeBuild: Build | null = null; - let activeRunUUID: string | null = null; try { - const initialClaim = await this.withCurrentBuildDeploymentLock( + const configuration = await this.withCurrentBuildDeploymentLock( buildId, - async () => Boolean(await this.claimDeploymentReconciliation(buildId, claim.generation)), - async () => { - const currentClaim = await this.claimDeploymentReconciliation(buildId, claim.generation); - if (!currentClaim) return; - build = await this.loadBuildDeploymentAuthority(buildId); - if (!build) return; + () => this.isIntentPending(buildId, token, claim.generation), + async (): Promise => { + const build = await this.loadBuildDeploymentAuthority(buildId); + if (!build) return null; if (build.uuid) updateLogContext({ buildUuid: build.uuid }); + state.build = build; const blocked = this.deploymentBlockReason(build); + if (blocked === 'tearing_down') { + getLogger(logContext).info('Build reconciliation: deferred reason=tearing_down'); + return { kind: 'deferred' }; + } if (blocked) { - if (blocked === 'tearing_down') { - getLogger({ buildId, generation: claim.generation }).info( - 'Build reconciliation: deferred reason=tearing_down' - ); - return; + getLogger(logContext).info(`Build reconciliation: observed without execution reason=${blocked}`); + await this.observeIntentDeploys(buildId, claim.generation); + return { kind: 'observed' }; + } + if (!envWide) { + const pendingRow = await this.db.models.Deploy.query() + .findOne({ buildId, desiredGeneration: claim.generation }) + .where('observedGeneration', '<', claim.generation) + .select('id'); + if (!pendingRow) { + getLogger(logContext).info('Build reconciliation: nothing pending reason=rows_settled_or_taken'); + return { kind: 'observed' }; } - getLogger({ buildId, generation: claim.generation }).info( - `Build reconciliation: observed without execution reason=${blocked}` - ); - await this.markDeploymentReconciliationObserved(buildId, claim.generation); - return; } - activeRunUUID = await this.claimDeploymentRun(build, claim.token, claim.generation); - } - ); + if (envWide && !(await this.claimDeploymentRun(build, token, claim.generation))) return null; - if (!initialClaim.admitted || !build || !activeRunUUID) return; - const reconciliationBuild = build; - activeBuild = reconciliationBuild; - const terminalOutcomes: Array<{ status: BuildStatus; error?: unknown }> = []; - const scopes = this.deploymentReconciliationScopes(claim.dirty); - getLogger({ stage: LogStage.BUILD_STARTING, buildId, generation: claim.generation }).info( - `Build reconciliation: started scopes=${scopes.length}` - ); - - for (const scope of scopes) { - const sourceRef = await this.resolveCurrentSourceRef(scope); - let preparation: DeploymentScopePreparation | null = null; - const configuration = await this.withCurrentBuildDeploymentLock( - buildId, - () => this.isDeploymentRunCurrent(buildId, claim.token, claim.generation), - async () => { - if (!(await this.isDeploymentRunCurrent(buildId, claim.token, claim.generation))) return; - - await this.importYamlConfigFile( - reconciliationBuild.environment!, - reconciliationBuild, + const sourceRef = await this.resolveCurrentSourceRef(scope); + const acceptedSourcePins = await this.resolveAcceptedSourcePins( + acceptedSourcePinsForIntent(claim.acceptedSourcePins, intent, rootSourcePinKey(build)) + ); + let imported: DeployableReconciliationResult | undefined; + let preparation: DeploymentScopePreparation | null; + try { + imported = await this.importYamlConfigFile( + build.environment!, + build, scope.githubRepositoryId ?? undefined, { skipDeletedServiceReconciliation: scope.skipDeletedServiceReconciliation, sourceRef, sourceBranch: scope.sourceBranch, sourceGithubRepositoryId: scope.sourceGithubRepositoryId, - runUUID: claim.token, + runUUID: token, expectedGeneration: claim.generation, + acceptedSourcePins, } ); - preparation = await this.prepareDeploymentScope( - reconciliationBuild, - scope.githubRepositoryId, - sourceRef, - { - runUUID: claim.token, - sourceBranch: scope.sourceBranch, - sourceGithubRepositoryId: scope.sourceGithubRepositoryId, - expectedGeneration: claim.generation, - } + // Every intent reads the root config, so any successful import proves it valid again. + if (build.status === BuildStatus.CONFIG_ERROR) { + await this.publishDerivedBuildStatus(buildId, { clearConfigError: true }); + } + preparation = await this.prepareDeploymentScope(build, scope.githubRepositoryId, sourceRef, { + runUUID: token, + sourceBranch: scope.sourceBranch, + sourceGithubRepositoryId: scope.sourceGithubRepositoryId, + expectedGeneration: claim.generation, + acceptedSourcePins, + }); + } catch (error) { + if (!(error instanceof ParsingError || error instanceof ValidationError)) throw error; + // A service's own broken YAML is reported per service, so an error that escapes the import is the root config's. + // Recorded while the configuration lock is held, so the status follows the order the config was read in. + await this.recordRootConfigError(build, error); + await this.takeIntentDeploys(buildId, claim.generation, token); + await this.observeIntentDeploys(buildId, claim.generation); + return { kind: 'observed' }; + } + if (!preparation) return null; + await this.adoptDeploysForIntent(buildId, intent, claim.generation); + let unresolvedServices: string[] = []; + if (imported?.configFailures && Object.keys(imported.configFailures).length > 0) { + unresolvedServices = await this.recordServiceConfigFailures( + buildId, + claim.generation, + token, + imported.configFailures ); } - ); - - if (!configuration.admitted || !preparation) return; - const outcome = await this.executeDeploymentScope(preparation, claim.generation); - if (!outcome) return; - terminalOutcomes.push(outcome); - if (!(await this.isDeploymentRunCurrent(buildId, claim.token, claim.generation))) return; + await recordIntentConfigFailures( + buildId, + claim.generation, + Object.fromEntries( + unresolvedServices.map((name) => [name, imported?.configFailures?.[name] ?? 'invalid lifecycle.yaml']) + ) + ); + preparation.unresolvedServices = unresolvedServices; + const owned = await this.claimIntentDeploys(buildId, claim.generation, token); + await this.markConfigurationsAsBuilt(build, token, scope.githubRepositoryId, scope.sourceBranch); + return { kind: 'ready', preparation, owned }; + } + ); + if (!configuration.admitted || !configuration.value) return; + const outcome = configuration.value; + if (outcome.kind === 'deferred') return; + if (outcome.kind === 'observed') { + await this.finishIntent(buildId, claim.generation); + return; } - - // Scope workers mutate one shared ingress snapshot. Publish it once, - // after every selected scope has settled, so an earlier scope cannot - // overwrite the final configuration for the same generation. - if (terminalOutcomes.some(({ status }) => status === BuildStatus.DEPLOYED)) { - await this.enqueueIngressManifest(buildId, claim.token, claim.generation); + if (outcome.owned === 0) { + getLogger(logContext).info('Build reconciliation: completed services=0'); + await this.finishIntent(buildId, claim.generation); + return; } - const failedOutcome = terminalOutcomes.find(({ status }) => status === BuildStatus.ERROR); - const finalStatus = failedOutcome - ? BuildStatus.ERROR - : isDeployEnabled(reconciliationBuild) - ? BuildStatus.DEPLOYED - : BuildStatus.BUILT; - await this.updateStatusAndComment( - reconciliationBuild, - finalStatus, - claim.token, - true, - true, - failedOutcome?.error instanceof Error ? failedOutcome.error : null, - claim.generation + getLogger({ stage: LogStage.BUILD_STARTING, ...logContext }).info( + `Build reconciliation: started services=${outcome.owned}` ); - - if (await this.markDeploymentReconciliationObserved(buildId, claim.generation, claim.token)) { - getLogger({ stage: LogStage.BUILD_COMPLETE, buildId, generation: claim.generation }).info( - 'Build reconciliation: completed' + await this.publishDerivedBuildStatus(buildId); + const summary = await this.executeDeploymentScope(outcome.preparation, claim.generation); + if (summary?.deployed) await this.enqueueIngressManifest(buildId, token, claim.generation); + const observed = await this.observeIntentDeploys(buildId, claim.generation, token); + await this.finishIntent(buildId, claim.generation); + if (observed > 0) { + getLogger({ stage: LogStage.BUILD_COMPLETE, ...logContext }).info( + `Build reconciliation: completed services=${observed}` ); } } catch (error) { - if (error instanceof AuthorityLockLostError) { - const stillCurrent = activeRunUUID - ? await this.isDeploymentRunCurrent(buildId, activeRunUUID, claim.generation) - : Boolean(await this.claimDeploymentReconciliation(buildId, claim.generation)); - if (stillCurrent) { - getLogger({ error, buildId, generation: claim.generation }).warn( - 'Build reconciliation: mutation lock lost; leaving generation pending for retry' - ); - throw error; - } + const stillCurrent = await this.isDeploymentRunCurrent(buildId, token, claim.generation).catch(() => false); + if (error instanceof AuthorityLockLostError && stillCurrent) { + getLogger({ error, ...logContext }).warn( + 'Build reconciliation: mutation lock lost; leaving generation pending for retry' + ); + throw error; + } + if (!stillCurrent) { + getLogger(logContext).info('Build reconciliation: stale failure ignored'); + return; } - const stillCurrent = activeRunUUID - ? await this.isDeploymentRunCurrent(buildId, activeRunUUID, claim.generation) - : Boolean(await this.claimDeploymentReconciliation(buildId, claim.generation)); - if (stillCurrent) { - const configError = error instanceof ParsingError || error instanceof ValidationError; - if (configError && activeBuild && activeRunUUID) { - await this.recordBuildFailure( - activeBuild, - BuildStatus.CONFIG_ERROR, - activeRunUUID, - error, - 'Lifecycle configuration failed validation.', - claim.generation - ); - await this.markDeploymentReconciliationObserved(buildId, claim.generation, activeRunUUID); - return; - } - - if (!this.isFinalDeploymentReconciliationAttempt(job)) { - getLogger({ error, buildId, generation: claim.generation }).warn( - 'Build reconciliation: transient failure; leaving generation pending for queue retry' - ); - throw error; - } - if ( - await this.recordFinalDeploymentReconciliationFailure(buildId, claim, activeBuild, activeRunUUID, error) - ) { - getLogger({ stage: LogStage.BUILD_FAILED, buildId, generation: claim.generation }).error( - { error }, - 'Build reconciliation: failed after final queue attempt' - ); - return; - } + if (error instanceof ParsingError || error instanceof ValidationError) { + if (state.build) await this.recordRootConfigError(state.build, error); + await this.takeIntentDeploys(buildId, claim.generation, token); + await this.observeIntentDeploys(buildId, claim.generation); + await this.finishIntent(buildId, claim.generation); + return; + } - // Authority may have changed while the final failure was being - // recorded. Stale failures are intentionally silent; a still-current - // failure must remain retryable if its terminal write could not land. - if (!(await this.claimDeploymentReconciliation(buildId, claim.generation))) return; + if (!this.isFinalDeploymentReconciliationAttempt(job)) { + getLogger({ error, ...logContext }).warn( + 'Build reconciliation: transient failure; leaving generation pending for queue retry' + ); throw error; } - getLogger({ buildId, generation: claim.generation }).info('Build reconciliation: stale failure ignored'); + + await this.recordFinalDeploymentReconciliationFailure(buildId, claim, error); + getLogger({ stage: LogStage.BUILD_FAILED, error, ...logContext }).error( + 'Build reconciliation: failed after final queue attempt' + ); } }); }); @@ -4413,6 +4906,66 @@ export default class BuildService extends BaseService { }; } +/** + * A full pass honours every floor. A narrower intent honours the root config + * floor plus, for a repository redeploy, that repository's own; a push carries + * its own revision. Floors are verified against GitHub, so a narrow intent + * must not pay for every repository's lookup. + */ +function acceptedSourcePinsForIntent( + pins: AcceptedSourcePins, + intent: DeploymentIntent, + rootKey: string | null +): AcceptedSourcePins { + if (deploymentIntentSelectsAllDeploys(intent)) return pins; + const ownPrefix = intent.type === 'repository' ? `${intent.githubRepositoryId}:` : null; + return Object.fromEntries( + Object.entries(pins).filter(([key]) => key === rootKey || (ownPrefix != null && key.startsWith(ownPrefix))) + ); +} + +function rootSourcePinKey(build: Build): string | null { + const source = getBuildSource(build); + if (source.githubRepositoryId == null || !source.branchName) return null; + return `${source.githubRepositoryId}:${source.branchName}`; +} + +/** + * Row-less services whose YAML failed in the newest import that evaluated them. + * An environment-wide import evaluates everything and replaces every older + * note; a newer service-scoped import adds its own. A service that has a row + * again carries its own state. + */ +function unresolvedServiceFailures( + refs: AcceptedDeploymentRefs | null | undefined, + active: Array> +): Record { + if (!refs || typeof refs !== 'object' || Array.isArray(refs)) return {}; + const imported = Object.values(refs).filter((entry) => entry?.configFailures && Number.isSafeInteger(entry.gen)); + const newestEnvWide = imported + .filter((entry) => deploymentIntentSelectsAllDeploys(entry)) + .sort((left, right) => right.gen - left.gen)[0]; + const carried = new Set(active.map((deploy) => deploy.deployable?.name).filter(Boolean)); + const failures: Record = {}; + for (const entry of imported) { + const counts = + entry === newestEnvWide || + (!deploymentIntentSelectsAllDeploys(entry) && (!newestEnvWide || entry.gen > newestEnvWide.gen)); + if (!counts) continue; + for (const [name, reason] of Object.entries(entry.configFailures ?? {})) { + if (!carried.has(name)) failures[name] = reason; + } + } + return failures; +} + +function unobservedAcceptedGenerations(refs: AcceptedDeploymentRefs | null | undefined): number[] { + if (!refs || typeof refs !== 'object' || Array.isArray(refs)) return []; + return Object.values(refs) + .filter((intent) => Number.isSafeInteger(intent?.gen) && intent.observedGen !== intent.gen) + .map((intent) => intent.gen); +} + function canonicalJson(value: unknown): unknown { if (Array.isArray(value)) return value.map(canonicalJson); if (value && typeof value === 'object') { diff --git a/src/server/services/deploy.ts b/src/server/services/deploy.ts index e14bf393..deb1bbb2 100644 --- a/src/server/services/deploy.ts +++ b/src/server/services/deploy.ts @@ -20,6 +20,7 @@ import * as codefresh from 'server/lib/codefresh'; import { getLogger, withLogContext, extractContextForQueue } from 'server/lib/logger'; import hash from 'object-hash'; import { BuildKind, DeployStatus, DeployTypes } from 'shared/constants'; +import { isDeployAuthorityCurrent } from 'server/lib/deploymentReconciliation/authority'; import * as cli from 'server/lib/cli'; import RDS from 'aws-sdk/clients/rds'; import resourceGroupsTagging from 'aws-sdk/clients/resourcegroupstaggingapi'; @@ -83,7 +84,8 @@ export default class DeployService extends BaseService { githubRepositoryId?: number, sourceRef?: string | null, sourceBranch?: string | null, - sourceGithubRepositoryId: number | null | undefined = githubRepositoryId + sourceGithubRepositoryId: number | null | undefined = githubRepositoryId, + acceptedSourcePins?: Record ): Promise { await build?.$fetchGraph('[deployables.[repository]]'); @@ -162,7 +164,9 @@ export default class DeployService extends BaseService { sourceRef, sourceGithubRepositoryId, sourceBranch - ) ?? (await getShaForDeploy(deploy)); + ) ?? + acceptedSourcePins?.[`${deployableRepositoryId}:${effectiveBranch}`] ?? + (await getShaForDeploy(deploy)); patchFields.sha = sha; } catch (error) { getLogger().debug({ error }, 'Deploy: SHA fetch failed continuing=true'); @@ -193,9 +197,10 @@ export default class DeployService extends BaseService { * Helper function to check if an Aurora database already exists in AWS * @param buildUuid The build UUID to search for * @param serviceName The service name to search for - * @returns The database cluster endpoint address if found (or instance endpoint if not clustered), null otherwise + * @returns The database cluster endpoint address if found (or instance endpoint if not clustered), null when + * none is found, undefined when the check itself failed */ - private async findExistingAuroraDatabase(buildUuid: string, serviceName: string): Promise { + private async findExistingAuroraDatabase(buildUuid: string, serviceName: string): Promise { try { const rds = new RDS(); const taggingApi = new resourceGroupsTagging(); @@ -250,7 +255,7 @@ export default class DeployService extends BaseService { return null; } catch (error) { getLogger().debug({ error }, 'Aurora: check failed'); - return null; + return undefined; } } @@ -281,6 +286,12 @@ export default class DeployService extends BaseService { } const existingDbEndpoint = await this.findExistingAuroraDatabase(deploy.build.uuid, deploy.deployable.name); + if (existingDbEndpoint === undefined && deploy.cname) { + // A failed check is no proof the database is gone; a new restore would replace a live endpoint. + getLogger().info('Aurora: skipped reason=checkFailed'); + await this.patchDeployForRun(deploy, runUUID, { status: DeployStatus.BUILT }); + return true; + } if (existingDbEndpoint) { getLogger().info('Aurora: skipped reason=exists'); await this.patchDeployForRun(deploy, runUUID, { @@ -330,7 +341,9 @@ export default class DeployService extends BaseService { runUUID: string, sourceRef?: string | null, sourceGithubRepositoryId?: number | null, - sourceBranch?: string | null + sourceBranch?: string | null, + acceptedSourcePins?: Record, + expectedGeneration?: number ): Promise { return withLogContext({ deployUuid: deploy.uuid, serviceName: deploy.deployable?.name }, async () => { let result: boolean = false; @@ -348,7 +361,8 @@ export default class DeployService extends BaseService { deploy.branchName, sourceRef, sourceGithubRepositoryId, - sourceBranch + sourceBranch, + acceptedSourcePins ); const shortSha = fullSha.substring(0, 7); const envSha = hash(merge(deploy.env || {}, build.commentRuntimeEnv)); @@ -366,26 +380,40 @@ export default class DeployService extends BaseService { let buildLogs: string; let codefreshBuildId: string; try { - const current = await this.patchDeployForRun(deploy, runUUID, { - buildLogs: null, - buildPipelineId: null, - buildOutput: null, - deployPipelineId: null, - deployOutput: null, - }); + const current = await this.patchDeployForRun( + deploy, + runUUID, + { + buildLogs: null, + buildPipelineId: null, + buildOutput: null, + deployPipelineId: null, + deployOutput: null, + }, + expectedGeneration + ); if (!current) { getLogger().info('Codefresh: skipped reason=superseded'); return true; } - const pinnedSourceRef = this.getPinnedBuildSourceRef( - build, - deploy.githubRepositoryId, - deploy.branchName, - sourceRef, - sourceGithubRepositoryId, - sourceBranch - ); + if ( + !(await this.patchDeployForRun(deploy, runUUID, { status: DeployStatus.DEPLOYING }, expectedGeneration)) + ) { + getLogger().info('Codefresh: deploy skipped reason=superseded'); + return true; + } + const pinnedSourceRef = + this.getPinnedBuildSourceRef( + build, + deploy.githubRepositoryId, + deploy.branchName, + sourceRef, + sourceGithubRepositoryId, + sourceBranch + ) ?? + acceptedSourcePins?.[`${deploy.githubRepositoryId}:${deploy.branchName}`] ?? + null; codefreshBuildId = await cli.codefreshDeploy(deploy, build, deployable, pinnedSourceRef).catch((error) => { getLogger().error({ error }, 'Codefresh: build id missing'); return null; @@ -455,13 +483,23 @@ export default class DeployService extends BaseService { runUUID: string, sourceRef?: string | null, sourceGithubRepositoryId?: number | null, - sourceBranch?: string | null + sourceBranch?: string | null, + acceptedSourcePins?: Record, + expectedGeneration?: number ): Promise { if (deploy.deployable != null) { if (deploy.deployable.type === DeployTypes.AURORA_RESTORE) { return this.deployAurora(deploy, runUUID); } else if (deploy.deployable.type === DeployTypes.CODEFRESH) { - return this.deployCodefresh(deploy, runUUID, sourceRef, sourceGithubRepositoryId, sourceBranch); + return this.deployCodefresh( + deploy, + runUUID, + sourceRef, + sourceGithubRepositoryId, + sourceBranch, + acceptedSourcePins, + expectedGeneration + ); } } } @@ -478,7 +516,8 @@ export default class DeployService extends BaseService { sourceGithubRepositoryId?: number | null, sourceBranch?: string | null, expectedGeneration?: number, - nativeServiceAccount?: string + nativeServiceAccount?: string, + acceptedSourcePins?: Record ): Promise { return withLogContext({ deployUuid: deploy.uuid, serviceName: deploy.deployable?.name }, async () => { try { @@ -499,7 +538,8 @@ export default class DeployService extends BaseService { sourceGithubRepositoryId, sourceBranch, expectedGeneration, - nativeServiceAccount + nativeServiceAccount, + acceptedSourcePins ); case DeployTypes.DOCKER: await this.patchAndUpdateActivityFeed( @@ -524,7 +564,8 @@ export default class DeployService extends BaseService { sourceGithubRepositoryId, sourceBranch, expectedGeneration, - nativeServiceAccount + nativeServiceAccount, + acceptedSourcePins ); } @@ -539,7 +580,8 @@ export default class DeployService extends BaseService { deploy.branchName, sourceRef, sourceGithubRepositoryId, - sourceBranch + sourceBranch, + acceptedSourcePins ); } catch (shaError) { getLogger().debug( @@ -654,16 +696,18 @@ export default class DeployService extends BaseService { branchName: string | null, sourceRef?: string | null, sourceGithubRepositoryId?: number | null, - sourceBranch?: string | null + sourceBranch?: string | null, + acceptedSourcePins?: Record ): Promise { - const pinned = this.getPinnedBuildSourceRef( - deploy.build, - deploy.githubRepositoryId, - deploy.branchName, - sourceRef, - sourceGithubRepositoryId, - sourceBranch - ); + const pinned = + this.getPinnedBuildSourceRef( + deploy.build, + deploy.githubRepositoryId, + deploy.branchName, + sourceRef, + sourceGithubRepositoryId, + sourceBranch + ) ?? acceptedSourcePins?.[`${deploy.githubRepositoryId}:${deploy.branchName}`]; if (pinned) return pinned; const [owner, name] = repo?.split('/') || []; @@ -693,9 +737,16 @@ export default class DeployService extends BaseService { private async patchDeployForRun( deploy: Deploy, runUUID: string, - params: Objection.PartialModelObject + params: Objection.PartialModelObject, + expectedGeneration?: number ): Promise { - const updated = await this.db.models.Deploy.query().where({ id: deploy.id, runUUID }).patch(params); + const updated = await this.db.models.Deploy.query() + .where({ + id: deploy.id, + runUUID, + ...(expectedGeneration != null ? { desiredGeneration: expectedGeneration } : {}), + }) + .patch(params); if (!updated) { getLogger().debug(`Deploy: stale write skipped deployId=${deploy.id} runUUID=${runUUID}`); return false; @@ -703,25 +754,8 @@ export default class DeployService extends BaseService { return true; } - private async isDeployRunCurrent(deploy: Deploy, runUUID: string): Promise { - const current = await this.db.models.Deploy.query().findOne({ id: deploy.id, runUUID }).select('id'); - return Boolean(current); - } - - private async isBuildRunCurrent( - buildId: number | null | undefined, - runUUID: string, - expectedGeneration?: number - ): Promise { - if (!buildId) return false; - let current = this.db.models.Build.query().findOne({ id: buildId, runUUID }).whereNull('deletedAt'); - if (expectedGeneration != null) current = current.where('desiredGeneration', expectedGeneration); - return Boolean(await current); - } - private async isDeploymentRunCurrent(deploy: Deploy, runUUID: string, expectedGeneration?: number): Promise { - if (!(await this.isDeployRunCurrent(deploy, runUUID))) return false; - return expectedGeneration == null || this.isBuildRunCurrent(deploy.buildId, runUUID, expectedGeneration); + return isDeployAuthorityCurrent(this.db.models, deploy, runUUID, expectedGeneration); } private async withDeploySecretMutation( @@ -1087,7 +1121,8 @@ export default class DeployService extends BaseService { sourceGithubRepositoryId?: number | null, sourceBranch?: string | null, expectedGeneration?: number, - nativeServiceAccount?: string + nativeServiceAccount?: string, + acceptedSourcePins?: Record ) { const { build, deployable } = deploy; const uuid = build?.uuid; @@ -1113,7 +1148,8 @@ export default class DeployService extends BaseService { deploy.branchName, sourceRef, sourceGithubRepositoryId, - sourceBranch + sourceBranch, + acceptedSourcePins ); const repositoryName: string = deployable.repository.fullName; diff --git a/src/server/services/deployable.ts b/src/server/services/deployable.ts index 2f2f8183..5705e42b 100644 --- a/src/server/services/deployable.ts +++ b/src/server/services/deployable.ts @@ -24,6 +24,8 @@ import { CAPACITY_TYPE, DeployTypes } from 'shared/constants'; import { Builder, Helm } from 'server/models/yaml'; import GlobalConfigService from './globalConfig'; +import { ParsingError } from 'server/lib/yamlConfigParser'; +import { ValidationError } from 'server/lib/yamlConfigValidator'; export type DeployableConfigSource = 'yaml'; @@ -44,6 +46,8 @@ export interface DeployableReconciliationResult { unresolvedServiceNames: string[]; /** Repositories whose YAML could not be read; their deployables are never reaped as stale. */ unresolvedRepositoryIds: number[]; + /** Services whose own lifecycle.yaml failed to parse or validate, by service name. */ + configFailures: Record; } export interface DeployableAttributes { @@ -444,7 +448,9 @@ export default class DeployableService extends BaseService { sourceBranch?: string | null, sourceGithubRepositoryId: number | null | undefined = filterGithubRepositoryId, unresolvedServiceNames: Set = new Set(), - unresolvedRepositoryIds: Set = new Set() + unresolvedRepositoryIds: Set = new Set(), + configFailures: Map = new Map(), + acceptedSourcePins?: Record ): Promise { try { let sourceRepository: Repository | null = null; @@ -571,7 +577,10 @@ export default class DeployableService extends BaseService { } if (filterGithubRepositoryId != null && sourceBranch != null) targetAttributionResolved = true; - const dependencyConfigRef = sourceRefTargetsDependency ? sourceRef : branchName; + // A full pass reads each service's YAML at the same accepted revision its image is built from. + const dependencyConfigRef = sourceRefTargetsDependency + ? sourceRef + : acceptedSourcePins?.[`${repository.githubRepositoryId}:${branchName}`] ?? branchName; dependencyYamlConfig = await YamlService.fetchLifecycleConfigByRepository( repository, dependencyConfigRef @@ -641,6 +650,12 @@ export default class DeployableService extends BaseService { service: yamlEnvService.name, error, }).error('Deployable: create/update from yaml failed'); + // A broken service lifecycle.yaml fails that service alone; only the root config fails the environment. + if (error instanceof ParsingError || error instanceof ValidationError) { + configFailures.set(yamlEnvService.name, (error as Error).message); + markUnresolved(yamlEnvService.name, undefined); + return; + } throw error; } }) @@ -671,7 +686,12 @@ export default class DeployableService extends BaseService { rootBaseConfigRef != null ) { const sourceRefTargetsRoot = sourceRef != null && matchesDeliveredSource(sourceRepository, rootBranch); - const rootConfigRef = sourceRefTargetsRoot ? sourceRef : rootBaseConfigRef; + // A branch-named root ref honours the newest accepted push to that branch; a pinned config SHA stays as is. + const rootPin = + rootBaseConfigRef === rootBranch + ? acceptedSourcePins?.[`${sourceRepository.githubRepositoryId}:${rootBranch}`] + : undefined; + const rootConfigRef = sourceRefTargetsRoot ? sourceRef : rootPin ?? rootBaseConfigRef; const yamlConfig: YamlService.LifecycleConfig = await YamlService.fetchLifecycleConfigByRepository( sourceRepository, rootConfigRef @@ -770,7 +790,8 @@ export default class DeployableService extends BaseService { filterGithubRepositoryId?: number, sourceRef?: string | null, sourceBranch?: string | null, - sourceGithubRepositoryId: number | null | undefined = filterGithubRepositoryId + sourceGithubRepositoryId: number | null | undefined = filterGithubRepositoryId, + acceptedSourcePins?: Record ): Promise { // We are going to ingest all the database and yaml configuration and process in the memory before writes into the database let deployables: Deployable[] = []; @@ -783,6 +804,7 @@ export default class DeployableService extends BaseService { const deployableServices: Map = new Map(); const unresolvedServiceNames = new Set(); const unresolvedRepositoryIds = new Set(); + const configFailures = new Map(); try { if (pullRequest != null || hasBuildSource) { if (pullRequest != null && pullRequest.branchName == null) { @@ -801,7 +823,9 @@ export default class DeployableService extends BaseService { sourceBranch, sourceGithubRepositoryId, unresolvedServiceNames, - unresolvedRepositoryIds + unresolvedRepositoryIds, + configFailures, + acceptedSourcePins ); // Finally, Upsert the deployables into the database @@ -828,6 +852,7 @@ export default class DeployableService extends BaseService { filterGithubRepositoryId: filterGithubRepositoryId ?? null, unresolvedServiceNames: Array.from(unresolvedServiceNames), unresolvedRepositoryIds: Array.from(unresolvedRepositoryIds), + configFailures: Object.fromEntries(configFailures), reconcileEligibleDeployables: Array.from(deployableServices.values()) .filter((deployable) => deployable.reconcileEligible) .map((deployable) => ({ diff --git a/src/server/services/github.ts b/src/server/services/github.ts index 1dc61e98..735ed8bd 100644 --- a/src/server/services/github.ts +++ b/src/server/services/github.ts @@ -563,17 +563,17 @@ export default class GithubService extends Service { getLogger({ error, githubRepositoryId, branchName }).error('Push: API auto-track processing failed'); }); - if (!allDeploys.length) { - // additional check for static env branch - await this.handlePushForStaticEnv({ - githubRepositoryId, - branchName, - headCommit: deployTriggerRef ?? null, - beforeCommit: !this.isVoidCommit(previousCommit) ? previousCommit : null, - }); - return; - } + // A config-repo push redeploys every static environment rooted at it, even when that repo also owns + // services; those environments are excluded from the repository-targeted path below. + const rootStaticBuildIds = await this.handlePushForStaticEnv({ + githubRepositoryId, + branchName, + headCommit: deployTriggerRef ?? null, + beforeCommit: !this.isVoidCommit(previousCommit) ? previousCommit : null, + }); + if (!allDeploys.length) return; const deploysToRebuild = allDeploys.filter((deploy) => { + if (rootStaticBuildIds.has(Number(deploy.buildId))) return false; if (!deploy?.build) return false; if (deploy.devMode) { getLogger().info(`Push: skipping dev mode service deployId=${deploy.id} service=${deploy.deployable?.name}`); @@ -756,9 +756,10 @@ export default class GithubService extends Service { branchName: string; headCommit?: string | null; beforeCommit?: string | null; - }): Promise => { + }): Promise> => { + const enqueued = new Set(); try { - const build = await this.db.models.Build.query() + const builds = await this.db.models.Build.query() .whereIn('pullRequestId', (prBuilder) => { prBuilder .from(this.db.models.PullRequest.tableName) @@ -773,24 +774,31 @@ export default class GithubService extends Service { }) .andWhere('isStatic', true) .andWhere('trackDefaultBranches', true) - .first(); + .whereNull('deletedAt'); - if (!build) return; - - getLogger().info(`Push: redeploying reason=staticEnv`); - await this.db.services.BuildService.enqueueResolveAndDeployBuild({ - buildId: build?.id, - ...(headCommit ? { sourceRef: headCommit } : {}), - ...(beforeCommit ? { sourceBeforeRef: beforeCommit } : {}), - sourceGithubRepositoryId: githubRepositoryId, - sourceBranch: branchName, - }); + for (const build of builds) { + // A root build is claimed here even when its enqueue fails: a repository-scoped intent is the wrong shape for it. + enqueued.add(Number(build.id)); + getLogger({ buildId: build.id }).info(`Push: redeploying reason=staticEnv`); + try { + await this.db.services.BuildService.enqueueResolveAndDeployBuild({ + buildId: build.id, + ...(headCommit ? { sourceRef: headCommit } : {}), + ...(beforeCommit ? { sourceBeforeRef: beforeCommit } : {}), + sourceGithubRepositoryId: githubRepositoryId, + sourceBranch: branchName, + }); + } catch (error) { + getLogger({ buildId: build.id }).error({ error }, 'Push: static env redeploy enqueue failed'); + } + } } catch (error) { getLogger({}).error( { error }, `Push: static env webhook failed branch=${branchName} repositoryId=${githubRepositoryId}` ); } + return enqueued; }; dispatchWebhook = async (req: NextApiRequest) => { diff --git a/src/server/services/ingress.ts b/src/server/services/ingress.ts index 02512298..6aeb5499 100644 --- a/src/server/services/ingress.ts +++ b/src/server/services/ingress.ts @@ -25,6 +25,7 @@ import yaml from 'js-yaml'; import { redisClient } from 'server/lib/dependencies'; import GlobalConfigService from './globalConfig'; import { buildLifecycleLabels } from 'server/lib/kubernetes/labels'; +import { BuildStatus } from 'shared/constants'; const MANIFEST_PATH = `${TMP_PATH}/ingress`; @@ -95,9 +96,17 @@ export default class IngressService extends BaseService { return withLogContext({ correlationId, buildUuid, sender, _ddTraceContext }, async () => { const isCurrent = async () => { if (!runUUID) return true; - let authority = this.db.models.Build.query().findOne({ id: buildId, runUUID }).whereNull('deletedAt'); - if (expectedGeneration != null) authority = authority.where('desiredGeneration', expectedGeneration); - return Boolean(await authority); + const build = await this.db.models.Build.query() + .findById(buildId) + .select('id', 'runUUID', 'status') + .whereNull('deletedAt'); + if (!build || build.status === BuildStatus.TEARING_DOWN || build.status === BuildStatus.TORN_DOWN) return false; + if (expectedGeneration == null) return build.runUUID === runUUID; + // Ingress is regenerated from every service's current row, so it stays valid while the run owns any of them. + const owned = await this.db.models.Deploy.query() + .findOne({ buildId, runUUID, desiredGeneration: expectedGeneration }) + .select('id'); + return Boolean(owned); }; if (!(await isCurrent())) { getLogger({ buildId }).info('Ingress: skipped reason=superseded'); @@ -105,39 +114,48 @@ export default class IngressService extends BaseService { } getLogger({ stage: LogStage.INGRESS_PROCESSING }).info('Ingress: creating'); - // We just want to create/update ingress for active services only - const configurations = await this.db.services.BuildService.configurationsForBuildId(buildId, false); - const namespace = await this.db.services.BuildService.getNamespace({ id: buildId }); - const { lifecycleDefaults, domainDefaults } = await GlobalConfigService.getInstance().getAllConfigs(); - const manifests = configurations.map((configuration) => { - return yaml.dump( - this.generateNginxManifestForConfiguration({ - configuration, - ingressClassName: lifecycleDefaults?.ingressClassName, - altHosts: domainDefaults?.altHttp || [], - }), - { - skipInvalid: true, - } - ); - }); - const apply = () => - Promise.all( + // Every service's current route is read under the promotion lock so a job that + // owns one service cannot publish a snapshot older than a sibling's rollout. + const apply = async () => { + const configurations = await this.db.services.BuildService.configurationsForBuildId(buildId, false); + const namespace = await this.db.services.BuildService.getNamespace({ id: buildId }); + const { lifecycleDefaults, domainDefaults } = await GlobalConfigService.getInstance().getAllConfigs(); + const manifests = configurations.map((configuration) => { + return yaml.dump( + this.generateNginxManifestForConfiguration({ + configuration, + ingressClassName: lifecycleDefaults?.ingressClassName, + altHosts: domainDefaults?.altHttp || [], + }), + { + skipInvalid: true, + } + ); + }); + const applied = await Promise.all( manifests.map((manifest, idx) => this.applyManifests(manifest, `${buildId}-${idx}-nginx`, namespace, buildId, runUUID, expectedGeneration) ) ); + return applied.every(Boolean); + }; + let applied = false; if (runUUID && this.db.services.BuildService.withCurrentBuildPromotionLock) { const result = await this.db.services.BuildService.withCurrentBuildPromotionLock(buildId, isCurrent, apply); if (!result.admitted) { getLogger({ buildId }).info('Ingress: skipped at promotion reason=superseded'); return; } + applied = result.value === true; } else { - await apply(); + applied = await apply(); } + // Only a run whose every manifest applied clears an earlier ingress failure note from the environment status. + if (applied && expectedGeneration != null) { + await this.db.services.BuildService.publishDerivedBuildStatus?.(buildId, { clearStatusMessage: true }); + } getLogger({ stage: LogStage.INGRESS_COMPLETE }).info('Ingress: created'); }); }; @@ -234,11 +252,13 @@ export default class IngressService extends BaseService { await shellPromise(`kubectl apply -f ${localPath} --namespace ${namespace}`, { timeout: 90_000, }); + return true; } catch (error) { getLogger({ stage: LogStage.INGRESS_FAILED }).warn({ error }, 'Ingress: manifest apply failed'); if (buildId !== undefined) { await this.recordIngressFailureOnBuild(buildId, error, runUUID, expectedGeneration).catch(() => undefined); } + return false; } }; @@ -252,17 +272,25 @@ export default class IngressService extends BaseService { const note = `Ingress apply failed: ${(error as Error)?.message || String(error)}` .replace(/\s+/g, ' ') .slice(0, 300); + if (runUUID && expectedGeneration != null) { + // A late failure from a run that no longer owns any service must not land on the current environment. + const owned = await this.db.models.Deploy.query() + .findOne({ buildId, runUUID, desiredGeneration: expectedGeneration }) + .select('id'); + if (!owned) return; + } let buildRead = this.db.models.Build.query().findById(buildId).whereNull('deletedAt'); - if (runUUID) buildRead = buildRead.where('runUUID', runUUID); - if (expectedGeneration != null) buildRead = buildRead.where('desiredGeneration', expectedGeneration); + if (runUUID && expectedGeneration == null) buildRead = buildRead.where('runUUID', runUUID); const build = await buildRead; if (!build || build.statusMessage?.includes(note)) { return; } const statusMessage = [build.statusMessage, note].filter(Boolean).join(' | ').slice(-500); - let buildPatch = this.db.models.Build.query().patch({ statusMessage }).where({ id: buildId }); - if (runUUID) buildPatch = buildPatch.where('runUUID', runUUID); - if (expectedGeneration != null) buildPatch = buildPatch.where('desiredGeneration', expectedGeneration); + let buildPatch = this.db.models.Build.query() + .patch({ statusMessage }) + .where({ id: buildId }) + .whereNull('deletedAt'); + if (runUUID && expectedGeneration == null) buildPatch = buildPatch.where('runUUID', runUUID); await buildPatch; }; } diff --git a/src/shared/config.ts b/src/shared/config.ts index 9fcf93b4..c3f06354 100644 --- a/src/shared/config.ts +++ b/src/shared/config.ts @@ -110,6 +110,7 @@ export const QUEUE_NAMES = { RESOLVE_AND_DEPLOY: `resolve_and_deploy_${JOB_VERSION}`, BUILD_QUEUE: `build_queue_${JOB_VERSION}`, DEPLOYMENT_RECONCILIATION: `deployment_reconciliation_${JOB_VERSION}`, + SERVICE_RECONCILIATION: `service_reconciliation_${JOB_VERSION}`, GITHUB_DEPLOYMENT: `github_deployment_${JOB_VERSION}`, DEPLOY_CLEANUP: `deploy_cleanup_${JOB_VERSION}`, LABEL: `label_${JOB_VERSION}`,