Deploy each service independently in static environments

src/server/db/migrations/037_add_deploy_reconciliation_generations.ts

@@ -0,0 +1,63 @@
+/**
+ * Copyright 2026 GoodRx, Inc.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import type { Knex } from 'knex';
+
+const PENDING_INDEX = 'deploys_reconcile_pending';
+
+// PostgreSQL concurrent index operations cannot run inside Knex's migration transaction.
+export const config = { transaction: false };
+
+// Runs outside a transaction, so every step is guarded to make a retry after a failed concurrent index build safe.
+export async function up(knex: Knex): Promise<void> {
+  for (const column of ['desiredGeneration', 'observedGeneration']) {
+    if (await knex.schema.hasColumn('deploys', column)) continue;
+    await knex.schema.alterTable('deploys', (table) => {
+      table.bigInteger(column).notNullable().defaultTo(0);
+    });
+  }
+
+  // Intents at or below the old watermark are finished; marking them keeps the sweep from replaying them.
+  await knex.raw(`
+    update builds
+       set "acceptedRefs" = (
+         select coalesce(jsonb_object_agg(
+                  entry.key,
+                  case
+                    when jsonb_typeof(entry.value -> 'gen') <> 'number' then entry.value
+                    when (entry.value ->> 'gen')::numeric <= builds."observedGeneration"
+                      then entry.value || jsonb_build_object('observedGen', entry.value -> 'gen')
+                    else entry.value
+                  end
+                ), '{}'::jsonb)
+           from jsonb_each("acceptedRefs") as entry
+       )
+     where "acceptedRefs" <> '{}'::jsonb
+       and "deletedAt" is null
+  `);
+
+  // A failed concurrent build leaves an invalid index with this name behind.
+  await knex.raw('drop index concurrently if exists ??', [PENDING_INDEX]);
+  await knex.raw(
+    `create index concurrently ?? on deploys ("buildId") where "desiredGeneration" > "observedGeneration"`,
+    [PENDING_INDEX]
+  );
+}
+
+export async function down(knex: Knex): Promise<void> {
+  await knex.raw('drop index concurrently if exists ??', [PENDING_INDEX]);
+
+  await knex.schema.alterTable('deploys', (table) => {
+    table.dropColumn('observedGeneration');
+    table.dropColumn('desiredGeneration');
+  });
+}

src/server/jobs/__tests__/apiEnvironmentJobWiring.test.ts

@@ -173,14 +173,16 @@ describe('bootstrapJobs', () => {
     const createWorker = registeredWorkers.find((worker) => worker.queue === QUEUE_NAMES.API_ENV_CREATE);
     const expiryWorker = registeredWorkers.find((worker) => worker.queue === QUEUE_NAMES.API_ENV_EXPIRY);
     const reconciliationWorker = registeredWorkers.find(
-      (worker) => worker.queue === QUEUE_NAMES.DEPLOYMENT_RECONCILIATION
+      (worker) => worker.queue === QUEUE_NAMES.SERVICE_RECONCILIATION
     );
     expect(createWorker).toMatchObject({
       handler: services.BuildService.processApiEnvironmentCreateQueue,
       options: { connection: { connection: 'redis' }, concurrency: 5 },
     });
     expect(expiryWorker?.handler).toBe(services.BuildService.processApiEnvironmentExpiryQueue);
     expect(reconciliationWorker?.handler).toBe(services.BuildService.processDeploymentReconciliationQueue);
+    // Old-queue jobs are drained by old workers only; leftovers are recovered by the sweep, never by a second model.
+    expect(registeredWorkers.some((worker) => worker.queue === QUEUE_NAMES.DEPLOYMENT_RECONCILIATION)).toBe(false);
     expect(workerOnHandlers[QUEUE_NAMES.API_ENV_CREATE]?.failed).toBe(
       services.BuildService.handleApiEnvironmentCreateFailure
     );

src/server/jobs/index.ts

@@ -166,7 +166,7 @@ export default function bootstrapJobs(services: IServices) {
   });
 
   queueManager.registerWorker(
-    QUEUE_NAMES.DEPLOYMENT_RECONCILIATION,
+    QUEUE_NAMES.SERVICE_RECONCILIATION,
     services.BuildService.processDeploymentReconciliationQueue,
     {
       connection: redisClient.getConnection(),