Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
110 changes: 110 additions & 0 deletions src/core/project/backends/cdk/toolkit.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -307,6 +307,116 @@ describe("Toolkit loading", () => {

expect(lines).toEqual(["CREATE_COMPLETE | AWS::IAM::Role"]);
});

test("reports a resource failure when Toolkit only throws NoStack after rollback", async () => {
const { loaded } = loadedToolkit();
const reason = "Agent with agentId: staging-agent not found";
const runner = createCdkRunner(createSilentLogger(), async (ioHost) => {
loaded.toolkit.deploy = async () => {
for (const event of [
{
ResourceType: "AWS::IAM::Role",
LogicalResourceId: "RuntimeRole",
ResourceStatus: "CREATE_FAILED",
ResourceStatusReason: "Resource creation cancelled",
},
{
ResourceType: "AWS::BedrockAgentCore::RuntimeEndpoint",
LogicalResourceId: "RuntimeEndpointProd",
ResourceStatus: "CREATE_FAILED",
ResourceStatusReason: reason,
},
{
ResourceType: "AWS::CloudFormation::Stack",
LogicalResourceId: "AgentCore-orders-staging",
ResourceStatus: "CREATE_FAILED",
ResourceStatusReason: "The following resource(s) failed",
},
]) {
await ioHost.notify({
...message("rollback progress"),
code: "CDK_TOOLKIT_I5502",
data: { event },
});
}
const error = new Error("CloudFormationStack object does not hold a stack");
error.name = "NoStack";
throw error;
};
return loaded;
});

await expect(
runner({ kind: "deploy", stackArtifactId: "AgentCore-orders-staging" }, runOptions()),
).rejects.toThrow(
`CloudFormationStack object does not hold a stack\n\n` +
`CloudFormation failure: RuntimeEndpointProd (CREATE_FAILED): ${reason}`,
);
});

test("reports a bootstrap resource failure when Toolkit throws NoStack", async () => {
const { loaded } = loadedToolkit();
const reason = "Bootstrap bucket could not be created";
const runner = createCdkRunner(createSilentLogger(), async (ioHost) => {
loaded.toolkit.bootstrap = async () => {
await ioHost.notify({
...message("bootstrap rollback progress"),
action: "bootstrap",
code: "CDK_TOOLKIT_I5502",
data: {
event: {
ResourceType: "AWS::S3::Bucket",
LogicalResourceId: "StagingBucket",
ResourceStatus: "CREATE_FAILED",
ResourceStatusReason: reason,
},
},
});
const error = new Error("CloudFormationStack object does not hold a stack");
error.name = "NoStack";
throw error;
};
return loaded;
});

await expect(
runner({ kind: "bootstrap", environments: ["aws://111122223333/us-east-1"] }, runOptions()),
).rejects.toThrow(
`CloudFormationStack object does not hold a stack\n\n` +
`CloudFormation failure: StagingBucket (CREATE_FAILED): ${reason}`,
);
});

test("does not repeat a resource failure already diagnosed by Toolkit", async () => {
const { loaded } = loadedToolkit();
const reason = `The resource failed: ${"validation error ".repeat(12)}`;
const original = new Error(
`The stack failed to deploy:\n ${reason.slice(0, 120)}\n ${reason.slice(120)}`,
);
const runner = createCdkRunner(createSilentLogger(), async (ioHost) => {
loaded.toolkit.deploy = async () => {
await ioHost.notify({
...message("update failed"),
code: "CDK_TOOLKIT_I5502",
data: {
event: {
ResourceType: "AWS::BedrockAgentCore::RuntimeEndpoint",
LogicalResourceId: "RuntimeEndpointProd",
ResourceStatus: "UPDATE_FAILED",
ResourceStatusReason: reason,
},
},
});
throw original;
};
return loaded;
});

await expect(
runner({ kind: "deploy", stackArtifactId: "AgentCore-orders-staging" }, runOptions()),
).rejects.toBe(original);
expect(original.message).not.toContain("CloudFormation failure:");
});
});

describe("bootstrap template loading", () => {
Expand Down
45 changes: 42 additions & 3 deletions src/core/project/backends/cdk/toolkit.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import { mkdtemp, rm, writeFile } from "node:fs/promises";
import { join } from "node:path";
import { tmpdir } from "node:os";
import type { StackEvent } from "@aws-sdk/client-cloudformation";
import type {
IActionAwareIoHost,
IIoHost,
Expand Down Expand Up @@ -120,7 +121,13 @@ export async function loadBootstrapTemplate(
};
}

export function createCdkIoHost(logger: Logger, onLine?: (line: string) => void): IIoHost {
type ResourceFailure = { logicalId: string; status: string; reason: string };

export function createCdkIoHost(
logger: Logger,
onLine?: (line: string) => void,
onResourceFailure?: (failure: ResourceFailure) => void,
): IIoHost {
const toolkitLogger = logger.child({ component: "cdk-toolkit" });
const notify = async (message: IoMessage<unknown>): Promise<void> => {
toolkitLogger
Expand All @@ -130,6 +137,26 @@ export function createCdkIoHost(logger: Logger, onLine?: (line: string) => void)
...(message.code && { code: message.code }),
})
.debug(message.message);
// I5502 carries each stack activity event. Toolkit 1.38.2 loses its resource
// diagnosis on first-create rollback when it reads a missing stack before
// formatting the collected failures; use these events for that NoStack case.
if (message.code === "CDK_TOOLKIT_I5502" && onResourceFailure) {

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can we add a why-comment here? Two things aren't obvious from the code:

  1. I5502 is Toolkit's per-event stack activity message.
  2. This whole path is a stopgap. In toolkit-lib 1.38.2, deploy-stack.js:411 reads finalState.wrapped before diagnoseFromErrorCollection, so a first create that rolls back throws NoStack and drops Toolkit's own resource diagnosis. The cancelled filter mirrors Toolkit's isCancellationEvent.

const event = (message.data as { event?: StackEvent } | undefined)?.event;
if (
event?.ResourceStatus?.endsWith("_FAILED") &&
event.ResourceType !== "AWS::CloudFormation::Stack" &&
event.LogicalResourceId &&
event.ResourceStatusReason &&
// Match Toolkit's cancellation filter so the initiating failure wins.
!event.ResourceStatusReason.includes("cancelled")
) {
onResourceFailure({
logicalId: event.LogicalResourceId,
status: event.ResourceStatus,
reason: event.ResourceStatusReason,
});
}
}
// Toolkit messages can span lines (diffs, stack traces); split so the sink
// always receives displayable single lines.
if (onLine) {
Expand Down Expand Up @@ -245,8 +272,20 @@ export function createCdkRunner(
return async (operation, options) => {
// A fresh ioHost per operation, so each operation's messages can flow into
// its own onOutput sink rather than only the construction-time logger.
const ioHost = createCdkIoHost(logger, options.onOutput);
let firstFailure: ResourceFailure | undefined;
const ioHost = createCdkIoHost(logger, options.onOutput, (failure) => {
firstFailure ??= failure;
});
const loaded = await load(ioHost, options.region, options.credentials);
return performCdkOperation(loaded, operation, options);
try {
return await performCdkOperation(loaded, operation, options);
} catch (error) {
if (error instanceof Error && error.name === "NoStack" && firstFailure) {
error.message +=
`\n\nCloudFormation failure: ${firstFailure.logicalId} ` +
`(${firstFailure.status}): ${firstFailure.reason}`;
}
throw error;
}
};
}
Loading