diff --git a/packages/@aws-cdk/toolkit-lib/lib/api/diagnosing/cloudtrail-investigation.ts b/packages/@aws-cdk/toolkit-lib/lib/api/diagnosing/cloudtrail-investigation.ts index e45941e2c..ef718e4f0 100644 --- a/packages/@aws-cdk/toolkit-lib/lib/api/diagnosing/cloudtrail-investigation.ts +++ b/packages/@aws-cdk/toolkit-lib/lib/api/diagnosing/cloudtrail-investigation.ts @@ -1,16 +1,10 @@ -import { - assumedRoleSessionName, - functionNameFromArnOrName, - identityMatches, - nameSegment, - serviceTokenReferencedLogicalId, -} from './resource-identifiers'; -import { getStackTemplate } from './stack-template'; +import { assumedRoleSessionName, identityMatches, nameSegment } from './resource-identifiers'; import type { AdditionalDiagnosticContext } from '../../actions/diagnose'; import type { ICloudFormationClient, ICloudTrailClient, SDK } from '../aws-auth/sdk'; import { isAccessDeniedError } from '../aws-auth/util'; import type { IoHelper } from '../io/private/io-helper'; -import type { ResourceError } from '../stack-events/resource-errors'; +import type { OperationPrincipal, ResourceError } from '../stack-events/resource-errors'; +import { PRINCIPAL_RESOURCE_TYPES } from '../stack-events/resource-errors'; /** * Stack-level CloudTrail investigation. @@ -26,10 +20,15 @@ import type { ResourceError } from '../stack-events/resource-errors'; * calls use unpredictable assumed-role session names, so there is no per-resource lookup key * we could compute. Instead we do one unfiltered sweep over the failure window and correlate * client-side: an event belongs to the stack when its *calling principal* carries a known - * stack identity (a physical ID, an IAM role referenced by resource properties, or a - * custom-resource backing function) as a whole ARN segment. Matching only the caller — never - * the resources a call touched — keeps unrelated principals' activity against stack resources - * out; a missed correlation degrades to "no events found", not a wrong attribution. + * stack identity (the physical ID of a role or Lambda function the stack operation involved) + * as a whole ARN segment. Matching only the caller — never the resources a + * call touched — keeps unrelated principals' activity against stack resources out; a missed + * correlation degrades to "no events found", not a wrong attribution. + * + * Roles the stack merely references (imported by ARN or name) are deliberately not + * identities: they may be shared with resources outside the stack, whose calls would then be + * attributed to it. Calls made with such a role still correlate when the caller itself is a + * stack resource (e.g. a stack Lambda, by its session name). */ /** @@ -159,6 +158,7 @@ export async function investigateStackViaCloudTrail( sdk: SDK, errors: readonly ResourceError[], ioHelper: IoHelper, + operationPrincipals: readonly OperationPrincipal[] = [], ): Promise { const debug = (msg: string) => ioHelper.defaults.debug(msg); @@ -178,7 +178,7 @@ export async function investigateStackViaCloudTrail( const cfn = sdk.cloudFormation(); const identities: StackIdentity[] = []; for (const stackArn of new Set(errors.map((e) => e.stackArn).filter((s) => !!s))) { - identities.push(...await buildStackIdentitySet(cfn, stackArn, debug)); + identities.push(...await buildStackIdentitySet(cfn, stackArn, operationPrincipals, debug)); } if (identities.length === 0) { await debug('CloudTrail investigation: no stack identities to correlate against'); @@ -201,8 +201,9 @@ export async function investigateStackViaCloudTrail( notes.push(`(${omitted} more distinct CloudTrail error event(s) not shown)`); } // Delivery latency applies whether or not something was already found: the fatal event may - // still be in flight while an earlier, benign error has already been delivered. - if (Date.now() - Math.max(...timestamps) < DELIVERY_LATENCY_MS) { + // still be in flight while an earlier, benign error has already been delivered. Once + // CloudTrail has delivered any event from after the window, assume the window is complete. + if (Date.now() - Math.max(...timestamps) < DELIVERY_LATENCY_MS && !await deliveredPast(sdk.cloudTrail(), windowEnd, ioHelper)) { notes.push( 'CloudTrail delivery can lag up to 15 minutes behind the failure; ' + 're-run `cdk diagnose` in a few minutes to check for additional control-plane errors (e.g. AccessDenied).', @@ -266,119 +267,67 @@ export function attributedCloudTrailContexts(investigation: CloudTrailInvestigat } /** - * Build the set of identities that tie CloudTrail events back to this stack: + * Build the set of identities that tie CloudTrail events back to this stack: the physical + * IDs of the roles and Lambda functions involved in the stack operation. + * + * - A role appears in the session ARN of whoever assumes it (a resource's own code, or a + * service acting on the resource's behalf with a generated session name). + * - A Lambda function is the session name, and `inScopeOf` resource, of calls its code + * makes; this also covers custom-resource handlers in the stack. * - * - physical IDs of all stack resources (a Lambda's role-session name is its function name; - * role physical IDs appear in assumed-role session ARNs); - * - IAM roles referenced by resource properties named `Role`/`RoleArn`/`ServiceToken` at any - * depth in the template (services acting on a resource's behalf assume these; the role - * *name* — not ARN — is embedded in the resulting session ARN, so bare names are indexed - * too). + * The set is the union of the roles and functions currently in the stack and those seen in + * the operation's stack events. The latter include resources the operation created and then + * rolled back, which no longer show up in the resource listing. Together they are an upper + * bound on the stack's own principals during the operation. * - * Best-effort: if the resource listing fails, template-derived identities (e.g. a literal - * ServiceToken ARN) are still collected. + * Roles the stack merely references (imported) never appear in either source, so they are + * not identities. Best-effort: if the resource listing fails, the event-derived principals + * are still used. */ async function buildStackIdentitySet( cfn: ICloudFormationClient, stackArn: string, + operationPrincipals: readonly OperationPrincipal[], debug: (msg: string) => Promise, ): Promise { const identities: StackIdentity[] = []; const seen = new Set(); - const add = (key: string | undefined, logicalId: string, description: string) => { + const add = (key: string | undefined, logicalId: string, resourceType: string) => { if (key && key.length >= MIN_IDENTITY_KEY_LENGTH && !seen.has(key)) { seen.add(key); - identities.push({ key, stackArn, logicalId, description }); + identities.push({ key, stackArn, logicalId, description: `${logicalId} (${resourceType})` }); } }; + const addPrincipal = (logicalId: string, resourceType: string, physicalId: string) => { + if (!PRINCIPAL_RESOURCE_TYPES.includes(resourceType)) { + return; + } + add(physicalId, logicalId, resourceType); + // ARN-shaped physical IDs also get their trailing name segment indexed: session ARNs + // embed names, not ARNs. + add(nameSegment(physicalId), logicalId, resourceType); + }; - const physicalIdByLogicalId = new Map(); try { const resp = await cfn.describeStackResources({ StackName: stackArn }); for (const r of resp.StackResources ?? []) { - if (r.LogicalResourceId && r.PhysicalResourceId) { - physicalIdByLogicalId.set(r.LogicalResourceId, r.PhysicalResourceId); - add(r.PhysicalResourceId, r.LogicalResourceId, `${r.LogicalResourceId} (${r.ResourceType})`); - // ARN-shaped physical IDs also get their trailing name segment indexed: session ARNs - // embed names, not ARNs. - add(nameSegment(r.PhysicalResourceId), r.LogicalResourceId, `${r.LogicalResourceId} (${r.ResourceType})`); + if (r.LogicalResourceId && r.PhysicalResourceId && r.ResourceType) { + addPrincipal(r.LogicalResourceId, r.ResourceType, r.PhysicalResourceId); } } } catch (e: any) { - // Not fatal: template-derived identities below don't need the physical IDs (literal - // ARNs/names resolve on their own; only Ref/GetAtt references do). await debug(`CloudTrail investigation: failed to list stack resources: ${e.message}`); } - const template = await getStackTemplate(cfn, stackArn, debug); - for (const [logicalId, resource] of Object.entries(template?.Resources ?? {})) { - for (const { propName, value } of collectPrincipalProperties(resource?.Properties)) { - const resolved = resolveToIdentity(value, physicalIdByLogicalId); - if (!resolved) { - continue; - } - const description = `${logicalId} (via ${propName})`; - if (propName === 'ServiceToken') { - // The backing function's role-session name is the function name. - add(functionNameFromArnOrName(resolved), logicalId, description); - } else { - add(resolved, logicalId, description); - add(nameSegment(resolved), logicalId, description); - } + for (const p of operationPrincipals) { + if (p.stackArn === stackArn) { + addPrincipal(p.logicalId, p.resourceType, p.physicalId); } } return identities; } -/** - * Collect properties that reference a calling principal — `Role`/`RoleArn` (any depth, since - * many resources nest role references, e.g. Firehose destination configurations or - * EventBridge targets) and the top-level `ServiceToken` of custom resources. - */ -function collectPrincipalProperties(props: any, depth = 0): Array<{ propName: string; value: any }> { - if (!props || typeof props !== 'object' || depth > 10) { - return []; - } - const ret: Array<{ propName: string; value: any }> = []; - for (const [key, value] of Object.entries(props)) { - if (/Role(Arn)?$/i.test(key) || key === 'ServiceToken') { - ret.push({ propName: key, value }); - } - ret.push(...collectPrincipalProperties(value, depth + 1)); - } - return ret; -} - -/** - * Resolve a template property value to a matchable identity string: literals pass through, - * `Ref`/`Fn::GetAtt` resolve via the resource's physical ID, and an `Fn::Sub` role ARN - * yields its role name when that segment is literal. Unresolvable expressions (e.g. - * `Fn::ImportValue`) return `undefined` — a missed identity degrades to "no events found". - */ -function resolveToIdentity(value: any, physicalIdByLogicalId: Map): string | undefined { - if (typeof value === 'string') { - return value; - } - const referenced = serviceTokenReferencedLogicalId(value); - if (referenced) { - return physicalIdByLogicalId.get(referenced); - } - const sub = value?.['Fn::Sub']; - if (typeof sub === 'string') { - if (!sub.includes('${')) { - return sub; - } - // Common shape: 'arn:aws:iam::${AWS::AccountId}:role/' — the name segment - // is all we need for matching, and it is often free of substitutions. - const name = nameSegment(sub); - if (name && !name.includes('${')) { - return name; - } - } - return undefined; -} - interface ParsedCloudTrailEvent { readonly eventTime?: Date; readonly eventSource?: string; @@ -392,6 +341,12 @@ interface ParsedCloudTrailEvent { * errored calls against stack resources to the stack. */ readonly callerIdentityStrings: string[]; + + /** + * ARN of the resource the caller's credentials were issued to + * (`userIdentity.inScopeOf.credentialsIssuedTo`, e.g. the Lambda function that made the call). + */ + readonly credentialsIssuedTo?: string; } interface SweepResult { @@ -463,6 +418,23 @@ async function sweepCloudTrail( return { events, eventsChecked, truncated }; } +/** + * Whether CloudTrail has delivered any event that happened at or after `time`. + * + * CloudTrail does not deliver events in strict `eventTime` order, so this is a heuristic for + * "events up to `time` have been delivered", not a guarantee. A failed probe answers `false`: + * the hint to re-run is the safe default. + */ +async function deliveredPast(cloudTrail: ICloudTrailClient, time: Date, ioHelper: IoHelper): Promise { + try { + const resp = await cloudTrail.lookupEvents({ StartTime: time, EndTime: new Date(), MaxResults: 1 }); + return (resp.Events ?? []).length > 0; + } catch (e: any) { + await ioHelper.defaults.debug(`CloudTrail investigation: delivery probe failed: ${e.message}`); + return false; + } +} + /** * Attribute errored events to stack resources. * @@ -504,10 +476,12 @@ function mostSpecificMatch(event: ParsedCloudTrailEvent, identities: StackIdenti .filter((s): s is string => !!s), ); + const callers = event.credentialsIssuedTo ? [event.credentialsIssuedTo] : event.callerIdentityStrings; + let best: StackIdentity | undefined; let bestRank = -1; for (const identity of identities) { - if (!event.callerIdentityStrings.some((s) => identityMatches(s, identity.key))) { + if (!callers.some((s) => identityMatches(s, identity.key))) { continue; } // Two-tier rank: session-name matches sort above everything else; key length breaks ties. @@ -579,8 +553,8 @@ function parseCloudTrailEvent(json: string | undefined): ParsedCloudTrailEvent | const callerIdentityStrings = [ obj.userIdentity?.arn, obj.userIdentity?.sessionContext?.sessionIssuer?.arn, - obj.userIdentity?.inScopeOf?.credentialsIssuedTo, ].filter((s: any): s is string => typeof s === 'string'); + const credentialsIssuedTo = obj.userIdentity?.inScopeOf?.credentialsIssuedTo; return { eventTime: typeof obj.eventTime === 'string' ? new Date(obj.eventTime) : undefined, eventSource: typeof obj.eventSource === 'string' ? obj.eventSource : undefined, @@ -588,6 +562,7 @@ function parseCloudTrailEvent(json: string | undefined): ParsedCloudTrailEvent | errorCode: typeof obj.errorCode === 'string' ? obj.errorCode : undefined, errorMessage: typeof obj.errorMessage === 'string' ? obj.errorMessage : undefined, callerIdentityStrings, + credentialsIssuedTo: typeof credentialsIssuedTo === 'string' ? credentialsIssuedTo : undefined, }; } catch { return undefined; diff --git a/packages/@aws-cdk/toolkit-lib/lib/api/diagnosing/stack-diagnoser.ts b/packages/@aws-cdk/toolkit-lib/lib/api/diagnosing/stack-diagnoser.ts index e8a4ae3d1..b56b88e5f 100644 --- a/packages/@aws-cdk/toolkit-lib/lib/api/diagnosing/stack-diagnoser.ts +++ b/packages/@aws-cdk/toolkit-lib/lib/api/diagnosing/stack-diagnoser.ts @@ -172,7 +172,7 @@ export class CloudFormationStackDiagnoser { stackStatus: stack.StackStatus ?? '', statusReason: stack.StackStatusReason ?? '', }, - await this.addCloudTrailContext(await this.enhanceErrors(errors.all), errors.all), + await this.addCloudTrailContext(await this.enhanceErrors(errors.all), errors), ); } @@ -187,7 +187,7 @@ export class CloudFormationStackDiagnoser { * level. When run right after a deployment, the events may not be delivered yet, in which * case the output notifies the user. */ - private async addCloudTrailContext(problems: TracedResourceError[], errs: readonly ResourceError[]): Promise { + private async addCloudTrailContext(problems: TracedResourceError[], errors: ResourceErrors): Promise { const sdk = await this.additionalExplorationSdk(); if (!sdk || problems.length === 0) { return problems; @@ -195,7 +195,7 @@ export class CloudFormationStackDiagnoser { let investigation; try { - investigation = await investigateStackViaCloudTrail(sdk, errs, this.props.ioHelper); + investigation = await investigateStackViaCloudTrail(sdk, errors.all, this.props.ioHelper, errors.principals); } catch (e: any) { await this.props.ioHelper.defaults.debug(`CloudTrail investigation failed: ${e.message}`); return problems; diff --git a/packages/@aws-cdk/toolkit-lib/lib/api/stack-events/resource-errors.ts b/packages/@aws-cdk/toolkit-lib/lib/api/stack-events/resource-errors.ts index b7a807ff3..3ff879454 100644 --- a/packages/@aws-cdk/toolkit-lib/lib/api/stack-events/resource-errors.ts +++ b/packages/@aws-cdk/toolkit-lib/lib/api/stack-events/resource-errors.ts @@ -101,6 +101,38 @@ export interface ResourceError { readonly warnings?: string[]; } +/** + * A stack resource that can act as an AWS principal, seen during a stack operation. + */ +export interface OperationPrincipal { + /** + * The stack the resource belongs to (a stack ID, not a name) + */ + readonly stackArn: string; + + /** + * Logical ID of the resource + */ + readonly logicalId: string; + + /** + * Resource type + */ + readonly resourceType: string; + + /** + * Physical ID of the resource (a role or function name) + */ + readonly physicalId: string; +} + +/** + * Resource types whose physical IDs identify the caller of an AWS API call in CloudTrail: + * a role appears in the session ARN of whoever assumes it, and a Lambda function is the + * session name (and `inScopeOf` resource) of calls made by its code. + */ +export const PRINCIPAL_RESOURCE_TYPES = ['AWS::IAM::Role', 'AWS::Lambda::Function']; + /** * Class used to send stack event errors into, to come up with root causes. */ @@ -122,6 +154,12 @@ export class ResourceErrors { */ private readonly seenHookFailures = new Map(); + /** + * Principal-capable resources seen in any event of the operation, keyed by stack ARN + + * logical ID + physical ID. + */ + private readonly seenPrincipals = new Map(); + public isEmpty() { return this._errors.length === 0; } @@ -130,6 +168,15 @@ export class ResourceErrors { return this._errors; } + /** + * Every role and Lambda function that appeared in an event of the operation, including + * ones that were created and have since been rolled back (and so no longer appear in the + * stack's resource list), and ones replaced by the update. + */ + public get principals(): ReadonlyArray { + return [...this.seenPrincipals.values()]; + } + /** * Update the error collection with the given stack activity * @@ -140,6 +187,7 @@ export class ResourceErrors { public update(...events: ResourceEvent[]) { for (const event of events) { this.trackHookFailure(event.event); + this.trackPrincipal(event.event); if (isErrorEvent(event.event)) { // Cancelled is not an interesting failure reason, nor is the stack message (stack // message will just say something like "stack failed to update") @@ -150,6 +198,14 @@ export class ResourceErrors { } } + private trackPrincipal(event: StackEvent) { + const { StackId: stackArn, LogicalResourceId: logicalId, ResourceType: resourceType, PhysicalResourceId: physicalId } = event; + if (!stackArn || !logicalId || !physicalId || !resourceType || !PRINCIPAL_RESOURCE_TYPES.includes(resourceType)) { + return; + } + this.seenPrincipals.set(`${stackArn}|${logicalId}|${physicalId}`, { stackArn, logicalId, resourceType, physicalId }); + } + private trackHookFailure(event: StackEvent) { if (!isFailedHookStatus(event.HookStatus) || !event.HookType || !event.LogicalResourceId || !event.StackId) { return; diff --git a/packages/@aws-cdk/toolkit-lib/test/api/diagnosing/cloudtrail-investigation.test.ts b/packages/@aws-cdk/toolkit-lib/test/api/diagnosing/cloudtrail-investigation.test.ts index 4b96ed2a6..bc52c93cc 100644 --- a/packages/@aws-cdk/toolkit-lib/test/api/diagnosing/cloudtrail-investigation.test.ts +++ b/packages/@aws-cdk/toolkit-lib/test/api/diagnosing/cloudtrail-investigation.test.ts @@ -1,8 +1,8 @@ import type { StackResource } from '@aws-sdk/client-cloudformation'; -import { DescribeStackResourcesCommand, GetTemplateCommand, ResourceStatus } from '@aws-sdk/client-cloudformation'; +import { DescribeStackResourcesCommand, ResourceStatus } from '@aws-sdk/client-cloudformation'; import { LookupEventsCommand } from '@aws-sdk/client-cloudtrail'; import { investigateStackViaCloudTrail } from '../../../lib/api/diagnosing/cloudtrail-investigation'; -import type { ResourceError } from '../../../lib/api/stack-events/resource-errors'; +import type { OperationPrincipal, ResourceError } from '../../../lib/api/stack-events/resource-errors'; import { mockCloudFormationClient, mockCloudTrailClient, @@ -42,18 +42,6 @@ beforeEach(() => { resource('MyCustomResource', 'Custom::MyThing', '2026/06/19/[$LATEST]abc123'), ], }); - mockCloudFormationClient.on(GetTemplateCommand).resolves({ - TemplateBody: JSON.stringify({ - Resources: { - CrHandler: { Type: 'AWS::Lambda::Function', Properties: { Role: { 'Fn::GetAtt': ['CrHandlerRole', 'Arn'] } } }, - VpcFn: { Type: 'AWS::Lambda::Function', Properties: { Role: { 'Fn::GetAtt': ['VpcFnRole', 'Arn'] } } }, - MyCustomResource: { - Type: 'Custom::MyThing', - Properties: { ServiceToken: { 'Fn::GetAtt': ['CrHandler', 'Arn'] } }, - }, - }, - }), - }); }); afterEach(() => { @@ -139,12 +127,13 @@ function unrelatedEvent() { } describe('correlation', () => { - test('attributes a handler AccessDenied to the custom resource via the role-session ARN', async () => { + test('attributes a custom-resource handler\'s AccessDenied to the handler function', async () => { mockCloudTrailClient.on(LookupEventsCommand).resolves({ Events: [handlerEvent(), unrelatedEvent()] }); const result = await investigateStackViaCloudTrail(sdk, [resourceError()], ioHost.asHelper('diagnose')); expect(result!.errors).toHaveLength(1); + expect(result!.errors[0].logicalId).toEqual('CrHandler'); expect(result!.errors[0].message).toMatch(/AccessDenied on s3\.amazonaws\.com:CreateBucket/); expect(result!.errors[0].message).toMatch(/not authorized to perform: s3:CreateBucket/); }); @@ -173,6 +162,66 @@ describe('correlation', () => { expect(result!.errors).toHaveLength(0); }); + test.each([ + ['imported by the stack', 'shared-role'], + ['created by the stack', CR_ROLE_NAME], + ])('excludes calls by a Lambda outside the stack that shares a role %s', async (_, roleName) => { + // Issue #2084: another stack's function runs with a role this stack also uses. + // `inScopeOf` names that function as the caller, so the shared role must not claim it. + mockCloudTrailClient.on(LookupEventsCommand).resolves({ + Events: [{ + CloudTrailEvent: JSON.stringify({ + eventTime: '2026-06-19T11:59:00Z', + eventSource: 'dynamodb.amazonaws.com', + eventName: 'DescribeTable', + errorCode: 'AccessDenied', + userIdentity: { + arn: `arn:aws:sts::${ACCOUNT}:assumed-role/${roleName}/other-stack-fn`, + sessionContext: { sessionIssuer: { arn: `arn:aws:iam::${ACCOUNT}:role/${roleName}` } }, + inScopeOf: { credentialsIssuedTo: `arn:aws:lambda:us-east-1:${ACCOUNT}:function:other-stack-fn` }, + }, + }), + }], + }); + + const result = await investigateStackViaCloudTrail(sdk, [resourceError()], ioHost.asHelper('diagnose')); + + expect(result!.errors).toHaveLength(0); + }); + + test('attributes calls by a stack Lambda running with an imported role', async () => { + mockCloudTrailClient.on(LookupEventsCommand).resolves({ + Events: [handlerEvent({ + userIdentity: { + arn: `arn:aws:sts::${ACCOUNT}:assumed-role/shared-role/${CR_FUNCTION_NAME}`, + sessionContext: { sessionIssuer: { arn: `arn:aws:iam::${ACCOUNT}:role/shared-role` } }, + inScopeOf: { credentialsIssuedTo: `arn:aws:lambda:us-east-1:${ACCOUNT}:function:${CR_FUNCTION_NAME}` }, + }, + })], + }); + + const result = await investigateStackViaCloudTrail(sdk, [resourceError()], ioHost.asHelper('diagnose')); + + expect(result!.errors).toHaveLength(1); + expect(result!.errors[0].logicalId).toEqual('CrHandler'); + }); + + test('attributes calls by a stack Lambda on an imported role via its session name, without inScopeOf', async () => { + mockCloudTrailClient.on(LookupEventsCommand).resolves({ + Events: [handlerEvent({ + userIdentity: { + arn: `arn:aws:sts::${ACCOUNT}:assumed-role/shared-role/${CR_FUNCTION_NAME}`, + sessionContext: { sessionIssuer: { arn: `arn:aws:iam::${ACCOUNT}:role/shared-role` } }, + }, + })], + }); + + const result = await investigateStackViaCloudTrail(sdk, [resourceError()], ioHost.asHelper('diagnose')); + + expect(result!.errors).toHaveLength(1); + expect(result!.errors[0].logicalId).toEqual('CrHandler'); + }); + test('excludes errored calls made by the deploying principal (already in StatusReason)', async () => { // CloudFormation's own denied calls carry no stack identity in userIdentity (and often an // empty resources list); their message is already surfaced via the resource StatusReason. @@ -264,25 +313,9 @@ describe('correlation', () => { expect(result!.errors).toHaveLength(0); }); - test('correlates roles referenced by nested template properties', async () => { - // Role references are often nested (Firehose destination configs, EventBridge targets); - // the identity scan must find them at any depth. The role here is NOT a stack resource - // (imported by ARN), so only the template scan can index it. - mockCloudFormationClient.on(GetTemplateCommand).resolves({ - TemplateBody: JSON.stringify({ - Resources: { - DeliveryStream: { - Type: 'AWS::KinesisFirehose::DeliveryStream', - Properties: { - S3DestinationConfiguration: { - BucketARN: 'arn:aws:s3:::some-bucket', - RoleARN: `arn:aws:iam::${ACCOUNT}:role/imported-firehose-role`, - }, - }, - }, - }, - }), - }); + test('does not correlate via a role that is not a stack resource', async () => { + // An imported role may be shared with resources outside the stack (issue #2084), so a + // service session on it cannot be told apart from another stack's use of the same role. mockCloudTrailClient.on(LookupEventsCommand).resolves({ Events: [{ CloudTrailEvent: JSON.stringify({ @@ -300,31 +333,28 @@ describe('correlation', () => { const result = await investigateStackViaCloudTrail(sdk, [resourceError()], ioHost.asHelper('diagnose')); - expect(result!.errors).toHaveLength(1); - expect(result!.errors[0].logicalId).toEqual('DeliveryStream'); + expect(result!.errors).toHaveLength(0); }); - test('correlates a role expressed as Fn::Sub with a literal name segment', async () => { - mockCloudFormationClient.on(GetTemplateCommand).resolves({ - TemplateBody: JSON.stringify({ - Resources: { - Project: { - Type: 'AWS::CodeBuild::Project', - Properties: { ServiceRole: { 'Fn::Sub': 'arn:aws:iam::${AWS::AccountId}:role/my-codebuild-role' } }, - }, - }, - }), + test('does not use physical IDs of non-principal resources as identities', async () => { + // A short physical ID (e.g. an API Gateway stage named `prod`) can equal a segment of an + // unrelated caller's ARN; only roles and functions identify callers. + mockCloudFormationClient.on(DescribeStackResourcesCommand).resolves({ + StackResources: [ + resource('CrHandler', 'AWS::Lambda::Function', CR_FUNCTION_NAME), + resource('Stage', 'AWS::ApiGateway::Stage', 'prod'), + ], }); mockCloudTrailClient.on(LookupEventsCommand).resolves({ Events: [{ CloudTrailEvent: JSON.stringify({ eventTime: '2026-06-19T11:59:00Z', - eventSource: 'ec2.amazonaws.com', - eventName: 'CreateNetworkInterface', - errorCode: 'Client.UnauthorizedOperation', + eventSource: 's3.amazonaws.com', + eventName: 'GetObject', + errorCode: 'AccessDenied', userIdentity: { - arn: `arn:aws:sts::${ACCOUNT}:assumed-role/my-codebuild-role/codebuild-session`, - sessionContext: { sessionIssuer: { arn: `arn:aws:iam::${ACCOUNT}:role/my-codebuild-role` } }, + arn: `arn:aws:sts::${ACCOUNT}:assumed-role/other-role/prod`, + sessionContext: { sessionIssuer: { arn: `arn:aws:iam::${ACCOUNT}:role/other-role` } }, }, }), }], @@ -332,32 +362,57 @@ describe('correlation', () => { const result = await investigateStackViaCloudTrail(sdk, [resourceError()], ioHost.asHelper('diagnose')); + expect(result!.errors).toHaveLength(0); + }); + + test('correlates via a role the operation created and then rolled back', async () => { + // The role no longer appears in DescribeStackResources, but the stack events recorded it. + const rolledBackRole = 'MyStack-NewFnRole-QWERTY123456'; + const principals: OperationPrincipal[] = [ + { stackArn: STACK_ARN, logicalId: 'NewFnRole', resourceType: 'AWS::IAM::Role', physicalId: rolledBackRole }, + ]; + mockCloudTrailClient.on(LookupEventsCommand).resolves({ + Events: [serviceOnBehalfEvent({ + userIdentity: { + arn: `arn:aws:sts::${ACCOUNT}:assumed-role/${rolledBackRole}/awslambda_320_20260619115800000`, + sessionContext: { sessionIssuer: { arn: `arn:aws:iam::${ACCOUNT}:role/${rolledBackRole}` } }, + }, + })], + }); + + const result = await investigateStackViaCloudTrail(sdk, [resourceError()], ioHost.asHelper('diagnose'), principals); + expect(result!.errors).toHaveLength(1); - expect(result!.errors[0].logicalId).toEqual('Project'); + expect(result!.errors[0].logicalId).toEqual('NewFnRole'); }); - test('still collects template-derived identities when DescribeStackResources fails', async () => { - // A denied/throttled resource listing must not abort the investigation: a literal - // ServiceToken ARN in the template is resolvable without physical IDs. + test('ignores operation principals of other stacks', async () => { + const principals: OperationPrincipal[] = [{ + stackArn: `arn:aws:cloudformation:us-east-1:${ACCOUNT}:stack/OtherStack/def`, + logicalId: 'OtherRole', + resourceType: 'AWS::IAM::Role', + physicalId: 'other-stack-OtherRole-yyy', + }]; + mockCloudTrailClient.on(LookupEventsCommand).resolves({ Events: [unrelatedEvent()] }); + + const result = await investigateStackViaCloudTrail(sdk, [resourceError()], ioHost.asHelper('diagnose'), principals); + + expect(result!.errors).toHaveLength(0); + }); + + test('still uses the operation\'s principals when DescribeStackResources fails', async () => { const denied = new Error('not authorized'); denied.name = 'AccessDeniedException'; mockCloudFormationClient.on(DescribeStackResourcesCommand).rejects(denied); - mockCloudFormationClient.on(GetTemplateCommand).resolves({ - TemplateBody: JSON.stringify({ - Resources: { - MyCustomResource: { - Type: 'Custom::MyThing', - Properties: { ServiceToken: `arn:aws:lambda:us-east-1:${ACCOUNT}:function:${CR_FUNCTION_NAME}` }, - }, - }, - }), - }); + const principals: OperationPrincipal[] = [ + { stackArn: STACK_ARN, logicalId: 'CrHandler', resourceType: 'AWS::Lambda::Function', physicalId: CR_FUNCTION_NAME }, + ]; mockCloudTrailClient.on(LookupEventsCommand).resolves({ Events: [handlerEvent()] }); - const result = await investigateStackViaCloudTrail(sdk, [resourceError()], ioHost.asHelper('diagnose')); + const result = await investigateStackViaCloudTrail(sdk, [resourceError()], ioHost.asHelper('diagnose'), principals); expect(result!.errors).toHaveLength(1); - expect(result!.errors[0].logicalId).toEqual('MyCustomResource'); + expect(result!.errors[0].logicalId).toEqual('CrHandler'); }); }); @@ -496,10 +551,37 @@ describe('sweep mechanics', () => { // must not be suppressed by partial delivery. jest.setSystemTime(FAILURE_TIME.valueOf() + 3 * 60 * 1000); mockCloudTrailClient.on(LookupEventsCommand).resolves({ Events: [handlerEvent()] }); + mockCloudTrailClient.on(LookupEventsCommand, { MaxResults: 1 }).resolves({ Events: [] }); + + const result = await investigateStackViaCloudTrail(sdk, [resourceError()], ioHost.asHelper('diagnose')); + + expect(result!.errors).toHaveLength(1); + expect(result!.notes.join('\n')).toMatch(/can lag up to 15 minutes/); + }); + + test('drops the delivery-latency hint once CloudTrail has delivered events from after the window', async () => { + jest.setSystemTime(FAILURE_TIME.valueOf() + 10 * 60 * 1000); + mockCloudTrailClient.on(LookupEventsCommand).resolves({ Events: [handlerEvent()] }); + mockCloudTrailClient.on(LookupEventsCommand, { MaxResults: 1 }).resolves({ Events: [unrelatedEvent()] }); const result = await investigateStackViaCloudTrail(sdk, [resourceError()], ioHost.asHelper('diagnose')); expect(result!.errors).toHaveLength(1); + expect(result!.notes).toHaveLength(0); + // The probe starts where the sweep window ends (2 minutes after the last failure). + expect(mockCloudTrailClient).toHaveReceivedCommandWith(LookupEventsCommand, { + StartTime: new Date(FAILURE_TIME.valueOf() + 2 * 60 * 1000), + MaxResults: 1, + }); + }); + + test('keeps the delivery-latency hint when the delivery probe fails', async () => { + jest.setSystemTime(FAILURE_TIME.valueOf() + 3 * 60 * 1000); + mockCloudTrailClient.on(LookupEventsCommand).resolves({ Events: [] }); + mockCloudTrailClient.on(LookupEventsCommand, { MaxResults: 1 }).rejects(new Error('ThrottlingException: rate exceeded')); + + const result = await investigateStackViaCloudTrail(sdk, [resourceError()], ioHost.asHelper('diagnose')); + expect(result!.notes.join('\n')).toMatch(/can lag up to 15 minutes/); }); diff --git a/packages/@aws-cdk/toolkit-lib/test/api/diagnosing/stack-diagnoser.test.ts b/packages/@aws-cdk/toolkit-lib/test/api/diagnosing/stack-diagnoser.test.ts index b3bd1b98a..6be889eeb 100644 --- a/packages/@aws-cdk/toolkit-lib/test/api/diagnosing/stack-diagnoser.test.ts +++ b/packages/@aws-cdk/toolkit-lib/test/api/diagnosing/stack-diagnoser.test.ts @@ -1036,6 +1036,51 @@ describe('CloudFormationStackDiagnoser', () => { expect(context!.messages[0]).toMatch(/AccessDenied on s3\.amazonaws\.com:CreateBucket/); }); + test('correlates via roles seen in the operation\'s stack events, even if no longer in the stack', async () => { + fakeCfn.createStackSync({ StackName: 'MyStack', StackStatus: 'UPDATE_ROLLBACK_COMPLETE', StackId: STACK_ARN }); + // The role was rolled back: the stack's resource list no longer has it. + mockCloudFormationClient.on(DescribeStackResourcesCommand).resolves({ StackResources: [] }); + mockCloudTrailClient.on(LookupEventsCommand).resolves({ + Events: [{ + CloudTrailEvent: JSON.stringify({ + eventTime: '2026-06-19T11:59:00Z', + eventSource: 'ec2.amazonaws.com', + eventName: 'CreateNetworkInterface', + errorCode: 'Client.UnauthorizedOperation', + userIdentity: { + arn: 'arn:aws:sts::123456789012:assumed-role/MyStack-NewRole-abc123def456/awslambda_1_2', + sessionContext: { sessionIssuer: { arn: 'arn:aws:iam::123456789012:role/MyStack-NewRole-abc123def456' } }, + }, + }), + }], + }); + const errors = makeErrors(); + errors.update({ + event: { + EventId: 'evt-0', + StackId: STACK_ARN, + StackName: 'MyStack', + LogicalResourceId: 'NewRole', + PhysicalResourceId: 'MyStack-NewRole-abc123def456', + ResourceType: 'AWS::IAM::Role', + ResourceStatus: 'CREATE_COMPLETE', + Timestamp: FAILURE_TIME, + }, + parentStackLogicalIds: [], + isRootStackEvent: false, + }); + + const result = await makeExploringDiagnoser().diagnoseFromErrorCollection(errors, { + StackName: 'MyStack', + StackStatus: 'UPDATE_ROLLBACK_COMPLETE', + CreationTime: new Date(), + }); + + const problem = assertProblem(result); + const context = problem.problems[0].additionalContext?.find((c) => c.source === 'CloudTrail Errors'); + expect(context!.messages[0]).toMatch(/^NewRole \(AWS::IAM::Role\): Client\.UnauthorizedOperation/); + }); + test('does not run the investigation without an exploration SDK', async () => { const result = await makeDiagnoser().diagnoseFromErrorCollection(makeErrors(), { StackName: 'MyStack', diff --git a/packages/@aws-cdk/toolkit-lib/test/api/stack-events/resource-errors.test.ts b/packages/@aws-cdk/toolkit-lib/test/api/stack-events/resource-errors.test.ts index df33295b0..36514192d 100644 --- a/packages/@aws-cdk/toolkit-lib/test/api/stack-events/resource-errors.test.ts +++ b/packages/@aws-cdk/toolkit-lib/test/api/stack-events/resource-errors.test.ts @@ -131,3 +131,40 @@ describe('hook failure correlation', () => { expect(errors.all[0].hookFailures).toBeUndefined(); }); }); + +describe('operation principals', () => { + test('records roles and functions seen in any event, including ones later rolled back', () => { + const errors = new ResourceErrors(); + errors.update( + // No physical ID yet on the first CREATE_IN_PROGRESS event + resourceEvent({ EventId: '1', LogicalResourceId: 'NewRole', ResourceType: 'AWS::IAM::Role', ResourceStatus: 'CREATE_IN_PROGRESS' }), + resourceEvent({ EventId: '2', LogicalResourceId: 'NewRole', ResourceType: 'AWS::IAM::Role', PhysicalResourceId: 'MyStack-NewRole-abc', ResourceStatus: 'CREATE_COMPLETE' }), + resourceEvent({ EventId: '3', LogicalResourceId: 'Fn', ResourceType: 'AWS::Lambda::Function', PhysicalResourceId: 'MyStack-Fn-def', ResourceStatus: 'CREATE_FAILED' }), + resourceEvent({ EventId: '4', LogicalResourceId: 'NewRole', ResourceType: 'AWS::IAM::Role', PhysicalResourceId: 'MyStack-NewRole-abc', ResourceStatus: 'DELETE_COMPLETE' }), + ); + + expect(errors.principals).toEqual([ + { stackArn: STACK_ARN, logicalId: 'NewRole', resourceType: 'AWS::IAM::Role', physicalId: 'MyStack-NewRole-abc' }, + { stackArn: STACK_ARN, logicalId: 'Fn', resourceType: 'AWS::Lambda::Function', physicalId: 'MyStack-Fn-def' }, + ]); + }); + + test('records both the old and the new physical ID of a replaced role', () => { + const errors = new ResourceErrors(); + errors.update( + resourceEvent({ EventId: '1', LogicalResourceId: 'Role', ResourceType: 'AWS::IAM::Role', PhysicalResourceId: 'old-role', ResourceStatus: 'UPDATE_IN_PROGRESS' }), + resourceEvent({ EventId: '2', LogicalResourceId: 'Role', ResourceType: 'AWS::IAM::Role', PhysicalResourceId: 'new-role', ResourceStatus: 'UPDATE_COMPLETE' }), + ); + + expect(errors.principals.map((p) => p.physicalId)).toEqual(['old-role', 'new-role']); + }); + + test('ignores resources that do not act as principals', () => { + const errors = new ResourceErrors(); + errors.update( + resourceEvent({ EventId: '1', LogicalResourceId: 'Bucket', ResourceType: 'AWS::S3::Bucket', PhysicalResourceId: 'my-bucket', ResourceStatus: 'CREATE_COMPLETE' }), + ); + + expect(errors.principals).toEqual([]); + }); +});