I am getting the following error while deploying Managed Workflow using Apache Airflow using Cloudformation. The error doesnt give out much information and hence difficult to debug
2022-09-16 15:25:42 UTC+0530 MwaaEnvironment CREATE_FAILED Resource handler returned message: "null (Service: Mwaa, Status Code: 403, Request ID: de70d88f-383a-43bb-91a7-4ebdee31223e)" (RequestToken: fe034e51-e5e2-73e9-3616-0bd928d64a1f, HandlerErrorCode: AccessDenied)
It shows access denied but not much information about which resource
Snippet of Cloudformation yaml for the mwaa execution policy
#####################################################################################################################
# CREATE MWAA
#####################################################################################################################
MwaaEnvironment:
Type: AWS::MWAA::Environment
DependsOn: MwaaExecutionPolicy
Properties:
Name: !Sub "${AWS::StackName}-MwaaEnvironment"
SourceBucketArn: !GetAtt EnvironmentBucket.Arn
ExecutionRoleArn: !GetAtt MwaaExecutionRole.Arn
DagS3Path: dags
NetworkConfiguration:
SecurityGroupIds:
- !GetAtt SecurityGroup.GroupId
SubnetIds:
- !Ref PrivateSubnet1
- !Ref PrivateSubnet2
WebserverAccessMode: PUBLIC_ONLY
MaxWorkers: !Ref MaxWorkerNodes
LoggingConfiguration:
DagProcessingLogs:
LogLevel: !Ref DagProcessingLogs
Enabled: true
SchedulerLogs:
LogLevel: !Ref SchedulerLogsLevel
Enabled: true
TaskLogs:
LogLevel: !Ref TaskLogsLevel
Enabled: true
WorkerLogs:
LogLevel: !Ref WorkerLogsLevel
Enabled: true
WebserverLogs:
LogLevel: !Ref WebserverLogsLevel
Enabled: true
SecurityGroup:
Type: AWS::EC2::SecurityGroup
Properties:
VpcId: !Ref VPC
GroupDescription: !Sub "Security Group for Amazon MWAA Environment ${AWS::StackName}-MwaaEnvironment"
GroupName: !Sub "airflow-security-group-${AWS::StackName}-MwaaEnvironment"
SecurityGroupIngress:
Type: AWS::EC2::SecurityGroupIngress
Properties:
GroupId: !Ref SecurityGroup
IpProtocol: "-1"
SourceSecurityGroupId: !Ref SecurityGroup
SecurityGroupEgress:
Type: AWS::EC2::SecurityGroupEgress
Properties:
GroupId: !Ref SecurityGroup
IpProtocol: "-1"
CidrIp: "0.0.0.0/0"
MwaaExecutionRole:
Type: AWS::IAM::Role
Properties:
AssumeRolePolicyDocument:
Version: 2012-10-17
Statement:
- Effect: Allow
Principal:
Service:
- airflow-env.amazonaws.com
- airflow.amazonaws.com
Action:
- "sts:AssumeRole"
Path: "/service-role/"
MwaaExecutionPolicy:
DependsOn: EnvironmentBucket
Type: AWS::IAM::ManagedPolicy
Properties:
Roles:
- !Ref MwaaExecutionRole
PolicyDocument:
Version: 2012-10-17
Statement:
- Effect: Allow
Action: airflow:PublishMetrics
Resource:
- !Sub "arn:aws:airflow:${AWS::Region}:${AWS::AccountId}:environment/${EnvironmentName}"
- Effect: Deny
Action: s3:ListAllMyBuckets
Resource:
- !Sub "${EnvironmentBucket.Arn}"
- !Sub "${EnvironmentBucket.Arn}/*"
- Effect: Allow
Action:
- "s3:GetObject*"
- "s3:GetBucket*"
- "s3:List*"
Resource:
- !Sub "${EnvironmentBucket.Arn}"
- !Sub "${EnvironmentBucket.Arn}/*"
- Effect: Allow
Action:
- logs:DescribeLogGroups
Resource: "*"
- Effect: Allow
Action:
- logs:CreateLogStream
- logs:CreateLogGroup
- logs:PutLogEvents
- logs:GetLogEvents
- logs:GetLogRecord
- logs:GetLogGroupFields
- logs:GetQueryResults
- logs:DescribeLogGroups
Resource:
- !Sub "arn:aws:logs:${AWS::Region}:${AWS::AccountId}:log-group:airflow-${AWS::StackName}*"
- Effect: Allow
Action: cloudwatch:PutMetricData
Resource: "*"
- Effect: Allow
Action:
- sqs:ChangeMessageVisibility
- sqs:DeleteMessage
- sqs:GetQueueAttributes
- sqs:GetQueueUrl
- sqs:ReceiveMessage
- sqs:SendMessage
Resource:
- !Sub "arn:aws:sqs:${AWS::Region}:*:airflow-celery-*"
- Effect: Allow
Action:
- kms:Decrypt
- kms:DescribeKey
- "kms:GenerateDataKey*"
- kms:Encrypt
NotResource: !Sub "arn:aws:kms:*:${AWS::AccountId}:key/*"
Condition:
StringLike:
"kms:ViaService":
- !Sub "sqs.${AWS::Region}.amazonaws.com"
- Action:
- cloudwatch:*
- cloudformation:CreateStack
- cloudformation:DescribeStackEvents
- ec2:AuthorizeSecurityGroupIngress
- ec2:AuthorizeSecurityGroupEgress
- ec2:CancelSpotInstanceRequests
- ec2:CreateRoute
- ec2:CreateSecurityGroup
- ec2:CreateTags
- ec2:DeleteRoute
- ec2:DeleteTags
- ec2:DeleteSecurityGroup
- ec2:DescribeAvailabilityZones
- ec2:DescribeAccountAttributes
- ec2:DescribeInstances
- ec2:DescribeKeyPairs
- ec2:DescribeRouteTables
- ec2:DescribeSecurityGroups
- ec2:DescribeSpotInstanceRequests
- ec2:DescribeSpotPriceHistory
- ec2:DescribeSubnets
- ec2:DescribeVpcAttribute
- ec2:DescribeVpcs
- ec2:DescribeRouteTables
- ec2:DescribeNetworkAcls
- ec2:CreateVpcEndpoint
- ec2:ModifyImageAttribute
- ec2:ModifyInstanceAttribute
- ec2:RequestSpotInstances
- ec2:RevokeSecurityGroupEgress
- ec2:RunInstances
- ec2:TerminateInstances
- elasticmapreduce:*
- iam:GetPolicy
- iam:GetPolicyVersion
- iam:ListRoles
- iam:PassRole
- kms:List*
- s3:*
- sdb:*
Effect: Allow
Resource: "*"
- Effect: Allow
Action: iam:CreateServiceLinkedRole
Resource: "*"
Condition:
StringLike:
iam:AWSServiceName:
- elasticmapreduce.amazonaws.com
- elasticmapreduce.amazonaws.com.cn