I have a step function where I am trying to run a Redshift query. I now have the entire step function running, the Lambda stage works perfectly, access at every stage seems to be fine. I am trying to read data from 1 table and write to another table. My query works perfectly when run from the console, but the step function does nothing.
Here is my step function code:
{
"Comment": "Run Redshift Queries",
"StartAt": "ReceiveMessage from SQS",
"States": {
"ReceiveMessage from SQS": {
"Type": "Task",
"Parameters": {
"QueueUrl": "https://sqs.us-east-2.amazonaws.com/***/dev-queryProcessingQueue"
},
"Resource": "arn:aws:states:::aws-sdk:sqs:receiveMessage",
"Next": "Lambda Invoke",
"ResultSelector": {
"body.$": "States.StringToJson($.Messages[0].Body)",
"receiptHandle.$": "$.Messages[0].ReceiptHandle"
}
},
"Lambda Invoke": {
"Type": "Task",
"Resource": "arn:aws:states:::lambda:invoke",
"OutputPath": "$.Payload",
"Parameters": {
"Payload.$": "$",
"FunctionName": "arn:aws:lambda:us-east-2:***:function:step-function-delete-message-from-sqs:$LATEST"
},
"Retry": [
{
"ErrorEquals": [
"Lambda.ServiceException",
"Lambda.AWSLambdaException",
"Lambda.SdkClientException"
],
"IntervalSeconds": 2,
"MaxAttempts": 6,
"BackoffRate": 2
}
],
"Next": "Run Analysis Queries"
},
"Run Analysis Queries": {
"Type": "Task",
"Parameters": {
"ClusterIdentifier": "test-***-redshift-cluster",
"Database": "prod",
"SecretArn": "arn:aws:secretsmanager:us-east-2:***:secret:prod/redshift/StepFn-*",
"Sql": "INSERT INTO unique_test1 (b, p, unique_r) (WITH data AS (SELECT DISTINCT t, p FROM inst WHERE b BETWEEN 1640879000 AND 1640879500) SELECT a.b AS b, b.p AS p, COUNT(DISTINCT d) AS unique_r FROM tt AS a INNER JOIN data AS b ON a.t = b.t WHERE a.err IS FALSE AND a.token = '0000' AND a.b BETWEEN 1640879000 AND 1640879500 GROUP BY b, b.p ORDER BY b ASC);"
},
"Resource": "arn:aws:states:::aws-sdk:redshiftdata:executeStatement",
"End": true
}
},
"TimeoutSeconds": 3600
}