AWS Step Function Run Redshift Query

Viewed 610

I have a step function where I am trying to run a Redshift query. I now have the entire step function running, the Lambda stage works perfectly, access at every stage seems to be fine. I am trying to read data from 1 table and write to another table. My query works perfectly when run from the console, but the step function does nothing.

Here is my step function code:

{
  "Comment": "Run Redshift Queries",
  "StartAt": "ReceiveMessage from SQS",
  "States": {
    "ReceiveMessage from SQS": {
      "Type": "Task",
      "Parameters": {
        "QueueUrl": "https://sqs.us-east-2.amazonaws.com/***/dev-queryProcessingQueue"
      },
      "Resource": "arn:aws:states:::aws-sdk:sqs:receiveMessage",
      "Next": "Lambda Invoke",
      "ResultSelector": {
        "body.$": "States.StringToJson($.Messages[0].Body)",
        "receiptHandle.$": "$.Messages[0].ReceiptHandle"
      }
    },
    "Lambda Invoke": {
      "Type": "Task",
      "Resource": "arn:aws:states:::lambda:invoke",
      "OutputPath": "$.Payload",
      "Parameters": {
        "Payload.$": "$",
        "FunctionName": "arn:aws:lambda:us-east-2:***:function:step-function-delete-message-from-sqs:$LATEST"
      },
      "Retry": [
        {
          "ErrorEquals": [
            "Lambda.ServiceException",
            "Lambda.AWSLambdaException",
            "Lambda.SdkClientException"
          ],
          "IntervalSeconds": 2,
          "MaxAttempts": 6,
          "BackoffRate": 2
        }
      ],
      "Next": "Run Analysis Queries"
    },
    "Run Analysis Queries": {
      "Type": "Task",
      "Parameters": {
        "ClusterIdentifier": "test-***-redshift-cluster",
        "Database": "prod",
        "SecretArn": "arn:aws:secretsmanager:us-east-2:***:secret:prod/redshift/StepFn-*",
        "Sql": "INSERT INTO unique_test1 (b, p, unique_r) (WITH data AS (SELECT DISTINCT t, p FROM inst WHERE b BETWEEN 1640879000 AND 1640879500) SELECT a.b AS b, b.p AS p, COUNT(DISTINCT d) AS unique_r FROM tt AS a INNER JOIN data AS b ON a.t = b.t WHERE a.err IS FALSE AND a.token = '0000' AND a.b BETWEEN 1640879000 AND 1640879500 GROUP BY b, b.p ORDER BY b ASC);"
      },
      "Resource": "arn:aws:states:::aws-sdk:redshiftdata:executeStatement",
      "End": true
    }
  },
  "TimeoutSeconds": 3600
}
0 Answers
Related