Step function with Redshift cluster

Viewed 112

Building a step function to orchestrate an ETL pipeline but keep getting this error. Here is my code and following below AWS docs. https://docs.aws.amazon.com/step-functions/latest/dg/sample-etl-orchestration.html

 "GetStateOfCluster": {
      "Type": "Task",
      "Resource": "lambda,
      "TimeoutSeconds": 180,
      "HeartbeatSeconds": 60,
      "Next": "IsClusterAvailable",
      "InputPath": "$",
      "ResultPath": "$.clusterStatus"
    },
    "IsClusterAvailable": {
      "Type": "Choice",
      "Choices": [
        {
          "Variable": "$.clusterStatus",
          "StringEquals": "available",
          "Next": "runetljobs"
        },
        {
          "Variable": "$.clusterStatus",
          "StringEquals": "unavailable",
          "Next": "ClusterUnavailable"
        },
        {
          "Variable": "$.clusterStatus",
          "StringEquals": "paused",
          "Next": "InitializeResumeCluster"
        },
        {
          "Variable": "$.clusterStatus",
          "StringEquals": "resuming",
          "Next": "ClusterWait"
        }
      ],
      "Default": "DefaultState"
    },
    "DefaultState": {
      "Type": "Fail",
      "Error": "DefaultStateError",
      "Cause": "No Matches!"
    },
    "ClusterUnavailable": {
      "Type": "Fail",
      "Cause": "Redshift cluster is not available",
      "Error": "Error"
    },
    "ClusterWait": {
      "Type": "Wait",
      "Seconds": 900,
      "Next": "InitializeCheckCluster"
    },

    "InitializeResumeCluster": {
      "Type": "Pass",
      "Next": "ResumeCluster",
      "Result": {
        "input": {
          "redshift_cluster_id": "redshift cluster id",
          "operation": "resume"
        }
      }
    },
    "ResumeCluster": {
      "Type": "Task",
      "Resource": "lambda",
      "TimeoutSeconds": 180,
      "HeartbeatSeconds": 60,
      "Next": "ClusterWait",
      "InputPath": "$",
      "ResultPath": "$"
    },

It's directly going to default even cluster status shows 'available', rather it should go to runetljob stage. In the doc, they dont have default, if we dont add default, error is,

"cause": "An error occurred while executing the state 'IsClusterAvailable' (entered at the event id #14). Failed to transition out of the state. The state does not point to a next state."
1 Answers

You don't see the state "runetljobs" defined in you state definition.

Related