AWS IAM Policies to connect AWS Cloudwatch Logs, Kinesis Firehose, S3 and ElasticSearch

Viewed 10505

I am trying to stream the AWS cloudwatch logs to ES via Kinesis Firehose. Below terraform code is giving an error. Any suggestions.. The error is:

  • aws_cloudwatch_log_subscription_filter.test_kinesis_logfilter: 1 error(s) occurred:
  • aws_cloudwatch_log_subscription_filter.test_kinesis_logfilter: InvalidParameterException: Could not deliver test message to specified Firehose stream. Check if the given Firehose stream is in ACTIVE state.

resource "aws_s3_bucket" "bucket" {
  bucket = "cw-kinesis-es-bucket"
  acl    = "private"
}

resource "aws_iam_role" "firehose_role" {
  name = "firehose_test_role"

  assume_role_policy = <<EOF
{
  "Version": "2012-10-17",
  "Statement": [
    {
      "Action": "sts:AssumeRole",
      "Principal": {
        "Service": "firehose.amazonaws.com"
      },
      "Effect": "Allow",
      "Sid": ""
    }
  ]
}
EOF
}

resource "aws_elasticsearch_domain" "es" {
  domain_name           = "firehose-es-test"
  elasticsearch_version = "1.5"
  cluster_config {
    instance_type = "t2.micro.elasticsearch"
  }
  ebs_options {
    ebs_enabled = true
    volume_size = 10
  }

  advanced_options {
    "rest.action.multi.allow_explicit_index" = "true"
  }

  access_policies = <<CONFIG
{
    "Version": "2012-10-17",
    "Statement": [
        {
            "Action": "es:*",
            "Principal": "*",
            "Effect": "Allow",
            "Condition": {
                "IpAddress": {"aws:SourceIp": ["xxxxx"]}
            }
        }
    ]
}
CONFIG

  snapshot_options {
    automated_snapshot_start_hour = 23
  }

  tags {
    Domain = "TestDomain"
  }
}

resource "aws_kinesis_firehose_delivery_stream" "test_stream" {
  name        = "terraform-kinesis-firehose-test-stream"
  destination = "elasticsearch"

  s3_configuration {
    role_arn           = "${aws_iam_role.firehose_role.arn}"
    bucket_arn         = "${aws_s3_bucket.bucket.arn}"
    buffer_size        = 10
    buffer_interval    = 400
    compression_format = "GZIP"
  }

  elasticsearch_configuration {
    domain_arn = "${aws_elasticsearch_domain.es.arn}"
    role_arn   = "${aws_iam_role.firehose_role.arn}"
    index_name = "test"
    type_name  = "test"
  }
}

resource "aws_iam_role" "iam_for_lambda" {
  name = "iam_for_lambda"
  assume_role_policy = <<EOF
  {
  "Version": "2012-10-17",
  "Statement": [
    {
      "Action": "sts:AssumeRole",
      "Principal": {
        "Service": "lambda.amazonaws.com"
      },
      "Effect": "Allow",
      "Sid": ""
    }
  ]
}
EOF
}

resource "aws_cloudwatch_log_subscription_filter" "test_kinesis_logfilter" {
  name            = "test_kinesis_logfilter"
  role_arn        = "${aws_iam_role.iam_for_lambda.arn}"
  log_group_name  = "loggorup.log"
  filter_pattern  = ""
  destination_arn = "${aws_kinesis_firehose_delivery_stream.test_stream.arn}"
}

1 Answers
Related