aws terraform error CannotPullContainerError: Error response from daemon - but the image url is valid

Viewed 306

I using terraform to deploy containers with fargate.

I got this error:

CannotPullContainerError: Error response from daemon: Get https://xxxxxxxxx.dkr.ecr.us-east-2.amazonaws.com/v2/: net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)

enter image description here

The terraform deploy pass successfully.

This image is just nginx (dockerfile):

FROM nginx:latest

WORKDIR /

COPY ./nginx.conf  /etc/nginx/nginx.conf

nginx.conf:

user  nginx;                                                                    

worker_processes 4;

events { worker_connections 1024; }

error_log  /var/log/nginx/error.log warn;                                       
pid        /var/run/nginx.pid;                                                  

http {                                                                          
    include       /etc/nginx/mime.types;                                        
    default_type  application/octet-stream;                                     
                                                                                
    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '   
                      '$status $body_bytes_sent "$http_referer" '               
                      '"$http_user_agent" "$http_x_forwarded_for"';             
                                                                                
    access_log  /var/log/nginx/access.log  main;                                
                                                                                
    sendfile        on;                                                         
    #tcp_nopush     on;                                                         
                                                                                
    keepalive_timeout  65;                                                      
  
    server {                                                                                                  
        listen       80;                                                                                      

        if ($host ~ ^(?!www\.)(?<domain>.+)$) {
            return  301 $scheme://www.$domain$request_uri;
        }

        location / {
            add_header Content-Type text/plain;
            
            return 200 "<H1>Nginx works!</H1>";
        }
    }                                                                                                         
}

I don't undestand why aws say it can't get response. the url is okay. I can download this image and run the container in my computer.

So why? where the problem is?

My terraform:

data "aws_iam_role" "ecs_task_execution_role" {
  name = "ecsTaskExecutionRole"
}

resource "aws_ecs_cluster" "cluster" {
  name = "${var.app}-ecs-cluster"
}

data "aws_iam_policy_document" "ecs_service_role" {
  statement {
    effect  = "Allow"
    actions = ["sts:AssumeRole"]
    principals {
      type        = "Service"
      identifiers = ["ecs.amazonaws.com"]
    }
  }
}

data "aws_iam_policy_document" "ecs_service_policy" {
  statement {
    effect = "Allow"
    resources = ["*"]
    actions = [
      "elasticloadbalancing:Describe*",
      "elasticloadbalancing:DeregisterInstancesFromLoadBalancer",
      "elasticloadbalancing:RegisterInstancesWithLoadBalancer",
      "ec2:Describe*",
      "ec2:AuthorizeSecurityGroupIngress"
    ]
  }
}


resource "aws_iam_role_policy" "ecs_service_role_policy" {
  name   = "ecs_service_role_policy"
  policy = "${data.aws_iam_policy_document.ecs_service_policy.json}"
  role   = "${aws_iam_role.ecs_role.id}"
}

resource "aws_iam_role" "ecs_role" {
  name               = "ecs_role"
  assume_role_policy = "${data.aws_iam_policy_document.ecs_service_role.json}"
}

resource "aws_ecs_task_definition" "nginx" {
  family = "nginx-${var.app}"

  network_mode             = "awsvpc"
  requires_compatibilities = ["FARGATE"]

  cpu    = "256"
  memory = "512"

  execution_role_arn = "${aws_iam_role.ecs_execution_role.arn}"
  task_role_arn      = "${aws_iam_role.ecs_execution_role.arn}"

  container_definitions = <<DEFINITION
  [
    {
      "image": "xxxxxx.dkr.ecr.us-east-2.amazonaws.com/org/prod/www-nginx:latest",
      "memory": 300,
      "name": "nginx-...-prod-www",
      "networkMode": "awsvpc",
      "essential": true,
      "portMappings": [
        {
          "containerPort": 80,
          "hostPort": 80
        }
      ],
      "logConfiguration": {
        "logDriver": "awslogs",
        "options": {
          "awslogs-group": "/xx-ecs-...-prod/nginx",
          "awslogs-region": "us-east-2",
          "awslogs-stream-prefix": "web"
        }
      }
    }    
  ]
  DEFINITION
}


resource "aws_cloudwatch_log_group" "nginx" {
  name = "/ecs-${var.app}/nginx"
}


resource "aws_ecs_service" "web" {
  name            = "nginx-${var.app}"

  task_definition = "${aws_ecs_task_definition.nginx.family}:${max("${aws_ecs_task_definition.nginx.revision}", "${aws_ecs_task_definition.nginx.revision}")}"
  
  desired_count   = 2
  launch_type     = "FARGATE"
  cluster =       "${aws_ecs_cluster.cluster.id}"

  network_configuration {
    subnets         = [aws_subnet.demo-private-1.id, aws_subnet.demo-private-2.id, aws_subnet.demo-private-3.id]
    security_groups = [aws_security_group.lb_sg.id]
  }


  load_balancer {
    target_group_arn = aws_alb_target_group.nginx.id
    container_name   = "nginx-${var.app}"
    container_port   = "80"
  }
  
  depends_on = ["aws_alb_target_group.nginx", "aws_iam_role_policy.ecs_service_role_policy"]
}

resource "aws_alb_target_group" "nginx" {
  name       = "nginx-${var.app}"
  port       = 80
  protocol   = "HTTP"
  vpc_id     = aws_vpc.demo-tf.id
  depends_on = [aws_alb.demo_eu_alb]
  target_type = "ip"

  stickiness {
    type            = "lb_cookie"
    cookie_duration = 86400
  }

  health_check {
    # path                = "/health"
    path                = "/"

    healthy_threshold   = 2
    unhealthy_threshold = 10 # 2 # 10
    timeout             = 60 # 5 # 60
    interval            = 300 # 8 # 300
    matcher             = "200,301,302"
  }
}


resource "aws_alb" "demo_eu_alb" {
  name            = "eu-alb-${var.app}"
  subnets         = [aws_subnet.demo-private-1.id, aws_subnet.demo-private-2.id, aws_subnet.demo-private-3.id]
  security_groups = [aws_security_group.lb_sg.id]
  enable_http2    = "true"
  idle_timeout    = 600
}



output "alb_output" {
  value = aws_alb.demo_eu_alb.dns_name
}


resource "aws_security_group" "lb_sg" {
  description = "controls access to the application ELB"

  vpc_id = aws_vpc.demo-tf.id
  name   = "ELB-${var.app}"

  ingress {
    protocol    = "tcp"
    from_port   = 80
    to_port     = 80
    cidr_blocks = ["0.0.0.0/0"]
  }

  ingress {
    protocol    = "tcp"
    from_port   = 443
    to_port     = 443
    cidr_blocks = ["0.0.0.0/0"]
  }

  egress {
    from_port = 0
    to_port   = 0
    protocol  = "-1"

    cidr_blocks = [
      "0.0.0.0/0",
    ]
  }
}



resource "aws_alb_listener" "front_end" {
  load_balancer_arn = aws_alb.demo_eu_alb.id
  port              = "80"
  protocol          = "HTTP"

  default_action {
    type = "redirect"

    redirect {
      port        = "443"
      protocol    = "HTTPS"
      status_code = "HTTP_301"
    }
  }
}


resource "aws_alb_listener" "alb_front_https" {
    load_balancer_arn   =   "${aws_alb.demo_eu_alb.arn}"
    port            =   "443"
    protocol        =   "HTTPS"
    ssl_policy      =   "ELBSecurityPolicy-TLS-1-2-Ext-2018-06"
    certificate_arn     =   "${aws_iam_server_certificate.lb_cert.arn}"
    default_action {
        target_group_arn    =   "${aws_alb_target_group.nginx.arn}"
        type            =   "forward"
    }
}


resource "aws_iam_server_certificate" "lb_cert" {
  name              = "lb_cert-${var.app}"
  certificate_body  = "${file("./www.____.com/cert.pem")}"
  private_key       = "${file("./www.____.com/privkey.pem")}"
  certificate_chain = "${file("./www.____.com/chain.pem")}"
}

resource "aws_iam_role" "ecs_execution_role" {
  name = "ecs_task_execution_role"
 
  assume_role_policy = <<EOF
{
  "Version": "2012-10-17",
  "Statement": [
    {
      "Sid": "",
        "Effect": "Allow",
        "Principal": {
         "Service": "ecs-tasks.amazonaws.com"
        },
        "Action": "sts:AssumeRole"
    }    
  ]
}
EOF
}

resource "aws_iam_policy" "ecs_permissions" {
  name        = "my_ecs_permissions"
  description = "Permissions to enable CT"

  policy = <<EOF
{
  "Version": "2012-10-17",
  "Statement": [
    {
      "Sid": "",
      "Effect": "Allow",
      "Action": [
        "ecs:CreateCluster",
        "ecs:DeregisterContainerInstance",
        "ecs:DiscoverPollEndpoint",
        "ecs:Poll",
        "ecs:RegisterContainerInstance",
        "ecs:StartTelemetrySession",
        "ecs:Submit*",
        "ecs:StartTask",
        "ecr:GetAuthorizationToken",
        "ecr:BatchCheckLayerAvailability",
        "ecr:GetDownloadUrlForLayer",
        "ecr:BatchGetImage",
        "logs:CreateLogStream",
        "logs:PutLogEvents"
      ],
      "Resource": "*"
    }    
  ]
}
EOF
}

resource "aws_iam_role_policy_attachment" "ecs_attachment" {
  role       = aws_iam_role.ecs_execution_role.name
  policy_arn = aws_iam_policy.ecs_permissions.arn
}









### VPC

### Network

# Internet VPC

resource "aws_vpc" "demo-tf" {
  cidr_block           = "172.21.0.0/16"
  instance_tenancy     = "default"
  enable_dns_support   = "true"
  enable_dns_hostnames = "true"
  enable_classiclink   = "false"

  tags = {
    Name = "vpc-${var.app}"
  }
}

# Subnets
resource "aws_subnet" "demo-public-1" {
  vpc_id                  = aws_vpc.demo-tf.id
  cidr_block              = "172.21.10.0/24"
  map_public_ip_on_launch = "true"
  availability_zone       = "us-east-2a"

  tags = {
    Name = "public-1-${var.app}"
  }
}

resource "aws_subnet" "demo-public-2" {
  vpc_id                  = aws_vpc.demo-tf.id
  cidr_block              = "172.21.20.0/24"
  map_public_ip_on_launch = "true"
  availability_zone       = "us-east-2b"

  tags = {
    Name = "public-2-${var.app}"
  }
}

resource "aws_subnet" "demo-public-3" {
  vpc_id                  = aws_vpc.demo-tf.id
  cidr_block              = "172.21.30.0/24"
  map_public_ip_on_launch = "true"
  availability_zone       = "us-east-2c"

  tags = {
    Name = "public-3-${var.app}"
  }
}

resource "aws_subnet" "demo-private-1" {
  vpc_id                  = aws_vpc.demo-tf.id
  cidr_block              = "172.21.40.0/24"
  map_public_ip_on_launch = "false"
  availability_zone       = "us-east-2a"

  tags = {
    Name = "private-1-${var.app}"
  }
}

resource "aws_subnet" "demo-private-2" {
  vpc_id                  = aws_vpc.demo-tf.id
  cidr_block              = "172.21.50.0/24"
  map_public_ip_on_launch = "false"
  availability_zone       = "us-east-2b"

  tags = {
    Name = "private-2-${var.app}"
  }
}

resource "aws_subnet" "demo-private-3" {
  vpc_id                  = aws_vpc.demo-tf.id
  cidr_block              = "172.21.60.0/24"
  map_public_ip_on_launch = "false"
  availability_zone       = "us-east-2c"

  tags = {
    Name = "private-3-${var.app}"
  }
}

# Internet GW
resource "aws_internet_gateway" "demo-gw" {
  vpc_id = aws_vpc.demo-tf.id

  tags = {
    Name = "IG-${var.app}"
  }
}

# route tables
resource "aws_route_table" "demo-private" {
  vpc_id = aws_vpc.demo-tf.id

  route {
    cidr_block = "0.0.0.0/0"
    gateway_id = aws_internet_gateway.demo-gw.id
  }

  tags = {
    Name = "private-1-${var.app}"
  }
}

# route tables
resource "aws_route_table" "demo-public" {
  vpc_id = aws_vpc.demo-tf.id

  route {
    cidr_block = "0.0.0.0/0"
    gateway_id = aws_internet_gateway.demo-gw.id
  }

  tags = {
    Name = "public-1-${var.app}"
  }
}


# route associations public
resource "aws_route_table_association" "demo-public-1-a" {
  subnet_id      = aws_subnet.demo-public-1.id
  route_table_id = aws_route_table.demo-public.id
}

resource "aws_route_table_association" "demo-public-2-a" {
  subnet_id      = aws_subnet.demo-public-2.id
  route_table_id = aws_route_table.demo-public.id
}

resource "aws_route_table_association" "demo-public-3-a" {
  subnet_id      = aws_subnet.demo-public-3.id
  route_table_id = aws_route_table.demo-public.id
}


# route associations private
resource "aws_route_table_association" "demo-private-1-a" {
  subnet_id      = aws_subnet.demo-private-1.id
  route_table_id = aws_route_table.demo-private.id
}

resource "aws_route_table_association" "demo-private-2-a" {
  subnet_id      = aws_subnet.demo-private-2.id
  route_table_id = aws_route_table.demo-private.id
}

resource "aws_route_table_association" "demo-private-3-a" {
  subnet_id      = aws_subnet.demo-private-3.id
  route_table_id = aws_route_table.demo-private.id
}
1 Answers

A possible reason could be lack of internet access for your tasks in Fargate.

Specifically, you are using in your aws_ecs_service:

  network_configuration {
    subnets         = [aws_subnet.demo-private-1.id, aws_subnet.demo-private-2.id, aws_subnet.demo-private-3.id]
    security_groups = [aws_security_group.lb_sg.id]
  }

However, it does not specify assign_public_ip, which by default is false. Subsequently, your tasks on Fargate don't have public ip and can't access ECR service resulting in the timeout observed.

Note that there could be other reasons, even if you add the public ip, related to your vpc or other settings, which I haven't verified.

Related