I'm using google Composer version 1.18.0 which is having Airflow version 2.2.3 and the worker node count is set to 4 with a Disk Size of 100 GB and machine type n1-standard-2 and
Web server machine type=composer-n1-webserver-2 (2 vCPU, 1.6 GB memory)
Cloud SQL machine type=db-n1-standard-2 (2 vCPU, 7.5 GB memory)
number of schedulers is 1 and the rest of the other configs are almost basic.
I have just moved 4 jobs into this and I'm already seeing POD evicted due to this below error
Container airflow-worker exceeded its local ephemeral storage limit "10137Mi".
I have tried contacting google support team, they told to create a new environment with enough resources and this one newly created as autopilot in another older environment is not working as expected and also pods are all evicted with the same error, so we creatd this above config environment with enough resources but still seeing this error and not sure how to get rid of this.
PFA screenshots for reference
Can anyone help me here
YAML FILE
apiVersion: v1
kind: Pod
metadata:
annotations:
composer.cloud.google.com/running-task: "true"
composer.cloud.google.com/template-version:
919cc331f4a25a332a1e0b6989f5fb56
seccomp.security.alpha.kubernetes.io/pod: runtime/default
creationTimestamp: "2022-01-25T17:02:00Z"
generateName: airflow-worker-
labels:
run: airflow-worker
managedFields:
- apiVersion: v1
fieldsType: FieldsV1
fieldsV1:
f:metadata:
f:annotations:
.: {}
f:composer.cloud.google.com/template-version: {}
f:generateName: {}
f:labels:
.: {}
f:run: {}
f:ownerReferences:
.: {}
k:{"uid":"7ece9e6c-e252-43d9-b989-239d765d375b"}:
.: {}
f:apiVersion: {}
f:blockOwnerDeletion: {}
f:controller: {}
f:kind: {}
f:name: {}
f:uid: {}
f:spec:
f:containers:
k:{"name":"airflow-worker"}:
.: {}
f:args: {}
f:env:
.: {}
k:{"name":"AIRFLOW__CORE__FERNET_KEY"}:
.: {}
f:name: {}
f:valueFrom:
.: {}
f:secretKeyRef:
.: {}
f:key: {}
f:name: {}
k:{"name":"AIRFLOW__CORE__SQL_ALCHEMY_CONN"}:
.: {}
f:name: {}
f:value: {}
k:{"name":"AIRFLOW__WEBSERVER__BASE_URL"}:
.: {}
f:name: {}
f:value: {}
k:{"name":"AIRFLOW_DATABASE_VERSION"}:
.: {}
f:name: {}
f:value: {}
k:{"name":"AIRFLOW_HOME"}:
.: {}
f:name: {}
f:value: {}
k:{"name":"AUTOGKE"}:
.: {}
f:name: {}
f:value: {}
k:{"name":"C_FORCE_ROOT"}:
.: {}
f:name: {}
f:value: {}
k:{"name":"CLOUDSDK_METRICS_ENVIRONMENT"}:
.: {}
f:name: {}
f:value: {}
k:{"name":"COMPOSER_ENVIRONMENT"}:
.: {}
f:name: {}
f:value: {}
k:{"name":"COMPOSER_GKE_LOCATION"}:
.: {}
f:name: {}
f:value: {}
k:{"name":"COMPOSER_GKE_NAME"}:
.: {}
f:name: {}
f:value: {}
k:{"name":"COMPOSER_GKE_ZONE"}:
.: {}
f:name: {}
k:{"name":"COMPOSER_LOCATION"}:
.: {}
f:name: {}
f:value: {}
k:{"name":"COMPOSER_PYTHON_VERSION"}:
.: {}
f:name: {}
f:value: {}
k:{"name":"COMPOSER_VERSION"}:
.: {}
f:name: {}
f:value: {}
k:{"name":"COMPOSER_VERSIONED_NAMESPACE"}:
.: {}
f:name: {}
f:value: {}
k:{"name":"DAGS_FOLDER"}:
.: {}
f:name: {}
f:value: {}
k:{"name":"GCP_PROJECT"}:
.: {}
f:name: {}
f:value: {}
k:{"name":"GCS_BUCKET"}:
.: {}
f:name: {}
f:value: {}
k:{"name":"GCSFUSE_EXTRACTED"}:
.: {}
f:name: {}
f:value: {}
k:{"name":"GRPC_POLL_STRATEGY"}:
.: {}
f:name: {}
f:value: {}
k:{"name":"SQL_DATABASE"}:
.: {}
f:name: {}
f:value: {}
k:{"name":"SQL_HOST"}:
.: {}
f:name: {}
f:value: {}
k:{"name":"SQL_PASSWORD"}:
.: {}
f:name: {}
f:valueFrom:
.: {}
f:secretKeyRef:
.: {}
f:key: {}
f:name: {}
k:{"name":"SQL_SUBNET"}:
.: {}
f:name: {}
f:value: {}
k:{"name":"SQL_USER"}:
.: {}
f:name: {}
f:value: {}
f:image: {}
f:imagePullPolicy: {}
f:livenessProbe:
.: {}
f:exec:
.: {}
f:command: {}
f:failureThreshold: {}
f:initialDelaySeconds: {}
f:periodSeconds: {}
f:successThreshold: {}
f:timeoutSeconds: {}
f:name: {}
f:ports:
.: {}
k:{"containerPort":8793,"protocol":"TCP"}:
.: {}
f:containerPort: {}
f:protocol: {}
f:resources:
.: {}
f:limits:
.: {}
f:cpu: {}
f:ephemeral-storage: {}
f:memory: {}
f:requests:
.: {}
f:cpu: {}
f:ephemeral-storage: {}
f:memory: {}
f:terminationMessagePath: {}
f:terminationMessagePolicy: {}
f:volumeMounts:
.: {}
k:{"mountPath":"/etc/airflow/airflow_cfg"}:
.: {}
f:mountPath: {}
f:name: {}
k:{"mountPath":"/home/airflow/container-comms"}:
.: {}
f:mountPath: {}
f:name: {}
k:{"mountPath":"/home/airflow/gcs"}:
.: {}
f:mountPath: {}
f:name: {}
k:{"mountPath":"/home/airflow/gcsfuse"}:
.: {}
f:mountPath: {}
f:mountPropagation: {}
f:name: {}
k:{"name":"gcs-syncd"}:
.: {}
f:args: {}
f:env:
.: {}
k:{"name":"AUTOGKE"}:
.: {}
f:name: {}
f:value: {}
k:{"name":"COMPOSER_GKE_LOCATION"}:
.: {}
f:name: {}
f:value: {}
k:{"name":"COMPOSER_GKE_NAME"}:
.: {}
f:name: {}
f:value: {}
k:{"name":"COMPOSER_GKE_ZONE"}:
.: {}
f:name: {}
k:{"name":"GCS_BUCKET"}:
.: {}
f:name: {}
f:value: {}
k:{"name":"SQL_DATABASE"}:
.: {}
f:name: {}
f:value: {}
k:{"name":"SQL_PASSWORD"}:
.: {}
f:name: {}
f:valueFrom:
.: {}
f:secretKeyRef:
.: {}
f:key: {}
f:name: {}
k:{"name":"SQL_SUBNET"}:
.: {}
f:name: {}
f:value: {}
k:{"name":"SQL_USER"}:
.: {}
f:name: {}
f:value: {}
f:image: {}
f:imagePullPolicy: {}
f:name: {}
f:resources:
.: {}
f:limits:
.: {}
f:cpu: {}
f:ephemeral-storage: {}
f:memory: {}
f:requests:
.: {}
f:cpu: {}
f:ephemeral-storage: {}
f:memory: {}
f:terminationMessagePath: {}
f:terminationMessagePolicy: {}
f:volumeMounts:
.: {}
k:{"mountPath":"/home/airflow/gcs"}:
.: {}
f:mountPath: {}
f:name: {}
f:dnsPolicy: {}
f:enableServiceLinks: {}
f:restartPolicy: {}
f:schedulerName: {}
f:securityContext: {}
f:terminationGracePeriodSeconds: {}
f:volumes:
.: {}
k:{"name":"airflow-config"}:
.: {}
f:configMap:
.: {}
f:defaultMode: {}
f:name: {}
f:name: {}
k:{"name":"container-comms"}:
.: {}
f:hostPath:
.: {}
f:path: {}
f:type: {}
f:name: {}
k:{"name":"gcsdir"}:
.: {}
f:emptyDir: {}
f:name: {}
k:{"name":"gcsfuse"}:
.: {}
f:hostPath:
.: {}
f:path: {}
f:type: {}
f:name: {}
manager: manager
operation: Update
time: "2022-01-25T17:02:00Z"
- apiVersion: v1
fieldsType: FieldsV1
fieldsV1:
f:metadata:
f:annotations:
f:composer.cloud.google.com/running-task: {}
manager: OpenAPI-Generator
operation: Update
time: "2022-01-25T17:02:33Z"
- apiVersion: v1
fieldsType: FieldsV1
fieldsV1:
f:status:
f:message: {}
f:phase: {}
f:reason: {}
f:startTime: {}
manager: kubelet
operation: Update
time: "2022-01-26T10:01:06Z"
name: airflow-worker-nlzdq
namespace: composer-2-0-0-airflow-2-1-4-2884adda
ownerReferences:
- apiVersion: composer.cloud.google.com/v1beta1
blockOwnerDeletion: true
controller: true
kind: AirflowWorkerSet
name: airflow-worker
uid: 7ece9e6c-e252-43d9-b989-239d765d375b
resourceVersion: "27047147"
uid: b904b45d-0001-4dc4-8f6c-88420d286973
spec:
containers:
- args:
- worker
env:
- name: GRPC_POLL_STRATEGY
value: epoll1
- name: CLOUDSDK_METRICS_ENVIRONMENT
value: 2.1.4+composer
- name: GCS_BUCKET
value: us-east4-XYZ-gcp-airflow-p-2884adda-bucket
- name: GCP_PROJECT
value: XYZ-259410
- name: COMPOSER_LOCATION
value: us-east4
- name: COMPOSER_GKE_ZONE
- name: COMPOSER_GKE_NAME
value: us-east4-XYZ-gcp-airflow-p-2884adda-gke
- name: AUTOGKE
value: "TRUE"
- name: COMPOSER_GKE_LOCATION
value: us-east4
- name: COMPOSER_PYTHON_VERSION
value: "3"
- name: COMPOSER_ENVIRONMENT
value: XYZ-gcp-airflow-prod-shared-vpc
- name: COMPOSER_VERSIONED_NAMESPACE
value: composer-2-0-0-airflow-2-1-4-2884adda
- name: AIRFLOW_HOME
value: /etc/airflow
- name: DAGS_FOLDER
value: /home/airflow/gcs/dags
- name: SQL_HOST
value: airflow-sqlproxy-service.composer-
system.svc.cluster.local
- name: SQL_DATABASE
value: composer-2-0-0-airflow-2-1-4-2884adda
- name: SQL_USER
value: root
- name: SQL_PASSWORD
valueFrom:
secretKeyRef:
key: sql_password
name: airflow-secrets
- name: GCSFUSE_EXTRACTED
value: "TRUE"
- name: COMPOSER_VERSION
value: 2.0.0
- name: AIRFLOW__WEBSERVER__BASE_URL
value: https://XYZ-dot-us-east4.composer.googleusercontent.com
- name: SQL_SUBNET
value: 172.16.2.0/29
- name: AIRFLOW_DATABASE_VERSION
value: POSTGRES_13
- name: AIRFLOW__CORE__SQL_ALCHEMY_CONN
value: postgresql+psycopg2://$(SQL_USER):$(SQL_PASSWORD)@airflow-sqlproxy-service.composer-system.svc.cluster.local:3306/$(SQL_DATABASE)
- name: AIRFLOW__CORE__FERNET_KEY
valueFrom:
secretKeyRef:
key: fernet_key
name: airflow-secrets
- name: C_FORCE_ROOT
value: "TRUE"
image: us-east4-docker.pkg.dev/XYZ-259410/composer-images-us-east4-XYZ-gcp-airflow-p-2884adda-gke/e77f16de-356f-40b5-b42b-f5482f02f793
imagePullPolicy: IfNotPresent
livenessProbe:
exec:
command:
- /var/local/worker_checker.py
failureThreshold: 6
initialDelaySeconds: 120
periodSeconds: 90
successThreshold: 1
timeoutSeconds: 30
name: airflow-worker
ports:
- containerPort: 8793
protocol: TCP
resources:
limits:
cpu: 1700m
ephemeral-storage: 4Gi
memory: 6963Mi
requests:
cpu: 1700m
ephemeral-storage: 2Gi
memory: 6963Mi
securityContext:
capabilities:
drop:
- NET_RAW
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File
volumeMounts:
- mountPath: /etc/airflow/airflow_cfg
name: airflow-config
- mountPath: /home/airflow/gcs
name: gcsdir
- mountPath: /home/airflow/container-comms
name: container-comms
- mountPath: /home/airflow/gcsfuse
mountPropagation: HostToContainer
name: gcsfuse
- mountPath: /var/run/secrets/kubernetes.io/serviceaccount
name: kube-api-access-59j75
readOnly: true
- args:
- /home/airflow/gcs
env:
- name: GCS_BUCKET
value: us-east4-XYZ-gcp-airflow-p-2884adda-bucket
- name: SQL_DATABASE
value: composer-2-0-0-airflow-2-1-4-2884adda
- name: SQL_USER
value: root
- name: SQL_PASSWORD
valueFrom:
secretKeyRef:
key: sql_password
name: airflow-secrets
- name: COMPOSER_GKE_ZONE
- name: COMPOSER_GKE_NAME
value: us-east4-XYZ-gcp-airflow-p-2884adda-gke
- name: SQL_SUBNET
value: 172.16.2.0/29
- name: AUTOGKE
value: "TRUE"
- name: COMPOSER_GKE_LOCATION
value: us-east4
image: us-docker.pkg.dev/cloud-airflow-releaser/gcs-syncd/gcs-syncd:cloud_composer_service_2021-12-08-RC1
imagePullPolicy: IfNotPresent
name: gcs-syncd
resources:
limits:
cpu: 300m
ephemeral-storage: 102Mi
memory: 1228Mi
requests:
cpu: 300m
ephemeral-storage: 102Mi
memory: 1228Mi
securityContext:
capabilities:
drop:
- NET_RAW
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File
volumeMounts:
- mountPath: /home/airflow/gcs
name: gcsdir
- mountPath: /var/run/secrets/kubernetes.io/serviceaccount
name: kube-api-access-59j75
readOnly: true
dnsPolicy: ClusterFirst
enableServiceLinks: true
nodeName: gk3-us-east4-XYZ-gcp-ai-nap-meqa92wv-4be2b93f-5kdv
preemptionPolicy: PreemptLowerPriority
priority: 0
restartPolicy: Always
schedulerName: gke.io/optimize-utilization-scheduler
securityContext:
seccompProfile:
type: RuntimeDefault
serviceAccount: default
serviceAccountName: default
terminationGracePeriodSeconds: 30
tolerations:
- effect: NoExecute
key: node.kubernetes.io/not-ready
operator: Exists
tolerationSeconds: 300
- effect: NoExecute
key: node.kubernetes.io/unreachable
operator: Exists
tolerationSeconds: 300
volumes:
- configMap:
defaultMode: 420
name: airflow-configmap
name: airflow-config
- emptyDir: {}
name: gcsdir
- hostPath:
path: /var/composer/gcs_mount_status
type: ""
name: container-comms
- hostPath:
path: /var/composer/gcs_mount
type: ""
name: gcsfuse
- name: kube-api-access-59j75
projected:
defaultMode: 420
sources:
- serviceAccountToken:
expirationSeconds: 3607
path: token
- configMap:
items:
- key: ca.crt
path: ca.crt
name: kube-root-ca.crt
- downwardAPI:
items:
- fieldRef:
apiVersion: v1
fieldPath: metadata.namespace
path: namespace
status:
message: 'Container airflow-worker exceeded its local ephemeral
storage limit "10137Mi". '
phase: Failed
reason: Evicted
startTime: "2022-01-25T17:02:01Z"