Kafka connection could not be established. Broker may not be available

Viewed 347

Docker containers connect on start to each other succesfully. But after some time an error occured in "connect" container

Connection to node -1 (broker/172.27.0.3:29092) could not be established. Broker may not be available.

Docker-compose file:

 version: "2.4"
    ### Common settings section ###
    x-common: &common
      logging:
        options:
          max-size: "1m"
          max-file: "10"
      env_file: .env
      restart: unless-stopped
    # network_mode: "host"
    
    services:
    
      zookeeper:
        <<: *common
        image: confluentinc/cp-zookeeper:6.0.0
        hostname: zookeeper
        container_name: zookeeper
        ports:
          - "2181:2181"
        volumes:
          - zookeeper_data:/var/lib/zookeeper/data
        environment:
          ZOOKEEPER_CLIENT_PORT: 2181
          ZOOKEEPER_TICK_TIME: 2000
    
      broker:
        <<: *common
        image: confluentinc/cp-kafka:6.0.0
        hostname: broker
        container_name: broker
        depends_on:
          - zookeeper
        ports:
          - "29092:29092"
          - "9092:9092"
          - "9101:9101"
        volumes:
          - broker_data:/var/lib/kafka/data
        environment:
          KAFKA_BROKER_ID: 1
          KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181'
          KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
          KAFKA_ADVERTISED_HOST_NAME: localhost
          KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://broker:29092,PLAINTEXT_HOST://localhost:9092
          KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
          KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
          KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
          KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
          KAFKA_JMX_PORT: 9101
          KAFKA_JMX_HOSTNAME: localhost
    
      schema-registry:
        image: confluentinc/cp-schema-registry:6.0.0
        hostname: schema-registry
        container_name: schema-registry
        depends_on:
          - zookeeper
          - broker
        ports:
          - "8081:8081"
        environment:
          SCHEMA_REGISTRY_HOST_NAME: schema-registry
          SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: 'broker:29092'
          SCHEMA_REGISTRY_LISTENERS: http://0.0.0.0:8081
    
      connect:
        <<: *common
        build:
          context: ""
          dockerfile: Dockerfile-connect-custom
          args:
            CP_VERSION: 6.0.0
        hostname: connect
        container_name: connect
        depends_on:
          - zookeeper
          - broker
          - schema-registry
        ports:
          - "8083:8083"
        environment:
          CONNECT_BOOTSTRAP_SERVERS: 'broker:29092'
          CONNECT_REST_ADVERTISED_HOST_NAME: connect
          CONNECT_REST_PORT: 8083
          CONNECT_GROUP_ID: compose-connect-group
          CONNECT_CONFIG_STORAGE_TOPIC: docker-connect-configs
          CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: 1
          CONNECT_OFFSET_FLUSH_INTERVAL_MS: 10000
          CONNECT_OFFSET_STORAGE_TOPIC: docker-connect-offsets
          CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: 1
          CONNECT_STATUS_STORAGE_TOPIC: docker-connect-status
          CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: 1
          CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: http://schema-registry:8081
          CONNECT_KEY_CONVERTER: org.apache.kafka.connect.storage.StringConverter
          CONNECT_VALUE_CONVERTER: io.confluent.connect.avro.AvroConverter
          CONNECT_INTERNAL_KEY_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
          CONNECT_INTERNAL_VALUE_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
          CONNECT_ZOOKEEPER_CONNECT: 'zookeeper:2181'
          # CLASSPATH required due to CC-2422
          CLASSPATH: /usr/share/java/monitoring-interceptors/monitoring-interceptors-6.0.0.jar
          CONNECT_PRODUCER_INTERCEPTOR_CLASSES: "io.confluent.monitoring.clients.interceptor.MonitoringProducerInterceptor"
          CONNECT_CONSUMER_INTERCEPTOR_CLASSES: "io.confluent.monitoring.clients.interceptor.MonitoringConsumerInterceptor"
          CONNECT_PLUGIN_PATH: "/usr/share/java,/usr/share/confluent-hub-components"
          CONNECT_LOG4J_LOGGERS: org.apache.zookeeper=ERROR,org.I0Itec.zkclient=ERROR,org.reflections=ERROR
    
      rest-proxy:
        <<: *common
        image: confluentinc/cp-kafka-rest:6.0.0
        depends_on:
          - schema-registry
          - broker
        ports:
          - 8082:8082
        hostname: rest-proxy
        container_name: rest-proxy
        environment:
          KAFKA_REST_HOST_NAME: rest-proxy
          KAFKA_REST_BOOTSTRAP_SERVERS: 'broker:29092'
          KAFKA_REST_LISTENERS: "http://0.0.0.0:8082"
          KAFKA_REST_SCHEMA_REGISTRY_URL: 'http://schema-registry:8081'
    
    volumes:
      zookeeper_data:
      broker_data:

Dockerfile-connect-custom:

FROM confluentinc/cp-kafka-connect-base:6.0.0
RUN confluent-hub install --no-prompt debezium/debezium-connector-mysql:1.3.1

How to fix this error? What is wrong in setup?

0 Answers
Related