I'm learning docker and trying to setup a Kafka Cluster with 3 Zookeepers instances and 3 brokers.
Sometimes the brokers runs fine, but sometimes they are exiting with status 1 and the logs shows the following message:
kafka.common.InconsistentClusterIdException: The Cluster ID p2ouSB1rTKyUddeL26jpxg doesn't match stored clusterId Some(4QuK9rOmTleiRryAOA5zwA) in meta.properties. The broker is trying to join the wrong cluster. Configured zookeeper.connect may be wrong.
I searched a lot before post this question and nothing I founded resolved the issue.
First a though it was the volumes I was using to persist Zookeeper data and logs on the host, but now that I got ride of them, I'm still getting the same error and have no idea of how to setup a Cluster Id through docker-compose and share it among all brokers. Is there a fix to this?
I'm running Docker version 20.10.10, build b485636 and Docker Compose version v2.1.1 on Windows.
This is my docker-compose.yml:
version: '3.9'
services:
zookeeper-1:
hostname: zookeeper-1
container_name: zookeeper-1
extends:
file: ./docker/services.yml
service: zookeeper
ports:
- '2181:2181'
environment:
ZOOKEEPER_SERVER_ID: 1
ZOOKEEPER_CLIENT_PORT: 2181
zookeeper-2:
hostname: zookeeper-2
container_name: zookeeper-2
extends:
file: ./docker/services.yml
service: zookeeper
ports:
- '2182:2182'
environment:
ZOOKEEPER_SERVER_ID: 2
ZOOKEEPER_CLIENT_PORT: 2182
zookeeper-3:
hostname: zookeeper-3
container_name: zookeeper-3
extends:
file: ./docker/services.yml
service: zookeeper
ports:
- '2183:2183'
environment:
ZOOKEEPER_SERVER_ID: 3
ZOOKEEPER_CLIENT_PORT: 2183
broker-1:
container_name: broker-1
extends:
file: ./docker/services.yml
service: broker
ports:
- '19092:9092'
environment:
KAFKA_BROKER_ID: 1
KAFKA_ZOOKEEPER_CONNECT: 'zookeeper-1:2181,zookeeper-2:2182,zookeeper-3:2183'
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://broker-1:9092,PLAINTEXT_HOST://localhost:19092
CONFLUENT_METRICS_REPORTER_BOOTSTRAP_SERVERS: broker-1:9092
depends_on:
- zookeeper-1
- zookeeper-2
- zookeeper-3
broker-2:
container_name: broker-2
extends:
file: ./docker/services.yml
service: broker
ports:
- '29092:9092'
environment:
KAFKA_BROKER_ID: 2
KAFKA_ZOOKEEPER_CONNECT: 'zookeeper-1:2181,zookeeper-2:2182,zookeeper-3:2183'
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://broker-2:9092,PLAINTEXT_HOST://localhost:29092
CONFLUENT_METRICS_REPORTER_BOOTSTRAP_SERVERS: broker-2:9092
depends_on:
- zookeeper-1
- zookeeper-2
- zookeeper-3
broker-3:
container_name: broker-3
extends:
file: ./docker/services.yml
service: broker
ports:
- '39092:9092'
environment:
KAFKA_BROKER_ID: 3
KAFKA_ZOOKEEPER_CONNECT: 'zookeeper-1:2181,zookeeper-2:2182,zookeeper-3:2183'
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://broker-3:9092,PLAINTEXT_HOST://localhost:39092
CONFLUENT_METRICS_REPORTER_BOOTSTRAP_SERVERS: broker-3:9092
depends_on:
- zookeeper-1
- zookeeper-2
- zookeeper-3
connect:
image: confluentinc/cp-kafka-connect:latest
hostname: connect
container_name: connect
depends_on:
- broker-1
- broker-2
- broker-3
ports:
- '8083:8083'
environment:
CONNECT_BOOTSTRAP_SERVERS: 'broker-1:9092,broker-2:9092,broker-3:9092'
CONNECT_REST_ADVERTISED_HOST_NAME: connect
CONNECT_REST_PORT: 8083
CONNECT_GROUP_ID: compose-connect-group
CONNECT_CONFIG_STORAGE_TOPIC: docker-connect-configs
CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: 1
CONNECT_OFFSET_FLUSH_INTERVAL_MS: 10000
CONNECT_OFFSET_STORAGE_TOPIC: docker-connect-offsets
CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: 1
CONNECT_STATUS_STORAGE_TOPIC: docker-connect-status
CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: 1
CONNECT_KEY_CONVERTER: org.apache.kafka.connect.storage.StringConverter
CONNECT_VALUE_CONVERTER: io.confluent.connect.avro.AvroConverter
CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: http://schema-registry:8081
# CLASSPATH required due to CC-2422
CLASSPATH: /usr/share/java/monitoring-interceptors/monitoring-interceptors-6.2.1.jar
CONNECT_PRODUCER_INTERCEPTOR_CLASSES: 'io.confluent.monitoring.clients.interceptor.MonitoringProducerInterceptor'
CONNECT_CONSUMER_INTERCEPTOR_CLASSES: 'io.confluent.monitoring.clients.interceptor.MonitoringConsumerInterceptor'
CONNECT_PLUGIN_PATH: '/usr/share/java,/usr/share/confluent-hub-components,/connect-plugins'
CONNECT_LOG4J_LOGGERS: org.apache.zookeeper=ERROR,org.I0Itec.zkclient=ERROR,org.reflections=ERROR
control-center:
image: confluentinc/cp-enterprise-control-center:6.2.1
hostname: control-center
container_name: control-center
depends_on:
- broker-1
- broker-2
- broker-3
- connect
ports:
- '9021:9021'
environment:
CONTROL_CENTER_BOOTSTRAP_SERVERS: 'broker-1:9092,broker-2:9092,broker-3:9092'
CONTROL_CENTER_CONNECT_CONNECT-DEFAULT_CLUSTER: 'connect:8083'
#CONTROL_CENTER_KSQL_KSQLDB1_URL: 'http://ksqldb-server:8088'
#CONTROL_CENTER_KSQL_KSQLDB1_ADVERTISED_URL: 'http://localhost:8088'
#CONTROL_CENTER_SCHEMA_REGISTRY_URL: 'http://schema-registry:8081'
CONTROL_CENTER_REPLICATION_FACTOR: 1
CONTROL_CENTER_INTERNAL_TOPICS_PARTITIONS: 1
CONTROL_CENTER_MONITORING_INTERCEPTOR_TOPIC_PARTITIONS: 1
CONFLUENT_METRICS_TOPIC_REPLICATION: 1
PORT: 9021
This is my services.yml file:
services:
zookeeper:
image: confluentinc/cp-zookeeper:6.2.1
environment:
ZOOKEEPER_TICK_TIME: 2000
ZOOKEEPER_PEER_PORT: 2888
ZOOKEEPER_LEADER_PORT: 3888
ZOOKEEPER_INIT_LIMIT: 5
ZOOKEEPER_SYNC_LIMIT: 2
broker:
image: confluentinc/cp-server:6.2.1
environment:
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
KAFKA_METRIC_REPORTERS: io.confluent.metrics.reporter.ConfluentMetricsReporter
KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
KAFKA_CONFLUENT_LICENSE_TOPIC_REPLICATION_FACTOR: 1
KAFKA_CONFLUENT_BALANCER_TOPIC_REPLICATION_FACTOR: 1
KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
KAFKA_JMX_PORT: 9101
KAFKA_JMX_HOSTNAME: localhost
# KAFKA_CONFLUENT_SCHEMA_REGISTRY_URL: http://schema-registry:8081
CONFLUENT_METRICS_REPORTER_TOPIC_REPLICAS: 1
CONFLUENT_METRICS_ENABLE: 'true'
CONFLUENT_SUPPORT_CUSTOMER_ID: 'anonymous'