Im trying to setup my cluster with elk and kafka inside docker containers, but always logstash cant consume data from the kafka. Producer based on my local machine, not inside docker. I appriciate any help.
docker-compose:
zoo1:
image: confluentinc/cp-zookeeper
restart: always
container_name: zoo1
ports:
- "2181:2181"
environment:
- ZOO_MY_ID=1
- ZOO_SERVERS=2181
- ZOOKEEPER_CLIENT_PORT=2181
- ALLOW_ANONYMOUS_LOGIN=yes
kafka:
image: confluentinc/cp-kafka
hostname: kafka
container_name: kafka
depends_on:
- zoo1
ports:
- "9092:9092"
environment:
KAFKA_BROKER_ID: 1
KAFKA_ZOOKEEPER_CONNECT: zoo1:2181
KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
KAFKA_LISTENERS: PLAINTEXT_HOST://0.0.0.0:9092, PLAINTEXT://kafka:9093
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9093,PLAINTEXT_HOST://kafka:9092
logstash:
build:
context: logstash/
args:
ELK_VERSION: $ELK_VERSION
volumes:
- type: bind
source: ./logstash/config/logstash.yml
target: /usr/share/logstash/config/logstash.yml
read_only: true
- type: bind
source: ./logstash/pipeline
target: /usr/share/logstash/pipeline
read_only: true
ports:
- "5044:5044"
- "5000:5000/tcp"
- "5000:5000/udp"
- "9600:9600"
environment:
LS_JAVA_OPTS: "-Xmx256m -Xms256m"
networks:
- elk
depends_on:
- elasticsearch
- kafka
logstash.conf:
input {
kafka {
topics => ["topic-ex"]
bootstrap_servers => "localhost:9092"
}
}
Trace:
logstash_1 | [2021-03-28T04:39:54,855][WARN ][org.apache.kafka.clients.NetworkClient][main][540d5db3f43043788c8c88c0e41536de536f338e7ba9b86852861fc54f459599] [Consumer clientId=logstash-0, groupId=logstash] Connection to node -1 (localhost/127.0.0.1:9092) could not be established. Broker may not be available.
logstash_1 | [2021-03-28T04:39:54,856][WARN ][org.apache.kafka.clients.NetworkClient][main][540d5db3f43043788c8c88c0e41536de536f338e7ba9b86852861fc54f459599] [Consumer clientId=logstash-0, groupId=logstash] Bootstrap broker localhost:9092 (id: -1 rack: null) disconnected
Edit: added logstash container and stack trace