When a broker host goes down/restarts, the kafka client keeps sending the new requests to the same hosts. As a result, the requests are failing with this error: Request timed out after 30000ms.
Node version: v6.8.1
Kafka-node version: 3.0.1
Kafka version: 2.11-2.0.0
Number of Brokers: 3
Number partitions for topic: 10
Some code pointers:
"clusterConfig" : {
"kafkaHost": "localhost:9092,localhost:9093,localhost:9094",
"autoConnect": true
}
...
let kafkaClient = new kafka.KafkaClient(clusterConfig);
producer = new kafka.HighLevelProducer(kafkaClient, cluster.producerConfig);
Promise.promisifyAll(producer);
...
producer.sendAsync([eventPayload])
.then(function (data) {
let topicName = eventPayload.topic;
let payLoadSize = (eventPayload || '').length;
logger.eventInfo(topicName, payLoadSize, source);
})
.catch(function (e) {
logger.produceFailedEvent(eventPayload, source);
throw Error.getErrorObject(errorType, e, topic, source);
});
I have the kept the other configurations to default. Please help in resolving the issue.
Thanks, Shashank