Enforce rebalance when a new topic is created

Viewed 100

I'm writing a chat application, with chat managed via Kafka topics, where a topic corresponds to a channel in the application. The application allows users to create their own channels, with their own unique names. To achieve this, I'm using a @KafkaListener with topicPattern = "chat_.*". When a user creates a cars channel, for example, the topic chat_cars will be created in Kafka. The listener class looks like this:

MessageListener.java

@Component
public class MessageListener {
    @Autowired
    SimpMessagingTemplate template;

    @KafkaListener(
            topicPattern = "chat_.*",
            groupId = "kafka-sandbox"
    )
    public void listen(ChatMessage message) {
        System.out.println("sending via kafka listener..");
        template.convertAndSend("/topic/" + message.getChannel(), message);
    }
}

The trouble is that the listener does not detect this new topic right away. I found this answer from a few years back explaining that there is a delay before the listener updates. The answer explains that there are a few ways to trigger a rebalance, however I'm wondering is there a way to automatically trigger the rebalance, and thus update the listener, as soon as I add the new topic?

I'm sending the message via a KafkaTemplate, the function looks like this:

@MessageMapping("/chat/{channel}")
    public void send(@DestinationVariable String channel, final ChatMessage message) {
        try {
            //Sending the message to kafka topic queue
            System.out.println("Sending message");
            kafkaTemplate.send("chat_" + channel, message).get();
            System.out.println("Message sent");
        } catch (InterruptedException | ExecutionException e) {
            throw new RuntimeException(e);
        }
    }

ListenerConfig.java

@EnableKafka
@Configuration
public class ListenerConfig {
    @Bean
    ConcurrentKafkaListenerContainerFactory<String, ChatMessage> kafkaListenerContainerFactory() {
        ConcurrentKafkaListenerContainerFactory<String, ChatMessage> factory = new ConcurrentKafkaListenerContainerFactory<>();
        factory.setConsumerFactory(consumerFactory());
        return factory;
    }

    @Bean
    public ConsumerFactory<String, ChatMessage> consumerFactory() {
        return new DefaultKafkaConsumerFactory<>(consumerConfigurations(), new StringDeserializer(), new JsonDeserializer<>(ChatMessage.class));
    }

    @Bean
    public Map<String, Object> consumerConfigurations() {
        Map<String, Object> configurations = new HashMap<>();
        configurations.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "kafka2:9092");
        configurations.put(ConsumerConfig.GROUP_ID_CONFIG, "kafka-sandbox");
        configurations.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
        configurations.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, JsonDeserializer.class);
        configurations.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "latest");
        return configurations;
    }
}

ProducerConfiguration.java

@EnableKafka
@Configuration
public class ProducerConfiguration {
    @Bean
    public ProducerFactory<String, ChatMessage> producerFactory() {
        return new DefaultKafkaProducerFactory<>(producerConfigurations());
    }

    @Bean
    public Map<String, Object> producerConfigurations() {
        Map<String, Object> configurations = new HashMap<>();
        configurations.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "kafka2:9092");
        configurations.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class);
        configurations.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, JsonSerializer.class);
        return configurations;
    }

    @Bean
    public KafkaTemplate<String, ChatMessage> kafkaTemplate() {
        return new KafkaTemplate<>(producerFactory());
    }
}
0 Answers
Related