Not resovalbe hostnames of RabbitMQ cluster when deployed on Docker Swarm using the replica parameter

Viewed 690

I use Docker Swarm to deploy 3 instances of RabbitMQ and Consul for the peer discovery.

version: '3.7'

services:
  rabbitmq-1:
    image: rabbitmq:3.8.9-alpine
    hostname: "rabbitmq-1"
    environment:
      - RABBITMQ_ERLANG_COOKIE="SECRET_COOKIE"
    networks:
      - prod-net
    configs:
      - source: rabbitmq-config
        target: /etc/rabbitmq/rabbitmq.conf
      - source: rabbitmq-plugins-config
        target: /etc/rabbitmq/enabled_plugins
    volumes:
      - /shared/rabbitmq/1/config:/etc/rabbitmq
      - /shared/rabbitmq/1/data:/var/lib/rabbitmq
    deploy:      
      replicas: 1

  rabbitmq-2:
    image: rabbitmq:3.8.9-alpine
    hostname: "rabbitmq-2"
    environment:
      - RABBITMQ_ERLANG_COOKIE="SECRET_COOKIE"
    networks:
      - prod-net
    configs:
      - source: rabbitmq-config
        target: /etc/rabbitmq/rabbitmq.conf
      - source: rabbitmq-plugins-config
        target: /etc/rabbitmq/enabled_plugins
    volumes:
      - /shared/rabbitmq/2/config:/etc/rabbitmq
      - /shared/rabbitmq/2/data:/var/lib/rabbitmq
    deploy:      
      replicas: 1

  rabbitmq-3:
    image: rabbitmq:3.8.9-alpine
    hostname: "rabbitmq-3"
    environment:
      - RABBITMQ_ERLANG_COOKIE="SECRET_COOKIE"
    networks:
      - prod-net
    configs:
      - source: rabbitmq-config
        target: /etc/rabbitmq/rabbitmq.conf
      - source: rabbitmq-plugins-config
        target: /etc/rabbitmq/enabled_plugins
    volumes:
      - /shared/rabbitmq/3/config:/etc/rabbitmq
      - /shared/rabbitmq/3/data:/var/lib/rabbitmq
    deploy:      
      replicas: 1

networks:
  consul-net:
    external: true
    name: prod-net

configs:
  rabbitmq-config:
    external: true
  rabbitmq-plugins-config:
    external: true

RabbitMQ registers itself with Consul by its hostname so that the nodes can discover each other. So all the hostnames must resolve on all nodes. In the example above I'm using the same value for both the hostname and the service name to achieve this. But I would like to use a more compact way to represent a cluster by using a replicas: 3 parameter.

version: '3.7'

services:
  rabbitmq:
    image: rabbitmq:3.8.9-alpine
    hostname: "rabbitmq-{{.Task.Slot}}"
    networks:
      - prod-net
    configs:
      - source: rabbitmq-config
        target: /etc/rabbitmq/rabbitmq.conf
      - source: rabbitmq-plugins-config
        target: /etc/rabbitmq/enabled_plugins
    deploy:      
      replicas: 3
      restart_policy:
        condition: any

networks:
  prod-net:
    external: true

configs:
  rabbitmq-config:
    external: true
  rabbitmq-plugins-config:
    external: true

Sadly in this case the hostnames are not anymore resolvable by docker DNS and the nodes cannot see each other. It there a way to achieve this ?

Edit.1:

Using these in rabbitmq-config gives the errors bellow:

cluster_formation.peer_discovery_backend = rabbit_peer_discovery_dns
cluster_formation.dns.hostname = tasks.rabbitmq
2020-10-15 22:06:01.573 [error] <0.272.0> attempted to contact: ['rabbit@rabbitmq-3','rabbit@rabbitmq-2']
2020-10-15 22:06:01.574 [error] <0.272.0> rabbit@rabbitmq-3:
2020-10-15 22:06:01.574 [error] <0.272.0>   * unable to connect to epmd (port 4369) on rabbitmq-3: couldn't resolve hostname
2020-10-15 22:06:01.574 [error] <0.272.0> rabbit@rabbitmq-2:
2020-10-15 22:06:01.575 [error] <0.272.0>   * unable to connect to epmd (port 4369) on rabbitmq-2: couldn't resolve hostname

Edit.2: rabbitmq.conf using Consul peer discovery

# Credentials
default_user = admin
default_pass = Pa$$w0rd1

loopback_users.admin = false
vm_memory_high_watermark.absolute = 1024MiB
disk_free_limit.absolute = 5GB
loopback_users.guest = false

# TLS Support in RabbitMQ
listeners.ssl.default = 5671

# Disables non-TLS listeners, only TLS-enabled clients will be able to connect
listeners.tcp = none

ssl_options.cacertfile = /etc/rabbitmq/ca_certificate.pem
ssl_options.certfile   = /etc/rabbitmq/server_certificate.pem
ssl_options.keyfile    = /etc/rabbitmq/server_key.pem
ssl_options.password   = Pa$$phr@se
ssl_options.verify     = verify_peer
ssl_options.fail_if_no_peer_cert = true

# TLS Support in RabbitMQ UI
management.ssl.port       = 15671
management.ssl.cacertfile = /etc/rabbitmq/ca_certificate.pem
management.ssl.certfile   = /etc/rabbitmq/server_certificate.pem
management.ssl.keyfile    = /etc/rabbitmq/server_key.pem
management.ssl.password   = Pa$$phr@se
management.ssl.verify     = verify_none
management.ssl.fail_if_no_peer_cert = false

management.ssl.honor_cipher_order   = true
management.ssl.honor_ecc_order      = true
management.ssl.client_renegotiation = false
management.ssl.secure_renegotiate   = true

cluster_partition_handling = autoheal

# Consul peer discovery
cluster_formation.peer_discovery_backend = rabbit_peer_discovery_consul

# Consul host (hostname or IP address)
cluster_formation.consul.host = consul_client

# Service name (as registered in Consul) defaults to "rabbitmq"
cluster_formation.consul.svc = rabbitmq

# Compute service address (if not specify it below) 
cluster_formation.consul.svc_addr_auto = true
cluster_formation.consul.use_longname = true
cluster_formation.consul.svc_ttl = 50
cluster_formation.consul.deregister_after = 100
cluster_formation.node_cleanup.only_log_warning = true

Or using DNS peer discovery:

#DNS Peer Discovery
cluster_formation.peer_discovery_backend = rabbit_peer_discovery_dns
cluster_formation.dns.hostname = tasks.rabbitmq
1 Answers

To achieve DNS resolve between RabbitMQ replicas, need to modify /etc/hosts of each RabbitMQ container.

This will allow to form RabbitMQ cluster via rabbtimq configuration file.

See demo on how to create RabbitMQ cluster using single docker swarm service. I removed references to Consul and TLS from rabbitmq.conf for simplicity.

Related