I know that there were similar questions but I couldn't find a solution to my problem. My problem is that I can't send messages from a ZMQ-publisher running on my local host, to a ZMQ-subscriber that is running inside a container that also runs an HTTP server.
I have a simple HTTP server written in python (Flask). All this HTTP server is doing is to create an instance of a ZMQ subscriber and start it(start it = make it start listening for incoming messages)
I am able to send GET messages to the HTTP server from my local host, but when the publisher sends its messages the subscriber never gets them.
My HTTP server looks like this:
import simple_subscriber
server = Flask(__name__)
import logging
from logging import getLogger
import json
'''
This is a simple http server that is responsible to get all the messages
that the simple_subscriber received and send them as a jason array to the http client
'''
logging.basicConfig(level=logging.DEBUG)
logger = getLogger(__name__)
simple_subscriber_instance = None
@server.route('/')
@server.route('/startListening')
def startSubscriber():
logger.info("starting to listen")
global simple_subscriber_instance
simple_subscriber_instance = simple_subscriber.SimpleSubscriber._start("tcp", "localhost", 5563, 5566, "topic_in",
"topic_out")
return jsonify(success=True)
@server.route('/killSubAndGetAllMessages')
def getAllMessages():
'''
make the simple_subscriber_instance to stop listening for incoming messages
and get all the messages that been received until this function executes.
:return: a json array of all the messages that been received
'''
all_meassages = simple_subscriber_instance.kill()
json_list = json.dumps(all_meassages)
logger.info("returning all messages")
return json_list
if __name__ == "__main__":
server.run(host="0.0.0.0")
my subscriber:
import zmq
import logging
from logging import getLogger
from typing import List, Tuple, Dict, ByteString, Callable
logging.basicConfig(level=logging.DEBUG)
class SimpleSubscriber:
def __init__(self, context: zmq.Context, socket_in, socket_out, poller, in_topic, out_topic):
self._logger = getLogger(__name__)
self._context = context
self._socket_in = socket_in
self._socket_out = socket_out
self._poller = poller
self.in_topic = in_topic
self.out_topic = out_topic
self._isAlive = None
self._worker = Thread(target=self._listener)
self._worker.setName("Simple-subscriber")
self._worker.isDaemon()
self._messages = []
@staticmethod
def _init_zmq(in_protocol: str, in_ip: str, in_port: str,
out_port, in_topic: str, out_topic: str):
out_addr = f"tcp://*:{out_port}"
in_addr = f"{in_protocol}://{in_ip}:{in_port}"
in_topic= in_topic.encode("utf-8")
out_topic= out_topic.encode("utf-8")
context = zmq.Context()
socket_in = context.socket(zmq.SUB)
socket_in.setsockopt(zmq.SUBSCRIBE, in_topic)
socket_in.connect(in_addr)
socket_out = context.socket(zmq.PUB)
socket_out.bind(out_addr)
poller = zmq.Poller()
poller.register(socket_in)
return context, socket_in, socket_out, poller, in_topic, out_topic
@classmethod
def _start(cls, in_protocol: str, in_ip: str, in_port: int, out_port: int, in_topic: str, out_topic: str):
logger = getLogger(__name__)
logger.info("starting simple-subscriber")
context, socket_in, socket_out, poller, in_topic, out_topic = SimpleSubscriber._init_zmq(
in_protocol, in_ip, in_port, out_port, in_topic, out_topic)
instance = cls(context, socket_in, socket_out, poller, in_topic, out_topic)
instance._isAlive = True
logger.info("starting listener function")
instance._worker.start()
return instance
def kill(self):
self._logger.info("killing worker")
self._isAlive = False
return self._messages
def _listener(self):
while self._isAlive:
msg_map = dict(self._poller.poll(10))
if self._socket_in in msg_map and msg_map[self._socket_in]:
msg = self._socket_in.recv()
self._logger.info("Received request: %s" % msg)
self._messages.append(msg)
self._socket_in.close()
self._socket_out.close()
self._logger.info("Worker is dead")
and my publisher that runs on the localhost(from outside the container)
import time
import zmq
out_addr = "tcp://localhost:5563"
context = zmq.Context()
socket_out = context.socket(zmq.PUB)
socket_out.bind(out_addr)
time.sleep(2)
for i in range(10):
topic = b"topic_in"
socket_out.send(topic+b" "+str(i))
In the case of out_addr = "tcp://localhost:5563" the messages are sent but never gets to the subscriber, and in the case of out_addr = "tcp://*:5563" I am getting this error:
socket_out.bind(out_addr)
File "zmq/backend/cython/socket.pyx", line 550, in zmq.backend.cython.socket.Socket.bind
File "zmq/backend/cython/checkrc.pxd", line 26, in zmq.backend.cython.checkrc._check_rc
zmq.error.ZMQError: Address already in use
Process finished with exit code 1
The flow is like this:
- running the container
- sending http://localhost:5000/startListening
- executing the publisher script
- sending http://localhost:5000/killSubAndGetAllMessages
I never gets the messages back (I know that step 3 doesn't reach the subscriber because it also should print the messages received)
Any ideas how can I make the messages, sent from the publisher on the local host, reach the subscriber running inside the container alongside the server?
This is my docker file:
FROM ubuntu:18.04
RUN apt-get update && apt-get install -y \
software-properties-common
RUN add-apt-repository ppa:deadsnakes/ppa
RUN apt-get update && apt-get install -y \
python3.7 \
python3-pip
RUN python3.7 -m pip install pip
RUN apt-get update && apt-get install -y \
python3-distutils \
python3-setuptools
COPY . /app
WORKDIR /app
RUN pip3 install -r req.txt
EXPOSE 5000
EXPOSE 5563
CMD ["python3","http_server.py"]
and this is the command I used to run the container docker run -p 5000:5000 -p 5563:5563 -it simple_subscriber_test
Everything is working fine when I run it all from my local host but I need the server and the subscriber inside a container.