I am trying to stream continuous data over the network to matplotlib. In principle this is working, but occasionally - esp. at startup of the UI or when I resize the window - the data gets chopped. I tried thread locking but no success. A little generator that I run over netcat:
from random import random as rand
while True:
print(f'{rand()};{rand()};{rand()};{rand()};{rand()}')
run over netcat:
python3 genrndcsv.py | nc localhost 5027
then I run my UI client:
import matplotlib
import collections
#selecting the right backend, change qt4agg to your desired backend
##matplotlib.use('qt4agg')
import matplotlib.pyplot as plt
import matplotlib.animation as animation
import socket
import threading
from random import random as rand
lock = threading.Lock()
#command to open the pipe
#datapipe = open('path to your pipe','r')
socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
socket.bind(('', 5027))
socket.listen(5)
client, address = socket.accept()
print("{} connected".format( address ))
#amount of data to be displayed at once, this is the size of the x axis
#increasing this amount also makes plotting slightly slower
data_amount = 1000
#set the size of the deque object
datalist = collections.deque([0]*data_amount,data_amount)
#configure the graph itself
fig, ax = plt.subplots()
line, = ax.plot([0,]*data_amount)
#size of the y axis is set here
ax.set_ylim(0,256)
def update(data):
line.set_ydata(data)
return line,
def data_gen1():
f = client.makefile()
while True:
#l = f'{rand()};{rand()};{rand()};{rand()};{rand()}'
l = f.readline()
eol = l.find('\n')
if eol < 0:
print(f'n0pe')
exit(-1)
l.strip('\n')
#print(f't {type(l)} {l}')
vals = [float(x) for x in l.split(';')]
#print(f'v {vals[0]}')
datalist.append(vals[0])
yield datalist
BLEN=256
def data_gen():
response = bytearray(b'')
while True:
lock.acquire()
if len(response) < BLEN/2:
rx = client.recv(BLEN)
print(f'type(rx){type(rx)}')
if len(rx) > 0:
print(f'len {len(rx)}')
response.extend(bytearray(rx))
eol = response.find(b'\n')
if eol != -1:
l = response[:eol]
##print(f'len={len(response)} eol={eol} r_pre {response}')
print(f'r_pre {response}')
response = response[eol+1:]
print(f'r_post {response}')
vals = [float(x) for x in l.split(b';')]
print(f'{vals[0]}')
print()
datalist.append(vals[0])
lock.release()
yield datalist
ani = animation.FuncAnimation(fig,update,data_gen1,interval=1, blit=True)
plt.show()
As you can see I tried datagen1() and a more manual datagen() but both behave the same. I captured the network data and it is OK. However I get huge spikes (which com from misinterpreted strings) like this when I resize the Window: spikes in data
the error becomes very obvious when I log r_pre and r_post in data_gen(): in the the error case r_pre is not equal to the previous r_post. If I exxagerate resizing the window I quickly run into a parsing error where not only large numbers but '' or ';' or '123.234.345' are interpreted as numbers, which obvously fails:
('127.0.0.1', 54281) connected
Exception in Tkinter callback
Traceback (most recent call last):
File "C:\Program Files\Python38\lib\tkinter\__init__.py", line 1892, in __call__
return self.func(*args)
File "C:\Program Files\Python38\lib\tkinter\__init__.py", line 814, in callit
func(*args)
File "C:\Program Files\Python38\lib\site-packages\matplotlib\backends\_backend_tk.py", line 139, in _on_timer
super()._on_timer()
File "C:\Program Files\Python38\lib\site-packages\matplotlib\backend_bases.py", line 1216, in _on_timer
ret = func(*args, **kwargs)
File "C:\Program Files\Python38\lib\site-packages\matplotlib\animation.py", line 1477, in _step
still_going = super()._step(*args)
File "C:\Program Files\Python38\lib\site-packages\matplotlib\animation.py", line 1188, in _step
framedata = next(self.frame_seq)
File ".\streamtestoffline.py", line 50, in data_gen1
vals = [float(x) for x in l.split(';')]
File ".\streamtestoffline.py", line 50, in <listcomp>
vals = [float(x) for x in l.split(';')]
ValueError: could not convert string to float: ''
Exception in Tkinter callback
Traceback (most recent call last):
File "C:\Program Files\Python38\lib\tkinter\__init__.py", line 1892, in __call__
return self.func(*args)
File "C:\Program Files\Python38\lib\tkinter\__init__.py", line 814, in callit
func(*args)
File "C:\Program Files\Python38\lib\site-packages\matplotlib\backends\_backend_tk.py", line 139, in _on_timer
super()._on_timer()
File "C:\Program Files\Python38\lib\site-packages\matplotlib\backend_bases.py", line 1216, in _on_timer
ret = func(*args, **kwargs)
File "C:\Program Files\Python38\lib\site-packages\matplotlib\animation.py", line 1477, in _step
still_going = super()._step(*args)
File "C:\Program Files\Python38\lib\site-packages\matplotlib\animation.py", line 1188, in _step
framedata = next(self.frame_seq)
File ".\streamtestoffline.py", line 50, in data_gen1
vals = [float(x) for x in l.split(';')]
File ".\streamtestoffline.py", line 50, in <listcomp>
vals = [float(x) for x in l.split(';')]
ValueError: could not convert string to float: '0.291439.7957812126795342'
any help to make this robust is very much appreciated