I want to :
1. Share deep-tensor(in a sub-class) i.e. multi process access
2. Feed data to processes, conditionally
3. Keep the class hierarchy
Below you can see the logic of the process. This NOW works, but what is left .....
1. self.ix should be internal counter i.e instead of
self.ix = val - 97
should be :
self.ix += 1
It should be shared across the processes, like the tensor.
import os
import torch
import torch.multiprocessing as mp
from torch.multiprocessing import Process
from time import sleep
import queue
import string
class Foo:
def __init__(self):
self.mem = torch.zeros(100, dtype=int)
self.ix = -1
def share(self): self.mem.share_memory_()
def add(self,val):
self.ix = val - 97
self.mem[self.ix] = val
print(f'{os.getpid()} : {self.ix} : {val}')
class Bar:
def __init__(self): self.foo = Foo()
def add(self,q) :
# sleep(int(torch.randint(1,5,(1,))[0]))
val = None
try :
val = q.get(timeout=10)
except queue.Empty :
return False
self.foo.add(val)
return True
class Blah:
def __init__(self):
self.bar = Bar()
def process(self,q):
#... code
status = True
while status :
status = self.bar.add(q)
#... code
def run(self):
#Share the Tensor across processes
self.bar.foo.share()
txt = string.ascii_lowercase[:5]
#every process recv its own data
q1 = mp.Queue()
q2 = mp.Queue()
p1 = Process(target=self.process, args=(q1,))
p2 = Process(target=self.process, args=(q2,))
#spawn .............
p1.start()
p2.start()
# p1.join()
# p2.join()
#... and then feed data based on conditions
for i,char in enumerate(txt):
sleep(1)
if i % 2 == 0 :
print(f'q2.put: {char}')
q2.put(ord(char))
else :
print(f'q1.put: {char}')
q1.put(ord(char))
sleep(20)
print(self.bar.foo.mem)
if __name__ == '__main__':
b = Blah()
b.run()