Sharing deep/buried tensor

Viewed 44

I want to :

 1. Share deep-tensor(in a sub-class) i.e. multi process access
 2. Feed data to processes, conditionally
 3. Keep the class hierarchy

Below you can see the logic of the process. This NOW works, but what is left .....

1. self.ix should be internal counter i.e instead of 

   self.ix = val - 97

 should be :

   self.ix += 1

It should be shared across the processes, like the tensor.


import os
import torch
import torch.multiprocessing as mp
from torch.multiprocessing import Process
from time import sleep
import queue
import string

class Foo:

    def __init__(self):
        self.mem = torch.zeros(100, dtype=int)
        self.ix = -1

    def share(self): self.mem.share_memory_()
    def add(self,val): 
        self.ix = val - 97
        self.mem[self.ix] = val
        print(f'{os.getpid()} : {self.ix} : {val}')


class Bar:

    def __init__(self): self.foo = Foo()

    def add(self,q) :
        # sleep(int(torch.randint(1,5,(1,))[0]))
        val = None
        try :
            val = q.get(timeout=10)
        except queue.Empty :
            return False    
        self.foo.add(val)
        return True


class Blah:

    def __init__(self):
        self.bar = Bar()

    def process(self,q):
        #... code
        status = True
        while status :
            status = self.bar.add(q)
        #... code

    def run(self):

        #Share the Tensor across processes
        self.bar.foo.share()

        txt = string.ascii_lowercase[:5]

        #every process recv its own data
        q1 = mp.Queue()
        q2 = mp.Queue()

        p1 = Process(target=self.process, args=(q1,))
        p2 = Process(target=self.process, args=(q2,))

        #spawn .............
        p1.start()
        p2.start()

        # p1.join()
        # p2.join()

        #... and then feed data based on conditions
        for i,char in enumerate(txt):
            sleep(1)
            if i % 2 == 0 : 
                print(f'q2.put: {char}')
                q2.put(ord(char))
            else : 
                print(f'q1.put: {char}')
                q1.put(ord(char))

        sleep(20)
        print(self.bar.foo.mem)



if __name__ == '__main__':
    b = Blah()      
    b.run()
0 Answers
Related