Running the following Python script parallel

Viewed 53

Hello Everyone

I have the following script :

   cluster = Cluster(['ip'])
    session = cluster.connect()
    session.row_factory = ordered_dict_factory
        
    result_find = session.execute("cassandra query (select)")
    jsonaki={"start":{}}
        
    for item in result_find :
      with open("giga_idea.json" , "w") as f:
        newItem=[]
        values = item.items()
        for value in values :
          newItem.append(value[1])
        
        if(newItem[0] == "DEVICE"):
          if newItem[2] in jsonaki:
            if str(newItem[1]) not in jsonaki[newItem[2]]:
              jsonaki[newItem[2]][str(newItem[1])]=[newItem[3]]
            else:
              if newItem[3] not in jsonaki[newItem[2]][str(newItem[1])]:
                jsonaki[newItem[2]][str(newItem[1])].append(newItem[3])
          else:
            jsonaki[newItem[2]]={str(newItem[1]):[newItem[3]]}
        f.write(str(jsonaki))
    f.close()       
         

The script runs with no problem but it is really slow (i am looking into a 90gb database table). I am trying to make it faster and i can not understand if i need to parallel the first loop (for item in result_find) or the second one with all the if statements.

I tried the following code as is and with small changes with no luck :

def complex_operation(item) :
  newItem=[]
  values = item.items()
  for value in values :
    newItem.append(value[1])
        
    if(newItem[0] == "DEVICE"):
      if newItem[2] in jsonaki:
        if str(newItem[1]) not in jsonaki[newItem[2]]:
          jsonaki[newItem[2]][str(newItem[1])]=[newItem[3]]
        else:
          if newItem[3] not in jsonaki[newItem[2]][str(newItem[1])]:
            jsonaki[newItem[2]][str(newItem[1])].append(newItem[3])
      else:
        jsonaki[newItem[2]]={str(newItem[1]):[newItem[3]]}
            

def run_complex_operations(operation, input, pool):
    pool.map(operation, input)

def main():
    
    cluster = Cluster(['ip'])
    session = cluster.connect()
    session.row_factory = ordered_dict_factory
   
    processes_count = 4  
    processes_pool = Pool(processes_count)
   
    result_find = session.execute("cassandra query (select)")
    jsonaki={"start":{}}
 
    for item in result_find :
        with open("giga_idea.json" , "w") as f:
        run_complex_operations(complex_operation, range(4), processes_pool)
        f.write(str(jsonaki))
    f.close()       
    
    cluster.shutdown()

Any idea on what should i do or which part should i try to parallel?

Thanks for your time!
Keep Coding :)

0 Answers
Related