Hello Everyone
I have the following script :
cluster = Cluster(['ip'])
session = cluster.connect()
session.row_factory = ordered_dict_factory
result_find = session.execute("cassandra query (select)")
jsonaki={"start":{}}
for item in result_find :
with open("giga_idea.json" , "w") as f:
newItem=[]
values = item.items()
for value in values :
newItem.append(value[1])
if(newItem[0] == "DEVICE"):
if newItem[2] in jsonaki:
if str(newItem[1]) not in jsonaki[newItem[2]]:
jsonaki[newItem[2]][str(newItem[1])]=[newItem[3]]
else:
if newItem[3] not in jsonaki[newItem[2]][str(newItem[1])]:
jsonaki[newItem[2]][str(newItem[1])].append(newItem[3])
else:
jsonaki[newItem[2]]={str(newItem[1]):[newItem[3]]}
f.write(str(jsonaki))
f.close()
The script runs with no problem but it is really slow (i am looking into a 90gb database table). I am trying to make it faster and i can not understand if i need to parallel the first loop (for item in result_find) or the second one with all the if statements.
I tried the following code as is and with small changes with no luck :
def complex_operation(item) :
newItem=[]
values = item.items()
for value in values :
newItem.append(value[1])
if(newItem[0] == "DEVICE"):
if newItem[2] in jsonaki:
if str(newItem[1]) not in jsonaki[newItem[2]]:
jsonaki[newItem[2]][str(newItem[1])]=[newItem[3]]
else:
if newItem[3] not in jsonaki[newItem[2]][str(newItem[1])]:
jsonaki[newItem[2]][str(newItem[1])].append(newItem[3])
else:
jsonaki[newItem[2]]={str(newItem[1]):[newItem[3]]}
def run_complex_operations(operation, input, pool):
pool.map(operation, input)
def main():
cluster = Cluster(['ip'])
session = cluster.connect()
session.row_factory = ordered_dict_factory
processes_count = 4
processes_pool = Pool(processes_count)
result_find = session.execute("cassandra query (select)")
jsonaki={"start":{}}
for item in result_find :
with open("giga_idea.json" , "w") as f:
run_complex_operations(complex_operation, range(4), processes_pool)
f.write(str(jsonaki))
f.close()
cluster.shutdown()
Any idea on what should i do or which part should i try to parallel?
Thanks for your time!
Keep Coding :)