I look to have an issue where values are being shard between different ThreadPoolExecutor submits
My code is like the below. In summary it gets a list of shops and then queries a model for a list of products to push to that shop. This is the code
from concurrent.futures import ThreadPoolExecutor
from catalogue_app.management.monitoring_and_reporting import *
from django.core.management.base import BaseCommand
from shop_push.models import ShopApiDefinition, ShopPushQueue
def push_a_product(update_product, shop):
destination_provider = shop.destination_provider
print "STEP 4 [**%s** %s] %s: %s: %s" % (shop.process_counter, shop, destination_provider, update_product.change_type, update_product)
def simple(update_product, shop, max_workers_num, product_counter):
print "STEP 2 [**%s** %s] START Simple: %s %s %s (in simple)" % (product_counter, shop, shop, update_product, max_workers_num)
process_product(shop, update_product)
print "[**%s** %s] END Simple: %s %s %s (in simple)" % (product_counter, shop, shop, update_product, max_workers_num)
def do_shop_task(shop):
print "do_shop_task: %s" % shop
product_counter = 0
django_query_fields = {}
django_query_fields['shopapi'] = shop
django_query_fields['processed'] = None
batch_size_shop_push = 5
got_products = ShopPushQueue.objects.filter(**django_query_fields).order_by('priority', 'pk')[
:batch_size_shop_push]
max_workers_num = 10
product_counter = 0
with ThreadPoolExecutor(max_workers=max_workers_num) as executor:
for update_product in got_products:
print "STEP 1 [**%s** %s]: %s. ID: %s (in do_shop_task)" % (product_counter, shop, update_product, update_product.id)
product_counter += 1
shop.process_counter = product_counter
future_list = executor.submit(simple, update_product, shop, max_workers_num, product_counter)
sleep_time = 3
print ("Sleeping (push_items) for %s" % sleep_time)
time.sleep(sleep_time)
def process_product(shop, update_product):
print("STEP 3 [**%s** %s] Starting process_product. With shop %s product id: %s product: %s (in process_product)" % (shop.process_counter, shop, shop, update_product.id, update_product))
#time.sleep(1)
target = push_a_product(update_product, shop)
print ("[**%s** %s] Finished process_product. With shop %s product id: %s product: %s (in process_product)" % (shop.process_counter,shop, shop, update_product.id, update_product))
def push_to_shops(self):
print ("Start of push_to_shops")
shops_to_do = ShopApiDefinition.objects.filter(published=True, definition_type="Queue")
max_workers_num = 5
with ThreadPoolExecutor(max_workers=max_workers_num) as executor:
#shops_to_do
results = executor.map(do_shop_task, shops_to_do)
for result in results:
print("Results push_to_shops: %s" % result)
print ("End of push_to_shops")
class Command(BaseCommand):
help = 'Send items from queue to shops'
def handle(self, *args, **options):
push_to_shops(self)
The issue is around "for update_product in got_products" The update_product is a product and data about that product is in update_product. I use that data in other commands
What looks to happen is that the value of update_product changes, as it uses a value from other parts of the queryset
Does that make any sense?
Thanks
Grant