ThreadPoolExecutor submit sharing value between submits

Viewed 18

I look to have an issue where values are being shard between different ThreadPoolExecutor submits

My code is like the below. In summary it gets a list of shops and then queries a model for a list of products to push to that shop. This is the code

from concurrent.futures import ThreadPoolExecutor
from catalogue_app.management.monitoring_and_reporting import *
from django.core.management.base import BaseCommand
from shop_push.models import ShopApiDefinition, ShopPushQueue


def push_a_product(update_product, shop):
    destination_provider = shop.destination_provider
    print "STEP 4 [**%s** %s] %s: %s: %s" % (shop.process_counter, shop, destination_provider, update_product.change_type, update_product)


def simple(update_product, shop, max_workers_num, product_counter):
    print "STEP 2 [**%s** %s] START Simple: %s %s %s (in simple)" % (product_counter, shop, shop, update_product, max_workers_num)
    process_product(shop, update_product)
    print "[**%s** %s] END Simple: %s %s %s (in simple)" % (product_counter, shop, shop, update_product, max_workers_num)


def do_shop_task(shop):
    print "do_shop_task: %s" % shop
    product_counter = 0
    django_query_fields = {}
    django_query_fields['shopapi'] = shop
    django_query_fields['processed'] = None
    batch_size_shop_push = 5

    got_products = ShopPushQueue.objects.filter(**django_query_fields).order_by('priority', 'pk')[
                   :batch_size_shop_push]
    max_workers_num = 10
    product_counter = 0
    with ThreadPoolExecutor(max_workers=max_workers_num) as executor:
        for update_product in got_products:

            print "STEP 1 [**%s** %s]: %s. ID: %s  (in do_shop_task)" % (product_counter, shop, update_product, update_product.id)
            product_counter += 1

            shop.process_counter = product_counter
            future_list = executor.submit(simple, update_product, shop, max_workers_num, product_counter)

    sleep_time = 3
    print ("Sleeping (push_items) for %s" % sleep_time)
    time.sleep(sleep_time)


def process_product(shop, update_product):
    print("STEP 3 [**%s** %s] Starting process_product. With shop %s product id: %s product: %s (in process_product)" % (shop.process_counter, shop, shop, update_product.id, update_product))
    #time.sleep(1)
    target = push_a_product(update_product, shop)
    print ("[**%s** %s] Finished process_product. With shop %s product id: %s product: %s (in process_product)" % (shop.process_counter,shop, shop, update_product.id, update_product))


def push_to_shops(self):
    print ("Start of push_to_shops")
    shops_to_do = ShopApiDefinition.objects.filter(published=True, definition_type="Queue")
    max_workers_num = 5
    with ThreadPoolExecutor(max_workers=max_workers_num) as executor:
        #shops_to_do
        results = executor.map(do_shop_task, shops_to_do)
        for result in results:
            print("Results push_to_shops: %s" % result)
    print ("End of push_to_shops")


class Command(BaseCommand):
    help = 'Send items from queue to shops'

    def handle(self, *args, **options):
        push_to_shops(self)

The issue is around "for update_product in got_products" The update_product is a product and data about that product is in update_product. I use that data in other commands

What looks to happen is that the value of update_product changes, as it uses a value from other parts of the queryset

Does that make any sense?

Thanks

Grant

0 Answers
Related