Efficiently distribute large number of events over a 1-year time span

Viewed 137

I am trying to efficiently and evenly distribute events over a year. For each year I have about 10^6 events. Each event should be assigned a date. Below I posted my approach which is quite slow. Do you see a way of speeding this up?

for year in range(start_year, end_year + 1):
    for evt in range(events_this_year):
        event_date = (datetime.datetime.strptime(str(year) + "-01-01", "%Y-%m-%d") + datetime.timedelta(days=365 * evt / events_this_year)).strftime("%Y-%m-%d")
2 Answers

You transition back and forth from datetime to str, that's the heavy part.

First of all, you can create the year datetime once in the outer loop and not everytime inside, this will already improves performance by ~3.5x (on my machine).

keeping the results as datetime values instead of strings (if that's OK for you) give a 110x(!!) performance boost:

import datetime
from timeit import timeit

start_year = 2010
end_year = 2020
events_this_year = 10 ** 5


def using_strptime():
    result = []
    for year in range(start_year, end_year + 1):
        for evt in range(events_this_year):
            event_date = (datetime.datetime.strptime(str(year) + "-01-01", "%Y-%m-%d") + datetime.timedelta(
                days=365 * evt / events_this_year)).strftime("%Y-%m-%d")
            result.append(event_date)
    return result


def using_delta():
    result = []
    sec_per_event = datetime.timedelta(seconds=365 * 24 * 60 * 60 // events_this_year)
    for year in range(start_year, end_year + 1):
        year_dt = datetime.datetime(year=year, month=1, day=1)
        cur_dt = year_dt
        for evt in range(events_this_year):
            cur_dt += sec_per_event
            result.append(cur_dt.strftime("%Y-%m-%d"))
    return result


def using_delta_nostring():
    result = []
    sec_per_event = datetime.timedelta(seconds=365 * 24 * 60 * 60 // events_this_year)
    for year in range(start_year, end_year + 1):
        year_dt = datetime.datetime(year=year, month=1, day=1)
        cur_dt = year_dt
        for evt in range(events_this_year):
            cur_dt += sec_per_event
            result.append(cur_dt) # no strftime
    return result


t1 = timeit('using_strptime()', globals=globals(), number=1)
t2 = timeit('using_delta()', globals=globals(), number=1)
t3 = timeit('using_delta_nostring()', globals=globals(), number=1)

print(t1)
print(t2)
print(t3)
print("Ratios:")
print(t1 / t2)
print(t1 / t3)

Output on my machine:

22.7066284
6.213773400000001
0.20198889999999992
Ratios:
3.654241463005393
112.4152287576199

There is no need to ever create dates by strptime.

Compute the delta between elements and create a list of events based on the enumeration of your events_this_year times the position in this range:

import datetime
start_year = 2021
end_year = 2021

events_this_year = 10**6

# using this to compute the available time handles leap years
day_first = datetime.datetime(start_year,1,1,0,0,0,0)
day_last  = datetime.datetime(start_year,12,31,23,59,59,999)

# delta time between events if spaced equally is 
# whole available time divided by number of events
ticks_per_event = (day_last - day_first) / events_this_year

# enumerate range of events and use index (i) to multiply the delta 
# and add that to the 1st day
events = [(day_first + ticks_per_event * i, f"Event: {e}") 
          for i,e in enumerate(range(events_this_year))]

print(events[:5], "...", events[-5:], sep="\n")

Output:

[(datetime.datetime(2021, 1, 1, 0, 0), 'Event: 0'), 
 (datetime.datetime(2021, 1, 1, 0, 0, 31, 535999), 'Event: 1'), 
 (datetime.datetime(2021, 1, 1, 0, 1, 3, 71998), 'Event: 2'), 
 (datetime.datetime(2021, 1, 1, 0, 1, 34, 607997), 'Event: 3'), 
 (datetime.datetime(2021, 1, 1, 0, 2, 6, 143996), 'Event: 4'), 
 (datetime.datetime(2021, 1, 1, 0, 2, 37, 679995), 'Event: 5')]
...
[(datetime.datetime(2021, 12, 31, 23, 57, 21, 320005), 'Event: 999995'), 
 (datetime.datetime(2021, 12, 31, 23, 57, 52, 856004), 'Event: 999996'), 
 (datetime.datetime(2021, 12, 31, 23, 58, 24, 392003), 'Event: 999997'), 
 (datetime.datetime(2021, 12, 31, 23, 58, 55, 928002), 'Event: 999998'), 
 (datetime.datetime(2021, 12, 31, 23, 59, 27, 464001), 'Event: 999999')]

Just collecting the dates (you can zip() them back together them with your eventlist) for

from timeit  import timeit

k ="""
e = []
for year in range(start_year, end_year + 1):
    for evt in range(events_this_year):
        e.append( (datetime.datetime.strptime(str(year) + "-01-01", "%Y-%m-%d") + datetime.timedelta(days=365 * evt / events_this_year)).strftime("%Y-%m-%d"))
"""
print(timeit(k, number=2, globals=globals()))

k = '[day_first + ticks_per_event*i for i,e in enumerate(range(events_this_year))]'
print(timeit(k, number=2, globals=globals()))

takes

100.682846539  # yours
  1.786883751  # mine
Related