Simplify two for loop to one loop

Viewed 15

I need modify my script. Here i use 2 loops.For finding keys of 2 files. I need inside of first "for" loop find all keys, and inside second "for" loop find _deltas for matched entries, but i not clearly understand how to do this. Could someone help me please ?

Both Json Data looks like : Where we have matched values and not matched. if 5 keys in both files matched for entry, so we take this entries and finding delta for time.

Problem that both files contains 5.000.000 entries

    {
"main": [
    {
        "platform_time" : "2022-09-12 00:00:00.034",
        "instrument_name" : "LTC-USDT|KucoinSpot",
        "aggressive_side" : "Bid",
        "exchange_time" : "2022-09-12 00:00:00.023",
        "price" : 62.177,
        "amount" : 1.575967
    },
    {
        "platform_time" : "2022-09-12 00:00:00.399",
        "instrument_name" : "LUNC-USDT|KucoinSpot",
        "aggressive_side" : "Ask",
        "exchange_time" : "2022-09-12 00:00:00.384",
        "price" : 4.3212E-4,
        "amount" : 11570.8599
    },

:

for md_idx, md_el in enumerate(md.get(md_key)):
        for main_idx, main_el in enumerate(main.get(main_key)):

Code

import datetime
import json
from typing import Tuple, List
import numpy as np


def matching(
    md_filename: str,
    md_key: str,
    main_filename: str,
    main_key: str,
    keys_to_compare: List[str],
    key_to_count_delta: str,
    datetime_pattern: str,
) -> Tuple[List[int], int, int, int]:
    with open(md_filename) as file:
        md = json.load(file)
    print('From def Matching#1')
    with open(main_filename) as file:
        main = json.load(file)
    print('From def Matching#2')

md_datetimes = {
    idx: datetime.datetime.strptime(
        el.get(key_to_count_delta), datetime_pattern
    ).timestamp()
    for idx, el in enumerate(md.get(md_key))
}
print('From def Matching#3')
main_datetimes = {
    idx: datetime.datetime.strptime(
        el.get(key_to_count_delta), datetime_pattern
    ).timestamp()
    for idx, el in enumerate(main.get(main_key))
}
print('From def Matching#4')
_deltas = []
print('From def Matching#5')
for md_idx, md_el in enumerate(md.get(md_key)):
    for main_idx, main_el in enumerate(main.get(main_key)):
        if (
            sum(
                1
                for key in keys_to_compare
                if md_el.get(key) == main_el.get(key)
            )
            >= 5
        ):
            _deltas.append(

                    main_datetimes.get(main_idx) -
                    md_datetimes.get(md_idx)


            )
print('From def Matching#6')
_main_bigger = len([el for el in _deltas if el > 0])
print('From def Matching#7')
_equal = len([el for el in _deltas if el == 0])
print('From def Matching#8')
_md_bigger = len([el for el in _deltas if el < 0])
print('From def Matching#9')

return _deltas, _md_bigger, _equal, _main_bigger


if __name__ == "__main__":
    deltas, md_bigger, equal, main_bigger = matching(
        md_filename="md_5K_cp_1.json", # md_all.json md_626K_cp_1.json md_5K_cp_1.json md_2400000.json
        md_key="md", #md
        main_filename="main_5K_cp_1.json",   # main_all.json  main_626K_cp_1.json main_5K_cp_1.json main_2400000.json
        main_key="main", #main
        keys_to_compare=["instrument_name", "aggressive_side", "price", "exchange_time", "amount"],
        key_to_count_delta="platform_time",
        datetime_pattern="%Y-%m-%d %H:%M:%S.%f",
    )

    print('deltas = ',deltas)
    print('main_bigger = ',main_bigger)
    print('equal = ',equal)
    print('md_bigger = ',md_bigger)
    print()
    print('main_bigger ',
          round((main_bigger / (main_bigger + md_bigger + equal)) * 100, 4), '% case')
    print('equal',
          round((equal / (main_bigger + md_bigger + equal)) * 100, 4), '% case')
    print('md_bigger',
          round((md_bigger / (main_bigger + md_bigger + equal)) * 100, 4),
          '% case', '\n')

    print("Q25 quantile of deltas_array  : ", round(np.quantile(deltas, .25), 4), 'seconds')
    print("Q50 quantile of deltas_array  : ", round(np.quantile(deltas, .50), 4), 'seconds')
    print("Q75 quantile of deltas_array  : ", round(np.quantile(deltas, .75), 4), 'seconds')
0 Answers
Related