how to post a json API with Parallel processing?

user_13646965 0
Viewed 175

I have code like this:

import requests
from multiprocessing import Pool
import json
import time
from datetime import datetime as dt

BASE_URL = 'http://localhost:3000/test'

with open('data.json', 'r') as f:
    list_dict = json.load(f)


def resource_post(post_data):
    stuff_got = []
    timestamp = dt.now().strftime('%Y%m%d %H:%M:%S:%f')[:-3]  # timestamp
    post_data["timestamp"] = timestamp
    response = requests.post(BASE_URL, json=post_data)
    stuff_got.append(response.json())
    print(stuff_got) # print hasil response
    df = pd.DataFrame.from_dict(stuff_got)
    df.to_csv("output.txt", sep='|', mode='a', index=False, header=False) 
    #time.sleep(0.1) #delay
    return stuff_got

if __name__ == '__main__':  
    start=time.time()
    pool = Pool()
    with pool as p:
        p.map(resource_post,list_dict)
        p.close()
        p.join()
    elapsed = (time.time() - start)  
    print("\n","time elapsed is :", elapsed)

in file data.json:

[{"first_name":"John","last_name":"Swen"},{"first_name":"Ricard","last_name":"Candra"}]

in BASE_URL there is data like this:

{
  "body": {
    "first_name": "Sebastian",
    "last_name": "Eschweiler"
}

I want to post dictionary from data.json with the format:

{
  "body": {
    "first_name":"John",
    "last_name":"Swen"
  }
}

each data above will be looped, and will generate response {"responseCode": "0006", "responseMessage": "success", "first_name": "John"} from the server. I will enter the response into the output.txt file with the format:

0006 | success | John.

so how to make each dictionary in data.json will be posted to the server with the format:

{
  "body": {
    "first_name":"John",
    "last_name":"Swen"
  }
}
parallel-processing post curl api python
1 Answers

I think something like this is what you're after.

  • Use a shared requests.Session() to make things faster.
  • Don't read the input data on the top level; it would be read in separately in all the multiprocessing children too (on Windows, anyway).
  • You weren't checking for error responses at all.
  • You don't need Pandas to write CSV-like data to a file.
  • Writing the data to the files is now a task done by the multiprocessing master, to avoid e.g. two processes messing with each others' writes.
  • Pool.imap_unordered() is faster but unordered. Use imap() if you need ordering.
import json
import time
from datetime import datetime as dt
from multiprocessing import Pool

import requests

BASE_URL = 'http://localhost:3000/test'

sess = requests.Session()


def resource_post(data_from_json):
    post_data = {
        "timestamp": dt.now().strftime('%Y%m%d %H:%M:%S:%f')[:-3],
        "body": data_from_json,
    }
    response = sess.post(BASE_URL, json=post_data)
    response.raise_for_status()
    return response.json()


def main():
    with open('data.json', 'r') as f:
        list_dict = json.load(f)
    start = time.time()
    with open("output.txt", "a") as outf:
        with Pool() as p:
            for resp in p.imap_unordered(resource_post, list_dict):
                print(resp["responseCode"], resp["responseMessage"], resp["first_name"], sep="|", file=outf)
    elapsed = (time.time() - start)
    print("\n", "time elapsed is :", elapsed)


if __name__ == '__main__':
    main()
AKX_516850
Related