Generating millions of rows using faker is very slow

Viewed 257

Am trying to generate 3.3 million fake rows using python as below snippet. generating the file is very very slow. any help speedup this?

Python version - 3.9.7

import os, csv, time, sys
from datetime import datetime
from faker import Faker
from time import sleep
from progress.bar import Bar


os.system('clear')
sCount = "distID.in"

fake = Faker()
startTime = datetime.now()

count = sum(1 for line in open(sCount))
fakeFile = open('fakeFile.csv', 'w')
bar = Bar('Processing', max=count)
with open(sCount) as piiFile:

    i=666000000
    for oldID in piiFile:
        i=i+1
        fn = fake.first_name()
        ln = fake.last_name()
        dob = (f'{fake.date_of_birth()}')
    
        fakeFile.write(f'{i},{fn},{ln},{dob},{oldID}'+'\n')
        bar.next()

fakeFile.close()
bar.finish()
0 Answers
Related