I have these csv files on hand that I have to upload to a remote database and I've used pyodbc with the csv library in python to do it.I don't know why but it's insanely slow(about 30 seconds for a 100 rows) and some of these csv files I have to upload have over 30k rows. I've tried using pandas as well but there was no change in speed. This is more or less my code.Unnecessary parts have been omitted.
if len(sys.argv) == 1:
print("This program needs needs an input state")
exit()
state_code = str(sys.argv[1])
f = open(state_code+".csv", "r")
reader = csv.reader(f, delimiter=',')
cnxn = pyodbc.connect('DRIVER={ODBC Driver 17 for SQL Server};SERVER='+server+';DATABASE='+database+';UID='+username+';PWD='+ password)
insert_query = '''INSERT INTO table (Zipcode, Pers_Property_Coverage, Deductible,
Liability,Average_Rate,
Highest_Rate, Lowest_Rate, CREATE_DATE, Active_Flag)
VALUES(?,?,?,?,?,?,?,?,?)'''
for row in reader:
zipcode = row[0]
if len(zipcode) == 4:
zipcode = "0" + zipcode
ppc=row[1][1:]
ppc=ppc.replace(',', '')
deductible = row[2][1:]
deductible = deductible.replace(',', '')
liability = row[3][1:]
liability = liability.replace(',', '')
average_rate = row[4][1:]
average_rate = average_rate.replace(',', '')
highest_rate = row[5][1:]
highest_rate = highest_rate.replace(',', '')
lowest_rate=row[6][1:]
lowest_rate = lowest_rate.replace(',', '')
ctr=ctr+1
if ctr % 100 == 0:
print("Time Elapsed = ", round(time.time() - start_time)," seconds")
values = (zipcode, ppc, deductible, liability, average_rate, highest_rate, lowest_rate, date, "Y")
print("Inserting "+zipcode ,ppc , deductible, liability, average_rate, highest_rate, lowest_rate,date, "Y")
cursor.execute(insert_query, values)
cnxn.commit()