I am looking to find a faster solution than pandas to_sql to ingest a pandas df into postgres database.
import time
import psycopg2
df = pd.read_csv('upload_test_data.csv')
conn = psycopg2.connect(user=username,password=password,
host=host,port=port,database=db)
cursor = conn.cursor()
#perform COPY test and print result
sql = '''
COPY copy_test
FROM 'C:\\Users\\Documents\\copy_test.csv'
DELIMITER ',' CSV;
'''
table_create_sql = '''
CREATE TABLE IF NOT EXISTS copy_test (id bigint,
quantity int,
cost double precision,
total_revenue double precision)
'''
cursor.execute(table_create_sql)
cursor.execute('TRUNCATE TABLE copy_test')
start_time = time.time()
df.to_csv('upload_test_data_from_copy.csv', index=False, header=False)
cursor.execute(sql)
conn.commit()
cursor.close()
print("COPY duration: {} seconds".format(time.time() - start_time))
The code above returns an error:
FeatureNotSupported: COPY from a file is not supported
As the goal is to automate the process, I prefer that the solution is fully in Python to switching between different software.
Version: PostgreSQL 13.7 on x86_64-pc-linux-gnu