INSERT to MySQL 8.0 Table too slow from Python program, New to python. Appreciate your time

Viewed 26
import mysql.connector
from datetime import datetime
from datetime import timedelta


try:
    def init_change_detector():
        print("Change Detector started at " + datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
        conn = mysql.connector.connect(user='user_app',
                                        password='xxxxxxxxxxxx',
                                        host='test.unit.com',
                                        database='devdb')
        cursor = conn.cursor()
        query = ('SELECT table_name, column_name, key_name '
                 'FROM csj_change_detect_table_column '
                 'ORDER BY table_name, column_name')
        cursor.execute(query)
        # get all records
        records = cursor.fetchall()
        for record in records:
            process_col_tab_chg(record[0], record[1], record[2])
        if  conn.is_connected():
            conn.close()
            cursor.close()


    def insert_change_log(table_name, key_name, attr_name, old_attr_value, new_attr_value):
        insert_query = """INSERT INTO csj_shipment_changelog(table_name, key_name,
                                                             attr_name, old_attr_value,
                                                             new_attr_value)
                                                           VALUES (%s, %s, %s, %s, %s)"""
        conn2 = mysql.connector.connect(user='new_user',
                                        password='xxxxxxxxxxxx',
                                        host='test.unit.com',
                                        database='devdb')
        cursor2 = conn2.cursor()
        tuple1 = (table_name, key_name, attr_name, old_attr_value, new_attr_value)
        cursor2.execute(insert_query, tuple1)
        conn2.commit()
        # if row is None:
        #     row = 1;
        # else:
        #     row= row+1;
        # if row == 100:
        #     quit()
        cursor2.close()
        conn2.close()


    # Look for Shipment, in past date
    def find_past_shipment(table_name,
                           key_name,
                           column_name,
                           before_date,
                           curr_key
                           ):
        saved_col_name=column_name
        saved_key_name=key_name
        conn4 = mysql.connector.connect(user='new_user',
                                        password='xxxxxxxxxxxx',
                                        host='test.unit.com',
                                        database='devdb')
        cursor4 = conn4.cursor()
        query4 = 'SELECT  ' + saved_key_name + ' , ' + saved_col_name + ' FROM ' + table_name \
                 + ' where rec_cre_dt_utc < ' + "'" + before_date.strftime('%Y-%m-%d 00:00:00') + "'" \
                 + ' and shipment_num = ' + "'" + curr_key + "'" + ' order by rec_cre_dt_utc desc LIMIT 1'
        cursor4.execute(query4)
        records = cursor4.fetchone()
        if records is not None:
            past_attr_val = records[1]
            return past_attr_val
        else:
            return 0

    def process_col_tab_chg(table_name, column_name, key_name):
        saved_key_name = key_name
        saved_col_name = column_name
        old_val = 0
        ini_time_for_now = datetime.now()
        date_before_1day = ini_time_for_now - timedelta(days=1)
        query = 'SELECT  ' + key_name + ' , ' + saved_col_name + ' , ' + ' rec_cre_dt_utc FROM ' + table_name \
                + ' where rec_cre_dt_utc >= ' + "'" + date_before_1day.strftime('%Y-%m-%d 00:00:00') + "'"
        conn3 = mysql.connector.connect(user='new_user',
                                        password='xxxxxxxxxxxx',
                                        host='test.unit.com',
                                        database='devdb')
        cursor3 = conn3.cursor()
        cursor3.execute(query)
        for (key_name, column_name, rec_cre_dt_utc) in cursor3:
            curr_attr_val = column_name
            curr_key_val = key_name
            old_val = find_past_shipment(table_name,
                                         saved_key_name,
                                         saved_col_name,
                                         rec_cre_dt_utc,
                                         curr_key_val
                                         )
            if   curr_attr_val != old_val \
                    and  old_val != 0:
                insert_change_log(table_name, key_name, saved_col_name, old_val, curr_attr_val )
            else:
                continue

        cursor3.close
        conn3.close()


    def cleanup():
        print("Change Detector stopped " + datetime.now().strftime("%Y-%m-%d %H:%M:%S"))


    def main():
        start = datetime.now()
        init_change_detector()
        end = datetime.now()
        time_diff = (end - start)
        execution_time = time_diff.total_seconds()
        print("Elapsed time(secs): " + str(execution_time))


    if __name__ == "__main__":
        main()

except Exception as e:
    print("Exception " + e)
finally:
    cleanup()
1 Answers

Switch to executemany(). I recommend limiting to 1000 rows per batch.

Related