XML Insert Performance into MYSQL

Viewed 91

I have some code which inserts the records on the database:

The code is supposed to insert 15M records on the database, right now, it takes 60 hours on a AWS t2.large instance. I'm looking for ways to make the insert on the DB faster while also not duplicating records.

Do you guys have suggestions for me?

I'm using Gorm and MYSQL.


// InsertJob will insert job into database, by checking its hash.
func InsertJob(job XMLJob, oid int, ResourceID int) (Job, error) {
    db := globalDBConnection
    cleanJobDescription := job.Body

    hashString := GetMD5Hash(job.Title + job.Body + job.Location + job.Zip)
    JobDescriptionHash := GetMD5Hash(job.Body)
    empty := sql.NullString{String: "", Valid: true}
    j := Job{
        CurrencyID:              1, //USD

        //other fields here elided for brevity

        PrimaryIndustry: sql.NullString{String: job.PrimaryIndustry, Valid: true},
    }

    err := db.Where("hash = ?", hashString).Find(&j).Error
    if err != nil {
        if err.Error() != "record not found" {
            return j, err
        }

        err2 := db.Create(&j).Error
        if err2 != nil {
            log.Println("Unable to create job:" + err.Error())
            return j, err2
        }
    }

    return j, nil
}
1 Answers
Related