I have a primary collection that I use directly with my app, and a secondary collection that has the same data but is updated from our source daily. Each document represents a real-life oil well (relevant for my code sample).
Each daily update to the secondary collection may have changes in some of the properties of the documents compared to the primary collection (e.g. "status" property may go from active to inactive), or there may be entirely new documents added that day.
I have a script where I compare the secondary collection to the primary collection that ultimately updates the primary collection with the changes or newly added documents, but at the same time creates "notification" objects, which I push to user documents to alert them of changes in documents that they "follow" (or any new documents added that are relevant to them for various reasons). I also consolidate them by user at the end, so that I'm pushing just 1-2 combined notifications to each user instead of possibly 10-20.
My question is, is there a way to make my script more efficient, maybe by converting it all to MongoDB Query Language (MQL) vs using Javascript for loops? It currently takes 1-2 hours on our collection of 39k documents.
(Note: I run this script manually in the mongosh shell, currently only once a month, but would like to eventually increase frequency, likely daily since that's what we're capable of.)
(Note 2: I'm currently only tracking document changes for the "operator" and "statusAbbr" properties, and then any "newWells". That is sufficient.)
(Note 3: I'm a newer programmer, and I'm happy to have made it "work", but now I want to learn how to make it better.)
let userNotifications = []
let secondaryCollection = db.secondaryCollection.find().toArray()
// GENERATE NOTIFICATIONS (PER WELL) (multiple notifications per user per well possible)
(async () => {
function notificationType(well, doc) {
if (well) {
if (well.statusAbbr != doc.statusAbbr) {
return "wellStatusChanges"
}
else if (well.operator != doc.operator) {
return "wellOperatorChanges"
}
else {
return ""
}
}
else {
return "newWells"
}
}
for (let doc of secondaryCollection) {
let well = null
let result = ""
if (db.primaryCollection.find({ndic: doc.ndic}).count() > 0) {
// UPDATE DB (well's status/statusHistory || operator/operatorId/operatorHistory)
well = db.primaryCollection.find({ndic: doc.ndic}).toArray()
well = well[0]
doc._id = well._id
result = notificationType(well, doc) // wellStatusChanges || wellOperatorChanges || ""
if (result == "wellStatusChanges") {
doc.statusHistory = [...well.statusHistory].unshift({
date: new Date(),
statusNew: doc.statusAbbr,
statusOld: well.statusAbbr
})
db.primaryCollection.findOneAndUpdate({_id: new ObjectId(doc._id)}, {$set: {statusHistory: doc.statusHistory, statusAbbr: doc.statusAbbr}})
}
else if (result == "wellOperatorChanges") {
if (!doc.operatorId) {
let result = await db.operators.find({name: doc.operator}).toArray()
if (result.length == 1) {
doc.operatorId = result._id
}
else {
let newOperator = await db.operators.insertOne({...operatorTemplate, name: doc.operator})
doc.operatorId = newOperator.insertedId
}
}
doc.operatorHistory = [...well.operatorHistory].unshift({
date: new Date(),
operatorNew: doc.operator,
operatorIdNew: doc.operatorId,
operatorOld: well.operator,
operatorIdOld: well.operatorId,
})
db.primaryCollection.findOneAndUpdate({_id: new ObjectId(doc._id)}, {$set: {operatorHistory: doc.operatorHistory, operator: doc.operator, operatorId: new ObjectId(doc.operatorId)}})
}
}
else {
result = notificationType(well, doc) // should be "newWells"
doc = {...wellTemplate, ...doc}
let createdDoc = await db.primaryCollection.insertOne(doc); //TODO: need to await ??
doc._id = createdDoc.insertedId // in ObjectId form
}
// (2) UPDATE USERNOTIFICATIONS (USER/ACCOUNT)
if (result != "") {
// script that determines who relevant users are for each change / new well added,
// generates an individual notification for each change, and pushes to userNotifications
}
}
})()
// AGGREGATE NOTIFICATIONS (FROM ALL WELLS) & UPDATE USERS' NOTIFICATIONS
(async () => {
// (1) script that consolidates notifications from userNotifications based on user
// (2) FOR EACH CONSOLIDATION/USER, UPDATE USER
await Promise.all(userNotificationsCombined.map(async (notif) => {
try {
let update = await db.users.findOneAndUpdate({_id: new ObjectId(notif.userId)},
{$push: {notifications: notif}}
)
}
catch {
console.log("couldn't find user ", notif.userId)
}
}))
})()