How to keep running an async function over and over again, once each task is completed

user_8497767 0
Viewed 167

this little task is giving me real headache.

I just want puppeteer to visit the gaming site, and then wait for the 6 ball result to appear in the Dom, then grab the class[1] of each ball element and their intertext then save it to mongo dB database.

when I run node index.js, it work fine just once. but what I want to do is that, once the async function runs and grab the data's and send it to the database, I want it to either wait for like 3-5 seconds and run again. And continue repeating the same process.

--- visit the gaming website --- scrape the 6 ball result --- save the result to the database --- then repeat from step 1 again.

I don't want to use setInterval because I really don't know, when the async function will complete it process.

So I was just looking for a way that I can make the async function return a value whenever it runs, so by the returned value, I can know that it has completed it task, then I will run it again.

I have included the solutions that I have tried, but the result are not stable. the iife method run just twice. the .then promise method did not work at all, i commented out the two solution below

please what am I doing wrong ?? or may be the scrapedata function itself is badly written. somebody pls help me correct it.

var http = require("http");
const mongoose = require("mongoose");
const puppeteer = require("puppeteer");

mongoose.connect("mongodb://localhost:27017/gameDb", { useNewUrlParser: true, useUnifiedTopology: true, });
const gameSchema = new mongoose.Schema({time: String, ball1color: String, ball1number: Number, ball2color: String, ball2number: Number, ball3color: String, ball3number: Number, ball4color: String, ball4number: Number, ball5color: String, ball5number: Number, ball6color: String, ball6number: Number});

async function scrapeData(url){
const browser = await puppeteer.launch({ headless:false, args: ['--no-sandbox'] });
const page = await browser.newPage();
await page.goto(url);

//wait for the elements that i want to scrape to become visible in the Dom
await page.waitForFunction(()=>{
  return document.querySelectorAll('.ball')[2] && document.querySelectorAll('.ball')[2].style.visibility != 'hidden' && document.querySelectorAll('.ball')[3] && document.querySelectorAll('.ball')[3].style.visibility != 'hidden' && document.querySelectorAll('.ball')[4] && document.querySelectorAll('.ball')[4].style.visibility != 'hidden' && document.querySelectorAll('.ball')[5] && document.querySelectorAll('.ball')[5].style.visibility != 'hidden' && document.querySelectorAll('.ball')[6] && document.querySelectorAll('.ball')[6].style.visibility != 'hidden' && document.querySelectorAll('.ball')[7] && document.querySelectorAll('.ball')[7].style.visibility != 'hidden';
}, {timeout: 100000 });

//wait for the innertext of the 6 balls to appear
await page.waitForFunction(() => {
return document.querySelectorAll(".ball")[2].innerText !== '' && document.querySelectorAll(".ball")[3].innerText !== '' && document.querySelectorAll(".ball")[4].innerText !== '' && document.querySelectorAll(".ball")[5].innerText !== '' && document.querySelectorAll(".ball")[6].innerText !== '' && document.querySelectorAll(".ball")[7].innerText !== '';
 }, { timeout: 100000 });

 //get the color and no of the six balls starting from the left
const [ball1, ball2, ball3, ball4, ball5, ball6] = await page.evaluate(() => {
const ball1 = document.querySelectorAll(".ball")[2]; const ball2 = document.querySelectorAll(".ball")[3];
const ball3 = document.querySelectorAll(".ball")[4]; const ball4 = document.querySelectorAll(".ball")[5]; 
const ball5 = document.querySelectorAll(".ball")[6]; const ball6 = document.querySelectorAll(".ball")[7];
 //return the color and their numbers
return [
{number: ball1.innerText,color: ball1.classList[1].split('-')[1]},{number: ball2.innerText,color: ball2.classList[1].split('-')[1]},
{number: ball3.innerText,color: ball3.classList[1].split('-')[1]},{number: ball4.innerText,color: ball4.classList[1].split('-')[1]},
{number: ball5.innerText,color: ball5.classList[1].split('-')[1]},{number: ball6.innerText,color: ball6.classList[1].split('-')[1]}
  ];
  });

  var gotData;

  //convert time to nigeria timezone 
  function convertTZ(date, tzString) {
    return new Date((typeof date === "string" ? new Date(date) : date).toLocaleString("en-US", {timeZone: tzString}));   
}
const convertDate = convertTZ(new Date(),"Africa/Lagos");

 //change convert getMonth() to human readable formart
 var thisMonth;
switch(convertDate.getMonth()){ case 0: thisMonth = "january"; break; case 1: thisMonth = "february"; break; case 2: thisMonth = "march"; break; case 3: thisMonth = "april"; break; case 4: thisMonth = "may"; break; case 5: thisMonth = "june"; break; case 6: thisMonth = "july"; break; case 7: thisMonth = "august"; break; case 8: thisMonth = "september"; break; case 9: thisMonth = "octomber"; break; case 10: thisMonth = "november"; break; case 11: thisMonth = "december"; break; default: thisMonth = "invalid_month";} 

//create a db collection, name it using today date & month in nigeria timezone
  var dbCollection = thisMonth+"_"+convertDate.getDate();

  //get the current time in Nigeria timezone & save it to a variable
const gameTime = convertDate.getHours()+":"+convertDate.getMinutes()+":"+convertDate.getSeconds();

  //write the result to the database
const collection = mongoose.model(dbCollection, gameSchema);

const Result = new collection({
time: gameTime, ball1color: ball1.color, ball1number: ball1.number,
ball2color: ball2.color, ball2number: ball2.number, 
ball3color: ball3.color, ball3number: ball3.number,
ball4color: ball4.color, ball4number: ball4.number,
ball5color: ball5.color, ball5number: ball5.number,
ball6color: ball6.color, ball6number: ball6.number,
});

Result.save((err, savedDoc, rowsAffected)=>{
if(err){
  gotData = false;
 //return gotData;
}else{
  gotData = true; 
//return gotData;
}
});

 //close the browser
 await browser.close();

 return gotData;
}

// scrapeData("https://logigames.bet9ja.com/Games/Launcher?gameId=11000&provider=0&sid=&pff=1&skin=201").then(isSaved)=>{
//  if(isSaved == true){
//    scrapeData("https://logigames.bet9ja.com/Games/Launcher?gameId=11000&provider=0&sid=&pff=1&skin=201");
//  }else if(isSaved == false){
//  scrapeData("https://logigames.bet9ja.com/Games/Launcher?gameId=11000&provider=0&sid=&pff=1&skin=201");
//  }else{
//    console.log(isSaved + " something is really wrong");
//  }
//    });

// (async ()=>{
//   const isSaved = await scrapeData("https://logigames.bet9ja.com/Games/Launcher?gameId=11000&provider=0&sid=&pff=1&skin=201");
//   if(isSaved == true){
//     scrapeData("https://logigames.bet9ja.com/Games/Launcher?gameId=11000&provider=0&sid=&pff=1&skin=201");
//   }else if(isSaved == false){
//   scrapeData("https://logigames.bet9ja.com/Games/Launcher?gameId=11000&provider=0&sid=&pff=1&skin=201");
//   }else{
//     console.log(isSaved + " something is really wrong");
//   }
// })();

http.createServer((request, response)=>{
  if(request.url == "/" && request.method == "GET"){
  response.writeHead(200, { 'Content-Type': 'text/html' });
  response.end("your app is working", "utf-8");
  }
  
}).listen(process.env.PORT || 3000,()=>{console.log("server working")});

puppeteer async-await node.js mongodb javascript
2 Answers

I want it to either wait for like 3-5 seconds and run again. And continue repeating the same process.

Just use a while loop and a promisified setTimeout:

const wait = ms => new Promise(r => setTimeout(r, ms));

(async => {
  while (true) {
    const isSaved = await scrapeData(url);
    // ...
    await wait(5000);
  }
})();
user_9189100

Here are four tasks. Note they are all async functions. The inner workings of the functions can be either synchronous or asynchronous.

The start() function is recursive. It uses chaining to put each task in sequence. The wait function is a simple timeout that can be adjusted.

In this example, I stop it after four iterations. It could run continuously by removing the conditional.

NOTE: It is imperative that you free up any resources you are creating within each of the task methods. This code is very prone to memory leaks if you are not careful about cleaning up after yourself. Just before the return statement of each task, set any objects created in the function to null.

async function doTask1() {
  console.log('doing task1...');
  return true;
}
async function doTask2() {
  console.log('doing task2...');
  return true;
}
async function doTask3() {
  console.log('doing task3...');
  return true;
}
async function doTask4() {
  console.log('doing task4...');
  return true;
}

function wait() {
  return new Promise((res, rej) => {
    if (counter < 3) {
      console.log('----- waiting -----');
      counter++;
      setTimeout(res, 2000);
    } else {
      console.log('----- DONE -----');
      rej();
    }
  });
}

let counter = 0;

function start() {
  doTask1().then(doTask2).then(doTask3).then(doTask4).then(wait).then(start);
}

start();

user_90783410
Related