Node js requests and cheerio wait for page to fully load

Admin 1
Viewed 15418

I'm trying to scrape images off a page but the page returns a placeholder source attr if that page isn't fully loaded, (takes about 0.5 seconds to fully load) how would I make request wait?

tried doing

function findCommonMovies(movie, callback){

    request('http://www.imdb.com/find?ref_=nv_sr_fn&q='+ movie +'&s=all', function (error, response, body) {
      if (error){
          return
      }else{
          var $ = cheerio.load(body);
          var title = $(".result_text").first().text().split("(")[0].split(" ").join('')
          var commonMovies = []
          // var endurl = $("a[name=tt] .result_text a").attr("href")
          var endurl = $('a[name=tt]').parent().parent().find(".findSection .findList .findResult .result_text a").attr("href");


          request('http://www.imdb.com' + endurl, function (err, response, body) {

              if (err){
                  console.log(err)
              }else{

                  setInterval(function(){var $ = cheerio.load(body)}, 2000)

                  $(".rec_page .rec_item a img").each(function(){


                    var title = $(this).attr("title")
                    var image = $(this).attr("src")

                    commonMovies.push({title: title, image: image})
                  });
              }
              callback(commonMovies)
          });
      }
    });

}
findCommonMovies("Gotham", function(common){
  console.log(common)
})
node.js request javascript
4 Answers

you can set timeout:

var options = {
    url : 'http://www.imdb.com/find?ref_=nv_sr_fn&q='+ movie +'&s=all',
    timeout: 10000 //set waiting time till 10 minutes.
  }
  request(options, function(err, response, body){
    if (err) {
      console.log(err);
    }
   //do what you want here
}
user_53101140

setTimeout(function, millseconds to wait) will pause for how many seconds you want. setTimeout(function(){var $ = cheerio.load(body)}, 2000)

user_86374640

It appears to me like your callback is located in the wrong place and there should be no need for any timer. When request() calls its callback, the whole response is ready so no need for a timer.

Here's the code with the callback in the right place and also changed so that it has an error argument so the caller can propagate and detect errors:

function findCommonMovies(movie, callback){
    request('http://www.imdb.com/find?ref_=nv_sr_fn&q='+ movie +'&s=all', function (error, response, body) {
      if (error) {
          callback(error);
          return;
      } else {
          var $ = cheerio.load(body);
          var title = $(".result_text").first().text().split("(")[0].split(" ").join('')
          var commonMovies = [];
          // var endurl = $("a[name=tt] .result_text a").attr("href")
          var endurl = $('a[name=tt]').parent().parent().find(".findSection .findList .findResult .result_text a").attr("href");
          request('http://www.imdb.com' + endurl, function (err, response, body) {
              if (err) {
                  console.log(err)
                  callback(err); 
              } else {
                  var $ = cheerio.load(body);
                  $(".rec_page .rec_item a img").each(function(){
                    var title = $(this).attr("title");
                    var image = $(this).attr("src");
                    commonMovies.push({title, image});
                  });
                  callback(null, commonMovies);
              }
          });
       }
    });
}

findCommonMovies("Gotham", function(err, common) {
  if (err) {
     console.log(err);
  } else {
     console.log(common)
  }
});

Note: This will access ONLY the HTML markup served by the server for the URLs you request. If those pages have content that is inserted by browser Javascript, that content will not be present in what you get here and no delay will make it appear. That's because cheerio does not run browser Javascript, it JUST parses the HTML that the server originally sends. To run browser Javascript, you need a more complete browser engine than cheerio provides such as PhantomJS that will actually run the page's Javascript.

user_8166200
Related