Cloud function does not write second argument to Firebase

Question

I have written a cloud function that runs every 5 minutes on my Firebase app. In essence, the function gathers trends data from the Google Trends website and parses the JSON into a variable. After doing so I want to then connect to the Twitter API and search for tweets using the trending topics fetched in the first part. My Issue seems to lie with the second part. It fetches the data but the remainder of the function does not wait for the result before writing to Firebase.

I have tried two different methods but both don't seem to work as intended. I am struggling to understand how the function should wait for the second part to gather and store the information before writing to Firebase.

Method 1

exports.callTo = functions.pubsub.schedule("5 * * * *").onRun((context) => {
  let searchTrends;
  const ts = Date.now();
  const dateOb = new Date(ts);
  const date = dateOb.getDate();
  const month = dateOb.getMonth() + 1;
  const year = dateOb.getFullYear();
  const twitterTrends = [];

  googleTrends.dailyTrends({
    trendDate: new Date(year + "-" + month + "-" + date),
    geo: "CA",
  }, function(err, res) {
    if (err) {
      functions.logger.error(err);
    } else {
      searchTrends = JSON.parse(res).default.trendingSearchesDays[0]
          .trendingSearches;
      functions.logger.info(searchTrends);
      for (let i = 0; i < searchTrends.length; i++) {
        functions.logger.log(searchTrends[i].title.query);
        T.get("search/tweets", {q: searchTrends[i].title.query, count: 1},
            function(err, data, response) {
              if (err) {
                functions.logger.error(err);
              }
              functions.logger.info("Twitter data" +
              JSON.stringify(data.statuses));
              twitterTrends[i] = JSON.stringify(data.statuses);
            });
      }
      
      const dbRef = admin.database().ref("searchTrends");
      dbRef.set({google: searchTrends, twitter: twitterTrends});
    }
  });
});

Method 2

exports.callTo = functions.pubsub.schedule("5 * * * *").onRun((context) => {
  let searchTrends;
  const ts = Date.now();
  const dateOb = new Date(ts);
  const date = dateOb.getDate();
  const month = dateOb.getMonth() + 1;
  const year = dateOb.getFullYear();
  const twitterTrends = [];

  async function getTrends(){
    googleTrends.dailyTrends({
      trendDate: new Date(year + "-" + month + "-" + date),
      geo: "CA",
    }, function(err, res) {
      if (err) {
        functions.logger.error(err);
      } else {
        searchTrends = JSON.parse(res).default.trendingSearchesDays[0]
            .trendingSearches;
        functions.logger.info(searchTrends);
      }
    });
    await getTwitterTrends();
  }
  
  async function getTwitterTrends(){
    for (let i = 0; i < 1; i++) {
      functions.logger.log(searchTrends[i].title.query);
      T.get("search/tweets", {q: searchTrends[i].title.query, count: 1},
          function(err, data, response) {
            if (err) {
              functions.logger.error(err);
            } else {
              functions.logger.info("Twitter data" +
              JSON.stringify(data.statuses));
              twitterTrends[i] = JSON.stringify(data.statuses);
            }
          });
    }
    return "done";
  }
      
  const dbRef = admin.database().ref("searchTrends");
  dbRef.set({google: searchTrends, twitter: twitterTrends});
    
});

user_16929894 · Answer

After checking your function it looks like a Promises issue. The reason you are seeing only the searchTrends data in Firestore is because the Firestore reference and upload is being done inside the callback for the dailyTrends method (taking for reference the method 1 code). However this does not wait for each request to the Twitter API to be resolved before writing to Firestore.

Based on the documentation for twit (which seems to be the wrapper you are using), it also supports standard promises. You could add each promise to an array, and then use Promise.all() to wait until they are all resolved to then write the data into Firestore. It would look something like this (which I haven’t tested since I don’t have Twitter API access).

exports.callTo = functions.pubsub.schedule("5 * * * *").onRun((context) => {
  const ts = Date.now();
  const dateOb = new Date(ts);
  const date = dateOb.getDate();
  const month = dateOb.getMonth() + 1;
  const year = dateOb.getFullYear();

  let searchTrends;
  const twitterTrends = [];
  const twPromises = [];


  googleTrends.dailyTrends({
    trendDate: new Date(year + "-" + month + "-" + date),
    geo: "CA",
  }, function(err, res) {
    if (err) {
      functions.logger.error(err);
    } else {
      searchTrends = JSON.parse(res).default.trendingSearchesDays[0]
          .trendingSearches;
      functions.logger.info(searchTrends);

      for (let i = 0; i < searchTrends.length; i++) {
        functions.logger.log(searchTrends[i].title.query);
        twPromises.push(T.get("search/tweets", {q: searchTrends[i].title.query, count: 1})); // adds promises to the array
      }
      
      Promise.all(twPromises).then((responses) => { // runs when all promises from the array are resolved
        responses.forEach((response) => {
            twitterTrends.push(JSON.stringify(response.statuses));
        })
        const dbRef = admin.database().ref("searchTrends");
        dbRef.set({google: searchTrends, twitter: twitterTrends});
      })
    }
  });
});

Cloud function does not write second argument to Firebase

1 Answers