I am making a bot to fetch the details of Facebook posts such as comments, reactions, and dates posted. I have done with the comments and reactions count but I couldn't find how can i get the timestamp of the post using puppeteer in node js
here is my full code
You can see I have done multiple actions here only need to add a get timestamp action bu hard to find a way of doing it
const puppeteer = require("puppeteer");
const ObjectsToCsv = require("objects-to-csv");
import { parse } from "node-html-parser";
const config = {
response_timeout: 10000,
headless: false,
base_url: "https://www.facebook.com/",
user: "https://www.facebook.com/abdul.aleem.58323",
username_field: 'input[name="email"]',
password_field: 'input[name="pass"]',
login_button: 'button[type="submit"]',
username_invalid_msg:
"The email address or mobile number you entered isn't connected to an account.",
password_invalid_msg: "The password that you've entered is incorrect",
credentials_invalid_msg_01: "Wrong Credentials",
credentials_invalid_msg_02: "Invalid username or password",
regex_remove_non_numeric: /[^0-9]/g,
page_end_msg: "search results only include things visible to you.",
page_not_available_msg: "this page isn't available",
content_not_available_msg: "this content isn't available right now",
content_not_available_msg_02: "this content isn't available at the moment",
content_not_available_msg_03: "we didn't find anything",
user_profile_check_msg: "add friend",
no_results_msg: "we didn't find any results",
end_of_results: "end of results",
date_time_format: "DD/MM/YYYY HH:mm:ss",
close_button_selector: "div[aria-label='Close']",
seven_days_timestamp: 7 * 24 * 60 * 60 * 1000,
login_button_name: "log in",
public_post_exclude_keyword: "see all public posts for",
};
const CRED = {
user: "",
pass: "",
};
const ID = {
login: "#email",
pass: "#pass",
};
export default async function handler(req, res) {
switch (req.method) {
case "POST":
// login to facebook
const sleep = async (ms) => {
return new Promise((res, rej) => {
setTimeout(() => {
res();
}, ms);
});
};
// let bodyObject = JSON.parse(req.body);
const browser = await puppeteer.launch({ headless: !true });
const context = browser.defaultBrowserContext();
context.overridePermissions("https://www.facebook.com/ZainMalik223/", [
"geolocation",
"notifications",
]);
const page = await browser.newPage();
// login
await page.goto(config.base_url, {
waitUntil: "networkidle2",
});
// -----------------------------------------------Login into Facebook----------------------------------
await page.waitForSelector(config.username_field, {
timeout: config.response_timeout,
});
await page.type(config.username_field, CRED.user, { delay: 50 });
await page.type(config.password_field, CRED.pass, { delay: 50 });
await page.click(config.login_button);
await page.waitForNavigation({ waitUntil: "networkidle2" });
await page.waitForTimeout(1000 + Math.floor(Math.random() * 500));
let bodyHTML = await page.evaluate(() => document.body.innerHTML);
let bodyString = bodyHTML.toString();
let loginResponse = {};
if (bodyString.includes(config.username_invalid_msg)) {
loginResponse.authenticated = false;
loginResponse.user = false;
} else if (bodyString.includes(config.password_invalid_msg)) {
loginResponse.authenticated = false;
loginResponse.user = true;
} else if (
bodyString.includes(config.credentials_invalid_msg_01) ||
bodyString.includes(config.credentials_invalid_msg_02)
) {
loginResponse.authenticated = false;
loginResponse.user = false;
} else {
loginResponse.success = true;
loginResponse.user = true;
loginResponse.authenticated = true;
}
await page.waitForTimeout(500 + Math.floor(Math.random() * 500));
// -----------------------------------------------Login into Facebook End----------------------------------
// -----------------------------------------------Check for page is exist----------------------------------
await page.goto(config.user, {
waitUntil: "networkidle2",
});
let is_not_found = false;
const page_spans = await page.$$eval("span", (spans) =>
spans.map((span) => {
return { text: span.textContent };
})
);
for (let i = 0; i < page_spans.length; i++) {
let page_text = page_spans[i].text;
if (
page_text.toLowerCase() === config.page_not_available_msg ||
page_text.toLowerCase() === config.user_profile_check_msg ||
page_text.toLowerCase() === config.no_results_msg ||
page_text.toLowerCase() === config.end_of_results ||
page_text.toLowerCase() === config.content_not_available_msg ||
page_text.toLowerCase() === config.content_not_available_msg_02 ||
page_text.toLowerCase() === config.content_not_available_msg_03
) {
is_not_found = true;
break;
}
}
const page_h2s = await page.$$eval("h2", (h2s) =>
h2s.map((h2) => {
return { text: h2.textContent };
})
);
for (let i = 0; i < page_h2s.length; i++) {
let page_text = page_h2s[i].text;
if (page_text.toLowerCase() === config.page_not_available_msg) {
is_not_found = true;
break;
}
}
// -----------------------------------------------Check for page is exist End----------------------------------
// -----------------------------------------------Get counted post----------------------------------
let post_count = 40;
let sevenDaysPostFetched = false;
let i = 0;
let total_filtered_list = [];
while (!sevenDaysPostFetched) {
page.evaluate(async () => {
await window.scrollBy(0, document.body.scrollHeight);
});
await page.waitForTimeout(3000 + Math.floor(Math.random() * 500));
const divs = await page.$$eval('div[role="article"]', (divs) =>
divs.map((div) => div.innerHTML)
);
await page.$$eval('div[role="button"]', (elements) => {
elements.find((e) => {
if (e.textContent === "See more") {
e.click();
}
});
});
await page.waitForTimeout(3000 + Math.floor(Math.random() * 500));
if (divs.length < 1) {
continue;
}
let is_search_finsihed = await checkFinishedSearch();
if (divs.length > post_count || is_search_finsihed) {
break;
}
total_filtered_list[i] = divs.length;
if (
total_filtered_list.length > 1 &&
total_filtered_list[i - 1] == total_filtered_list[i]
) {
sevenDaysPostFetched = true;
}
i++;
}
async function checkFinishedSearch() {
let is_search_results_finish = false;
const page_spans = await page.$$eval("span", (spans) =>
spans.map((span) => {
return { text: span.textContent };
})
);
for (let i = 0; i < page_spans.length; i++) {
let page_text = page_spans[i].text;
if (page_text.toLowerCase() === config.page_end_msg) {
is_search_results_finish = true;
break;
}
}
return is_search_results_finish;
}
// -----------------------------------------------Get counted post End----------------------------------
// -----------------------------------------------Get Filtered Posts----------------------------------
const divs = await page.$$eval('div[role="article"]', (divs) =>
divs.map((div) => {
return {
text: div.textContent,
html: div.innerHTML,
ariaLabel: div.ariaLabel,
};
})
);
let filtered_list = [];
for (let i = 0; i < divs.length; i++) {
if (divs[i].ariaLabel == null) {
filtered_list.push(divs[i]);
}
}
// -----------------------------------------------Get Filtered Posts ENd----------------------------------
function formatDataCounts(data_count) {
if (data_count.includes("K")) {
return parseInt(data_count) * 1000;
} else if (data_count.includes("M")) {
return parseInt(data_count) * 1000 * 1000;
} else {
return parseInt(data_count);
}
}
// -----------------------------------------------Extract Post Comments Counts----------------------------------
function getCommentsCount(root_html) {
const spans = root_html.querySelectorAll('span[dir="auto"]');
let comment_regex = /^[0-9 ]+\scomment/;
let comment_text = 0;
let data_count = 0;
for (let x = 0; x < spans.length; x++) {
let comment_count = spans[x].innerText.toLowerCase();
if (comment_count.match(comment_regex)) {
comment_text = spans[x].innerText;
break;
}
}
if (comment_text !== 0) {
data_count = formatDataCounts(
comment_text.replace(config.regex_remove_non_numeric, "")
);
}
return data_count;
}
// -----------------------------------------------Extract Post Comments Counts End----------------------------------
// -----------------------------------------------Extract Post Reactions Counts Start----------------------------------
function getReactions(root_html) {
const spans2 = root_html.querySelectorAll('span[aria-hidden="true"]');
let reaction_regex = /\d/;
let reaction_text = 0;
for (let x = 0; x < spans2.length; x++) {
const reaction_count = spans2[x].innerText.toLowerCase();
if (reaction_count.match(reaction_regex)) {
reaction_text = spans2[x].innerText;
break;
}
}
const totalReactions = formatDataCounts(reaction_text.toString());
const reactions = { totalReactions };
const hahaSpan = root_html.querySelectorAll(
'span[aria-label="See who reacted to this"] div'
);
for (let x = 0; x < hahaSpan.length; x++) {
const text = hahaSpan[x].getAttribute("aria-label");
const key = text.slice(0, text.indexOf(":"));
const numbers = text.match(/\d+/)[0];
reactions[key] = parseInt(numbers);
}
return reactions;
}
// -----------------------------------------------Extract Post Reactions Counts End----------------------------------
// -----------------------------------------------Get Share Count----------------------------------
function getShareCount(root_html) {
const spans = root_html.querySelectorAll('span[dir="auto"]');
let share_regex = /^[0-9 ]+\sshare/;
let share_text = 0;
let data_count = 0;
for (let x = 0; x < spans.length; x++) {
const shareCount = spans[x].innerText.toLowerCase();
if (shareCount.match(share_regex)) {
share_text = spans[x].innerText;
break;
}
}
if (share_text !== 0) {
data_count = formatDataCounts(
share_text.replace(config.regex_remove_non_numeric, "")
);
}
return data_count;
}
// -----------------------------------------------Get Share Count End----------------------------------
// -----------------------------------------------Get Timesitemap----------------------------------
// -----------------------------------------------Get Post url and id ----------------------------------
async function getOEmbedContent(iterator) {
await page.evaluate(function (arg) {
let actions = document.querySelectorAll(
'div[aria-label = "Actions for this post"]'
)[arg];
if (actions !== undefined) {
actions.click();
}
}, iterator);
await page.waitForTimeout(1500 + Math.floor(Math.random() * 500));
await page.$$eval("span", (elements) => {
elements.find((e) => {
if (e.textContent.trim() === "Embed") {
e.click();
}
});
});
await page.waitForTimeout(1000 + Math.floor(Math.random() * 500));
await page.$$eval('input[name="checkbox"]', (elements) => {
if (elements[0] !== undefined) {
elements[0].click();
}
});
await page.waitForSelector('input[aria-label = "Sample code input"]', {
delay: 40000,
});
await page.waitForTimeout(1000 + Math.floor(Math.random() * 500));
const embeds = await page.$$eval(
'input[aria-label = "Sample code input"]',
(embeds) => embeds.map((embed) => embed.defaultValue)
);
await page.waitForTimeout(1000 + Math.floor(Math.random() * 500));
if (embeds !== undefined) {
if (embeds[embeds.length - 1] !== undefined) {
} else {
}
return embeds[embeds.length - 1];
} else {
return null;
}
}
function extractPostUrl(oEmbed) {
const src = oEmbed.split("href=")[1].split(/[ >]/)[0];
let url = src.replace(/%2F/gi, "/");
url = url.replace(/%3A/gi, ":");
let post_url = url.split("&")[0];
if (post_url.includes("/photos")) {
let first_part = post_url.split("/photos/")[0];
let second_part = post_url.split("/photos/")[1];
let post_id = second_part.split("/")[1];
post_url = `${first_part}/photos/${post_id}`;
}
return post_url;
}
function extractPostId(post_url) {
if (post_url.includes("posts/")) {
return post_url.split("posts/")[1];
} else if (post_url.includes("videos/")) {
return post_url.split("videos/")[1];
} else if (post_url.includes("photos/")) {
return post_url.split("photos/")[1];
}
}
// -----------------------------------------------Get Post url and id End----------------------------------
const fullData = [];
for (let index = 0; index < filtered_list.length; index++) {
const root_html = parse(filtered_list[index].html);
const comments = getCommentsCount(root_html);
const reactions = getReactions(root_html);
const shares = getShareCount(root_html);
const postOEmbedContent = await getOEmbedContent(index);
const postUrl = extractPostUrl(postOEmbedContent);
const postId = extractPostId(postUrl);
if (true) {
fullData.push({
comments,
reactions,
shares,
postUrl,
postId,
// CreatedAt,
postOEmbedContent,
});
}
}
const csv = new ObjectsToCsv(fullData);
await csv.toDisk("./list.csv");
await csv.toDisk("./list.csv", { append: true });
res.json({ status: 200, data: fullData });
// await browser.close();
break;
} }