Basically, the problem is that I take a screenshot of a page, then extract the HTML and they really don't match.
I am trying to get the list of new messages from the new Facebook Messenger (website version) using puppeteer. I didn't have problems before Facebook's latest version but now I do. Here is the code:
var util = require('util');
const puppeteer = require('puppeteer');
(async () => {
const browser = await puppeteer.launch({args: ['--no-sandbox', '--shm-size=2gb']});
const page = await browser.newPage();
await page.setViewport({width:1940, height:1500});
await page.setUserAgent('Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_5) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.1.1 Safari/605.1.15');
await page.goto('https://facebook.com');
await page.screenshot({path: 'captures/1-login.png'});
await page.waitFor(3000);
await page.waitForSelector('#email');
await page.type('#email', 'your username', {delay:100});
await page.type('#pass', 'your password', {delay:100});
await page.screenshot({path: 'captures/2-email.png'});
await page.waitFor('input[data-testid="royal_login_button"]');
await page.click('input[data-testid="royal_login_button"]');
await page.waitFor(2000);
await page.screenshot({path: 'captures/3-login.png'});
var ok=true;
await page.waitFor('a[href="https://www.facebook.com/messages/t/"]').catch(function(error) {
console.log(error);
ok=false;
});
if (!ok) {
await browser.close();
}
await page.waitFor(5000);
await page.screenshot({path: 'captures/6-main-page.png'});
await page.waitFor('a[href="https://www.facebook.com/messages/t/"]');
await page.click('a[href="https://www.facebook.com/messages/t/"]');
await page.waitFor(5000);
await page.screenshot({path: 'captures/7-messenger.png'});
const bodyHTML = await page.evaluate(() => document.querySelector('*').outerHTML);
console.log('START HTML: '+bodyHTML);
let messageLinksText = await page.evaluate((sel) => {
var unreads=document.querySelectorAll(sel);
var links='';
if (unreads!==null) {
for (i = 0; i < unreads.length; i++) {
links+=unreads[i].getAttribute('data-href')+';';
}
}
return links;
}, 'li[aria-live="polite"] > div > a').catch(function(error) {
console.log('ERROR messageLinks: '+error);
});
console.log('unread: '+messageLinksText+' messages');
let messageLinks=messageLinksText.split(';');
console.log('checking '+(messageLinks.length-1)+' unread message(s)');
await browser.close();
})();
The new messages have the li[aria-live="polite"] selector, if I use the web developper console from any browser, I can see them in the Messenger page. The screenshot of the messages (you'll see it in 7-messenger.png if you run the code) shows all the messages but when I look at the HTML from bodyHTML in the code, it doesn't have any of the messages, just the page's top part. I've tried 5-6 different ways of getting the HTML, everything from page.content() to various versions of evaluate like in this code.
Am I missing something?
Thanks!
Note: the "personal profile" messenger does not have an API, it's just for business pages.