I am using the code below to scroll all the way to the bottom of a YouTube page and it works. My question is after the site is scrolled down to the bottom how can I console.log that the bottom have been reached? Thanks in advance.
note: the solution should work with youtube.com. I have already tried getting the document height and compared it with the scroll height but that didn't work!
const puppeteer = require('puppeteer');
let thumbArr = []
const scrapeInfiniteScrollItems = async(page) => {
while (true) {
const previousHeight = await page.evaluate(
"document.querySelector('ytd-app').scrollHeight"
);
await page.evaluate(() => {
const youtubeScrollHeight =
document.querySelector("ytd-app").scrollHeight;
window.scrollTo(0, youtubeScrollHeight);
});
await page.waitForFunction(
`document.querySelector('ytd-app').scrollHeight > ${previousHeight}`, {
timeout: 0
}
);
const thumbnailLength = (await page.$$('ytd-grid-video-renderer')).length
//this logs the amount of thumbnails every loop but once bottom scroll has been reached it stops logging (obviously) but the question is how am I supposed to compare the last amount of thumbnail's found with total thumbnails once the loop has stopped running. Take a look below to better understand my question.
thumbArr.push(thumbnailLength)
if (thumbnailLength == thumbArr.at(-1)) {
console.log('bottom has been reached')
}
await page.waitForTimeout(1000)
}
};
(async() => {
const browser = await puppeteer.launch({
headless: false
});
const page = await browser.newPage();
await page.goto('https://www.youtube.com', {
waitUntil: 'networkidle2',
});
await scrapeInfiniteScrollItems(page)
})();UPDATE:
let clientHeightArr = []
let clientHeightArrTracker = []
const scrapeInfiniteScrollItems = async(browser, page) => {
var infiniteScrollTrackerInterval = setInterval(async() => {
clientHeightArrTracker.push(clientHeightArr.length)
if (clientHeightArrTracker.some((e, i, arr) => arr.indexOf(e) !== i) == true) {
clearInterval(infiniteScrollTrackerInterval)
console.log('Bottom is reached')
//causes error "ProtocolError: Protocol error (Runtime.callFunctionOn): Target closed."
await browser.close()
}
}, 2000)
while (true) {
const previousHeight = await page.evaluate(
"document.querySelector('ytd-app').scrollHeight"
);
await page.evaluate(() => {
const youtubeScrollHeight =
document.querySelector("ytd-app").scrollHeight;
window.scrollTo(0, youtubeScrollHeight);
});
await page.waitForFunction(
`document.querySelector('ytd-app').scrollHeight > ${previousHeight}`, {
timeout: 0
},
);
const clientHeight = await page.$$eval("ytd-app", el => el.map(x => x.clientHeight));
clientHeightArr.push(clientHeight[0])
await page.waitForTimeout(1000)
}
};
(async() => {
const browser = await puppeteer.launch({
headless: false
});
const page = await browser.newPage();
await page.goto('https://www.youtube.com/c/mkbhd/videos', {
waitUntil: 'networkidle2',
});
await scrapeInfiniteScrollItems(browser, page)
})();