I'm scraping the trending articles from Medium using Puppeteer.
But, after going to the targeted URL (page of single article e.g. https://royals.mlblogs.com/?source=home---------0---------------------54c56bca_541c_4664_80aa_ea1a69f93942-------7) after 1-2 seconds it's auto redirecting to wanted URL: https://royals.mlblogs.com/
I don't know why this kind of behavior is happening. How to keep into the targeted URL?
Here is my code:
'use strict';
const puppeteer = require('puppeteer');
(async () => {
try {
const browser = await puppeteer.launch({
args: [`Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/%s Safari/537.36`],
headless: false,
devtools: true,
defaultViewport: {
width: 1920,
height: 1080
}
});
const page = await browser.newPage();
const initialTargetedURL = 'https://medium.com/';
await page.setBypassCSP(true);
await page.goto(initialTargetedURL);
await page.waitForSelector('div[class="iw ix z"]')
const trendingArticles = await page.$$('div[class="iw ix z"] div[class="an do"]');
console.log(trendingArticles.length);
for (let i = 0; i < trendingArticles.length; i++) {
await page.goto(initialTargetedURL);
await page.waitForSelector('div[class="iw ix z"]');
const trendingArticles = await page.$$('div[class="iw ix z"] div[class="an do"]');
const trendingArticle = trendingArticles[i];
const link = await trendingArticle.$('div[class="an br ay"] > a');
// redirect way 1
const linkToVisit = await page.evaluate(link => link.getAttribute('href'), link)
console.log('linkToVisit', linkToVisit)
await page.goto(linkToVisit);
// redirect way 2
// await link.click()
await page.waitForSelector('div[class="n p"]')
const article = await page.$('div[class="n p"]')
console.log('Hi, there...', article)
}
// await browser.close();
} catch (e) {
if (e instanceof puppeteer.errors.TimeoutError) {
console.log('Oppsss! Timeout.', e)
} else {
console.log('::Error::', e)
}
}
})();