Puppeteer JS product page web scraping

Viewed 22

I am able to scrape product data for a given link in the ecommerce website. My problem is that my code only scrapes the first image link on the product page. It does not scrape the image links for the other images. When I inspect the image element, the image HTML selector is the same for all images on the product page if there is more than one image.

Here is the category link: https://www.ciceksepeti.com/hediye-sepetleri?page=2

and here is a product page: https://www.ciceksepeti.com/porselen-beyaz-fincan-lavi-cikolata-kurutulmus-cicek-buketi-hediye-seti-kcm36216623

Here is my code:

const puppeteer = require('puppeteer');
const xlsx = require('xlsx');

async function prodData(url, page) {

  await page.goto(url);
 
  
  const title = await page.$eval(".js-product-title", title => title.textContent);
  const price = await page.$eval(".product__info__new-price__integer", price => price.textContent);
  const desc = await page.$eval(".product__description-text", desc => desc.textContent.trim());
  const sku = await page.$eval(".product__code", sku => sku.textContent.trim());
  const images = await page.$eval('#productDetailSend > div > div > div:nth-child(2) > div.product__main-info-left.js-product-image-content > div.product__image-container.js-product-image-container > div.easyzoom.easyzoom--overlay.easyzoom--with-thumbnails.zoom-image.js-easyzoom-clickable.br-6 > a', images => images.href);

  return {
    title: title,
    price: price,
    desc: desc,
    sku: sku,
    images: images
  }
  
  
  //const links = await page.$$eval('body > main > div > div.products.products--category.js-ajax-category-products > div > div > div > a', allAs => allAs.map(a => a.href));
  //await page.keyboard.press("Enter");
  //console.log(links);
  //await browser.close();
};

async function getLinks() {

  const browser = await puppeteer.launch({headless: false});
  const page = await browser.newPage();
  await page.goto("https://www.ciceksepeti.com/hediye-sepetleri?page=2");
  const links = await page.$$eval('body > main > div > div.products.products--category.js-ajax-category-products > div > div > div > a', allAs => allAs.map(a => a.href));
  
  return links;

}


async function main() {

  const allLinks = await getLinks();
  const browser = await puppeteer.launch({headless: false});
  const page = await browser.newPage();
  const scrapedData = [];

  for (link of allLinks) {
    const data = await prodData(link, page);
    scrapedData.push(data);
    const wb = xlsx.utils.book_new();
    const ws = xlsx.utils.json_to_sheet(scrapedData);
    xlsx.utils.book_append_sheet(wb,ws), {origin: -1};
    xlsx.writeFile(wb,"hediye.xlsx");
    console.log(scrapedData);
  }

};

main()
0 Answers
Related