How to associate array values ​to a given object regardless of quantity

user_11424363 0
Viewed 60

I'm working with Web Scraping using puppeteer, iterating with DOM elements for example:

<div class="entry">
    <div class="information info_0"> <p> Close </p> </div>
    <div class="details"> bla bla bla bla </div>
    <div class="slide"> <img src="0_path1" /> </div>
    <div class="slide"> <img src="1_path2" /> </div>
    <div class="slide"> <img src="2_path3" /> </div>
    <div class="slide"> <img src="3_path4" /> </div>
    <div class="slide"> <img src="4_path5" /> </div>

    <div class="information info_1"> <p> Rextry </p> </div>
    <div class="details"> ble ble ble </div>
    <div class="slide"> <img src="1_path1" /> </div>
    <div class="slide"> <img src="1_path2" /> </div>
    <div class="slide"> <img src="1_path3" /> </div>
</div>

so I have to wait this entry load so, waitForSelector('div.entry').

const result = await page.evaluate(() => {
 ... Iterate Images ...
}

Iterate Images:

    const items = document.querySelectorAll('entry');
    let information = []
    let details = []
    items.forEach(function(el) { 
      el.querySelectorAll('div.information p').forEach(function(el) { 
               information.push(el.innerHTML)
      }

      el.querySelectorAll('div.details').forEach(function(el) { 
               details.push(el.innerHTML)
      }

}

now I could create an object for these array:

let ob = []

      information.forEach((el, index) => {
      ob.push({
        info: information[index],
        detail: details[index],
      });
});

result:

[
   { info: 'Close', details: 'bla bla bla bla bla' },
   { info: 'Rextry', details: 'ble ble ble' },
]

But now I would to like to associate the images src with respective information. How could I do that?

I thought iterate images and push inside info array:

el.querySelectorAll('div.slide').forEach(function(el) { 
               info.push(el.innerHTML)
      }

Results would be:

info: ['Close','0_path1',0_path2',0_path3',0_path4',0_path5','Rextry','1_path1','1_path2','1_path3']

puppeteer javascript
2 Answers

Instead of doing :

  el.querySelectorAll('div.information p').forEach(function(el) { 
           information.push(el.innerHTML)
  }

  el.querySelectorAll('div.details').forEach(function(el) { 
           details.push(el.innerHTML)
  }

You could first create your let ob = [] array, then loop over all divs and check their classes

let ob = [];
let idx = -1;
// [...]
  el.querySelectorAll('div').forEach(function(el) { 
    if (el.classList.contains('information') {
      idx++;
    }
    // populate ob|idx] information, detail, slide or whatever you want
  }
user_13481240

As all elements for an object entry are children of one div from one key element till another, you can iterate over all the children, creating a new object entry on every key element occurrence. Here is a full test example:

import puppeteer from 'puppeteer';

const browser = await puppeteer.launch(/* { headless: false, defaultViewport: null } */);

const html = `
  <!doctype html>
  <html>
    <head><meta charset='UTF-8'><title>Test</title></head>
    <body>
      <div class="entry">
          <div class="information info_0"> <p> Close 1</p> </div>
          <div class="details"> bla bla bla bla </div>
          <div class="slide"> <img src="0_path1" /> </div>
          <div class="slide"> <img src="1_path2" /> </div>
          <div class="slide"> <img src="2_path3" /> </div>
          <div class="slide"> <img src="3_path4" /> </div>
          <div class="slide"> <img src="4_path5" /> </div>

          <div class="information info_1"> <p> Rextry 1</p> </div>
          <div class="details"> ble ble ble </div>
          <div class="slide"> <img src="1_path1" /> </div>
          <div class="slide"> <img src="1_path2" /> </div>
          <div class="slide"> <img src="1_path3" /> </div>
      </div>
      <div class="entry">
          <div class="information info_0"> <p> Close 2</p> </div>
          <div class="details"> bla bla bla bla </div>
          <div class="slide"> <img src="0_path1" /> </div>
          <div class="slide"> <img src="1_path2" /> </div>
          <div class="slide"> <img src="2_path3" /> </div>
          <div class="slide"> <img src="3_path4" /> </div>
          <div class="slide"> <img src="4_path5" /> </div>

          <div class="information info_1"> <p> Rextry 2</p> </div>
          <div class="details"> ble ble ble </div>
          <div class="slide"> <img src="1_path1" /> </div>
          <div class="slide"> <img src="1_path2" /> </div>
          <div class="slide"> <img src="1_path3" /> </div>
      </div>
    </body>
  </html>`;

try {
  const [page] = await browser.pages();

  await page.goto(`data:text/html,${html}`);

  const entries = await page.evaluate(() => {
    const data = [];

    for (const entry of document.querySelectorAll('.entry')) {
      let section = null;
      for (const element of entry.querySelectorAll('.information p, div.details, .slide img')) {
        if (element.tagName === 'P') {
          if (section) data.push(section);
          section = { info: element.innerText, images: [] };
        } else if (element.tagName === 'DIV') {
          section.details = element.innerText;
        } else {
          section.images.push(element.getAttribute('src'));
        }
      }
      if (section) data.push(section);
    }

    return data;
  });
  console.log(entries);
} catch(err) { console.error(err); } finally { await browser.close(); }

Output:

[
  {
    info: 'Close 1',
    images: [ '0_path1', '1_path2', '2_path3', '3_path4', '4_path5' ],
    details: 'bla bla bla bla'
  },
  {
    info: 'Rextry 1',
    images: [ '1_path1', '1_path2', '1_path3' ],
    details: 'ble ble ble'
  },
  {
    info: 'Close 2',
    images: [ '0_path1', '1_path2', '2_path3', '3_path4', '4_path5' ],
    details: 'bla bla bla bla'
  },
  {
    info: 'Rextry 2',
    images: [ '1_path1', '1_path2', '1_path3' ],
    details: 'ble ble ble'
  }
]
user_26255600
Related