When using Node Crawler, once the crawl has finished and there are no further links. How do I then fire off code to output results

Viewed 29

this is the first time I've ever posted here so please be gentle :)

I am using a node crawler (found here: https://www.npmjs.com/package/crawler) to crawl a site and find all internal links. The script runs and outputs the links to the console. What I'm trying to do is detect once it has finished and finds no further links, then I want to do stuff such as output all the links, maybe add to text file or Google sheet. I have tried async but I'm a bit out of my depth. Code is as follows

const Crawler = require("crawler");

let obselete = []; // Array of what was crawled already

let c = new Crawler();

const crawlAllUrls = async function (url) {
    
    c.queue({
        uri: url,
        callback: function (err, res, done) {
            if (err) throw err;
            let $ = res.$;
            try {
                let urls = $("a"); // all urls
                Object.keys(urls).forEach((item) => {
                    if (urls[item].type === 'tag') {
                        let href = urls[item].attribs.href;
                        if (href && !obselete.includes(href)) {
                            href = href.trim();
                            console.log(`href is ${href}`);
                            obselete.push(href);
                            // Slow down the
                            setTimeout(function() {
                                href.startsWith(url) ? crawlAllUrls(href) : crawlAllUrls(`${url}${href}`)
                            }, 5000)

                        }
                    }
                });
            } catch (e) {
                console.error(`Encountered an error crawling ${url}. Aborting crawl.`);
                done()

            }
            done();
        }
    })
}

c.on('drain',function(){
    // For example, release a connection to database.
    console.log("Done");
});
// Start function
const start = async function(a, b) {
    const result = await crawlAllUrls('http://www.five12.co.uk/');
}

  start();
0 Answers
Related