this is the first time I've ever posted here so please be gentle :)
I am using a node crawler (found here: https://www.npmjs.com/package/crawler) to crawl a site and find all internal links. The script runs and outputs the links to the console. What I'm trying to do is detect once it has finished and finds no further links, then I want to do stuff such as output all the links, maybe add to text file or Google sheet. I have tried async but I'm a bit out of my depth. Code is as follows
const Crawler = require("crawler");
let obselete = []; // Array of what was crawled already
let c = new Crawler();
const crawlAllUrls = async function (url) {
c.queue({
uri: url,
callback: function (err, res, done) {
if (err) throw err;
let $ = res.$;
try {
let urls = $("a"); // all urls
Object.keys(urls).forEach((item) => {
if (urls[item].type === 'tag') {
let href = urls[item].attribs.href;
if (href && !obselete.includes(href)) {
href = href.trim();
console.log(`href is ${href}`);
obselete.push(href);
// Slow down the
setTimeout(function() {
href.startsWith(url) ? crawlAllUrls(href) : crawlAllUrls(`${url}${href}`)
}, 5000)
}
}
});
} catch (e) {
console.error(`Encountered an error crawling ${url}. Aborting crawl.`);
done()
}
done();
}
})
}
c.on('drain',function(){
// For example, release a connection to database.
console.log("Done");
});
// Start function
const start = async function(a, b) {
const result = await crawlAllUrls('http://www.five12.co.uk/');
}
start();