The following code is scraping "title" and "url" of every article on theguardian frontpage, but I want it this to crawl the entire site for every article "title" and "url"
And I cannot wrap my head around how to do so, the following code is what does the scraping but not the crawling.
const PORT = 8000
const axios = require('axios')
const cheerio = require('cheerio')
const express = require('express')
const app = express()
const cors = require('cors')
app.use(cors())
const url = 'https://www.theguardian.com/uk'
app.get('/', function (req, res) {
res.json('This is my webscraper')
})
app.get('/results', (req, res) => {
axios(url)
.then(response => {
const html = response.data
const $ = cheerio.load(html)
const articles = []
$('.fc-item__title', html).each(function () { //<-- cannot be a function expression
const title = $(this).text()
const url = $(this).find('a').attr('href')
articles.push({
title,
url
})
})
res.json(articles)
}).catch(err => console.log(err))
})
app.listen(PORT, () => console.log(`server running on PORT ${PORT}`))
I am not very familiar with node in general and only just began learning the basics of scraping for specific html tags and text. Thanks in advance.