I am trying to save the data that I am scarping from a website with node and mongoDB but I cannot save it in the database. I am able to save the data in the array (bdata) and GET the data to show in my localhost but not save to it.
Would really appreciate any help. Thank you in advance!
const axios = require('axios');
const cheerio = require('cheerio');
const express = require('express');
const port = 4000;
const app = express();
const bodyParser = require('body-parser');
const mongoose = require("mongoose");
const Phones = require('./models/phones');
mongoose.Promise = global.Promise;
mongoose.connect("mongodb://0.0.0.0:27017/testDatabase");
app.set('view engine', 'ejs');
app.use(express.json())
app.use(bodyParser.json());
app.use(bodyParser.urlencoded({ extended: true }));
app.use(express.urlencoded({ extended: false }));
const url = 'https://scrapewebsite.com';
const bdata = [];
async function scrapeSite(url) {
try {
const response = await axios.get(url);
const $ = cheerio.load(response.data);
const phones = $('.product-card');
// loop through products
phones.each((i, phone) => {
if ($(phone).find("ins span.woocommerce-Price-amount.amount")) {
price = $(phone).find("ins span.woocommerce-Price-amount.amount").text();
} else if ($(phone).find(".price.font-bold.text-lg.md:text-xl.text-black")) {
price = $(phone).find(".price.font-bold.text-lg.md:text-xl.text-black").text();
}
article_name = $(phone).find(".woocommerce-loop-product__title").text();
link = $(phone).find("a").attr("href");
bdata.push({ "article": article_name, "link": link, "price": price });
});
//console.log(bdata);
return;
// iterate through all pages
const hasNext = $(".next").text();
if (hasNext.length > 0) {
next_page = $(".next").attr('href');
scrapeSite(next_page);
}
console.log(next_page);
} catch (error) {
console.error(error);
}
}
scrapeSite(url);
app.get('/', (req, res) => {
res.render('index', { bdata: bdata });
});
app.listen(port, () => console.log('Example app listening on port ' + port));
And my schema
const mongoose = require('mongoose');
const Schema = mongoose.Schema;
const PhonesSchema = new Schema({
article: String,
link: String,
price: String
});
module.exports = mongoose.model('Phones', PhonesSchema);