I am trying to save the data that I am scarping from a website with node and mongoDB but I cannot save it in the database. I am able to save the data in the array (bdata) and GET the data to show in my localhost but not save to it.
Would really appreciate any help.
Thank you in advance!
const axios = require('axios');
const cheerio = require('cheerio');
const express = require('express');
const port = 4000;
const app = express();
const bodyParser = require('body-parser');
const mongoose = require("mongoose");
const Phones = require('./models/phones');
mongoose.Promise = global.Promise;
mongoose.connect("mongodb://0.0.0.0:27017/testDatabase");
app.set('view engine', 'ejs');
app.use(express.json())
app.use(bodyParser.json());
app.use(bodyParser.urlencoded({ extended: true }));
app.use(express.urlencoded({ extended: false }));
const url = 'https://scrapewebsite.com';
const bdata = [];
async function scrapeSite(url) {
try {
const response = await axios.get(url);
const $ = cheerio.load(response.data);
const phones = $('.product-card');
// loop through products
phones.each((i, phone) => {
if ($(phone).find("ins span.woocommerce-Price-amount.amount")) {
price = $(phone).find("ins span.woocommerce-Price-amount.amount").text();
} else if ($(phone).find(".price.font-bold.text-lg.md:text-xl.text-black")) {
price = $(phone).find(".price.font-bold.text-lg.md:text-xl.text-black").text();
}
article_name = $(phone).find(".woocommerce-loop-product__title").text();
link = $(phone).find("a").attr("href");
bdata.push({ "article": article_name, "link": link, "price": price });
});
//console.log(bdata);
return;
// iterate through all pages
const hasNext = $(".next").text();
if (hasNext.length > 0) {
next_page = $(".next").attr('href');
scrapeSite(next_page);
}
console.log(next_page);
} catch (error) {
console.error(error);
}
}
scrapeSite(url);
app.get('/', (req, res) => {
res.render('index', { bdata: bdata });
});
app.listen(port, () => console.log('Example app listening on port ' + port));
And my schema
const mongoose = require('mongoose');
const Schema = mongoose.Schema;
const PhonesSchema = new Schema({
article: String,
link: String,
price: String
});
module.exports = mongoose.model('Phones', PhonesSchema);
2
Answers
Okay so i managed to save the data. In place of pushing into the array i just saved it with the mongoose method
.save()
Thank you for the efforts!
What made you use the below lines in your code? You are trying to do the same work with inbuilt
middlewares
and also using the separate packagebodyParser
that doesn’t make any sense. And hence the problem.bodyParser
was separated out from the earlier inversion 4.0
but they re-introduced it in theExpress release 4.16.0
. So, for the latest version ofExpress
i.erelease >= 4.16.0
you just need to do the following:and you do not need the below lines in your code:
and also get rid of the package
body-parser
.