skip to Main Content

I am trying to save the data that I am scarping from a website with node and mongoDB but I cannot save it in the database. I am able to save the data in the array (bdata) and GET the data to show in my localhost but not save to it.

Would really appreciate any help.
Thank you in advance!

    const axios = require('axios');
const cheerio = require('cheerio');
const express = require('express');
const port = 4000;
const app = express();
const bodyParser = require('body-parser');
const mongoose = require("mongoose");
const Phones = require('./models/phones');

mongoose.Promise = global.Promise;
mongoose.connect("mongodb://0.0.0.0:27017/testDatabase");

app.set('view engine', 'ejs');
app.use(express.json())
app.use(bodyParser.json());
app.use(bodyParser.urlencoded({ extended: true }));
app.use(express.urlencoded({ extended: false }));

const url = 'https://scrapewebsite.com';
const bdata = [];

async function scrapeSite(url) {
    try {
        const response = await axios.get(url);
        const $ = cheerio.load(response.data);

        const phones = $('.product-card');
        // loop through products
        phones.each((i, phone) => {
            if ($(phone).find("ins span.woocommerce-Price-amount.amount")) {
                price = $(phone).find("ins span.woocommerce-Price-amount.amount").text();
            } else if ($(phone).find(".price.font-bold.text-lg.md:text-xl.text-black")) {
                price = $(phone).find(".price.font-bold.text-lg.md:text-xl.text-black").text();
            }
            article_name = $(phone).find(".woocommerce-loop-product__title").text();
            link = $(phone).find("a").attr("href");

            bdata.push({ "article": article_name, "link": link, "price": price });
        });

        //console.log(bdata);
        return;
        // iterate through all pages
        const hasNext = $(".next").text();
        if (hasNext.length > 0) {
            next_page = $(".next").attr('href');
            scrapeSite(next_page);
        }
        console.log(next_page);

    } catch (error) {
        console.error(error);
    }
}

scrapeSite(url);

app.get('/', (req, res) => {
    res.render('index', { bdata: bdata });
});


app.listen(port, () => console.log('Example app listening on port ' + port));

And my schema

const mongoose = require('mongoose');
const Schema = mongoose.Schema;

const PhonesSchema = new Schema({
    article: String,
    link: String,
    price: String
});

module.exports = mongoose.model('Phones', PhonesSchema);

2

Answers


  1. Chosen as BEST ANSWER

    Okay so i managed to save the data. In place of pushing into the array i just saved it with the mongoose method .save()

    bdata.save({ "article": article_name, "link": link, "price": price });
    

    Thank you for the efforts!


  2. What made you use the below lines in your code? You are trying to do the same work with inbuilt middlewares and also using the separate package bodyParser that doesn’t make any sense. And hence the problem.

    app.use(express.json())
    app.use(bodyParser.json())
    app.use(bodyParser.urlencoded({ extended: true }));
    app.use(express.urlencoded({ extended: false }));
    

    bodyParser was separated out from the earlier in version 4.0 but they re-introduced it in the Express release 4.16.0. So, for the latest version of Express i.e release >= 4.16.0 you just need to do the following:

    app.use(express.json())
    app.use(express.urlencoded({ extended: true }));
    

    and you do not need the below lines in your code:

    app.use(bodyParser.json())
    app.use(bodyParser.urlencoded({ extended: true }));
    

    and also get rid of the package body-parser.

    Login or Signup to reply.
Please signup or login to give your own answer.
Back To Top
Search