Twitter API Question posted in
The official Twitter API documentation can be found here.

Node closes DB connection either before inserts or does not close at all – Twitter API

I have the following code that does the following:

  • Opens DB connection (using pg).
  • Call Twitter Trends API (once or more for different places).
  • Go through the trends items and store each one into DB.
  • Close DB connection.

I tried a straight forward code. I faced the following error first.

Error: Connection terminated by user

Here is the code.

client.connect();

const text = 'INSERT INTO TRENDS(AS_OF, location_name, name, tweet_volume, rank) VALUES($1, $2, $3, $4, $5) RETURNING *';

T.get('trends/place', { id: 2295424 }, function(err, data, response) {
    data_json = data[0];
    var rankNum = 0;
    console.log(data_json.locations[0].name);
    for (var item of data_json.trends) {
        rankNum++;
        var values = [data_json.as_of, data_json.locations[0].name, item.name, item.tweet_volume, rankNum];
        client.query(text, values)
            .then(res => {
                console.log(res.rows[0])
            })
            .catch(e => console.error(e.stack));
    };
    client.end();
});

It appeared, the DB connection (client.end()) was called and closed before the client.query() could run. So, I rewrote my code to wait for the client.query to complete and then call client.end using async/await.

client.connect();

const text = 'INSERT INTO TRENDS(AS_OF, location_name, name, tweet_volume, rank) VALUES($1, $2, $3, $4, $5) RETURNING *';

T.get('trends/place', { id: 23424848 }, function (err, data, response) {
    data_json = data[0];
    var rankNum = 0;
    console.log(data_json.locations[0].name);

    function insRows(){
    return new Promise( resolve => {
        for (var item of data_json.trends) {
        rankNum++;
        var values = [data_json.as_of, data_json.locations[0].name, item.name, item.tweet_volume, rankNum];
        client.query(text, values)
            .then(res => {
                console.log(res.rows[0])
            })
            .catch(e => console.error(e.stack));
        };
    });
    };

    async function closeDBConnection(){
        const a = await insRows();
        client.end();
        return 0;
    };

    closeDBConnection().then( v => {
         console.log(v);
    });
});

This code with async/await does run fine, however the DB connection is not closed and node just hangs for me to CTRL+C.

  • Node version: v9.2.1
  • I run the program using command: node app.js

Update 1

Updated code with Promise array and async/await. This does not work:

client.connect();

const text = 'INSERT INTO TRENDS(AS_OF, location_name, name, tweet_volume, rank) VALUES($1, $2, $3, $4, $5) RETURNING *';
T.get('trends/place', { id: 23424848 }, function(err, data, response) {
    const data_json = data[0];
    let rankNum = 0;
    const queryPromises = [];
    console.log(data_json.locations[0].name);
    async function insRows() {
        for (let item of data_json.trends) {
            rankNum++;
            const values = [data_json.as_of, data_json.locations[0].name, item.name, item.tweet_volume, rankNum];
            queryPromises.push(client.query(text, values)
                .then(res => console.log(res.rows[0]))
                .catch(e => console.error(e.stack)));
        };
    };

    (async = () => {
        await Promise.all(queryPromises);
        client.end();
    });
});

I also tried:

    async function insRows(){
     /* same as above*/
    }
    await Promise.all(queryPromises);
    client.end();

I get the following error (node app_ans.js):

        await Promise.all(queryPromises);
        ^^^^^

SyntaxError: await is only valid in async function
Tags:

2

Answers


  1. 0

    Your code never calls resolve of the promise you create, but you don’t need to create your own promise since client.query already returns a promise. I would write it like this:

    const data_json = data[0];
let rankNum = 0;
const queryPromises = [];

for (let item of data_json.trends) {
    rankNum++;
    const values = [data_json.as_of, data_json.locations[0].name, item.name, item.tweet_volume, rankNum];
    queryPromises.push(client.query(text, values)
        .then(res => console.log(res.rows[0]))
        .catch(e => console.error(e.stack)));
};
await Promise.all(queryPromises);
client.end();
    Login or Signup to reply.
  2. 0

    First thing, I assume you are using npm-request package to place GET request to twitter trends/places api, if yes, I suggest using request-promise; which allows us to use Async/Await directly as it supports Promises instead of callback. That being said, the issue could be with the way you are framing the GET request. use T.get('trends/place?id=23424848' instead of T.get('trends/place', { id: 23424848 }

    You may also consider the below code:

    const T = require('request-promise')
const { Client } = require('pg')
const client = new Client()

await client.connect()

const getTrendsAndUpdate = async (id) => {
    const text = 'INSERT INTO TRENDS(AS_OF, location_name, name, tweet_volume, rank) VALUES($1, $2, $3, $4, $5) RETURNING *';

    const closeDBConnection = async () => {
        await client.end();
        console.log('DB Connection closed')
        return 0;
    }

    try{
        const data_json = await T.get('trends/place?id='+id, {json:true});
        let trends = data_json['trends'];
        let as_of = data_json['as_of'];
        let location_name = data_json['locations']['name'];
        let rankNum = 0;
        let a = await trends.forEach(async item => {
            rankNum++;
            let values = [data_json.as_of, data_json.locations[0].name, item.name, item.tweet_volume, rankNum];
            let res = await client.query(text, values)
            console.log(res.rows[0])
        });
        if(a){
            await closeDBConnection()
            return
        }
    } catch(e){ console.error(e.stack) }
}

getTrendsAndUpdate(23424848)

    Hope this resolves the issue and points you in right direction.

    Login or Signup to reply.
Please signup or login to give your own answer.

Back To Top
Search