Question 52068 in Javascript Question posted in
A very good W3school tutorial can be found here.

How do I get Axios to get data from multiple sources simultaneously without data mixing? – Javascript

When I try to run multiple simultaneous axios operations of getting data from different data sources using a loop, the data received gets mixed, and therefore corrupted.

I tried creating a different axios instance for each data source, but even still, it seems axios global parameters seem to overwrite individual instance URLs, as far as I can tell.

I’ve written a simplified example of what generally reproduces the problem:

const axios = require('axios');
const axiosObjs = {};

async function get_data(dataSetName) {

    let dataToSave = [];

    if (!axiosObjs[dataSetName]) {
        axiosObjs[dataSetName] = axios.create({
            method: 'get',
            url: 'https://example.com/dataSets/',
        });
    }

    for (let i = 0; i < 10; i++) {
        const urlString = 'https://example.com/dataSets/' + dataSetName + "_" + i;

        const dataSetChunk = await axiosObjs[dataSetName](urlString)
            .then(result => result) 
            .catch((err) => { return "this is just a simplified example"; });

        dataToSave = [].concat(dataToSave, dataSetChunk.data);

        await wait_milliseconds(500);  // made up function, waits half a second
    }
    
    save_data_to_file(dataSetName, dataToSave);  // made up function, saves data to file
}

// these all result in data mixed up between saved data sets
get_data("dataSet1");
get_data("dataSet2");
get_data("dataSet3");

The resulting files include data mixed up from each data source, apparently because the loops are happening simultaneously where each iteration of the loops overwrites each other’s parameters within the separate function calls.

How can I get around this without having to run get_data as separate synchronous calls?

Tags:

2

Answers


  1. Chosen as BEST ANSWER
    0

    I solved my problem, and it actually was embarrassingly neither axios nor server-related.

    The problem stems from the single const keyword I put in bold within the below example (explanation underneath the example):

    const axios = require('axios');
const axiosObjs = {};

async function get_data(dataSetName) {

    let dataToSave = [];

    if (!axiosObjs[dataSetName]) {
        axiosObjs[dataSetName] = axios.create({
            method: 'get',
            url: 'https://example.com/dataSets/',
        });
    }

    for (let i = 0; i < 10; i++) {
        const urlString = 'https://example.com/dataSets/' + dataSetName + "_" + i;

        **const** dataSetChunk = await axiosObjs[dataSetName](urlString)
            .then(result => result) 
            .catch((err) => { return "this is just a simplified example"; });

        dataToSave = [].concat(dataToSave, dataSetChunk.data);

        await wait_milliseconds(500);  // made up function, waits half a second
    }
    
    save_data_to_file(dataSetName, dataToSave);  // made up function, saves data to file
}

// these all result in data mixed up between saved data sets
get_data("dataSet1");
get_data("dataSet2");
get_data("dataSet3");

    When I created the simplified example to post on StackOverflow, I included the const keyword when defining the dataSetChunk constant. But in the actual code, I accidentally did not include that const keyword, and JavaScript being JavaScript, it just went ahead and made dataSetChunk a global variable. Of course, it being a global variable meant that all the simultaneous function calls shared that same variable while populating the data from the server.


  2. 0 
    const axios = require('axios');

async function get_data(dataSetName) {
  let unordered_data_chunks = [];
  let promises_to_await = [];

  for (let i = 0; i < 10; i++) {
    let data_source_url = 'https://example.com/dataSets/' + dataSetName + "_" + i;

    let promise = axios.get(data_source_url)
      .then((res) => {
        let data_chunk = {
          index: i,
          data: res.data
        };
        unordered_data_chunks.push(data_chunk);
      })
      .catch((err) => {
        // Do something to handle error.
      });

    promises_to_await.push(promise);
  }

  await Promise.all(promises_to_await);

  let ordered_data_chunks = unordered_data_chunks.sort(
    (a, b) => a.index - b.index
  );

  let data = ordered_data_chunks.reduce(
    (accumulator, current_value) => {
      accumulator.push(current_value.data);
      return accumulator;
    },
    []
  );

  save_data_to_file(dataSetName, data);
}

get_data("dataSet1");
    Login or Signup to reply.
Please signup or login to give your own answer.

Back To Top
Search