I have a nodeJS application with a mongoDB database and recently after uploading data into the images database, my api is starting to take 40+ seconds to load data. For some, even if I search through the collection with limits or cursors, the data still takes time to load. Even if I only fetch back some data. If I look for items in mongoDb, I get the error. I have looked at various examples and even researched workers but got confused on its use and was not able to get it working with my code. Any help would be appreciated and thanks in advance.
So although creating an index did improve some of the api’s using images, I found that once a couple hundred records are pulled, the items[i].imageData = LZString.decompressFromBase64(image.imageData);
is also taking long to decompress all those images. is there a faster way or better way of doing this?
my controller for an api taking too long that fetches images from the database is:
function itemsByCategoryWithImages(req, res, next) {
cloverService.getAllItemsByCategoryWithImages( req.params.id)
.then(users => res.send(users))
.catch(err => next(err));
}
my service is:
async function getAllItemsByCategoryWithImages(category_id) {
const options = {
method: 'GET',
url: some_url,
headers: {
Authorization: 'Bearer ' + token,
ContentType: 'application/json'
},
retry: {limit: 3, methods: ["GET", "POST"]},
};
const response = await got(options);
const items = JSON.parse(response.body).elements.map(item=> item.name);
let promises = [];
promises.push(response);
promises.push(imageService.getImageByName(items));
// const items = JSON.parse(response.body).elements.map(item=> item.id);
const responses = await Promise.all(promises);
let images = [];
const itemsResponse = JSON.parse(await responses[0].body).elements;
images = responses[1];
return await this.matchItemImages(itemsResponse, images);
}
my image service funtion is:
async function getImageByName(names, isCategory) {
if(isCategory){
return Image.find({$or: [{cloverDataName: { $in: names }}], isCategory: isCategory})
}
return Image.find({$or: [{cloverDataName: { $in: names }}]})
}
finally, the match image function:
async function matchItemImages(items, images) {
console.log(' about to match images', images.length, items[0].name)
let imagesById = []
if(images.length > 0) {
imagesById = new Map(images.map(image => [image.cloverDataId, image]));
var i = 0, len = items.length;
while (i < len) {
const image = imagesById.get(items[i].id);
if (image) {
items[i].imageData = LZString.decompressFromBase64(image.imageData);
}
;
i++;
}
}
return items.sort((a, b) => {
var textA = a.name.toUpperCase();
var textB = b.name.toUpperCase();
return (textA < textB) ? -1 : (textA > textB) ? 1 : 0;
});
}
when i comment the promises.push(imageService.getImageByName(items));
, the speed is restored.
2
Answers
I solved the issue with pagination.
you need to create index on {cloverDataName: 1}