I have to upload a 90 MB CSV file and then analyze it with chart.js as a chart. The CSV file has measured values that are recorded per minute. The 90 MB are then almost a year’s worth of data. I have already set the website response time to a high value. But my code is going down the drain. That’s why I’ve made do and only display a certain number of data values and then click through the diagram at intervals. Even that is still very slow and not nice. For the evaluation, at least a monthly overview would be nicer. But I have no idea what adjustments I can still make. Do you have any ideas?
HTML
<!DOCTYPE html>
<html lang="de">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>CSV Diagramm mit Chart.js</title>
<link rel="stylesheet" href="styles.css">
</head>
<body>
<div id="drop-area" class="drop-area" style="width: 100%;" ondrop="handleDrop(event)" ondragover="handleDragOver(event)">
<p>Datei hier ablegen</p>
<input type="file" id="csvFileInput" accept=".csv" style="display:none;" onchange="handleUpload()">
</div>
<div class="chart-container" style="width: 100%;">
<canvas id="myChart"></canvas>
</div>
<button onclick="showPreviousData()">Vorheriger Tag</button>
<button onclick="showNextData()">Nächster Tag</button>
<script src="https://cdn.jsdelivr.net/npm/chart.js"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/hammer.js/2.0.8/hammer.min.js" integrity="sha512-UXumZrZNiOwnTcZSHLOfcTs0aos2MzBWHXOHOuB0J/R44QB0dwY5JgfbvljXcklVf65Gc4El6RjZ+lnwd2az2g==" crossorigin="anonymous" referrerpolicy="no-referrer"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/chartjs-plugin-zoom/2.0.1/chartjs-plugin-zoom.min.js" integrity="sha512-wUYbRPLV5zs6IqvWd88HIqZU/b8TBx+I8LEioQ/UC0t5EMCLApqhIAnUg7EsAzdbhhdgW07TqYDdH3QEXRcPOQ==" crossorigin="anonymous" referrerpolicy="no-referrer"></script>
<script src="script.js"></script>
</body>
</html>
JS
let startIndex = 0;
const displayCount = 1440;
let labels = [];
let datasets = [];
let originalDatasetVisibility = [];
function handleUpload() {
const fileInput = document.getElementById('csvFileInput');
const file = fileInput.files[0];
handleFile(file);
}
function processData(csvData) {
const rows = csvData.split('n');
labels = [];
datasets = [];
originalDatasetVisibility = [];
const colors = ['rgba(255, 0, 0, 1)', 'rgba(0, 255, 0, 1)', 'rgba(255, 255, 0, 1)', 'rgba(0, 0, 255, 1)'];
const columns = rows[0].split(';');
for (let i = 1; i < columns.length; i++) {
const data = [];
const currentLabel = columns[i];
const color = colors[i - 1];
for (let j = 1; j < rows.length; j++) {
const cols = rows[j].split(';');
if (i === 1) {
labels.push(cols[0]);
}
data.push(parseFloat(cols[i]));
}
const dataset = {
label: currentLabel,
data: data,
backgroundColor: color,
borderColor: color,
fill: false,
borderWidth: 1,
pointRadius: 1,
};
datasets.push(dataset);
originalDatasetVisibility.push(true);
}
createChart(labels.slice(startIndex, startIndex + displayCount), datasets, function() {
console.log('Diagramm wurde erstellt');
});
}
function createChart(labels, datasets, callback) {
const chartContainer = document.querySelector('.chart-container');
const canvasElement = document.getElementById('myChart');
if (canvasElement) {
chartContainer.removeChild(canvasElement);
}
chartContainer.innerHTML = '<canvas id="myChart"></canvas>';
const ctx = document.getElementById('myChart').getContext('2d');
window.myChart = new Chart(ctx, {
type: 'line',
data: {
labels: labels,
datasets: datasets.map((dataset, index) => ({
...dataset,
data: dataset.data.slice(startIndex, startIndex + displayCount),
hidden: !originalDatasetVisibility[index],
})),
},
options: {
scales: {
x: {
stacked: true,
min: labels[startIndex],
max: labels[startIndex + displayCount - 1],
},
y: {},
},
plugins: {
zoom: {
pan: {
enabled: true,
mode: 'x'
},
zoom: {
wheel: {
enabled: true,
},
pinch: {
enabled: true
},
mode: 'x',
}
}
}
}
});
if (callback && typeof callback === 'function') {
callback();
}
window.myChart.resetZoom();
window.myChart.ctx.canvas.addEventListener('wheel', handleZoom);
}
function handleZoom(event) {
const chart = window.myChart;
const chartArea = chart.chartArea;
const originalDatasets = chart.data.datasets;
const zoomEnabled = chart.options.plugins.zoom.zoom.wheel.enabled;
const deltaY = event.deltaY;
if (zoomEnabled && deltaY !== 0) {
const deltaMode = event.deltaMode;
const scaleDelta = deltaY > 0 ? 0.9 : 1.1;
let newMinIndex = chart.getDatasetMeta(0).data.findIndex(
(d) => d.x >= chartArea.left
);
let newMaxIndex = chart.getDatasetMeta(0).data.findIndex(
(d) => d.x >= chartArea.right
);
if (deltaMode === 0) {
newMinIndex = Math.max(0, newMinIndex - Math.abs(deltaY));
newMaxIndex = Math.min(
originalDatasets[0].data.length - 1,
newMaxIndex + Math.abs(deltaY)
);
} else if (deltaMode === 1) {
newMinIndex = Math.max(0, newMinIndex - Math.abs(deltaY) * 10);
newMaxIndex = Math.min(
originalDatasets[0].data.length - 1,
newMaxIndex + Math.abs(deltaY) * 10
);
}
const newMinLabel = originalDatasets[0].data[newMinIndex].label;
const newMaxLabel = originalDatasets[0].data[newMaxIndex].label;
chart.options.scales.x.min = newMinLabel;
chart.options.scales.x.max = newMaxLabel;
chart.update();
}
}
function handleFile(file) {
if (file) {
const reader = new FileReader();
reader.onload = function (e) {
const csvData = e.target.result;
processData(csvData);
};
reader.readAsText(file);
} else {
alert('Bitte eine CSV-Datei auswählen.');
}
}
function handleDrop(event) {
event.preventDefault();
const file = event.dataTransfer.files[0];
handleFile(file);
}
function handleDragOver(event) {
event.preventDefault();
}
function showPreviousData() {
if (startIndex - displayCount >= 0) {
startIndex -= displayCount;
updateChart();
}
}
function showNextData() {
if (startIndex + displayCount < labels.length) {
startIndex += displayCount;
updateChart();
}
}
function updateChart() {
const endIndex = Math.min(startIndex + displayCount, labels.length);
const updatedLabels = labels.slice(startIndex, endIndex);
const updatedDatasets = datasets.map((dataset, index) => ({
...dataset,
data: dataset.data.slice(startIndex, endIndex),
hidden: !originalDatasetVisibility[index],
}));
window.myChart.data.labels = updatedLabels;
window.myChart.data.datasets = updatedDatasets;
window.myChart.options.scales.x.min = updatedLabels[0];
window.myChart.options.scales.x.max = updatedLabels[updatedLabels.length - 1];
window.myChart.update();
}
function removeZoomEventListener() {
window.myChart.ctx.canvas.removeEventListener('wheel', handleZoom);
}
2
Answers
Well it depends on your use case, how the data has to visualized, how exact the representation has to be and how fast it needs to be.
That said here are some tips to improve the performance:
ChartJs related
Checkout Chartjs official Performance "Tips and Tricks"
Image 1: Minor tweaks ~ 70 Sec (5255999 data rows)
Image 2: Minor tweaks + fixed Scaling ~ 3 Sec (5255999 data rows)
prepare the data for easy use
You can load a small chunk of data and async load the data, and update the chart data, checkout this update chart example
On the Data/Web side
Bonus Tipp: if you don’t have to use chartjs check out this SO question/answer here, it recommends using Highcharts for bigdata instead of chartjs.
After writing my last answer, I tried to optimize your (long) code, I noticed:
So sadly my last answer, would not be helpful in your specific case, so here is a more tailored solution.
So what could be a solution:
JSON.parse
or so.for
-loops to something like this:Here the result chart of my dataset: