Large file upload/download to Azure Container App using memory

Thanks @Magnus As Magnus suggested streaming directly to Azure Blob Storage and saving to temporary local storage with chunked processing is good approach but here you wanted to open the SQLite connection to the file.

You can’t directly open an SQLite connection to a file stored in Azure Blob Storage as if it were a local file.

• Download the SQLite file from Blob Storage to a local file system (into the container or VM), open it, process it.

using Azure.Storage.Blobs;
using System.Data.SQLite;
using System.IO;

// Define your Blob Storage connection details
string connectionString = "<your_connection_string>";
string containerName = "<container_name>";
string blobName = "<sqlite_file_name>";

// Create the BlobClient to download the SQLite file
BlobClient blobClient = new BlobClient(connectionString, containerName, blobName);

// Define a temporary local path to store the downloaded SQLite file
string tempFilePath = Path.Combine(Path.GetTempPath(), blobName);

// Download the SQLite file to the local temp path
await blobClient.DownloadToAsync(tempFilePath);

// Open the SQLite connection to the downloaded file
using (SQLiteConnection conn = new SQLiteConnection($"Data Source={tempFilePath}"))
{
    await conn.OpenAsync();

    // Perform operations on the SQLite database
    // Example: Execute a query
    using (SQLiteCommand command = new SQLiteCommand("SELECT * FROM YourTable", conn))
    {
        using (SQLiteDataReader reader = await command.ExecuteReaderAsync())
        {
            while (await reader.ReadAsync())
            {
                Console.WriteLine(reader.GetString(0));  // Example: read first column as string
            }
        }
    }
}

• Next steps is like cleaning up resources, uploading the processed file to a storage solution (e.g., Blob Storage), and possibly updating the database or notifying clients about the result of the file processing.

The below controller handles the large file uploads and streams them directly to a temporary file.

Complete code:

using Microsoft.AspNetCore.Http.Features;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Net.Http.Headers;
using Microsoft.AspNetCore.WebUtilities;

namespace LargeFileUploadApi.Controllers
{
    [ApiController]
    [Route("api/files")]
    public class FileUploadController : ControllerBase
    {
        private const int BufferSize = 81920; // 80KB buffer size for CopyToAsync
        private readonly ILogger<FileUploadController> _logger;

        public FileUploadController(ILogger<FileUploadController> logger)
        {
            _logger = logger;
        }

        [HttpPost("upload")]
        [DisableFormValueModelBinding] // Prevent ASP.NET from buffering form values
        [RequestSizeLimit(long.MaxValue)] // Allow very large uploads
        public async Task<IActionResult> UploadLargeFileAsync()
        {
            _logger.LogInformation("File upload started...");

            // Ensure the request has multipart content type
            if (!Request.HasFormContentType || !MediaTypeHeaderValue.TryParse(Request.ContentType, out var contentType))
            {
                return BadRequest("Invalid Content-Type. Expected multipart/form-data.");
            }

            // Extract the boundary
            var boundary = MultipartRequestHelper.GetBoundary(contentType, int.MaxValue);
            var multipartReader = new MultipartReader(boundary, Request.Body);

            MultipartSection section;
            while ((section = await multipartReader.ReadNextSectionAsync()) != null)
            {
                // Check if the section contains a file
                var contentDisposition = section.GetContentDispositionHeader();
                if (contentDisposition == null || !contentDisposition.DispositionType.Equals("form-data") || string.IsNullOrEmpty(contentDisposition.FileName.Value))
                    continue;

                // Create a temporary file path
                var tempFilePath = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName());

                try
                {
                    // Stream the file directly to disk
                    await using (var targetStream = System.IO.File.Create(tempFilePath))
                    {
                        _logger.LogInformation($"Writing file to temporary path: {tempFilePath}");
                        await section.Body.CopyToAsync(targetStream, BufferSize);
                    }

                    // Process the file (example: simulate processing time)
                    await ProcessFileAsync(tempFilePath);

                    _logger.LogInformation("File processed successfully.");

                    // Clean up: delete the temporary file
                    System.IO.File.Delete(tempFilePath);
                    _logger.LogInformation("Temporary file deleted.");
                }
                catch (Exception ex)
                {
                    _logger.LogError(ex, "Error during file upload or processing.");
                    return StatusCode(500, "An error occurred while processing the file.");
                }
            }

            return Ok(new { message = "File uploaded and processed successfully." });
        }

        private async Task ProcessFileAsync(string filePath)
        {
            _logger.LogInformation($"Processing file: {filePath}");

            // Simulate file processing (replace this with real logic)
            await Task.Delay(2000); // Simulating some processing time

            _logger.LogInformation($"File {filePath} processing complete.");
        }
    }

    // Helper class to get multipart content disposition
    internal static class MultipartRequestHelper
    {
        public static string GetBoundary(MediaTypeHeaderValue contentType, int lengthLimit)
        {
            var boundary = HeaderUtilities.RemoveQuotes(contentType.Boundary).Value;
            if (string.IsNullOrWhiteSpace(boundary))
            {
                throw new InvalidDataException("Missing content-type boundary.");
            }

            return boundary;
        }

        public static ContentDispositionHeaderValue? GetContentDispositionHeader(this MultipartSection section)
        {
            if (ContentDispositionHeaderValue.TryParse(section.ContentDisposition, out var contentDisposition))
            {
                return contentDisposition;
            }

            return null;
        }
    }
}