skip to Main Content

I’m attempting to upload files through a web form to a Google Drive backend using the Resumable Uplaods feature. I’m starting with a script from https://github.com/tanaikech/AsynchronousResumableUploadForGoogleDrive/

I’ve spent much time on this without success. What I’d like to do is upload 1,000 files. The way that the async script works now is that all 1,000 will upload concurrently. What I’d like to do is upload (for example) 10 files concurrently, then repeat for the next 10, and so on until done. I believe my issue is that I’ll need to modify https://cdn.jsdelivr.net/gh/tanaikech/ResumableUploadForGoogleDrive_js@master/resumableupload_js.min.js (included in index.html). I’ve already gotten the files split into chunks of 10, but when doing a chunk.forEach..., the async nature of https://cdn.jsdelivr.net/gh/tanaikech/ResumableUploadForGoogleDrive_js@master/resumableupload_js.min.js still uploads them all in parallel.

What should I be focusing on to change to only limit 10 async responses, and wait until done, then repeat?

To reproduce, use the setup instructions in the README.md from the repo I linked above. Except, use the code below for index.html.
Note the <insert folder ID> in the index.html file below would need to be updated to your own folder ID for this to work.

<input type="file" id="file" multiple="true" />
<input type="button" onclick="run()" value="Upload" />
<div id="progress"></div>

<script>src="https://cdn.jsdelivr.net/gh/tanaikech/ResumableUploadForGoogleDrive_js@master/resumableupload_js.min.js"</script>
<script>

function run() {
  google.script.run.withSuccessHandler(accessToken => ResumableUploadForGoogleDrive(accessToken)).getAuth();
}

function ResumableUploadForGoogleDrive(accessToken) {
  const f = document.getElementById("file");
  const chunkSize = 10;
  const totalChunks = Math.ceil(f.files.length / chunkSize);
  console.log("The totalChunks are: " + totalChunks);
  for (let i = 0; i < totalChunks; i++) {
    const startIndex = i * chunkSize;
    console.log("The startIndex is: " + startIndex);
    const endIndex = startIndex + chunkSize;
    console.log("The endIndex is: " + endIndex);
    const chunk = Array.from(f.files).slice(startIndex, endIndex);
    console.log("The chunk is: " + chunk);
    chunk.forEach((file, i) => {
      if (!file) return;
      let fr = new FileReader();
      fr.fileName = file.name;
      fr.fileSize = file.size;
      fr.fileType = file.type;
      fr.readAsArrayBuffer(file);
      fr.onload = e => {
        var id = "p" + ++i;
        var div = document.createElement("div");
        div.id = id;
        document.getElementById("progress").appendChild(div);
        document.getElementById(id).innerHTML = "Initializing.";
        const f = e.target;
        const resource = { fileName: fr.fileName, fileSize: fr.fileSize, fileType: fr.fileType, fileBuffer: fr.result, accessToken: accessToken, folderId: "<insert folder ID>" };
        const ru = new ResumableUploadToGoogleDrive();
        ru.Do(resource, function (res, err) {
          if (err) {
            console.log(err);
            return;
          }
          console.log(res);
          let msg = "";
          if (res.status == "Uploading") {
            msg = Math.round((res.progressNumber.current / res.progressNumber.end) * 100) + "% (" + fr.fileName + ")";
          } else {
            msg = res.status + " (" + fr.fileName + ")";
          }

          document.getElementById(id).innerText = msg;
        });
      };
    });
  };
};
</script>

2

Answers


  1. In your situation, how about the following sample script?

    Before you use this script, please enable Drive API at Advanced Google services.

    Google Apps Script: code.gs

    function getAuth() {
      // DriveApp.createFile(blob) // This is used for adding the scope of "https://www.googleapis.com/auth/drive".
      return ScriptApp.getOAuthToken();
    }
    
    function showSidebar() {
      var html = HtmlService.createHtmlOutputFromFile("index");
      SpreadsheetApp.getUi().showSidebar(html);
    }
    

    HTML & Javascript: index.html

    Please modify your folder ID to folderId: "root",.

    <input type="file" id="file" multiple="true" />
    <input type="button" onclick="run()" value="Upload" />
    <div id="progress"></div>
    
    
    <script src="https://cdn.jsdelivr.net/gh/tanaikech/[email protected]/resumableupload_js.min.js"></script>
    </script>
    <script>
    function run() {
      google.script.run.withSuccessHandler(accessToken => ResumableUploadForGoogleDrive(accessToken)).getAuth();
    }
    
    function upload({ accessToken, file, idx }) {
      return new Promise((resolve, reject) => {
        let fr = new FileReader();
        fr.fileName = file.name;
        fr.fileSize = file.size;
        fr.fileType = file.type;
        fr.readAsArrayBuffer(file);
        fr.onload = e => {
          var id = `p_${idx}`;
          var div = document.createElement("div");
          div.id = id;
          document.getElementById("progress").appendChild(div);
          document.getElementById(id).innerHTML = "Initializing.";
          const f = e.target;
          const resource = {
            fileName: f.fileName,
            fileSize: f.fileSize,
            fileType: f.fileType,
            fileBuffer: f.result,
            accessToken: accessToken,
            folderId: "root",
          };
          const ru = new ResumableUploadToGoogleDrive();
          ru.Do(resource, function (res, err) {
            if (err) {
              reject(err);
              return;
            }
            console.log(res);
            let msg = "";
            if (res.status == "Uploading") {
              msg = Math.round((res.progressNumber.current / res.progressNumber.end) * 100) + `% (${f.fileName})`;
            } else {
              msg = `${res.status} (${f.fileName})`;
            }
            if (res.status == "Done") {
              resolve(res.result);
            }
            document.getElementById(id).innerText = msg;
          });
        };
      });
    }
    
    
    async function ResumableUploadForGoogleDrive(accessToken) {
    
      const n = 10; // You can adjust the chunk size.
    
      const f = document.getElementById("file");
      const files = [...f.files];
      const splitFiles = [...Array(Math.ceil(files.length / n))].map((_) => files.splice(0, n));
      for (let i = 0; i < splitFiles.length; i++) {
        const res = await Promise.all(splitFiles[i].map(async (file, j) => await upload({ accessToken, file, idx: `${i}_${j}` })));
        console.log(res);
      }
    }
    </script>
    

    Testing:

    When this script is run, the following result is obtained. In this case, const n = 2; is used. So, the files are uploaded every 2 files with the asynchronous process.

    enter image description here

    References:

    Login or Signup to reply.
  2. ALTERNATIVE SOLUTION

    This is another method by which you may try to upload 1,000 files, but limit the script to 10 files concurrently at a time.

    Here’s an example:

    SAMPLE1

    SAMPLE2

    After choosing 11 files, the first 10 start downloading, and when they’re finished, the 11th file will start to download and complete.

    This is the modified version of index.html:

    <input type="file" id="file" multiple="true" />
    <input type="button" onclick="run()" value="Upload" />
    <div id="progress"></div>
    
    <script src="https://cdn.jsdelivr.net/gh/tanaikech/ResumableUploadForGoogleDrive_js@master/resumableupload_js.min.js"></script>
    
    <script>
      var q = [];
      var c = 10;
    
      function run() {
        google.script.run
          .withSuccessHandler(accessToken => {
            [...document.getElementById("file").files].forEach(file => {
              q.push({ file, accessToken });
            });
            upload();
          })
          .getAuth();
      }
    
      function upload() {
        while (q.length > 0 && c > 0) {
          const { file, accessToken } = q.shift();
          ResumableUploadForGoogleDrive(file, accessToken);
          c--;
        }
      }
    
      function ResumableUploadForGoogleDrive(file, accessToken) {
        let fr = new FileReader();
        fr.fileName = file.name;
        fr.fileSize = file.size;
        fr.fileType = file.type;
        fr.readAsArrayBuffer(file);
        fr.onload = e => {
          const id = "p" + Date.now();
          const div = document.createElement("div");
          div.id = id;
          document.getElementById("progress").appendChild(div);
          document.getElementById(id).innerHTML = "Initializing.";
          const resource = {
            fileName: e.target.fileName,
            fileSize: e.target.fileSize,
            fileType: e.target.fileType,
            fileBuffer: e.target.result,
            accessToken
          };
          const ru = new ResumableUploadToGoogleDrive();
          ru.Do(resource, (res, err) => {
            if (err) {
              console.log(err);
              return;
            }
            console.log(res);
            let msg = "";
            if (res.status === "Uploading") {
              msg =
                Math.round(
                  (res.progressNumber.current / res.progressNumber.end) * 100
                ) +
                "% (" +
                e.target.fileName +
                ")";
            } else {
              msg = res.status + " (" + e.target.fileName + ")";
            }
            if (res.status === "Done") google.script.run.putFileInf(res.result);
            document.getElementById(id).innerText = msg;
            if (res.status === "Done") {
              c++;
              upload();
            }
          });
        };
      }
    </script>
    

    The q variable stores the files selected, the c variable serves as the concurrent counter, and the upload() function processes the download 10 at a time.

    REFERENCE

    Login or Signup to reply.
Please signup or login to give your own answer.
Back To Top
Search