I am trying to upload very large files (some are +30GB) by chunks. I was using Vue.JS and the dropzone library but noticed that when using Firefox (but not with Chrome) and uploading large files, the memory would blow up (the process "GeckoMain" seems to grow as big as the file itself). After writing my own chunk uploading code, the same problem appeared.
So I reproduced the chunk uploading part in a smaller setup with just a bit of html/javascript and a flask server in the backend. You can see by uploading a large file the memory consumption growing in Firefox whereas using Chrome it does not. Does someone have an idea of what is going on here ? Should I use something different than file.slice ?
While the process "GeckoMain" is growing in memory, if I use the developer tools and snapshot the memory, before, during and after the upload, it does not change.
The frontend code :
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8" />
<title>File Uploader</title>
</head>
<body>
<input
id="file"
type="file"
name="static_file"
onchange="handleFiles(this.files)"
/>
<script>
function onChunkLoad(oEvent, oreq, file) {
let chunk_start = parseInt(sessionStorage.chunk_start)
let chunk_size = parseInt(sessionStorage.chunk_size)
let resp = JSON.parse(oreq.response)
chunk_start += chunk_size
sessionStorage.chunk_start = chunk_start
if (chunk_start < file.size) {
createChunk(file)
} else {
console.log('Upload ended')
}
}
function uploadChunk(file, chunk_form) {
let oreq = new XMLHttpRequest()
let upload_chunk_url = '/upload_static_file'
oreq.open('POST', upload_chunk_url, true)
oreq.onload = event => onChunkLoad(event, oreq, file)
oreq.send(chunk_form)
}
function createChunk(file) {
let nb_chunks = parseInt(sessionStorage.nb_chunks)
let chunk_start = parseInt(sessionStorage.chunk_start)
let chunk_size = parseInt(sessionStorage.chunk_size)
let chunk_counter = parseInt(sessionStorage.chunk_counter)
chunk_counter += 1
let chunk_end = Math.min(chunk_start + chunk_size, file.size)
let chunk = file.slice(chunk_start, chunk_end)
let chunk_form = new FormData()
chunk_form.set('file', chunk, file.name)
chunk_form.set('chunkbyteoffset', chunk_start)
sessionStorage.chunk_counter = chunk_counter
sessionStorage.chunk_end = chunk_end
uploadChunk(file, chunk_form)
}
function handleFiles(files) {
let file = files[0]
let chunk_size = 256 * 1024 * 1024
let nb_chunks = Math.ceil(file.size / chunk_size)
let chunk_start = 0
let chunk_counter = -1
sessionStorage.chunk_size = chunk_size
sessionStorage.nb_chunks = nb_chunks
sessionStorage.chunk_start = chunk_start
sessionStorage.chunk_counter = chunk_counter
createChunk(file)
}
</script>
</body>
</html>
Here is the backend code as well :
import io
from flask import Flask, request, render_template, jsonify
from flask_cors import CORS
app = Flask(__name__)
CORS(app)
@app.route("/", methods=["GET", "POST"])
def get_message():
return render_template("index.html")
@app.route("/upload_static_file", methods=["POST"])
def upload_static_file():
file_form = request.form
file = request.files.get("file")
filepath = file.filename
chunk_offset = int(file_form["chunkbyteoffset"])
with io.open(filepath, "ab") as f:
f.seek(chunk_offset)
chunk = file.stream.read()
f.write(chunk)
resp = {"success": True, "response": "file saved!"}
return jsonify(resp), 200
if __name__ == "__main__":
app.run(host="0.0.0.0", debug=True)
UPDATE : TL;DR : Memory leak might come from very fast repeated calls to "file.slice" in Firefox, ReadableStream solves the memory problem but you lose the ability to read the file "anywhere" quickly.
Following Keith advice to use a ReadableStream instead of file.slice, it is indeed possible to then upload chunks without the memory leak. This could suggest that fast repeated calls to file.slice in Firefox may cause some kind of memory leak. I couldn't find any issue or statements about that behavior. ReadableStream solves the memory issue but then you lose the ability to retrieve any "slice" of the file you want without having to read through the beginning and discard it. Is there another way that could help me with that ? (For resumable uploads, I need to be able to read any slice of the file I want)