How to get progress of file upload using requests.post() if the file was just uploaded via form-data? (Not sure whether this is a streaming process)

user_18577900 0
Viewed 458

Suppose I'm running some kind of web service with python & flask aiming to provide file upload. It should be noted that my service is only a kind of transfer station, which means I'll "repost" the file elsewhere, say, a massive centralized file storage, as soon as I receive a file from frontend form-data. The flask code looks like this:

@admin.route('/data', methods=['POST'])
def data_upload():
    if 'file' in request.files:
        f = request.files['file']
        try:
            r = Requests.post(DataConfig.server + '/' + '/upload.php', files={'file': (f.filename, f)})
            return {'e': 0, 'msg': r.content.decode('utf-8')}
        except RequestException:
            return ReturnCode.ERR_NETWORK_FAILURE
    return ReturnCode.ERR_BAD_DATA

It is not allowed that the frontend directly send file or data to "upload.php", since the server key should not be exposed to users.

Now I've got two problems that confuse me.

  • Is this process streaming or streaming-ready? I mean, whether my python server will receive the file and store it somewhere temporarily, or the file will be repost in chunks, like iter_block? If streaming is not readily enabled, how can I do so to enable large-file uploading user experience?
  • Is it possible to provide the user (frontend, browser) information about the progress of uploading?

For question 2, there exists some posts indicating that tqdm will help, but tqdm is a cli tool and printing anything in the python server's terminal makes no sense, and document of tqdm doesn't show me any obvious reference about their apis to get a percentage. What's more, I think that maybe sending this percentage to frontend may require some xhr techniques, which I believe my code can't deal with.

python-requests flask python ajax
1 Answers

PART 1 OF MY ANSWER : THE JAVASCRIPT

async function ubox_send_chucks (divid) {
    
    var chunstart = 0;
    var chunend = 0;
    var chunksize = 1024 * 1024 * 9;
    var newblobchun_id = parseInt(Math.floor(Math.random() * 100000000) + 1);
    fi = $("#"+divid).files;
    stubuploads = fi.length;

    for (var i = 0; i < fi.length; i++) { 
        var thismediaby_id = (newblobchun_id + i + 3); 
        $("#progressx").append("<div id=\'loaderz_"+thismediaby_id+"_message\' class=\'padding3 margin3\' >FILE "+i+" (PREP)</div>"); 
        $("#phpx").append("<div id=\'phpz_"+thismediaby_id+"_message\' class=\'padding3 margin3\' >FILE "+i+" (PREP)</div>"); 
    }

    for (var i = 0; i < fi.length; i++) {
    
        if ( fi[i].size > 0 ) {
        
            var numberofchunks = Math.ceil( fi[i].size / chunksize );
            var thismediaby_id = (newblobchun_id + i + 3);
            
            logx("FILE "+i+" -- size: "+fi[i].size+" name: "+fi[i].name+" n of chunks to send: "+numberofchunks );
        
            // SEND EACH CHUNKS
            for (var c = 0; c <= numberofchunks; c++) {
            
                chunstart = (c * chunksize); chunend = chunstart + chunksize + 1; if ( chunend > fi[i].size ){ chunend = fi[i].size; }
                
                var thischunk = fi[i].slice(chunstart, chunend);
                var thismediaby_name = thismediaby_id+"--chunk_"+c;
                
                console.log("FILE "+i+" send chunk: "+c+" start: "+chunstart+" end: "+chunend);
            
                upload_percent = ( c / numberofchunks ) * 100;
                $("#loaderz_"+thismediaby_id+"_message").html("FILE "+i+" : " + Math.round(upload_percent) + " %");
        
                var fd = new FormData(); 
                fd.append("data", thischunk, encodeURIComponent(thismediaby_name)); 
                fd.append("thismedia_id", encodeURIComponent(thismediaby_id)); 
                fd.append("thismedia_name", encodeURIComponent(fi[i].name)); 
                fd.append("numberofchunks", encodeURIComponent(numberofchunks)); 
                fd.append("thischunk_number", encodeURIComponent(c)); 
                fd.append("thisfilex", encodeURIComponent(i)); 
                fd.append("thissession", encodeURIComponent(thissession)); 
                fd.append("howmanyfiles", encodeURIComponent(fi.length)); 
                
                var pcache = (Math.floor(Math.random() * 100000000) + 1); 
                await fetch("/templates/tests_ubox_slice/slice_receiver.php?pcache="+pcache, { method: "POST", body: fd })
                    .then(function (response) { return response.text(); })
                    .then(function (html) { $("#php_message").html(html); })
        
            }
            
            // WHEN ALL CHUNKS SENT TRIGGER A RECOMBINE (SAFER)
            // AJAX FUNCTION HERE https://stubs.s3.filebase.com/media/stubs/202111100393/recookies.js
            var combinex = [];
            combinex["thismedia_id"] = encodeURIComponent(thismediaby_id); 
            combinex["thissession"] = encodeURIComponent(thissession); 
            combinex["thismedia_name"] = encodeURIComponent(fi[i].name); 
            stubajax("combiner","/templates/tests_ubox_slice/slice_combiner.php?pcache="+pcache,combinex);
                
        }
    
    }

}

PART II PHP RECEIVER

function clean () { ....stuff to make user input more secure like rawurldecode, html_entity_decode, stripslashes, etc }

$pcache = clean( $_REQUEST['pcache'] ?? '' ) ;
$thismedia_id = clean( $_REQUEST['thismedia_id'] ?? '' ) ;
$thismedia_name = clean( $_REQUEST['thismedia_name'] ?? '' );
$thismedia_ext = pathinfo($exif_file)['extension'] ?? '';
$numberofchunks = clean( $_REQUEST['numberofchunks'] ?? '' ) ;
$thischunk_number = clean( $_REQUEST['thischunk_number'] ?? '' ) ;
$thisfilex = clean( $_REQUEST['thisfilex'] ?? '' ) ;
$howmanyfiles = clean( $_REQUEST['howmanyfiles'] ?? '' ) ;
$thissession = clean( $_REQUEST['thissession'] ?? '' ) ;

if ( $pcache != '' ) {
    
    // DEV
    // var_dump(['thismedia_id'=>$thismedia_id,'thismedia_name'=>$thismedia_name,'numberofchunks'=>$numberofchunks,'thischunk_number'=>$thischunk_number,'thisfilex'=>$thisfilex]);

    // WHERE TO SAVE CHUNKS
    $received_chunk_dir = '/temporary_path_here_you_receiver_chunks/'.$thissession.'/'.$thismedia_id.'/';
    $received_chunk_path = $received_chunk_dir.$thismedia_id.'_chunk_'.$thischunk_number;

    // IF DIRECTORY NOT THERE CREATE IT
    if ( !file_exists($received_chunk_dir) ) { shell_exec('mkdir -p "'.$received_chunk_dir.'"'); }

    // MOVE_UPLOADED_FILE
    foreach ($_FILES as $thisfilekey => $thisfilechun) { 
        if ( isset($thisfilechun['name']) && isset($thisfilechun['tmp_name']) ) { 
            if ( filesize($thisfilechun['tmp_name']) > 1 ) { move_uploaded_file($thisfilechun['tmp_name'],$received_chunk_path); }
        }
    }
    
    // ECHO PERCENT PROGRESSION FOR THAT FILE
    echo '<script>$("#phpz_'.$thismedia_id.'_message").html("FILE '.$thisfilex.' : received chunk number '.$thischunk_number.' of '.$numberofchunks.' chunks");</script>';

}

PART III PHP COMBINER

$pcache = clean( $_REQUEST['pcache'] ?? '' ) ;
$thismedia_id = clean( $_REQUEST['thismedia_id'] ?? '' ) ;
$thismedia_name = accentx(clean( $_REQUEST['thismedia_name'] ?? '' ),'unlock');
$thismedia_ext = exiftensionx(['exif_file'=>$thismedia_name]);
$numberofchunks = clean( $_REQUEST['numberofchunks'] ?? '' ) ;
$thischunk_number = clean( $_REQUEST['thischunk_number'] ?? '' ) ;
$thisfilex = clean( $_REQUEST['thisfilex'] ?? '' ) ;
$howmanyfiles = clean( $_REQUEST['howmanyfiles'] ?? '' ) ;
$thissession = clean( $_REQUEST['thissession'] ?? '' ) ;

if ( $thissession != '' ) {

    // PATH
    $received_chunk_dir = '/temporary_path_here_you_receiver_chunks/'.$thissession;
    $received_final_path = $received_chunk_dir.'/'.$thismedia_id.'/'.$thismedia_id.'.'.$thismedia_ext;

    // GET OF SORTED FILES -V because chunk_1, chunk_2, ...., chunk_9, chunk_10, chunk_11 ==> dont want chunk_1,chunk_10,chunk_2 but chunk_1,chunk_2,...,chunk_10
    $all_chunks_raw = shell_exec('ls '.$received_chunk_dir.'/'.$thismedia_id.'/* | sort -V');
            
    if ( $all_chunks_raw != '' ) {

        // GET LS OF ALL CHUNKS
        $all_chunks_explo = array_filter(explode(PHP_EOL,$all_chunks_raw));
        
        // IF ONLY 1 CHUNK JUST RENAME
        if ( count($all_chunks_explo) == 1 ) { rename($all_chunks_explo[0],$received_final_path); @unlink($all_chunks_explo[0]); }
        else {
            // RECOMBINE ALL CHUNKS WITH FOPEN FREAD chunksize = 1024 * 1024 * 9 = 9437184 from javascript HAS TO BE THE SAME VALUE
            foreach ( $all_chunks_explo as $chunkey => $chunx ){    
                $file = fopen($chunx, 'rb'); $buff = fread($file, 9437184); fclose($file);
                $final = fopen($received_final_path, 'ab'); $write = fwrite($final, $buff); fclose($final);
            }
            // DELETE CHUNKS AFTER COMBINE
            shell_exec('ls '.$received_chunk_dir.'/'.$thismedia_id.' -name "*_chunk_*" -delete');
        }

    }
    
    // HERE YOU CAN FFMPEG, IMAGEMAGICK, ETC TO CONVERT TO WEB COMPATIBLE FORMAT
    
    // HERE YOU CAN SEND FILES TO S3 BUCKET (services like filebase.com)

    // DELETE FILE AFTER SENDING TO S3 BUCKET IF YOU NEED TO CLEAR SPACE

    echo '<script>$("#phpz_'.$thismedia_id.'_message").append(" ---COMBINATION DONE---");</script>';

}

******** NOTE : because of async: it's important to WAIT for all chuncks BEFORE combining because of networks (internet) factors, chunks don't always arrive one after an other, some times, you get chunk 1, then 3 then 6 then 2. it's why a ajax call sends a signal to combiner to "tell" ok, all chuncks are sent

enter image description here

user_180680280
Related