Scrapping memory issue (Out Of Memory)

Viewed 16

Good day,

I have cameras that are saving videos by clips on a cloud service, with a limit on N clips. In order to save more clips, I made a scrapping script that gets all my video links from the website where I can preview them. (to be executed on page load with a JS extension)

Note that these are M3U8 files. Here is my script:

var cameraScrapper = {
    playerReady: false,
    videos: {},
    
    cameraCurrent: 0,
    cameras: [
        'cid1', // #1
        'cid2', // #3
        'cid3', // #2
    ],
    
    gui: function()  {
        return {
            calendarPreviousMonth: document.getElementById( 'event_calendar_section' ).getElementsByClassName( 'navigation' )[0].getElementsByClassName( 'prev' )[0],
            calendarNextMonth: document.getElementById( 'event_calendar_section' ).getElementsByClassName( 'navigation' )[0].getElementsByClassName( 'next' )[0],
            calendarSelectedDate: document.getElementById( 'event_selected_date' ),
            calendarMonthDays: document.getElementById( 'event_calendar_section' ).getElementsByClassName( 'month' )[0].children,
            cameras: document.getElementById( 'camera_list_dropdown' ).children,
            dayClips: document.getElementById( 'event_clips_wrap' ).children,
            noRecords: document.getElementsByClassName( 'no_recording' )[0]
        };
    },
    
    init: function() {
        document.getElementById( 'hls_player' ).addEventListener( 'canplay', function() { this.playerReady = true; }.bind( this ) );
        
        for( let i = 0; i < this.cameras.length; i++ ) {
            if( window.location.href.includes( '=' + this.cameras[i] + '&' ) ) {
                this.cameraCurrent = i;
                break;
            }
        }
        
        while( !this.gui().calendarPreviousMonth.children[0].classList.contains( 'disabled' ) ) {
            this.gui().calendarPreviousMonth.click();
        }
        
        this.scrap();
    },
    
    getVideo: function( pCamera, pTimeStamp ) {
        let performance = window.performance || window.mozPerformance || window.msPerformance || window.webkitPerformance || {};
        let requests = ( performance.getEntries() || {} );
        
        for( let i = 0; i < requests.length; i++ ) {
            if( requests[i].name.includes( '.ts?' ) ) {
                if( this.videos[encodeURIComponent( requests[i].name )] === undefined ) {
                    this.videos[encodeURIComponent( requests[i].name )] = { 'camera': pCamera, 'timestamp': this.formatDate( pTimeStamp ) };
                }
            }
        }
        
        if( Object.entries( this.videos ).length > 25 ) {
            this.save();
            
            this.videos = {};
            performance.clearMarks();
            performance.clearMeasures()
            performance.clearResourceTimings();
        }
    },
    
    formatDate( pTimestamp ) {
        let date = pTimestamp.split( ' ' )[0];
        let time = pTimestamp.split( ' ' )[1];
        
        let month = {
            JAN: '01',
            FEB: '02',
            MAR: '03',
            APR: '04',
            MAY: '05',
            JUN: '06',
            JUL: '07',
            AUG: '08',
            SEP: '09',
            OCT: '10',
            NOV: '11',
            DEC: '12'
        }
        
        return date.split( '.' )[2] + '-' + month[date.split( '.' )[1]] + '-' + date.split( '.' )[0] + ' ' + time;
    },
    
    sleep: function( pMs ) {
        return new Promise( resolve => setTimeout( resolve, pMs ) );
    },
    
    scrap: async function() {
        let scrapDays = true;
        
        while( scrapDays ) {
            for( let i = 0; i < this.gui().calendarMonthDays.length; i++ ) {
                if( !this.gui().calendarMonthDays[i].classList.contains( 'is-disabled' ) && !this.gui().calendarMonthDays[i].classList.contains( 'is-expired' ) && !this.gui().calendarMonthDays[i].classList.contains( 'is-future' ) ) {
                    // console.log( 'day ' + this.gui().calendarMonthDays[i].innerText );
                    
                    this.gui().calendarMonthDays[i].click();
                    
                    let clipsLoaded = false;
                    
                    if( this.gui().noRecords !== undefined ) {
                        clipsLoaded = !this.gui().noRecords.classList.contains( 'displayOff' )
                    }
                    
                    while( !clipsLoaded && ( this.gui().dayClips.length == 0 ) ) {
                        await this.sleep( 99 );
                        
                        if( this.gui().noRecords !== undefined ) {
                            clipsLoaded = !this.gui().noRecords.classList.contains( 'displayOff' )
                        }
                    }
                    
                    // console.log( 'clips ' + this.gui().dayClips.length );
                    
                    for( let j = 0; j < this.gui().dayClips.length; j++ ) {
                        this.playerReady = false;
                        this.gui().dayClips[j].children[0].click();
                        
                        while( !this.playerReady ) {
                            await this.sleep( 99 );
                        }
                        
                        this.getVideo( ( '' + this.cameraCurrent ), this.gui().calendarSelectedDate.innerText + ' ' + this.gui().dayClips[j].children[0].children[1].children[1].innerText );
                    }
                }
            }
            
            if( !this.gui().calendarNextMonth.children[0].classList.contains( 'disabled' ) ) {
                this.gui().calendarNextMonth.click();
            } else {
                scrapDays = false;
            }
        }
        
        this.save();
        
        this.sleep( 4999 );
        
        let cameraNext = this.cameraCurrent + 1;
        if( this.cameras.length >= cameraNext ) {
            cameraNext = 0;
        }
        
        window.location.href = 'https://cloudcamerasservice.com/camera?no=' + this.cameras[cameraNext] + '&model=CAM';
    },
    
    save() {
        let xhr = new XMLHttpRequest();
        xhr.open( 'POST', 'http://localhost/videomonitoring/stack.php', true );
        xhr.setRequestHeader( 'Content-type', 'application/x-www-form-urlencoded' );
        
        xhr.onload = ( e ) => { /* console.log( 'saved' ); */ };
        xhr.onerror = ( e ) => {};
        xhr.send( 'stack=' + encodeURIComponent( JSON.stringify( this.videos ) ) );
    }
};

setTimeout( function() { cameraScrapper.init() }.bind( this ), 4999 );

stack.php will handle file download from provided list, my script is mainly about getting the video links and date of recording.

But when launching my script, my browser run out of memory after 75-100 clips saved. Can you help me with this?

Not sure if it's comming from my script, indeed, I have the feeling this is due to preloading many videos on the same page.

0 Answers
Related