<commit_before>package pman.async;

import tannus.io.*;
import tannus.ds.*;
import tannus.geom.*;
import tannus.sys.*;
import tannus.sys.FileSystem in Fs;
import tannus.node.*;
import tannus.node.Fs as Nfs;

import gryffin.core.*;
import gryffin.display.*;
import gryffin.media.MediaObject;
import gryffin.audio.*;
import gryffin.Tools.now;

import pman.core.*;
import pman.media.*;
import pman.tools.mp4box.MP4Box;

import js.Browser.window;
import electron.Tools.defer;
import Std.*;
import tannus.math.TMath.*;

using tannus.math.TMath;
using StringTools;
using tannus.ds.StringUtils;
using Lambda;
using tannus.ds.ArrayTools;
using Slambda;

class Mp4InfoLoader {
    /* Constructor Function */
    public function new():Void {
        box = new MP4Box();
    }

/* === Instance Methods === */

    /**
      * initiate loading of the metadata on [path]
      */
    public function loadSync(path : Path):MP4Info {
        var bytes:ByteArray = Fs.read( path );
        var buffer:Buffer = bytes.getData();
        sfs(buffer, 0);

        box.appendBuffer( buffer );
        var info = box.getInfo();
        return info;
    }

    /**
      * initiate loading of the metadata
      */
    public function load(path : Path):Promise<MP4Info> {
        return Promise.create({
            var infoParsed:Bool = false;

            // register events
            box.onReady = (function(info : MP4Info) {
                infoParsed = true;

                return info;
            });
            box.onError = (function(error : Dynamic) {
                throw error;
            });

            var fileStart:Int = 0;
            // chunk size (1MB)
            var chunkSize:Int = (1 * 1024 * 1024);

            // prepare chunked read
            function readStep(data : Buffer):Bool {
                sfs(data, fileStart);
                fileStart = box.appendBuffer( data );
                @ignore return !infoParsed;
            }
            function readComplete():Void {
                trace('chunked read has stopped');
            }

            chunkedRead(path, chunkSize, readStep, readComplete);
        });
    }

    /**
      * chunked file-read
      */
    private function chunkedRead(path:Path, chunkSize:Int, step:Buffer->Bool, complete:Void->Void):Void {
        Nfs.open(path.toString(), 'r', function(err, fid:Int) {
            if (err != null) {
                throw err;
            }

            function readNextChunk():Void {
                var buf:Buffer = new Buffer( chunkSize );
                Nfs.read(fid, buf, 0, chunkSize, null, function(err, nread, b) {
                    if (err != null) {
                        throw err;
                    }

                    if (nread == 0) {
                        Nfs.close(fid, function(err) {
                            if (err != null) {
                                throw err;
                            }
                            complete();
                        });
                    }

                    var data:Buffer;
                    if (nread < chunkSize) {
                        data = buf.slice(0, nread);
                    }
                    else {
                        data = buf;
                    }

                    var continu = step( data );
                    if ( continu ) {
                        readNextChunk();
                    }
                    else {
                        Nfs.close(fid, function(err) {
                            if (err != null) {
                                throw err;
                            }
                            complete();
                        });
                    }
                });
            }

            readNextChunk();
        });
    }

    /**
      * when [this] has finished loading
      */
    private function _loaded(info : Dynamic):Void {
        trace( info );
    }

    /**
      * set the magical 'fileStart' property
      */
    private function sfs(buffer:Buffer, fileStart:Int):Void {
        var ab:Dynamic = buffer.buffer;
        ab.fileStart = fileStart;
    }<commit_msg>made pman.async.MP4InfoLoader a little more dope<commit_after>package pman.async;

import tannus.io.*;
import tannus.ds.*;
import tannus.geom.*;
import tannus.sys.*;
import tannus.sys.FileSystem in Fs;
import tannus.node.*;
import tannus.node.Fs as Nfs;

import gryffin.core.*;
import gryffin.display.*;
import gryffin.media.MediaObject;
import gryffin.audio.*;
import gryffin.Tools.now;

import pman.core.*;
import pman.media.*;
import pman.tools.mp4box.MP4Box;
import pman.tools.mp4box.MP4Metadata;

import js.Browser.window;
import electron.Tools.defer;
import Std.*;
import tannus.math.TMath.*;

using tannus.math.TMath;
using StringTools;
using tannus.ds.StringUtils;
using Lambda;
using tannus.ds.ArrayTools;
using Slambda;

class Mp4InfoLoader {
    /* Constructor Function */
    public function new():Void {
        box = new MP4Box();
    }

/* === Instance Methods === */

    /**
      * initiate loading of the metadata on [path]
      */
    public function loadSync(path : Path):MP4Metadata {
        var bytes:ByteArray = Fs.read( path );
        var buffer:Buffer = bytes.getData();
        sfs(buffer, 0);

        box.appendBuffer( buffer );
        var info = box.getInfo();
        return info;
    }

    /**
      * initiate loading of the metadata
      */
    public function load(path : Path):Promise<MP4Metadata> {
        return Promise.create({
            var infoParsed:Bool = false;

            // register events
            box.onReady = (function(info : MP4Info) {
                infoParsed = true;

                return info;
            });
            box.onError = (function(error : Dynamic) {
                throw error;
            });

            var fileStart:Int = 0;
            // chunk size (1MB)
            var chunkSize:Int = (1 * 1024 * 1024);

            // handle chunked-read errors
            function readReject(error : Dynamic):Void {
                throw error;
            }
            // prepare chunked read
            function readStep(data : Buffer):Bool {
                sfs(data, fileStart);
                fileStart = box.appendBuffer( data );
                @ignore return !infoParsed;
            }
            // when read is complete
            function readComplete():Void {
                trace('chunked read has stopped');
            }

            // initiate chunked-read
            chunkedRead(path, chunkSize, readStep, readComplete, readReject);
        });
    }

    /**
      * chunked file-read
      */
    private function chunkedRead(path:Path, chunkSize:Int, step:Buffer->Bool, complete:Void->Void, reject:Dynamic->Void):Void {
        Nfs.open(path.toString(), 'r', function(err, fid:Int) {
            if (err != null) {
                throw err;
            }

            function readNextChunk():Void {
                var buf:Buffer = new Buffer( chunkSize );
                Nfs.read(fid, buf, 0, chunkSize, null, function(err, nread, b) {
                    if (err != null) {
                        reject( err );
                        return ;
                    }

                    if (nread == 0) {
                        Nfs.close(fid, function(err) {
                            if (err != null) {
                                reject( err );
                                return ;
                            }
                            complete();
                        });
                    }

                    var data:Buffer;
                    if (nread < chunkSize) {
                        data = buf.slice(0, nread);
                    }
                    else {
                        data = buf;
                    }

                    var continu = step( data );
                    if ( continu ) {
                        readNextChunk();
                    }
                    else {
                        Nfs.close(fid, function(err) {
                            if (err != null) {
                                reject( err );
                                return ;
                            }
                            complete();
                        });
                    }
                });
            }

            readNextChunk();
        });
    }

    /**
      * when [this] has finished loading
      */
    private function _loaded(info : Dynamic):Void {
        trace( info );
    }

    /**
      * set the magical 'fileStart' property
      */
    private function sfs(buffer:Buffer, fileStart:Int):Void {
        var ab:Dynamic = buffer.buffer;
        ab.fileStart = fileStart;
    }
{'fraction_non_alphanumeric': 0.1063195320564285, 'fraction_numerical': 0.005390526436517949, 'mean_word_length': 1.234178836792211, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4531026', 'n_tokens_mistral': 2392, 'n_tokens_neox': 2270, 'n_words': 544}