<commit_before>/**
 * @file Loader
 * @author <NAME> <<EMAIL>>
 */


///////////////
// Datasource

NGL.DatasourceRegistry = {

    sourceDict: {},

    listing: undefined,
    trajectory: undefined,

    __passThrough: {
        getUrl: function( path ){
            return path;
        }
    },

    add: function( name, datasource ){
        name = name.toLowerCase();
        if( name in this.sourceDict ){
            NGL.warn( "overwriting datasource named '" + name + "'" );
        }
        this.sourceDict[ name ] = datasource;
    },

    get: function( name ){
        name = name || "";
        name = name.toLowerCase();
        if( name in this.sourceDict ){
            return this.sourceDict[ name ];
        }else if( [ "http", "https", "ftp" ].indexOf( name ) !== -1 ){
            return this.__passThrough;
        }else if( !name ){
            return this.__passThrough;
        }else{
            NGL.error( "no datasource named '" + name + "' found" );
        }
    }

};


NGL.getDataInfo = function( src ){

    var info = NGL.getFileInfo( src );
    var datasource = NGL.DatasourceRegistry.get( info.protocol );
    var url = datasource.getUrl( info.src );

    return NGL.getFileInfo( url );

};


NGL.StaticDatasource = function( baseUrl ){

    baseUrl = baseUrl || "";

    this.getUrl = function( src ){
        var info = NGL.getFileInfo( src );
        return NGL.getAbsolutePath( baseUrl + info.path );
    };

};


NGL.RcsbDatasource = function(){

    var baseUrl = "http://www.rcsb.org/pdb/files/";

    this.getUrl = function( src ){
        // valid path are
        // XXXX.pdb, XXXX.pdb.gz, XXXX.cif, XXXX.cif.gz
        // XXXX defaults to XXXX.cif.gz
        var info = NGL.getFileInfo( src );
        var file;
        if( [ "pdb", "cif" ].indexOf( info.ext ) !== -1 &&
            ( info.compressed === false || info.compressed === "gz" )
        ){
            file = info.path;
        }else{
            file = info.name + ".cif.gz";
        }
        return baseUrl + file;
    };

};

NGL.DatasourceRegistry.add(
    "rcsb", new NGL.RcsbDatasource()
);


///////////
// Loader

NGL.Loader = function( src, params ){

    var p = Object.assign( {}, params );

    var binaryExtList = [ "msgpack", "dcd", "mrc", "ccp4", "map" ];
    var binary = binaryExtList.indexOf( p.ext ) !== -1;

    this.compressed = p.compressed || false;
    this.binary = p.binary !== undefined ? p.binary : binary;
    this.name = p.name || "";
    this.ext = p.ext || "";
    this.dir = p.dir || "";
    this.path = p.path || "";
    this.protocol = p.protocol || "";

    this.params = params;

    //

    var streamerParams = {
        compressed: this.compressed,
        binary: this.binary
    };

    if( src instanceof File || src instanceof Blob ){
        this.streamer = new NGL.FileStreamer( src, streamerParams );
    }else{
        this.streamer = new NGL.NetworkStreamer( src, streamerParams );
    }

    if( typeof p.onProgress === "function" ){
        this.streamer.onprogress = p.onprogress;
    }

};<commit_msg>added .dxbin to the list of binary files<commit_after>/**
 * @file Loader
 * @author <NAME> <<EMAIL>>
 */


///////////////
// Datasource

NGL.DatasourceRegistry = {

    sourceDict: {},

    listing: undefined,
    trajectory: undefined,

    __passThrough: {
        getUrl: function( path ){
            return path;
        }
    },

    add: function( name, datasource ){
        name = name.toLowerCase();
        if( name in this.sourceDict ){
            NGL.warn( "overwriting datasource named '" + name + "'" );
        }
        this.sourceDict[ name ] = datasource;
    },

    get: function( name ){
        name = name || "";
        name = name.toLowerCase();
        if( name in this.sourceDict ){
            return this.sourceDict[ name ];
        }else if( [ "http", "https", "ftp" ].indexOf( name ) !== -1 ){
            return this.__passThrough;
        }else if( !name ){
            return this.__passThrough;
        }else{
            NGL.error( "no datasource named '" + name + "' found" );
        }
    }

};


NGL.getDataInfo = function( src ){

    var info = NGL.getFileInfo( src );
    var datasource = NGL.DatasourceRegistry.get( info.protocol );
    var url = datasource.getUrl( info.src );

    return NGL.getFileInfo( url );

};


NGL.StaticDatasource = function( baseUrl ){

    baseUrl = baseUrl || "";

    this.getUrl = function( src ){
        var info = NGL.getFileInfo( src );
        return NGL.getAbsolutePath( baseUrl + info.path );
    };

};


NGL.RcsbDatasource = function(){

    var baseUrl = "http://www.rcsb.org/pdb/files/";

    this.getUrl = function( src ){
        // valid path are
        // XXXX.pdb, XXXX.pdb.gz, XXXX.cif, XXXX.cif.gz
        // XXXX defaults to XXXX.cif.gz
        var info = NGL.getFileInfo( src );
        var file;
        if( [ "pdb", "cif" ].indexOf( info.ext ) !== -1 &&
            ( info.compressed === false || info.compressed === "gz" )
        ){
            file = info.path;
        }else{
            file = info.name + ".cif.gz";
        }
        return baseUrl + file;
    };

};

NGL.DatasourceRegistry.add(
    "rcsb", new NGL.RcsbDatasource()
);


///////////
// Loader

NGL.Loader = function( src, params ){

    var p = Object.assign( {}, params );

    var binaryExtList = [ "msgpack", "dcd", "mrc", "ccp4", "map", "dxbin" ];
    var binary = binaryExtList.indexOf( p.ext ) !== -1;

    this.compressed = p.compressed || false;
    this.binary = p.binary !== undefined ? p.binary : binary;
    this.name = p.name || "";
    this.ext = p.ext || "";
    this.dir = p.dir || "";
    this.path = p.path || "";
    this.protocol = p.protocol || "";

    this.params = params;

    //

    var streamerParams = {
        compressed: this.compressed,
        binary: this.binary
    };

    if( src instanceof File || src instanceof Blob ){
        this.streamer = new NGL.FileStreamer( src, streamerParams );
    }else{
        this.streamer = new NGL.NetworkStreamer( src, streamerParams );
    }

    if( typeof p.onProgress === "function" ){
        this.streamer.onprogress = p.onprogress;
    }

};
{'fraction_non_alphanumeric': 0.16125892277741727, 'fraction_numerical': 0.0012978585334198572, 'mean_word_length': 1.9469407265774379, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4285705', 'n_tokens_mistral': 2044, 'n_tokens_neox': 1977, 'n_words': 460}