<commit_before>/*
* MIT Licensed
* http://www.23developer.com/opensource
* http://github.com/23/resumable.js
* <NAME>, <EMAIL>
*/

(function(){
"use strict";

  var Resumable = function(opts){
    if ( !(this instanceof Resumable) ) {
      return new Resumable(opts);
    }
    this.version = 1.0;
    // SUPPORTED BY BROWSER?
    // Check if these features are support by the browser:
    // - File object type
    // - Blob object type
    // - FileList object type
    // - slicing files
    this.support = (
                   (typeof(File)!=='undefined')
                   &&
                   (typeof(Blob)!=='undefined')
                   &&
                   (typeof(FileList)!=='undefined')
                   &&
                   (!!Blob.prototype.webkitSlice||!!Blob.prototype.mozSlice||!!Blob.prototype.slice||false)
                   );
    if(!this.support) return(false);


    // PROPERTIES
    var $ = this;
    $.files = [];
    $.defaults = {
      chunkSize:1*1024*1024,
      forceChunkSize:false,
      simultaneousUploads:3,
      fileParameterName:'file',
      throttleProgressCallbacks:0.5,
      query:{},
      headers:{},
      preprocess:null,
      method:'multipart',
      prioritizeFirstAndLastChunk:false,
      target:'/',
      testChunks:true,
      generateUniqueIdentifier:null,
      maxChunkRetries:undefined,
      chunkRetryInterval:undefined,
      permanentErrors:[400, 404, 415, 500, 501],
      maxFiles:undefined,
      withCredentials:false,
      xhrTimeout:0,
      maxFilesErrorCallback:function (files, errorCount) {
        var maxFiles = $.getOpt('maxFiles');
        alert('Please upload ' + maxFiles + ' file' + (maxFiles === 1 ? '' : 's') + ' at a time.');
      },
      minFileSize:1,
      minFileSizeErrorCallback:function(file, errorCount) {
        alert(file.fileName||file.name +' is too small, please upload files larger than ' + $h.formatSize($.getOpt('minFileSize')) + '.');
      },
      maxFileSize:undefined,
      maxFileSizeErrorCallback:function(file, errorCount) {
        alert(file.fileName||file.name +' is too large, please upload files less than ' + $h.formatSize($.getOpt('maxFileSize')) + '.');
      },
      fileType: [],
      fileTypeErrorCallback: function(file, errorCount) {
        alert(file.fileName||file.name +' has type not allowed, please upload files of type ' + $.getOpt('fileType') + '.');
      }
    };
    $.opts = opts||{};
    $.getOpt = function(o) {
      var $opt = this;
      // Get multiple option if passed an array
      if(o instanceof Array) {
        var options = {};
        $h.each(o, function(option){
          options[option] = $opt.getOpt(option);
        });
        return options;
      }
      // Otherwise, just return a simple option
      if ($opt instanceof ResumableChunk) {
        if (typeof $opt.opts[o] !== 'undefined') { return $opt.opts[o]; }
        else { $opt = $opt.fileObj; }
      }
      if ($opt instanceof ResumableFile) {
        if (typeof $opt.opts[o] !== 'undefined') { return $opt.opts[o]; }
        else { $opt = $opt.resumableObj; }
      }
      if ($opt instanceof Resumable) {
        if (typeof $opt.opts[o] !== 'undefined') { return $opt.opts[o]; }
        else { return $opt.defaults[o]; }
      }
    };
    
    // EVENTS
    // catchAll(event, ...)
    // fileSuccess(file), fileProgress(file), fileAdded(file, event), fileRetry(file), fileError(file, message),
    // complete(), progress(), error(message, file), pause()
    $.events = [];
    $.on = function(event,callback){
      $.events.push(event.toLowerCase(), callback);
    };
    $.fire = function(){
      // `arguments` is an object, not array, in FF, so:
      var args = [];
      for (var i=0; i<arguments.length; i++) args.push(arguments[i]);
      // Find event listeners, and support pseudo-event `catchAll`
      var event = args[0].toLowerCase();
      for (var i=0; i<=$.events.length; i+=2) {
        if($.events[i]==event) $.events[i+1].apply($,args.slice(1));
        if($.events[i]=='catchall') $.events[i+1].apply(null,args);
      }
      if(event=='fileerror') $.fire('error', args[2], args[1]);
      if(event=='fileprogress') $.fire('progress');
    };
    
    
    // INTERNAL HELPER METHODS (handy, but ultimately not part of uploading)
    var $h = {
      stopEvent: function(e){
        e.stopPropagation();
        e.preventDefault();
      },
      each: function(o,callback){
        if(typeof(o.length)!=='undefined') {
          for (var i=0; i<o.length; i++) {
            // Array or FileList
            if(callback(o[i])===false) return;
          }
        } else {
          for (i in o) {
            // Object
            if(callback(i,o[i])===false) return;
          }
        }
      },
      generateUniqueIdentifier:function(file){
        var custom = $.getOpt('generateUniqueIdentifier');
        if(typeof custom === 'function') {
          return custom(file);
        }
        var relativePath = file.webkitRelativePath||file.fileName||file.name; // Some confusion in different versions of Firefox
        var size = file.size;
        return(size + '-' + relativePath.replace(/[^0-9a-zA-Z_-]/img, ''));
      },
      contains:function(array,test) {
        var result = false;

        $h.each(array, function(value) {
          if (value == test) {
            result = true;
            return false;
          }
          return true;
        });

        return result;
      },
      formatSize:function(size){
        if(size<1024) {
          return size + ' bytes';
        } else if(size<1024*1024) {
          return (size/1024.0).toFixed(0) + ' KB';
        } else if(size<1024*1024*1024) {
          return (size/1024.0/1024.0).toFixed(1) + ' MB';
        } else {
          return (size/1024.0/1024.0/1024.0).toFixed(1) + ' GB';
        }
      },
      getTarget:function(params){
        var target = $.getOpt('target');
        if(target.indexOf('?') < 0) {
          target += '?';
        } else {
          target += '&';
        }
        return target + params.join('&');
      }
    };

    var onDrop = function(event){
      $h.stopEvent(event);

      //handle dropped things as items if we can (Chrome only at the moment)
      if (event.dataTransfer && event.dataTransfer.items) {
        loadFiles(event.dataTransfer.items, event);
      }
      //else fall back on treating them as files
      else if (event.dataTransfer && event.dataTransfer.files) {
        loadFiles(event.dataTransfer.files, event);
      }
    };
    var onDragOver = function(e) {
      e.preventDefault();
    };

    // INTERNAL METHODS (both handy and responsible for the heavy load)
    var loadFiles = function (files, event, queue, path){
      //initialize the queue object if it doesn't exist
      if (!queue) {
        queue = {
          total: 0,
          files: [],
          event: event
        };
      }

      //update the total number of things we plan to process
      queue.total += files.length;

      for (var i = 0; i < files.length; i++) {
        var file = files[i];
        var entry, reader;

        if (file.isFile || file.isDirectory) {
          entry = file;
        }
        else if (file.getAsEntry) {
          entry = file.getAsEntry();
        }
        else if (file.webkitGetAsEntry) {
          entry = file.webkitGetAsEntry();
        }
        else if (typeof file.getAsFile === 'function') {
          enqueueFileAddition(file.getAsFile(), queue, path);
          continue;
        }
        else if (File && file instanceof File) {
          enqueueFileAddition(file, queue, path);
          continue;
        }
        else {
          queue.total -= 1;
          continue;
        }

        if (!entry) {
          //there isn't anything we can do with this so shrink the total expected by 1
          queue.total -= 1;
        }
        else if (entry.isFile) {
          entry.file(function(file) {
            enqueueFileAddition(file, queue, path);
          }, function(err) {
            console.warn(err);
          });
        }
        else if (entry.isDirectory) {
          reader = entry.createReader();

          reader.readEntries(function(entries) {
            //process each thing in this directory recursively
            loadFiles(entries, event, queue, entry.fullPath);
            //this was a directory rather than a file so decrement the expected file count
            queue.total -= 1;
          }, function(err) {
            console.warn(err);
          });
        }
      }
    };

    var enqueueFileAddition = function(file, queue, path) {
      file.relativePath = path + '/' + file.name;
      queue.files.push(file);

      // If all the files we expect have shown up, then flush the queue.
      if (queue.files.length === queue.total) {
        appendFilesFromFileList(queue.files, queue.event);
      }<commit_msg>fix issues with relative path<commit_after>/*
* MIT Licensed
* http://www.23developer.com/opensource
* http://github.com/23/resumable.js
* <NAME>, <EMAIL>
*/

(function(){
"use strict";

  var Resumable = function(opts){
    if ( !(this instanceof Resumable) ) {
      return new Resumable(opts);
    }
    this.version = 1.0;
    // SUPPORTED BY BROWSER?
    // Check if these features are support by the browser:
    // - File object type
    // - Blob object type
    // - FileList object type
    // - slicing files
    this.support = (
                   (typeof(File)!=='undefined')
                   &&
                   (typeof(Blob)!=='undefined')
                   &&
                   (typeof(FileList)!=='undefined')
                   &&
                   (!!Blob.prototype.webkitSlice||!!Blob.prototype.mozSlice||!!Blob.prototype.slice||false)
                   );
    if(!this.support) return(false);


    // PROPERTIES
    var $ = this;
    $.files = [];
    $.defaults = {
      chunkSize:1*1024*1024,
      forceChunkSize:false,
      simultaneousUploads:3,
      fileParameterName:'file',
      throttleProgressCallbacks:0.5,
      query:{},
      headers:{},
      preprocess:null,
      method:'multipart',
      prioritizeFirstAndLastChunk:false,
      target:'/',
      testChunks:true,
      generateUniqueIdentifier:null,
      maxChunkRetries:undefined,
      chunkRetryInterval:undefined,
      permanentErrors:[400, 404, 415, 500, 501],
      maxFiles:undefined,
      withCredentials:false,
      xhrTimeout:0,
      maxFilesErrorCallback:function (files, errorCount) {
        var maxFiles = $.getOpt('maxFiles');
        alert('Please upload ' + maxFiles + ' file' + (maxFiles === 1 ? '' : 's') + ' at a time.');
      },
      minFileSize:1,
      minFileSizeErrorCallback:function(file, errorCount) {
        alert(file.fileName||file.name +' is too small, please upload files larger than ' + $h.formatSize($.getOpt('minFileSize')) + '.');
      },
      maxFileSize:undefined,
      maxFileSizeErrorCallback:function(file, errorCount) {
        alert(file.fileName||file.name +' is too large, please upload files less than ' + $h.formatSize($.getOpt('maxFileSize')) + '.');
      },
      fileType: [],
      fileTypeErrorCallback: function(file, errorCount) {
        alert(file.fileName||file.name +' has type not allowed, please upload files of type ' + $.getOpt('fileType') + '.');
      }
    };
    $.opts = opts||{};
    $.getOpt = function(o) {
      var $opt = this;
      // Get multiple option if passed an array
      if(o instanceof Array) {
        var options = {};
        $h.each(o, function(option){
          options[option] = $opt.getOpt(option);
        });
        return options;
      }
      // Otherwise, just return a simple option
      if ($opt instanceof ResumableChunk) {
        if (typeof $opt.opts[o] !== 'undefined') { return $opt.opts[o]; }
        else { $opt = $opt.fileObj; }
      }
      if ($opt instanceof ResumableFile) {
        if (typeof $opt.opts[o] !== 'undefined') { return $opt.opts[o]; }
        else { $opt = $opt.resumableObj; }
      }
      if ($opt instanceof Resumable) {
        if (typeof $opt.opts[o] !== 'undefined') { return $opt.opts[o]; }
        else { return $opt.defaults[o]; }
      }
    };
    
    // EVENTS
    // catchAll(event, ...)
    // fileSuccess(file), fileProgress(file), fileAdded(file, event), fileRetry(file), fileError(file, message),
    // complete(), progress(), error(message, file), pause()
    $.events = [];
    $.on = function(event,callback){
      $.events.push(event.toLowerCase(), callback);
    };
    $.fire = function(){
      // `arguments` is an object, not array, in FF, so:
      var args = [];
      for (var i=0; i<arguments.length; i++) args.push(arguments[i]);
      // Find event listeners, and support pseudo-event `catchAll`
      var event = args[0].toLowerCase();
      for (var i=0; i<=$.events.length; i+=2) {
        if($.events[i]==event) $.events[i+1].apply($,args.slice(1));
        if($.events[i]=='catchall') $.events[i+1].apply(null,args);
      }
      if(event=='fileerror') $.fire('error', args[2], args[1]);
      if(event=='fileprogress') $.fire('progress');
    };
    
    
    // INTERNAL HELPER METHODS (handy, but ultimately not part of uploading)
    var $h = {
      stopEvent: function(e){
        e.stopPropagation();
        e.preventDefault();
      },
      each: function(o,callback){
        if(typeof(o.length)!=='undefined') {
          for (var i=0; i<o.length; i++) {
            // Array or FileList
            if(callback(o[i])===false) return;
          }
        } else {
          for (i in o) {
            // Object
            if(callback(i,o[i])===false) return;
          }
        }
      },
      generateUniqueIdentifier:function(file){
        var custom = $.getOpt('generateUniqueIdentifier');
        if(typeof custom === 'function') {
          return custom(file);
        }
        var relativePath = file.webkitRelativePath||file.fileName||file.name; // Some confusion in different versions of Firefox
        var size = file.size;
        return(size + '-' + relativePath.replace(/[^0-9a-zA-Z_-]/img, ''));
      },
      contains:function(array,test) {
        var result = false;

        $h.each(array, function(value) {
          if (value == test) {
            result = true;
            return false;
          }
          return true;
        });

        return result;
      },
      formatSize:function(size){
        if(size<1024) {
          return size + ' bytes';
        } else if(size<1024*1024) {
          return (size/1024.0).toFixed(0) + ' KB';
        } else if(size<1024*1024*1024) {
          return (size/1024.0/1024.0).toFixed(1) + ' MB';
        } else {
          return (size/1024.0/1024.0/1024.0).toFixed(1) + ' GB';
        }
      },
      getTarget:function(params){
        var target = $.getOpt('target');
        if(target.indexOf('?') < 0) {
          target += '?';
        } else {
          target += '&';
        }
        return target + params.join('&');
      }
    };

    var onDrop = function(event){
      $h.stopEvent(event);

      //handle dropped things as items if we can (Chrome only at the moment)
      if (event.dataTransfer && event.dataTransfer.items) {
        loadFiles(event.dataTransfer.items, event);
      }
      //else fall back on treating them as files
      else if (event.dataTransfer && event.dataTransfer.files) {
        loadFiles(event.dataTransfer.files, event);
      }
    };
    var onDragOver = function(e) {
      e.preventDefault();
    };

    // INTERNAL METHODS (both handy and responsible for the heavy load)
    var loadFiles = function (files, event, queue, path){
      //initialize the queue object if it doesn't exist
      if (!queue) {
        queue = {
          total: 0,
          files: [],
          event: event
        };
      }

      //update the total number of things we plan to process
      queue.total += files.length;

      for (var i = 0; i < files.length; i++) {
        var file = files[i];
        var entry, reader;

        if (file.isFile || file.isDirectory) {
          entry = file;
        }
        else if (file.getAsEntry) {
          entry = file.getAsEntry();
        }
        else if (file.webkitGetAsEntry) {
          entry = file.webkitGetAsEntry();
        }
        else if (typeof file.getAsFile === 'function') {
          enqueueFileAddition(file.getAsFile(), queue, path);
          continue;
        }
        else if (File && file instanceof File) {
          enqueueFileAddition(file, queue, path);
          continue;
        }
        else {
          queue.total -= 1;
          continue;
        }

        if (!entry) {
          //there isn't anything we can do with this so shrink the total expected by 1
          queue.total -= 1;
        }
        else if (entry.isFile) {
          entry.file(function(file) {
            enqueueFileAddition(file, queue, path);
          }, function(err) {
            console.warn(err);
          });
        }
        else if (entry.isDirectory) {
          var directory=entry;
          reader = entry.createReader();

          reader.readEntries(function(entries) {
            //process each thing in this directory recursively
            loadFiles(entries, event, queue, directory.fullPath);
            //this was a directory rather than a file so decrement the expected file count
            queue.total -= 1;
          }, function(err) {
            console.warn(err);
          });
        }
      }
    };

    var enqueueFileAddition = function(file, queue, path) {
      //store the path to this file if it came in as part of a folder
      if (path) file.relativePath = path + '/' + file.name;
      queue.files.push(file);

      // If all the files we expect have shown up, then flush the queue.
      if (queue.files.length === queue.total) {
        appendFilesFromFileList(queue.files, queue.event);
      }
{'fraction_non_alphanumeric': 0.14477870179159458, 'fraction_numerical': 0.01250209298431657, 'mean_word_length': 1.9774011299435028, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4064508', 'n_tokens_mistral': 5438, 'n_tokens_neox': 5216, 'n_words': 1233}