<commit_before>var Loader = (function(){
  var eventNames = {
    AUDIO: 'canplaythrough',
    IMG: 'load'
  };
  var itemsToAjax = [];
  var itemsToLoad = [];
  var rootPath = '';
  return {
    setRootPath: function(path){
      rootPath = path;
    },
    loadAjax: function(filepath, callback) {
      itemsToAjax.push({
        filepath: filepath,
        callback: callback
      });
    },
    loadTexture: function(filepath, callback) {
      var image = new Image();
      image.crossOrigin = "Anonymous";
      var texture = new THREE.Texture();
      texture.image = image;
      texture.sourceFile = filepath;
      Loader.load(filepath, image, function() {
        texture.needsUpdate = true;
        callback && callback();
      });
      return texture;
    },
    load: function(filepath, element, callback) {
      console.log('starting to load', filepath);
      itemsToLoad.push({
        filepath: filepath,
        element: element,
        callback: callback
      });
    },
    start: function(onprogress, oncomplete) {
      var maxWaitingCount = itemsToAjax.length + itemsToLoad.length;
      var waitingCount = maxWaitingCount;
      function registerAsLoaded(item) {
        onprogress(100 - waitingCount / maxWaitingCount * 100);
        console.log('finished loading', item.filepath);
        if(!--waitingCount) {
          oncomplete();  
        }
      }
      itemsToLoad.forEach(function(item) {
        var eventName = eventNames[item.element.tagName];
        item.element.addEventListener(eventName, listener);
        function listener() {
          item.element.removeEventListener(eventName, listener);
          item.callback && item.callback(); 
          registerAsLoaded(item); 
        };

        if(window.FILES) {
          var prefix = {
            'jpg': 'data:image/jpg;base64,',
            'jpeg': 'data:image/jpg;base64,',
            'png': 'data:image/png;base64,',
            'mp3': 'data:audio/mp3;base64,',
          }[item.filepath.slice(-3)];
          console.log(item.filepath, prefix + (FILES[item.filepath] && FILES[item.filepath].slice(0, 10)));
          item.element.src = prefix + FILES[item.filepath];
        } else {
          item.element.src = rootPath + item.filepath;
        }
      });
      itemsToAjax.forEach(function(item) {
        if(window.FILES) {
          console.log(item.filepath, FILES[item.filepath] && atob(FILES[item.filepath]).slice(0, 10));<commit_msg>Add mp4 and svg support to the b64 loader<commit_after>var Loader = (function(){
  var eventNames = {
    AUDIO: 'canplaythrough',
    IMG: 'load'
  };
  var itemsToAjax = [];
  var itemsToLoad = [];
  var rootPath = '';
  return {
    setRootPath: function(path){
      rootPath = path;
    },
    loadAjax: function(filepath, callback) {
      itemsToAjax.push({
        filepath: filepath,
        callback: callback
      });
    },
    loadTexture: function(filepath, callback) {
      var image = new Image();
      image.crossOrigin = "Anonymous";
      var texture = new THREE.Texture();
      texture.image = image;
      texture.sourceFile = filepath;
      Loader.load(filepath, image, function() {
        texture.needsUpdate = true;
        callback && callback();
      });
      return texture;
    },
    load: function(filepath, element, callback) {
      console.log('starting to load', filepath);
      itemsToLoad.push({
        filepath: filepath,
        element: element,
        callback: callback
      });
    },
    start: function(onprogress, oncomplete) {
      var maxWaitingCount = itemsToAjax.length + itemsToLoad.length;
      var waitingCount = maxWaitingCount;
      function registerAsLoaded(item) {
        onprogress(100 - waitingCount / maxWaitingCount * 100);
        console.log('finished loading', item.filepath);
        if(!--waitingCount) {
          oncomplete();  
        }
      }
      itemsToLoad.forEach(function(item) {
        var eventName = eventNames[item.element.tagName];
        item.element.addEventListener(eventName, listener);
        function listener() {
          item.element.removeEventListener(eventName, listener);
          item.callback && item.callback(); 
          registerAsLoaded(item); 
        };

        if(window.FILES) {
          var prefix = {
            'jpg': 'data:image/jpg;base64,',
            'jpeg': 'data:image/jpg;base64,',
            'png': 'data:image/png;base64,',
            'mp3': 'data:audio/mp3;base64,',
            'mp4': 'data:video/mp4;base64,',
            'svg': 'data:image/svg+xml;base64,',
          }[item.filepath.slice(-3)];
          console.log(item.filepath, prefix + (FILES[item.filepath] && FILES[item.filepath].slice(0, 10)));
          item.element.src = prefix + FILES[item.filepath];
        } else {
          item.element.src = rootPath + item.filepath;
        }
      });
      itemsToAjax.forEach(function(item) {
        if(window.FILES) {
          console.log(item.filepath, FILES[item.filepath] && atob(FILES[item.filepath]).slice(0, 10));
{'fraction_non_alphanumeric': 0.1358392999204455, 'fraction_numerical': 0.010938743038981702, 'mean_word_length': 2.341528239202658, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 1}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '315055', 'n_tokens_mistral': 1541, 'n_tokens_neox': 1481, 'n_words': 277}