<commit_before>
import urlToBase64 from 'imageurl-base64-revolunet';
import { defer } from 'Q';

class ImagesCache {
  constructor() {
    this.cache = {};
  }
  /**
   * use imageurl-base64 to convert an url to its base64 and return promise.
   * @method _addUrlToCache
   * @param  {String}       url url to the ressource
   * @return {Promise}          return a promise that resolve on success and rejects on error
   */
  _addUrlToCache(url) {
    let deferred = defer();
    urlToBase64({
      url: url,
      withCredentials: false
    }, function(err, data) {
      if (err) {
        //console.error('unable to load ' + url, err);
        deferred.reject(err);
        return;
      }
      deferred.resolve(data);
    });
    return deferred.promise;
  }
  /**
   * load in cache a bunch of urls
   * @method loa
   * @param  {Array|String} urls list of urls, or single url to cache
   * @return {Promise}  return a promise with progress({loaded: x, total: y}) and resolved when all images loaded
   */
  load(urls) {
    if (typeof urls === 'string') {
      urls = [urls];
    }
    const nbImages = urls.length;
    let nbImagesLoaded = 0;
    let deferred = defer();

    function urlLoaded() {
      nbImagesLoaded++;
      deferred.notify({
        loaded: nbImagesLoaded,
        total: nbImages
      });
      if (nbImagesLoaded === nbImages) {
        deferred.resolve();
      }
    }

    urls.forEach(url => {
      if (!this.cache[url]) {
        // add resource to cache
        // always increment counter, even when image load failed
        this._addUrlToCache(url).then(result => {
          this.cache[url] = result.dataUri;
          urlLoaded();
        }).catch(e => {
          urlLoaded();
        })
      } else {
        // also notify when image already in cache
        urlLoaded();
      }
    });
    return deferred.promise;
    // optionaly cache on FS or whatever
    // handle http:// urls
    // handle bundle:// urls for cordova bundled files
  }
  get(url, addToCache=true) {
    let cached;
    if (cached = this.cache[url]) {
      return cached;
    } else {<commit_msg>fix: use DOM and Image+canvas for b64 conversion<commit_after>
import { defer } from 'q';
import request from 'request';

import image2base64 from 'image2base64';

require('request').debug = true

// most little pixel to prevent safari leak, see http://www.fngtps.com/2010/mobile-safari-image-resource-limit-workaround/
const PICO_PIXEL = 'data:image/gif;base64,R0lGODlhAQABAAD/ACwAAAAAAQABAAACADs=';


/**
 * convert an url to its base64 value and return promise
 * we use canvas here due to strange browserify+buffer issue, see https://gist.github.com/revolunet/4417e4168ae2879a3604
 * @method getUrlAsBase64
 * @param  {String}       url url if the ressource
 * @return {Promise}          return a promise that resolve on success with b64 data and rejects on error
 */
function getUrlAsBase64(url) {
  let deferred = defer();
  var img = new Image();
  img.onload = function() {
    deferred.resolve(image2base64(img));
    img.onload = img.onerror = null;
    img.src = PICO_PIXEL;
    img = null;
  }
  img.onerror = function() {
    deferred.reject();
    img.onload = img.onerror = null;
    img.src = PICO_PIXEL;
    img = null;
  };
  img.src = url;
  return deferred.promise;
}



class ImagesCache {
  constructor() {
    this.cache = {};
  }

  /**
   * load in cache a bunch of urls
   * @method loa
   * @param  {Array|String} urls list of urls, or single url to cache
   * @return {Promise}  return a promise with progress({loaded: x, total: y}) and resolved when all images loaded
   */
  load(urls) {
    if (typeof urls === 'string') {
      urls = [urls];
    }
    const nbImages = urls.length;
    let nbImagesLoaded = 0;
    let deferred = defer();

    function urlLoaded() {
      nbImagesLoaded++;
      deferred.notify({
        loaded: nbImagesLoaded,
        total: nbImages
      });
      if (nbImagesLoaded === nbImages) {
        deferred.resolve();
      }
    }

    urls.forEach(url => {
      if (!this.cache[url]) {
        // add resource to cache
        // always increment counter, even when image load failed
        getUrlAsBase64(url).then(b64 => {
          this.cache[url] = b64;
          urlLoaded();
        }).catch(e => {
          urlLoaded();
        })
      } else {
        // also notify when image already in cache
        urlLoaded();
      }
    });
    return deferred.promise;
    // optionaly cache on FS or whatever
    // handle http:// urls
    // handle bundle:// urls for cordova bundled files
  }
  get(url, addToCache=true) {
    let cached;
    if (cached = this.cache[url]) {
      return cached;
    } else {
{'fraction_non_alphanumeric': 0.11907293217095472, 'fraction_numerical': 0.01275781416117372, 'mean_word_length': 2.3196894848270997, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2520624', 'n_tokens_mistral': 1529, 'n_tokens_neox': 1468, 'n_words': 431}