<commit_before>import * as utils from 'base/utils';
import Promise from 'base/promise';
import Reader from 'base/reader';

var FILE_CACHED = {}; //caches files from this reader
var FILE_REQUESTED = {}; //caches files from this reader

var CSVReader = Reader.extend({

  /**
   * Initializes the reader.
   * @param {Object} reader_info Information about the reader
   */
  init: function(reader_info) {
    this._name = 'csv';
    this._data = [];
    this._basepath = reader_info.path;
    this._formatters = reader_info.formatters;
    if(!this._basepath) {
      utils.error("Missing base path for csv reader");
    }
  },

  /**
   * Reads from source
   * @param {Object} query to be performed
   * @param {String} language language
   * @returns a promise that will be resolved when data is read
   */
  read: function(query, language) {
    var _this = this;
    var p = new Promise();

    //this specific reader has support for the tag {{LANGUAGE}}
    var path = this._basepath.replace("{{LANGUAGE}}", language);

    //if no year
    if(!query.where.time) {
      path = path.replace(".csv", "-properties.csv");
    }
    //if only one year, files ending in "-YYYY.csv"
    else if(query.where.time && query.where.time[0].length === 1) {
      path = path.replace(".csv", "-" + query.where.time[0][0] + ".csv");
    }

    //replace conditional tags {{<any conditional>}}
    path = path.replace(/{{(.*?)}}/g, function(match, capture) {
      capture = capture.toLowerCase();
      if(utils.isArray(query.where[capture])) {
        return query.where[capture].sort().join('-');
      }
      return query.where[capture];
    });

    _this._data = [];

    (function(query, p) {

      //if cached, retrieve and parse
      if(FILE_CACHED.hasOwnProperty(path)) {
        parse(FILE_CACHED[path]);
      }
      //if requested by another hook, wait for the response
      else if(FILE_REQUESTED.hasOwnProperty(path)) {
        FILE_REQUESTED[path].then(function() {
          parse(FILE_CACHED[path]);
        });<commit_msg>Extend reader hack to support pop by age<commit_after>import * as utils from 'base/utils';
import Promise from 'base/promise';
import Reader from 'base/reader';

var FILE_CACHED = {}; //caches files from this reader
var FILE_REQUESTED = {}; //caches files from this reader

var CSVReader = Reader.extend({

  /**
   * Initializes the reader.
   * @param {Object} reader_info Information about the reader
   */
  init: function(reader_info) {
    this._name = 'csv';
    this._data = [];
    this._basepath = reader_info.path;
    this._formatters = reader_info.formatters;
    if(!this._basepath) {
      utils.error("Missing base path for csv reader");
    }
  },

  /**
   * Reads from source
   * @param {Object} query to be performed
   * @param {String} language language
   * @returns a promise that will be resolved when data is read
   */
  read: function(query, language) {
    var _this = this;
    var p = new Promise();

    //this specific reader has support for the tag {{LANGUAGE}}
    var path = this._basepath.replace("{{LANGUAGE}}", language);

    //hack: for time and age
    if(!query.where.time && !query.where.age) {
      path = path.replace(".csv", "-properties.csv");
    }
    //if only one year, files ending in "-YYYY.csv"
    else if(query.where.time && query.where.time[0].length === 1) {
      path = path.replace(".csv", "-" + query.where.time[0][0] + ".csv");
    }

    //replace conditional tags {{<any conditional>}}
    path = path.replace(/{{(.*?)}}/g, function(match, capture) {
      capture = capture.toLowerCase();
      if(utils.isArray(query.where[capture])) {
        return query.where[capture].sort().join('-');
      }
      return query.where[capture];
    });

    _this._data = [];

    (function(query, p) {

      //if cached, retrieve and parse
      if(FILE_CACHED.hasOwnProperty(path)) {
        parse(FILE_CACHED[path]);
      }
      //if requested by another hook, wait for the response
      else if(FILE_REQUESTED.hasOwnProperty(path)) {
        FILE_REQUESTED[path].then(function() {
          parse(FILE_CACHED[path]);
        });
{'fraction_non_alphanumeric': 0.14870428675224026, 'fraction_numerical': 0.0019375151368370065, 'mean_word_length': 2.845437616387337, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6300075', 'n_tokens_mistral': 1270, 'n_tokens_neox': 1257, 'n_words': 340}