<commit_before>'use strict';
module.exports = MultiVector;

var assert = require('assert');

function MultiVector(vectors) {
  if (arguments.length > 1) {
    vectors = Array.prototype.slice.call(arguments);
  }
  if (!(this instanceof MultiVector)) {
    return new MultiVector(vectors);
  }

  this._store = makeStore();
  this._vectorIndex = makeStore();
  this._forEachFns = makeStore();
  this._vectors = vectors.map(function(vector, index) {
    validateVector(vector);
    this._vectorIndex[vector] = index;
    return vector;
  }, this);
}

var mvp = MultiVector.prototype;

mvp.get = function get(vectorObj) {
  this.validateVectorObject(vectorObj);
  var vectors = this._vectors;
  var store = this._store;

  for (var i = 0, len = vectors.length; store && i < len; ++i) {
    store = store[vectorObj[vectors[i]]];
  }
  return store;
};

mvp.set = function set(vectorObj, value) {
  this.validateVectorObject(vectorObj);
  var vectors = this._vectors;
  var store = this._store;
  var len = vectors.length - 1;

  for (var i = 0; i < len; ++i) {
    var vector = vectorObj[vectors[i]];
    store = store[vector] || (store[vector] = makeStore());
  }

  return (store[vectorObj[vectors[len]]] = value);
};

mvp.export = function exportFn(vectors) {
  if (arguments.length > 1) {
    vectors = Array.prototype.slice.call(arguments);
    vectors.forEach(validateVector);
  } else if (!vectors) {
    vectors = this._vectors;
  } else {
    vectors.forEach(validateVector);
  }

  var indexes = vectors.map(function(vector) {
    return this._vectorIndex[vector];
  }, this);
  var len = indexes.length - 1;

  var copy = makeStore();
  this.forEach(function(value){
    var store = copy;
    for (var i = 0; i < len; i++) {
      var vector = arguments[indexes[i] + 1];
      store = store[vector] || (store[vector] = makeStore());
    }
    store[arguments[indexes[len] + 1]] = value;
  });
  return copy;
};

mvp.forEach = function(fn, vectors) {
  assert.equal('function', typeof fn);
  if (arguments.length > 2) {
    vectors = Array.prototype.slice.call(arguments, 1);
    vectors.forEach(validateVector);
  } else if (!vectors) {
    vectors = this._vectors;
  } else {
    vectors.forEach(validateVector);
  }

  var indexes = vectors.map(function(vector) {
    return this._vectorIndex[vector];
  }, this);

  var fnKey = indexes.join(',');

  /* jshint evil:true */
  var fe = this._forEachFns[fnKey] ||
    (this._forEachFns[fnKey] = new Function('fn', generateForEachCode(indexes)));
  /* jshint evil:false */

  fe.call(this, fn);
};

function generateForEachCode(indexes) {
  var lines = [];
  var len = indexes.length;
  var i;
  for (i = 0; i < len; i++) {
    lines.push(
      'var s' + i + ' = ' + prevStore(i) + ';',
      'for (var v' + i + ' in s' + i + ') {',
        'if (s' + i + '.hasOwnProperty(v' + i + ')) {'
    );
  }
  lines.push(
    'var s' + len + ' = ' + prevStore(len) + ';'
  );
  var varNames = [];
  for (i = 0; i < len; i ++) {
    varNames.push('v' + indexes[i]);
  }
  lines.push(
    'fn(s' + len + ', ' + varNames.join(', ') + ');'
  );<commit_msg>move common argument handling logic to it's own function<commit_after>'use strict';
module.exports = MultiVector;

var assert = require('assert');

function MultiVector(vectors) {
  if (arguments.length > 1) {
    vectors = Array.prototype.slice.call(arguments);
  }
  if (!(this instanceof MultiVector)) {
    return new MultiVector(vectors);
  }

  this._store = makeStore();
  this._vectorIndex = makeStore();
  this._forEachFns = makeStore();
  this._vectors = vectors.map(function(vector, index) {
    validateVector(vector);
    this._vectorIndex[vector] = index;
    return vector;
  }, this);
}

var mvp = MultiVector.prototype;

mvp.get = function get(vectorObj) {
  this.validateVectorObject(vectorObj);
  var vectors = this._vectors;
  var store = this._store;

  for (var i = 0, len = vectors.length; store && i < len; ++i) {
    store = store[vectorObj[vectors[i]]];
  }
  return store;
};

mvp.set = function set(vectorObj, value) {
  this.validateVectorObject(vectorObj);
  var vectors = this._vectors;
  var store = this._store;
  var len = vectors.length - 1;

  for (var i = 0; i < len; ++i) {
    var vector = vectorObj[vectors[i]];
    store = store[vector] || (store[vector] = makeStore());
  }

  return (store[vectorObj[vectors[len]]] = value);
};

mvp.export = function exportFn(vectors) {
  vectors = this._normalizeArgs(arguments, 0);

  var indexes = vectors.map(function(vector) {
    return this._vectorIndex[vector];
  }, this);
  var len = indexes.length - 1;

  var copy = makeStore();
  this.forEach(function(value){
    var store = copy;
    for (var i = 0; i < len; i++) {
      var vector = arguments[indexes[i] + 1];
      store = store[vector] || (store[vector] = makeStore());
    }
    store[arguments[indexes[len] + 1]] = value;
  });
  return copy;
};

mvp.forEach = function(fn, vectors) {
  assert.equal('function', typeof fn);
  vectors = this._normalizeArgs(arguments, 1);

  var indexes = vectors.map(function(vector) {
    return this._vectorIndex[vector];
  }, this);

  var fnKey = indexes.join(',');

  /* jshint evil:true */
  var fe = this._forEachFns[fnKey] ||
    (this._forEachFns[fnKey] = new Function('fn', generateForEachCode(indexes)));
  /* jshint evil:false */

  fe.call(this, fn);
};

mvp._normalizeArgs = function(args, i) {
  var vectors;
  if (args.length > i + 1) {
    vectors = Array.prototype.slice.call(args, i);
  } else if (!args[i]) {
    return this._vectors;
  } else {
    vectors = args[i];
  }
  vectors.forEach(validateVector);
  return vectors;
};

function generateForEachCode(indexes) {
  var lines = [];
  var len = indexes.length;
  var i;
  for (i = 0; i < len; i++) {
    lines.push(
      'var s' + i + ' = ' + prevStore(i) + ';',
      'for (var v' + i + ' in s' + i + ') {',
        'if (s' + i + '.hasOwnProperty(v' + i + ')) {'
    );
  }
  lines.push(
    'var s' + len + ' = ' + prevStore(len) + ';'
  );
  var varNames = [];
  for (i = 0; i < len; i ++) {
    varNames.push('v' + indexes[i]);
  }
  lines.push(
    'fn(s' + len + ', ' + varNames.join(', ') + ');'
  );
{'fraction_non_alphanumeric': 0.17164179104477612, 'fraction_numerical': 0.004218040233614536, 'mean_word_length': 3.2400275103163687, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6373867', 'n_tokens_mistral': 2196, 'n_tokens_neox': 2113, 'n_words': 492}