<commit_before>/*!
 * Flow Package: Core v0.X / <NAME> (c) 2011 / MIT
 *
 * Depends on the Flow platform
 * http://github.com/bemson/Flow/tree/nextgen
 */
!function (window, Object, Array, Math, Flow, isNaN, undefined) {
  // init vars
  var core = Flow.pkg('core'), // define core package
    /*
    this generator handles any nesting and combination of _var component values...
    ...strings
      > _var: 'foo'
    ...objects
      > _var: {foo: 'bar'}
    ...arrays of strings, arrays and objects
      > _var: ['f', 'o', 'b']
      > _var: [['f'], ['o'], ['b]]
      > _var: [{foo: 'bar'}, {hello: 'world'}]
      > _var: [['g',{foo: 'bar'}], 'alpha', {accts: 9}] // mixed
    */
    generateVariableConfigurationObjects = new genData(function (name, value, parent, dataset, flags, shared) { // generator to extract variable pairs from a _var component
      // init vars
      var data = this, // alias self
        keep = 1, // flag when this data will be a variable configuration
        obj = { // variable configuration object
          name: data.name, // name of variable
          value: data.value, // initial variable value
          use: 1 // flag to use this value (true by default)
        };
      // omit everything
      flags.omit = 1;
      // flag when this is an object
      data.O = typeof value === 'object';
      // flag when this is an Array
      data.A = value instanceof Array;
      // if there is a parent...
      if (parent) {
        // if the parent is an array...
        if (parent.A) {
          // if this is an object...
          if (data.O) {
            // exclude from resulting array
            keep = 0;
          }
        } else { // otherwise, when the parent is not an array (assume the parent is an object)...
          // don't scan the children of this object (because it's the value of this _vars config, not a new one)
          flags.scan = 0;
        }
      } else { // otherwise, when there is no parent...
        // if there is no keys in shared...
        if (!shared.keys) {
          // init shared.keys
          shared.keys = {};
        }
        // if the first item is an object...
        if (data.O) {
          // exclude from the result
          keep = 0;
        }
      }
      // if keeping this data...
      if (keep) {
        // if there is no parent, or the parent is an array...
        if (!parent || parent.A) {
          // use the value as the name
          obj.name = value;
          // set the value to undefined
          obj.value = undefined;
          // flag that this data has no value
          obj.use = 0;
        }
        // if the (resolved) name is valid...
        if (isVariableNameValid(obj.name)) {
          // convert name to string
          obj.name += '';
          // if this key name exists...
          if (shared.keys.hasOwnProperty(obj.name)) {
            // remove existing variable configuration
            dataset.splice(shared.keys[obj.name], 1);
          }
          // add to dataset and capture index
          shared.keys[obj.name] = dataset.push(obj) - 1;
        }
      }
    }),
    generateTokens = new genData(function (name, value, parent, dataset, flags) { // return tokens found in the given string
      // init vars
      var data = this, // alias self
        slash = '/'; // shorthand forward-slash character
      // init set property (default is false, or "not a set")
      data.set = 0;
      // capture parent
      data.parent = parent;
      // init done property
      data.done = 0;
      // if this is a string...
      if (typeof value === 'string') {
        // if slashes exist...
        if (~value.indexOf(slash)) {
          // split into slashes
          data.value = value.split(slash);
        } else if (value.charAt(0) === '[') { // or, when a match-set...
          // remove brackets and split by the match-set delimiter
          data.value = value.slice(1,-1).split('|');
          // flag that this is a set
          data.set = 1;
        }
      }
      // if the value is (still) a string...
      if (typeof data.value === 'string') {
        // if the parent exists and is a set...
        if (parent && parent.set) {
          // identify this as part of that set
          data.set = 1;
          // identify this as the last option in the set
          parent.last = data;
        }
        // set "first" property, based on whether items already exist in the dataset
        data.first = !dataset.length;
      } else { // otherwise, when not a string...
        // exclude from dataset
        flags.omit = 1;
      }
    }),
    activeFlows = []; // collection of active flows

  if (!activeFlows.every) {
    Array.prototype.every = function(fun /*, thisp */) {
      "use strict";

      if (this === void 0 || this === null)
        throw new TypeError();

      var t = Object(this);
      var len = t.length >>> 0;
      if (typeof fun !== "function")
        throw new TypeError();

      var thisp = arguments[1];
      for (var i = 0; i < len; i++) {
        if (!fun.call(thisp, t[i], i, t))
          return false;
      }

      return true;
    };
  }

  if (!activeFlows.indexOf) {
    Array.prototype.indexOf = function (searchElement /*, fromIndex */ ) {
      "use strict";
      if (this === void 0 || this === null) {
        throw new TypeError();
      }
      var t = Object(this);
      var len = t.length >>> 0;
      if (len === 0) {
        return -1;
      }
      var n = 0;
      if (arguments.length > 0) {
        n = Number(arguments[1]);
        if (n !== n) { // shortcut for verifying if it's NaN
          n = 0;
        } else if (n !== 0 && n !== Infinity && n !== -Infinity) {
          n = (n > 0 || -1) * Math.floor(Math.abs(n));
        }
      }
      if (n >= len) {
        return -1;
      }
      var k = n >= 0 ? n : Math.max(len - Math.abs(n), 0);
      for (; k < len; k++) {
        if (k in t && t[k] === searchElement) {
          return k;
        }
      }
      return -1;
    }
  }

  // returns true when the argument is a valid variable name
  function isVariableNameValid(name) {
    return name != null && /\w/.test(name);<commit_msg>Optimized Array.indexOf() and Array.every() shims<commit_after>/*!
 * Flow Package: Core v0.X / <NAME> (c) 2011 / MIT
 *
 * Depends on the Flow platform
 * http://github.com/bemson/Flow/tree/nextgen
 */
!function (window, Object, Array, Math, Flow, isNaN, undefined) {
  // init vars
  var core = Flow.pkg('core'), // define core package
    /*
    this generator handles any nesting and combination of _var component values...
    ...strings
      > _var: 'foo'
    ...objects
      > _var: {foo: 'bar'}
    ...arrays of strings, arrays and objects
      > _var: ['f', 'o', 'b']
      > _var: [['f'], ['o'], ['b]]
      > _var: [{foo: 'bar'}, {hello: 'world'}]
      > _var: [['g',{foo: 'bar'}], 'alpha', {accts: 9}] // mixed
    */
    generateVariableConfigurationObjects = new genData(function (name, value, parent, dataset, flags, shared) { // generator to extract variable pairs from a _var component
      // init vars
      var data = this, // alias self
        keep = 1, // flag when this data will be a variable configuration
        obj = { // variable configuration object
          name: data.name, // name of variable
          value: data.value, // initial variable value
          use: 1 // flag to use this value (true by default)
        };
      // omit everything
      flags.omit = 1;
      // flag when this is an object
      data.O = typeof value === 'object';
      // flag when this is an Array
      data.A = value instanceof Array;
      // if there is a parent...
      if (parent) {
        // if the parent is an array...
        if (parent.A) {
          // if this is an object...
          if (data.O) {
            // exclude from resulting array
            keep = 0;
          }
        } else { // otherwise, when the parent is not an array (assume the parent is an object)...
          // don't scan the children of this object (because it's the value of this _vars config, not a new one)
          flags.scan = 0;
        }
      } else { // otherwise, when there is no parent...
        // if there is no keys in shared...
        if (!shared.keys) {
          // init shared.keys
          shared.keys = {};
        }
        // if the first item is an object...
        if (data.O) {
          // exclude from the result
          keep = 0;
        }
      }
      // if keeping this data...
      if (keep) {
        // if there is no parent, or the parent is an array...
        if (!parent || parent.A) {
          // use the value as the name
          obj.name = value;
          // set the value to undefined
          obj.value = undefined;
          // flag that this data has no value
          obj.use = 0;
        }
        // if the (resolved) name is valid...
        if (isVariableNameValid(obj.name)) {
          // convert name to string
          obj.name += '';
          // if this key name exists...
          if (shared.keys.hasOwnProperty(obj.name)) {
            // remove existing variable configuration
            dataset.splice(shared.keys[obj.name], 1);
          }
          // add to dataset and capture index
          shared.keys[obj.name] = dataset.push(obj) - 1;
        }
      }
    }),
    generateTokens = new genData(function (name, value, parent, dataset, flags) { // return tokens found in the given string
      // init vars
      var data = this, // alias self
        slash = '/'; // shorthand forward-slash character
      // init set property (default is false, or "not a set")
      data.set = 0;
      // capture parent
      data.parent = parent;
      // init done property
      data.done = 0;
      // if this is a string...
      if (typeof value === 'string') {
        // if slashes exist...
        if (~value.indexOf(slash)) {
          // split into slashes
          data.value = value.split(slash);
        } else if (value.charAt(0) === '[') { // or, when a match-set...
          // remove brackets and split by the match-set delimiter
          data.value = value.slice(1,-1).split('|');
          // flag that this is a set
          data.set = 1;
        }
      }
      // if the value is (still) a string...
      if (typeof data.value === 'string') {
        // if the parent exists and is a set...
        if (parent && parent.set) {
          // identify this as part of that set
          data.set = 1;
          // identify this as the last option in the set
          parent.last = data;
        }
        // set "first" property, based on whether items already exist in the dataset
        data.first = !dataset.length;
      } else { // otherwise, when not a string...
        // exclude from dataset
        flags.omit = 1;
      }
    }),
    activeFlows = [], // collection of active flows
    arrayPrototype = Array.prototype; // alias for minification purporses

  if (!arrayPrototype.every) {
    arrayPrototype.every = function(fnc, scope) {
      for (var i = 0, j = this.length; i < j; i++) {
        if (!fnc.call(scope, this[i], i, this)) {
          return false;
        }
      }
      return true;
    };
  }

  if (!arrayPrototype.indexOf) {
    arrayPrototype.indexOf = function (searchElement, fromIndex) {
      for (var i = (fromIndex | 0), j = this.length; i < j; i++) {
        if (this[i] === searchElement) {
          return i;
        }
      }
      return -1;
    }
  }

  // returns true when the argument is a valid variable name
  function isVariableNameValid(name) {
    return name != null && /\w/.test(name);
{'fraction_non_alphanumeric': 0.1335681154440818, 'fraction_numerical': 0.005840920803985569, 'mean_word_length': 1.6117092866756393, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7016065', 'n_tokens_mistral': 3379, 'n_tokens_neox': 3288, 'n_words': 1222}