<commit_before>/**
 * Come from Vue.js v1.0.24
 *
 *   https://github.com/vuejs/vue
 *
 */

import Cache from './cache.js'

const defaultAllowedKeywords =
        'Math,Date,this,true,false,null,undefined,Infinity,NaN,'
      + 'isNaN,isFinite,decodeURI,decodeURIComponent,encodeURI,'
      + 'encodeURIComponent,parseInt,parseFloat'

// keywords that don't make sense inside expressions
const improperKeywordsRE =
  new RegExp(
    '^('
    + ('break,case,class,catch,const,continue,debugger,default,'
      + 'delete,do,else,export,extends,finally,for,function,if,'
      + 'import,in,instanceof,let,return,super,switch,throw,try,'
      + 'var,while,with,yield,enum,await,implements,package,'
      + 'protected,static,interface,private,public'
    ).replace(/,/g, '\\b|')
    + '\\b)'
  )

const wsRE = /\s/g
const newlineRE = /\n/g
const saveRE = /[\{,]\s*[\w\$_]+\s*:|('(?:[^'\\]|\\.)*'|"(?:[^"\\]|\\.)*"|`(?:[^`\\]|\\.)*\$\{|\}(?:[^`\\]|\\.)*`|`(?:[^`\\]|\\.)*`)|new |typeof |void /g
const restoreRE = /"(\d+)"/g
const pathTestRE = /^[A-Za-z_$][\w$]*(?:\.[A-Za-z_$][\w$]*|\['.*?'\]|\[".*?"\]|\[\d+\]|\[[A-Za-z_$][\w$]*\])*$/
const identRE = /[^\w$\.](?:[A-Za-z_$][\w$]*)/g
const booleanLiteralRE = /^(?:true|false)$/

/**
 * Save / Rewrite / Restore
 *
 * When rewriting paths found in an expression, it is
 * possible for the same letter sequences to be found in
 * strings and Object literal property keys. Therefore we
 * remove and store these parts in a temporary array, and
 * restore them after the path rewrite.
 */

let saved = []

/**
 * Save replacer
 *
 * The save regex can match two possible cases:
 * 1. An opening object literal
 * 2. A string
 * If matched as a plain string, we need to escape its
 * newlines, since the string needs to be preserved when
 * generating the function body.
 *
 * @param {String} str
 * @param {String} isString - str if matched as a string
 * @return {String} - placeholder with index
 */

function save (str, isString) {
  let i = saved.length
  saved[i] = isString
    ? str.replace(newlineRE, '\\n')
    : str
  return '"' + i + '"'
}

/**
 * Restore replacer
 *
 * @param {String} str
 * @param {String} i - matched save index
 * @return {String}
 */

function restore (str, i) {
  return saved[i]
}

/**
 * Check if an expression is a simple path.
 *
 * @param {String} expr
 * @return {Boolean}
 */

function isSimplePath (expr) {
  return pathTestRE.test(expr)
    // don't treat true/false as paths
    && !booleanLiteralRE.test(expr)// &&
    // Math constants e.g. Math.PI, Math.E etc.
    // && expr.slice(0, 5) !== 'Math.'
}

/**
 * Check if an expression is a simple path.
 *
 * @param {String} expr
 * @return {Boolean}
 */

function parseKeywordsToRE (keywords) {
  return new RegExp(
        '^(?:'
      + keywords
        .replace(wsRE, '')
        .replace(/\$/g, '\\$')
        .replace(/,/g, '\\b|')
      + '\\b)'
    )
}

/**
 * @param {Object} config
 *   - {Number} cache, default 1000
 *              limited for Cache
 *   - {Array} params, default ['$']
 *              first one is for scope and you can add more params
 * @constructor
 */

export default class Gep {
  constructor ({
    cache = 1000,
    scope = '$',
    scopes,
    params = scopes
           ? Object.keys(scopes).unshift(scope)
           : [scope],
  } = {}) {
    this._cache = new Cache(cache)

    this._funcParams = params.join(',').replace(wsRE, '')
    this._funcBefore = 'function(' + this._funcParams + '){return '

    this.scope = scope

    if (scopes) {
      let keywords
      Object.keys(scopes).forEach(key => {
        keywords = scopes[key]
        if (Array.isArray(keywords)) {
          keywords = keywords.join(',')
        }
        scopes[key] = parseKeywordsToRE(keywords)
      })
      this._scopeREs = scopes
    }
<commit_msg>Fix params<commit_after>/**
 * Come from Vue.js v1.0.24
 *
 *   https://github.com/vuejs/vue
 *
 */

import Cache from './cache.js'

const defaultAllowedKeywords =
        'Math,Date,this,true,false,null,undefined,Infinity,NaN,'
      + 'isNaN,isFinite,decodeURI,decodeURIComponent,encodeURI,'
      + 'encodeURIComponent,parseInt,parseFloat'

// keywords that don't make sense inside expressions
const improperKeywordsRE =
  new RegExp(
    '^('
    + ('break,case,class,catch,const,continue,debugger,default,'
      + 'delete,do,else,export,extends,finally,for,function,if,'
      + 'import,in,instanceof,let,return,super,switch,throw,try,'
      + 'var,while,with,yield,enum,await,implements,package,'
      + 'protected,static,interface,private,public'
    ).replace(/,/g, '\\b|')
    + '\\b)'
  )

const wsRE = /\s/g
const newlineRE = /\n/g
const saveRE = /[\{,]\s*[\w\$_]+\s*:|('(?:[^'\\]|\\.)*'|"(?:[^"\\]|\\.)*"|`(?:[^`\\]|\\.)*\$\{|\}(?:[^`\\]|\\.)*`|`(?:[^`\\]|\\.)*`)|new |typeof |void /g
const restoreRE = /"(\d+)"/g
const pathTestRE = /^[A-Za-z_$][\w$]*(?:\.[A-Za-z_$][\w$]*|\['.*?'\]|\[".*?"\]|\[\d+\]|\[[A-Za-z_$][\w$]*\])*$/
const identRE = /[^\w$\.](?:[A-Za-z_$][\w$]*)/g
const booleanLiteralRE = /^(?:true|false)$/

/**
 * Save / Rewrite / Restore
 *
 * When rewriting paths found in an expression, it is
 * possible for the same letter sequences to be found in
 * strings and Object literal property keys. Therefore we
 * remove and store these parts in a temporary array, and
 * restore them after the path rewrite.
 */

let saved = []

/**
 * Save replacer
 *
 * The save regex can match two possible cases:
 * 1. An opening object literal
 * 2. A string
 * If matched as a plain string, we need to escape its
 * newlines, since the string needs to be preserved when
 * generating the function body.
 *
 * @param {String} str
 * @param {String} isString - str if matched as a string
 * @return {String} - placeholder with index
 */

function save (str, isString) {
  let i = saved.length
  saved[i] = isString
    ? str.replace(newlineRE, '\\n')
    : str
  return '"' + i + '"'
}

/**
 * Restore replacer
 *
 * @param {String} str
 * @param {String} i - matched save index
 * @return {String}
 */

function restore (str, i) {
  return saved[i]
}

/**
 * Check if an expression is a simple path.
 *
 * @param {String} expr
 * @return {Boolean}
 */

function isSimplePath (expr) {
  return pathTestRE.test(expr)
    // don't treat true/false as paths
    && !booleanLiteralRE.test(expr)// &&
    // Math constants e.g. Math.PI, Math.E etc.
    // && expr.slice(0, 5) !== 'Math.'
}

/**
 * Check if an expression is a simple path.
 *
 * @param {String} expr
 * @return {Boolean}
 */

function parseKeywordsToRE (keywords) {
  return new RegExp(
        '^(?:'
      + keywords
        .replace(wsRE, '')
        .replace(/\$/g, '\\$')
        .replace(/,/g, '\\b|')
      + '\\b)'
    )
}

/**
 * @param {Object} config
 *   - {Number} cache, default 1000
 *              limited for Cache
 *   - {Array} params, default ['$']
 *              first one is for scope and you can add more params
 * @constructor
 */

export default class Gep {
  constructor ({
    cache = 1000,
    scope = '$',
    scopes,
    params,
  } = {}) {
    if (!params) {
      if (scopes) {
        params = Object.keys(scopes)
        params.unshift(scope)
      } else {
        params = [scope]
      }
    }
    this._cache = new Cache(cache)

    this._funcParams = params.join(',').replace(wsRE, '')
    this._funcBefore = 'function(' + this._funcParams + '){return '

    this.scope = scope

    if (scopes) {
      let keywords
      Object.keys(scopes).forEach(key => {
        keywords = scopes[key]
        if (Array.isArray(keywords)) {
          keywords = keywords.join(',')
        }
        scopes[key] = parseKeywordsToRE(keywords)
      })
      this._scopeREs = scopes
    }

{'fraction_non_alphanumeric': 0.19317149162663896, 'fraction_numerical': 0.004154225626379333, 'mean_word_length': 2.6967370441458733, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1245723', 'n_tokens_mistral': 2726, 'n_tokens_neox': 2659, 'n_words': 648}