<commit_before>import every from 'lodash/every'
import isArray from 'lodash/isArray'
import isPlainObject from 'lodash/isPlainObject'
import isString from 'lodash/isString'
import some from 'lodash/some'
import { invoke } from 'utils'

// and = term+
// term = ws* (not | property | string)
// not = "!" term
// property = string ":" term
// string = /[a-z0-9-_.]+/i
export const parse = invoke(() => {
  let i
  let n
  let pattern

  // -----

  const rule = parser => () => {
    const pos = i
    const node = parser()
    if (node) {
      return node
    }
    i = pos
  }

  // -----

  const parseAnd = rule(() => {
    const children = []

    while (i < n) { // eslint-disable-line no-unmodified-loop-condition
      const child = parseTerm()
      if (!child) {
        break
      }
      children.push(child)
    }

    const { length } = children
    if (!length) {
      return
    }
    if (length === 1) {
      return children[0]
    }
    return { type: 'and', children }
  })
  const parseTerm = rule(() => {
    while (i < n && pattern[i] === ' ') {
      ++i
    }

    return parseNot() || parseProperty() || parseString()
  })
  const parseNot = rule(() => {
    let child
    if (
      pattern[i++] === '!' &&
      (child = parseTerm())
    ) {
      return { type: 'not', child }
    }
  })
  const parseProperty = rule(() => {
    let name, child
    if (
      (name = parseString()) &&
      pattern[i++] === ':' &&
      (child = parseTerm())
    ) {
      return { type: 'property', name: name.value, child }
    }
  })
  const parseString = rule(() => {
    const matches = pattern.slice(i).match(/^[a-z0-9-_.]+/i)
    if (!matches) {
      return
    }

    const value = matches[0]
    i += value.length

    return { type: 'string', value }
  })

  return pattern_ => {
    i = 0
    n = pattern_.length
    pattern = pattern_

    try {
      return parseAnd()
    } finally {
      pattern = null
    }
  }
})

export const execute = invoke(() => {
  const visitors = {
    and: ({ children }, value) => (
      every(children, child => execute(child, value))
    ),
    not: ({ child }, value) => (
      !execute(child, value)
    ),
    or: ({ children }, value) => (
      some(children, child => execute(child, value))
    ),
    property: ({ name, child }, value) => (
      value != null && execute(child, value[name])
    ),
    string: invoke(() => {
      const match = (pattern, value) => {
        if (isString(value)) {
          return value.toLowerCase().indexOf(pattern) !== -1
        }

        if (isArray(value) || isPlainObject(value)) {
          return some(value, value => match(pattern, value))
        }

        return false
      }

      return ({ value: pattern }, value) => (
        match(pattern.toLowerCase(), value)
      )
    })
  }

  return (node, value) => visitors[node.type](node, value)
})

export const create = pattern => {
  pattern = parse(pattern)
  if (!pattern) {
    return
  }

  return value => execute(pattern, value)
}<commit_msg>feat(home): add group/or syntax<commit_after>import every from 'lodash/every'
import isArray from 'lodash/isArray'
import isPlainObject from 'lodash/isPlainObject'
import isString from 'lodash/isString'
import map from 'lodash/map'
import some from 'lodash/some'

import invoke from './invoke'

// term      = ws (and | or | not | property | string) ws
// ws        = ' '*
// *and      = group
// group     = "(" term+ ")"
// *or       = "|" ws group
// *not      = "!" term
// *property = string ws ":" term
// *string   = /[a-z0-9-_.]+/i
export const parse = invoke(() => {
  let i
  let n
  let pattern

  // -----

  const rule = parser => () => {
    const pos = i
    const node = parser()
    if (node) {
      return node
    }
    i = pos
  }

  // -----

  const parseTerm = rule(() => {
    parseWs()

    const child = (
      parseAnd() ||
      parseOr() ||
      parseNot() ||
      parseProperty() ||
      parseString()
    )
    if (child) {
      parseWs()
      return child
    }
  })
  const parseWs = () => {
    while (i < n && pattern[i] === ' ') {
      ++i
    }

    return true
  }
  const parseAnd = rule(() => {
    const children = parseGroup()
    if (children) {
      return children.length === 1
        ? children[0]
        : { type: 'and', children }
    }
  })
  const parseGroup = rule(() => {
    if (pattern[i++] !== '(') {
      return
    }

    const terms = []
    while (i < n) { // eslint-disable-line no-unmodified-loop-condition
      const term = parseTerm()
      if (!term) {
        break
      }
      terms.push(term)
    }

    if (
      terms.length &&
      pattern[i++] === ')'
    ) {
      return terms
    }
  })
  const parseOr = rule(() => {
    let children
    if (
      pattern[i++] === '|' &&
      parseWs() &&
      (children = parseGroup())
    ) {
      return children.length === 1
        ? children[0]
        : { type: 'or', children }
    }
  })
  const parseNot = rule(() => {
    let child
    if (
      pattern[i++] === '!' &&
      (child = parseTerm())
    ) {
      return { type: 'not', child }
    }
  })
  const parseProperty = rule(() => {
    let name, child
    if (
      (name = parseString()) &&
      parseWs() &&
      (pattern[i++] === ':') &&
      (child = parseTerm())
    ) {
      return { type: 'property', name: name.value, child }
    }
  })
  const parseString = rule(() => {
    const matches = pattern.slice(i).match(/^[a-z0-9-_.]+/i)
    if (!matches) {
      return
    }

    const value = matches[0]
    i += value.length

    return { type: 'string', value }
  })

  return pattern_ => {
    i = 0
    pattern = `(${pattern_})` // There is an implicit “and”.
    n = pattern.length

    try {
      return parseTerm()
    } finally {
      pattern = null
    }
  }
})

export const execute = invoke(() => {
  const visitors = {
    and: ({ children }, value) => (
      every(children, child => execute(child, value))
    ),
    not: ({ child }, value) => (
      !execute(child, value)
    ),
    or: ({ children }, value) => (
      some(children, child => execute(child, value))
    ),
    property: ({ name, child }, value) => (
      value != null && execute(child, value[name])
    ),
    string: invoke(() => {
      const match = (pattern, value) => {
        if (isString(value)) {
          return value.toLowerCase().indexOf(pattern) !== -1
        }

        if (isArray(value) || isPlainObject(value)) {
          return some(value, value => match(pattern, value))
        }

        return false
      }

      return ({ value: pattern }, value) => (
        match(pattern.toLowerCase(), value)
      )
    })
  }

  return (node, value) => visitors[node.type](node, value)
})

export const toString = invoke(() => {
  const toStringTerms = terms => map(terms, toString).join(' ')
  const toStringGroup = terms => `(${toStringTerms(terms)})`

  const visitors = {
    and: ({ children }) => toStringGroup(children),
    not: ({ child }) => `!${toString(child)}`,
    or: ({ children }) => `|${toStringGroup(children)}`,
    property: ({ name, child }) => `${name}:${toString(child)}`,
    string: ({ value }) => value
  }

  const toString = node => visitors[node.type](node)

  // Special case for a root “and”: do not add braces.
  return node => node.type === 'and'
    ? toStringTerms(node.children)
    : toString(node)
})

export const create = pattern => {
  pattern = parse(pattern)
  if (!pattern) {
    return
  }

  return value => execute(pattern, value)
}
{'fraction_non_alphanumeric': 0.17641584684924222, 'fraction_numerical': 0.0026588673225206062, 'mean_word_length': 2.0681076672104406, 'pattern_counts': {'":': 2, '<': 7, '<?xml version=': 0, '>': 62, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2204046', 'n_tokens_mistral': 2439, 'n_tokens_neox': 2385, 'n_words': 608}