<commit_before>_ = require 'underscore-plus'
settings = require './settings'

wildCardTable = {
  '*': '.*'
  '\\*': '\\*'
}

expandWildCard = (word) ->
  # Replace double '**' to escaped *
  word = word.replace(/\*\*/g, '\\*')
  segments = []
  for segment in word.split(/(\\?\*)/) when segment isnt ''
    segments.push(wildCardTable[segment] ? _.escapeRegExp(segment))
  segments.join('')

getRegExpForWord = (word, {wildcard, sensitivity}={}) ->
  isIncludeUpperCase = /[A-Z]/.test(word)

  if not word.startsWith('|') and not word.endsWith('|')
    words = word.split('|')
  else
    words = [word]

  patterns = []
  for word in words
    if word.length > 1 # don't expand wildcard for sole `*`.
      pattern = expandWildCard(word)
    else
      pattern = _.escapeRegExp(word)

    # Translate
    # - ">word<" to "\bword\b"
    # - ">word" to "\bword"
    # - "word<" to "word\b"
    if /^>./.test(pattern)
      pattern = pattern[1...]
      pattern = "\\b" + pattern if /^\w/.test(pattern)
    if /.<$/.test(pattern)
      pattern = pattern[...-1]
      pattern = pattern + "\\b" if /\w$/.test(pattern)
    patterns.push(pattern)

  options = ''
  if (sensitivity is 'sensitive') or (sensitivity is 'smartcase' and isIncludeUpperCase)
    options += 'i'

  new RegExp(patterns.join('|'), options)

module.exports = (query, options={}) ->
  include = []
  exclude = []
  {negateByEndingExclamation} = options
  delete options.negateByEndingExclamation
  words = _.compact(query.split(/\s+/))
  for word in words
    if word isnt '!' and word.startsWith('!')
      exclude.push(getRegExpForWord(word[1...], options))
    else if word isnt '!' and negateByEndingExclamation and word.endsWith('!')
      exclude.push(getRegExpForWord(word[...-1], options))
    else
      include.push(getRegExpForWord(word, options))<commit_msg>CRITICAL, smart case is inverted by careless mistake.<commit_after>_ = require 'underscore-plus'
settings = require './settings'

wildCardTable = {
  '*': '.*'
  '\\*': '\\*'
}

expandWildCard = (word) ->
  # Replace double '**' to escaped *
  word = word.replace(/\*\*/g, '\\*')
  segments = []
  for segment in word.split(/(\\?\*)/) when segment isnt ''
    segments.push(wildCardTable[segment] ? _.escapeRegExp(segment))
  segments.join('')

getRegExpForWord = (word, {wildcard, sensitivity}={}) ->
  isIncludeUpperCase = /[A-Z]/.test(word)

  if not word.startsWith('|') and not word.endsWith('|')
    words = word.split('|')
  else
    words = [word]

  patterns = []
  for word in words
    if word.length > 1 # don't expand wildcard for sole `*`.
      pattern = expandWildCard(word)
    else
      pattern = _.escapeRegExp(word)

    # Translate
    # - ">word<" to "\bword\b"
    # - ">word" to "\bword"
    # - "word<" to "word\b"
    if /^>./.test(pattern)
      pattern = pattern[1...]
      pattern = "\\b" + pattern if /^\w/.test(pattern)
    if /.<$/.test(pattern)
      pattern = pattern[...-1]
      pattern = pattern + "\\b" if /\w$/.test(pattern)
    patterns.push(pattern)

  options = ''
  if (sensitivity is 'sensitive') or (sensitivity is 'smartcase' and not isIncludeUpperCase)
    options += 'i'

  new RegExp(patterns.join('|'), options)

module.exports = (query, options={}) ->
  include = []
  exclude = []
  {negateByEndingExclamation} = options
  delete options.negateByEndingExclamation
  words = _.compact(query.split(/\s+/))
  for word in words
    if word isnt '!' and word.startsWith('!')
      exclude.push(getRegExpForWord(word[1...], options))
    else if word isnt '!' and negateByEndingExclamation and word.endsWith('!')
      exclude.push(getRegExpForWord(word[...-1], options))
    else
      include.push(getRegExpForWord(word, options))
{'fraction_non_alphanumeric': 0.18353067814854682, 'fraction_numerical': 0.002691065662002153, 'mean_word_length': 3.0053879310344827, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4994612', 'n_tokens_mistral': 1318, 'n_tokens_neox': 1269, 'n_words': 289}