<commit_before># Scans for occurrences of a key in a line of text and returns any associated
# values.
#
# Looks for the following patterns of key-value pairs:
# KEY=value
# KEY='value'
# KEY="value"
# KEY value
# KEY 'value'
# KEY "value"
# KEY: value
# KEY => value
class Carwash::ValueDiscoverer < Struct.new(:key)
  ESCAPE_CHARACTERS = {
    "\\0" => "\0",
    "\\a" => "\a",
    "\\b" => "\b",
    "\\f" => "\f",
    "\\n" => "\n",
    "\\r" => "\r",
    "\\t" => "\t",
    "\\v" => "\v",
  }

  def discover(line)
    patterns.flat_map { |pattern|
      line.scan(pattern).map(&:first)
    }.reject(&:nil?).reject(&:empty?).map(&:downcase).map { |val|
      unescape_value(val)
    }
  end

  def patterns
    @patterns ||= [
      %r{#{key}['"]?\s*(?:=>|=|:|\s+)\s*'((?:\\'|[^'])+)'}i,
      %r{#{key}['"]?\s*(?:=>|=|:|\s+)\s*"((?:\\"|[^"])+)"}i,
      %r{#{key}['"]?\s*(?:=>|=|:|\s+)\s*((?:\\\s|\\"|\\'|[^\s'"])+)}i
    ]
  end

  def unescape_value(value)
    value.gsub(/\\(.)/) { |match|
      ESCAPE_CHARACTERS.fetch(match, $1)
    }
  end
end
<commit_msg>Docs update.<commit_after># Scans for occurrences of a key in a line of text and returns any associated
# values.
#
# Looks for the following patterns of key-value pairs:
# * KEY=value
# * KEY value
# * KEY: value
# * KEY => value
# Where the value may be single or double quoted.
class Carwash::ValueDiscoverer < Struct.new(:key)
  ESCAPE_CHARACTERS = {
    "\\0" => "\0",
    "\\a" => "\a",
    "\\b" => "\b",
    "\\f" => "\f",
    "\\n" => "\n",
    "\\r" => "\r",
    "\\t" => "\t",
    "\\v" => "\v",
  }

  def discover(line)
    patterns.flat_map { |pattern|
      line.scan(pattern).map(&:first)
    }.reject(&:nil?).reject(&:empty?).map(&:downcase).map { |val|
      unescape_value(val)
    }
  end

  def patterns
    @patterns ||= [
      %r{#{key}['"]?\s*(?:=>|=|:|\s+)\s*'((?:\\'|[^'])+)'}i,
      %r{#{key}['"]?\s*(?:=>|=|:|\s+)\s*"((?:\\"|[^"])+)"}i,
      %r{#{key}['"]?\s*(?:=>|=|:|\s+)\s*((?:\\\s|\\"|\\'|[^\s'"])+)}i
    ]
  end

  def unescape_value(value)
    value.gsub(/\\(.)/) { |match|
      ESCAPE_CHARACTERS.fetch(match, $1)
    }
  end
end

{'fraction_non_alphanumeric': 0.30837622835751055, 'fraction_numerical': 0.0028076743097800653, 'mean_word_length': 2.623728813559322, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4602748', 'n_tokens_mistral': 939, 'n_tokens_neox': 910, 'n_words': 168}