<commit_before>require "regexp_parser"

module RegXing
  class Regex

    # Class Methods
    # -------------

    class << self
      def matchers
        {
          /(?<!\\)\./ => random_letter,
          /\\d/       => random_number,
          /\\w/       => random_letter,
          /\\W/       => random_non_word_character,
          /\\D/       => random_letter,
          /\\h/       => random_hexdigit_character,
          /\\H/       => random_non_hexdigit_character,
          /\\s/       => " ",
          /\\S/       => random_letter
        }
      end

      def groupings
        [ /\(.*\)/, /\[.*\]/ ]
      end

      def count_indicators
        [ /(?<!\\)\*/, /(?<!\\)\+/, /(?<!\\)\?/, /\{\d*\,?\d*\}/ ]
      end

      def anchors
        [ /^\^/, /\$$/, /^\\A/, /\\z$/ ]
      end

      def process_count_indicator(indicator)
        if indicator.match count_indicators.last

          minimum = indicator.match(/(?<=\{)\d+/) ? indicator.match(/(?<=\{)\d+/)[0].to_i : 1

          return minimum
        else
          return 1
        end
      end

      private

      def random_letter
        ("a".."z").to_a.sample
      end

      alias_method :random_lowercase_letter, :random_letter

      def random_uppercase_letter
        ("A".."Z").to_a.sample
      end

      def random_number
        (0..9).to_a.sample.to_s
      end

      def random_non_word_character
        non_word_characters.sample
      end

      def random_hexdigit_character
        [(0..9).to_a.map(&:to_s), ("A".."F").to_a, ("a".."f").to_a].flatten.sample
      each_with_indexbe

      def random_non_hexdigit_character
        ("h".."z").to_a.sample
      end

      def non_word_characters
        [
          "!", "@", "#", "%", "&", "*", "(", ")", "-", "{", "}",
          "[", "]", "\\", "'", "\"", ":", ";", ",", ".", "?", "/"
        ]
      end
    end

    # Instance Methods
    # ----------------

    attr_reader :expression

    def initialize(exp)
      @expression = exp
    end

    def extract_groupings
      return_matches(expression.source).reject {|str| str.nil? || str == '' }
    end

    def is_anchor(char)
      RegXing::Regex.anchors.any? {|exp| char.match(exp) }
    end

    def return_matches(string)
      string.scan(/\\\||\\\?|[^\\]?\?|\\\.|[^\\]?\.|\\\+|[^\\]?\+|\\\*|[^\\]?\*|\\[a-zA-Z]|(?<!\\)[a-zA-Z]|\{\d*\,?\d*\}|\([^\)]*\)|\[[^\]]*\]|./).flatten
    end

    def is_indicator(first, second=nil)
      RegXing::Regex.count_indicators.any? {|exp| second && second.match(exp) }
    end

    def split
      arr = extract_groupings.map {|item| return_matches item }.flatten

      arr.each_with_index do |item, index|
        if is_indicator(item, arr[index + 1])
          arr[index] = [ item, RegXing::Regex.process_count_indicator(arr.delete_at(index + 1)) ]
        elsif is_anchor(item)
          arr[index] = nil
        else
          arr[index] = [item, 1]
        end
      end

      arr.compact
    end<commit_msg>Make some progress on the groupings thing using the regexp_parsing gem<commit_after>require "regexp_parser"

module RegXing
  class Regex

    # Class Methods
    # -------------

    class << self
      def matchers
        {
          /(?<!\\)\./ => random_letter,
          /\\d/       => random_number,
          /\\w/       => random_letter,
          /\\W/       => random_non_word_character,
          /\\D/       => random_letter,
          /\\h/       => random_hexdigit_character,
          /\\H/       => random_non_hexdigit_character,
          /\\s/       => " ",
          /\\S/       => random_letter
        }
      end

      def groupings
        [ /\(.*\)/, /\[.*\]/ ]
      end

      def count_indicators
        [ /(?<!\\)\*/, /(?<!\\)\+/, /(?<!\\)\?/, /\{\d*\,?\d*\}/ ]
      end

      def anchors
        [ /^\^/, /\$$/, /^\\A/, /\\z$/ ]
      end

      def process_count_indicator(indicator)
        if indicator.match count_indicators.last

          minimum = indicator.match(/(?<=\{)\d+/) ? indicator.match(/(?<=\{)\d+/)[0].to_i : 1

          return minimum
        else
          return 1
        end
      end

      private

      def random_letter
        ("a".."z").to_a.sample
      end

      alias_method :random_lowercase_letter, :random_letter

      def random_uppercase_letter
        ("A".."Z").to_a.sample
      end

      def random_number
        (0..9).to_a.sample.to_s
      end

      def random_non_word_character
        non_word_characters.sample
      end

      def random_hexdigit_character
        [(0..9).to_a.map(&:to_s), ("A".."F").to_a, ("a".."f").to_a].flatten.sample
      end

      def random_non_hexdigit_character
        ("h".."z").to_a.sample
      end

      def non_word_characters
        [
          "!", "@", "#", "%", "&", "*", "(", ")", "-", "{", "}",
          "[", "]", "\\", "'", "\"", ":", ";", ",", ".", "?", "/"
        ]
      end
    end

    # Instance Methods
    # ----------------

    attr_reader :expression

    def initialize(exp)
      @expression = exp
    end

    def extract_groupings
      tree = Regexp::Parser.parse(expression, 'ruby/2.1')
      groupings = []

      tree.each_expression(0) do |exp, level_index|
        groupings << exp if exp.is_a?(Regexp::Expression::Group::Capture) || exp.is_a?(Regexp::Expression::Literal)
      end

      groupings.map(&:to_s)
    end

    def is_anchor(char)
      RegXing::Regex.anchors.any? {|exp| char.match(exp) }
    end

    def return_matches(string)
      string.scan(/\\\||\\\?|[^\\]?\?|\\\.|[^\\]?\.|\\\+|[^\\]?\+|\\\*|[^\\]?\*|\\[a-zA-Z]|(?<!\\)[a-zA-Z]|\{\d*\,?\d*\}|\([^\)]*\)|\[[^\]]*\]|./).flatten
    end

    def is_indicator(first, second=nil)
      RegXing::Regex.count_indicators.any? {|exp| second && second.match(exp) }
    end

    def split
      arr = return_matches(expression.source)

      arr.each_with_index do |item, index|
        if is_indicator(item, arr[index + 1])
          arr[index] = [ item, RegXing::Regex.process_count_indicator(arr.delete_at(index + 1)) ]
        elsif is_anchor(item)
          arr[index] = nil
        else
          arr[index] = [item, 1]
        end
      end

      arr.compact
    end
{'fraction_non_alphanumeric': 0.20974025974025973, 'fraction_numerical': 0.0037337662337662337, 'mean_word_length': 1.59957805907173, 'pattern_counts': {'":': 2, '<': 23, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2840750', 'n_tokens_mistral': 2246, 'n_tokens_neox': 2090, 'n_words': 330}