<commit_before>require_relative './node'

module UrlRegexp
  class Set < ::Set
    def include?(o)
      # Comparing keys directly is faster than rehash everytime for few items
      # @hash.rehash
      # super(o)
      @hash.keys.include?(o)
    end
  end

  class PathSet < Node
    extend Forwardable

    def_delegators :@set, :size, *Enumerable.public_instance_methods(false)

    def initialize(set = Set.new)
      @set = set
    end

    def append(path)
      set << path
    end

    def ==(other)
      self.class == other.class &&
        @set.to_a == other.set.to_a
    end

    alias eql? ==

    def hash
      @set.hash
    end

    def &(other)
      self.class.new(@set & other.set)
    end

    def |(other)
      self.class.new(@set & other.set)
    end

    def include?(path)
      @set.include?(path)
    end

    def to_regexp_s
      if 5 < size
        '([^#?]*)'
      elsif 1 < size
        children_paths = map(&:paths).reduce { |a, e| a & e }
        if children_paths.size == 1 && all? { |p| !p.path_end }
          "(#{map(&:label).join('|')})/#{children_paths.to_regexp_s}"
        else
          regexps = map(&:to_regexp_s)
          match = ''
          if regexps.size > 1
            base = regexps[0]
            base.split(//).each do |c|
              tmp = match + c
              break unless regexps[1..-1].all? { |r| r.start_with? tmp }
              match = _match
            end
            regexps = regexps.map { |r| r[match.size..-1] }
          end
          "#{match}(#{regexps.join('|')})"
        end
      elsif 1 == size
        to_a.first.to_regexp_s
      end
    end

    protected

    attr_reader :set
  end
end<commit_msg>Fix wrong variable<commit_after>require_relative './node'

module UrlRegexp
  class Set < ::Set
    def include?(o)
      # Comparing keys directly is faster than rehash everytime for few items
      # @hash.rehash
      # super(o)
      @hash.keys.include?(o)
    end
  end

  class PathSet < Node
    extend Forwardable

    def_delegators :@set, :size, *Enumerable.public_instance_methods(false)

    def initialize(set = Set.new)
      @set = set
    end

    def append(path)
      set << path
    end

    def ==(other)
      self.class == other.class &&
        @set.to_a == other.set.to_a
    end

    alias eql? ==

    def hash
      @set.hash
    end

    def &(other)
      self.class.new(@set & other.set)
    end

    def |(other)
      self.class.new(@set & other.set)
    end

    def include?(path)
      @set.include?(path)
    end

    def to_regexp_s
      if 5 < size
        '([^#?]*)'
      elsif 1 < size
        children_paths = map(&:paths).reduce { |a, e| a & e }
        if children_paths.size == 1 && all? { |p| !p.path_end }
          "(#{map(&:label).join('|')})/#{children_paths.to_regexp_s}"
        else
          regexps = map(&:to_regexp_s)
          match = ''
          if regexps.size > 1
            base = regexps[0]
            base.split(//).each do |c|
              tmp = match + c
              break unless regexps[1..-1].all? { |r| r.start_with? tmp }
              match = tmp
            end
            regexps = regexps.map { |r| r[match.size..-1] }
          end
          "#{match}(#{regexps.join('|')})"
        end
      elsif 1 == size
        to_a.first.to_regexp_s
      end
    end

    protected

    attr_reader :set
  end
end
{'fraction_non_alphanumeric': 0.13218731475992887, 'fraction_numerical': 0.005334914048606995, 'mean_word_length': 1.6849642004773269, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7190198', 'n_tokens_mistral': 1170, 'n_tokens_neox': 1147, 'n_words': 271}