<commit_before>module CommentParser::Scanner::Concerns::SimpleScanner
  include CommentParser::CodeObject::Comment::Type

  def self.included(k)
    k.class_eval do |klass|
      extend ClassMethods
    end
  end

  module ClassMethods
    include CommentParser::CodeObject::Comment::Type

    def define_rule(open: nil, close: nil, type: ONE_LINER_COMMENT)
      @comment_regexp ||= []
      raise ArgumentError unless [type, open].all?

      definition = { open: build_regexp(open), type: type }

      if type == BLOCK_COMMENT
        definition[:close] = build_regexp(close, Regexp::MULTILINE)
      end
      @comment_regexp << definition
    end

    def define_bracket(bracket, options = 0)
      open_regexp = build_regexp(bracket)
      close_regexp = if bracket.is_a?(Regexp)
                 join_regexp(/(?<!\\)/, bracket)
               else
                 /(?<!\\)#{bracket}/
               end
      close_regexp = Regexp.new(close_regexp.source, options)
      append_bracket(open_regexp, close_regexp)
    end

    def define_regexp_bracket
      append_bracket(%r!/(?=[^/])!, /(?<!\\)\//)
    end

    def define_default_bracket
      define_bracket('"', Regexp::MULTILINE)
      define_bracket("'", Regexp::MULTILINE)
    end

    def append_bracket(open, close)
      @brackets ||= []
      @brackets << { open: open, close: close }
    end

    private

    def join_regexp(*regexp)
      # [review] - Should I ignore regexp options?
      Regexp.new(regexp.map { |v| v.source }.inject(:+))
    end

    def build_regexp(str_or_reg, type = 0)
      str_or_reg = str_or_reg.source if str_or_reg.respond_to?(:source)
      Regexp.new(str_or_reg, type)
    end
  end

  def self.attr_definition(*keys)
    keys.each do |key|
      define_method key do
        self.class.instance_variable_get("@#{key}") || []
      end
    end
  end
  attr_definition :brackets, :comment_regexp

  def scan
    until scanner.eos?
      case
      when scan_bracket
        next
      when scan_comment<commit_msg>Improves simple_scanner<commit_after>module CommentParser::Scanner::Concerns::SimpleScanner
  include CommentParser::CodeObject::Comment::Type

  def self.included(k)
    k.class_eval do |klass|
      extend ClassMethods
    end
  end

  module ClassMethods
    include CommentParser::CodeObject::Comment::Type

    def define_rule(open: nil, close: nil, type: ONE_LINER_COMMENT)
      @comment_regexp ||= []
      raise ArgumentError unless [type, open].all?

      definition = { open: build_regexp(open), type: type }

      if type == BLOCK_COMMENT
        definition[:close] = build_regexp(close, Regexp::MULTILINE)
      end
      @comment_regexp << definition
    end

    def define_bracket(bracket, options = 0)
      open_regexp = build_regexp(bracket)
      close_regexp = if bracket.is_a?(Regexp)
                 join_regexp(/(?<!\\)/, bracket)
               else
                 /(?<!\\)#{bracket}/
               end
      close_regexp = Regexp.new(close_regexp.source, options)
      append_bracket(open_regexp, close_regexp)
    end

    def define_regexp_bracket
      append_bracket(%r!/(?=[^/])!, /(?<!\\)\//)
    end

    def define_default_bracket
      define_bracket('"', Regexp::MULTILINE)
      define_bracket("'", Regexp::MULTILINE)
    end

    def append_bracket(open, close)
      @brackets ||= []
      @brackets << { open: open, close: close }
    end

    private

    def join_regexp(*regexp)
      # [review] - Should I ignore regexp options?
      Regexp.new(regexp.map { |v| v.source }.inject(:+))
    end

    def build_regexp(str_or_reg, type = 0)
      str_or_reg = str_or_reg.source if str_or_reg.respond_to?(:source)
      Regexp.new(str_or_reg, type)
    end
  end

  def self.attr_definition(*keys)
    keys.each do |key|
      define_method key do
        self.class.instance_variable_get("@#{key}") || []
      end
    end
  end
  attr_definition :brackets, :comment_regexp

  def scan
    until scanner.eos?
      case
      when scan_comment
        next
      when scan_bracket
{'fraction_non_alphanumeric': 0.11566979733069699, 'fraction_numerical': 0.0009886307464162135, 'mean_word_length': 2.6724137931034484, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6267095', 'n_tokens_mistral': 1391, 'n_tokens_neox': 1306, 'n_words': 274}