<commit_before># encoding: utf-8

module Rubocop
  module Cop
    class Position < Struct.new :lineno, :column
      # Does a recursive search and replaces each [lineno, column] array
      # in the sexp with a Position object.
      def self.make_position_objects(sexp)
        if sexp[0] =~ /^@/
          sexp[2] = Position.new(*sexp[2])
        else
          sexp.grep(Array).each { |s| make_position_objects(s) }
        end
      end

      # The point of this class is to provide named attribute access.
      # So we don't want backwards compatibility with array indexing.
      undef_method :[]
    end

    class Token
      attr_reader :pos, :type, :text

      def initialize(pos, type, text)
        @pos, @type, @text = Position.new(*pos), type, text
      end

      def to_s
        "[[#{@pos.lineno}, #{@pos.column}], #{@type}, #{@text.inspect}]"
      end
    end

    class Cop
      attr_accessor :offences
      attr_accessor :debug
      attr_writer :correlations, :disabled_lines

      @all = []
      @config = {}

      class << self
        attr_accessor :all
        attr_accessor :config
      end

      def self.inherited(subclass)
        all << subclass
      end

      def self.cop_name
        name.to_s.split('::').last
      end

      def initialize
        @offences = []
        @debug = false
      end

      def has_report?
        !@offences.empty?
      end

      def add_offence(severity, line_number, message)
        unless @disabled_lines && @disabled_lines.include?(line_number)
          message = debug ? "#{name}: #{message}" : message
          @offences << Offence.new(severity, line_number, message)
        end
      end

      def name
        self.class.cop_name
      end

      private

      def each_parent_of(sym, sexp)
        parents = []
        sexp.each do |elem|
          if Array === elem
            if elem[0] == sym
              parents << sexp unless parents.include?(sexp)
              elem = elem[1..-1]
            end
            each_parent_of(sym, elem) do |parent|
              parents << parent unless parents.include?(parent)
            end
          end
        end
        parents.each { |parent| yield parent }
      end

      def each(sym, sexp)
        yield sexp if sexp[0] == sym
        sexp.each do |elem|
          each(sym, elem) { |s| yield s } if Array === elem
        end
      end

      def find_all(sym, sexp)
        result = []
        each(sym, sexp) { |s| result << s }
        result
      end

      def find_first(sym, sexp)
        find_all(sym, sexp).first
      end

      def whitespace?(token)
        [:on_sp, :on_ignored_nl, :on_nl].include?(token.type)
      end

      def all_positions(sexp)
        return [sexp[2]] if sexp[0] =~ /^@/
        sexp.grep(Array).reduce([]) { |a, e| a + all_positions(e) }
      end

      def keywords(tokens)
        # we need to keep track of the previous token to avoid
        # interpreting :some_keyword as the keyword some_keyword
        prev = Token.new(0, :init, '')
        keywords = []

        tokens.each do |t|
          keywords << t if prev.type != :on_symbeg && t.type == :on_kw
          prev = t
        end

        keywords
      end

      def each_keyword(keyword, tokens)
        keywords(tokens).select { |t| t.text == keyword }.each { |t| yield t }
      end
    end
  end
end<commit_msg>Handle defs with names coinciding with keywords<commit_after># encoding: utf-8

module Rubocop
  module Cop
    class Position < Struct.new :lineno, :column
      # Does a recursive search and replaces each [lineno, column] array
      # in the sexp with a Position object.
      def self.make_position_objects(sexp)
        if sexp[0] =~ /^@/
          sexp[2] = Position.new(*sexp[2])
        else
          sexp.grep(Array).each { |s| make_position_objects(s) }
        end
      end

      # The point of this class is to provide named attribute access.
      # So we don't want backwards compatibility with array indexing.
      undef_method :[]
    end

    class Token
      attr_reader :pos, :type, :text

      def initialize(pos, type, text)
        @pos, @type, @text = Position.new(*pos), type, text
      end

      def to_s
        "[[#{@pos.lineno}, #{@pos.column}], #{@type}, #{@text.inspect}]"
      end
    end

    class Cop
      attr_accessor :offences
      attr_accessor :debug
      attr_writer :correlations, :disabled_lines

      @all = []
      @config = {}

      class << self
        attr_accessor :all
        attr_accessor :config
      end

      def self.inherited(subclass)
        all << subclass
      end

      def self.cop_name
        name.to_s.split('::').last
      end

      def initialize
        @offences = []
        @debug = false
      end

      def has_report?
        !@offences.empty?
      end

      def add_offence(severity, line_number, message)
        unless @disabled_lines && @disabled_lines.include?(line_number)
          message = debug ? "#{name}: #{message}" : message
          @offences << Offence.new(severity, line_number, message)
        end
      end

      def name
        self.class.cop_name
      end

      private

      def each_parent_of(sym, sexp)
        parents = []
        sexp.each do |elem|
          if Array === elem
            if elem[0] == sym
              parents << sexp unless parents.include?(sexp)
              elem = elem[1..-1]
            end
            each_parent_of(sym, elem) do |parent|
              parents << parent unless parents.include?(parent)
            end
          end
        end
        parents.each { |parent| yield parent }
      end

      def each(sym, sexp)
        yield sexp if sexp[0] == sym
        sexp.each do |elem|
          each(sym, elem) { |s| yield s } if Array === elem
        end
      end

      def find_all(sym, sexp)
        result = []
        each(sym, sexp) { |s| result << s }
        result
      end

      def find_first(sym, sexp)
        find_all(sym, sexp).first
      end

      def whitespace?(token)
        [:on_sp, :on_ignored_nl, :on_nl].include?(token.type)
      end

      def all_positions(sexp)
        return [sexp[2]] if sexp[0] =~ /^@/
        sexp.grep(Array).reduce([]) { |a, e| a + all_positions(e) }
      end

      def keywords(tokens)
        # we need to keep track of the previous token to avoid
        # interpreting :some_keyword as the keyword some_keyword
        prev = Token.new(0, :init, '')
        # same goes for defs so we need to track those as well
        keywords = []

        tokens.each do |t|
          keywords << t if prev.type != :on_symbeg && t.type == :on_kw
          # def with name that's a kw confuses Ripper.lex
          penultimate = keywords[-2]
          keywords.pop if penultimate && penultimate.text == 'def'
          prev = t
        end

        keywords
      end

      def each_keyword(keyword, tokens)
        keywords(tokens).select { |t| t.text == keyword }.each { |t| yield t }
      end
    end
  end
end
{'fraction_non_alphanumeric': 0.10583368991584653, 'fraction_numerical': 0.0032805591213806874, 'mean_word_length': 1.7814359381197937, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6922486', 'n_tokens_mistral': 2193, 'n_tokens_neox': 2127, 'n_words': 618}