<commit_before># encoding: utf-8

module Rubocop
  module Cop
    class Cop
      attr_accessor :offences

      @all = []
      @enabled = []
      @config = {}

      class << self
        attr_accessor :all
        attr_accessor :enabled
        attr_accessor :config
      end

      def self.inherited(subclass)
        all << subclass
      end

      def self.enabled
        all.select(&:enabled?)
      end

      def self.enabled?
        true
      end

      def initialize
        @offences = []
      end

      def has_report?
        !@offences.empty?
      end

      def inspect_source(file, source)
        case method(:inspect).arity
        when 2
          inspect(file, source)
        else
          tokens = Ripper.lex(source.join("\n"))
          sexp = Ripper.sexp(source.join("\n"))
          inspect(file, source, tokens, sexp)
        end
      end

      def add_offence(file, line_number, line, message)
        @offences << Offence.new(file, line_number, line, message)
      end

      private

      def each_parent_of(sym, sexp)
        parents = []
        sexp.each do |elem|
          if Array === elem
            if elem[0] == sym
              parents << sexp
              elem = elem[1..-1]
            end
            each_parent_of(sym, elem) { |parent| parents << parent }
          end
        end
        parents.uniq.each { |parent| yield parent }
      end

      def each(sym, sexp)
        yield sexp if sexp[0] == sym
        sexp.each do |elem|
          each(sym, elem) { |s| yield s } if Array === elem
        end
      end

      def whitespace?(token)
        [:on_sp, :on_ignored_nl, :on_nl].include?(token[1])
      end
    end
  end<commit_msg>Added elements to array only if they're not already in there instead of calling uniq afterwards.<commit_after># encoding: utf-8

module Rubocop
  module Cop
    class Cop
      attr_accessor :offences

      @all = []
      @enabled = []
      @config = {}

      class << self
        attr_accessor :all
        attr_accessor :enabled
        attr_accessor :config
      end

      def self.inherited(subclass)
        all << subclass
      end

      def self.enabled
        all.select(&:enabled?)
      end

      def self.enabled?
        true
      end

      def initialize
        @offences = []
      end

      def has_report?
        !@offences.empty?
      end

      def inspect_source(file, source)
        case method(:inspect).arity
        when 2
          inspect(file, source)
        else
          tokens = Ripper.lex(source.join("\n"))
          sexp = Ripper.sexp(source.join("\n"))
          inspect(file, source, tokens, sexp)
        end
      end

      def add_offence(file, line_number, line, message)
        @offences << Offence.new(file, line_number, line, message)
      end

      private

      def each_parent_of(sym, sexp)
        parents = []
        sexp.each do |elem|
          if Array === elem
            if elem[0] == sym
              parents << sexp unless parents.include?(sexp)
              elem = elem[1..-1]
            end
            each_parent_of(sym, elem) do |parent|
              parents << parent unless parents.include?(parent)
            end
          end
        end
        parents.each { |parent| yield parent }
      end

      def each(sym, sexp)
        yield sexp if sexp[0] == sym
        sexp.each do |elem|
          each(sym, elem) { |s| yield s } if Array === elem
        end
      end

      def whitespace?(token)
        [:on_sp, :on_ignored_nl, :on_nl].include?(token[1])
      end
    end
  end
{'fraction_non_alphanumeric': 0.09740440971253139, 'fraction_numerical': 0.003907340217694669, 'mean_word_length': 1.5673352435530086, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1839979', 'n_tokens_mistral': 1087, 'n_tokens_neox': 1063, 'n_words': 290}