<commit_before># encoding: utf-8

require 'open3'

module Rubocop
  module Cop
    class Syntax < Cop
      def inspect(file, source, tokens, sexp)
        _, stderr, _ =
          Open3.capture3('ruby -wc', stdin_data: source.join("\n"))

        stderr.each_line do |line|
          line_no, severity, message = process_line(line)
          add_offence(severity, line_no, message)
        end
      end

      def process_line(line)
        line_no, message = line.match(/.+:(\d+): (.+)/).captures
        if message.start_with?('warning: ')
          [line_no.to_i, :warning, message.sub(/warning: /, '').capitalize]
        else
          [line_no.to_i, :error, message.capitalize]
        end
      end
    end
  end
end
<commit_msg>Discard uninformative lines in the syntax cop<commit_after># encoding: utf-8

require 'open3'

module Rubocop
  module Cop
    class Syntax < Cop
      def inspect(file, source, tokens, sexp)
        _, stderr, _ =
          Open3.capture3('ruby -wc', stdin_data: source.join("\n"))

        stderr.each_line do |line|
          # discard lines that are not containing relevant info
          if line =~ /.+:(\d+): (.+)/
            line_no, severity, message = process_line(line)
            add_offence(severity, line_no, message)
          end
        end
      end

      def process_line(line)
        line_no, message = line.match(/.+:(\d+): (.+)/).captures
        if message.start_with?('warning: ')
          [line_no.to_i, :warning, message.sub(/warning: /, '').capitalize]
        else
          [line_no.to_i, :error, message.capitalize]
        end
      end
    end
  end
end

{'fraction_non_alphanumeric': 0.12714987714987716, 'fraction_numerical': 0.004914004914004914, 'mean_word_length': 2.126679462571977, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6479387', 'n_tokens_mistral': 550, 'n_tokens_neox': 525, 'n_words': 128}