<commit_before>require 'set'

class Array
  def join2(sep = $,, last_sep = nil)
    if last_sep && 1 < self.size
      self[0..-2].join(sep) + last_sep + self[-1]
    else
      self.join(sep)
    end
  end
end

desc "Alias to whitespace:check"
task :whitespace => "whitespace:check"

namespace :whitespace do
  violations = {
    "tabs" => {
      :expr => /\t/,
      :excl => /(^Changelog)|(\.diff$)/,
    },
    "trailing spaces" => {
      :expr     => /\s+$/,
      :neg_expr => /^\s*#/,
    },
  }

  desc "List files with " + violations.keys.sort.join2(", ", " or ")
  task :check do
    files = {}

    Bones.config.gem.files.each do |name|
      files[name] = {}
      linenum     = 0

      open(name, "r") do |file|
        while (!file.eof?)
          line    = file.gets.chomp
          linenum = linenum.next

          violations.each do |n, v|
            next if v[:excl] =~ name

            if v[:expr] =~ line && v[:neg_expr] !~ line
              (files[name][n] ||= []) << linenum
            end
          end
        end
      end

      files.delete(name) if files[name].empty?
    end

    unless files.empty?
      pad_to = files.keys.max { |a, b| a.length <=> b.length }.length

      files.sort.each do |name, vs|
        puts(sprintf("%-*s %s",
                     pad_to + 1,
                     name + ":",
                     vs.sort.map { |v, ls|
                       "#{v} (line #{ls.join(", ")})"
                     }.join(", ")))
      end
    end
  end
end<commit_msg>Changed output of whitespace task to print "filename:linenumber: whitespace type". This matches the way ruby prints warnings and errors.<commit_after>require 'set'

class Array
  def join2(sep = $,, last_sep = nil)
    if last_sep && 1 < self.size
      self[0..-2].join(sep) + last_sep + self[-1]
    else
      self.join(sep)
    end
  end
end

desc "Alias to whitespace:check"
task :whitespace => "whitespace:check"

namespace :whitespace do
  violations = {
    "tabs" => {
      :expr => /\t/,
      :excl => /(^Changelog)|(\.diff$)/,
    },
    "trailing spaces" => {
      :expr     => /\s+$/,
      :neg_expr => /^\s*#/,
    },
  }

  desc "List files with " + violations.keys.sort.join2(", ", " or ")
  task :check do
    files = {}

    Bones.config.gem.files.each do |name|
      files[name] = {}
      linenum     = 0

      open(name, "r") do |file|
        while (!file.eof?)
          line    = file.gets.chomp
          linenum = linenum.next

          violations.each do |n, v|
            next if v[:excl] =~ name

            if v[:expr] =~ line && v[:neg_expr] !~ line
              (files[name][linenum] ||= []) << n
            end
          end
        end
      end

      files.delete(name) if files[name].empty?
    end

    unless files.empty?
      pad_to = files.keys.max { |a, b| a.length <=> b.length }.length

      puts

      files.sort.each do |file, linenumbers|
        linenumbers.keys.sort.each do |linenumber|
          linenumbers[linenumber].sort.each do |validation_name|
            puts "%s:%d: %s" % ["./" + file, linenumber, validation_name]
          end
        end
      end

      puts
    end
  end
end
{'fraction_non_alphanumeric': 0.15746292205743137, 'fraction_numerical': 0.004733354370463868, 'mean_word_length': 1.6887192536047497, 'pattern_counts': {'":': 1, '<': 11, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1267795', 'n_tokens_mistral': 1123, 'n_tokens_neox': 1081, 'n_words': 261}