<commit_before>module Loco
  class Counter
    include Loco::Util

    EXCLUDE = [
               /\.gem/,
               /^db\/schema.rb/,
               /^db\/backups/,
               /^db\/migrate/,
               /^vendor/,
               /^public\/system/,
               /^public\/images/,
               /^public\/stylesheets/,
               /^config\/newrelic.yml/,
               /^tmp/,
               /^coverage/,
               /^log/,
               /^doc/,
               /^script/,
               /^stories/,
               /^static/,
               /^public\/fonts/,
               /\.#.+/,
               /\.png/,
               /\.pdf/,
               /\.ico/,
               /lib\/tasks\/.+\.txt/,
               /^README/,
               /^Gemfile.lock/,
               /^public\/javascripts\/jquery/
              ]

    def exclude? path
      EXCLUDE.each { |regex|
        if path =~ regex
          return true
        end
      }
      false
    end

    def analyse_loc opts={ }
      regexp = opts[:pattern]
      @stats = []
      @exts = Hash.new { |h, k| h[k] = 0 }
      @ext_lines = Hash.new { |h, k| h[k] = 0 }
      @sizes = Hash.new { |h, k| h[k] = [] }
      total_lines = 0

      here = File.expand_path "."
      Dir.glob("#{here}/**/*").each do |full_path|
        unless File.stat(full_path).directory?
          path = full_path.sub("#{here}/", '')
          unless exclude?(path) || !regexp.match(path)
            s = Stats.new(path)
            if s.matches(opts)
              @stats << s
              @exts[File.extname(path)] += 1
              @ext_lines[File.extname(path)] += s.lines
              @sizes[s.lines] << path
              total_lines += s.lines
            end
          end
        end
      end

      puts "\n### extensions ###"
      @exts.each { |ext, count| puts "#{align ext} - #{align count} files, #{align @ext_lines[ext]} lines"}

      puts "\n### distribution ###"
      @sizes = @sizes.to_a.sort { |a, b|
        a[0] <=> b[0]
      }

      @sizes.each { |size, files|
        puts "#{align size} lines #{align files.size} #{"*" * files.size}"
      }

      total = @stats.size
      @stats = @stats.sort { |a, b| b.lines <=> a.lines }
      @stats = @stats[0..50] if @stats.size > 50

      puts "\n### longest files ###"
      @stats.each { |s| puts s }

      puts "\n### Total #{regexp.inspect} : #{total} files, #{total_lines} lines, #{"%.3f" % (1.0 * total_lines / total)} avg loc per file ###"
    end
  end
end
<commit_msg>exclude .sqlite3 files by default<commit_after>module Loco
  class Counter
    include Loco::Util

    EXCLUDE = [
               /\.sqlite3/,
               /\.gem/,
               /^db\/schema.rb/,
               /^db\/backups/,
               /^db\/migrate/,
               /^vendor/,
               /^public\/system/,
               /^public\/images/,
               /^public\/stylesheets/,
               /^config\/newrelic.yml/,
               /^tmp/,
               /^coverage/,
               /^log/,
               /^doc/,
               /^script/,
               /^stories/,
               /^static/,
               /^public\/fonts/,
               /\.#.+/,
               /\.png/,
               /\.pdf/,
               /\.ico/,
               /lib\/tasks\/.+\.txt/,
               /^README/,
               /^Gemfile.lock/,
               /^public\/javascripts\/jquery/
              ]

    def exclude? path
      EXCLUDE.each { |regex|
        if path =~ regex
          return true
        end
      }
      false
    end

    def analyse_loc opts={ }
      regexp = opts[:pattern]
      @stats = []
      @exts = Hash.new { |h, k| h[k] = 0 }
      @ext_lines = Hash.new { |h, k| h[k] = 0 }
      @sizes = Hash.new { |h, k| h[k] = [] }
      total_lines = 0

      here = File.expand_path "."
      Dir.glob("#{here}/**/*").each do |full_path|
        unless File.stat(full_path).directory?
          path = full_path.sub("#{here}/", '')
          unless exclude?(path) || !regexp.match(path)
            s = Stats.new(path)
            if s.matches(opts)
              @stats << s
              @exts[File.extname(path)] += 1
              @ext_lines[File.extname(path)] += s.lines
              @sizes[s.lines] << path
              total_lines += s.lines
            end
          end
        end
      end

      puts "\n### extensions ###"
      @exts.each { |ext, count| puts "#{align ext} - #{align count} files, #{align @ext_lines[ext]} lines"}

      puts "\n### distribution ###"
      @sizes = @sizes.to_a.sort { |a, b|
        a[0] <=> b[0]
      }

      @sizes.each { |size, files|
        puts "#{align size} lines #{align files.size} #{"*" * files.size}"
      }

      total = @stats.size
      @stats = @stats.sort { |a, b| b.lines <=> a.lines }
      @stats = @stats[0..50] if @stats.size > 50

      puts "\n### longest files ###"
      @stats.each { |s| puts s }

      puts "\n### Total #{regexp.inspect} : #{total} files, #{total_lines} lines, #{"%.3f" % (1.0 * total_lines / total)} avg loc per file ###"
    end
  end
end

{'fraction_non_alphanumeric': 0.17572892040977148, 'fraction_numerical': 0.00591016548463357, 'mean_word_length': 1.1959342560553634, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2714692', 'n_tokens_mistral': 1664, 'n_tokens_neox': 1619, 'n_words': 351}