<commit_before># Various reports, aggregated here for convenience
class Report
  # Returns a report of issues found per repository
  #
  # @param run [Run] the Run being reported over
  # @return [Hash<Hash<String>>] the report
  def self.issues_per_repo(run)
    Repository.
      joins(finding_aids: {
              finding_aid_versions: {
                concrete_issues: [:issue,:run]
              }
            }).
      where(runs: {id: run.id}).
      group('repositories.id', 'issues.id').
      order('code, substr').
      pluck('code', Arel.sql('substr(issues.message, 0, 120)'), 'COUNT(concrete_issues.id)').
      group_by {|el| el[1]}.
      map do |k, v|
        [k, v.map {|el| [el[0], el[2]]}.to_h] # Drop message from vals, then hashify
      end.to_h
  end
end
<commit_msg>Report on manual issues<commit_after>require 'set'
# Various reports, aggregated here for convenience
class Report
  # Returns a report of issues found per repository
  #
  # @param run [Run] the Run being reported over
  # @return [Hash<Hash<String>>] the report
  def self.issues_per_repo(run)
    Repository.
      joins(finding_aids: {
              finding_aid_versions: {
                concrete_issues: [:issue,:run]
              }
            }).
      where(runs: {id: run.id}).
      group('repositories.id', 'issues.id').
      order('code, substr').
      pluck('code', Arel.sql('substr(issues.message, 0, 120)'), 'COUNT(concrete_issues.id)').
      group_by {|el| el[1]}.
      map do |k, v|
        [k, v.map {|el| [el[0], el[2]]}.to_h] # Drop message from vals, then hashify
      end.to_h
  end

  def self.manual_issues(run)
    manual_issues = Set.new(run.schematron.issues.where("identifier NOT IN (?)", Fixes.to_h.keys).pluck('identifier'))
    Run.
      joins(finding_aid_versions: [
              {concrete_issues: :issue},
              :finding_aid
            ]).
      where({'runs.id' => run.id, 'issues.identifier' => manual_issues}).
      group('eadid', 'issues.id', Arel.sql("issues.identifier || ' - ' || issues.message")).
      order('eadid', 'issues.id').
      pluck('eadid', Arel.sql("issues.identifier || ' - ' || issues.message"), Arel.sql('array_agg(DISTINCT concrete_issues.line_number)'))

    # Todo: put this into some kind of usable data format and display it somewhere?

  end

end

{'fraction_non_alphanumeric': 0.14931153184165233, 'fraction_numerical': 0.006024096385542169, 'mean_word_length': 2.696343402225755, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3907126', 'n_tokens_mistral': 782, 'n_tokens_neox': 754, 'n_words': 184}