<commit_before>require 'csv'

module Hdo
  module Stats
    class AgreementScorer
      VALID_UNITS = [:propositions, :votes]

      COMBINATIONS = (
        parties = Party.all.to_a
        2.upto(parties.size).flat_map do |n|
          parties.combination(n).to_a
        end
      )

      def self.by_category(parliament_issues = ParliamentIssue.all)
        category_votes = Hash.new { |h, k| h[k] = [] }

        parliament_issues.each do |pi|
          pi.categories.each do |cat|
            cat = cat.main? ? cat : cat.parent
            category_votes[cat.human_name].concat pi.vote_ids
          end
        end

        result = {}

        category_votes.each do |category_name, vote_ids|
          votes = Vote.with_results.find(vote_ids.uniq)
          result[category_name] = new(votes: votes).result
        end

        result
      end

      def self.summary(opts = {})
        data, total = new(opts).result.values_at(:data, :total)
        data.map do |combo, count|
          {combination: combo, percentage: (count * 100 / total.to_f).round(1), total: total, count: count }
        end.sort_by { |e| e[:percentage] }
      end

      def self.csv_summary(opts = {})
        CSV.generate do |csv|
          csv << ["combination", "agreed_count", "total_count", "percentage"]
          summary(opts).sort_by { |e| e[:percentage] }.each do |entry|
            csv << entry.values_at(:combination, :count, :total, :percentage)
          end
        end
      end

      def self.csv_by_month(votes = Vote.all)
        groups = votes.group_by { |e| e.time.strftime("%Y-%m") }

        combos       = [%w[FrP H], %w[FrP H KrF V], %w[A Sp SV]].map { |e| e.sort }
        party_combos = combos.map { |combo| combo.map { |e| Party.find_by_external_id(e) } }

        result = Hash.new { |hash, key| hash[key] = {} }

        groups.each do |time, votes|
          data, total_count = new(votes: votes, combinations: party_combos).result.values_at(:data, :total)
          data.each do |combo, vote_count|
            result[time][combo] = {count: vote_count, total: total_count}
          end
        end

        CSV.generate do |csv|
          csv << ['month', 'unit_count', *combos.map { |e| e.join(',') }]

          result.sort_by { |time, _| time.split('-').map(&:to_i) }.each do |time, data|
            month_totals      = []
            month_percentages = []

            combos.each do |combo|
              d = data.fetch(combo.join(','))

              month_totals << d[:total]
              month_percentages << (d[:count] / d[:total].to_f) * 100
            end

            if month_totals.uniq.size > 1
              raise "total units varies by party combo for #{time.inspect}: #{month_totals.inspect}"
            end

            csv << [time, month_totals.first, *month_percentages]
          end
        end
      end

      def self.csv
        all        = new.result
        categories = by_category
<commit_msg>AgreementScorer: make category csv more configurable.<commit_after>require 'csv'

module Hdo
  module Stats
    class AgreementScorer
      VALID_UNITS = [:propositions, :votes]

      COMBINATIONS = (
        parties = Party.all.to_a
        2.upto(parties.size).flat_map do |n|
          parties.combination(n).to_a
        end
      )

      def self.by_category(parliament_issues = ParliamentIssue.all)
        category_votes = Hash.new { |h, k| h[k] = [] }

        parliament_issues.each do |pi|
          pi.categories.each do |cat|
            cat = cat.main? ? cat : cat.parent
            category_votes[cat.human_name].concat pi.vote_ids
          end
        end

        result = {}

        category_votes.each do |category_name, vote_ids|
          votes = Vote.with_results.find(vote_ids.uniq)
          result[category_name] = new(votes: votes).result
        end

        result
      end

      def self.summary(opts = {})
        data, total = new(opts).result.values_at(:data, :total)
        data.map do |combo, count|
          {combination: combo, percentage: (count * 100 / total.to_f).round(1), total: total, count: count }
        end.sort_by { |e| e[:percentage] }
      end

      def self.csv_summary(opts = {})
        CSV.generate do |csv|
          csv << ["combination", "agreed_count", "total_count", "percentage"]
          summary(opts).sort_by { |e| e[:percentage] }.each do |entry|
            csv << entry.values_at(:combination, :count, :total, :percentage)
          end
        end
      end

      def self.csv_by_month(votes = Vote.all)
        groups = votes.group_by { |e| e.time.strftime("%Y-%m") }

        combos       = [%w[FrP H], %w[FrP H KrF V], %w[A Sp SV]].map { |e| e.sort }
        party_combos = combos.map { |combo| combo.map { |e| Party.find_by_external_id(e) } }

        result = Hash.new { |hash, key| hash[key] = {} }

        groups.each do |time, votes|
          data, total_count = new(votes: votes, combinations: party_combos).result.values_at(:data, :total)
          data.each do |combo, vote_count|
            result[time][combo] = {count: vote_count, total: total_count}
          end
        end

        CSV.generate do |csv|
          csv << ['month', 'unit_count', *combos.map { |e| e.join(',') }]

          result.sort_by { |time, _| time.split('-').map(&:to_i) }.each do |time, data|
            month_totals      = []
            month_percentages = []

            combos.each do |combo|
              d = data.fetch(combo.join(','))

              month_totals << d[:total]
              month_percentages << (d[:count] / d[:total].to_f) * 100
            end

            if month_totals.uniq.size > 1
              raise "total units varies by party combo for #{time.inspect}: #{month_totals.inspect}"
            end

            csv << [time, month_totals.first, *month_percentages]
          end
        end
      end

      def self.csv(opts = {})
        all               = new(opts).result
        parliament_issues = opts[:votes] ? opts[:votes].flat_map(&:parliament_issues) : ParliamentIssue.all
        categories        = by_category(parliament_issues)

{'fraction_non_alphanumeric': 0.13040629095674966, 'fraction_numerical': 0.002948885976408912, 'mean_word_length': 1.9780487804878049, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7228143', 'n_tokens_mistral': 2027, 'n_tokens_neox': 1955, 'n_words': 431}