<commit_before># models a summary of the answers for a question on a form
class Report::QuestionSummary
  # the questioning we're summarizing
  attr_reader :questioning
    
  # array containing the individual items of information in the summary
  attr_reader :items

  # the column headers, if any
  attr_reader :headers

  # the number of null answers we encountered
  attr_reader :null_count

  # the number of choices as opposed to answers, we encountered (only set for select_multiple questions)
  attr_reader :choice_count

  attr_reader :display_type
  attr_reader :overall_header

  def self.generate_for(questionings)
    @summaries = []

    # split questionings by type
    type_to_group = {
      'integer' => 'stat',
      'decimal' => 'stat',
      'time' => 'stat',
      'datetime' => 'stat',
      'select_one' => 'select',
      'select_multiple' => 'select',
      'date' => 'date',
      'text' => 'raw',
      'tiny_text' => 'raw',
      'long_text' => 'raw'
    }

    grouped = {'stat' => [], 'select' => [], 'date' => [], 'raw' => []}

    questionings.each{|qing| grouped[type_to_group[qing.qtype_name]] << qing}

    # take all statistic type questions and get data
    @summaries += generate_for_statistic_questionings(grouped['stat'])

    @summaries += generate_for_select_questionings(grouped['select'])

    # @summaries += generate_for_date_questionings(date_qings)

    # @summaries += generate_for_raw_questionings(raw_qings)

    @summaries
  end

  def self.generate_for_statistic_questionings(questionings)
    return [] if questionings.empty?

    qing_ids = questionings.map(&:id).join(',')
    qings_by_id = questionings.index_by(&:id)

    # build big query
    query = <<-eos
      SELECT qing.id AS qing_id, q.qtype_name AS qtype_name,
        SUM(
          CASE q.qtype_name
            WHEN 'integer' THEN IF(a.value IS NULL OR a.value = '', 1, 0)
            WHEN 'decimal' THEN IF(a.value IS NULL OR a.value = '', 1, 0)
            WHEN 'time' THEN IF(a.time_value IS NULL, 1, 0)
            WHEN 'datetime' THEN IF(a.datetime_value IS NULL, 1, 0)
          END
        ) AS null_count,
        CASE q.qtype_name
          WHEN 'integer' THEN AVG(CONVERT(a.value, SIGNED INTEGER)) 
          WHEN 'decimal' THEN AVG(CONVERT(a.value, DECIMAL(9,6)))
          WHEN 'time' THEN SEC_TO_TIME(AVG(TIME_TO_SEC(a.time_value)))
          WHEN 'datetime' THEN FROM_UNIXTIME(AVG(UNIX_TIMESTAMP(a.datetime_value)))
        END AS mean,
        CASE q.qtype_name
          WHEN 'integer' THEN MIN(CONVERT(a.value, SIGNED INTEGER)) 
          WHEN 'decimal' THEN MIN(CONVERT(a.value, DECIMAL(9,6)))
          WHEN 'time' THEN MIN(a.time_value)
          WHEN 'datetime' THEN MIN(a.datetime_value)
        END AS min,
        CASE q.qtype_name
          WHEN 'integer' THEN MAX(CONVERT(a.value, SIGNED INTEGER)) 
          WHEN 'decimal' THEN MAX(CONVERT(a.value, DECIMAL(9,6)))
          WHEN 'time' THEN MAX(a.time_value)
          WHEN 'datetime' THEN MAX(a.datetime_value)
        END AS max
      FROM answers a INNER JOIN questionings qing ON a.questioning_id = qing.id AND qing.id IN (#{qing_ids}) 
        INNER JOIN questions q ON q.id = qing.question_id 
      WHERE q.qtype_name in ('integer', 'decimal', 'time', 'datetime') GROUP BY qing.id, q.qtype_name
    eos

    res = ActiveRecord::Base.connection.execute(query)
    stats = %w(mean min max)

    # build headers
    headers = stats.map{|s| {:name => I18n.t("report/report.standard_form_report.stat_headers.#{s}"), :stat => s.to_sym}}

    summaries = res.each(:as => :hash).map do |row|
      qing = qings_by_id[row['qing_id']]

      # if mean is nil, means no non-nil values
      if row['mean'].nil?

        items = {}
      else
        # convert stats to appropriate type
        case qing.qtype_name
        when 'integer'
          row['mean'] = row['mean'].to_f
          %w(max min).each{|s| row[s] = row[s].to_i}
        when 'decimal'
          %w(mean max min).each{|s| row[s] = row[s].to_f}
        when 'time'
          %w(mean max min).each{|s| row[s] = I18n.l(Time.parse(row[s]), :format => :time_only)}
        when 'datetime'
          %w(mean max min).each{|s| row[s] = I18n.l(Time.zone.parse(row[s] + ' UTC'))}
        end

        # build items
        items = stats.map{|stat| Report::SummaryItem.new(:qtype_name => row['qtype_name'], :stat => row[stat])}
      end

      # build summary
      new(:questioning => qings_by_id[row['qing_id']], :display_type => :structured, :headers => headers, :items => items, :null_count => row['null_count'])
    end

    # build blank summaries for missing qings
    already_summarized = summaries.map(&:questioning)
    summaries += (questionings - already_summarized).map do |qing|
      new(:questioning => qing, :display_type => :structured, :headers => headers, :items => {}, :null_count => 0)
    end

    summaries
  end

  def self.generate_for_select_questionings(questionings)
    return [] if questionings.empty?

    qing_ids = questionings.map(&:id).join(',')
    qings_by_id = questionings.index_by(&:id)

    # build big query
    query = <<-eos
      SELECT qings.id AS qing_id, a.option_id AS option_id, COUNT(a.id) AS answer_count 
      FROM questionings qings 
        INNER JOIN questions q ON qings.question_id = q.id 
        LEFT OUTER JOIN answers a ON qings.id = a.questioning_id
        WHERE q.qtype_name IN ('select_one', 'select_multiple') 
          AND qings.id IN (#{qing_ids})
        GROUP BY qings.id, a.option_id 
    eos

    # run query
    res = ActiveRecord::Base.connection.execute(query)

    # read tallies into hash
    tallies = {}
    res.each(:as => :hash).map do |row|
      tallies[[row['qing_id'], row['option_id']]] = row['answer_count']
    end

    # loop over each questioning and generate summary
    questionings.map do |qing|

      # build headers
      headers = qing.options.map{|option| {:name => option.name, :option => option}}

      # build tallies, keeping a running sum
      non_null_answer_count = 0
      items = qing.options.map do |o|
        count = tallies[[qing.id, o.id]] || 0
        non_null_answer_count += count
        Report::SummaryItem.new(:count => count)
      end

      # compute percentages
      items.each do |item|
        item.pct = non_null_answer_count == 0 ? 0 : item.count.to_f / non_null_answer_count * 100
      end

      # null count
      null_count = tallies[[qing.id, nil]]

      new(:questioning => qing, :display_type => :structured, :overall_header => qing.option_set.name, :headers => headers, :items => items, :null_count => null_count)
    end
  end

  def initialize(attribs)
    # save attribs
    attribs.each{|k,v| instance_variable_set("@#{k}", v)}

    qtype_name = questioning.qtype_name
    case qtype_name

    # these types all get descriptive statistics
    when nil<commit_msg>got basic sel mult working<commit_after># models a summary of the answers for a question on a form
class Report::QuestionSummary
  # the questioning we're summarizing
  attr_reader :questioning
    
  # array containing the individual items of information in the summary
  attr_reader :items

  # the column headers, if any
  attr_reader :headers

  # the number of null answers we encountered
  attr_reader :null_count

  # the number of choices as opposed to answers, we encountered (only set for select_multiple questions)
  attr_reader :choice_count

  attr_reader :display_type
  attr_reader :overall_header

  def self.generate_for(questionings)
    @summaries = []

    # split questionings by type
    type_to_group = {
      'integer' => 'stat',
      'decimal' => 'stat',
      'time' => 'stat',
      'datetime' => 'stat',
      'select_one' => 'select',
      'select_multiple' => 'select',
      'date' => 'date',
      'text' => 'raw',
      'tiny_text' => 'raw',
      'long_text' => 'raw'
    }

    grouped = {'stat' => [], 'select' => [], 'date' => [], 'raw' => []}

    questionings.each{|qing| grouped[type_to_group[qing.qtype_name]] << qing}

    # take all statistic type questions and get data
    @summaries += generate_for_statistic_questionings(grouped['stat'])

    @summaries += generate_for_select_questionings(grouped['select'])

    # @summaries += generate_for_date_questionings(date_qings)

    # @summaries += generate_for_raw_questionings(raw_qings)

    @summaries
  end

  def self.generate_for_statistic_questionings(questionings)
    return [] if questionings.empty?

    qing_ids = questionings.map(&:id).join(',')
    qings_by_id = questionings.index_by(&:id)

    # build big query
    query = <<-eos
      SELECT qing.id AS qing_id, q.qtype_name AS qtype_name,
        SUM(
          CASE q.qtype_name
            WHEN 'integer' THEN IF(a.value IS NULL OR a.value = '', 1, 0)
            WHEN 'decimal' THEN IF(a.value IS NULL OR a.value = '', 1, 0)
            WHEN 'time' THEN IF(a.time_value IS NULL, 1, 0)
            WHEN 'datetime' THEN IF(a.datetime_value IS NULL, 1, 0)
          END
        ) AS null_count,
        CASE q.qtype_name
          WHEN 'integer' THEN AVG(CONVERT(a.value, SIGNED INTEGER)) 
          WHEN 'decimal' THEN AVG(CONVERT(a.value, DECIMAL(9,6)))
          WHEN 'time' THEN SEC_TO_TIME(AVG(TIME_TO_SEC(a.time_value)))
          WHEN 'datetime' THEN FROM_UNIXTIME(AVG(UNIX_TIMESTAMP(a.datetime_value)))
        END AS mean,
        CASE q.qtype_name
          WHEN 'integer' THEN MIN(CONVERT(a.value, SIGNED INTEGER)) 
          WHEN 'decimal' THEN MIN(CONVERT(a.value, DECIMAL(9,6)))
          WHEN 'time' THEN MIN(a.time_value)
          WHEN 'datetime' THEN MIN(a.datetime_value)
        END AS min,
        CASE q.qtype_name
          WHEN 'integer' THEN MAX(CONVERT(a.value, SIGNED INTEGER)) 
          WHEN 'decimal' THEN MAX(CONVERT(a.value, DECIMAL(9,6)))
          WHEN 'time' THEN MAX(a.time_value)
          WHEN 'datetime' THEN MAX(a.datetime_value)
        END AS max
      FROM answers a INNER JOIN questionings qing ON a.questioning_id = qing.id AND qing.id IN (#{qing_ids}) 
        INNER JOIN questions q ON q.id = qing.question_id 
      WHERE q.qtype_name in ('integer', 'decimal', 'time', 'datetime') GROUP BY qing.id, q.qtype_name
    eos

    res = ActiveRecord::Base.connection.execute(query)
    stats = %w(mean min max)

    # build headers
    headers = stats.map{|s| {:name => I18n.t("report/report.standard_form_report.stat_headers.#{s}"), :stat => s.to_sym}}

    summaries = res.each(:as => :hash).map do |row|
      qing = qings_by_id[row['qing_id']]

      # if mean is nil, means no non-nil values
      if row['mean'].nil?

        items = {}
      else
        # convert stats to appropriate type
        case qing.qtype_name
        when 'integer'
          row['mean'] = row['mean'].to_f
          %w(max min).each{|s| row[s] = row[s].to_i}
        when 'decimal'
          %w(mean max min).each{|s| row[s] = row[s].to_f}
        when 'time'
          %w(mean max min).each{|s| row[s] = I18n.l(Time.parse(row[s]), :format => :time_only)}
        when 'datetime'
          %w(mean max min).each{|s| row[s] = I18n.l(Time.zone.parse(row[s] + ' UTC'))}
        end

        # build items
        items = stats.map{|stat| Report::SummaryItem.new(:qtype_name => row['qtype_name'], :stat => row[stat])}
      end

      # build summary
      new(:questioning => qings_by_id[row['qing_id']], :display_type => :structured, :headers => headers, :items => items, :null_count => row['null_count'])
    end

    # build blank summaries for missing qings
    already_summarized = summaries.map(&:questioning)
    summaries += (questionings - already_summarized).map do |qing|
      new(:questioning => qing, :display_type => :structured, :headers => headers, :items => {}, :null_count => 0)
    end

    summaries
  end

  def self.generate_for_select_questionings(questionings)
    return [] if questionings.empty?

    qing_ids = questionings.map(&:id).join(',')
    qings_by_id = questionings.index_by(&:id)

    # build and run queries for select_one and _multiple
    query = <<-eos
      SELECT qings.id AS qing_id, a.option_id AS option_id, COUNT(a.id) AS answer_count 
      FROM questionings qings 
        INNER JOIN questions q ON qings.question_id = q.id 
        LEFT OUTER JOIN answers a ON qings.id = a.questioning_id
        WHERE q.qtype_name IN ('select_one', 'select_multiple')
          AND qings.id IN (#{qing_ids})
        GROUP BY qings.id, a.option_id
    eos
    sel_one_res = ActiveRecord::Base.connection.execute(query)
    query = <<-eos
      SELECT qings.id AS qing_id, c.option_id AS option_id, COUNT(c.id) AS choice_count 
      FROM questionings qings 
        INNER JOIN questions q ON qings.question_id = q.id 
        LEFT OUTER JOIN answers a ON qings.id = a.questioning_id
        LEFT OUTER JOIN choices c ON a.id = c.answer_id
        WHERE q.qtype_name = 'select_multiple'
          AND qings.id IN (#{qing_ids})
        GROUP BY qings.id, c.option_id
    eos
    sel_mult_res = ActiveRecord::Base.connection.execute(query)

    # read tallies into hashes
    tallies = {}
    sel_one_res.each(:as => :hash).map do |row|
      tallies[[row['qing_id'], row['option_id']]] = row['answer_count']
    end
    sel_mult_res.each(:as => :hash).map do |row|
      tallies[[row['qing_id'], row['option_id']]] = row['choice_count']
    end

    # loop over each questioning and generate summary
    questionings.map do |qing|

      # build headers
      headers = qing.options.map{|option| {:name => option.name, :option => option}}

      # build tallies, keeping a running sum
      non_null_count = 0
      items = qing.options.map do |o|
        count = tallies[[qing.id, o.id]] || 0
        non_null_count += count
        Report::SummaryItem.new(:count => count)
      end

      # compute percentages
      items.each do |item|
        item.pct = non_null_count == 0 ? 0 : item.count.to_f / non_null_count * 100
      end

      # null count
      null_count = tallies[[qing.id, nil]]

      new(:questioning => qing, :display_type => :structured, :overall_header => qing.option_set.name, 
        :headers => headers, :items => items, :null_count => null_count)
    end
  end

  def initialize(attribs)
    # save attribs
    attribs.each{|k,v| instance_variable_set("@#{k}", v)}

    qtype_name = questioning.qtype_name
    case qtype_name

    # these types all get descriptive statistics
    when nil
{'fraction_non_alphanumeric': 0.12448795389849338, 'fraction_numerical': 0.003888078872457127, 'mean_word_length': 2.8186638388123013, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 84, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6067569', 'n_tokens_mistral': 4993, 'n_tokens_neox': 4733, 'n_words': 1216}