<commit_before># TODO: refactor as AssignmentEvent type Aggregate::Count
class Analytics::AssignmentEvent
  include Analytics::Aggregate

  field :assignment_id, type: Integer
  field :events, type: Hash

  # assignment_id: 1,
  # events: {
  #   "_all": {
  #     all_time: %,
  #     yearly: {
  #       key: %
  #     },
  #     monthly: {
  #       key: %
  #     },
  #     weekly: {
  #       key: %
  #     },
  #     daily: {
  #       key: %
  #     },
  #     hourly: {
  #       key: %
  #     },
  #     minutely: {
  #       key: %
  #     }
  #   },
  #   "predictor": {
  #     all_time: %,
  #     yearly: {
  #       key: %
  #     },
  #     monthly: {
  #       key: %
  #     },
  #     weekly: {
  #       key: %
  #     },
  #     daily: {
  #       key: %
  #     },
  #     hourly: {
  #       key: %
  #     },
  #     minutely: {
  #       key: %
  #     }
  #   }
  # }

  def self.aggregate_scope(event)
    self.where(assignment_id: event.data['assignment_id'])
  end

  def self.upsert_hash(event)
    upsert_hash = Hash.new.tap do |hash|
      GRANULARITIES.each do |granularity, interval|
        event_key = ["events", event.event_type]
        granular_key = [granularity, self.time_key(event.created_at, interval)].compact

        hash[ (event_key + granular_key).join('.') ] = 1
        hash[ (["events", "_all"] + granular_key).join('.') ] = 1
      end
    end
  end

  def self.data(granularity, from, to, assignments, event="_all")
    interval = GRANULARITIES[granularity]
    start_at = self.time_key(from, interval)
    end_at = to.to_i
    range = (start_at..end_at).step(interval)

    keys = range.map { |i| :"events.#{event}.#{granularity}.#{i}"}

    all_data = self.
      in(assignment_id: assignments.keys).
      where('$or' => keys.map{ |k| {k => { '$exists' => true}} }).
      only("assignment_id", *keys).to_a

    events = all_data.collect { |d| d.events.keys }.flatten.uniq

    return {
      range: range,
      events: events,
      assignments: assignments,
      data: all_data
    }
  end
end<commit_msg>Refactored formatted data returned from analytics aggregate.<commit_after># TODO: refactor as AssignmentEvent type Aggregate::Count
class Analytics::AssignmentEvent
  include Analytics::Aggregate

  field :assignment_id, type: Integer
  field :events, type: Hash

  # assignment_id: 1,
  # events: {
  #   "_all": {
  #     all_time: %,
  #     yearly: {
  #       key: %
  #     },
  #     monthly: {
  #       key: %
  #     },
  #     weekly: {
  #       key: %
  #     },
  #     daily: {
  #       key: %
  #     },
  #     hourly: {
  #       key: %
  #     },
  #     minutely: {
  #       key: %
  #     }
  #   },
  #   "predictor": {
  #     all_time: %,
  #     yearly: {
  #       key: %
  #     },
  #     monthly: {
  #       key: %
  #     },
  #     weekly: {
  #       key: %
  #     },
  #     daily: {
  #       key: %
  #     },
  #     hourly: {
  #       key: %
  #     },
  #     minutely: {
  #       key: %
  #     }
  #   }
  # }

  def self.aggregate_scope(event)
    self.where(assignment_id: event.data['assignment_id'])
  end

  def self.upsert_hash(event)
    upsert_hash = Hash.new.tap do |hash|
      GRANULARITIES.each do |granularity, interval|
        event_key = ["events", event.event_type]
        granular_key = [granularity, self.time_key(event.created_at, interval)].compact

        hash[ (event_key + granular_key).join('.') ] = 1
        hash[ (["events", "_all"] + granular_key).join('.') ] = 1
      end
    end
  end

  def self.data(granularity, from, to, assignments, event="_all")
    interval = GRANULARITIES[granularity]
    start_at = self.time_key(from, interval)
    end_at = to.to_i
    range = (start_at..end_at).step(interval)

    keys = range.map { |i| :"events.#{event}.#{granularity}.#{i}"}

    count_data = self.
      in(assignment_id: assignments.keys).
      where('$or' => keys.map{ |k| {k => { '$exists' => true}} }).
      only("assignment_id", *keys).to_a

    count_data.each { |d| d[:name] = assignments[d.assignment_id] }

    return {
      range: range,
      key: "events.#{event}.#{granularity}",
      data: count_data
    }
  end
end
{'fraction_non_alphanumeric': 0.16136743963662442, 'fraction_numerical': 0.0014343772412144394, 'mean_word_length': 1.9238294898672257, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5701242', 'n_tokens_mistral': 1581, 'n_tokens_neox': 1555, 'n_words': 264}