<commit_before>class Profile::SubProcessor

  # need to serialize

  def query(student, batch_size, offset)
    all = Profile::Query.new.query(student, batch_size, offset)
    is_last_page = (all.count < batch_size)
    by_unit = group_by_unit(all)
    by_unit_by_state = group_by_state_within_unit(by_unit)
    sorted = sort_sessions(by_unit_by_state)
    return [sorted, is_last_page]
  end



  private

  def group_by_unit(all)
    all.group_by{|s| s.classroom_activity.unit.name}
  end

  def group_by_state_within_unit(by_unit)
    result = by_unit.reduce({}) do |acc, (k, v)|
      acc[k] = group_by_state(v)
      acc
    end
    result<commit_msg>filter out classroom_activities with no unit from the student profile<commit_after>class Profile::SubProcessor

  # need to serialize

  def query(student, batch_size, offset)
    all = Profile::Query.new.query(student, batch_size, offset)
    is_last_page = (all.count < batch_size)
    by_unit = group_by_unit(all)
    by_unit_by_state = group_by_state_within_unit(by_unit)
    sorted = sort_sessions(by_unit_by_state)
    return [sorted, is_last_page]
  end



  private

  def group_by_unit(all)
    all.select{|s| s.classroom_activity.unit.present?}
       .group_by{|s| s.classroom_activity.unit.name}
  end

  def group_by_state_within_unit(by_unit)
    result = by_unit.reduce({}) do |acc, (k, v)|
      acc[k] = group_by_state(v)
      acc
    end
    result
{'fraction_non_alphanumeric': 0.09768095572733661, 'fraction_numerical': 0.0, 'mean_word_length': 3.893470790378007, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4536688', 'n_tokens_mistral': 533, 'n_tokens_neox': 513, 'n_words': 100}