<commit_before># This is for manually importing a large spreadsheet, with historical data
# about students across a range of reading assessments.
# It outputs JSON to the console.
#
# Usage:
# file_text = <<EOD
# ...
# EOD
# educator = Educator.find_by_login_name('...')
# output = MegaReadingProcessor.new(educator.id).import(file_text);nil
class MegaReadingProcessor
  def initialize(educator_id, options = {})
    @educator_id = educator_id
    @log = options.fetch(:log, Rails.env.test? ? LogHelper::Redirect.instance.file : STDOUT)
    @matcher = options.fetch(:matcher, ImportMatcher.new)
    @header_rows_count = options.fetch(:header_rows_count, 1)

    @fuzzy_student_matcher = FuzzyStudentMatcher.new
    reset_counters!
  end

  def process(file_text, options = {})
    reset_counters!

    # parse
    rows = []
    #last row before data is the header
    string = (@header_rows_count > 1) ? file_text.lines[@header_rows_count-1..-1].join : file_text
    StreamingCsvTransformer.from_text(@log, string).each_with_index do |row, index|
      flattened_rows = flat_map_rows(row, index)
      next if flattened_rows.nil?

      rows += flattened_rows
      @matcher.count_valid_row
    end
    log "matcher#stats: #{@matcher.stats}"
    log "MegaReadingProcessor#stats: #{stats}"

    # deprecated, use ReadingBenchmarkData model for F&P instead
    # write to database for F&P only
    # f_and_ps = nil
    # FAndPAssessment.transaction do
    #   FAndPAssessment.where(benchmark_date: @benchmark_date).destroy_all
    #   f_and_ps = rows.map {|row| FAndPAssessment.create!(row) }
    # end
    # f_and_ps
    [rows, stats]
  end

  private
  def reset_counters!
    @missing_data_point = 0
    @missing_data_point_because_student_moved_school = 0
    @invalid_student_name_count = 0
    @invalid_student_names_list = []
    @valid_data_points = 0
    @valid_student_name = 0
  end

  def stats
    {
      invalid_student_name_count: @invalid_student_name_count,
      invalid_student_names_list_size: @invalid_student_names_list.size,
      valid_student_name: @valid_student_name,
      missing_data_point: @missing_data_point,
      missing_data_point_because_student_moved_school: @missing_data_point_because_student_moved_school,
      valid_data_points: @valid_data_points
    }
  end

  def flat_map_rows(row, index)
    # match student
    student_id, student_match_failure = match_student(row)
    if student_id.nil?
      @invalid_student_name_count += 1
      @invalid_student_names_list << student_match_failure
      return nil
    end
    @valid_student_name += 1

    shared = {
      student_id: student_id,
      imported_by_educator_id: @educator_id
    }

    # data points for each grade
    (
      data_points_for_kindergarten(shared, row) +
      data_points_for_first(shared, row) +
      data_points_for_second(shared, row) +
      data_points_for_third(shared, row)
    )
  end

  # returns [student_id, debug_match_failure_text]
  def match_student(row)
    # exactprimary key
    student = if row.has_key?('LASID')
      Student.find_by_local_id(row['LASID'])
    elsif row.has_key?('local_id')
      Student.find_by_local_id(row['local_id'])
    end
    return [student.id, nil] if student.present?

    # fuzzy
    last_first_name = row["First Name "] + " " + row['Last Name']

    fuzzy_match = @fuzzy_student_matcher.match_from_full_name(last_first_name)
    return [fuzzy_match[:student_id], nil] if fuzzy_match.present?

    # no match
    return [nil, last_first_name]
  end

  def data_points_for_kindergarten(shared, row)
    import_data_points(shared, row, [
      ['KF', :fall, :dibels_fsf, 'fall K - FSF'],
      ['KF', :fall, :dibels_lnf, 'fall K - LNF'],
      ['KF', :winter, :dibels_fsf, 'winter K - FSF'],
      ['KF', :winter, :dibels_lnf, 'winter K - LNF'],
      ['KF', :winter, :dibels_psf, 'winter K - PSF'],
      ['KF', :winter, :dibels_nwf_cls, 'winter K - NWF CLS'],
      ['KF', :winter, :dibels_nwf_growth, 'LNF Growth'],
    ])
  end

  def data_points_for_first(shared, row)
    import_data_points(shared, row, [
      ['1', :fall, :dibels_lnf, '1 / FALL / LNF'],
      ['1', :fall, :dibels_psf, '1 / FALL / PSF'],
      ['1', :fall, :dibels_nwf_cls, '1 / FALL / NWF CLS'],
      ['1', :fall, :dibels_nwf_wwr, '1 / FALL / NWF WWR'],
      ['1', :fall, :f_and_p_english, '1 / FALL / F&P Level English'],
      ['1', :fall, :f_and_p_spanish, '1 / FALL / F&P Level Spanish'],
      ['1', :fall, :instructional_needs, '1 / FALL / Instructional needs'],
      ['1', :winter, :dibels_nwf_cls, '1 / WINTER / NWF CLS'],
      ['1', :winter, :dibels_nwf_wwr, '1 / WINTER / NWF WWR'],
      ['1', :winter, :dibels_dorf_wpm, '1 / WINTER / DORF WPM'],
      ['1', :winter, :dibels_dorf_errors, '1 / WINTER / DORF Errors'],<commit_msg>revising for new sheet<commit_after># This is for manually importing a large spreadsheet, with historical data
# about students across a range of reading assessments.
# It outputs JSON to the console.
#
# Usage:
# file_text = <<EOD
# ...
# EOD
# educator = Educator.find_by_login_name('...')
# output = MegaReadingProcessor.new(educator.id).import(file_text);nil
class MegaReadingProcessor
  def initialize(educator_id, options = {})
    @educator_id = educator_id
    @log = options.fetch(:log, Rails.env.test? ? LogHelper::Redirect.instance.file : STDOUT)
    @matcher = options.fetch(:matcher, ImportMatcher.new)
    @header_rows_count = options.fetch(:header_rows_count, 1)

    @fuzzy_student_matcher = FuzzyStudentMatcher.new
    reset_counters!
  end

  def process(file_text, options = {})
    reset_counters!

    # parse
    rows = []
    #last row before data is the header
    string = (@header_rows_count > 1) ? file_text.lines[@header_rows_count-1..-1].join : file_text
    StreamingCsvTransformer.from_text(@log, string).each_with_index do |row, index|
      flattened_rows = flat_map_rows(row, index)
      next if flattened_rows.nil?

      rows += flattened_rows
      @matcher.count_valid_row
    end
    log "matcher#stats: #{@matcher.stats}"
    log "MegaReadingProcessor#stats: #{stats}"

    # deprecated, use ReadingBenchmarkData model for F&P instead
    # write to database for F&P only
    # f_and_ps = nil
    # FAndPAssessment.transaction do
    #   FAndPAssessment.where(benchmark_date: @benchmark_date).destroy_all
    #   f_and_ps = rows.map {|row| FAndPAssessment.create!(row) }
    # end
    # f_and_ps
    [rows, stats]
  end

  private
  def reset_counters!
    @missing_data_point = 0
    @missing_data_point_because_student_moved_school = 0
    @invalid_student_name_count = 0
    @invalid_student_names_list = []
    @valid_data_points = 0
    @valid_student_name = 0
  end

  def stats
    {
      invalid_student_name_count: @invalid_student_name_count,
      invalid_student_names_list_size: @invalid_student_names_list.size,
      valid_student_name: @valid_student_name,
      missing_data_point: @missing_data_point,
      missing_data_point_because_student_moved_school: @missing_data_point_because_student_moved_school,
      valid_data_points: @valid_data_points
    }
  end

  def flat_map_rows(row, index)
    # match student
    student_id, student_match_failure = match_student(row)
    if student_id.nil?
      @invalid_student_name_count += 1
      @invalid_student_names_list << student_match_failure
      return nil
    end
    @valid_student_name += 1

    shared = {
      student_id: student_id,
      imported_by_educator_id: @educator_id
    }

    # data points for each grade
    (
      data_points_for_kindergarten(shared, row) +
      data_points_for_first(shared, row) +
      data_points_for_second(shared, row) +
      data_points_for_third(shared, row)
    )
  end

  # returns [student_id, debug_match_failure_text]
  def match_student(row)
    # exactprimary key
    student = if row.has_key?('LASID')
      Student.find_by_local_id(row['LASID'])
    elsif row.has_key?('local_id')
      Student.find_by_local_id(row['local_id'])
    end
    return [student.id, nil] if student.present?

    # fuzzy
    last_first_name = row["First Name "] + " " + row['Last Name']

    fuzzy_match = @fuzzy_student_matcher.match_from_full_name(last_first_name)
    return [fuzzy_match[:student_id], nil] if fuzzy_match.present?

    # no match
    return [nil, last_first_name]
  end

  def data_points_for_kindergarten(shared, row)
    import_data_points(shared, row, [
      ['KF', :fall, :dibels_fsf, 'K / FALL / FSF'],
      ['KF', :fall, :dibels_lnf, 'K / FALL / LNF'],
      ['KF', :fall, :instructional_needs, 'K / FALL / Instructional needs'],
      ['KF', :winter, :dibels_fsf, 'K / WINTER / FSF'],
      ['KF', :winter, :dibels_lnf, 'K / WINTER / LNF'],
      ['KF', :winter, :dibels_psf, 'K / WINTER / PSF'],
      ['KF', :winter, :instructional_needs, '1 / WINTER / Instructional needs']
      ['KF', :winter, :f_and_p_english, '1 / WINTER / F&P Level English'],
      ['KF', :winter, :f_and_p_spanish, '1 / WINTER / F&P Level Spanish'],
      ['KF', :spring, :dibels_lnf, 'K / SPRING / LNF'],
      ['KF', :spring, :dibels_psf, 'K / SPRING / PSF'],
      ['KF', :spring, :dibels_nwf_cls, 'K / SPRING / NWF CLS'],
      ['KF', :spring, :dibels_nwf_wwr, 'K / SPRING / NWF WWR'],
      ['KF', :spring, :instructional_needs, '1 / SPRING / Instructional needs']
      ['KF', :spring, :f_and_p_english, '1 / SPRING / F&P Level English'],
      ['KF', :spring, :f_and_p_spanish, '1 / SPRING / F&P Level Spanish'],
      ['KF', :spring, :instructional_needs, '1 / WINTER / LAS Links Speaking']
      ['KF', :spring, :f_and_p_english, '1 / WINTER / LAS Links Listening']
    ])
  end

  def data_points_for_first(shared, row)
    import_data_points(shared, row, [
      ['1', :fall, :dibels_lnf, '1 / FALL / LNF'],
      ['1', :fall, :dibels_psf, '1 / FALL / PSF'],
      ['1', :fall, :dibels_nwf_cls, '1 / FALL / NWF CLS'],
      ['1', :fall, :dibels_nwf_wwr, '1 / FALL / NWF WWR'],
      ['1', :fall, :f_and_p_english, '1 / FALL / F&P Level English'],
      ['1', :fall, :f_and_p_spanish, '1 / FALL / F&P Level Spanish'],
      ['1', :fall, :instructional_needs, '1 / FALL / Instructional needs'],
      ['1', :winter, :dibels_nwf_cls, '1 / WINTER / NWF CLS'],
      ['1', :winter, :dibels_nwf_wwr, '1 / WINTER / NWF WWR'],
      ['1', :winter, :dibels_dorf_wpm, '1 / WINTER / DORF WPM'],
      ['1', :winter, :dibels_dorf_errors, '1 / WINTER / DORF Errors'],
{'fraction_non_alphanumeric': 0.12905703860188208, 'fraction_numerical': 0.0071058190896869594, 'mean_word_length': 3.479569892473118, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2359666', 'n_tokens_mistral': 3938, 'n_tokens_neox': 3800, 'n_words': 805}