<commit_before># Tasks for importing data
require "digilys/importer"

namespace :app do
  namespace :import do

    task evaluation_templates_from_csv: :environment do
      title = true

      data = []
      CSV.foreach(ENV["file"]) do |row|
        # Skip first row
        if title
          title = false
          next
        end

        max_result = row[3].to_i

        red, yellow, green = parse_color_intervals(row[4], row[5], row[6], max_result)

        attributes = {
          name:          row[0].try(:gsub, "\u00A0", " "),
          description:   row[1].try(:gsub, "\u00A0", " "),
          category_list: row[2].try(:gsub, "\u00A0", " "),
          max_result:    max_result,
          red:           red,
          yellow:        yellow,
          green:         green,
          stanine1:      parse_interval(row[7],  max_result),
          stanine2:      parse_interval(row[8],  max_result),
          stanine3:      parse_interval(row[9],  max_result),
          stanine4:      parse_interval(row[10], max_result),
          stanine5:      parse_interval(row[11], max_result),
          stanine6:      parse_interval(row[12], max_result),
          stanine7:      parse_interval(row[13], max_result),
          stanine8:      parse_interval(row[14], max_result),
          stanine9:      parse_interval(row[15], max_result)
        }

        data << {
          original_row: row,
          attributes:   attributes
        }
      end

      valid   = []
      invalid = []

      data.each do |d|
        attributes = d[:attributes]
        evaluation = Evaluation.new

        evaluation.imported      = true
        evaluation.value_type    = :numeric
        evaluation.target        = :all
        evaluation.type          = :template

        evaluation.name          = attributes[:name]
        evaluation.description   = attributes[:description]
        evaluation.category_list = attributes[:category_list]
        evaluation.max_result    = attributes[:max_result]

        evaluation.red_min       = attributes[:red].try(:min)
        evaluation.red_max       = attributes[:red].try(:max)
        evaluation.yellow_min    = attributes[:yellow].try(:min)
        evaluation.yellow_max    = attributes[:yellow].try(:max)
        evaluation.green_min     = attributes[:green].try(:min)
        evaluation.green_max     = attributes[:green].try(:max)

        evaluation.stanine1_min  = attributes[:stanine1].try(:min)
        evaluation.stanine1_max  = attributes[:stanine1].try(:max)
        evaluation.stanine2_min  = attributes[:stanine2].try(:min)
        evaluation.stanine2_max  = attributes[:stanine2].try(:max)
        evaluation.stanine3_min  = attributes[:stanine3].try(:min)
        evaluation.stanine3_max  = attributes[:stanine3].try(:max)
        evaluation.stanine4_min  = attributes[:stanine4].try(:min)
        evaluation.stanine4_max  = attributes[:stanine4].try(:max)
        evaluation.stanine5_min  = attributes[:stanine5].try(:min)
        evaluation.stanine5_max  = attributes[:stanine5].try(:max)
        evaluation.stanine6_min  = attributes[:stanine6].try(:min)
        evaluation.stanine6_max  = attributes[:stanine6].try(:max)
        evaluation.stanine7_min  = attributes[:stanine7].try(:min)
        evaluation.stanine7_max  = attributes[:stanine7].try(:max)
        evaluation.stanine8_min  = attributes[:stanine8].try(:min)
        evaluation.stanine8_max  = attributes[:stanine8].try(:max)
        evaluation.stanine9_min  = attributes[:stanine9].try(:min)
        evaluation.stanine9_max  = attributes[:stanine9].try(:max)

        evaluation.instance_id   = ENV["instance_id"]

        if evaluation.valid?
          valid << evaluation
        else
          invalid << d.merge(model: evaluation)
        end
      end

      if !invalid.blank?
        puts "Invalid rows:\n"
        invalid.each { |d| puts d[:original_row].to_json + "\n" + d[:model].errors.to_json + "\n\n" }
      end

      puts "\nEnter 'yes' to import #{valid.length} valid rows, or 'no' to abort:"

      if get_input()
        puts "\nImporting..."

        valid.collect(&:save!)

        puts "\nNew ids:"
        puts valid.collect(&:id).join(",")
      else
        puts "\nAborting..."
      end
    end

    def parse_color_intervals(red_str, yellow_str, green_str, max_result)
      red    = parse_interval(red_str,    max_result)
      yellow = parse_interval(yellow_str, max_result)
      green  = parse_interval(green_str,  max_result)
<commit_msg>Added updating functionality to evaluation template import task<commit_after># Tasks for importing data
require "digilys/importer"

namespace :app do
  namespace :import do

    task evaluation_templates_from_csv: :environment do
      title = true

      data = []
      CSV.foreach(ENV["file"]) do |row|
        # Skip first row
        if title
          title = false
          next
        end

        max_result = row[3].to_i

        red, yellow, green = parse_color_intervals(row[4], row[5], row[6], max_result)

        attributes = {
          name:          row[0].try(:gsub, "\u00A0", " "),
          description:   row[1].try(:gsub, "\u00A0", " "),
          category_list: row[2].try(:gsub, "\u00A0", " "),
          max_result:    max_result,
          red:           red,
          yellow:        yellow,
          green:         green,
          stanine1:      parse_interval(row[7],  max_result),
          stanine2:      parse_interval(row[8],  max_result),
          stanine3:      parse_interval(row[9],  max_result),
          stanine4:      parse_interval(row[10], max_result),
          stanine5:      parse_interval(row[11], max_result),
          stanine6:      parse_interval(row[12], max_result),
          stanine7:      parse_interval(row[13], max_result),
          stanine8:      parse_interval(row[14], max_result),
          stanine9:      parse_interval(row[15], max_result)
        }

        data << {
          original_row: row,
          attributes:   attributes
        }
      end

      valid       = []
      invalid     = []
      instance_id = ENV["instance_id"]
      update      = ENV["update"]
      updated     = 0

      data.each do |d|
        attributes = d[:attributes]

        evaluation = Evaluation.where(
          imported:    true,
          instance_id: instance_id,
          name:        attributes[:name],
          description: attributes[:description]
        ).first if update

        evaluation ||= Evaluation.new

        evaluation.imported      = true
        evaluation.value_type    = :numeric
        evaluation.target        = :all
        evaluation.type          = :template

        evaluation.name          = attributes[:name]
        evaluation.description   = attributes[:description]
        evaluation.category_list = attributes[:category_list]
        evaluation.max_result    = attributes[:max_result]

        evaluation.red_min       = attributes[:red].try(:min)
        evaluation.red_max       = attributes[:red].try(:max)
        evaluation.yellow_min    = attributes[:yellow].try(:min)
        evaluation.yellow_max    = attributes[:yellow].try(:max)
        evaluation.green_min     = attributes[:green].try(:min)
        evaluation.green_max     = attributes[:green].try(:max)

        evaluation.stanine1_min  = attributes[:stanine1].try(:min)
        evaluation.stanine1_max  = attributes[:stanine1].try(:max)
        evaluation.stanine2_min  = attributes[:stanine2].try(:min)
        evaluation.stanine2_max  = attributes[:stanine2].try(:max)
        evaluation.stanine3_min  = attributes[:stanine3].try(:min)
        evaluation.stanine3_max  = attributes[:stanine3].try(:max)
        evaluation.stanine4_min  = attributes[:stanine4].try(:min)
        evaluation.stanine4_max  = attributes[:stanine4].try(:max)
        evaluation.stanine5_min  = attributes[:stanine5].try(:min)
        evaluation.stanine5_max  = attributes[:stanine5].try(:max)
        evaluation.stanine6_min  = attributes[:stanine6].try(:min)
        evaluation.stanine6_max  = attributes[:stanine6].try(:max)
        evaluation.stanine7_min  = attributes[:stanine7].try(:min)
        evaluation.stanine7_max  = attributes[:stanine7].try(:max)
        evaluation.stanine8_min  = attributes[:stanine8].try(:min)
        evaluation.stanine8_max  = attributes[:stanine8].try(:max)
        evaluation.stanine9_min  = attributes[:stanine9].try(:min)
        evaluation.stanine9_max  = attributes[:stanine9].try(:max)

        evaluation.instance_id   = instance_id

        if evaluation.valid?
          valid << evaluation
        else
          invalid << d.merge(model: evaluation)
        end

        updated += 1 unless evaluation.new_record?
      end

      if !invalid.blank?
        puts "Invalid rows:\n"
        invalid.each { |d| puts d[:original_row].to_json + "\n" + d[:model].errors.to_json + "\n\n" }
      end

      puts "\nTotal: #{valid.length + invalid.length}, to be updated: #{updated}" if update

      puts "\nEnter 'yes' to import #{valid.length} valid rows, or 'no' to abort:"

      if get_input()
        puts "\nImporting..."
        valid.collect(&:save!)
      else
        puts "\nAborting..."
      end
    end

    def parse_color_intervals(red_str, yellow_str, green_str, max_result)
      red    = parse_interval(red_str,    max_result)
      yellow = parse_interval(yellow_str, max_result)
      green  = parse_interval(green_str,  max_result)

{'fraction_non_alphanumeric': 0.11817604739737622, 'fraction_numerical': 0.01629284807448159, 'mean_word_length': 2.289144050104384, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5247723', 'n_tokens_mistral': 2836, 'n_tokens_neox': 2838, 'n_words': 497}