<commit_before>require_dependency 'importers'

module Importers
  class CourseContentImporter < Importer

    self.item_class = Course
    Importers.register_content_importer(self)

    def self.process_migration_files(course, data, migration)
      data['all_files_export'] ||= {}
      data['all_files_export']['file_path'] ||= data['all_files_zip']
      return unless data['all_files_export']['file_path'] && File.exist?(data['all_files_export']['file_path'])

      migration.attachment_path_id_lookup ||= {}
      migration.attachment_path_id_lookup_lower ||= {}

      params = migration.migration_settings[:migration_ids_to_import]
      valid_paths = []
      (data['file_map'] || {}).each do |id, file|
        path = file['path_name'].starts_with?('/') ? file['path_name'][1..-1] : file['path_name']
        migration.add_attachment_path(path, file['migration_id'])
        if migration.import_object?("attachments", file['migration_id']) || migration.import_object?("files", file['migration_id'])
          if file['errored']
            migration.add_warning(t(:file_import_warning, "File %{file} could not be found", :file => File.basename(file['path_name'])))
          else
            valid_paths << path
          end
        end
      end
      valid_paths = [0] if valid_paths.empty? && params[:copy] && params[:copy][:files]
      logger.debug "adding #{valid_paths.length} files"
      total = valid_paths.length
      if valid_paths != [0]
        current = 0
        last = current
        callback = Proc.new do
          current += 1
          if (current - last) > 10
            last = current
            migration.update_import_progress((current.to_f/total) * 18.0)
          end
        end
        unzip_opts = {
            :course => migration.context,
            :filename => data['all_files_export']['file_path'],
            :valid_paths => valid_paths,
            :callback => callback,
            :logger => logger,
            :rename_files => migration.migration_settings[:files_import_allow_rename],
            :migration_id_map => migration.attachment_path_id_lookup,
        }
        if root_path = migration.migration_settings[:files_import_root_path]
          unzip_opts[:root_directory] = Folder.assert_path(
              root_path, migration.context)
        end
        unzipper = UnzipAttachment.new(unzip_opts)
        migration.update_import_progress(1.0)
        unzipper.process
      end
    end

    def self.import_media_objects(mo_attachments, migration)
      wait_for_completion = migration && migration.canvas_import?
      unless mo_attachments.blank?
        MediaObject.add_media_files(mo_attachments, wait_for_completion)
      end
    end

    def self.import_content(course, data, params, migration)
      params ||= {:copy=>{}}
      logger.debug "starting import"

      Importers::ContentImporterHelper.add_assessment_id_prepend(course, data, migration)

      course.full_migration_hash = data
      course.external_url_hash = {}
      course.migration_results = []

      migration.check_cross_institution
      logger.debug "migration is cross-institution; external references will not be used" if migration.cross_institution?

      (data['web_link_categories'] || []).map{|c| c['links'] }.flatten.each do |link|
        course.external_url_hash[link['link_id']] = link
      end
      ActiveRecord::Base.skip_touch_context

      if !migration.for_course_copy?
        Importers::ContextModuleImporter.select_all_linked_module_items(data, migration)
        Importers::GradingStandardImporter.select_course_grading_standard(data, migration)
        # These only need to be processed once
        Attachment.skip_media_object_creation do
          self.process_migration_files(course, data, migration); migration.update_import_progress(18)
          Importers::AttachmentImporter.process_migration(data, migration); migration.update_import_progress(20)
          mo_attachments = migration.imported_migration_items_by_class(Attachment).find_all { |i| i.media_entry_id.present? }
          begin
            self.import_media_objects(mo_attachments, migration)
          rescue => e
            er = Canvas::Errors.capture_exception(:import_media_objects, e)[:error_report]
            error_message = t('Failed to import media objects')
            migration.add_error(error_message, error_report_id: er)
          end
        end
        if migration.canvas_import?
          migration.update_import_progress(30)
          Importers::MediaTrackImporter.process_migration(data[:media_tracks], migration)
        end
      end

      migration.update_import_progress(35)
      question_data = Importers::AssessmentQuestionImporter.process_migration(data, migration); migration.update_import_progress(45)
      Importers::GroupImporter.process_migration(data, migration); migration.update_import_progress(48)
      Importers::LearningOutcomeImporter.process_migration(data, migration); migration.update_import_progress(50)
      Importers::RubricImporter.process_migration(data, migration); migration.update_import_progress(52)
      course.assignment_group_no_drop_assignments = {}
      Importers::AssignmentGroupImporter.process_migration(data, migration); migration.update_import_progress(54)
      Importers::ExternalFeedImporter.process_migration(data, migration); migration.update_import_progress(56)
      Importers::GradingStandardImporter.process_migration(data, migration); migration.update_import_progress(58)
      Importers::ContextExternalToolImporter.process_migration(data, migration); migration.update_import_progress(60)
      Importers::QuizImporter.process_migration(data, migration, question_data); migration.update_import_progress(65)
      Importers::DiscussionTopicImporter.process_migration(data, migration); migration.update_import_progress(70)
      Importers::WikiPageImporter.process_migration(data, migration); migration.update_import_progress(75)
      Importers::AssignmentImporter.process_migration(data, migration); migration.update_import_progress(80)
      Importers::ContextModuleImporter.process_migration(data, migration); migration.update_import_progress(85)
      Importers::WikiPageImporter.process_migration_course_outline(data, migration)
      Importers::CalendarEventImporter.process_migration(data, migration)

      everything_selected = !migration.copy_options || migration.is_set?(migration.copy_options[:everything])
      if everything_selected || migration.is_set?(migration.copy_options[:all_course_settings])
        self.import_settings_from_migration(course, data, migration)
      end
      migration.update_import_progress(90)

      # be very explicit about draft state courses, but be liberal toward legacy courses
      if course.wiki.has_no_front_page
        if migration.for_course_copy? && (source = migration.source_course || Course.where(id: migration.migration_settings[:source_course_id]).first)
          mig_id = CC::CCHelper.create_key(source.wiki.front_page)
          if new_front_page = course.wiki.wiki_pages.where(migration_id: mig_id).first
            course.wiki.set_front_page_url!(new_front_page.url)
          end
        end
      end
      front_page = course.wiki.front_page
      course.wiki.unset_front_page! if front_page.nil? || front_page.new_record?

      syllabus_should_be_added = everything_selected || migration.copy_options[:syllabus_body] || migration.copy_options[:all_syllabus_body]
      if syllabus_should_be_added
        syllabus_body = data[:course][:syllabus_body] if data[:course]
        self.import_syllabus_from_migration(course, syllabus_body, migration) if syllabus_body
      end

      migration.resolve_content_links!
      migration.update_import_progress(95)

      if data['external_content']
        Canvas::Migration::ExternalContent::Migrator.send_imported_content(migration, data['external_content'])
      end

      begin
        #Adjust dates
        if shift_options = migration.date_shift_options
          shift_options = self.shift_date_options(course, shift_options)

          migration.imported_migration_items_by_class(Assignment).each do |event|
            event.reload # just in case
            event.due_at = shift_date(event.due_at, shift_options)
            event.lock_at = shift_date(event.lock_at, shift_options)
            event.unlock_at = shift_date(event.unlock_at, shift_options)<commit_msg>save course settings changes early on import<commit_after>require_dependency 'importers'

module Importers
  class CourseContentImporter < Importer

    self.item_class = Course
    Importers.register_content_importer(self)

    def self.process_migration_files(course, data, migration)
      data['all_files_export'] ||= {}
      data['all_files_export']['file_path'] ||= data['all_files_zip']
      return unless data['all_files_export']['file_path'] && File.exist?(data['all_files_export']['file_path'])

      migration.attachment_path_id_lookup ||= {}
      migration.attachment_path_id_lookup_lower ||= {}

      params = migration.migration_settings[:migration_ids_to_import]
      valid_paths = []
      (data['file_map'] || {}).each do |id, file|
        path = file['path_name'].starts_with?('/') ? file['path_name'][1..-1] : file['path_name']
        migration.add_attachment_path(path, file['migration_id'])
        if migration.import_object?("attachments", file['migration_id']) || migration.import_object?("files", file['migration_id'])
          if file['errored']
            migration.add_warning(t(:file_import_warning, "File %{file} could not be found", :file => File.basename(file['path_name'])))
          else
            valid_paths << path
          end
        end
      end
      valid_paths = [0] if valid_paths.empty? && params[:copy] && params[:copy][:files]
      logger.debug "adding #{valid_paths.length} files"
      total = valid_paths.length
      if valid_paths != [0]
        current = 0
        last = current
        callback = Proc.new do
          current += 1
          if (current - last) > 10
            last = current
            migration.update_import_progress((current.to_f/total) * 18.0)
          end
        end
        unzip_opts = {
            :course => migration.context,
            :filename => data['all_files_export']['file_path'],
            :valid_paths => valid_paths,
            :callback => callback,
            :logger => logger,
            :rename_files => migration.migration_settings[:files_import_allow_rename],
            :migration_id_map => migration.attachment_path_id_lookup,
        }
        if root_path = migration.migration_settings[:files_import_root_path]
          unzip_opts[:root_directory] = Folder.assert_path(
              root_path, migration.context)
        end
        unzipper = UnzipAttachment.new(unzip_opts)
        migration.update_import_progress(1.0)
        unzipper.process
      end
    end

    def self.import_media_objects(mo_attachments, migration)
      wait_for_completion = migration && migration.canvas_import?
      unless mo_attachments.blank?
        MediaObject.add_media_files(mo_attachments, wait_for_completion)
      end
    end

    def self.import_content(course, data, params, migration)
      params ||= {:copy=>{}}
      logger.debug "starting import"

      Importers::ContentImporterHelper.add_assessment_id_prepend(course, data, migration)

      course.full_migration_hash = data
      course.external_url_hash = {}
      course.migration_results = []

      migration.check_cross_institution
      logger.debug "migration is cross-institution; external references will not be used" if migration.cross_institution?

      (data['web_link_categories'] || []).map{|c| c['links'] }.flatten.each do |link|
        course.external_url_hash[link['link_id']] = link
      end
      ActiveRecord::Base.skip_touch_context

      if !migration.for_course_copy?
        Importers::ContextModuleImporter.select_all_linked_module_items(data, migration)
        Importers::GradingStandardImporter.select_course_grading_standard(data, migration)
        # These only need to be processed once
        Attachment.skip_media_object_creation do
          self.process_migration_files(course, data, migration); migration.update_import_progress(18)
          Importers::AttachmentImporter.process_migration(data, migration); migration.update_import_progress(20)
          mo_attachments = migration.imported_migration_items_by_class(Attachment).find_all { |i| i.media_entry_id.present? }
          begin
            self.import_media_objects(mo_attachments, migration)
          rescue => e
            er = Canvas::Errors.capture_exception(:import_media_objects, e)[:error_report]
            error_message = t('Failed to import media objects')
            migration.add_error(error_message, error_report_id: er)
          end
        end
        if migration.canvas_import?
          migration.update_import_progress(30)
          Importers::MediaTrackImporter.process_migration(data[:media_tracks], migration)
        end
      end

      migration.update_import_progress(35)
      question_data = Importers::AssessmentQuestionImporter.process_migration(data, migration); migration.update_import_progress(45)
      Importers::GroupImporter.process_migration(data, migration); migration.update_import_progress(48)
      Importers::LearningOutcomeImporter.process_migration(data, migration); migration.update_import_progress(50)
      Importers::RubricImporter.process_migration(data, migration); migration.update_import_progress(52)
      course.assignment_group_no_drop_assignments = {}
      Importers::AssignmentGroupImporter.process_migration(data, migration); migration.update_import_progress(54)
      Importers::ExternalFeedImporter.process_migration(data, migration); migration.update_import_progress(56)
      Importers::GradingStandardImporter.process_migration(data, migration); migration.update_import_progress(58)
      Importers::ContextExternalToolImporter.process_migration(data, migration); migration.update_import_progress(60)
      Importers::QuizImporter.process_migration(data, migration, question_data); migration.update_import_progress(65)
      Importers::DiscussionTopicImporter.process_migration(data, migration); migration.update_import_progress(70)
      Importers::WikiPageImporter.process_migration(data, migration); migration.update_import_progress(75)
      Importers::AssignmentImporter.process_migration(data, migration); migration.update_import_progress(80)
      Importers::ContextModuleImporter.process_migration(data, migration); migration.update_import_progress(85)
      Importers::WikiPageImporter.process_migration_course_outline(data, migration)
      Importers::CalendarEventImporter.process_migration(data, migration)

      everything_selected = !migration.copy_options || migration.is_set?(migration.copy_options[:everything])
      if everything_selected || migration.is_set?(migration.copy_options[:all_course_settings])
        self.import_settings_from_migration(course, data, migration)
      end
      migration.update_import_progress(90)

      # be very explicit about draft state courses, but be liberal toward legacy courses
      if course.wiki.has_no_front_page
        if migration.for_course_copy? && (source = migration.source_course || Course.where(id: migration.migration_settings[:source_course_id]).first)
          mig_id = CC::CCHelper.create_key(source.wiki.front_page)
          if new_front_page = course.wiki.wiki_pages.where(migration_id: mig_id).first
            course.wiki.set_front_page_url!(new_front_page.url)
          end
        end
      end
      front_page = course.wiki.front_page
      course.wiki.unset_front_page! if front_page.nil? || front_page.new_record?

      syllabus_should_be_added = everything_selected || migration.copy_options[:syllabus_body] || migration.copy_options[:all_syllabus_body]
      if syllabus_should_be_added
        syllabus_body = data[:course][:syllabus_body] if data[:course]
        self.import_syllabus_from_migration(course, syllabus_body, migration) if syllabus_body
      end

      course.save! if course.changed?

      migration.resolve_content_links!
      migration.update_import_progress(95)

      if data['external_content']
        Canvas::Migration::ExternalContent::Migrator.send_imported_content(migration, data['external_content'])
      end

      begin
        #Adjust dates
        if shift_options = migration.date_shift_options
          shift_options = self.shift_date_options(course, shift_options)

          migration.imported_migration_items_by_class(Assignment).each do |event|
            event.reload # just in case
            event.due_at = shift_date(event.due_at, shift_options)
            event.lock_at = shift_date(event.lock_at, shift_options)
            event.unlock_at = shift_date(event.unlock_at, shift_options)
{'fraction_non_alphanumeric': 0.09589689015017146, 'fraction_numerical': 0.006030507272082299, 'mean_word_length': 3.8040329451860266, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1362113', 'n_tokens_mistral': 4899, 'n_tokens_neox': 4820, 'n_words': 746}