<commit_before>require "#{Rails.root}/lib/wiki"
require "#{Rails.root}/lib/importers/user_importer"

#= Imports and updates courses from Wikipedia into the dashboard database
class CourseImporter
  ################
  # Entry points #
  ################
  def self.get_course_info(course_id)
    Wiki.get_course_info course_id
  end

  ##############
  # API Access #
  ##############
  def self.update_all_courses(initial=false, raw_ids={})
    raw_ids = Wiki.course_list if raw_ids.empty?
    listed_ids = raw_ids.values.flatten
    course_ids = listed_ids | Course.where(listed: true).pluck(:id).map(&:to_s)

    if initial
      _minimum = course_ids.map(&:to_i).min
      maximum = course_ids.map(&:to_i).max
      course_ids = (0..maximum).to_a.map(&:to_s)
    end

    # Break up course_ids into smaller groups that Wikipedia's API can handle.
    data = Utils.chunk_requests(course_ids) { |c| get_course_info c }
    import_courses(raw_ids, data)
  end

  ###########
  # Helpers #
  ###########
  def self.import_courses(raw_ids, data)
    courses = []
    participants = {}
    listed_ids = raw_ids.values.flatten

    # Encountered an API error; cancel course import for today
    if data.include? nil
      Rails.logger.warn 'Network error. Course import cancelled.'
      return
    end

    # Delist courses that have been deleted
    Course.where(listed: true).each do |c|
      c.delist unless listed_ids.include?(c.id)
    end

    # Update courses from new data
    data.each do |c|
      c['course']['listed'] = listed_ids.include?(c['course']['id'])
      course = Course.new(id: c['course']['id'])
      course.update(c, false)
      courses.push course
      participants[c['course']['id']] = c['participants']
    end
    Course.import courses, on_duplicate_key_update: [:start, :end, :listed]<commit_msg>Fix for bad course listings<commit_after>require "#{Rails.root}/lib/wiki"
require "#{Rails.root}/lib/importers/user_importer"

#= Imports and updates courses from Wikipedia into the dashboard database
class CourseImporter
  ################
  # Entry points #
  ################
  def self.get_course_info(course_id)
    Wiki.get_course_info course_id
  end

  ##############
  # API Access #
  ##############
  def self.update_all_courses(initial=false, raw_ids={})
    raw_ids = Wiki.course_list if raw_ids.empty?
    listed_ids = raw_ids.values.flatten
    course_ids = listed_ids | Course.where(listed: true).pluck(:id).map(&:to_s)

    if initial
      _minimum = course_ids.map(&:to_i).min
      maximum = course_ids.map(&:to_i).max
      course_ids = (0..maximum).to_a.map(&:to_s)
    end

    # Break up course_ids into smaller groups that Wikipedia's API can handle.
    data = Utils.chunk_requests(course_ids) { |c| get_course_info c }
    import_courses(raw_ids, data)
  end

  ###########
  # Helpers #
  ###########
  def self.import_courses(raw_ids, data)
    courses = []
    participants = {}
    listed_ids = raw_ids.values.flatten.map(&:to_i)

    # Encountered an API error; cancel course import for today
    if data.include? nil
      Rails.logger.warn 'Network error. Course import cancelled.'
      return
    end

    # Delist courses that have been deleted
    Course.where(listed: true).each do |c|
      c.delist unless listed_ids.include?(c.id)
    end

    # Update courses from new data
    data.each do |c|
      c['course']['listed'] = listed_ids.include?(c['course']['id'])
      course = Course.new(id: c['course']['id'])
      course.update(c, false)
      courses.push course
      participants[c['course']['id']] = c['participants']
    end
    Course.import courses, on_duplicate_key_update: [:start, :end, :listed]
{'fraction_non_alphanumeric': 0.1578804347826087, 'fraction_numerical': 0.0005434782608695652, 'mean_word_length': 3.522113022113022, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3691525', 'n_tokens_mistral': 1219, 'n_tokens_neox': 1185, 'n_words': 271}