<commit_before>require 'mediawiki_api'
require 'crack'
require 'json'

#= This class is for getting data directly from the Wikipedia API.
class Wiki
  ###################
  # Parsing methods #
  ###################

  # Based on the cohorts and wiki pages defined in application.yml, get the list
  # of courses for each cohort.
  def self.course_list
    response = {}
    Cohort.all.each do |cohort|
      content = get_page_content(cohort.url)
      response[cohort.slug] = content.split(/\n/) unless content.nil?
    end
    response
  end

  def self.get_course_info(course_ids)
    raw = get_course_info_raw(course_ids)
    return [nil] if raw.nil? # This indicates a failure to get the course data.
    return [] unless raw # This indicates that the course(s) don't exist.

    info = []
    (0...raw.count).each do |course|
      raw_course_info = raw[course.to_s]
      course_info = parse_course_info(raw_course_info)
      info.append(course_info)
    end
    info
  end

  def self.parse_course_info(course)
    parsed = { 'course' => {}, 'participants' => {} }
    append = course['name'][-1, 1] != ')' ? ' ()' : ''
    course_info = (course['name'] + append).split(/(.*)\/(.*)\s\(([^\)]+)?\)/)
    parsed.tap do |p|
      p['course']['id'] = course['id']
      p['course']['slug'] = course['name'].gsub(' ', '_')
      p['course']['school'] = course_info[1]
      p['course']['title'] = course_info[2]
      p['course']['term'] = course_info[3]
      p['course']['start'] = course['start'].to_date
      p['course']['end'] = course['end'].to_date

      roles = %w(student instructor online_volunteer campus_volunteer)
      roles.each do |r|
        p['participants'][r] = course[r + 's']
      end
    end
  end

  def self.get_article_rating(titles)
    titles = [titles] unless titles.is_a?(Array)
    titles = titles.sort_by(&:downcase)

    talk_titles = titles.map { |at| 'Talk:' + at }
    raw = get_raw_page_content(talk_titles)
    return [] unless raw

    # Pages that are missing get returned before pages that exist, so we cannot
    # count on our array being in the same order as titles.
    raw.map do |article_id, article|
      # Remove "Talk:" from the "title" value to get the title.
      { article['title'][5..-1] => parse_article_rating(article) }
    end
  end

  # Try to find the Wikipedia 1.0 rating of an article by parsing its talk page
  # contents.
  #
  # Adapted from https://en.wikipedia.org/wiki/User:Pyrospirit/metadata.js
  # alt https://en.wikipedia.org/wiki/MediaWiki:Gadget-metadata.js
  # We simplify this parser by removing folding the nonstandard ratings
  # into the corresponding standard ones. We don't want to deal with edge cases
  # like bplus and a/ga.
  def self.parse_article_rating(raw_article)
    # Handle MediaWiki API errors
    return nil if raw_article.nil?

    # Handle the case of nonexistent talk pages.
    return nil if raw_article['missing']

    article = raw_article['revisions'][0]['*']

    # Handle empty talk page
    return nil if article.is_a? Hash

    # rubocop:disable Metrics/LineLength
    if article.match(/\|\s*(class|currentstatus)\s*=\s*fa\b/i)
      'fa'
    elsif article.match(/\|\s*(class|currentstatus)\s*=\s*fl\b/i)
      'fl'
    elsif article.match(/\|\s*class\s*=\s*a\b/i)
      'a' # Treat all forms of A, including A/GA, as simple A.
    elsif article.match(/\|\s*class\s*=\s*ga\b|\|\s*currentstatus\s*=\s*(ffa\/)?ga\b|\{\{\s*ga\s*\|/i) && !article.match(/\|\s*currentstatus\s*=\s*dga\b/i)
      'ga'
    elsif article.match(/\|\s*class\s*=\s*b\b/i)
      'b'
    elsif article.match(/\|\s*class\s*=\s*bplus\b/i)
      'b' # Treat B-plus as regular B.
    elsif article.match(/\|\s*class\s*=\s*c\b/i)<commit_msg>Refactor complex article ratings parsing method<commit_after>require 'mediawiki_api'
require 'crack'
require 'json'

#= This class is for getting data directly from the Wikipedia API.
class Wiki
  ###################
  # Parsing methods #
  ###################

  # Based on the cohorts and wiki pages defined in application.yml, get the list
  # of courses for each cohort.
  def self.course_list
    response = {}
    Cohort.all.each do |cohort|
      content = get_page_content(cohort.url)
      response[cohort.slug] = content.split(/\n/) unless content.nil?
    end
    response
  end

  def self.get_course_info(course_ids)
    raw = get_course_info_raw(course_ids)
    return [nil] if raw.nil? # This indicates a failure to get the course data.
    return [] unless raw # This indicates that the course(s) don't exist.

    info = []
    (0...raw.count).each do |course|
      raw_course_info = raw[course.to_s]
      course_info = parse_course_info(raw_course_info)
      info.append(course_info)
    end
    info
  end

  def self.parse_course_info(course)
    parsed = { 'course' => {}, 'participants' => {} }
    append = course['name'][-1, 1] != ')' ? ' ()' : ''
    course_info = (course['name'] + append).split(/(.*)\/(.*)\s\(([^\)]+)?\)/)
    parsed.tap do |p|
      p['course']['id'] = course['id']
      p['course']['slug'] = course['name'].gsub(' ', '_')
      p['course']['school'] = course_info[1]
      p['course']['title'] = course_info[2]
      p['course']['term'] = course_info[3]
      p['course']['start'] = course['start'].to_date
      p['course']['end'] = course['end'].to_date

      roles = %w(student instructor online_volunteer campus_volunteer)
      roles.each do |r|
        p['participants'][r] = course[r + 's']
      end
    end
  end

  def self.get_article_rating(titles)
    titles = [titles] unless titles.is_a?(Array)
    titles = titles.sort_by(&:downcase)

    talk_titles = titles.map { |at| 'Talk:' + at }
    raw = get_raw_page_content(talk_titles)
    return [] unless raw

    # Pages that are missing get returned before pages that exist, so we cannot
    # count on our array being in the same order as titles.
    raw.map do |article_id, article|
      # Remove "Talk:" from the "title" value to get the title.
      { article['title'][5..-1] => parse_article_rating(article) }
    end
  end

  def self.parse_article_rating(raw_article)
    # Handle MediaWiki API errors
    return nil if raw_article.nil?
    # Handle the case of nonexistent talk pages.
    return nil if raw_article['missing']

    article = raw_article['revisions'][0]['*']
    find_article_class article
  end

  # Try to find the Wikipedia 1.0 rating of an article by parsing its talk page
  # contents.
  #
  # Adapted from https://en.wikipedia.org/wiki/User:Pyrospirit/metadata.js
  # alt https://en.wikipedia.org/wiki/MediaWiki:Gadget-metadata.js
  # We simplify this parser by removing folding the nonstandard ratings
  # into the corresponding standard ones. We don't want to deal with edge cases
  # like bplus and a/ga.
  def self.find_article_class(article)
    # Handle empty talk page
    return nil if article.is_a? Hash
        # rubocop:disable Metrics/LineLength
    if article.match(/\|\s*(class|currentstatus)\s*=\s*fa\b/i)
      'fa'
    elsif article.match(/\|\s*(class|currentstatus)\s*=\s*fl\b/i)
      'fl'
    elsif article.match(/\|\s*class\s*=\s*a\b/i)
      'a' # Treat all forms of A, including A/GA, as simple A.
    elsif article.match(/\|\s*class\s*=\s*ga\b|\|\s*currentstatus\s*=\s*(ffa\/)?ga\b|\{\{\s*ga\s*\|/i) && !article.match(/\|\s*currentstatus\s*=\s*dga\b/i)
      'ga'
    elsif article.match(/\|\s*class\s*=\s*b\b/i)
      'b'
    elsif article.match(/\|\s*class\s*=\s*bplus\b/i)
      'b' # Treat B-plus as regular B.
    elsif article.match(/\|\s*class\s*=\s*c\b/i)
{'fraction_non_alphanumeric': 0.16401906274821287, 'fraction_numerical': 0.0029123643102991793, 'mean_word_length': 3.0926327193932828, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6758909', 'n_tokens_mistral': 2628, 'n_tokens_neox': 2569, 'n_words': 637}