<commit_before>require 'mediawiki_api'

#= Imports revision scoring data from ores.wmflabs.org
class RevisionScoreImporter
  ################
  # Entry points #
  ################
  def self.update_revision_scores(revisions=nil)
    # Unscored mainspace, userspace, and Draft revisions, by default
    revisions ||= unscored_mainspace_userspace_and_draft_revisions
    revisions.each_slice(50) do |rev_batch|
      rev_ids = rev_batch.map(&:id)
      scores = get_revision_scores rev_ids
      save_scores scores
    end
  end

  def self.update_all_revision_scores_for_articles(article_ids = nil)
    article_ids ||= Article.namespace(0).pluck(:id)
    revisions = Revision.where(article_id: article_ids)
    update_revision_scores revisions

    first_revisions = []
    article_ids.each do |id|
      first_revisions << Revision.where(article_id: id).first
    end

    first_revisions.each do |revision|
      parent_id = get_parent_id revision
      score = get_revision_scores [parent_id]<commit_msg>Handle network errors in RevisionScoreImporter without breaking a batch<commit_after>require 'mediawiki_api'

#= Imports revision scoring data from ores.wmflabs.org
class RevisionScoreImporter
  ################
  # Entry points #
  ################
  def self.update_revision_scores(revisions=nil)
    # Unscored mainspace, userspace, and Draft revisions, by default
    revisions ||= unscored_mainspace_userspace_and_draft_revisions
    revisions.each_slice(50) do |rev_batch|
      rev_ids = rev_batch.map(&:id)
      scores = get_revision_scores rev_ids
      next if scores.blank?
      save_scores scores
    end
  end

  def self.update_all_revision_scores_for_articles(article_ids = nil)
    article_ids ||= Article.namespace(0).pluck(:id)
    revisions = Revision.where(article_id: article_ids)
    update_revision_scores revisions

    first_revisions = []
    article_ids.each do |id|
      first_revisions << Revision.where(article_id: id).first
    end

    first_revisions.each do |revision|
      parent_id = get_parent_id revision
      score = get_revision_scores [parent_id]
{'fraction_non_alphanumeric': 0.10104861773117255, 'fraction_numerical': 0.002859866539561487, 'mean_word_length': 4.313924050632911, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4613294', 'n_tokens_mistral': 712, 'n_tokens_neox': 651, 'n_words': 136}