<commit_before># frozen_string_literal: true

class WikidataSummaryImporter
  def self.import_all_missing_summaries
    import_missing_summaries Revision.where(wiki: wikidata, summary: nil, deleted: false)
  end

  def self.import_missing_summaries(revisions)
    revisions.find_in_batches(batch_size: 50) do |revision_batch|
      summaries = fetch_summaries(revision_batch)

      revision_batch.each do |rev|
        summary = summaries[rev.mw_rev_id]
        next if summary.nil?

        rev.update(summary: CGI.escape(summary))
      end
    end
  end

  def self.wikidata
    @wikidata ||= Wiki.get_or_create(language: nil, project: 'wikidata')
  end

  def self.fetch_summaries(revisions)
    revids = revisions.map(&:mw_rev_id).join('|')
    query = {
      prop: 'revisions',
      rvprop: 'comment|ids',
      revids:
    }
    data = WikiApi.new(wikidata).query(query)
    page_data = data.data['pages']
    # Deleted revisions return data without a 'pages' key, like:
    # {"batchcomplete":"","query":{"badrevids":{"968242606":{"revid":968242606}}}}
    return {} unless page_data
<commit_msg>Try to figure out what's going wrong in CI<commit_after># frozen_string_literal: true

class WikidataSummaryImporter
  def self.import_all_missing_summaries
    import_missing_summaries Revision.where(wiki: wikidata, summary: nil, deleted: false)
  end

  def self.import_missing_summaries(revisions)
    revisions.find_in_batches(batch_size: 50) do |revision_batch|
      summaries = fetch_summaries(revision_batch)

      revision_batch.each do |rev|
        summary = summaries[rev.mw_rev_id]
        next if summary.nil?

        rev.update!(summary: CGI.escape(summary))
      end
    end
  end

  def self.wikidata
    @wikidata ||= Wiki.get_or_create(language: nil, project: 'wikidata')
  end

  def self.fetch_summaries(revisions)
    revids = revisions.map(&:mw_rev_id).join('|')
    query = {
      prop: 'revisions',
      rvprop: 'comment|ids',
      revids:
    }
    data = WikiApi.new(wikidata).query(query)
    page_data = data.data['pages']
    # Deleted revisions return data without a 'pages' key, like:
    # {"batchcomplete":"","query":{"badrevids":{"968242606":{"revid":968242606}}}}
    return {} unless page_data

{'fraction_non_alphanumeric': 0.11675579322638147, 'fraction_numerical': 0.017825311942959002, 'mean_word_length': 3.827956989247312, 'pattern_counts': {'":': 10, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '965534', 'n_tokens_mistral': 820, 'n_tokens_neox': 759, 'n_words': 150}