<commit_before>$LOAD_PATH << '..'
require 'musikbot'

module PermClerk
  COMMENT_INDENT = "\n::"
  COMMENT_PREFIX = '{{comment|Automated comment}} '
  SPLIT_KEY = '====[[User:'
  PREREQ_EXPIRY = 90
  AWB_CHECKPAGE = 'Wikipedia:AutoWikiBrowser/CheckPage'

  PERMISSION_KEYS = {
    'Account creator' => 'accountcreator',
    'Autopatrolled' => 'autoreviewer',
    'Confirmed' => '(?=>auto)?confirmed',
    'File mover' => 'filemover',
    'Mass message sender' => 'massmessage-sender',
    'Pending changes reviewer' => 'reviewer',
    'Rollback' => 'rollbacker',
    'Template editor' => 'templateeditor'
  }

  def self.run
    @mb = MusikBot::Session.new(inspect)

    @denied_cache = {}
    @user_info_cache = {}
    @user_links_cache = {}
    @archive_changes = {}
    @errors = {}
    @total_user_count = 0

    config['pages'].each do |permission|
      @permission = permission
      @edit_summaries = []
      @headers_removed = {}
      @users_count = 0

      begin
        process_permission
        run_status[permission] = @mb.now.to_s
        @total_user_count += @users_count
      rescue => e
        @mb.report_error("Failed to process #{permission}", e)
      end
    end

    archive_requests if @archive_changes.any?

    generate_report

    run_file = @mb.local_storage('lastrun', 'r+')
    run_file.write(run_status.inspect)
    run_file.close

    info("#{'~' * 25} Task complete #{'~' * 25}")
  rescue => e
    @mb.report_error('Fatal error', e)
  end

  def self.process_permission(throttle = 0)
    info("Processing #{@permission}...")
    page_name = "Wikipedia:Requests for permissions/#{@permission}"

    old_wikitext = page_props(page_name)
    return unless formatting_check(old_wikitext)

    last_run = @mb.parse_date(run_status[@permission])

    if prereqs # if prereqs enabled for this permission
      has_prereq_data = old_wikitext.match(/\<!-- mb-\w*(?:Count|Age) --\>/)
      should_check_prereq_data = has_prereq_data.present?
    else
      should_check_prereq_data = false
    end

    # only process if there's data to update or the page has changed since the last run
    if @mb.env == :production && !should_check_prereq_data && last_run > @last_edit
      return info('  Less than 90 minutes since last run without changes, and no prerequisites to update')
    end

    if config['run']['autoformat']
      debug('Checking for extraneous headers')
      old_wikitext = remove_headers(old_wikitext)
    end<commit_msg>PermClerk2: run after 90 minutes<commit_after>$LOAD_PATH << '..'
require 'musikbot'

module PermClerk
  COMMENT_INDENT = "\n::"
  COMMENT_PREFIX = '{{comment|Automated comment}} '
  SPLIT_KEY = '====[[User:'
  PREREQ_EXPIRY = 90
  AWB_CHECKPAGE = 'Wikipedia:AutoWikiBrowser/CheckPage'

  PERMISSION_KEYS = {
    'Account creator' => 'accountcreator',
    'Autopatrolled' => 'autoreviewer',
    'Confirmed' => '(?=>auto)?confirmed',
    'File mover' => 'filemover',
    'Mass message sender' => 'massmessage-sender',
    'Pending changes reviewer' => 'reviewer',
    'Rollback' => 'rollbacker',
    'Template editor' => 'templateeditor'
  }

  def self.run
    @mb = MusikBot::Session.new(inspect)

    @denied_cache = {}
    @user_info_cache = {}
    @user_links_cache = {}
    @archive_changes = {}
    @errors = {}
    @total_user_count = 0

    config['pages'].each do |permission|
      @permission = permission
      @edit_summaries = []
      @headers_removed = {}
      @users_count = 0

      begin
        process_permission
        run_status[permission] = @mb.now.to_s
        @total_user_count += @users_count
      rescue => e
        @mb.report_error("Failed to process #{permission}", e)
      end
    end

    archive_requests if @archive_changes.any?

    generate_report

    run_file = @mb.local_storage('lastrun', 'r+')
    run_file.write(run_status.inspect)
    run_file.close

    info("#{'~' * 25} Task complete #{'~' * 25}")
  rescue => e
    @mb.report_error('Fatal error', e)
  end

  def self.process_permission(throttle = 0)
    info("Processing #{@permission}...")
    page_name = "Wikipedia:Requests for permissions/#{@permission}"

    old_wikitext = page_props(page_name)
    return unless formatting_check(old_wikitext)

    last_run = @mb.parse_date(run_status[@permission])

    if prereqs # if prereqs enabled for this permission
      has_prereq_data = old_wikitext.match(/\<!-- mb-\w*(?:Count|Age) --\>/)
      should_check_prereq_data = has_prereq_data.present?
    else
      should_check_prereq_data = false
    end

    # only process if there's data to update, the page has changed since the last run, or 90 minutes has passed
    if @mb.env == :production && !should_check_prereq_data && last_run > @last_edit && last_run + Rational(90, 1440) > @mb.now
      return info('  Less than 90 minutes since last run without changes, and no prerequisites to update')
    end

    if config['run']['autoformat']
      debug('Checking for extraneous headers')
      old_wikitext = remove_headers(old_wikitext)
    end
{'fraction_non_alphanumeric': 0.13112793941809486, 'fraction_numerical': 0.00657632522917497, 'mean_word_length': 3.352992194275802, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 30, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7619348', 'n_tokens_mistral': 1723, 'n_tokens_neox': 1634, 'n_words': 363}