<commit_before>module PermClerk
  require 'date'
  require 'logger'
  require 'uri'

  COMMENT_INDENT = "\n::"
  AWB_COMMENT_INDENT = "\n*:"
  COMMENT_PREFIX = '{{comment|Automated comment}} '
  EDIT_THROTTLE = 3
  SPLIT_KEY = '====[[User:'
  AWB_SPLIT_KEY = '*{{AWBUser|'

  PERMISSION_KEYS = {
    'Account creator' => 'accountcreator',
    'Autopatrolled' => 'autoreviewer',
    'Confirmed' => '(?=>auto)?confirmed',
    'File mover' => 'filemover',
    'Mass message sender' => 'massmessage-sender',
    'Pending changes reviewer' => 'reviewer',
    'Rollback' => 'rollbacker',
    'Template editor' => 'templateeditor',
    'AWB' => 'awb'
  }

  def self.init(mw, repl_client, config)
    @mw = mw
    @repl_client = repl_client
    @config = config

    @denied_cache = {}
    @user_info_cache = {}
    @user_links_cache = {}

    `stat perm_clerk.log`
    @logger = Logger.new('perm_clerk.log')
    @logger.level = Logger::DEBUG

    @run_status = eval(File.open('lastrun', 'r').read) rescue {}
    @run_file = File.open('lastrun', 'r+')

    @archive_changes = {}
    @errors = {}

    if @config[:env] == :production
      @PREREQ_EXPIRY = 90
      @PERMISSIONS = [
        'Account creator',
        'Autopatrolled',
        'AWB',
        'Confirmed',
        'File mover',
        'Mass message sender',
        'Pending changes reviewer',
        'Rollback',
        'Template editor'
      ]
    else
      @PREREQ_EXPIRY = 0
      @PERMISSIONS = ['AWB']
    end

    start

    @run_file.write(@run_status.inspect)
    @run_file.close
  end

  def self.start
    total_user_count = 0
    for @permission in @PERMISSIONS
      sleep 2

      @base_timestamp = nil
      @perm_edit_throttle = 0
      @edit_summaries = []
      @headers_removed = {}
      @users_count = 0
      if process
        info("Processing of #{@permission} complete")
        @run_status[@permission] = current_time.to_s
        total_user_count += @users_count
      else
        error('Failed to process')
      end
    end

    @permission = ''
    archive_requests if @archive_changes.length

    report_errors

    info("#{'~' * 25} Task complete #{'~' * 25}")
  end

  def self.process
    info("Processing #{@permission}...")

    if @permission == 'AWB'
      @page_name = 'Wikipedia talk:AutoWikiBrowser/CheckPage'
    else
      @page_name = "Wikipedia:Requests for permissions/#{@permission}"
    end

    old_wikitext = set_page_props

    return false unless old_wikitext

    @last_edit = parse_date_time(@base_timestamp)
    @last_run = parse_date_time(@run_status[@permission]) rescue DateTime.new

    prereqs = @config['prerequisites_config'][@permission.downcase.gsub(/ /, '_')]
    if @config['prerequisites'] && prereqs # if prereqs enabled for this permission
      has_prereq_data = old_wikitext.match(/&lt;!-- mb-\w*(?:Count|Age) --&gt;/)
      should_update_prereq_data = @config['prerequisites'] ? !!has_prereq_data : false
    else
      should_update_prereq_data = false
    end

    # only process if there's data to update, the page has changed since the last run or it's been 90 minutes since the last run
    if @config[:env] == :production && !should_update_prereq_data && @last_run > @last_edit && @last_run + Rational(90, 1440) > current_time
      info('  Less than 90 minutes since last run without changes, and no prerequisites to update')
      return false
    end

    new_wikitext = []

    if @config['autoformat'] && @permission != 'AWB'
      debug('Checking for extraneous headers')
      old_wikitext = remove_headers(old_wikitext)
    end

    @split_key = @permission == 'AWB' ? AWB_SPLIT_KEY : SPLIT_KEY
    @num_open_requests = 0

    return false unless formatting_check(old_wikitext)

    # first make fixes to confirm to what split_key looks for
    old_wikitext.gsub!(/\*\s+{{AWBUser\|/, '*{{AWBUser|')
    old_wikitext.gsub(/\=\=\=\=\s+\[\[User:/, '====[[User:')

    sections = old_wikitext.split(@split_key)
    new_wikitext << sections.shift

    sections.each do |section|
      bot_section = ''
      if @permission == 'AWB'
        bot_split = section.split(/====\s*Bots/i)
        section = bot_split[0]
        bot_section = '==== Bots' + bot_split[1] if bot_split[1]
      end

      request_changes = []

      if @permission == 'AWB'
        username = section.scan(/^(.*?)}}/).flatten[0]
      else
        username = section.scan(/{{(?:template\:)?rfplinks\|1=(.*?)}}/i).flatten[0]
      end

      if !username || username == 'username' || username == 'bot username'
        new_wikitext << @split_key + section + bot_section
        next
      end

      info("Checking section for User:#{username}...")

      username[0] = username[0].capitalize
      username.gsub!('_', ' ')
      # TODO: if capitalization happens, update page as part of autoformat task

      timestamps = section.scan(/(?<!&lt;!-- mbdate --&gt; )\d\d:\d\d.*\d{4} \(UTC\)/)
      newest_timestamp = timestamps.min { |a, b| parse_date_time(b) <=> parse_date_time(a) }
      if overriden_resolution = section.match(/\{\{User:MusikBot\/override\|d\}\}/i) ? 'done' : section.match(/\{\{User:MusikBot\/override\|nd\}\}/i) ? 'notdone' : false
        info('  Resolution override found')
      end
      resolution = overriden_resolution || (section.match(/#{@config["regex_done"]}/i) ? 'done' : section.match(/#{@config["regex_notdone"]}/i) ? 'notdone' : false)
      resolution_date = Date.parse(section.scan(/#{@config["regex_#{resolution}"]}.*(\d\d:\d\d, \d+ \w+ \d{4} \(UTC\))/i).flatten.last) rescue nil

      # use newest timestamp when forcing resolution and no resolution template exists
      if resolution_date.nil? && overriden_resolution
        resolution_date = Date.parse(newest_timestamp)
      end

      @num_open_requests += 1 unless resolution

      if should_update_prereq_data = section.scan(/&lt;!-- mb-/).length > 0
        prereq_sig_regex = section.scan(/(&lt;!-- mbsig --&gt;.*&lt;!-- mbdate --&gt; (\d\d:\d\d.*\d{4} \(UTC\)))/)
        prereq_signature = prereq_sig_regex.flatten[0]
        prereq_timestamp = prereq_sig_regex.flatten[1]
        if should_update_prereq_data = current_time > parse_date_time(prereq_timestamp) + Rational(@PREREQ_EXPIRY, 1440) rescue false
          debug('  Found expired prerequisite data')
        else
          debug("  Prerequisite data under #{@PREREQ_EXPIRY} minutes old")
        end
      end

      should_archive_now = section.match(/\{\{User:MusikBot\/archivenow\}\}/)

      # <ARCHIVING>
      if resolution && @config['archive'] && resolution_date.nil?
        record_error(
          group: 'archive',
          message: "User:#{username} - Resolution template not dated",
          log_message: "    User:#{username}: Resolution template not dated"
        ) and next<commit_msg>fix bug with regex<commit_after>module PermClerk
  require 'date'
  require 'logger'
  require 'uri'

  COMMENT_INDENT = "\n::"
  AWB_COMMENT_INDENT = "\n*:"
  COMMENT_PREFIX = '{{comment|Automated comment}} '
  EDIT_THROTTLE = 3
  SPLIT_KEY = '====[[User:'
  AWB_SPLIT_KEY = '*{{AWBUser|'

  PERMISSION_KEYS = {
    'Account creator' => 'accountcreator',
    'Autopatrolled' => 'autoreviewer',
    'Confirmed' => '(?=>auto)?confirmed',
    'File mover' => 'filemover',
    'Mass message sender' => 'massmessage-sender',
    'Pending changes reviewer' => 'reviewer',
    'Rollback' => 'rollbacker',
    'Template editor' => 'templateeditor',
    'AWB' => 'awb'
  }

  def self.init(mw, repl_client, config)
    @mw = mw
    @repl_client = repl_client
    @config = config

    @denied_cache = {}
    @user_info_cache = {}
    @user_links_cache = {}

    `stat perm_clerk.log`
    @logger = Logger.new('perm_clerk.log')
    @logger.level = Logger::DEBUG

    @run_status = eval(File.open('lastrun', 'r').read) rescue {}
    @run_file = File.open('lastrun', 'r+')

    @archive_changes = {}
    @errors = {}

    if @config[:env] == :production
      @PREREQ_EXPIRY = 90
      @PERMISSIONS = [
        'Account creator',
        'Autopatrolled',
        'AWB',
        'Confirmed',
        'File mover',
        'Mass message sender',
        'Pending changes reviewer',
        'Rollback',
        'Template editor'
      ]
    else
      @PREREQ_EXPIRY = 0
      @PERMISSIONS = ['AWB']
    end

    start

    @run_file.write(@run_status.inspect)
    @run_file.close
  end

  def self.start
    total_user_count = 0
    for @permission in @PERMISSIONS
      sleep 2

      @base_timestamp = nil
      @perm_edit_throttle = 0
      @edit_summaries = []
      @headers_removed = {}
      @users_count = 0
      if process
        info("Processing of #{@permission} complete")
        @run_status[@permission] = current_time.to_s
        total_user_count += @users_count
      else
        error('Failed to process')
      end
    end

    @permission = ''
    archive_requests if @archive_changes.length

    report_errors

    info("#{'~' * 25} Task complete #{'~' * 25}")
  end

  def self.process
    info("Processing #{@permission}...")

    if @permission == 'AWB'
      @page_name = 'Wikipedia talk:AutoWikiBrowser/CheckPage'
    else
      @page_name = "Wikipedia:Requests for permissions/#{@permission}"
    end

    old_wikitext = set_page_props

    return false unless old_wikitext

    @last_edit = parse_date_time(@base_timestamp)
    @last_run = parse_date_time(@run_status[@permission]) rescue DateTime.new

    prereqs = @config['prerequisites_config'][@permission.downcase.gsub(/ /, '_')]
    if @config['prerequisites'] && prereqs # if prereqs enabled for this permission
      has_prereq_data = old_wikitext.match(/&lt;!-- mb-\w*(?:Count|Age) --&gt;/)
      should_update_prereq_data = @config['prerequisites'] ? !!has_prereq_data : false
    else
      should_update_prereq_data = false
    end

    # only process if there's data to update, the page has changed since the last run or it's been 90 minutes since the last run
    if @config[:env] == :production && !should_update_prereq_data && @last_run > @last_edit && @last_run + Rational(90, 1440) > current_time
      info('  Less than 90 minutes since last run without changes, and no prerequisites to update')
      return false
    end

    new_wikitext = []

    if @config['autoformat'] && @permission != 'AWB'
      debug('Checking for extraneous headers')
      old_wikitext = remove_headers(old_wikitext)
    end

    @split_key = @permission == 'AWB' ? AWB_SPLIT_KEY : SPLIT_KEY
    @num_open_requests = 0

    return false unless formatting_check(old_wikitext)

    # first make fixes to confirm to what split_key looks for
    old_wikitext.gsub!(/\*\s+{{AWBUser\|/, '*{{AWBUser|')
    old_wikitext.gsub(/\=\=\=\=\s+\[\[User:/, '====[[User:')

    sections = old_wikitext.split(@split_key)
    new_wikitext << sections.shift

    sections.each do |section|
      bot_section = ''
      if @permission == 'AWB'
        bot_split = section.split(/====\s*Bots/i)
        section = bot_split[0]
        bot_section = '==== Bots' + bot_split[1] if bot_split[1]
      end

      request_changes = []

      if @permission == 'AWB'
        username = section.scan(/^(.*?)}}/).flatten[0]
      else
        username = section.scan(/{{(?:template\:)?rfplinks\|1=(.*?)}}/i).flatten[0]
      end

      if !username || username == 'username' || username == 'bot username'
        new_wikitext << @split_key + section + bot_section
        next
      end

      info("Checking section for User:#{username}...")

      username[0] = username[0].capitalize
      username.gsub!('_', ' ')
      # TODO: if capitalization happens, update page as part of autoformat task

      timestamps = section.scan(/(?<!&lt;!-- mbdate --&gt; )\d\d:\d\d.*\d{4} \(UTC\)/)
      newest_timestamp = timestamps.min { |a, b| parse_date_time(b) <=> parse_date_time(a) }
      if overriden_resolution = section.match(/\{\{User:MusikBot\/override\|d\}\}/i) ? 'done' : section.match(/\{\{User:MusikBot\/override\|nd\}\}/i) ? 'notdone' : false
        info('  Resolution override found')
      end
      resolution = overriden_resolution || (section.match(/(?:#{@config["regex_done"]})/i) ? 'done' : section.match(/(?:#{@config["regex_notdone"]})/i) ? 'notdone' : false)
      resolution_date = Date.parse(section.scan(/(?:#{@config["regex_#{resolution}"]}).*(\d\d:\d\d, \d+ \w+ \d{4} \(UTC\))/i).flatten.drop(1).last) rescue nil

      # use newest timestamp when forcing resolution and no resolution template exists
      if resolution_date.nil? && overriden_resolution
        resolution_date = Date.parse(newest_timestamp)
      end

      @num_open_requests += 1 unless resolution

      if should_update_prereq_data = section.scan(/&lt;!-- mb-/).length > 0
        prereq_sig_regex = section.scan(/(&lt;!-- mbsig --&gt;.*&lt;!-- mbdate --&gt; (\d\d:\d\d.*\d{4} \(UTC\)))/)
        prereq_signature = prereq_sig_regex.flatten[0]
        prereq_timestamp = prereq_sig_regex.flatten[1]
        if should_update_prereq_data = current_time > parse_date_time(prereq_timestamp) + Rational(@PREREQ_EXPIRY, 1440) rescue false
          debug('  Found expired prerequisite data')
        else
          debug("  Prerequisite data under #{@PREREQ_EXPIRY} minutes old")
        end
      end

      should_archive_now = section.match(/\{\{User:MusikBot\/archivenow\}\}/)

      # <ARCHIVING>
      if resolution && @config['archive'] && resolution_date.nil?
        record_error(
          group: 'archive',
          message: "User:#{username} - Resolution template not dated",
          log_message: "    User:#{username}: Resolution template not dated"
        ) and next
{'fraction_non_alphanumeric': 0.15040082371111274, 'fraction_numerical': 0.006251378980657498, 'mean_word_length': 2.9176029962546814, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3044018', 'n_tokens_mistral': 4697, 'n_tokens_neox': 4461, 'n_words': 958}