<commit_before>import datetime
import itertools
import mwparserfromhell
import pywikibot
import re
import sys

BOTREQ = "Wikipedia:Bot requests"
BOTOP_CAT = "Wikipedia bot operators"
REPORT_PAGE = "User:EnterpriseyBot/BOTREQ status"
TABLE_HEADER = """<noinclude>{{botnav}}This is a table of current [[WP:BOTREQ|]] discussions, updated automatically by {{user|EnterpriseyBot}}.</noinclude>
{| border="1" class="sortable wikitable plainlinks"
! # !! Title !! Replies !! Last editor !! Date/Time !! Last botop editor !! Date/Time
"""
SUMMARY = "Bot updating BOTREQ status table ({} requests)"

USER = re.compile(r"\[\[User.*?:(.*?)(?:\||(?:\]\]))")
TIMESTAMP = re.compile(r"\d{2}:\d{2}, \d{1,2} [A-Za-z]* \d{4}")
SIGNATURE = re.compile(r"\[\[User.*?\]\].*?\(UTC\)")
SECTION_HEADER = re.compile(r"== ?([^=]+) ?==")

SIGNATURE_TIME_FORMAT = "%H:%M, %d %B %Y"
TIME_FORMAT_STRING = "%Y-%m-%d, %H:%M"

class Request:
    pass

def print_log(what_to_print):
    print(datetime.datetime.utcnow().strftime("[%Y-%m-%dT%H:%M:%SZ] ") + what_to_print)

def make_table_row(r):
    replies = ('style="background: red;" | ' if r.replies == 0 else '') + str(r.replies)

    # Utility function for processing
    def take_inner(regex, text):
        """
	Given a regex with exactly one capturing group and some text,
	return the text after all occurrences of the regex have been
	replaced with the group.

	Example: take_inner("a(.)a", "aba") == "b"
	"""
	return re.sub(regex, r"\1", text)

    # Row number
    row_number = r.row_number

    # We'll be putting r.title in a wikilink, so we can't have nested wikilinks
    title = take_inner(r"\[\[(?:.+?\|)?(.+?)\]\]", r.title)

    # Escape some characters in the link target
    encodings = {"#": "%23", "<": "%3C", ">": "%3E", "[": "%5B", "]": "%5D", "|": "%7C", "{": "%7B", "}": "%7D", "_": "%7C"}
    target = re.sub("[{}]".format("".join(map(re.escape, encodings.keys()))), lambda match: encodings[match.group(0)], title)

    # Remove formatting in the link target
    target = take_inner(r"''([^']+)''", take_inner(r"'''([^']+)'''", target))

    if type(r.last_edit_time) is datetime.datetime:
        old = (datetime.datetime.now() - r.last_edit_time).days > 60
        r.last_edit_time = ('style="background: red;" | ' if old else '') + r.last_edit_time.strftime(TIME_FORMAT_STRING)

    if type(r.last_botop_time) is datetime.datetime:
        r.last_botop_time = r.last_botop_time.strftime(TIME_FORMAT_STRING)

    elements = map(unicode, [row_number, target, title, replies, r.last_editor, r.last_edit_time, r.last_botop_editor, r.last_botop_time])
    return u"|-\n| {} || [[WP:Bot requests#{}|{}]] || {} || {} || {} || {} || {}".format(*elements)<commit_msg>[botreq-status] Fix encoding bug<commit_after>import datetime
import itertools
import mwparserfromhell
import pywikibot
import re
import sys

BOTREQ = "Wikipedia:Bot requests"
BOTOP_CAT = "Wikipedia bot operators"
REPORT_PAGE = "User:EnterpriseyBot/BOTREQ status"
TABLE_HEADER = """<noinclude>{{botnav}}This is a table of current [[WP:BOTREQ|]] discussions, updated automatically by {{user|EnterpriseyBot}}.</noinclude>
{| border="1" class="sortable wikitable plainlinks"
! # !! Title !! Replies !! Last editor !! Date/Time !! Last botop editor !! Date/Time
"""
SUMMARY = "Bot updating BOTREQ status table ({} requests)"

USER = re.compile(r"\[\[User.*?:(.*?)(?:\||(?:\]\]))")
TIMESTAMP = re.compile(r"\d{2}:\d{2}, \d{1,2} [A-Za-z]* \d{4}")
SIGNATURE = re.compile(r"\[\[User.*?\]\].*?\(UTC\)")
SECTION_HEADER = re.compile(r"== ?([^=]+) ?==")

SIGNATURE_TIME_FORMAT = "%H:%M, %d %B %Y"
TIME_FORMAT_STRING = "%Y-%m-%d, %H:%M"

class Request:
    pass

def print_log(what_to_print):
    print(datetime.datetime.utcnow().strftime("[%Y-%m-%dT%H:%M:%SZ] ") + what_to_print)

def make_table_row(r):
    replies = ('style="background: red;" | ' if r.replies == 0 else '') + str(r.replies)

    # Utility function for processing
    def take_inner(regex, text):
        """
	Given a regex with exactly one capturing group and some text,
	return the text after all occurrences of the regex have been
	replaced with the group.

	Example: take_inner("a(.)a", "aba") == "b"
	"""
	return re.sub(regex, r"\1", text)

    # Row number
    row_number = r.row_number

    # We'll be putting r.title in a wikilink, so we can't have nested wikilinks
    title = take_inner(r"\[\[(?:.+?\|)?(.+?)\]\]", r.title)

    # Escape some characters in the link target
    encodings = {"#": "%23", "<": "%3C", ">": "%3E", "[": "%5B", "]": "%5D", "|": "%7C", "{": "%7B", "}": "%7D"}
    target = re.sub("[{}]".format("".join(map(re.escape, encodings.keys()))), lambda match: encodings[match.group(0)], title)

    # Remove formatting in the link target
    target = take_inner(r"''([^']+)''", take_inner(r"'''([^']+)'''", target))

    if type(r.last_edit_time) is datetime.datetime:
        old = (datetime.datetime.now() - r.last_edit_time).days > 60
        r.last_edit_time = ('style="background: red;" | ' if old else '') + r.last_edit_time.strftime(TIME_FORMAT_STRING)

    if type(r.last_botop_time) is datetime.datetime:
        r.last_botop_time = r.last_botop_time.strftime(TIME_FORMAT_STRING)

    elements = map(unicode, [row_number, target, title, replies, r.last_editor, r.last_edit_time, r.last_botop_editor, r.last_botop_time])
    return u"|-\n| {} || [[WP:Bot requests#{}|{}]] || {} || {} || {} || {} || {}".format(*elements)
{'fraction_non_alphanumeric': 0.21192541997415543, 'fraction_numerical': 0.0075687649990769795, 'mean_word_length': 4.380337636544191, 'pattern_counts': {'":': 17, '<': 9, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1530147', 'n_tokens_mistral': 2035, 'n_tokens_neox': 1954, 'n_words': 445}