<commit_before>from datetime import datetime
import re

import mwparserfromhell
import pywikibot


pywikibot.config.family = "wikidata"
pywikibot.config.mylang = "wikidata"

site = pywikibot.Site()
site.login()


class RFORArchiverBot(object):
    """Archives closed requests on [[Wikidata:Requests for permissions/Other rights]]"""

    def __init__(self):
        self.base_page = pywikibot.Page(site, "Wikidata:Requests for permissions")
        self.requests_page = pywikibot.Page(site, self.base_page.title() + "/Other rights")
        self.archive_titles = {
            "confirmed": self.base_page.title() + "/RfConfirmed/%B %Y",
            "ipblock-exempt": self.base_page.title() + "/RfIPBE/%Y",  # archived by the year
            "propertycreator": self.base_page.title() + "/RfPropertyCreator/%B %Y",
            "rollbacker": self.base_page.title() + "/RfRollback/%B %Y"
        }
        self.closed_regex = "\n\{\{\s*(?:A(?:rchive)?|D(?:iscussion)?) ?top\|.*?\}\}.*?"
        self.closed_regex += "\{\{\s*(?:A(?:rchive)?|D(?:iscussion)?) ?bottom\}\}\n"
        self.archive_text = "{{Archive|category=Archived requests for permissions}}"
        self.archive_text += "\n__TOC__\n{{Discussion top}}"

    def run(self):
        text = self.requests_page.get()
        code = mwparserfromhell.parser.Parser().parse(text, skip_style_tags=True)
        for section in code.get_sections(levels=[2]):
            heading = section.filter_headings()[0].title.lower()
            if "confirmed" in heading:
                group = "confirmed"
            elif "ipbe" in heading:
                group = "ipblock-exempt"
            elif "property" in heading:
                group = "propertycreator"
            elif "rollback" in heading:
                group = "rollbacker"
            else:
                continue
            archivable = []
            for discussion in section.get_sections(levels=[3]):
                templates = [template.name.lower().strip() for template in discussion.ifilter_templates()]
                if not ("done" in templates or "not done" in templates or "notdone" in templates):
                    continue
                timestamps = re.findall(
                    "\d{1,2}:\d{2},\s\d{1,2}\s\D{3,9}\s\d{4}\s\(UTC\)", unicode(discussion)
                )
                timestamps = sorted(
                    datetime.strptime(timestamp[:-6], "%H:%M, %d %B %Y") for timestamp in timestamps
                )
                if (datetime.utcnow() - timestamps[-1]).days >= 5:
                    archivable.append(discussion)
            if not archivable:
                continue<commit_msg>Change unicode to str for Python 2->3<commit_after>from datetime import datetime
import re

import mwparserfromhell
import pywikibot


pywikibot.config.family = "wikidata"
pywikibot.config.mylang = "wikidata"

site = pywikibot.Site()
site.login()


class RFORArchiverBot:
    """Archives closed requests on [[Wikidata:Requests for permissions/Other rights]]"""

    def __init__(self):
        self.base_page = pywikibot.Page(site, "Wikidata:Requests for permissions")
        self.requests_page = pywikibot.Page(site, self.base_page.title() + "/Other rights")
        self.archive_titles = {
            "confirmed": self.base_page.title() + "/RfConfirmed/%B %Y",
            "ipblock-exempt": self.base_page.title() + "/RfIPBE/%Y",  # archived by the year
            "propertycreator": self.base_page.title() + "/RfPropertyCreator/%B %Y",
            "rollbacker": self.base_page.title() + "/RfRollback/%B %Y"
        }
        self.closed_regex = "\n\{\{\s*(?:A(?:rchive)?|D(?:iscussion)?) ?top\|.*?\}\}.*?"
        self.closed_regex += "\{\{\s*(?:A(?:rchive)?|D(?:iscussion)?) ?bottom\}\}\n"
        self.archive_text = "{{Archive|category=Archived requests for permissions}}"
        self.archive_text += "\n__TOC__\n{{Discussion top}}"

    def run(self):
        text = self.requests_page.get()
        code = mwparserfromhell.parser.Parser().parse(text, skip_style_tags=True)
        for section in code.get_sections(levels=[2]):
            heading = section.filter_headings()[0].title.lower()
            if "confirmed" in heading:
                group = "confirmed"
            elif "ipbe" in heading:
                group = "ipblock-exempt"
            elif "property" in heading:
                group = "propertycreator"
            elif "rollback" in heading:
                group = "rollbacker"
            else:
                continue
            archivable = []
            for discussion in section.get_sections(levels=[3]):
                templates = [template.name.lower().strip() for template in discussion.ifilter_templates()]
                if not ("done" in templates or "not done" in templates or "notdone" in templates):
                    continue
                timestamps = re.findall(
                    "\d{1,2}:\d{2},\s\d{1,2}\s\D{3,9}\s\d{4}\s\(UTC\)", str(discussion)
                )
                timestamps = sorted(
                    datetime.strptime(timestamp[:-6], "%H:%M, %d %B %Y") for timestamp in timestamps
                )
                if (datetime.utcnow() - timestamps[-1]).days >= 5:
                    archivable.append(discussion)
            if len(archivable) == 0:
                continue
{'fraction_non_alphanumeric': 0.14527662777568687, 'fraction_numerical': 0.005833646970267219, 'mean_word_length': 2.165574746873139, 'pattern_counts': {'":': 8, '<': 3, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2986301', 'n_tokens_mistral': 1659, 'n_tokens_neox': 1578, 'n_words': 329}