<commit_before>import getpass
import json
import re
from wikitools.wiki import Wiki
from wikitools.page import Page
from wikitools import api

class DYKNotifier():
    """
    A Wikipedia bot to notify an editor if an article they had created/expanded
    was nominated for DYK by someone else.
    """

    def __init__(self):
        self._wiki = Wiki("http://en.wikipedia.org/w/api.php")
        self._ttdyk = Page(self._wiki, title="Template talk:Did you know")
        self._dyk_noms = self.get_list_of_dyk_noms_from_ttdyk()
        self._people_to_notify = dict()

        # CONFIGURATION
        self.actually_editing = raw_input("Actually edit (y/n)? ") == "y"
        self._summary = "[[Wikipedia:Bots/Requests for approval/APersonBot " +\
                        "2|Robot]] notification about the DYK nomination of" +\
                        "%(nom_name)s."

        # LOGIN
        def attempt_login():
            username = raw_input("Username: ")
            password = <PASSWORD>()
            self._wiki.login(username, password)
        if self.actually_editing:
            attempt_login()
            while not self._wiki.isLoggedIn():
                print "Error logging in. Try again."
                attempt_login()
            print "Successfully logged in as " + self._wiki.username + "."
        else:
            print "Won't be editing, so no need to log in."

    #################
    ##
    ## MAIN FUNCTIONS
    ##
    #################

    def get_list_of_dyk_noms_from_ttdyk(self):
        """
        Returns a list of subpages of T:DYKN nominated for DYK.
        """
        dyk_noms = []
        all_templates = self._ttdyk.getTemplates()
        print "Got all " + str(len(all_templates)) + " templates from T:DYKN."
        for template in all_templates:
            if template.startswith("Template:Did you know nominations/"):
                dyk_noms.append(template)
        print "Read " + str(len(dyk_noms)) + " noms from T:DYKN."
        return dyk_noms

    def run(self):
        """
        Runs the task.
        """
        self.remove_resolved_noms()
        self.remove_self_nominated_noms()
        self.get_people_to_notify()
        self.prune_list_of_people()
        self.notify_people()
        print "[run()] Notified people."

    def remove_resolved_noms(self):
        """
        Removes all resolved noms from the list of DYK noms.
        """
        def resolved_handler(page):
            if self.should_prune_as_resolved(page):
                self._dyk_noms.remove(page["title"])
        dyk_noms_strings = self.list_to_pipe_separated_query(self._dyk_noms)
        self.run_query(dyk_noms_strings, {"prop":"categories"},
                       resolved_handler)
        print "[remove_resolved_noms()] Done. " + str(len(self._dyk_noms)) +\
              " noms left."

    def remove_self_nominated_noms(self):
        """
        Removes all resolved noms from the list of DYK noms.
        """
        def resolved_handler(page):
            if self.should_prune_as_self_nom(page):
                self._dyk_noms.remove(page["title"])
        dyk_noms_strings = self.list_to_pipe_separated_query(self._dyk_noms)
        self.run_query(dyk_noms_strings,
                       {"prop":"revisions", "rvprop":"content"},
                       resolved_handler)
        print "[remove_self_nominated_noms()] Done. " +\
              str(len(self._dyk_noms)) + " noms left."

    def should_prune_as_resolved(self, page):
        """
        Given a page, should it be pruned from the list of DYK noms
        since it's already been passed or failed?
        """
        try:
            test = page["categories"]
        except KeyError:
            return False
        for category in page["categories"]:
            if "Category:Passed DYK nominations" in category["title"] or\
               "Category:Failed DYK nominations" in category["title"]:
                return True
        return False

    def should_prune_as_self_nom(self, page):
        wikitext = ""
        try:
            wikitext = page["revisions"][0]["*"]
        except KeyError:
            return False
        return "Self nominated" in wikitext          
                
    def get_people_to_notify(self):
        """
        Returns a dict of user talkpages to notify about their creations and
        the noms about which they should be notified.
        """
        print "Getting whom to notify for " + str(len(self._dyk_noms)) +\
              " noms..."
        dyk_noms_strings = self.list_to_pipe_separated_query(self._dyk_noms)
        eventual_count = (len(self._dyk_noms) // 50) +\
                         (cmp(len(self._dyk_noms), 0))
        count = 1
        for dyk_noms_string in dyk_noms_strings:
            params = {"action":"query", "titles":dyk_noms_string,\
                      "prop":"revisions", "rvprop":"content"}
            api_request = api.APIRequest(self._wiki, params)
            api_result = api_request.query()
            print "Processing results from query number " + str(count) +\
                  " out of " + str(eventual_count) + "..."
            for wikitext, title in [(page["revisions"][0]["*"], page["title"])\
                                    for page in\
                                    api_result["query"]["pages"].values()]:
                success, talkpages = self._get_who_to_nominate_from_wikitext(\
                    wikitext, title)
                if success:
                    self._people_to_notify.update(talkpages)
            count += 1
        print "There are" + str(len(self._people_to_notify)) +\
              " people to notify."

    def prune_list_of_people(self):
        """
        Removes three types of people who shouldn't be notified from the list:
        people with blank usernames or nomination names; people with {{bots}}
        indicating that they shouldn't be notified by this bot; and people
        who have already been notified about the specific nomination (i.e. not
        people who've already been notified about a different nomination)
        """
        initial_count = len(self._people_to_notify)
        # Remove entries with empty keys
        self._people_to_notify = dict([(x, y) for x, y in\
                                       self._people_to_notify.iteritems() if x])
        
        # Remove people using {{bots}} to exclude this bot AND already-notified
        # people in one go, since both use wikitext.
        def handler(page):
            wikitext, title = "", ""
            try:
                wikitext, title = page["revisions"][0]["*"], page["title"]
            except KeyError:
                return
            if self._is_excluded_given_wikitext(wikitext) or\
               self._is_already_notified(wikitext,\
                                         self.people_to_notify[title]):
                del self._people_to_notify[title]
        titles_string = self.list_to_pipe_separated_query(\
            self._people_to_notify.keys())
        self.run_query(titles_string, {"prop":"revisions", "rvprop":"content"},\
                       handler)
        print "Removed " + str(initial_count - len(self._people_to_notify)) +\
              " people. " + str(len(self._people_to_notify)) + " people left."

    def notify_people(self):
        """
        Substitutes User:APersonBot/DYKNotice at the end of each page in a list
        of user talkpages, given a list of usernames.
        """
        for person in self._people_to_notify:
            nom_name = self._people_to_notify[person]
            template = "{{subst:DYKNom|" +\
                       nom_name + "}}"
            print "ABOUT TO NOTIFY " + str(person)
            talkpage = Page(self._wiki, title="User talk:" + person)
            if self.actually_editing:
                result = talkpage.edit(appendtext=template, bot=True,\
                                       summary=self._summary %\
                                       {"nom_name":nom_name})
                print result
            print "Notified " + person + " because of " + nom_name + "."

    #################
    ##
    ## IMPORTANT HELPER FUNCTIONS<commit_msg>fixes<commit_after>import getpass
import json
import re
from wikitools.wiki import Wiki
from wikitools.page import Page
from wikitools import api

class DYKNotifier():
    """
    A Wikipedia bot to notify an editor if an article they had created/expanded
    was nominated for DYK by someone else.
    """

    def __init__(self):
        self._wiki = Wiki("http://en.wikipedia.org/w/api.php")
        self._ttdyk = Page(self._wiki, title="Template talk:Did you know")
        self._dyk_noms = self.get_list_of_dyk_noms_from_ttdyk()
        self._people_to_notify = dict()

        # CONFIGURATION
        self.actually_editing = raw_input("Actually edit (y/n)? ") == "y"
        self._summary = "[[Wikipedia:Bots/Requests for approval/APersonBot " +\
                        "2|Robot]] notification about the DYK nomination of" +\
                        "%(nom_name)s."

        # LOGIN
        def attempt_login():
            username = raw_input("Username: ")
            password = getpass.get<PASSWORD>()
            self._wiki.login(username, password)
        if self.actually_editing:
            attempt_login()
            while not self._wiki.isLoggedIn():
                print "Error logging in. Try again."
                attempt_login()
            print "Successfully logged in as " + self._wiki.username + "."
        else:
            print "Won't be editing, so no need to log in."

    #################
    ##
    ## MAIN FUNCTIONS
    ##
    #################

    def get_list_of_dyk_noms_from_ttdyk(self):
        """
        Returns a list of subpages of T:DYKN nominated for DYK.
        """
        dyk_noms = []
        all_templates = self._ttdyk.getTemplates()
        print "Got all " + str(len(all_templates)) + " templates from T:DYKN."
        for template in all_templates:
            if template.startswith("Template:Did you know nominations/"):
                dyk_noms.append(template)
        print "Read " + str(len(dyk_noms)) + " noms from T:DYKN."
        return dyk_noms

    def run(self):
        """
        Runs the task.
        """
        self.remove_resolved_noms()
        self.remove_self_nominated_noms()
        self.get_people_to_notify()
        self.prune_list_of_people()
        self.notify_people()
        print "[run()] Notified people."

    def remove_resolved_noms(self):
        """
        Removes all resolved noms from the list of DYK noms.
        """
        def resolved_handler(page):
            if self.should_prune_as_resolved(page):
                self._dyk_noms.remove(page["title"])
        dyk_noms_strings = self.list_to_pipe_separated_query(self._dyk_noms)
        self.run_query(dyk_noms_strings, {"prop":"categories"},
                       resolved_handler)
        print "[remove_resolved_noms()] Done. " + str(len(self._dyk_noms)) +\
              " noms left."

    def remove_self_nominated_noms(self):
        """
        Removes all resolved noms from the list of DYK noms.
        """
        def resolved_handler(page):
            if self.should_prune_as_self_nom(page):
                self._dyk_noms.remove(page["title"])
        dyk_noms_strings = self.list_to_pipe_separated_query(self._dyk_noms)
        self.run_query(dyk_noms_strings,
                       {"prop":"revisions", "rvprop":"content"},
                       resolved_handler)
        print "[remove_self_nominated_noms()] Done. " +\
              str(len(self._dyk_noms)) + " noms left."

    def should_prune_as_resolved(self, page):
        """
        Given a page, should it be pruned from the list of DYK noms
        since it's already been passed or failed?
        """
        try:
            test = page["categories"]
        except KeyError:
            return False
        for category in page["categories"]:
            if "Category:Passed DYK nominations" in category["title"] or\
               "Category:Failed DYK nominations" in category["title"]:
                return True
        return False

    def should_prune_as_self_nom(self, page):
        wikitext = ""
        try:
            wikitext = page["revisions"][0]["*"]
        except KeyError:
            return False
        return "Self nominated" in wikitext          
                
    def get_people_to_notify(self):
        """
        Returns a dict of user talkpages to notify about their creations and
        the noms about which they should be notified.
        """
        print "Getting whom to notify for " + str(len(self._dyk_noms)) +\
              " noms..."
        dyk_noms_strings = self.list_to_pipe_separated_query(self._dyk_noms)
        eventual_count = (len(self._dyk_noms) // 50) +\
                         (cmp(len(self._dyk_noms), 0))
        count = 1
        for dyk_noms_string in dyk_noms_strings:
            params = {"action":"query", "titles":dyk_noms_string,\
                      "prop":"revisions", "rvprop":"content"}
            api_request = api.APIRequest(self._wiki, params)
            api_result = api_request.query()
            print "Processing results from query number " + str(count) +\
                  " out of " + str(eventual_count) + "..."
            for wikitext, title in [(page["revisions"][0]["*"], page["title"])\
                                    for page in\
                                    api_result["query"]["pages"].values()]:
                success, talkpages = self._get_who_to_nominate_from_wikitext(\
                    wikitext, title)
                if success:
                    self._people_to_notify.update(talkpages)
            count += 1
        print "There are " + str(len(self._people_to_notify)) +\
              " people to notify."

    def prune_list_of_people(self):
        """
        Removes three types of people who shouldn't be notified from the list:
        people with blank usernames or nomination names; people with {{bots}}
        indicating that they shouldn't be notified by this bot; and people
        who have already been notified about the specific nomination (i.e. not
        people who've already been notified about a different nomination)
        """
        initial_count = len(self._people_to_notify)
        # Remove entries with empty keys
        self._people_to_notify = dict([(x, y) for x, y in\
                                       self._people_to_notify.iteritems() if x])
        
        # Remove people using {{bots}} to exclude this bot AND already-notified
        # people in one go, since both use wikitext.
        def handler(page):
            wikitext, title = "", ""
            try:
                wikitext, title = page["revisions"][0]["*"], page["title"]
            except KeyError:
                return
            if self._is_excluded_given_wikitext(wikitext) or\
               self._is_already_notified(wikitext,\
                                         self._people_to_notify[title]):
                del self._people_to_notify[title]
        titles_string = self.list_to_pipe_separated_query(\
            self._people_to_notify.keys())
        self.run_query(titles_string, {"prop":"revisions", "rvprop":"content"},\
                       handler)
        print "Removed " + str(initial_count - len(self._people_to_notify)) +\
              " people. " + str(len(self._people_to_notify)) + " people left."

    def notify_people(self):
        """
        Substitutes User:APersonBot/DYKNotice at the end of each page in a list
        of user talkpages, given a list of usernames.
        """
        for person in self._people_to_notify:
            nom_name = self._people_to_notify[person]
            template = "\n\n{{subst:DYKNom|" +\
                       nom_name[34:] + "|passive=yes}}"
            print "ABOUT TO NOTIFY " + str(person)
            talkpage = Page(self._wiki, title="User talk:" + person)
            if self.actually_editing:
                # cross fingers here
                result = talkpage.edit(appendtext=template, bot=True,\
                                       summary=self._summary %\
                                       {"nom_name":nom_name})
                print result
            print "Notified " + person + " because of " + nom_name + "."

    #################
    ##
    ## IMPORTANT HELPER FUNCTIONS
{'fraction_non_alphanumeric': 0.10642168455521193, 'fraction_numerical': 0.0012127827299739252, 'mean_word_length': 1.8756756756756756, 'pattern_counts': {'":': 20, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3277563', 'n_tokens_mistral': 4828, 'n_tokens_neox': 4606, 'n_words': 1127}