<commit_before># -*- coding: utf-8 -*-

"""
Project Index -- prepares an index of WikiProjects and their in-scope pages for use by scripts
Copyright (C) 2015 <NAME>
Licensed under MIT License: http://mitlicense.org
"""

from collections import namedtuple
import re

from BTrees.IIBTree import IIBTree
from oursql import ProgrammingError

from reportsbot.task import Task
from reportsbot.util import to_wiki_format

__all__ = ["UpdateProjectIndex"]

Project = namedtuple("Project", ["id", "title", "categories"])
Page = namedtuple("Page", ["id", "talkid", "ns", "title", "isredir"])

class UpdateProjectIndex(Task):
    """Updates the index of articles associated with each WikiProject."""

    def __init__(self, bot):
        super().__init__(bot)
        self._page_table = bot.wikiid + "_page"
        self._project_table = bot.wikiid + "_project"
        self._index_table = bot.wikiid + "_index"

    def _create_tables(self, cursor):
        """Create this wiki's various tables, using base_* as references."""
        self._logger.info("Creating index tables for %s", self._bot.wikiid)

        query1 = "DROP TABLE IF EXISTS {}"
        query2 = "CREATE TABLE {} LIKE {}"

        cursor.execute(query1.format(self._index_table))
        cursor.execute(query1.format(self._project_table))
        cursor.execute(query1.format(self._page_table))

        cursor.execute(query2.format(self._page_table, "base_page"))
        cursor.execute(query2.format(self._project_table, "base_project"))
        cursor.execute(query2.format(self._index_table, "base_index"))

    def _ensure_tables(self):
        """Ensure that all necessary tables exist for this wiki."""
        query = "SELECT 1 FROM {} LIMIT 1"
        with self._bot.localdb as cursor:
            try:
                cursor.execute(query.format(self._page_table))
                cursor.execute(query.format(self._project_table))
                cursor.execute(query.format(self._index_table))
            except ProgrammingError:
                self._create_tables(cursor)

    def _get_project_categories(self):
        """Return a list of Wikiproject classification categories."""
        query = r"""SELECT page_title
            FROM page
            WHERE page_namespace = 14 AND (
                page_title LIKE "%-Class\_%\_articles" OR
                page_title LIKE "Unassessed\_%\_articles" OR
                page_title LIKE "WikiProject\_%\_articles"
            )"""

        with self._bot.wikidb as cursor:
            cursor.execute(query)
            return [cat.decode("utf8") for (cat,) in cursor.fetchall()]

    def _find_project(self, cursor, name):
        """Return the ID and title of the given WikiProject's root page."""
        query1 = """SELECT page_id, page_namespace, page_title,
                rd_namespace, rd_title
            FROM page LEFT JOIN redirect ON rd_from = page_id
            WHERE page_title = ? AND page_namespace = 4"""
        query2 = """SELECT page_id
            FROM page
            WHERE page_namespace = ? AND page_title = ?"""

        candidates = ("WikiProject_%s" % name, "WikiProject_%ss" % name, name)
        for candidate in candidates:
            cursor.execute(query1, (candidate,))
            results = cursor.fetchall()
            if results:
                break
        else:
            raise ValueError(name)

        pid, ns, title, rns, rtitle = results[0]
        if rns and rtitle:
            cursor.execute(query2, (rns, rtitle))
            result = cursor.fetchall()
            if not result:  # Broken redirect?
                raise ValueError(name)
            pid = result[0][0]
            ns, title = rns, rtitle

        ns_name = self._bot.site.namespaces[ns].custom_name
        fulltitle = to_wiki_format(ns_name + ":" + title.decode("utf8"))
        return pid, fulltitle

    def _get_projects(self):
        """Return a list of valid WikiProjects.

        Each project is a 3-tuple: (project_id, project_title, categories).
        """
        self._logger.info("Building project list")
        categories = self._get_project_categories()
        catmap = {}

        # Extract and map project names to their categories:
        regex = r"""^
            (?:.*?-Class_|Unassessed_)?
            (?:.*?-importance_)?
            (?:WikiProject_)?
            (.*?)                               # Actual project name fragment
            (?:-related|_task_force)?
            _articles$"""
        for cat in categories:
            name = re.search(regex, cat, flags=re.VERBOSE).group(1)
            if not name:  # Shouldn't happen, but would break us if it did
                continue
            name = name[0].upper() + name[1:]  # Normalize
            if name in catmap:
                catmap[name].append(cat)
            else:
                catmap[name] = [cat]

        # Build the Project objects:
        projects = {}
        with self._bot.wikidb as cursor:
            for name, cats in catmap.items():
                try:
                    pid, title = self._find_project(cursor, name)
                except ValueError:
                    msg = "Rejecting project: %s (%s categories, first: %s)"
                    self._logger.debug(msg, name, len(cats), cats[0])
                    continue

                if pid in projects:
                    projects[pid].categories.extend(cats)
                else:
                    projects[pid] = Project(pid, title, cats)

        msg = "%s projects with %s total categories"
        total_cats = sum(len(proj.categories) for proj in projects.values())
        self._logger.info(msg, len(projects), total_cats)
        return list(projects.values())

    def _sync_projects(self, cursor, projects):
        """Synchronize the given projects with the database."""
        self._logger.info("Synchronizing projects")

        query1 = "SELECT project_id, project_title FROM {}"
        query2 = """DELETE {0}, {1}
            FROM {0} LEFT JOIN {1} ON project_id = index_project
            WHERE project_id = ?"""
        query3 = "INSERT INTO {} (project_id, project_title) VALUES (?, ?)"
        query4 = "UPDATE {} SET project_title = ? WHERE project_id = ?"

        query1 = query1.format(self._project_table)
        query2 = query2.format(self._project_table, self._index_table)
        query3 = query3.format(self._project_table)
        query4 = query4.format(self._project_table)

        cursor.execute(query1)
        old = dict(cursor.fetchall())
        new = {proj.id: proj.title for proj in projects}

        to_remove = old.keys() - new.keys()
        to_add = new.keys() - old.keys()
        to_update = [pid for pid in new.keys() & old.keys()
                     if new[pid] != old[pid]]

        msg = "Remove/add/update: %s/%s/%s"
        self._logger.info(msg, len(to_remove), len(to_add), len(to_update))

        cursor.executemany(query2, [(pid,) for pid in to_remove])
        cursor.executemany(query3, [(pid, new[pid]) for pid in to_add])
        cursor.executemany(query4, [(new[pid], pid) for pid in to_update])

    def _get_talkmap(self, cursor):
        """Return a dict mapping talk page IDs to subject page IDs.

        Actually, this is misleading. The map also stores whether the subject
        page is a redirect, and for maximum space efficiency, we do this by
        shifting the subject page ID right by one bit and storing the boolean
        in the least significant bit.
        """
        self._logger.info("Building talkmap")

        query = """SELECT talk.page_id, subj.page_id, subj.page_is_redirect
        FROM page AS talk
        INNER JOIN page AS subj ON talk.page_title = subj.page_title
            AND talk.page_namespace = subj.page_namespace + 1
        WHERE talk.page_namespace % 2 = 1"""

        talkmap = IIBTree()
        cursor.execute(query)
        self._logger.debug("Fetching result chunks")
        while True:<commit_msg>Should be able to detect last few projects now.<commit_after># -*- coding: utf-8 -*-

"""
Project Index -- prepares an index of WikiProjects and their in-scope pages for use by scripts
Copyright (C) 2015 James Hare
Licensed under MIT License: http://mitlicense.org
"""

from collections import namedtuple
import re

from BTrees.IIBTree import IIBTree
from oursql import ProgrammingError

from reportsbot.task import Task
from reportsbot.util import to_wiki_format

__all__ = ["UpdateProjectIndex"]

Project = namedtuple("Project", ["id", "title", "categories"])
Page = namedtuple("Page", ["id", "talkid", "ns", "title", "isredir"])

class UpdateProjectIndex(Task):
    """Updates the index of articles associated with each WikiProject."""

    def __init__(self, bot):
        super().__init__(bot)
        self._page_table = bot.wikiid + "_page"
        self._project_table = bot.wikiid + "_project"
        self._index_table = bot.wikiid + "_index"

    def _create_tables(self, cursor):
        """Create this wiki's various tables, using base_* as references."""
        self._logger.info("Creating index tables for %s", self._bot.wikiid)

        query1 = "DROP TABLE IF EXISTS {}"
        query2 = "CREATE TABLE {} LIKE {}"

        cursor.execute(query1.format(self._index_table))
        cursor.execute(query1.format(self._project_table))
        cursor.execute(query1.format(self._page_table))

        cursor.execute(query2.format(self._page_table, "base_page"))
        cursor.execute(query2.format(self._project_table, "base_project"))
        cursor.execute(query2.format(self._index_table, "base_index"))

    def _ensure_tables(self):
        """Ensure that all necessary tables exist for this wiki."""
        query = "SELECT 1 FROM {} LIMIT 1"
        with self._bot.localdb as cursor:
            try:
                cursor.execute(query.format(self._page_table))
                cursor.execute(query.format(self._project_table))
                cursor.execute(query.format(self._index_table))
            except ProgrammingError:
                self._create_tables(cursor)

    def _get_project_categories(self):
        """Return a list of Wikiproject classification categories."""
        query = r"""SELECT page_title
            FROM page
            WHERE page_namespace = 14 AND (
                page_title LIKE "%-Class\_%\_articles" OR
                page_title LIKE "Unassessed\_%\_articles" OR
                page_title LIKE "WikiProject\_%\_articles"
            )"""

        with self._bot.wikidb as cursor:
            cursor.execute(query)
            return [cat.decode("utf8") for (cat,) in cursor.fetchall()]

    def _find_project(self, cursor, name):
        """Return the ID and title of the given WikiProject's root page."""
        query1 = """SELECT page_id, page_namespace, page_title,
                rd_namespace, rd_title
            FROM page LEFT JOIN redirect ON rd_from = page_id
            WHERE page_title LIKE ? ESCAPE "|" AND page_namespace = 4"""
        query2 = """SELECT page_id
            FROM page
            WHERE page_namespace = ? AND page_title = ?"""

        escname = name.replace("_", "|_")
        candidates = (
            "WikiProject|_%s" % escname,
            "WikiProject|_%ss" % escname,
            escname,
            "WikiProject|_?/%s|_task|_force" % escname,
            "WikiProject|_?/%s" % escname
        )

        for candidate in candidates:
            cursor.execute(query1, (candidate,))
            results = cursor.fetchall()
            if results and len(results) == 1:
                break
        else:
            raise ValueError(name)

        pid, ns, title, rns, rtitle = results[0]
        if rns and rtitle:
            cursor.execute(query2, (rns, rtitle))
            result = cursor.fetchall()
            if not result:  # Broken redirect?
                raise ValueError(name)
            pid = result[0][0]
            ns, title = rns, rtitle

        ns_name = self._bot.site.namespaces[ns].custom_name
        fulltitle = to_wiki_format(ns_name + ":" + title.decode("utf8"))
        return pid, fulltitle

    def _get_projects(self):
        """Return a list of valid WikiProjects.

        Each project is a 3-tuple: (project_id, project_title, categories).
        """
        self._logger.info("Building project list")
        categories = self._get_project_categories()
        catmap = {}

        # Extract and map project names to their categories:
        regex = r"""^
            (?:.*?-Class_|Unassessed_)?
            (?:.*?-importance_)?
            (?:WikiProject_)?
            (.*?)                               # Actual project name fragment
            (?:-related|_task_force)?
            _articles$"""
        for cat in categories:
            name = re.search(regex, cat, flags=re.VERBOSE).group(1)
            if not name:  # Shouldn't happen, but would break us if it did
                continue
            name = name[0].upper() + name[1:]  # Normalize
            if name in catmap:
                catmap[name].append(cat)
            else:
                catmap[name] = [cat]

        # Build the Project objects:
        projects = {}
        with self._bot.wikidb as cursor:
            for name, cats in catmap.items():
                try:
                    pid, title = self._find_project(cursor, name)
                except ValueError:
                    msg = "Rejecting project: %s (%s categories, first: %s)"
                    self._logger.debug(msg, name, len(cats), cats[0])
                    continue

                if pid in projects:
                    projects[pid].categories.extend(cats)
                else:
                    projects[pid] = Project(pid, title, cats)

        msg = "%s projects with %s total categories"
        total_cats = sum(len(proj.categories) for proj in projects.values())
        self._logger.info(msg, len(projects), total_cats)
        return list(projects.values())

    def _sync_projects(self, cursor, projects):
        """Synchronize the given projects with the database."""
        self._logger.info("Synchronizing projects")

        query1 = "SELECT project_id, project_title FROM {}"
        query2 = """DELETE {0}, {1}
            FROM {0} LEFT JOIN {1} ON project_id = index_project
            WHERE project_id = ?"""
        query3 = "INSERT INTO {} (project_id, project_title) VALUES (?, ?)"
        query4 = "UPDATE {} SET project_title = ? WHERE project_id = ?"

        query1 = query1.format(self._project_table)
        query2 = query2.format(self._project_table, self._index_table)
        query3 = query3.format(self._project_table)
        query4 = query4.format(self._project_table)

        cursor.execute(query1)
        old = dict(cursor.fetchall())
        new = {proj.id: proj.title for proj in projects}

        to_remove = old.keys() - new.keys()
        to_add = new.keys() - old.keys()
        to_update = [pid for pid in new.keys() & old.keys()
                     if new[pid] != old[pid]]

        for pid in to_remove:
            self._logger.debug("Remove: %s: %s", pid, old[pid])
        for pid in to_add:
            self._logger.debug("Add:    %s: %s", pid, new[pid])
        for pid in to_update:
            self._logger.debug("Update: %s: %s -> %s", pid, old[pid], new[pid])

        msg = "Remove/add/update: %s/%s/%s"
        self._logger.info(msg, len(to_remove), len(to_add), len(to_update))

        cursor.executemany(query2, [(pid,) for pid in to_remove])
        cursor.executemany(query3, [(pid, new[pid]) for pid in to_add])
        cursor.executemany(query4, [(new[pid], pid) for pid in to_update])

    def _get_talkmap(self, cursor):
        """Return a dict mapping talk page IDs to subject page IDs.

        Actually, this is misleading. The map also stores whether the subject
        page is a redirect, and for maximum space efficiency, we do this by
        shifting the subject page ID right by one bit and storing the boolean
        in the least significant bit.
        """
        self._logger.info("Building talkmap")

        query = """SELECT talk.page_id, subj.page_id, subj.page_is_redirect
        FROM page AS talk
        INNER JOIN page AS subj ON talk.page_title = subj.page_title
            AND talk.page_namespace = subj.page_namespace + 1
        WHERE talk.page_namespace % 2 = 1"""

        talkmap = IIBTree()
        cursor.execute(query)
        self._logger.debug("Fetching result chunks")
        while True:
{'fraction_non_alphanumeric': 0.11074008864064112, 'fraction_numerical': 0.006739117236354806, 'mean_word_length': 2.1689111196614084, 'pattern_counts': {'":': 2, '<': 4, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2109587', 'n_tokens_mistral': 4789, 'n_tokens_neox': 4541, 'n_words': 1296}