<commit_before>from datetime import datetime
from pathlib import Path

import os.path
import phpserialize


class Page(object):
    """Metadata and content of a single DokuWiki page."""

    def __init__(self, dokuwiki_base_dir, path, lazy_load_content=True):
        """
        Represents a single DokuWiki page.

        :param dokuwiki_base_dir: The base directory of the DokuWiki install.
            It's the one that contains the ``data`` directory.
        :param path: The URL path identifying the page, e.g. ``docs:dev:setup``.
        :param lazy_load_content: By default, only metadata is initially loaded.
            The full page text is loaded lazily. Change this behavior by setting
            this to False.
        """
        self._text = None
        self._changes = None
        self.path = path
        self._lazy_load = lazy_load_content
        self._base_dir = dokuwiki_base_dir
        self.reload()

    def reload(self):
        """
        Purges the page's metadata and, if loaded, text content, and re-reads it
        from storage.
        """
        self._load_metadata()
        if self._lazy_load:
            self.purge_text()
        else:
            self._text = self._load_text()
            self._load_changes()

    def purge_text(self):
        """
        Purges the page text from memory. This is useful when lazy loading and
        trying to keep memory consumption low.
        """
        self._text = None

    @property
    def text(self):
        """
        Returns the text content of the page. May be loaded lazily if onfigured
        to do so.
        """
        if self._text is None and self._lazy_load:
            self._text = self._load_text()
        return self._text

    @property
    def deleted(self):
        """
        Checks the history of a page to see if it was deleted in its last
        change. Pre-installed pages may not have a history, so they are assumed
        to not have been deleted.

        :return: True if the page is currently deleted.
        """
        if self._changes is None and self._lazy_load:
            self._load_changes()
        # The last line contains the most recent change. The third column holds
        # a single character indicator of the change's nature, 'C' for creation,
        # 'E' for editing, and 'D' for deletion.
        return len(self._changes) == 0 or self._changes[-1][2] == 'D'

    def _load_metadata(self):
        metadata_file_path = self._base_dir + '/data/meta/' + \
                             self.path.replace(':', '/') + '.meta'
        if not os.path.isfile(metadata_file_path):
            print(metadata_file_path)
            raise ValueError('The requested page does not exist.')
        with open(metadata_file_path, 'r') as metadata_file:
            metadata = phpserialize.loads(metadata_file.read().encode('utf-8'))

        self.title = self._get_title(metadata)
        self.description = self._get_description(metadata)
        self.creator = self._get_creator(metadata)
        self.contributors = self._get_contributors(metadata)
        self.created_at = self._get_create_date(metadata)
        self.updated_at = self._get_modify_date(metadata)

    def _load_changes(self):
        change_file_path = self._base_dir + '/data/meta/' + \
                           self.path.replace(':', '/') + '.changes'
        if not os.path.isfile(change_file_path):
            # Pages without change history are pre-installed and have not been
            # deleted yet.
            self._changes = []
            return

        with open(change_file_path, 'r') as change_file:
            raw_changes = change_file.readlines()
        self._changes = [line.split('\t') for line in raw_changes]

    @staticmethod
    def _get_title(metadata):
        try:
            return metadata[b'current'].get('title', None)
        except KeyError:
            return None

    @staticmethod
    def _get_description(metadata):
        """
        Retrieves the page description from metadata, but only if it's not
        empty.
        """
        try:
            if len(metadata[b'current'][b'description']) > 0:<commit_msg>If not available in metadata, guess the title from content<commit_after>from datetime import datetime
from pathlib import Path
import re

import os.path
import phpserialize


class Page(object):
    """Metadata and content of a single DokuWiki page."""

    def __init__(self, dokuwiki_base_dir, path, lazy_load_content=True):
        """
        Represents a single DokuWiki page.

        :param dokuwiki_base_dir: The base directory of the DokuWiki install.
            It's the one that contains the ``data`` directory.
        :param path: The URL path identifying the page, e.g. ``docs:dev:setup``.
        :param lazy_load_content: By default, only metadata is initially loaded.
            The full page text is loaded lazily. Change this behavior by setting
            this to False.
        """
        self._text = None
        self._changes = None
        self.path = path
        self._lazy_load = lazy_load_content
        self._base_dir = dokuwiki_base_dir
        self.reload()

    def reload(self):
        """
        Purges the page's metadata and, if loaded, text content, and re-reads it
        from storage.
        """
        self._load_metadata()
        if self._lazy_load:
            self.purge_text()
        else:
            self._text = self._load_text()
            self._load_changes()

    def purge_text(self):
        """
        Purges the page text from memory. This is useful when lazy loading and
        trying to keep memory consumption low.
        """
        self._text = None

    @property
    def text(self):
        """
        Returns the text content of the page. May be loaded lazily if onfigured
        to do so.
        """
        if self._text is None and self._lazy_load:
            self._text = self._load_text()
        return self._text

    @property
    def deleted(self):
        """
        Checks the history of a page to see if it was deleted in its last
        change. Pre-installed pages may not have a history, so they are assumed
        to not have been deleted.

        :return: True if the page is currently deleted.
        """
        if self._changes is None and self._lazy_load:
            self._load_changes()
        # The last line contains the most recent change. The third column holds
        # a single character indicator of the change's nature, 'C' for creation,
        # 'E' for editing, and 'D' for deletion.
        return len(self._changes) == 0 or self._changes[-1][2] == 'D'

    def _load_metadata(self):
        metadata_file_path = self._base_dir + '/data/meta/' + \
                             self.path.replace(':', '/') + '.meta'
        if not os.path.isfile(metadata_file_path):
            print(metadata_file_path)
            raise ValueError('The requested page does not exist.')
        with open(metadata_file_path, 'r') as metadata_file:
            metadata = phpserialize.loads(metadata_file.read().encode('utf-8'))

        self.title = self._get_title(metadata)
        self.description = self._get_description(metadata)
        self.creator = self._get_creator(metadata)
        self.contributors = self._get_contributors(metadata)
        self.created_at = self._get_create_date(metadata)
        self.updated_at = self._get_modify_date(metadata)

    def _load_changes(self):
        change_file_path = self._base_dir + '/data/meta/' + \
                           self.path.replace(':', '/') + '.changes'
        if not os.path.isfile(change_file_path):
            # Pages without change history are pre-installed and have not been
            # deleted yet.
            self._changes = []
            return

        with open(change_file_path, 'r') as change_file:
            raw_changes = change_file.readlines()
        self._changes = [line.split('\t') for line in raw_changes]

    def _get_title(self, metadata):
        """
        Retrieves the title from metadata. If none exists there, the first
        heading of the content is used. If that one doesn't exist either, the
        title is None.

        :param metadata: The metadata file to read.
        :return: Page title or None.
        """
        try:
            title = metadata[b'current'].get('title', None)
            if title is None:
                title = self._extract_title_from_content()
            return title
        except KeyError:
            return None

    def _extract_title_from_content(self):
        """
        Extracts the content of the first heading in the content, to be used as
        a title. This should be done in case the metadata does not contain it.

        :return: The text of the first heading in the content, or None if no
            headings are found.
        """
        content = self.text.splitlines()
        title = None
        for line in content:
            match_result = re.match('={2,6}(.*?)={2,6}', line)
            if match_result is not None:
                title = match_result.group(1).strip()
                break
        return title


    @staticmethod
    def _get_description(metadata):
        """
        Retrieves the page description from metadata, but only if it's not
        empty.
        """
        try:
            if len(metadata[b'current'][b'description']) > 0:
{'fraction_non_alphanumeric': 0.08857264231096007, 'fraction_numerical': 0.0015930331350892098, 'mean_word_length': 2.0475728155339805, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2104518', 'n_tokens_mistral': 2583, 'n_tokens_neox': 2530, 'n_words': 845}