<commit_before>import os
from os.path import isdir, join

from page import Page, ParentPage


class Category(object):
    def __init__(self, site, path):
        self.site = site
        self.path = path
        self._pages = None

    def load(self):
        pages = [p for p in self._list_dir() if p.endswith('.page')]
        self._pages = [self._new_page(p) for p in pages]

    @property
    def pages(self):
        if self._pages is None:
            self.load()
        return self._pages

    @property
    def categories(self):
        return [Category(self.site, p) for p in self._list_dir() if isdir(p)]

    def _list_dir(self):
        return [join(self.path, f) for f in os.listdir(self.path)]

    @property
    def url_path(self):<commit_msg>respect include_drafts flag<commit_after>import os
from os.path import isdir, join

from page import Page, ParentPage


class Category(object):
    def __init__(self, site, path):
        self.site = site
        self.path = path
        self._pages = None

    def load(self):
        pages = [p for p in self._list_dir() if p.endswith('.page')]
        pages = [self._new_page(p) for p in pages]
        if not self.site.include_drafts:
            pages = [p for p in pages if p.published]
        self._pages = pages

    @property
    def pages(self):
        if self._pages is None:
            self.load()
        return self._pages

    @property
    def categories(self):
        return [Category(self.site, p) for p in self._list_dir() if isdir(p)]

    def _list_dir(self):
        return [join(self.path, f) for f in os.listdir(self.path)]

    @property
    def url_path(self):
{'fraction_non_alphanumeric': 0.10673135233474833, 'fraction_numerical': 0.0, 'mean_word_length': 2.3333333333333335, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5355523', 'n_tokens_mistral': 527, 'n_tokens_neox': 512, 'n_words': 162}