<commit_before>import os
import re

from datetime import datetime
from houdini import escape_html
from tornado.util import ObjectDict

from catsup.logger import logger
from catsup.options import g
from catsup.parser import markdown
from catsup.utils import to_unicode
from .utils import Pagination


class CatsupPage(object):
    @property
    def class_name(self):
        return self.__class__.__name__.lower()

    def get_permalink_args(self):
        return {}

    @property
    def permalink(self):
        kwargs = self.__dict__.copy()
        kwargs.update(self.get_permalink_args())
        return g.permalink[self.class_name].format(**kwargs).replace(" ", "-")

    def render(self, renderer, **kwargs):
        if hasattr(self, "template_name"):
            template_name = self.template_name
        else:
            template_name = self.class_name + ".html"
        kwargs[self.class_name] = self
        kwargs.update(self.__dict__)
        renderer.render_to(template_name, self.permalink, **kwargs)


class Tag(CatsupPage):
    def __init__(self, name):
        self.name = name
        self.posts = []

    def add_post(self, post):
        self.posts.append(post)

    @property
    def count(self):
        return len(self.posts)

    def __iter__(self):
        for post in self.posts:
            yield post


class Tags(CatsupPage):
    def __init__(self, tags=None):
        if tags is None:
            tags = {}
        self.tags = tags

    def get(self, name):
        return self.tags.setdefault(
            name,
            Tag(name)
        )

    def render(self, renderer, **kwargs):
        self.tags = self.tags.values()
        self.tags.sort(
            key=lambda x: x.count,
            reverse=True
        )
        for tag in self.tags:
            tag.render(renderer)
        super(Tags, self).render(renderer, **kwargs)

    def __iter__(self):
        for tag in self.tags:
            yield tag


class Archive(CatsupPage):
    def __init__(self, year):
        self.year = int(year)
        self.posts = []

    def add_post(self, post):
        self.posts.append(post)

    @property
    def count(self):
        return len(self.posts)

    def __iter__(self):
        for post in self.posts:
            yield post


class Archives(CatsupPage):
    def __init__(self, archives=None):
        if archives is None:
            archives = {}
        self.archives = archives

    def get(self, year):
        return self.archives.setdefault(
            year,
            Archive(year)
        )

    def render(self, renderer, **kwargs):
        self.archives = self.archives.values()
        self.archives.sort(
            key=lambda x: x.year,
            reverse=True
        )
        for tag in self.archives:
            tag.render(renderer)
        super(Archives, self).render(renderer, **kwargs)

    def __iter__(self):
        for tag in self.archives:
            yield tag


class Post(CatsupPage):
    DATE_RE = re.compile('\d{4}\-\d{2}\-\d{2}')

    def __init__(self, filename, ext):
        self.meta = ObjectDict()
        path = os.path.join(g.source, "%s%s" % (filename, ext))
        self.filename = filename
        self.parse(path)

    def get_permalink_args(self):
        return self.meta

    def parse(self, path):
        try:
            with open(path, "r") as f:
                lines = f.readlines()
        except IOError:
            logger.error("Can't open file %s" % path)
            exit(1)

        def invailed_post():
            logger.error("%s is not a vailed catsup post" % self.filename)
            exit(1)

        title = lines.pop(0)
        if title.startswith("#"):
            self.title = escape_html(title[1:].strip())
        else:
            invailed_post()<commit_msg>improve Archives model and Tags model<commit_after>import os
import re

from datetime import datetime
from houdini import escape_html
from tornado.util import ObjectDict

from catsup.logger import logger
from catsup.options import g
from catsup.parser import markdown
from catsup.utils import to_unicode
from .utils import Pagination


class CatsupPage(object):
    @property
    def class_name(self):
        return self.__class__.__name__.lower()

    def get_permalink_args(self):
        return {}

    @property
    def permalink(self):
        kwargs = self.__dict__.copy()
        kwargs.update(self.get_permalink_args())
        return g.permalink[self.class_name].format(**kwargs).replace(" ", "-")

    def render(self, renderer, **kwargs):
        if hasattr(self, "template_name"):
            template_name = self.template_name
        else:
            template_name = self.class_name + ".html"
        kwargs[self.class_name] = self
        kwargs.update(self.__dict__)
        renderer.render_to(template_name, self.permalink, **kwargs)


class Tag(CatsupPage):
    def __init__(self, name):
        self.name = name
        self.posts = []

    def add_post(self, post):
        self.posts.append(post)

    @property
    def count(self):
        return len(self.posts)

    def __iter__(self):
        for post in self.posts:
            yield post


class Tags(CatsupPage):
    def __init__(self, tags=None):
        if tags is None:
            tags = {}
        self.tags_dict = tags

    def get(self, name):
        return self.tags_dict.setdefault(
            name,
            Tag(name)
        )

    def render(self, renderer, **kwargs):
        for tag in self.tags:
            tag.render(renderer)
        super(Tags, self).render(renderer, **kwargs)

    @property
    def tags(self):
        if not hasattr(self, "_tags"):
            self._tags = self.tags_dict.values()
            self._tags.sort(
                key=lambda x: x.count,
                reverse=True
            )
        return self._tags

    def __iter__(self):
        for tag in self.tags:
            yield tag


class Archive(CatsupPage):
    def __init__(self, year):
        self.year = int(year)
        self.posts = []

    def add_post(self, post):
        self.posts.append(post)

    @property
    def count(self):
        return len(self.posts)

    def __iter__(self):
        for post in self.posts:
            yield post


class Archives(CatsupPage):
    def __init__(self, archives=None):
        if archives is None:
            archives = {}
        self.archives_dict = archives

    def get(self, year):
        return self.archives_dict.setdefault(
            year,
            Archive(year)
        )

    def render(self, renderer, **kwargs):
        for tag in self.archives:
            tag.render(renderer)
        super(Archives, self).render(renderer, **kwargs)

    @property
    def archives(self):
        if not hasattr(self, "_archives"):
            self._archives = self.archives_dict.values()
            self._archives.sort(
                key=lambda x: x.year,
                reverse=True
            )
        return self._archives

    def __iter__(self):
        for archive in self.archives:
            yield archive


class Post(CatsupPage):
    DATE_RE = re.compile('\d{4}\-\d{2}\-\d{2}')

    def __init__(self, filename, ext):
        self.meta = ObjectDict()
        path = os.path.join(g.source, "%s%s" % (filename, ext))
        self.filename = filename
        self.parse(path)

    def get_permalink_args(self):
        return self.meta

    def parse(self, path):
        try:
            with open(path, "r") as f:
                lines = f.readlines()
        except IOError:
            logger.error("Can't open file %s" % path)
            exit(1)

        def invailed_post():
            logger.error("%s is not a vailed catsup post" % self.filename)
            exit(1)

        title = lines.pop(0)
        if title.startswith("#"):
            self.title = escape_html(title[1:].strip())
        else:
            invailed_post()
{'fraction_non_alphanumeric': 0.10809439228622177, 'fraction_numerical': 0.0017761989342806395, 'mean_word_length': 1.9747169811320755, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4502269', 'n_tokens_mistral': 2396, 'n_tokens_neox': 2326, 'n_words': 572}