<commit_before>from __future__ import with_statement
import os.path, re, datetime, urllib, codecs, functools, textwrap, tempfile, shutil
import html, model, doc, utils, template
import rssgen
import cubictemp


class Links:
    """
        A template processor for maintaining a link log. A link log is set of
        entries of the following format, separated by one or more empty line:

        http://url/
        Multiple line title

        Description that maybe multiple lines, or indeed multiple
        paragraphs.
    """
    def __init__(self, markup):
        self.markup = markup

    def parse(self, txt):
        txt = txt.expandtabs()
        txt = textwrap.dedent(txt)

        entries = []
        current = None
        whiltespace = 0
        r = re.compile(r"^\s*\n*", re.MULTILINE)
        for i in re.split(r, txt):
            si = i.strip()
            if not si:
                continue
            elif si.startswith("http://") or si.startswith("https://"):
                if current:
                    entries.append(current)
                current = {}
                parts = si.split("\n", 1)
                current["link"] = parts[0].strip()
                current["title"] = parts[1]
                current["body"] = None
            else:
                if current:
                    c = current.get("body")
                    if not c:
                        current["body"] = si
                    else:
                        current["body"] = c + "\n\n" + si
        if current:
            entries.append(current)
        return entries

    def __call__(self, str):
        parts = self.parse(str)
        links = []
        for i in parts:
            links.append(
                dict(
                    title = template.Template(self.markup, i["title"]),
                    body = template.Template(self.markup, i["body"]) if i["body"] else None,
                    link = i["link"],
                )
            )
        t = template.Template(
                self.markup,
                file(utils.data.path("resources/links.html")).read(),
                links = links
            )
        return unicode(t)


class _Postfix:
    """
        Defines a postfix added to blog posts and the index page.
    """
    def index(self, page):
        """
            Page postfix for the index page - will appear once at the bottom of
            the index page.
        """
        return ""

    def inline(self, post):
        """
            Postfix for blog posts appearing inline on a page with other posts
            (i.e. the index page). Will appear after each post.
        """
        return ""

    def solo(self, post):
        """
            Postfix for blog posts appearing alone on a page (i.e. the
            permalink destination).
        """
        return ""


class _PostList(_Postfix):
    def __init__(self, num):
        self.num = num

    def _makeList(self, posts, num, title):
        monthyear = None
        output = html.DIV(_class=self.CSS_PREFIX)
        output.addChild(html.H1(title))
        postlst = []
        for i in posts:
            if not num:
                break
            if not i.url:
                postlst.append(
                    html.Group(
                        html.SPAN(
                            model.LinkTo(i),
                            _class="%s-post"%self.CSS_PREFIX
                        ),
                        " ",
                        html.SPAN(
                            i.time.strftime("%d %b %Y"),
                            _class="%s-date"%self.CSS_PREFIX
                        )
                    )
                )
                num -= 1
        if postlst:
            output.addChild(html.UL(postlst))
            return output
        else:
            return ""

    

class RecentPosts(_PostList):
    """
        A postfix that shows a list of recent top posts, excluding link posts (i.e.
        posts with a specified url option).
    """
    RECENT_TITLE = "More posts:"
    RELATED_TITLE = "Related:"
    CSS_PREFIX = "recent"
    def __init__(self, recent, related):
        self.recent, self.related = recent, related

    def index(self, idx):
        posts = idx.blog.blogdir.sortedPosts()
        return self._makeList(posts[idx.posts:], self.recent, self.RECENT_TITLE)

    def solo(self, post):
        posts = list(post.blog.blogdir.sortedPosts())
        posts.remove(post)
        parts = []
        related_posts = []
        if self.related:
            related_posts = post.related(posts, self.related)
            parts.append(self._makeList(related_posts, self.related, self.RELATED_TITLE))
        if self.recent:
            for i in related_posts:
                if i in posts:
                    posts.remove(i)
            parts.append(
                self._makeList(
                    [i for i in posts if "top" in i.options],
                    self.recent,
                    self.RECENT_TITLE
                )
            )
        return html.DIV(*parts)


class Disqus(_Postfix):
    def __init__(self, account):
        self.account = account

    def index(self, page):
        return html.rawstr(cubictemp.File(
            utils.data.path("resources/disqus_index.html"),
            account = self.account
        ))

    def inline(self, post):
        return html.rawstr("<a href=\"%s#disqus_thread\">Comments</a>"%model.UrlTo(post))

    def solo(self, post):
        return html.rawstr(cubictemp.File(
            utils.data.path("resources/disqus_solo.html"),
            permalink = post.permalink,
            account = self.account
        ))


class _PostRenderer(html._Renderable):
    """
        Lazy post renderer.
    """
    def __init__(self, post, *postfixes):
        """
            postfixes: A list of callables, which will be called with no
            arguments.
        """
        self.post, self.postfixes = post, postfixes
        self.src = os.path.abspath(post.src)

    def __unicode__(self):
        with utils.InDir(os.path.dirname(self.src)):
            if self.post.url:
                title = html.A(self.post.title, href=self.post.url)
            else:
                title = model.LinkTo(self.post)
            posttime = self.post.time.strftime("%Y %B %d %H:%M %p")
            links = Links(self.post.findAttr("markup"))
            postbody = template.Template(
                            self.post.findAttr("markup"),
                            self.post.data,
                            links = links
                        )
            postfixes = self.postfixes
            t = template.Template(
                self.post.findAttr("markup"),
                file(utils.data.path("resources/post.html")).read(),
                title = title, 
                posttime = posttime, 
                postdata = postbody, 
                postfixes = [i() for i in postfixes]
                )
            return unicode(t)
        
class Post(doc._DocHTMLPage):
    """
        Each post is housed in a separate file, 

            First line is a multi-word title 
            Time: Time

            The rest of the file is the post. <commit_msg>Fix a bug that caused blog post content to be processed twice by the markdown engine.<commit_after>from __future__ import with_statement
import os.path, re, datetime, urllib, codecs, functools, textwrap, tempfile, shutil
import html, model, doc, utils, template
import rssgen
import cubictemp


class Links:
    """
        A template processor for maintaining a link log. A link log is set of
        entries of the following format, separated by one or more empty line:

        http://url/
        Multiple line title

        Description that maybe multiple lines, or indeed multiple
        paragraphs.
    """
    def __init__(self, markup):
        self.markup = markup

    def parse(self, txt):
        txt = txt.expandtabs()
        txt = textwrap.dedent(txt)

        entries = []
        current = None
        whiltespace = 0
        r = re.compile(r"^\s*\n*", re.MULTILINE)
        for i in re.split(r, txt):
            si = i.strip()
            if not si:
                continue
            elif si.startswith("http://") or si.startswith("https://"):
                if current:
                    entries.append(current)
                current = {}
                parts = si.split("\n", 1)
                current["link"] = parts[0].strip()
                current["title"] = parts[1]
                current["body"] = None
            else:
                if current:
                    c = current.get("body")
                    if not c:
                        current["body"] = si
                    else:
                        current["body"] = c + "\n\n" + si
        if current:
            entries.append(current)
        return entries

    def __call__(self, str):
        parts = self.parse(str)
        links = []
        for i in parts:
            links.append(
                dict(
                    title = template.Template(self.markup, i["title"]),
                    body = template.Template(self.markup, i["body"]) if i["body"] else None,
                    link = i["link"],
                )
            )
        t = template.Template(
                self.markup,
                file(utils.data.path("resources/links.html")).read(),
                links = links
            )
        return unicode(t)


class _Postfix:
    """
        Defines a postfix added to blog posts and the index page.
    """
    def index(self, page):
        """
            Page postfix for the index page - will appear once at the bottom of
            the index page.
        """
        return ""

    def inline(self, post):
        """
            Postfix for blog posts appearing inline on a page with other posts
            (i.e. the index page). Will appear after each post.
        """
        return ""

    def solo(self, post):
        """
            Postfix for blog posts appearing alone on a page (i.e. the
            permalink destination).
        """
        return ""


class _PostList(_Postfix):
    def __init__(self, num):
        self.num = num

    def _makeList(self, posts, num, title):
        monthyear = None
        output = html.DIV(_class=self.CSS_PREFIX)
        output.addChild(html.H1(title))
        postlst = []
        for i in posts:
            if not num:
                break
            if not i.url:
                postlst.append(
                    html.Group(
                        html.SPAN(
                            model.LinkTo(i),
                            _class="%s-post"%self.CSS_PREFIX
                        ),
                        " ",
                        html.SPAN(
                            i.time.strftime("%d %b %Y"),
                            _class="%s-date"%self.CSS_PREFIX
                        )
                    )
                )
                num -= 1
        if postlst:
            output.addChild(html.UL(postlst))
            return output
        else:
            return ""

    

class RecentPosts(_PostList):
    """
        A postfix that shows a list of recent top posts, excluding link posts (i.e.
        posts with a specified url option).
    """
    RECENT_TITLE = "More posts:"
    RELATED_TITLE = "Related:"
    CSS_PREFIX = "recent"
    def __init__(self, recent, related):
        self.recent, self.related = recent, related

    def index(self, idx):
        posts = idx.blog.blogdir.sortedPosts()
        return self._makeList(posts[idx.posts:], self.recent, self.RECENT_TITLE)

    def solo(self, post):
        posts = list(post.blog.blogdir.sortedPosts())
        posts.remove(post)
        parts = []
        related_posts = []
        if self.related:
            related_posts = post.related(posts, self.related)
            parts.append(self._makeList(related_posts, self.related, self.RELATED_TITLE))
        if self.recent:
            for i in related_posts:
                if i in posts:
                    posts.remove(i)
            parts.append(
                self._makeList(
                    [i for i in posts if "top" in i.options],
                    self.recent,
                    self.RECENT_TITLE
                )
            )
        return html.DIV(*parts)


class Disqus(_Postfix):
    def __init__(self, account):
        self.account = account

    def index(self, page):
        return html.rawstr(cubictemp.File(
            utils.data.path("resources/disqus_index.html"),
            account = self.account
        ))

    def inline(self, post):
        return html.rawstr("<a href=\"%s#disqus_thread\">Comments</a>"%model.UrlTo(post))

    def solo(self, post):
        return html.rawstr(cubictemp.File(
            utils.data.path("resources/disqus_solo.html"),
            permalink = post.permalink,
            account = self.account
        ))


class _PostRenderer(html._Renderable):
    """
        Lazy post renderer.
    """
    def __init__(self, post, *postfixes):
        """
            postfixes: A list of callables, which will be called with no
            arguments.
        """
        self.post, self.postfixes = post, postfixes
        self.src = os.path.abspath(post.src)

    def __unicode__(self):
        with utils.InDir(os.path.dirname(self.src)):
            if self.post.url:
                title = html.A(self.post.title, href=self.post.url)
            else:
                title = model.LinkTo(self.post)
            posttime = self.post.time.strftime("%Y %B %d %H:%M %p")
            links = Links(self.post.findAttr("markup"))
            postbody = template.Template(
                            self.post.findAttr("markup"),
                            self.post.data,
                            links = links
                        )
            postfixes = self.postfixes
            t = template.Template(
                None,
                file(utils.data.path("resources/post.html")).read(),
                title = title, 
                posttime = posttime, 
                postdata = postbody, 
                postfixes = [i() for i in postfixes]
                )

            return unicode(t)
        
class Post(doc._DocHTMLPage):
    """
        Each post is housed in a separate file, 

            First line is a multi-word title 
            Time: Time

            The rest of the file is the post. 
{'fraction_non_alphanumeric': 0.10210562651018294, 'fraction_numerical': 0.0008284432171211598, 'mean_word_length': 1.3569801496908558, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2316521', 'n_tokens_mistral': 4049, 'n_tokens_neox': 3835, 'n_words': 1027}