<commit_before># -*- coding: utf-8 -*-
"""
flaskext.blogpit

Blogpit storage for markdown

To use:

    app.register_blueprint( 
            create_blogpit_blueprint('/srv/git/source.git',
                'refs/heads/master', None,
                ContentHandler))

"""
from __future__ import print_function, unicode_literals
from __future__ import absolute_import, division
from flask import Blueprint, render_template, abort, request, redirect, \
                    url_for, flash, current_app

import blogpit
import mimetypes
from BeautifulSoup import BeautifulSoup
#import forms
from .forms import CommentForm

class ContentHandler(object):
    """
    This class intercepts all operations involving
    data coming from, or going to the blogpit storage.
    This involves:
    1. Reading content
    2. Storing content

    """

    def filter_sections(self, pathlist):
        """
        Filter out sections you don't want to be visible

        This method takes a list of paths and returs a
        subset of the given list - removing those that should
        be concealed from lists.

        This implementation returns the given list as is.
        """

        return pathlist

    def filter_articles(self, pathlist):
        """
        Filter out articles you don't want to be visible

        This method takes a list of paths and returs a
        subset of the given list - removing those that should
        be concealed from lists.

        This implementation returns the given list as is.
        """
        return pathlist

    def decode(self, path, content):
        """
        Decode byte content as retrieved from the store
        and if need be, convert it.

        The outcome of this method is what you will get in templates, so
        act accordingly. If the result of this call is a byte string (str)
        them blogpit will serve directly as static content, otherwise this
        will be passed upwards into the proper template.

        This particular implementation decodes ALL content as utf-8
        and returns an unicode object
        """

        return content.decode('utf-8')

    def append_comment_from_form(self, rawdata, form):
        """
        Given a comment form convert it into some data we safely store away.
        Return a new version to be stored, based on the current rawdata plus
        the data extracted from the form.

        This method should:
        1. Append information that identifies the owner of the comment
        2. Escape comment content
        3. Return a string object

        This particular implementation:
        1. Prepends the message with the author name
        2. <Does nothing to escape the content>
        3. Encodes the result as an utf-8 string
        """

        name_esc = ''.join(BeautifulSoup(form.name.data).findAll(text=True))
        content_esc = ''.join(BeautifulSoup(form.name.data).findAll(text=True))

        content = u'\n<br/>%s<br/>\n%s' % (name_esc, content_esc)
        return rawdata + content.encode('utf-8')


def create_blogpit_blueprint(path, branch, cache, handler, **kwargs):
    """
    Bluepring factory for blogpit
    """
    __cache = cache
    __blogpit = blogpit.Blogpit(path, branch)
    if handler:
        __handler = handler
    else:
        __handler = ContentHandler()


    name = 'blogpit-' + str(id(__blogpit))
    blogpit_blueprint = Blueprint(name, __name__, 
                                template_folder='templates', **kwargs)

    def from_cache(fun):
        """
        A decorator to cache function results.

        All decorated functions must take a single argument, a path.
        """
        def newfun(path):
            if not __cache:
                return fun(path)

            key = '%s:%s:%s' % (fun.func_name, __blogpit.version(), path)
            data =  __cache.get(key)
            if data is None:
                data = fun(path)
                __cache.set(key, data)
            return data

        return newfun

    @from_cache
    def getarticle(path):
        """
        Get content
        """
        return __handler.decode(path, get_article_from_store(path) )

    def get_article_from_store(path):
        """
        Get content - skipping cache and content handler
        """
        r = __blogpit.getarticle(path)
        return r

    @from_cache<commit_msg>Add missing check for non-existing files<commit_after># -*- coding: utf-8 -*-
"""
flaskext.blogpit

Blogpit storage for markdown

To use:

    app.register_blueprint( 
            create_blogpit_blueprint('/srv/git/source.git',
                'refs/heads/master', None,
                ContentHandler))

"""
from __future__ import print_function, unicode_literals
from __future__ import absolute_import, division
from flask import Blueprint, render_template, abort, request, redirect, \
                    url_for, flash, current_app

import blogpit
import mimetypes
from BeautifulSoup import BeautifulSoup
#import forms
from .forms import CommentForm

class ContentHandler(object):
    """
    This class intercepts all operations involving
    data coming from, or going to the blogpit storage.
    This involves:
    1. Reading content
    2. Storing content

    """

    def filter_sections(self, pathlist):
        """
        Filter out sections you don't want to be visible

        This method takes a list of paths and returs a
        subset of the given list - removing those that should
        be concealed from lists.

        This implementation returns the given list as is.
        """

        return pathlist

    def filter_articles(self, pathlist):
        """
        Filter out articles you don't want to be visible

        This method takes a list of paths and returs a
        subset of the given list - removing those that should
        be concealed from lists.

        This implementation returns the given list as is.
        """
        return pathlist

    def decode(self, path, content):
        """
        Decode byte content as retrieved from the store
        and if need be, convert it.

        The outcome of this method is what you will get in templates, so
        act accordingly. If the result of this call is a byte string (str)
        them blogpit will serve directly as static content, otherwise this
        will be passed upwards into the proper template.

        This particular implementation decodes ALL content as utf-8
        and returns an unicode object
        """

        return content.decode('utf-8')

    def append_comment_from_form(self, rawdata, form):
        """
        Given a comment form convert it into some data we safely store away.
        Return a new version to be stored, based on the current rawdata plus
        the data extracted from the form.

        This method should:
        1. Append information that identifies the owner of the comment
        2. Escape comment content
        3. Return a string object

        This particular implementation:
        1. Prepends the message with the author name
        2. <Does nothing to escape the content>
        3. Encodes the result as an utf-8 string
        """

        name_esc = ''.join(BeautifulSoup(form.name.data).findAll(text=True))
        content_esc = ''.join(BeautifulSoup(form.name.data).findAll(text=True))

        content = u'\n<br/>%s<br/>\n%s' % (name_esc, content_esc)
        return rawdata + content.encode('utf-8')


def create_blogpit_blueprint(path, branch, cache, handler, **kwargs):
    """
    Bluepring factory for blogpit
    """
    __cache = cache
    __blogpit = blogpit.Blogpit(path, branch)
    if handler:
        __handler = handler
    else:
        __handler = ContentHandler()


    name = 'blogpit-' + str(id(__blogpit))
    blogpit_blueprint = Blueprint(name, __name__, 
                                template_folder='templates', **kwargs)

    def from_cache(fun):
        """
        A decorator to cache function results.

        All decorated functions must take a single argument, a path.
        """
        def newfun(path):
            if not __cache:
                return fun(path)

            key = '%s:%s:%s' % (fun.func_name, __blogpit.version(), path)
            data =  __cache.get(key)
            if data is None:
                data = fun(path)
                __cache.set(key, data)
            return data

        return newfun

    @from_cache
    def getarticle(path):
        """
        Get content
        """
        raw = get_article_from_store(path)
        if not raw:
            return raw
        return __handler.decode(path, raw)

    def get_article_from_store(path):
        """
        Get content - skipping cache and content handler
        """
        r = __blogpit.getarticle(path)
        return r

    @from_cache
{'fraction_non_alphanumeric': 0.07301010559781991, 'fraction_numerical': 0.0029521971159305097, 'mean_word_length': 2.159253945480631, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5540363', 'n_tokens_mistral': 2361, 'n_tokens_neox': 2246, 'n_words': 877}