<commit_before>#
# Flask-SPF
#
# Copyright (C) 2017 <NAME>
# All rights reserved
#


import flask

from bs4 import BeautifulSoup
from flask import current_app, jsonify, render_template, request
from htmlmin import Minifier


class SPF(object):
    """
    Flask-SPF

    Documentation:

    https://flask-spf.readthedocs.io

    Usage:

    `<link rel="spf-url" href="/foo/bar">`

    `<link rel="stylesheet" href="/assets/css/page.css" class="spf-head" name="page">`

    `<div id="some-div" data-foo="foo" data-bar="bar" data-spf-attr="data-foo data-bar">`

    `<main id="main" class="spf-body">...</main>`

    `<script src="https://platform.twitter.com/widgets.js" class="spf-foot" name="twitter" async defer></script>`

    :param app: Flask app to initialize with. Defaults to `None`
    """

    minifier = None

    def __init__(self, app=None, **kwargs):
        if app is not None:
            self.init_app(app, **kwargs)

    def init_app(self, app, **kwargs):

        # Config
        app.config.setdefault('SPF_URL_IDENTIFIER', 'spf')

        # Minifier
        defaults = dict(
            remove_comments=True,
            remove_empty_space=True,
            reduce_empty_attributes=True,
            reduce_boolean_attributes=True,
            remove_optional_attribute_quotes=False,
        )
        defaults.update(kwargs)
        self.minifier = Minifier(defaults)

        app.extensions['spf'] = self


def _render_template(template_name_or_list, **context):
    """"""

    response = render_template(template_name_or_list, **context)

    criteria = (
        request.args.get(current_app.config.get('SPF_URL_IDENTIFIER')) in ('load', 'navigate'),
        'X-SPF-REFERER' in request.headers,
    )
    if any(criteria):
        response = _render_fragment(response)

    return response


def _render_fragment(html_doc):
    """
    https://youtube.github.io/spfjs/documentation/responses
    ---
    http://atodorov.org/blog/2014/02/21/skip-or-render-specific-blocks-from-jinja2-templates/
    http://stackoverflow.com/questions/32512568/how-to-render-only-given-block-using-jinja2-with-flask-and-pjax
    """

    soup = BeautifulSoup(html_doc, 'html.parser')

    minifier = current_app.extensions['spf'].minifier

    response = {}

    # `title`: Update document title
    tag = soup.title
    if tag is not None:
        response['title'] = tag.string.strip()<commit_msg>Configurable HTML parser<commit_after>#
# Flask-SPF
#
# Copyright (C) 2017 <NAME>
# All rights reserved
#


import flask

from bs4 import BeautifulSoup
from flask import current_app, jsonify, render_template, request
from htmlmin import Minifier


class SPF(object):
    """
    Flask-SPF

    Documentation:

    https://flask-spf.readthedocs.io

    Usage:

    `<link rel="spf-url" href="/foo/bar">`

    `<link rel="stylesheet" href="/assets/css/page.css" class="spf-head" name="page">`

    `<div id="some-div" data-foo="foo" data-bar="bar" data-spf-attr="data-foo data-bar">`

    `<main id="main" class="spf-body">...</main>`

    `<script src="https://platform.twitter.com/widgets.js" class="spf-foot" name="twitter" async defer></script>`

    :param app: Flask app to initialize with. Defaults to `None`
    """

    minifier = None

    def __init__(self, app=None, **kwargs):
        if app is not None:
            self.init_app(app, **kwargs)

    def init_app(self, app, **kwargs):

        # Config
        app.config.setdefault('SPF_HTML_PARSER', 'html.parser')
        app.config.setdefault('SPF_URL_IDENTIFIER', 'spf')

        # Minifier
        defaults = dict(
            remove_comments=True,
            remove_empty_space=True,
            reduce_empty_attributes=True,
            reduce_boolean_attributes=True,
            remove_optional_attribute_quotes=False,
        )
        defaults.update(kwargs)
        self.minifier = Minifier(defaults)

        app.extensions['spf'] = self


def _render_template(template_name_or_list, **context):
    """"""

    response = render_template(template_name_or_list, **context)

    criteria = (
        request.args.get(current_app.config.get('SPF_URL_IDENTIFIER')) in ('load', 'navigate'),
        'X-SPF-REFERER' in request.headers,
    )
    if any(criteria):
        response = _render_fragment(response)

    return response


def _render_fragment(html_doc):
    """
    https://youtube.github.io/spfjs/documentation/responses
    ---
    http://atodorov.org/blog/2014/02/21/skip-or-render-specific-blocks-from-jinja2-templates/
    http://stackoverflow.com/questions/32512568/how-to-render-only-given-block-using-jinja2-with-flask-and-pjax
    """

    soup = BeautifulSoup(html_doc, current_app.config['SPF_HTML_PARSER'])

    minifier = current_app.extensions['spf'].minifier

    response = {}

    # `title`: Update document title
    tag = soup.title
    if tag is not None:
        response['title'] = tag.string.strip()
{'fraction_non_alphanumeric': 0.13968641824475667, 'fraction_numerical': 0.009366727754021585, 'mean_word_length': 2.987012987012987, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 19, 'https://': 6, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5671800', 'n_tokens_mistral': 1689, 'n_tokens_neox': 1586, 'n_words': 287}