<commit_before>from django import template
from django.conf import settings
from django.utils.importlib import import_module
from django.utils.safestring import mark_safe

from micawber.parsers import full_handler, inline_handler, parse_text, \
    parse_html, extract, extract_html


def _load_from_module(path):
    package, attr = path.rsplit('.', 1)
    module = import_module(package)
    return getattr(module, attr)


PROVIDERS = getattr(settings, 'MICAWBER_PROVIDERS', 'micawber.contrib.mcdjango.providers.bootstrap_basic')

providers = _load_from_module(PROVIDERS)
if callable(providers):
    providers = providers()


register = template.Library()


def extension(filter_name, providers=providers, urlize_all=True, html=False, handler=full_handler, 
                  block_handler=inline_handler, **kwargs):
    if html:
        fn = parse_html
    else:
        fn = parse_text
    def _extension(s, width_height=None):
        params = getattr(settings, 'MICAWBER_DEFAULT_SETTINGS', {})
        params.update(kwargs)
        if width_height:
            if 'x' in width_height:
                params['maxwidth'], params['maxheight'] = map(int, width_height.split('x'))
            else:
                params['maxwidth'] = int(width_height[0])
                params.pop('maxheight', None)
        return mark_safe(fn(s, providers, urlize_all, handler, block_handler, **params))
    register.filter(filter_name, _extension)
    return _extension

oembed = extension('oembed')
oembed_html = extension('oembed_html')

user_extensions = getattr(settings, 'MICAWBER_TEMPLATE_EXTENSIONS', [])
if isinstance(user_extensions, basestring):
    user_extensions = _load_from_module(user_extensions)

for filter_name, filter_params in user_extensions:
    extension(filter_name, **filter_params)<commit_msg>Adding extract template filters for parity w/flask<commit_after>from django import template
from django.conf import settings
from django.utils.importlib import import_module
from django.utils.safestring import mark_safe

from micawber.parsers import full_handler, inline_handler, parse_text, \
    parse_html, extract, extract_html


def _load_from_module(path):
    package, attr = path.rsplit('.', 1)
    module = import_module(package)
    return getattr(module, attr)


PROVIDERS = getattr(settings, 'MICAWBER_PROVIDERS', 'micawber.contrib.mcdjango.providers.bootstrap_basic')

providers = _load_from_module(PROVIDERS)
if callable(providers):
    providers = providers()


register = template.Library()


def extension(filter_name, providers=providers, urlize_all=True, html=False, handler=full_handler, 
              block_handler=inline_handler, text_fn=parse_text, html_fn=parse_html, **kwargs):
    if html:
        fn = html_fn
    else:
        fn = text_fn
    def _extension(s, width_height=None):
        params = getattr(settings, 'MICAWBER_DEFAULT_SETTINGS', {})
        params.update(kwargs)
        if width_height:
            if 'x' in width_height:
                params['maxwidth'], params['maxheight'] = map(int, width_height.split('x'))
            else:
                params['maxwidth'] = int(width_height[0])
                params.pop('maxheight', None)
        return mark_safe(fn(s, providers, urlize_all, handler, block_handler, **params))
    register.filter(filter_name, _extension)
    return _extension

oembed = extension('oembed')
oembed_html = extension('oembed_html', html=True)

def _extract_oembed(text, width_height=None, html=False):
    if html:
        fn = extract_html
    else:
        fn = extract
    params = getattr(settings, 'MICAWBER_DEFAULT_SETTINGS', {})
    if width_height:
        if 'x' in width_height:
            params['maxwidth'], params['maxheight'] = map(int, width_height.split('x'))
        else:
            params['maxwidth'] = int(width_height[0])
            params.pop('maxheight', None)
    url_list, url_data = fn(text, providers, **params)
    return [(u, url_data[u]) for u in url_list if u in url_data]

@register.filter
def extract_oembed(text, width_height=None):
    return _extract_oembed(text, width_height)

@register.filter
def extract_oembed_html(text, width_height=None):
    return _extract_oembed(text, width_height, True)

user_extensions = getattr(settings, 'MICAWBER_TEMPLATE_EXTENSIONS', [])
if isinstance(user_extensions, basestring):
    user_extensions = _load_from_module(user_extensions)

for filter_name, filter_params in user_extensions:
    extension(filter_name, **filter_params)
{'fraction_non_alphanumeric': 0.10344827586206896, 'fraction_numerical': 0.0011123470522803114, 'mean_word_length': 4.103291713961408, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '394541', 'n_tokens_mistral': 1387, 'n_tokens_neox': 1369, 'n_words': 298}