<commit_before># -*- coding: utf-8 -*-

# Copyright 2013 django-htmlmin authors. All rights reserved.
# Use of this source code is governed by a BSD-style
# license that can be found in the LICENSE file.

import re

import six

import bs4

from .util import force_text

EXCLUDE_TAGS = set(["pre", "script", "textarea"])
# element list coming from
# https://developer.mozilla.org/en-US/docs/Web/Guide/HTML/HTML5/HTML5_element_list
# combining text-level semantics & edits
TEXT_FLOW = {
    "a", "em", "strong", "small", "s", "cite", "q", "dfn", "abbr", "data",
    "time", "code", "var", "samp", "kbd", "sup", "sub", "i", "b", "u", "mark",
    "ruby", "rt", "rp", "bdi", "bdo", "span", "br", "wbr", "ins", "del",
}

# fold the doctype element, if True then no newline is added after the
# doctype element. If False, a newline will be insterted
FOLD_DOCTYPE = True
re_space = six.u('((?=\\s)[^\xa0])')
re_multi_space = re.compile(re_space + '+', re.MULTILINE | re.UNICODE)
re_single_nl = re.compile(r'^\n$', re.MULTILINE | re.UNICODE)
re_only_space = re.compile(r'^' + re_space + r'+$', re.MULTILINE | re.UNICODE)
re_start_space = re.compile(r'^' + re_space + '+', re.MULTILINE | re.UNICODE)
re_end_space = re.compile(re_space + r'+$', re.MULTILINE | re.UNICODE)
# see http://en.wikipedia.org/wiki/Conditional_comment
re_cond_comment = re.compile(r'\[if .*\]>.*<!\[endif\]',
                             re.MULTILINE | re.DOTALL | re.UNICODE)
re_cond_comment_start_space = re.compile(r'(\[if .*\]>)\s+',
                                         re.MULTILINE | re.DOTALL | re.UNICODE)
re_cond_comment_end_space = re.compile(r'\s+(<!\[endif\])',
                                       re.MULTILINE | re.DOTALL | re.UNICODE)


def html_minify(html_code, ignore_comments=True, parser="html5lib"):
    html_code = force_text(html_code)
    soup = bs4.BeautifulSoup(html_code, parser)
    mini_soup = space_minify(soup, ignore_comments)
    if FOLD_DOCTYPE is True:
        # monkey patching to remove new line after doctype
        bs4.element.Doctype.SUFFIX = six.u('>')
    return six.text_type(mini_soup)


def space_minify(soup, ignore_comments=True):
    """recursive function to reduce space characters in html code.

    :param soup: a BeautifulSoup of the code to reduce
    :type soup: bs4.BeautifulSoup
    :param ignore_comments: whether or not to keep comments in the
                            result
    :type ignore_comments: bool
    """
    # if tag excluded from minification, just pass
    if str(soup.name) in EXCLUDE_TAGS:<commit_msg>Adding max_lenght parameter to limit size of lines of minified html (useful when rendering emails because mails contents are/canbe line wrapped by smtp) This splits line before an '<'.<commit_after># -*- coding: utf-8 -*-

# Copyright 2013 django-htmlmin authors. All rights reserved.
# Use of this source code is governed by a BSD-style
# license that can be found in the LICENSE file.

import re

import six

import bs4

from .util import force_text

EXCLUDE_TAGS = set(["pre", "script", "textarea"])
# element list coming from
# https://developer.mozilla.org/en-US/docs/Web/Guide/HTML/HTML5/HTML5_element_list
# combining text-level semantics & edits
TEXT_FLOW = {
    "a", "em", "strong", "small", "s", "cite", "q", "dfn", "abbr", "data",
    "time", "code", "var", "samp", "kbd", "sup", "sub", "i", "b", "u", "mark",
    "ruby", "rt", "rp", "bdi", "bdo", "span", "br", "wbr", "ins", "del",
}

# fold the doctype element, if True then no newline is added after the
# doctype element. If False, a newline will be insterted
FOLD_DOCTYPE = True
re_space = six.u('((?=\\s)[^\xa0])')
re_multi_space = re.compile(re_space + '+', re.MULTILINE | re.UNICODE)
re_single_nl = re.compile(r'^\n$', re.MULTILINE | re.UNICODE)
re_only_space = re.compile(r'^' + re_space + r'+$', re.MULTILINE | re.UNICODE)
re_start_space = re.compile(r'^' + re_space + '+', re.MULTILINE | re.UNICODE)
re_end_space = re.compile(re_space + r'+$', re.MULTILINE | re.UNICODE)
# see http://en.wikipedia.org/wiki/Conditional_comment
re_cond_comment = re.compile(r'\[if .*\]>.*<!\[endif\]',
                             re.MULTILINE | re.DOTALL | re.UNICODE)
re_cond_comment_start_space = re.compile(r'(\[if .*\]>)\s+',
                                         re.MULTILINE | re.DOTALL | re.UNICODE)
re_cond_comment_end_space = re.compile(r'\s+(<!\[endif\])',
                                       re.MULTILINE | re.DOTALL | re.UNICODE)


def html_minify(html_code, ignore_comments=True, parser="html5lib", max_length=0):
    html_code = force_text(html_code)
    soup = bs4.BeautifulSoup(html_code, parser)
    mini_soup = space_minify(soup, ignore_comments)
    if FOLD_DOCTYPE is True:
        # monkey patching to remove new line after doctype
        bs4.element.Doctype.SUFFIX = six.u('>')
    s = six.text_type(mini_soup)

    if max_length > 0:
        pos = 0
        l = []
        while len(s) - pos > max_length:
            p = s.rfind('<', pos, pos+max_length)
            if p < 0:
                p = s.find('<', pos+max_length)
                if p < 0:
                    l.append(s[pos:])
                    break
            l.append(s[pos:p])
            pos = p
        s = '\n'.join(l)
    
    return s 


def space_minify(soup, ignore_comments=True):
    """recursive function to reduce space characters in html code.

    :param soup: a BeautifulSoup of the code to reduce
    :type soup: bs4.BeautifulSoup
    :param ignore_comments: whether or not to keep comments in the
                            result
    :type ignore_comments: bool
    """
    # if tag excluded from minification, just pass
    if str(soup.name) in EXCLUDE_TAGS:
{'fraction_non_alphanumeric': 0.14715719063545152, 'fraction_numerical': 0.00545678577715191, 'mean_word_length': 2.943095072866065, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 11, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7415207', 'n_tokens_mistral': 2072, 'n_tokens_neox': 1952, 'n_words': 520}