<commit_before>import re
import cgi
import logging

from lxml.html import fragment_fromstring

from adhocracy.lib.text import diff
from adhocracy.lib.text.tag import (tag_normalize, tag_split,
                                    tag_cloud_normalize, tag_split_last)
from normalize import *
from render import render, render_line_based


META_RE = re.compile("(\n|\t|\")", re.MULTILINE)


log = logging.getLogger(__name__)


def meta_escape(text, markdown=True):
    '''
    Transform *text* to be usable in a html meta header.
    This replaces problematic characters with a " " (space).
    If *markdown is `True`, it will do a markdown->plain text
    transformation too.
    '''
    if markdown:
        text = markdown_to_plain_text(text)
    text = META_RE.sub(" ", text)
    text = text.strip()
    return text


def markdown_to_plain_text(markup, safe_mode):
    html = render(markup, substitutions=False, safe_mode=safe_mode)
    try:
        return fragment_fromstring(html, create_parent=True).text_content()
    except Exception, e:
        log.exception(e)
        return markup


def text_rows(text):
    rows = len(list(text.lines))
    return max(min(30, rows), 10)


def revision_rows(revision):
    if revision.text is None:
        return 5
    rows = int((len([ch for ch in revision.text if ch == "\n"]) +<commit_msg>Restore default safe_mode argument in markdown_to_plain_text<commit_after>import re
import cgi
import logging

from lxml.html import fragment_fromstring

from adhocracy.lib.text import diff
from adhocracy.lib.text.tag import (tag_normalize, tag_split,
                                    tag_cloud_normalize, tag_split_last)
from normalize import *
from render import render, render_line_based


META_RE = re.compile("(\n|\t|\")", re.MULTILINE)


log = logging.getLogger(__name__)


def meta_escape(text, markdown=True):
    '''
    Transform *text* to be usable in a html meta header.
    This replaces problematic characters with a " " (space).
    If *markdown is `True`, it will do a markdown->plain text
    transformation too.
    '''
    if markdown:
        text = markdown_to_plain_text(text)
    text = META_RE.sub(" ", text)
    text = text.strip()
    return text


def markdown_to_plain_text(markup, safe_mode='escape'):
    html = render(markup, substitutions=False, safe_mode=safe_mode)
    try:
        return fragment_fromstring(html, create_parent=True).text_content()
    except Exception, e:
        log.exception(e)
        return markup


def text_rows(text):
    rows = len(list(text.lines))
    return max(min(30, rows), 10)


def revision_rows(revision):
    if revision.text is None:
        return 5
    rows = int((len([ch for ch in revision.text if ch == "\n"]) +
{'fraction_non_alphanumeric': 0.1000733137829912, 'fraction_numerical': 0.0036656891495601175, 'mean_word_length': 3.2774294670846396, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3284612', 'n_tokens_mistral': 886, 'n_tokens_neox': 839, 'n_words': 243}