<commit_before>import codecs
import pygments.lexers
import re
import types

from pygments.lexer import inherit

def dq(x):
    if isinstance(x, str) or isinstance(x, unicode):
        return '"%s"' % "".join([esc(y) for y in x])
    elif isinstance(x, list) or isinstance(x, tuple):
        return "[" + ", ".join([dq(y) for y in x]) + "]"
    else:
        raise Exception("Cannot dq: %s" % repr(x))

def rr(x):
    # in a range?
    in_range = False

    # just after an opening [? (implies in_range)
    opened = False

    # next character escaped?
    escaped = False

    sane = ""

    # fix up []foo[] as [\]foo\[]
    for c in x:
        if c == '[' and not escaped:
            if in_range:
                sane += '\\'
            else:
                in_range = True
                opened = True
        elif c == ']' and not escaped:
            if opened:
                sane += '\\'
            elif in_range:
                in_range = False
        elif c == '\\':
            escaped = not escaped
            opened = False
        else:
            escaped = False
            opened = False

        sane += c

    return rq(sane)

def rq(x):
    if isinstance(x, str) or isinstance(x, unicode):
        return '"%s"' % "".join([reg_esc(y) for y in x])
    elif isinstance(x, list) or isinstance(x, tuple):
        return "[" + ", ".join([rq(y) for y in x]) + "]"
    else:
        raise Exception("Cannot rq: %s" % repr(x))

def esc(c):
    if c == '"':
        return '\\"'
    elif c == '\\':
        return '\\\\'<commit_msg>fix up regexp (?s) rules<commit_after>import codecs
import pygments.lexers
import re
import types

from pygments.lexer import inherit

def dq(x):
    if isinstance(x, str) or isinstance(x, unicode):
        return '"%s"' % "".join([esc(y) for y in x])
    elif isinstance(x, list) or isinstance(x, tuple):
        return "[" + ", ".join([dq(y) for y in x]) + "]"
    else:
        raise Exception("Cannot dq: %s" % repr(x))

def rr(x):
    # in a range?
    in_range = False

    # just after an opening [? (implies in_range)
    opened = False

    # next character escaped?
    escaped = False

    sane = ""

    # fix up []foo[] as [\]foo\[]
    for c in x:
        if c == '[' and not escaped:
            if in_range:
                sane += '\\'
            else:
                in_range = True
                opened = True
        elif c == ']' and not escaped:
            if opened:
                sane += '\\'
            elif in_range:
                in_range = False
        elif c == '\\':
            escaped = not escaped
            opened = False
        else:
            escaped = False
            opened = False

        sane += c

    # replace (?s) with (?m)
    sane = re.sub(r"\(\?([a-z]*)s([a-z]*)\)", r"(?\1m\2)", sane)

    return rq(sane)

def rq(x):
    if isinstance(x, str) or isinstance(x, unicode):
        return '"%s"' % "".join([reg_esc(y) for y in x])
    elif isinstance(x, list) or isinstance(x, tuple):
        return "[" + ", ".join([rq(y) for y in x]) + "]"
    else:
        raise Exception("Cannot rq: %s" % repr(x))

def esc(c):
    if c == '"':
        return '\\"'
    elif c == '\\':
        return '\\\\'
{'fraction_non_alphanumeric': 0.15597758405977585, 'fraction_numerical': 0.0006226650062266501, 'mean_word_length': 1.6553719008264463, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3671420', 'n_tokens_mistral': 1090, 'n_tokens_neox': 1041, 'n_words': 326}