<commit_before>import itertools
import os
import re
from typing import (
    Any,
    AnyStr,
    Dict,
    List,
    MutableMapping,
    MutableSequence,
    Optional,
    Pattern,
    Tuple,
    Type,
    Union,
)

import ruamel.yaml
from ruamel.yaml.comments import CommentedBase, CommentedMap, CommentedSeq


lineno_re = re.compile("^(.*?:[0-9]+:[0-9]+: )(( *)(.*))")


def regex_chunk(lines, regex):
    # type: (List[str], Pattern[str]) -> List[List[str]]
    lst = list(itertools.dropwhile(lambda x: not regex.match(x), lines))
    arr = []
    while lst:
        ret = [lst[0]] + list(
            itertools.takewhile(lambda x: not regex.match(x), lst[1:])
        )
        arr.append(ret)
        lst = list(itertools.dropwhile(lambda x: not regex.match(x), lst[1:]))
    return arr


def chunk_messages(message):  # type: (str) -> List[Tuple[int, str]]
    file_regex = re.compile(r"^(.+:\d+:\d+:)(\s+)(.+)$")
    item_regex = re.compile(r"^\s*\*\s+")
    arr = []
    for chun in regex_chunk(message.splitlines(), file_regex):
        fst = chun[0]
        mat = file_regex.match(fst)
        if mat:
            place = mat.group(1)
            indent = len(mat.group(2))

            lst = [mat.group(3)] + chun[1:]
            if [x for x in lst if item_regex.match(x)]:
                for item in regex_chunk(lst, item_regex):
                    msg = re.sub(item_regex, "", "\n".join(item))
                    arr.append((indent, place + " " + re.sub(r"[\n\s]+", " ", msg)))
            else:
                msg = re.sub(item_regex, "", "\n".join(lst))
                arr.append((indent, place + " " + re.sub(r"[\n\s]+", " ", msg)))
    return arr


def reformat_yaml_exception_message(message):  # type: (str) -> str
    line_regex = re.compile(r'^\s+in "(.+)", line (\d+), column (\d+)$')
    fname_regex = re.compile(r"^file://" + re.escape(os.getcwd()) + "/")
    msgs = message.splitlines()
    ret = []

    if len(msgs) == 3:
        msgs = msgs[1:]
        nblanks = 0
    elif len(msgs) == 4:
        c_msg = msgs[0]
        match = line_regex.match(msgs[1])
        if match:
            c_file, c_line, c_column = match.groups()
            c_file = re.sub(fname_regex, "", c_file)
            ret.append("{}:{}:{}: {}".format(c_file, c_line, c_column, c_msg))

        msgs = msgs[2:]
        nblanks = 2

    p_msg = msgs[0]
    match = line_regex.match(msgs[1])
    if match:
        p_file, p_line, p_column = match.groups()
        p_file = re.sub(fname_regex, "", p_file)
        ret.append(
            "{}:{}:{}:{} {}".format(p_file, p_line, p_column, " " * nblanks, p_msg)
        )
    return "\n".join(ret)


def _add_lc_filename(
    r, source
):  # type: (ruamel.yaml.comments.CommentedBase, AnyStr) -> None
    if isinstance(r, ruamel.yaml.comments.CommentedBase):
        r.lc.filename = source
    if isinstance(r, MutableSequence):
        for d in r:
            _add_lc_filename(d, source)
    elif isinstance(r, MutableMapping):<commit_msg>Remove unused functions (#295)<commit_after>import os
import re
from typing import (
    Any,
    AnyStr,
    Dict,
    List,
    MutableMapping,
    MutableSequence,
    Optional,
    Tuple,
    Type,
    Union,
)

import ruamel.yaml
from ruamel.yaml.comments import CommentedBase, CommentedMap, CommentedSeq


lineno_re = re.compile("^(.*?:[0-9]+:[0-9]+: )(( *)(.*))")


def _add_lc_filename(
    r, source
):  # type: (ruamel.yaml.comments.CommentedBase, AnyStr) -> None
    if isinstance(r, ruamel.yaml.comments.CommentedBase):
        r.lc.filename = source
    if isinstance(r, MutableSequence):
        for d in r:
            _add_lc_filename(d, source)
    elif isinstance(r, MutableMapping):
{'fraction_non_alphanumeric': 0.16675690308608554, 'fraction_numerical': 0.00785056848944234, 'mean_word_length': 2.408671586715867, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3596368', 'n_tokens_mistral': 1357, 'n_tokens_neox': 1306, 'n_words': 271}