<commit_before># -*- coding: utf-8 -*-
from __future__ import print_function, division, absolute_import, unicode_literals
import functools

__all__ = ['simplify', 'memoize', 'trim', 'indent']

def simplify(x):
    if isinstance(x, list) and len(x) == 1:
        return simplify(x[0])
    return x

def trim(docstring):
    """
    Definition of the trim algorithm from Python's PEP 257. It is used
    to trim the templates used by the nodes.

    http://www.python.org/dev/peps/pep-0257/
    """
    if not docstring:
        return ''
    # Convert tabs to spaces (following the normal Python rules)
    # and split into a list of lines:
    lines = docstring.expandtabs().splitlines()
    # Determine minimum indentation (first line doesn't count):
    maxindent = len(docstring)  # a reasonable large value
    indent = maxindent
    for line in lines[1:]:
        stripped = line.lstrip()
        if stripped:
            indent = min(indent, len(line) - len(stripped))
    # Remove indentation (first line is special):
    trimmed = [lines[0].strip()]
    if indent < maxindent:
        for line in lines[1:]:
            trimmed.append(line[indent:].rstrip())
    # Strip off trailing and leading blank lines:
#    while trimmed and not trimmed[-1]:
#        trimmed.pop()
    while trimmed and not trimmed[0]:
        trimmed.pop(0)
    # Return a single string:
    return '\n'.join(trimmed)

def indent(text, indent=1):
    """ Indent the given block of text by indent*4 spaces
    """
    if text is None:
        return ''
    text = str(text)
    if indent >= 0:
        lines = [' ' * 4 * indent + t for t in text.split('\n')]
        text = '\n'.join(lines)
    return text
<commit_msg>functools no longer used.<commit_after># -*- coding: utf-8 -*-
from __future__ import print_function, division, absolute_import, unicode_literals

__all__ = ['simplify', 'memoize', 'trim', 'indent']

def simplify(x):
    if isinstance(x, list) and len(x) == 1:
        return simplify(x[0])
    return x

def trim(docstring):
    """
    Definition of the trim algorithm from Python's PEP 257. It is used
    to trim the templates used by the nodes.

    http://www.python.org/dev/peps/pep-0257/
    """
    if not docstring:
        return ''
    # Convert tabs to spaces (following the normal Python rules)
    # and split into a list of lines:
    lines = docstring.expandtabs().splitlines()
    # Determine minimum indentation (first line doesn't count):
    maxindent = len(docstring)  # a reasonable large value
    indent = maxindent
    for line in lines[1:]:
        stripped = line.lstrip()
        if stripped:
            indent = min(indent, len(line) - len(stripped))
    # Remove indentation (first line is special):
    trimmed = [lines[0].strip()]
    if indent < maxindent:
        for line in lines[1:]:
            trimmed.append(line[indent:].rstrip())
    # Strip off trailing and leading blank lines:
#    while trimmed and not trimmed[-1]:
#        trimmed.pop()
    while trimmed and not trimmed[0]:
        trimmed.pop(0)
    # Return a single string:
    return '\n'.join(trimmed)

def indent(text, indent=1):
    """ Indent the given block of text by indent*4 spaces
    """
    if text is None:
        return ''
    text = str(text)
    if indent >= 0:
        lines = [' ' * 4 * indent + t for t in text.split('\n')]
        text = '\n'.join(lines)
    return text

{'fraction_non_alphanumeric': 0.11981159846923756, 'fraction_numerical': 0.011775095672652341, 'mean_word_length': 2.6187433439829606, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1916385', 'n_tokens_mistral': 1061, 'n_tokens_neox': 1020, 'n_words': 353}