<commit_before>import ast
import inspect
import warnings
import linecache
from uuid import uuid4
from collections import OrderedDict
from astmonkey.transformers import ParentNodeTransformer
from astmonkey.visitors import SourceGeneratorNodeVisitor


def split(func, name=None):
    return func, name


def get_sub(lst, pred):
    for subnode in lst:
        if pred(subnode):
            lst.remove(subnode)
            return subnode


class ToSource(SourceGeneratorNodeVisitor):
    '''
    The changes in here are primarily to patch
    around differences between python versions
    (I've tested with Python 3.5)
    '''

    @classmethod
    def to_source(cls, node):
        generator = cls('    ')
        generator.visit(node)
        return ''.join(generator.result)

    def visit_Call(self, node):
        node.starargs = get_sub(
            node.args,
            lambda t: isinstance(t, ast.Starred)
        )
        node.kwargs = get_sub(
            node.keywords,
            lambda t: t.arg is None
        )
        return super().visit_Call(node)

    def visit_Lambda(self, node):
        # until i can get the ast compiling directly,
        # we need to make sure precendence is correct
        self.write('(')
        super().visit_Lambda(node)
        self.write(')')

    def signature(self, node):
        if node.vararg:
            node.vararg = node.vararg.arg
        if node.kwarg:
            node.kwarg = node.kwarg.arg
        return super().signature(node)


class Visitor(ast.NodeVisitor):

    def __init__(self):
        self.parts = {}
        self.current = None
        self.name = None
        self.last_idx = -1

    def visit_FunctionDef(self, node):
        self.current = self.parts[node] = OrderedDict()
        self.name = node.name
        self.args = node.args
        return super().generic_visit(node)

    def visit_Call(self, node):
        if getattr(node.func, 'id', None) == 'split':
            if len(node.args) == 2:
                name = node.args[1].s
            else:
                name = None

            # this is the expression that contains the call,
            # or basically the value of the assignment/return
            expr = node.parent
            user = expr.parent

            # it's possible more are actually supported,
            # but i'm hesitant to just allow them without<commit_msg>Explanatory comment<commit_after>import ast
import inspect
import warnings
import linecache
from uuid import uuid4
from collections import OrderedDict
from astmonkey.transformers import ParentNodeTransformer
from astmonkey.visitors import SourceGeneratorNodeVisitor


def split(func, name=None):
    return func, name


def get_sub(lst, pred):
    for subnode in lst:
        if pred(subnode):
            lst.remove(subnode)
            return subnode


class ToSource(SourceGeneratorNodeVisitor):
    '''
    The changes in here are primarily to patch
    around differences between python versions
    (I've tested with Python 3.5)
    '''

    @classmethod
    def to_source(cls, node):
        generator = cls('    ')
        generator.visit(node)
        return ''.join(generator.result)

    def visit_Call(self, node):
        node.starargs = get_sub(
            node.args,
            lambda t: isinstance(t, ast.Starred)
        )
        node.kwargs = get_sub(
            node.keywords,
            lambda t: t.arg is None
        )
        return super().visit_Call(node)

    def visit_Lambda(self, node):
        # until i can get the ast compiling directly,
        # we need to make sure precendence is correct
        self.write('(')
        super().visit_Lambda(node)
        self.write(')')

    def signature(self, node):
        if node.vararg:
            node.vararg = node.vararg.arg
        if node.kwarg:
            node.kwarg = node.kwarg.arg
        return super().signature(node)


class Visitor(ast.NodeVisitor):

    def __init__(self):
        self.parts = {}
        self.current = None
        self.name = None
        self.last_idx = -1

    def visit_FunctionDef(self, node):
        self.current = self.parts[node] = OrderedDict()
        self.name = node.name
        self.args = node.args
        return super().generic_visit(node)

    def visit_Call(self, node):
        if getattr(node.func, 'id', None) == 'split':
            if len(node.args) == 2:
                name = node.args[1].s
            else:
                name = None

            # this is the expression that contains the call,
            # or basically the value of the assignment/return
            expr = node.parent
            user = expr.parent  # by user we mean user of the value

            # it's possible more are actually supported,
            # but i'm hesitant to just allow them without
{'fraction_non_alphanumeric': 0.08779979144942648, 'fraction_numerical': 0.002502606882168926, 'mean_word_length': 2.0277777777777777, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '50622', 'n_tokens_mistral': 1354, 'n_tokens_neox': 1311, 'n_words': 404}