<commit_before>import json

import six

import django

def get_filtered_selection_from_tree(tree, selection=[], include_level=-1, exclude_parent=True, final_selection=None, filter={}, exclude={}):

    # print 'get_filtered_selection TREE', tree
    if selection is None:
        return []



    if isinstance(selection, six.string_types):
        selection = selection.split(',')

    _selection = []
    for s in selection:
        s = str(s).strip()
        if s != "":
            _selection.append(int(s))


    _final_selection = final_selection if final_selection else {}
    root_empty = True

    if isinstance(tree, list):
        parent_id = None
        children = tree

    else:
        parent_id = tree.get('id')
        children = tree.get('children', [])

    for child in children:
        if child.get('id') in _selection:
            _final_selection[child.get('id')] = child
            root_empty = False

            if exclude_parent and parent_id and _final_selection.get(parent_id):
                del _final_selection[parent_id]

            get_filtered_selection_from_tree(child, selection, include_level=include_level, exclude_parent=exclude_parent, final_selection=_final_selection, filter=filter, exclude=exclude)

    if include_level and root_empty:
        for child in tree.get('children', []):
            _final_selection[child.get('id')] = child
            if include_level >= 1 or include_level == -1:
                get_filtered_selection_from_tree(child, selection, include_level=include_level-1, exclude_parent=exclude_parent, final_selection=_final_selection, filter=filter, exclude=exclude)

    return _final_selection.keys()

def get_tree_from_queryset(queryset, on_create_node=None, max_level=None, with_instance=False):
    """
    Return tree data that is suitable for jqTree.
    The queryset must be sorted by 'tree_id' and 'left' fields.
    """
    pk_attname = queryset.model._meta.pk.attname

    def serialize_id(pk):
        if isinstance(pk, six.integer_types + six.string_types):
            return pk
        else:
            # Nb. special case for uuid field
            return str(pk)

    # Result tree
    tree = []

    # Dict of all nodes; used for building the tree
    # - key is node id<commit_msg>improve tree<commit_after>import json

import six

import django

def get_filtered_selection_from_tree(tree, selection=[], include_level=-1, exclude_parent=True, final_selection=None, filter={}, exclude={}):

    # print 'get_filtered_selection TREE', tree
    if selection is None:
        return []



    if isinstance(selection, six.string_types):
        selection = selection.split(',')

    _selection = []
    for s in selection:
        s = str(s).strip()
        if s != "":
            _selection.append(int(s))


    _final_selection = final_selection if final_selection else {}
    root_empty = True

    print tree
    if isinstance(tree, list):
        parent_id = None
        children = tree

    else:
        parent_id = tree.get('id')
        children = tree.get('children', [])

    for child in children:
        if child.get('id') in _selection:
            _final_selection[child.get('id')] = child
            root_empty = False

            if exclude_parent and parent_id and _final_selection.get(parent_id):
                del _final_selection[parent_id]

            get_filtered_selection_from_tree(child, selection, include_level=include_level, exclude_parent=exclude_parent, final_selection=_final_selection, filter=filter, exclude=exclude)

    if include_level and root_empty:
        for child in children:
            _final_selection[child.get('id')] = child
            if include_level >= 1 or include_level == -1:
                get_filtered_selection_from_tree(child, selection, include_level=include_level-1, exclude_parent=exclude_parent, final_selection=_final_selection, filter=filter, exclude=exclude)

    return _final_selection.keys()

def get_tree_from_queryset(queryset, on_create_node=None, max_level=None, with_instance=False):
    """
    Return tree data that is suitable for jqTree.
    The queryset must be sorted by 'tree_id' and 'left' fields.
    """
    pk_attname = queryset.model._meta.pk.attname

    def serialize_id(pk):
        if isinstance(pk, six.integer_types + six.string_types):
            return pk
        else:
            # Nb. special case for uuid field
            return str(pk)

    # Result tree
    tree = []

    # Dict of all nodes; used for building the tree
    # - key is node id
{'fraction_non_alphanumeric': 0.09286971830985916, 'fraction_numerical': 0.0017605633802816902, 'mean_word_length': 2.891267123287671, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2333481', 'n_tokens_mistral': 1346, 'n_tokens_neox': 1291, 'n_words': 336}