<commit_before># -*- coding: utf-8 -*-
from __future__ import print_function
from __future__ import unicode_literals
from __future__ import division

import json
import psycopg2

from datetime import datetime
from functools import partial

from django.db.models import Q
from django.db.models.query import ValuesListQuerySet
from django.db import connection
from django.utils.tree import Node

from django.contrib.gis.geos import Point, MultiPolygon

from treemap.tests import (make_instance, make_commander_user,
                           make_simple_polygon, set_write_permissions)
from treemap.tests.base import OTMTestCase
from treemap.tests.test_udfs import make_collection_udf
from treemap.models import (Tree, Plot, Boundary, Species)
from treemap.udf import UserDefinedFieldDefinition
from treemap import search

COLLECTION_UDF_DATATYPE = [{'type': 'choice',
                            'choices': ['water', 'prune'],
                            'name': 'action'},
                           {'type': 'date',
                            'name': 'date'}]


def destructure_query_set(node):
    """
    Django query objects are not comparable by themselves, but they
    are built from a tree (django.util.tree) and stored in nodes

    This function generates a canonical representation using sets and
    tuples of a query tree

    This can be used to verify that query structures are made correctly
    """
    if isinstance(node, Node):
        children = [destructure_query_set(c) for c in node.children]
        try:
            child_hash = frozenset(children)
        except:
            # Necessary for udf collection queries, which contain
            # (selector, dict) pairs, and the dicts are unhashable
            child_hash = dict(children)
        n = (node.connector, child_hash)

        if node.negated:
            n = ('NOT', n)

        return n
    elif isinstance(node, tuple):
        # Lists are unhashable, so convert ValuesListQuerySets into tuples
        # for easy comparison
        return tuple(tuple(c) if isinstance(c, ValuesListQuerySet) else c
                     for c in node)
    else:
        return node
<commit_msg>Remove reference to ValuesListQuerySet<commit_after># -*- coding: utf-8 -*-
from __future__ import print_function
from __future__ import unicode_literals
from __future__ import division

import json
import psycopg2

from datetime import datetime
from functools import partial

from django.db.models import Q
from django.db import connection
from django.db.models.query import QuerySet
from django.utils.tree import Node

from django.contrib.gis.geos import Point, MultiPolygon

from treemap.tests import (make_instance, make_commander_user,
                           make_simple_polygon, set_write_permissions)
from treemap.tests.base import OTMTestCase
from treemap.tests.test_udfs import make_collection_udf
from treemap.models import (Tree, Plot, Boundary, Species)
from treemap.udf import UserDefinedFieldDefinition
from treemap import search

COLLECTION_UDF_DATATYPE = [{'type': 'choice',
                            'choices': ['water', 'prune'],
                            'name': 'action'},
                           {'type': 'date',
                            'name': 'date'}]


def destructure_query_set(node):
    """
    Django query objects are not comparable by themselves, but they
    are built from a tree (django.util.tree) and stored in nodes

    This function generates a canonical representation using sets and
    tuples of a query tree

    This can be used to verify that query structures are made correctly
    """
    if isinstance(node, Node):
        children = [destructure_query_set(c) for c in node.children]
        try:
            child_hash = frozenset(children)
        except:
            # Necessary for udf collection queries, which contain
            # (selector, dict) pairs, and the dicts are unhashable
            child_hash = dict(children)
        n = (node.connector, child_hash)

        if node.negated:
            n = ('NOT', n)

        return n
    elif isinstance(node, tuple):
        # Lists are unhashable, so convert ValuesListQuerySets into tuples
        # for easy comparison
        return tuple(tuple(c) if isinstance(c, QuerySet) else c for c in node)
    else:
        return node

{'fraction_non_alphanumeric': 0.06898148148148148, 'fraction_numerical': 0.000925925925925926, 'mean_word_length': 2.6587637595258258, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4925979', 'n_tokens_mistral': 1175, 'n_tokens_neox': 1148, 'n_words': 415}