<commit_before>from __future__ import print_function
from __future__ import unicode_literals
from __future__ import division

from json import loads

from django.db.models import Q

from django.contrib.gis.measure import Distance
from django.contrib.gis.geos import Point

from treemap.models import Plot, Boundary


class ParseException (Exception):
    def __init__(self, message):
        super(Exception, self).__init__(message)
        self.message = message


MODEL_MAPPING = {'plot': '',
                 'tree': 'tree__',
                 'species': 'tree__species__'}


def create_filter(filterstr):
    """
    A filter is a string that must be valid json and conform to
    the following grammar:
    literal        = json literal | GMT date string in 'YYYY-MM-DD HH:MM:SS'
    model          = 'plot' | 'tree' | 'species'
    value-property = 'MIN'
                   | 'MAX'
                   | 'EXCLUSIVE'
                   | 'IN'
                   | 'IS'
                   | 'WITHIN_RADIUS'
                   | 'IN_BOUNDARY'
    combinator     = 'AND' | 'OR'
    predicate      = { model.field: literal }
                   | { model.field: { (value-property: literal)* }}
    filter         = predicate
                   | [combinator, filter*]

    Returns a lazy query set of plot objects
    """
    query = loads(filterstr)

    q = _parse_filter(query)

    return Plot.objects.filter(q)


def _parse_filter(query):
    if type(query) is dict:
        return _parse_predicate(query)
    elif type(query) is list:
        predicates = [_parse_filter(p) for p in query[1:]]
        return _apply_combinator(query[0], predicates)


def _parse_predicate(query):
    qs = [_parse_predicate_pair(*kv) for kv in query.iteritems()]
    return _apply_combinator('AND', qs)


def _parse_predicate_key(key, mapping=MODEL_MAPPING):
    parts = key.split('.')

    if len(parts) != 2:
        raise ParseException(
            'Keys must be in the form of "model.field", not "%s"' %
            key)

    model, field = parts

    if model not in mapping:
        raise ParseException(
            'Valid models are: %s, not "%s"' %
            (mapping.keys(), model))

    return mapping[model] + field


#TODO: Date not supported
def _parse_value(value):
    """
    A value can be either:
    * A date
    * A literal
    * A list of other values
    """
    if type(value) is list:
        return [_parse_value(v) for v in value]
    else:
        return value


def _parse_min_max_value_fn(operator):
    """
    returns a function that produces predicate pairs
    for the given operator.
    """

    def fn(predicate_value):
        # assign closure value to a local
        # in order to mutate it later
        key = operator

        # a min/max predicate can either take
        # a value or a dictionary that provides
        # a VALUE and EXCLUSIVE flag.
        if type(predicate_value) == dict:
            raw_value = predicate_value.get('VALUE')
            exclusive = predicate_value.get('EXCLUSIVE')
        else:
            raw_value = predicate_value
            exclusive = False

        if not exclusive:
            # django use lt/lte and gt/gte
            # to handle inclusive/exclusive
            key += 'e'

        value = _parse_value(raw_value)

        return {key: value}

    return fn<commit_msg>Cleanup code and clarify documentation<commit_after>from __future__ import print_function
from __future__ import unicode_literals
from __future__ import division

from json import loads

from django.db.models import Q

from django.contrib.gis.measure import Distance
from django.contrib.gis.geos import Point

from treemap.models import Plot, Boundary


class ParseException (Exception):
    def __init__(self, message):
        super(Exception, self).__init__(message)
        self.message = message


MODEL_MAPPING = {'plot': '',
                 'tree': 'tree__',
                 'species': 'tree__species__'}


def create_filter(filterstr):
    """
    A filter is a string that must be valid json and conform to
    the following grammar:
    literal        = json literal | GMT date string in 'YYYY-MM-DD HH:MM:SS'
    model          = 'plot' | 'tree' | 'species'
    value-property = 'MIN'
                   | 'MAX'
                   | 'EXCLUSIVE'
                   | 'IN'
                   | 'IS'
                   | 'WITHIN_RADIUS'
                   | 'IN_BOUNDARY'
    combinator     = 'AND' | 'OR'
    predicate      = { model.field: literal }
                   | { model.field: { (value-property: literal)* }}
    filter         = predicate
                   | [combinator, filter*]

    Returns a lazy query set of plot objects
    """
    query = loads(filterstr)

    q = _parse_filter(query)

    return Plot.objects.filter(q)


def _parse_filter(query):
    if type(query) is dict:
        return _parse_predicate(query)
    elif type(query) is list:
        predicates = [_parse_filter(p) for p in query[1:]]
        return _apply_combinator(query[0], predicates)


def _parse_predicate(query):
    qs = [_parse_predicate_pair(*kv) for kv in query.iteritems()]
    return _apply_combinator('AND', qs)


def _parse_predicate_key(key, mapping=MODEL_MAPPING):
    parts = key.split('.')

    if len(parts) != 2:
        raise ParseException(
            'Keys must be in the form of "model.field", not "%s"' %
            key)

    model, field = parts

    if model not in mapping:
        raise ParseException(
            'Valid models are: %s, not "%s"' %
            (mapping.keys(), model))

    return mapping[model] + field


#TODO: Date not supported
def _parse_value(value):
    """
    A value can be either:
    * A date
    * A literal
    * A list of other values
    """
    if type(value) is list:
        return [_parse_value(v) for v in value]
    else:
        return value


def _parse_min_max_value_fn(operator):
    """
    returns a function that produces singleton
    dictionary of django operands for the given
    query operator.
    """

    def fn(predicate_value):
        # a min/max predicate can either take
        # a value or a dictionary that provides
        # a VALUE and EXCLUSIVE flag.
        if type(predicate_value) == dict:
            raw_value = predicate_value.get('VALUE')
            exclusive = predicate_value.get('EXCLUSIVE')
        else:
            raw_value = predicate_value
            exclusive = False

        if exclusive:
            key = operator
        else:
            # django use lt/lte and gt/gte
            # to handle inclusive/exclusive
            key = operator + 'e'

        value = _parse_value(raw_value)

        return {key: value}

    return fn
{'fraction_non_alphanumeric': 0.09197972570065593, 'fraction_numerical': 0.0008944543828264759, 'mean_word_length': 2.0888581952117864, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3076676', 'n_tokens_mistral': 2017, 'n_tokens_neox': 1906, 'n_words': 598}