<commit_before>from __future__ import print_function, unicode_literals

import ply.lex as lex
import pytz
import operator
import re
import six

from collections import OrderedDict
from datetime import timedelta
from decimal import Decimal
from django.core.exceptions import ObjectDoesNotExist
from django.db.models import Q
from functools import reduce
from ply import yacc
from temba.locations.models import AdminBoundary
from temba.utils import str_to_datetime
from temba.values.models import Value

PROPERTY_ALIASES = None  # initialised in contact_search to avoid circular import

NON_FIELD_PROPERTIES = ('name', 'urns__path')  # identifiers which are not contact fields

TEXT_LOOKUP_ALIASES = LOCATION_LOOKUP_ALIASES = {
    '=': 'iexact',
    'is': 'iexact',
    '~': 'icontains',
    'has': 'icontains'
}

DECIMAL_LOOKUP_ALIASES = {
    '=': 'exact',
    'is': 'exact',
    '>': 'gt',
    '>=': 'gte',
    '<': 'lt',
    '<=': 'lte'
}

DATETIME_LOOKUP_ALIASES = {
    '=': '<equal>',
    'is': '<equal>',
    '>': 'gt',
    '>=': 'gte',
    '<': 'lt',
    '<=': 'lte'
}


class SearchException(Exception):
    """
    Exception class for unparseable search queries
    """
    def __init__(self, message):
        self.message = message


class SearchLexer(object):
    """
    Lexer for complex search queries
    """
    t_LPAREN = r'\('
    t_RPAREN = r'\)'
    t_ignore = ' \t'  # ignore tabs and spaces

    tokens = ('AND', 'OR', 'COMPARATOR', 'TEXT', 'STRING', 'LPAREN', 'RPAREN')

    literals = '()'

    # treat reserved words specially http://www.dabeaz.com/ply/ply.html#ply_nn4
    reserved = {
        'and': 'AND',
        'or': 'OR',
        'has': 'COMPARATOR',
        'is': 'COMPARATOR',
    }

    def __init__(self, **kwargs):
        self.lexer = lex.lex(module=self, reflags=re.UNICODE, **kwargs)

    def input(self, s):
        return self.lexer.input(s)

    def token(self):
        return self.lexer.token()

    def t_COMPARATOR(self, t):
        r"""(?i)~|=|[<>]=?|~~?"""
        return t

    def t_STRING(self, t):
        r"""("[^"]*")"""
        t.value = t.value[1:-1]
        return t

    def t_TEXT(self, t):
        r"""[\w_\.\+\-\/]+"""
        t.type = self.reserved.get(t.value.lower(), 'TEXT')
        return t

    def t_error(self, t):
        raise SearchException("Invalid character %s" % t.value[0])

    def test(self, data):  # pragma: no cover
        self.lexer.input(data)
        while True:
            tok = self.lexer.token()
            if not tok:
                break
            print(tok)


@six.python_2_unicode_compatible
class ContactQuery(object):
    def __init__(self, root):
        self.root = root.simplify().split_by_prop()

    def as_query(self, org):
        prop_map = self.get_prop_map(org)

        return self.root.as_query(prop_map)

    def get_prop_map(self, org):
        from temba.contacts.models import ContactField

        prop_map = {p: None for p in self.root.get_prop_names()}

        for field in ContactField.objects.filter(org=org, key__in=prop_map.keys(), is_active=True):
            prop_map[field.key] = field

        # TODO schemes, attributes

        for prop, prop_obj in prop_map.items():
            if not prop_obj:
                raise SearchException("Unrecognized field: %s" % prop)

        return prop_map

    def __str__(self):
        return six.text_type(self.root)


class QueryNode(object):
    def simplify(self):
        return self

    def split_by_prop(self):
        return self

    def as_query(self, prop_map):
        pass


@six.python_2_unicode_compatible
class Condition(QueryNode):
    def __init__(self, prop, comparator, value):
        self.prop = prop
        self.comparator = comparator
        self.value = value

    def get_prop_names(self):
        return [self.prop]

    def as_query(self, prop_map):
        from temba.contacts.models import ContactField

        prop_obj = prop_map[self.prop]

        if isinstance(prop_obj, ContactField):
            value_query = self.get_value_query(prop_obj)

            return Q(id__in=Value.objects.filter(**value_query).values('contact_id'))
        else:
            # TODO if not a field?
            raise ValueError("TODO")

    def get_value_query(self, field):
        # TODO different value types, lookups

        lookup = 'iexact'
        return {'contact_field': field, 'string_value__%s' % lookup: self.value}

    def __str__(self):
        return '%s%s%s' % (self.prop, self.comparator, self.value)


@six.python_2_unicode_compatible
class BoolCombination(QueryNode):
    """<commit_msg>WIP (value based queries for all types)<commit_after>from __future__ import print_function, unicode_literals

import ply.lex as lex
import pytz
import operator
import re
import six

from collections import OrderedDict
from datetime import timedelta
from decimal import Decimal
from django.db.models import Q
from functools import reduce
from ply import yacc
from temba.locations.models import AdminBoundary
from temba.utils import str_to_datetime
from temba.values.models import Value

PROPERTY_ALIASES = None  # initialised in contact_search to avoid circular import

NON_FIELD_PROPERTIES = ('name', 'urns__path')  # identifiers which are not contact fields

TEXT_LOOKUP_ALIASES = LOCATION_LOOKUP_ALIASES = {
    '=': 'iexact',
    'is': 'iexact',
    '~': 'icontains',
    'has': 'icontains'
}

DECIMAL_LOOKUP_ALIASES = {
    '=': 'exact',
    'is': 'exact',
    '>': 'gt',
    '>=': 'gte',
    '<': 'lt',
    '<=': 'lte'
}

DATETIME_LOOKUP_ALIASES = {
    '=': '<equal>',
    'is': '<equal>',
    '>': 'gt',
    '>=': 'gte',
    '<': 'lt',
    '<=': 'lte'
}


class SearchException(Exception):
    """
    Exception class for unparseable search queries
    """
    def __init__(self, message):
        self.message = message


class SearchLexer(object):
    """
    Lexer for complex search queries
    """
    t_LPAREN = r'\('
    t_RPAREN = r'\)'
    t_ignore = ' \t'  # ignore tabs and spaces

    tokens = ('AND', 'OR', 'COMPARATOR', 'TEXT', 'STRING', 'LPAREN', 'RPAREN')

    literals = '()'

    # treat reserved words specially http://www.dabeaz.com/ply/ply.html#ply_nn4
    reserved = {
        'and': 'AND',
        'or': 'OR',
        'has': 'COMPARATOR',
        'is': 'COMPARATOR',
    }

    def __init__(self, **kwargs):
        self.lexer = lex.lex(module=self, reflags=re.UNICODE, **kwargs)

    def input(self, s):
        return self.lexer.input(s)

    def token(self):
        return self.lexer.token()

    def t_COMPARATOR(self, t):
        r"""(?i)~|=|[<>]=?|~~?"""
        return t

    def t_STRING(self, t):
        r"""("[^"]*")"""
        t.value = t.value[1:-1]
        return t

    def t_TEXT(self, t):
        r"""[\w_\.\+\-\/]+"""
        t.type = self.reserved.get(t.value.lower(), 'TEXT')
        return t

    def t_error(self, t):
        raise SearchException("Invalid character %s" % t.value[0])

    def test(self, data):  # pragma: no cover
        self.lexer.input(data)
        while True:
            tok = self.lexer.token()
            if not tok:
                break
            print(tok)


@six.python_2_unicode_compatible
class ContactQuery(object):
    def __init__(self, root):
        self.root = root.simplify().split_by_prop()

    def as_query(self, org):
        prop_map = self.get_prop_map(org)

        return self.root.as_query(prop_map)

    def get_prop_map(self, org):
        from temba.contacts.models import ContactField

        prop_map = {p: None for p in self.root.get_prop_names()}

        for field in ContactField.objects.filter(org=org, key__in=prop_map.keys(), is_active=True):
            prop_map[field.key] = field

        # TODO schemes, attributes

        for prop, prop_obj in prop_map.items():
            if not prop_obj:
                raise SearchException("Unrecognized field: %s" % prop)

        return prop_map

    def __str__(self):
        return six.text_type(self.root)


class QueryNode(object):
    def simplify(self):
        return self

    def split_by_prop(self):
        return self

    def as_query(self, prop_map):
        pass


@six.python_2_unicode_compatible
class Condition(QueryNode):
    def __init__(self, prop, comparator, value):
        self.prop = prop
        self.comparator = comparator
        self.value = value

    def get_prop_names(self):
        return [self.prop]

    def as_query(self, prop_map):
        from temba.contacts.models import ContactField

        prop_obj = prop_map[self.prop]

        if isinstance(prop_obj, ContactField):
            # empty string equality means contacts without that field set
            if self.comparator.lower() in ('=', 'is') and self.value == "":
                return ~Q(id__in=Value.objects.filter(contact__field=prop_obj).values('contact_id'))
            else:
                value_query = self.build_value_query(prop_obj)

                return Q(id__in=Value.objects.filter(**value_query).values('contact_id'))
        else:
            # TODO if not a field?
            raise ValueError("TODO")

    def build_value_query(self, field):
        if field.value_type == Value.TYPE_TEXT:
            return self._build_text_field_query(field)
        elif field.value_type == Value.TYPE_DECIMAL:
            return self._build_decimal_field_comparison(field)
        elif field.value_type == Value.TYPE_DATETIME:
            return self._build_datetime_field_comparison(field)
        elif field.value_type in (Value.TYPE_STATE, Value.TYPE_DISTRICT, Value.TYPE_WARD):
            return self._build_location_field_comparison(field)
        else:  # pragma: no cover
            raise ValueError("Unrecognized contact field type '%s'" % field.value_type)

    def _build_text_field_query(self, field):
        lookup = TEXT_LOOKUP_ALIASES.get(self.comparator)
        if not lookup:
            raise SearchException("Unsupported comparator %s for text field" % self.comparator)

        return {'contact_field': field, 'string_value__%s' % lookup: self.value}

    def _build_decimal_field_query(self, field):
        lookup = DECIMAL_LOOKUP_ALIASES.get(self.comparator)
        if not lookup:
            raise SearchException("Unsupported comparator %s for decimal field" % self.comparator)

        try:
            value = Decimal(self.value)
        except Exception:
            raise SearchException("Can't convert '%s' to a decimal" % self.value)

        return {'contact_field': field, 'decimal_value__%s' % lookup: value}

    def _build_datetime_field_query(self, org, field):
        lookup = DATETIME_LOOKUP_ALIASES.get(self.comparator)
        if not lookup:
            raise SearchException("Unsupported comparator %s for datetime field" % self.comparator)

        # parse as localized date and then convert to UTC
        local_date = str_to_datetime(self.value, org.timezone, org.get_dayfirst(), fill_time=False)
        if not local_date:
            raise SearchException("Unable to parse date: %s" % self.value)

        value = local_date.astimezone(pytz.utc)

        if lookup == '<equal>':
            # check if datetime is between date and date + 1d, i.e. anytime in that 24 hour period
            return {'contact_field__id': field.id,
                    'datetime_value__gte': value, 'datetime_value__lt': value + timedelta(days=1)}
        elif lookup == 'lte':
            # check if datetime is less then date + 1d, i.e. that day and all previous
            return {'contact_field__id': field.id, 'datetime_value__lt': value + timedelta(days=1)}
        elif lookup == 'gt':
            # check if datetime is greater than or equal to date + 1d, i.e. day after and subsequent
            return {'contact_field__id': field.id, 'datetime_value__gte': value + timedelta(days=1)}
        else:
            return {'contact_field__id': field.id, 'datetime_value__%s' % lookup: value}

    def _build_location_field_query(self, field):
        lookup = LOCATION_LOOKUP_ALIASES.get(self.comparator)
        if not lookup:
            raise SearchException("Unsupported comparator %s for location field" % self.comparator)

        locations = AdminBoundary.objects.filter(**{'name__%s' % lookup: self.value}).values('id')

        return {'contact_field': field, 'values__location_value__in': locations}

    def __str__(self):
        return '%s%s%s' % (self.prop, self.comparator, self.value)


@six.python_2_unicode_compatible
class BoolCombination(QueryNode):
    """
{'fraction_non_alphanumeric': 0.12281825460368295, 'fraction_numerical': 0.0017614091273018414, 'mean_word_length': 2.692284954182678, 'pattern_counts': {'":': 1, '<': 18, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7217012', 'n_tokens_mistral': 4070, 'n_tokens_neox': 3924, 'n_words': 960}