<commit_before># -*- coding: utf-8 -*-
from __future__ import unicode_literals

import json
import decimal
import datetime
import copy
import time
from collections import OrderedDict
import re
import sympy
from threading import Thread

from django.contrib.auth.models import User
from django.contrib.postgres.fields import JSONField
from django.db.models import *

from aggregator.models import *

from query_designer.formula_functions import *
from query_designer.query_processors.utils import SolrResultEncoder, PostgresResultEncoder


class Query(Model):
    user = ForeignKey(User, related_name='queries')
    title = TextField(default='Untitled query')
    created = DateTimeField(auto_now_add=True)
    updated = DateTimeField(auto_now=True)
    document = JSONField()
    design = JSONField(blank=True, null=True, default=None)
    count = IntegerField(blank=True, null=True, default=None)
    headers = JSONField(blank=True, null=True, default=None)

    def __unicode__(self):
        return '<#%d "%s"%s>' % (self.pk, self.title, ' (%d results)' % self.count if self.count is not None else '')

    @staticmethod
    def operator_to_str(op, mode='postgres'):
        return {
            # comparison
            'eq': ('=', ':'),
            'neq': ('!=', None),
            'gt': ('>', None),
            'gte': ('>=', None),
            'lt': ('<', None),
            'lte': ('<=', None),
            'mod': ('%', None),

            # boolean
            '&&': ('AND', 'AND'),
            'and': ('AND', 'AND'),
            '||': ('OR', 'OR'),
            'or': ('OR', 'OR'),
            '!': ('NOT', None),
            'not': ('NOT', None),
        }[op.lower()][0 if mode == 'postgres' else 1]

    @staticmethod
    def process_filters(self, filters, mode='postgres', quote=False):
        # end value
        if type(filters) in [int, float]:
            try:
                col_name = ''
                from_order = int(filters[filters.find('i')+1:filters.find('_')])
                if from_order >= 0:
                    table_name = self.document['from'][from_order]['name']
                    for x in self.document['from'][from_order]['select']:
                        if x['name'] == filters:
                            if x['type'] != 'VALUE':
                                col_name = Dimension.objects.get(pk=x['type']).data_column_name
                            else:
                                col_name = 'value'
                    filters = table_name + '.' + col_name
            except:
                return filters
            return filters

        if type(filters) in [str, unicode]:
            if quote and (mode == 'solr') and filters.strip() != '*' and (not filters.startswith('"')) and filters:
                return '"%s"' % filters
            else:
                return filters

        # Special case: parsing location filters
        # inside_rect|outside_rect <<lat_south,lng_west>,<lat_north,lng_east>>

        if filters['op'] in ['inside_rect', 'outside_rect', ]:
            rect_start = filters['b'].split('<')[2].split('>,')[0].split(',')
            rect_end = filters['b'].split('>,<')[1].split('>')[0].split(',')

            #lat = filters['a'] + '_latitude'
            #lng = filters['a'] + '_longitude'

            lat_col_name = ''
            lon_col_name = ''
            from_order = int(filters['a'][1])
            table_name = self.document['from'][from_order]['name']
            for x in self.document['from'][from_order]['select']:
                if x['name'] == (filters['a'] + '_latitude'):
                    lat_col_name = Dimension.objects.get(pk=x['type']).data_column_name
                if x['name'] == (filters['a'] + '_longitude'):
                    lon_col_name = Dimension.objects.get(pk=x['type']).data_column_name

            lat = table_name+'.'+lat_col_name
            lng = table_name+'.'+lon_col_name

            result = '%s >= %s AND %s <= %s' % (lat, rect_start[0], lat, rect_end[0])
            result += ' AND %s >= %s AND %s <= %s' % (lng, rect_start[1], lng, rect_end[1])
            lat = filters['a'] + '_latitude'
            lng = filters['a'] + '_longitude'

            if mode == 'solr':
                result = '%s:[%s TO %s]' % (lat, rect_start[0], rect_end[0])
                result += ' AND %s:[%s TO %s]' % (lng, rect_start[1], rect_end[1])
            else:
                result = '%s >= %s AND %s <= %s' % (lat, rect_start[0], lat, rect_end[0])
                result += ' AND %s >= %s AND %s <= %s' % (lng, rect_start[1], lng, rect_end[1])

            if filters['op'] == 'outside_rect':
                if mode == 'postgres':
                    result = 'NOT(%s)' % result
                else:
                    result = '-(%s)' % result

            return result

        result = ''
        _op = filters['op'].lower()
        if mode == 'solr' and _op in ['neq', 'gt', 'gte', 'lt', 'lte', 'mod', '!', 'not']:
            if _op == 'neq':
                result = '-%s:%s' % (Query.process_filters(filters['a']), Query.process_filters(filters['b']))
            elif _op in ['gt', 'gte']:
                result = '%s:[%s TO *]' % (Query.process_filters(filters['a']), Query.process_filters(filters['b']))
            elif _op in ['lt', 'lte']:
                result = '%s:[* TO %s]' % (Query.process_filters(self, filters['a']), Query.process_filters(filters['b']))
            elif _op == 'mod':
                result = 'mod(%s, %s)' % (Query.process_filters(filters['a']), Query.process_filters(filters['b']))
            elif _op in ['!', 'not']:
                raise NotImplementedError('TODO fix missing NOT operator in solr')

        else:
            _a = Query.process_filters(filters['a'], mode=mode)
            _b = Query.process_filters(self, filters['b'], mode=mode, quote=True)

            result = '%s %s %s' % \<commit_msg>Refactored process_filters from staticmethod to method<commit_after># -*- coding: utf-8 -*-
from __future__ import unicode_literals

import json
import decimal
import datetime
import copy
import time
from collections import OrderedDict
import re
import sympy
from threading import Thread

from django.contrib.auth.models import User
from django.contrib.postgres.fields import JSONField
from django.db.models import *

from aggregator.models import *

from query_designer.formula_functions import *
from query_designer.query_processors.utils import SolrResultEncoder, PostgresResultEncoder


class Query(Model):
    user = ForeignKey(User, related_name='queries')
    title = TextField(default='Untitled query')
    created = DateTimeField(auto_now_add=True)
    updated = DateTimeField(auto_now=True)
    document = JSONField()
    design = JSONField(blank=True, null=True, default=None)
    count = IntegerField(blank=True, null=True, default=None)
    headers = JSONField(blank=True, null=True, default=None)

    def __unicode__(self):
        return '<#%d "%s"%s>' % (self.pk, self.title, ' (%d results)' % self.count if self.count is not None else '')

    @staticmethod
    def operator_to_str(op, mode='postgres'):
        return {
            # comparison
            'eq': ('=', ':'),
            'neq': ('!=', None),
            'gt': ('>', None),
            'gte': ('>=', None),
            'lt': ('<', None),
            'lte': ('<=', None),
            'mod': ('%', None),

            # boolean
            '&&': ('AND', 'AND'),
            'and': ('AND', 'AND'),
            '||': ('OR', 'OR'),
            'or': ('OR', 'OR'),
            '!': ('NOT', None),
            'not': ('NOT', None),
        }[op.lower()][0 if mode == 'postgres' else 1]

    def process_filters(self, filters, mode='postgres', quote=False):
        # end value
        if type(filters) in [int, float]:
            try:
                col_name = ''
                from_order = int(filters[filters.find('i')+1:filters.find('_')])
                if from_order >= 0:
                    table_name = self.document['from'][from_order]['name']
                    for x in self.document['from'][from_order]['select']:
                        if x['name'] == filters:
                            if x['type'] != 'VALUE':
                                col_name = Dimension.objects.get(pk=x['type']).data_column_name
                            else:
                                col_name = 'value'
                    filters = table_name + '.' + col_name
            except:
                return filters
            return filters

        if type(filters) in [str, unicode]:
            if quote and (mode == 'solr') and filters.strip() != '*' and (not filters.startswith('"')) and filters:
                return '"%s"' % filters
            else:
                return filters

        # Special case: parsing location filters
        # inside_rect|outside_rect <<lat_south,lng_west>,<lat_north,lng_east>>

        if filters['op'] in ['inside_rect', 'outside_rect', ]:
            rect_start = filters['b'].split('<')[2].split('>,')[0].split(',')
            rect_end = filters['b'].split('>,<')[1].split('>')[0].split(',')

            #lat = filters['a'] + '_latitude'
            #lng = filters['a'] + '_longitude'

            lat_col_name = ''
            lon_col_name = ''
            from_order = int(filters['a'][1])
            table_name = self.document['from'][from_order]['name']
            for x in self.document['from'][from_order]['select']:
                if x['name'] == (filters['a'] + '_latitude'):
                    lat_col_name = Dimension.objects.get(pk=x['type']).data_column_name
                if x['name'] == (filters['a'] + '_longitude'):
                    lon_col_name = Dimension.objects.get(pk=x['type']).data_column_name

            lat = table_name+'.'+lat_col_name
            lng = table_name+'.'+lon_col_name

            result = '%s >= %s AND %s <= %s' % (lat, rect_start[0], lat, rect_end[0])
            result += ' AND %s >= %s AND %s <= %s' % (lng, rect_start[1], lng, rect_end[1])
            lat = filters['a'] + '_latitude'
            lng = filters['a'] + '_longitude'

            if mode == 'solr':
                result = '%s:[%s TO %s]' % (lat, rect_start[0], rect_end[0])
                result += ' AND %s:[%s TO %s]' % (lng, rect_start[1], rect_end[1])
            else:
                result = '%s >= %s AND %s <= %s' % (lat, rect_start[0], lat, rect_end[0])
                result += ' AND %s >= %s AND %s <= %s' % (lng, rect_start[1], lng, rect_end[1])

            if filters['op'] == 'outside_rect':
                if mode == 'postgres':
                    result = 'NOT(%s)' % result
                else:
                    result = '-(%s)' % result

            return result

        result = ''
        _op = filters['op'].lower()
        if mode == 'solr' and _op in ['neq', 'gt', 'gte', 'lt', 'lte', 'mod', '!', 'not']:
            if _op == 'neq':
                result = '-%s:%s' % (self.process_filters(filters['a']), self.process_filters(filters['b']))
            elif _op in ['gt', 'gte']:
                result = '%s:[%s TO *]' % (self.process_filters(filters['a']), self.process_filters(filters['b']))
            elif _op in ['lt', 'lte']:
                result = '%s:[* TO %s]' % (self.process_filters(filters['a']), self.process_filters(filters['b']))
            elif _op == 'mod':
                result = 'mod(%s, %s)' % (self.process_filters(filters['a']), self.process_filters(filters['b']))
            elif _op in ['!', 'not']:
                raise NotImplementedError('TODO fix missing NOT operator in solr')

        else:
            _a = self.process_filters(filters['a'], mode=mode)
            _b = self.process_filters(filters['b'], mode=mode, quote=True)

            result = '%s %s %s' % \
{'fraction_non_alphanumeric': 0.17179421984913976, 'fraction_numerical': 0.003729129587253157, 'mean_word_length': 2.056203056203056, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '779774', 'n_tokens_mistral': 3632, 'n_tokens_neox': 3512, 'n_words': 822}