<commit_before>"""Simple Python Wrapper of the Yahoo! Query Language
"""

from __future__ import absolute_import

import re
import logging

import requests
from myql import errors 


logging.basicConfig(level=logging.DEBUG,format="[%(asctime)s %(levelname)s] [%(name)s.%(module)s.%(funcName)s] %(message)s \n")
logger = logging.getLogger('mYQL')

logging.getLogger('requests').disabled = True # Disabling requests default logger


class YQL(object):
    '''Yet another Python Yahoo! Query Language Wrapper
    Attributes:
    - url : data provider url
    - table : default table, so you won't have to specify a table later
    - format : default format of the responses
    - diagnostics : set to <True> to see diagnostics on queries
    - community : set to <True> to have access to community tables
    '''
    PUBLIC_URL = 'https://query.yahooapis.com/v1/public/yql'
    PRIVATE_URL = 'https://query.yahooapis.com/v1/yql'
    COMMUNITY_DATA  = "env 'store://datatables.org/alltableswithkeys'; " #Access to community table 

    FUNC_FILTERS = ['sort', 'tail', 'truncate', 'reverse', 'unique', 'sanitize']
  
    def __init__(self, community=True, format='json', jsonCompact=False, crossProduct=None, debug=False, diagnostics=False, oauth=None):
        self.community = community # True means access to community data
        self.format = format
        self._table = None
        self._query = None # used to build query when using methods such as <select>, <insert>, ...
        self._payload = {} # Last payload
        self.diagnostics = diagnostics # Who knows, someone would like to turn it ON lol
        self._limit = None
        self._offset = None
        self.crossProduct = crossProduct
        self.jsonCompact = jsonCompact
        self.debug = debug
    
        if oauth:
            self.oauth = oauth

    def __repr__(self):
        '''Returns information on the current instance
        '''
        return "<Community>: {0} - <Format>: {1} ".format(self.community, self.format)

    def _payload_builder(self, query, format=None):
        '''Build the payload'''
        if self.community :
            query = self.COMMUNITY_DATA + query # access to community data tables

        if vars(self).get('yql_table_url') : # Attribute only defined when MYQL.use has been called before
            query = "use '{0}' as {1}; ".format(self.yql_table_url, self.yql_table_name) + query

        if vars(self).get('_func'): # if post query function filters
            query = '| '.join((query, self._func))

        self._query = query
        logger.info("QUERY = %s" %(query,))

        payload = {
            'q': query,
            'callback': '',#This is not javascript
            'diagnostics': self.diagnostics,
            'format': format if format else self.format,
            'debug': self.debug,
            'jsonCompact': self.jsonCompact
        }
        if self.crossProduct:
            payload['crossProduct'] = self.crossProduct

        self._payload = payload
        logger.info("PAYLOAD = %s " %(payload, ))

        return payload

    def raw_query(self, query, format=None, pretty=False):
        '''Executes a YQL query and returns a response
        >>>...
        >>> resp = yql.raw_query('select * from weather.forecast where woeid=2502265')
        >>>
        '''
        if format:
            format = format
        else:
            format = self.format

        payload = self._payload_builder(query, format=format)
        response = self.execute_query(payload)
        if pretty:
            response = self.response_builder(response)

        return response

    def execute_query(self, payload):
        '''Execute the query and returns and response'''
        if vars(self).get('oauth'):
            if not self.oauth.token_is_valid(): # Refresh token if token has expired
                self.oauth.refresh_token()
            response = self.oauth.session.get(self.PRIVATE_URL, params= payload, header_auth=True)
        else:
            response = requests.get(self.PUBLIC_URL, params= payload)

        self._response = response # Saving last response object.
        return response

    def _clause_formatter(self, cond):
        '''Formats conditions
        args is a list of ['field', 'operator', 'value']
        '''

        if len(cond) == 2 :
            cond = ' '.join(cond)
            return cond

        if 'in' in cond[1].lower() :
            if not isinstance(cond[2], tuple):
                    raise TypeError('("{0}") must be of type <tuple>'.format(cond[2]))

            if not isinstance(cond[2], str) and 'select' not in cond[2][0].lower() :
                cond[2] = "({0})".format(','.join(map(str,["'{0}'".format(e) for e in cond[2]])))
            elif not isinstance(cond[2], str) and 'select' in cond[2][0].lower() :
                cond[2] = "({0})".format(','.join(map(str,["{0}".format(e) for e in cond[2]])))
            else:
                cond[2] = "({0})".format(','.join(map(str,["{0}".format(e) for e in cond[2]])))

            cond = " ".join(cond)
        else: 
            if isinstance(cond[2], str):
                var = re.match('^@(\w+)$', cond[2])
            else:
                var = None
            if var :
                cond[2] = "{0}".format(var.group(1))
            else :
                cond[2] = "'{0}'".format(cond[2])
            cond = ' '.join(cond)

        return cond
    
    def response_builder(self, response):
        '''Try to return a pretty formatted response object
        '''
        try:
            r = response.json()
            result = r['query']['results']
            response = {
                'num_result': r['query']['count'] ,
                'result': result
            }
        except (Exception,) as e:
            print(e)
            return response.content

        return response

    def _func_filters(self, filters):
        '''Build post query filters
        '''
        if not isinstance(filters, list):
            raise TypeError('func_filters must be a <type list>')

        for i, func in enumerate(filters) :
            if isinstance(func, str) and func == 'reverse':
                filters[i] = 'reverse()'
            elif isinstance(func, tuple) and func[0] in YQL.FUNC_FILTERS:
                filters[i] = '{:s}(count={:d})'.format(*func)
            elif isinstance(func, dict) :
                func_stmt = ''
                try:
                    func_name = func.keys()[0]
                except: #Py3
                    func_name = list(func)[0]

                value = [ "{0}='{1}'".format(v[0], v[1]) for v in func[func_name] ]
                func_stmt = ','.join(value)<commit_msg>fix issue when List are passed to where with IN<commit_after>"""Simple Python Wrapper of the Yahoo! Query Language
"""

from __future__ import absolute_import

import re
import logging

import requests
from myql import errors 


logging.basicConfig(level=logging.DEBUG,format="[%(asctime)s %(levelname)s] [%(name)s.%(module)s.%(funcName)s] %(message)s \n")
logger = logging.getLogger('mYQL')

logging.getLogger('requests').disabled = True # Disabling requests default logger


class YQL(object):
    '''Yet another Python Yahoo! Query Language Wrapper
    Attributes:
    - url : data provider url
    - table : default table, so you won't have to specify a table later
    - format : default format of the responses
    - diagnostics : set to <True> to see diagnostics on queries
    - community : set to <True> to have access to community tables
    '''
    PUBLIC_URL = 'https://query.yahooapis.com/v1/public/yql'
    PRIVATE_URL = 'https://query.yahooapis.com/v1/yql'
    COMMUNITY_DATA  = "env 'store://datatables.org/alltableswithkeys'; " #Access to community table 

    FUNC_FILTERS = ['sort', 'tail', 'truncate', 'reverse', 'unique', 'sanitize']
  
    def __init__(self, community=True, format='json', jsonCompact=False, crossProduct=None, debug=False, diagnostics=False, oauth=None):
        self.community = community # True means access to community data
        self.format = format
        self._table = None
        self._query = None # used to build query when using methods such as <select>, <insert>, ...
        self._payload = {} # Last payload
        self.diagnostics = diagnostics # Who knows, someone would like to turn it ON lol
        self._limit = None
        self._offset = None
        self.crossProduct = crossProduct
        self.jsonCompact = jsonCompact
        self.debug = debug
    
        if oauth:
            self.oauth = oauth

    def __repr__(self):
        '''Returns information on the current instance
        '''
        return "<Community>: {0} - <Format>: {1} ".format(self.community, self.format)

    def _payload_builder(self, query, format=None):
        '''Build the payload'''
        if self.community :
            query = self.COMMUNITY_DATA + query # access to community data tables

        if vars(self).get('yql_table_url') : # Attribute only defined when MYQL.use has been called before
            query = "use '{0}' as {1}; ".format(self.yql_table_url, self.yql_table_name) + query

        if vars(self).get('_func'): # if post query function filters
            query = '| '.join((query, self._func))

        self._query = query
        logger.info("QUERY = %s" %(query,))

        payload = {
            'q': query,
            'callback': '',#This is not javascript
            'diagnostics': self.diagnostics,
            'format': format if format else self.format,
            'debug': self.debug,
            'jsonCompact': self.jsonCompact
        }
        if self.crossProduct:
            payload['crossProduct'] = self.crossProduct

        self._payload = payload
        logger.info("PAYLOAD = %s " %(payload, ))

        return payload

    def raw_query(self, query, format=None, pretty=False):
        '''Executes a YQL query and returns a response
        >>>...
        >>> resp = yql.raw_query('select * from weather.forecast where woeid=2502265')
        >>>
        '''
        if format:
            format = format
        else:
            format = self.format

        payload = self._payload_builder(query, format=format)
        response = self.execute_query(payload)
        if pretty:
            response = self.response_builder(response)

        return response

    def execute_query(self, payload):
        '''Execute the query and returns and response'''
        if vars(self).get('oauth'):
            if not self.oauth.token_is_valid(): # Refresh token if token has expired
                self.oauth.refresh_token()
            response = self.oauth.session.get(self.PRIVATE_URL, params= payload, header_auth=True)
        else:
            response = requests.get(self.PUBLIC_URL, params= payload)

        self._response = response # Saving last response object.
        return response

    def _clause_formatter(self, cond):
        '''Formats conditions
        args is a list of ['field', 'operator', 'value']
        '''

        if len(cond) == 2 :
            cond = ' '.join(cond)
            return cond

        if 'in' in cond[1].lower() :
            if not isinstance(cond[2], (tuple, list)):
                    raise TypeError('("{0}") must be of type <tuple>'.format(cond[2]))

            if not isinstance(cond[2], str) and 'select' not in cond[2][0].lower() :
                cond[2] = "({0})".format(','.join(map(str,["'{0}'".format(e) for e in cond[2]])))
            elif not isinstance(cond[2], str) and 'select' in cond[2][0].lower() :
                cond[2] = "({0})".format(','.join(map(str,["{0}".format(e) for e in cond[2]])))
            else:
                cond[2] = "({0})".format(','.join(map(str,["{0}".format(e) for e in cond[2]])))

            cond = " ".join(cond)
        else: 
            if isinstance(cond[2], str):
                var = re.match('^@(\w+)$', cond[2])
            else:
                var = None
            if var :
                cond[2] = "{0}".format(var.group(1))
            else :
                cond[2] = "'{0}'".format(cond[2])
            cond = ' '.join(cond)

        return cond
    
    def response_builder(self, response):
        '''Try to return a pretty formatted response object
        '''
        try:
            r = response.json()
            result = r['query']['results']
            response = {
                'num_result': r['query']['count'] ,
                'result': result
            }
        except (Exception,) as e:
            print(e)
            return response.content

        return response

    def _func_filters(self, filters):
        '''Build post query filters
        '''
        if not isinstance(filters, (list,tuple)):
            raise TypeError('func_filters must be a <type list> or <type tuple>')

        for i, func in enumerate(filters) :
            if isinstance(func, str) and func == 'reverse':
                filters[i] = 'reverse()'
            elif isinstance(func, tuple) and func[0] in YQL.FUNC_FILTERS:
                filters[i] = '{:s}(count={:d})'.format(*func)
            elif isinstance(func, dict) :
                func_stmt = ''
                try:
                    func_name = func.keys()[0]
                except: #Py3
                    func_name = list(func)[0]

                value = [ "{0}='{1}'".format(v[0], v[1]) for v in func[func_name] ]
                func_stmt = ','.join(value)
{'fraction_non_alphanumeric': 0.1320053515683068, 'fraction_numerical': 0.007730043109855805, 'mean_word_length': 2.1002304147465436, 'pattern_counts': {'":': 0, '<': 20, '<?xml version=': 0, '>': 38, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '447645', 'n_tokens_mistral': 3921, 'n_tokens_neox': 3757, 'n_words': 1062}