<commit_before>'''
Core and Base objects for the heliosSDK. 

@author: <NAME>
'''
import json
import os
from io import BytesIO
from itertools import repeat
from math import ceil
from multiprocessing.dummy import Pool as ThreadPool

import numpy as np
from PIL import Image

from heliosSDK import BASE_API_URL


class SDKCore(object):
    """
    Core class for Python interface to Helios Core APIs.
    This class must be inherited by any additional Core API classes.
    """
    BASE_API_URL = BASE_API_URL

    def _parseInputsForQuery(self, input_dict):
        # Check for unique case: sensors
        if 'sensors' in input_dict.keys():
            query_str = input_dict.pop('sensors') + '&'
        else:
            query_str = ''

        # Parse input_dict into a str
        for key in input_dict.keys():
            if isinstance(input_dict[key], (list, tuple)):
                query_str += str(key) + '=' + ','.join([str(x) for x in input_dict[key]]) + '&'
            else:
                query_str += str(key) + '=' + str(input_dict[key]) + '&'
        query_str = query_str[:-1]

        return query_str


class IndexMixin(object):
    def index(self, **kwargs):
        max_skip = 4000
        limit = kwargs.get('limit', 100)
        skip = kwargs.get('skip', 0)

        # Log start
        self.logger.info('Entering index(kwargs={})'.format(kwargs))

        # Establish all queries.
        params_str = self._parseInputsForQuery(kwargs)
        queries = []
        for i in range(skip, max_skip, limit):
            if i + limit > max_skip:
                temp_limit = max_skip - i
            else:
                temp_limit = limit

            query_str = '{}/{}?{}&limit={}&skip={}'.format(self.BASE_API_URL,
                                                           self.CORE_API,
                                                           params_str,
                                                           temp_limit,
                                                           i)

            queries.append(query_str)

        # Do first query to find total number of results to expect.
        initial_resp = self.requestManager.get(queries.pop(0))
        initial_resp_json = initial_resp.json()

        try:
            total = initial_resp_json['properties']['total']
        except KeyError:
            total = initial_resp_json['total']

        # Warn the user when truncation occurs. (max_skip is hit)
        if total > max_skip:
            # Log truncation warning
            self.logger.warn('Maximum skip level. Truncated results will be returned for: {}'.format(kwargs))

        # Get number of results in initial query.
        try:
            n = len(initial_resp_json['features'])
        except KeyError:
            n = len(initial_resp_json['results'])

        # If only one query was necessary, return immediately.
        if n < limit:
            return [initial_resp_json]

        # Determine number of iterations that will be needed.
        n_queries_needed = int(ceil((total - skip) / float(limit))) - 1
        queries = queries[0:n_queries_needed]

        # Log number of queries required.
        self.logger.info('{} index queries required for: {}'.format(n_queries_needed, kwargs))

        # Create thread pool and get results
        num_threads = min(self.MAX_THREADS, n_queries_needed)
        if num_threads > 1:
            with ThreadPool(num_threads) as POOL:
                results = POOL.map(self.__indexWorker, queries)
        else:
            results = [self.__indexWorker(queries[0])]

        # Put initial query back in list.
        results.insert(0, initial_resp_json)

        # Log success
        self.logger.info('Leaving index(N={})'.format(total))

        return results

    def __indexWorker(self, args):
        query_str = args
<commit_msg>Fix for not catching when the number of features equals the limit. Should return right away<commit_after>'''
Core and Base objects for the heliosSDK. 

@author: <NAME>
'''
import json
import os
from io import BytesIO
from itertools import repeat
from math import ceil
from multiprocessing.dummy import Pool as ThreadPool

import numpy as np
from PIL import Image

from heliosSDK import BASE_API_URL


class SDKCore(object):
    """
    Core class for Python interface to Helios Core APIs.
    This class must be inherited by any additional Core API classes.
    """
    BASE_API_URL = BASE_API_URL

    def _parseInputsForQuery(self, input_dict):
        # Check for unique case: sensors
        if 'sensors' in input_dict.keys():
            query_str = input_dict.pop('sensors') + '&'
        else:
            query_str = ''

        # Parse input_dict into a str
        for key in input_dict.keys():
            if isinstance(input_dict[key], (list, tuple)):
                query_str += str(key) + '=' + ','.join([str(x) for x in input_dict[key]]) + '&'
            else:
                query_str += str(key) + '=' + str(input_dict[key]) + '&'
        query_str = query_str[:-1]

        return query_str


class IndexMixin(object):
    def index(self, **kwargs):
        max_skip = 4000
        limit = kwargs.get('limit', 100)
        skip = kwargs.get('skip', 0)

        # Log start
        self.logger.info('Entering index(kwargs={})'.format(kwargs))

        # Establish all queries.
        params_str = self._parseInputsForQuery(kwargs)
        queries = []
        for i in range(skip, max_skip, limit):
            if i + limit > max_skip:
                temp_limit = max_skip - i
            else:
                temp_limit = limit

            query_str = '{}/{}?{}&limit={}&skip={}'.format(self.BASE_API_URL,
                                                           self.CORE_API,
                                                           params_str,
                                                           temp_limit,
                                                           i)

            queries.append(query_str)

        # Do first query to find total number of results to expect.
        initial_resp = self.requestManager.get(queries.pop(0))
        initial_resp_json = initial_resp.json()

        try:
            total = initial_resp_json['properties']['total']
        except KeyError:
            total = initial_resp_json['total']

        # Warn the user when truncation occurs. (max_skip is hit)
        if total > max_skip:
            # Log truncation warning
            self.logger.warn('Maximum skip level. Truncated results will be returned for: {}'.format(kwargs))

        # Get number of results in initial query.
        try:
            n = len(initial_resp_json['features'])
        except KeyError:
            n = len(initial_resp_json['results'])

        # If only one query was necessary, return immediately.
        if n <= limit:
            return [initial_resp_json]

        # Determine number of iterations that will be needed.
        n_queries_needed = int(ceil((total - skip) / float(limit))) - 1
        queries = queries[0:n_queries_needed]

        # Log number of queries required.
        self.logger.info('{} index queries required for: {}'.format(n_queries_needed, kwargs))

        # Create thread pool and get results
        num_threads = min(self.MAX_THREADS, n_queries_needed)
        if num_threads > 1:
            with ThreadPool(num_threads) as POOL:
                results = POOL.map(self.__indexWorker, queries)
        else:
            results = [self.__indexWorker(queries[0])]

        # Put initial query back in list.
        results.insert(0, initial_resp_json)

        # Log success
        self.logger.info('Leaving index(N={})'.format(total))

        return results

    def __indexWorker(self, args):
        query_str = args

{'fraction_non_alphanumeric': 0.09484642076853875, 'fraction_numerical': 0.003855545559696697, 'mean_word_length': 1.885428253615128, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2335779', 'n_tokens_mistral': 2165, 'n_tokens_neox': 2076, 'n_words': 615}