<commit_before>import requests, urllib
import logging
from time import sleep


request_size_limit = 0
LOGGER = logging.getLogger(__name__)


def byteify(input):
    if isinstance(input, dict):
        return {byteify(key):byteify(value) for key,value in input.iteritems()}
    elif isinstance(input, list):
        return [byteify(element) for element in input]
    elif isinstance(input, unicode):
        return input.encode('utf-8')
    else:
        return input


def _get_data(api_name, params={}):
    params_get_string = ''
    for k, v in params.iteritems():
        params_get_string += '{}={}&'.format(k, urllib.quote(v))
    if params_get_string:
        params_get_string = '?' + params_get_string
    r = requests.get('http://api.rutracker.org/v1/' + api_name + params_get_string,
        headers={'content-type': 'application/json'})
    return byteify(r.json()['result'])


def get_limit():
    try:
        res = _get_data('get_limit')
        request_size_limit = res['limit']
        assert isinstance(request_size_limit, int),\
            'request_size_limit is not int! ({})'.format(request_size_limit)
        return request_size_limit
    except Exception:
        LOGGER.warning('failed to get request_size_limit', exc_info=True)
        return 100


def _iterative_get_data(api_name, full_list, param_generator):
    if not isinstance(full_list, list):
        full_list = [full_list]
    current = 0
    ret = {}
    while current < len(full_list):
        ret.update(_get_data(api_name, param_generator(
            full_list[current : current + request_size_limit])))
        current += request_size_limit
    return ret


def _get_data_by(api_name, filter_name, data):
    response = _iterative_get_data(api_name,
        data,
        lambda data_list: {
            'by': filter_name,
            'val': ','.join(data_list)
        })<commit_msg>cache responses from api<commit_after>import requests, urllib
import logging
from time import sleep
import pickle


request_size_limit = 0
LOGGER = logging.getLogger(__name__)

cache = {}

try:
    with open('/tmp/rutracker_cache.pickle', 'rb') as f:
        cache = pickle.load(f)
except Exception:
    pass


def byteify(input):
    if isinstance(input, dict):
        return {byteify(key):byteify(value) for key,value in input.iteritems()}
    elif isinstance(input, list):
        return [byteify(element) for element in input]
    elif isinstance(input, unicode):
        return input.encode('utf-8')
    else:
        return input


def _get_data(api_name, params={}):
    params_get_string = ''
    for k, v in params.iteritems():
        params_get_string += '{}={}&'.format(k, urllib.quote(v))
    if params_get_string:
        params_get_string = '?' + params_get_string
    r = requests.get('http://api.rutracker.org/v1/' + api_name + params_get_string,
        headers={'content-type': 'application/json'})
    return byteify(r.json()['result'])


def get_limit():
    try:
        res = _get_data('get_limit')
        request_size_limit = res['limit']
        assert isinstance(request_size_limit, int),\
            'request_size_limit is not int! ({})'.format(request_size_limit)
        return request_size_limit
    except Exception:
        LOGGER.warning('failed to get request_size_limit', exc_info=True)
        return 100


def _iterative_get_data(api_name, full_list, param_generator):
    if not isinstance(full_list, list):
        full_list = [full_list]

    # initialize cache
    global cache
    if not api_name in cache:
        cache[api_name] = {}

    api_cache = cache[api_name]
    # search in cache, first
    cache_keys = set(api_cache.keys())
    cache_hit = cache_keys.intersection(full_list)
    ret = { field: api_cache[field] for field in cache_hit }
    full_list = list(set(full_list) - cache_hit)
    if not full_list:
        return ret

    current = 0
    while current < len(full_list):
        ret.update(_get_data(api_name, param_generator(
            full_list[current : current + request_size_limit])))
        current += request_size_limit

    # update cache with new data
    api_cache.update(ret)
    with open('/tmp/rutracker_cache.pickle', 'wb') as f:
        pickle.dump(cache, f)

    return ret


def _get_data_by(api_name, filter_name, data):
    response = _iterative_get_data(api_name,
        data,
        lambda data_list: {
            'by': filter_name,
            'val': ','.join(data_list)
        })
{'fraction_non_alphanumeric': 0.11295755670334606, 'fraction_numerical': 0.003143947900291938, 'mean_word_length': 3.0712979890310788, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2463623', 'n_tokens_mistral': 1427, 'n_tokens_neox': 1401, 'n_words': 322}