<commit_before>"""connections.py: general tools for all cronjobs: db connection and requests"""
from os import path
from datetime import datetime
import warnings
import json  # TODO: ujson?
import concurrent
import time

import requests
import pymongo
from plumbum import cli
from requests_futures.sessions import FuturesSession

import navitron_crons.exceptions as exceptions
import navitron_crons.cli_core as cli_core

DEFAULT_HEADER = {
    'User-Agent': 'Navitron-cron: https://github.com/j9ac9k/NavitronEve'
}
HERE = path.abspath(path.dirname(__file__))

def rate_limited(requests_per_second):
    """rate limiting decorator

    Notes:
        Stolen from: https://github.com/fuzzysteve/FuzzMarket/

    """
    minimum_interval = 1.0 / float(requests_per_second)
    def decorate(func):
        last_time_called = [0.0]
        def rate_limited_func(*args,**kargs):
            elapsed = time.clock() - last_time_called[0]
            wait_remining = minimum_interval - elapsed
            if wait_remining > 0:
                time.sleep(wait_remining)
            ret = func(*args,**kargs)
            last_time_called[0] = time.clock()
            return ret
        return rate_limited_func
    return decorate


@rate_limited(200)
def async_request(
        request_obj,
        url,
        retry
):
    """rate limited async requests

    Notes:
        Stolen from: https://github.com/fuzzysteve/FuzzMarket/

    """
    future = request_obj.get(url)
    future.url = url
    future.retry = retry
    return future

def fetch_bulk_data_async(
        base_url,
        id_list,
        workers=20,
        retry=0,
        logger=cli_core.DEFAULT_LOGGER
):
    """fetch bulk data from ESI using async methods

    Notes:
        Adapted from: https://github.com/fuzzysteve/FuzzMarket/

    Args:
        base_url (str): endpoint address to map onto id_list
        id_list (:obj:`list`): list of id's for requesting
        workers (int, optional): number of async workers to apply to job
        retry (int, optional): retry failure attempts
        logger (:obj:`logging.logger`, optional): logging handle

    Returns:
        :obj:`list`: data from all enpoints

    """
    session = FuturesSession(max_workers=workers)
    logger.info('--building async request queue for: %s', base_url)
    url_list = [f'{base_url}{id_val}' for id_val in id_list]
    request_queue = []
    for url in cli.terminal.Progress(url_list):
        # logger.debug(url)  # spammy AF
        request_queue.append(async_request(
            session,
            url,
            retry
        ))

    logger.info('--reading async request results')
    results = []<commit_msg>moving import into async fetch to avoid pytest+plumbum+requests_futures bugs<commit_after>"""connections.py: general tools for all cronjobs: db connection and requests"""
from os import path
from datetime import datetime
import warnings
import json  # TODO: ujson?
import concurrent
import time

import requests
import pymongo
from plumbum import cli


import navitron_crons.exceptions as exceptions
import navitron_crons.cli_core as cli_core

DEFAULT_HEADER = {
    'User-Agent': 'Navitron-cron: https://github.com/j9ac9k/NavitronEve'
}
HERE = path.abspath(path.dirname(__file__))

def rate_limited(requests_per_second):
    """rate limiting decorator

    Notes:
        Stolen from: https://github.com/fuzzysteve/FuzzMarket/

    """
    minimum_interval = 1.0 / float(requests_per_second)
    def decorate(func):
        last_time_called = [0.0]
        def rate_limited_func(*args,**kargs):
            elapsed = time.clock() - last_time_called[0]
            wait_remining = minimum_interval - elapsed
            if wait_remining > 0:
                time.sleep(wait_remining)
            ret = func(*args,**kargs)
            last_time_called[0] = time.clock()
            return ret
        return rate_limited_func
    return decorate


@rate_limited(200)
def async_request(
        request_obj,
        url,
        retry
):
    """rate limited async requests

    Notes:
        Stolen from: https://github.com/fuzzysteve/FuzzMarket/

    """
    future = request_obj.get(url)
    future.url = url
    future.retry = retry
    return future

def fetch_bulk_data_async(
        base_url,
        id_list,
        workers=20,
        retry=0,
        logger=cli_core.DEFAULT_LOGGER
):
    """fetch bulk data from ESI using async methods

    Notes:
        Adapted from: https://github.com/fuzzysteve/FuzzMarket/

    Args:
        base_url (str): endpoint address to map onto id_list
        id_list (:obj:`list`): list of id's for requesting
        workers (int, optional): number of async workers to apply to job
        retry (int, optional): retry failure attempts
        logger (:obj:`logging.logger`, optional): logging handle

    Returns:
        :obj:`list`: data from all enpoints

    """
    from requests_futures.sessions import FuturesSession  # pytest & plumbum & pytest = FAIL
    session = FuturesSession(max_workers=workers)
    logger.info('--building async request queue for: %s', base_url)
    url_list = [f'{base_url}{id_val}' for id_val in id_list]
    request_queue = []
    for url in cli.terminal.Progress(url_list):
        # logger.debug(url)  # spammy AF
        request_queue.append(async_request(
            session,
            url,
            retry
        ))

    logger.info('--reading async request results')
    results = []
{'fraction_non_alphanumeric': 0.09324522760646109, 'fraction_numerical': 0.005506607929515419, 'mean_word_length': 2.7042828008157715, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 8, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4607566', 'n_tokens_mistral': 1691, 'n_tokens_neox': 1630, 'n_words': 430}