<commit_before>import operator
import time
import urllib
from http_helper import HttpHelper
from multiprocessing.dummy import Pool as ThreadPool


class TimeSeries(object):
    """Wrap time-series object retrieval in a generator.

    Args:
        session(object): HaloSession object.
        start_time(str): ISO 8601-formatted timestamp.
        start_url(str): Path from URL, no hostname and no URL-encoded params.
        item_key(str): Top-level key, below which is a list of target items.
        params(dict): Parameters for URL, which will be URL-encoded.
    """
    def __init__(self, session, start_time, start_url, item_key, params={}):
        self.url = start_url
        self.params = params
        self.start_url = start_url
        self.batch_size = 10
        self.page_size = 50
        self.session = session
        self.max_threads = self.session.threads
        self.item_key = item_key
        self.sort_key = "created_at"
        self.last_item_id = None
        self.params["since"] = start_time
        self.params["per_page"] = self.page_size
        return

    def __iter__(self):
        """Yields one item from a time-series query against Halo. Forever."""
        while True:
            for item in self.get_next_batch():
                yield item

    def adjust_batch_size(self, adjustment_factor):
        """Adjust the batch size for subsequent queries against Halo API.

        We will never take the batch size below 1 or over the original maximum
        thread count.

        Args:
            adjustent_factor(int): Number, positive or negative, by which we
                should increase or decrease the number of concurrent requsts
                agains the Halo API.<commit_msg>chg: dev: Institute a 'ramp-up' design in TimeSeries where threads start at 1 and never exceed 10.<commit_after>import operator
import time
import urllib
from http_helper import HttpHelper
from multiprocessing.dummy import Pool as ThreadPool


class TimeSeries(object):
    """Wrap time-series object retrieval in a generator.

    Args:
        session(object): HaloSession object.
        start_time(str): ISO 8601-formatted timestamp.
        start_url(str): Path from URL, no hostname and no URL-encoded params.
        item_key(str): Top-level key, below which is a list of target items.
        params(dict): Parameters for URL, which will be URL-encoded.
    """
    def __init__(self, session, start_time, start_url, item_key, params={}):
        self.url = start_url
        self.params = params
        self.start_url = start_url
        self.batch_size = 1
        self.page_size = 50
        self.session = session
        self.max_threads = 10
        self.item_key = item_key
        self.sort_key = "created_at"
        self.last_item_id = None
        self.params["since"] = start_time
        self.params["per_page"] = self.page_size
        return

    def __iter__(self):
        """Yields one item from a time-series query against Halo. Forever."""
        while True:
            for item in self.get_next_batch():
                yield item

    def adjust_batch_size(self, adjustment_factor):
        """Adjust the batch size for subsequent queries against Halo API.

        We will never take the batch size below 1 or over the original maximum
        thread count.

        Args:
            adjustent_factor(int): Number, positive or negative, by which we
                should increase or decrease the number of concurrent requsts
                agains the Halo API.
{'fraction_non_alphanumeric': 0.07142857142857142, 'fraction_numerical': 0.006235827664399093, 'mean_word_length': 2.4295432458697763, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6171512', 'n_tokens_mistral': 978, 'n_tokens_neox': 926, 'n_words': 345}