<commit_before># coding: utf-8
"""This class provides an iterator.  Under the covers it does multi-threaded
consumption of events, only providing information to the iterator when it's
been ordered correctly."""

import cloudpassage
import operator
import time
import urllib
import utility
from multiprocessing.dummy import Pool as ThreadPool
from requests.exceptions import ConnectionError
from cloudpassage import CloudPassageGeneral


class HaloEvents(object):
    """Instantiate with a donlib.ConfigHelper() object as an argument."""
    def __init__(self, config):
        self.halo_key = config.halo_api_key
        self.halo_secret = config.halo_api_secret_key
        self.start_timestamp = utility.Utility.iso8601_now()
        self.max_threads = config.max_threads
        self.halo_batch_size = config.halo_batch_size
        self.last_event_timestamp = None
        self.events = []
        self.halo_session = None
        self.last_event_id = ""
        self.ua = config.ua
        print("Event Collector: Starting timestamp: " + self.start_timestamp)

    def __iter__(self):
        """This allows us to iterate through the events stream."""
        while True:
            try:
                for event in self.get_next_batch():
                    yield event
            except IndexError:
                pass

    def get_next_batch(self):
        """Gets the next batch of events for the iterator."""
        url_list = self.create_url_list()
        try:
            pages = self.get_pages(url_list)
        except ConnectionError:  # Sometimes connection abort happens
            now_string = unicode(utility.Utility.iso8601_now())
            print(u'EventCollector: ConnectionError 🌧️ %s' % now_string)
            pages = [{"events": []}]
        except CloudPassageGeneral:  # We wait if this happens...
            now_string = unicode(utility.Utility.iso8601_now())
            print(u'EventCollector: Caught Halo error 🌩️ %s' % now_string)
            pages = [{"events": []}]
            time.sleep(15)
        events = self.events_from_pages(pages)
        if events[0]["id"] == self.last_event_id:
            del events[0]
        try:
            last_event_timestamp = events[-1]['created_at']<commit_msg>chg: usr: Logging tweaks !wip<commit_after>"""This class provides an iterator.  Under the covers it does multi-threaded
consumption of events, only providing information to the iterator when it's
been ordered correctly."""

import cloudpassage
import operator
import time
import urllib
import utility
from multiprocessing.dummy import Pool as ThreadPool
from requests.exceptions import ConnectionError
from cloudpassage import CloudPassageGeneral


class HaloEvents(object):
    """Instantiate with a donlib.ConfigHelper() object as an argument."""
    def __init__(self, config):
        self.halo_key = config.halo_api_key
        self.halo_secret = config.halo_api_secret_key
        self.start_timestamp = utility.Utility.iso8601_now()
        self.max_threads = config.max_threads
        self.halo_batch_size = config.halo_batch_size
        self.last_event_timestamp = None
        self.events = []
        self.halo_session = None
        self.last_event_id = ""
        self.ua = config.ua
        print("Event Collector: Starting timestamp: " + self.start_timestamp)

    def __iter__(self):
        """This allows us to iterate through the events stream."""
        while True:
            try:
                for event in self.get_next_batch():
                    yield event
            except IndexError:
                pass

    def get_next_batch(self):
        """Gets the next batch of events for the iterator."""
        url_list = self.create_url_list()
        try:
            pages = self.get_pages(url_list)
        except ConnectionError:  # Sometimes connection abort happens
            now_string = unicode(utility.Utility.iso8601_now())
            print("EventCollector: ConnectionError %s" % now_string)
            pages = [{"events": []}]
        except CloudPassageGeneral:  # We wait if this happens...
            now_string = unicode(utility.Utility.iso8601_now())
            print("EventCollector: Caught Halo API error %s" % now_string)
            pages = [{"events": []}]
            time.sleep(15)
        events = self.events_from_pages(pages)
        if events[0]["id"] == self.last_event_id:
            del events[0]
        try:
            last_event_timestamp = events[-1]['created_at']
{'fraction_non_alphanumeric': 0.08835341365461848, 'fraction_numerical': 0.007809013833110219, 'mean_word_length': 2.5635930047694755, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '86692', 'n_tokens_mistral': 1260, 'n_tokens_neox': 1226, 'n_words': 341}