<commit_before>#
#
#

from __future__ import absolute_import, division, print_function, \
    unicode_literals

from StringIO import StringIO
from concurrent.futures import Future, ThreadPoolExecutor
from importlib import import_module
from os import environ
import logging

from .provider.base import BaseProvider
from .provider.yaml import YamlProvider
from .yaml import safe_load
from .zone import Zone


class _AggregateTarget(object):
    id = 'aggregate'

    def __init__(self, targets):
        self.targets = targets

    def supports(self, record):
        for target in self.targets:
            if not target.supports(record):
                return False
        return True

    @property
    def SUPPORTS_GEO(self):
        for target in self.targets:
            if not target.SUPPORTS_GEO:
                return False
        return True


class MainThreadExecutor(object):
    '''
    Dummy executor that runs things on the main call during the involcation of
    submit, but still returns a future object with the result. This allows code
    to be written to handle async, even in the case where we don't want to use
    multiple threads/workers and would prefer that things flow as if
    traditionally written.
    '''

    def submit(self, func, *args, **kwargs):
        future = Future()
        try:
            future.set_result(func(*args, **kwargs))
        except Exception as e:
            future.set_exception(e)
        return future


class Manager(object):
    log = logging.getLogger('Manager')

    def __init__(self, config_file, max_workers=None):<commit_msg>"main thread" not "main call"<commit_after>#
#
#

from __future__ import absolute_import, division, print_function, \
    unicode_literals

from StringIO import StringIO
from concurrent.futures import Future, ThreadPoolExecutor
from importlib import import_module
from os import environ
import logging

from .provider.base import BaseProvider
from .provider.yaml import YamlProvider
from .yaml import safe_load
from .zone import Zone


class _AggregateTarget(object):
    id = 'aggregate'

    def __init__(self, targets):
        self.targets = targets

    def supports(self, record):
        for target in self.targets:
            if not target.supports(record):
                return False
        return True

    @property
    def SUPPORTS_GEO(self):
        for target in self.targets:
            if not target.SUPPORTS_GEO:
                return False
        return True


class MainThreadExecutor(object):
    '''
    Dummy executor that runs things on the main thread during the involcation
    of submit, but still returns a future object with the result. This allows
    code to be written to handle async, even in the case where we don't want to
    use multiple threads/workers and would prefer that things flow as if
    traditionally written.
    '''

    def submit(self, func, *args, **kwargs):
        future = Future()
        try:
            future.set_result(func(*args, **kwargs))
        except Exception as e:
            future.set_exception(e)
        return future


class Manager(object):
    log = logging.getLogger('Manager')

    def __init__(self, config_file, max_workers=None):
{'fraction_non_alphanumeric': 0.06587942821628341, 'fraction_numerical': 0.0, 'mean_word_length': 2.8275862068965516, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7275980', 'n_tokens_mistral': 870, 'n_tokens_neox': 843, 'n_words': 325}