<commit_before>"""

concurrent.futures implementation over sqs


Scatter/Gather or Map/Reduce style over two sqs queues.

"""

import random
from threading import Condition
import logging
import inspect

from janitor import utils

from concurrent.futures import Executor, Future

log = logging.getLogger('janitor.sqsexec')


def named(o):
    assert inspect.isfunction(o)
    return "%s:%s" % (o.__module__, o.__name__)


def resolve(o):
    name, func = o.rsplit(':', 1)
    module = __import__(name, fromlist=[True])
    return getattr(module, func)
    

class SQSExecutor(Executor):

    def __init__(self, session_factory, map_queue, reduce_queue):
        self.session_factory = session_factory<commit_msg>remove stray import<commit_after>"""

concurrent.futures implementation over sqs


Scatter/Gather or Map/Reduce style over two sqs queues.

"""

import random
import logging
import inspect

from janitor import utils

from concurrent.futures import Executor, Future

log = logging.getLogger('janitor.sqsexec')


def named(o):
    assert inspect.isfunction(o)
    return "%s:%s" % (o.__module__, o.__name__)


def resolve(o):
    name, func = o.rsplit(':', 1)
    module = __import__(name, fromlist=[True])
    return getattr(module, func)
    

class SQSExecutor(Executor):

    def __init__(self, session_factory, map_queue, reduce_queue):
        self.session_factory = session_factory
{'fraction_non_alphanumeric': 0.10300429184549356, 'fraction_numerical': 0.001430615164520744, 'mean_word_length': 4.360153256704981, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7013689', 'n_tokens_mistral': 448, 'n_tokens_neox': 434, 'n_words': 123}