<commit_before>import signal
import asyncio

from pprint import pprint
from collections import defaultdict, Counter

from .proxy import Proxy
from .judge import Judge, judgesList
from .checker import ProxyChecker
from .negotiators import BaseNegotiator
from .utils import log, set_my_ip, IPPortPatternLine
from .providers import Provider, providersList


class Broker:
    def __init__(self,
                 queue,
                 timeout=8,
                 attempts_conn=3,
                 max_concurrent_conn=200,
                 judges=None,
                 providers=None,
                 verify_ssl=False,
                 loop=None):
        self._qResult = queue
        self._allFoundProxies = []
        self._allFoundProxyPairs = set()

        self._providers = []
        if providers:
            for p in providers:
                if isinstance(p, Provider):
                    self._providers.append(p)
                else:
                    self._providers.append(Provider(p))
        else:
            self._providers = providersList

        self._judges = []
        if judges:
            for j in judges:
                if isinstance(j, Judge):
                    self._judges.append(j)
                else:
                    self._judges.append(Judge(j))
        else:
            self._judges = judgesList

        self._loop = loop or asyncio.get_event_loop()
        self._limit = None
        self._countries= None
        self._on_check = []
        self._to_check = []
        self._isDone = False
        self._tasks = []
        self._all_checked = asyncio.Event(loop=self._loop)
        self._all_checked.set()
        self._checker = ProxyChecker(broker=self,
                                     judges=self._judges,
                                     loop=self._loop)
        self._timeout = timeout
        self._cycle_lock = asyncio.Lock()
        self._setup(timeout, attempts_conn, max_concurrent_conn, verify_ssl)

    def _setup(self, timeout, attempts_conn, max_concurrent_conn, verify_ssl):
        if isinstance(max_concurrent_conn, asyncio.Semaphore):
            sem = max_concurrent_conn
        else:
            sem = asyncio.Semaphore(max_concurrent_conn)

        self._loop.add_signal_handler(signal.SIGINT, self._done)

        Judge.clear()
        Judge._sem = sem
        Judge._timeout = timeout
        Judge._loop = self._loop
        Judge._verifySSL = verify_ssl
        Proxy._sem = sem
        Proxy._loop = self._loop
        Proxy._timeout = timeout
        Proxy._verifySSL = verify_ssl
        Provider._sem = sem
        Provider._loop = self._loop
        BaseNegotiator._sem = sem
        BaseNegotiator._attemptsConnect = attempts_conn

    async def grab(self, *, countries=None, limit=None):
        self._countries = countries
        self._limit = limit
        await self._run(self._grab(types=None, push=self.push_to_result))

    async def find(self, *, data=None, types=None, countries=None, limit=None):
        await set_my_ip(self._timeout, self._loop)<commit_msg>fix handle signal on Windows<commit_after>import signal
import asyncio

from pprint import pprint
from collections import defaultdict, Counter

from .proxy import Proxy
from .judge import Judge, judgesList
from .checker import ProxyChecker
from .negotiators import BaseNegotiator
from .utils import log, set_my_ip, IPPortPatternLine
from .providers import Provider, providersList


class Broker:
    def __init__(self,
                 queue,
                 timeout=8,
                 attempts_conn=3,
                 max_concurrent_conn=200,
                 judges=None,
                 providers=None,
                 verify_ssl=False,
                 loop=None):
        self._qResult = queue
        self._allFoundProxies = []
        self._allFoundProxyPairs = set()

        self._providers = []
        if providers:
            for p in providers:
                if isinstance(p, Provider):
                    self._providers.append(p)
                else:
                    self._providers.append(Provider(p))
        else:
            self._providers = providersList

        self._judges = []
        if judges:
            for j in judges:
                if isinstance(j, Judge):
                    self._judges.append(j)
                else:
                    self._judges.append(Judge(j))
        else:
            self._judges = judgesList

        self._loop = loop or asyncio.get_event_loop()
        self._limit = None
        self._countries= None
        self._on_check = []
        self._to_check = []
        self._isDone = False
        self._tasks = []
        self._all_checked = asyncio.Event(loop=self._loop)
        self._all_checked.set()
        self._checker = ProxyChecker(broker=self,
                                     judges=self._judges,
                                     loop=self._loop)
        self._timeout = timeout
        self._cycle_lock = asyncio.Lock()
        self._setup(timeout, attempts_conn, max_concurrent_conn, verify_ssl)

    def _setup(self, timeout, attempts_conn, max_concurrent_conn, verify_ssl):
        if isinstance(max_concurrent_conn, asyncio.Semaphore):
            sem = max_concurrent_conn
        else:
            sem = asyncio.Semaphore(max_concurrent_conn)

        try:
            self._loop.add_signal_handler(signal.SIGINT, self._done)
        except NotImplementedError:
            # add_signal_handler() is not implemented on Win
            # https://docs.python.org/3.5/library/asyncio-eventloops.html#windows
            pass

        Judge.clear()
        Judge._sem = sem
        Judge._timeout = timeout
        Judge._loop = self._loop
        Judge._verifySSL = verify_ssl
        Proxy._sem = sem
        Proxy._loop = self._loop
        Proxy._timeout = timeout
        Proxy._verifySSL = verify_ssl
        Provider._sem = sem
        Provider._loop = self._loop
        BaseNegotiator._sem = sem
        BaseNegotiator._attemptsConnect = attempts_conn

    async def grab(self, *, countries=None, limit=None):
        self._countries = countries
        self._limit = limit
        await self._run(self._grab(types=None, push=self.push_to_result))

    async def find(self, *, data=None, types=None, countries=None, limit=None):
        await set_my_ip(self._timeout, self._loop)
{'fraction_non_alphanumeric': 0.07992448080553807, 'fraction_numerical': 0.0018879798615481435, 'mean_word_length': 1.8303650934995548, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2312587', 'n_tokens_mistral': 1747, 'n_tokens_neox': 1695, 'n_words': 376}