<commit_before>from collections import defaultdict, deque
from functools import partial
import logging
import os
import socket
import sys
from threading import Event, Lock, Thread
import time
import traceback
import Queue
from errno import EALREADY, EINPROGRESS, EWOULDBLOCK, EINVAL, EISCONN, errorcode

import asyncore

try:
    from cStringIO import StringIO
except ImportError:
    from StringIO import StringIO  # ignore flake8 warning: # NOQA

try:
    import ssl
except ImportError:
    ssl = None  # NOQA

from cassandra import OperationTimedOut
from cassandra.connection import (Connection, ResponseWaiter, ConnectionShutdown,
                                  ConnectionBusy, ConnectionException, NONBLOCKING,
                                  MAX_STREAM_PER_CONNECTION)
from cassandra.decoder import RegisterMessage
from cassandra.marshal import int32_unpack

log = logging.getLogger(__name__)

_loop_started = False
_loop_lock = Lock()

_starting_conns = set()
_starting_conns_lock = Lock()


def _run_loop():
    global _loop_started
    log.debug("Starting asyncore event loop")
    with _loop_lock:
        while True:
            try:
                asyncore.loop(timeout=0.001, use_poll=True, count=None)
            except Exception:
                log.debug("Asyncore event loop stopped unexepectedly", exc_info=True)
                break

            with _starting_conns_lock:
                if not _starting_conns:
                    break

        _loop_started = False
        if log:
            # this can happen during interpreter shutdown
            log.debug("Asyncore event loop ended")


def _start_loop():
    global _loop_started
    should_start = False
    did_acquire = False
    try:
        did_acquire = _loop_lock.acquire(False)
        if did_acquire and not _loop_started:
            _loop_started = True
            should_start = True
    finally:
        if did_acquire:
            _loop_lock.release()

    if should_start:
        t = Thread(target=_run_loop, name="event_loop")
        t.daemon = True
        t.start()


class AsyncoreConnection(Connection, asyncore.dispatcher):
    """
    An implementation of :class:`.Connection` that utilizes the ``asyncore``
    module in the Python standard library for its event loop.
    """

    _total_reqd_bytes = 0
    _writable = False
    _readable = False
    _have_listeners = False

    @classmethod
    def factory(cls, *args, **kwargs):
        timeout = kwargs.pop('timeout', 5.0)
        conn = cls(*args, **kwargs)
        conn.connected_event.wait(timeout)
        if conn.last_error:
            raise conn.last_error
        elif not conn.connected_event.is_set():
            conn.close()
            raise OperationTimedOut("Timed out creating connection")
        else:
            return conn

    def __init__(self, *args, **kwargs):
        Connection.__init__(self, *args, **kwargs)
        asyncore.dispatcher.__init__(self)

        self.connected_event = Event()
        self._iobuf = StringIO()

        self._callbacks = {}
        self._push_watchers = defaultdict(set)
        self.deque = deque()
        self.deque_lock = Lock()

        with _starting_conns_lock:
            _starting_conns.add(self)

        self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
        self.connect((self.host, self.port))

        if self.sockopts:
            for args in self.sockopts:
                self.socket.setsockopt(*args)

        self._writable = True
        self._readable = True

        # start the global event loop if needed
        _start_loop()

    def create_socket(self, family, type):
        # copied from asyncore, but with the line to set the socket in
        # non-blocking mode removed (we will do that after connecting)
        self.family_and_type = family, type
        sock = socket.socket(family, type)
        if self.ssl_options:
            if not ssl:
                raise Exception("This version of Python was not compiled with SSL support")
            sock = ssl.wrap_socket(sock, **self.ssl_options)
        self.set_socket(sock)

    def connect(self, address):
        # this is copied directly from asyncore.py, except that
        # a timeout is set before connecting
        self.connected = False
        self.connecting = True
        self.socket.settimeout(1.0)
        err = self.socket.connect_ex(address)
        if err in (EINPROGRESS, EALREADY, EWOULDBLOCK) \
        or err == EINVAL and os.name in ('nt', 'ce'):
            raise ConnectionException("Timed out connecting to %s" % (address[0]))
        if err in (0, EISCONN):
            self.addr = address
            self.setblocking(0)
            self.handle_connect_event()<commit_msg>remove asyncore DeprecationWarning<commit_after>from collections import defaultdict, deque
from functools import partial
import logging
import os
import socket
import sys
from threading import Event, Lock, Thread
import time
import traceback
import Queue
from errno import EALREADY, EINPROGRESS, EWOULDBLOCK, EINVAL, EISCONN, errorcode

import asyncore

try:
    from cStringIO import StringIO
except ImportError:
    from StringIO import StringIO  # ignore flake8 warning: # NOQA

try:
    import ssl
except ImportError:
    ssl = None  # NOQA

from cassandra import OperationTimedOut
from cassandra.connection import (Connection, ResponseWaiter, ConnectionShutdown,
                                  ConnectionBusy, ConnectionException, NONBLOCKING,
                                  MAX_STREAM_PER_CONNECTION)
from cassandra.decoder import RegisterMessage
from cassandra.marshal import int32_unpack

log = logging.getLogger(__name__)

_loop_started = False
_loop_lock = Lock()

_starting_conns = set()
_starting_conns_lock = Lock()


def _run_loop():
    global _loop_started
    log.debug("Starting asyncore event loop")
    with _loop_lock:
        while True:
            try:
                asyncore.loop(timeout=0.001, use_poll=True, count=None)
            except Exception:
                log.debug("Asyncore event loop stopped unexepectedly", exc_info=True)
                break

            with _starting_conns_lock:
                if not _starting_conns:
                    break

        _loop_started = False
        if log:
            # this can happen during interpreter shutdown
            log.debug("Asyncore event loop ended")


def _start_loop():
    global _loop_started
    should_start = False
    did_acquire = False
    try:
        did_acquire = _loop_lock.acquire(False)
        if did_acquire and not _loop_started:
            _loop_started = True
            should_start = True
    finally:
        if did_acquire:
            _loop_lock.release()

    if should_start:
        t = Thread(target=_run_loop, name="event_loop")
        t.daemon = True
        t.start()


class AsyncoreConnection(Connection, asyncore.dispatcher):
    """
    An implementation of :class:`.Connection` that utilizes the ``asyncore``
    module in the Python standard library for its event loop.
    """

    _total_reqd_bytes = 0
    _writable = False
    _readable = False
    _have_listeners = False

    @classmethod
    def factory(cls, *args, **kwargs):
        timeout = kwargs.pop('timeout', 5.0)
        conn = cls(*args, **kwargs)
        conn.connected_event.wait(timeout)
        if conn.last_error:
            raise conn.last_error
        elif not conn.connected_event.is_set():
            conn.close()
            raise OperationTimedOut("Timed out creating connection")
        else:
            return conn

    def __init__(self, *args, **kwargs):
        Connection.__init__(self, *args, **kwargs)
        asyncore.dispatcher.__init__(self)

        self.connected_event = Event()
        self._iobuf = StringIO()

        self._callbacks = {}
        self._push_watchers = defaultdict(set)
        self.deque = deque()
        self.deque_lock = Lock()

        with _starting_conns_lock:
            _starting_conns.add(self)

        self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
        self.connect((self.host, self.port))

        if self.sockopts:
            for args in self.sockopts:
                self.socket.setsockopt(*args)

        self._writable = True
        self._readable = True

        # start the global event loop if needed
        _start_loop()

    def create_socket(self, family, type):
        # copied from asyncore, but with the line to set the socket in
        # non-blocking mode removed (we will do that after connecting)
        self.family_and_type = family, type
        sock = socket.socket(family, type)
        if self.ssl_options:
            if not ssl:
                raise Exception("This version of Python was not compiled with SSL support")
            sock = ssl.wrap_socket(sock, **self.ssl_options)
        self.set_socket(sock)

    def connect(self, address):
        # this is copied directly from asyncore.py, except that
        # a timeout is set before connecting
        self.connected = False
        self.connecting = True
        self.socket.settimeout(1.0)
        err = self.socket.connect_ex(address)
        if err in (EINPROGRESS, EALREADY, EWOULDBLOCK) \
        or err == EINVAL and os.name in ('nt', 'ce'):
            raise ConnectionException("Timed out connecting to %s" % (address[0]))
        if err in (0, EISCONN):
            self.addr = address
            self.socket.setblocking(0)
            self.handle_connect_event()
{'fraction_non_alphanumeric': 0.07587589700295483, 'fraction_numerical': 0.0031658927817644574, 'mean_word_length': 2.329936753338018, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1532118', 'n_tokens_mistral': 2741, 'n_tokens_neox': 2633, 'n_words': 759}