<commit_before>import time
import errno
import socket

from weakref import ref as weakref
from itertools import izip

from redis import StrictRedis
from redis.client import list_or_args
from redis.exceptions import ConnectionError, TimeoutError

from rb.promise import Promise
from rb.poll import poll


AUTO_BATCH_COMMANDS = {
    'GET': ('MGET', True),
    'SET': ('MSET', False),
}


def assert_open(client):
    if client.closed:
        raise ValueError('I/O operation on closed file')


def send_buffer(buf, sock, block=False):
    try:
        if not block:
            timeout = sock.gettimeout()
            sock.setblocking(False)
        try:
            for idx, item in enumerate(buf):
                sent = 0
                while 1:
                    try:
                        sent = sock.send(item)
                    except IOError as e:
                        if e.errno == errno.EAGAIN:
                            continue
                        elif e.errno == errno.EWOULDBLOCK:
                            break
                        raise
                    break
                if sent < len(item):
                    del buf[:idx + 1]
                    buf.insert(0, item[sent:])
                    break<commit_msg>Slight improvement on the buffer handling.<commit_after>import time
import errno
import socket

from weakref import ref as weakref
from itertools import izip

from redis import StrictRedis
from redis.client import list_or_args
from redis.exceptions import ConnectionError, TimeoutError

from rb.promise import Promise
from rb.poll import poll


AUTO_BATCH_COMMANDS = {
    'GET': ('MGET', True),
    'SET': ('MSET', False),
}


def assert_open(client):
    if client.closed:
        raise ValueError('I/O operation on closed file')


def send_buffer(buf, sock, block=False):
    """Utility function that sends the buffer into the provided socket.
    The buffer itself will slowly clear out and is modified in place.
    """
    try:
        if not block:
            timeout = sock.gettimeout()
            sock.setblocking(False)
        try:
            for idx, item in enumerate(buf):
                sent = 0
                while 1:
                    try:
                        sent = sock.send(item)
                    except IOError as e:
                        if e.errno == errno.EAGAIN:
                            continue
                        elif e.errno == errno.EWOULDBLOCK:
                            break
                        raise
                    break
                if sent < len(item):
                    buf[:idx + 1] = [item[sent:]]
                    break
{'fraction_non_alphanumeric': 0.07212622088655146, 'fraction_numerical': 0.002629601803155522, 'mean_word_length': 1.4657407407407408, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1491554', 'n_tokens_mistral': 709, 'n_tokens_neox': 666, 'n_words': 226}