<commit_before>import socket
from .. import _PROTOCOL_IDENTIFIER_

_SOCKET_IDENTIFIER_ = '<agutil.io.socket:1.0.0>'
class Socket:
    def __init__(self, address, port, _socket=None, _skipIdentifier=False, _useIdentifier=_PROTOCOL_IDENTIFIER_+_SOCKET_IDENTIFIER_):
        self.addr = address
        self.port = port
        if _socket!=None:
            self.sock = _socket
        else:
            self.sock = socket.socket()
            self.sock.connect((address, port))
        self.rollover = b""
        if not _skipIdentifier:
            self.send(_useIdentifier)
            remoteID = self.recv(True)
            if remoteID != _useIdentifier:
                self.close()
                raise ValueError("The remote socket provided an invalid identifier at the Socket level")

    def send(self, msg):
        if type(msg)==str:
            msg=msg.encode()
        elif type(msg)!=bytes:
            raise TypeError("msg argument must be str or bytes")
        payload_size = len(msg)
        # print("Sending: <", payload_size, ">",msg)
        self.sock.send(format(payload_size, 'x').encode()+b'|')
        # self.sock.send(b"|")
        while payload_size > 0:
            payload_size -= self.sock.send(msg)
            msg = msg[len(msg)-payload_size:]

    def recv(self, decode=False):
        msg = ""
        found_size = False
        size = ""
        while not found_size:
            intake = self.rollover + self.sock.recv(4096)
            for i in range(len(intake)):
                current = intake[i:i+1]
                if current == b'|':
                    size = int(size, 16)
                    msg = intake[i+1:i+1+size]
                    self.rollover = intake[i+1+size:]
                    found_size = True
                    break
                else:
                    size+=current.decode()

        while len(msg) < size:
            msg += self.sock.recv(min(4096, size-len(msg)))

        if decode:
            return msg.decode()
        # print("Received: <", size, ">", msg)
        return msg

    def settimeout(self, time):
        self.sock.settimeout(time)

    def gettimeout(self):
        return self.sock.gettimeout()

    def close(self):
        try:
            self.sock.setblocking(True)
            self.sock.shutdown(socket.SHUT_RDWR)
        except OSError:
            pass<commit_msg>Fixed an issue with socket rollover<commit_after>import socket
from .. import _PROTOCOL_IDENTIFIER_

_SOCKET_IDENTIFIER_ = '<agutil.io.socket:1.0.0>'
class Socket:
    def __init__(self, address, port, _socket=None, _skipIdentifier=False, _useIdentifier=_PROTOCOL_IDENTIFIER_+_SOCKET_IDENTIFIER_):
        self.addr = address
        self.port = port
        if _socket!=None:
            self.sock = _socket
        else:
            self.sock = socket.socket()
            self.sock.connect((address, port))
        self.rollover = b""
        if not _skipIdentifier:
            self.send(_useIdentifier)
            remoteID = self.recv(True)
            if remoteID != _useIdentifier:
                self.close()
                raise ValueError("The remote socket provided an invalid identifier at the Socket level")

    def send(self, msg):
        if type(msg)==str:
            msg=msg.encode()
        elif type(msg)!=bytes:
            raise TypeError("msg argument must be str or bytes")
        payload_size = len(msg)
        # print("Sending: <", payload_size, ">",msg)
        self.sock.send(format(payload_size, 'x').encode()+b'|')
        # self.sock.send(b"|")
        while payload_size > 0:
            payload_size -= self.sock.send(msg)
            msg = msg[len(msg)-payload_size:]

    def recv(self, decode=False):
        msg = ""
        found_size = False
        size = ""
        while not found_size:
            if len(self.rollover):
                intake = self.rollover
            else:
                intake = self.sock.recv(4096)
            for i in range(len(intake)):
                current = intake[i:i+1]
                if current == b'|':
                    size = int(size, 16)
                    msg = intake[i+1:i+1+size]
                    self.rollover = intake[i+1+size:]
                    found_size = True
                    break
                else:
                    size+=current.decode()

        while len(msg) < size:
            msg += self.sock.recv(min(4096, size-len(msg)))

        if decode:
            return msg.decode()
        # print("Received: <", size, ">", msg)
        return msg

    def settimeout(self, time):
        self.sock.settimeout(time)

    def gettimeout(self):
        return self.sock.gettimeout()

    def close(self):
        try:
            self.sock.setblocking(True)
            self.sock.shutdown(socket.SHUT_RDWR)
        except OSError:
            pass
{'fraction_non_alphanumeric': 0.1135705419942077, 'fraction_numerical': 0.007447248655357882, 'mean_word_length': 1.7147669848399776, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '885167', 'n_tokens_mistral': 1445, 'n_tokens_neox': 1384, 'n_words': 306}