<commit_before>from webapi_pb2 import *
import sys
import socket
import time
import logging
import uuid
import threading

logger = logging.getLogger(__name__)

def delimitProtobuf(src):
    """Python protobuf bindings are missing writeDelimited, this is a workaround to add a delimiter"""
    from google.protobuf.internal import encoder

    serializedMessage = src.SerializeToString()
    delimiter = encoder._VarintBytes(len(serializedMessage))

    return delimiter + serializedMessage

def readDelimited(sock):
    """Read a delmited protobuf from a socket"""
    # FIXME - current implementation is very expensive in terms of heap thrash and it will fail for protobufs bigger than  32768 bytes
    from google.protobuf.internal import decoder
    delimsize = 2
    delimbytes = sock.recv(delimsize)
    (payloadsize, pos) = decoder._DecodeVarint(delimbytes, 0)
    # We now know size of our msg, but we might have unused bytes from the delimeter
    numextra = delimsize - pos
    numtoread = payloadsize - numextra
    payload = delimbytes[pos:] + sock.recv(numtoread, socket.MSG_WAITALL)
    msg = Envelope()
    #print "payloadsize=%d, pos=%d, numextra=%d, numtoread=%d, paylen=%d" % (payloadsize, pos, numextra, numtoread, len(payload))
    msg.ParseFromString(payload)
    return msg

class GCSHooks(object):
    def __init__(self, host = "api.3drobotics.com", port = 5555):
        connecttimeout = 15
        readtimeout = 30
        self.sock = socket.create_connection((host, port), connecttimeout)
        self.sock.settimeout(60 * 60)
        self.sock.setblocking(True)
        self.startTime = long(round(time.time() * 1e6)) # time in usecs since 1970
<commit_msg>Fixes MSG_WAITALL implementation for Windows.<commit_after>from webapi_pb2 import *
import sys
import socket
import time
import logging
import uuid
import threading

logger = logging.getLogger(__name__)

# Not defined on Windows in Python for some reason.
# https://github.com/Perl/perl5/blob/blead/win32/include/sys/socket.h#L35
if sys.platform == 'win32':
    MSG_WAITALL = 0x8
else:
    MSG_WAITALL = socket.MSG_WAITALL

def delimitProtobuf(src):
    """Python protobuf bindings are missing writeDelimited, this is a workaround to add a delimiter"""
    from google.protobuf.internal import encoder

    serializedMessage = src.SerializeToString()
    delimiter = encoder._VarintBytes(len(serializedMessage))

    return delimiter + serializedMessage

def readDelimited(sock):
    """Read a delmited protobuf from a socket"""
    # FIXME - current implementation is very expensive in terms of heap thrash and it will fail for protobufs bigger than  32768 bytes
    from google.protobuf.internal import decoder
    delimsize = 2
    delimbytes = sock.recv(delimsize)
    (payloadsize, pos) = decoder._DecodeVarint(delimbytes, 0)
    # We now know size of our msg, but we might have unused bytes from the delimeter
    numextra = delimsize - pos
    numtoread = payloadsize - numextra
    payload = delimbytes[pos:] + sock.recv(numtoread, MSG_WAITALL)
    msg = Envelope()
    #print "payloadsize=%d, pos=%d, numextra=%d, numtoread=%d, paylen=%d" % (payloadsize, pos, numextra, numtoread, len(payload))
    msg.ParseFromString(payload)
    return msg

class GCSHooks(object):
    def __init__(self, host = "api.3drobotics.com", port = 5555):
        connecttimeout = 15
        readtimeout = 30
        self.sock = socket.create_connection((host, port), connecttimeout)
        self.sock.settimeout(60 * 60)
        self.sock.setblocking(True)
        self.startTime = long(round(time.time() * 1e6)) # time in usecs since 1970

{'fraction_non_alphanumeric': 0.0905566325117696, 'fraction_numerical': 0.01744669066740515, 'mean_word_length': 3.894308943089431, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '483819', 'n_tokens_mistral': 1139, 'n_tokens_neox': 1065, 'n_words': 332}