<commit_before>import json
import time
from log import logger
from communication import messages

TERMINATION_SEQUENCE = 'f1a5238b-ec60-430e-a0a8-5fe7442273a0'

def receive_loop(sock, socket_broken_event=None):
    leftover = ''
    while True:
        msg = sock.recv(656536)
        if msg:
            leftover = receive_message(leftover, msg)
        else:
            if socket_broken_event is not None:
                socket_broken_event.set()
            return

def receive_message(prefix, message_bytes):<commit_msg>socket timeout handling<commit_after>import json
import time
import threading
from log import logger
from communication import messages

TERMINATION_SEQUENCE = 'f1a5238b-ec60-430e-a0a8-5fe7442273a0'

def receive_loop(sock, socket_broken_event=None):
    socket_broken_event = threading.Event() if socket_broken_event is None else socket_broken_event
    leftover = ''
    while True:
        try:
            msg = sock.recv(656536)
        except Exception as e:
            print('erer', e)
            return socket_broken_event.set()
        if msg:
            leftover = receive_message(leftover, msg)
        else:
            return socket_broken_event.set()

def receive_message(prefix, message_bytes):
{'fraction_non_alphanumeric': 0.07033144704931285, 'fraction_numerical': 0.043654001616814875, 'mean_word_length': 2.9935483870967743, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '340010', 'n_tokens_mistral': 417, 'n_tokens_neox': 370, 'n_words': 91}