<commit_before>import json
from .webwait import follow_events

def render_message(msg):
    lines = []
    lines.append('== %d: from %s (cid=%d #%d):' % (
        msg["id"], msg["petname"], msg["cid"], msg["seqnum"]))
    payload = json.loads(msg["payload_json"])
    if "basic" in payload:
        lines.append(" basic: " + payload["basic"])
    else:
        lines.append(str(msg["payload_json"]))
    return "\n".join(lines)+"\n"

class EOFError(Exception):
    pass

def get_line(r):
    buf = ""
    while True:
        data = r.read(1)
        if not data:
            raise EOFError
        if data == "\n":
            return buf
        buf += data

def get_field(r):
    first_line = get_line(r)
    fieldname, data = first_line.split(": ", 1)
    lines = [data]
    while True:
        line = get_line(r)
        if not line:
            return fieldname, "\n".join(lines)
        lines.append(line)


def follow_messages(so, stdout, stderr):
    resp = follow_events(so["basedir"], "messages")
    if resp.status == 200:
        # httplib is not really built to read a stream of lines
        try:
            while True:
                fieldname, data = get_field(resp)
                if fieldname == "data":
                    stdout.write(render_message(json.loads(data)["new_value"]))
                    stdout.flush()
        except EOFError:
            return 0
    else:
        print >>stderr, "Error:", resp.status, resp.reason
        return 1<commit_msg>follow-messages: shut down less messily upon SIGINT<commit_after>import json
from .webwait import follow_events

def render_message(msg):
    lines = []
    lines.append('== %d: from %s (cid=%d #%d):' % (
        msg["id"], msg["petname"], msg["cid"], msg["seqnum"]))
    payload = json.loads(msg["payload_json"])
    if "basic" in payload:
        lines.append(" basic: " + payload["basic"])
    else:
        lines.append(str(msg["payload_json"]))
    return "\n".join(lines)+"\n"

class EOFError(Exception):
    pass

def get_line(r):
    buf = ""
    while True:
        data = r.read(1)
        if not data:
            raise EOFError
        if data == "\n":
            return buf
        buf += data

def get_field(r):
    first_line = get_line(r)
    fieldname, data = first_line.split(": ", 1)
    lines = [data]
    while True:
        line = get_line(r)
        if not line:
            return fieldname, "\n".join(lines)
        lines.append(line)


def follow_messages(so, stdout, stderr):
    try:
        resp = follow_events(so["basedir"], "messages")
        if resp.status != 200:
            print >>stderr, "Error:", resp.status, resp.reason
            return 1
        # httplib is not really built to read a stream of lines
        while True:
            fieldname, data = get_field(resp)
            if fieldname == "data":
                stdout.write(render_message(json.loads(data)["new_value"]))
                stdout.flush()
    except (KeyboardInterrupt, EOFError):
        return 0
{'fraction_non_alphanumeric': 0.13813813813813813, 'fraction_numerical': 0.004671338004671338, 'mean_word_length': 2.149159663865546, 'pattern_counts': {'":': 6, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1654539', 'n_tokens_mistral': 945, 'n_tokens_neox': 929, 'n_words': 243}