<commit_before>import socket

from tornado import ioloop, iostream

delimiter = '\r\n\r\n'

def read_command():
    stream.read_until(delimiter, on_headers)

def on_headers(data):
    data = data.replace('\r\n\r\n', '')
    print '<< ' + data
    tokens = data.split(' ')
    ts = tokens[1]
    resource = tokens[2]
    content = open(resource).read()
    response = 'CONTENT %s %s' % (ts, content)
    print '>> ' + response
    stream.write(response + delimiter)
    read_command()

s = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0)
stream = iostream.IOStream(s)
stream.connect(('localhost', 8890), read_command)
ioloop.IOLoop.instance().start()<commit_msg>auth + command handling in client<commit_after>import socket

from tornado import ioloop, iostream

delimiter = '\r\n\r\n'

def recv_data(callback):
    while stream.reading() == True:
        continue

    def onrecv(data):
        data = data.replace(delimiter, '')
        print '<< ' + data
        callback(data)

    stream.read_until(delimiter, onrecv)

def send_data(data, callback):
    print '>> '  + data
    stream.write(data + delimiter, callback)

def process_commands(data):
    tokens = data.split(' ')
    cmd = tokens[0]
    args = tokens[1:]

    if cmd == 'ACK':
        read_command()

    elif cmd == 'FETCH':
        on_fetch(args)

    else:
        read_command()

def authenticate():
    send_data('IDENT ashish locker', read_command)

def read_command():
    recv_data(process_commands)

def on_fetch(args):
    ts = args[0]
    resource = args[1]
    try:
        content = open(resource).read()
    except:
        content = 'Not found.'
        
    response = 'CONTENT {0} {1}'.format(ts, content)
    send_data(response, read_command)

s = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0)
stream = iostream.IOStream(s)
stream.connect(('localhost', 8890), authenticate)
ioloop.IOLoop.instance().start()
{'fraction_non_alphanumeric': 0.1418812401471361, 'fraction_numerical': 0.009458749343142408, 'mean_word_length': 3.2690582959641254, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1404552', 'n_tokens_mistral': 659, 'n_tokens_neox': 633, 'n_words': 140}