<commit_before>import errno
import socket

import tornado.ioloop
import tornado.iostream

class WebConnectionTable(object):
    def __init__(self, app):
        # Keep a link to the owning application.
        self.app = app
        self.log = self.app.log

        self.ioloop = None
        self.listensock = None
        self.webconns = set()   # set of IOStreams (normally just one)

    def listen(self):
        self.ioloop = tornado.ioloop.IOLoop.current()
        
        sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0)
        self.listensock = sock
        #sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
        sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR,
            (sock.getsockopt (socket.SOL_SOCKET, socket.SO_REUSEADDR) | 1))
        sock.setblocking(0)
        sock.bind( ('localhost', self.app.opts.tworld_port) )
        sock.listen(32)
        
        self.ioloop.add_handler(
            sock.fileno(),
            self.listen_ready,
            tornado.ioloop.IOLoop.READ)
        
        self.log.info('Listening on port %d', self.app.opts.tworld_port)

    def listen_ready(self, fd, events):
        while True:
            sock = None
            try:
                (sock, addr) = self.listensock.accept()
                (host, port) = addr
            except socket.error as ex:
                if ex.args[0] not in (errno.EWOULDBLOCK, errno.EAGAIN):
                    raise
                return
            
            sock.setblocking(0)
            
            stream = tornado.iostream.IOStream(sock)
            stream.twhost = host
            self.webconns.add(stream)
            self.log.info('Accepted web connection from %s (now %d connected)', stream.twhost, len(self.webconns))

            stream.read_until_close(
                lambda dat:self.stream_closed(stream, dat),
                lambda dat:self.stream_read(stream, dat))

    def stream_read(self, stream, dat):
        self.log.info('### stream_read %s', dat)

    def stream_closed(self, stream, dat):
        try:
            self.webconns.remove(stream)
        except:
            pass
        self.log.info('End of web connection from %s (now %d connected)', stream.twhost, len(self.webconns))
        
        
<commit_msg>WebConn streams are their own class now.<commit_after>import errno
import socket

import tornado.ioloop
import tornado.iostream

class WebConnectionTable(object):
    def __init__(self, app):
        # Keep a link to the owning application.
        self.app = app
        self.log = self.app.log

        self.ioloop = None
        self.listensock = None
        self.webconns = set()   # set of IOStreams (normally just one)

    def listen(self):
        self.ioloop = tornado.ioloop.IOLoop.current()
        
        sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0)
        self.listensock = sock
        #sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
        sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR,
            (sock.getsockopt (socket.SOL_SOCKET, socket.SO_REUSEADDR) | 1))
        sock.setblocking(0)
        sock.bind( ('localhost', self.app.opts.tworld_port) )
        sock.listen(32)
        
        self.ioloop.add_handler(
            sock.fileno(),
            self.listen_ready,
            tornado.ioloop.IOLoop.READ)
        
        self.log.info('Listening on port %d', self.app.opts.tworld_port)

    def listen_ready(self, fd, events):
        while True:
            sock = None
            try:
                (sock, addr) = self.listensock.accept()
                (host, port) = addr
            except socket.error as ex:
                if ex.args[0] not in (errno.EWOULDBLOCK, errno.EAGAIN):
                    raise
                return
            
            sock.setblocking(0)
            
            stream = WebConnIOStream(self, sock, host)
            self.webconns.add(stream)
            self.log.info('Accepted web connection from %s (now %d connected)', stream, len(self.webconns))

            stream.read_until_close(stream.twclose, stream.twread)

class WebConnIOStream(tornado.iostream.IOStream):
    def __init__(self, table, socket, host):
        tornado.iostream.IOStream.__init__(self, socket)
        self.twhost = host
        self.twtable = table

    def __repr__(self):
        return '<WebConnIOStream (%s)>' % (self.twhost,)
        
    def twread(self, dat):
        self.twtable.log.info('### stream_read %s', dat)

    def twclose(self, dat):
        try:
            self.twtable.webconns.remove(self)
        except:
            pass
        self.twtable.log.info('End of web connection from %s (now %d connected)', self, len(self.twtable.webconns))

{'fraction_non_alphanumeric': 0.10443037974683544, 'fraction_numerical': 0.0033755274261603376, 'mean_word_length': 1.9319727891156462, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2312381', 'n_tokens_mistral': 1499, 'n_tokens_neox': 1427, 'n_words': 310}