<commit_before>import logging
import re
import socket
import socketserver
import struct

import aluigi
from . import models, tasks, settings
from .statsd import statsd


logger = logging.getLogger(__name__)


def encode_list(servers):
    byte_string = bytearray()

    for ip, port in servers:
        byte_string.extend(socket.inet_aton(ip))
        byte_string.extend(struct.pack('>h', port))

    byte_string.extend(b'\\final\\')

    return aluigi.encode_list(settings.GAMESPY_KEY, memoryview(byte_string).tobytes())


def decode_list(encrypted):
    decrypted = aluigi.decode_list(settings.GAMESPY_KEY, encrypted)

    for i in range(0, len(decrypted) - 1, 6):
        if decrypted[i:].startswith(b'\\final'):
            break

        ip = socket.inet_ntoa(decrypted[i:i+4])
        port = struct.unpack('>h', decrypted[i+4:i+6])[0]
        yield ip, port


# TODO cache for some small amount of time
def get_encoded_server_list():
    servers = []

    for server in models.Server.query.filter_by(online=True).all():
        servers.append((server.ip, server.info_port))

    return encode_list(servers)


class MasterService(socketserver.TCPServer):

    allow_reuse_address = True

    def __init__(self):
        super().__init__(('', settings.MASTER_PORT), MasterHandler)


class MasterHandler(socketserver.BaseRequestHandler):

    def handle(self):
        logger.debug(f'Responding to {self.request.getpeername()[0]}...')

        self.request.sendall('\\basic\\\\secure\\'.encode('latin1'))
        self.request.send(get_encoded_server_list())

        statsd.incr('master_pulled')


class HeartbeatService(socketserver.UDPServer):

    allow_reuse_address = True

    def __init__(self):
        super().__init__(('', settings.HEARTBEAT_PORT), HeartbeatHandler)


class HeartbeatHandler(socketserver.BaseRequestHandler):

    def handle(self):
        logger.debug(f'Got heartbeat from {self.client_address[0]}...')

        msg = self.request[0].decode('ascii').split('\\')

        if msg[1] != 'heartbeat':
            return

        tasks.register(ip=self.client_address[0], port=msg[2], force_pull=True)

        statsd.incr('heartbeat_registered')


def fetch_from_master(ip):
    logger.debug(f'Fetching new servers from {ip}...')

    client = socket.create_connection((ip, settings.MASTER_PORT))
<commit_msg>#8: refreshing DB session as often as possible.<commit_after>import logging
import re
import socket
import socketserver
import struct

import aluigi
from . import models, tasks, settings
from .database import db_session
from .statsd import statsd


logger = logging.getLogger(__name__)


def encode_list(servers):
    byte_string = bytearray()

    for ip, port in servers:
        byte_string.extend(socket.inet_aton(ip))
        byte_string.extend(struct.pack('>h', port))

    byte_string.extend(b'\\final\\')

    return aluigi.encode_list(settings.GAMESPY_KEY, memoryview(byte_string).tobytes())


def decode_list(encrypted):
    decrypted = aluigi.decode_list(settings.GAMESPY_KEY, encrypted)

    for i in range(0, len(decrypted) - 1, 6):
        if decrypted[i:].startswith(b'\\final'):
            break

        ip = socket.inet_ntoa(decrypted[i:i+4])
        port = struct.unpack('>h', decrypted[i+4:i+6])[0]
        yield ip, port


# TODO cache for some small amount of time
def get_encoded_server_list():
    servers = []

    for server in models.Server.query.filter_by(online=True).all():
        servers.append((server.ip, server.info_port))

    return encode_list(servers)


class MasterService(socketserver.TCPServer):

    allow_reuse_address = True

    def __init__(self):
        super().__init__(('', settings.MASTER_PORT), MasterHandler)


class MasterHandler(socketserver.BaseRequestHandler):

    def handle(self):
        logger.debug(f'Responding to {self.request.getpeername()[0]}...')

        # so we're not stuck in a previous invalid transaction
        db_session.remove()

        self.request.sendall('\\basic\\\\secure\\'.encode('latin1'))
        self.request.send(get_encoded_server_list())

        statsd.incr('master_pulled')


class HeartbeatService(socketserver.UDPServer):

    allow_reuse_address = True

    def __init__(self):
        super().__init__(('', settings.HEARTBEAT_PORT), HeartbeatHandler)


class HeartbeatHandler(socketserver.BaseRequestHandler):

    def handle(self):
        logger.debug(f'Got heartbeat from {self.client_address[0]}...')

        # so we're not stuck in a previous invalid transaction
        db_session.remove()

        msg = self.request[0].decode('ascii').split('\\')

        if msg[1] != 'heartbeat':
            return

        tasks.register(ip=self.client_address[0], port=msg[2], force_pull=True)

        statsd.incr('heartbeat_registered')


def fetch_from_master(ip):
    logger.debug(f'Fetching new servers from {ip}...')

    client = socket.create_connection((ip, settings.MASTER_PORT))

{'fraction_non_alphanumeric': 0.12797074954296161, 'fraction_numerical': 0.005890717042453788, 'mean_word_length': 3.5592592592592593, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5964417', 'n_tokens_mistral': 1585, 'n_tokens_neox': 1523, 'n_words': 305}