<commit_before>import time
import json
import sqlite3
import socket
from collections import deque

class safesocket(socket.socket):
    def __init__(self, *args):
        self.floodQueue = deque([0] * 10, maxlen=10)
        super().__init__(*args)

    def send(self, message, *args):
        try:
            elapsed = int(time.time() * 1000) - self.floodQueue[0]
            if not isinstance(message, bytes):
                message = message.encode()
            if elapsed / 10 < 100:
                # Start rate limiting after 10 messages within 100ms
                # to avoid IRC kicking us for flooding
                time.sleep((100 - elapsed / 10) / 1000)
            super().send(message, *args)
            self.floodQueue.append(int(time.time() * 1000))
        except Exception as e:
            print('Could not write to socket: ', e)

starttime = int(time.time())
lastDisconnect = 0

with open('config/config.json') as config_file:
    config = json.load(config_file)

channels = config['irc']['channels']
silent_channels = config['irc']['silent_channels']

db = sqlite3.connect(config['db']['location'])
cursor = db.cursor()
ircsock = None
<commit_msg>Move ratelimit functionality into its own class<commit_after>import time
import json
import sqlite3
import socket
from collections import deque

class ratelimit:
    def __init__(self, max, duration):
        self.max = max
        self.duration = duration # in milliseconds
        self.rateQueue = deque([0] * max, maxlen=max)

    def queue(self, job):
        elapsed = int(time.time() * 1000) - self.rateQueue[0]
        if elapsed / self.max < self.duration:
            time.sleep((self.duration - elapsed / self.max) / 1000)
        self.rateQueue.append(int(time.time() * 1000))
        # TODO need to adapt this for the event loop so we can yield on delays

class safesocket(socket.socket):
    def __init__(self, *args):
        self.floodQueue = ratelimit(10, 100)
        super().__init__(*args)

    def send(self, message, *args):
        try:
            if not isinstance(message, bytes):
                message = message.encode()
            # Start rate limiting after 10 messages within 100ms
            # to avoid IRC kicking us for flooding
            self.floodQueue.queue()
            super().send(message, *args)
        except Exception as e:
            print('Could not write to socket: ', e)

starttime = int(time.time())
lastDisconnect = 0

with open('config/config.json') as config_file:
    config = json.load(config_file)

channels = config['irc']['channels']
silent_channels = config['irc']['silent_channels']

db = sqlite3.connect(config['db']['location'])
cursor = db.cursor()
ircsock = None

{'fraction_non_alphanumeric': 0.11645101663585952, 'fraction_numerical': 0.023290203327171903, 'mean_word_length': 2.6816326530612247, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4779699', 'n_tokens_mistral': 848, 'n_tokens_neox': 762, 'n_words': 228}