<commit_before>import urllib.request
import json
from dateutil import parser

from coinotomy.watchers.common import Watcher
from coinotomy.utils.ticketsystem import TicketSystem
from coinotomy.config import config

NORMAL_TIMEOUT = 1000
FAST_TIMEOUT = 5

# maximum trades received per call call
MAX_TRADES = 500


class WatcherBinance(Watcher):
    ticket_system = TicketSystem(6)

    def __init__(self, name: str, symbol: str):
        Watcher.__init__(self, "binance." + name, NORMAL_TIMEOUT)

        self.backend = None
        self.api = BinanceAPI(symbol, self.log)
        self.interval = FAST_TIMEOUT
        self.newest_tid = 0

    def setup(self, backend):
        self.backend = backend

        # inefficient
        self.newest_tid = len([_ for _ in self.backend.lines()])

    def tick(self):
        trades, self.newest_tid = self.api.more_since_tid(self.newest_tid)

        for ts, p, v in trades:
            self.backend.append(ts, p, v)

        if len(trades) == MAX_TRADES:
            self.interval = FAST_TIMEOUT
        else:
            self.interval = NORMAL_TIMEOUT

    def unload(self):
        if self.backend:
            self.backend.unload()
        self.backend = None

    def wait(self, first):
        self.ticket_system.get_ticket(self.interval)


class BinanceAPI(object):
    def __init__(self, symbol, log):
        self.symbol = symbol
        self.log = log
<commit_msg>improve efficiency of binance startup<commit_after>import urllib.request
import json
from dateutil import parser

from coinotomy.watchers.common import Watcher
from coinotomy.utils.ticketsystem import TicketSystem
from coinotomy.config import config

NORMAL_TIMEOUT = 1000
FAST_TIMEOUT = 5

# maximum trades received per call call
MAX_TRADES = 500


class WatcherBinance(Watcher):
    ticket_system = TicketSystem(6)

    def __init__(self, name: str, symbol: str):
        Watcher.__init__(self, "binance." + name, NORMAL_TIMEOUT)

        self.backend = None
        self.api = BinanceAPI(symbol, self.log)
        self.interval = FAST_TIMEOUT
        self.newest_tid = 0

    def setup(self, backend):
        self.backend = backend

        # inefficient
        self.newest_tid = 0
        for line in self.backend.lines():
            self.newest_tid += 1

    def tick(self):
        trades, self.newest_tid = self.api.more_since_tid(self.newest_tid)

        for ts, p, v in trades:
            self.backend.append(ts, p, v)

        if len(trades) == MAX_TRADES:
            self.interval = FAST_TIMEOUT
        else:
            self.interval = NORMAL_TIMEOUT

    def unload(self):
        if self.backend:
            self.backend.unload()
        self.backend = None

    def wait(self, first):
        self.ticket_system.get_ticket(self.interval)


class BinanceAPI(object):
    def __init__(self, symbol, log):
        self.symbol = symbol
        self.log = log

{'fraction_non_alphanumeric': 0.08707671043538355, 'fraction_numerical': 0.007601935038009675, 'mean_word_length': 2.5608856088560885, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '334887', 'n_tokens_mistral': 927, 'n_tokens_neox': 884, 'n_words': 225}