<commit_before># -*- coding: utf-8 -*-
from concurrent.futures import ThreadPoolExecutor, CancelledError, Future
from collections import deque
from datetime import datetime
from functools import partial
from sqlalchemy.exc import IntegrityError
from geopy.distance import great_circle
from queue import Queue
from multiprocessing.managers import SyncManager
import argparse
import asyncio
import logging
import os
import random
import sys
import threading
import time
import pickle

from pgoapi import (
    exceptions as pgoapi_exceptions,
    PGoApi,
    utilities as pgoapi_utils,
)

import config
import db
import utils


START_TIME = time.time()
GLOBAL_VISITS = 0
GLOBAL_SEEN = 0
NOTIFICATIONS_SENT = 0
LAST_LOGIN = 0


# Check whether config has all necessary attributes
REQUIRED_SETTINGS = (
    'DB_ENGINE',
    'MAP_START',
    'MAP_END',
    'GRID',
    'ACCOUNTS',
    'COMPUTE_THREADS',
    'NETWORK_THREADS'
)
for setting_name in REQUIRED_SETTINGS:
    if not hasattr(config, setting_name):
        raise RuntimeError('Please set "{}" in config'.format(setting_name))

# Set defaults for missing config options
OPTIONAL_SETTINGS = {
    'LONGSPAWNS': False,
    'PROXIES': None,
    'SCAN_RADIUS': 70,
    'SCAN_DELAY': (10, 12, 11),
    'NOTIFY_IDS': None,
    'NOTIFY_RANKING': None,
    'CONTROL_SOCKS': None,
    'ENCRYPT_PATH': None,
    'HASH_PATH': None
}
for setting_name, default in OPTIONAL_SETTINGS.items():
    if not hasattr(config, setting_name):
        setattr(config, setting_name, default)

if config.CONTROL_SOCKS:
    from stem import Signal
    from stem.control import Controller
    import stem.util.log
    stem.util.log.get_logger().level = 40
    CIRCUIT_TIME = dict()
    CIRCUIT_FAILURES = dict()
    CIRCUIT_LATENCIES = dict()
    for proxy in config.PROXIES:
        CIRCUIT_TIME[proxy] = time.time()
        CIRCUIT_FAILURES[proxy] = 0
        CIRCUIT_LATENCIES[proxy] = deque(maxlen=30)
else:
    CIRCUIT_TIME = None
    CIRCUIT_FAILURES = None
    CIRCUIT_LATENCIES = None

BAD_STATUSES = (
    'LOGIN FAIL',
    'EXCEPTION',
    'BAD LOGIN',
    'RETRYING',
    'THROTTLE',
)


class MalformedResponse(Exception):
    """Raised when server response is malformed"""


class BannedAccount(Exception):
    """Raised when account is banned"""


def configure_logger(filename='worker.log'):
    logging.basicConfig(
        filename=filename,
        format=(
            '[%(asctime)s][%(levelname)8s][%(name)s] '
            '%(message)s'
        ),
        style='%',
        level=logging.INFO,
    )

try:
    with open('cells.pickle', 'rb') as f:
        CELL_IDS = pickle.load(f)
    if config.COMPUTE_THREADS > 2:
        config.NETWORK_THREADS += config.COMPUTE_THREADS - 2
        config.COMPUTE_THREADS = 2
except Exception:
    CELL_IDS = dict()


class Spawns:
    def __init__(self):
        self.spawns = None
        self.session = db.Session()

    def update_spawns(self):
        self.spawns = db.get_spawns(self.session)

    def have_id(self, spawn_id):
        return spawn_id in self.spawns

    def get_despawn_seconds(self, spawn_id):
        if self.have_id(spawn_id):
            return self.spawns[spawn_id][2]
        else:
            return None

    def get_despawn_time(self, spawn_id):
        if self.have_id(spawn_id):
            current_hour = utils.get_current_hour()
            despawn_time = self.get_despawn_seconds(spawn_id) + current_hour
            return despawn_time
        else:
            return None

    def get_time_till_hidden(self, spawn_id):
        if not self.have_id(spawn_id):
            return None
        despawn_seconds = self.spawns[spawn_id][2]
        return utils.time_until_time(despawn_seconds)

SPAWNS = Spawns()

if config.NOTIFY_IDS or config.NOTIFY_RANKING:
    import notification
    notifier = notification.Notifier(SPAWNS)

class Slave:
    """Single worker walking on the map"""

    def __init__(
            self,
            worker_no,
            db_processor,
            cell_ids_executor,
            network_executor,
            extra_queue,
            captcha_queue,
            worker_dict,
            device_info=None,
            proxy=None
    ):
        self.worker_no = worker_no
        self.visits = 0
        # asyncio/thread references
        self.future = None  # worker's own future
        self.db_processor = db_processor
        self.cell_ids_executor = cell_ids_executor
        self.network_executor = network_executor
        # Some handy counters
        self.total_seen = 0
        # State variables
        self.busy = False
        self.killed = False  # killed worker will stay killed
        self.logged_in = False
        # Other variables
        self.last_visit = 0
        self.last_api_latency = 0
        self.error_code = 'INIT'
        self.empty_visits = 0
        self.location = utils.get_start_coords(self.worker_no, altitude=True)
        # And now, configure logger and PGoApi
        self.logger = logging.getLogger('worker-{}'.format(worker_no))
        self.device_info = device_info
        self.after_spawn = None
        self.speed = 0
        self.api = PGoApi(device_info=device_info)
        if config.ENCRYPT_PATH:
            self.api.set_signature_lib(config.ENCRYPT_PATH)
        if config.HASH_PATH:
            self.api.set_hash_lib(config.HASH_PATH)
        self.api.set_position(*self.location)
        self.api.set_logger(self.logger)
        self.extra_queue = extra_queue
        self.captcha_queue = captcha_queue
        self.worker_dict = worker_dict
        if proxy:
            self.set_proxy(proxy)
        else:
            self.proxy = None

    def call_api(self, method, *args, **kwargs):
        """Returns decorated function that measures execution time

        This works exactly like functools.partial does.
        """
        def inner():
            start = time.time()
            result = method(*args, **kwargs)
            self.last_api_latency = time.time() - start
            if CIRCUIT_LATENCIES:
                self.add_circuit_latency()
            return result
        return inner

    def add_circuit_latency(self):
        CIRCUIT_LATENCIES[self.proxy].append(self.last_api_latency)
        samples = len(CIRCUIT_LATENCIES[self.proxy])
        if samples > 10:
            average = sum(CIRCUIT_LATENCIES[self.proxy]) / samples
            if average > 10:
                self.swap_circuit('average latency of ' + str(round(average)) + 's')

    def set_proxy(self, proxy):
        self.proxy = proxy
        self.api.set_proxy({'http': proxy, 'https': proxy})


    async def bench_account(self):
        self.logged_in = False
        self.error_code = 'BENCHING'
        self.empty_visits = 0
        new_account = self.extra_queue.get()
        self.logger.warning('Swapping ' +
                            config.ACCOUNTS[self.worker_no][0] +
                            ' due to CAPTCHA.')
        self.captcha_queue.put(config.ACCOUNTS[self.worker_no])
        config.ACCOUNTS[self.worker_no] = new_account
        self.device_info = utils.get_worker_device(self.worker_no)


    async def swap_account(self, reason=''):
        self.logged_in = False
        self.error_code = 'SWAPPING'
        self.empty_visits = 0
        new_account = self.extra_queue.get()
        self.logger.warning(new_account)
        self.logger.warning('Swapping out ' +
                            config.ACCOUNTS[self.worker_no][0] +
                            ' because ' + reason + '.')
        self.extra_queue.put(config.ACCOUNTS[self.worker_no])
        config.ACCOUNTS[self.worker_no] = new_account
        self.device_info = utils.get_worker_device(self.worker_no)

    def swap_proxy(self, reason=''):
        self.set_proxy(random.choice(config.PROXIES))
        self.logger.warning('Swapped out ' + self.proxy +
                            ' due to ' + reason)

    def swap_circuit(self, reason=''):
        if not config.CONTROL_SOCKS:
            if config.PROXIES:
                swap_proxy(self, reason=reason)
            return
        time_passed = time.time() - CIRCUIT_TIME[self.proxy]
        if time_passed > 180:
            socket = config.CONTROL_SOCKS[self.proxy]
            with Controller.from_socket_file(path=socket) as controller:
                controller.authenticate()
                controller.signal(Signal.NEWNYM)
            CIRCUIT_TIME[self.proxy] = time.time()
            CIRCUIT_FAILURES[self.proxy] = 0
            CIRCUIT_LATENCIES[self.proxy] = deque(maxlen=30)
            self.logger.warning('Changed circuit on ' + self.proxy +
                                ' due to ' + reason)
        else:
            self.logger.info('Skipped changing circuit on ' + self.proxy +
                             ' because it was changed ' + str(time_passed)
                             + ' seconds ago.')

    async def login(self, initial=True):
        """Logs worker in and prepares for scanning"""
        self.error_code = 'LOGIN'
        loop = asyncio.get_event_loop()
        self.logger.info('Trying to log in')
        global LAST_LOGIN
        time_required = random.uniform(3, 5)
        while (time.time() - LAST_LOGIN) < time_required:
            await asyncio.sleep(1.5)
        LAST_LOGIN = time.time()
        for attempts in range(0,5):
            try:
                await loop.run_in_executor(
                    self.network_executor,
                    self.call_api(
                        self.api.set_authentication,
                        username=config.ACCOUNTS[self.worker_no][0],
                        password=config.ACCOUNTS[self.worker_no][1],
                        provider=config.ACCOUNTS[self.worker_no][2],
                    )
                )
                self.logged_in = True
                self.error_code = 'READY'
                return
            except pgoapi_exceptions.ServerSideAccessForbiddenException:
                self.logger.error('Banned IP: ' + self.proxy)
                self.error_code = 'IP BANNED'
                self.swap_circuit(reason='ban')
                await self.random_sleep(sleep_min=15, sleep_max=20)
            except pgoapi_exceptions.AuthException:
                self.logger.warning('Login failed: ' + config.ACCOUNTS[self.worker_no][0])
                self.error_code = 'LOGIN FAIL'
                await self.swap_account(reason='login failed')
                await self.random_sleep()
            except pgoapi_exceptions.NotLoggedInException:
                self.logger.error('Invalid credentials: ' + config.ACCOUNTS[self.worker_no][0])
                self.error_code = 'BAD LOGIN'
                await self.swap_account(reason='bad login')
                await self.random_sleep()
            except pgoapi_exceptions.ServerBusyOrOfflineException:
                self.logger.info('Server too busy - restarting')
                self.error_code = 'RETRYING'
                await self.random_sleep()
            except pgoapi_exceptions.ServerSideRequestThrottlingException:
                self.logger.info('Server throttling - sleeping for a bit')
                self.error_code = 'THROTTLE'
                await self.random_sleep(sleep_min=10)
            except CancelledError:
                self.kill()
                return
            except Exception as err:
                self.logger.exception('A wild exception appeared! ' + str(err))
                self.error_code = 'EXCEPTION'
                await self.random_sleep()
        return False


    async def visit(self, point, i):
        """Wrapper for self.visit_point - runs it a few times before giving up

        Also is capable of restarting in case an error occurs.
        """
        visited = False
        for attempts in range(0,5):
            try:
                if not self.logged_in:
                    await self.login()
                visited = await self.visit_point(point, i)
            except pgoapi_exceptions.ServerSideAccessForbiddenException:
                self.logger.error('Banned IP.')
                self.error_code = 'IP BANNED'
                self.swap_circuit(reason='ban')
                await self.random_sleep(sleep_min=15, sleep_max=20)
            except pgoapi_exceptions.AuthException:
                self.logger.warning('Login failed: ' + config.ACCOUNTS[self.worker_no][0])
                self.error_code = 'LOGIN FAIL'
                self.logged_in = False
                await self.swap_account(reason='failed login')
                await self.random_sleep()
            except pgoapi_exceptions.NotLoggedInException:
                self.logger.error('Invalid credentials: ' + config.ACCOUNTS[self.worker_no][0])
                self.error_code = 'BAD LOGIN'
                await self.swap_account(reason='bad login')
                self.logged_in = False
                await self.random_sleep()
            except pgoapi_exceptions.ServerBusyOrOfflineException:
                self.logger.info('Server too busy - restarting')
                self.error_code = 'RETRYING'
                await self.random_sleep()
            except pgoapi_exceptions.ServerSideRequestThrottlingException:
                self.logger.info('Server throttling - sleeping for a bit')
                self.error_code = 'THROTTLE'
                await self.random_sleep(sleep_min=10)
            except MalformedResponse:
                self.logger.warning('Malformed response received!')
                self.error_code = 'RESTART'
                await self.random_sleep()
            except BannedAccount:
                self.error_code = 'BANNED?'
                await self.swap_account(reason='code 3')
            except Exception as err:
                self.logger.exception('A wild exception appeared!')
                self.error_code = 'EXCEPTION'
                await self.random_sleep()
            else:
                if visited:<commit_msg>Create account variable in worker class<commit_after># -*- coding: utf-8 -*-
from concurrent.futures import ThreadPoolExecutor, CancelledError, Future
from collections import deque
from datetime import datetime
from functools import partial
from sqlalchemy.exc import IntegrityError
from geopy.distance import great_circle
from queue import Queue
from multiprocessing.managers import SyncManager
import argparse
import asyncio
import logging
import os
import random
import sys
import threading
import time
import pickle

from pgoapi import (
    exceptions as pgoapi_exceptions,
    PGoApi,
    utilities as pgoapi_utils,
)

import config
import db
import utils


START_TIME = time.time()
GLOBAL_VISITS = 0
GLOBAL_SEEN = 0
NOTIFICATIONS_SENT = 0
LAST_LOGIN = 0


# Check whether config has all necessary attributes
REQUIRED_SETTINGS = (
    'DB_ENGINE',
    'MAP_START',
    'MAP_END',
    'GRID',
    'ACCOUNTS',
    'COMPUTE_THREADS',
    'NETWORK_THREADS'
)
for setting_name in REQUIRED_SETTINGS:
    if not hasattr(config, setting_name):
        raise RuntimeError('Please set "{}" in config'.format(setting_name))

# Set defaults for missing config options
OPTIONAL_SETTINGS = {
    'LONGSPAWNS': False,
    'PROXIES': None,
    'SCAN_RADIUS': 70,
    'SCAN_DELAY': (10, 12, 11),
    'NOTIFY_IDS': None,
    'NOTIFY_RANKING': None,
    'CONTROL_SOCKS': None,
    'ENCRYPT_PATH': None,
    'HASH_PATH': None
}
for setting_name, default in OPTIONAL_SETTINGS.items():
    if not hasattr(config, setting_name):
        setattr(config, setting_name, default)

if config.CONTROL_SOCKS:
    from stem import Signal
    from stem.control import Controller
    import stem.util.log
    stem.util.log.get_logger().level = 40
    CIRCUIT_TIME = dict()
    CIRCUIT_FAILURES = dict()
    CIRCUIT_LATENCIES = dict()
    for proxy in config.PROXIES:
        CIRCUIT_TIME[proxy] = time.time()
        CIRCUIT_FAILURES[proxy] = 0
        CIRCUIT_LATENCIES[proxy] = deque(maxlen=30)
else:
    CIRCUIT_TIME = None
    CIRCUIT_FAILURES = None
    CIRCUIT_LATENCIES = None

BAD_STATUSES = (
    'LOGIN FAIL',
    'EXCEPTION',
    'BAD LOGIN',
    'RETRYING',
    'THROTTLE',
)


class MalformedResponse(Exception):
    """Raised when server response is malformed"""


class BannedAccount(Exception):
    """Raised when account is banned"""


def configure_logger(filename='worker.log'):
    logging.basicConfig(
        filename=filename,
        format=(
            '[%(asctime)s][%(levelname)8s][%(name)s] '
            '%(message)s'
        ),
        style='%',
        level=logging.INFO,
    )

try:
    with open('cells.pickle', 'rb') as f:
        CELL_IDS = pickle.load(f)
    if config.COMPUTE_THREADS > 2:
        config.NETWORK_THREADS += config.COMPUTE_THREADS - 2
        config.COMPUTE_THREADS = 2
except Exception:
    CELL_IDS = dict()


class Spawns:
    def __init__(self):
        self.spawns = None
        self.session = db.Session()

    def update_spawns(self):
        self.spawns = db.get_spawns(self.session)

    def have_id(self, spawn_id):
        return spawn_id in self.spawns

    def get_despawn_seconds(self, spawn_id):
        if self.have_id(spawn_id):
            return self.spawns[spawn_id][2]
        else:
            return None

    def get_despawn_time(self, spawn_id):
        if self.have_id(spawn_id):
            current_hour = utils.get_current_hour()
            despawn_time = self.get_despawn_seconds(spawn_id) + current_hour
            return despawn_time
        else:
            return None

    def get_time_till_hidden(self, spawn_id):
        if not self.have_id(spawn_id):
            return None
        despawn_seconds = self.spawns[spawn_id][2]
        return utils.time_until_time(despawn_seconds)

SPAWNS = Spawns()

if config.NOTIFY_IDS or config.NOTIFY_RANKING:
    import notification
    notifier = notification.Notifier(SPAWNS)

class Slave:
    """Single worker walking on the map"""

    def __init__(
            self,
            worker_no,
            db_processor,
            cell_ids_executor,
            network_executor,
            extra_queue,
            captcha_queue,
            worker_dict,
            device_info=None,
            proxy=None
    ):
        self.worker_no = worker_no
        self.visits = 0
        self.account = config.ACCOUNTS[self.worker_no]
        # asyncio/thread references
        self.future = None  # worker's own future
        self.db_processor = db_processor
        self.cell_ids_executor = cell_ids_executor
        self.network_executor = network_executor
        # Some handy counters
        self.total_seen = 0
        # State variables
        self.busy = False
        self.killed = False  # killed worker will stay killed
        self.logged_in = False
        # Other variables
        self.last_visit = 0
        self.last_api_latency = 0
        self.error_code = 'INIT'
        self.empty_visits = 0
        self.location = utils.get_start_coords(self.worker_no, altitude=True)
        # And now, configure logger and PGoApi
        self.logger = logging.getLogger('worker-{}'.format(worker_no))
        self.device_info = device_info
        self.after_spawn = None
        self.speed = 0
        self.api = PGoApi(device_info=device_info)
        if config.ENCRYPT_PATH:
            self.api.set_signature_lib(config.ENCRYPT_PATH)
        if config.HASH_PATH:
            self.api.set_hash_lib(config.HASH_PATH)
        self.api.set_position(*self.location)
        self.api.set_logger(self.logger)
        self.extra_queue = extra_queue
        self.captcha_queue = captcha_queue
        self.worker_dict = worker_dict
        if proxy:
            self.set_proxy(proxy)
        else:
            self.proxy = None

    def call_api(self, method, *args, **kwargs):
        """Returns decorated function that measures execution time

        This works exactly like functools.partial does.
        """
        def inner():
            start = time.time()
            result = method(*args, **kwargs)
            self.last_api_latency = time.time() - start
            if CIRCUIT_LATENCIES:
                self.add_circuit_latency()
            return result
        return inner

    def add_circuit_latency(self):
        CIRCUIT_LATENCIES[self.proxy].append(self.last_api_latency)
        samples = len(CIRCUIT_LATENCIES[self.proxy])
        if samples > 10:
            average = sum(CIRCUIT_LATENCIES[self.proxy]) / samples
            if average > 10:
                self.swap_circuit('average latency of ' + str(round(average)) + 's')

    def set_proxy(self, proxy):
        self.proxy = proxy
        self.api.set_proxy({'http': proxy, 'https': proxy})


    async def bench_account(self):
        self.logged_in = False
        self.error_code = 'BENCHING'
        self.empty_visits = 0
        new_account = self.extra_queue.get()
        self.logger.warning('Swapping ' + self.account[0] + ' for ' +
                            new_account[0] + ' due to CAPTCHA.')
        self.captcha_queue.put(self.account)
        self.account = new_account
        self.device_info = utils.get_worker_device(self.worker_no)


    async def swap_account(self, reason=''):
        self.logged_in = False
        self.error_code = 'SWAPPING'
        self.empty_visits = 0
        new_account = self.extra_queue.get()
        self.logger.warning('Swapping out ' + self.account[0] + ' for ' +
                            new_account[0] + ' because ' + reason + '.')
        self.extra_queue.put(self.account)
        self.account = new_account
        self.device_info = utils.get_worker_device(self.worker_no)

    def swap_proxy(self, reason=''):
        self.set_proxy(random.choice(config.PROXIES))
        self.logger.warning('Swapped out ' + self.proxy +
                            ' due to ' + reason)

    def swap_circuit(self, reason=''):
        if not config.CONTROL_SOCKS:
            if config.PROXIES:
                swap_proxy(self, reason=reason)
            return
        time_passed = time.time() - CIRCUIT_TIME[self.proxy]
        if time_passed > 180:
            socket = config.CONTROL_SOCKS[self.proxy]
            with Controller.from_socket_file(path=socket) as controller:
                controller.authenticate()
                controller.signal(Signal.NEWNYM)
            CIRCUIT_TIME[self.proxy] = time.time()
            CIRCUIT_FAILURES[self.proxy] = 0
            CIRCUIT_LATENCIES[self.proxy] = deque(maxlen=30)
            self.logger.warning('Changed circuit on ' + self.proxy +
                                ' due to ' + reason)
        else:
            self.logger.info('Skipped changing circuit on ' + self.proxy +
                             ' because it was changed ' + str(time_passed)
                             + ' seconds ago.')

    async def login(self, initial=True):
        """Logs worker in and prepares for scanning"""
        self.error_code = 'LOGIN'
        loop = asyncio.get_event_loop()
        self.logger.info('Trying to log in')
        global LAST_LOGIN
        time_required = random.uniform(3, 5)
        while (time.time() - LAST_LOGIN) < time_required:
            await asyncio.sleep(1.5)
        LAST_LOGIN = time.time()
        for attempts in range(0,5):
            try:
                await loop.run_in_executor(
                    self.network_executor,
                    self.call_api(
                        self.api.set_authentication,
                        username=self.account[0],
                        password=<PASSWORD>.account[1],
                        provider=self.account[2],
                    )
                )
                self.logged_in = True
                self.error_code = 'READY'
                return
            except pgoapi_exceptions.ServerSideAccessForbiddenException:
                self.logger.error('Banned IP: ' + self.proxy)
                self.error_code = 'IP BANNED'
                self.swap_circuit(reason='ban')
                await self.random_sleep(sleep_min=15, sleep_max=20)
            except pgoapi_exceptions.AuthException:
                self.logger.warning('Login failed: ' + self.account[0])
                self.error_code = 'LOGIN FAIL'
                await self.swap_account(reason='login failed')
                await self.random_sleep()
            except pgoapi_exceptions.NotLoggedInException:
                self.logger.error('Invalid credentials: ' + self.account[0])
                self.error_code = 'BAD LOGIN'
                await self.swap_account(reason='bad login')
                await self.random_sleep()
            except pgoapi_exceptions.ServerBusyOrOfflineException:
                self.logger.info('Server too busy - restarting')
                self.error_code = 'RETRYING'
                await self.random_sleep()
            except pgoapi_exceptions.ServerSideRequestThrottlingException:
                self.logger.info('Server throttling - sleeping for a bit')
                self.error_code = 'THROTTLE'
                await self.random_sleep(sleep_min=10)
            except CancelledError:
                self.kill()
                return
            except Exception as err:
                self.logger.exception('A wild exception appeared! ' + str(err))
                self.error_code = 'EXCEPTION'
                await self.random_sleep()
        return False


    async def visit(self, point, i):
        """Wrapper for self.visit_point - runs it a few times before giving up

        Also is capable of restarting in case an error occurs.
        """
        visited = False
        for attempts in range(0,5):
            try:
                if not self.logged_in:
                    await self.login()
                visited = await self.visit_point(point, i)
            except pgoapi_exceptions.ServerSideAccessForbiddenException:
                self.logger.error('Banned IP.')
                self.error_code = 'IP BANNED'
                self.swap_circuit(reason='ban')
                await self.random_sleep(sleep_min=15, sleep_max=20)
            except pgoapi_exceptions.AuthException:
                self.logger.warning('Login failed: ' + self.account[0])
                self.error_code = 'LOGIN FAIL'
                self.logged_in = False
                await self.swap_account(reason='failed login')
                await self.random_sleep()
            except pgoapi_exceptions.NotLoggedInException:
                self.logger.error('Invalid credentials: ' + self.account[0])
                self.error_code = 'BAD LOGIN'
                await self.swap_account(reason='bad login')
                self.logged_in = False
                await self.random_sleep()
            except pgoapi_exceptions.ServerBusyOrOfflineException:
                self.logger.info('Server too busy - restarting')
                self.error_code = 'RETRYING'
                await self.random_sleep()
            except pgoapi_exceptions.ServerSideRequestThrottlingException:
                self.logger.info('Server throttling - sleeping for a bit')
                self.error_code = 'THROTTLE'
                await self.random_sleep(sleep_min=10)
            except MalformedResponse:
                self.logger.warning('Malformed response received!')
                self.error_code = 'RESTART'
                await self.random_sleep()
            except BannedAccount:
                self.error_code = 'BANNED?'
                await self.swap_account(reason='code 3')
            except Exception as err:
                self.logger.exception('A wild exception appeared!')
                self.error_code = 'EXCEPTION'
                await self.random_sleep()
            else:
                if visited:
{'fraction_non_alphanumeric': 0.0896616473477052, 'fraction_numerical': 0.0052720904199617235, 'mean_word_length': 2.0260052447552446, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 7, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7091872', 'n_tokens_mistral': 8400, 'n_tokens_neox': 8053, 'n_words': 1689}