<commit_before># -*- coding: utf-8 -*-
from datetime import datetime
import argparse
import logging
import os
import random
import sys
import threading
import time

from pgoapi import (
    exceptions as pgoapi_exceptions,
    PGoApi,
    utilities as pgoapi_utils,
)

import config
import db
import utils


# Check whether config has all necessary attributes
REQUIRED_SETTINGS = (
    'DB_ENGINE',
    'ENCRYPT_PATH',
    'CYCLES_PER_WORKER',
    'MAP_START',
    'MAP_END',
    'GRID',
    'ACCOUNTS',
    'SCAN_RADIUS',
    'SCAN_DELAY',
)
for setting_name in REQUIRED_SETTINGS:
    if not hasattr(config, setting_name):
        raise RuntimeError('Please set "{}" in config'.format(setting_name))


workers = {}
local_data = threading.local()


class CannotProcessStep(Exception):
    """Raised when servers are too busy"""


def configure_logger(filename='worker.log'):
    logging.basicConfig(
        filename=filename,
        format=(
            '[%(asctime)s][%(threadName)10s][%(levelname)8s][L%(lineno)4d] '
            '%(message)s'
        ),
        style='%',
        level=logging.INFO,
    )

logger = logging.getLogger()


class Slave(threading.Thread):
    """Single worker walking on the map"""
    def __init__(
        self,
        group=None,
        target=None,
        name=None,
        worker_no=None,
        points=None,
    ):
        super(Slave, self).__init__(group, target, name)
        self.worker_no = worker_no
        local_data.worker_no = worker_no
        self.points = points
        self.count_points = len(self.points)
        self.step = 0
        self.cycle = 0
        self.seen_per_cycle = 0
        self.total_seen = 0
        self.error_code = None
        self.running = True
        center = self.points[0]
        self.api = PGoApi()
        self.api.activate_signature(config.ENCRYPT_PATH)
        self.api.set_position(center[0], center[1], 100)  # lat, lon, alt
        if hasattr(config, 'PROXIES') and config.PROXIES:
            self.api.set_proxy(config.PROXIES)

    def run(self):
        """Wrapper for self.main - runs it a few times before restarting

        Also is capable of restarting in case an error occurs.
        """
        self.cycle = 1
        self.error_code = None

        service = config.ACCOUNTS[self.worker_no][2]
        while True:
            try:
                loginsuccess = self.api.login(
                    provider=service,
                    username=config.ACCOUNTS[self.worker_no][0],
                    password=config.ACCOUNTS[self.worker_no][1],
                )
                if not loginsuccess:
                    self.error_code = 'LOGIN FAIL'
                    self.restart()
                    return
            except pgoapi_exceptions.AuthException:
                self.error_code = 'LOGIN FAIL'
                self.restart()
                return
            except pgoapi_exceptions.NotLoggedInException:
                self.error_code = 'BAD LOGIN'
                self.restart()
                return
            except pgoapi_exceptions.ServerBusyOrOfflineException:
                self.error_code = 'RETRYING'
                self.restart()
                return
            except pgoapi_exceptions.ServerSideRequestThrottlingException:
                time.sleep(random.uniform(1, 5))
                continue
            except Exception:
                logger.exception('A wild exception appeared!')
                self.error_code = 'EXCEPTION'
                self.restart()
                return
            break
        while self.cycle <= config.CYCLES_PER_WORKER:
            if not self.running:
                self.restart()
                return
            try:
                self.main()
            except CannotProcessStep:
                self.error_code = 'RESTART'
                self.restart()
            except Exception:
                logger.exception('A wild exception appeared!')
                self.error_code = 'EXCEPTION'
                self.restart()
                return
            if not self.running:
                self.restart()
                return
            self.cycle += 1
            if self.cycle <= config.CYCLES_PER_WORKER:
                self.error_code = 'SLEEP'
                self.running = False
                time.sleep(random.randint(30, 60))
                self.running = True
                self.error_code = None
        self.error_code = 'RESTART'
        self.restart()

    def main(self):
        """Heart of the worker - goes over each point and reports sightings"""
        session = db.Session()
        self.seen_per_cycle = 0
        self.step = 0
        for i, point in enumerate(self.points):
            if not self.running:
                return
            logger.info('Visiting point %d (%s %s)', i, point[0], point[1])
            self.api.set_position(point[0], point[1], 0)
            cell_ids = pgoapi_utils.get_cell_ids(point[0], point[1])
            self.api.set_position(point[0], point[1], 100)
            response_dict = self.api.get_map_objects(
                latitude=pgoapi_utils.f2i(point[0]),
                longitude=pgoapi_utils.f2i(point[1]),
                cell_id=cell_ids
            )
            logger.debug('Response: %s', response_dict)
            if response_dict is False:
                raise CannotProcessStep
            map_objects = response_dict['responses'].get('GET_MAP_OBJECTS', {})
            pokemons = []
            forts = []
            if map_objects.get('status') == 1:
                for map_cell in map_objects['map_cells']:
                    for pokemon in map_cell.get('wild_pokemons', []):
                        # Care only about 15 min spawns
                        # 30 and 45 min ones (negative) will be just put after
                        # time_till_hidden is below 15 min
                        # As of 2016.08.14 we don't know what values over
                        # 60 minutes are, so ignore them too
                        invalid_time = (
                            pokemon['time_till_hidden_ms'] < 0 or
                            pokemon['time_till_hidden_ms'] > 900000
                        )
                        if invalid_time:
                            continue
                        pokemons.append(
                            self.normalize_pokemon(
                                pokemon, map_cell['current_timestamp_ms']
                            )<commit_msg>Make sure that what comes from backend is a dict<commit_after># -*- coding: utf-8 -*-
from datetime import datetime
import argparse
import logging
import os
import random
import sys
import threading
import time

from pgoapi import (
    exceptions as pgoapi_exceptions,
    PGoApi,
    utilities as pgoapi_utils,
)

import config
import db
import utils


# Check whether config has all necessary attributes
REQUIRED_SETTINGS = (
    'DB_ENGINE',
    'ENCRYPT_PATH',
    'CYCLES_PER_WORKER',
    'MAP_START',
    'MAP_END',
    'GRID',
    'ACCOUNTS',
    'SCAN_RADIUS',
    'SCAN_DELAY',
)
for setting_name in REQUIRED_SETTINGS:
    if not hasattr(config, setting_name):
        raise RuntimeError('Please set "{}" in config'.format(setting_name))


workers = {}
local_data = threading.local()


class MalformedResponse(Exception):
    """Raised when server response is malformed"""


def configure_logger(filename='worker.log'):
    logging.basicConfig(
        filename=filename,
        format=(
            '[%(asctime)s][%(threadName)10s][%(levelname)8s][L%(lineno)4d] '
            '%(message)s'
        ),
        style='%',
        level=logging.INFO,
    )

logger = logging.getLogger()


class Slave(threading.Thread):
    """Single worker walking on the map"""
    def __init__(
        self,
        group=None,
        target=None,
        name=None,
        worker_no=None,
        points=None,
    ):
        super(Slave, self).__init__(group, target, name)
        self.worker_no = worker_no
        local_data.worker_no = worker_no
        self.points = points
        self.count_points = len(self.points)
        self.step = 0
        self.cycle = 0
        self.seen_per_cycle = 0
        self.total_seen = 0
        self.error_code = None
        self.running = True
        center = self.points[0]
        self.api = PGoApi()
        self.api.activate_signature(config.ENCRYPT_PATH)
        self.api.set_position(center[0], center[1], 100)  # lat, lon, alt
        if hasattr(config, 'PROXIES') and config.PROXIES:
            self.api.set_proxy(config.PROXIES)

    def run(self):
        """Wrapper for self.main - runs it a few times before restarting

        Also is capable of restarting in case an error occurs.
        """
        self.cycle = 1
        self.error_code = None

        service = config.ACCOUNTS[self.worker_no][2]
        while True:
            try:
                loginsuccess = self.api.login(
                    provider=service,
                    username=config.ACCOUNTS[self.worker_no][0],
                    password=config.ACCOUNTS[self.worker_no][1],
                )
                if not loginsuccess:
                    self.error_code = 'LOGIN FAIL'
                    self.restart()
                    return
            except pgoapi_exceptions.AuthException:
                self.logger.warning('Login failed!')
                self.error_code = 'LOGIN FAIL'
                self.restart()
                return
            except pgoapi_exceptions.NotLoggedInException:
                self.logger.error('Invalid credentials')
                self.error_code = 'BAD LOGIN'
                self.restart()
                return
            except pgoapi_exceptions.ServerBusyOrOfflineException:
                self.logger.info('Server too busy - restarting')
                self.error_code = 'RETRYING'
                self.restart()
                return
            except pgoapi_exceptions.ServerSideRequestThrottlingException:
                self.logger.info('Server throttling - sleeping for a bit')
                time.sleep(random.uniform(1, 5))
                continue
            except Exception:
                logger.exception('A wild exception appeared!')
                self.error_code = 'EXCEPTION'
                self.restart()
                return
            break
        while self.cycle <= config.CYCLES_PER_WORKER:
            if not self.running:
                self.restart()
                return
            try:
                self.main()
            except MalformedResponse:
                self.logger.warning('Malformed response received!')
                self.error_code = 'RESTART'
                self.restart()
            except Exception:
                logger.exception('A wild exception appeared!')
                self.error_code = 'EXCEPTION'
                self.restart()
                return
            if not self.running:
                self.restart()
                return
            self.cycle += 1
            if self.cycle <= config.CYCLES_PER_WORKER:
                self.logger.info('Going to sleep for a bit')
                self.error_code = 'SLEEP'
                self.running = False
                time.sleep(random.randint(30, 60))
                self.logger.info('AWAKEN MY MASTERS')
                self.running = True
                self.error_code = None
        self.error_code = 'RESTART'
        self.restart()

    def main(self):
        """Heart of the worker - goes over each point and reports sightings"""
        session = db.Session()
        self.seen_per_cycle = 0
        self.step = 0
        for i, point in enumerate(self.points):
            if not self.running:
                return
            logger.info('Visiting point %d (%s %s)', i, point[0], point[1])
            self.api.set_position(point[0], point[1], 0)
            cell_ids = pgoapi_utils.get_cell_ids(point[0], point[1])
            self.api.set_position(point[0], point[1], 100)
            response_dict = self.api.get_map_objects(
                latitude=pgoapi_utils.f2i(point[0]),
                longitude=pgoapi_utils.f2i(point[1]),
                cell_id=cell_ids
            )
            if not isinstance(response_dict, dict):
                logger.warning('Response: %s', response_dict)
                raise MalformedResponse
            responses = response_dict.get('responses')
            if not responses:
                logger.warning('Response: %s', response_dict)
                raise MalformedResponse
            map_objects = response_dict['responses'].get('GET_MAP_OBJECTS', {})
            pokemons = []
            forts = []
            if map_objects.get('status') == 1:
                for map_cell in map_objects['map_cells']:
                    for pokemon in map_cell.get('wild_pokemons', []):
                        # Care only about 15 min spawns
                        # 30 and 45 min ones (negative) will be just put after
                        # time_till_hidden is below 15 min
                        # As of 2016.08.14 we don't know what values over
                        # 60 minutes are, so ignore them too
                        invalid_time = (
                            pokemon['time_till_hidden_ms'] < 0 or
                            pokemon['time_till_hidden_ms'] > 900000
                        )
                        if invalid_time:
                            continue
                        pokemons.append(
                            self.normalize_pokemon(
                                pokemon, map_cell['current_timestamp_ms']
                            )
{'fraction_non_alphanumeric': 0.08987286277948268, 'fraction_numerical': 0.010229431535876079, 'mean_word_length': 1.552116352787619, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1481680', 'n_tokens_mistral': 3933, 'n_tokens_neox': 3677, 'n_words': 843}