<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-

'''
Search Architecture:
 - Have a list of accounts
 - Create an "overseer" thread
 - Search Overseer:
   - Tracks incoming new location values
   - Tracks "paused state"
   - During pause or new location will clears current search queue
   - Starts search_worker threads
 - Search Worker Threads each:
   - Have a unique API login
   - Listens to the same Queue for areas to scan
   - Can re-login as needed
   - Pushes finds to db queue and webhook queue
'''

import logging
import math
import os
import random
import sys
import time
import timeit
import threading
import traceback

from collections import deque
from datetime import datetime
from distutils.version import StrictVersion
from sets import Set
from threading import Thread, Lock

import requests
from mrmime import mrmime_pgpool_enabled
from mrmime.pogoaccount import POGOAccount
from pgoapi.hash_server import (HashServer)
from queue import Queue, Empty
from requests.adapters import HTTPAdapter
from requests.packages.urllib3.util.retry import Retry
from cachetools import TTLCache

import schedulers
import terminalsize
from .account import (AccountSet,
                      setup_mrmime_account, get_account, account_failed, account_revive)
from .captcha import captcha_overseer_thread, handle_captcha
from .models import (parse_map, GymDetails, parse_gyms, MainWorker,
                     WorkerStatus, HashKeys)
from .proxy import get_new_proxy
from .transform import get_new_coords
from .utils import now, clear_dict_response, get_args, distance

log = logging.getLogger(__name__)

loginDelayLock = Lock()
gym_cache_lock = threading.Lock()


# Thread to handle user input.
def switch_status_printer(display_type, current_page, mainlog,
                          loglevel, logmode):
    # Disable logging of the first handler - the stream handler, and disable
    # it's output.
    if (logmode != 'logs'):
        mainlog.handlers[0].setLevel(logging.CRITICAL)

    while True:
        # Wait for the user to press a key.
        command = raw_input()

        if command == '':
            # Switch between logging and display.
            if display_type[0] != 'logs':
                # Disable display, enable on screen logging.
                mainlog.handlers[0].setLevel(loglevel)
                display_type[0] = 'logs'
                # If logs are going slowly, sometimes it's hard to tell you
                # switched.  Make it clear.
                print 'Showing logs...'
            elif display_type[0] == 'logs':
                # Enable display, disable on screen logging (except for
                # critical messages).
                mainlog.handlers[0].setLevel(logging.CRITICAL)
                display_type[0] = 'workers'
        elif command.isdigit():
            current_page[0] = int(command)
            mainlog.handlers[0].setLevel(logging.CRITICAL)
        elif command.lower() == 'a':
            mainlog.handlers[0].setLevel(logging.CRITICAL)
            display_type[0] = 'account_stats'
        elif command.lower() == 'f':
            mainlog.handlers[0].setLevel(logging.CRITICAL)
            display_type[0] = 'failedaccounts'
        elif command.lower() == 'h':
            mainlog.handlers[0].setLevel(logging.CRITICAL)
            display_type[0] = 'hashstatus'


# Thread to print out the status of each worker.
def status_printer(threadStatus, account_queue, account_captchas, account_failures, logmode, hash_key,
                   key_scheduler):

    if (logmode == 'logs'):
        display_type = ['logs']
    else:
        display_type = ['workers']

    current_page = [1]
    # Grab current log / level.
    mainlog = logging.getLogger()
    loglevel = mainlog.getEffectiveLevel()

    # Start another thread to get user input.
    t = Thread(target=switch_status_printer,
               name='switch_status_printer',
               args=(display_type, current_page, mainlog, loglevel, logmode))
    t.daemon = True
    t.start()

    while True:
        time.sleep(1)

        if display_type[0] == 'logs':
            # In log display mode, we don't want to show anything.
            continue

        # Create a list to hold all the status lines, so they can be printed
        # all at once to reduce flicker.
        status_text = []

        if display_type[0] == 'workers':

            # Get the terminal size.
            width, height = terminalsize.get_terminal_size()
            # Queue and overseer take 2 lines.  Switch message takes up 2
            # lines.  Remove an extra 2 for things like screen status lines.
            usable_height = height - 6
            # Prevent people running terminals only 6 lines high from getting a
            # divide by zero.
            if usable_height < 1:
                usable_height = 1<commit_msg>GeneralFixes: initialize total_pages to prevent UnboundLocalError<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-

'''
Search Architecture:
 - Have a list of accounts
 - Create an "overseer" thread
 - Search Overseer:
   - Tracks incoming new location values
   - Tracks "paused state"
   - During pause or new location will clears current search queue
   - Starts search_worker threads
 - Search Worker Threads each:
   - Have a unique API login
   - Listens to the same Queue for areas to scan
   - Can re-login as needed
   - Pushes finds to db queue and webhook queue
'''

import logging
import math
import os
import random
import sys
import time
import timeit
import threading
import traceback

from collections import deque
from datetime import datetime
from distutils.version import StrictVersion
from sets import Set
from threading import Thread, Lock

import requests
from mrmime import mrmime_pgpool_enabled
from mrmime.pogoaccount import POGOAccount
from pgoapi.hash_server import (HashServer)
from queue import Queue, Empty
from requests.adapters import HTTPAdapter
from requests.packages.urllib3.util.retry import Retry
from cachetools import TTLCache

import schedulers
import terminalsize
from .account import (AccountSet,
                      setup_mrmime_account, get_account, account_failed, account_revive)
from .captcha import captcha_overseer_thread, handle_captcha
from .models import (parse_map, GymDetails, parse_gyms, MainWorker,
                     WorkerStatus, HashKeys)
from .proxy import get_new_proxy
from .transform import get_new_coords
from .utils import now, clear_dict_response, get_args, distance

log = logging.getLogger(__name__)

loginDelayLock = Lock()
gym_cache_lock = threading.Lock()


# Thread to handle user input.
def switch_status_printer(display_type, current_page, mainlog,
                          loglevel, logmode):
    # Disable logging of the first handler - the stream handler, and disable
    # it's output.
    if (logmode != 'logs'):
        mainlog.handlers[0].setLevel(logging.CRITICAL)

    while True:
        # Wait for the user to press a key.
        command = raw_input()

        if command == '':
            # Switch between logging and display.
            if display_type[0] != 'logs':
                # Disable display, enable on screen logging.
                mainlog.handlers[0].setLevel(loglevel)
                display_type[0] = 'logs'
                # If logs are going slowly, sometimes it's hard to tell you
                # switched.  Make it clear.
                print 'Showing logs...'
            elif display_type[0] == 'logs':
                # Enable display, disable on screen logging (except for
                # critical messages).
                mainlog.handlers[0].setLevel(logging.CRITICAL)
                display_type[0] = 'workers'
        elif command.isdigit():
            current_page[0] = int(command)
            mainlog.handlers[0].setLevel(logging.CRITICAL)
        elif command.lower() == 'a':
            mainlog.handlers[0].setLevel(logging.CRITICAL)
            display_type[0] = 'account_stats'
        elif command.lower() == 'f':
            mainlog.handlers[0].setLevel(logging.CRITICAL)
            display_type[0] = 'failedaccounts'
        elif command.lower() == 'h':
            mainlog.handlers[0].setLevel(logging.CRITICAL)
            display_type[0] = 'hashstatus'


# Thread to print out the status of each worker.
def status_printer(threadStatus, account_queue, account_captchas, account_failures, logmode, hash_key,
                   key_scheduler):

    if (logmode == 'logs'):
        display_type = ['logs']
    else:
        display_type = ['workers']

    current_page = [1]
    # Grab current log / level.
    mainlog = logging.getLogger()
    loglevel = mainlog.getEffectiveLevel()

    # Start another thread to get user input.
    t = Thread(target=switch_status_printer,
               name='switch_status_printer',
               args=(display_type, current_page, mainlog, loglevel, logmode))
    t.daemon = True
    t.start()

    while True:
        time.sleep(1)

        if display_type[0] == 'logs':
            # In log display mode, we don't want to show anything.
            continue

        # Create a list to hold all the status lines, so they can be printed
        # all at once to reduce flicker.
        status_text = []

        # Initialize to prevent UnboundLocalError
        total_pages = 1
        
        if display_type[0] == 'workers':

            # Get the terminal size.
            width, height = terminalsize.get_terminal_size()
            # Queue and overseer take 2 lines.  Switch message takes up 2
            # lines.  Remove an extra 2 for things like screen status lines.
            usable_height = height - 6
            # Prevent people running terminals only 6 lines high from getting a
            # divide by zero.
            if usable_height < 1:
                usable_height = 1
{'fraction_non_alphanumeric': 0.07742394953708413, 'fraction_numerical': 0.005799165734052294, 'mean_word_length': 2.472271282232427, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6387976', 'n_tokens_mistral': 2768, 'n_tokens_neox': 2664, 'n_words': 900}