<commit_before>import os
import netmiko
import orionsdk
from getpass import (getpass, getuser)
from distutils.util import strtobool
import urllib3
import multiprocessing
from tqdm import tqdm
import re
from time import sleep
import socket

# Usage of globals is bad, I know. I use them so I don't have to keep
# re-entering my creds with every function call, and without needing
# to store plaintext creds. I can't use keyring because I haven't
# figured out how to use it on my headless jumphost.
network_username = ''
network_password = ''
telnet_password = ''
secret = ''
orion_username = ''
orion_password = ''

# Number of threads for concurrent sessions.
# Typically, multiprocessing pools are set to use the number of cores
# available on the server. However, in our case, the threads are not
# CPU-intensive, so we can get away with a lot more. Experimentation has shown
# that 20 strikes a good balance. Fewer means slower performance, more means
# maxing out system resources.
THREADS = 20


class NetError(Exception):
    """ Base class for exceptions in this module. """
    pass

class HostOfflineError(NetError):
    """ Raised when an attempt is made to connect to a host that is not
       network reachable.

    Attributes:
        host -- the hostname or IP that was attempted
        msg  -- explanation of the error
    """

    def __init__(self, host, msg):
        self.host = host
        self.msg = msg

class NoOpenPortError(NetError):
    """ Raised when an attempt is made to connect to a host that does
        not have either SSH or telnet available (ports 22 or 23, respectively).

    Attributes:
        host -- the hostname or IP that was attempted
        msg  -- explanation of the error
    """

    def __init__(self, host, msg):
        self.host = host
        self.msg = msg

class InvalidCommandTypeError(NetError):
    """ Raised when a command type is given that is not either 'show' or 'config'
    """

    def __init__(self, msg):
        self.msg = msg

# http://mattoc.com/python-yes-no-prompt-cli.html
def prompt(query):
    """ Returns boolean for y/n input """
    print '%s [y/n]: ' % query
    val = raw_input()
    try:
        ret = strtobool(val)
    except ValueError:
        print 'Reply with y/n'
        return prompt(query)
    return ret

def get_creds():
    """ Prompts for credentials that are stored in global variables for reuse
    """
    global default_username
    global network_username
    global network_password
    global telnet_password
    global secret
    default_username = getuser()
    if not network_username or not network_password or not telnet_password or not secret:
        network_username = raw_input('Network username [' + default_username + ']: ') or default_username
        network_password = getpass('Network password: ')
        telnet_password = getpass('Telnet password: ') or None
        secret = getpass('Enable secret: ')

def orion_init():
    """ Prompts for Orion credentials and returns a SwisClient object
    """
    global orion_server
    global orion_username
    global orion_password
    if not orion_username:
        default_username = getuser()
        orion_username = raw_input('Orion username [' + default_username + ']: ') or default_username
    if not orion_password:
        orion_password = getpass('Orion password: ')
    # SolarWinds-Orion is a special hostname in /etc/hosts
    # This was necessary to implement SSL checking
    # https://github.com/solarwinds/orionsdk-python#ssl-certificate-verification
    # this disables the SubjectAltNameWarning
    urllib3.disable_warnings()
    return orionsdk.SwisClient('SolarWinds-Orion', orion_username, orion_password, verify='../server.pem')

def get_devices(*args, **kwargs):
    """ Retrieve a list of hosts for later use.
    
    TODO:
    Ideally this will be able to handle:
     - An SWQL query directly from Orion
     - A CSV
     - A flat text file of hostnames/IPs
    """

    if args:
        # Single host
        if type(args[0]) is str:
            # Returns as a single-item list
            # This is important since most methods below iterate,
            # and a string is not iterable.
            return args[0].split()
        # If hosts given as list, simply return that list again
        elif type(args[0]) is list or type(args[0]) is dict:
            return args[0]
    # TODO: If source is a file, read that
    elif args and isfile(str(args[0])):
        pass
    # Otherwise assume the source is a SWQL query
    else:
        # Initialize Orion connection
        swis = orion_init()
     
        # Read base query from file
        # TODO: Ensure file exists
        base_query_file = open('base-query.swql')
        query = base_query_file.readlines()
        base_query_file.close()

        # Can't append this to the query until we have all filters in place
        query_order = "ORDER BY caption\n"

        # Special keyword 'all' retrieves all devices returned by the base query
        # without further filtering.
        if args and str(args[0]) == 'all':
            query += query_order
            return (swis.query(query))['results']
        # Otherwise, check to see if we are parsing by district/site
        elif kwargs:
            query += 'AND '
            query_filter = []<commit_msg>Fixed get_devices()<commit_after>import os
import netmiko
import orionsdk
from getpass import (getpass, getuser)
from distutils.util import strtobool
import urllib3
import multiprocessing
from tqdm import tqdm
import re
from time import sleep
import socket

# Usage of globals is bad, I know. I use them so I don't have to keep
# re-entering my creds with every function call, and without needing
# to store plaintext creds. I can't use keyring because I haven't
# figured out how to use it on my headless jumphost.
network_username = ''
network_password = ''
telnet_password = ''
secret = ''
orion_username = ''
orion_password = ''

# Number of threads for concurrent sessions.
# Typically, multiprocessing pools are set to use the number of cores
# available on the server. However, in our case, the threads are not
# CPU-intensive, so we can get away with a lot more. Experimentation has shown
# that 20 strikes a good balance. Fewer means slower performance, more means
# maxing out system resources.
THREADS = 20


class NetError(Exception):
    """ Base class for exceptions in this module. """
    pass

class HostOfflineError(NetError):
    """ Raised when an attempt is made to connect to a host that is not
       network reachable.

    Attributes:
        host -- the hostname or IP that was attempted
        msg  -- explanation of the error
    """

    def __init__(self, host, msg):
        self.host = host
        self.msg = msg

class NoOpenPortError(NetError):
    """ Raised when an attempt is made to connect to a host that does
        not have either SSH or telnet available (ports 22 or 23, respectively).

    Attributes:
        host -- the hostname or IP that was attempted
        msg  -- explanation of the error
    """

    def __init__(self, host, msg):
        self.host = host
        self.msg = msg

class InvalidCommandTypeError(NetError):
    """ Raised when a command type is given that is not either 'show' or 'config'
    """

    def __init__(self, msg):
        self.msg = msg

# http://mattoc.com/python-yes-no-prompt-cli.html
def prompt(query):
    """ Returns boolean for y/n input """
    print '%s [y/n]: ' % query
    val = raw_input()
    try:
        ret = strtobool(val)
    except ValueError:
        print 'Reply with y/n'
        return prompt(query)
    return ret

def get_creds():
    """ Prompts for credentials that are stored in global variables for reuse
    """
    global default_username
    global network_username
    global network_password
    global telnet_password
    global secret
    default_username = getuser()
    if not network_username or not network_password or not telnet_password or not secret:
        network_username = raw_input('Network username [' + default_username + ']: ') or default_username
        network_password = getpass('Network password: ')
        telnet_password = getpass('Telnet password: ') or None
        secret = getpass('Enable secret: ')

def orion_init():
    """ Prompts for Orion credentials and returns a SwisClient object
    """
    global orion_server
    global orion_username
    global orion_password
    if not orion_username:
        default_username = getuser()
        orion_username = raw_input('Orion username [' + default_username + ']: ') or default_username
    if not orion_password:
        orion_password = getpass('Orion password: ')
    # SolarWinds-Orion is a special hostname in /etc/hosts
    # This was necessary to implement SSL checking
    # https://github.com/solarwinds/orionsdk-python#ssl-certificate-verification
    # this disables the SubjectAltNameWarning
    urllib3.disable_warnings()
    return orionsdk.SwisClient('SolarWinds-Orion', orion_username, orion_password, verify='../server.pem')

def get_devices(*args, **kwargs):
    """ Retrieve a list of hosts for later use.
    
    TODO:
    Ideally this will be able to handle:
     - An SWQL query directly from Orion
     - A CSV
     - A flat text file of hostnames/IPs
    """

    if args:
        # Single host
        if type(args[0]) is str:
            # Returns as a single-item list
            # This is important since most methods below iterate,
            # and a string is not iterable.
            return args[0].split()
        # If hosts given as list, simply return that list again
        elif type(args[0]) is list or type(args[0]) is dict:
            return args[0]
    # TODO: If source is a file, read that
    elif args and isfile(str(args[0])):
        pass
    # Otherwise assume the source is a SWQL query
    else:
        # Initialize Orion connection
        swis = orion_init()
     
        # Read base query from file
        # TODO: Ensure file exists
        base_query_file = open('base-query.swql')
        query = base_query_file.read()
        base_query_file.close()

        # Can't append this to the query until we have all filters in place
        query_order = "ORDER BY caption\n"

        # Special keyword 'all' retrieves all devices returned by the base query
        # without further filtering.
        if args and str(args[0]) == 'all':
            query += query_order
            return (swis.query(query))['results']
        # Otherwise, check to see if we are parsing by district/site
        elif kwargs:
            query += 'AND '
            query_filter = []
{'fraction_non_alphanumeric': 0.07812940182176731, 'fraction_numerical': 0.0031927880552164523, 'mean_word_length': 2.710801393728223, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '48548', 'n_tokens_mistral': 3012, 'n_tokens_neox': 2931, 'n_words': 1165}