<commit_before>import argparse
from getpass import getpass
import os
from os.path import expanduser
import shelve
import requests
import sys
import time
import yaml
from yaml import SafeDumper


class APIError(Exception):
    def __init__(self, message, context):
        super(APIError, self).__init__(message)
        self.context = context


class HighwindsClient:
    def __init__(self, base_url, token=None):
        self.base_url = base_url
        self.token = token

    def version(self):
        response = requests.get(self.base_url + '/version')
        return response.headers['X-Cdnws-Version']

    def me(self):
        user_response = requests.get(
            self.base_url + '/api/v1/users/me', headers={'Authorization': 'Bearer %s' % self.token})
        if user_response.status_code == 200:
            return user_response.json()
        else:
            raise APIError('Could not fetch user details', user_response)

    def create_token(self, username, password, application=None):
        if application is None:
            application = 'Highwinds Python client'

        # Grab an access token to use to fetch user
        response = requests.post(self.base_url + '/auth/token', data={
            "username": username, "password": password, "grant_type": "password"
        }, headers={
            'User-Agent': application
        })
        auth = response.json()
        access_token = auth['access_token']

        # Grab user's id and root account hash
        user_response = requests.get(self.base_url + '/api/v1/users/me', headers={'Authorization': 'Bearer %s' % access_token})
        account_hash = user_response.json()['accountHash']
        user_id = user_response.json()['id']

        # Generate a new API token
        token_response = requests.post(self.base_url + ('/api/v1/accounts/{account_hash}/users/{user_id}/tokens'.format(
            account_hash=account_hash, user_id=user_id
        )), json={
            "password": password, "application": application
        }, headers={
            'Authorization': 'Bearer %s' % access_token,
            'Content-Type': 'application/json'
        })
        if 'token' not in token_response.json():
            raise APIError('Could not generate API token', token_response)
        self.token = token_response.json()['token']
        return self.token

    def purge(self, account_hash, urls):
        purge_response = requests.post(self.base_url + ('/api/v1/accounts/%s/purge' % account_hash), json={
            "list": urls
        }, headers={
            'Content-Type': 'application/json',
            'Authorization': 'Bearer %s' % self.token
        })
        if 'id' not in purge_response.json():
            raise APIError('Could not send purge batch', purge_response)
        return purge_response.json()['id']

    def purge_status(self, account_hash, job_id):
        status_response = requests.get(self.base_url + ('/api/v1/accounts/%s/purge/%s' % (account_hash, job_id,)), headers={
            'Authorization': 'Bearer %s' % self.token,
            })
        if 'progress' not in status_response.json():
            raise APIError('Could not fetch purge status', status_response)
        return float(status_response.json()['progress'])



def command(arguments=()):
    def apply_args(fn):
        def wrapper(self, *args, **kwargs):
            # Apply arguments
            for arg in arguments:
                name = arg['name']
                del arg['name']
                self.parser.add_argument(name, **arg)
            self.args = self.parser.parse_args()

            # Load token store
            if not self.args.token:
                self.cache = shelve.open(os.path.join(expanduser('~'), '.highwinds'))
                self.client.token = self.cache.get('token')
            else:
                self.client.token = self.args.token

            # Call original function
            fn(self, *args, **kwargs)
        return wrapper
    return apply_args


class HighwindsCommand:
    def __init__(self):
        # Instantiate library
        base_url = os.environ.get('HIGHWINDS_BASE_URL', 'https://striketracker.highwinds.com')
        self.client = HighwindsClient(base_url)

        # Read in command line arguments
        self.parser = argparse.ArgumentParser(description='Interface to the Highwinds CDN Web Services')
        methodList = [method for method in dir(self) if callable(getattr(self, method)) and '_' not in method]
        methodList.sort()
        self.parser.add_argument('action', help=",".join(methodList))
        self.parser.add_argument('--token', help='Token to use for this action')
<commit_msg>Add option for verbose logging<commit_after>import argparse
from getpass import getpass
import os
from os.path import expanduser
import shelve
import requests
import sys
import time
import yaml
from yaml import SafeDumper
import logging


class APIError(Exception):
    def __init__(self, message, context):
        super(APIError, self).__init__(message)
        self.context = context


class HighwindsClient:
    def __init__(self, base_url, token=None):
        self.base_url = base_url
        self.token = token

    def version(self):
        response = requests.get(self.base_url + '/version')
        return response.headers['X-Cdnws-Version']

    def me(self):
        user_response = requests.get(
            self.base_url + '/api/v1/users/me', headers={'Authorization': 'Bearer %s' % self.token})
        if user_response.status_code == 200:
            return user_response.json()
        else:
            raise APIError('Could not fetch user details', user_response)

    def create_token(self, username, password, application=None):
        if application is None:
            application = 'Highwinds Python client'

        # Grab an access token to use to fetch user
        response = requests.post(self.base_url + '/auth/token', data={
            "username": username, "password": password, "grant_type": "password"
        }, headers={
            'User-Agent': application
        })
        auth = response.json()
        access_token = auth['access_token']

        # Grab user's id and root account hash
        user_response = requests.get(self.base_url + '/api/v1/users/me', headers={'Authorization': 'Bearer %s' % access_token})
        account_hash = user_response.json()['accountHash']
        user_id = user_response.json()['id']

        # Generate a new API token
        token_response = requests.post(self.base_url + ('/api/v1/accounts/{account_hash}/users/{user_id}/tokens'.format(
            account_hash=account_hash, user_id=user_id
        )), json={
            "password": password, "application": application
        }, headers={
            'Authorization': 'Bearer %s' % access_token,
            'Content-Type': 'application/json'
        })
        if 'token' not in token_response.json():
            raise APIError('Could not generate API token', token_response)
        self.token = token_response.json()['token']
        return self.token

    def purge(self, account_hash, urls):
        purge_response = requests.post(self.base_url + ('/api/v1/accounts/%s/purge' % account_hash), json={
            "list": urls
        }, headers={
            'Content-Type': 'application/json',
            'Authorization': 'Bearer %s' % self.token
        })
        if 'id' not in purge_response.json():
            raise APIError('Could not send purge batch', purge_response)
        return purge_response.json()['id']

    def purge_status(self, account_hash, job_id):
        status_response = requests.get(self.base_url + ('/api/v1/accounts/%s/purge/%s' % (account_hash, job_id,)), headers={
            'Authorization': 'Bearer %s' % self.token,
            })
        if 'progress' not in status_response.json():
            raise APIError('Could not fetch purge status', status_response)
        return float(status_response.json()['progress'])



def command(arguments=()):
    def apply_args(fn):
        def wrapper(self, *args, **kwargs):
            # Apply arguments
            for arg in arguments:
                name = arg['name']
                del arg['name']
                self.parser.add_argument(name, **arg)
            self.args = self.parser.parse_args()

            # Optionally turn on verbose logging
            if self.args.verbose:
                try:
                    import http.client as http_client
                except ImportError:
                    import httplib as http_client
                    http_client.HTTPConnection.debuglevel = 1

                # You must initialize logging, otherwise you'll not see debug output.
                logging.basicConfig()
                logging.getLogger().setLevel(logging.DEBUG)
                requests_log = logging.getLogger("requests.packages.urllib3")
                requests_log.setLevel(logging.DEBUG)
                requests_log.propagate = True

            # Load token store
            if not self.args.token:
                self.cache = shelve.open(os.path.join(expanduser('~'), '.highwinds'))
                self.client.token = self.cache.get('token')
            else:
                self.client.token = self.args.token

            # Call original function
            fn(self, *args, **kwargs)
        return wrapper
    return apply_args


class HighwindsCommand:
    def __init__(self):
        # Instantiate library
        base_url = os.environ.get('HIGHWINDS_BASE_URL', 'https://striketracker.highwinds.com')
        self.client = HighwindsClient(base_url)

        # Read in command line arguments
        self.parser = argparse.ArgumentParser(description='Interface to the Highwinds CDN Web Services')
        methodList = [method for method in dir(self) if callable(getattr(self, method)) and '_' not in method]
        methodList.sort()
        self.parser.add_argument('action', help=",".join(methodList))
        self.parser.add_argument('--token', help='Token to use for this action')
        self.parser.add_argument('-v', '--verbose', help='Turn on verbose logging', action='store_true')

{'fraction_non_alphanumeric': 0.11512638230647709, 'fraction_numerical': 0.0017772511848341231, 'mean_word_length': 2.411586392724823, 'pattern_counts': {'":': 12, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6234933', 'n_tokens_mistral': 2852, 'n_tokens_neox': 2762, 'n_words': 698}