<commit_before>#!/usr/bin/env python

import colorama
from colorama import Fore, Back, Style
import json
import os
import re
from screeps import ScreepsConnection
from time import sleep
import websocket
import sys
import yaml

colorama.init()

class ScreepsConsole(object):

    def __init__(self, user, password, ptr=False):
        self.user = user
        self.password = password
        self.ptr = ptr

    def on_message(self, ws, message):
        print message

    def on_error(self, ws, error):
        print error

    def on_close(self, ws):
        print "### closed ###"

    def on_open(self, ws):
        screepsConnection = ScreepsConnection(u=self.user,p=self.password,ptr=self.ptr)
        me = screepsConnection.me()
        self.user_id = me['_id']
        ws.send('auth ' + screepsConnection.token)


    def on_message(self, ws, message):

        if (message.startswith('auth ok')):
            ws.send('subscribe user:' + self.user_id + '/console')
            return

        if (message.startswith('time')):
            return

        SEVERITY_RE = re.compile(r'<.*severity="(\d)">')
        TAG_RE = re.compile(r'<[^>]+>')

        data = json.loads(message)

        if 'messages' in data[1]:
            if 'log' in data[1]['messages']:

                message_count = len(data[1]['messages']['log'])

                # Make sure the delay doesn't cause an overlap into other ticks
                message_delay = 1.0 / message_count
                if message_delay > 0.10:
                    message_delay = 0.10

                for line in data[1]['messages']['log']:

                    # Extract severity from html tag
                    if '<' in line:
                        try:
                            match_return = SEVERITY_RE.match(line)
                            groups = match_return.groups()
                            if len(groups) > 0:
                                severity = int(groups[0])
                            else:
                                severity = false
                        except:
                            severity = false
                    else:
                        severity = 3

                    # Add color based on severity
                    if 'severity' not in locals():
                        severity = 3

                    if severity == 0:
                        color = Style.DIM + Fore.WHITE
                    elif severity == 1:
                        color = Style.NORMAL + Fore.BLUE
                    elif severity == 2:
                        color = Style.NORMAL + Fore.CYAN
                    elif severity == 3:
                        color = Style.NORMAL + Fore.WHITE
                    elif severity == 4:
                        color = Style.NORMAL + Fore.RED
                    elif severity == 5:
                        color = Style.NORMAL + Fore.BLACK + Back.RED
                    else:
                        color = Style.NORMAL + Fore.BLACK + Back.YELLOW


                    # Replace html tab entity with actual tabs
                    line = line.replace('&#09;', "\t")
                    print color + TAG_RE.sub('', line) + Style.RESET_ALL
                    sleep(message_delay) # sleep for smoother scrolling
            return

        print('on_message', message)


    def connect(self):
        url = 'wss://screeps.com/socket/websocket'
        #websocket.enableTrace(True)
        ws = websocket.WebSocketApp(url=url,
                                    on_message=self.on_message,
                                    on_error=self.on_error,
                                    on_close=self.on_close,
                                    on_open=self.on_open)
        ws.run_forever(ping_interval=1)

    def start(self):
        self.connect()



def getSettings():
    if not getSettings.settings:
        cwd = os.getcwd()
        path = cwd + '/.settings.yaml'
        if not os.path.isfile(path):
            print 'no settings file found'
            sys.exit(-1)
            return False
        with open(path, 'r') as f:
            getSettings.settings = yaml.load(f)
    return getSettings.settings
getSettings.settings = False


if __name__ == "__main__":
    settings = getSettings()
    screepsconsole = ScreepsConsole(user=settings['screeps_username'], password=settings['screeps_password'], ptr=settings['screeps_ptr'])
    screepsconsole.start()<commit_msg>Remove settings and formatting code, support json output<commit_after>#!/usr/bin/env python

from base64 import b64decode
import getopt
from gzip import GzipFile
import json
import logging
from outputparser import parseLine
from outputparser import tagLine
from screeps import ScreepsConnection
from settings import getSettings
from time import sleep
import websocket
import sys


class ScreepsConsole(object):

    def __init__(self, user, password, ptr=False, logging=False):
        self.format = 'color'
        self.user = user
        self.password = password
        self.ptr = ptr
        self.logging = False

    def on_message(self, ws, message):
        print message

    def on_error(self, ws, error):
        print error

    def on_close(self, ws):
        print "### closed ###"

    def on_open(self, ws):
        screepsConnection = ScreepsConnection(u=self.user,p=self.password,ptr=self.ptr)
        me = screepsConnection.me()
        self.user_id = me['_id']
        ws.send('auth ' + screepsConnection.token)


    def on_message(self, ws, message):

        if (message.startswith('auth ok')):
            ws.send('subscribe user:' + self.user_id + '/console')
            return

        if (message.startswith('time')):
            return

        if (message.startswith('gz')):
            message = GzipFile(fileobj=StringIO(b64decode(message[3:])))

        data = json.loads(message)

        if 'messages' in data[1]:
            if 'log' in data[1]['messages']:

                message_count = len(data[1]['messages']['log'])

                # Make sure the delay doesn't cause an overlap into other ticks
                message_delay = 1.0 / message_count
                if message_delay > 0.10:
                    message_delay = 0.10

                for line in data[1]['messages']['log']:

                    if self.format == 'color':
                        line_parsed = parseLine(line)
                    elif self.format == 'json':
                        line_parsed = json.dumps(line)
                    else:
                        line_parsed = tagLine(line)

                    print line_parsed
                    sleep(message_delay) # sleep for smoother scrolling
            return
        else:

            if 'error' in data[1]:
                #line = '<'+data[1]['error']
                line = "<severity=\"5\">" + data[1]['error'] + "</font>"
                if self.format == 'color':
                    line_parsed = parseLine(line)
                elif self.format == 'json':
                    line_parsed = json.dumps(line)
                else:
                    line_parsed = tagLine(line)

                print line_parsed
                return
            else:
                print 'undiscovered protocol feature'
                print json.dumps(message)

        print('on_message', message)


    def connect(self):

        if self.logging:
            logging.getLogger('websocket').addHandler(logging.StreamHandler())
            websocket.enableTrace(True)
        else:
            logging.getLogger('websocket').addHandler(logging.NullHandler())
            websocket.enableTrace(False)

        url = 'wss://screeps.com/socket/websocket'

        ws = websocket.WebSocketApp(url=url,
                                    on_message=self.on_message,
                                    on_error=self.on_error,
                                    on_close=self.on_close,
                                    on_open=self.on_open)
        ws.run_forever(ping_interval=1)

    def start(self):
        self.connect()


if __name__ == "__main__":

    opts, args = getopt.getopt(sys.argv[1:], "hi:o:",["ifile=","ofile="])
    settings = getSettings()
    screepsconsole = ScreepsConsole(user=settings['screeps_username'], password=settings['screeps_password'], ptr=settings['screeps_ptr'])

    if len(sys.argv) > 1:
        if sys.argv[1] == 'interactive':
            screepsconsole.format = 'tag'
        if sys.argv[1] == 'json':
            screepsconsole.format = 'json'

    screepsconsole.start()
{'fraction_non_alphanumeric': 0.10262417994376757, 'fraction_numerical': 0.006326148078725398, 'mean_word_length': 1.572160289243748, 'pattern_counts': {'":': 2, '<': 9, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6138255', 'n_tokens_mistral': 2421, 'n_tokens_neox': 2216, 'n_words': 538}