<commit_before>#!/usr/bin/env python

"""Query and aggregate data from log files using SQL-like syntax"""

import sys

if sys.version < '2.7':
    print "%s requires python2 version 2.7 or higher" % sys.argv[0]
    sys.exit(1)

import argparse
import os
import re
import ast
import readline
import atexit
import time
from multiprocessing import cpu_count
try:
    from collections import OrderedDict
except ImportError:
    # python < 2.7 compatability
    from compat.OrderedDict import OrderedDict

from ply import yacc

import parser
import parallel
import screen
import sqlfuncs
import logformat
from util import NoTokenError, parse_format_string, Complete, Table, pretty_print

DEBUG = False
log_regex = None

class LogQuery(object):
    def __init__(self, data, query):
        self.data = data
        self.query = query
        try:
            self.ast = parser.parse(query)
        except NoTokenError, e:
            print "ERROR: %s" % e.message
            print query
            return
        except SyntaxError:
            return
        if DEBUG:
            # pretty-printer
            sq = str(self.ast)
            pretty_print(sq)
            print sq
            print '-'*screen.width
            pass
    
    def run(self):
        start_time = time.time()
        op_data = sqlfuncs.do(self.ast, self.data[:]) # COPY!!! 
        response = OrderedDict()
        for row in op_data:
            for key in row.keys():
                if not response.has_key(key):
                    response[key] = []
                response[key].append(row[key])
        Table(response, start_time).prnt()

class LoGrok(object):
    def __init__(self, args, interactive=False, curses=False, chunksize=10000):
        if curses:
            screen.init_curses()
        elif interactive:
            screen.init_linebased()
        self.interactive = interactive
        self.args = args
        self.processed_rows = 0
        self.oldpct = 0
        self.data = []
        self.chunksize = chunksize
        self.complete = Complete()
        self.crunchlogs()
        self.interact()

    def crunchlogs(self):
        global log_regex
        if self.args.format is not None:
            logformat = self.args.format
        else:
            logformat = logformat.TYPES[self.args.type]

        print
        lines = []
        for logfile in self.args.logfile:
            screen.print_mutable("Reading lines from %s:" % logfile.name)
            lines += logfile.readlines()
            screen.print_mutable("Reading lines from %s: %d" % (logfile.name, len(lines)))
            logfile.close()
            screen.print_mutable("", True)

        log_regex = re.compile(parse_format_string(logformat))
        if self.args.lines:
            lines = lines[:self.args.lines]
        st = time.time()
        self.data = parallel.run(log_match, lines, _print=True)
        et = time.time()
        print "%d lines crunched in %0.3f seconds" % (len(lines), (et-st))

    def interact(self):
        if screen.is_curses():
            screen.draw_curses_screen(self.data)
            self.main_loop()
        elif self.interactive:
            self.shell()
        else:
            print self.query(self.args.query)

    def shell(self):
        try:
            history = os.path.expanduser('~/.logrok_history')
            readline.read_history_file(history)
        except IOError:
            pass
        atexit.register(readline.write_history_file, history)
        readline.set_history_length(1000)
        readline.parse_and_bind('tab: complete')
        readline.set_completer(self.complete.complete)
        # XXX This is ugly and needs to be more intelligent. Ideally, the <commit_msg>I think i have removed the deps on python 2.7+<commit_after>#!/usr/bin/env python

"""Query and aggregate data from log files using SQL-like syntax"""

import sys
import argparse
import os
import re
import ast
import readline
import atexit
import time
from multiprocessing import cpu_count
try:
    from collections import OrderedDict
except ImportError:
    # python < 2.7 compatability
    from compat.OrderedDict import OrderedDict

from ply import yacc

import parser
import parallel
import screen
import sqlfuncs
import logformat
from util import NoTokenError, parse_format_string, Complete, Table, pretty_print

DEBUG = False
log_regex = None

class LogQuery(object):
    def __init__(self, data, query):
        self.data = data
        self.query = query
        try:
            self.ast = parser.parse(query)
        except NoTokenError, e:
            print "ERROR: %s" % e.message
            print query
            return
        except SyntaxError:
            return
        if DEBUG:
            # pretty-printer
            sq = str(self.ast)
            pretty_print(sq)
            print sq
            print '-'*screen.width
            pass
    
    def run(self):
        start_time = time.time()
        op_data = sqlfuncs.do(self.ast, self.data[:]) # COPY!!! 
        response = OrderedDict()
        for row in op_data:
            for key in row.keys():
                if not response.has_key(key):
                    response[key] = []
                response[key].append(row[key])
        Table(response, start_time).prnt()

class LoGrok(object):
    def __init__(self, args, interactive=False, curses=False, chunksize=10000):
        if curses:
            screen.init_curses()
        elif interactive:
            screen.init_linebased()
        self.interactive = interactive
        self.args = args
        self.processed_rows = 0
        self.oldpct = 0
        self.data = []
        self.chunksize = chunksize
        self.complete = Complete()
        self.crunchlogs()
        self.interact()

    def crunchlogs(self):
        global log_regex
        if self.args.format is not None:
            logformat = self.args.format
        else:
            logformat = logformat.TYPES[self.args.type]

        print
        lines = []
        for logfile in self.args.logfile:
            screen.print_mutable("Reading lines from %s:" % logfile.name)
            lines += logfile.readlines()
            screen.print_mutable("Reading lines from %s: %d" % (logfile.name, len(lines)))
            logfile.close()
            screen.print_mutable("", True)

        log_regex = re.compile(parse_format_string(logformat))
        if self.args.lines:
            lines = lines[:self.args.lines]
        st = time.time()
        self.data = parallel.run(log_match, lines, _print=True)
        et = time.time()
        print "%d lines crunched in %0.3f seconds" % (len(lines), (et-st))

    def interact(self):
        if screen.is_curses():
            screen.draw_curses_screen(self.data)
            self.main_loop()
        elif self.interactive:
            self.shell()
        else:
            self.query(self.args.query)

    def shell(self):
        try:
            history = os.path.expanduser('~/.logrok_history')
            readline.read_history_file(history)
        except IOError:
            pass
        atexit.register(readline.write_history_file, history)
        readline.set_history_length(1000)
        readline.parse_and_bind('tab: complete')
        readline.set_completer(self.complete.complete)
        # XXX This is ugly and needs to be more intelligent. Ideally, the 
{'fraction_non_alphanumeric': 0.09355849107994008, 'fraction_numerical': 0.005311180716328476, 'mean_word_length': 2.0347107438016527, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '716606', 'n_tokens_mistral': 2131, 'n_tokens_neox': 2086, 'n_words': 545}