<commit_before>#!/usr/bin/python

import argparse, re
from filters import *
import sys


class MongoLogFilter(object):
    def __init__(self):
        self.filters = [] 

    def addFilter(self, filterClass):
        """ adds a filter class to the parser. """
        if not filterClass in self.filters:
            self.filters.append(filterClass)

    def _arrayToString(self, arr):
        """ if arr is of type list, join elements with space delimiter. """
        if isinstance(arr, list):
            return " ".join(arr)
        else:
            return arr

    def parse(self):
        """ parses the logfile and asks each filter if it accepts the line.
            it will only be printed if all filters accept the line.
        """

        # create parser object
        parser = argparse.ArgumentParser(description='mongod/mongos log file parser.')
        
        # only create default argument if not using stdin
        if sys.stdin.isatty():
            parser.add_argument('logfile', action='store', help='logfile to parse.')
        
        parser.add_argument('--verbose', action='store_true', help='outputs information about the parser and arguments.')

        # add arguments from filter classes
        for f in self.filters:
            for fa in f.filterArgs:
                parser.add_argument(fa[0], **fa[1])

        args = vars(parser.parse_args())
        args = dict((k, self._arrayToString(args[k])) for k in args)
        
        if args['verbose']:
            print args

        # create filter objects from classes and pass args
        self.filters = [f(args) for f in self.filters]

        # remove non-active filter objects
        self.filters = [f for f in self.filters if f.active]

        # open logfile
        if sys.stdin.isatty():
            logfile = open(args['logfile'], 'r')
        else:
            logfile = sys.stdin


        if args['verbose']:
            print "mlogfilter> command line arguments"
            for a in args:
                print "mlogfilter> %8s: %s" % (a, args[a])

        # go through each line and ask each filter if it accepts
        for line in logfile:

            # special case: if line starts with ***, always print (server restart)
            if line.startswith('***'):
                print line,
                continue

            # only print line if all filters agree
            if all([f.accept(line) for f in self.filters]):
                print line,

            # if at least one filter refuses to print remaining lines, stop
            if any([f.skipRemaining() for f in self.filters]):
                # if called from shell, break
                if sys.stdin.isatty():
                    break



if __name__ == '__main__':

    # create MongoLogParser instance
    mlogfilter = MongoLogFilter()

    # add filters
    mlogfilter.addFilter(SlowFilter)
    mlogfilter.addFilter(WordFilter)<commit_msg>added --shorten parameter to crop log lines to a certain max length.<commit_after>#!/usr/bin/python

import argparse, re
from filters import *
import sys


class MongoLogFilter(object):
    def __init__(self):
        self.filters = [] 

    def addFilter(self, filterClass):
        """ adds a filter class to the parser. """
        if not filterClass in self.filters:
            self.filters.append(filterClass)

    def _arrayToString(self, arr):
        """ if arr is of type list, join elements with space delimiter. """
        if isinstance(arr, list):
            return " ".join(arr)
        else:
            return arr

    def _outputLine(self, line, length=None):
        if length:
            if len(line) > length:
                line = line[:length/2-2] + '...' + line[-length/2:]
        print line, 

    def parse(self):
        """ parses the logfile and asks each filter if it accepts the line.
            it will only be printed if all filters accept the line.
        """

        # create parser object
        parser = argparse.ArgumentParser(description='mongod/mongos log file parser.')
        
        # only create default argument if not using stdin
        if sys.stdin.isatty():
            parser.add_argument('logfile', action='store', help='logfile to parse.')
        
        parser.add_argument('--verbose', action='store_true', help='outputs information about the parser and arguments.')
        parser.add_argument('--shorten', action='store', type=int, default=False, nargs='?', metavar='LENGTH', help='shortens long lines by cutting characters out of the middle until the length is <= LENGTH.')

        # add arguments from filter classes
        for f in self.filters:
            for fa in f.filterArgs:
                parser.add_argument(fa[0], **fa[1])

        self.args = vars(parser.parse_args())
        self.args = dict((k, self._arrayToString(self.args[k])) for k in self.args)
        
        # if self.args['verbose']:
        #     print self.args

        # create filter objects from classes and pass args
        self.filters = [f(self.args) for f in self.filters]

        # remove non-active filter objects
        self.filters = [f for f in self.filters if f.active]

        # open logfile
        if sys.stdin.isatty():
            logfile = open(self.args['logfile'], 'r')
        else:
            logfile = sys.stdin

        if self.args['shorten'] != False:
            if self.args['shorten'] == None:
                self.args['shorten'] = 200        

        if self.args['verbose']:
            print "mlogfilter> command line arguments"
            for a in self.args:
                print "mlogfilter> %8s: %s" % (a, self.args[a])

        # go through each line and ask each filter if it accepts
        for line in logfile:

            # special case: if line starts with ***, always print (server restart)
            if line.startswith('***'):
                self._outputLine(line, self.args['shorten'])
                continue

            # only print line if all filters agree
            if all([f.accept(line) for f in self.filters]):
                self._outputLine(line, self.args['shorten'])

            # if at least one filter refuses to print remaining lines, stop
            if any([f.skipRemaining() for f in self.filters]):
                # if called from shell, break
                if sys.stdin.isatty():
                    break



if __name__ == '__main__':

    # create MongoLogParser instance
    mlogfilter = MongoLogFilter()

    # add filters
    mlogfilter.addFilter(SlowFilter)
    mlogfilter.addFilter(WordFilter)
{'fraction_non_alphanumeric': 0.10109789569990851, 'fraction_numerical': 0.0018298261665141812, 'mean_word_length': 1.9867941712204007, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '85006', 'n_tokens_mistral': 1808, 'n_tokens_neox': 1724, 'n_words': 597}