<commit_before>#!/usr/bin/env python3
# -*- coding: utf-8 -*-

# Perform syntax highlighting on Scribe logs.

import sys, os, re, io
import time
import errno

import instabot

# Hard-coded ANSI escape sequences for coloring.
COLORS = {None: '\033[0m', 'bold': '\033[1m', 'black': '\033[30m',
    'red': '\033[31m', 'green': '\033[32m', 'orange': '\033[33m',
    'blue': '\033[34m', 'magenta': '\033[35m', 'cyan': '\033[36m',
    'gray': '\033[37m'}

def highlight(line):
    def highlight_scalar(val):
        if val in instabot.CONSTANTS:
            return (COLORS['magenta'], val)
        elif instabot.INTEGER.match(val):
            return (COLORS['cyan'], val)
        else:
            return (COLORS[None], val)
    m = instabot.LOGLINE.match(line)
    if not m: return line
    ret = [line[:m.start(2)], COLORS['bold'], m.group(2), COLORS[None],
           line[m.end(2):m.start(3)]]
    idx = m.start(3)
    if idx != -1:
        while idx < len(line):
            # Skip whitespace
            m = instabot.WHITESPACE.match(line, idx)
            if m:
                ret.extend((COLORS[None], m.group()))
                idx = m.end()
                if idx == len(line): break
            # Match the next parameter; output name
            m = instabot.PARAM.match(line, idx)
            if not m: break
            name, val = m.group(1, 2)
            ret.extend((COLORS['green'], name, '='))
            # Output value
            if val[0] == '(':
                # Output tuple prefix
                ret.extend((COLORS['orange'], '('))
                sm = instabot.WHITESPACE.match(val, 1)
                if sm:
                    ret.extend((COLORS[None], sm.group()))
                    subidx = sm.end()
                else:
                    subidx = 1
                # Output tuple values
                while subidx < len(val):
                    sm = instabot.TUPLE_ENTRY.match(val, subidx)
                    if not sm: break
                    ret.extend(highlight_scalar(sm.group(1)))
                    sp = val[sm.end(1):sm.start(2)]
                    if sp: ret.extend((COLORS[None], sp))
                    ret.extend((COLORS['orange'], ','))
                    sp = val[sm.end(2):sm.end()]
                    if sp: ret.extend((COLORS[None], sp))
                    subidx = sm.end()
                # Output trailing entry
                sm = instabot.SCALAR.match(val, subidx)
                if sm:
                    ret.extend(highlight_scalar(sm.group()))
                    ret.extend((COLORS[None], val[sm.end():-1]))
                else:
                    ret.extend((COLORS[None], val[subidx:-1]))
                # Output suffix
                ret.extend((COLORS['orange'], ')'))
            else:
                ret.extend(highlight_scalar(val))
            idx = m.end()
        ret.extend((COLORS[None], line[idx:]))
    return ''.join(ret)

def highlight_stream(it, newlines=False):
    if not newlines:
        for line in it:
            yield highlight(line)
    else:
        for line in it:
            yield highlight(line.rstrip('\n')) + '\n'

def open_file(path, mode):
    if path == '-':
        if mode[:1] in ('a', 'w'):
            return io.open(sys.stdout.fileno(), mode)
        else:
            return io.open(sys.stdin.fileno(), mode)
    else:
        return io.open(path, mode)

def main():
    p = instabot.OptionParser(sys.argv[0])
    p.help_action(desc='A syntax highlighter for Scribe logs.')
    p.option('out', short='o', default='-',
             help='File to write output to (- is standard output and '
                 'the default)')
    p.flag('append', short='a',
           help='Append to output file instead of overwriting it')
    p.flag('line-buffered', short='u',
           help='Flush output after each input line')
    p.argument('in', default='-',
               help='File to read from (- is standard input and '
                   'the default)')
    p.parse(sys.argv[1:])
    inpath, outpath, append = p.get('in', 'out', 'append')
    linebuf = p.get('line-buffered')
    outm = ('a' if append else 'w')
    try:
        with open_file(inpath, 'r') as fi, open_file(outpath, outm) as fo:
            for l in highlight_stream(fi, True):
                fo.write(l)
                if linebuf: fo.flush()
    except KeyboardInterrupt:
        # Suppress noisy stack traces.
        pass

if __name__ == '__main__': main()
<commit_msg>[Scripts] Implement line type ignoring in colorlogs.py<commit_after>#!/usr/bin/env python3
# -*- coding: utf-8 -*-

# Perform syntax highlighting on Scribe logs.

import sys, os, re, io
import time
import errno

import instabot

# Hard-coded ANSI escape sequences for coloring.
COLORS = {None: '\033[0m', 'bold': '\033[1m', 'black': '\033[30m',
    'red': '\033[31m', 'green': '\033[32m', 'orange': '\033[33m',
    'blue': '\033[34m', 'magenta': '\033[35m', 'cyan': '\033[36m',
    'gray': '\033[37m'}

def highlight(line, filt=None):
    def highlight_scalar(val):
        if val in instabot.CONSTANTS:
            return (COLORS['magenta'], val)
        elif instabot.INTEGER.match(val):
            return (COLORS['cyan'], val)
        else:
            return (COLORS[None], val)
    m = instabot.LOGLINE.match(line)
    if not m: return line
    if filt and not filt(m.group(2)): return None
    ret = [line[:m.start(2)], COLORS['bold'], m.group(2), COLORS[None],
           line[m.end(2):m.start(3)]]
    idx = m.start(3)
    if idx != -1:
        while idx < len(line):
            # Skip whitespace
            m = instabot.WHITESPACE.match(line, idx)
            if m:
                ret.extend((COLORS[None], m.group()))
                idx = m.end()
                if idx == len(line): break
            # Match the next parameter; output name
            m = instabot.PARAM.match(line, idx)
            if not m: break
            name, val = m.group(1, 2)
            ret.extend((COLORS['green'], name, '='))
            # Output value
            if val[0] == '(':
                # Output tuple prefix
                ret.extend((COLORS['orange'], '('))
                sm = instabot.WHITESPACE.match(val, 1)
                if sm:
                    ret.extend((COLORS[None], sm.group()))
                    subidx = sm.end()
                else:
                    subidx = 1
                # Output tuple values
                while subidx < len(val):
                    sm = instabot.TUPLE_ENTRY.match(val, subidx)
                    if not sm: break
                    ret.extend(highlight_scalar(sm.group(1)))
                    sp = val[sm.end(1):sm.start(2)]
                    if sp: ret.extend((COLORS[None], sp))
                    ret.extend((COLORS['orange'], ','))
                    sp = val[sm.end(2):sm.end()]
                    if sp: ret.extend((COLORS[None], sp))
                    subidx = sm.end()
                # Output trailing entry
                sm = instabot.SCALAR.match(val, subidx)
                if sm:
                    ret.extend(highlight_scalar(sm.group()))
                    ret.extend((COLORS[None], val[sm.end():-1]))
                else:
                    ret.extend((COLORS[None], val[subidx:-1]))
                # Output suffix
                ret.extend((COLORS['orange'], ')'))
            else:
                ret.extend(highlight_scalar(val))
            idx = m.end()
        ret.extend((COLORS[None], line[idx:]))
    return ''.join(ret)

def highlight_stream(it, newlines=False, filt=None):
    if not newlines:
        for line in it:
            yield highlight(line, filt)
    else:
        for line in it:
            yield highlight(line.rstrip('\n'), filt) + '\n'

def open_file(path, mode):
    if path == '-':
        if mode[:1] in ('a', 'w'):
            return io.open(sys.stdout.fileno(), mode)
        else:
            return io.open(sys.stdin.fileno(), mode)
    else:
        return io.open(path, mode)

def main():
    p = instabot.OptionParser(sys.argv[0])
    p.help_action(desc='A syntax highlighter for Scribe logs.')
    p.option('ignore', short='i', default=[], accum=True,
             help='Filter out lines of this type (may be repeated)')
    p.option('out', short='o', default='-',
             help='File to write output to (- is standard output and '
                 'the default)')
    p.flag('append', short='a',
           help='Append to output file instead of overwriting it')
    p.flag('line-buffered', short='u',
           help='Flush output after each input line')
    p.argument('in', default='-',
               help='File to read from (- is standard input and '
                   'the default)')
    p.parse(sys.argv[1:])
    ignore, inpath, outpath, append = p.get('ignore', 'in', 'out', 'append')
    linebuf = p.get('line-buffered')
    outm = ('a' if append else 'w')
    try:
        filt = (lambda t: t not in ignore) if ignore else None
        with open_file(inpath, 'r') as fi, open_file(outpath, outm) as fo:
            for l in highlight_stream(fi, True, filt):
                fo.write(l)
                if linebuf: fo.flush()
    except KeyboardInterrupt:
        # Suppress noisy stack traces.
        pass

if __name__ == '__main__': main()

{'fraction_non_alphanumeric': 0.1494004537107054, 'fraction_numerical': 0.015231716538835476, 'mean_word_length': 1.7374334713187463, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2715089', 'n_tokens_mistral': 2974, 'n_tokens_neox': 2815, 'n_words': 718}