<commit_before>#!/usr/bin/env python
# encoding: utf-8
import sys
import re
import argparse
import fileinput

from argparse import RawDescriptionHelpFormatter

from patterns import pre_patterns


def prepare(input_stream, outfile=sys.stdout):
    """
    Apply pre_patterns from patterns to input_stream

    :input_stream: input stream

    """

    try:
        for line in input_stream:
            result = line
            for pattern in pre_patterns:
                result = re.sub(pattern[0], pattern[1], result, re.VERBOSE)
            outfile.write(result)
    except (KeyboardInterrupt):
        pass
    except:
        raise


def main():
    parser = argparse.ArgumentParser(description=\
    """
    Parse file[s]\n\n
    examlpe: cat error_log | tail -n 1000 | ./direlog.py
    """, formatter_class=RawDescriptionHelpFormatter)
    parser.add_argument('file', nargs='*', default=[],
                        help='file[s] to do some work')
    args = parser.parse_args()

    input_stream = fileinput.input(args.file)


    prepare(input_stream)

    pass

if __name__ == '__main__':
    main()<commit_msg>Small fixe in help<commit_after>#!/usr/bin/env python
# encoding: utf-8
import sys
import re
import argparse
import fileinput

from argparse import RawDescriptionHelpFormatter

from patterns import pre_patterns


def prepare(input_stream, outfile=sys.stdout):
    """
    Apply pre_patterns from patterns to input_stream

    :input_stream: input stream

    """

    try:
        for line in input_stream:
            result = line
            for pattern in pre_patterns:
                result = re.sub(pattern[0], pattern[1], result, re.VERBOSE)
            outfile.write(result)
    except (KeyboardInterrupt):
        pass
    except:
        raise


def main():
    parser = argparse.ArgumentParser(description=\
    """
        Prepare file[s]\n\n
        examlpe: cat error_log | tail -n 1000 | ./prepare.py
    """, formatter_class=RawDescriptionHelpFormatter)
    parser.add_argument('file', nargs='*', default=[],
                        help='file[s] to do some work')
    args = parser.parse_args()

    input_stream = fileinput.input(args.file)


    prepare(input_stream)

    pass

if __name__ == '__main__':
    main()
{'fraction_non_alphanumeric': 0.10101905183872396, 'fraction_numerical': 0.006202924235711121, 'mean_word_length': 2.641935483870968, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3037663', 'n_tokens_mistral': 691, 'n_tokens_neox': 661, 'n_words': 174}