<commit_before>#!/usr/bin/env python

from bundler import Bundler

if __name__ == '__main__':

    import sys
#    import os
    import argparse
    import codecs
#    import chardet


    arg_parser = argparse.ArgumentParser(description="bundle a HTML file with multiple external (js|css) files " +
                                                     "to one big HTML file ")

    # input
    arg_parser.add_argument('-i', '--input-file', metavar='<file>', type=str,
                            help='a file path - if none given, listen on stdin',
                            dest="infile")

    """ @TODO add support
    arg_parser.add_argument('-p', '--path-dir', metavar='<dir>', type=str,
                            # help='',
                            dest="path")
    """

    """ @TODO add support
    arg_parser.add_argument('-r', '--htroot-dir', metavar='<dir>', type=str,
                            # help='',
                            dest="htroot")
    """

    # output
    arg_parser.add_argument('-o', '--output-file', metavar='<file>', type=str,
                            help='a file path - if none given, output is sent to stdout',
                            dest="outfile")


    # strip HTML comments
    arg_parser.add_argument('--strip-comments-html',
                            # help='',
                            dest="flags", action="append_const", const=Bundler.FLAG_STRIP_COMMENTS_HTML)

    # strip JS flags
    arg_parser.add_argument('--strip-comments-javascript-block',
                            # help='',
                            dest="flags", action="append_const", const=Bundler.FLAG_STRIP_COMMENTS_JS_BLOCK)
    arg_parser.add_argument('--strip-comments-javascript-endline',
                            # help='',
                            dest="flags", action="append_const", const=Bundler.FLAG_STRIP_COMMENTS_JS_ENDLINE)
    arg_parser.add_argument('--strip-comments-javascript',
                            help='forces --strip-javascript-block AND --strip-javascript-endline',
                            dest="flags", action="append_const", const=Bundler.FLAG_STRIP_COMMENTS_JS)
    arg_parser.add_argument('--strip-comments-javascript-keep-first',
                            # help='forces --strip-javascript-block AND --strip-javascript-endline',
                            dest="flags", action="append_const", const=Bundler.FLAG_STRIP_COMMENTS_JS_KEEP_FIRST)

    # strip css flags
    arg_parser.add_argument('--strip-comments-css',
                            #help='forces --strip-javascript-block AND --strip-javascript-endline',
                            dest="flags", action="append_const", const=Bundler.FLAG_STRIP_COMMENTS_CSS)
    arg_parser.add_argument('--strip-comments-css-keep-first',
                            #help='forces --strip-javascript-block AND --strip-javascript-endline',
                            dest="flags", action="append_const", const=Bundler.FLAG_STRIP_COMMENTS_CSS_KEEP_FIRST)

    # strip flags combined
    arg_parser.add_argument('--strip-comments',
                            help='forces --strip-comments-html AND --strip-comments-javascript AND --strip-comments-css',
                            dest="flags", action="append_const", const=Bundler.FLAG_STRIP_COMMENTS)
    arg_parser.add_argument('--strip-comments-keep-first',
                            help='forces --strip-comments-js-keep-first AND --strip-comments-css-keep-first',
                            dest="flags", action="append_const", const=Bundler.FLAG_STRIP_COMMENTS_KEEP_FIRST)

    # strip HTML tags
    arg_parser.add_argument('--strip-tags', metavar='<tags>', type=str,
                            help='comma separated list of HTML tags to strip from the document',
                            dest="strip_tags")

    # strip markers JS
    arg_parser.add_argument('--strip-markers-js', metavar='<markers>', type=str,
                            help='comma separated list of markers. ' +
                                 'entire marked lines will ge stripped of any JavaScript',
                            dest="strip_inline_js")

    # strip markers CSS
    arg_parser.add_argument('--strip-markers-css', metavar='<markers>', type=str,
                            help='comma separated list of markers. ' +
                                 'entire marked lines will ge stripped of any CSS style definition',
                            dest="strip_inline_css")

    # strip markers JS and CSS combined
    arg_parser.add_argument('--strip-markers', metavar='<markers>', type=str,
                            help='comma separated list of markers. ' +
                                 'entire marked lines will ge stripped of any JavaScript or CSS style definition',
                            dest="strip_inline_js_css")

    # compress
    arg_parser.add_argument('--compress',
                            help='compress the output',
                            dest="flags", action="append_const", const=Bundler.FLAG_COMPRESS)

    # parse the args
    args = arg_parser.parse_args()
    del arg_parser

    #defaults and consts
    source = ""
    path = ""
    htroot = ""
    flags = 0
    compress_len = 120
    encoding = "utf-8"
    strip_tags = []
    strip_inline_js = []
    strip_inline_css = []

    infile = sys.stdin
    outfile = sys.stdout

    # @TODO add support for encoding argument

    if args.infile:
        infile = codecs.open(args.infile, 'rb', encoding=encoding)
        if not infile:
            raise Exception('can not read input file "' + args.infile + '"')

    source_lines = infile.readlines()
    infile.close()
    del infile

    # join the lines ...
    source = "".join(source_lines)
    del source_lines

    """ @TODO add support
    if args.path:
        path = args.path
    elif args.infile:
        path = os.path.abspath(os.path.dirname(args.infile))
    """

    """ @TODO add support
    if args.htroot:
        htroot = args.htroot
    elif args.path:
        htroot = path
    """
<commit_msg>added --path-dir<commit_after>#!/usr/bin/env python

from bundler import Bundler

if __name__ == '__main__':

    import sys
    import os
    import argparse
    import codecs
#    import chardet


    arg_parser = argparse.ArgumentParser(description="bundle a HTML file with multiple external (js|css) files " +
                                                     "to one big HTML file ")

    # input
    arg_parser.add_argument('-i', '--input-file', metavar='<file>', type=str,
                            help='a file path - if none given, listen on stdin',
                            dest="infile")

    arg_parser.add_argument('-p', '--path-dir', metavar='<dir>', type=str,
                            # help='',
                            dest="path")

    """ @TODO add support
    arg_parser.add_argument('-r', '--htroot-dir', metavar='<dir>', type=str,
                            # help='',
                            dest="htroot")
    """

    # output
    arg_parser.add_argument('-o', '--output-file', metavar='<file>', type=str,
                            help='a file path - if none given, output is sent to stdout',
                            dest="outfile")


    # strip HTML comments
    arg_parser.add_argument('--strip-comments-html',
                            # help='',
                            dest="flags", action="append_const", const=Bundler.FLAG_STRIP_COMMENTS_HTML)

    # strip JS flags
    arg_parser.add_argument('--strip-comments-javascript-block',
                            # help='',
                            dest="flags", action="append_const", const=Bundler.FLAG_STRIP_COMMENTS_JS_BLOCK)
    arg_parser.add_argument('--strip-comments-javascript-endline',
                            # help='',
                            dest="flags", action="append_const", const=Bundler.FLAG_STRIP_COMMENTS_JS_ENDLINE)
    arg_parser.add_argument('--strip-comments-javascript',
                            help='forces --strip-javascript-block AND --strip-javascript-endline',
                            dest="flags", action="append_const", const=Bundler.FLAG_STRIP_COMMENTS_JS)
    arg_parser.add_argument('--strip-comments-javascript-keep-first',
                            # help='forces --strip-javascript-block AND --strip-javascript-endline',
                            dest="flags", action="append_const", const=Bundler.FLAG_STRIP_COMMENTS_JS_KEEP_FIRST)

    # strip css flags
    arg_parser.add_argument('--strip-comments-css',
                            #help='forces --strip-javascript-block AND --strip-javascript-endline',
                            dest="flags", action="append_const", const=Bundler.FLAG_STRIP_COMMENTS_CSS)
    arg_parser.add_argument('--strip-comments-css-keep-first',
                            #help='forces --strip-javascript-block AND --strip-javascript-endline',
                            dest="flags", action="append_const", const=Bundler.FLAG_STRIP_COMMENTS_CSS_KEEP_FIRST)

    # strip flags combined
    arg_parser.add_argument('--strip-comments',
                            help='forces --strip-comments-html AND --strip-comments-javascript AND --strip-comments-css',
                            dest="flags", action="append_const", const=Bundler.FLAG_STRIP_COMMENTS)
    arg_parser.add_argument('--strip-comments-keep-first',
                            help='forces --strip-comments-js-keep-first AND --strip-comments-css-keep-first',
                            dest="flags", action="append_const", const=Bundler.FLAG_STRIP_COMMENTS_KEEP_FIRST)

    # strip HTML tags
    arg_parser.add_argument('--strip-tags', metavar='<tags>', type=str,
                            help='comma separated list of HTML tags to strip from the document',
                            dest="strip_tags")

    # strip markers JS
    arg_parser.add_argument('--strip-markers-js', metavar='<markers>', type=str,
                            help='comma separated list of markers. ' +
                                 'entire marked lines will ge stripped of any JavaScript',
                            dest="strip_inline_js")

    # strip markers CSS
    arg_parser.add_argument('--strip-markers-css', metavar='<markers>', type=str,
                            help='comma separated list of markers. ' +
                                 'entire marked lines will ge stripped of any CSS style definition',
                            dest="strip_inline_css")

    # strip markers JS and CSS combined
    arg_parser.add_argument('--strip-markers', metavar='<markers>', type=str,
                            help='comma separated list of markers. ' +
                                 'entire marked lines will ge stripped of any JavaScript or CSS style definition',
                            dest="strip_inline_js_css")

    # compress
    arg_parser.add_argument('--compress',
                            help='compress the output',
                            dest="flags", action="append_const", const=Bundler.FLAG_COMPRESS)

    # parse the args
    args = arg_parser.parse_args()
    del arg_parser

    #defaults and consts
    source = ""
    path = os.getcwd()
    htroot = os.getcwd()
    flags = 0
    compress_len = 120
    encoding = "utf-8"
    strip_tags = []
    strip_inline_js = []
    strip_inline_css = []

    infile = sys.stdin
    outfile = sys.stdout

    # @TODO add support for encoding argument

    if args.infile:
        infile = codecs.open(args.infile, 'rb', encoding=encoding)
        if not infile:
            raise Exception('can not read input file "' + args.infile + '"')

    source_lines = infile.readlines()
    infile.close()
    del infile

    # join the lines ...
    source = "".join(source_lines)
    del source_lines

    if args.path:
        path = args.path
    elif args.infile:
        path = os.path.dirname(args.infile)

    """ @TODO add support
    if args.htroot:
        htroot = args.htroot
    elif args.path:
        htroot = path
    """

{'fraction_non_alphanumeric': 0.11556073599200732, 'fraction_numerical': 0.000832570144034635, 'mean_word_length': 1.8310157907141174, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5575469', 'n_tokens_mistral': 3366, 'n_tokens_neox': 3267, 'n_words': 679}