<commit_before>#!/usr/bin/env python
"""usage: %prog [options] filename

Parse a document to a simpletree tree, with optional profiling
"""
#RELEASE move ./examples/

import sys
import os
from optparse import OptionParser

#RELEASE remove
from src import html5parser, liberalxmlparser
from src import treebuilders
#END RELEASE
#RELEASE add
#from html5lib import html5parser, liberalxmlparser
#from html5lib import treebuilders
#END RELEASE

def convertTreeDump(treedump):
    """convert the output of str(document) to something more readable
    """
    treedump = treedump.split("\n")[1:]
    rv = []
    for line in treedump:
        if line.startswith("|"):
            rv.append(line[3:])
        else:
            rv.append(line)
    return "\n".join(rv)

def parse():
    optParser = getOptParser()
    opts,args = optParser.parse_args()

    try:
        f = args[-1]
        # Try opening from the internet
        if f.startswith('http://'):
            try:
                import urllib
                f = urllib.urlopen(f).read()
            except: pass
        else:
            try:
                # Try opening from file system
                f = open(f)
            except IOError: pass
    except IndexError:
        sys.stderr.write("No filename provided. Use -h for help")
        sys.exit(1)

    if opts.treebuilder is not None:
        try:
            treebuilder = eval("treebuilders." + opts.treebuilder).TreeBuilder
        except ImportError, name:
            sys.stderr.write("Treebuilder %s not found"%name)
            raise
        except Exception, foo:
            treebuilder = treebuilders.simpletree.TreeBuilder
    else:
        treebuilder = treebuilders.simpletree.TreeBuilder

    if opts.xml:
        p = liberalxmlparser.XHTMLParser(tree=treebuilder)
    else:
        p = html5parser.HTMLParser(tree=treebuilder)

    if opts.profile:
        import hotshot
        import hotshot.stats
        prof = hotshot.Profile('stats.prof')
        prof.runcall(p.parse, f)
        prof.close()
        # XXX - We should use a temp file here
        stats = hotshot.stats.load('stats.prof')
        stats.strip_dirs()
        stats.sort_stats('time')
        stats.print_stats()
    elif opts.time:
        import time
        t0 = time.time()
        document = p.parse(f)
        t1 = time.time()
        printOutput(p, document, opts)
        t2 = time.time()
        sys.stdout.write("\n\nRun took: %fs (plus %fs to print the output)"%(t1-t0, t2-t1))
    else:
        document = p.parse(f)
        printOutput(p, document, opts)

def printOutput(parser, document, opts):
    if opts.encoding:
        print "Encoding:", parser.tokenizer.stream.charEncoding
    if not opts.no_tree:
        if opts.xml:
            sys.stdout.write(document.toxml("utf-8"))
        elif opts.hilite:
            sys.stdout.write(document.hilite("utf-8"))
        else:
            sys.stdout.write(parser.tree.testSerializer(document).encode("utf-8"))
    if opts.error:
        errList=[]
        for pos, message in parser.errors:
            errList.append("Line %i Col %i"%pos + " " + message)
        sys.stderr.write("\nParse errors:\n" + "\n".join(errList))

def getOptParser():
    parser = OptionParser(usage=__doc__)

    parser.add_option("-p", "--profile", action="store_true", default=False,
                      dest="profile", help="Use the hotshot profiler to "
                      "produce a detailed log of the run")
    
    parser.add_option("-t", "--time",
                      action="store_true", default=False, dest="time",<commit_msg>Add newlines to parse.py output<commit_after>#!/usr/bin/env python
"""usage: %prog [options] filename

Parse a document to a simpletree tree, with optional profiling
"""
#RELEASE move ./examples/

import sys
import os
from optparse import OptionParser

#RELEASE remove
from src import html5parser, liberalxmlparser
from src import treebuilders
#END RELEASE
#RELEASE add
#from html5lib import html5parser, liberalxmlparser
#from html5lib import treebuilders
#END RELEASE

def convertTreeDump(treedump):
    """convert the output of str(document) to something more readable
    """
    treedump = treedump.split("\n")[1:]
    rv = []
    for line in treedump:
        if line.startswith("|"):
            rv.append(line[3:])
        else:
            rv.append(line)
    return "\n".join(rv)

def parse():
    optParser = getOptParser()
    opts,args = optParser.parse_args()

    try:
        f = args[-1]
        # Try opening from the internet
        if f.startswith('http://'):
            try:
                import urllib
                f = urllib.urlopen(f).read()
            except: pass
        else:
            try:
                # Try opening from file system
                f = open(f)
            except IOError: pass
    except IndexError:
        sys.stderr.write("No filename provided. Use -h for help\n")
        sys.exit(1)

    if opts.treebuilder is not None:
        try:
            treebuilder = eval("treebuilders." + opts.treebuilder).TreeBuilder
        except ImportError, name:
            sys.stderr.write("Treebuilder %s not found\n"%name)
            raise
        except Exception, foo:
            treebuilder = treebuilders.simpletree.TreeBuilder
    else:
        treebuilder = treebuilders.simpletree.TreeBuilder

    if opts.xml:
        p = liberalxmlparser.XHTMLParser(tree=treebuilder)
    else:
        p = html5parser.HTMLParser(tree=treebuilder)

    if opts.profile:
        import hotshot
        import hotshot.stats
        prof = hotshot.Profile('stats.prof')
        prof.runcall(p.parse, f)
        prof.close()
        # XXX - We should use a temp file here
        stats = hotshot.stats.load('stats.prof')
        stats.strip_dirs()
        stats.sort_stats('time')
        stats.print_stats()
    elif opts.time:
        import time
        t0 = time.time()
        document = p.parse(f)
        t1 = time.time()
        printOutput(p, document, opts)
        t2 = time.time()
        sys.stdout.write("\n\nRun took: %fs (plus %fs to print the output)"%(t1-t0, t2-t1))
    else:
        document = p.parse(f)
        printOutput(p, document, opts)

def printOutput(parser, document, opts):
    if opts.encoding:
        print "Encoding:", parser.tokenizer.stream.charEncoding
    if not opts.no_tree:
        if opts.xml:
            sys.stdout.write(document.toxml("utf-8"))
        elif opts.hilite:
            sys.stdout.write(document.hilite("utf-8"))
        else:
            sys.stdout.write(parser.tree.testSerializer(document).encode("utf-8"))
    if opts.error:
        errList=[]
        for pos, message in parser.errors:
            errList.append("Line %i Col %i"%pos + " " + message)
        sys.stderr.write("\nParse errors:\n" + "\n".join(errList)+"\n")

def getOptParser():
    parser = OptionParser(usage=__doc__)

    parser.add_option("-p", "--profile", action="store_true", default=False,
                      dest="profile", help="Use the hotshot profiler to "
                      "produce a detailed log of the run")
    
    parser.add_option("-t", "--time",
                      action="store_true", default=False, dest="time",
{'fraction_non_alphanumeric': 0.11530456500624393, 'fraction_numerical': 0.005272651588733176, 'mean_word_length': 2.165568730786122, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 12}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '689790', 'n_tokens_mistral': 2147, 'n_tokens_neox': 2116, 'n_words': 547}