<commit_before>import sys
from freequery.repository.formats import WARCParser

inpaths = sys.argv[1:]

for path in inpaths:
    print "@ %s" % path
    for doc in WARCParser(open(path, 'rb')):
        print doc.uri
        doc.links_lxml_html()
        for uri in doc.link_uris:
            pass#print " - %s" % uri<commit_msg>only try to instantiate html parser<commit_after>import sys
from freequery.repository.formats import WARCParser

inpaths = sys.argv[1:]

for path in inpaths:
    print "@ %s" % path
    for doc in WARCParser(open(path, 'rb')):
        print doc.uri
        doc.html_parser
        #doc.links_lxml_html()
        #for uri in doc.link_uris:
        #    pass#print " - %s" % uri
{'fraction_non_alphanumeric': 0.10810810810810811, 'fraction_numerical': 0.002844950213371266, 'mean_word_length': 2.6666666666666665, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1168615', 'n_tokens_mistral': 239, 'n_tokens_neox': 239, 'n_words': 65}