<commit_before>#!/usr/bin/env python
# vim:fileencoding=utf-8
# License: Apache 2.0 Copyright: 2017, <NAME> <kovid at kovidgoyal.net>

from __future__ import absolute_import, division, print_function, unicode_literals

from functools import partial

import html5lib

import html5_parser

try:
    from time import monotonic
except ImportError:
    from time import time as monotonic

TF = 'test/large.html'
try:
    raw = open(TF, 'rb').read()
except Exception:
    try:
        from urllib.request import urlopen
    except ImportError:
        from urllib import urlopen
    print('Downloading large HTML file...')
    raw = urlopen('https://www.w3.org/TR/html5/single-page.html').read()
    open(TF, 'wb').write(raw)


print('Testing with HTML file of', '{:,}'.format(len(raw)), 'bytes')


def timeit(func, number=1):
    total = 0
    for i in range(number):
        st = monotonic()
        r = func()
        t = monotonic() - st
        total += t
        del r
    return total / number


def doit(name, func, num=5):
    print('Parsing repeatedly with', name)
    t = timeit(func, num)
    print(name, 'took an average of : {:,.3f} seconds to parse it'.format(t))


doit('html5-parser', partial(html5_parser.parse, raw, transport_encoding="utf-8"))
doit('html5lib', partial(html5lib.parse, raw, transport_encoding="utf-8", treebuilder="lxml"))<commit_msg>Turn on namespacing in the benchmark script for fair comparison<commit_after>#!/usr/bin/env python
# vim:fileencoding=utf-8
# License: Apache 2.0 Copyright: 2017, <NAME> <kovid at kovidgoyal.net>

from __future__ import absolute_import, division, print_function, unicode_literals

from functools import partial

import html5lib

import html5_parser

try:
    from time import monotonic
except ImportError:
    from time import time as monotonic

TF = 'test/large.html'
try:
    raw = open(TF, 'rb').read()
except Exception:
    try:
        from urllib.request import urlopen
    except ImportError:
        from urllib import urlopen
    print('Downloading large HTML file...')
    raw = urlopen('https://www.w3.org/TR/html5/single-page.html').read()
    open(TF, 'wb').write(raw)

print('Testing with HTML file of', '{:,}'.format(len(raw)), 'bytes')


def timeit(func, number=1):
    total = 0
    for i in range(number):
        st = monotonic()
        r = func()
        t = monotonic() - st
        total += t
        del r
    return total / number


def doit(name, func, num=5):
    print('Parsing repeatedly with', name)
    t = timeit(func, num)
    print(name, 'took an average of : {:,.3f} seconds to parse it'.format(t))


doit(
    'html5-parser',
    partial(html5_parser.parse, raw, transport_encoding="utf-8", namespace_elements=True))
doit('html5lib', partial(html5lib.parse, raw, transport_encoding="utf-8", treebuilder="lxml"))
{'fraction_non_alphanumeric': 0.13020277481323372, 'fraction_numerical': 0.014941302027748132, 'mean_word_length': 3.54281098546042, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '287560', 'n_tokens_mistral': 947, 'n_tokens_neox': 900, 'n_words': 264}