<commit_before>import pkgutil

from pybtex.bibtex import bst
from io import StringIO



test_data = (
    'plain',
    'apacite',
    'jurabib',
)


def check_bst_parser(dataset_name):
    module = __import__('pybtex.tests.bst_parser_test.{0}'.format(dataset_name), globals(), locals(), 'bst')
    correct_result = module.bst
    bst_data = pkgutil.get_data('pybtex.tests.bst_parser_test', dataset_name + '.bst').decode('latin1')
    actual_result = list(bst.parse_stream(StringIO(bst_data)))

    # XXX pyparsing return list-like object which are not equal to plain lists
    for correct_element, actual_element in zip(actual_result, correct_result):
        assert repr(correct_element) == repr(actual_element), '\n{0}\n{1}'.format(correct_element, actual_element)

 
def test_bst_parser():
    for dataset_name in test_data:
        yield check_bst_parser, dataset_name<commit_msg>Do not rely on repr() when comparing BST ASTs.<commit_after>import pkgutil

from pybtex.bibtex import bst
from io import StringIO



test_data = (
    'plain',
    'apacite',
    'jurabib',
)


def check_bst_parser(dataset_name):
    module = __import__('pybtex.tests.bst_parser_test.{0}'.format(dataset_name), globals(), locals(), 'bst')
    correct_result = module.bst
    bst_data = pkgutil.get_data('pybtex.tests.bst_parser_test', dataset_name + '.bst').decode('latin1')
    actual_result = list(bst.parse_stream(StringIO(bst_data)))

    # XXX pyparsing return list-like object which are not equal to plain lists
    for correct_element, actual_element in zip(actual_result, correct_result):
        assert correct_element == actual_element, '\n{0}\n{1}'.format(correct_element, actual_element)

 
def test_bst_parser():
    for dataset_name in test_data:
        yield check_bst_parser, dataset_name
{'fraction_non_alphanumeric': 0.11123532699832309, 'fraction_numerical': 0.004471771939631079, 'mean_word_length': 4.7006369426751595, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5988350', 'n_tokens_mistral': 603, 'n_tokens_neox': 576, 'n_words': 129}