<commit_before>from pathlib import Path
import sys

from panflute import convert_text

from pantable.ast import PanTable


def test_from_panflute_ast():
    '''test parsing native table into Pantable
    '''
    paths = (Path(__file__).parent / 'native').glob('*.native')
    for path in paths:
        try:
            with open(path, 'r') as f:
                native = f.read()
            temp = convert_text(native, input_format='native')
            # input files should only have 1 single outter block
            assert len(temp) == 1
            table = temp[0]
            pan_table = PanTable.from_panflute_ast(table)
            table_idem = pan_table.to_panflute_ast()
            # check for idempotence
            native_orig = convert_text(table, input_format='panflute', output_format='native')
            native_idem = convert_text(table_idem, input_format='panflute', output_format='native')
            if native_orig == native_idem:
                print(f'{path} is identical after round trip.', file=sys.stderr)
            else:
                raise AssertionError(f'{path} is not identical after round trip.')
        except AssertionError as e:
            raise e
        except Exception as e:
            print(f'Cannot parse native file {path} into PanTable.', file=sys.stderr)
            raise e<commit_msg>native_test.py: refactor into one test per input file<commit_after>from pathlib import Path
import sys
import inspect

from panflute import convert_text

from pantable.ast import PanTable


def routine():
    '''test parsing native table into Pantable
    '''
    # c.f. https://stackoverflow.com/a/5067654
    name = inspect.stack()[1][3].split('_')[1]

    path = Path(__file__).parent / 'native' / f'{name}.native'
    print(f'Testing case {name} in {path}...', file=sys.stderr)
    with open(path, 'r') as f:
        native = f.read()
    doc = convert_text(native, input_format='native')
    # input files should only have 1 single outter block
    assert len(doc) == 1
    table = doc[0]
    pan_table = PanTable.from_panflute_ast(table)
    table_idem = pan_table.to_panflute_ast()
    # check for idempotence
    native_orig = convert_text(table, input_format='panflute', output_format='native')
    native_idem = convert_text(table_idem, input_format='panflute', output_format='native')
    assert native_orig == native_idem

# test_NAME will test against the file NAME.native

def test_nordics():
    routine()


def test_planets():
    routine()


def test_students():
    routine()
{'fraction_non_alphanumeric': 0.10295857988165681, 'fraction_numerical': 0.00631163708086785, 'mean_word_length': 2.9075500770416025, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '91367', 'n_tokens_mistral': 774, 'n_tokens_neox': 747, 'n_words': 210}