<commit_before>import os, logging
import pytest
from mappyfile.parser import Parser
from mappyfile.pprint import PrettyPrinter
import mappyfile
from mappyfile.transformer import MapfileToDict

import cProfile


def output():
    """
    Parse, transform, and pretty print 
    the result
    """
    p = Parser()
    m = MapfileToDict()
    
    fn = r"D:\Temp\large.map"
    with open(fn) as f:
        s = f.read()

    ast = p.parse(s)
    ##print(ast)
    d = m.transform(ast)
    ##print(d)
    pp = PrettyPrinter(indent=0, newlinechar=" ", quote="'")
    pp.pprint(d)

if __name__ == "__main__":
    
    pr = cProfile.Profile()
    pr.enable()

    #cProfile.run('output()')
    output()
    pr.disable()
    pr.print_stats(sort='time')<commit_msg>Update performance test<commit_after>import os, logging
import pytest
from mappyfile.parser import Parser
from mappyfile.pprint import PrettyPrinter
import mappyfile
from mappyfile.transformer import MapfileToDict

import cProfile


def output(fn):
    """
    Parse, transform, and pretty print 
    the result
    """
    p = Parser()
    m = MapfileToDict()

    ast = p.parse_file(fn)
    #print(ast)
    d = m.transform(ast)
    #print(d)
    pp = PrettyPrinter(indent=0, newlinechar=" ", quote="'")
    pp.pprint(d)

if __name__ == "__main__":
    #fn = r"D:\Temp\large_map1.txt"
    fn = r"D:\Temp\large_map2.txt"
    pr = cProfile.Profile()
    pr.enable()
    output(fn)
    pr.disable()
    pr.print_stats(sort='time')
{'fraction_non_alphanumeric': 0.1301416048550236, 'fraction_numerical': 0.0026972353337828725, 'mean_word_length': 3.076923076923077, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7116645', 'n_tokens_mistral': 528, 'n_tokens_neox': 522, 'n_words': 122}