<jupyter_start><jupyter_text>Basic wiki template parser using parsimonious> "First step towards wiktionary extraction"- toc: false- branch: master- hidden: true- categories: [wiki, parsimonious]<jupyter_code>from parsimonious.grammar import Grammar
grammar = Grammar(
"""
tplcall = tplopen tpltext (tplinner)+ tplclose
tplinner = named / positional
named = bar tpltext eq tpltext
positional = bar tpltext
tpltext = ("{{=}}" / ~r"[^\}\|=]")*
tplchar = ~r"[^\}\|=]"
tplopen = "{{"
tplclose = "}}"
eq = "="
bar = "|"
"""
)
test1 = "{{tpl|foo|bar=baz||fooish{{=}}bar}}"
print(grammar.parse(test1))<jupyter_output><Node called "tplcall" matching "{{tpl|foo|bar=baz||fooish{{=}}bar}}">
    <Node called "tplopen" matching "{{">
    <Node called "tpltext" matching "tpl">
        <Node matching "t">
            <RegexNode matching "t">
        <Node matching "p">
            <RegexNode matching "p">
        <Node matching "l">
            <RegexNode matching "l">
    <Node matching "|foo|bar=baz||fooish{{=}}bar">
        <Node called "tplinner" matching "|foo">
            <Node called "positional" matching "|foo">
                <Node called "bar" matching "|">
                <Node called "tpltext" matching "foo">
                    <Node matching "f">
                        <RegexNode matching "f">
                    <Node matching "o">
                        <RegexNode matching "o">
                    <Node matching "o">
                        <RegexNode matching "o">
        <Node called "tplinner" matching "|bar=baz">
            <Node called "named" matching "|bar=baz">
                [...]
{'fraction_non_alphanumeric': 0.16047087980173483, 'fraction_numerical': 0.0012391573729863693, 'mean_word_length': 2.4731182795698925, 'pattern_counts': {'":': 0, '<': 26, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '575732', 'license': 'Apache-2.0', 'n_tokens_mistral': 512, 'n_tokens_neox': 502, 'n_words': 133, 'path': '_notebooks/2021-11-08-basic-wiki-template-parser-with-parsimonious.ipynb', 'repo_name': 'jimregan/notes', 'size': '1614'}