generate a parser load the generated parser class run a parser on a file stream run the parser on a string combine parsestring and generateparser import a python module from a path spec importlib util specfromfilelocationfullname path assert spec is not none mod importlib util modulefromspecspec we assume this is not none and has an execmodule method see https docs python org3referenceimport html highlightexecmoduleloading loader castany spec loader loader execmodulemod return mod def generatecparsersourcegrammar grammar str out io stringio genr cparsergeneratorgrammar alltokens exacttokens nonexacttokens out genr generatestring return out getvalue def generateparsercextension grammar grammar path pathlib purepath debug bool false librarydir optionalstr none any make sure that the working directory is empty reusing nonempty temporary directories when generating extensions can lead to segmentation faults check issue 95 https github comgvanrossumpegenissues95 for more context assert not os listdirpath source path parse c with opensource w encodingutf8 as file genr cparsergenerator grammar alltokens exacttokens nonexacttokens file debugdebug genr generateparse c compilecextension strsource builddirstrpath significant testpeggenerator speedups disableoptimizationtrue librarydirlibrarydir def printmemstats bool mib final 220 try import psutil except importerror return false printmemory stats process psutil process meminfo process memoryinfo res resrss meminfo rss mib resvms meminfo vms mib if sys platform win32 resmaxrss meminfo peakwset mib else see https stackoverflow comquestions938733totalmemoryusedbypythonprocess import resource since it doesn t exist on windows rusage resource getrusageresource rusageself if sys platform darwin factor 1 else factor 1024 linux resmaxrss rusage rumaxrss factor mib for key value in res items printf key 12 12s value 10 0f mib return true generate a parser load the generated parser class run a parser on a file stream type ignore arg type typeshed issue 3515 run the parser on a string type ignore arg type typeshed issue 3515 combine parse_string and generate_parser import a python module from a path we assume this is not none and has an exec_module method see https docs python org 3 reference import html highlight exec_module loading generate a parser c extension for the given grammar in the given path returns a module object with a parse_string method todo express that using a protocol make sure that the working directory is empty reusing non empty temporary directories when generating extensions can lead to segmentation faults check issue 95 https github com gvanrossum pegen issues 95 for more context significant test_peg_generator speedups see https stackoverflow com questions 938733 total memory used by python process since it doesn t exist on windows linux
import importlib.util
import io
import os
import pathlib
import sys
import textwrap
import token
import tokenize
from typing import IO, Any, Dict, Final, Optional, Type, cast

from pegen.build import compile_c_extension
from pegen.c_generator import CParserGenerator
from pegen.grammar import Grammar
from pegen.grammar_parser import GeneratedParser as GrammarParser
from pegen.parser import Parser
from pegen.python_generator import PythonParserGenerator
from pegen.tokenizer import Tokenizer

ALL_TOKENS = token.tok_name
EXACT_TOKENS = token.EXACT_TOKEN_TYPES
NON_EXACT_TOKENS = {
    name for index, name in token.tok_name.items() if index not in EXACT_TOKENS.values()
}


def generate_parser(grammar: Grammar) -> Type[Parser]:
    
    out = io.StringIO()
    genr = PythonParserGenerator(grammar, out)
    genr.generate("<string>")

    
    ns: Dict[str, Any] = {}
    exec(out.getvalue(), ns)
    return ns["GeneratedParser"]


def run_parser(file: IO[bytes], parser_class: Type[Parser], *, verbose: bool = False) -> Any:
    
    tokenizer = Tokenizer(tokenize.generate_tokens(file.readline))  
    parser = parser_class(tokenizer, verbose=verbose)
    result = parser.start()
    if result is None:
        raise parser.make_syntax_error("invalid syntax")
    return result


def parse_string(
    source: str, parser_class: Type[Parser], *, dedent: bool = True, verbose: bool = False
) -> Any:
    
    if dedent:
        source = textwrap.dedent(source)
    file = io.StringIO(source)
    return run_parser(file, parser_class, verbose=verbose)  


def make_parser(source: str) -> Type[Parser]:
    
    grammar = parse_string(source, GrammarParser)
    return generate_parser(grammar)


def import_file(full_name: str, path: str) -> Any:
    

    spec = importlib.util.spec_from_file_location(full_name, path)
    assert spec is not None
    mod = importlib.util.module_from_spec(spec)

    
    
    loader = cast(Any, spec.loader)
    loader.exec_module(mod)
    return mod


def generate_c_parser_source(grammar: Grammar) -> str:
    out = io.StringIO()
    genr = CParserGenerator(grammar, ALL_TOKENS, EXACT_TOKENS, NON_EXACT_TOKENS, out)
    genr.generate("<string>")
    return out.getvalue()


def generate_parser_c_extension(
    grammar: Grammar,
    path: pathlib.PurePath,
    debug: bool = False,
    library_dir: Optional[str] = None,
) -> Any:
    
    
    
    
    
    assert not os.listdir(path)
    source = path / "parse.c"
    with open(source, "w", encoding="utf-8") as file:
        genr = CParserGenerator(
            grammar, ALL_TOKENS, EXACT_TOKENS, NON_EXACT_TOKENS, file, debug=debug
        )
        genr.generate("parse.c")
    compile_c_extension(
        str(source),
        build_dir=str(path),
        
        disable_optimization=True,
        library_dir=library_dir,
    )


def print_memstats() -> bool:
    MiB: Final = 2**20
    try:
        import psutil
    except ImportError:
        return False
    print("Memory stats:")
    process = psutil.Process()
    meminfo = process.memory_info()
    res = {}
    res["rss"] = meminfo.rss / MiB
    res["vms"] = meminfo.vms / MiB
    if sys.platform == "win32":
        res["maxrss"] = meminfo.peak_wset / MiB
    else:
        
        import resource  

        rusage = resource.getrusage(resource.RUSAGE_SELF)
        if sys.platform == "darwin":
            factor = 1
        else:
            factor = 1024  
        res["maxrss"] = rusage.ru_maxrss * factor / MiB
    for key, value in res.items():
        print(f"  {key:12.12s}: {value:10.0f} MiB")
    return True