def resolvefilenamesfilenames if filenames resolved files resolvefilenamef for f in filenames else resolved files iterfilenames return resolved the formats def fmtsummaryanalysis xxx support sorting and grouping supported unsupported for item in analysis if item supported supported appenditem else unsupported appenditem total 0 def sectionname groupitems nonlocal total items render canalyzer buildsectionname groupitems relrootreporoot yield from render total lenitems yield yield yield supported yield yield from section types supported yield from section variables supported yield yield yield unsupported yield yield from section types unsupported yield from section variables unsupported yield yield f grand total total the checks checks dictcanalyzer checks globals analyzer checkglobals the commands fileskwargs dictexcludedparser excluded nargs def cliparseparser processoutput cparser addoutputcliparser processkind addkindfilteringcliparser processpreprocessor cparser addpreprocessorcli parser getpreprocessorparser getpreprocessor processfiles addfilescliparser fileskwargs return processoutput processkind processpreprocessor processfiles def cmdparsefilenamesnone kwargs filenames resolvefilenamesfilenames if getfilepreprocessor not in kwargs kwargs getfilepreprocessor parser getpreprocessor cparser cmdparse filenames relrootreporoot filemaxsizesparser maxsizes kwargs def clicheckparser kwargs return canalyzer clicheckparser checks kwargs fileskwargs def cmdcheckfilenamesnone kwargs filenames resolvefilenamesfilenames kwargs getfilepreprocessor parser getpreprocessorlogerrprint try canalyzer cmdcheck filenames relrootreporoot analyzeanalyzer analyze checkschecks filemaxsizesparser maxsizes kwargs except systemexit as exc numfailed exc args0 if getattrexc args none else none if isinstancenumfailed int if numfailed 0 sys stderr flush printcheckexplanation flushtrue raise reraise except exception sys stderr flush printcheckexplanation flushtrue raise reraise def cmdanalyzefilenamesnone kwargs formats dictcanalyzer formats formats summary fmtsummary filenames resolvefilenamesfilenames kwargs getfilepreprocessor parser getpreprocessorlogerrprint canalyzer cmdanalyze filenames relrootreporoot analyzeanalyzer analyze formatsformats filemaxsizesparser maxsizes kwargs def clidataparser filenames false known true return canalyzer clidataparser filenames known def cmddatadatacmd kwargs formats dictcanalyzer formats formats summary fmtsummary filenames file for file in resolvefilenamesnone if file not in parser excluded kwargs getfilepreprocessor parser getpreprocessorlogerrprint if datacmd show types analyzer readknown results for decl info in types items if info is unknown if decl kind in kind struct kind union extra unsupported type unknown lendecl members else extra unsupported type unknown info info extra results appenddecl info if decl shortkey struct object tempinfo info known analyzer analysis fromresultsresults analyze none elif datacmd dump known analyzer knownfile def analyzefiles kwargs decls for decl in analyzer iterdeclsfiles kwargs if not kind istypedecldecl kind continue if not decl filename endswith h if decl shortkey not in analyzer knownindotc continue decls appenddecl results canalyzer analyzedecls decls known analyzeresolvedanalyzer analyzeresolved return analyzer analysis fromresultsresults else check known analyzer readknown def analyzefiles kwargs return analyzer iterdeclsfiles kwargs extracolumns none canalyzer cmddata datacmd filenames known analyzeanalyze formatsformats extracolumnsextracolumns relrootreporoot kwargs def clicapiparser parser addargument levels action append metavar level parser addargumentf public dest levels action appendconst const public parser addargumentf nopublic dest levels action appendconst const nopublic for level in capi levels parser addargumentf level dest levels action appendconst constlevel def processlevelsargs argvnone levels for raw in args levels or for level in raw replace strip split if level public levels append stable levels append cpython elif level nopublic levels append private levels append internal elif level in capi levels levels appendlevel else parser errorf expected level to be one of sortedcapi levels got level r args levels setlevels parser addargument kinds action append metavar kind for kind in capi kinds parser addargumentf kind dest kinds action appendconst constkind def processkindsargs argvnone kinds for raw in args kinds or for kind in raw replace strip split if kind in capi kinds kinds appendkind else parser errorf expected kind to be one of sortedcapi kinds got kind r args kinds setkinds parser addargument groupby dest groupby choices level kind parser addargument format default table parser addargument summary dest format action storeconst const summary def processformatargs argvnone orig args format args format capi resolveformatargs format if isinstanceargs format str if args format not in capi formats parser errorf unsupported format orig r parser addargument showempty dest showempty action storetrue parser addargument noshowempty dest showempty action storefalse parser setdefaultsshowemptynone xxx add sortby sort and nosort parser addargument ignore dest ignored action append def processignoredargs argvnone ignored for raw in args ignored or ignored extendraw replace strip split args ignored ignored or none parser addargument filenames nargs metavar filename processprogress addprogresscliparser return processlevels processkinds processformat processignored processprogress def cmdcapifilenamesnone levelsnone kindsnone groupby kind format table showemptynone ignorednone trackprogressnone verbosityverbosity kwargs render capi getrendererformat filenames files iterheaderfilesfilenames levelslevels filenames file for file in mainforfilenamesfilenames if trackprogress filenames trackprogressfilenames items capi itercapifilenames if levels items item for item in items if item level in levels if kinds items item for item in items if item kind in kinds filter capi resolvefilterignored if filter items item for item in items if filteritem loglambda msg logger log1 msg lines render items groupbygroupby showemptyshowempty verboseverbosity verbosity print for line in lines printline def clibuiltintypesparser parser addargument format dest fmt default table parser addargument summary dest format action storeconst const summary def processformatargs argvnone orig args fmt args fmt builtintypes resolveformatargs fmt if isinstanceargs fmt str if args fmt not in builtintypes formats parser errorf unsupported format orig r parser addargument includemodules dest showmodules action storetrue def processmodulesargs argvnone pass return processformat processmodules def cmdbuiltintypesfmt showmodulesfalse verbosityverbosity render builtintypes getrendererfmt types builtintypes iterbuiltintypes match builtintypes resolvematchershowmodules if match types t for t in types if matcht loglambda msg logger log1 msg lines render types verboseverbosity verbosity print for line in lines printline we do not define any other cmd handlers here favoring those defined elsewhere commands check analyze and fail if the cpython source code has any problems clicheck cmdcheck analyze report on the state of the cpython source code lambda p canalyzer clianalyzep fileskwargs cmdanalyze parse parse the cpython source files cliparse cmdparse data checkmanage local data e g known types ignored vars caches clidata cmddata capi inspect the capi clicapi cmdcapi builtintypes show the builtin types clibuiltintypes cmdbuiltintypes the script def parseargsargvsys argv1 prognone subsetnone import argparse parser argparse argumentparser progprog or getprog if subset check or subset check if checks is not none commands dictcommands commands check listcommands check cli commands check 10 commands check 10 lambda p clip checkschecks processors addcommandscli parser commandscommands commonspecs addverbositycli addtracebackcli subsetsubset args parser parseargsargv ns varsargs cmd ns pop cmd verbosity tracebackcm processargsbykey args argv processorscmd verbosity tracebackcm if cmd parse verbosity is sent to the commands so we put it back args verbosity verbosity return cmd ns verbosity tracebackcm def maincmd cmdkwargs try runcmd commandscmd1 except keyerror raise valueerrorf unsupported cmd cmd r runcmdcmdkwargs if name main cmd cmdkwargs verbosity tracebackcm parseargs configureloggerverbosity with tracebackcm maincmd cmdkwargs non constant global variables are generally not supported in the cpython repo we use a tool to analyze the c code and report if any unsupported globals are found the tool may be run manually with python tools c analyzer check c globals py format summary file occasionally the tool is unable to parse updated code if this happens then add the file to the excluded list in tools c analyzer cpython _parser py and create a new issue for fixing the tool and cc ericsnowcurrently on the issue if the tool reports an unsupported global variable and it is actually const and thus supported then first try fixing the declaration appropriately in the code if that doesn t work then add the variable to the should be const section of tools c analyzer cpython ignored tsv if the tool otherwise reports an unsupported global variable then first try to make it non global possibly adding to pyinterpreterstate for core code or module state for extension modules in an emergency you can add the variable to tools c analyzer cpython globals to fix tsv to get ci passing but doing so should be avoided if this course it taken be sure to create an issue for eliminating the global and cc ericsnowcurrently the formats xxx support sorting and grouping the checks the commands re raise re raise check xxx add sort by sort and no sort filenames file for file _ in main_for_filenames filenames parser add_argument summary dest format action store_const const summary verbose verbosity verbosity we do not define any other cmd_ handlers here favoring those defined elsewhere the script if subset check or subset check if checks is not none commands dict commands commands check list commands check cli commands check 1 0 commands check 1 0 lambda p cli p checks checks verbosity is sent to the commands so we put it back
import logging
import sys
import textwrap

from c_common.scriptutil import (
    VERBOSITY,
    add_verbosity_cli,
    add_traceback_cli,
    add_commands_cli,
    add_kind_filtering_cli,
    add_files_cli,
    add_progress_cli,
    process_args_by_key,
    configure_logger,
    get_prog,
)
from c_parser.info import KIND
import c_parser.__main__ as c_parser
import c_analyzer.__main__ as c_analyzer
import c_analyzer as _c_analyzer
from c_analyzer.info import UNKNOWN
from . import _analyzer, _builtin_types, _capi, _files, _parser, REPO_ROOT


logger = logging.getLogger(__name__)


CHECK_EXPLANATION = textwrap.dedent()


def _resolve_filenames(filenames):
    if filenames:
        resolved = (_files.resolve_filename(f) for f in filenames)
    else:
        resolved = _files.iter_filenames()
    return resolved





def fmt_summary(analysis):
    
    supported = []
    unsupported = []
    for item in analysis:
        if item.supported:
            supported.append(item)
        else:
            unsupported.append(item)
    total = 0

    def section(name, groupitems):
        nonlocal total
        items, render = c_analyzer.build_section(name, groupitems,
                                                 relroot=REPO_ROOT)
        yield from render()
        total += len(items)

    yield ''
    yield '===================='
    yield 'supported'
    yield '===================='

    yield from section('types', supported)
    yield from section('variables', supported)

    yield ''
    yield '===================='
    yield 'unsupported'
    yield '===================='

    yield from section('types', unsupported)
    yield from section('variables', unsupported)

    yield ''
    yield f'grand total: {total}'





CHECKS = dict(c_analyzer.CHECKS, **{
    'globals': _analyzer.check_globals,
})




FILES_KWARGS = dict(excluded=_parser.EXCLUDED, nargs='*')


def _cli_parse(parser):
    process_output = c_parser.add_output_cli(parser)
    process_kind = add_kind_filtering_cli(parser)
    process_preprocessor = c_parser.add_preprocessor_cli(
        parser,
        get_preprocessor=_parser.get_preprocessor,
    )
    process_files = add_files_cli(parser, **FILES_KWARGS)
    return [
        process_output,
        process_kind,
        process_preprocessor,
        process_files,
    ]


def cmd_parse(filenames=None, **kwargs):
    filenames = _resolve_filenames(filenames)
    if 'get_file_preprocessor' not in kwargs:
        kwargs['get_file_preprocessor'] = _parser.get_preprocessor()
    c_parser.cmd_parse(
        filenames,
        relroot=REPO_ROOT,
        file_maxsizes=_parser.MAX_SIZES,
        **kwargs
    )


def _cli_check(parser, **kwargs):
    return c_analyzer._cli_check(parser, CHECKS, **kwargs, **FILES_KWARGS)


def cmd_check(filenames=None, **kwargs):
    filenames = _resolve_filenames(filenames)
    kwargs['get_file_preprocessor'] = _parser.get_preprocessor(log_err=print)
    try:
        c_analyzer.cmd_check(
            filenames,
            relroot=REPO_ROOT,
            _analyze=_analyzer.analyze,
            _CHECKS=CHECKS,
            file_maxsizes=_parser.MAX_SIZES,
            **kwargs
        )
    except SystemExit as exc:
        num_failed = exc.args[0] if getattr(exc, 'args', None) else None
        if isinstance(num_failed, int):
            if num_failed > 0:
                sys.stderr.flush()
                print(CHECK_EXPLANATION, flush=True)
        raise  
    except Exception:
        sys.stderr.flush()
        print(CHECK_EXPLANATION, flush=True)
        raise  


def cmd_analyze(filenames=None, **kwargs):
    formats = dict(c_analyzer.FORMATS)
    formats['summary'] = fmt_summary
    filenames = _resolve_filenames(filenames)
    kwargs['get_file_preprocessor'] = _parser.get_preprocessor(log_err=print)
    c_analyzer.cmd_analyze(
        filenames,
        relroot=REPO_ROOT,
        _analyze=_analyzer.analyze,
        formats=formats,
        file_maxsizes=_parser.MAX_SIZES,
        **kwargs
    )


def _cli_data(parser):
    filenames = False
    known = True
    return c_analyzer._cli_data(parser, filenames, known)


def cmd_data(datacmd, **kwargs):
    formats = dict(c_analyzer.FORMATS)
    formats['summary'] = fmt_summary
    filenames = (file
                 for file in _resolve_filenames(None)
                 if file not in _parser.EXCLUDED)
    kwargs['get_file_preprocessor'] = _parser.get_preprocessor(log_err=print)
    if datacmd == 'show':
        types = _analyzer.read_known()
        results = []
        for decl, info in types.items():
            if info is UNKNOWN:
                if decl.kind in (KIND.STRUCT, KIND.UNION):
                    extra = {'unsupported': ['type unknown'] * len(decl.members)}
                else:
                    extra = {'unsupported': ['type unknown']}
                info = (info, extra)
            results.append((decl, info))
            if decl.shortkey == 'struct _object':
                tempinfo = info
        known = _analyzer.Analysis.from_results(results)
        analyze = None
    elif datacmd == 'dump':
        known = _analyzer.KNOWN_FILE
        def analyze(files, **kwargs):
            decls = []
            for decl in _analyzer.iter_decls(files, **kwargs):
                if not KIND.is_type_decl(decl.kind):
                    continue
                if not decl.filename.endswith('.h'):
                    if decl.shortkey not in _analyzer.KNOWN_IN_DOT_C:
                        continue
                decls.append(decl)
            results = _c_analyzer.analyze_decls(
                decls,
                known={},
                analyze_resolved=_analyzer.analyze_resolved,
            )
            return _analyzer.Analysis.from_results(results)
    else:  
        known = _analyzer.read_known()
        def analyze(files, **kwargs):
            return _analyzer.iter_decls(files, **kwargs)
    extracolumns = None
    c_analyzer.cmd_data(
        datacmd,
        filenames,
        known,
        _analyze=analyze,
        formats=formats,
        extracolumns=extracolumns,
        relroot=REPO_ROOT,
        **kwargs
    )


def _cli_capi(parser):
    parser.add_argument('--levels', action='append', metavar='LEVEL[,...]')
    parser.add_argument(f'--public', dest='levels',
                        action='append_const', const='public')
    parser.add_argument(f'--no-public', dest='levels',
                        action='append_const', const='no-public')
    for level in _capi.LEVELS:
        parser.add_argument(f'--{level}', dest='levels',
                            action='append_const', const=level)
    def process_levels(args, *, argv=None):
        levels = []
        for raw in args.levels or ():
            for level in raw.replace(',', ' ').strip().split():
                if level == 'public':
                    levels.append('stable')
                    levels.append('cpython')
                elif level == 'no-public':
                    levels.append('private')
                    levels.append('internal')
                elif level in _capi.LEVELS:
                    levels.append(level)
                else:
                    parser.error(f'expected LEVEL to be one of {sorted(_capi.LEVELS)}, got {level!r}')
        args.levels = set(levels)

    parser.add_argument('--kinds', action='append', metavar='KIND[,...]')
    for kind in _capi.KINDS:
        parser.add_argument(f'--{kind}', dest='kinds',
                            action='append_const', const=kind)
    def process_kinds(args, *, argv=None):
        kinds = []
        for raw in args.kinds or ():
            for kind in raw.replace(',', ' ').strip().split():
                if kind in _capi.KINDS:
                    kinds.append(kind)
                else:
                    parser.error(f'expected KIND to be one of {sorted(_capi.KINDS)}, got {kind!r}')
        args.kinds = set(kinds)

    parser.add_argument('--group-by', dest='groupby',
                        choices=['level', 'kind'])

    parser.add_argument('--format', default='table')
    parser.add_argument('--summary', dest='format',
                        action='store_const', const='summary')
    def process_format(args, *, argv=None):
        orig = args.format
        args.format = _capi.resolve_format(args.format)
        if isinstance(args.format, str):
            if args.format not in _capi._FORMATS:
                parser.error(f'unsupported format {orig!r}')

    parser.add_argument('--show-empty', dest='showempty', action='store_true')
    parser.add_argument('--no-show-empty', dest='showempty', action='store_false')
    parser.set_defaults(showempty=None)

    

    parser.add_argument('--ignore', dest='ignored', action='append')
    def process_ignored(args, *, argv=None):
        ignored = []
        for raw in args.ignored or ():
            ignored.extend(raw.replace(',', ' ').strip().split())
        args.ignored = ignored or None

    parser.add_argument('filenames', nargs='*', metavar='FILENAME')
    process_progress = add_progress_cli(parser)

    return [
        process_levels,
        process_kinds,
        process_format,
        process_ignored,
        process_progress,
    ]


def cmd_capi(filenames=None, *,
             levels=None,
             kinds=None,
             groupby='kind',
             format='table',
             showempty=None,
             ignored=None,
             track_progress=None,
             verbosity=VERBOSITY,
             **kwargs
             ):
    render = _capi.get_renderer(format)

    filenames = _files.iter_header_files(filenames, levels=levels)
    
    if track_progress:
        filenames = track_progress(filenames)
    items = _capi.iter_capi(filenames)
    if levels:
        items = (item for item in items if item.level in levels)
    if kinds:
        items = (item for item in items if item.kind in kinds)

    filter = _capi.resolve_filter(ignored)
    if filter:
        items = (item for item in items if filter(item, log=lambda msg: logger.log(1, msg)))

    lines = render(
        items,
        groupby=groupby,
        showempty=showempty,
        verbose=verbosity > VERBOSITY,
    )
    print()
    for line in lines:
        print(line)


def _cli_builtin_types(parser):
    parser.add_argument('--format', dest='fmt', default='table')


    def process_format(args, *, argv=None):
        orig = args.fmt
        args.fmt = _builtin_types.resolve_format(args.fmt)
        if isinstance(args.fmt, str):
            if args.fmt not in _builtin_types._FORMATS:
                parser.error(f'unsupported format {orig!r}')

    parser.add_argument('--include-modules', dest='showmodules',
                        action='store_true')
    def process_modules(args, *, argv=None):
        pass

    return [
        process_format,
        process_modules,
    ]


def cmd_builtin_types(fmt, *,
                      showmodules=False,
                      verbosity=VERBOSITY,
                      ):
    render = _builtin_types.get_renderer(fmt)
    types = _builtin_types.iter_builtin_types()
    match = _builtin_types.resolve_matcher(showmodules)
    if match:
        types = (t for t in types if match(t, log=lambda msg: logger.log(1, msg)))

    lines = render(
        types,

    )
    print()
    for line in lines:
        print(line)





COMMANDS = {
    'check': (
        'analyze and fail if the CPython source code has any problems',
        [_cli_check],
        cmd_check,
    ),
    'analyze': (
        'report on the state of the CPython source code',
        [(lambda p: c_analyzer._cli_analyze(p, **FILES_KWARGS))],
        cmd_analyze,
    ),
    'parse': (
        'parse the CPython source files',
        [_cli_parse],
        cmd_parse,
    ),
    'data': (
        'check/manage local data (e.g. known types, ignored vars, caches)',
        [_cli_data],
        cmd_data,
    ),
    'capi': (
        'inspect the C-API',
        [_cli_capi],
        cmd_capi,
    ),
    'builtin-types': (
        'show the builtin types',
        [_cli_builtin_types],
        cmd_builtin_types,
    ),
}





def parse_args(argv=sys.argv[1:], prog=None, *, subset=None):
    import argparse
    parser = argparse.ArgumentParser(
        prog=prog or get_prog(),
    )







    processors = add_commands_cli(
        parser,
        commands=COMMANDS,
        commonspecs=[
            add_verbosity_cli,
            add_traceback_cli,
        ],
        subset=subset,
    )

    args = parser.parse_args(argv)
    ns = vars(args)

    cmd = ns.pop('cmd')

    verbosity, traceback_cm = process_args_by_key(
        args,
        argv,
        processors[cmd],
        ['verbosity', 'traceback_cm'],
    )
    if cmd != 'parse':
        
        args.verbosity = verbosity

    return cmd, ns, verbosity, traceback_cm


def main(cmd, cmd_kwargs):
    try:
        run_cmd = COMMANDS[cmd][-1]
    except KeyError:
        raise ValueError(f'unsupported cmd {cmd!r}')
    run_cmd(**cmd_kwargs)


if __name__ == '__main__':
    cmd, cmd_kwargs, verbosity, traceback_cm = parse_args()
    configure_logger(verbosity)
    with traceback_cm:
        main(cmd, cmd_kwargs)