<jupyter_start><jupyter_text>This notebook is used to rip the output scope from a VisionEval "Query-Spec.R" file.It is not compatible with all possible query specs.<jupyter_code>import emat
import os
qs_file = "./VERSPM/Query-Spec.R"

with open(qs_file, 'rt') as qs_f:
    qs = qs_f.read()

# change from R syntax to Python syntax
qs = qs.replace(" <- "," = ")
qs = qs.replace("list(","r_list(")
qs = qs.replace("c(","r_list(")
# R has named list types, Python uses seperate dicts and lists.

def r_list(*args, **kwargs):
    if args and kwargs:
        raise ValueError('all unnamed or named arguments only')
    if args:
        return list(args)
    if kwargs:
        return dict(**kwargs)
exec(qs)
PMSpecifications
pm_scope = emat.Scope(None)

filename = 'Measures_VERSPM_2010,2038_Marea=RVMPO.csv'

def add_measure(*args, **kwargs):
    try:
        pm_scope.add_measure(*args, **kwargs)
    except ValueError:
        pass

for measure in PMSpecifications:
    
    if 'Require' in measure:
        if measure.get('Require',{}).get("Table") == 'Bzone':
            continue
    
    summarize = measure.get('Summarize',{})
    name = measure['Name'].replace("_MaAz","_Az").replace("_Ma","")
    if 'BreakNames' in summarize:
        breaks = list(summarize['BreakNames'].values())[0]
        add_measure(
            f"{name}.min", 
            desc=measure['Description']+f" (up to {breaks[0]})",
            parser=dict(file=filename, loc=[f"{name}.min", 2038]),
        )
        for b0,b1 in zip(breaks[:-1], breaks[1:]):
            add_measure(
                f"{name}.{b0}", 
                desc=measure['Description']+f" ({b0} to {b1})",
                parser=dict(file=filename, loc=[f"{name}.{b0}", 2038]),
            )
        b0 = breaks[-1]
        add_measure(
            f"{name}.{b0}", 
            desc=measure['Description']+f" ({b0} and up)",
            parser=dict(file=filename, loc=[f"{name}.{b0}", 2038]),
        )
    else:
        add_measure(
            name, 
            desc=measure['Description'],
            parser=dict(file=filename, loc=[name, 2038]),
        )
print(pm_scope.dump(indent=4))
from emat.model.core_files.parsers import TableParser, MappingParser, loc, key

loc['MareaHouseholdCarSvcDvmt','2038'] - loc['MareaHouseholdCarSvcDvmt','2010']
eval("""(
loc['MareaHouseholdCarSvcDvmt','2038'] 
- loc['MareaHouseholdCarSvcDvmt','2010']
)""")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16268533772652388, 'fraction_numerical': 0.023064250411861616, 'mean_word_length': 2.968954248366013, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '221249', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 865, 'n_tokens_neox': 824, 'n_words': 172, 'path': 'verspm-scope-tool.ipynb', 'repo_name': 'tmip-emat/tmip-emat-ve', 'size': '2428'}