<commit_before>import json

def get_terms(marcframe):
    terms = set()
    for k, v in marcframe['entityTypeMap'].items():
        terms.add(k)
        terms.update(v.get('instanceTypes', []))
    dfn_keys = {'property', 'addProperty', 'link', 'addLink', 'domainEntity', 'rangeEntity'}
    def add_terms(dfn):
        for k, v in dfn.items():
            if k in dfn_keys:
                terms.add(v)
            elif isinstance(v, dict):
                add_terms(v)
            elif k == 'defaults':
                terms.update(v)
    for part in ['bib', 'auth', 'hold']:
        for field in marcframe[part].values():
            add_terms(field)
    return terms

if __name__ == '__main__':
    import sys
    source = sys.argv[1]
    with open(source) as fp:
        marcframe = json.load(fp)
    terms = get_terms(marcframe)
    for term in sorted(terms):
        print term<commit_msg>Find missing definitions using terms_from_marcframe.py by providing context data<commit_after>import json

def get_terms(marcframe):
    terms = set()
    for k, v in marcframe['entityTypeMap'].items():
        terms.add(k)
        terms.update(v.get('instanceTypes', []))
    dfn_keys = {'property', 'addProperty', 'link', 'addLink', 'domainEntity', 'rangeEntity'}
    def add_terms(dfn):
        for k, v in dfn.items():
            if k in dfn_keys and v:
                terms.add(v)
            elif isinstance(v, dict):
                add_terms(v)
            elif k == 'defaults':
                terms.update(v)
    for part in ['bib', 'auth', 'hold']:
        for field in marcframe[part].values():
            add_terms(field)
    return terms

if __name__ == '__main__':
    import sys
    args = sys.argv[1:]
    source = args.pop(0)
    context_source = args.pop(0) if args else None

    with open(source) as fp:
        marcframe = json.load(fp)

    terms = get_terms(marcframe)

    if context_source:
        with open(context_source) as fp:
            contexts = json.load(fp)['@context']
            if not isinstance(contexts, list):
                contexts = [contexts]
        context_terms = {term for defs in contexts for term in defs if term[0] != '@'}
        print "# terms missing in context:"
        terms -= context_terms

    for term in sorted(terms):
        print term
{'fraction_non_alphanumeric': 0.12337098175499565, 'fraction_numerical': 0.002172024326672459, 'mean_word_length': 2.2164804469273744, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2260336', 'n_tokens_mistral': 734, 'n_tokens_neox': 704, 'n_words': 198}