<commit_before>import json
from etk.etk import ETK
from etk.extractors.glossary_extractor import GlossaryExtractor

sample_input = {
    "projects": [
        {
            "name": "etk",
            "description": "version 2 of etk, implemented by Runqi, Dongyu, Sylvia and others."
        }
    ]
}

etk = ETK()
doc = etk.create_document(sample_input)

name_extractor = GlossaryExtractor(etk.load_glossary("./names.txt"))

description_segments = doc.select_segments("projects[*].description")
names = doc.invoke_extractor(name_extractor, description_segments)

root_segment = doc.select_segments("$").items()[0]
root_segment.store_extractions(names, "all_names")

print(json.dumps(sample_input, indent=2))
<commit_msg>refactor example<commit_after>import json
from etk.etk import ETK
from etk.extractors.glossary_extractor import GlossaryExtractor

sample_input = {
    "projects": [
        {
            "name": "etk",
            "description": "version 2 of etk, implemented by Runqi, Dongyu, Sylvia and others."
        }
    ]
}

etk = ETK()
doc = etk.create_document(sample_input)

name_extractor = GlossaryExtractor(etk.load_glossary("./names.txt"))

descriptions = doc.select_segments("projects[*].description")
projects = doc.select_segments("projects[*]")
for d, p in zip(descriptions, projects):
    names = doc.invoke_extractor(name_extractor, input)
    p.store_extractions(names, "members")

print(json.dumps(sample_input, indent=2))
{'fraction_non_alphanumeric': 0.12405237767057202, 'fraction_numerical': 0.0034458993797381117, 'mean_word_length': 4.479245283018868, 'pattern_counts': {'":': 6, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4535695', 'n_tokens_mistral': 494, 'n_tokens_neox': 482, 'n_words': 93}