<commit_before>import json

class ETK(object):

    def __init__(self):
        pass

    def invoke_extractor(self, extractor=None, doc=None, json_path=None, input_key=None, output_key=None):

        containers = doc.cdr_document['__content_strict']
        # containers = doc.select_containers(json_path)
        if isinstance(containers, list):
            for c in containers:
                segment = c.get(input_key)
                tokens = doc.get_tokens(segment)
                for i in tokens:
                    print(i.orth_)
        else:
            segment = containers.get(input_key)
            tokens = doc.get_tokens(segment)
            for i in tokens:
                print(i.orth_)

        fake_extraction = doc.tokenizer.reconstruct_text(tokens)
        doc.store_extraction(extractor, fake_extraction, containers, output_key)
        print(json.dumps(doc.cdr_document, indent=2))
            # if extractor.requires_tokens():
            #     tokens = doc.get_tokens(segment, tokenizer=extractor.preferred_tokenizer())
            #     if tokens:
            #         extraction = extractor.extract(tokens, doc)
            #         doc.store_extraction(extractor, extraction, c, output_key)<commit_msg>remove print lines<commit_after>import json

class ETK(object):

    def __init__(self):
        pass

    def invoke_extractor(self, extractor=None, doc=None, json_path=None, input_key=None, output_key=None):

        containers = doc.cdr_document['__content_strict']
        # containers = doc.select_containers(json_path)
        if isinstance(containers, list):
            for c in containers:
                segment = c.get(input_key)
                tokens = doc.get_tokens(segment)
        else:
            segment = containers.get(input_key)
            tokens = doc.get_tokens(segment)

        fake_extraction = doc.tokenizer.reconstruct_text(tokens)
        doc.store_extraction(extractor, fake_extraction, containers, output_key)
        print(json.dumps(doc.cdr_document, indent=2))
            # if extractor.requires_tokens():
            #     tokens = doc.get_tokens(segment, tokenizer=extractor.preferred_tokenizer())
            #     if tokens:
            #         extraction = extractor.extract(tokens, doc)
            #         doc.store_extraction(extractor, extraction, c, output_key)
{'fraction_non_alphanumeric': 0.0918757975329647, 'fraction_numerical': 0.0008507018290089324, 'mean_word_length': 2.3173483779971793, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3541154', 'n_tokens_mistral': 674, 'n_tokens_neox': 655, 'n_words': 123}