<commit_before>import os, sys, json, codecs
sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
sys.path.append(os.path.join(os.path.dirname(__file__), '../..'))

from etk.etk import ETK
from etk.extractors.html_content_extractor import HTMLContentExtractor, Strategy
from etk.extractors.html_metadata_extractor import HTMLMetadataExtractor


sample_html = json.load(codecs.open('sample_html.json', 'r')) # read sample file from disk


etk = ETK()
doc = etk.create_document(sample_html, mime_type="text/html", url="http://ex.com/123")

metadata_extractor = HTMLMetadataExtractor()
content_extractor = HTMLContentExtractor()

root = doc.select_segments("$")[0]

# Passing arguments to extractors using keyword arguments in invoke_extractor is causing warnings.
# Is there a pythonic way to do this?
# **options are defined in invoke_extractor() to pass the arguments to extractor.extract(), is that ok?
# root.store_extractions(doc.invoke_extractor(metadata_extractor, extract_title=True), "title")
# root.store_extractions(doc.invoke_extractor(metadata_extractor, extract_meta=True), "metadata")
root.store_extractions(doc.invoke_extractor(content_extractor, strategy=Strategy.ALL_TEXT), "etk2_text")
root.store_extractions(doc.invoke_extractor(metadata_extractor), "etk2_metadata")


print(json.dumps(doc.cdr_document, indent=2))<commit_msg>add content strict and relaxed to example<commit_after>import os, sys, json, codecs
sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
sys.path.append(os.path.join(os.path.dirname(__file__), '../..'))

from etk.etk import ETK
from etk.extractors.html_content_extractor import HTMLContentExtractor, Strategy
from etk.extractors.html_metadata_extractor import HTMLMetadataExtractor


sample_html = json.load(codecs.open('sample_html.json', 'r')) # read sample file from disk


etk = ETK()
doc = etk.create_document(sample_html, mime_type="text/html", url="http://ex.com/123")

metadata_extractor = HTMLMetadataExtractor()
content_extractor = HTMLContentExtractor()

root = doc.select_segments("$")[0]

# Passing arguments to extractors using keyword arguments in invoke_extractor is causing warnings.
# Is there a pythonic way to do this?
# **options are defined in invoke_extractor() to pass the arguments to extractor.extract(), is that ok?
# root.store_extractions(doc.invoke_extractor(metadata_extractor, extract_title=True), "title")
# root.store_extractions(doc.invoke_extractor(metadata_extractor, extract_meta=True), "metadata")
root.store_extractions(doc.invoke_extractor(content_extractor, strategy=Strategy.ALL_TEXT), "etk2_text")
root.store_extractions(doc.invoke_extractor(content_extractor, strategy=Strategy.MAIN_CONTENT_STRICT), "etk2_content_strict")
root.store_extractions(doc.invoke_extractor(content_extractor, strategy=Strategy.MAIN_CONTENT_RELAXED), "etk2_content_relaxed")
root.store_extractions(doc.invoke_extractor(metadata_extractor), "etk2_metadata")


print(json.dumps(doc.cdr_document, indent=2))
{'fraction_non_alphanumeric': 0.11914323962516733, 'fraction_numerical': 0.00535475234270415, 'mean_word_length': 7.975975975975976, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2263593', 'n_tokens_mistral': 963, 'n_tokens_neox': 951, 'n_words': 185}