<commit_before>#
# Example of configuration for running a Focused Crawl
#
target_storage.use_classifier: true
target_storage.store_negative_pages: false
target_storage.data_format.type: FILESYSTEM_JSON
target_storage.data_format.filesystem.compress_data: true
target_storage.english_language_detection_enabled: false

link_storage.max_pages_per_domain: 1000
link_storage.link_strategy.use_scope: false
link_storage.link_strategy.outlinks: true
link_storage.link_strategy.backlinks: false
link_storage.link_classifier.type: LinkClassifierBaseline
link_storage.online_learning.enabled: true
link_storage.online_learning.type: FORWARD_CLASSIFIER_BINARY
link_storage.online_learning.learning_limit: 1000
link_storage.link_selector: TopkLinkSelector
link_storage.scheduler.host_min_access_interval: 5000
link_storage.scheduler.max_links: 10000

crawler_manager.downloader.user_agent.name: ACHE
crawler_manager.downloader.user_agent.url: https://github.com/ViDA-NYU/ache<commit_msg>Changed value for store_negative_pages in focused crawl sample config<commit_after>#
# Example of configuration for running a Focused Crawl
#
target_storage.use_classifier: true
target_storage.store_negative_pages: true
target_storage.data_format.type: FILESYSTEM_JSON
target_storage.data_format.filesystem.compress_data: true
target_storage.english_language_detection_enabled: false

link_storage.max_pages_per_domain: 1000
link_storage.link_strategy.use_scope: false
link_storage.link_strategy.outlinks: true
link_storage.link_strategy.backlinks: false
link_storage.link_classifier.type: LinkClassifierBaseline
link_storage.online_learning.enabled: true
link_storage.online_learning.type: FORWARD_CLASSIFIER_BINARY
link_storage.online_learning.learning_limit: 1000
link_storage.link_selector: TopkLinkSelector
link_storage.scheduler.host_min_access_interval: 5000
link_storage.scheduler.max_links: 10000

crawler_manager.downloader.user_agent.name: ACHE
crawler_manager.downloader.user_agent.url: https://github.com/ViDA-NYU/ache
{'fraction_non_alphanumeric': 0.06477329347284504, 'fraction_numerical': 0.016940707523667164, 'mean_word_length': 9.854054054054053, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5145102', 'n_tokens_mistral': 662, 'n_tokens_neox': 624, 'n_words': 97}