<commit_before>##################### ACHE Configuration Example #####################

#
# Configurations for Target Storage
#

# Change to false if you don't want to store pages classified as irrelevant
target_storage.store_negative_pages: false

# Configuration for data format used to store crawled data

# Enable one of the following lines to use one of the FILESYSTEM_* data format
#target_storage.data_format.type: FILESYSTEM_HTML
#target_storage.data_format.type: FILESYSTEM_CBOR
#target_storage.data_format.type: FILESYSTEM_JSON
#
# Enable this to name files using a fixed-lenght hash instead of the percent-encoded URL
#target_storage.data_format.filesystem.hash_file_name: true
#
# Enable this to compress the file content
#target_storage.data_format.filesystem.compress_data: true

# Enable this to use the FILES data format
target_storage.data_format.type: FILES
target_storage.data_format.files.max_file_size: 134217728 # 128mb in bytes


# Enable following line to index pages in Elasticsearch
#target_storage.data_format.type: ELASTICSEARCH
#
# (Transport client ES 1.5)
#target_storage.data_format.elasticsearch.host: localhost
#target_storage.data_format.elasticsearch.port: 9300
#target_storage.data_format.elasticsearch.cluster_name: elasticsearch
#
# (REST client ES 1.x and 5.x)
#target_storage.data_format.elasticsearch.rest.hosts:
#  - http://localhost:9200
#target_storage.data_format.elasticsearch.rest.connect_timeout: 30000
#target_storage.data_format.elasticsearch.rest.socket_timeout: 30000
#target_storage.data_format.elasticsearch.rest.max_retry_timeout_millis: 90000


# Instead of configuring a single data format, you can also configure multiple
# data formats in a list as follows. The settings for each data format should
# be configured independently, as if you were configuring a single data format.
# In the following config, data will be pushed to both FILES and ELASTICSEARCH.
#
#target_storage.data_formats:
#  - FILES
#  - ELASTICSEARCH


# Performs hard focus or soft focus. When hard focus is enabled,
# the crawler only follows links from pages classified as relevant
target_storage.hard_focus: true

# Run bipartite crawler
target_storage.bipartite: false

# Relevance threshold for classified pages. Pages with probability of being
# relevant above this threshold are considered relevant
target_storage.relevance_threshold: 0.5

# Maximum number of pages to visit
target_storage.visited_page_limit: 10000000

# Store only pages that contain english text using language detector
target_storage.english_language_detection_enabled: true

# Configurations for target storage's server
target_storage.server.host: localhost
target_storage.server.port: 1987

#
# Configurations for Link Storage
#

# Max number of pages to be crawled from each web domain
link_storage.max_pages_per_domain: 100
# Restricts the crawler to crawl the websites provided as seeds
link_storage.link_strategy.use_scope: false
# Allows the crawler to follow forward links
link_storage.link_strategy.outlinks: true
# Gets backlinks of the pages from a search engine used by the bipartite crawling
link_storage.link_strategy.backlinks: false<commit_msg>Removed legacy config from sample_config/ache.yml<commit_after>##################### ACHE Configuration Example #####################

#
# Configurations for Target Storage
#

# Change to false if you don't want to store pages classified as irrelevant
target_storage.store_negative_pages: false

# Configuration for data format used to store crawled data

# Enable one of the following lines to use one of the FILESYSTEM_* data format
#target_storage.data_format.type: FILESYSTEM_HTML
#target_storage.data_format.type: FILESYSTEM_CBOR
#target_storage.data_format.type: FILESYSTEM_JSON
#
# Enable this to name files using a fixed-lenght hash instead of the percent-encoded URL
#target_storage.data_format.filesystem.hash_file_name: true
#
# Enable this to compress the file content
#target_storage.data_format.filesystem.compress_data: true

# Enable this to use the FILES data format
target_storage.data_format.type: FILES
target_storage.data_format.files.max_file_size: 134217728 # 128mb in bytes


# Enable following line to index pages in Elasticsearch
#target_storage.data_format.type: ELASTICSEARCH
#
# (Transport client ES 1.5)
#target_storage.data_format.elasticsearch.host: localhost
#target_storage.data_format.elasticsearch.port: 9300
#target_storage.data_format.elasticsearch.cluster_name: elasticsearch
#
# (REST client ES 1.x and 5.x)
#target_storage.data_format.elasticsearch.rest.hosts:
#  - http://localhost:9200
#target_storage.data_format.elasticsearch.rest.connect_timeout: 30000
#target_storage.data_format.elasticsearch.rest.socket_timeout: 30000
#target_storage.data_format.elasticsearch.rest.max_retry_timeout_millis: 90000


# Instead of configuring a single data format, you can also configure multiple
# data formats in a list as follows. The settings for each data format should
# be configured independently, as if you were configuring a single data format.
# In the following config, data will be pushed to both FILES and ELASTICSEARCH.
#
#target_storage.data_formats:
#  - FILES
#  - ELASTICSEARCH


# Performs hard focus or soft focus. When hard focus is enabled,
# the crawler only follows links from pages classified as relevant
target_storage.hard_focus: true

# Run bipartite crawler
target_storage.bipartite: false

# Maximum number of pages to visit
target_storage.visited_page_limit: 10000000

# Store only pages that contain english text using language detector
target_storage.english_language_detection_enabled: true

# Configurations for target storage's server
target_storage.server.host: localhost
target_storage.server.port: 1987

#
# Configurations for Link Storage
#

# Max number of pages to be crawled from each web domain
link_storage.max_pages_per_domain: 100
# Restricts the crawler to crawl the websites provided as seeds
link_storage.link_strategy.use_scope: false
# Allows the crawler to follow forward links
link_storage.link_strategy.outlinks: true
# Gets backlinks of the pages from a search engine used by the bipartite crawling
link_storage.link_strategy.backlinks: false
{'fraction_non_alphanumeric': 0.06971115055672099, 'fraction_numerical': 0.017750524447313214, 'mean_word_length': 5.995485327313769, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6513230', 'n_tokens_mistral': 1830, 'n_tokens_neox': 1730, 'n_words': 604}