<commit_before>; This file provides a centralized configuration mechanism for those wishing to
; use sorter.py and crawler.py without tapping into the API. The configuration
; options in this file are still intended to be powerful enough to allow one to
; conduct website fingerprinting research on a completely different site or set
; of sites than SecureDrop.

[sorter]
; Space delimited list of hidden service directories
onion_dirs = http://secrdrop5wyphb5x.onion/sites/securedrop.org/files/securedrop_list.txt http://msydqstlz2kzerdg.onion/address/ http://skunksworkedp2cg.onion/sites.html
; Time to wait for a onion service to load before timing out
page_load_timeout = 20
; Number of asynchronous connections to have open at a time. The higher this
; value is, the faster the sorter works, but if set too high, connection errors
; may increase significantly.
max_tasks = 10
; The Sorter sorts sites based on user-defined boolean expressions that operate
; on the HTML body of pages. Create %%-delimited dictionaries, where the key is
; the class name and the value is some boolean expression that operates on the
; object reference `text`. Order matters; a site will be sorted into the first
; class for which the boolean expression evaluates to true.
class_tests = {"sd_038": "'Powered by SecureDrop 0.3.8.' in text"}%%{"non_monitored": "'SecureDrop' not in text"}
; Do you want this data to be inserted into the database?
use_database = true

[crawler]
; Whether the crawler should use the database
use_database = true
; How far back in our sorting history should the crawler look,
; accepts integer and single character unit: '4w', '1d', '1h'
; This only works if you use the database (e.g., if we scraped/"sorted" every
; day last week, we might want to try a site that was up on W but not Th).
hs_history_lookback = 1w
; If use_database is set to false then the crawler will use a pickle file
class_data = class-data_07-28_18:15:40.pickle
; Time to wait for a onion service to load before timing out
page_load_timeout = 20
; Time to wait after a page has succesfully loaded to catch traffic after
; initial onload event
wait_on_page = 5
; Time to wait between closing all open circuits and starting collection of the
; next trace
wait_after_closing_circuits = 5
; Experimental: whether or not to restart tor in the case that an exception
; known to crash the Crawler is encountered. Still not confirmed if this
; prevents Crawler crash.<commit_msg>Bump SecureDrop version to 0.3.9<commit_after>; This file provides a centralized configuration mechanism for those wishing to
; use sorter.py and crawler.py without tapping into the API. The configuration
; options in this file are still intended to be powerful enough to allow one to
; conduct website fingerprinting research on a completely different site or set
; of sites than SecureDrop.

[sorter]
; Space delimited list of hidden service directories
onion_dirs = http://secrdrop5wyphb5x.onion/sites/securedrop.org/files/securedrop_list.txt http://msydqstlz2kzerdg.onion/address/ http://skunksworkedp2cg.onion/sites.html
; Time to wait for a onion service to load before timing out
page_load_timeout = 20
; Number of asynchronous connections to have open at a time. The higher this
; value is, the faster the sorter works, but if set too high, connection errors
; may increase significantly.
max_tasks = 10
; The Sorter sorts sites based on user-defined boolean expressions that operate
; on the HTML body of pages. Create %%-delimited dictionaries, where the key is
; the class name and the value is some boolean expression that operates on the
; object reference `text`. Order matters; a site will be sorted into the first
; class for which the boolean expression evaluates to true.
class_tests = {"sd_039": "'Powered by SecureDrop 0.3.9.' in text"}%%{"non_monitored": "'SecureDrop' not in text"}
; Do you want this data to be inserted into the database?
use_database = true

[crawler]
; Whether the crawler should use the database
use_database = true
; How far back in our sorting history should the crawler look,
; accepts integer and single character unit: '4w', '1d', '1h'
; This only works if you use the database (e.g., if we scraped/"sorted" every
; day last week, we might want to try a site that was up on W but not Th).
hs_history_lookback = 1w
; If use_database is set to false then the crawler will use a pickle file
class_data = class-data_07-28_18:15:40.pickle
; Time to wait for a onion service to load before timing out
page_load_timeout = 20
; Time to wait after a page has succesfully loaded to catch traffic after
; initial onload event
wait_on_page = 5
; Time to wait between closing all open circuits and starting collection of the
; next trace
wait_after_closing_circuits = 5
; Experimental: whether or not to restart tor in the case that an exception
; known to crash the Crawler is encountered. Still not confirmed if this
; prevents Crawler crash.
{'fraction_non_alphanumeric': 0.05790645879732739, 'fraction_numerical': 0.01356549908888439, 'mean_word_length': 4.632839224629419, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2433097', 'n_tokens_mistral': 1416, 'n_tokens_neox': 1321, 'n_words': 707}