<commit_before><?php
/* Remove unwanted documents from the index. This is a bit of a hack: Ideally those would already 
 * be excluded during crawling/indexing, but sometimes this is not possible for technical reasons or for sake of simplicity.
 */
require_once('./lib/http_post/http_post.php');
include('./config.php');

print do_post_request(SOLR_URL . '/update', '<delete><query>(title:"ATTRACT | HOME") OR (title:"Page Not Found") (title:wikipedia.org/wiki/ATC_code_*) OR (id:web*) </query></delete>');
print do_post_request(SOLR_URL . '/update', '<delete><query>title:"Welcome to BestBETs"</query></delete>');
print do_post_request(SOLR_URL . '/update', '<delete><query>title:(Articles AND "Medscape Reference")</query></delete>');

print do_post_request(SOLR_URL . '/update', '<commit/>');
print do_post_request(SOLR_URL . '/update', '<optimize/>');
	
?>
}<commit_msg>fix of issue #12 (ATC codes in Wikipedia)<commit_after><?php
/* Remove unwanted documents from the index. This is a bit of a hack: Ideally those would already 
 * be excluded during crawling/indexing, but sometimes this is not possible for technical reasons or for sake of simplicity.
 */
require_once('./lib/http_post/http_post.php');
include('./config.php');

print do_post_request(SOLR_URL . '/update', '<delete><query>(title:"ATTRACT | HOME") OR (title:"Page Not Found") '
        . 'OR (id:*wikipedia.org/wiki/ATC_code_*) OR (id:*wikipedia.org/wiki/ATCvet_code_*) OR (id:web*) </query></delete>');
print do_post_request(SOLR_URL . '/update', '<delete><query>title:"Welcome to BestBETs"</query></delete>');
print do_post_request(SOLR_URL . '/update', '<delete><query>title:(Articles AND "Medscape Reference")</query></delete>');

print do_post_request(SOLR_URL . '/update', '<commit/>');
print do_post_request(SOLR_URL . '/update', '<optimize/>');
	
?>
}
{'fraction_non_alphanumeric': 0.17412663755458516, 'fraction_numerical': 0.001091703056768559, 'mean_word_length': 5.763837638376383, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2542268', 'n_tokens_mistral': 592, 'n_tokens_neox': 562, 'n_words': 154}