<commit_before>

models: \
		dewiki_models \
		enwiki_models \
		eswiki_models \
		etwiki_models \
		fawiki_models \
		frwiki_models \
		hewiki_models \
		idwiki_models \
		itwiki_models \
		nlwiki_models \
		ptwiki_models \
		trwiki_models \
		ukwiki_models \
		viwiki_models \
		wikidatawiki_models
		#jawiki_models
		#ruwiki_models
		#urwiki_models

tuning_reports: \
		dewiki_tuning_reports \
		enwiki_tuning_reports \
		eswiki_tuning_reports \
		etwiki_tuning_reports \
		fawiki_tuning_reports \
		frwiki_tuning_reports \
		hewiki_tuning_reports \
		idwiki_tuning_reports \
		itwiki_tuning_reports \
		nlwiki_tuning_reports \
		ptwiki_tuning_reports \
		trwiki_tuning_reports \
		ukwiki_tuning_reports \
		viwiki_tuning_reports \
		wikidatawiki_tuning_reports
		#jawiki_tuning_reports
		#ruwiki_tuning_reports
		#urwiki_tuning_reports

test_statistics = \
		-s 'table' -s 'accuracy' -s 'precision' -s 'recall' \
		-s 'pr' -s 'roc' \
		-s 'recall_at_fpr(max_fpr=0.10)' \
		-s 'filter_rate_at_recall(min_recall=0.90)' \
		-s 'filter_rate_at_recall(min_recall=0.75)'

############################# Arabic Wikipedia ################################

datasets/arwiki.sampled_revisions.20k_2016.tsv:
	wget -qO- http://quarry.wmflabs.org/run/65713/output/0/tsv?download=true > \
	datasets/arwiki.sampled_revisions.20k_2016.tsv<commit_msg>Adds arwiki and plwiki models to the main rules in Makefile<commit_after>

models: \
		arwiki_models \
		dewiki_models \
		enwiki_models \
		eswiki_models \
		etwiki_models \
		fawiki_models \
		frwiki_models \
		hewiki_models \
		idwiki_models \
		itwiki_models \
		nlwiki_models \
		plwiki_models \
		ptwiki_models \
		trwiki_models \
		ukwiki_models \
		viwiki_models \
		wikidatawiki_models
		#jawiki_models
		#ruwiki_models
		#urwiki_models

tuning_reports: \
		arwiki_tuning_reports \
		dewiki_tuning_reports \
		enwiki_tuning_reports \
		eswiki_tuning_reports \
		etwiki_tuning_reports \
		fawiki_tuning_reports \
		frwiki_tuning_reports \
		hewiki_tuning_reports \
		idwiki_tuning_reports \
		itwiki_tuning_reports \
		nlwiki_tuning_reports \
		plwiki_tuning_reports \
		ptwiki_tuning_reports \
		trwiki_tuning_reports \
		ukwiki_tuning_reports \
		viwiki_tuning_reports \
		wikidatawiki_tuning_reports
		#jawiki_tuning_reports
		#ruwiki_tuning_reports
		#urwiki_tuning_reports

test_statistics = \
		-s 'table' -s 'accuracy' -s 'precision' -s 'recall' \
		-s 'pr' -s 'roc' \
		-s 'recall_at_fpr(max_fpr=0.10)' \
		-s 'filter_rate_at_recall(min_recall=0.90)' \
		-s 'filter_rate_at_recall(min_recall=0.75)'

############################# Arabic Wikipedia ################################

datasets/arwiki.sampled_revisions.20k_2016.tsv:
	wget -qO- http://quarry.wmflabs.org/run/65713/output/0/tsv?download=true > \
	datasets/arwiki.sampled_revisions.20k_2016.tsv
{'fraction_non_alphanumeric': 0.1240199572344975, 'fraction_numerical': 0.01924447612259444, 'mean_word_length': 4.09437386569873, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '385662', 'n_tokens_mistral': 1293, 'n_tokens_neox': 1124, 'n_words': 143}