<commit_before>#! /usr/bin/env perl
#
#  wiki-scrub.pl
#
# Ad-hoc script to scrub wikipedia xml dumps, outputing only valid
# english-language sentences.  This  script removes wiki markup, URL's
# tables, images, & etc.  It currently seems to be pretty darned
# bullet-proof, although it might handle multi-line refs incorrectly.
#
# The output is in the form of one big single file; it should be
# straight-forward to modify this script to dump output into individual
# files.
#
# Copyright (c) 2008 <NAME> <<EMAIL>>
#

# Need to specify the binmodes, in order for \w to match utf8 chars
use utf8;
binmode STDIN, ':encoding(UTF-8)'; 
binmode STDOUT, ':encoding(UTF-8)';

$have_text = 0;
$have_infobox = 0;
$have_table = 0;
$have_ptable = 0;
$notfirst = 0;
while (<>)
{
	if (/<text xml:space/) { $have_text = 1; }
	if (/<\/text>/) { $have_text = 0; }

	chop;

	# remove the text xml
	s/.*<text xml:space="preserve">//;

	# kill redirect pages
	if (/#REDIRECT/) { $have_text = 0; next; }
	if (/#redirect/) { $have_text = 0; next; }

	# kill photo galleries
	if (/&lt;gallery&gt;/) { $have_text = 0; }
	if (/&lt;\/gallery&gt;/) { $have_text = 1; next; }

	# remove stuff that's commented out. Don't be greedy(?)!
	# do this before start of artcle processing ... <commit_msg>reset the state variables at the end of a wiki page.<commit_after>#! /usr/bin/env perl
#
#  wiki-scrub.pl
#
# Ad-hoc script to scrub wikipedia xml dumps, outputing only valid
# english-language sentences.  This  script removes wiki markup, URL's
# tables, images, & etc.  It currently seems to be pretty darned
# bullet-proof, although it might handle multi-line refs incorrectly.
#
# The output is in the form of one big single file; it should be
# straight-forward to modify this script to dump output into individual
# files.
#
# Copyright (c) 2008 <NAME> <<EMAIL>>
#

# Need to specify the binmodes, in order for \w to match utf8 chars
use utf8;
binmode STDIN, ':encoding(UTF-8)'; 
binmode STDOUT, ':encoding(UTF-8)';

$have_text = 0;
$have_infobox = 0;
$have_table = 0;
$have_ptable = 0;
$notfirst = 0;
while (<>)
{
	if (/<text xml:space/) { $have_text = 1; }

	# End of a wiki page.
	# If there are any badly-formed tables, etc. then reset the state
	# variables as we exit from a wiki page.
	if (/<\/text>/) {
		$have_text = 0;
		$have_infobox = 0;
		$have_table = 0;
		$have_ptable = 0;
		$notfirst = 0;
	}

	chop;

	# remove the text xml
	s/.*<text xml:space="preserve">//;

	# kill redirect pages
	if (/#REDIRECT/) { $have_text = 0; next; }
	if (/#redirect/) { $have_text = 0; next; }

	# kill photo galleries
	if (/&lt;gallery&gt;/) { $have_text = 0; }
	if (/&lt;\/gallery&gt;/) { $have_text = 1; next; }

	# remove stuff that's commented out. Don't be greedy(?)!
	# do this before start of artcle processing ... 
{'fraction_non_alphanumeric': 0.14484978540772533, 'fraction_numerical': 0.015021459227467811, 'mean_word_length': 3.570261437908497, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 8}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3270486', 'n_tokens_mistral': 1096, 'n_tokens_neox': 989, 'n_words': 358}