<commit_before>#! /usr/bin/env perl
#
# This file just stripes out the sentences from the semcor files.

# Loop -- read from stdin, and put quotes around the attributes.
$leading_quote = 0;
while(<>)
{
	chop;
	s/<[\/\:\$\'\w\s=-]+>//g;  # strip out all xml markup

	s/_/ /g;  # replace underscores.

	s/\'\'/\"/g;
	s/\`\`/\"/g;

	if ((!(/^[\'\"\,\.]/)) && (leading_quote == 0))
	{ 
		print " ";
	}
	print $_;
	if (/\./) { print "\n"; }

	$leading_quote = 0;
	if (/^\"/) { $leading_quote = 1; }

}

<commit_msg>more layout fixes<commit_after>#! /usr/bin/env perl
#
# This utility just strips out the sentences from the semcor files.
#

# Loop -- read from stdin, and put quotes around the attributes.
$leading_quote = 0;
while(<>)
{
	chop;
	s/<[\/\:\$\'\(\)\w\s=-]+>//g;  # strip out all xml markup

	s/^ //;
	s/_/ /g;  # replace underscores.

	s/\'\'/\"/g;
	s/\`\`/\"/g;

	if (!($_)) { next; }

	if ((!(/^[\'\"\,\.]/)) && ($leading_quote == 0))
	{ 
		print " ";
	}

	$leading_quote = 0;
	if (/^\"/) { $leading_quote = 1; }

	print $_;
	if (/\?/) {print " "; }
	if (/\./) { print "\n"; $leading_quote = 1; }


}


{'fraction_non_alphanumeric': 0.2715827338129496, 'fraction_numerical': 0.008093525179856115, 'mean_word_length': 2.3125, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3369535', 'n_tokens_mistral': 510, 'n_tokens_neox': 465, 'n_words': 115}