<commit_before>#! /usr/bin/perl -w

# List source element from a shex schema.
# Reads stdin; writes stdout.

my $all = join("", <>);


# Strip everything up through the first brace:
# <http://hokukahu.com/Absent_Status-8131> {
$all =~ s/\A[^\{\}]*\{//ms;
# Strip everything from the last brace to the end:
$all =~ s/\}[^\{\}]*\Z//ms;

# Delete everything between braces, in order to leave only
# top-level properties:
while (1) {
	my $old = $all;
	$all =~ s/\{[^\{\}]*\}//ms;
	last if $old eq $all;
	}
# print "$all\n";

# Collect the words into a hash, for uniqueness
my %words = ();
while ($all =~ s/\w+\:[\w\-]+//) {
	my $w = $&;
	# Skip datatypes:
	next if $w =~ m/^xsd\:/;
	$words{$w} = 1;
	# warn "word: $w\n";
	}

# Sort and output:
my @words = sort keys %words;
my $words = join("\n", @words);
print "$words\n";

exit 0;

<commit_msg>Create CSV for all shex files<commit_after>#! /usr/bin/perl -w

# List source elements from shex schemas.  Generates TSV.
# Use like this:
# ./grep-source-properties.perl  ../src/shex/cmumps/common/*.shex > /tmp/properties.csv

# Header line:
# print "CHCS Class Name\tCHCS Attributes Mapped to Common Schema\n";
# CSV instead of TSV:
print "CHCS Class Name,CHCS Attributes Mapped to Common Schema\n";

foreach my $f ( @ARGV ) {
	open(my $fh, "<$f") || die;
	my $all = join("", <$fh>);

	# Strip everything up through the first brace:
	# <http://hokukahu.com/Absent_Status-8131> {
	$all =~ s/\A[^\{\}]*\{//ms;
	# Strip everything from the last brace to the end:
	$all =~ s/\}[^\{\}]*\Z//ms;

	# Delete everything between braces, in order to leave only
	# top-level properties:
	while (1) {
		my $old = $all;
		$all =~ s/\{[^\{\}]*\}//ms;
		last if $old eq $all;
		}
	# print "$all\n";

	# Collect the words into a hash, for uniqueness
	my %words = ();
	while ($all =~ s/\w+\:[\w\-]+//) {
		my $w = $&;
		# Skip datatypes:
		next if $w =~ m/^xsd\:/;

		# Strip the prefix:
		$w =~ s/^\w+\://;

		# Skip cmumps_type
		next if $w eq "cmumps_type";

		# Save it (uniquely):
		$words{$w} = 1;
		# warn "word: $w\n";
		}

	# Shorten the filename for display. Strip the path:
	$f =~ s/^.*\///;
	# and the extension:
	$f =~ s/\.[^\.]*$//;

	# Sort and output:
	my @words = sort keys %words;
	my $words = join(", ", @words);
	# print "$f\t$words\n";
	# CSV instead of TSV:
	print "$f,\"$words\"\n";
	}

exit 0;


{'fraction_non_alphanumeric': 0.21748400852878466, 'fraction_numerical': 0.005970149253731343, 'mean_word_length': 2.923076923076923, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2339603', 'n_tokens_mistral': 1001, 'n_tokens_neox': 947, 'n_words': 287}