<commit_before>#!/usr/bin/perl

open (IN,"ktest.pl");
while (<IN>) {
    if (/\$opt\{"?([A-Z].*?)(\[.*\])?"?\}/ ||
	/set_test_option\("(.*?)"/) {
	$opt{$1} = 1;
    }
}
close IN;

open (IN, "sample.conf");
while (<IN>) {
    if (/^\s*#?\s*(\S+)\s*=/) {
	$samp{$1} = 1;
    }
}
close IN;

foreach $opt (keys %opt) {
    if (!defined($samp{$opt})) {
	print "opt = $opt\n";
    }
}

foreach $samp (keys %samp) {
    if (!defined($opt{$samp})) {
	print "samp = $samp\n";
    }
}
<commit_msg>ktest: Fix compare script to test if options are not documented<commit_after>#!/usr/bin/perl

open (IN,"ktest.pl");
while (<IN>) {
    # hashes are now used
    if (/\$opt\{"?([A-Z].*?)(\[.*\])?"?\}/ ||
	/^\s*"?([A-Z].*?)"?\s*=>\s*/ ||
	/set_test_option\("(.*?)"/) {
	$opt{$1} = 1;
    }
}
close IN;

open (IN, "sample.conf");
while (<IN>) {
    if (/^\s*#?\s*([A-Z]\S*)\s*=/) {
	$samp{$1} = 1;
    }
}
close IN;

foreach $opt (keys %opt) {
    if (!defined($samp{$opt})) {
	print "opt = $opt\n";
    }
}

foreach $samp (keys %samp) {
    if (!defined($opt{$samp})) {
	print "samp = $samp\n";
    }
}

{'fraction_non_alphanumeric': 0.32169117647058826, 'fraction_numerical': 0.007352941176470588, 'mean_word_length': 2.7422680412371134, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '242582', 'n_tokens_mistral': 533, 'n_tokens_neox': 509, 'n_words': 97}