<commit_before>#! /usr/bin/perl

# Attributes
$bold = "\033[1m";

# Colors
$white  = "\033[37m";
$yellow  = "\033[33m";
$magenta  = "\033[35m";
$blue  = "\033[34m";
$red  = "\033[31m";
$reset = "\033[0m";

# Combinations
$info   = $white.$bold;
$warn   = $yellow.$bold;
$error  = $red.$bold;
$file   = $magenta.$bold;
$lineno = $blue.$bold;

while(<STDIN>)
{
     $line = $_;
     chomp $line;
     # Skip entering/leaving directories and incomplete lines
     if(
	$line =~ /make\[([0-9]*)\]:.*/ ||
#       $line =~ /.*\s\\$/ ||
        $line =~ /^test\s\-z\s/ ||
        $line =~ /^Making\sclean\sin\s\./ ||
	$line =~ /then\smv\s-f/ ||
	$line =~ /.*make\s\s.*/ ||
        $line =~ /make\s\sall-recursive/ ||
        $line =~ /[A-z0-9-]*ar\s[A-z0-9]*\s([A-z0-9\-_\/\.]*)\s.*/ ||
        $line =~ /^[A-z0-9-]*ranlib\s[A-z0-9-_]*plugin(.*)/ ||
        $line =~ /^touch.*/ ||
        $line =~ /^srcdir=.*/ ||
        $line =~ /^.* (lib[A-z0-9-_]*plugin.so).*/ ||
	$line =~ /^\s*gcc\s-std=.*/ ||
        $line =~ /^\sgcc\s-mmacosx.*/ ||
        $line =~ /^\sg\+\+\s.*/ ||
#    $line =~ /^.*moc.*/ ||
	$line =~ /^.*libtool.*\-o\s(lib.*\.la).*/ ||
        $line =~ /^.*rm\s\-f\s(.*)/ ||
	$line =~ /^rm\s-fr\s(.*)/ ||
	$line =~ /^mv\s-f\s(.*)/ ||
	$line =~ /^ln\s-s\s(.*)/ ||
	$line =~ /^echo\s/ ||
	$line =~ /^mkdir\s/ ||
	$line =~ /^cat\s/ ||
	$line =~ /^grep\s/ ||
	$line =~ /^cd\s/ ||
	$line =~ /^sed\s/ ||
	$line =~ /^bindir=\s/ ||
	$line =~ /^libtool:\s/ ||
	$line =~ /^creating lib.*/ ||
        $line =~ /^\s*\// )
     {}
     # Info<commit_msg>make.pl: matches gcc-xx and g++-xx<commit_after>#! /usr/bin/perl

# Attributes
$bold = "\033[1m";

# Colors
$white  = "\033[37m";
$yellow  = "\033[33m";
$magenta  = "\033[35m";
$blue  = "\033[34m";
$red  = "\033[31m";
$reset = "\033[0m";

# Combinations
$info   = $white.$bold;
$warn   = $yellow.$bold;
$error  = $red.$bold;
$file   = $magenta.$bold;
$lineno = $blue.$bold;

while(<STDIN>)
{
     $line = $_;
     chomp $line;
     # Skip entering/leaving directories and incomplete lines
     if(
	$line =~ /make\[([0-9]*)\]:.*/ ||
#       $line =~ /.*\s\\$/ ||
        $line =~ /^test\s\-z\s/ ||
        $line =~ /^Making\sclean\sin\s\./ ||
	$line =~ /then\smv\s-f/ ||
	$line =~ /.*make\s\s.*/ ||
        $line =~ /make\s\sall-recursive/ ||
        $line =~ /[A-z0-9-]*ar\s[A-z0-9]*\s([A-z0-9\-_\/\.]*)\s.*/ ||
        $line =~ /^[A-z0-9-]*ranlib\s[A-z0-9-_]*plugin(.*)/ ||
        $line =~ /^touch.*/ ||
        $line =~ /^srcdir=.*/ ||
        $line =~ /^.* (lib[A-z0-9-_]*plugin.so).*/ ||
	$line =~ /^\s*gcc(-.*)?\s-std=.*/ ||
        $line =~ /^\sgcc(-.*)?\s-mmacosx.*/ ||
        $line =~ /^\sg\+\+(-.*)?\s.*/ ||
#    $line =~ /^.*moc.*/ ||
	$line =~ /^.*libtool.*\-o\s(lib.*\.la).*/ ||
        $line =~ /^.*rm\s\-f\s(.*)/ ||
	$line =~ /^rm\s-fr\s(.*)/ ||
	$line =~ /^mv\s-f\s(.*)/ ||
	$line =~ /^ln\s-s\s(.*)/ ||
	$line =~ /^echo\s/ ||
	$line =~ /^mkdir\s/ ||
	$line =~ /^cat\s/ ||
	$line =~ /^grep\s/ ||
	$line =~ /^cd\s/ ||
	$line =~ /^sed\s/ ||
	$line =~ /^bindir=\s/ ||
	$line =~ /^libtool:\s/ ||
	$line =~ /^creating lib.*/ ||
        $line =~ /^\s*\// )
     {}
     # Info
{'fraction_non_alphanumeric': 0.3521531100478469, 'fraction_numerical': 0.029984051036682616, 'mean_word_length': 1.8201438848920863, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4980357', 'n_tokens_mistral': 1631, 'n_tokens_neox': 1561, 'n_words': 211}