<commit_before>#!/usr/bin/perl -w
use strict;

my $usage = qq{add-features.pl [-f dict] < input
Add features from dict for each word on the left side of the two column input.
-f file that contains a word and one or more features on each line.
};

use Getopt::Std;
our($opt_f);
getopts('f:');
die $usage unless $opt_f;
$opt_f = "zcat $opt_f |" if $opt_f =~ /\.gz$/;

my %feat;
open(FP, $opt_f) or die $!;
while(<FP>) {
    my @a = split;
    my $w = shift(@a);
    $feat{$w}{$_}++ for @a;
}
close(FP);

my $prev;
while(<>) {
    my ($w, $x) = split;
    next if $w eq '</s>';
    print;
    if (defined $feat{$w}) {
	for my $f (keys %{$feat{$w}}) {
	    if (($f eq '/INITCAP1/') and
		((not defined $prev) or
		 ($prev =~ /^[.`'?!:;]+$/) or
		 ($prev =~ /^-[LR][RC]B-$/))) {
		next;
	    }
	    print "$w\t$f\n";
	}
    }
    $prev = $w;
}<commit_msg>*** empty log message ***<commit_after>#!/usr/bin/perl -w
use strict;

my $usage = qq{add-features.pl [-f dict] < input
Add features from dict for each word on the left side of the two column input.
-f file that contains a word and one or more features on each line.
};

use Getopt::Std;
our($opt_f);
getopts('f:');
die $usage unless $opt_f;
$opt_f = "zcat $opt_f |" if $opt_f =~ /\.gz$/;

my %feat;
open(FP, $opt_f) or die $!;
while(<FP>) {
    my @a = split;
    my $w = shift(@a);
    $feat{$w}{$_}++ for @a;
}
close(FP);

my $prev;
while(<>) {
    my ($w, $x) = split;
    next if $w eq '</s>';
    print;
    if (defined $feat{$w}) {
	for my $f (keys %{$feat{$w}}) {
	    if (($f eq '/IC/') and
		((not defined $prev) or
		 ($prev =~ /^[.`'?!:;]+$/) or
		 ($prev =~ /^-[LR][RC]B-$/))) {
		next;
	    }
	    print "$w\t$f\n";
	}
    }
    $prev = $w;
}
{'fraction_non_alphanumeric': 0.23563892145369286, 'fraction_numerical': 0.0005861664712778429, 'mean_word_length': 2.735229759299781, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '896484', 'n_tokens_mistral': 755, 'n_tokens_neox': 717, 'n_words': 224}