<commit_before>#!/usr/bin/env perl 
#===============================================================================
#
#         FILE: wc_stop.pl
#
#        USAGE: ./wc_stop.pl  
#
#  DESCRIPTION: 
#
#      OPTIONS: ---
# REQUIREMENTS: ---
#         BUGS: ---
#        NOTES: ---
#       AUTHOR: <NAME> (agoldst), <EMAIL>
# ORGANIZATION: Rutgers University, New Brunswick
#      VERSION: 1.0
#      CREATED: 12/04/2012 07:59:30
#     REVISION: ---
#===============================================================================
use v5.14;                                  # entails strict, unicode_strings 
use autodie;
use utf8;                                   # source code itself is in utf-8
use warnings;
use warnings FATAL => "utf8";               # Unicode encode errors are fatal
use open qw( :std :utf8 );                  # default utf8 layer

my $USAGE = <<EOM;
Usage:
wc_stop -s stoplist_file file1 file2 file3 file4...
EOM

my $first = shift;
unless $first eq "-s" {
    say $USAGE;
    exit;
}
my $stoplist = shift;
open STOP, $stoplist or die;
my %STOPLIST = ();
while(<STOP>) {
    chomp;
    $STOPLIST{$_} = 1; 
}
close STOP;

my $count;
foreach(@ARGV) {
    open my $fh, "$_" or die;
    if(/\.csv$/i) {
        $count = count_csv($fh);
    }
    else {
        $count = count($fh);
    }
    close $fh;
    print "$_,$count\n";
}

sub count_csv {
    my $fh = shift;
    my $header = <$fh>;

    unless($header && $header eq "WORDCOUNTS,WEIGHT\n") {
        die "unexpected header found in csv file";
    }

    my $result = 0;
    while(<$fh>) {
        chomp;
        my ($word,$count) = split /,/;
        $result++ unless $STOPLIST($word);
    }

    return $result;
}

sub count {
    my $fh = shift;
    my $result = 0;
    while(<$fh>) {
        chomp;
        my @words = split;
        foreach(@words) {
            $result++ unless $STOPLIST{$_}; <commit_msg>fix typos, runs fine now<commit_after>#!/usr/bin/env perl 
#===============================================================================
#
#         FILE: wc_stop.pl
#
#        USAGE: ./wc_stop.pl  
#
#  DESCRIPTION: 
#
#      OPTIONS: ---
# REQUIREMENTS: ---
#         BUGS: ---
#        NOTES: ---
#       AUTHOR: <NAME> (agoldst), <EMAIL>
# ORGANIZATION: Rutgers University, New Brunswick
#      VERSION: 1.0
#      CREATED: 12/04/2012 07:59:30
#     REVISION: ---
#===============================================================================
use v5.14;                                  # entails strict, unicode_strings 
use autodie;
use utf8;                                   # source code itself is in utf-8
use warnings;
use warnings FATAL => "utf8";               # Unicode encode errors are fatal
use open qw( :std :utf8 );                  # default utf8 layer

my $USAGE = <<EOM;
Usage:
wc_stop -s stoplist_file file1 file2 file3 file4...
EOM

my $first = shift;
unless ($first && $first eq "-s") {
    say $USAGE;
    exit;
}
my $stoplist = shift;
open STOP, $stoplist or die;
my %STOPLIST = ();
while(<STOP>) {
    chomp;
    $STOPLIST{$_} = 1; 
}
close STOP;

my $count;
foreach my $filename (@ARGV) {
    open my $fh, "$filename" or die;
    if($filename =~ /\.csv$/i) {
        $count = count_csv($fh);
    }
    else {
        $count = count($fh);
    }
    close $fh;
    print "$filename,$count\n";
}

sub count_csv {
    my $fh = shift;
    my $header = <$fh>;

    unless($header && $header eq "WORDCOUNTS,WEIGHT\n") {
        die "unexpected header found in csv file";
    }

    my $result = 0;
    while(<$fh>) {
        chomp;
        my ($word,$count) = split /,/;
        $result++ unless $STOPLIST{$word};
    }

    return $result;
}

sub count {
    my $fh = shift;
    my $result = 0;
    while(<$fh>) {
        chomp;
        my @words = split;
        foreach(@words) {
            $result++ unless $STOPLIST{$_}; 
{'fraction_non_alphanumeric': 0.2247658688865765, 'fraction_numerical': 0.016129032258064516, 'mean_word_length': 2.1158833063209075, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3767268', 'n_tokens_mistral': 1384, 'n_tokens_neox': 1256, 'n_words': 340}