<commit_before>#!/usr/bin/perl
#
# Read in two reports of AUids and status. Generate a count
# of category crossings.
# 
# To create report, comparing two points in time.
# svn update -r 20110101
# ./scripts/tdb/tdbout -t auid,status tdb/prod/*.tdb | sort -u > file1.txt
# svn update -r 20111231
# ./scripts/tdb/tdbout -t auid,status tdb/prod/*.tdb | sort -u > file2.txt
# ./report_buckets file1.txt file2.txt
# 
# To create a report, comparing status1 and status2 in clockssingest
# ./scripts/tdb/tdbout -t auid,status tdb/clockssingest/*.tdb | sort -u > file1.txt
# ./scripts/tdb/tdbout -t auid,status2 tdb/clockssingest/*.tdb | sort -u > file2.txt
# ./report_buckets.pl file1.txt file2.txt


my %code = ("notPresent" => 0,
    "expected" => 1,
    "exists" => 2,
    "manifest" => 3,
    "wanted" => 4,
    "testing" => 5,
    "notReady" => 6,
    "ready" => 7,
    "crawling" => 8,
    "deepCrawl" => 9,
    "frozen" => 10,
    "ingNotReady" => 11,
    "finished" => 12,
    "released" => 13,
    "down" => 14,
    "superseded" => 15,
    "zapped" => 16,
    "superseded" => 17,
    "doNotProcess" => 18,
    "doesNotExist" => 19,
    "other" => 20,
    "deleted" => 21);

my %auid_status = ();

my $file1_name = shift(@ARGV);
my $file2_name = shift(@ARGV);

# Read in first file. (Should be the report from the earlier
# date.
open(IFILE, "<$file1_name");
while (my $line = <IFILE>) {
    my ($auid, $status) = split(/\s+/, $line);
    my $status_code = $code{"other"};
    if (exists($code{$status})) {
        $status_code = $code{$status};
    }
    $auid_status{$auid}{start} = $status_code;
}
close(IFILE);

# Debug report
#foreach my $auid (keys(%auid_status)) {
#    printf("%s %d\n", $auid, $auid_status{$auid}{start});<commit_msg>Typo.<commit_after>#!/usr/bin/perl
#
# Read in two reports of AUids and status. Generate a count
# of category crossings.
# 
# To create report, comparing two points in time.
# svn update -r 20110101
# ./scripts/tdb/tdbout -t auid,status tdb/prod/*.tdb | sort -u > file1.txt
# svn update -r 20111231
# ./scripts/tdb/tdbout -t auid,status tdb/prod/*.tdb | sort -u > file2.txt
# ./report_buckets file1.txt file2.txt
# 
# To create a report, comparing status1 and status2 in clockssingest
# ./scripts/tdb/tdbout -t auid,status tdb/clockssingest/*.tdb | sort -u > file1.txt
# ./scripts/tdb/tdbout -t auid,status2 tdb/clockssingest/*.tdb | sort -u > file2.txt
# ./report_buckets.pl file1.txt file2.txt


my %code = ("notPresent" => 0,
    "expected" => 1,
    "exists" => 2,
    "manifest" => 3,
    "wanted" => 4,
    "testing" => 5,
    "notReady" => 6,
    "ready" => 7,
    "crawling" => 8,
    "deepCrawl" => 9,
    "frozen" => 10,
    "ingNotReady" => 11,
    "finished" => 12,
    "released" => 13,
    "down" => 14,
    "superseded" => 15,
    "zapped" => 16,
    "doNotProcess" => 17,
    "doesNotExist" => 18,
    "other" => 19,
    "deleted" => 20);

my %auid_status = ();

my $file1_name = shift(@ARGV);
my $file2_name = shift(@ARGV);

# Read in first file. (Should be the report from the earlier
# date.
open(IFILE, "<$file1_name");
while (my $line = <IFILE>) {
    my ($auid, $status) = split(/\s+/, $line);
    my $status_code = $code{"other"};
    if (exists($code{$status})) {
        $status_code = $code{$status};
    }
    $auid_status{$auid}{start} = $status_code;
}
close(IFILE);

# Debug report
#foreach my $auid (keys(%auid_status)) {
#    printf("%s %d\n", $auid, $auid_status{$auid}{start});
{'fraction_non_alphanumeric': 0.1875, 'fraction_numerical': 0.036458333333333336, 'mean_word_length': 3.130227001194743, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 56, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '750697', 'n_tokens_mistral': 1475, 'n_tokens_neox': 1355, 'n_words': 341}