<commit_before>#!/usr/bin/perl -w

use URI::Escape;
use Getopt::Long;
use LWP::UserAgent;
use HTTP::Request;
use HTTP::Cookies;

my $lockss_tag  = "LOCKSS system has permission to collect, preserve, and serve this Archival Unit";
my $oa_tag      = "LOCKSS system has permission to collect, preserve, and serve this open access Archival Unit";
my $clockss_tag = "CLOCKSS system has permission to ingest, preserve, and serve this Archival Unit";
my $cc_license_tag = "rel..license";
my $cc_by_tag = "href..http://creativecommons.org/licenses/by";
my $bmc_tag = "<span>Archive</span>";
my $igi_tag = "/gateway/issue/";
my $total_manifests = 0;
my $total_missing = 0;
my $total_missing_plugin = 0;

# Set up "cookie jar" to hold session cookies for Web access.
# Don't save these cookies from run to run.
my $cjar = HTTP::Cookies->new();

# Create user agent.
my $ua = LWP::UserAgent->new( cookie_jar => $cjar, agent => "LOCKSS cache" );
$ua->proxy('http', 'http://lockss.org:8888/');
$ua->no_proxy('localhost', '127.0.0.1');

while (my $line = <>) {
  chomp $line;
  my @input_rec = split(/\|/, $line);
  my $num_elements = int(@input_rec);
  my $auid = $input_rec[$num_elements-1];
  my @auid_rec = split(/\&/, $auid);
  my $plugin = shift(@auid_rec);
  my %param = ();
  foreach my $param_entry (@auid_rec) {
    if ($param_entry =~ m/^([^\~]+)\~([^\~]+)$/) {
      $param{$1} = $2;
    }
  }
  my $url = "cant_create_url";
  my $url_permission = "cant_create_url";
  my $man_url = "cant_create_url";
  my $vol_title = "NO TITLE FOUND";
  my $result = "Plugin Unknown";
  if ($plugin eq "HighWirePressH20Plugin" || $plugin eq "HighWirePressPlugin") {
    $url = sprintf("%slockss-manifest/vol_%s_manifest.dtl",
      $param{base_url}, $param{volume_name});
    $man_url = uri_unescape($url);
    my $req = HTTP::Request->new(GET, $man_url);
    my $resp = $ua->request($req);
    if ($resp->is_success) {
      my $man_contents = $resp->content;
      if (defined($man_contents) && (($man_contents =~ m/$lockss_tag/) || ($man_contents =~ m/$oa_tag/)) && ($man_contents =~ m/\/content\/$param{volume_name}\//)) {
        if ($man_contents =~ m/<title>\s*(.*)\s+C?LOCKSS\s+Manifest\s+Page.*<\/title>/si) {
          $vol_title = $1;
          $vol_title =~ s/\s*\n\s*/ /g;
          if (($vol_title =~ m/</) || ($vol_title =~ m/>/)) {
            $vol_title = "\"" . $vol_title . "\"";
          }
        } 
        $result = "Manifest"
      } else {
        $result = "--"
      }
    } else {
      $result = "--"
    }
    sleep(5);

  } elsif ($plugin eq "ClockssHighWirePressH20Plugin" || $plugin eq "ClockssHighWirePlugin") {
        $url = sprintf("%sclockss-manifest/vol_%s_manifest.dtl",
            $param{base_url}, $param{volume_name});
        $man_url = uri_unescape($url);
  my $req = HTTP::Request->new(GET, $man_url);
  my $resp = $ua->request($req);
  if ($resp->is_success) {
      my $man_contents = $resp->content;
      if (defined($man_contents) && ($man_contents =~ m/$clockss_tag/) && (($man_contents =~ m/\/content\/$param{volume_name}\//) || ($man_contents =~ m/\/content\/vol$param{volume_name}\//))) {
    if ($man_contents =~ m/<title>\s*(.*)\s+C?LOCKSS\s+Manifest\s+Page.*<\/title>/si) {
        $vol_title = $1;
        $vol_title =~ s/\s*\n\s*/ /g;
        if (($vol_title =~ m/</) || ($vol_title =~ m/>/)) {
      $vol_title = "\"" . $vol_title . "\"";
        }
    } 
    $result = "Manifest"
      } else {
    $result = "--"
      }
  } else {
      $result = "--"
  }
        sleep(5);

  } elsif ($plugin eq "ProjectMusePlugin") {
        $url = sprintf("%sjournals/%s/v%03d/", 
            $param{base_url}, $param{journal_dir}, $param{volume});
        $man_url = uri_unescape($url);
    my $req = HTTP::Request->new(GET, $man_url);
    my $resp = $ua->request($req);
    if ($resp->is_success) {
      my $man_contents = $resp->content;
      if (defined($man_contents) && (($man_contents =~ m/$lockss_tag/) || ($man_contents =~ m/$oa_tag/))) {
    if ($man_contents =~ m/<title>Project MUSE -\s*(.*)<\/title>/si) {
        $vol_title = $1;
        $vol_title =~ s/\s*\n\s*/ /g;
        if (($vol_title =~ m/</) || ($vol_title =~ m/>/)) {
      $vol_title = "\"" . $vol_title . "\"";
        }
    } 
    $result = "Manifest"
      } else {
    $result = "--"
      }
  } else {
      $result = "--"
  }
        sleep(5);

  } elsif ($plugin eq "BePressPlugin") {
        $url = sprintf("%s%s/lockss-volume%d.html", 
            $param{base_url}, $param{journal_abbr}, $param{volume});
        $man_url = uri_unescape($url);
    my $req = HTTP::Request->new(GET, $man_url);
    my $resp = $ua->request($req);
    if ($resp->is_success) {
      my $man_contents = $resp->content;
      if (defined($man_contents) && (($man_contents =~ m/$lockss_tag/) || ($man_contents =~ m/$oa_tag/))) {
    if ($man_contents =~ m/<title>(.*) --.*<\/title>/si) {
        $vol_title = $1;
        $vol_title =~ s/\s*\n\s*/ /g;
        if (($vol_title =~ m/</) || ($vol_title =~ m/>/)) {
      $vol_title = "\"" . $vol_title . "\"";
        }
    } 
    $result = "Manifest"
      } else {
    $result = "--"
      }
  } else {
      $result = "--"
  }
        sleep(5);
        
  } elsif ($plugin eq "ClockssBerkeleyElectronicPressPlugin") {
        $url = sprintf("%s%s/lockss-volume%d.html", 
            $param{base_url}, $param{journal_abbr}, $param{volume});
        $man_url = uri_unescape($url);
    my $req = HTTP::Request->new(GET, $man_url);
    my $resp = $ua->request($req);
    if ($resp->is_success) {
      my $man_contents = $resp->content;
      if (defined($man_contents) && (($man_contents =~ m/$clockss_tag/))) {
    if ($man_contents =~ m/<title>(.*) --.*<\/title>/si) {
        $vol_title = $1;
        $vol_title =~ s/\s*\n\s*/ /g;
        if (($vol_title =~ m/</) || ($vol_title =~ m/>/)) {
      $vol_title = "\"" . $vol_title . "\"";
        }
    } 
    $result = "Manifest"
      } else {
    $result = "--"
      }
  } else {
      $result = "--"
  }
        sleep(5);
        
  } elsif ($plugin eq "OJS2Plugin" || $plugin eq "CoActionPublishingPlugin") {
        $url = sprintf("%sindex.php/%s/gateway/lockss?year=%d", 
            $param{base_url}, $param{journal_id}, $param{year});
        $man_url = uri_unescape($url);
    my $req = HTTP::Request->new(GET, $man_url);
    my $resp = $ua->request($req);
    if ($resp->is_success) {
      my $man_contents = $resp->content;
      if (defined($man_contents) && (($man_contents =~ m/$lockss_tag/) || ($man_contents =~ m/$oa_tag/)) && (($man_contents =~ m/\($param{year}\)/) || ($man_contents =~ m/: $param{year}/))) {
    if ($man_contents =~ m/<title>(.*)<\/title>/si) {
        $vol_title = $1;
        $vol_title =~ s/\s*\n\s*/ /g;
        if (($vol_title =~ m/</) || ($vol_title =~ m/>/)) {
      $vol_title = "\"" . $vol_title . "\"";
        }
    } 
    $result = "Manifest"
      } else {
    $result = "--"
      }
  } else {
      $result = "--"
  }
        sleep(5);
        
  } elsif ($plugin eq "ClockssOJS2Plugin" || $plugin eq "ClockssCoActionPublishingPlugin") {
        #Url with permission statement
        $url_m = sprintf("%sindex.php/%s/about/editorialPolicies", 
            $param{base_url}, $param{journal_id});
        $man_url = uri_unescape($url_m);
        my $req_m = HTTP::Request->new(GET, $man_url);
        my $resp_m = $ua->request($req_m);
        #Url with list of urls for issues
        $url_s = sprintf("%sindex.php/%s/gateway/lockss?year=%d", 
            $param{base_url}, $param{journal_id}, $param{year});
        $start_url = uri_unescape($url_s);
        my $req_s = HTTP::Request->new(GET, $start_url);
        my $resp_s = $ua->request($req_s);
        #For reporting at the end
        $man_url = $start_url ;
    if (($resp_m->is_success) && ($resp_m->is_success)) {
      my $man_contents = $resp_m->content;
      my $start_contents = $resp_s->content;
      if (defined($man_contents) && defined($start_contents) && (($man_contents =~ m/$clockss_tag/) || ($man_contents =~ m/$oa_tag/)) && (($start_contents =~ m/\($param{year}\)/) || ($start_contents =~ m/: $param{year}/))) {
         $result = "Manifest"
      } else {
         $result = "--"
      }
    } else {
      $result = "--"
    }
        sleep(5);
        
  } elsif (($plugin eq "TaylorAndFrancisPlugin") || 
           ($plugin eq "SiamPlugin") || 
           ($plugin eq "FutureSciencePlugin")) {
        $url = sprintf("%slockss/%s/%s/index.html", 
            $param{base_url}, $param{journal_id}, $param{volume_name});
        $man_url = uri_unescape($url);
    my $req = HTTP::Request->new(GET, $man_url);
    my $resp = $ua->request($req);
    if ($resp->is_success) {
      my $man_contents = $resp->content;
      if (defined($man_contents) && (($man_contents =~ m/$lockss_tag/) && ($man_contents =~ m/$param{journal_id}\/$param{volume_name}/))) {
    if ($man_contents =~ m/<title>(.*) LOCKSS Manifest Page<\/title>/si) {
        $vol_title = $1;
        $vol_title =~ s/\s*\n\s*/ /g;
        $vol_title =~ s/2013/Volume $param{volume_name}/g;
        if (($vol_title =~ m/</) || ($vol_title =~ m/>/)) {
          $vol_title = "\"" . $vol_title . "\"";
        }
    } 
    $result = "Manifest";
      } else {
    $result = "--"
      }
  } else {
      $result = "--"
  }
        sleep(5);
        
  } elsif (($plugin eq "ClockssTaylorAndFrancisPlugin") || 
           ($plugin eq "ClockssNRCResearchPressPlugin") || 
           ($plugin eq "ClockssASCEPlugin") || 
           ($plugin eq "ClockssSiamPlugin") || 
           ($plugin eq "ClockssAmmonsScientificPlugin") || 
           ($plugin eq "ClockssFutureSciencePlugin")) {
        $url = sprintf("%sclockss/%s/%s/index.html", 
            $param{base_url}, $param{journal_id}, $param{volume_name});
        $man_url = uri_unescape($url);
    my $req = HTTP::Request->new(GET, $man_url);
    my $resp = $ua->request($req);
    if ($resp->is_success) {
      my $man_contents = $resp->content;
      if (defined($man_contents) && (($man_contents =~ m/$clockss_tag/) && ($man_contents =~ m/$param{journal_id}\/$param{volume_name}/))) {
    if ($man_contents =~ m/<title>(.*) CLOCKSS Manifest Page<\/title>/si) {
        $vol_title = $1;
        $vol_title =~ s/\s*\n\s*/ /g;
        $vol_title =~ s/2013/Volume $param{volume_name}/g;
        if (($vol_title =~ m/</) || ($vol_title =~ m/>/)) {
          $vol_title = "\"" . $vol_title . "\"";
        }
    } 
    $result = "Manifest";
      } else {
    $result = "--"
      }
  } else {
      $result = "--"
  }
        sleep(5);
        
  } elsif ($plugin eq "EdinburghUniversityPressPlugin") {
    $url = sprintf("%slockss/%s/%s/index.html", 
      $param{base_url}, $param{journal_id}, $param{volume_name});
    $man_url = uri_unescape($url);
    #printf("\nUrl: %s\n", $man_url);
    my $req = HTTP::Request->new(GET, $man_url);
    my $resp = $ua->request($req);
    if ($resp->is_success) {
      my $man_contents = $resp->content;
      if (defined($man_contents) && (($man_contents =~ m/$lockss_tag/) || ($man_contents =~ m/$oa_tag/))) {
        if ($man_contents =~ m/<title>(.*)LOCKSS Manifest Page<\/title>/si) {
          $vol_title = $1;
          $vol_title =~ s/\s*\n\s*/ /g;
          if (($vol_title =~ m/</) || ($vol_title =~ m/>/)) {
            $vol_title = "\"" . $vol_title . "\"";
          }
        } 
        $result = "Manifest";
      } else {
        $result = "--"
      }
    } else {
      $result = "--"
    }
    sleep(5);
        
  } elsif ($plugin eq "ClockssEdinburghUniversityPressPlugin") {
    $url = sprintf("%sclockss/%s/%s/index.html", 
      $param{base_url}, $param{journal_id}, $param{volume_name});
    $man_url = uri_unescape($url);
    #printf("\nUrl: %s\n", $man_url);
    my $req = HTTP::Request->new(GET, $man_url);
    my $resp = $ua->request($req);
    if ($resp->is_success) {
      my $man_contents = $resp->content;
      if (defined($man_contents) && (($man_contents =~ m/$clockss_tag/))) {
        if ($man_contents =~ m/<title>(.*)CLOCKSS Manifest Page<\/title>/si) {
          $vol_title = $1;
          $vol_title =~ s/\s*\n\s*/ /g;
          if (($vol_title =~ m/</) || ($vol_title =~ m/>/)) {
            $vol_title = "\"" . $vol_title . "\"";
          }
        } 
        $result = "Manifest";
      } else {
        $result = "--"
      }
    } else {
      $result = "--"
    }
    sleep(5);
        
  } elsif ($plugin eq "ClockssPionPlugin") {
    $url = sprintf("%scontents.cgi?journal=%s&amp;volume=%s", 
      $param{base_url}, $param{short_journal_code}, $param{volume_name});
    $man_url = uri_unescape($url);
    #printf("\nUrl: %s\n", $man_url);
    my $req = HTTP::Request->new(GET, $man_url);
    my $resp = $ua->request($req);
    if ($resp->is_success) {
      my $man_contents = $resp->content;
      if (defined($man_contents) && ($man_contents =~ m/$clockss_tag/) && ($man_contents =~ m/href="abstract\.cgi\?id>/) {
        if ($man_contents =~ m/<title>(.*)<\/title>/si) {
          $vol_title = $1;
          $vol_title =~ s/\s*\n\s*/ /g;
          if (($vol_title =~ m/</) || ($vol_title =~ m/>/)) {
            $vol_title = "\"" . $vol_title . "\"";
          }
        } 
        $result = "Manifest";
      } else {
        $result = "--"
      }
    } else {
      $result = "--"
    }
    sleep(5);
        
  } elsif ($plugin eq "BioOneAtyponPlugin") {
    $url = sprintf("%slockss/%s/%s/index.html", 
      $param{base_url}, $param{journal_id}, $param{volume_name});
    $man_url = uri_unescape($url);
    my $req = HTTP::Request->new(GET, $man_url);
    my $resp = $ua->request($req);
    if ($resp->is_success) {
      my $man_contents = $resp->content;
      if (defined($man_contents) && (($man_contents =~ m/$lockss_tag/) || ($man_contents =~ m/$oa_tag/))) {
        if ($man_contents =~ m/<title>(.*)\s*LOCKSS Manifest Page<\/title>/si) {<commit_msg>Syntax error.<commit_after>#!/usr/bin/perl -w

use URI::Escape;
use Getopt::Long;
use LWP::UserAgent;
use HTTP::Request;
use HTTP::Cookies;

my $lockss_tag  = "LOCKSS system has permission to collect, preserve, and serve this Archival Unit";
my $oa_tag      = "LOCKSS system has permission to collect, preserve, and serve this open access Archival Unit";
my $clockss_tag = "CLOCKSS system has permission to ingest, preserve, and serve this Archival Unit";
my $cc_license_tag = "rel..license";
my $cc_by_tag = "href..http://creativecommons.org/licenses/by";
my $bmc_tag = "<span>Archive</span>";
my $igi_tag = "/gateway/issue/";
my $total_manifests = 0;
my $total_missing = 0;
my $total_missing_plugin = 0;

# Set up "cookie jar" to hold session cookies for Web access.
# Don't save these cookies from run to run.
my $cjar = HTTP::Cookies->new();

# Create user agent.
my $ua = LWP::UserAgent->new( cookie_jar => $cjar, agent => "LOCKSS cache" );
$ua->proxy('http', 'http://lockss.org:8888/');
$ua->no_proxy('localhost', '127.0.0.1');

while (my $line = <>) {
  chomp $line;
  my @input_rec = split(/\|/, $line);
  my $num_elements = int(@input_rec);
  my $auid = $input_rec[$num_elements-1];
  my @auid_rec = split(/\&/, $auid);
  my $plugin = shift(@auid_rec);
  my %param = ();
  foreach my $param_entry (@auid_rec) {
    if ($param_entry =~ m/^([^\~]+)\~([^\~]+)$/) {
      $param{$1} = $2;
    }
  }
  my $url = "cant_create_url";
  my $url_permission = "cant_create_url";
  my $man_url = "cant_create_url";
  my $vol_title = "NO TITLE FOUND";
  my $result = "Plugin Unknown";
  if ($plugin eq "HighWirePressH20Plugin" || $plugin eq "HighWirePressPlugin") {
    $url = sprintf("%slockss-manifest/vol_%s_manifest.dtl",
      $param{base_url}, $param{volume_name});
    $man_url = uri_unescape($url);
    my $req = HTTP::Request->new(GET, $man_url);
    my $resp = $ua->request($req);
    if ($resp->is_success) {
      my $man_contents = $resp->content;
      if (defined($man_contents) && (($man_contents =~ m/$lockss_tag/) || ($man_contents =~ m/$oa_tag/)) && ($man_contents =~ m/\/content\/$param{volume_name}\//)) {
        if ($man_contents =~ m/<title>\s*(.*)\s+C?LOCKSS\s+Manifest\s+Page.*<\/title>/si) {
          $vol_title = $1;
          $vol_title =~ s/\s*\n\s*/ /g;
          if (($vol_title =~ m/</) || ($vol_title =~ m/>/)) {
            $vol_title = "\"" . $vol_title . "\"";
          }
        } 
        $result = "Manifest"
      } else {
        $result = "--"
      }
    } else {
      $result = "--"
    }
    sleep(5);

  } elsif ($plugin eq "ClockssHighWirePressH20Plugin" || $plugin eq "ClockssHighWirePlugin") {
        $url = sprintf("%sclockss-manifest/vol_%s_manifest.dtl",
            $param{base_url}, $param{volume_name});
        $man_url = uri_unescape($url);
  my $req = HTTP::Request->new(GET, $man_url);
  my $resp = $ua->request($req);
  if ($resp->is_success) {
      my $man_contents = $resp->content;
      if (defined($man_contents) && ($man_contents =~ m/$clockss_tag/) && (($man_contents =~ m/\/content\/$param{volume_name}\//) || ($man_contents =~ m/\/content\/vol$param{volume_name}\//))) {
    if ($man_contents =~ m/<title>\s*(.*)\s+C?LOCKSS\s+Manifest\s+Page.*<\/title>/si) {
        $vol_title = $1;
        $vol_title =~ s/\s*\n\s*/ /g;
        if (($vol_title =~ m/</) || ($vol_title =~ m/>/)) {
      $vol_title = "\"" . $vol_title . "\"";
        }
    } 
    $result = "Manifest"
      } else {
    $result = "--"
      }
  } else {
      $result = "--"
  }
        sleep(5);

  } elsif ($plugin eq "ProjectMusePlugin") {
        $url = sprintf("%sjournals/%s/v%03d/", 
            $param{base_url}, $param{journal_dir}, $param{volume});
        $man_url = uri_unescape($url);
    my $req = HTTP::Request->new(GET, $man_url);
    my $resp = $ua->request($req);
    if ($resp->is_success) {
      my $man_contents = $resp->content;
      if (defined($man_contents) && (($man_contents =~ m/$lockss_tag/) || ($man_contents =~ m/$oa_tag/))) {
    if ($man_contents =~ m/<title>Project MUSE -\s*(.*)<\/title>/si) {
        $vol_title = $1;
        $vol_title =~ s/\s*\n\s*/ /g;
        if (($vol_title =~ m/</) || ($vol_title =~ m/>/)) {
      $vol_title = "\"" . $vol_title . "\"";
        }
    } 
    $result = "Manifest"
      } else {
    $result = "--"
      }
  } else {
      $result = "--"
  }
        sleep(5);

  } elsif ($plugin eq "BePressPlugin") {
        $url = sprintf("%s%s/lockss-volume%d.html", 
            $param{base_url}, $param{journal_abbr}, $param{volume});
        $man_url = uri_unescape($url);
    my $req = HTTP::Request->new(GET, $man_url);
    my $resp = $ua->request($req);
    if ($resp->is_success) {
      my $man_contents = $resp->content;
      if (defined($man_contents) && (($man_contents =~ m/$lockss_tag/) || ($man_contents =~ m/$oa_tag/))) {
    if ($man_contents =~ m/<title>(.*) --.*<\/title>/si) {
        $vol_title = $1;
        $vol_title =~ s/\s*\n\s*/ /g;
        if (($vol_title =~ m/</) || ($vol_title =~ m/>/)) {
      $vol_title = "\"" . $vol_title . "\"";
        }
    } 
    $result = "Manifest"
      } else {
    $result = "--"
      }
  } else {
      $result = "--"
  }
        sleep(5);
        
  } elsif ($plugin eq "ClockssBerkeleyElectronicPressPlugin") {
        $url = sprintf("%s%s/lockss-volume%d.html", 
            $param{base_url}, $param{journal_abbr}, $param{volume});
        $man_url = uri_unescape($url);
    my $req = HTTP::Request->new(GET, $man_url);
    my $resp = $ua->request($req);
    if ($resp->is_success) {
      my $man_contents = $resp->content;
      if (defined($man_contents) && (($man_contents =~ m/$clockss_tag/))) {
    if ($man_contents =~ m/<title>(.*) --.*<\/title>/si) {
        $vol_title = $1;
        $vol_title =~ s/\s*\n\s*/ /g;
        if (($vol_title =~ m/</) || ($vol_title =~ m/>/)) {
      $vol_title = "\"" . $vol_title . "\"";
        }
    } 
    $result = "Manifest"
      } else {
    $result = "--"
      }
  } else {
      $result = "--"
  }
        sleep(5);
        
  } elsif ($plugin eq "OJS2Plugin" || $plugin eq "CoActionPublishingPlugin") {
        $url = sprintf("%sindex.php/%s/gateway/lockss?year=%d", 
            $param{base_url}, $param{journal_id}, $param{year});
        $man_url = uri_unescape($url);
    my $req = HTTP::Request->new(GET, $man_url);
    my $resp = $ua->request($req);
    if ($resp->is_success) {
      my $man_contents = $resp->content;
      if (defined($man_contents) && (($man_contents =~ m/$lockss_tag/) || ($man_contents =~ m/$oa_tag/)) && (($man_contents =~ m/\($param{year}\)/) || ($man_contents =~ m/: $param{year}/))) {
    if ($man_contents =~ m/<title>(.*)<\/title>/si) {
        $vol_title = $1;
        $vol_title =~ s/\s*\n\s*/ /g;
        if (($vol_title =~ m/</) || ($vol_title =~ m/>/)) {
      $vol_title = "\"" . $vol_title . "\"";
        }
    } 
    $result = "Manifest"
      } else {
    $result = "--"
      }
  } else {
      $result = "--"
  }
        sleep(5);
        
  } elsif ($plugin eq "ClockssOJS2Plugin" || $plugin eq "ClockssCoActionPublishingPlugin") {
        #Url with permission statement
        $url_m = sprintf("%sindex.php/%s/about/editorialPolicies", 
            $param{base_url}, $param{journal_id});
        $man_url = uri_unescape($url_m);
        my $req_m = HTTP::Request->new(GET, $man_url);
        my $resp_m = $ua->request($req_m);
        #Url with list of urls for issues
        $url_s = sprintf("%sindex.php/%s/gateway/lockss?year=%d", 
            $param{base_url}, $param{journal_id}, $param{year});
        $start_url = uri_unescape($url_s);
        my $req_s = HTTP::Request->new(GET, $start_url);
        my $resp_s = $ua->request($req_s);
        #For reporting at the end
        $man_url = $start_url ;
    if (($resp_m->is_success) && ($resp_m->is_success)) {
      my $man_contents = $resp_m->content;
      my $start_contents = $resp_s->content;
      if (defined($man_contents) && defined($start_contents) && (($man_contents =~ m/$clockss_tag/) || ($man_contents =~ m/$oa_tag/)) && (($start_contents =~ m/\($param{year}\)/) || ($start_contents =~ m/: $param{year}/))) {
         $result = "Manifest"
      } else {
         $result = "--"
      }
    } else {
      $result = "--"
    }
        sleep(5);
        
  } elsif (($plugin eq "TaylorAndFrancisPlugin") || 
           ($plugin eq "SiamPlugin") || 
           ($plugin eq "FutureSciencePlugin")) {
        $url = sprintf("%slockss/%s/%s/index.html", 
            $param{base_url}, $param{journal_id}, $param{volume_name});
        $man_url = uri_unescape($url);
    my $req = HTTP::Request->new(GET, $man_url);
    my $resp = $ua->request($req);
    if ($resp->is_success) {
      my $man_contents = $resp->content;
      if (defined($man_contents) && (($man_contents =~ m/$lockss_tag/) && ($man_contents =~ m/$param{journal_id}\/$param{volume_name}/))) {
    if ($man_contents =~ m/<title>(.*) LOCKSS Manifest Page<\/title>/si) {
        $vol_title = $1;
        $vol_title =~ s/\s*\n\s*/ /g;
        $vol_title =~ s/2013/Volume $param{volume_name}/g;
        if (($vol_title =~ m/</) || ($vol_title =~ m/>/)) {
          $vol_title = "\"" . $vol_title . "\"";
        }
    } 
    $result = "Manifest";
      } else {
    $result = "--"
      }
  } else {
      $result = "--"
  }
        sleep(5);
        
  } elsif (($plugin eq "ClockssTaylorAndFrancisPlugin") || 
           ($plugin eq "ClockssNRCResearchPressPlugin") || 
           ($plugin eq "ClockssASCEPlugin") || 
           ($plugin eq "ClockssSiamPlugin") || 
           ($plugin eq "ClockssAmmonsScientificPlugin") || 
           ($plugin eq "ClockssFutureSciencePlugin")) {
        $url = sprintf("%sclockss/%s/%s/index.html", 
            $param{base_url}, $param{journal_id}, $param{volume_name});
        $man_url = uri_unescape($url);
    my $req = HTTP::Request->new(GET, $man_url);
    my $resp = $ua->request($req);
    if ($resp->is_success) {
      my $man_contents = $resp->content;
      if (defined($man_contents) && (($man_contents =~ m/$clockss_tag/) && ($man_contents =~ m/$param{journal_id}\/$param{volume_name}/))) {
    if ($man_contents =~ m/<title>(.*) CLOCKSS Manifest Page<\/title>/si) {
        $vol_title = $1;
        $vol_title =~ s/\s*\n\s*/ /g;
        $vol_title =~ s/2013/Volume $param{volume_name}/g;
        if (($vol_title =~ m/</) || ($vol_title =~ m/>/)) {
          $vol_title = "\"" . $vol_title . "\"";
        }
    } 
    $result = "Manifest";
      } else {
    $result = "--"
      }
  } else {
      $result = "--"
  }
        sleep(5);
        
  } elsif ($plugin eq "EdinburghUniversityPressPlugin") {
    $url = sprintf("%slockss/%s/%s/index.html", 
      $param{base_url}, $param{journal_id}, $param{volume_name});
    $man_url = uri_unescape($url);
    #printf("\nUrl: %s\n", $man_url);
    my $req = HTTP::Request->new(GET, $man_url);
    my $resp = $ua->request($req);
    if ($resp->is_success) {
      my $man_contents = $resp->content;
      if (defined($man_contents) && (($man_contents =~ m/$lockss_tag/) || ($man_contents =~ m/$oa_tag/))) {
        if ($man_contents =~ m/<title>(.*)LOCKSS Manifest Page<\/title>/si) {
          $vol_title = $1;
          $vol_title =~ s/\s*\n\s*/ /g;
          if (($vol_title =~ m/</) || ($vol_title =~ m/>/)) {
            $vol_title = "\"" . $vol_title . "\"";
          }
        } 
        $result = "Manifest";
      } else {
        $result = "--"
      }
    } else {
      $result = "--"
    }
    sleep(5);
        
  } elsif ($plugin eq "ClockssEdinburghUniversityPressPlugin") {
    $url = sprintf("%sclockss/%s/%s/index.html", 
      $param{base_url}, $param{journal_id}, $param{volume_name});
    $man_url = uri_unescape($url);
    #printf("\nUrl: %s\n", $man_url);
    my $req = HTTP::Request->new(GET, $man_url);
    my $resp = $ua->request($req);
    if ($resp->is_success) {
      my $man_contents = $resp->content;
      if (defined($man_contents) && (($man_contents =~ m/$clockss_tag/))) {
        if ($man_contents =~ m/<title>(.*)CLOCKSS Manifest Page<\/title>/si) {
          $vol_title = $1;
          $vol_title =~ s/\s*\n\s*/ /g;
          if (($vol_title =~ m/</) || ($vol_title =~ m/>/)) {
            $vol_title = "\"" . $vol_title . "\"";
          }
        } 
        $result = "Manifest";
      } else {
        $result = "--"
      }
    } else {
      $result = "--"
    }
    sleep(5);
        
  } elsif ($plugin eq "ClockssPionPlugin") {
    $url = sprintf("%scontents.cgi?journal=%s&amp;volume=%s", 
      $param{base_url}, $param{short_journal_code}, $param{volume_name});
    $man_url = uri_unescape($url);
    #printf("\nUrl: %s\n", $man_url);
    my $req = HTTP::Request->new(GET, $man_url);
    my $resp = $ua->request($req);
    if ($resp->is_success) {
      my $man_contents = $resp->content;
      if (defined($man_contents) && ($man_contents =~ m/$clockss_tag/) && ($man_contents =~ m/href=.abstract\.cgi\?id>/)) {
        if ($man_contents =~ m/<title>(.*)<\/title>/si) {
          $vol_title = $1;
          $vol_title =~ s/\s*\n\s*/ /g;
          if (($vol_title =~ m/</) || ($vol_title =~ m/>/)) {
            $vol_title = "\"" . $vol_title . "\"";
          }
        } 
        $result = "Manifest";
      } else {
        $result = "--"
      }
    } else {
      $result = "--"
    }
    sleep(5);
        
  } elsif ($plugin eq "BioOneAtyponPlugin") {
    $url = sprintf("%slockss/%s/%s/index.html", 
      $param{base_url}, $param{journal_id}, $param{volume_name});
    $man_url = uri_unescape($url);
    my $req = HTTP::Request->new(GET, $man_url);
    my $resp = $ua->request($req);
    if ($resp->is_success) {
      my $man_contents = $resp->content;
      if (defined($man_contents) && (($man_contents =~ m/$lockss_tag/) || ($man_contents =~ m/$oa_tag/))) {
        if ($man_contents =~ m/<title>(.*)\s*LOCKSS Manifest Page<\/title>/si) {
{'fraction_non_alphanumeric': 0.2161073342685515, 'fraction_numerical': 0.003956692205316355, 'mean_word_length': 2.417158308751229, 'pattern_counts': {'":': 0, '<': 79, '<?xml version=': 0, '>': 205, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '942446', 'n_tokens_mistral': 10317, 'n_tokens_neox': 10287, 'n_words': 1805}