<commit_before>#!/usr/bin/env perl


#
# INCLUDES
#
use v5.10;
use strict;
use warnings;
use Getopt::Long;
use File::Basename;
use LWP::UserAgent;
use Time::HiRes qw(gettimeofday);
use HTML::TreeBuilder;
use Data::Dump qw(dd);
use IPC::System::Simple qw(system);
use Try::Tiny;

#
# VARIABLE SCOPE
#
my $all;
my $genus;
my $species;
my $outfile; ## log
my $help;
my $man;
my $sequences;
my $alignments;
my $assemblies;
my $cgp_response = "CGP_DB_response.html"; # HTML

#
# Opts
#
GetOptions(
           'all'              => \$all,
	   'g|genus=s'        => \$genus,
	   's|species=s'      => \$species,
	   'o|outfile=s'      => \$outfile,
	   'seq|sequences'  => \$sequences,
           'aln|alignments' => \$alignments,
           'asm|assemblies' => \$assemblies,
	   'h|help'           => \$help,
	   'm|man'            => \$man,
	  );

pod2usage( -verbose => 1 ) if $help;
pod2usage( -verbose => 2 ) if $man;

#
# Check @ARGV
#
if (!$assemblies && !$sequences && !$alignments) {
   print "\nERROR: Command line not parsed correctly. Exiting.\n";
   usage();
   exit(1);
}


#
# Counters
#
my $t0 = gettimeofday();
my $records = 0;

#
# Set terms for search
#
my ($gen, $sp);
if ($genus) {
    $gen = substr($genus, 0, 4);
}
if ($species) {
    $sp = substr($species, 0, 4);
}

#
# Create the UserAgent
# <commit_msg>add code for pod, modify pragma usage<commit_after>#!/usr/bin/env perl

#TODO: add POD

#
# INCLUDES
#
use 5.010;
use strict;
use warnings;
use Getopt::Long;
use File::Basename;
use LWP::UserAgent;
use Time::HiRes qw(gettimeofday);
use HTML::TreeBuilder;
use Data::Dump qw(dd);
use IPC::System::Simple qw(system);
use Try::Tiny;
use Pod::Usage;

#
# VARIABLE SCOPE
#
my $all;
my $genus;
my $species;
my $outfile; ## log
my $help;
my $man;
my $sequences;
my $alignments;
my $assemblies;
my $cgp_response = "CGP_DB_response.html"; # HTML

#
# Opts
#
GetOptions(
           'all'              => \$all,
	   'g|genus=s'        => \$genus,
	   's|species=s'      => \$species,
	   'o|outfile=s'      => \$outfile,
	   'seq|sequences'  => \$sequences,
           'aln|alignments' => \$alignments,
           'asm|assemblies' => \$assemblies,
	   'h|help'           => \$help,
	   'm|man'            => \$man,
	  );

#pod2usage( -verbose => 1 ) if $help;
#pod2usage( -verbose => 2 ) if $man;

#
# Check @ARGV
#
if (!$assemblies && !$sequences && !$alignments) {
   say "\nERROR: Command line not parsed correctly. Exiting.";
   usage();
   exit(1);
}


#
# Counters
#
my $t0 = gettimeofday();
my $records = 0;

#
# Set terms for search
#
my ($gen, $sp);
if ($genus) {
    $gen = substr($genus, 0, 4);
}
if ($species) {
    $sp = substr($species, 0, 4);
}

#
# Create the UserAgent
# 
{'fraction_non_alphanumeric': 0.18932938125231566, 'fraction_numerical': 0.010744720266765468, 'mean_word_length': 2.9416058394160585, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2352913', 'n_tokens_mistral': 1151, 'n_tokens_neox': 1071, 'n_words': 267}