<commit_before>#!/usr/bin/perl -w

use strict;
use LWP::Simple;
use Data::Dumper;
use Parallel::ForkManager;

# for understanding accession nomeclature from NCBI see http://www.ncbi.nlm.nih.gov/RefSeq/key.html#accessions
# genbank genlist.cgi type codes:
# 0 == Chromosomes and plasmids (NC_)
# 1 == Chromosomes (NC_)
# 2 == plasmids (NC_)
# 3 == Whole shotgun sequences (NZ_)
# 4 == mito, plastids, plasmids, nucleomorphs
# 5 == viruses
# 6 == phages
# 7 == all plasmids for all proks? (NC_)
# 8 == Chromosomes (NC_ and NS_)
# 9 == chromosomes and plasmids (NC_ NS_ NZ_)
my @pages = qw(
http://www.ncbi.nlm.nih.gov/genomes/genlist.cgi?taxid=2157&type=1&name=Archaea%20Complete%20Chromosomes
http://www.ncbi.nlm.nih.gov/genomes/genlist.cgi?taxid=2&type=0
http://www.ncbi.nlm.nih.gov/genomes/genlist.cgi?taxid=2759&type=8&name=Eukaryotae%20Complete%20Chromosomes
http://www.ncbi.nlm.nih.gov/genomes/genlist.cgi?taxid=10239&type=5&name=Viruses
http://www.ncbi.nlm.nih.gov/genomes/genlist.cgi?taxid=10239&type=6&name=Phages
http://www.ncbi.nlm.nih.gov/genomes/genlist.cgi?taxid=12884&type=0&name=Viroids
);
my $base_url = "http://www.ncbi.nlm.nih.gov/genomes/genlist.cgi?";

#taxids
#2157 == archaea
#2    == bacteria
#2759 == euks

#taxid type
my @taxids =(
	     [2157, 9], #archaea, get WGS
	     [2,    9], #bacteria, get WGS
	     [2759, 0], #euks, not ready for WGS -- lots of data with minimal annotations.  Will get there though!
	     [10239,9], #viruses, phages, 
	     [12884,9], #viroids
	    );

my $pm = new Parallel::ForkManager(20);
foreach my $item (@taxids)
  {
    my ($id, $type) = @$item;
    my $url = $base_url."type=$type"."&taxid=$id";
    my $content =get($url);
    my @tables = split /<\/table>/, $content;
    my @rows = split /<tr>/,$tables[8];
    my $prog = '/home/elyons/projects/CoGeX/scripts/genbank_genome_loader.pl';
    foreach my $row (@rows)
      {
	$pm->start and next;
	next unless $row =~/^<td>/;
	$row =~ s/\n|\r//g;
	my @cols = split /<td.*?>/, $row;
	#    print Dumper \@cols;
	foreach my $item (@cols)
	  {
	    $item =~ s/<.*?>//g;
	  }
	next unless $cols[3];
	next if $cols[3] =~ /accession/;
	my $run = $prog." -accn $cols[3]";
	$cols[2] =~ s/chromosome//i;
	$cols[2] =~ s/chr//i;
	$cols[2] =~ s/^\s+//;
	$cols[2] =~ s/\s+$//;
	$cols[2] = undef if $cols[2] && ($cols[2] eq " " || $cols[2] =~ /nbsp/);
	$run .= " -chr '$cols[2]'" if $cols[2];
	$run .= " -td '/tmp/gb/'";
	$run .= " -go";
	print "\n",$run,"\n";
	open (IN, $run." |");
	while (<IN>)
	  {
	    print $_;
	  }
	close IN;<commit_msg>added options for specifying number of forks, and pausing after a new genome is loaded from the command line<commit_after>#!/usr/bin/perl -w

use strict;
use LWP::Simple;
use Data::Dumper;
use Parallel::ForkManager;
use Getopt::Long;


my ($DEBUG, $forks, $pause);

GetOptions (
	    "debug"=>\$DEBUG,
	    "forks|f=i"=>\$forks,
	    "pause|p"=>\$pause,
	    );
$forks = 20 unless defined $forks;

# for understanding accession nomeclature from NCBI see http://www.ncbi.nlm.nih.gov/RefSeq/key.html#accessions
# genbank genlist.cgi type codes:
# 0 == Chromosomes and plasmids (NC_)
# 1 == Chromosomes (NC_)
# 2 == plasmids (NC_)
# 3 == Whole shotgun sequences (NZ_)
# 4 == mito, plastids, plasmids, nucleomorphs
# 5 == viruses
# 6 == phages
# 7 == all plasmids for all proks? (NC_)
# 8 == Chromosomes (NC_ and NS_)
# 9 == chromosomes (complete and WGS) and plasmids (NC_ NS_ NZ_)

my $base_url = "http://www.ncbi.nlm.nih.gov/genomes/genlist.cgi?";

#taxid type
my @taxids =(
	     [2157, 9], #archaea, get WGS
	     [2,    9], #bacteria, get WGS
	     [2759, 0], #euks, not ready for WGS -- lots of data with minimal annotations.  Will get there though!
	     [10239,9], #viruses, phages, 
	     [12884,9], #viroids
	    );

my $pm = new Parallel::ForkManager($forks);
foreach my $item (@taxids)
  {
    my ($id, $type) = @$item;
    my $url = $base_url."type=$type"."&taxid=$id";
    my $content =get($url);
    my @tables = split /<\/table>/, $content;
    my @rows = split /<tr>/,$tables[8];
    my $prog = '/home/elyons/projects/CoGeX/scripts/genbank_genome_loader.pl';
    foreach my $row (@rows)
      {
	$pm->start and next;
	next unless $row =~/^<td>/;
	$row =~ s/\n|\r//g;
	my @cols = split /<td.*?>/, $row;
	#    print Dumper \@cols;
	foreach my $item (@cols)
	  {
	    $item =~ s/<.*?>//g;
	  }
	next unless $cols[3];
	next if $cols[3] =~ /accession/;
	my $run = $prog." -accn $cols[3]";
	$cols[2] =~ s/chromosome//i;
	$cols[2] =~ s/chr//i;
	$cols[2] =~ s/^\s+//;
	$cols[2] =~ s/\s+$//;
	$cols[2] = undef if $cols[2] && ($cols[2] eq " " || $cols[2] =~ /nbsp/);
	$run .= " -chr '$cols[2]'" if $cols[2];
	$run .= " -td '/tmp/gb/'";
	$run .= " -go";
	print "\n",$run,"\n";
	my $previously_loaded = 0;
	open (IN, $run." |");
	while (<IN>)
	  {
	    $previously_loaded =1 if /already inserted/;
	    print $_;
	  }
	close IN;
	if ($pause && !$previously_loaded)
	  {
	    print "Paused:  press enter to continue\n";
	    <STDIN>;
	  }
{'fraction_non_alphanumeric': 0.2091949407749448, 'fraction_numerical': 0.029913671953423007, 'mean_word_length': 3.2690659811482434, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 10, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6195681', 'n_tokens_mistral': 2261, 'n_tokens_neox': 2018, 'n_words': 489}