<commit_before>#!/usr/bin/perl -w

use strict;
use LWP::Simple;
use Data::Dumper;
use XML::Simple;
use Parallel::ForkManager;


my $bpids = get_NCBI_bioprj();

my $pm = new Parallel::ForkManager(10);
my $out = "Load_all_NCBI.".sprintf( "%04d-%02d-%02d-%02d:%02d:%02d",
                 sub { ($_[5]+1900, $_[4]+1, $_[3]),$_[2],$_[1],$_[0] }->(localtime)).".log";


foreach my $bpid (@$bpids)
  { 
    my $pid = $pm->start and next;
    my $output;
    $output.= "BioProject $bpid Started\n";
    if (check_bioprj_genome($bpid))
      {
	my $ntids = get_bioprj_nt($bpid);
	my @accns;
	foreach my $ntid (@$ntids)
	  {
	    my $accn = get_nt_accn($ntid);
	    push @accns, $accn;
	  }
	if (@accns)
	  {
	    my $processed_accns = process_accns(\@accns, \$output);
	    run_load_genome($processed_accns, \$output);
	  }<commit_msg>update fork manager<commit_after>#!/usr/bin/perl -w

use strict;
use LWP::Simple;
use Data::Dumper;
use XML::Simple;
use Parallel::ForkManager;


my $bpids = get_NCBI_bioprj();

my $pm = new Parallel::ForkManager(20);
my $out = "Load_all_NCBI.".sprintf( "%04d-%02d-%02d-%02d:%02d:%02d",
                 sub { ($_[5]+1900, $_[4]+1, $_[3]),$_[2],$_[1],$_[0] }->(localtime)).".log";


foreach my $bpid (@$bpids)
  { 
    my $pid = $pm->start and next;
    my $output;
    $output.= "BioProject $bpid Started\n";
    if (check_bioprj_genome($bpid))
      {
	my $ntids = get_bioprj_nt($bpid);
	my @accns;
	foreach my $ntid (@$ntids)
	  {
	    my $accn = get_nt_accn($ntid);
	    push @accns, $accn;
	  }
	if (@accns)
	  {
	    my $processed_accns = process_accns(\@accns, \$output);
	    run_load_genome($processed_accns, \$output);
	  }
{'fraction_non_alphanumeric': 0.2036144578313253, 'fraction_numerical': 0.030120481927710843, 'mean_word_length': 3.5013550135501355, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1467624', 'n_tokens_mistral': 767, 'n_tokens_neox': 707, 'n_words': 138}