<commit_before>package Bio::SUPERSMART::App::smrt::Command::Taxize;

use strict;
use warnings;

use List::MoreUtils qw(firstidx uniq);

use Bio::Phylo::PhyLoTA::Config;
use Bio::Phylo::PhyLoTA::Service::MarkersAndTaxaSelector;
use Bio::Phylo::PhyLoTA::Service::ParallelService 'pfm'; # can be either 'pthreads' or 'mpi';

use base 'Bio::SUPERSMART::App::smrt::SubCommand';
use Bio::SUPERSMART::App::smrt qw(-command);

# ABSTRACT: writes taxa table for given list of taxon names or root taxa

=head1 NAME

Taxize.pm - reconciles taxon names with information from NCBI taxonomy

=head1 SYNOPSYS

$ smrt taxize [-v ] [-w <dir>] -i <file> [-o <file>] [-e <rank>] 

=head1 DESCRIPTION

Performs taxonomic name resolution by mapping a provided list of input names onto the 
NCBI taxonomy. Optionally expands the higher input taxa to the specified lower taxon, 
e.g. to expand a named Order to its constituent species. Produces a table that lists 
all resolved (and expanded) taxa and their higher classification.

=cut

sub options {
	my ($self, $opt, $args) = @_;		
	my $outfile_default = "species.tsv";
	return (
		["infile|i=s", "file with list of taxon names", { arg => "file", mandatory => 1}],
		["outfile|o=s", "name of the output file, defaults to '$outfile_default'", {default => $outfile_default, arg => "filename"}],
		["expand_rank|e=s", "rank to which root taxa are expanded", { default => 0, arg => "rank"}],	
	);	
}

sub validate {
	my ($self, $opt, $args) = @_;		

	# We only have to check the 'infile' argument. 
	#  If the infile is absent or empty, abort  
	my $file = $opt->infile;
	$self->usage_error("no infile argument given") if not $file;
	$self->usage_error("file $file does not exist") unless (-e $file);
	$self->usage_error("file $file is empty") unless (-s $file);			
}

sub run {
	my ($self, $opt, $args) = @_;
		
	# collect command-line arguments
	my $infile = $opt->infile;
	my $expand_rank = $opt->expand_rank;
	my $outfile = $self->outfile;
	(my $workdir = $opt->workdir) =~ s/\/$//g;
	
	# instantiate helper objects
	my $log = $self->logger; 
    my $config = Bio::Phylo::PhyLoTA::Config->new;
	my $mts = Bio::Phylo::PhyLoTA::Service::MarkersAndTaxaSelector->new;
    
	# read names from file or STDIN, clean line breaks
	open my $fh, '<', $infile or die $!;
	my @names = <$fh>;
	chomp(@names);
	close $fh;
	$log->info("read ".scalar(@names)." species names from $infile");
	
	# expand root taxa if argument is provided
	if ( $expand_rank ) {
		@names = $mts->expand_taxa( \@names , $expand_rank);
	}
		
	# get all possible taxonomic ranks
	my @levels = reverse $mts->get_taxonomic_ranks;
		
	# this will take some time to do the taxonomic name resolution in the
	# database and with webservices. The below code runs in parallel
	my @result = pmap {
	        my $name = $_; 
	        # replace consecutive whitespaces with one
	        $name =~ s/\s+/ /g;               
	        my @res = ();
	        my @nodes = $mts->get_nodes_for_names($name);
	        if ( @nodes ) {
	                if ( @nodes > 1 ) {
	                        $log->warn("found more than one taxon for name $name");
	                }
	                else {
	                        $log->info("found exactly one taxon for name $name");
	                }
	                
	                # for each node, fetch the IDs of all taxonomic levels of interest
	                for my $node ( @nodes ) {
	      
	                        # create hash of taxonomic levels so that when we walk up the
	                        # taxonomy tree we can more easily check to see if we are at a
	                        # level of interest
	                        my %level = map { $_ => "NA" } @levels;
	                        
	                        # traverse up the tree
	                        while ( $node ) {
	                                my $tn = $node->taxon_name;
	                                my $ti = $node->ti;
	                                my $rank = $node->rank;
	                                if ( exists $level{$rank} ) {
	                                        $level{$rank} = $node->get_id;
	                                } 
	                                $node = $node->get_parent;
	                        }                        
	                        my @entry = ($name, @level{@levels});
	                        push @res, \@entry;
	        		}
	        }
	        else {
	                $log->warn("couldn't resolve name $name");
	        }                
	        return  @res;
	
	} @names; 
	
	# clean the results for duplicates and rows that represent taxa with levels higher 
	# than 'Species', then write the results to the output file
	sequential {
		
		open my $out, '>', $outfile or die $!;
	
		# print table header 
		print $out join ("\t", 'name', @levels), "\n";
	
		my %seen = ();
		foreach my $res ( @result ) {
			my @entry = @{$res};
			my $name =  shift @entry;
			my $ids = join "\t", @entry;
			
			# omit all taxa with higher taxonomic rank than 'Species' 
			my $highest_rank = "Species";
			my $idx = firstidx { lc ($_) eq lc ($highest_rank) } @levels;			<commit_msg>changing input arg for parallel code block to @_, otherwise parallel execution with Parallel fork manager won't work. <commit_after>package Bio::SUPERSMART::App::smrt::Command::Taxize;

use strict;
use warnings;

use List::MoreUtils qw(firstidx uniq);

use Bio::Phylo::PhyLoTA::Config;
use Bio::Phylo::PhyLoTA::Service::MarkersAndTaxaSelector;
use Bio::Phylo::PhyLoTA::Service::ParallelService 'pfm';

use base 'Bio::SUPERSMART::App::smrt::SubCommand';
use Bio::SUPERSMART::App::smrt qw(-command);

# ABSTRACT: writes taxa table for given list of taxon names or root taxa

=head1 NAME

Taxize.pm - reconciles taxon names with information from NCBI taxonomy

=head1 SYNOPSYS

$ smrt taxize [-v ] [-w <dir>] -i <file> [-o <file>] [-e <rank>] 

=head1 DESCRIPTION

Performs taxonomic name resolution by mapping a provided list of input names onto the 
NCBI taxonomy. Optionally expands the higher input taxa to the specified lower taxon, 
e.g. to expand a named Order to its constituent species. Produces a table that lists 
all resolved (and expanded) taxa and their higher classification.

=cut

sub options {
	my ($self, $opt, $args) = @_;		
	my $outfile_default = "species.tsv";
	return (
		["infile|i=s", "file with list of taxon names", { arg => "file", mandatory => 1}],
		["outfile|o=s", "name of the output file, defaults to '$outfile_default'", {default => $outfile_default, arg => "filename"}],
		["expand_rank|e=s", "rank to which root taxa are expanded", { default => 0, arg => "rank"}],	
	);	
}

sub validate {
	my ($self, $opt, $args) = @_;		

	# We only have to check the 'infile' argument. 
	#  If the infile is absent or empty, abort  
	my $file = $opt->infile;
	$self->usage_error("no infile argument given") if not $file;
	$self->usage_error("file $file does not exist") unless (-e $file);
	$self->usage_error("file $file is empty") unless (-s $file);			
}

sub run {
	my ($self, $opt, $args) = @_;
		
	# collect command-line arguments
	my $infile = $opt->infile;
	my $expand_rank = $opt->expand_rank;
	my $outfile = $self->outfile;
	(my $workdir = $opt->workdir) =~ s/\/$//g;
	
	# instantiate helper objects
	my $log = $self->logger; 
    my $config = Bio::Phylo::PhyLoTA::Config->new;
	my $mts = Bio::Phylo::PhyLoTA::Service::MarkersAndTaxaSelector->new;
    
	# read names from file or STDIN, clean line breaks
	open my $fh, '<', $infile or die $!;
	my @names = <$fh>;
	chomp(@names);
	close $fh;
	$log->info("read ".scalar(@names)." species names from $infile");
	
	# expand root taxa if argument is provided
	if ( $expand_rank ) {
		@names = $mts->expand_taxa( \@names , $expand_rank);
	}
		
	# get all possible taxonomic ranks
	my @levels = reverse $mts->get_taxonomic_ranks;
		
	# this will take some time to do the taxonomic name resolution in the
	# database and with webservices. The below code runs in parallel
	my @result = pmap {
	        my ($name) = @_; 
	        # replace consecutive whitespaces with one
	        $name =~ s/\s+/ /g;               
	        my @res = ();
	        my @nodes = $mts->get_nodes_for_names($name);
	        if ( @nodes ) {
	                if ( @nodes > 1 ) {
	                        $log->warn("found more than one taxon for name $name");
	                }
	                else {
	                        $log->info("found exactly one taxon for name $name");
	                }
	                
	                # for each node, fetch the IDs of all taxonomic levels of interest
	                for my $node ( @nodes ) {
	      
	                        # create hash of taxonomic levels so that when we walk up the
	                        # taxonomy tree we can more easily check to see if we are at a
	                        # level of interest
	                        my %level = map { $_ => "NA" } @levels;
	                        
	                        # traverse up the tree
	                        while ( $node ) {
	                                my $tn = $node->taxon_name;
	                                my $ti = $node->ti;
	                                my $rank = $node->rank;
	                                if ( exists $level{$rank} ) {
	                                        $level{$rank} = $node->get_id;
	                                } 
	                                $node = $node->get_parent;
	                        }                        
	                        my @entry = ($name, @level{@levels});
	                        push @res, \@entry;
	        		}
	        }
	        else {
	                $log->warn("couldn't resolve name $name");
	        }                
	        return  @res;
	
	} @names; 
	
	$log->info("Data received, writing outfile!");
	
	# clean the results for duplicates and rows that represent taxa with levels higher 
	# than 'Species', then write the results to the output file
	sequential {
		
		open my $out, '>', $outfile or die $!;
	
		# print table header 
		print $out join ("\t", 'name', @levels), "\n";
	
		my %seen = ();
		foreach my $res ( @result ) {
			my @entry = @{$res};
			my $name =  shift @entry;
			my $ids = join "\t", @entry;
			
			# omit all taxa with higher taxonomic rank than 'Species' 
			my $highest_rank = "Species";
			my $idx = firstidx { lc ($_) eq lc ($highest_rank) } @levels;			
{'fraction_non_alphanumeric': 0.1260100038476337, 'fraction_numerical': 0.0011542901115813775, 'mean_word_length': 2.0338488473883864, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 78, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1780199', 'n_tokens_mistral': 3437, 'n_tokens_neox': 3089, 'n_words': 1061}