<commit_before>#!/usr/bin/env perl

use strict;
use warnings;
use Getopt::Long;


# Default for variables
my $drisee_path = "/home/ubuntu/DRISEE/drisee.py";
my $file_in;
my $file_type = "fastq"; # fasta or fastq
my $num_proc = 1;
#my $percent = 1;
#my $verbose = 1;
my $stat_file;
my $drisee_log;
my $drisee_stdout;
my $data_log = "DRISEE_data_log.txt";
my $command_log = "DRISEE_command_log.txt";
my $help;
my $debug;

if( $debug ){ print STDOUT "made it here"."\n"; }

# check input args and display usage if not suitable
# if ( (@ARGV > 0) && ($ARGV[0] =~ /-h/) ) { &usage(); }
if ( $ARGV[0] =~ /-h/ ) { &usage(); }

unless ( @ARGV > 0 || $file_in ) { &usage(); }
unless ( @ARGV > 0 ) { &usage(); }

if ( ! GetOptions (
		   "p|drisee_path=s"   => \$drisee_path,
		   "f|file_in=s"       => \$file_in,
		   "t|file_type=s"     => \$file_type,
		   "n|num_proc=i"      => \$num_proc,
		   #"p|percent!"        => \$percent,
		   #"v|verbose!"        => \$verbose,
		   "s|stat_file=s"     => \$stat_file,
		   "l|drisee_log=s"    => \$drisee_log,
		   "o|drisee_stdout=s" => \$drisee_stdout,
		   "c|data_log=s"      => \$data_log,
		   "b|command_log=s"   => \$command_log,
		   "h|help!"           => \$help,
		   "d|debug!"          => \$debug
		  )
   ) { &usage(); }


# generate default names for output files 

unless ( defined $stat_file && length $stat_file > 0 )         { $stat_file = $file_in."."."drisee_STAT.txt" };
unless ( defined $drisee_log  && length $drisee_log > 0 )      { $drisee_log = $file_in."."."drisee_log.txt" };
unless ( defined $drisee_stdout && length $drisee_stdout > 0 ) { $drisee_stdout = $file_in."."."drisee_stdout.txt" };
unless ( defined $data_log  && length $data_log > 0 )          { $data_log = "DRISEE_data_log.txt" };
unless ( defined $command_log && length $command_log > 0 )     { $command_log = "DRISEE_command_log.txt" };

# create a cummulative log of the data - add header if the file does not exist
unless (-e $data_log){ 
  open(DATA_LOG, ">", $data_log) or die "\n\n"."can't open DATA_LOG $data_log"."\n\n";
			    
  # print a header
  print DATA_LOG (
		  "# This is a cummulative summary log of the data that process_DRISEE2.pl has produced."."\n".
		  "file_in"."\t".
		  "Version"."\t".
		  "bp_count"."\t".
		  "sequence_count"."\t".
		  "average_length"."\t".
		  "standard_deviation_length"."\t".
		  "length_min"."\t".
		  "length_max"."\t".
		  "input_seqs"."\t".
		  "procesed_bins"."\t".
		  "processed_seqs"."\t".
		  "OG_DRISEE_score"."\t".
		  "Contam_bins"."\t".
		  "Contam_seqs"."\t".
		  "Contam_DRISEE_score"."\t".
		  "Non-contam_bins"."\t".
		  "Non-contam_seqs"."\t".
		  "Non-contam-DRISEE_score"."\n"
		 );
  close (DATA_LOG);
  }

# create a log for the DRISEE commands that the script issues
unless ( -e $command_log ){
  open(COMMAND_LOG, ">", $command_log) or die "\n\n"."can't open COMMAND_LOG $command_log"."\n\n";			    
  # print a header
  print COMMAND_LOG "# This is a log of the system commands that process_DRISEE2.p has produced."."\n";		    
  close (COMMAND_LOG);
}


my $system_command = (
		      $drisee_path.
		      " -t ".$file_type.
		      " -l ".$drisee_log.
		      " --percent -v".
		      " -p ".$num_proc.
		      " ".$file_in.
		      " ".$stat_file.
		      " > ".$drisee_stdout
		     );
open(COMMAND_LOG, ">>", $command_log) or die "\n\n"."can't RE-open COMMAND_LOG $command_log"."\n\n";
print COMMAND_LOG "DRISEE_command:"."\n".$system_command."\n"."\n";
close(COMMAND_LOG);
# system($system_command);



my @summary_values = ($file_in); # array

while (my $line = <DATA_LOG>){
    chomp $line;
    if ($line =~ m/^V||^bp||^se||^av||^st||^le||^In||^Pr||^Dr||^Con||^Dr/){ # skip comment lines
      chomp $line;
      my @line_array = split("\t", $line);
      my $array_value = $line_array[1];
      push (@summary_values, $array_value);
    }
  }
	
my $summary_line = join("\t", @summary_values);

open(DATA_LOG, ">>", $data_log) or die "\n\n"."can't RE-open DATA_LOG $data_log"."\n\n";
print DATA_LOG $summary_line."\n";
close(DATA_LOG);

# python ~/DRISEE/drisee.py -t fastq -l v1p5_mgm4473069_log --percent -v mgm4473069.fq mgm4473069.v1p5.STAT> v1.5_mgm4473069_stdout
<commit_msg>edit process_DRISEE2.pl<commit_after>#!/usr/bin/env perl

use strict;
use warnings;
use Getopt::Long;


# Default for variables
my $drisee_path = "/home/ubuntu/DRISEE/drisee.py";
my $file_in;
my $file_type = "fastq"; # fasta or fastq
my $num_proc = 1;
#my $percent = 1;
#my $verbose = 1;
my $stat_file;
my $drisee_log;
my $drisee_stdout;
my $data_log = "DRISEE_data_log.txt";
my $command_log = "DRISEE_command_log.txt";
my $help;
my $debug;

if( $debug ){ print STDOUT "made it here"."\n"; }

# check input args and display usage if not suitable
# if ( (@ARGV > 0) && ($ARGV[0] =~ /-h/) ) { &usage(); }
if ( $ARGV[0] =~ /-h/ ) { &usage(); }

unless ( @ARGV > 0 || $file_in ) { &usage(); }
unless ( @ARGV > 0 ) { &usage(); }

if ( ! GetOptions (
		   "p|drisee_path=s"   => \$drisee_path,
		   "f|file_in=s"       => \$file_in,
		   "t|file_type=s"     => \$file_type,
		   "n|num_proc=i"      => \$num_proc,
		   #"p|percent!"        => \$percent,
		   #"v|verbose!"        => \$verbose,
		   "s|stat_file=s"     => \$stat_file,
		   "l|drisee_log=s"    => \$drisee_log,
		   "o|drisee_stdout=s" => \$drisee_stdout,
		   "c|data_log=s"      => \$data_log,
		   "b|command_log=s"   => \$command_log,
		   "h|help!"           => \$help,
		   "d|debug!"          => \$debug
		  )
   ) { &usage(); }


# generate default names for output files 

unless ( defined $stat_file && length $stat_file > 0 )         { $stat_file = $file_in."."."drisee_STAT.txt" };
unless ( defined $drisee_log  && length $drisee_log > 0 )      { $drisee_log = $file_in."."."drisee_log.txt" };
unless ( defined $drisee_stdout && length $drisee_stdout > 0 ) { $drisee_stdout = $file_in."."."drisee_stdout.txt" };
unless ( defined $data_log  && length $data_log > 0 )          { $data_log = "DRISEE_data_log.txt" };
unless ( defined $command_log && length $command_log > 0 )     { $command_log = "DRISEE_command_log.txt" };

# create a cummulative log of the data - add header if the file does not exist
unless (-e $data_log){ 
  open(DATA_LOG, ">", $data_log) or die "\n\n"."can't open DATA_LOG $data_log"."\n\n";
			    
  # print a header
  print DATA_LOG (
		  "# This is a cummulative summary log of the data that process_DRISEE2.pl has produced."."\n".
		  "file_in"."\t".
		  "Version"."\t".
		  "bp_count"."\t".
		  "sequence_count"."\t".
		  "average_length"."\t".
		  "standard_deviation_length"."\t".
		  "length_min"."\t".
		  "length_max"."\t".
		  "input_seqs"."\t".
		  "procesed_bins"."\t".
		  "processed_seqs"."\t".
		  "OG_DRISEE_score"."\t".
		  "Contam_bins"."\t".
		  "Contam_seqs"."\t".
		  "Contam_DRISEE_score"."\t".
		  "Non-contam_bins"."\t".
		  "Non-contam_seqs"."\t".
		  "Non-contam-DRISEE_score"."\n"
		 );
  close (DATA_LOG);
  }

# create a log for the DRISEE commands that the script issues
unless ( -e $command_log ){
  open(COMMAND_LOG, ">", $command_log) or die "\n\n"."can't open COMMAND_LOG $command_log"."\n\n";			    
  # print a header
  print COMMAND_LOG "# This is a log of the system commands that process_DRISEE2.p has produced."."\n";		    
  close (COMMAND_LOG);
}


my $system_command = (
		      $drisee_path.
		      " -t ".$file_type.
		      " -l ".$drisee_log.
		      " --percent -v".
		      " -p ".$num_proc.
		      " ".$file_in.
		      " ".$stat_file.
		      " > ".$drisee_stdout
		     );
open(COMMAND_LOG, ">>", $command_log) or die "\n\n"."can't RE-open COMMAND_LOG $command_log"."\n\n";
print COMMAND_LOG "DRISEE_command:"."\n".$system_command."\n"."\n";
close(COMMAND_LOG);
# system($system_command);



my @summary_values = ($file_in); # array

open(DRISEE_STDOUT, "<", $drisee_stdout) or die "\n\n"."can't open DRISEE_STDOUT $drisee_stdout"."\n\n";
while (my $line = <DRISEE_STDOUT>){
    chomp $line;
    if ($line =~ m/^V||^bp||^se||^av||^st||^le||^In||^Pr||^Dr||^Con||^Dr/){ # skip comment lines
      chomp $line;
      my @line_array = split("\t", $line);
      my $array_value = $line_array[1];
      push (@summary_values, $array_value);
    }
  }
	
my $summary_line = join("\t", @summary_values);

open(DATA_LOG, ">>", $data_log) or die "\n\n"."can't RE-open DATA_LOG $data_log"."\n\n";
print DATA_LOG $summary_line."\n";
close(DATA_LOG);

# python ~/DRISEE/drisee.py -t fastq -l v1p5_mgm4473069_log --percent -v mgm4473069.fq mgm4473069.v1p5.STAT> v1.5_mgm4473069_stdout

{'fraction_non_alphanumeric': 0.1962902089692416, 'fraction_numerical': 0.011857243484386007, 'mean_word_length': 2.929428044280443, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 63, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2356501', 'n_tokens_mistral': 3799, 'n_tokens_neox': 3408, 'n_words': 697}