<commit_before>package PipelineAWE;

use strict;
use warnings;
no warnings('once');

use JSON;
use POSIX;
use DateTime;
use DateTime::Format::ISO8601;
use Data::Dumper;
use LWP::UserAgent;
use HTTP::Request;
use Capture::Tiny qw(:all);

our $post_attempt = 0;
our $debug = 1;
our $layout = '[%d] [%-5p] %m%n';
our $shock_api = "http://shock-internal.metagenomics.anl.gov";
our $default_api = "http://api-internal.metagenomics.anl.gov";
our $proxy_url = "http://proxy.metagenomics.anl.gov";
our $host_api = "api-internal.metagenomics.anl.gov";
our $mg_email = '"Metagenomics Analysis Server" <<EMAIL>>';
our $mg_smtp = 'smtp.mcs.anl.gov';
our $global_attr = "userattr.json";
our $agent = LWP::UserAgent->new();
$agent->timeout(3600);
our $json = JSON->new;
$json = $json->utf8();
$json->max_size(0);
$json->allow_nonref;


######### Set logging  ##########

use Log::Log4perl qw(:easy);
if ($debug) {
    Log::Log4perl->easy_init({level => $DEBUG, layout => $layout});
} else {
    Log::Log4perl->easy_init({level => $INFO, layout => $layout});
}
our $logger = Log::Log4perl->get_logger();

######### Helper Functions ##########

sub logger {
    my ($type, $msg) = @_;
    # replace line breaks
    $msg =~ s/\n/, /g;
    # find logger channel
    if ($type eq 'debug') {
        $logger->debug($msg);
    } elsif ($type eq 'info') {
        $logger->info($msg);
    } elsif ($type eq 'warn') {
        $logger->warn($msg);
    } elsif ($type eq 'error') {
        $logger->error($msg);
    }
}

sub run_cmd {
    my ($cmd, $shell) = @_;
    my $status = undef;
    my $stderr = undef;
    my @parts  = split(/ /, $cmd);
    my $start  = time;
    logger('info', "exec: ".$cmd);
    eval {
        if ($shell) {
            (undef, $stderr, $status) = capture {
                system($cmd);
            };
        } else {
            (undef, $stderr, $status) = capture {
                system(@parts);
            };
        }
    };
    my $done = time;
    logger('info', "time: ".($done - $start)." secs");
    if ($@ || $status) {
        logger('error', "died running child process ".$parts[0]);
        logger('debug', $parts[0].": ".$@);
        if ($stderr) {
            logger('debug', "STDERR: ".$stderr);
        }
        if (defined($status) && ($status != 0)) {
            logger('error', $parts[0]." returns value $status");
            exit $status >> 8;
        }
        exit 1;
    }
}

sub file_to_array {
    my ($file) = @_;
    my $data = [];
    unless ($file && (-s $file)) {
        return $data;
    }
    open(FILE, "<$file") || return $data;
    while (my $line = <FILE>) {
        chomp $line;
        my @parts = split(/\t/, $line);
        push @$data, [ @parts ];
    }<commit_msg>sortmerna runs OOM, exit failed-permanent<commit_after>package PipelineAWE;

use strict;
use warnings;
no warnings('once');

use JSON;
use POSIX;
use DateTime;
use DateTime::Format::ISO8601;
use Data::Dumper;
use LWP::UserAgent;
use HTTP::Request;
use Capture::Tiny qw(:all);

our $post_attempt = 0;
our $debug = 1;
our $layout = '[%d] [%-5p] %m%n';
our $shock_api = "http://shock-internal.metagenomics.anl.gov";
our $default_api = "http://api-internal.metagenomics.anl.gov";
our $proxy_url = "http://proxy.metagenomics.anl.gov";
our $host_api = "api-internal.metagenomics.anl.gov";
our $mg_email = '"Metagenomics Analysis Server" <<EMAIL>>';
our $mg_smtp = 'smtp.mcs.anl.gov';
our $global_attr = "userattr.json";
our $agent = LWP::UserAgent->new();
$agent->timeout(3600);
our $json = JSON->new;
$json = $json->utf8();
$json->max_size(0);
$json->allow_nonref;


######### Set logging  ##########

use Log::Log4perl qw(:easy);
if ($debug) {
    Log::Log4perl->easy_init({level => $DEBUG, layout => $layout});
} else {
    Log::Log4perl->easy_init({level => $INFO, layout => $layout});
}
our $logger = Log::Log4perl->get_logger();

######### Helper Functions ##########

sub logger {
    my ($type, $msg) = @_;
    # replace line breaks
    $msg =~ s/\n/, /g;
    # find logger channel
    if ($type eq 'debug') {
        $logger->debug($msg);
    } elsif ($type eq 'info') {
        $logger->info($msg);
    } elsif ($type eq 'warn') {
        $logger->warn($msg);
    } elsif ($type eq 'error') {
        $logger->error($msg);
    }
}

sub run_cmd {
    my ($cmd, $shell) = @_;
    my $status = undef;
    my $stderr = undef;
    my @parts  = split(/ /, $cmd);
    my $start  = time;
    logger('info', "exec: ".$cmd);
    eval {
        if ($shell) {
            (undef, $stderr, $status) = capture {
                system($cmd);
            };
        } else {
            (undef, $stderr, $status) = capture {
                system(@parts);
            };
        }
    };
    my $done = time;
    logger('info', "time: ".($done - $start)." secs");
    if ($@ || $status) {
        logger('error', "died running child process ".$parts[0]);
        logger('debug', $parts[0].": ".$@);
        if ($stderr) {
            logger('debug', "STDERR: ".$stderr);
        }
        # special case, sortmerna runs OOM, exit failed-permanent
        if (($parts[0] eq "sortmerna") && ($stderr =~ /Segmentation/)) {
            exit 42;
        }
        if (defined($status) && ($status != 0)) {
            logger('error', $parts[0]." returns value $status");
            exit $status >> 8;
        }
        exit 1;
    }
}

sub file_to_array {
    my ($file) = @_;
    my $data = [];
    unless ($file && (-s $file)) {
        return $data;
    }
    open(FILE, "<$file") || return $data;
    while (my $line = <FILE>) {
        chomp $line;
        my @parts = split(/\t/, $line);
        push @$data, [ @parts ];
    }
{'fraction_non_alphanumeric': 0.20911670805250088, 'fraction_numerical': 0.008691025186236254, 'mean_word_length': 2.5221736414740787, 'pattern_counts': {'":': 2, '<': 11, '<?xml version=': 0, '>': 47, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4467815', 'n_tokens_mistral': 2042, 'n_tokens_neox': 1964, 'n_words': 473}