<commit_before>package EASIH::JMS;
# 
# JobManagementSystem frame for running pipelines everywhere!
# 
# 
# <NAME> (23 Apr 2010), contact: <EMAIL>

use strict;
use warnings;
use Data::Dumper;
use Storable;
use File::Temp;
use Time::HiRes;

use EASIH::JMS::Hive;

my $last_save      =   0;
my $save_interval  = 300;
my $verbose_level  =   0;
my $max_retry      =   3;
my $jobs_submitted =   0;
my $sleep_time     =   5;
my $current_logic_name;
my $use_storing    =   1; # debugging purposes
my $max_jobs       =  -1; # to control that we do not flood Darwin, or if local, block the machine. -1 is no limit
my @argv; # the argv from main is fetched at load time, and a copy kept here so we can store it later

my $no_restart     =   0; # failed jobs that cannot be restarted. 

# default dummy hive that will fail gracefully, and the class that every other hive
# should inherit from.
my $hive           = "EASIH::JMS::Hive";

my @delete_files;
my %jms_hash;
my @jms_ids;

my @retained_jobs;
my %analysis_order;

my $job_counter = 1; # This is for generating internal jms_id (JobManamentSystem_Id)

our $cwd      = `pwd`;
chomp($cwd);

my $dry_run  = 0;
my %dependencies;


our $FINISHED    =    1;
our $FAILED      =    2;
our $RUNNING     =    3;
our $QUEUEING    =    4;
our $RESUBMITTED =    5;
our $SUBMITTED   =    6;
our $UNKNOWN     =  100;

my %s2status = ( 1   =>  "Finished",
		 2   =>  "Failed",
		 3   =>  "Running",
		 4   =>  "Queueing",
		 5   =>  "Resubmitted",
		 6   =>  "Submitted",
		 100 =>  "Unknown");


# 
# 
# 
# <NAME> (23 Apr 2010)
sub verbosity {
  $verbose_level = shift || 0;
}



# 
# 
# 
# <NAME> (05 Jul 2010)
sub verbose {
  my ( $message, $level) = @_;
  $level ||= 0;
  
  return if ( $level > $verbose_level);
  $message =~ s/\Z//;
  print " ::::" . " $message\n";

}


# 
# 
# 
# <NAME> (23 Apr 2010)
sub fail {
  my ($message ) = @_;

  print STDERR "ERROR :: $message\n";
  store_state();
  exit;
  
}



# 
# 
# 
# <NAME> (24 Jun 2010)
sub hive {
  $hive = shift;
  
  if ( $hive ) {
    # strip away the the expected class
    $hive =~ s/EASIH::JMS::Hive:://;
    # and (re)append it (again);
    $hive = "EASIH::JMS::Hive::".$hive;
  }

  return $hive;
}


# 
# 
# 
# <NAME> (23 Apr 2010)
sub sleep_time {
  $sleep_time = shift || 60;
}


# 
# -1 is never or only on crashes
# 
# <NAME> (23 Apr 2010)
sub save_interval {
  $save_interval = shift || -1;
}



# 
# Checks and see if the current state of the run should be stored
# The inverval of this happening is set with save_interval
#
# <NAME> (04 May 2010)
sub check_n_store_state {

  return if ( $save_interval == -1 );
  
  my $now = Time::HiRes::gettimeofday();
  store_state() if ($now - $last_save > $save_interval );
}


# 
# 
# 
# <NAME> (23 Apr 2010)
sub max_retry {
  $max_retry = shift || 0;
  $max_retry = 0 if ( $max_retry < 0);
}


# 
# Setting the working directory, if different than the cwd
# 
# <NAME> (23 Apr 2010)
sub cwd {
  my ($new_cwd) = @_;
  $cwd = $new_cwd;
}



# 
# 
# 
# <NAME> (05 Jul 2010)
sub submit_system_job {
  my ($cmd, $output, $system) = @_;
  submit_job($cmd, $output, 1);
}


# 
# submit a single job if $system is then a single system call is doing the work!
# 
# <NAME> (22 Apr 2010)
sub submit_job {
  my ($cmd, $output, $system) = @_;
<commit_msg>removed useless call parameter<commit_after>package EASIH::JMS;
# 
# JobManagementSystem frame for running pipelines everywhere!
# 
# 
# <NAME> (23 Apr 2010), contact: <EMAIL>

use strict;
use warnings;
use Data::Dumper;
use Storable;
use File::Temp;
use Time::HiRes;

use EASIH::JMS::Hive;

my $last_save      =   0;
my $save_interval  = 300;
my $verbose_level  =   0;
my $max_retry      =   3;
my $jobs_submitted =   0;
my $sleep_time     =   5;
my $current_logic_name;
my $use_storing    =   1; # debugging purposes
my $max_jobs       =  -1; # to control that we do not flood Darwin, or if local, block the machine. -1 is no limit
my @argv; # the argv from main is fetched at load time, and a copy kept here so we can store it later

my $no_restart     =   0; # failed jobs that cannot be restarted. 

# default dummy hive that will fail gracefully, and the class that every other hive
# should inherit from.
my $hive           = "EASIH::JMS::Hive";

my @delete_files;
my %jms_hash;
my @jms_ids;

my @retained_jobs;
my %analysis_order;

my $job_counter = 1; # This is for generating internal jms_id (JobManamentSystem_Id)

our $cwd      = `pwd`;
chomp($cwd);

my $dry_run  = 0;
my %dependencies;


our $FINISHED    =    1;
our $FAILED      =    2;
our $RUNNING     =    3;
our $QUEUEING    =    4;
our $RESUBMITTED =    5;
our $SUBMITTED   =    6;
our $UNKNOWN     =  100;

my %s2status = ( 1   =>  "Finished",
		 2   =>  "Failed",
		 3   =>  "Running",
		 4   =>  "Queueing",
		 5   =>  "Resubmitted",
		 6   =>  "Submitted",
		 100 =>  "Unknown");


# 
# 
# 
# <NAME> (23 Apr 2010)
sub verbosity {
  $verbose_level = shift || 0;
}



# 
# 
# 
# <NAME> (05 Jul 2010)
sub verbose {
  my ( $message, $level) = @_;
  $level ||= 0;
  
  return if ( $level > $verbose_level);
  $message =~ s/\Z//;
  print " ::::" . " $message\n";

}


# 
# 
# 
# <NAME> (23 Apr 2010)
sub fail {
  my ($message ) = @_;

  print STDERR "ERROR :: $message\n";
  store_state();
  exit;
  
}



# 
# 
# 
# <NAME> (24 Jun 2010)
sub hive {
  $hive = shift;
  
  if ( $hive ) {
    # strip away the the expected class
    $hive =~ s/EASIH::JMS::Hive:://;
    # and (re)append it (again);
    $hive = "EASIH::JMS::Hive::".$hive;
  }

  return $hive;
}


# 
# 
# 
# <NAME> (23 Apr 2010)
sub sleep_time {
  $sleep_time = shift || 60;
}


# 
# -1 is never or only on crashes
# 
# <NAME> (23 Apr 2010)
sub save_interval {
  $save_interval = shift || -1;
}



# 
# Checks and see if the current state of the run should be stored
# The inverval of this happening is set with save_interval
#
# <NAME> (04 May 2010)
sub check_n_store_state {

  return if ( $save_interval == -1 );
  
  my $now = Time::HiRes::gettimeofday();
  store_state() if ($now - $last_save > $save_interval );
}


# 
# 
# 
# <NAME> (23 Apr 2010)
sub max_retry {
  $max_retry = shift || 0;
  $max_retry = 0 if ( $max_retry < 0);
}


# 
# Setting the working directory, if different than the cwd
# 
# <NAME> (23 Apr 2010)
sub cwd {
  my ($new_cwd) = @_;
  $cwd = $new_cwd;
}



# 
# 
# 
# <NAME> (05 Jul 2010)
sub submit_system_job {
  my ($cmd, $output) = @_;
  submit_job($cmd, $output, 1);
}


# 
# submit a single job if $system is then a single system call is doing the work!
# 
# <NAME> (22 Apr 2010)
sub submit_job {
  my ($cmd, $output, $system) = @_;

{'fraction_non_alphanumeric': 0.14728211357424842, 'fraction_numerical': 0.035226237473428484, 'mean_word_length': 2.805314846909301, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 47, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4585133', 'n_tokens_mistral': 2873, 'n_tokens_neox': 2619, 'n_words': 796}