<commit_before>#!/usr/bin/perl

use strict;
use warnings;
use List::Util qw(max);
use JSON qw(encode_json);
use Capture::Tiny qw(capture);
use Cwd 'abs_path';
use DateTime;

my $dt_format = '%F %T';

my $base_path = shift @ARGV;
my $mt_name = shift @ARGV;
my $wt_name = shift @ARGV;
my $timezone = shift @ARGV;
my $outfile = shift @ARGV;

my $min_epoch = time;
my $max_cpus = max_cpu();

my @algs = qw(ascat pindel caveman brass);
my %alg_elements = (ascat => [qw( allele_count
                                  ascat
                                  finalise)],
                    pindel => [qw(input
                                  pindel
                                  pindel_to_vcf
                                  merge_and_bam
                                  flag)],
                    caveman => [qw( caveman_setup
                                    caveman_split
                                    concat
                                    caveman_mstep
                                    caveman_merge
                                    caveman_estep
                                    caveman_merge_results
                                    caveman_add_ids
                                    caveman_flag)],
                    brass => [qw( input
                                  cover
                                  merge
                                  group
                                  isize
                                  normcn
                                  filter
                                  split
                                  assemble
                                  grass
                                  tabix)],
                    );

my $cgpbox_ver = q{-};
$cgpbox_ver = $ENV{CGPBOX_VERSION} if(exists $ENV{CGPBOX_VERSION});

my $load_trend = [[],[],[],[]];

my $started_at = DateTime->now->set_time_zone($timezone)->strftime($dt_format);

while (1) {
  my ($ref_status, $ref_mod) = setup_status($base_path);
  my ($testdata_status, $testdata_mod) = testdata_status($base_path);
  my ($qc_status, $qc_mod) = qc_status($base_path, $mt_name, $wt_name);

  my @mods = ($min_epoch, # this ensures that archive files can't give daft time-points
              $ref_mod,$testdata_mod,$qc_mod);

  my $complete_dt = completed($base_path, $mt_name, $wt_name);

  open my $OUT, '>', $outfile or die "$!: $outfile";

  # all the simple string variables
  print $OUT sprintf qq{%s = "%s";\n}, 'cgpbox_ver', $cgpbox_ver;
  print $OUT sprintf qq{%s = "%s";\n}, 'mt_name', $mt_name;
  print $OUT sprintf qq{%s = "%s";\n}, 'wt_name', $wt_name;
  print $OUT sprintf qq{%s = "%s";\n}, 'setup_status', $ref_status;
  print $OUT sprintf qq{%s = "%s";\n}, 'testdata_status', $testdata_status;
  print $OUT sprintf qq{%s = "%s";\n}, 'qc_status', $qc_status;
  print $OUT sprintf qq{%s = "%s";\n}, 'last_change', recent_date_from_epoch( \@mods );
  print $OUT sprintf qq{%s = "%s";\n}, 'total_cpus', $max_cpus;
  print $OUT sprintf qq{%s = "%s";\n}, 'started_at', $started_at;
  print $OUT sprintf qq{%s = "%s";\n}, 'completed_at', $complete_dt;
  print $OUT sprintf qq{%s = "%s";\n}, 'load_avg', load_avg($load_trend);

  # encoded variables
  for my $alg(@algs) {
    my (@running, @completed, @labels);
    for my $element(@{$alg_elements{$alg}}) {
      my ($started, $done, $recent) = alg_counts($base_path, $alg, $element, $mt_name, $wt_name);
      push @running, $started;
      push @completed, $done;
      push @mods, $recent;
      $element =~ s/^caveman_//;
      push @labels, $element;
    }
    print $OUT sprintf "%s = %s\n;", $alg, encode_json ${progress_struct($alg, \@running, \@completed, \@labels)};
  }


  print $OUT sprintf qq{load_trend = %s;\n}, encode_json ${trend_struct($load_trend)};

  close $OUT;

  if($complete_dt ne q{-}) {
    print "Workflow completed, shutting down monitoring\n";
    exit 0;
  }

  sleep 30;
}

sub completed {
  my ($base_path, $mt_name, $wt_name) = @_;
  my $logs_moved = 0;
  for my $alg(@algs) {
    my $alg_base = sprintf "%s/output/%s_vs_%s/%s", $base_path, $mt_name, $wt_name, $alg;
    $logs_moved++ if(-e "$alg_base/logs");
  }
  my $ret = q{-};
  $ret = DateTime->now->set_time_zone($timezone)->strftime($dt_format) if($logs_moved == @algs);
  return $ret;
}

sub trend_struct {<commit_msg>new line in daft place<commit_after>#!/usr/bin/perl

use strict;
use warnings;
use List::Util qw(max);
use JSON qw(encode_json);
use Capture::Tiny qw(capture);
use Cwd 'abs_path';
use DateTime;

my $dt_format = '%F %T';

my $base_path = shift @ARGV;
my $mt_name = shift @ARGV;
my $wt_name = shift @ARGV;
my $timezone = shift @ARGV;
my $outfile = shift @ARGV;

my $min_epoch = time;
my $max_cpus = max_cpu();

my @algs = qw(ascat pindel caveman brass);
my %alg_elements = (ascat => [qw( allele_count
                                  ascat
                                  finalise)],
                    pindel => [qw(input
                                  pindel
                                  pindel_to_vcf
                                  merge_and_bam
                                  flag)],
                    caveman => [qw( caveman_setup
                                    caveman_split
                                    concat
                                    caveman_mstep
                                    caveman_merge
                                    caveman_estep
                                    caveman_merge_results
                                    caveman_add_ids
                                    caveman_flag)],
                    brass => [qw( input
                                  cover
                                  merge
                                  group
                                  isize
                                  normcn
                                  filter
                                  split
                                  assemble
                                  grass
                                  tabix)],
                    );

my $cgpbox_ver = q{-};
$cgpbox_ver = $ENV{CGPBOX_VERSION} if(exists $ENV{CGPBOX_VERSION});

my $load_trend = [[],[],[],[]];

my $started_at = DateTime->now->set_time_zone($timezone)->strftime($dt_format);

while (1) {
  my ($ref_status, $ref_mod) = setup_status($base_path);
  my ($testdata_status, $testdata_mod) = testdata_status($base_path);
  my ($qc_status, $qc_mod) = qc_status($base_path, $mt_name, $wt_name);

  my @mods = ($min_epoch, # this ensures that archive files can't give daft time-points
              $ref_mod,$testdata_mod,$qc_mod);

  my $complete_dt = completed($base_path, $mt_name, $wt_name);

  open my $OUT, '>', $outfile or die "$!: $outfile";

  # all the simple string variables
  print $OUT sprintf qq{%s = "%s";\n}, 'cgpbox_ver', $cgpbox_ver;
  print $OUT sprintf qq{%s = "%s";\n}, 'mt_name', $mt_name;
  print $OUT sprintf qq{%s = "%s";\n}, 'wt_name', $wt_name;
  print $OUT sprintf qq{%s = "%s";\n}, 'setup_status', $ref_status;
  print $OUT sprintf qq{%s = "%s";\n}, 'testdata_status', $testdata_status;
  print $OUT sprintf qq{%s = "%s";\n}, 'qc_status', $qc_status;
  print $OUT sprintf qq{%s = "%s";\n}, 'last_change', recent_date_from_epoch( \@mods );
  print $OUT sprintf qq{%s = "%s";\n}, 'total_cpus', $max_cpus;
  print $OUT sprintf qq{%s = "%s";\n}, 'started_at', $started_at;
  print $OUT sprintf qq{%s = "%s";\n}, 'completed_at', $complete_dt;
  print $OUT sprintf qq{%s = "%s";\n}, 'load_avg', load_avg($load_trend);

  # encoded variables
  for my $alg(@algs) {
    my (@running, @completed, @labels);
    for my $element(@{$alg_elements{$alg}}) {
      my ($started, $done, $recent) = alg_counts($base_path, $alg, $element, $mt_name, $wt_name);
      push @running, $started;
      push @completed, $done;
      push @mods, $recent;
      $element =~ s/^caveman_//;
      push @labels, $element;
    }
    print $OUT sprintf "%s = %s;\n", $alg, encode_json ${progress_struct($alg, \@running, \@completed, \@labels)};
  }


  print $OUT sprintf qq{load_trend = %s;\n}, encode_json ${trend_struct($load_trend)};

  close $OUT;

  if($complete_dt ne q{-}) {
    print "Workflow completed, shutting down monitoring\n";
    exit 0;
  }

  sleep 30;
}

sub completed {
  my ($base_path, $mt_name, $wt_name) = @_;
  my $logs_moved = 0;
  for my $alg(@algs) {
    my $alg_base = sprintf "%s/output/%s_vs_%s/%s", $base_path, $mt_name, $wt_name, $alg;
    $logs_moved++ if(-e "$alg_base/logs");
  }
  my $ret = q{-};
  $ret = DateTime->now->set_time_zone($timezone)->strftime($dt_format) if($logs_moved == @algs);
  return $ret;
}

sub trend_struct {
{'fraction_non_alphanumeric': 0.1434696952138139, 'fraction_numerical': 0.0011588828369451848, 'mean_word_length': 1.843492586490939, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 48, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4359498', 'n_tokens_mistral': 2963, 'n_tokens_neox': 2854, 'n_words': 664}