<commit_before>#!/usr/bin/env perl

# Gets the activity of each analysis along time, in a CSV file or in an image (see list of formats supported by GNUplot)

use strict;
use warnings;

    # Finding out own path in order to reference own components (including own modules):
use Cwd            ();
use File::Basename ();
BEGIN {
    $ENV{'EHIVE_ROOT_DIR'} ||= File::Basename::dirname( File::Basename::dirname( Cwd::realpath($0) ) );
    unshift @INC, $ENV{'EHIVE_ROOT_DIR'}.'/modules';
}


use Getopt::Long;
use List::Util qw(sum);
use POSIX;
use Data::Dumper;

use Bio::EnsEMBL::Hive::DBSQL::DBAdaptor;
use Bio::EnsEMBL::Hive::Utils ('script_usage');

no warnings qw{qw};

main();
exit(0);

sub main {

    my ($url, $reg_conf, $reg_type, $reg_alias, $nosqlvc, $help, $verbose, $mode, $start_date, $end_date, $output, $top, $default_memory, $default_cores);

    GetOptions(
                # connect to the database:
            'url=s'                      => \$url,
            'reg_conf|regfile=s'         => \$reg_conf,
            'reg_type=s'                 => \$reg_type,
            'reg_alias|regname=s'        => \$reg_alias,
            'nosqlvc=i'                  => \$nosqlvc,      # using "=i" instead of "!" for consistency with scripts where it is a propagated option

            'verbose!'                   => \$verbose,
            'h|help'                     => \$help,

            'start_date=s'               => \$start_date,
            'end_date=s'                 => \$end_date,
            'mode=s'                     => \$mode,
            'top=f'                      => \$top,
            'mem=i'                      => \$default_memory,
            'n_core=i'                   => \$default_cores,
            'output=s'                   => \$output,
    );

    if ($help) { script_usage(0); }

    my $hive_dba;
    if($url or $reg_alias) {
        $hive_dba = Bio::EnsEMBL::Hive::DBSQL::DBAdaptor->new(
                -url                            => $url,
                -reg_conf                       => $reg_conf,
                -reg_type                       => $reg_type,
                -reg_alias                      => $reg_alias,
                -no_sql_schema_version_check    => $nosqlvc,
        );
    } else {
        warn "\nERROR: Connection parameters (url or reg_conf+reg_alias) need to be specified\n";
        script_usage(1);
    }

    # Check whether $mode is valid
    my %allowed_modes = (
        workers => 'Number of workers',
        memory => 'Memory asked (Gb)',
        cores => 'Number of CPU cores',
        unused_memory => 'Unused memory (Gb)',
        unused_cores => 'Number of unused CPU cores',
        pending_workers => 'Number of pending workers',
    );
    if ($mode) {
        die "Unknown mode '$mode'. Allowed modes are: ".join(", ", keys %allowed_modes) unless exists $allowed_modes{$mode};
        $default_memory = 100 unless $default_memory;
        $default_cores = 1 unless $default_cores;
    } else {
        $mode = 'workers';
    }

    # Palette generated with R: c(brewer.pal(9, "Set1"), brewer.pal(12, "Set3")). #FFFFB3 is removed because it is too close to white
    my @palette = qw(#E41A1C #377EB8 #4DAF4A #984EA3 #FF7F00 #FFFF33 #A65628 #F781BF #999999     #8DD3C7 #BEBADA #FB8072 #80B1D3 #FDB462 #B3DE69 #FCCDE5 #D9D9D9 #BC80BD #CCEBC5 #FFED6F    #2F4F4F);

    my %terminal_mapping = (
        'emf' => 'emf',
        'png' => 'png',
        'svg' => 'svg',
        'jpg' => 'jpeg',
        'gif' => 'gif',
        'ps'  => 'postscript eps enhanced color',
        'pdf' => 'pdf color enhanced',
    );
    my $gnuplot_terminal = undef;
    if ($output and $output =~ /\.(\w+)$/) {
        $gnuplot_terminal = $1;
        die "The format '$gnuplot_terminal' is not currently supported." if not exists $terminal_mapping{$gnuplot_terminal};
        require Chart::Gnuplot;

    }


    my $dbh = $hive_dba->dbc->db_handle();

    # Get the memory usage from each resource_class
    my %mem_resources = ();
    my %cpu_resources = ();
    {
        foreach my $rd (@{$hive_dba->get_ResourceDescriptionAdaptor->fetch_all()}) {
            if ($rd->meadow_type eq 'LSF') {
                $mem_resources{$rd->resource_class_id} = $1 if $rd->submission_cmd_args =~ m/mem=(\d+)/;
                $cpu_resources{$rd->resource_class_id} = $1 if $rd->submission_cmd_args =~ m/-n\s*(\d+)/;
            }
        }
    }
    warn "mem_resources: ", Dumper \%mem_resources if $verbose;
    warn "cpu_resources: ", Dumper \%cpu_resources if $verbose;

    # Get the memory used by each worker
    my %used_res = ();
    if (($mode eq 'unused_memory') or ($mode eq 'unused_cores')) {
        my $sql_used_res = 'SELECT worker_id, mem_megs, cpu_sec/lifespan_sec FROM worker_resource_usage';
        foreach my $db_entry (@{$dbh->selectall_arrayref($sql_used_res)}) {
            my ($worker_id, $mem_megs, $cpu_usage) = @$db_entry;
            $used_res{$worker_id} = [$mem_megs, $cpu_usage];
        }
        warn scalar(keys %used_res), " worker info loaded from worker_resource_usage\n" if $verbose;
    }

    # Get the info about the analysis
    my %default_resource_class = ();
    my %analysis_name = ();
    {
        foreach my $analysis (@{$hive_dba->get_AnalysisAdaptor->fetch_all()}) {
            $analysis_name{$analysis->dbID} = $analysis->logic_name;
            $default_resource_class{$analysis->dbID} = $analysis->resource_class_id;
        }
    }
    warn "default_resource_class: ", Dumper \%default_resource_class if $verbose;
    warn "analysis_name: ", Dumper \%analysis_name if $verbose;
    warn scalar(keys %analysis_name), " analysis\n" if $verbose;

    # Get the events from the database
    my %events = ();
    if ($mode ne 'pending_workers') {
        my @tmp_dates = @{$dbh->selectall_arrayref('SELECT DATE_FORMAT(when_started, "%Y-%m-%dT%T"), DATE_FORMAT(when_finished, "%Y-%m-%dT%T"), analysis_id, worker_id FROM role')};
        warn scalar(@tmp_dates), " events\n" if $verbose;

        foreach my $db_entry (@tmp_dates) {
            my ($birth_date, $death_date, $analysis_id, $worker_id) = @$db_entry;
            my $resource_class_id = $default_resource_class{$analysis_id};  # ToDo: fetch it from the Worker object first, but if not available - take it from the default_resource_class hash
            my $offset = 0;

            if ($mode eq 'workers') {
                $offset = 1;
            } elsif ($mode eq 'memory') {
                $offset = ($mem_resources{$resource_class_id} || $default_memory) / 1024.;
            } elsif ($mode eq 'cores') {<commit_msg>a more correct treatment of resource_class_id (first call is from a specific worker, and then from the analysis)<commit_after>#!/usr/bin/env perl

# Gets the activity of each analysis along time, in a CSV file or in an image (see list of formats supported by GNUplot)

use strict;
use warnings;

    # Finding out own path in order to reference own components (including own modules):
use Cwd            ();
use File::Basename ();
BEGIN {
    $ENV{'EHIVE_ROOT_DIR'} ||= File::Basename::dirname( File::Basename::dirname( Cwd::realpath($0) ) );
    unshift @INC, $ENV{'EHIVE_ROOT_DIR'}.'/modules';
}


use Getopt::Long;
use List::Util qw(sum);
use POSIX;
use Data::Dumper;

use Bio::EnsEMBL::Hive::DBSQL::DBAdaptor;
use Bio::EnsEMBL::Hive::Utils ('script_usage');

no warnings qw{qw};

main();
exit(0);

sub main {

    my ($url, $reg_conf, $reg_type, $reg_alias, $nosqlvc, $help, $verbose, $mode, $start_date, $end_date, $output, $top, $default_memory, $default_cores);

    GetOptions(
                # connect to the database:
            'url=s'                      => \$url,
            'reg_conf|regfile=s'         => \$reg_conf,
            'reg_type=s'                 => \$reg_type,
            'reg_alias|regname=s'        => \$reg_alias,
            'nosqlvc=i'                  => \$nosqlvc,      # using "=i" instead of "!" for consistency with scripts where it is a propagated option

            'verbose!'                   => \$verbose,
            'h|help'                     => \$help,

            'start_date=s'               => \$start_date,
            'end_date=s'                 => \$end_date,
            'mode=s'                     => \$mode,
            'top=f'                      => \$top,
            'mem=i'                      => \$default_memory,
            'n_core=i'                   => \$default_cores,
            'output=s'                   => \$output,
    );

    if ($help) { script_usage(0); }

    my $hive_dba;
    if($url or $reg_alias) {
        $hive_dba = Bio::EnsEMBL::Hive::DBSQL::DBAdaptor->new(
                -url                            => $url,
                -reg_conf                       => $reg_conf,
                -reg_type                       => $reg_type,
                -reg_alias                      => $reg_alias,
                -no_sql_schema_version_check    => $nosqlvc,
        );
    } else {
        warn "\nERROR: Connection parameters (url or reg_conf+reg_alias) need to be specified\n";
        script_usage(1);
    }

    # Check whether $mode is valid
    my %allowed_modes = (
        workers => 'Number of workers',
        memory => 'Memory asked (Gb)',
        cores => 'Number of CPU cores',
        unused_memory => 'Unused memory (Gb)',
        unused_cores => 'Number of unused CPU cores',
        pending_workers => 'Number of pending workers',
    );
    if ($mode) {
        die "Unknown mode '$mode'. Allowed modes are: ".join(", ", keys %allowed_modes) unless exists $allowed_modes{$mode};
        $default_memory = 100 unless $default_memory;
        $default_cores = 1 unless $default_cores;
    } else {
        $mode = 'workers';
    }

    # Palette generated with R: c(brewer.pal(9, "Set1"), brewer.pal(12, "Set3")). #FFFFB3 is removed because it is too close to white
    my @palette = qw(#E41A1C #377EB8 #4DAF4A #984EA3 #FF7F00 #FFFF33 #A65628 #F781BF #999999     #8DD3C7 #BEBADA #FB8072 #80B1D3 #FDB462 #B3DE69 #FCCDE5 #D9D9D9 #BC80BD #CCEBC5 #FFED6F    #2F4F4F);

    my %terminal_mapping = (
        'emf' => 'emf',
        'png' => 'png',
        'svg' => 'svg',
        'jpg' => 'jpeg',
        'gif' => 'gif',
        'ps'  => 'postscript eps enhanced color',
        'pdf' => 'pdf color enhanced',
    );
    my $gnuplot_terminal = undef;
    if ($output and $output =~ /\.(\w+)$/) {
        $gnuplot_terminal = $1;
        die "The format '$gnuplot_terminal' is not currently supported." if not exists $terminal_mapping{$gnuplot_terminal};
        require Chart::Gnuplot;

    }


    my $dbh = $hive_dba->dbc->db_handle();

    # Get the memory usage from each resource_class
    my %mem_resources = ();
    my %cpu_resources = ();
    {
        foreach my $rd (@{$hive_dba->get_ResourceDescriptionAdaptor->fetch_all()}) {
            if ($rd->meadow_type eq 'LSF') {
                $mem_resources{$rd->resource_class_id} = $1 if $rd->submission_cmd_args =~ m/mem=(\d+)/;
                $cpu_resources{$rd->resource_class_id} = $1 if $rd->submission_cmd_args =~ m/-n\s*(\d+)/;
            }
        }
    }
    warn "mem_resources: ", Dumper \%mem_resources if $verbose;
    warn "cpu_resources: ", Dumper \%cpu_resources if $verbose;

    # Get the memory used by each worker
    my %used_res = ();
    if (($mode eq 'unused_memory') or ($mode eq 'unused_cores')) {
        my $sql_used_res = 'SELECT worker_id, mem_megs, cpu_sec/lifespan_sec FROM worker_resource_usage';
        foreach my $db_entry (@{$dbh->selectall_arrayref($sql_used_res)}) {
            my ($worker_id, $mem_megs, $cpu_usage) = @$db_entry;
            $used_res{$worker_id} = [$mem_megs, $cpu_usage];
        }
        warn scalar(keys %used_res), " worker info loaded from worker_resource_usage\n" if $verbose;
    }

    # Get the info about the analysis
    my %default_resource_class = ();
    my %analysis_name = ();
    {
        foreach my $analysis (@{$hive_dba->get_AnalysisAdaptor->fetch_all()}) {
            $analysis_name{$analysis->dbID} = $analysis->logic_name;
            $default_resource_class{$analysis->dbID} = $analysis->resource_class_id;
        }
    }
    warn "default_resource_class: ", Dumper \%default_resource_class if $verbose;
    warn "analysis_name: ", Dumper \%analysis_name if $verbose;
    warn scalar(keys %analysis_name), " analysis\n" if $verbose;

    # Get the events from the database
    my %events = ();
    if ($mode ne 'pending_workers') {
        my @tmp_dates = @{$dbh->selectall_arrayref('SELECT DATE_FORMAT(when_started, "%Y-%m-%dT%T"), DATE_FORMAT(when_finished, "%Y-%m-%dT%T"), analysis_id, worker_id, resource_class_id FROM role JOIN worker USING (worker_id)')};
        warn scalar(@tmp_dates), " events\n" if $verbose;

        foreach my $db_entry (@tmp_dates) {
            my ($birth_date, $death_date, $analysis_id, $worker_id, $resource_class_id) = @$db_entry;
            $resource_class_id //= $default_resource_class{$analysis_id};
            my $offset = 0;

            if ($mode eq 'workers') {
                $offset = 1;
            } elsif ($mode eq 'memory') {
                $offset = ($mem_resources{$resource_class_id} || $default_memory) / 1024.;
            } elsif ($mode eq 'cores') {
{'fraction_non_alphanumeric': 0.1411693882158176, 'fraction_numerical': 0.012491534351719466, 'mean_word_length': 2.218697021070477, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 103, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4984054', 'n_tokens_mistral': 4477, 'n_tokens_neox': 4215, 'n_words': 1052}