<commit_before>#!/usr/bin/env perl

use strict;
use warnings;

    # Finding out own path in order to reference own components (including own modules):
use Cwd            ();
use File::Basename ();
BEGIN {
    $ENV{'EHIVE_ROOT_DIR'} ||= File::Basename::dirname( File::Basename::dirname( Cwd::realpath($0) ) );
    unshift @INC, $ENV{'EHIVE_ROOT_DIR'}.'/modules';
}


use Getopt::Long;

use Bio::EnsEMBL::Hive::HivePipeline;
use Bio::EnsEMBL::Hive::Utils ('destringify');
use Bio::EnsEMBL::Hive::Utils::GraphViz;


my $self = {};

main();


sub main {

    my $pipeline;

    GetOptions(
            # connection parameters
        'url=s'                 => \$self->{'url'},
        'reg_conf|reg_file=s'   => \$self->{'reg_conf'},
        'reg_type=s'            => \$self->{'reg_type'},
        'reg_alias|reg_name=s'  => \$self->{'reg_alias'},
        'nosqlvc=i'             => \$self->{'nosqlvc'},             # using "=i" instead of "!" for consistency with scripts where it is a propagated option

        'job_id=s@'             => \$self->{'job_ids'},             # jobs to start from
        'start_analysis_name=s' => \$self->{'start_analysis_name'}, # if given, first trace the graph up to the given analysis or the seed_jobs, and then start visualization
        'stop_analysis_name=s'  => \$self->{'stop_analysis_name'},  # if given, the visualization is aborted at that analysis and doesn't go any further

        'o|out|output=s'        => \$self->{'output'},
        'dot_input=s'           => \$self->{'dot_input'},   # filename to store the intermediate dot input (valuable for debugging)

        'h|help'                => \$self->{'help'},
    );

    if($self->{'url'} or $self->{'reg_alias'}) {
        $pipeline = Bio::EnsEMBL::Hive::HivePipeline->new(
            -url                            => $self->{'url'},
            -reg_conf                       => $self->{'reg_conf'},
            -reg_type                       => $self->{'reg_type'},
            -reg_alias                      => $self->{'reg_alias'},
            -no_sql_schema_version_check    => $self->{'nosqlvc'},
        );

    } else {
        die "\nERROR : Connection parameters (url or reg_conf+reg_alias) need to be specified\n\n";
    }

    if($self->{'output'}) {

        if(!$self->{'format'}) {
            if($self->{'output'}=~/\.(\w+)$/) {
                $self->{'format'} = $1;
            } else {
                die "Format was not set and could not guess from ".$self->{'output'}.". Please use either way to select it.\n";
            }
        }

        $self->{'graph'} = Bio::EnsEMBL::Hive::Utils::GraphViz->new(
                    'name'          => 'JobDependencyGraph',
                    'concentrate'   => 'true',
                    'pad'           => 1,
        );

        $self->{'graph'}->cluster_2_nodes( {} );
        $self->{'graph'}->main_pipeline_name( $pipeline->hive_pipeline_name );
        $self->{'graph'}->other_pipeline_bgcolour( [ 'pastel19', 3 ] );
        $self->{'graph'}->display_cluster_names( 1 );

        my $job_adaptor     = $pipeline->hive_dba->get_AnalysisJobAdaptor;
        my $anchor_jobs     = $self->{'job_ids'} && $job_adaptor->fetch_all( 'job_id IN ('.join(',', @{$self->{'job_ids'}} ).')' );
        my $start_analysis  = $self->{'start_analysis_name'} && $pipeline->find_by_query( {'object_type' => 'Analysis', 'logic_name' => $self->{'start_analysis_name'} } );

        my $start_jobs  =   $start_analysis
                                ? ( $anchor_jobs
                                        ? find_the_top( $anchor_jobs, $start_analysis )                     # perform a per-jobs scan to the start_analysis
                                        : $job_adaptor->fetch_all_by_analysis_id( $start_analysis->dbID )   # take all jobs of the top analysis
                                ) : ( $anchor_jobs
                                        ? $anchor_jobs                                                      # just start from the given anchor_jobs
                                        : $job_adaptor->fetch_all_by_prev_job_id( undef )                   # by default start from the seed jobs
                                );

        foreach my $start_job ( @$start_jobs ) {
            my $job_node_name   = add_family_tree( $start_job, $self->{'stop_analysis_name'} );
        }

            ## If you need to take a look at the intermediate dot file:
        if( $self->{'dot_input'} ) {
            $self->{'graph'}->dot_input_filename( $self->{'dot_input'} );
        }

        my $call = 'as_'.$self->{'format'};

        $self->{'graph'}->$call($self->{'output'});

    } else {
        die "\nERROR : -output filename has to be defined\n\n";
    }

}


sub find_the_top {
    my ($anchor_jobs, $start_analysis) = @_;

    my @starters    = ();

        # first try to find the start_analysis on the way up:
    foreach my $anchor_job ( @$anchor_jobs ) {
        my $job;

        for($job = $anchor_job; ($job->analysis != $start_analysis) and ($job->prev_job) ; $job = $job->prev_job) {}

        push @starters, $job;
    }

    return \@starters;
}


my %job_node_hash = ();

sub add_job_node {
    my $job = shift @_;

    my $job_id          = $job->dbID;
    my $job_node_name   = 'job_'.$job_id;

    unless($job_node_hash{$job_node_name}++) {
        my $job_shape           = 'record';
        my $job_status_colour   = {'DONE' => 'DeepSkyBlue', 'READY' => 'green', 'SEMAPHORED' => 'grey', 'FAILED' => 'red'}->{$job->status} // 'blue';
        my $job_id              = $job->dbID;
        my $job_params          = destringify($job->input_id);

        my $job_label           = qq{<<table border="0" cellborder="0" cellspacing="0" cellpadding="1">}
                                 .qq{<tr><td><u><i>job_id:</i></u></td><td><i>$job_id</i></td></tr>};

        if(my $param_id_stack = $job->param_id_stack) {
            $job_label  .=  qq{<tr><td><u><i>params from:</i></u></td><td><i>$param_id_stack</i></td></tr>};
        }

        foreach my $param_key (sort keys %$job_params) {
            my $param_value = $job_params->{$param_key};
            $job_label  .= "<tr><td>$param_key:</td><td> $param_value</td></tr>";
        }

        my $accu_adaptor    = $job->adaptor->db->get_AccumulatorAdaptor;
        my $accu            = $accu_adaptor->fetch_structures_for_job_ids( $job_id )->{ $job_id };

        foreach my $accu_name (sort keys %$accu) {
            $job_label  .=  qq{<tr><td><u><i>accumulated:</i></u></td><td><i>$accu_name</i></td></tr>};
        }

        $job_label  .= "</table>>";


        $self->{'graph'}->add_node( $job_node_name,
            shape       => $job_shape,
            style       => 'filled',
            fillcolor   => $job_status_colour,
            label       => $job_label,
        );

            # adding the job to the corresponding analysis' cluster:
        push @{$self->{'graph'}->cluster_2_nodes->{ $job->analysis->logic_name }}, $job_node_name;
    }

    return $job_node_name;
}


my %semaphore_node_hash = ();

sub add_semaphore_node {<commit_msg>colour analysis-clusters based on analysis->status<commit_after>#!/usr/bin/env perl

use strict;
use warnings;

    # Finding out own path in order to reference own components (including own modules):
use Cwd            ();
use File::Basename ();
BEGIN {
    $ENV{'EHIVE_ROOT_DIR'} ||= File::Basename::dirname( File::Basename::dirname( Cwd::realpath($0) ) );
    unshift @INC, $ENV{'EHIVE_ROOT_DIR'}.'/modules';
}


use Getopt::Long;

use Bio::EnsEMBL::Hive::HivePipeline;
use Bio::EnsEMBL::Hive::Utils ('destringify');
use Bio::EnsEMBL::Hive::Utils::GraphViz;


my $self = {};

main();


sub main {

    my $pipeline;

    GetOptions(
            # connection parameters
        'url=s'                 => \$self->{'url'},
        'reg_conf|reg_file=s'   => \$self->{'reg_conf'},
        'reg_type=s'            => \$self->{'reg_type'},
        'reg_alias|reg_name=s'  => \$self->{'reg_alias'},
        'nosqlvc=i'             => \$self->{'nosqlvc'},             # using "=i" instead of "!" for consistency with scripts where it is a propagated option

        'job_id=s@'             => \$self->{'job_ids'},             # jobs to start from
        'start_analysis_name=s' => \$self->{'start_analysis_name'}, # if given, first trace the graph up to the given analysis or the seed_jobs, and then start visualization
        'stop_analysis_name=s'  => \$self->{'stop_analysis_name'},  # if given, the visualization is aborted at that analysis and doesn't go any further

        'o|out|output=s'        => \$self->{'output'},
        'dot_input=s'           => \$self->{'dot_input'},   # filename to store the intermediate dot input (valuable for debugging)

        'h|help'                => \$self->{'help'},
    );

    if($self->{'url'} or $self->{'reg_alias'}) {
        $pipeline = Bio::EnsEMBL::Hive::HivePipeline->new(
            -url                            => $self->{'url'},
            -reg_conf                       => $self->{'reg_conf'},
            -reg_type                       => $self->{'reg_type'},
            -reg_alias                      => $self->{'reg_alias'},
            -no_sql_schema_version_check    => $self->{'nosqlvc'},
        );

    } else {
        die "\nERROR : Connection parameters (url or reg_conf+reg_alias) need to be specified\n\n";
    }

    if($self->{'output'}) {

        if(!$self->{'format'}) {
            if($self->{'output'}=~/\.(\w+)$/) {
                $self->{'format'} = $1;
            } else {
                die "Format was not set and could not guess from ".$self->{'output'}.". Please use either way to select it.\n";
            }
        }

        $self->{'graph'} = Bio::EnsEMBL::Hive::Utils::GraphViz->new(
                    'name'          => 'JobDependencyGraph',
                    'concentrate'   => 'true',
                    'pad'           => 1,
        );

        $self->{'graph'}->cluster_2_nodes( {} );
        $self->{'graph'}->cluster_2_colour_pair( {} );
        $self->{'graph'}->display_cluster_names( 1 );

        my $job_adaptor     = $pipeline->hive_dba->get_AnalysisJobAdaptor;
        my $anchor_jobs     = $self->{'job_ids'} && $job_adaptor->fetch_all( 'job_id IN ('.join(',', @{$self->{'job_ids'}} ).')' );
        my $start_analysis  = $self->{'start_analysis_name'} && $pipeline->find_by_query( {'object_type' => 'Analysis', 'logic_name' => $self->{'start_analysis_name'} } );

        my $start_jobs  =   $start_analysis
                                ? ( $anchor_jobs
                                        ? find_the_top( $anchor_jobs, $start_analysis )                     # perform a per-jobs scan to the start_analysis
                                        : $job_adaptor->fetch_all_by_analysis_id( $start_analysis->dbID )   # take all jobs of the top analysis
                                ) : ( $anchor_jobs
                                        ? $anchor_jobs                                                      # just start from the given anchor_jobs
                                        : $job_adaptor->fetch_all_by_prev_job_id( undef )                   # by default start from the seed jobs
                                );

        foreach my $start_job ( @$start_jobs ) {
            my $job_node_name   = add_family_tree( $start_job, $self->{'stop_analysis_name'} );
        }

            ## If you need to take a look at the intermediate dot file:
        if( $self->{'dot_input'} ) {
            $self->{'graph'}->dot_input_filename( $self->{'dot_input'} );
        }

        my $call = 'as_'.$self->{'format'};

        $self->{'graph'}->$call($self->{'output'});

    } else {
        die "\nERROR : -output filename has to be defined\n\n";
    }

}


sub find_the_top {
    my ($anchor_jobs, $start_analysis) = @_;

    my @starters    = ();

        # first try to find the start_analysis on the way up:
    foreach my $anchor_job ( @$anchor_jobs ) {
        my $job;

        for($job = $anchor_job; ($job->analysis != $start_analysis) and ($job->prev_job) ; $job = $job->prev_job) {}

        push @starters, $job;
    }

    return \@starters;
}


my %job_node_hash = ();

sub add_job_node {
    my $job = shift @_;

    my $job_id          = $job->dbID;
    my $job_node_name   = 'job_'.$job_id;

    unless($job_node_hash{$job_node_name}++) {
        my $job_shape           = 'record';
        my $job_status_colour   = {'DONE' => 'DeepSkyBlue', 'READY' => 'green', 'SEMAPHORED' => 'grey', 'FAILED' => 'red'}->{$job->status} // 'yellow';
        my $analysis_status_colour = {
            "EMPTY"       => "white",
            "BLOCKED"     => "grey",
            "LOADING"     => "green",
            "ALL_CLAIMED" => "grey",
            "SYNCHING"    => "green",
            "READY"       => "green",
            "WORKING"     => "yellow",
            "DONE"        => "DeepSkyBlue",
            "FAILED"      => "red",
        };

        my $job_id              = $job->dbID;
        my $job_params          = destringify($job->input_id);

        my $job_label           = qq{<<table border="0" cellborder="0" cellspacing="0" cellpadding="1">}
                                 .qq{<tr><td><u><i>job_id:</i></u></td><td><i>$job_id</i></td></tr>};

        if(my $param_id_stack = $job->param_id_stack) {
            $job_label  .=  qq{<tr><td><u><i>params from:</i></u></td><td><i>$param_id_stack</i></td></tr>};
        }

        foreach my $param_key (sort keys %$job_params) {
            my $param_value = $job_params->{$param_key};
            $job_label  .= "<tr><td>$param_key:</td><td> $param_value</td></tr>";
        }

        my $accu_adaptor    = $job->adaptor->db->get_AccumulatorAdaptor;
        my $accu            = $accu_adaptor->fetch_structures_for_job_ids( $job_id )->{ $job_id };

        foreach my $accu_name (sort keys %$accu) {
            $job_label  .=  qq{<tr><td><u><i>accumulated:</i></u></td><td><i>$accu_name</i></td></tr>};
        }

        $job_label  .= "</table>>";


        $self->{'graph'}->add_node( $job_node_name,
            shape       => $job_shape,
            style       => 'filled',
            fillcolor   => $job_status_colour,
            label       => $job_label,
        );

            # adding the job to the corresponding analysis' cluster:
        my $analysis_name   = $job->analysis->logic_name;
        my $analysis_status = $job->analysis->status;
        push @{$self->{'graph'}->cluster_2_nodes->{ $analysis_name }}, $job_node_name;
        $self->{'graph'}->cluster_2_colour_pair->{ $analysis_name } = [ $analysis_status_colour->{$analysis_status} ];
    }

    return $job_node_name;
}


my %semaphore_node_hash = ();

sub add_semaphore_node {
{'fraction_non_alphanumeric': 0.17101704084392752, 'fraction_numerical': 0.0016905599134433324, 'mean_word_length': 1.9221497727721795, 'pattern_counts': {'":': 0, '<': 93, '<?xml version=': 0, '>': 318, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 28, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1972300', 'n_tokens_mistral': 4844, 'n_tokens_neox': 4609, 'n_words': 905}