<commit_before># This module contains several methods that I would like to see
# in the Hive API but it is unlikely that they get into it.
# 
# They are convenient to simplify constructs like:
# $obj->$method(@args);
#
# and work with (for example):
# $analysis_stats->hive_capacity(555);
# and
# $analysis->delete_param("mlss_id");
# or
# $analysis->add_param("mlss_id", 40086);
#
# The cost is not being coded in the class itself
#

package hive_extended;

use strict;
use warnings;
use Data::Dumper;
no warnings "once";

use Bio::EnsEMBL::Hive::Utils qw/stringify destringify/;

# Analysis' module should check that the module exists and is compilable
*Bio::EnsEMBL::Hive::Analysis::update_module = sub {
    my $self = shift;
    $self->{'_module'} = shift if(@_ && do{
	my $module = $_[0];

	 ## The problem with this, is that we are doing these checks *locally*, but the module is suppose to be used *remotely* (in the farm, etc)
	## So chances are that the module only exists (or is fully functional) remotely, where we don't have change to test it
	## TODO: use the regular "module" method instead
#	eval "require $module";
#	die "The module '$module' can't be loaded: $@\n" if ($@);
#	die "Problem accessing methods in '$module'. Please check that it inherits from Bio::EnsEMBL::Hive::Process and is named correctly\n" unless ($module->isa('Bio::EnsEMBL::Hive::Process'));
	1;
				  });
    return $self->{'_module'};
};

# add_input_id_key adds a new key with empty value
*Bio::EnsEMBL::Hive::AnalysisJob::add_input_id_key = sub {
    my ($self, $key) = @_;
    my $value = "";
    $self->add_input_id($key, $value);
    return $key;
};

# add_input_id adds/change a single input_id key/value pair in the AnalysisJob table
# if no new value is provided, the current value is returned
*Bio::EnsEMBL::Hive::AnalysisJob::add_input_id = sub {
    my ($self, $key, $value) = @_;
    my $curr_raw_input_id = $self->input_id;
    $curr_raw_input_id = '{}' unless ($curr_raw_input_id);
    my $curr_input_id = Bio::EnsEMBL::Hive::Utils::destringify($curr_raw_input_id);
    return $curr_input_id->{$key} unless (defined $value);
    $curr_input_id->{$key} = $value;
    my $new_raw_input_id = Bio::EnsEMBL::Hive::Utils::stringify($curr_input_id);
    $self->input_id($new_raw_input_id);
    return $value;
};

# delete_input_id deletes a single input_id key/value pair in the AnalysisJob table
*Bio::EnsEMBL::Hive::AnalysisJob::delete_input_id = sub {
    my ($self, $key) = @_;
    my $curr_raw_input_id = $self->input_id;
    my $curr_input_id = Bio::EnsEMBL::Hive::Utils::destringify($curr_raw_input_id);
    delete $curr_input_id->{$key};
    my $new_raw_parameters = Bio::EnsEMBL::Hive::Utils::stringify($curr_input_id);
    $self->input_id($new_raw_parameters);
    return;
};

# delete_param deletes a single param in the param table by key
# It is injected in the Analysis object directly so it works as a 
# native method of the object
*Bio::EnsEMBL::Hive::Analysis::delete_param = sub {
    my ($self, $key) = @_;
    my $curr_raw_parameters = $self->parameters;
    my $curr_parameters = Bio::EnsEMBL::Hive::Utils->destringify($curr_raw_parameters);
    delete $curr_parameters->{$key};
    my $new_raw_parameters = Bio::EnsEMBL::Hive::Utils->stringify($curr_parameters);
    $self->parameters($new_raw_parameters);
    return;
};

# add_param adds/change a single param in the param table by key
# It is injected in the Analysis object directly so it works as a 
# native method of the object
*Bio::EnsEMBL::Hive::Analysis::add_param = sub {
    my ($self, $key, $value) = @_;
    my $var = eval $value;
    $var = $value unless(defined $var);
    my $curr_raw_parameters = $self->parameters;
    my $curr_parameters = Bio::EnsEMBL::Hive::Utils->destringify($curr_raw_parameters);
    $curr_parameters->{$key} = $var;
    my $new_raw_parameters = Bio::EnsEMBL::Hive::Utils->stringify($curr_parameters);
    $self->parameters($new_raw_parameters);
    return;
};

# description returns the ResourceDescription object
# of a given ResourceClass
*Bio::EnsEMBL::Hive::ResourceClass::description = sub {
    my ($self) = @_;
    my $description = $self->adaptor->db->get_ResourceDescriptionAdaptor->fetch_all_by_resource_class_id($self->dbID)->[0];
    return $description;
};

# Method for retrieving the ResourceDescription adaptor from a dbID
*Bio::EnsEMBL::Hive::DBSQL::ResourceDescriptionAdaptor::fetch_by_dbID = sub {
    my ($self, $id) = @_;
    my $obj = $self->fetch_all_by_resource_class_id($id)->[0];
    return $obj;
};

# This should be fetched correctly by AnalysisStatsAdaptor now
# TODO: Test that this natively in the Adaptor and if so,
# remove this injected method from here
*Bio::EnsEMBL::Hive::DBSQL::AnalysisStatsAdaptor::fetch_by_dbID = sub {
  my ($self, $id) = @_;
  my $obj = $self->fetch_by_analysis_id($id);
  return $obj;
};

## To allow the creation of new resources (class + description) in one call
*Bio::EnsEMBL::Hive::DBSQL::ResourceClassAdaptor::create_full_description = sub {
  my ($self, $rc_name, $meadow_type, $parameters) = @_;
  # If we don't have parameters, the resource_class will be created but the description would
  # raise an exception
  # It is better to raise the exception here and let the client deal with it.
  # This way we don't create an orphan resource_class
  die "It is not allowed to create a class without parameters\n" unless (defined $parameters);

  print STDERR "$rc_name, $meadow_type, $parameters\n";

  for my $rc (@{$self->fetch_all}) {
    # if ($rc_name eq $rc->name) {
    #   throw("This resource name exists in the database\n");
    # }
  }
  my ($rc) = $self->create_new(-NAME => $rc_name);
  my $rc_id = $rc->dbID();

  $self->db->get_ResourceDescriptionAdaptor->create_new(
      -RESOURCE_CLASS_ID => $rc_id,
      -MEADOW_TYPE       => $meadow_type,
      -PARAMETERS        => $parameters,
      );
};

## AnalysisJobAdaptor doesn't have a generic update method
## and it doesn't inherits from Hive::DBSQL::BaseAdaptor either
## I guess that this will change in the future,
## but for now I am injecting a generic update for that adaptor.
*Bio::EnsEMBL::Hive::DBSQL::AnalysisJobAdaptor::update = sub {
  my ($self, $job) = @_;
  my $sql = "UPDATE job SET input_id='" . $job->input_id . "'";
  $sql .= ",status='" . $job->status . "'";
  $sql .= ",retry_count=" . $job->retry_count;
  $sql .= ",semaphore_count=" . $job->semaphore_count;
  $sql .= ",semaphored_job_id=" . $job->semaphored_job_id if (defined $job->semaphored_job_id);
  $sql .= " WHERE job_id=" . $job->dbID;
  my $sth = $self->prepare($sql);
  $sth->execute();
  $sth->finish();

  unless ($job->completed) {
    $self->update_status($job);
  }
};

## Method to get all the jobs in the database
## Too tricky. It would be better to have a _generic_count directly in the AnalysisJob adaptor
## or better still in Hive's BaseAdaptor
*Bio::EnsEMBL::Hive::DBSQL::AnalysisJobAdaptor::_generic_count = sub {
  my ($self, $constraints) = @_;

  # We save previous _columns and _objs_from_sth methods
  my $old_columns = *Bio::EnsEMBL::Hive::DBSQL::AnalysisJobAdaptor::_columns;
  my $old_objs_from_sth = *Bio::EnsEMBL::Hive::DBSQL::AnalysisJobAdaptor::_objs_from_sth;<commit_msg>Added methods to deal with job's input_ids stored in analysis_data<commit_after># This module contains several methods that I would like to see
# in the Hive API but it is unlikely that they get into it.
# 
# They are convenient to simplify constructs like:
# $obj->$method(@args);
#
# and work with (for example):
# $analysis_stats->hive_capacity(555);
# and
# $analysis->delete_param("mlss_id");
# or
# $analysis->add_param("mlss_id", 40086);
#
# The cost is not being coded in the class itself
#

package hive_extended;

use strict;
use warnings;
use Data::Dumper;
no warnings "once";

use Bio::EnsEMBL::Hive::Utils qw/stringify destringify/;

# Analysis' module should check that the module exists and is compilable
*Bio::EnsEMBL::Hive::Analysis::update_module = sub {
    my $self = shift;
    $self->{'_module'} = shift if(@_ && do{
	my $module = $_[0];

	 ## The problem with this, is that we are doing these checks *locally*, but the module is suppose to be used *remotely* (in the farm, etc)
	## So chances are that the module only exists (or is fully functional) remotely, where we don't have change to test it
	## TODO: use the regular "module" method instead
#	eval "require $module";
#	die "The module '$module' can't be loaded: $@\n" if ($@);
#	die "Problem accessing methods in '$module'. Please check that it inherits from Bio::EnsEMBL::Hive::Process and is named correctly\n" unless ($module->isa('Bio::EnsEMBL::Hive::Process'));
	1;
				  });
    return $self->{'_module'};
};

# add_input_id_key adds a new key with empty value
*Bio::EnsEMBL::Hive::AnalysisJob::add_input_id_key = sub {
    my ($self, $key) = @_;
    my $value = "";
    $self->add_input_id($key, $value);
    return $key;
};

# add_input_id adds/change a single input_id key/value pair in the AnalysisJob table
# if no new value is provided, the current value is returned
*Bio::EnsEMBL::Hive::AnalysisJob::add_input_id = sub {
    my ($self, $key, $value) = @_;
    my $curr_raw_input_id = $self->input_id;
    $curr_raw_input_id = '{}' unless ($curr_raw_input_id);
    my $curr_input_id = Bio::EnsEMBL::Hive::Utils::destringify($curr_raw_input_id);
    return $curr_input_id->{$key} unless (defined $value);
    $curr_input_id->{$key} = $value;
    my $new_raw_input_id = Bio::EnsEMBL::Hive::Utils::stringify($curr_input_id);
    $self->input_id($new_raw_input_id);
    return $value;
};

# delete_input_id deletes a single input_id key/value pair in the AnalysisJob table
*Bio::EnsEMBL::Hive::AnalysisJob::delete_input_id = sub {
    my ($self, $key) = @_;
    my $curr_raw_input_id = $self->input_id;
    my $curr_input_id = Bio::EnsEMBL::Hive::Utils::destringify($curr_raw_input_id);
    delete $curr_input_id->{$key};
    my $new_raw_parameters = Bio::EnsEMBL::Hive::Utils::stringify($curr_input_id);
    $self->input_id($new_raw_parameters);
    return;
};

# delete_param deletes a single param in the param table by key
# It is injected in the Analysis object directly so it works as a 
# native method of the object
*Bio::EnsEMBL::Hive::Analysis::delete_param = sub {
    my ($self, $key) = @_;
    my $curr_raw_parameters = $self->parameters;
    my $curr_parameters = Bio::EnsEMBL::Hive::Utils->destringify($curr_raw_parameters);
    delete $curr_parameters->{$key};
    my $new_raw_parameters = Bio::EnsEMBL::Hive::Utils->stringify($curr_parameters);
    $self->parameters($new_raw_parameters);
    return;
};

# add_param adds/change a single param in the param table by key
# It is injected in the Analysis object directly so it works as a 
# native method of the object
*Bio::EnsEMBL::Hive::Analysis::add_param = sub {
    my ($self, $key, $value) = @_;
    my $var = eval $value;
    $var = $value unless(defined $var);
    my $curr_raw_parameters = $self->parameters;
    my $curr_parameters = Bio::EnsEMBL::Hive::Utils->destringify($curr_raw_parameters);
    $curr_parameters->{$key} = $var;
    my $new_raw_parameters = Bio::EnsEMBL::Hive::Utils->stringify($curr_parameters);
    $self->parameters($new_raw_parameters);
    return;
};

# description returns the ResourceDescription object
# of a given ResourceClass
*Bio::EnsEMBL::Hive::ResourceClass::description = sub {
    my ($self) = @_;
    my $description = $self->adaptor->db->get_ResourceDescriptionAdaptor->fetch_all_by_resource_class_id($self->dbID)->[0];
    return $description;
};

# Method for retrieving the ResourceDescription adaptor from a dbID
*Bio::EnsEMBL::Hive::DBSQL::ResourceDescriptionAdaptor::fetch_by_dbID = sub {
    my ($self, $id) = @_;
    my $obj = $self->fetch_all_by_resource_class_id($id)->[0];
    return $obj;
};

# This should be fetched correctly by AnalysisStatsAdaptor now
# TODO: Test that this natively in the Adaptor and if so,
# remove this injected method from here
*Bio::EnsEMBL::Hive::DBSQL::AnalysisStatsAdaptor::fetch_by_dbID = sub {
  my ($self, $id) = @_;
  my $obj = $self->fetch_by_analysis_id($id);
  return $obj;
};

## To allow the creation of new resources (class + description) in one call
*Bio::EnsEMBL::Hive::DBSQL::ResourceClassAdaptor::create_full_description = sub {
  my ($self, $rc_name, $meadow_type, $parameters) = @_;
  # If we don't have parameters, the resource_class will be created but the description would
  # raise an exception
  # It is better to raise the exception here and let the client deal with it.
  # This way we don't create an orphan resource_class
  die "It is not allowed to create a class without parameters\n" unless (defined $parameters);

  for my $rc (@{$self->fetch_all}) {
    # if ($rc_name eq $rc->name) {
    #   throw("This resource name exists in the database\n");
    # }
  }
  my ($rc) = $self->create_new(-NAME => $rc_name);
  my $rc_id = $rc->dbID();

  $self->db->get_ResourceDescriptionAdaptor->create_new(
      -RESOURCE_CLASS_ID => $rc_id,
      -MEADOW_TYPE       => $meadow_type,
      -PARAMETERS        => $parameters,
      );
};

## _input_id_is_extended determines if the input_id of a job is too large in the
## database. If it is, returns the extended_data_id
*Bio::EnsEMBL::Hive::DBSQL::AnalysisJobAdaptor::_input_id_is_extended = sub {
  my ($self, $job) = @_;

  my $sql = "SELECT input_id FROM job WHERE job_id = ?";
  my $sth = $self->prepare($sql);
  $sth->execute($job->dbID);
  my ($real_input_id) = $sth->fetchrow_array();
  $sth->finish();
  if ($real_input_id =~ /^_ext(?:\w+)_data_id (\d+)$/) {
    return $1;
  }
  return 0; # 0 is not used as an extended_data_id
};

## If the input_id of a job has grown over its size limit (255 characters)
## _move_input_id_to_analysis_data puts it into analysis_data and update the job's input_id
## to link to that entry in analysis_data
## The method returns the newly created dbID in the analysis_data table
*Bio::EnsEMBL::Hive::DBSQL::AnalysisJobAdaptor::_move_input_id_to_analysis_data = sub {
  my ($self, $job) = @_;
  my $extended_data_id = $self->db->get_AnalysisDataAdaptor->store($job->input_id);
  my $extended_input_id = "_extended_data_id $extended_data_id";
  $self->_update_input_id_in_job_table($job, $extended_input_id);
  return $extended_data_id;
};


*Bio::EnsEMBL::Hive::DBSQL::AnalysisJobAdaptor::_move_input_id_from_analysis_data = sub {
  my ($self, $job, $extended_data_id) = @_;
  my $input_id = $job->input_id;
  $self->throw("input_id is too large to be moved into the job table (it has to remain in analysis_data)")
    if (length($input_id)>=255);
  $self->db->get_AnalysisDataAdaptor->remove($extended_data_id);
  $self->_update_input_id_in_job_table($job, $input_id);
  return;
};

*Bio::EnsEMBL::Hive::DBSQL::AnalysisJobAdaptor::_update_input_id_in_job_table = sub {
  my ($self, $job, $input_id) = @_;

  my $sql = "UPDATE job SET input_id = ? WHERE job_id = ?";
  my $sth = $self->prepare($sql);
  $sth->execute($input_id, $job->dbID);
  $sth->finish();
  return;
};

## update updates the corresponding entry in analysis_data with the values
## provided as arguments
*Bio::EnsEMBL::Hive::DBSQL::AnalysisDataAdaptor::update = sub {
  my ($self, $analysis_data_id, $data) = @_;
  my $sql = "UPDATE analysis_data SET data = ? WHERE analysis_data_id = ?";
  my $sth = $self->prepare($sql);
  $sth->execute($data, $analysis_data_id);
  $sth->finish();
  return;
};

## remove deletes the corresponding entry fort the given analysis_data_id in the
## analysis_data table
*Bio::EnsEMBL::Hive::DBSQL::AnalysisDataAdaptor::remove = sub {
  my ($self, $analysis_data_id) = @_;
  my $sql = "DELETE FROM analysis_data WHERE analysis_data_id = ?";
  my $sth = $self->prepare($sql);
  $sth->execute($analysis_data_id);
  return;
};

## AnalysisJobAdaptor doesn't have a generic update method
## and it doesn't inherits from Hive::DBSQL::BaseAdaptor either
## I guess that this will change in the future,
## but for now I am injecting a generic update for that adaptor.
*Bio::EnsEMBL::Hive::DBSQL::AnalysisJobAdaptor::update = sub {
  my ($self, $job) = @_;

  my $curr_data_id = $self->_input_id_is_extended($job);

  if (length($job->input_id) >= 255) {
    if ($curr_data_id) {
      $self->db->get_AnalysisDataAdaptor->update($curr_data_id, $job->input_id);
    } else {
      $self->_move_input_id_to_analysis_data($job)
    }
  } else {
    if ($curr_data_id) {
      $self->_move_input_id_from_analysis_data($job, $curr_data_id);
    } else {
      $self->_update_input_id_in_job_table($job, $job->input_id);
    }
  }
  my $sql = "UPDATE job SET ";
  $sql .= "status='" . $job->status . "'";
  $sql .= ",retry_count=" . $job->retry_count;
  $sql .= ",semaphore_count=" . $job->semaphore_count;
  $sql .= ",semaphored_job_id=" . $job->semaphored_job_id if (defined $job->semaphored_job_id);
  $sql .= " WHERE job_id=" . $job->dbID;
  my $sth = $self->prepare($sql);
  $sth->execute();
  $sth->finish();

  unless ($job->completed) {
    $self->update_status($job);
  }
};

## Method to get all the jobs in the database
## Too tricky. It would be better to have a _generic_count directly in the AnalysisJob adaptor
## or better still in Hive's BaseAdaptor
*Bio::EnsEMBL::Hive::DBSQL::AnalysisJobAdaptor::_generic_count = sub {
  my ($self, $constraints) = @_;

  # We save previous _columns and _objs_from_sth methods
  my $old_columns = *Bio::EnsEMBL::Hive::DBSQL::AnalysisJobAdaptor::_columns;
  my $old_objs_from_sth = *Bio::EnsEMBL::Hive::DBSQL::AnalysisJobAdaptor::_objs_from_sth;
{'fraction_non_alphanumeric': 0.1441661923733637, 'fraction_numerical': 0.0020489470688673877, 'mean_word_length': 4.742156862745098, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 153, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6860131', 'n_tokens_mistral': 6222, 'n_tokens_neox': 6045, 'n_words': 1614}