<commit_before>package Bio::EnsEMBL::DBLoader::PipeConfig::LoadDBs_conf;

use strict;
use warnings;
use base qw/Bio::EnsEMBL::Hive::PipeConfig::HiveGeneric_conf/;

use Bio::EnsEMBL::ApiVersion;

sub default_options {
  my ($self) = @_;
  my $parent_options = $self->SUPER::default_options();
  return {
    %{$parent_options},

    #FTP Settings
    ftp_host => 'ftp.ensembl.org',
    ftp_port => 21,
    ftp_user => 'anonymous',
#    ftp_pass => ''

    #Required DBs; leave blank for all DBs
    required_dbs => [],

    #Set to the actual release or the current which
    release => software_version(),

    #Only required when working with EnsemblGenomes servers
    division => '',

    #Work directory - location of the files where we can download to
    #work_directory => '',

    #Automatically set name
    pipeline_name => 'mirror_ensembl_'.$self->o('release'),

    #Target DB
    target_db => {
      -host => $self->o('target_db_host'),
      -port => $self->o('target_db_port'),
      -user => $self->o('target_db_user'),
      -pass => $self->o('target_db_pass')
    },

    #Setting the Hive DB which we will use SQLite for
    pipeline_db => {
      -driver => 'sqlite',
      -dbname => $self->o('pipeline_name'),
      -host   => '',
      -port   => '',
      -user   => '',
      -pass   => '',
    },

    #Priority listing
    priority => {
      species => [qw/homo_sapiens mus_musculus danio_rerio/],
      group => [qw/core variation/],
    }
  };
}

sub pipeline_wide_parameters {
  my ($self) = @_;
  return {
    %{$self->SUPER::pipeline_wide_parameters()},
    'work_directory' => $self->o('work_directory'),
  };
}

sub pipeline_create_commands {
  my ($self) = @_;
  my $parent_cmds = $self->SUPER::pipeline_create_commands();
  return [
    @{$parent_cmds},
    'mkdir -p '.$self->o('work_directory'),
  ];
}

#1 job - Figure out if we want all DBs or a subset
#1-3 jobs - Download all required directories (database and datafiles)
#10 jobs - Checksum all files
#1 job - Sub-divide DBs into high priority & low priority by DB type & species
#2 jobs - Load DB SQL, Gunzip data & load tables

sub pipeline_analyses {
  my ($self) = @_;
  return [
    {
      -logic_name => 'find_dbs',
      -module => 'Bio::EnsEMBL::DBLoader::RunnableDB::FindDbs',
      -parameters => {
        required_dbs  => $self->o('required_dbs'),
        release       => $self->o('release'),
        division      => $self->o('division'),
        ftp_host      => $self->o('ftp_host'),
        ftp_port      => $self->o('ftp_port'),
        ftp_user      => $self->o('ftp_user'),
        ftp_pass      => $self->o('ftp_pass'),
      },
      -input_ids => [{}],
      -flow_into => { 1 => [qw/download_files/] },
    },

    {
      -logic_name => 'download_files',
      -module => 'Bio::EnsEMBL::DBLoader::RunnableDB::DownloadFiles',
      -parameters => {
        ftp_host  => $self->o('ftp_host'),
        ftp_port  => $self->o('ftp_port'),
        ftp_user  => $self->o('ftp_user'),
        ftp_pass  => $self->o('ftp_<PASSWORD>'),
      },
      -flow_into => { 1 => [qw/checksum_files/] },
      -hive_capacity => 1,
      -failed_job_tolerance => 10,
    },

    {
      -logic_name => 'checksum_files',
      -module => 'Bio::EnsEMBL::DBLoader::RunnableDB::ChecksumFiles',
      -parameters => {},
      -hive_capacity => 10,
      -flow_into => { 1 => [qw/prioritise/] },
      -failed_job_tolerance => 10,
    },

    {
      -logic_name => 'prioritise',
      -module => 'Bio::EnsEMBL::DBLoader::RunnableDB::Prioritise',
      -parameters => { priority => $self->o('priority') },
      -flow_into => { 2 => [qw/high_priority_load_files/], 3 => [qw/load_files/] },
    },

    {
      -logic_name => 'high_priority_load_files',
      -module => 'Bio::EnsEMBL::DBLoader::RunnableDB::LoadFiles',
      -parameters => { target_db => $self->o('target_db') },
      -hive_capacity => 2,
      -failed_job_tolerance => 100,
      -can_be_empty => 1,
    },

    {
      -logic_name => 'load_files',
      -module => 'Bio::EnsEMBL::DBLoader::RunnableDB::LoadFiles',
      -parameters => { target_db => $self->o('target_db') },
      -hive_capacity => 2,
      -wait_for => [qw/prioritise high_priority_load_files/],
      -retry_count => 1,
      -failed_job_tolerance => 50,
      -can_be_empty => 1,
    },
  ];
}

1;<commit_msg>Bringing capacity for loads to 1<commit_after>package Bio::EnsEMBL::DBLoader::PipeConfig::LoadDBs_conf;

use strict;
use warnings;
use base qw/Bio::EnsEMBL::Hive::PipeConfig::HiveGeneric_conf/;

use Bio::EnsEMBL::ApiVersion;

sub default_options {
  my ($self) = @_;
  my $parent_options = $self->SUPER::default_options();
  return {
    %{$parent_options},

    #FTP Settings
    ftp_host => 'ftp.ensembl.org',
    ftp_port => 21,
    ftp_user => 'anonymous',
#    ftp_pass => ''

    #Required DBs; leave blank for all DBs
    required_dbs => [],

    #Set to the actual release or the current which
    release => software_version(),

    #Only required when working with EnsemblGenomes servers
    division => '',

    #Work directory - location of the files where we can download to
    #work_directory => '',

    #Automatically set name
    pipeline_name => 'mirror_ensembl_'.$self->o('release'),

    #Target DB
    target_db => {
      -host => $self->o('target_db_host'),
      -port => $self->o('target_db_port'),
      -user => $self->o('target_db_user'),
      -pass => $self->o('target_db_pass')
    },

    #Setting the Hive DB which we will use SQLite for
    pipeline_db => {
      -driver => 'sqlite',
      -dbname => $self->o('pipeline_name'),
      -host   => '',
      -port   => '',
      -user   => '',
      -pass   => '',
    },

    #Priority listing
    priority => {
      species => [qw/homo_sapiens mus_musculus danio_rerio/],
      group => [qw/core variation/],
    }
  };
}

sub pipeline_wide_parameters {
  my ($self) = @_;
  return {
    %{$self->SUPER::pipeline_wide_parameters()},
    'work_directory' => $self->o('work_directory'),
  };
}

sub pipeline_create_commands {
  my ($self) = @_;
  my $parent_cmds = $self->SUPER::pipeline_create_commands();
  return [
    @{$parent_cmds},
    'mkdir -p '.$self->o('work_directory'),
  ];
}

#1 job - Figure out if we want all DBs or a subset
#1-3 jobs - Download all required directories (database and datafiles)
#10 jobs - Checksum all files
#1 job - Sub-divide DBs into high priority & low priority by DB type & species
#2 jobs - Load DB SQL, Gunzip data & load tables

sub pipeline_analyses {
  my ($self) = @_;
  return [
    {
      -logic_name => 'find_dbs',
      -module => 'Bio::EnsEMBL::DBLoader::RunnableDB::FindDbs',
      -parameters => {
        required_dbs  => $self->o('required_dbs'),
        release       => $self->o('release'),
        division      => $self->o('division'),
        ftp_host      => $self->o('ftp_host'),
        ftp_port      => $self->o('ftp_port'),
        ftp_user      => $self->o('ftp_user'),
        ftp_pass      => $self->o('ftp_pass'),
      },
      -input_ids => [{}],
      -flow_into => { 1 => [qw/download_files/] },
    },

    {
      -logic_name => 'download_files',
      -module => 'Bio::EnsEMBL::DBLoader::RunnableDB::DownloadFiles',
      -parameters => {
        ftp_host  => $self->o('ftp_host'),
        ftp_port  => $self->o('ftp_port'),
        ftp_user  => $self->o('ftp_user'),
        ftp_pass  => $self->o('ftp_<PASSWORD>'),
      },
      -flow_into => { 1 => [qw/checksum_files/] },
      -hive_capacity => 1,
      -failed_job_tolerance => 10,
    },

    {
      -logic_name => 'checksum_files',
      -module => 'Bio::EnsEMBL::DBLoader::RunnableDB::ChecksumFiles',
      -parameters => {},
      -hive_capacity => 10,
      -flow_into => { 1 => [qw/prioritise/] },
      -failed_job_tolerance => 10,
    },

    {
      -logic_name => 'prioritise',
      -module => 'Bio::EnsEMBL::DBLoader::RunnableDB::Prioritise',
      -parameters => { priority => $self->o('priority') },
      -flow_into => { 2 => [qw/high_priority_load_files/], 3 => [qw/load_files/] },
    },

    {
      -logic_name => 'high_priority_load_files',
      -module => 'Bio::EnsEMBL::DBLoader::RunnableDB::LoadFiles',
      -parameters => { target_db => $self->o('target_db') },
      -hive_capacity => 1,
      -failed_job_tolerance => 100,
      -can_be_empty => 1,
    },

    {
      -logic_name => 'load_files',
      -module => 'Bio::EnsEMBL::DBLoader::RunnableDB::LoadFiles',
      -parameters => { target_db => $self->o('target_db') },
      -hive_capacity => 1,
      -wait_for => [qw/prioritise high_priority_load_files/],
      -retry_count => 1,
      -failed_job_tolerance => 50,
      -can_be_empty => 1,
    },
  ];
}

1;
{'fraction_non_alphanumeric': 0.17522103571018485, 'fraction_numerical': 0.0074635434607876906, 'mean_word_length': 2.6658249158249157, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 213, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '823721', 'n_tokens_mistral': 3193, 'n_tokens_neox': 3033, 'n_words': 574}