<commit_before>
=pod 

=head1 NAME

  Bio::EnsEMBL::Compara::PipeConfig::ImportNCBItaxonomy_conf

=head1 SYNOPSIS

    init_pipeline.pl Bio::EnsEMBL::Compara::PipeConfig::ImportNCBItaxonomy_conf -password <<PASSWORD>> -ensembl_cvs_root_dir <path_to_your_ensembl_cvs_root>
    init_pipeline.pl Bio::EnsEMBL::Compara::PipeConfig::ImportNCBItaxonomy_conf -password <<PASSWORD>>

=head1 DESCRIPTION  

    A pipeline to import NCBI taxonomy database into ncbi_taxonomy@ens-livemirror database

=head1 CONTACT

  Please contact <EMAIL> mailing list with questions/suggestions.

=cut

package Bio::EnsEMBL::Compara::PipeConfig::ImportNCBItaxonomy_conf;

use strict;
use warnings;

use base ('Bio::EnsEMBL::Hive::PipeConfig::HiveGeneric_conf');      # we want to treat it as a 'pure' Hive pipeline


=head2 default_options

    Description : Implements default_options() interface method of Bio::EnsEMBL::Hive::PipeConfig::HiveGeneric_conf that is used to initialize default options.
                
                  There are rules dependent on two options that do not have defaults (this makes them mandatory):
                    o('password')       your read-write password for creation and maintenance of the hive database

=cut

sub default_options {
    my ($self) = @_;
    return {
         %{$self->SUPER::default_options},

        'pipeline_name' => 'ncbi_taxonomy',            # name used by the beekeeper to prefix job names on the farm

        'name_prefix'   => $self->o('ENV', 'USER').'_', # use a non-empty value if you want to test the pipeline
        'name_suffix'   => '_66c',                      # use a non-empty value if you want to test the pipeline

        'pipeline_db' => {
#            -host   => 'ens-livemirror',
            -host   => 'compara3',
            -port   => 3306,
            -user   => 'ensadmin',
            -pass   => $self->o('password'),
            -dbname => $self->o('name_prefix').$self->o('pipeline_name').$self->o('name_suffix'),
        },

        'taxdump_loc'   => 'ftp://ftp.ncbi.nih.gov/pub/taxonomy',   # the original location of the dump
        'taxdump_file'  => 'taxdump.tar.gz',                        # the filename of the dump

        'work_dir'      => $ENV{'HOME'}.'/ncbi_taxonomy_loading',
    };
}

=head2 pipeline_create_commands

    Description : Implements pipeline_create_commands() interface method of Bio::EnsEMBL::Hive::PipeConfig::HiveGeneric_conf that lists the commands that will create and set up the Hive database.
                  In addition to the standard creation of the database and populating it with Hive tables and procedures it also creates a working directory to store intermediate files.

=cut

sub pipeline_create_commands {
    my ($self) = @_;
    return [
        @{$self->SUPER::pipeline_create_commands},  # inheriting database and hive tables' creation

            # additional tables that we use here (taken from ensembl-compara schema):

        $self->db_execute_command('pipeline_db', qq{
            CREATE TABLE ncbi_taxa_node (
              taxon_id                        INT(10) UNSIGNED NOT NULL,
              parent_id                       INT(10) UNSIGNED NOT NULL,

              rank                            CHAR(32) DEFAULT \"\" NOT NULL,
              genbank_hidden_flag             TINYINT(1) DEFAULT 0 NOT NULL,

              left_index                      INT(10) NOT NULL,
              right_index                     INT(10) NOT NULL,
              root_id                         INT(10) DEFAULT 1 NOT NULL,

              PRIMARY KEY (taxon_id),
              KEY (parent_id),
              KEY (rank),
              KEY (left_index),
              KEY (right_index)
            )
        }),

        $self->db_execute_command('pipeline_db', qq{
            CREATE TABLE ncbi_taxa_name (
              taxon_id                    INT(10) UNSIGNED NOT NULL,

              name                        VARCHAR(255),
              name_class                  VARCHAR(50),

              KEY (taxon_id),
              KEY (name),
              KEY (name_class)
            )
        }),

        'mkdir '.$self->o('work_dir'),
    ];
}

sub resource_classes {
    my ($self) = @_;
    return {
         0 => { -desc => 'default',          'LSF' => '' },
         1 => { -desc => 'urgent',           'LSF' => '-q yesterday' },
         2 => { -desc => 'himem',            'LSF' => '-q yesterday -R"select[mem>3000] rusage[mem=3000]" -M3000000' },
    };
}


=head2 pipeline_analyses

    Description : Implements pipeline_analyses() interface method of Bio::EnsEMBL::Hive::PipeConfig::HiveGeneric_conf that defines the structure of the pipeline: analyses, jobs, rules, etc.


=cut

sub pipeline_analyses {
    my ($self) = @_;
    return [
        {   -logic_name    => 'download_tarball',
            -module        => 'Bio::EnsEMBL::Hive::RunnableDB::SystemCmd',
            -parameters    => {
                'cmd'       => 'curl '.$self->o('taxdump_loc').'/'.$self->o('taxdump_file').' > #work_dir#/'.$self->o('taxdump_file'),
            },
            -input_ids     => [
                { 'work_dir' => $self->o('work_dir') }
            ],
            -hive_capacity  => 10,  # to allow parallel branches
            -flow_into => {
                1 => [ 'untar' ],
            },
            -rc_id => 1,
        },

        {   -logic_name    => 'untar',
            -module        => 'Bio::EnsEMBL::Hive::RunnableDB::SystemCmd',
            -parameters    => {
                'cmd'       => 'cd #work_dir# ; tar -xzf #work_dir#/'.$self->o('taxdump_file'),
            },
            -hive_capacity  => 10,  # to allow parallel branches
            -flow_into => {
                1 => [ 'load_nodes', 'load_names' ],
            },
            -rc_id => 1,
        },

        {   -logic_name => 'load_nodes',
            -module     => 'Bio::EnsEMBL::Hive::RunnableDB::JobFactory',
            -parameters => {
                'inputfile'       => '#work_dir#/nodes.dmp',
                'delimiter'       => "\t\Q|\E\t?",
                'input_id'        => { 'taxon_id' => '#_0#', 'parent_id' => '#_1#', 'rank' => '#_2#', 'genbank_hidden_flag' => '#_10#'},
                'fan_branch_code' => 2,
            },
            -hive_capacity  => 10,  # to allow parallel branches
            -flow_into => {
                1 => [ 'zero_parent_id' ],
                2 => [ ':////ncbi_taxa_node' ],
            },
            -rc_id => 2,
        },

        {   -logic_name    => 'zero_parent_id',
            -module        => 'Bio::EnsEMBL::Hive::RunnableDB::SqlCmd',
            -parameters    => {
                'sql'         => "update ncbi_taxa_node set parent_id=0 where parent_id=taxon_id",
            },
            -hive_capacity  => 10,  # to allow parallel branches
            -flow_into => {
                1 => [ 'build_left_right_indices' ],
            },
            -rc_id => 1,
        },

        {   -logic_name    => 'build_left_right_indices',
            -module        => 'Bio::EnsEMBL::Hive::RunnableDB::SystemCmd',
            -parameters    => {
                'cmd'       => $self->o('ensembl_cvs_root_dir').'/ensembl-compara/scripts/taxonomy/taxonTreeTool.pl -url '.$self->dbconn_2_url('pipeline_db').' -index',
            },
            -hive_capacity  => 10,  # to allow parallel branches
            -wait_for => ['load_names'],
            -rc_id => 1,
        },



        {   -logic_name => 'load_names',
            -module     => 'Bio::EnsEMBL::Hive::RunnableDB::JobFactory',
            -parameters => {
                'inputfile'       => '#work_dir#/names.dmp',
                'delimiter'       => "\t\Q|\E\t?",
                'input_id'        => { 'taxon_id' => '#_0#', 'name' => '#_1#', 'name_class' => '#_3#'},
                'fan_branch_code' => 2,
            },
            -hive_capacity  => 10,  # to allow parallel branches
            -flow_into => {
                1 => [ 'load_merged_names' ],
                2 => [ ':////ncbi_taxa_name' ],
            },
            -rc_id => 2,
        },

        {   -logic_name => 'load_merged_names',
            -module     => 'Bio::EnsEMBL::Hive::RunnableDB::JobFactory',
            -parameters => {
                'inputfile'       => '#work_dir#/merged.dmp',<commit_msg>More memory for the left/right indices computation<commit_after>
=pod 

=head1 NAME

  Bio::EnsEMBL::Compara::PipeConfig::ImportNCBItaxonomy_conf

=head1 SYNOPSIS

    init_pipeline.pl Bio::EnsEMBL::Compara::PipeConfig::ImportNCBItaxonomy_conf -password <<PASSWORD>> -ensembl_cvs_root_dir <path_to_your_ensembl_cvs_root>
    init_pipeline.pl Bio::EnsEMBL::Compara::PipeConfig::ImportNCBItaxonomy_conf -password <your_password>

=head1 DESCRIPTION  

    A pipeline to import NCBI taxonomy database into ncbi_taxonomy@ens-livemirror database

=head1 CONTACT

  Please contact <EMAIL> mailing list with questions/suggestions.

=cut

package Bio::EnsEMBL::Compara::PipeConfig::ImportNCBItaxonomy_conf;

use strict;
use warnings;

use base ('Bio::EnsEMBL::Hive::PipeConfig::HiveGeneric_conf');      # we want to treat it as a 'pure' Hive pipeline


=head2 default_options

    Description : Implements default_options() interface method of Bio::EnsEMBL::Hive::PipeConfig::HiveGeneric_conf that is used to initialize default options.
                
                  There are rules dependent on two options that do not have defaults (this makes them mandatory):
                    o('password')       your read-write password for creation and maintenance of the hive database

=cut

sub default_options {
    my ($self) = @_;
    return {
         %{$self->SUPER::default_options},

        'pipeline_name' => 'ncbi_taxonomy',            # name used by the beekeeper to prefix job names on the farm

        'name_prefix'   => $self->o('ENV', 'USER').'_', # use a non-empty value if you want to test the pipeline
        'name_suffix'   => '_66c',                      # use a non-empty value if you want to test the pipeline

        'pipeline_db' => {
#            -host   => 'ens-livemirror',
            -host   => 'compara3',
            -port   => 3306,
            -user   => 'ensadmin',
            -pass   => $self->o('password'),
            -dbname => $self->o('name_prefix').$self->o('pipeline_name').$self->o('name_suffix'),
        },

        'taxdump_loc'   => 'ftp://ftp.ncbi.nih.gov/pub/taxonomy',   # the original location of the dump
        'taxdump_file'  => 'taxdump.tar.gz',                        # the filename of the dump

        'work_dir'      => $ENV{'HOME'}.'/ncbi_taxonomy_loading',
    };
}

=head2 pipeline_create_commands

    Description : Implements pipeline_create_commands() interface method of Bio::EnsEMBL::Hive::PipeConfig::HiveGeneric_conf that lists the commands that will create and set up the Hive database.
                  In addition to the standard creation of the database and populating it with Hive tables and procedures it also creates a working directory to store intermediate files.

=cut

sub pipeline_create_commands {
    my ($self) = @_;
    return [
        @{$self->SUPER::pipeline_create_commands},  # inheriting database and hive tables' creation

            # additional tables that we use here (taken from ensembl-compara schema):

        $self->db_execute_command('pipeline_db', qq{
            CREATE TABLE ncbi_taxa_node (
              taxon_id                        INT(10) UNSIGNED NOT NULL,
              parent_id                       INT(10) UNSIGNED NOT NULL,

              rank                            CHAR(32) DEFAULT \"\" NOT NULL,
              genbank_hidden_flag             TINYINT(1) DEFAULT 0 NOT NULL,

              left_index                      INT(10) NOT NULL,
              right_index                     INT(10) NOT NULL,
              root_id                         INT(10) DEFAULT 1 NOT NULL,

              PRIMARY KEY (taxon_id),
              KEY (parent_id),
              KEY (rank),
              KEY (left_index),
              KEY (right_index)
            )
        }),

        $self->db_execute_command('pipeline_db', qq{
            CREATE TABLE ncbi_taxa_name (
              taxon_id                    INT(10) UNSIGNED NOT NULL,

              name                        VARCHAR(255),
              name_class                  VARCHAR(50),

              KEY (taxon_id),
              KEY (name),
              KEY (name_class)
            )
        }),

        'mkdir '.$self->o('work_dir'),
    ];
}

sub resource_classes {
    my ($self) = @_;
    return {
         0 => { -desc => 'default',          'LSF' => '' },
         1 => { -desc => 'urgent',           'LSF' => '-q yesterday' },
         2 => { -desc => 'himem',            'LSF' => '-q yesterday -R"select[mem>3000] rusage[mem=3000]" -M3000000' },
    };
}


=head2 pipeline_analyses

    Description : Implements pipeline_analyses() interface method of Bio::EnsEMBL::Hive::PipeConfig::HiveGeneric_conf that defines the structure of the pipeline: analyses, jobs, rules, etc.


=cut

sub pipeline_analyses {
    my ($self) = @_;
    return [
        {   -logic_name    => 'download_tarball',
            -module        => 'Bio::EnsEMBL::Hive::RunnableDB::SystemCmd',
            -parameters    => {
                'cmd'       => 'curl '.$self->o('taxdump_loc').'/'.$self->o('taxdump_file').' > #work_dir#/'.$self->o('taxdump_file'),
            },
            -input_ids     => [
                { 'work_dir' => $self->o('work_dir') }
            ],
            -hive_capacity  => 10,  # to allow parallel branches
            -flow_into => {
                1 => [ 'untar' ],
            },
            -rc_id => 1,
        },

        {   -logic_name    => 'untar',
            -module        => 'Bio::EnsEMBL::Hive::RunnableDB::SystemCmd',
            -parameters    => {
                'cmd'       => 'cd #work_dir# ; tar -xzf #work_dir#/'.$self->o('taxdump_file'),
            },
            -hive_capacity  => 10,  # to allow parallel branches
            -flow_into => {
                1 => [ 'load_nodes', 'load_names' ],
            },
            -rc_id => 1,
        },

        {   -logic_name => 'load_nodes',
            -module     => 'Bio::EnsEMBL::Hive::RunnableDB::JobFactory',
            -parameters => {
                'inputfile'       => '#work_dir#/nodes.dmp',
                'delimiter'       => "\t\Q|\E\t?",
                'input_id'        => { 'taxon_id' => '#_0#', 'parent_id' => '#_1#', 'rank' => '#_2#', 'genbank_hidden_flag' => '#_10#'},
                'fan_branch_code' => 2,
            },
            -hive_capacity  => 10,  # to allow parallel branches
            -flow_into => {
                1 => [ 'zero_parent_id' ],
                2 => [ ':////ncbi_taxa_node' ],
            },
            -rc_id => 2,
        },

        {   -logic_name    => 'zero_parent_id',
            -module        => 'Bio::EnsEMBL::Hive::RunnableDB::SqlCmd',
            -parameters    => {
                'sql'         => "update ncbi_taxa_node set parent_id=0 where parent_id=taxon_id",
            },
            -hive_capacity  => 10,  # to allow parallel branches
            -flow_into => {
                1 => [ 'build_left_right_indices' ],
            },
            -rc_id => 1,
        },

        {   -logic_name    => 'build_left_right_indices',
            -module        => 'Bio::EnsEMBL::Hive::RunnableDB::SystemCmd',
            -parameters    => {
                'cmd'       => $self->o('ensembl_cvs_root_dir').'/ensembl-compara/scripts/taxonomy/taxonTreeTool.pl -url '.$self->dbconn_2_url('pipeline_db').' -index',
            },
            -hive_capacity  => 10,  # to allow parallel branches
            -wait_for => ['load_names'],
            -rc_id => 2,
        },



        {   -logic_name => 'load_names',
            -module     => 'Bio::EnsEMBL::Hive::RunnableDB::JobFactory',
            -parameters => {
                'inputfile'       => '#work_dir#/names.dmp',
                'delimiter'       => "\t\Q|\E\t?",
                'input_id'        => { 'taxon_id' => '#_0#', 'name' => '#_1#', 'name_class' => '#_3#'},
                'fan_branch_code' => 2,
            },
            -hive_capacity  => 10,  # to allow parallel branches
            -flow_into => {
                1 => [ 'load_merged_names' ],
                2 => [ ':////ncbi_taxa_name' ],
            },
            -rc_id => 2,
        },

        {   -logic_name => 'load_merged_names',
            -module     => 'Bio::EnsEMBL::Hive::RunnableDB::JobFactory',
            -parameters => {
                'inputfile'       => '#work_dir#/merged.dmp',
{'fraction_non_alphanumeric': 0.1347971760201029, 'fraction_numerical': 0.010889075026923538, 'mean_word_length': 1.7038175347783888, 'pattern_counts': {'":': 0, '<': 14, '<?xml version=': 0, '>': 232, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1889980', 'n_tokens_mistral': 5306, 'n_tokens_neox': 4937, 'n_words': 1075}