<commit_before>=head1 LICENSE

Copyright [1999-2015] Wellcome Trust Sanger Institute and the EMBL-European Bioinformatics Institute
Copyright [2016-2017] EMBL-European Bioinformatics Institute

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

=cut


=pod 

=head1 NAME

Bio::EnsEMBL::Compara::PipeConfig::Families_conf

=head1 SYNOPSIS

    #0. make sure that ProteinTree pipeline (whose EnsEMBL peptide members you want to incorporate) is already past member loading stage

    #1. update ensembl-hive, ensembl and ensembl-compara GIT repositories before each new release

    #3. make sure that all default_options are set correctly

    #4. Run init_pipeline.pl script:
        init_pipeline.pl Bio::EnsEMBL::Compara::PipeConfig::Families_conf -password <your_password>

    #5. Run the "beekeeper.pl ... -loop" command suggested by init_pipeline.pl

    #6. Please remember that mapping_session, stable_id_history, member and sequence tables will have to be MERGED in an intelligent way, and not just written over.
        ReleaseCoordination.txt document explains how to do the merge correctly.

=head1 DESCRIPTION  

The PipeConfig file for Families pipeline that should automate most of the tasks

=head1 CONTACT

Please email comments or questions to the public Ensembl
developers list at <http://lists.ensembl.org/mailman/listinfo/dev>.

Questions may also be sent to the Ensembl help desk at
<http://www.ensembl.org/Help/Contact>.

=cut



package Bio::EnsEMBL::Compara::PipeConfig::Families_conf;

use strict;
use warnings;

use Bio::EnsEMBL::Hive::Version 2.4;

use Bio::EnsEMBL::Hive::PipeConfig::HiveGeneric_conf;
use base ('Bio::EnsEMBL::Compara::PipeConfig::ComparaGeneric_conf');

sub default_options {
    my ($self) = @_;
    return {
        %{$self->SUPER::default_options},

        #'mlss_id'         => 30047,         # it is very important to check that this value is current (commented out to make it obligatory to specify)
        #'host'            => 'compara2',    # where the pipeline database will be created
        'file_basename'   => 'metazoa_families_'.$self->o('rel_with_suffix'),

        # HMM clustering
        #'hmm_clustering'  => 0,
        #'hmm_library_basedir'       => '/lustre/scratch109/sanger/fs9/treefam8_hmms',
        #'pantherScore_path'         => '/software/ensembl/compara/pantherScore1.03',
        #'hmmer2_home'               => '/software/ensembl/compara/hmmer-2.3.2/src/',

        # code directories:
        #'blast_bin_dir'   => '/software/ensembl/compara/ncbi-blast-2.2.30+/bin',
        #'mcl_bin_dir'     => '/software/ensembl/compara/mcl-14-137/bin',
        #'mafft_root_dir'  => '/software/ensembl/compara/mafft-7.221',

        # data directories:
        #'work_dir'        => '/lustre/scratch110/ensembl/'.$self->o('ENV', 'USER').'/'.$self->o('pipeline_name'),
        #'warehouse_dir'   => '/warehouse/ensembl05/lg4/families/',      # ToDo: move to a Compara-wide warehouse location
        'uniprot_dir'     => $self->o('work_dir').'/uniprot',
        'blastdb_dir'     => $self->o('work_dir').'/blast_db',
        'blastdb_name'    => $self->o('file_basename').'.pep',

        'uniprot_rel_url' => 'ftp://ftp.uniprot.org/pub/databases/uniprot/current_release/knowledgebase/taxonomic_divisions/reldate.txt',
        'uniprot_ftp_url' => 'ftp://ftp.uniprot.org/pub/databases/uniprot/current_release/knowledgebase/taxonomic_divisions/uniprot_#uniprot_source#_#tax_div#.dat.gz',

        #'blast_params'    => '', # By default C++ binary has composition stats on and -seg masking off

        #resource requirements:
        'blast_minibatch_size'  => 25,  # we want to reach the 1hr average runtime per minibatch
        'blast_gigs'      =>  4,
        'blast_hm_gigs'   =>  6,
        'mcl_gigs'        => 72,
        'mcl_threads'     => 12,
        'lomafft_gigs'    =>  4,
        'himafft_gigs'    => 14,
        'blast_capacity'  => 5000,                                  # work both as hive_capacity and resource-level throttle
        'mafft_capacity'  =>  400,
        'cons_capacity'   =>  100,
        'HMMer_classify_capacity' => 100,

    };
}


sub pipeline_create_commands {
    my ($self) = @_;
    return [
        @{$self->SUPER::pipeline_create_commands},  # here we inherit creation of database, hive tables and compara tables
        
        'mkdir -p '.$self->o('work_dir'),
        'mkdir -p '.$self->o('blastdb_dir'),
        'mkdir -p '.$self->o('uniprot_dir'),

            # perform "lfs setstripe" only if lfs is runnable and the directory is on lustre:
        'which lfs && lfs getstripe '.$self->o('blastdb_dir').' >/dev/null 2>/dev/null && lfs setstripe '.$self->o('blastdb_dir').' -c -1 || echo "Striping is not available on this system" ',
    ];
}


sub pipeline_wide_parameters {  # these parameter values are visible to all analyses, can be overridden by parameters{} and input_id{}
    my ($self) = @_;
    return {
        %{$self->SUPER::pipeline_wide_parameters},          # here we inherit anything from the base class

        'email'             => $self->o('email'),                   # for automatic notifications (may be unsupported by your Meadows)
        'mlss_id'           => $self->o('mlss_id'),
        'blast_params'      => $self->o('blast_params'),

        'work_dir'          => $self->o('work_dir'),                # data directories and filenames
        'warehouse_dir'     => $self->o('warehouse_dir'),
        'blastdb_dir'       => $self->o('blastdb_dir'),
        'uniprot_dir'       => $self->o('uniprot_dir'),
        'file_basename'     => $self->o('file_basename'),
        'blastdb_name'      => $self->o('blastdb_name'),

        'blast_bin_dir'     => $self->o('blast_bin_dir'),           # binary & script directories
        'mcl_bin_dir'       => $self->o('mcl_bin_dir'),
        'mafft_root_dir'    => $self->o('mafft_root_dir'),
        'mafft_threads'    => $self->o('mafft_threads'),

        'master_db'         => $self->o('master_db'),               # databases
        'member_db'         => $self->o('member_db'),

        'hmm_clustering'    => $self->o('hmm_clustering'),
    };
}

sub hive_meta_table {
    my ($self) = @_;
    return {
        %{$self->SUPER::hive_meta_table},       # here we inherit anything from the base class
        'hive_use_param_stack'  => 1,           # switch on the new param_stack mechanism
    }
}


sub pipeline_analyses {
    my ($self) = @_;
    return [

        {   -logic_name => 'copy_table_factory',
            -module     => 'Bio::EnsEMBL::Hive::RunnableDB::JobFactory',
            -input_ids => [ {} ],
            -parameters => {
                'inputlist'     => [
                                        [ '#member_db#'     => 'genome_db' ],       # we need them in "located" state
                                        [ '#master_db#'     => 'ncbi_taxa_node' ],
                                        [ '#master_db#'     => 'ncbi_taxa_name' ],
                                        [ '#master_db#'     => 'method_link' ],
                                        [ '#master_db#'     => 'species_set_header' ],
                                        [ '#master_db#'     => 'species_set' ],
                                        [ '#master_db#'     => 'method_link_species_set' ],
                                    ],
                'column_names'  => [ 'src_db_conn', 'table' ],
            },
            -flow_into => {
                '2->A' => [ 'copy_table' ],
                'A->1' => [ 'offset_tables' ],  # backbone
            },
        },

        {   -logic_name    => 'copy_table',
            -module        => 'Bio::EnsEMBL::Hive::RunnableDB::MySQLTransfer',
            -parameters    => {
                'mode'          => 'overwrite',
                'filter_cmd'    => 'sed "s/ENGINE=MyISAM/ENGINE=InnoDB/"',
            },
            -analysis_capacity => 10,
        },

        {   -logic_name => 'offset_tables',
            -module     => 'Bio::EnsEMBL::Compara::RunnableDB::GeneTrees::OffsetTables',
            -parameters => {
                'range_index'   => 2,
            },
            -flow_into => [ 'genomedb_factory' ],
        },

        {   -logic_name => 'genomedb_factory',
            -module     => 'Bio::EnsEMBL::Compara::RunnableDB::GenomeDBFactory',
            -flow_into => {
                '2->A' => [ 'genome_member_copy' ],
                'A->1' => [ 'hc_nonref_members' ],
            },
        },

        {   -logic_name        => 'genome_member_copy',
            -module            => 'Bio::EnsEMBL::Compara::RunnableDB::Families::CopyMembersByGenomeDB',
            -parameters        => {
                'reuse_db'              => '#member_db#',
                'biotype_filter'        => 'biotype_group IN ("coding","LRG")',
            },
            -analysis_capacity => 10,
            -rc_name           => '250Mb_job',
        },

        {   -logic_name         => 'hc_nonref_members',
            -module             => 'Bio::EnsEMBL::Compara::RunnableDB::Families::SqlHealthChecks',
            -parameters         => {
                mode            => 'nonref_members',
            },
            -flow_into  => [ 'save_uniprot_release_date' ],
        },<commit_msg>GeneTrees::OffsetTables is now exclusively for gene-tree tables. Here we need to offset the member tables<commit_after>=head1 LICENSE

Copyright [1999-2015] Wellcome Trust Sanger Institute and the EMBL-European Bioinformatics Institute
Copyright [2016-2017] EMBL-European Bioinformatics Institute

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

=cut


=pod 

=head1 NAME

Bio::EnsEMBL::Compara::PipeConfig::Families_conf

=head1 SYNOPSIS

    #0. make sure that ProteinTree pipeline (whose EnsEMBL peptide members you want to incorporate) is already past member loading stage

    #1. update ensembl-hive, ensembl and ensembl-compara GIT repositories before each new release

    #3. make sure that all default_options are set correctly

    #4. Run init_pipeline.pl script:
        init_pipeline.pl Bio::EnsEMBL::Compara::PipeConfig::Families_conf -password <<PASSWORD>>

    #5. Run the "beekeeper.pl ... -loop" command suggested by init_pipeline.pl

    #6. Please remember that mapping_session, stable_id_history, member and sequence tables will have to be MERGED in an intelligent way, and not just written over.
        ReleaseCoordination.txt document explains how to do the merge correctly.

=head1 DESCRIPTION  

The PipeConfig file for Families pipeline that should automate most of the tasks

=head1 CONTACT

Please email comments or questions to the public Ensembl
developers list at <http://lists.ensembl.org/mailman/listinfo/dev>.

Questions may also be sent to the Ensembl help desk at
<http://www.ensembl.org/Help/Contact>.

=cut



package Bio::EnsEMBL::Compara::PipeConfig::Families_conf;

use strict;
use warnings;

use Bio::EnsEMBL::Hive::Version 2.4;

use Bio::EnsEMBL::Hive::PipeConfig::HiveGeneric_conf;
use base ('Bio::EnsEMBL::Compara::PipeConfig::ComparaGeneric_conf');

sub default_options {
    my ($self) = @_;
    return {
        %{$self->SUPER::default_options},

        #'mlss_id'         => 30047,         # it is very important to check that this value is current (commented out to make it obligatory to specify)
        #'host'            => 'compara2',    # where the pipeline database will be created
        'file_basename'   => 'metazoa_families_'.$self->o('rel_with_suffix'),

        # HMM clustering
        #'hmm_clustering'  => 0,
        #'hmm_library_basedir'       => '/lustre/scratch109/sanger/fs9/treefam8_hmms',
        #'pantherScore_path'         => '/software/ensembl/compara/pantherScore1.03',
        #'hmmer2_home'               => '/software/ensembl/compara/hmmer-2.3.2/src/',

        # code directories:
        #'blast_bin_dir'   => '/software/ensembl/compara/ncbi-blast-2.2.30+/bin',
        #'mcl_bin_dir'     => '/software/ensembl/compara/mcl-14-137/bin',
        #'mafft_root_dir'  => '/software/ensembl/compara/mafft-7.221',

        # data directories:
        #'work_dir'        => '/lustre/scratch110/ensembl/'.$self->o('ENV', 'USER').'/'.$self->o('pipeline_name'),
        #'warehouse_dir'   => '/warehouse/ensembl05/lg4/families/',      # ToDo: move to a Compara-wide warehouse location
        'uniprot_dir'     => $self->o('work_dir').'/uniprot',
        'blastdb_dir'     => $self->o('work_dir').'/blast_db',
        'blastdb_name'    => $self->o('file_basename').'.pep',

        'uniprot_rel_url' => 'ftp://ftp.uniprot.org/pub/databases/uniprot/current_release/knowledgebase/taxonomic_divisions/reldate.txt',
        'uniprot_ftp_url' => 'ftp://ftp.uniprot.org/pub/databases/uniprot/current_release/knowledgebase/taxonomic_divisions/uniprot_#uniprot_source#_#tax_div#.dat.gz',

        #'blast_params'    => '', # By default C++ binary has composition stats on and -seg masking off

        #resource requirements:
        'blast_minibatch_size'  => 25,  # we want to reach the 1hr average runtime per minibatch
        'blast_gigs'      =>  4,
        'blast_hm_gigs'   =>  6,
        'mcl_gigs'        => 72,
        'mcl_threads'     => 12,
        'lomafft_gigs'    =>  4,
        'himafft_gigs'    => 14,
        'blast_capacity'  => 5000,                                  # work both as hive_capacity and resource-level throttle
        'mafft_capacity'  =>  400,
        'cons_capacity'   =>  100,
        'HMMer_classify_capacity' => 100,

    };
}


sub pipeline_create_commands {
    my ($self) = @_;
    return [
        @{$self->SUPER::pipeline_create_commands},  # here we inherit creation of database, hive tables and compara tables
        
        'mkdir -p '.$self->o('work_dir'),
        'mkdir -p '.$self->o('blastdb_dir'),
        'mkdir -p '.$self->o('uniprot_dir'),

            # perform "lfs setstripe" only if lfs is runnable and the directory is on lustre:
        'which lfs && lfs getstripe '.$self->o('blastdb_dir').' >/dev/null 2>/dev/null && lfs setstripe '.$self->o('blastdb_dir').' -c -1 || echo "Striping is not available on this system" ',
    ];
}


sub pipeline_wide_parameters {  # these parameter values are visible to all analyses, can be overridden by parameters{} and input_id{}
    my ($self) = @_;
    return {
        %{$self->SUPER::pipeline_wide_parameters},          # here we inherit anything from the base class

        'email'             => $self->o('email'),                   # for automatic notifications (may be unsupported by your Meadows)
        'mlss_id'           => $self->o('mlss_id'),
        'blast_params'      => $self->o('blast_params'),

        'work_dir'          => $self->o('work_dir'),                # data directories and filenames
        'warehouse_dir'     => $self->o('warehouse_dir'),
        'blastdb_dir'       => $self->o('blastdb_dir'),
        'uniprot_dir'       => $self->o('uniprot_dir'),
        'file_basename'     => $self->o('file_basename'),
        'blastdb_name'      => $self->o('blastdb_name'),

        'blast_bin_dir'     => $self->o('blast_bin_dir'),           # binary & script directories
        'mcl_bin_dir'       => $self->o('mcl_bin_dir'),
        'mafft_root_dir'    => $self->o('mafft_root_dir'),
        'mafft_threads'    => $self->o('mafft_threads'),

        'master_db'         => $self->o('master_db'),               # databases
        'member_db'         => $self->o('member_db'),

        'hmm_clustering'    => $self->o('hmm_clustering'),
    };
}

sub hive_meta_table {
    my ($self) = @_;
    return {
        %{$self->SUPER::hive_meta_table},       # here we inherit anything from the base class
        'hive_use_param_stack'  => 1,           # switch on the new param_stack mechanism
    }
}


sub pipeline_analyses {
    my ($self) = @_;
    return [

        {   -logic_name => 'copy_table_factory',
            -module     => 'Bio::EnsEMBL::Hive::RunnableDB::JobFactory',
            -input_ids => [ {} ],
            -parameters => {
                'inputlist'     => [
                                        [ '#member_db#'     => 'genome_db' ],       # we need them in "located" state
                                        [ '#master_db#'     => 'ncbi_taxa_node' ],
                                        [ '#master_db#'     => 'ncbi_taxa_name' ],
                                        [ '#master_db#'     => 'method_link' ],
                                        [ '#master_db#'     => 'species_set_header' ],
                                        [ '#master_db#'     => 'species_set' ],
                                        [ '#master_db#'     => 'method_link_species_set' ],
                                    ],
                'column_names'  => [ 'src_db_conn', 'table' ],
            },
            -flow_into => {
                '2->A' => [ 'copy_table' ],
                'A->1' => [ 'offset_tables' ],  # backbone
            },
        },

        {   -logic_name    => 'copy_table',
            -module        => 'Bio::EnsEMBL::Hive::RunnableDB::MySQLTransfer',
            -parameters    => {
                'mode'          => 'overwrite',
                'filter_cmd'    => 'sed "s/ENGINE=MyISAM/ENGINE=InnoDB/"',
            },
            -analysis_capacity => 10,
        },

        {   -logic_name => 'offset_tables',
            -module     => 'Bio::EnsEMBL::Hive::RunnableDB::SqlCmd',
            -parameters => {
                'sql'   => [
                    'ALTER TABLE sequence                   AUTO_INCREMENT=800000001',
                    'ALTER TABLE seq_member                 AUTO_INCREMENT=8000000001',
                ],
            },
            -flow_into => [ 'genomedb_factory' ],
        },

        {   -logic_name => 'genomedb_factory',
            -module     => 'Bio::EnsEMBL::Compara::RunnableDB::GenomeDBFactory',
            -flow_into => {
                '2->A' => [ 'genome_member_copy' ],
                'A->1' => [ 'hc_nonref_members' ],
            },
        },

        {   -logic_name        => 'genome_member_copy',
            -module            => 'Bio::EnsEMBL::Compara::RunnableDB::Families::CopyMembersByGenomeDB',
            -parameters        => {
                'reuse_db'              => '#member_db#',
                'biotype_filter'        => 'biotype_group IN ("coding","LRG")',
            },
            -analysis_capacity => 10,
            -rc_name           => '250Mb_job',
        },

        {   -logic_name         => 'hc_nonref_members',
            -module             => 'Bio::EnsEMBL::Compara::RunnableDB::Families::SqlHealthChecks',
            -parameters         => {
                mode            => 'nonref_members',
            },
            -flow_into  => [ 'save_uniprot_release_date' ],
        },
{'fraction_non_alphanumeric': 0.1336573511543135, 'fraction_numerical': 0.012150668286755772, 'mean_word_length': 2.0729620410703173, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 264, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 34, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '133125', 'n_tokens_mistral': 6496, 'n_tokens_neox': 5926, 'n_words': 1301}