<commit_before>=pod 

=head1 NAME

  Bio::EnsEMBL::Compara::StableId::Adaptor

=head1 SYNOPSIS

=head1 DESCRIPTION

    All database I/O should be done via this class,
    which has a general functionality of an adaptor, but strictly speaking isn't.

=cut

package Bio::EnsEMBL::Compara::StableId::Adaptor;

use strict;
use DBI;
use Treefam::Tree;
use Bio::EnsEMBL::Utils::Argument;  # import 'rearrange()'
use Bio::EnsEMBL::Compara::StableId::NamedClusterSet;
use Bio::EnsEMBL::Compara::StableId::Map;
use Bio::EnsEMBL::Utils::Exception qw(throw);

sub new {
    my $class = shift @_;

    my $self = bless { }, $class;

    my ($release_in_progress, $dg_suffix_in_progress, $dbname_in_progress, $hist_contrib_unit) =
         rearrange([qw(release_in_progress dg_suffix_in_progress dbname_in_progress hist_contrib_unit) ], @_);

    $self->release_in_progress($release_in_progress) if(defined($release_in_progress));
    $self->dg_suffix_in_progress($dg_suffix_in_progress) if(defined($dg_suffix_in_progress));
    $self->dbname_in_progress($dbname_in_progress) if(defined($dbname_in_progress));
    $self->hist_contrib_unit($hist_contrib_unit || 100);

    return $self;
}

sub treefam_dbh {
    my ($self, $release) = @_;

    my $dbh = DBI->connect("DBI:mysql:mysql_use_result=1;host=db.treefam.org;port=3308;database=treefam_${release}", 'anonymous', '');
    
    return $dbh;
}

sub dbh_from_dgsuffix_dbname {
    my ($self, $dg_suffix, $dbname) = @_;

    my $dbh = DBI->connect("DBI:mysql:mysql_read_default_group=client${dg_suffix};mysql_use_result=1;database=${dbname}") or die "Connection Error: $DBI::errstr\n";

    return $dbh;
}

sub guess_dbh {     # only works if you have .my.cnf properly pre-filled with connection parameters
    my ($self, $release, $type) = @_;

    if(($type eq 'c') or ($type eq 'w')) {

        warn "${type}${release} - going to load the data from 'treefam_${release}'\n";

        return $self->treefam_dbh($release);

    } elsif (($type eq 'f') or ($type eq 't')) {
        
        my($dg_suffix, $dbname);

        if($self->release_in_progress() and ($release==$self->release_in_progress())) {
            $dg_suffix = $self->dg_suffix_in_progress();
        } elsif ($release<=47) {
            $dg_suffix = '_archive';
        } else {
            $dg_suffix = '_narchive';
        }

        if($self->release_in_progress() and ($release==$self->release_in_progress())) {
            $dbname = $self->dbname_in_progress();
        } elsif($release>28) {
            $dbname = "ensembl_compara_${release}";
        } else {
            $dbname = "ensembl_compara_${release}_1";
        }

        warn "${type}${release} - going to load the data from '$dbname'\n";

        return $self->dbh_from_dgsuffix_dbname($dg_suffix, $dbname);
    }
}

sub fetch_ncs {
    my ($self, $release, $type, $dbh) = @_;

    $dbh ||= $self->guess_dbh($release, $type);

    my $ncs = Bio::EnsEMBL::Compara::StableId::NamedClusterSet->new(-TYPE => $type, -RELEASE => $release);

    if( ($type eq 'f') or ($type eq 't') ) {
        $self->load_compara_ncs($ncs, $dbh);
    } elsif( ($type eq 'c') or ($type eq 'w') ) {
        $self->load_treefam_ncs($ncs, $dbh);
    }

    return $ncs;
}

sub load_treefam_ncs {
    my ($self, $ncs, $dbh) = @_;

    my $step = 30000;
    my $tree_type = ($ncs->type() eq 'c') ? 'CLEAN' : 'FULL';
    my $sql = qq{ SELECT TRIM(LEADING 'TF' FROM ac), ac, tree FROM trees WHERE type = ?};
    my $sth = $dbh->prepare($sql);
    warn "\t- waiting for the data to start coming\n";
    $sth->execute($tree_type);
    warn "\t- done waiting\n";
    my $counter = 0;

    my $dummy_dbc = {}; # let's see if it works

    while(my($tree_id, $tree_name, $tree_code) = $sth->fetchrow()) {
        next if (!$tree_code or ($tree_code eq ';') or ($tree_code eq ',_null_;') or ($tree_code eq '_null_;;[&&NHX:O=_null_;;];') );

        eval {
            my $tree = Treefam::Tree->new($dummy_dbc, $tree_name, $tree_type, $tree_code);
            foreach my $leaf ($tree->get_leaves()) {
                if(my $member = $leaf->sequence_id()) {
                    $member=~s/\.\d+$//;

                    $ncs->mname2clid($member, $tree_id);
                    $ncs->clid2clname($tree_id, $tree_name);

                    unless(++$counter % $step) {
                        warn "\t- $counter\n";
                    }
                }
            }
        };
        if($@) {
            warn "Problem with tree '$tree_name' ($@) \n TreeCode = [ $tree_code ]\n";
        }
    }
    $sth->finish();
    warn "\t- total of $counter members fetched\n";

    return $ncs;
}

sub load_compara_ncs {
    my ($self, $ncs, $dbh) = @_;
    
    #Need the schema's version not the reported version from the 
    #NamedClusterSet as people like EG's releases are not the same as Ensembl's
    my $schema_version = $self->_get_db_version($dbh);
    warn "\t- Detected DB Version is ${schema_version}\n";

    my $step = ($ncs->type() eq 'f') ? 100000 : 30000;
    my $sql = ($ncs->type() eq 'f')
    ? qq{
        SELECT f.family_id, }.(($schema_version<53)?"f.stable_id":"CONCAT(f.stable_id,'.',f.version)").qq{,
            IF(m.source_name='ENSEMBLPEP', SUBSTRING_INDEX(TRIM(LEADING 'Transcript:' FROM m.description),' ',1), m.stable_id)
        FROM family f, family_member fm, member m
        WHERE f.family_id=fm.family_id
        AND   fm.member_id=m.member_id
        AND   m.source_name <> 'ENSEMBLGENE'
    } : qq{
        SELECT ptn.node_id, }.(($schema_version<53)?"CONCAT('Node_',ptn.node_id)":"IFNULL(CONCAT(ptsi.stable_id,'.',ptsi.version), CONCAT('Node_',ptn.node_id))").qq{,
            IF(m.source_name='ENSEMBLPEP', SUBSTRING_INDEX(TRIM(LEADING 'Transcript:' FROM m.description),' ',1), m.stable_id)
        FROM protein_tree_node ptn
        LEFT JOIN protein_tree_member n2m ON ptn.node_id=n2m.node_id
        LEFT JOIN member m ON n2m.member_id=m.member_id
        }.(($schema_version<53) ? q{} :q{ LEFT JOIN protein_tree_stable_id ptsi ON ptn.node_id=ptsi.node_id}).
        ( ($schema_version < 55) ? q{ WHERE (ptn.parent_id = ptn.root_id } : q{ WHERE (ptn.node_id = ptn.root_id }).
        q{ OR m.stable_id IS NOT NULL) AND left_index AND right_index ORDER BY
        }.(($schema_version < 65) ? q{} : q{root_id,}).
        q{left_index };

    my $sth = $dbh->prepare($sql);
    warn "\t- waiting for the data to start coming\n";
    $sth->execute();
    warn "\t- done waiting\n";
    my $counter = 0;

    my $cached_id = 0;
    while(my($cluster_id, $cluster_name, $member)=$sth->fetchrow()) {

        $cluster_name ||= 'NoName'; # we need some name here however bogus (for formatting purposes)

        if($ncs->type() eq 'f') {
            $ncs->mname2clid($member, $cluster_id);
            $ncs->clid2clname($cluster_id, $cluster_name);
        } elsif($member) {<commit_msg>typo fix<commit_after>=pod 

=head1 NAME

  Bio::EnsEMBL::Compara::StableId::Adaptor

=head1 SYNOPSIS

=head1 DESCRIPTION

    All database I/O should be done via this class,
    which has a general functionality of an adaptor, but strictly speaking isn't.

=cut

package Bio::EnsEMBL::Compara::StableId::Adaptor;

use strict;
use DBI;
use Treefam::Tree;
use Bio::EnsEMBL::Utils::Argument;  # import 'rearrange()'
use Bio::EnsEMBL::Compara::StableId::NamedClusterSet;
use Bio::EnsEMBL::Compara::StableId::Map;
use Bio::EnsEMBL::Utils::Exception qw(throw);

sub new {
    my $class = shift @_;

    my $self = bless { }, $class;

    my ($release_in_progress, $dg_suffix_in_progress, $dbname_in_progress, $hist_contrib_unit) =
         rearrange([qw(release_in_progress dg_suffix_in_progress dbname_in_progress hist_contrib_unit) ], @_);

    $self->release_in_progress($release_in_progress) if(defined($release_in_progress));
    $self->dg_suffix_in_progress($dg_suffix_in_progress) if(defined($dg_suffix_in_progress));
    $self->dbname_in_progress($dbname_in_progress) if(defined($dbname_in_progress));
    $self->hist_contrib_unit($hist_contrib_unit || 100);

    return $self;
}

sub treefam_dbh {
    my ($self, $release) = @_;

    my $dbh = DBI->connect("DBI:mysql:mysql_use_result=1;host=db.treefam.org;port=3308;database=treefam_${release}", 'anonymous', '');
    
    return $dbh;
}

sub dbh_from_dgsuffix_dbname {
    my ($self, $dg_suffix, $dbname) = @_;

    my $dbh = DBI->connect("DBI:mysql:mysql_read_default_group=client${dg_suffix};mysql_use_result=1;database=${dbname}") or die "Connection Error: $DBI::errstr\n";

    return $dbh;
}

sub guess_dbh {     # only works if you have .my.cnf properly pre-filled with connection parameters
    my ($self, $release, $type) = @_;

    if(($type eq 'c') or ($type eq 'w')) {

        warn "${type}${release} - going to load the data from 'treefam_${release}'\n";

        return $self->treefam_dbh($release);

    } elsif (($type eq 'f') or ($type eq 't')) {
        
        my($dg_suffix, $dbname);

        if($self->release_in_progress() and ($release==$self->release_in_progress())) {
            $dg_suffix = $self->dg_suffix_in_progress();
        } elsif ($release<=47) {
            $dg_suffix = '_archive';
        } else {
            $dg_suffix = '_narchive';
        }

        if($self->release_in_progress() and ($release==$self->release_in_progress())) {
            $dbname = $self->dbname_in_progress();
        } elsif($release>28) {
            $dbname = "ensembl_compara_${release}";
        } else {
            $dbname = "ensembl_compara_${release}_1";
        }

        warn "${type}${release} - going to load the data from '$dbname'\n";

        return $self->dbh_from_dgsuffix_dbname($dg_suffix, $dbname);
    }
}

sub fetch_ncs {
    my ($self, $release, $type, $dbh) = @_;

    $dbh ||= $self->guess_dbh($release, $type);

    my $ncs = Bio::EnsEMBL::Compara::StableId::NamedClusterSet->new(-TYPE => $type, -RELEASE => $release);

    if( ($type eq 'f') or ($type eq 't') ) {
        $self->load_compara_ncs($ncs, $dbh);
    } elsif( ($type eq 'c') or ($type eq 'w') ) {
        $self->load_treefam_ncs($ncs, $dbh);
    }

    return $ncs;
}

sub load_treefam_ncs {
    my ($self, $ncs, $dbh) = @_;

    my $step = 30000;
    my $tree_type = ($ncs->type() eq 'c') ? 'CLEAN' : 'FULL';
    my $sql = qq{ SELECT TRIM(LEADING 'TF' FROM ac), ac, tree FROM trees WHERE type = ?};
    my $sth = $dbh->prepare($sql);
    warn "\t- waiting for the data to start coming\n";
    $sth->execute($tree_type);
    warn "\t- done waiting\n";
    my $counter = 0;

    my $dummy_dbc = {}; # let's see if it works

    while(my($tree_id, $tree_name, $tree_code) = $sth->fetchrow()) {
        next if (!$tree_code or ($tree_code eq ';') or ($tree_code eq ',_null_;') or ($tree_code eq '_null_;;[&&NHX:O=_null_;;];') );

        eval {
            my $tree = Treefam::Tree->new($dummy_dbc, $tree_name, $tree_type, $tree_code);
            foreach my $leaf ($tree->get_leaves()) {
                if(my $member = $leaf->sequence_id()) {
                    $member=~s/\.\d+$//;

                    $ncs->mname2clid($member, $tree_id);
                    $ncs->clid2clname($tree_id, $tree_name);

                    unless(++$counter % $step) {
                        warn "\t- $counter\n";
                    }
                }
            }
        };
        if($@) {
            warn "Problem with tree '$tree_name' ($@) \n TreeCode = [ $tree_code ]\n";
        }
    }
    $sth->finish();
    warn "\t- total of $counter members fetched\n";

    return $ncs;
}

sub load_compara_ncs {
    my ($self, $ncs, $dbh) = @_;
    
    #Need the schema's version not the reported version from the 
    #NamedClusterSet as people like EG's releases are not the same as Ensembl's
    my $schema_version = $self->_get_db_version($dbh);
    warn "\t- Detected DB Version is ${schema_version}\n";

    my $step = ($ncs->type() eq 'f') ? 100000 : 30000;
    my $sql = ($ncs->type() eq 'f')
    ? qq{
        SELECT f.family_id, }.(($schema_version<53)?"f.stable_id":"CONCAT(f.stable_id,'.',f.version)").qq{,
            IF(m.source_name='ENSEMBLPEP', SUBSTRING_INDEX(TRIM(LEADING 'Transcript:' FROM m.description),' ',1), m.stable_id)
        FROM family f, family_member fm, member m
        WHERE f.family_id=fm.family_id
        AND   fm.member_id=m.member_id
        AND   m.source_name <> 'ENSEMBLGENE'
    } : qq{
        SELECT ptn.node_id, }.(($schema_version<53)?"CONCAT('Node_',ptn.node_id)":"IFNULL(CONCAT(ptsi.stable_id,'.',ptsi.version), CONCAT('Node_',ptn.node_id))").qq{,
            IF(m.source_name='ENSEMBLPEP', SUBSTRING_INDEX(TRIM(LEADING 'Transcript:' FROM m.description),' ',1), m.stable_id)
        FROM protein_tree_node ptn
        LEFT JOIN protein_tree_member n2m ON ptn.node_id=n2m.node_id
        LEFT JOIN member m ON n2m.member_id=m.member_id
        }.(($schema_version<53) ? q{} :q{ LEFT JOIN protein_tree_stable_id ptsi ON ptn.node_id=ptsi.node_id}).
        ( ($schema_version < 55) ? q{ WHERE (ptn.parent_id = ptn.root_id } : q{ WHERE (ptn.node_id = ptn.root_id }).
        q{ OR m.stable_id IS NOT NULL) AND left_index AND right_index ORDER BY
        }.(($schema_version < 65) ? q{} : q{ptn.root_id,}).
        q{left_index };

    my $sth = $dbh->prepare($sql);
    warn "\t- waiting for the data to start coming\n";
    $sth->execute();
    warn "\t- done waiting\n";
    my $counter = 0;

    my $cached_id = 0;
    while(my($cluster_id, $cluster_name, $member)=$sth->fetchrow()) {

        $cluster_name ||= 'NoName'; # we need some name here however bogus (for formatting purposes)

        if($ncs->type() eq 'f') {
            $ncs->mname2clid($member, $cluster_id);
            $ncs->clid2clname($cluster_id, $cluster_name);
        } elsif($member) {
{'fraction_non_alphanumeric': 0.16578007724258545, 'fraction_numerical': 0.008015739998542593, 'mean_word_length': 2.9814331302581953, 'pattern_counts': {'":': 4, '<': 17, '<?xml version=': 0, '>': 85, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1880160', 'n_tokens_mistral': 5170, 'n_tokens_neox': 4911, 'n_words': 1051}