<commit_before>=head1 LICENSE

Copyright [1999-2015] Wellcome Trust Sanger Institute and the EMBL-European Bioinformatics Institute
Copyright [2016-2020] EMBL-European Bioinformatics Institute

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

=cut

package Bio::EnsEMBL::Production::Pipeline::ProteinFeatures::ChecksumProteins;

use strict;
use warnings;
use base ('Bio::EnsEMBL::Production::Pipeline::Common::Base');

use Bio::SeqIO;
use DBI qw(:sql_types);
use Digest::MD5 qw(md5_hex);
use File::Spec::Functions qw(catdir);
use Path::Tiny;

sub run {
  my ($self) = @_;

  my $fasta_file         = $self->param_required('fasta_file');
  my $uniparc_xrefs      = $self->param_required('uniparc_xrefs');
  my $uniprot_xrefs      = $self->param_required('uniprot_xrefs');
  my $uniparc_logic_name = $self->param_required('uniparc_logic_name');
  my $uniprot_logic_name = $self->param_required('uniprot_logic_name');

  my $fasta = path($fasta_file);
  my $out_dir = $fasta->parent;
  my $basename = $fasta->basename;
  my $checksum_file = catdir($out_dir, 'checksum', $basename);
  my $nochecksum_file = catdir($out_dir, 'nochecksum', $basename);
  path($checksum_file)->parent->mkpath;
  path($nochecksum_file)->parent->mkpath;

  $self->param('checksum_file', $checksum_file);
  $self->param('nochecksum_file', $nochecksum_file);
  
  my $all        = Bio::SeqIO->new(-format => 'Fasta', -file => $fasta_file);
  my $checksum   = Bio::SeqIO->new(-format => 'Fasta', -file => ">$checksum_file");
  my $nochecksum = Bio::SeqIO->new(-format => 'Fasta', -file => ">$nochecksum_file");

  my $uniparc_sql = "SELECT upi FROM uniparc WHERE md5sum = ?";
  my $dbh = $self->hive_dbh;
  my $uniparc_sth = $dbh->prepare($uniparc_sql) or $self->throw($dbh->errstr);

  my ($uniparc_analysis, $uniprot_analysis, $uniprot_sth, $dbea);
  if ($uniparc_xrefs) {
    my $dba  = $self->get_DBAdaptor('core');
    my $aa   = $dba->get_adaptor('Analysis');
    my $dbea = $dba->get_adaptor('DBEntry');

    $uniparc_analysis = $aa->fetch_by_logic_name($uniparc_logic_name);

    if ($uniprot_xrefs) {
      $uniprot_analysis = $aa->fetch_by_logic_name($uniprot_logic_name);

      my $uniprot_sql = "SELECT acc FROM uniprot WHERE upi = ?";
      $uniprot_sth = $dbh->prepare($uniprot_sql) or $self->throw($dbh->errstr);
    }

    $dba->dbc && $dba->dbc->disconnect_if_idle();
  }

  while (my $seq = $all->next_seq) {
    my $upi = $self->fetch_upi($uniparc_sth, md5_hex($seq->seq));

    if (defined $upi) {
      my $success = $checksum->write_seq($seq);
      $self->throw("Failed to write sequence to '$checksum_file'") unless $success;

      if ($uniparc_xrefs) {
        my $uniparc_xref = $self->add_uniparc_xref($upi, $uniparc_analysis, $dbea, $seq->id);
        if ($uniprot_xrefs) {
          my $accs = $self->fetch_acc($uniprot_sth, $upi);
          if (scalar(@$accs)) {
            $self->add_uniprot_xref($accs, $uniprot_analysis, $dbea, $seq->id, $uniparc_xref);
          }
        }
      }
    } else {
      my $success = $nochecksum->write_seq($seq);
      $self->throw("Failed to write sequence to '$nochecksum_file'") unless $success;
    }
  }
}

sub write_output {
  my ($self) = @_;
  
  $self->dataflow_output_id({'checksum_file' => $self->param('checksum_file')}, 3);
  $self->dataflow_output_id({'nochecksum_file' => $self->param('nochecksum_file')}, 4);
}

sub fetch_upi {
  my ($self, $sth, $md5sum) = @_;

  $sth->bind_param(1, $md5sum, SQL_CHAR);
  $sth->execute();

  my $upi;
  my $results = $sth->fetchall_arrayref;

  if (scalar(@$results)) {
    $upi = $$results[0][0];
    if (scalar(@$results) > 1) {
      $self->warning("Multiple UPIs found for $md5sum");
    }
  }

  return $upi;
}

sub add_uniparc_xref {
  my ($self, $upi, $analysis, $dbea, $translation_id) = @_;<commit_msg>Set external_db release to null, which is required in order to assign xrefs for the relevant source.<commit_after>=head1 LICENSE

Copyright [1999-2015] Wellcome Trust Sanger Institute and the EMBL-European Bioinformatics Institute
Copyright [2016-2020] EMBL-European Bioinformatics Institute

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

=cut

package Bio::EnsEMBL::Production::Pipeline::ProteinFeatures::ChecksumProteins;

use strict;
use warnings;
use base ('Bio::EnsEMBL::Production::Pipeline::Common::Base');

use Bio::SeqIO;
use DBI qw(:sql_types);
use Digest::MD5 qw(md5_hex);
use File::Spec::Functions qw(catdir);
use Path::Tiny;

sub run {
  my ($self) = @_;

  my $fasta_file         = $self->param_required('fasta_file');
  my $uniparc_xrefs      = $self->param_required('uniparc_xrefs');
  my $uniprot_xrefs      = $self->param_required('uniprot_xrefs');
  my $uniparc_logic_name = $self->param_required('uniparc_logic_name');
  my $uniprot_logic_name = $self->param_required('uniprot_logic_name');

  my $fasta = path($fasta_file);
  my $out_dir = $fasta->parent;
  my $basename = $fasta->basename;
  my $checksum_file = catdir($out_dir, 'checksum', $basename);
  my $nochecksum_file = catdir($out_dir, 'nochecksum', $basename);
  path($checksum_file)->parent->mkpath;
  path($nochecksum_file)->parent->mkpath;

  $self->param('checksum_file', $checksum_file);
  $self->param('nochecksum_file', $nochecksum_file);
  
  my $all        = Bio::SeqIO->new(-format => 'Fasta', -file => $fasta_file);
  my $checksum   = Bio::SeqIO->new(-format => 'Fasta', -file => ">$checksum_file");
  my $nochecksum = Bio::SeqIO->new(-format => 'Fasta', -file => ">$nochecksum_file");

  my $uniparc_sql = "SELECT upi FROM uniparc WHERE md5sum = ?";
  my $dbh = $self->hive_dbh;
  my $uniparc_sth = $dbh->prepare($uniparc_sql) or $self->throw($dbh->errstr);

  my ($uniparc_analysis, $uniprot_analysis, $uniprot_sth, $dbea);
  if ($uniparc_xrefs) {
    my $dba = $self->get_DBAdaptor('core');
    my $aa = $dba->get_adaptor('Analysis');
    $dbea = $dba->get_adaptor('DBEntry');

    $uniparc_analysis = $aa->fetch_by_logic_name($uniparc_logic_name);
    $self->external_db_reset($dba, 'UniParc');

    if ($uniprot_xrefs) {
      $uniprot_analysis = $aa->fetch_by_logic_name($uniprot_logic_name);
      $self->external_db_reset($dba, 'UniProtKB_all');
      $self->external_db_reset($dba, 'Uniprot/Varsplic');

      my $tax_id = $dba->get_adaptor('MetaContainer')->get_taxonomy_id();
      my $uniprot_sql = "SELECT acc FROM uniprot WHERE upi = ? and tax_id = $tax_id";
      $uniprot_sth = $dbh->prepare($uniprot_sql) or $self->throw($dbh->errstr);
    }

    $dba->dbc && $dba->dbc->disconnect_if_idle();
  }

  while (my $seq = $all->next_seq) {
    my $upi = $self->fetch_upi($uniparc_sth, md5_hex($seq->seq));

    if (defined $upi) {
      my $success = $checksum->write_seq($seq);
      $self->throw("Failed to write sequence to '$checksum_file'") unless $success;

      if ($uniparc_xrefs) {
        my $uniparc_xref = $self->add_uniparc_xref($upi, $uniparc_analysis, $dbea, $seq->id);
        if ($uniprot_xrefs) {
          my $accs = $self->fetch_acc($uniprot_sth, $upi);
          if (scalar(@$accs)) {
            $self->add_uniprot_xref($accs, $uniprot_analysis, $dbea, $seq->id, $uniparc_xref);
          }
        }
      }
    } else {
      my $success = $nochecksum->write_seq($seq);
      $self->throw("Failed to write sequence to '$nochecksum_file'") unless $success;
    }
  }
}

sub write_output {
  my ($self) = @_;
  
  $self->dataflow_output_id({'checksum_file' => $self->param('checksum_file')}, 3);
  $self->dataflow_output_id({'nochecksum_file' => $self->param('nochecksum_file')}, 4);
}

sub external_db_reset {
  my ($self, $dba, $db_name) = @_;

  my $dbh = $dba->dbc->db_handle();
  my $sql = "UPDATE external_db SET db_release = NULL WHERE db_name = ?;";
  my $sth = $dbh->prepare($sql);
  $sth->execute($db_name) or $self->throw("Failed to execute ($db_name): $sql");
}

sub fetch_upi {
  my ($self, $sth, $md5sum) = @_;

  $sth->bind_param(1, $md5sum, SQL_CHAR);
  $sth->execute();

  my $upi;
  my $results = $sth->fetchall_arrayref;

  if (scalar(@$results)) {
    $upi = $$results[0][0];
    if (scalar(@$results) > 1) {
      $self->warning("Multiple UPIs found for $md5sum");
    }
  }

  return $upi;
}

sub add_uniparc_xref {
  my ($self, $upi, $analysis, $dbea, $translation_id) = @_;
{'fraction_non_alphanumeric': 0.16202090592334495, 'fraction_numerical': 0.007404181184668989, 'mean_word_length': 4.281771132834963, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 137, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2563868', 'n_tokens_mistral': 3531, 'n_tokens_neox': 3409, 'n_words': 725}