<commit_before>"""
Copyright [2009-2017] EMBL-European Bioinformatics Institute
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
     http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

import six
import operator as op
import itertools as it
from collections import Counter, defaultdict

from caching.base import CachingMixin, CachingManager
from django.core.exceptions import ObjectDoesNotExist
from django.core.urlresolvers import reverse
from django.db import models
from django.db.models import Prefetch, Min, Max, Q
from django.utils.functional import cached_property

from .database import Database
from .genomic_coordinates import GenomicCoordinates
from .modification import Modification
from .rna_precomputed import RnaPrecomputed
from .reference import Reference
from .xref import Xref
from .rfam import RfamHit, RfamAnalyzedSequences
from .accession import Accession
from .formatters import Gff3Formatter, GffFormatter, _xref_to_bed_format
from portal.utils import descriptions as desc
from portal.rfam_matches import check_issues
from portal.config.expert_databases import expert_dbs


class Rna(CachingMixin, models.Model):
    id = models.IntegerField(db_column='id')
    upi = models.CharField(max_length=13, db_index=True, primary_key=True)
    timestamp = models.DateField()
    userstamp = models.CharField(max_length=30)
    crc64 = models.CharField(max_length=16)
    length = models.IntegerField(db_column='len')
    seq_short = models.CharField(max_length=4000)
    seq_long = models.TextField()
    md5 = models.CharField(max_length=32, unique=True, db_index=True)

    objects = CachingManager()

    class Meta:
        db_table = 'rna'

    def get_absolute_url(self):
        """Get a URL for an RNA object. Used for generating sitemaps."""
        return reverse('unique-rna-sequence', kwargs={'upi': self.upi})

    def get_publications(self, taxid=None):
        """
        Get all publications associated with a Unique RNA Sequence.
        Use raw SQL query for better performance.

        Normally, querysets are lazily evaluated, which (among other benefits)
        allows code that receives queryset to paginate it later on (this is
        used by Django and DRF views).

        Here we want to do 2 things that require immediate evaluation of
        queryset:
         - filter out INSDC submissions only if there are no other papers
         - order publications, so that expert database releases go last

        Hopefully, the number of publications per RNA is not huge, so
        this shouldn't make our server too slow.
        """
        query = """
        SELECT b.id, b.location, b.title, b.pmid as pubmed, b.doi, b.authors
        FROM
            (SELECT DISTINCT t3.id
            FROM xref t1, rnc_reference_map t2, RNC_REFERENCES t3
            WHERE t1.ac = t2.accession AND
                  t1.upi = %s AND
                  {taxid_clause}
                  {deleted_clause}
                  t2.reference_id = t3.id) a
        JOIN
            rnc_references b
        ON a.id = b.id
        {where_clause}
        ORDER BY b.title
        """

        where_clause = "WHERE NOT ((b.title is NULL OR b.title = '') AND b.location LIKE 'Submitted%%')"
        taxid_clause = 't1.taxid = %s AND' % taxid
        deleted_clause = "t1.deleted = 'N' AND"

        # filter-out INSDC submissions with where_clause, deleted with deleted_clause; filter by taxid, if it's given
        if taxid:
            formatted_query = query.format(taxid_clause=taxid_clause, where_clause=where_clause, deleted_clause=deleted_clause)
        else:
            formatted_query = query.format(taxid_clause='', where_clause=where_clause, deleted_clause=deleted_clause)

        queryset = list(Reference.objects.raw(formatted_query, [self.upi]))

        # if queryset is empty, try finding at least INSDC submissions
        if len(queryset) == 0:
            if taxid:
                formatted_query = query.format(taxid_clause=taxid_clause, where_clause='', deleted_clause=deleted_clause)
            else:
                formatted_query = query.format(taxid_clause='', where_clause='', deleted_clause=deleted_clause)

            queryset = list(Reference.objects.raw(formatted_query, [self.upi]))

        # if queryset is still empty, try displaying deleted papers
        if len(queryset) == 0:
            if taxid:
                formatted_query = query.format(taxid_clause=taxid_clause, where_clause='', deleted_clause='')
            else:
                formatted_query = query.format(taxid_clause='', where_clause='', deleted_clause='')

            queryset = list(Reference.objects.raw(formatted_query, [self.upi]))

        # find expert dbs and move them to the end of the list, apply filtration
        references = {}
        for expert_db in expert_dbs:
            for reference in expert_db['references']:
                pubmed_id = reference['pubmed_id']
                references[pubmed_id] = expert_db

        expert_db_publications = []
        non_expert_db_publications = []

        for publication in queryset:
            if publication.pubmed in references:
                expert_db_publications.append(publication)
                publication.expert_db = True  # flags that it's an expert_db, UI should display a special label
            else:
                non_expert_db_publications.append(publication)
                publication.expert_db = False

        return non_expert_db_publications + expert_db_publications

    def is_active(self):
        """A sequence is considered active if it has at least one active cross_reference."""
        values_list = self.xrefs.values_list('deleted', flat=True)
        return 'N' in self.xrefs.values_list('deleted', flat=True).distinct()  # deleted xrefs are marked with N

    def has_genomic_coordinates(self, taxid=None):
        """Return True if at least one cross-reference has genomic coordinates."""
        xrefs = self.xrefs
        if taxid:
            xrefs = xrefs.filter(taxid=taxid)
        chromosomes = xrefs.all().values_list('accession__coordinates__chromosome', flat=True)
        for chromosome in chromosomes:
            if chromosome:
                return True
        return False

    def get_sequence(self):
        """
        Sequences of up to 4000 nucleotides are stored in seq_short, while the
        longer ones are in stored in seq_long.
        This was due to Oracle column size restrictions.
        """
        if self.seq_short:
            sequence = self.seq_short<commit_msg>Do not show redundant publications<commit_after>"""
Copyright [2009-2017] EMBL-European Bioinformatics Institute
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
     http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

import six
import operator as op
import itertools as it
from collections import Counter, defaultdict

from caching.base import CachingMixin, CachingManager
from django.core.exceptions import ObjectDoesNotExist
from django.core.urlresolvers import reverse
from django.db import models
from django.db.models import Prefetch, Min, Max, Q
from django.utils.functional import cached_property

from .database import Database
from .genomic_coordinates import GenomicCoordinates
from .modification import Modification
from .rna_precomputed import RnaPrecomputed
from .reference import Reference
from .xref import Xref
from .rfam import RfamHit, RfamAnalyzedSequences
from .accession import Accession
from .formatters import Gff3Formatter, GffFormatter, _xref_to_bed_format
from portal.utils import descriptions as desc
from portal.rfam_matches import check_issues
from portal.config.expert_databases import expert_dbs


class Rna(CachingMixin, models.Model):
    id = models.IntegerField(db_column='id')
    upi = models.CharField(max_length=13, db_index=True, primary_key=True)
    timestamp = models.DateField()
    userstamp = models.CharField(max_length=30)
    crc64 = models.CharField(max_length=16)
    length = models.IntegerField(db_column='len')
    seq_short = models.CharField(max_length=4000)
    seq_long = models.TextField()
    md5 = models.CharField(max_length=32, unique=True, db_index=True)

    objects = CachingManager()

    class Meta:
        db_table = 'rna'

    def get_absolute_url(self):
        """Get a URL for an RNA object. Used for generating sitemaps."""
        return reverse('unique-rna-sequence', kwargs={'upi': self.upi})

    def get_publications(self, taxid=None):
        """
        Get all publications associated with a Unique RNA Sequence.
        Use raw SQL query for better performance.

        Normally, querysets are lazily evaluated, which (among other benefits)
        allows code that receives queryset to paginate it later on (this is
        used by Django and DRF views).

        Here we want to do 2 things that require immediate evaluation of
        queryset:
         - filter out INSDC submissions only if there are no other papers
         - order publications, so that expert database releases go last

        Hopefully, the number of publications per RNA is not huge, so
        this shouldn't make our server too slow.
        """
        query = """
        SELECT b.id, b.location, b.title, b.pmid as pubmed, b.doi, b.authors
        FROM
            (SELECT DISTINCT t3.id
            FROM xref t1, rnc_reference_map t2, RNC_REFERENCES t3
            WHERE t1.ac = t2.accession AND
                  t1.upi = %s AND
                  {taxid_clause}
                  {deleted_clause}
                  t2.reference_id = t3.id) a
        JOIN
            rnc_references b
        ON a.id = b.id
        {where_clause}
        ORDER BY b.title
        """

        where_clause = "WHERE NOT ((b.title is NULL OR b.title = '') AND b.location LIKE 'Submitted%%')"
        taxid_clause = 't1.taxid = %s AND' % taxid
        deleted_clause = "t1.deleted = 'N' AND"

        # filter-out INSDC submissions with where_clause, deleted with deleted_clause; filter by taxid, if it's given
        if taxid:
            formatted_query = query.format(taxid_clause=taxid_clause, where_clause=where_clause, deleted_clause=deleted_clause)
        else:
            formatted_query = query.format(taxid_clause='', where_clause=where_clause, deleted_clause=deleted_clause)

        queryset = list(Reference.objects.raw(formatted_query, [self.upi]))

        # if queryset is empty, try finding at least INSDC submissions
        if len(queryset) == 0:
            if taxid:
                formatted_query = query.format(taxid_clause=taxid_clause, where_clause='', deleted_clause=deleted_clause)
            else:
                formatted_query = query.format(taxid_clause='', where_clause='', deleted_clause=deleted_clause)

            queryset = list(Reference.objects.raw(formatted_query, [self.upi]))

        # if queryset is still empty, try displaying deleted papers
        if len(queryset) == 0:
            if taxid:
                formatted_query = query.format(taxid_clause=taxid_clause, where_clause='', deleted_clause='')
            else:
                formatted_query = query.format(taxid_clause='', where_clause='', deleted_clause='')

            queryset = list(Reference.objects.raw(formatted_query, [self.upi]))

        # find expert dbs and move them to the end of the list, apply filtration
        references = {}
        for expert_db in expert_dbs:
            for reference in expert_db['references']:
                pubmed_id = reference['pubmed_id']
                references[pubmed_id] = expert_db

        expert_db_publications = []
        non_expert_db_publications = []

        titles = []
        for publication in queryset:
            if publication.pubmed in references and publication.title.lower() not in titles:
                expert_db_publications.append(publication)
                titles.append(publication.title.lower())
                publication.expert_db = True  # flags that it's an expert_db, UI should display a special label
            elif publication.title.lower() not in titles:
                non_expert_db_publications.append(publication)
                titles.append(publication.title.lower())
                publication.expert_db = False

        return non_expert_db_publications + expert_db_publications

    def is_active(self):
        """A sequence is considered active if it has at least one active cross_reference."""
        values_list = self.xrefs.values_list('deleted', flat=True)
        return 'N' in self.xrefs.values_list('deleted', flat=True).distinct()  # deleted xrefs are marked with N

    def has_genomic_coordinates(self, taxid=None):
        """Return True if at least one cross-reference has genomic coordinates."""
        xrefs = self.xrefs
        if taxid:
            xrefs = xrefs.filter(taxid=taxid)
        chromosomes = xrefs.all().values_list('accession__coordinates__chromosome', flat=True)
        for chromosome in chromosomes:
            if chromosome:
                return True
        return False

    def get_sequence(self):
        """
        Sequences of up to 4000 nucleotides are stored in seq_short, while the
        longer ones are in stored in seq_long.
        This was due to Oracle column size restrictions.
        """
        if self.seq_short:
            sequence = self.seq_short
{'fraction_non_alphanumeric': 0.07481841901135322, 'fraction_numerical': 0.006487553769127706, 'mean_word_length': 2.8216114254917812, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7463739', 'n_tokens_mistral': 4129, 'n_tokens_neox': 3937, 'n_words': 1259}