<commit_before># -*- coding: utf-8 -*-

"""
Copyright [2009-2017] EMBL-European Bioinformatics Institute
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

import itertools as it
import collections as coll

import attr

from rnacentral_pipeline.databases import data
from rnacentral_pipeline.databases.helpers import phylogeny as phy
from rnacentral_pipeline.databases.helpers import publications as pub


def secondary_structure(record):
    """
    Fetches the secondary structure, if any, of the given JSON schema entry.
    """

    dot_bracket = record.get('secondary_structure', None)
    if dot_bracket:
        return data.SecondaryStructure(dot_bracket=dot_bracket)
    return data.SecondaryStructure.empty()


def xrefs(record):
    """
    Fetches the cross references between this sequence and any other known
    databases.
    """

    grouped = coll.defaultdict(list)
    for xref in record.get('crossReferenceIds', []):
        dbid, accession = xref.split(':', 1)
        grouped[dbid].append(accession)
    return dict(grouped)


def taxid(entry):
    """
    Gets the NCBI taxon id as an integer.
    """

    base, tid = entry['taxonId'].split(':', 1)
    assert base == 'NCBITaxon'
    return int(tid)


def as_exon(assembly, exon):
    """
    Turn a raw exon into one we can store in the rnc_coordinates table.
    """

    complement = None
    if exon['strand'] == '+':
        complement = False
    elif exon['strand'] == '-':
        complement = True
    else:
        raise ValueError("Invalid strand %s" % exon)

    chromosome = exon['chromosome']
    if chromosome.startswith('chr'):
        chromosome = chromosome[3:]

    return data.Exon(
        chromosome_name=chromosome,
        # Input is 0 based, but we store 1 based
        primary_start=int(exon['startPosition']) + 1,
        primary_end=int(exon['endPosition']),
        assembly_id=assembly,
        complement=complement,
    )


def exons(genome_location):
    """
    Get all genomic locations this record is in.
    """
    assembly = genome_location['assembly']
    return [as_exon(assembly, e) for e in genome_location['exons']]


def gene_info(_):
    """
    Extract the gene level information from the given record if present.
    """
    pass


def as_reference(ref):
    """
    Turn a raw reference (just a pmid) into a reference we can import.
    """
    if ref.startswith('PMID:'):
        pmid = int(ref[5:])
        return pub.reference(pmid=pmid)
    return None
<commit_msg>Try to correct miRBase coordinates<commit_after># -*- coding: utf-8 -*-

"""
Copyright [2009-2017] EMBL-European Bioinformatics Institute
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

import itertools as it
import collections as coll

import attr

from rnacentral_pipeline.databases import data
from rnacentral_pipeline.databases.helpers import phylogeny as phy
from rnacentral_pipeline.databases.helpers import publications as pub


def secondary_structure(record):
    """
    Fetches the secondary structure, if any, of the given JSON schema entry.
    """

    dot_bracket = record.get('secondary_structure', None)
    if dot_bracket:
        return data.SecondaryStructure(dot_bracket=dot_bracket)
    return data.SecondaryStructure.empty()


def xrefs(record):
    """
    Fetches the cross references between this sequence and any other known
    databases.
    """

    grouped = coll.defaultdict(list)
    for xref in record.get('crossReferenceIds', []):
        dbid, accession = xref.split(':', 1)
        grouped[dbid].append(accession)
    return dict(grouped)


def taxid(entry):
    """
    Gets the NCBI taxon id as an integer.
    """

    base, tid = entry['taxonId'].split(':', 1)
    assert base == 'NCBITaxon'
    return int(tid)


def as_exon(assembly, exon):
    """
    Turn a raw exon into one we can store in the rnc_coordinates table.
    """

    complement = None
    if exon['strand'] == '+':
        complement = False
    elif exon['strand'] == '-':
        complement = True
    else:
        raise ValueError("Invalid strand %s" % exon)

    chromosome = exon['chromosome']
    if chromosome.startswith('chr'):
        chromosome = chromosome[3:]

    return data.Exon(
        chromosome_name=chromosome,
        # Input is 0 based, but we store 1 based
        primary_start=int(exon['startPosition']) + 1,
        primary_end=int(exon['endPosition']) + 1,
        assembly_id=assembly,
        complement=complement,
    )


def exons(genome_location):
    """
    Get all genomic locations this record is in.
    """
    assembly = genome_location['assembly']
    return [as_exon(assembly, e) for e in genome_location['exons']]


def gene_info(_):
    """
    Extract the gene level information from the given record if present.
    """
    pass


def as_reference(ref):
    """
    Turn a raw reference (just a pmid) into a reference we can import.
    """
    if ref.startswith('PMID:'):
        pmid = int(ref[5:])
        return pub.reference(pmid=pmid)
    return None

{'fraction_non_alphanumeric': 0.09894171006215353, 'fraction_numerical': 0.006887283722492861, 'mean_word_length': 3.2558970693352394, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1550556', 'n_tokens_mistral': 1818, 'n_tokens_neox': 1693, 'n_words': 589}