<commit_before>import logging
import os
from stat import ST_SIZE
import re
from zipfile import ZipFile
from datetime import datetime
import time
from dipper.sources.Source import Source
from dipper.models.Model import Model
from dipper.models.Dataset import Dataset
from dipper.models.assoc.Association import Assoc
from dipper.models.Genotype import Genotype
import ftplib
import pandas as pd

logger = logging.getLogger(__name__)


class Bgee(Source):
    """
    Bgee is a database to retrieve and compare gene expression
    patterns between animal species.

    Bgee first maps heterogeneous expression data (currently RNA-Seq,
    Affymetrix, in situ hybridization, and EST data) to anatomy and
    development of different species.

    Then, in order to perform automated cross species comparisons,
    homology relationships across anatomies, and comparison criteria
    between developmental stages, are designed.
    """
    BGEE_FTP = 'ftp.bgee.org'
    DEFAULT_TAXA = [10090, 10116, 13616, 28377, 6239,
                    7227, 7955, 8364, 9031, 9258,
                    9544, 9593, 9597, 9598, 9606,
                    9823, 9913]
    files = {
        'anat_entity': {
            'path': 'current/download/ranks/anat_entity/',
            'pattern': re.compile(r'.*_all_data_.*')
        }
    }

    def __init__(self, graph_type, are_bnodes_skolemized, tax_ids=None):
        """
        :param tax_ids: [int,], List of taxa
        :return:
        """
        super().__init__(graph_type, are_bnodes_skolemized, 'bgee')
        if tax_ids is None:
            self.tax_ids = Bgee.DEFAULT_TAXA
        else:
            logger.info("Filtering on taxa {}".format(tax_ids))
            self.tax_ids = tax_ids

        self.dataset = Dataset(
            'Bgee', 'Bgee Gene expression data in animals',
            'http://bgee.org/')

    def fetch(self, is_dl_forced=False):
        """
        :param is_dl_forced: boolean, force download
        :return:
        """
        for group in self.files:
            files_to_download, ftp = \
                self._get_file_list(self.files[group]['path'],
                                    self.files[group]['pattern'])
            for name, info in files_to_download:
                localfile = '/'.join((self.rawdir, name))
                if not os.path.exists(localfile)\
                        or is_dl_forced\
                        or self.checkIfRemoteIsNewer(localfile, info['size'],
                                                     info['modify']):
                    logger.info("Fetching {}".format(name))
                    logger.info("Writing to {}".format(localfile))
                    ftp.retrbinary('RETR {}'.format(name), open(localfile, 'wb').write)
                    remote_dt = Bgee._convert_ftp_time_to_iso(info['modify'])
                    os.utime(localfile, (time.mktime(remote_dt.timetuple()),
                                         time.mktime(remote_dt.timetuple())))

        ftp.quit()

        return

    def parse(self, limit=None):
        """
        Given the input taxa, expects files in the raw directory
        with the name {tax_id}_anat_entity_all_data_Pan_troglodytes.tsv.zip
        :param limit: int Limit to top ranked anatomy associations per group
        :return: None
        """
        files_to_download, ftp = \
            self._get_file_list(self.files['anat_entity']['path'],
                                self.files['anat_entity']['pattern'])
        for name, info in files_to_download:
            localfile = '/'.join((self.rawdir, name))
            with ZipFile(localfile, 'r') as zip_file:
                fh = zip_file.open(re.sub(r'\.zip$', '', name))
                logger.info("Processing {}".format(name))
                self._parse_gene_anatomy(fh, limit)

        return

    def _parse_gene_anatomy(self, fh, limit):
        """
        Process anat_entity files with columns:
        Ensembl gene ID,gene name, anatomical entity ID,
        anatomical entity name, rank score, XRefs to BTO
        :param fh: filehandle
        :param limit: int, limit per group
        :return: None
        """
        dataframe = pd.read_csv(fh, sep='\t')
        gene_groups = dataframe.sort_values('rank score', ascending=False)\
                               .groupby('Ensembl gene ID')

        if limit is not None:
            gene_groups = gene_groups.head(limit).groupby('Ensembl gene ID')

        for gene, group in gene_groups:
            for index, row in group.iterrows():
                self._add_gene_anatomy_association(
                    row['Ensembl gene ID'], row['anatomical entity ID'],
                    row['rank score']
                )
        return

    def _add_gene_anatomy_association(self, gene_id, anatomy_curie, rank):
        """
        :param gene_id: str Non curified ID
        :param gene_label: str Gene symbol
        :param anatomy_curie: str curified anatomy term
        :param rank: str rank
        :return: None
        """
        g2a_association = Assoc(self.graph, self.name)
        genotype = Genotype(self.graph)
        model = Model(self.graph)
        if re.match(r'^FBgn', gene_id):
            gene_curie = "FlyBase:{}".format(gene_id)
        else:
            gene_curie = "ENSEMBL:{}".format(gene_id)
        rank = re.sub(r',', '', rank)
        model.addIndividualToGraph(ind_id=gene_curie, label=None,
                                   ind_type=genotype.genoparts['gene'])
        g2a_association.sub = gene_curie
        g2a_association.obj = anatomy_curie
        g2a_association.rel = Assoc.object_properties['expressed_in']
        g2a_association.add_association_to_graph()
        g2a_association.add_predicate_object(
            Assoc.datatype_properties['has_quantifier'],
            float(rank), 'Literal', 'xsd:float')
        return

    # Override
    def checkIfRemoteIsNewer(self, localfile, remote_size, remote_modify):
        """<commit_msg>switch back to ensembl prefix<commit_after>import logging
import os
from stat import ST_SIZE
import re
from zipfile import ZipFile
from datetime import datetime
import time
from dipper.sources.Source import Source
from dipper.models.Model import Model
from dipper.models.Dataset import Dataset
from dipper.models.assoc.Association import Assoc
from dipper.models.Genotype import Genotype
import ftplib
import pandas as pd

logger = logging.getLogger(__name__)


class Bgee(Source):
    """
    Bgee is a database to retrieve and compare gene expression
    patterns between animal species.

    Bgee first maps heterogeneous expression data (currently RNA-Seq,
    Affymetrix, in situ hybridization, and EST data) to anatomy and
    development of different species.

    Then, in order to perform automated cross species comparisons,
    homology relationships across anatomies, and comparison criteria
    between developmental stages, are designed.
    """
    BGEE_FTP = 'ftp.bgee.org'
    DEFAULT_TAXA = [10090, 10116, 13616, 28377, 6239,
                    7227, 7955, 8364, 9031, 9258,
                    9544, 9593, 9597, 9598, 9606,
                    9823, 9913]
    files = {
        'anat_entity': {
            'path': 'current/download/ranks/anat_entity/',
            'pattern': re.compile(r'.*_all_data_.*')
        }
    }

    def __init__(self, graph_type, are_bnodes_skolemized, tax_ids=None):
        """
        :param tax_ids: [int,], List of taxa
        :return:
        """
        super().__init__(graph_type, are_bnodes_skolemized, 'bgee')
        if tax_ids is None:
            self.tax_ids = Bgee.DEFAULT_TAXA
        else:
            logger.info("Filtering on taxa {}".format(tax_ids))
            self.tax_ids = tax_ids

        self.dataset = Dataset(
            'Bgee', 'Bgee Gene expression data in animals',
            'http://bgee.org/')

    def fetch(self, is_dl_forced=False):
        """
        :param is_dl_forced: boolean, force download
        :return:
        """
        for group in self.files:
            files_to_download, ftp = \
                self._get_file_list(self.files[group]['path'],
                                    self.files[group]['pattern'])
            for name, info in files_to_download:
                localfile = '/'.join((self.rawdir, name))
                if not os.path.exists(localfile)\
                        or is_dl_forced\
                        or self.checkIfRemoteIsNewer(localfile, info['size'],
                                                     info['modify']):
                    logger.info("Fetching {}".format(name))
                    logger.info("Writing to {}".format(localfile))
                    ftp.retrbinary('RETR {}'.format(name), open(localfile, 'wb').write)
                    remote_dt = Bgee._convert_ftp_time_to_iso(info['modify'])
                    os.utime(localfile, (time.mktime(remote_dt.timetuple()),
                                         time.mktime(remote_dt.timetuple())))

        ftp.quit()

        return

    def parse(self, limit=None):
        """
        Given the input taxa, expects files in the raw directory
        with the name {tax_id}_anat_entity_all_data_Pan_troglodytes.tsv.zip
        :param limit: int Limit to top ranked anatomy associations per group
        :return: None
        """
        files_to_download, ftp = \
            self._get_file_list(self.files['anat_entity']['path'],
                                self.files['anat_entity']['pattern'])
        for name, info in files_to_download:
            localfile = '/'.join((self.rawdir, name))
            with ZipFile(localfile, 'r') as zip_file:
                fh = zip_file.open(re.sub(r'\.zip$', '', name))
                logger.info("Processing {}".format(name))
                self._parse_gene_anatomy(fh, limit)

        return

    def _parse_gene_anatomy(self, fh, limit):
        """
        Process anat_entity files with columns:
        Ensembl gene ID,gene name, anatomical entity ID,
        anatomical entity name, rank score, XRefs to BTO
        :param fh: filehandle
        :param limit: int, limit per group
        :return: None
        """
        dataframe = pd.read_csv(fh, sep='\t')
        gene_groups = dataframe.sort_values('rank score', ascending=False)\
                               .groupby('Ensembl gene ID')

        if limit is not None:
            gene_groups = gene_groups.head(limit).groupby('Ensembl gene ID')

        for gene, group in gene_groups:
            for index, row in group.iterrows():
                self._add_gene_anatomy_association(
                    row['Ensembl gene ID'], row['anatomical entity ID'],
                    row['rank score']
                )
        return

    def _add_gene_anatomy_association(self, gene_id, anatomy_curie, rank):
        """
        :param gene_id: str Non curified ID
        :param gene_label: str Gene symbol
        :param anatomy_curie: str curified anatomy term
        :param rank: str rank
        :return: None
        """
        g2a_association = Assoc(self.graph, self.name)
        genotype = Genotype(self.graph)
        model = Model(self.graph)
        gene_curie = "ENSEMBL:{}".format(gene_id)
        rank = re.sub(r',', '', rank)
        model.addIndividualToGraph(ind_id=gene_curie, label=None,
                                   ind_type=genotype.genoparts['gene'])
        g2a_association.sub = gene_curie
        g2a_association.obj = anatomy_curie
        g2a_association.rel = Assoc.object_properties['expressed_in']
        g2a_association.add_association_to_graph()
        g2a_association.add_predicate_object(
            Assoc.datatype_properties['has_quantifier'],
            float(rank), 'Literal', 'xsd:float')
        return

    # Override
    def checkIfRemoteIsNewer(self, localfile, remote_size, remote_modify):
        """
{'fraction_non_alphanumeric': 0.09971366009769243, 'fraction_numerical': 0.013137948458817585, 'mean_word_length': 2.040194572452637, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 24, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3672854', 'n_tokens_mistral': 3654, 'n_tokens_neox': 3374, 'n_words': 834}