<commit_before>import logging
import os
import sys
from urllib.request import urlopen
from xml.etree.ElementTree import fromstring

import pandas as pd
import requests
import xlrd

from django.conf import settings
from django.core.management.base import BaseCommand
from django.db import IntegrityError
from protein.models import (Gene, Protein, ProteinAlias, ProteinConformation,
                            ProteinFamily, ProteinArrestinPair, ProteinSegment,
                            ProteinSequenceType, ProteinSource, ProteinState, Species)
from residue.models import (Residue, ResidueGenericNumber,
                            ResidueGenericNumberEquivalent,
                            ResidueNumberingScheme)
from signprot.models import SignprotStructure
from structure.models import Structure


class Command(BaseCommand):
    help = 'Build Arrestin proteins'

    # source files
    arrestin_data_file = os.sep.join([settings.DATA_DIR, 'arrestin_data', 'ortholog_alignment.xlsx'])
    bouvier_file = os.sep.join([settings.DATA_DIR, 'g_protein_data', '201025_bouvier_gloriam.xlsx'])
    local_uniprot_dir = os.sep.join([settings.DATA_DIR, 'protein_data', 'uniprot'])
    remote_uniprot_dir = 'https://uniprot.org/uniprot/'

    logger = logging.getLogger(__name__)

    def add_arguments(self, parser):
        parser.add_argument('--filename',
                            action='append',
                            dest='filename',
                            help='Filename to import. Can be used multiple times')
        parser.add_argument('--coupling',
                            default=False,
                            action='store_true',
                            help='Purge and import GPCR-Arrestin coupling data')

    def handle(self, *args, **options):
        self.options = options
        if options['filename']:
            filenames = options['filename']
        else:
            filenames = False
        if self.options['coupling']:
            self.purge_coupling_data()
            self.logger.info('PASS: purge_coupling_data')
            if os.path.exists(self.bouvier_file):
                self.add_bouvier_coupling_data()
                self.logger.info('PASS: add_bouvier_coupling_data')
            else:
                self.logger.warning('Bouvier source data ' + self.bouvier_file + ' not found')

        else:
            try:
                self.purge_can_residues()
                self.logger.info('PASS: purge_can_residues')
                self.purge_can_proteins()
                self.logger.info('PASS: purge_can_proteins')

                # add proteins
                self.can_create_families()
                self.logger.info('PASS: can_create_families')
                self.can_add_proteins()
                self.logger.info('PASS: can_add_proteins')

                # add residues
                self.add_can_residues()
                self.logger.info('PASS: add_can_residues')

                # add coupling data
                self.purge_coupling_data()
                self.logger.info('PASS: purge_coupling_data')
                if os.path.exists(self.bouvier_file):
                    self.add_bouvier_coupling_data()
                    self.logger.info('PASS: add_bouvier_coupling_data')
                else:
                    self.logger.warning('Bouvier source data ' + self.bouvier_file + ' not found')

            except Exception as msg:
                exc_type, exc_obj, exc_tb = sys.exc_info()
                fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
                print(exc_type, exc_obj, fname, exc_tb.tb_lineno)
                self.logger.error(msg)

    def purge_can_residues(self):
        """Purge residues."""
        try:
            Residue.objects.filter(generic_number_id__scheme__slug="can").delete()
        except Exception as msg:
            self.logger.warning('Existing Residue data cannot be deleted', msg)

    def purge_can_proteins(self):
        """Purge proteins."""
        try:
            Protein.objects.filter(residue_numbering_scheme__slug='can').delete()
        except Exception as msg:
            self.logger.warning('Protein to delete not found' + str(msg))

    def purge_coupling_data(self):
        """DROP data from the protein_arrestin_pair table."""
        try:
            ProteinArrestinPair.objects.filter().delete()
        except Exception as msg:
            self.logger.warning('Existing protein_arrestin_pair data cannot be deleted' + str(msg))

    @staticmethod
    def read_coupling(filenames=False):
        """Function to read Coupling data from Excel files.
        The ideal would be for the excel organization to hopefully be fixed in the same way for data
        coming from different groups. For now the data comes from Bouvier and has been processed by <NAME>.
        """
        book = xlrd.open_workbook(filenames)
        sheet1 = book.sheet_by_name("plain")
        rows = sheet1.nrows
        beglogmaxec50 = 36
        endlogmaxec50 = 38
        begpec50 = 52
        endpec50 = 54
        begemax = 68
        endemax = 70

        data = {}<commit_msg>CODACY: We'll see what codacy thinks now.<commit_after>import logging
import os
import sys
from urllib.request import urlopen
from xml.etree.ElementTree import fromstring

import pandas as pd
import requests
import xlrd

from django.conf import settings
from django.core.management.base import BaseCommand
from django.db import IntegrityError
from protein.models import (Gene, Protein, ProteinAlias, ProteinConformation,
                            ProteinFamily, ProteinArrestinPair, ProteinSegment,
                            ProteinSequenceType, ProteinSource, ProteinState, Species)
from residue.models import (Residue, ResidueGenericNumber,
                            ResidueGenericNumberEquivalent,
                            ResidueNumberingScheme)
from signprot.models import SignprotStructure
from structure.models import Structure


class Command(BaseCommand):
    help = 'Build Arrestin proteins'

    # source files
    arrestin_data_file = os.sep.join([settings.DATA_DIR, 'arrestin_data', 'ortholog_alignment.xlsx'])
    bouvier_file = os.sep.join([settings.DATA_DIR, 'g_protein_data', '201025_bouvier_gloriam.xlsx'])
    local_uniprot_dir = os.sep.join([settings.DATA_DIR, 'protein_data', 'uniprot'])
    remote_uniprot_dir = 'https://uniprot.org/uniprot/'

    logger = logging.getLogger(__name__)

    def add_arguments(self, parser):
        parser.add_argument('--filename',
                            action='append',
                            dest='filename',
                            help='Filename to import. Can be used multiple times')
        parser.add_argument('--coupling',
                            default=False,
                            action='store_true',
                            help='Purge and import GPCR-Arrestin coupling data')

    def handle(self, *args, **options):
        self.options = options
        if options['filename']:
            filenames = options['filename']
        else:
            filenames = False
        if self.options['coupling']:
            self.purge_coupling_data()
            self.logger.info('PASS: purge_coupling_data')
            if os.path.exists(self.bouvier_file):
                self.add_bouvier_coupling_data()
                self.logger.info('PASS: add_bouvier_coupling_data')
            else:
                self.logger.warning('Bouvier source data ' + self.bouvier_file + ' not found')

        else:
            try:
                self.purge_can_residues()
                self.logger.info('PASS: purge_can_residues')
                self.purge_can_proteins()
                self.logger.info('PASS: purge_can_proteins')

                # add proteins
                self.can_create_families()
                self.logger.info('PASS: can_create_families')
                self.can_add_proteins()
                self.logger.info('PASS: can_add_proteins')

                # add residues
                self.add_can_residues()
                self.logger.info('PASS: add_can_residues')

                # add coupling data
                self.purge_coupling_data()
                self.logger.info('PASS: purge_coupling_data')
                if os.path.exists(self.bouvier_file):
                    self.add_bouvier_coupling_data()
                    self.logger.info('PASS: add_bouvier_coupling_data')
                else:
                    self.logger.warning('Bouvier source data ' + self.bouvier_file + ' not found')

            except Exception as msg:
                exc_type, exc_obj, exc_tb = sys.exc_info()
                fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
                print(exc_type, exc_obj, fname, exc_tb.tb_lineno)
                self.logger.error(msg)

    def purge_can_residues(self):
        """Purge residues."""
        try:
            Residue.objects.filter(generic_number_id__scheme__slug="can").delete()
        except Exception as msg:
            self.logger.warning('Existing Residue data cannot be deleted', msg)

    def purge_can_proteins(self):
        """Purge proteins."""
        try:
            Protein.objects.filter(residue_numbering_scheme__slug='can').delete()
        except Exception as msg:
            self.logger.warning('Protein to delete not found' + str(msg))

    def purge_coupling_data(self):
        """DROP data from the protein_arrestin_pair table."""
        try:
            ProteinArrestinPair.objects.filter().delete()
        except Exception as msg:
            self.logger.warning('Existing protein_arrestin_pair data cannot be deleted' + str(msg))

    @staticmethod
    def read_coupling(filenames=False):
        """
        Function to read Coupling data from Excel files.

        The ideal would be for the excel organization to hopefully be fixed in the same way for data
        coming from different groups. For now the data comes from Bouvier and has been processed by <NAME>.
        """
        book = xlrd.open_workbook(filenames)
        sheet1 = book.sheet_by_name("plain")
        rows = sheet1.nrows
        beglogmaxec50 = 36
        endlogmaxec50 = 38
        begpec50 = 52
        endpec50 = 54
        begemax = 68
        endemax = 70

        data = {}
{'fraction_non_alphanumeric': 0.08720480061943477, 'fraction_numerical': 0.0056136275648470775, 'mean_word_length': 2.0844776119402986, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2756574', 'n_tokens_mistral': 3011, 'n_tokens_neox': 2764, 'n_words': 636}