<commit_before>from django.core.management.base import BaseCommand, CommandError
from django.conf import settings
from django.utils.text import slugify
from django.utils.html import strip_tags
from django.db import IntegrityError

from build.management.commands.base_build import Command as BaseBuild
from protein.models import (Protein, ProteinConformation, ProteinState, ProteinSequenceType, ProteinSegment, ProteinSource)
from residue.models import Residue
from structure.functions import ParseStructureCSV

import os
import logging
import yaml


class Command(BaseBuild):
    help = 'Reads source data and creates protein records for constructs'

    def add_arguments(self, parser):
        parser.add_argument('-p', '--proc',
            type=int,
            action='store',
            dest='proc',
            default=1,
            help='Number of processes to run')
        parser.add_argument('-c', '--construct',
            # action='append',
            dest='construct',
            help='Constructs to import (PDB ID)',
            nargs='+')
        parser.add_argument('-u', '--purge',
            action='store_true',
            dest='purge',
            default=False,
            help='Purge existing construct records')

    def handle(self, *args, **options):
        # delete any existing construct data
        if options['purge']:
            try:
                self.purge_constructs()
            except Exception as msg:
                print(msg)
                self.logger.error(msg)
        self.parsed_structures = ParseStructureCSV()

        # where pdb ids specified?
        if options['construct']:
            self.parsed_structures.pdb_ids = [i for i in self.parsed_structures.pdb_ids if i in options['construct'] or i.lower() in options['construct']]
        try:
            self.logger.info('CREATING CONSTRUCTS')
            self.prepare_input(options['proc'], self.parsed_structures.pdb_ids)
            self.logger.info('COMPLETED CREATING CONSTRUCTS')
        except Exception as msg:
            print(msg)
            self.logger.error(msg)

    def purge_constructs(self):
        try:
            pst = ProteinSequenceType.objects.get(slug='mod')
            Protein.objects.filter(sequence_type=pst).delete()
        except ProteinSequenceType.DoesNotExist:
            self.logger.warning('ProteinSequenceType mod not found: nothing to delete.')

    def main_func(self, positions, iteration,count,lock):
        # setting up processes
        if not positions[1]:
            constructs = self.parsed_structures.pdb_ids[positions[0]:]
        else:
            constructs = self.parsed_structures.pdb_ids[positions[0]:positions[1]]

        for sd in constructs:
            sd = self.parsed_structures.structures[sd]
            # is a protein specified?
            if 'protein' not in sd:
                self.logger.error('Protein not specified for construct, skipping')
                continue

            # fetch the parent protein
            try:
                ppc = ProteinConformation.objects.prefetch_related('protein__family', 'protein__species',
                    'protein__residue_numbering_scheme').get(protein__entry_name=sd['protein'],
                    state__slug=settings.DEFAULT_PROTEIN_STATE)
            except ProteinConformation.DoesNotExist:
                # abort if parent protein is not found
                print('Parent protein {} for construct {} not found, aborting!'.format(
                    sd['protein'], sd['name']))
                self.logger.error('Parent protein {} for construct {} not found, aborting!'.format(
                    sd['protein'], sd['name']))
                continue
            # sequence type
            try:
                sequence_type, created = ProteinSequenceType.objects.get_or_create(slug='mod',
                    defaults={'name': 'Modified'})
                if created:
                    self.logger.info('Created sequence type {}'.format(sequence_type))
            except IntegrityError:
                sequence_type = ProteinSequenceType.objects.get(slug='mod')
<commit_msg>FIX: remove dependency on correct case of protein name<commit_after>from django.core.management.base import BaseCommand, CommandError
from django.conf import settings
from django.utils.text import slugify
from django.utils.html import strip_tags
from django.db import IntegrityError

from build.management.commands.base_build import Command as BaseBuild
from protein.models import (Protein, ProteinConformation, ProteinState, ProteinSequenceType, ProteinSegment, ProteinSource)
from residue.models import Residue
from structure.functions import ParseStructureCSV

import os
import logging
import yaml


class Command(BaseBuild):
    help = 'Reads source data and creates protein records for constructs'

    def add_arguments(self, parser):
        parser.add_argument('-p', '--proc',
            type=int,
            action='store',
            dest='proc',
            default=1,
            help='Number of processes to run')
        parser.add_argument('-c', '--construct',
            # action='append',
            dest='construct',
            help='Constructs to import (PDB ID)',
            nargs='+')
        parser.add_argument('-u', '--purge',
            action='store_true',
            dest='purge',
            default=False,
            help='Purge existing construct records')

    def handle(self, *args, **options):
        # delete any existing construct data
        if options['purge']:
            try:
                self.purge_constructs()
            except Exception as msg:
                print(msg)
                self.logger.error(msg)
        self.parsed_structures = ParseStructureCSV()

        # where pdb ids specified?
        if options['construct']:
            self.parsed_structures.pdb_ids = [i for i in self.parsed_structures.pdb_ids if i in options['construct'] or i.lower() in options['construct']]
        try:
            self.logger.info('CREATING CONSTRUCTS')
            self.prepare_input(options['proc'], self.parsed_structures.pdb_ids)
            self.logger.info('COMPLETED CREATING CONSTRUCTS')
        except Exception as msg:
            print(msg)
            self.logger.error(msg)

    def purge_constructs(self):
        try:
            pst = ProteinSequenceType.objects.get(slug='mod')
            Protein.objects.filter(sequence_type=pst).delete()
        except ProteinSequenceType.DoesNotExist:
            self.logger.warning('ProteinSequenceType mod not found: nothing to delete.')

    def main_func(self, positions, iteration,count,lock):
        # setting up processes
        if not positions[1]:
            constructs = self.parsed_structures.pdb_ids[positions[0]:]
        else:
            constructs = self.parsed_structures.pdb_ids[positions[0]:positions[1]]

        for sd in constructs:
            sd = self.parsed_structures.structures[sd]
            # is a protein specified?
            if 'protein' not in sd:
                self.logger.error('Protein not specified for construct, skipping')
                continue

            # fetch the parent protein
            try:
                ppc = ProteinConformation.objects.prefetch_related('protein__family', 'protein__species',
                    'protein__residue_numbering_scheme').get(protein__entry_name=sd['protein'].lower(),
                    state__slug=settings.DEFAULT_PROTEIN_STATE)
            except ProteinConformation.DoesNotExist:
                # abort if parent protein is not found
                print('Parent protein {} for construct {} not found, aborting!'.format(
                    sd['protein'], sd['name']))
                self.logger.error('Parent protein {} for construct {} not found, aborting!'.format(
                    sd['protein'], sd['name']))
                continue
            # sequence type
            try:
                sequence_type, created = ProteinSequenceType.objects.get_or_create(slug='mod',
                    defaults={'name': 'Modified'})
                if created:
                    self.logger.info('Created sequence type {}'.format(sequence_type))
            except IntegrityError:
                sequence_type = ProteinSequenceType.objects.get(slug='mod')

{'fraction_non_alphanumeric': 0.09610983981693363, 'fraction_numerical': 0.0012043839576056847, 'mean_word_length': 2.269291338582677, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '975072', 'n_tokens_mistral': 2178, 'n_tokens_neox': 2055, 'n_words': 524}