<commit_before>from django.conf import settings
from django.db.models import Q

from protein.models import ProteinAnomaly
from residue.models import Residue, ResidueGenericNumber, ResidueNumberingScheme, ResidueGenericNumberEquivalent

import logging
from collections import OrderedDict
import yaml
import shlex
import os
from Bio import AlignIO
from Bio.Align.Applications import ClustalOmegaCommandline

def parse_scheme_tables(path):
    # get generic residue numbering schemes
    rnss = ResidueNumberingScheme.objects.all()
    
    # dictionary for mapping tables
    schemes = OrderedDict()

    for scheme in rnss:
        schemes[scheme.slug] = {}
        schemes[scheme.slug]['generic_numbers'] = {}
        if scheme.parent:
            schemes[scheme.slug]['parent'] = scheme.parent.slug
        schemes[scheme.slug]['obj'] = scheme
        mapping_file = os.sep.join([path, 'mapping_' + scheme.slug + '.txt'])
        if os.path.isfile(mapping_file):
            with open(mapping_file, "r", encoding='UTF-8') as scheme_table_file:
                schemes[scheme.slug]['table'] = {}
                for row in scheme_table_file:
                    split_row = shlex.split(row)
                    schemes[scheme.slug]['table'][split_row[0]] = split_row[1]
    return schemes

def load_reference_positions(path):
    try:
        with open(path, 'r') as ref_position_file:
            ref_positions = yaml.load(ref_position_file)
        return ref_positions
    except:
        # if the file does not exists, create an empty file
        ref_position_file = open(path, 'w')
        ref_position_file.close()
<commit_msg>Bugfix in residue building<commit_after>from django.conf import settings
from django.db.models import Q

from protein.models import ProteinAnomaly
from residue.models import Residue, ResidueGenericNumber, ResidueNumberingScheme, ResidueGenericNumberEquivalent

import logging
from collections import OrderedDict
import yaml
import shlex
import os
from Bio import AlignIO
from Bio.Align.Applications import ClustalOmegaCommandline

def parse_scheme_tables(path):
    # get generic residue numbering schemes
    rnss = ResidueNumberingScheme.objects.all()
    
    # dictionary for mapping tables
    schemes = OrderedDict()

    for scheme in rnss:
        schemes[scheme.slug] = {}
        schemes[scheme.slug]['generic_numbers'] = {}
        if scheme.parent:
            schemes[scheme.slug]['parent'] = scheme.parent.slug
        schemes[scheme.slug]['obj'] = scheme
        mapping_file = os.sep.join([path, 'mapping_' + scheme.slug + '.txt'])
        if os.path.isfile(mapping_file):
            with open(mapping_file, "r", encoding='UTF-8') as scheme_table_file:
                schemes[scheme.slug]['table'] = {}
                for row in scheme_table_file:
                    split_row = shlex.split(row)
                    schemes[scheme.slug]['table'][split_row[0]] = split_row[1]
    return schemes

def load_reference_positions(path):
    try:
        with open(path, 'r') as ref_position_file:
            ref_positions = yaml.load(ref_position_file)
        return ref_positions
    except:
        return False

{'fraction_non_alphanumeric': 0.09134007585335019, 'fraction_numerical': 0.0018963337547408343, 'mean_word_length': 3.0629011553273426, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5284514', 'n_tokens_mistral': 910, 'n_tokens_neox': 905, 'n_words': 225}