<commit_before># -*- coding: utf-8 -*-
"""
Created on Sun Dec 23 18:37:36 2012

@author: niklas
"""
import os
import logging
import subprocess
import tempfile
import argparse
import atexit
import signal

from modeller import ModellerError

import errors
import sql_db

import domain_template
import domain_model
import domain_mutation

from Bio.SubsMat import MatrixInfo
from ConfigParser import SafeConfigParser
import helper_functions as hf

blacklisted_uniprots = [
    'Q8WZ42', # Titin has a large number of domains that are joined in messed up ways
]


class ProteinDomainData(object):

    def __init__(self, domain, template, model, mutations=[]):
        self.d = domain
        self.t = template
        self.m = model
        self.mut = mutations


class Pipeline(object):

    def __init__(self, configFile):
        #######################################################################
        # Read the configuration file and set the variables
        configParser = SafeConfigParser(
            defaults={
                'global_temp_path': '/tmp/',
                'temp_path': '/tmp/elaspic/',
                'DEBUG': True,
                'path_to_archive': '/home/kimlab1/database_data/elaspic/',
                'look_for_interactions': 'True',
                'n_cores': 1,
                'db_type': sql_db.sql_flavor,
                'db_path': hf.path_to_pipeline_code() + '/../db/pipeline.db',
                'web_server': False,
            })
        configParser.read(configFile)

        # From [DEFAULT]
        self.global_temp_path = configParser.get('DEFAULT', 'global_temp_path')
        temp_path = configParser.get('DEFAULT', 'temp_path').strip('/') + '/' # So it never starts with '/';
        self.debug = configParser.getboolean('DEFAULT', 'DEBUG')
        self.path_to_archive = configParser.get('DEFAULT', 'path_to_archive')
        self.look_for_interactions = configParser.getboolean('DEFAULT', 'look_for_interactions')
        self.n_cores = configParser.get('DEFAULT', 'n_cores')
        self.db_type = configParser.get('DEFAULT', 'db_type')
        self.db_path = configParser.get('DEFAULT', 'db_path')
        self.web_server = configParser.get('DEFAULT', 'web_server')

        # From [SETTINGS]
        self.blast_db_path = configParser.get('SETTINGS', 'blast_db_path')
        self.pdb_path = configParser.get('SETTINGS', 'pdb_path')
        self.output_path = configParser.get('SETTINGS', 'output_path')
        self.bin_path = configParser.get('SETTINGS', 'bin_path')
        self.max_runtime = configParser.get('SETTINGS', 'max_runtime')

        # From [GET_TEMPLATE]

        # From [GET_MODEL]
        self.modeller_runs = configParser.getint('GET_MODEL', 'modeller_runs')

        # From [GET_MUTATION]
        self.foldx_water = configParser.get('GET_MUTATION', 'foldx_water')
        self.foldx_num_of_runs = configParser.get('GET_MUTATION', 'foldx_num_of_runs')
        self.matrix_type = configParser.get('GET_MUTATION', 'matrix_type')
        self.gap_start = configParser.getint('GET_MUTATION', 'gap_start')
        self.gap_extend = configParser.getint('GET_MUTATION', 'gap_extend')
        self.matrix = getattr(MatrixInfo, self.matrix_type)

        #######################################################################
        # If a TMPDIR is given as environment variable the tmp directory
        # is created relative to that. This is useful when running on banting
        # (the cluster in the ccbr) and also on Scinet (I might have set the
        # environment variable on Scinet myself though..). Make sure that it
        # points to '/dev/shm/' on Scinet
        tmpdir_cluster = hf.get_echo('$TMPDIR')
        if tmpdir_cluster:
            self.temp_path = os.path.join(tmpdir_cluster, temp_path)
        else:
            self.temp_path = os.path.join(self.global_temp_path, temp_path)
        subprocess.check_call('mkdir -p ' + self.temp_path, shell=True)

        #######################################################################
        # Copy the PDBFam database file
        if self.db_type == 'sqlite_file' and not os.path.isfile(self.temp_path + '/' + self.db_path.strip().split('/')[-1]):
            print self.temp_path
            print self.db_path
            system_command = 'cp -u ' + self.db_path + ' ' + self.temp_path + '/'
            childProcess = subprocess.Popen(system_command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
            result, error_message = childProcess.communicate()
            if childProcess.returncode != 0:
                print result
                print error_message
                raise Exception('Could not copy the sqlite3 database!')

        #######################################################################
        # If running on the cluster copy the database to the tmp DIR for
        # speedup and to avoid killing the network. BLAST is very IO intensive
        # and you don't want that to be run over the network!
        # I can distinguish where the pipeline is running by checking the username
        # you will have to adjust that!
        # my usernames are:
        # local: niklas
        # banting: nberliner
        # Scinet: joan
        username  = hf.get_username()
        hostname = hf.get_hostname()
        if username == 'strokach' and (('beagle' in hostname) or ('grendel' in hostname)):
            # Copy the blast database if running on beagle or Grendel
            system_command = (
                'mkdir -p ' + self.global_temp_path + 'blast/pdbaa_db/ && ' +
                'mkdir -p ' + self.global_temp_path + 'blast/db/ && ' +
                'rsync -rzu ' + self.blast_db_path + 'pdbaa_db/ ' + self.global_temp_path + 'blast/pdbaa_db/ && ' +
                'rsync -rzu ' + self.blast_db_path + 'db/ ' + self.global_temp_path + 'blast/db/')
        elif username == 'strokach' or username == 'alexey':
            # Use a symbolic link to the blast database and use the home folder for temporary storage
            system_command = (
                'rm -rf ' + self.global_temp_path + 'blast/ && ' +
                'mkdir -p ' + self.global_temp_path + 'blast/ && ' +
                'ln -sf ' + self.blast_db_path + 'pdbaa_db ' + self.global_temp_path + 'blast/ && ' +
                'ln -sf ' + self.blast_db_path + 'db ' +  self.global_temp_path + 'blast/')
        elif username == 'witvliet':
            system_command = (<commit_msg>Added 'or username == kimadmin' for virtual machine.<commit_after># -*- coding: utf-8 -*-
"""
Created on Sun Dec 23 18:37:36 2012

@author: niklas
"""
import os
import logging
import subprocess
import tempfile
import argparse
import atexit
import signal

from modeller import ModellerError

import errors
import sql_db

import domain_template
import domain_model
import domain_mutation

from Bio.SubsMat import MatrixInfo
from ConfigParser import SafeConfigParser
import helper_functions as hf

blacklisted_uniprots = [
    'Q8WZ42', # Titin has a large number of domains that are joined in messed up ways
]


class ProteinDomainData(object):

    def __init__(self, domain, template, model, mutations=[]):
        self.d = domain
        self.t = template
        self.m = model
        self.mut = mutations


class Pipeline(object):

    def __init__(self, configFile):
        #######################################################################
        # Read the configuration file and set the variables
        configParser = SafeConfigParser(
            defaults={
                'global_temp_path': '/tmp/',
                'temp_path': '/tmp/elaspic/',
                'DEBUG': True,
                'path_to_archive': '/home/kimlab1/database_data/elaspic/',
                'look_for_interactions': 'True',
                'n_cores': 1,
                'db_type': sql_db.sql_flavor,
                'db_path': hf.path_to_pipeline_code() + '/../db/pipeline.db',
                'web_server': False,
            })
        configParser.read(configFile)

        # From [DEFAULT]
        self.global_temp_path = configParser.get('DEFAULT', 'global_temp_path')
        temp_path = configParser.get('DEFAULT', 'temp_path').strip('/') + '/' # So it never starts with '/';
        self.debug = configParser.getboolean('DEFAULT', 'DEBUG')
        self.path_to_archive = configParser.get('DEFAULT', 'path_to_archive')
        self.look_for_interactions = configParser.getboolean('DEFAULT', 'look_for_interactions')
        self.n_cores = configParser.get('DEFAULT', 'n_cores')
        self.db_type = configParser.get('DEFAULT', 'db_type')
        self.db_path = configParser.get('DEFAULT', 'db_path')
        self.web_server = configParser.get('DEFAULT', 'web_server')

        # From [SETTINGS]
        self.blast_db_path = configParser.get('SETTINGS', 'blast_db_path')
        self.pdb_path = configParser.get('SETTINGS', 'pdb_path')
        self.output_path = configParser.get('SETTINGS', 'output_path')
        self.bin_path = configParser.get('SETTINGS', 'bin_path')
        self.max_runtime = configParser.get('SETTINGS', 'max_runtime')

        # From [GET_TEMPLATE]

        # From [GET_MODEL]
        self.modeller_runs = configParser.getint('GET_MODEL', 'modeller_runs')

        # From [GET_MUTATION]
        self.foldx_water = configParser.get('GET_MUTATION', 'foldx_water')
        self.foldx_num_of_runs = configParser.get('GET_MUTATION', 'foldx_num_of_runs')
        self.matrix_type = configParser.get('GET_MUTATION', 'matrix_type')
        self.gap_start = configParser.getint('GET_MUTATION', 'gap_start')
        self.gap_extend = configParser.getint('GET_MUTATION', 'gap_extend')
        self.matrix = getattr(MatrixInfo, self.matrix_type)

        #######################################################################
        # If a TMPDIR is given as environment variable the tmp directory
        # is created relative to that. This is useful when running on banting
        # (the cluster in the ccbr) and also on Scinet (I might have set the
        # environment variable on Scinet myself though..). Make sure that it
        # points to '/dev/shm/' on Scinet
        tmpdir_cluster = hf.get_echo('$TMPDIR')
        if tmpdir_cluster:
            self.temp_path = os.path.join(tmpdir_cluster, temp_path)
        else:
            self.temp_path = os.path.join(self.global_temp_path, temp_path)
        subprocess.check_call('mkdir -p ' + self.temp_path, shell=True)

        #######################################################################
        # Copy the PDBFam database file
        if self.db_type == 'sqlite_file' and not os.path.isfile(self.temp_path + '/' + self.db_path.strip().split('/')[-1]):
            print self.temp_path
            print self.db_path
            system_command = 'cp -u ' + self.db_path + ' ' + self.temp_path + '/'
            childProcess = subprocess.Popen(system_command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
            result, error_message = childProcess.communicate()
            if childProcess.returncode != 0:
                print result
                print error_message
                raise Exception('Could not copy the sqlite3 database!')

        #######################################################################
        # If running on the cluster copy the database to the tmp DIR for
        # speedup and to avoid killing the network. BLAST is very IO intensive
        # and you don't want that to be run over the network!
        # I can distinguish where the pipeline is running by checking the username
        # you will have to adjust that!
        # my usernames are:
        # local: niklas
        # banting: nberliner
        # Scinet: joan
        username  = hf.get_username()
        hostname = hf.get_hostname()
        if username == 'strokach' and (('beagle' in hostname) or ('grendel' in hostname)):
            # Copy the blast database if running on beagle or Grendel
            system_command = (
                'mkdir -p ' + self.global_temp_path + 'blast/pdbaa_db/ && ' +
                'mkdir -p ' + self.global_temp_path + 'blast/db/ && ' +
                'rsync -rzu ' + self.blast_db_path + 'pdbaa_db/ ' + self.global_temp_path + 'blast/pdbaa_db/ && ' +
                'rsync -rzu ' + self.blast_db_path + 'db/ ' + self.global_temp_path + 'blast/db/')
        elif username == 'strokach' or username == 'alexey':
            # Use a symbolic link to the blast database and use the home folder for temporary storage
            system_command = (
                'rm -rf ' + self.global_temp_path + 'blast/ && ' +
                'mkdir -p ' + self.global_temp_path + 'blast/ && ' +
                'ln -sf ' + self.blast_db_path + 'pdbaa_db ' + self.global_temp_path + 'blast/ && ' +
                'ln -sf ' + self.blast_db_path + 'db ' +  self.global_temp_path + 'blast/')
        elif username == 'witvliet' or username == 'kimadmin':
            system_command = (
{'fraction_non_alphanumeric': 0.14835630148587267, 'fraction_numerical': 0.003233505273693125, 'mean_word_length': 2.6013307457721098, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1148836', 'n_tokens_mistral': 3901, 'n_tokens_neox': 3747, 'n_words': 937}