<commit_before>import os
import zipfile
from pathlib import Path

import pkg_resources
import yaml

from Datasnakes.Manager.management import ProjectManagement
from Datasnakes.Manager import config
from Datasnakes.Manager.database_management import BaseDatabaseManagement
from Datasnakes.Orthologs.Align import MultipleSequenceAlignment as MSA
from Datasnakes.Orthologs.Blast.standard_blastn import StandardBlastN
from Datasnakes.Orthologs.Blast.comparative_genetics_objects import CompGenObjects
from Datasnakes.Orthologs.GenBank.genbank import GenBank


#import configparser
#from slacker import Slacker
#import argparse
#import textwrap
# TODO-ROB: Use this class to move data back and forth between local and remote servers
# TODO-ROB:  Mirror the directory creation on MCSR's servers
# TODO-ROB:  ^^ This will allow the transfer of data

# TODO-ROB:  Add FTP and s2s inheritance


class DataMana(object):

    def __init__(self, config_file=None, pipeline=None, new=False, start=False, **kwargs):
        """Initialize the attributes that can be used as keys in the config_file."""
        # Full configuration for the pipeline's YAML based variables
        self.Management_config = self.Database_config = self.CompGenAnalysis_config = self.BLASTn_config = \
            self.GenBank_config = self.Alignment_config = None
        # Alignment configuration
        self.Guidance_config = self.Clustalo_config = self.Pal2Nal_config = None
        self.pm = self.bl = self.gb = self.al = self.db = None
        if pipeline == 'Ortho_CDS_1':
            if new is True:
                config_file = pkg_resources.resource_filename(config.yaml.__name__, 'config_template_new.yml')
            else:
                config_file = pkg_resources.resource_filename(config.yaml.__name__, 'config_template_existing.yml')
        if config_file is not None:
            if start is True:
                self.configure(config_file)

    def configure(self, config_file):
        '''
        This method uses YAML configuration in order to initialize different classes.
        :param config_file: A YAML file that is used to create a dictionary(kwargs) for each class.
        :return:
        '''
        with open(config_file, 'r') as ymlfile:
            configuration = yaml.safe_load(ymlfile)
            # TODO-ROB:  Set up configuratioin to parse this full list.  For each sub configuration do something.
            # TODO-ROB:  Before doing the above set up Airflow.
            setattr(self, "CONFIGURATION", configuration)
            for key, value in configuration.items():
                setattr(self, key, value)
                print('key:' + str(key) + '\nvalue: ' + str(value))

                # Project Management
            if self.Management_config is not None:
                self.pm = ProjectManagement(**self.Management_config)
                print('mana_config')
                print(self.pm)
            else:
                self.pm = self.Management_config

            if self.Database_config is not None:
                self.database(self.pm, self.Database_config)
            else:
                self.db = self.Database_config

                # CompGenAnalysis and BLASTn Configuration
            if self.BLASTn_config is not None and self.CompGenAnalysis_config is not None:
                self.BLASTn_config.update(self.CompGenAnalysis_config)
            if self.BLASTn_config is not None:
                # Blast has not taken place so it will happen here
                self.blast(self.pm, self.BLASTn_config)
            elif self.CompGenAnalysis_config is not None:
                self.blast(self.pm, self.CompGenAnalysis_config)
            else:
                # Blast has taken place so
                self.bl = self.BLASTn_config

                # GenBank
            if self.GenBank_config is not None:
                self.genbank(self.pm, self.bl)
            else:
                self.gb = self.GenBank_config

                # Alignment
            if self.Alignment_config is not None:
                self.align(self.gb)
            else:
                self.al = self.Alignment_config

    def database(self, proj_mana, database_config):
        self.db = BaseDatabaseManagement(proj_mana=proj_mana, **database_config)
        for config_type, database_config_list in self.db.database_dict.items():
            implementation = database_config_list[0]
            configuration = database_config_list[1]
            if isinstance(implementation, dict):
                for sub_config_type, sub_implementation in implementation.items():
                    if isinstance(configuration, dict):
                        sub_implementation(**configuration[sub_config_type])
                    else:
                        sub_implementation(**configuration)
            else:
                implementation(**configuration)


            # TODO-ROB parse the config options
        pass

    def blast(self, proj_mana, blast_config):
        self.bl = StandardBlastN(proj_mana=proj_mana, **self.Management_config, **blast_config)
        self.bl.blast_config(self.bl.blast_human, 'Homo_sapiens', auto_start=True)
        # TODO-Create directories for the blast data
        # Do the blasting here using CompGenBLASTn

    def genbank(self, proj_mana, blast):
        if blast is not None:
            self.gb = GenBank(blast=blast, **self.Management_config, **self.GenBank_config)
        else:
            self.gb = GenBank(blast=blast, **self.Management_config, **self.GenBank_config)
        if blast is not None:
            if issubclass(type(blast), StandardBlastN):
                self.gb.create_post_blast_gbk_records(blast.org_list, blast.gene_dict)
        else:
            print(proj_mana.__dict__)
            cga = CompGenObjects(proj_mana=proj_mana, **self.CompGenAnalysis_config)

            # Parse the tier_frame_dict to get the tier
            for G_KEY in cga.tier_frame_dict.keys():
                tier = G_KEY
                # Parse the tier based transformed dataframe to get the gene
                for GENE in cga.tier_frame_dict[tier].T:
                    # Parse the organism list to get the desired accession number
                    for ORGANISM in cga.org_list:
                        accession = str(cga.gene_dict[GENE][ORGANISM])
                        parts = list(accession.partition('.'))
                        accession = parts[0]
                        accession = accession.upper()
                        server_flag = False
                        self.gb.get_gbk_file(accession, GENE, ORGANISM, server_flag=server_flag)

    def align(self, genbank):
        self.al = MSA(genbank=genbank, **self.Management_config, **self.Alignment_config)
        for _program, aligner_config_list in self.al.alignment_dict.items():
            aligner = aligner_config_list[0]
            configuration = aligner_config_list[1]
            aligner(**configuration)


class ZipUtils(DataMana):
    """The ZipUtil class allows easy compression/zipping of file folders.
    Inspired by http://stackoverflow.com/a/670635/7351746
    """

    def __init__(self, zip_filename, zip_path):
        """Initialize the input files and path.

        :param comp_filename (string):  This is the name of the compressed file that will be generated (eg 'test.zip')
        :param zip_path: This is the absolute path of the directory (or file) to be zipped.
        :returns:  A zip file that is created inside of the zip_path.  The path string is returned.
        """
        self.zip_filename = zip_filename
        self.zip_path = zip_path
        self.ignore_parts = Path(zip_path).parent.parts

    def to_zip(self):
        """Zip a folder."""
        comp_path = os.path.join(self.zip_path, self.zip_filename)
        zip_handle = zipfile.ZipFile(comp_path, 'w', zipfile.ZIP_DEFLATED)
        if os.path.isfile(self.zip_path):
            zip_handle.write(self.zip_path)
        else:
            print('skipped')
            self.add_folder_to_zip(zip_handle, self.zip_path)
        zip_handle.close()
        return comp_path

    def add_folder_to_zip(self, zip_handle, folder):
        """Not meant to be used explicitly.  Use compress.

        :param zip_handle: An initialized zipfile.ZipFile handle.
        :param folder: A path that represents an entire folder to be zipped.
        :return: Recursively zips nested directories.
        """
        for file in os.listdir(folder):
            full_path = os.path.join(folder, file)
            rel_path = Path(full_path)
            rel_path = rel_path.relative_to(Path(self.zip_path))
            if os.path.isfile(full_path):
                if str(file) == str(self.comp_filename):
                    continue
                print('File added: ' + str(full_path))
                zip_handle.write(full_path, rel_path)
            elif os.path.isdir(full_path):
                if str(file) in self.ignore_parts:
                    continue
                print('Entering folder: ' + str(full_path))
                self.add_folder_to_zip(zip_handle, full_path)
<commit_msg>Refactored data_management.py OrthoEvol imports and updated Blast imports.<commit_after>import os
import zipfile
from pathlib import Path

import pkg_resources
import yaml

from OrthoEvol.Manager.management import ProjectManagement
from OrthoEvol.Manager import config
from OrthoEvol.Manager.database_management import BaseDatabaseManagement
from OrthoEvol.Orthologs.Align import MultipleSequenceAlignment as MSA
from OrthoEvol.Orthologs.Blast.orthologs_blastn import OrthoBlastN
from OrthoEvol.Orthologs.Blast.comparative_genetics import BaseComparativeGenetics
from OrthoEvol.Orthologs.GenBank.genbank import GenBank


#import configparser
#from slacker import Slacker
#import argparse
#import textwrap
# TODO-ROB: Use this class to move data back and forth between local and remote servers
# TODO-ROB:  Mirror the directory creation on MCSR's servers
# TODO-ROB:  ^^ This will allow the transfer of data

# TODO-ROB:  Add FTP and s2s inheritance


class DataMana(object):

    def __init__(self, config_file=None, pipeline=None, new=False, start=False, **kwargs):
        """Initialize the attributes that can be used as keys in the config_file."""
        # Full configuration for the pipeline's YAML based variables
        self.Management_config = self.Database_config = self.CompGenAnalysis_config = self.BLASTn_config = \
            self.GenBank_config = self.Alignment_config = None
        # Alignment configuration
        self.Guidance_config = self.Clustalo_config = self.Pal2Nal_config = None
        self.pm = self.bl = self.gb = self.al = self.db = None
        if pipeline == 'Ortho_CDS_1':
            if new is True:
                config_file = pkg_resources.resource_filename(config.yaml.__name__, 'config_template_new.yml')
            else:
                config_file = pkg_resources.resource_filename(config.yaml.__name__, 'config_template_existing.yml')
        if config_file is not None:
            if start is True:
                self.configure(config_file)

    def configure(self, config_file):
        '''
        This method uses YAML configuration in order to initialize different classes.
        :param config_file: A YAML file that is used to create a dictionary(kwargs) for each class.
        :return:
        '''
        with open(config_file, 'r') as ymlfile:
            configuration = yaml.safe_load(ymlfile)
            # TODO-ROB:  Set up configuratioin to parse this full list.  For each sub configuration do something.
            # TODO-ROB:  Before doing the above set up Airflow.
            setattr(self, "CONFIGURATION", configuration)
            for key, value in configuration.items():
                setattr(self, key, value)
                print('key:' + str(key) + '\nvalue: ' + str(value))

                # Project Management
            if self.Management_config is not None:
                self.pm = ProjectManagement(**self.Management_config)
                print('mana_config')
                print(self.pm)
            else:
                self.pm = self.Management_config

            if self.Database_config is not None:
                self.database(self.pm, self.Database_config)
            else:
                self.db = self.Database_config

                # CompGenAnalysis and BLASTn Configuration
            if self.BLASTn_config is not None and self.CompGenAnalysis_config is not None:
                self.BLASTn_config.update(self.CompGenAnalysis_config)
            if self.BLASTn_config is not None:
                # Blast has not taken place so it will happen here
                self.blast(self.pm, self.BLASTn_config)
            elif self.CompGenAnalysis_config is not None:
                self.blast(self.pm, self.CompGenAnalysis_config)
            else:
                # Blast has taken place so
                self.bl = self.BLASTn_config

                # GenBank
            if self.GenBank_config is not None:
                self.genbank(self.pm, self.bl)
            else:
                self.gb = self.GenBank_config

                # Alignment
            if self.Alignment_config is not None:
                self.align(self.gb)
            else:
                self.al = self.Alignment_config

    def database(self, proj_mana, database_config):
        self.db = BaseDatabaseManagement(proj_mana=proj_mana, **database_config)
        for config_type, database_config_list in self.db.database_dict.items():
            implementation = database_config_list[0]
            configuration = database_config_list[1]
            if isinstance(implementation, dict):
                for sub_config_type, sub_implementation in implementation.items():
                    if isinstance(configuration, dict):
                        sub_implementation(**configuration[sub_config_type])
                    else:
                        sub_implementation(**configuration)
            else:
                implementation(**configuration)


            # TODO-ROB parse the config options
        pass

    def blast(self, proj_mana, blast_config):
        self.bl = OrthoBlastN(proj_mana=proj_mana, **self.Management_config, **blast_config)
        self.bl.blast_config(self.bl.blast_human, 'Homo_sapiens', auto_start=True)
        # TODO-Create directories for the blast data
        # Do the blasting here using CompGenBLASTn

    def genbank(self, proj_mana, blast):
        if blast is not None:
            self.gb = GenBank(blast=blast, **self.Management_config, **self.GenBank_config)
        else:
            self.gb = GenBank(blast=blast, **self.Management_config, **self.GenBank_config)
        if blast is not None:
            if issubclass(type(blast), OrthoBlastN):
                self.gb.create_post_blast_gbk_records(blast.org_list, blast.gene_dict)
        else:
            print(proj_mana.__dict__)
            cga = BaseComparativeGenetics(proj_mana=proj_mana, **self.CompGenAnalysis_config)

            # Parse the tier_frame_dict to get the tier
            for G_KEY in cga.tier_frame_dict.keys():
                tier = G_KEY
                # Parse the tier based transformed dataframe to get the gene
                for GENE in cga.tier_frame_dict[tier].T:
                    # Parse the organism list to get the desired accession number
                    for ORGANISM in cga.org_list:
                        accession = str(cga.gene_dict[GENE][ORGANISM])
                        parts = list(accession.partition('.'))
                        accession = parts[0]
                        accession = accession.upper()
                        server_flag = False
                        self.gb.get_gbk_file(accession, GENE, ORGANISM, server_flag=server_flag)

    def align(self, genbank):
        self.al = MSA(genbank=genbank, **self.Management_config, **self.Alignment_config)
        for _program, aligner_config_list in self.al.alignment_dict.items():
            aligner = aligner_config_list[0]
            configuration = aligner_config_list[1]
            aligner(**configuration)


class ZipUtils(DataMana):
    """The ZipUtil class allows easy compression/zipping of file folders.
    Inspired by http://stackoverflow.com/a/670635/7351746
    """

    def __init__(self, zip_filename, zip_path):
        """Initialize the input files and path.

        :param comp_filename (string):  This is the name of the compressed file that will be generated (eg 'test.zip')
        :param zip_path: This is the absolute path of the directory (or file) to be zipped.
        :returns:  A zip file that is created inside of the zip_path.  The path string is returned.
        """
        self.zip_filename = zip_filename
        self.zip_path = zip_path
        self.ignore_parts = Path(zip_path).parent.parts

    def to_zip(self):
        """Zip a folder."""
        comp_path = os.path.join(self.zip_path, self.zip_filename)
        zip_handle = zipfile.ZipFile(comp_path, 'w', zipfile.ZIP_DEFLATED)
        if os.path.isfile(self.zip_path):
            zip_handle.write(self.zip_path)
        else:
            print('skipped')
            self.add_folder_to_zip(zip_handle, self.zip_path)
        zip_handle.close()
        return comp_path

    def add_folder_to_zip(self, zip_handle, folder):
        """Not meant to be used explicitly.  Use compress.

        :param zip_handle: An initialized zipfile.ZipFile handle.
        :param folder: A path that represents an entire folder to be zipped.
        :return: Recursively zips nested directories.
        """
        for file in os.listdir(folder):
            full_path = os.path.join(folder, file)
            rel_path = Path(full_path)
            rel_path = rel_path.relative_to(Path(self.zip_path))
            if os.path.isfile(full_path):
                if str(file) == str(self.comp_filename):
                    continue
                print('File added: ' + str(full_path))
                zip_handle.write(full_path, rel_path)
            elif os.path.isdir(full_path):
                if str(file) in self.ignore_parts:
                    continue
                print('Entering folder: ' + str(full_path))
                self.add_folder_to_zip(zip_handle, full_path)

{'fraction_non_alphanumeric': 0.07747708424268879, 'fraction_numerical': 0.002291575731121781, 'mean_word_length': 2.22580077437522, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '231067', 'n_tokens_mistral': 5253, 'n_tokens_neox': 4958, 'n_words': 1333}