<commit_before>"""
Copyright 2015-2017 Knights Lab, Regents of the University of Minnesota.

This software is released under the GNU Affero General Public License (AGPL) v3.0 License.
"""

import os
from yaml import load
from collections import defaultdict, Counter
import pandas as pd
from cytoolz import valfilter
import csv
import numpy as np

from shogun.wrappers import embalmer_align, utree_search_gg, bowtie2_align
from shogun.utils.last_common_ancestor import build_lca_map
from shogun.parsers import yield_alignments_from_sam_inf
from shogun.redistribute import Taxonomy
from shogun import logger


class Aligner:
    _name = None

    def __init__(self, database_dir, threads=1, shell=False):
        self.threads = threads
        self.shell = shell
        check, msg = self.check_database(database_dir)

        with open(os.path.join(database_dir, 'metadata.yaml')) as stream:
            self.data_files = load(stream)

        if not check:
            raise Exception("Database %s is not formatted correctly: %s" % (database_dir, msg))

        self.database_dir = database_dir

        self.tax = os.path.join(database_dir, self.data_files['general']['taxonomy'])
        self.fasta = os.path.join(database_dir, self.data_files['general']['fasta'])
        self.outfile = None
        logger.debug("Initiate Logger %s" % self._name)

    @classmethod
    def check_database(cls, dir):
        with open(os.path.join(dir, 'metadata.yaml'), 'r') as stream:
            data_files = load(stream)

        SUFFICES = {
            'embalmer': ['.edx'],
            'utree': ['.ctr'],
            'bowtie2': ['*']
        }
        for value in SUFFICES[cls._name]:
            if cls._name == "bowtie2":
                from glob import glob
                files = glob(os.path.abspath(os.path.join(dir, data_files[cls._name] + value)))
                if len(files) == 0:
                    return False, "%s not found" % ("Prefix not found: %s"  % os.path.abspath(os.path.join(dir, data_files[cls._name])))
            elif not os.path.exists(os.path.abspath(os.path.join(dir, data_files[cls._name] + value))):
                return False, '%s not found' % (os.path.abspath(os.path.join(dir, data_files[cls._name] + value)))
        return True, ''

    def align(self, infile, outdir):
        raise NotImplementedError

    def _post_align(self, outf):
        raise NotImplementedError

class EmbalmerAligner(Aligner):
    _name = 'embalmer'

    def __init__(self, database_dir, post_align='capitalist', **kwargs):
        super().__init__(database_dir,**kwargs)

        # Setup the embalmer database
        prefix = self.data_files[self._name]
        self.database = os.path.join(self.database_dir, prefix)

        if os.path.exists(self.database + '.acx'):
            self.accelerator = self.database + '.acx'
        else:
            self.accelerator = False
        self.tree = Taxonomy(self.tax)
        self.post_align = 'capitalist'


    def _post_align(self, outf):
        if self.post_align == 'capitalist':
            return self._post_align_capitalist(outf)
        else:
            return self._post_align_taxonomy(outf)


    def align(self, infile, outdir):<commit_msg>Properly set post_align<commit_after>"""
Copyright 2015-2017 Knights Lab, Regents of the University of Minnesota.

This software is released under the GNU Affero General Public License (AGPL) v3.0 License.
"""

import os
from yaml import load
from collections import defaultdict, Counter
import pandas as pd
from cytoolz import valfilter
import csv
import numpy as np

from shogun.wrappers import embalmer_align, utree_search_gg, bowtie2_align
from shogun.utils.last_common_ancestor import build_lca_map
from shogun.parsers import yield_alignments_from_sam_inf
from shogun.redistribute import Taxonomy
from shogun import logger


class Aligner:
    _name = None

    def __init__(self, database_dir, threads=1, shell=False):
        self.threads = threads
        self.shell = shell
        check, msg = self.check_database(database_dir)

        with open(os.path.join(database_dir, 'metadata.yaml')) as stream:
            self.data_files = load(stream)

        if not check:
            raise Exception("Database %s is not formatted correctly: %s" % (database_dir, msg))

        self.database_dir = database_dir

        self.tax = os.path.join(database_dir, self.data_files['general']['taxonomy'])
        self.fasta = os.path.join(database_dir, self.data_files['general']['fasta'])
        self.outfile = None
        logger.debug("Initiate Logger %s" % self._name)

    @classmethod
    def check_database(cls, dir):
        with open(os.path.join(dir, 'metadata.yaml'), 'r') as stream:
            data_files = load(stream)

        SUFFICES = {
            'embalmer': ['.edx'],
            'utree': ['.ctr'],
            'bowtie2': ['*']
        }
        for value in SUFFICES[cls._name]:
            if cls._name == "bowtie2":
                from glob import glob
                files = glob(os.path.abspath(os.path.join(dir, data_files[cls._name] + value)))
                if len(files) == 0:
                    return False, "%s not found" % ("Prefix not found: %s"  % os.path.abspath(os.path.join(dir, data_files[cls._name])))
            elif not os.path.exists(os.path.abspath(os.path.join(dir, data_files[cls._name] + value))):
                return False, '%s not found' % (os.path.abspath(os.path.join(dir, data_files[cls._name] + value)))
        return True, ''

    def align(self, infile, outdir):
        raise NotImplementedError

    def _post_align(self, outf):
        raise NotImplementedError

class EmbalmerAligner(Aligner):
    _name = 'embalmer'

    def __init__(self, database_dir, post_align='capitalist', **kwargs):
        super().__init__(database_dir,**kwargs)

        # Setup the embalmer database
        prefix = self.data_files[self._name]
        self.database = os.path.join(self.database_dir, prefix)

        if os.path.exists(self.database + '.acx'):
            self.accelerator = self.database + '.acx'
        else:
            self.accelerator = False
        self.tree = Taxonomy(self.tax)
        self.post_align = post_align


    def _post_align(self, outf):
        if self.post_align == 'capitalist':
            return self._post_align_capitalist(outf)
        else:
            return self._post_align_taxonomy(outf)


    def align(self, infile, outdir):
{'fraction_non_alphanumeric': 0.11149068322981366, 'fraction_numerical': 0.004658385093167702, 'mean_word_length': 2.7209705372616986, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6482912', 'n_tokens_mistral': 2045, 'n_tokens_neox': 1959, 'n_words': 485}