# Path: lib/log.py
# class Logger:
#     """Common logging functions."""
# 
#     def __init__(self, log_file, log_level):
#         self.log_file = log_file
#         self.log_level = LEVEL[log_level]
#         self.file_handle = open(log_file, 'a') if log_file else None
# 
#     def __del__(self):
#         if self.file_handle:
#             self.file_handle.close()
# 
#     def header(self):
#         """Log header information."""
#         self.info('#' * 80)
#         self.info('aTRAM version: {}'.format(db.ATRAM_VERSION))
#         self.info('Python version: {}'.format(' '.join(sys.version.split())))
#         self.info(' '.join(sys.argv[:]))
# 
#     def subcommand(self, cmd, temp_dir, timeout=None):
#         """
#         Call a subprocess and log the output.
# 
#         Note: stdout=PIPE is blocking and large logs cause a hang.
#         So we don't use it.
#         """
#         self.debug(cmd)
# 
#         error = None
# 
#         with tempfile.NamedTemporaryFile(mode='w', dir=temp_dir) as log_output:
#             with subprocess.Popen(
#                     cmd, shell=True, stdout=log_output, stderr=log_output) as proc:
#                 try:
#                     proc.communicate(timeout=timeout)
# 
#                 # Catch any error and kill all child processes
#                 except Exception as err:  # pylint: disable=broad-except
#                     error = err
#                     self.error('Exception: {}'.format(err))
# 
#                 # On success or failure log what we can
#                 finally:
#                     wait = 5
# 
#                     killed, alive = util.kill_proc_tree(proc.pid, timeout=wait)
# 
#                     if killed:
#                         self.error('Killed {} processes with SIGTERM.'.format(killed))
# 
#                     if alive > 0:
#                         self.error('{} alive processes after SIGTERM'.format(alive))
# 
#                         killed, alive = util.kill_proc_tree(
#                             proc.pid, timeout=wait, sig=signal.SIGKILL)
#                         self.error('Killed {} processes with SIGKILL.'.format(killed))
#                         self.error('{} alive processes after SIGKILL'.format(alive))
# 
#                     with open(log_output.name) as log_input:
#                         for line in log_input:
#                             line = line.strip()
#                             if line:
#                                 self.debug(line)
#                     if error:
#                         raise error
# 
#     def _output(self, msg, level):
#         timestamp = datetime.now().strftime('%Y-%m-%d %H:%M:%S.%f')
#         msg = '({:8d}) {} {:<5}: {}'.format(os.getpid(), timestamp, level, msg)
#         print(msg)
#         if self.file_handle:
#             self.file_handle.write(msg)
#             self.file_handle.write('\n')
#             self.file_handle.flush()
# 
#     def debug(self, msg):
#         """Log an info message."""
#         if self.log_level <= DEBUG:
#             self._output(msg, 'DEBUG')
# 
#     def info(self, msg):
#         """Log an info message."""
#         if self.log_level <= INFO:
#             self._output(msg, 'INFO')
# 
#     def error(self, msg):
#         """Log an error message."""
#         if self.log_level <= ERROR:
#             self._output(msg, 'ERROR')
# 
#     def fatal(self, msg):
#         """Log an error message and exit."""
#         if self.log_level <= FATAL:
#             self._output(msg, 'FATAL')
#         sys.exit(1)


import multiprocessing
import sys
import numpy as np
from os.path import basename, join, splitext
from Bio.SeqIO.FastaIO import SimpleFastaParser
from Bio.SeqIO.QualityIO import FastqGeneralIterator
from . import blast, db, db_preprocessor, util
from .log import Logger
            fasta_path = fill_shuffled_fasta(args, cxn, shard_count, shard_idx)
            results.append(pool.apply_async(
                create_one_shuffled_shard,
                (args, fasta_path, shard_idx)))

        all_results = [result.get() for result in results]
    db_preprocessor.aux_detach(cxn)
    log.info('Finished making all {} blast DBs'.format(len(all_results)))


def fill_shuffled_fasta(args, cxn, shard_count, shard_index):
    """Fill the shard input files with sequences."""
    exe_name, _ = splitext(basename(sys.argv[0]))
    fasta_name = '{}_{:03d}.fasta'.format(exe_name, shard_index + 1)
    fasta_path = join(args['temp_dir'], fasta_name)

    with open(fasta_path, 'w') as fasta_file:
        for row in db_preprocessor.get_shuffled_sequences_in_shard(
                cxn, shard_count, shard_index):
            util.write_fasta_record(fasta_file, row[0], row[2], row[1])

    return fasta_path


def create_one_blast_shard(args, shard_params, shard_index):
    """Create a blast DB from the shard.

    We fill a fasta file with the appropriate sequences and hand things off
    to the makeblastdb program.
    """

# Path: lib/log.py
# class Logger:
#     """Common logging functions."""
# 
#     def __init__(self, log_file, log_level):
#         self.log_file = log_file
#         self.log_level = LEVEL[log_level]
#         self.file_handle = open(log_file, 'a') if log_file else None
# 
#     def __del__(self):
#         if self.file_handle:
#             self.file_handle.close()
# 
#     def header(self):
#         """Log header information."""
#         self.info('#' * 80)
#         self.info('aTRAM version: {}'.format(db.ATRAM_VERSION))
#         self.info('Python version: {}'.format(' '.join(sys.version.split())))
#         self.info(' '.join(sys.argv[:]))
# 
#     def subcommand(self, cmd, temp_dir, timeout=None):
#         """
#         Call a subprocess and log the output.
# 
#         Note: stdout=PIPE is blocking and large logs cause a hang.
#         So we don't use it.
#         """
#         self.debug(cmd)
# 
#         error = None
# 
#         with tempfile.NamedTemporaryFile(mode='w', dir=temp_dir) as log_output:
#             with subprocess.Popen(
#                     cmd, shell=True, stdout=log_output, stderr=log_output) as proc:
#                 try:
#                     proc.communicate(timeout=timeout)
# 
#                 # Catch any error and kill all child processes
#                 except Exception as err:  # pylint: disable=broad-except
#                     error = err
#                     self.error('Exception: {}'.format(err))
# 
#                 # On success or failure log what we can
#                 finally:
#                     wait = 5
# 
#                     killed, alive = util.kill_proc_tree(proc.pid, timeout=wait)
# 
#                     if killed:
#                         self.error('Killed {} processes with SIGTERM.'.format(killed))
# 
#                     if alive > 0:
#                         self.error('{} alive processes after SIGTERM'.format(alive))
# 
#                         killed, alive = util.kill_proc_tree(
#                             proc.pid, timeout=wait, sig=signal.SIGKILL)
#                         self.error('Killed {} processes with SIGKILL.'.format(killed))
#                         self.error('{} alive processes after SIGKILL'.format(alive))
# 
#                     with open(log_output.name) as log_input:
#                         for line in log_input:
#                             line = line.strip()
#                             if line:
#                                 self.debug(line)
#                     if error:
#                         raise error
# 
#     def _output(self, msg, level):
#         timestamp = datetime.now().strftime('%Y-%m-%d %H:%M:%S.%f')
#         msg = '({:8d}) {} {:<5}: {}'.format(os.getpid(), timestamp, level, msg)
#         print(msg)
#         if self.file_handle:
#             self.file_handle.write(msg)
#             self.file_handle.write('\n')
#             self.file_handle.flush()
# 
#     def debug(self, msg):
#         """Log an info message."""
#         if self.log_level <= DEBUG:
#             self._output(msg, 'DEBUG')
# 
#     def info(self, msg):
#         """Log an info message."""
#         if self.log_level <= INFO:
#             self._output(msg, 'INFO')
# 
#     def error(self, msg):
#         """Log an error message."""
#         if self.log_level <= ERROR:
#             self._output(msg, 'ERROR')
# 
#     def fatal(self, msg):
#         """Log an error message and exit."""
#         if self.log_level <= FATAL:
#             self._output(msg, 'FATAL')
#         sys.exit(1)


# Path: lib/core_preprocessor.py
import multiprocessing
import sys
import numpy as np
from os.path import basename, join, splitext
from Bio.SeqIO.FastaIO import SimpleFastaParser
from Bio.SeqIO.QualityIO import FastqGeneralIterator
from . import blast, db, db_preprocessor, util
from .log import Logger

            fasta_path = fill_shuffled_fasta(args, cxn, shard_count, shard_idx)
            results.append(pool.apply_async(
                create_one_shuffled_shard,
                (args, fasta_path, shard_idx)))

        all_results = [result.get() for result in results]
    db_preprocessor.aux_detach(cxn)
    log.info('Finished making all {} blast DBs'.format(len(all_results)))


def fill_shuffled_fasta(args, cxn, shard_count, shard_index):
    """Fill the shard input files with sequences."""
    exe_name, _ = splitext(basename(sys.argv[0]))
    fasta_name = '{}_{:03d}.fasta'.format(exe_name, shard_index + 1)
    fasta_path = join(args['temp_dir'], fasta_name)

    with open(fasta_path, 'w') as fasta_file:
        for row in db_preprocessor.get_shuffled_sequences_in_shard(
                cxn, shard_count, shard_index):
            util.write_fasta_record(fasta_file, row[0], row[2], row[1])

    return fasta_path


def create_one_blast_shard(args, shard_params, shard_index):
    """Create a blast DB from the shard.

    We fill a fasta file with the appropriate sequences and hand things off
    to the makeblastdb program.
    """

    log = Logger(args['log_file'], args['log_level'])