# Path: core/kover/utils.py
# def _minimum_uint_size(max_value):
#     """
#     Find the minimum size unsigned integer type that can store values of at most max_value
#     """
#     if max_value <= np.iinfo(np.uint8).max:
#         return np.uint8
#     elif max_value <= np.iinfo(np.uint16).max:
#         return np.uint16
#     elif max_value <= np.iinfo(np.uint32).max:
#         return np.uint32
#     elif max_value <= np.iinfo(np.uint64).max:
#         return np.uint64
#     else:
#         return np.uint128
# 
# def _pack_binary_bytes_to_ints(a, pack_size):
#     """
#     Packs binary values stored in bytes into ints
#     """
#     if pack_size == 64:
#         type = np.uint64
#     elif pack_size == 32:
#         type = np.uint32
#     else:
#         raise ValueError("Supported data types are 32-bit and 64-bit integers.")
# 
#     b = np.zeros((int(ceil(1.0 * a.shape[0] / pack_size)), a.shape[1]), dtype=type)
#     packed_rows = 0
#     packing_row = 0
#     for i in xrange(a.shape[0]):
#         if packed_rows == pack_size:
#             packed_rows = 0
#             packing_row += 1
#         tmp = np.asarray(a[i], dtype=type)
#         tmp = np.left_shift(tmp, type(pack_size - packed_rows - 1))
#         np.bitwise_or(b[packing_row], tmp, out=b[packing_row])
#         packed_rows += 1
# 
#     return b
# 
# Path: core/kover/dataset/tools/kmer_count.py
# def contigs_count_kmers(file_path, out_dir, kmer_size, out_compress, nb_cores, verbose, progress):
# 
# 	dir_path = dirname(abspath(__file__))
# 
# 	# Calling MultiDSK tool
# 	call([str(join(dir_path, "kmer_tools", "multidsk")),
# 					"-file", str(file_path),
# 					"-out-dir", str(out_dir),
# 					"-kmer-size", str(kmer_size),
# 					"-abundance-min", "1",
# 					"-out-compress", str(out_compress),
# 					"-nb-cores", str(nb_cores),
# 					"-out-tmp", str(out_dir),
# 					"-verbose", str(verbose),
# 					"-progress", str(progress)])
# 
# def reads_count_kmers(file_path, out_dir, kmer_size, abundance_min, out_compress, nb_cores, verbose, progress):
# 	dir_path = dirname(abspath(__file__))
# 
# 	# Calling MultiDSK tool
# 	call([str(join(dir_path, "kmer_tools", "multidsk")),
# 					"-file", str(file_path),
# 					"-out-dir", str(out_dir),
# 					"-kmer-size", str(kmer_size),
# 					"-abundance-min", str(abundance_min),
# 					"-out-compress", str(out_compress),
# 					"-nb-cores", str(nb_cores),
# 					"-out-tmp", str(out_dir),
# 					"-verbose", str(verbose),
# 					"-progress", str(progress)])
# 
# Path: core/kover/dataset/tools/kmer_pack.py
# def contigs_pack_kmers(file_path, out_path, filter_singleton, kmer_length, compression, chunk_size, nb_genomes, progress):


import gc
import h5py as h
import logging
import numpy as np
import pandas as pd
from math import ceil
from os import getpid, mkdir, listdir
from os.path import basename, exists, getsize, join, splitext
from shutil import rmtree
from time import time
from uuid import uuid1
from ..utils import _minimum_uint_size, _pack_binary_bytes_to_ints
from .tools.kmer_count import contigs_count_kmers, reads_count_kmers
from .tools.kmer_pack import contigs_pack_kmers, reads_pack_kmers
        # XXX: This is can break if the file format changes!
        #      Assumes that each line has the format kmer_seq\tV\tV\t...V\n with one binary V per genome
        total_size = getsize(tsv)
        with open(tsv, "r") as f:
            header_size = len(f.next())
            line_size = len(f.next())
        content_size = float(total_size) - header_size
        if content_size % line_size != 0:
            raise Exception()
        return int(content_size / line_size)

    # Execution callback functions
    if warning_callback is None:
        warning_callback = lambda w: logging.warning(w)
    if error_callback is None:
        def normal_raise(exception):
            raise exception

        error_callback = normal_raise
    if progress_callback is None:
        progress_callback = lambda t, p: None

    if (phenotype_description is None and phenotype_metadata_path is not None) or (
                    phenotype_description is not None and phenotype_metadata_path is None):
        raise ValueError("If a phenotype is specified, it must have a description and a metadata file.")

    kmer_len = get_kmer_length(tsv_path)
    kmer_count = get_kmer_count(tsv_path)

    kmer_dtype = 'S' + str(kmer_len)

# Path: core/kover/utils.py
# def _minimum_uint_size(max_value):
#     """
#     Find the minimum size unsigned integer type that can store values of at most max_value
#     """
#     if max_value <= np.iinfo(np.uint8).max:
#         return np.uint8
#     elif max_value <= np.iinfo(np.uint16).max:
#         return np.uint16
#     elif max_value <= np.iinfo(np.uint32).max:
#         return np.uint32
#     elif max_value <= np.iinfo(np.uint64).max:
#         return np.uint64
#     else:
#         return np.uint128
# 
# def _pack_binary_bytes_to_ints(a, pack_size):
#     """
#     Packs binary values stored in bytes into ints
#     """
#     if pack_size == 64:
#         type = np.uint64
#     elif pack_size == 32:
#         type = np.uint32
#     else:
#         raise ValueError("Supported data types are 32-bit and 64-bit integers.")
# 
#     b = np.zeros((int(ceil(1.0 * a.shape[0] / pack_size)), a.shape[1]), dtype=type)
#     packed_rows = 0
#     packing_row = 0
#     for i in xrange(a.shape[0]):
#         if packed_rows == pack_size:
#             packed_rows = 0
#             packing_row += 1
#         tmp = np.asarray(a[i], dtype=type)
#         tmp = np.left_shift(tmp, type(pack_size - packed_rows - 1))
#         np.bitwise_or(b[packing_row], tmp, out=b[packing_row])
#         packed_rows += 1
# 
#     return b
# 
# Path: core/kover/dataset/tools/kmer_count.py
# def contigs_count_kmers(file_path, out_dir, kmer_size, out_compress, nb_cores, verbose, progress):
# 
# 	dir_path = dirname(abspath(__file__))
# 
# 	# Calling MultiDSK tool
# 	call([str(join(dir_path, "kmer_tools", "multidsk")),
# 					"-file", str(file_path),
# 					"-out-dir", str(out_dir),
# 					"-kmer-size", str(kmer_size),
# 					"-abundance-min", "1",
# 					"-out-compress", str(out_compress),
# 					"-nb-cores", str(nb_cores),
# 					"-out-tmp", str(out_dir),
# 					"-verbose", str(verbose),
# 					"-progress", str(progress)])
# 
# def reads_count_kmers(file_path, out_dir, kmer_size, abundance_min, out_compress, nb_cores, verbose, progress):
# 	dir_path = dirname(abspath(__file__))
# 
# 	# Calling MultiDSK tool
# 	call([str(join(dir_path, "kmer_tools", "multidsk")),
# 					"-file", str(file_path),
# 					"-out-dir", str(out_dir),
# 					"-kmer-size", str(kmer_size),
# 					"-abundance-min", str(abundance_min),
# 					"-out-compress", str(out_compress),
# 					"-nb-cores", str(nb_cores),
# 					"-out-tmp", str(out_dir),
# 					"-verbose", str(verbose),
# 					"-progress", str(progress)])
# 
# Path: core/kover/dataset/tools/kmer_pack.py
# def contigs_pack_kmers(file_path, out_path, filter_singleton, kmer_length, compression, chunk_size, nb_genomes, progress):


# Path: core/kover/dataset/create.py
import gc
import h5py as h
import logging
import numpy as np
import pandas as pd
from math import ceil
from os import getpid, mkdir, listdir
from os.path import basename, exists, getsize, join, splitext
from shutil import rmtree
from time import time
from uuid import uuid1
from ..utils import _minimum_uint_size, _pack_binary_bytes_to_ints
from .tools.kmer_count import contigs_count_kmers, reads_count_kmers
from .tools.kmer_pack import contigs_pack_kmers, reads_pack_kmers

        # XXX: This is can break if the file format changes!
        #      Assumes that each line has the format kmer_seq\tV\tV\t...V\n with one binary V per genome
        total_size = getsize(tsv)
        with open(tsv, "r") as f:
            header_size = len(f.next())
            line_size = len(f.next())
        content_size = float(total_size) - header_size
        if content_size % line_size != 0:
            raise Exception()
        return int(content_size / line_size)

    # Execution callback functions
    if warning_callback is None:
        warning_callback = lambda w: logging.warning(w)
    if error_callback is None:
        def normal_raise(exception):
            raise exception

        error_callback = normal_raise
    if progress_callback is None:
        progress_callback = lambda t, p: None

    if (phenotype_description is None and phenotype_metadata_path is not None) or (
                    phenotype_description is not None and phenotype_metadata_path is None):
        raise ValueError("If a phenotype is specified, it must have a description and a metadata file.")

    kmer_len = get_kmer_length(tsv_path)
    kmer_count = get_kmer_count(tsv_path)

    kmer_dtype = 'S' + str(kmer_len)

    kmer_by_matrix_column_dtype = _minimum_uint_size(kmer_count)