# Path: metamat.py
# class MetaMat(object):
#     """
#     Iterator that returns a large matrix of floats in chunks of rows, where the
#     number of rows in a chunk is a parameter passed to the constructor.
#     Assumes all elements are double-precision 8-byte floating-point numbers.
#     """
# 
#     def __init__(self, prefix, chunk_size=1000000):
#         """ Parse metadata, check that files exist and initialize members """
#         self.prefix = prefix
#         self.chunk_size = chunk_size
#         self.fh = None
#         self.cur = 0
#         self.done = False
# 
#         meta_fn = prefix + '.meta'
#         if not os.path.exists(meta_fn):
#             raise RuntimeError('Metadata file does not exist: "%s"' % meta_fn)
# 
#         self.data_fn = prefix + '.npy'
#         if not os.path.exists(self.data_fn):
#             raise RuntimeError('Data file does not exist: "%s"' % self.data_fn)
# 
#         with open(meta_fn) as fh:
#             fields = fh.readline().rstrip().split(',')
#             self.nrow = int(fields[-1])
#             self.cols = fields[:-1]
# 
#         # Start at first chunk
#         self.fh = open(self.data_fn, 'rb')
#         self.cur = 0
#         self.done = False
# 
#     def __iter__(self):
#         return self
# 
#     def next(self):
#         return self.__next__()
# 
#     def __next__(self):
#         """ Return next chunk """
#         if self.done:
#             self.fh.close()
#             raise StopIteration
#         if self.chunk_size > 0:
#             row_i, row_f = self.cur, min(self.cur + self.chunk_size, self.nrow)
#         else:
#             assert self.cur == 0
#             row_i, row_f = 0, self.nrow
#         self.done = row_f == self.nrow
#         self.cur = row_f
#         nelt = (row_f - row_i) * len(self.cols)
#         assert self.fh.tell() == row_i * 8 * len(self.cols)
#         m = numpy.fromfile(self.fh, dtype=numpy.float64, count=nelt, sep='')
#         expected_pos = row_f * 8 * len(self.cols)
#         assert self.fh.tell() == expected_pos
#         assert m.size == nelt, (row_i, row_f, len(self.cols), m.size, nelt)
#         m = m.reshape((row_f - row_i, len(self.cols)))
#         return pandas.DataFrame(data=m, columns=self.cols)
# 
#     def reset(self):
#         if self.fh is not None:
#             self.fh.close()
#         self.fh = open(self.data_fn, 'rb')
#         self.cur = 0
#         self.done = False
# 
#     @staticmethod
#     def write_metamat(prefix, col_names, floats=None, append=False):
#         """ Caution: this doesn't use numpy tofile, and so is slow for writing
#             lots of floats """
#         if not append:
#             with open(prefix + '.meta', 'wb') as ofh:
#                 n_col = len(col_names)
#                 assert len(floats) % n_col == 0
#                 n_row = len(floats) // n_col
#                 ofh.write(b','.join(col_names + [str(n_row).encode()]))
# 
#         if floats is not None:
#             with open(prefix + '.npy', 'ab' if append else 'wb') as ofh:
#                 for f in floats:
#                     ofh.write(struct.pack('d', f))


import os
import warnings
import math
import numpy as np
    from itertools import imap
from metamat import MetaMat
FeatureTableReader
"""

try:
except ImportError:
    imap = map

# qtip imports


__author__ = 'langmead'


class FeatureTableReader(object):
    """ Reads a table of information describing alignments.  These are tables
        output by qtip.  Tables might describe training or test alignments. """

    #            short
    #            name   suffix
    datasets = [('u',   '_rec_u'),
                ('d',   '_rec_d'),
                ('c',   '_rec_c'),
                ('b',   '_rec_b')]

    def __init__(self, prefix, chunksize=100000):
        self.prefix = prefix
        self.dfs = {}
        self.readers = {}
        nonempty = False
        fns = []

# Path: metamat.py
# class MetaMat(object):
#     """
#     Iterator that returns a large matrix of floats in chunks of rows, where the
#     number of rows in a chunk is a parameter passed to the constructor.
#     Assumes all elements are double-precision 8-byte floating-point numbers.
#     """
# 
#     def __init__(self, prefix, chunk_size=1000000):
#         """ Parse metadata, check that files exist and initialize members """
#         self.prefix = prefix
#         self.chunk_size = chunk_size
#         self.fh = None
#         self.cur = 0
#         self.done = False
# 
#         meta_fn = prefix + '.meta'
#         if not os.path.exists(meta_fn):
#             raise RuntimeError('Metadata file does not exist: "%s"' % meta_fn)
# 
#         self.data_fn = prefix + '.npy'
#         if not os.path.exists(self.data_fn):
#             raise RuntimeError('Data file does not exist: "%s"' % self.data_fn)
# 
#         with open(meta_fn) as fh:
#             fields = fh.readline().rstrip().split(',')
#             self.nrow = int(fields[-1])
#             self.cols = fields[:-1]
# 
#         # Start at first chunk
#         self.fh = open(self.data_fn, 'rb')
#         self.cur = 0
#         self.done = False
# 
#     def __iter__(self):
#         return self
# 
#     def next(self):
#         return self.__next__()
# 
#     def __next__(self):
#         """ Return next chunk """
#         if self.done:
#             self.fh.close()
#             raise StopIteration
#         if self.chunk_size > 0:
#             row_i, row_f = self.cur, min(self.cur + self.chunk_size, self.nrow)
#         else:
#             assert self.cur == 0
#             row_i, row_f = 0, self.nrow
#         self.done = row_f == self.nrow
#         self.cur = row_f
#         nelt = (row_f - row_i) * len(self.cols)
#         assert self.fh.tell() == row_i * 8 * len(self.cols)
#         m = numpy.fromfile(self.fh, dtype=numpy.float64, count=nelt, sep='')
#         expected_pos = row_f * 8 * len(self.cols)
#         assert self.fh.tell() == expected_pos
#         assert m.size == nelt, (row_i, row_f, len(self.cols), m.size, nelt)
#         m = m.reshape((row_f - row_i, len(self.cols)))
#         return pandas.DataFrame(data=m, columns=self.cols)
# 
#     def reset(self):
#         if self.fh is not None:
#             self.fh.close()
#         self.fh = open(self.data_fn, 'rb')
#         self.cur = 0
#         self.done = False
# 
#     @staticmethod
#     def write_metamat(prefix, col_names, floats=None, append=False):
#         """ Caution: this doesn't use numpy tofile, and so is slow for writing
#             lots of floats """
#         if not append:
#             with open(prefix + '.meta', 'wb') as ofh:
#                 n_col = len(col_names)
#                 assert len(floats) % n_col == 0
#                 n_row = len(floats) // n_col
#                 ofh.write(b','.join(col_names + [str(n_row).encode()]))
# 
#         if floats is not None:
#             with open(prefix + '.npy', 'ab' if append else 'wb') as ofh:
#                 for f in floats:
#                     ofh.write(struct.pack('d', f))


# Path: feature_table.py
import os
import warnings
import math
import numpy as np
    from itertools import imap
from metamat import MetaMat

FeatureTableReader
"""

try:
except ImportError:
    imap = map

# qtip imports


__author__ = 'langmead'


class FeatureTableReader(object):
    """ Reads a table of information describing alignments.  These are tables
        output by qtip.  Tables might describe training or test alignments. """

    #            short
    #            name   suffix
    datasets = [('u',   '_rec_u'),
                ('d',   '_rec_d'),
                ('c',   '_rec_c'),
                ('b',   '_rec_b')]

    def __init__(self, prefix, chunksize=100000):
        self.prefix = prefix
        self.dfs = {}
        self.readers = {}
        nonempty = False
        fns = []

        for sn, suf in self.datasets: