# Path: picsom_bin_data.py
# class picsom_bin_data:
# 
#   def __init__(self, path) :
#     self._path = path
#     self._fp = open(path)
#     self._header = self._fp.read(64)
#     # print self._header[0:4]
#     if self._header[0:4] != "PSBD" :
#       raise Exception('Not a picsom_bin_file "%s"' % self._path)
#     hdr = unpack('if7Q', self._header)
#     self._version  = hdr[1]
#     self._hsize    = hdr[2]
#     self._rlength  = hdr[3]
#     self._vdim     = hdr[4]
#     self._format   = hdr[5]
#     self._nobjects = (os.stat(self._path).st_size-self._hsize)/self._rlength
# 
#   def __del__(self) :
#     self._fp.close()
#     pass
# 
#   def vdim(self) :
#     return self._vdim
# 
#   def nobjects(self) :
#     return self._nobjects
# 
#   def format(self) :
#     return self._format
# 
#   def format_str(self) :
#     if self._format==0 :
#       return 'undef'
#     if self._format==1 :
#       return 'float'
#     if self._format==9 :
#       return 'size_t'
#     return 'unimplemented'
# 
#   def get_size_t(self, i) :
#     if (i<0 or i>=self._nobjects) :
#       raise Exception('Index %i exceeds size of "%s"' % (i, self._path))
#     self._fp.seek(self._hsize+i*self._rlength)
#     vec = self._fp.read(self._rlength)
#     return list(unpack('%iL' % self._vdim, vec))
#   
#   def get_float(self, i) :
#     if (i<0 or i>=self._nobjects) :
#       raise Exception('Index %i exceeds size of "%s"' % (i, self._path))
#     self._fp.seek(self._hsize+i*self._rlength)
#     vec = self._fp.read(self._rlength)
#     return list(unpack('%if' % self._vdim, vec))
#   
#   def get_float_list(self, iL) :
#     if iL == -1 :
# 	  iL = xrange(self._nobjects)
#           vec = np.fromfile(self._fp, dtype = np.float32, count=self._nobjects*self._vdim )
#           vec = vec.reshape(self._nobjects,self._vdim)
# 	  print vec.shape
#     else:
# 	
#       vec = [[]]*len(iL)
#       for idx, i in enumerate(iL):
#         if (i<0 or i>=self._nobjects) :
#           raise Exception('Index %i exceeds size of "%s"' % (i, self._path))
#         self._fp.seek(self._hsize+i*self._rlength)
#         vec[idx] = list(unpack('%if' % self._vdim, self._fp.read(self._rlength)))
#     return vec
# 
#   def str(self) :
#     print "Hello world!"


import json
import os
import h5py
import scipy.io
import codecs
import numpy as np
import theano
import random
from collections import defaultdict
from picsom_bin_data import picsom_bin_data
      elif features_path.rsplit('.',1)[1] == 'bin':
        features_struct = picsom_bin_data(features_path) 
        features = np.array(features_struct.get_float_list(idxes)).T.astype(theano.config.floatX) # this is a 4096 x N numpy array of features
        print "Working on Bin file now"
      elif features_path.rsplit('.',1)[1] == 'txt':
          #This is for feature concatenation.
          # NOTE: Assuming same order of features in all the files listed in the txt file
          feat_Flist = open(features_path, 'r').read().splitlines()
          feat_list = []
          for f in feat_Flist:
              f_struct = picsom_bin_data(f) 
              feat_list.append(np.array(f_struct.get_float_list(idxes)).T)
              print feat_list[-1].shape
      	  # this is a 4096 x N numpy array of features
          features = np.concatenate(feat_list, axis=0).astype(theano.config.floatX)
          print "Combined all the features. Final size is %d %d"%(features.shape[0],features.shape[1])
      
      aux_inp = []
      aux_inp_file = params['aux_inp_file'][i] if params.get('multi_model',0) else params.get('aux_inp_file',None)
      if aux_inp_file != None:
          # Load Auxillary input file, one vec per image
          # NOTE: Assuming same order as feature file
          auxidxes = idxes if auxidxes == [] else auxidxes
          if aux_inp_file.rsplit('.',1)[1] == 'bin':
              f_struct = picsom_bin_data(aux_inp_file)
              aux_inp = np.array(f_struct.get_float_list(auxidxes)).T.astype(theano.config.floatX)
          elif aux_inp_file.rsplit('.',1)[1] == 'txt':
              feat_Flist = open(aux_inp_file, 'r').read().splitlines()
              feat_list = []
              for f in feat_Flist:

# Path: picsom_bin_data.py
# class picsom_bin_data:
# 
#   def __init__(self, path) :
#     self._path = path
#     self._fp = open(path)
#     self._header = self._fp.read(64)
#     # print self._header[0:4]
#     if self._header[0:4] != "PSBD" :
#       raise Exception('Not a picsom_bin_file "%s"' % self._path)
#     hdr = unpack('if7Q', self._header)
#     self._version  = hdr[1]
#     self._hsize    = hdr[2]
#     self._rlength  = hdr[3]
#     self._vdim     = hdr[4]
#     self._format   = hdr[5]
#     self._nobjects = (os.stat(self._path).st_size-self._hsize)/self._rlength
# 
#   def __del__(self) :
#     self._fp.close()
#     pass
# 
#   def vdim(self) :
#     return self._vdim
# 
#   def nobjects(self) :
#     return self._nobjects
# 
#   def format(self) :
#     return self._format
# 
#   def format_str(self) :
#     if self._format==0 :
#       return 'undef'
#     if self._format==1 :
#       return 'float'
#     if self._format==9 :
#       return 'size_t'
#     return 'unimplemented'
# 
#   def get_size_t(self, i) :
#     if (i<0 or i>=self._nobjects) :
#       raise Exception('Index %i exceeds size of "%s"' % (i, self._path))
#     self._fp.seek(self._hsize+i*self._rlength)
#     vec = self._fp.read(self._rlength)
#     return list(unpack('%iL' % self._vdim, vec))
#   
#   def get_float(self, i) :
#     if (i<0 or i>=self._nobjects) :
#       raise Exception('Index %i exceeds size of "%s"' % (i, self._path))
#     self._fp.seek(self._hsize+i*self._rlength)
#     vec = self._fp.read(self._rlength)
#     return list(unpack('%if' % self._vdim, vec))
#   
#   def get_float_list(self, iL) :
#     if iL == -1 :
# 	  iL = xrange(self._nobjects)
#           vec = np.fromfile(self._fp, dtype = np.float32, count=self._nobjects*self._vdim )
#           vec = vec.reshape(self._nobjects,self._vdim)
# 	  print vec.shape
#     else:
# 	
#       vec = [[]]*len(iL)
#       for idx, i in enumerate(iL):
#         if (i<0 or i>=self._nobjects) :
#           raise Exception('Index %i exceeds size of "%s"' % (i, self._path))
#         self._fp.seek(self._hsize+i*self._rlength)
#         vec[idx] = list(unpack('%if' % self._vdim, self._fp.read(self._rlength)))
#     return vec
# 
#   def str(self) :
#     print "Hello world!"


# Path: imagernn/data_provider.py
import json
import os
import h5py
import scipy.io
import codecs
import numpy as np
import theano
import random
from collections import defaultdict
from picsom_bin_data import picsom_bin_data

      elif features_path.rsplit('.',1)[1] == 'bin':
        features_struct = picsom_bin_data(features_path) 
        features = np.array(features_struct.get_float_list(idxes)).T.astype(theano.config.floatX) # this is a 4096 x N numpy array of features
        print "Working on Bin file now"
      elif features_path.rsplit('.',1)[1] == 'txt':
          #This is for feature concatenation.
          # NOTE: Assuming same order of features in all the files listed in the txt file
          feat_Flist = open(features_path, 'r').read().splitlines()
          feat_list = []
          for f in feat_Flist:
              f_struct = picsom_bin_data(f) 
              feat_list.append(np.array(f_struct.get_float_list(idxes)).T)
              print feat_list[-1].shape
      	  # this is a 4096 x N numpy array of features
          features = np.concatenate(feat_list, axis=0).astype(theano.config.floatX)
          print "Combined all the features. Final size is %d %d"%(features.shape[0],features.shape[1])
      
      aux_inp = []
      aux_inp_file = params['aux_inp_file'][i] if params.get('multi_model',0) else params.get('aux_inp_file',None)
      if aux_inp_file != None:
          # Load Auxillary input file, one vec per image
          # NOTE: Assuming same order as feature file
          auxidxes = idxes if auxidxes == [] else auxidxes
          if aux_inp_file.rsplit('.',1)[1] == 'bin':
              f_struct = picsom_bin_data(aux_inp_file)
              aux_inp = np.array(f_struct.get_float_list(auxidxes)).T.astype(theano.config.floatX)
          elif aux_inp_file.rsplit('.',1)[1] == 'txt':
              feat_Flist = open(aux_inp_file, 'r').read().splitlines()
              feat_list = []
              for f in feat_Flist:

                  f_struct = picsom_bin_data(f)