# Path: picsom_bin_data.py
# class picsom_bin_data:
# 
#   def __init__(self, path) :
#     self._path = path
#     self._fp = open(path)
#     self._header = self._fp.read(64)
#     # print self._header[0:4]
#     if self._header[0:4] != "PSBD" :
#       raise Exception('Not a picsom_bin_file "%s"' % self._path)
#     hdr = unpack('if7Q', self._header)
#     self._version  = hdr[1]
#     self._hsize    = hdr[2]
#     self._rlength  = hdr[3]
#     self._vdim     = hdr[4]
#     self._format   = hdr[5]
#     self._nobjects = (os.stat(self._path).st_size-self._hsize)/self._rlength
# 
#   def __del__(self) :
#     self._fp.close()
#     pass
# 
#   def vdim(self) :
#     return self._vdim
# 
#   def nobjects(self) :
#     return self._nobjects
# 
#   def format(self) :
#     return self._format
# 
#   def format_str(self) :
#     if self._format==0 :
#       return 'undef'
#     if self._format==1 :
#       return 'float'
#     if self._format==9 :
#       return 'size_t'
#     return 'unimplemented'
# 
#   def get_size_t(self, i) :
#     if (i<0 or i>=self._nobjects) :
#       raise Exception('Index %i exceeds size of "%s"' % (i, self._path))
#     self._fp.seek(self._hsize+i*self._rlength)
#     vec = self._fp.read(self._rlength)
#     return list(unpack('%iL' % self._vdim, vec))
#   
#   def get_float(self, i) :
#     if (i<0 or i>=self._nobjects) :
#       raise Exception('Index %i exceeds size of "%s"' % (i, self._path))
#     self._fp.seek(self._hsize+i*self._rlength)
#     vec = self._fp.read(self._rlength)
#     return list(unpack('%if' % self._vdim, vec))
#   
#   def get_float_list(self, iL) :
#     if iL == -1 :
# 	  iL = xrange(self._nobjects)
#           vec = np.fromfile(self._fp, dtype = np.float32, count=self._nobjects*self._vdim )
#           vec = vec.reshape(self._nobjects,self._vdim)
# 	  print vec.shape
#     else:
# 	
#       vec = [[]]*len(iL)
#       for idx, i in enumerate(iL):
#         if (i<0 or i>=self._nobjects) :
#           raise Exception('Index %i exceeds size of "%s"' % (i, self._path))
#         self._fp.seek(self._hsize+i*self._rlength)
#         vec[idx] = list(unpack('%if' % self._vdim, self._fp.read(self._rlength)))
#     return vec
# 
#   def str(self) :
#     print "Hello world!"


import json
import os
import h5py
import scipy.io
import codecs
import numpy as np
import theano
import random
from collections import defaultdict
from picsom_bin_data import picsom_bin_data
        self.edata_prob = params['ed_sample_prob']

    # load the image features into memory
    features_path = os.path.join(self.dataset_root, feature_file)
    print 'BasicDataProvider: reading %s' % (features_path, )
    
    ## A hook to hack, no longer needed as dataset.json for lsmdc has been fixed
    offset = 0 if dataset == 'lsmdc2015' else 0
    print offset

    self.capperimage = self.dataset.get('capperimage',5)
    print self.capperimage
    
    if feature_file.rsplit('.',1)[1] == 'mat':
        if mat_new_ver == 1:
            features_struct = h5py.File(features_path)
            self.features = np.array(features_struct[features_struct.keys()[0]],dtype=theano.config.floatX)
        else:
            features_struct = scipy.io.loadmat(features_path)
            self.features = features_struct['feats']
    #The condition below makes consuming HDF5 features easy
    #   This is what I (Vik) use for features extracted in an unsupervised
    #   manner
    elif feature_file.rsplit('.',1)[1] == 'hdf5':
        self.hdf5Flag = 1
        features_struct = h5py.File(features_path)
        self.features = features_struct['features'] #The dataset in the HDF5 file is named 'features'
    elif feature_file.rsplit('.',1)[1] == 'bin':
        #This is for feature concatenation.
        # NOTE: Assuming same order of features in all the files listed in the txt file

# Path: picsom_bin_data.py
# class picsom_bin_data:
# 
#   def __init__(self, path) :
#     self._path = path
#     self._fp = open(path)
#     self._header = self._fp.read(64)
#     # print self._header[0:4]
#     if self._header[0:4] != "PSBD" :
#       raise Exception('Not a picsom_bin_file "%s"' % self._path)
#     hdr = unpack('if7Q', self._header)
#     self._version  = hdr[1]
#     self._hsize    = hdr[2]
#     self._rlength  = hdr[3]
#     self._vdim     = hdr[4]
#     self._format   = hdr[5]
#     self._nobjects = (os.stat(self._path).st_size-self._hsize)/self._rlength
# 
#   def __del__(self) :
#     self._fp.close()
#     pass
# 
#   def vdim(self) :
#     return self._vdim
# 
#   def nobjects(self) :
#     return self._nobjects
# 
#   def format(self) :
#     return self._format
# 
#   def format_str(self) :
#     if self._format==0 :
#       return 'undef'
#     if self._format==1 :
#       return 'float'
#     if self._format==9 :
#       return 'size_t'
#     return 'unimplemented'
# 
#   def get_size_t(self, i) :
#     if (i<0 or i>=self._nobjects) :
#       raise Exception('Index %i exceeds size of "%s"' % (i, self._path))
#     self._fp.seek(self._hsize+i*self._rlength)
#     vec = self._fp.read(self._rlength)
#     return list(unpack('%iL' % self._vdim, vec))
#   
#   def get_float(self, i) :
#     if (i<0 or i>=self._nobjects) :
#       raise Exception('Index %i exceeds size of "%s"' % (i, self._path))
#     self._fp.seek(self._hsize+i*self._rlength)
#     vec = self._fp.read(self._rlength)
#     return list(unpack('%if' % self._vdim, vec))
#   
#   def get_float_list(self, iL) :
#     if iL == -1 :
# 	  iL = xrange(self._nobjects)
#           vec = np.fromfile(self._fp, dtype = np.float32, count=self._nobjects*self._vdim )
#           vec = vec.reshape(self._nobjects,self._vdim)
# 	  print vec.shape
#     else:
# 	
#       vec = [[]]*len(iL)
#       for idx, i in enumerate(iL):
#         if (i<0 or i>=self._nobjects) :
#           raise Exception('Index %i exceeds size of "%s"' % (i, self._path))
#         self._fp.seek(self._hsize+i*self._rlength)
#         vec[idx] = list(unpack('%if' % self._vdim, self._fp.read(self._rlength)))
#     return vec
# 
#   def str(self) :
#     print "Hello world!"


# Path: imagernn/data_provider.py
import json
import os
import h5py
import scipy.io
import codecs
import numpy as np
import theano
import random
from collections import defaultdict
from picsom_bin_data import picsom_bin_data

        self.edata_prob = params['ed_sample_prob']

    # load the image features into memory
    features_path = os.path.join(self.dataset_root, feature_file)
    print 'BasicDataProvider: reading %s' % (features_path, )
    
    ## A hook to hack, no longer needed as dataset.json for lsmdc has been fixed
    offset = 0 if dataset == 'lsmdc2015' else 0
    print offset

    self.capperimage = self.dataset.get('capperimage',5)
    print self.capperimage
    
    if feature_file.rsplit('.',1)[1] == 'mat':
        if mat_new_ver == 1:
            features_struct = h5py.File(features_path)
            self.features = np.array(features_struct[features_struct.keys()[0]],dtype=theano.config.floatX)
        else:
            features_struct = scipy.io.loadmat(features_path)
            self.features = features_struct['feats']
    #The condition below makes consuming HDF5 features easy
    #   This is what I (Vik) use for features extracted in an unsupervised
    #   manner
    elif feature_file.rsplit('.',1)[1] == 'hdf5':
        self.hdf5Flag = 1
        features_struct = h5py.File(features_path)
        self.features = features_struct['features'] #The dataset in the HDF5 file is named 'features'
    elif feature_file.rsplit('.',1)[1] == 'bin':
        #This is for feature concatenation.
        # NOTE: Assuming same order of features in all the files listed in the txt file

        features_struct = picsom_bin_data(features_path) 