# Path: mozi/utils/utils.py
# def get_file(fpath, origin, untar=False):
#     datadir = os.path.dirname(fpath)
#     if not os.path.exists(datadir):
#         os.makedirs(datadir)
# 
#     if not os.path.exists(fpath):
#         print('Downloading data from',  origin)
# 
#         global progbar
#         progbar = None
#         def dl_progress(count, block_size, total_size):
#             global progbar
#             if progbar is None:
#                 progbar = Progbar(total_size)
#             else:
#                 progbar.update(count*block_size)
# 
#         urlretrieve(origin, fpath, dl_progress)
#         progbar = None
# 
#     dirname = ""
#     if untar:
#         tfile = tarfile.open(fpath, 'r:*')
#         names = tfile.getnames()
#         dirname = names[0]
#         not_exists = [int(not os.path.exists("{}/{}".format(datadir, fname))) for fname in names]
#         if sum(not_exists) > 0:
#             print('Untaring file...')
#             tfile.extractall(path=datadir)
#         else:
#             print('Files already downloaded and untarred')
#         tfile.close()
# 
#     return "{}/{}".format(datadir, dirname)
# 
# def make_one_hot(X, onehot_size):
#     """
#     DESCRIPTION:
#         Make a one-hot version of X
#     PARAM:
#         X: 1d numpy with each value in X representing the class of X
#         onehot_size: length of the one hot vector
#     RETURN:
#         2d numpy tensor, with each row been the onehot vector
#     """
# 
#     rX = np.zeros((len(X), onehot_size), dtype=theano.config.floatX)
#     for i in xrange(len(X)):
#         rX[i, X[i]] = 1
# 
#     return rX
# 
# Path: mozi/datasets/dataset.py
# class SingleBlock(Dataset):
# 
#     def __init__(self, X=None, y=None, train_valid_test_ratio=[8,1,1], log=None, **kwargs):
#         '''
#         All the data is loaded into memory for one go training
#         '''
#         super(SingleBlock, self).__init__(log=log)
#         self.ratio = train_valid_test_ratio
#         self.train = IterMatrix(X=None, y=None, **kwargs)
#         self.valid = IterMatrix(X=None, y=None, **kwargs)
#         self.test = IterMatrix(X=None, y=None, **kwargs)
# 
#         assert len(self.ratio) == 3, 'the size of list is not 3'
# 
#         if X is not None and y is not None:
#             self.set_Xy(X, y)
# 
#     def __iter__(self):
#         self.iter = True
#         return self
# 
#     def next(self):
#         if self.iter:
#             # only one iteration since there is only one data block
#             self.iter = False
#             return self
#         else:
#             raise StopIteration
# 
#     @property
#     def nblocks(self):
#         return 1
# 
#     def set_Xy(self, X, y):
#         num_examples = len(X)
#         total_ratio = sum(self.ratio)
#         num_train = int(self.ratio[0] * 1.0 * num_examples / total_ratio)
#         num_valid = int(self.ratio[1] * 1.0 * num_examples / total_ratio)
# 
#         train_X = X[:num_train]
#         train_y = y[:num_train]
# 
#         valid_X = X[num_train:num_train+num_valid]
#         valid_y = y[num_train:num_train+num_valid]
# 
#         test_X = X[num_train+num_valid:]
#         test_y = y[num_train+num_valid:]
# 
#         self.train.X = train_X
#         self.train.y = train_y
# 
#         if self.ratio[1] == 0:
#             self.log.info('Valid set is empty! It is needed for early stopping and saving best model')
# 
#         self.valid.X = valid_X
#         self.valid.y = valid_y
# 
#         if self.ratio[2] == 0:
#             self.log.info('Test set is empty! It is needed for testing the best model')
# 
#         self.test.X = test_X
#         self.test.y = test_y
# 
# 
#     def get_train(self):
#         return self.train
# 
#     def get_valid(self):
#         return self.valid
# 
#     def get_test(self):
#         return self.test
# 
#     def set_train(self, X, y):
#         self.train.X = X
#         self.train.y = y
# 
#     def set_valid(self, X, y):
#         self.valid.X = X
#         self.valid.y = y
# 
#     def set_test(self, X, y):
#         self.test.X = X
#         self.test.y = y


from mozi.utils.utils import get_file, make_one_hot
from mozi.datasets.dataset import SingleBlock
from skimage.transform import resize
from skimage.io import imread
import xml.etree.ElementTree as ET
import os
import glob
import cPickle
import marshal
import numpy as np



class VOC(SingleBlock):

    def __init__(self, resized_shape=(222,222,3), **kwargs):
        '''
        using only voc 2012 for actions classification, total 2154 images
        resized_shape is of (height, width, channel)
        '''
        im_dir = os.environ['MOZI_DATA_PATH'] + '/voc'
        path = 'http://host.robots.ox.ac.uk/pascal/VOC/voc2012/VOCtrainval_11-May-2012.tar'

# Path: mozi/utils/utils.py
# def get_file(fpath, origin, untar=False):
#     datadir = os.path.dirname(fpath)
#     if not os.path.exists(datadir):
#         os.makedirs(datadir)
# 
#     if not os.path.exists(fpath):
#         print('Downloading data from',  origin)
# 
#         global progbar
#         progbar = None
#         def dl_progress(count, block_size, total_size):
#             global progbar
#             if progbar is None:
#                 progbar = Progbar(total_size)
#             else:
#                 progbar.update(count*block_size)
# 
#         urlretrieve(origin, fpath, dl_progress)
#         progbar = None
# 
#     dirname = ""
#     if untar:
#         tfile = tarfile.open(fpath, 'r:*')
#         names = tfile.getnames()
#         dirname = names[0]
#         not_exists = [int(not os.path.exists("{}/{}".format(datadir, fname))) for fname in names]
#         if sum(not_exists) > 0:
#             print('Untaring file...')
#             tfile.extractall(path=datadir)
#         else:
#             print('Files already downloaded and untarred')
#         tfile.close()
# 
#     return "{}/{}".format(datadir, dirname)
# 
# def make_one_hot(X, onehot_size):
#     """
#     DESCRIPTION:
#         Make a one-hot version of X
#     PARAM:
#         X: 1d numpy with each value in X representing the class of X
#         onehot_size: length of the one hot vector
#     RETURN:
#         2d numpy tensor, with each row been the onehot vector
#     """
# 
#     rX = np.zeros((len(X), onehot_size), dtype=theano.config.floatX)
#     for i in xrange(len(X)):
#         rX[i, X[i]] = 1
# 
#     return rX
# 
# Path: mozi/datasets/dataset.py
# class SingleBlock(Dataset):
# 
#     def __init__(self, X=None, y=None, train_valid_test_ratio=[8,1,1], log=None, **kwargs):
#         '''
#         All the data is loaded into memory for one go training
#         '''
#         super(SingleBlock, self).__init__(log=log)
#         self.ratio = train_valid_test_ratio
#         self.train = IterMatrix(X=None, y=None, **kwargs)
#         self.valid = IterMatrix(X=None, y=None, **kwargs)
#         self.test = IterMatrix(X=None, y=None, **kwargs)
# 
#         assert len(self.ratio) == 3, 'the size of list is not 3'
# 
#         if X is not None and y is not None:
#             self.set_Xy(X, y)
# 
#     def __iter__(self):
#         self.iter = True
#         return self
# 
#     def next(self):
#         if self.iter:
#             # only one iteration since there is only one data block
#             self.iter = False
#             return self
#         else:
#             raise StopIteration
# 
#     @property
#     def nblocks(self):
#         return 1
# 
#     def set_Xy(self, X, y):
#         num_examples = len(X)
#         total_ratio = sum(self.ratio)
#         num_train = int(self.ratio[0] * 1.0 * num_examples / total_ratio)
#         num_valid = int(self.ratio[1] * 1.0 * num_examples / total_ratio)
# 
#         train_X = X[:num_train]
#         train_y = y[:num_train]
# 
#         valid_X = X[num_train:num_train+num_valid]
#         valid_y = y[num_train:num_train+num_valid]
# 
#         test_X = X[num_train+num_valid:]
#         test_y = y[num_train+num_valid:]
# 
#         self.train.X = train_X
#         self.train.y = train_y
# 
#         if self.ratio[1] == 0:
#             self.log.info('Valid set is empty! It is needed for early stopping and saving best model')
# 
#         self.valid.X = valid_X
#         self.valid.y = valid_y
# 
#         if self.ratio[2] == 0:
#             self.log.info('Test set is empty! It is needed for testing the best model')
# 
#         self.test.X = test_X
#         self.test.y = test_y
# 
# 
#     def get_train(self):
#         return self.train
# 
#     def get_valid(self):
#         return self.valid
# 
#     def get_test(self):
#         return self.test
# 
#     def set_train(self, X, y):
#         self.train.X = X
#         self.train.y = y
# 
#     def set_valid(self, X, y):
#         self.valid.X = X
#         self.valid.y = y
# 
#     def set_test(self, X, y):
#         self.test.X = X
#         self.test.y = y


# Path: mozi/datasets/voc.py
from mozi.utils.utils import get_file, make_one_hot
from mozi.datasets.dataset import SingleBlock
from skimage.transform import resize
from skimage.io import imread
import xml.etree.ElementTree as ET
import os
import glob
import cPickle
import marshal
import numpy as np




class VOC(SingleBlock):

    def __init__(self, resized_shape=(222,222,3), **kwargs):
        '''
        using only voc 2012 for actions classification, total 2154 images
        resized_shape is of (height, width, channel)
        '''
        im_dir = os.environ['MOZI_DATA_PATH'] + '/voc'
        path = 'http://host.robots.ox.ac.uk/pascal/VOC/voc2012/VOCtrainval_11-May-2012.tar'

        im_dir = get_file(fpath="{}/VOCtrainval_11-May-2012.tar".format(im_dir), origin=path, untar=True)