# Path: mozi/utils/utils.py
# def get_file(fpath, origin, untar=False):
#     datadir = os.path.dirname(fpath)
#     if not os.path.exists(datadir):
#         os.makedirs(datadir)
# 
#     if not os.path.exists(fpath):
#         print('Downloading data from',  origin)
# 
#         global progbar
#         progbar = None
#         def dl_progress(count, block_size, total_size):
#             global progbar
#             if progbar is None:
#                 progbar = Progbar(total_size)
#             else:
#                 progbar.update(count*block_size)
# 
#         urlretrieve(origin, fpath, dl_progress)
#         progbar = None
# 
#     dirname = ""
#     if untar:
#         tfile = tarfile.open(fpath, 'r:*')
#         names = tfile.getnames()
#         dirname = names[0]
#         not_exists = [int(not os.path.exists("{}/{}".format(datadir, fname))) for fname in names]
#         if sum(not_exists) > 0:
#             print('Untaring file...')
#             tfile.extractall(path=datadir)
#         else:
#             print('Files already downloaded and untarred')
#         tfile.close()
# 
#     return "{}/{}".format(datadir, dirname)
# 
# def make_one_hot(X, onehot_size):
#     """
#     DESCRIPTION:
#         Make a one-hot version of X
#     PARAM:
#         X: 1d numpy with each value in X representing the class of X
#         onehot_size: length of the one hot vector
#     RETURN:
#         2d numpy tensor, with each row been the onehot vector
#     """
# 
#     rX = np.zeros((len(X), onehot_size), dtype=theano.config.floatX)
#     for i in xrange(len(X)):
#         rX[i, X[i]] = 1
# 
#     return rX
# 
# Path: mozi/datasets/dataset.py
# class SingleBlock(Dataset):
# 
#     def __init__(self, X=None, y=None, train_valid_test_ratio=[8,1,1], log=None, **kwargs):
#         '''
#         All the data is loaded into memory for one go training
#         '''
#         super(SingleBlock, self).__init__(log=log)
#         self.ratio = train_valid_test_ratio
#         self.train = IterMatrix(X=None, y=None, **kwargs)
#         self.valid = IterMatrix(X=None, y=None, **kwargs)
#         self.test = IterMatrix(X=None, y=None, **kwargs)
# 
#         assert len(self.ratio) == 3, 'the size of list is not 3'
# 
#         if X is not None and y is not None:
#             self.set_Xy(X, y)
# 
#     def __iter__(self):
#         self.iter = True
#         return self
# 
#     def next(self):
#         if self.iter:
#             # only one iteration since there is only one data block
#             self.iter = False
#             return self
#         else:
#             raise StopIteration
# 
#     @property
#     def nblocks(self):
#         return 1
# 
#     def set_Xy(self, X, y):
#         num_examples = len(X)
#         total_ratio = sum(self.ratio)
#         num_train = int(self.ratio[0] * 1.0 * num_examples / total_ratio)
#         num_valid = int(self.ratio[1] * 1.0 * num_examples / total_ratio)
# 
#         train_X = X[:num_train]
#         train_y = y[:num_train]
# 
#         valid_X = X[num_train:num_train+num_valid]
#         valid_y = y[num_train:num_train+num_valid]
# 
#         test_X = X[num_train+num_valid:]
#         test_y = y[num_train+num_valid:]
# 
#         self.train.X = train_X
#         self.train.y = train_y
# 
#         if self.ratio[1] == 0:
#             self.log.info('Valid set is empty! It is needed for early stopping and saving best model')
# 
#         self.valid.X = valid_X
#         self.valid.y = valid_y
# 
#         if self.ratio[2] == 0:
#             self.log.info('Test set is empty! It is needed for testing the best model')
# 
#         self.test.X = test_X
#         self.test.y = test_y
# 
# 
#     def get_train(self):
#         return self.train
# 
#     def get_valid(self):
#         return self.valid
# 
#     def get_test(self):
#         return self.test
# 
#     def set_train(self, X, y):
#         self.train.X = X
#         self.train.y = y
# 
#     def set_valid(self, X, y):
#         self.valid.X = X
#         self.valid.y = y
# 
#     def set_test(self, X, y):
#         self.test.X = X
#         self.test.y = y


from mozi.utils.utils import get_file, make_one_hot
from mozi.datasets.dataset import SingleBlock
from skimage.transform import resize
from skimage.io import imread
import xml.etree.ElementTree as ET
import os
import glob
import cPickle
import marshal
import numpy as np
            files = glob.glob(annote + '/2012*xml')
            labels = []
            rimage = []
            for f in files:
                bname = os.path.basename(f).rstrip('.xml')
                image = imread('{}/{}.jpg'.format(images, bname))
                rimage.append(resize(image, resized_shape))
                tree = ET.parse(f)
                root = tree.getroot()
                actions = root.find('object').find('actions')

                for act in actions:
                    if act.text == '1':
                        labels.append(actls.index(act.tag))
                        # only restrict to one action per photo
                        break

            print 'saving data'
            with open(X_path, 'wb') as Xout, open(y_path, 'wb') as yout:
                X = np.asarray(rimage)
                y = np.asarray(labels)
                np.save(Xout, X)
                np.save(yout, y)

        else:
            print X_path + ' exists, loading..'
            with open(X_path, 'rb') as Xin, open(y_path, 'rb') as yin:
                X = np.load(Xin)
                y = np.load(yin)
                

# Path: mozi/utils/utils.py
# def get_file(fpath, origin, untar=False):
#     datadir = os.path.dirname(fpath)
#     if not os.path.exists(datadir):
#         os.makedirs(datadir)
# 
#     if not os.path.exists(fpath):
#         print('Downloading data from',  origin)
# 
#         global progbar
#         progbar = None
#         def dl_progress(count, block_size, total_size):
#             global progbar
#             if progbar is None:
#                 progbar = Progbar(total_size)
#             else:
#                 progbar.update(count*block_size)
# 
#         urlretrieve(origin, fpath, dl_progress)
#         progbar = None
# 
#     dirname = ""
#     if untar:
#         tfile = tarfile.open(fpath, 'r:*')
#         names = tfile.getnames()
#         dirname = names[0]
#         not_exists = [int(not os.path.exists("{}/{}".format(datadir, fname))) for fname in names]
#         if sum(not_exists) > 0:
#             print('Untaring file...')
#             tfile.extractall(path=datadir)
#         else:
#             print('Files already downloaded and untarred')
#         tfile.close()
# 
#     return "{}/{}".format(datadir, dirname)
# 
# def make_one_hot(X, onehot_size):
#     """
#     DESCRIPTION:
#         Make a one-hot version of X
#     PARAM:
#         X: 1d numpy with each value in X representing the class of X
#         onehot_size: length of the one hot vector
#     RETURN:
#         2d numpy tensor, with each row been the onehot vector
#     """
# 
#     rX = np.zeros((len(X), onehot_size), dtype=theano.config.floatX)
#     for i in xrange(len(X)):
#         rX[i, X[i]] = 1
# 
#     return rX
# 
# Path: mozi/datasets/dataset.py
# class SingleBlock(Dataset):
# 
#     def __init__(self, X=None, y=None, train_valid_test_ratio=[8,1,1], log=None, **kwargs):
#         '''
#         All the data is loaded into memory for one go training
#         '''
#         super(SingleBlock, self).__init__(log=log)
#         self.ratio = train_valid_test_ratio
#         self.train = IterMatrix(X=None, y=None, **kwargs)
#         self.valid = IterMatrix(X=None, y=None, **kwargs)
#         self.test = IterMatrix(X=None, y=None, **kwargs)
# 
#         assert len(self.ratio) == 3, 'the size of list is not 3'
# 
#         if X is not None and y is not None:
#             self.set_Xy(X, y)
# 
#     def __iter__(self):
#         self.iter = True
#         return self
# 
#     def next(self):
#         if self.iter:
#             # only one iteration since there is only one data block
#             self.iter = False
#             return self
#         else:
#             raise StopIteration
# 
#     @property
#     def nblocks(self):
#         return 1
# 
#     def set_Xy(self, X, y):
#         num_examples = len(X)
#         total_ratio = sum(self.ratio)
#         num_train = int(self.ratio[0] * 1.0 * num_examples / total_ratio)
#         num_valid = int(self.ratio[1] * 1.0 * num_examples / total_ratio)
# 
#         train_X = X[:num_train]
#         train_y = y[:num_train]
# 
#         valid_X = X[num_train:num_train+num_valid]
#         valid_y = y[num_train:num_train+num_valid]
# 
#         test_X = X[num_train+num_valid:]
#         test_y = y[num_train+num_valid:]
# 
#         self.train.X = train_X
#         self.train.y = train_y
# 
#         if self.ratio[1] == 0:
#             self.log.info('Valid set is empty! It is needed for early stopping and saving best model')
# 
#         self.valid.X = valid_X
#         self.valid.y = valid_y
# 
#         if self.ratio[2] == 0:
#             self.log.info('Test set is empty! It is needed for testing the best model')
# 
#         self.test.X = test_X
#         self.test.y = test_y
# 
# 
#     def get_train(self):
#         return self.train
# 
#     def get_valid(self):
#         return self.valid
# 
#     def get_test(self):
#         return self.test
# 
#     def set_train(self, X, y):
#         self.train.X = X
#         self.train.y = y
# 
#     def set_valid(self, X, y):
#         self.valid.X = X
#         self.valid.y = y
# 
#     def set_test(self, X, y):
#         self.test.X = X
#         self.test.y = y


# Path: mozi/datasets/voc.py
from mozi.utils.utils import get_file, make_one_hot
from mozi.datasets.dataset import SingleBlock
from skimage.transform import resize
from skimage.io import imread
import xml.etree.ElementTree as ET
import os
import glob
import cPickle
import marshal
import numpy as np

            files = glob.glob(annote + '/2012*xml')
            labels = []
            rimage = []
            for f in files:
                bname = os.path.basename(f).rstrip('.xml')
                image = imread('{}/{}.jpg'.format(images, bname))
                rimage.append(resize(image, resized_shape))
                tree = ET.parse(f)
                root = tree.getroot()
                actions = root.find('object').find('actions')

                for act in actions:
                    if act.text == '1':
                        labels.append(actls.index(act.tag))
                        # only restrict to one action per photo
                        break

            print 'saving data'
            with open(X_path, 'wb') as Xout, open(y_path, 'wb') as yout:
                X = np.asarray(rimage)
                y = np.asarray(labels)
                np.save(Xout, X)
                np.save(yout, y)

        else:
            print X_path + ' exists, loading..'
            with open(X_path, 'rb') as Xin, open(y_path, 'rb') as yin:
                X = np.load(Xin)
                y = np.load(yin)
                

        super(VOC, self).__init__(X=np.rollaxis(X,3,1), y=make_one_hot(y,len(actls)), **kwargs)