オートエンコーダー(AutoEncoder)について質問があり投稿しました。  
初めての投稿で不備があったら教えて頂けると幸いです。

以下のプログラムを実装したのですが、今、横160縦120pixの画像を入力すると、「ResourceExhaustedError」が発生して、学習に進むことができません。  
具体的には、130行目のところでErrorが発生します。  
一方、解像度を半分の横80縦60pixにすると、EPOCが進み学習が進んでいるように見えます。  
(プログラムで画像を割る2して小さくしています。)

画像サイズ(横160縦120pix)や枚数(約700枚)が特に多くないと思っているのですが、なぜエラーが発生するのかと解決方法をご教授頂けないでしょうか。  
メインメモリ不足が影響している可能性を考えてメモリを128GBにしましたが同様のエラーが発生します。

以上、よろしくお願いいたします。

実装環境を以下に記します。  
CPU:Xeon E5-1620v4 4core/8t  
マザーボード：ASUS X99-E WS  
メモリ：DDR4-2400 64GB(8G×8)  
GPU:NVIDIA Quadro GP100 16GB 2個  
OS:ubuntu16.04LTS

ソースコード

```

    import numpy as np
    import tensorflow as tf
    from tensorflow.python.framework import ops
    import cv2
    import os
    
    DATASET_PATH = "/home/densos/workspaces/autoencoder"
    DIR_PATH = "input_gray_160*120"
    IMAGE_PATH = os.path.join(DATASET_PATH, DIR_PATH)
    X_PIXEL, Y_PIXEL = 160, 120
    M = 1
    N_HIDDENS = np.array(np.array([1.5]) * X_PIXEL * Y_PIXEL // (M*M), dtype = np.int)
    TRANCE_FRAME_NUM = 700
    
    ops.reset_default_graph()
    
    def xavier_init(fan_in, fan_out, constant = 1):
        low = -constant * np.sqrt(6.0 / (fan_in + fan_out))
        high = constant * np.sqrt(6.0 / (fan_in + fan_out))
        return tf.random_uniform((fan_in, fan_out), minval = low, maxval = high, dtype = tf.float32)
    
    class AdditiveGaussianNoiseAutoencoder(object):
        def __init__(self, n_input, n_hidden, transfer_function = tf.nn.sigmoid, optimizer = tf.train.AdamOptimizer(), scale = 0.1):
            self.n_input = n_input
            self.n_hidden = n_hidden
            self.transfer = transfer_function
            self.scale = tf.placeholder(tf.float32)
            self.training_scale = scale
            network_weights = self._initialize_weights()
            self.weights = network_weights
            self.sparsity_level = np.repeat([0.05], self.n_hidden).astype(np.float32)
            self.sparse_reg = 0.1
    
            # model
            self.x = tf.placeholder(tf.float32, [None, self.n_input])
            self.hidden = self.transfer(tf.add(tf.matmul(self.x + scale * tf.random_normal((n_input,)),
                    self.weights['w1']),
                    self.weights['b1']))
            self.reconstruction = tf.add(tf.matmul(self.hidden, self.weights['w2']), self.weights['b2'])
    
            # cost
            self.cost = 0.5 * tf.reduce_sum(tf.pow(tf.subtract(self.reconstruction, self.x), 2.0)) + self.sparse_reg \
                            * self.kl_divergence(self.sparsity_level, self.hidden)
    
            self.optimizer = optimizer.minimize(self.cost)
    
            init = tf.global_variables_initializer()
            self.sess = tf.Session()
            self.sess.run(init)
    
        def _initialize_weights(self):
            all_weights = dict()
            all_weights['w1'] = tf.Variable(xavier_init(self.n_input, self.n_hidden))
            all_weights['b1'] = tf.Variable(tf.zeros([self.n_hidden], dtype = tf.float32))
            all_weights['w2'] = tf.Variable(tf.zeros([self.n_hidden, self.n_input], dtype = tf.float32))
            all_weights['b2'] = tf.Variable(tf.zeros([self.n_input], dtype = tf.float32))
            return all_weights
    
        def partial_fit(self, X):
            cost, opt = self.sess.run((self.cost, self.optimizer), feed_dict = {self.x: X,
                                                                                self.scale: self.training_scale
                                                                                })
            return cost
    
        def kl_divergence(self, p, p_hat):
            return tf.reduce_mean(p * tf.log(p) - p * tf.log(p_hat) + (1 - p) * tf.log(1 - p) - (1 - p) * tf.log(1 - p_hat))
    
        def calc_total_cost(self, X):
            return self.sess.run(self.cost, feed_dict = {self.x: X,
                                                         self.scale: self.training_scale
                                                         })
    
        def transform(self, X):
            return self.sess.run(self.hidden, feed_dict = {self.x: X,
                                                           self.scale: self.training_scale
                                                           })
    
        def generate(self, hidden = None):
            if hidden is None:
                hidden = np.random.normal(size = self.weights["b1"])
            return self.sess.run(self.reconstruction, feed_dict = {self.hidden: hidden})
    
        def reconstruct(self, X):
            return self.sess.run(self.reconstruction, feed_dict = {self.x: X,
                                                                   self.scale: self.training_scale
                                                                   })
    
        def getWeights(self):
            return self.sess.run(self.weights['w1'])
    
        def getBiases(self):
            return self.sess.run(self.weights['b1'])
    
    def get_random_block_from_data(data, batch_size):
        start_index = np.random.randint(0, len(data) - batch_size)
        return data[start_index:(start_index + batch_size)]
    
    
    if __name__ == '__main__':
    #get input data lists
        lists = []
        for file in os.listdir(IMAGE_PATH):
            if file.endswith(".jpeg"):
                lists.append(file)
            lists.sort()
    
    #read input data    
        input_images = []
        for image in lists:
            tmp = cv2.imread(os.path.join(IMAGE_PATH, image), cv2.IMREAD_GRAYSCALE)
            tmp = cv2.resize(tmp, (X_PIXEL // M, Y_PIXEL // M))
            tmp = tmp.reshape(tmp.shape[0] * tmp.shape[1])
            input_images.append(tmp)
    
    #preprocess images    
        input_images = np.array(input_images) / 255.
    
    #convert data to float16
        input_images = np.array(input_images, dtype = np.float16)
    
    #set train and test data
        X_train = input_images[:500]
        X_test = input_images[500:]
    
        n_samples = X_train.shape[0]
        training_epochs = 200
        batch_size = X_train.shape[0] // 4
        display_step = 10
    
        autoencoder = AdditiveGaussianNoiseAutoencoder(n_input = X_train.shape[1],
                                                       n_hidden = N_HIDDENS[0],
                                                       transfer_function = tf.nn.relu6,
                                                       optimizer = tf.train.AdamOptimizer(learning_rate = 0.001),
                                                       scale = 0.01)
    
        for epoch in range(training_epochs):
            avg_cost = 0.
            total_batch = int(n_samples / batch_size)
            # Loop over all batches
            for i in range(total_batch):
                batch_xs = get_random_block_from_data(X_train, batch_size)
    
                # Fit training using batch data
                cost = autoencoder.partial_fit(X_train)
                # Compute average loss
                avg_cost += cost / n_samples * batch_size
    
            # Display logs per epoch step
            if epoch % display_step == 0:
                print("Epoch:", '%04d' % (epoch + 1), "cost=", avg_cost)
    
        print("Finish Train")
    
    predicted_imgs = autoencoder.reconstruct(X_test)
    predicted_imgs = np.array((predicted_imgs) * 255, dtype = np.uint8)
    input_imgs = np.array((X_test) * 255, dtype = np.uint8)
    
    # plot the reconstructed images
    for i in range(100):
        im1 = predicted_imgs[i].reshape((Y_PIXEL//M, X_PIXEL//M))
        im2 = input_imgs[i].reshape((Y_PIXEL//M, X_PIXEL//M))
    
        img_v_union = cv2.vconcat([im1, im2])
        cv2.moveWindow('result.jpg', 100, 200)
        cv2.imshow('result.jpg', img_v_union)
    
        cv2.waitKey(33)
    
```
CC BY-SA 4.0
2018-06-05T00:06:58.483
44522
2018-06-05T00:06:58.483
28774
question
python tensorflow 深層学習
ResourceExhaustedErrorについて。（AutoEncoderの実装）