ResourceExhaustedErrorについて。（AutoEncoderの実装）

前提・実現したいこと

オートエンコーダー(AutoEncoder)について質問があり投稿しました。
初めての投稿で不備があったら教えて頂けると幸いです。

以下のプログラムを実装したのですが、今、横160縦120pixの画像を入力すると、「ResourceExhaustedError」が発生して、学習に進むことができません。
具体的には、130行目のところでErrorが発生します。
一方、解像度を半分の横80縦60pixにすると、EPOCが進み学習が進んでいるように見えます。
(プログラムで画像を割る2して小さくしています。)

画像サイズ(横160縦120pix)や枚数(約700枚)が特に多くないと思っているのですが、なぜエラーが発生するのかと解決方法をご教授頂けないでしょうか。

以上、よろしくお願いいたします。

実装環境を以下に記します。
CPU:Xeon E5-1620v4 4core/8t
マザーボード：ASUS X99-E WS
メモリ：DDR4-2400 64GB(8G×8)
GPU:NVIDIA Quadro GP100 16GB 2個
OS:ubuntu16.04LTS

発生している問題・エラーメッセージ

ResourceExhaustedError (see above for traceback): OOM when allocating tensor with shape[28800,19200] and type float on /job:localhost/replica:0/task:0/device:GPU:0 by allocator GPU_0_bfc
	 [[Node: Variable_2/Adam/Assign = Assign[T=DT_FLOAT, _class=["loc:@Variable_2"], use_locking=true, validate_shape=true, _device="/job:localhost/replica:0/task:0/device:GPU:0"](Variable_2/Adam, zeros_1)]]
Hint: If you want to see a list of allocated tensors when OOM happens, add report_tensor_allocations_upon_oom to RunOptions for current allocation info.

該当のソースコード

python
1import numpy as np
2import tensorflow as tf
3from tensorflow.python.framework import ops
4import cv2
5import os
6
7DATASET_PATH = "/home/densos/workspaces/autoencoder"
8DIR_PATH = "input_gray_160*120"
9IMAGE_PATH = os.path.join(DATASET_PATH, DIR_PATH)
10X_PIXEL, Y_PIXEL = 160, 120
11M = 1
12N_HIDDENS = np.array(np.array([1.5]) * X_PIXEL * Y_PIXEL // (M*M), dtype = np.int)
13TRANCE_FRAME_NUM = 700
14
15ops.reset_default_graph()
16
17def xavier_init(fan_in, fan_out, constant = 1):
18    low = -constant * np.sqrt(6.0 / (fan_in + fan_out))
19    high = constant * np.sqrt(6.0 / (fan_in + fan_out))
20    return tf.random_uniform((fan_in, fan_out), minval = low, maxval = high, dtype = tf.float32)
21
22class AdditiveGaussianNoiseAutoencoder(object):
23    def __init__(self, n_input, n_hidden, transfer_function = tf.nn.sigmoid, optimizer = tf.train.AdamOptimizer(), scale = 0.1):
24        self.n_input = n_input
25        self.n_hidden = n_hidden
26        self.transfer = transfer_function
27        self.scale = tf.placeholder(tf.float32)
28        self.training_scale = scale
29        network_weights = self._initialize_weights()
30        self.weights = network_weights
31        self.sparsity_level = np.repeat([0.05], self.n_hidden).astype(np.float32)
32        self.sparse_reg = 0.1
33
34        # model
35        self.x = tf.placeholder(tf.float32, [None, self.n_input])
36        self.hidden = self.transfer(tf.add(tf.matmul(self.x + scale * tf.random_normal((n_input,)),
37                self.weights['w1']),
38                self.weights['b1']))
39        self.reconstruction = tf.add(tf.matmul(self.hidden, self.weights['w2']), self.weights['b2'])
40
41        # cost
42        self.cost = 0.5 * tf.reduce_sum(tf.pow(tf.subtract(self.reconstruction, self.x), 2.0)) + self.sparse_reg \
43                        * self.kl_divergence(self.sparsity_level, self.hidden)
44            
45        self.optimizer = optimizer.minimize(self.cost)
46
47        init = tf.global_variables_initializer()
48        self.sess = tf.Session()
49        self.sess.run(init)
50
51    def _initialize_weights(self):
52        all_weights = dict()
53        all_weights['w1'] = tf.Variable(xavier_init(self.n_input, self.n_hidden))
54        all_weights['b1'] = tf.Variable(tf.zeros([self.n_hidden], dtype = tf.float32))
55        all_weights['w2'] = tf.Variable(tf.zeros([self.n_hidden, self.n_input], dtype = tf.float32))
56        all_weights['b2'] = tf.Variable(tf.zeros([self.n_input], dtype = tf.float32))
57        return all_weights
58
59    def partial_fit(self, X):
60        cost, opt = self.sess.run((self.cost, self.optimizer), feed_dict = {self.x: X,
61                                                                            self.scale: self.training_scale
62                                                                            })
63        return cost
64
65    def kl_divergence(self, p, p_hat):
66        return tf.reduce_mean(p * tf.log(p) - p * tf.log(p_hat) + (1 - p) * tf.log(1 - p) - (1 - p) * tf.log(1 - p_hat))
67
68    def calc_total_cost(self, X):
69        return self.sess.run(self.cost, feed_dict = {self.x: X,
70                                                     self.scale: self.training_scale
71                                                     })
72
73    def transform(self, X):
74        return self.sess.run(self.hidden, feed_dict = {self.x: X,
75                                                       self.scale: self.training_scale
76                                                       })
77
78    def generate(self, hidden = None):
79        if hidden is None:
80            hidden = np.random.normal(size = self.weights["b1"])
81        return self.sess.run(self.reconstruction, feed_dict = {self.hidden: hidden})
82
83    def reconstruct(self, X):
84        return self.sess.run(self.reconstruction, feed_dict = {self.x: X,
85                                                               self.scale: self.training_scale
86                                                               })
87
88    def getWeights(self):
89        return self.sess.run(self.weights['w1'])
90
91    def getBiases(self):
92        return self.sess.run(self.weights['b1'])
93
94def get_random_block_from_data(data, batch_size):
95    start_index = np.random.randint(0, len(data) - batch_size)
96    return data[start_index:(start_index + batch_size)]
97
98
99if __name__ == '__main__':
100#get input data lists
101    lists = []
102    for file in os.listdir(IMAGE_PATH):
103        if file.endswith(".jpeg"):
104            lists.append(file)
105        lists.sort()
106
107#read input data    
108    input_images = []
109    for image in lists:
110        tmp = cv2.imread(os.path.join(IMAGE_PATH, image), cv2.IMREAD_GRAYSCALE)
111        tmp = cv2.resize(tmp, (X_PIXEL // M, Y_PIXEL // M))
112        tmp = tmp.reshape(tmp.shape[0] * tmp.shape[1])
113        input_images.append(tmp)
114
115#preprocess images    
116    input_images = np.array(input_images) / 255.
117
118#convert data to float16
119    input_images = np.array(input_images, dtype = np.float16)
120
121#set train and test data
122    X_train = input_images[:500]
123    X_test = input_images[500:]
124
125    n_samples = X_train.shape[0]
126    training_epochs = 200
127    batch_size = X_train.shape[0] // 4
128    display_step = 10
129
130    autoencoder = AdditiveGaussianNoiseAutoencoder(n_input = X_train.shape[1],
131                                                   n_hidden = N_HIDDENS[0],
132                                                   transfer_function = tf.nn.relu6,
133                                                   optimizer = tf.train.AdamOptimizer(learning_rate = 0.001),
134                                                   scale = 0.01)
135
136    for epoch in range(training_epochs):
137        avg_cost = 0.
138        total_batch = int(n_samples / batch_size)
139        # Loop over all batches
140        for i in range(total_batch):
141            batch_xs = get_random_block_from_data(X_train, batch_size)
142
143            # Fit training using batch data
144            cost = autoencoder.partial_fit(X_train)
145            # Compute average loss
146            avg_cost += cost / n_samples * batch_size
147
148        # Display logs per epoch step
149        if epoch % display_step == 0:
150            print("Epoch:", '%04d' % (epoch + 1), "cost=", avg_cost)
151
152    print("Finish Train")
153
154predicted_imgs = autoencoder.reconstruct(X_test)
155predicted_imgs = np.array((predicted_imgs) * 255, dtype = np.uint8)
156input_imgs = np.array((X_test) * 255, dtype = np.uint8)
157
158# plot the reconstructed images
159for i in range(100):
160    im1 = predicted_imgs[i].reshape((Y_PIXEL//M, X_PIXEL//M))
161    im2 = input_imgs[i].reshape((Y_PIXEL//M, X_PIXEL//M))
162
163    img_v_union = cv2.vconcat([im1, im2])
164    cv2.moveWindow('result.jpg', 100, 200)
165    cv2.imshow('result.jpg', img_v_union)
166
167    cv2.waitKey(33)