SENet(CNN)がanaconda上でエラーが出ずに停止してしまう問題について

Question

SENet(CNN)を実装し、caifar10のデータをサンプルに読み込ませました。訓練画像1万枚です。

エラーは出ずに実装上は問題ないようなのですが、sess.run()を実行すると、エラーが出ずに止まってしまいます。

ローカルで動かしているのですが、
１、SENet自体が層が深く計算量が多いから
または
２、実装に問題がある

のどちらかだと思うのですが、エラーが出ずに止まる理由として何が考えられるのでしょうか？

思い当たることでもいいので、ご教授お願いします。

import tensorflow as tf
import os
import math
import numpy as np
from tflearn.layers.conv import global_avg_pool
from tensorflow.contrib.layers import batch_norm, flatten
from tensorflow.contrib.framework import arg_scope

weight_decay = 0.0005
momentum = 0.9

init_learning_rate = 0.1
cardinality = 8 # how many split ?
blocks = 3 # res_block ! (split + transition)
depth = 64 # out channel

reduction_ratio = 4

batch_size = 128
iteration = 391
# 128 * 391 ~ 50,000

test_iteration = 10

training_flag = tf.placeholder(tf.bool)
learning_rate = tf.placeholder(tf.float32, name='learning_rate')


def conv_layer(input, filter, kernel, stride, padding='SAME', layer_name="conv"):
    with tf.name_scope(layer_name):
        network = tf.layers.conv2d(inputs=input, use_bias=False, filters=filter, kernel_size=kernel, strides=stride, padding=padding)
        return network

def Global_Average_Pooling(x):
    return global_avg_pool(x, name='Global_avg_pooling')

def Average_pooling(x, pool_size=[2,2], stride=2, padding='SAME'):
    return tf.layers.average_pooling2d(inputs=x, pool_size=pool_size, strides=stride, padding=padding)

def Fully_connected(x, units=9, layer_name='fully_connected') :
    with tf.name_scope(layer_name) :
        return tf.layers.dense(inputs=x, use_bias=False, units=units)
    
def Batch_Normalization(x):
    shape = x.get_shape().as_list()
    assert len(shape) in [2, 4]

    if len(shape) == 2:
        batch_mean, batch_var = tf.nn.moments(x, [0])
    else:
        batch_mean, batch_var = tf.nn.moments(x, [0, 1, 2])

    mean0, variance0 = tf.nn.moments(x, [0, 1, 2])
    return tf.nn.batch_normalization(x, mean0, variance0, None, None, 1e-5)


class SE_ResNet():
    def __init__(self, x, training):
        self.training = training
        self.model = self.Build_SEnet(x)
        
    def first_layer(self, x, scope):
        with tf.name_scope(scope) :
            x = conv_layer(x, filter=64, kernel=[3, 3], stride=1, layer_name=scope+'_conv1')
            x = Batch_Normalization(x)
            return tf.nn.relu(x)

    def transform_layer(self, x, stride, scope):
        with tf.name_scope(scope) :
            x = conv_layer(x, filter=depth, kernel=[1,1], stride=1, layer_name=scope+'_conv1')
            x = Batch_Normalization(x)
            x = tf.nn.relu(x)

            x = conv_layer(x, filter=depth, kernel=[3,3], stride=stride, layer_name=scope+'_conv2')
            x = Batch_Normalization(x)
            return tf.nn.relu(x)
        
        
    def transition_layer(self, x, out_dim, scope):
        with tf.name_scope(scope):
            x = conv_layer(x, filter=out_dim, kernel=[1,1], stride=1, layer_name=scope+'_conv1')
            return Batch_Normalization(x)

    def split_layer(self, input_x, stride, layer_name):
        with tf.name_scope(layer_name) :
            layers_split = list()
            for i in range(cardinality) :
                splits = self.transform_layer(input_x, stride=stride, scope=layer_name + '_splitN_' + str(i))
                layers_split.append(splits)

            return tf.concat(layers_split, axis=3)

    def squeeze_excitation_layer(self, input_x, out_dim, ratio, layer_name):
        with tf.name_scope(layer_name) :


            squeeze = Global_Average_Pooling(input_x)

            excitation = Fully_connected(squeeze, units=out_dim / ratio, layer_name=layer_name+'_fully_connected1')
            excitation = tf.nn.relu(excitation)
            excitation = Fully_connected(excitation, units=out_dim, layer_name=layer_name+'_fully_connected2')
            excitation = tf.nn.sigmoid(excitation)

            excitation = tf.reshape(excitation, [-1,1,1,out_dim])
            return input_x * excitation

    def residual_layer(self, input_x, out_dim, layer_num, res_block=blocks):
        # split + transform(bottleneck) + transition + merge
        # input_dim = input_x.get_shape().as_list()[-1]

        for i in range(res_block):
            input_dim = int(np.shape(input_x)[-1])

            if input_dim * 2 == out_dim:
                flag = True
                stride = 2
                channel = input_dim // 2
            else:
                flag = False
                stride = 1

            x = self.split_layer(input_x, stride=stride, layer_name='split_layer_'+layer_num+'_'+str(i))
            x = self.transition_layer(x, out_dim=out_dim, scope='trans_layer_'+layer_num+'_'+str(i))
            x = self.squeeze_excitation_layer(x, out_dim=out_dim, ratio=reduction_ratio, layer_name='squeeze_layer_'+layer_num+'_'+str(i))

            if flag is True :
                pad_input_x = Average_pooling(input_x)
                pad_input_x = tf.pad(pad_input_x, [[0, 0], [0, 0], [0, 0], [channel, channel]]) # [?, height, width, channel]
            else :
                pad_input_x = input_x

            input_x = tf.nn.relu(x + pad_input_x)

        return input_x


    def Build_SEnet(self, input_x):
        # only cifar10 architecture

        input_x = self.first_layer(input_x, scope='first_layer')

        x = self.residual_layer(input_x, out_dim=64, layer_num='1')
        x = self.residual_layer(x, out_dim=128, layer_num='2')
        x = self.residual_layer(x, out_dim=256, layer_num='3')

        x = Global_Average_Pooling(x)
        x = flatten(x)

        return Fully_connected(x, layer_name='final_fully_connected')
    


def loss(logits, labels):
    labels = tf.cast(labels, tf.int64)
    cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits, labels=labels, name='cross_entropy_per_example')
    cross_entropy_mean = tf.reduce_mean(cross_entropy, name='cross_entropy__raw_')
    tf.add_to_collection('losses', cross_entropy_mean)
    # The total loss is defined as the cross entropy loss plus all of the weight(L2 loss)
    return tf.add_n(tf.get_collection('losses'), name='total_loss')


# In[4]: # train using AdamOptimizer


def train(total_loss, global_step):
    
    loss_averages = tf.train.ExponentialMovingAverage(0.9, name='avg')
    losses = tf.get_collection('losses')
    loss_averages_op = loss_averages.apply(losses + [total_loss])
    
        
    # Compute gradients.
    with tf.control_dependencies([loss_averages_op]):
        opt = tf.train.AdamOptimizer() # instead of tf.train.AdamOptimizer()
        grads = opt.compute_gradients(total_loss)

    # Apply gradients.
    apply_gradient_op = opt.apply_gradients(grads, global_step=global_step)
    for var in tf.trainable_variables():
        tf.summary.histogram(var.op.name, var)
        
    for grad, var in grads:
        if grad is not None:
            tf.summary.histogram(var.op.name + '/gradients', grad)

    # Track the moving averages of all trainable variables.
    variable_averages = tf.train.ExponentialMovingAverage(0.9999, global_step)
    variables_averages_op = variable_averages.apply(tf.trainable_variables())

    with tf.control_dependencies([apply_gradient_op, variables_averages_op]):
        train_op = tf.no_op(name='train')

    return train_op # Op for training.


# 実行
with tf.Graph().as_default():
    global_step = tf.contrib.framework.get_or_create_global_step()
    # for train
    train_image, train_label = distorted_inputs('/Users/Downloads/train.tfrecords',128) # データ読み込み関数で画像とラベルを返す。

    c_logits = SE_ResNet(train_image, training=training_flag).model
    loss = loss(c_logits, train_label)
    train_op = train(loss, global_step)
    
    class _LoggerHook(tf.train.SessionRunHook):
        def begin(self):
            self._step = -1
            self._start_time = time.time()

        def before_run(self, run_context):
            self._step += 1
            return tf.train.SessionRunArgs(loss)  # Asks for loss value.

        def after_run(self, run_context, run_values):
            if self._step % 10 == 0:
                current_time = time.time()
                duration = current_time - self._start_time
                self._start_time = current_time

                loss_value = run_values.results
                examples_per_sec = 10 * 128 / duration
                sec_per_batch = float(duration / 10)

                format_str = ('%s: step %d, loss = %.2f (%.1f examples/sec; %.3f '
                        'sec/batch)')
                print (format_str % (datetime.now(), self._step, loss_value,
                               examples_per_sec, sec_per_batch))
            with tf.train.MonitoredTrainingSession(checkpoint_dir='/Users/Downloads/senet_dir',
                                           hooks=[tf.train.StopAtStepHook(last_step=4000),
               tf.train.NanTensorHook(loss), _LoggerHook()], config=tf.ConfigProto(log_device_placement=False)) as mon_sess:
                while not mon_sess.should_stop():
                    mon_sess.run(train_op)


>>>エラーが出ずに止まる
Filling queue with 4000 CIFAR images before starting to train. This will take a few minutes. (データ読み込み関数に書いたメッセージ)

Comments

[mkgrei]

とりあえず停止している時間によるかと思います。メモリーが足りないと環境によってはスワップが発生してより一層処理が遅くなります。

[trafalbad]

実は重い時と違って止まり方が特殊で重いときは左のワクの中が、アスタリスクマークになったままなんですが、今回はすぐに処理完了の数字になるんです。コード見て思ったのですが、monsessのとこで定義してあるtime()を書いてないからスタートしてないだけなのでしょうか？

[mkgrei]

distorted_inputsでロードしていますが、この関数の中身が載せられていません。どのような処理をしているのでしょうか。

[trafalbad]

このURLに乗ってるのと同様の内容です https://github.com/bakuratozoku/DCGANs/blob/master/VGGnet_CNN.py

Answer 1

最後の実行を行うwith文のインデントが_LoggerHookのメソッドの中に入ってしまっているので、実行されていません。

---

実行されるとリソースエラーが出ます。
バッチ処理が実装されていないように見えます。

Comments

[trafalbad]

ありがとうございます！