cifar-10で90%到達する方法。

Question

今TensorFlowを使ってCifar-10の識別率を90%に到達させようとしています。
今はResNetを使っているのですがどうもうまくいきません。
良くて86%といった感じでそれ以上は望めないという状況です。
どこに着目してモデルを改善していくべきなのかわかりません。
もしわかる方がいれば教えてください。
ResNetは２層ごとに２層前に出力を合計していくもので、
8層ごとにフィルターのサイズを２倍ずつにしました。

フィルターの数	フィルターのサイズ	層の数
16	3×3	1
16	3×3	12
32	3×3	12
64	3×3	12
128	3×3	12

画像の前処理は画像の左右を反転させるものと明るさとコントラストをランダムに変更してから標準化しています。
以下、コードになります。
文字数の関係でNumPyの配列に画像データを落とし込むところはカットしました。

import tensorflow as tf
import numpy as np
import pickle
import time

dtype = 'float64'

dtype2 = tf.float32
x = tf.placeholder(dtype2, shape = [None, 32, 32, 3], name='x')
with tf.name_scope('Const'):
    keep_prob = tf.placeholder(dtype2, name='keep_prob') # CNN内でのドロップアウト率
with tf.name_scope('Input'):
    mm = tf.placeholder(shape = [32, 32, 3], dtype=dtype2, name = 'input')
    distorted_image = tf.image.random_flip_left_right(mm)
    distorted_image = tf.image.random_brightness(distorted_image,
                                                 max_delta=63)
    distorted_image = tf.image.random_contrast(distorted_image,lower=0.2, upper=1.8)
    
    float_image = tf.image.per_image_standardization(distorted_image)


with tf.name_scope('Input_for_Test'):
    test_image = tf.image.per_image_standardization(mm)

def weight_variable(shape, name=None):
    initial = tf.truncated_normal(shape,stddev=0.1, dtype=dtype2)
    return tf.Variable(initial, name=name)

def softmax_layer(inpt, shape):
    fc_w = weight_variable(shape)
    fc_b = tf.Variable(tf.constant(0.1, shape=[shape[1]]))
    fc_h = tf.nn.softmax(tf.matmul(inpt, fc_w) + fc_b)

    return fc_h
def relu_layer(inpt, shape):
    fc_w_1 = weight_variable(shape)
    fc_b_1 = tf.Variable(tf.constant(0.1, shape=[shape[1]]))

    fc_h_1 = tf.nn.relu(tf.matmul(inpt, fc_w_1) + fc_b_1)

    return fc_h_1
def conv_layer(inpt, filter_shape, stride):
    out_channels = filter_shape[3]

    filter_ = weight_variable(filter_shape)
    conv = tf.nn.conv2d(inpt, filter=filter_, strides=[1, stride, stride, 1], padding='SAME')
    mean, var = tf.nn.moments(conv, axes=[0,1,2])
    beta = tf.Variable(tf.zeros([out_channels], name="beta"))
    gamma = weight_variable([out_channels], name="gamma")

    batch_norm = tf.nn.batch_norm_with_global_normalization(
        conv, mean, var, beta, gamma, 0.001,
        scale_after_normalization=True)
    return batch_norm

def residual_block(inpt, output_depth, down_sample, projection=False):
    input_depth = inpt.get_shape().as_list()[3]
    if down_sample:
        filter_ = [1,2,2,1]
        inpt = tf.nn.max_pool(inpt, ksize=filter_, strides=filter_, padding='SAME')

    conv1 = conv_layer(inpt, [3,3, input_depth, output_depth], 1)
    conv2 = conv_layer(conv1, [3,3, output_depth, output_depth],1)

    if input_depth != output_depth:
        if projection:
            input_layer = conv_layer(inpt, [1,1, input_depth, output_depth], 2)
        else:
            input_layer = tf.pad(inpt, [[0,0], [0,0],[0,0],[0, output_depth-input_depth]])

    else:
        input_layer = inpt

    res = conv2 + input_layer
    out = tf.nn.relu(res)
    return out


def resnet(inpt, n):
    num_conv = (n-1)//16 + 1
    layers = []

    with tf.variable_scope('conv1'):
        conv1 = conv_layer(inpt, [3,3,3,8], 1)
        layers.append(conv1)

    for i in range(num_conv):
        with tf.variable_scope('conv2_%d' % (i+1)):
            conv2_x = residual_block(layers[-1], 8, False)
            conv2 = residual_block(conv2_x, 8, False)
            conv2_dropout = tf.nn.dropout(conv2, keep_prob=keep_prob)
            layers.append(conv2_x)
            layers.append(conv2)
            layers.append(conv2_dropout)

        assert conv2.get_shape().as_list()[1:] == [32, 32, 8]

    for i in range (num_conv):
        down_sample = True if i==0 else False
        with tf.variable_scope('conv3_%d' % (i+1)):
            conv3_x = residual_block(layers[-1], 16, down_sample)
            conv3 = residual_block(conv3_x, 16, False)
            conv3_dropout = tf.nn.dropout(conv3, keep_prob=keep_prob)
            layers.append(conv3_x)
            layers.append(conv3)
            layers.append(conv3_dropout)

        assert conv3.get_shape().as_list()[1:] == [16, 16, 16]

    for i in range(num_conv):
        down_sample = True if i==0 else False
        with tf.variable_scope('conv4_%d' % (i+1)):
            conv4_x = residual_block(layers[-1], 32, down_sample)
            conv4 = residual_block(conv4_x, 32, False)
            conv4_dropout = tf.nn.dropout(conv4, keep_prob = keep_prob)
            layers.append(conv4_x)
            layers.append(conv4)
            layers.append(conv4_dropout)
        print(conv4.get_shape())
        assert conv4.get_shape().as_list()[1:] == [8, 8, 32]
    for i in range(num_conv):
        down_sample = True if i==0 else False
        with tf.variable_scope('conv5_%d' % (i+1)):
            conv5_x = residual_block(layers[-1], 64, down_sample)
            conv5 = residual_block(conv5_x, 64, False)
            conv5_dropout = tf.nn.dropout(conv5, keep_prob = keep_prob)
            layers.append(conv5_x)
            layers.append(conv5)
            layers.append(conv5_dropout)
        print(conv5.get_shape())
        assert conv5.get_shape().as_list()[1:] == [4, 4, 64]

    with tf.variable_scope('fc'):
        global_pool = tf.reduce_mean(layers[-1], [1,2])
        assert global_pool.get_shape().as_list()[1:] == [64]
        conv6 = conv_layer(inpt, [1,1,3,64], 1)
        relu_conv6 = tf.nn.relu(conv6)
        global_pool2 = tf.reduce_mean(relu_conv6, [1, 2])
        global_pool3 = tf.add(global_pool, global_pool2)
        hidden = relu_layer(global_pool3, [64, 1000])
        out = softmax_layer(hidden, [1000, 10])
        layers.append(out)

    return layers[-1]

p = resnet(inpt=x, n=49)
with tf.name_scope('Loss'):
        t = tf.placeholder(dtype2, [None, 10], name='labels')
        loss = -tf.reduce_sum(t * tf.log(tf.clip_by_value(p, 1e-10, 1.0)), name='loss')

with tf.name_scope('Train'):
        learning_rate = tf.placeholder(dtype2)
        train_step = tf.train.MomentumOptimizer(learning_rate,momentum=0.85).minimize(loss)

with tf.name_scope('Accuracy'):
        correct_prediction = tf.equal(tf.argmax(p, 1), tf.argmax(t, 1))
        accuracy = tf.reduce_mean(tf.cast(correct_prediction, dtype=dtype2), name='accuracy')

sess = tf.Session()
sess.run(tf.global_variables_initializer())
loss_list  = []
test_acc_list = []
test_loss_list = []
i = 0
saver = tf.train.Saver()
batch_size = 25
height = width = 32
rate = 0.00004
decay = np.exp(1/80000)
start_time = time.time()
max_epoch = 150
training_time = 50000//batch_size

for epoch in range(max_epoch):
    for i in range(training_time):
        ra = np.random.randint(50000, size= batch_size)
        batch_xs, batch_ts = X1[ra], Y_train[ra]
        batch_xs2 = np.empty([batch_size, height, width, 3], dtype='float32')

        for k in range(batch_size):
            batch_xs2[k] = sess.run(float_image, feed_dict={mm: batch_xs[k]})

        sess.run(train_step, feed_dict = {x: batch_xs2,t:batch_ts, learning_rate: rate,keep_prob:0.95})
        if epoch==0 and i%50==0:
            print("Training step is %d" % (i))
    if epoch == 100 or epoch == 130:
        rate /= 10
        print("Learning rate turned to be %e" % (rate))
    if epoch >= 0:
        loss_val = np.empty(400)
        acc_val = np.empty(400)
        for k in range(400):
            X_test2 = np.empty((25, 32, 32, 3))
            for l in range(25):
                X_test2[l] = sess.run(test_image, feed_dict={mm:X1_test[(25*k)+l]})
            loss_val[k], acc_val[k] = sess.run([loss, accuracy],
                            feed_dict={x: X_test2,
                            t: Y_test2[25*k:25*(k+1)], keep_prob:1.0})
        loss_val = np.sum(loss_val)
        acc_val = np.mean(acc_val)
        test_loss_list.append(loss_val)
        test_acc_list.append(acc_val)
        print('Epoch: %d, Loss: %f, Accuracy: %f'
                        % (epoch, loss_val, acc_val))
    if epoch % 5 == 0:
        saver = tf.train.Saver()
        saver.save(sess, "0119ver10_resnet49_normalize_set_0.0015_%d.ckpt" %
                            (epoch))

    if epoch%5==0:
        loss_val = np.empty(2000)
        acc_val = np.empty(2000)
        for k in range(2000):
            X_test2 = np.empty((25, 32, 32, 3))
            for l in range(25):
                X_test2[l] = sess.run(test_image, feed_dict={mm:X1[(25*k)+l]})
            loss_val[k], acc_val[k] = sess.run([loss, accuracy],
                                        feed_dict={x: X_test2,
                                                t: Y_train[25*k:25*(k+1)], keep_prob:1.0})
        loss_val = np.sum(loss_val)
        acc_val = np.mean(acc_val)
        print('Train data --','Epoch: %d, Loss: %f, Accuracy: %f'
                            % (epoch, loss_val, acc_val))
  
    

Comments

[wakame]

目的はタイトル通りcifar10で精度90%を達成することですか？でしたら既に達成しているモデルがあるのでそちらをダウンロードして使えばよいと思います。

[Sunshine96]

その通りです。Tensorflowを用いたものがなかなか見つからず、苦労しています。私の調べ不足なのは重々承知しているのですが実際にTensorFlowで達成したTensorFlowで記述されたモデルのコードを知っているのなら教えていただけると幸いです。

Answer 1

とりあえずこんな感じでしょうか

Tensorflow

Reproduced ResNet on CIFAR-10 and CIFAR-100 dataset.

Caffe

ResNets experiments on cifar10 with caffe

Chainer

Various CNN models including for CIFAR10 with Chainer

PyTorch

Train CIFAR10 with PyTorch

Answer 2

https://qiita.com/dsanno/items/ad84f078520f9c9c3ed1

http://tensorflow.classcat.com/category/cifar-10/

https://github.com/kuangliu/pytorch-cifar

https://twitter.com/aiskoaskosd/status/894074415461179392
https://qiita.com/aiskoaskosd/items/59c49f2e2a6d76d62798

https://qiita.com/takedarts/items/fc6f6e96f2d0b7b55630