分類モデルでバリデーションの精度が上昇しない

前提・実現したいこと

testデータではEpochが増えるにしたがって損失は減少し、精度は向上しているのですが
バリデーションはEpochが増えるにしたがって損失は上昇し、精度低下していってしまいます。
どのようなことが原因としてこのようなことが起こるのか教えていただきたいです。

発生している問題・エラーメッセージ

4800/4800 [==============================] - 19s 4ms/step - loss: 1.0837 - acc: 0.4135 - val_loss: 1.0239 - val_acc: 0.4917
Epoch 2/20
4800/4800 [==============================] - 17s 3ms/step - loss: 0.9937 - acc: 0.4940 - val_loss: 1.0219 - val_acc: 0.4917
Epoch 3/20
4800/4800 [==============================] - 18s 4ms/step - loss: 0.9191 - acc: 0.5535 - val_loss: 1.3023 - val_acc: 0.4500
Epoch 4/20
4800/4800 [==============================] - 18s 4ms/step - loss: 0.8838 - acc: 0.5988 - val_loss: 1.1906 - val_acc: 0.4667
Epoch 5/20
4800/4800 [==============================] - 18s 4ms/step - loss: 0.8181 - acc: 0.6275 - val_loss: 1.1910 - val_acc: 0.5083
Epoch 6/20
4800/4800 [==============================] - 18s 4ms/step - loss: 0.7781 - acc: 0.6542 - val_loss: 1.0617 - val_acc: 0.5667
Epoch 7/20
4800/4800 [==============================] - 17s 4ms/step - loss: 0.7222 - acc: 0.6852 - val_loss: 1.4210 - val_acc: 0.5000
Epoch 8/20
4800/4800 [==============================] - 18s 4ms/step - loss: 0.6638 - acc: 0.7131 - val_loss: 1.5748 - val_acc: 0.5167
Epoch 9/20
4800/4800 [==============================] - 17s 4ms/step - loss: 0.6089 - acc: 0.7379 - val_loss: 1.2671 - val_acc: 0.5417
Epoch 10/20
4800/4800 [==============================] - 16s 3ms/step - loss: 0.5622 - acc: 0.7712 - val_loss: 1.7027 - val_acc: 0.5000
Epoch 11/20
4800/4800 [==============================] - 16s 3ms/step - loss: 0.5092 - acc: 0.7892 - val_loss: 1.5143 - val_acc: 0.5333
Epoch 12/20
4800/4800 [==============================] - 17s 4ms/step - loss: 0.4602 - acc: 0.8163 - val_loss: 1.3911 - val_acc: 0.5917
Epoch 13/20
4800/4800 [==============================] - 17s 3ms/step - loss: 0.4247 - acc: 0.8298 - val_loss: 2.4402 - val_acc: 0.5083
Epoch 14/20
4800/4800 [==============================] - 17s 3ms/step - loss: 0.4087 - acc: 0.8413 - val_loss: 1.8062 - val_acc: 0.5417
Epoch 15/20
4800/4800 [==============================] - 16s 3ms/step - loss: 0.3773 - acc: 0.8483 - val_loss: 2.1479 - val_acc: 0.5417
Epoch 16/20
4800/4800 [==============================] - 19s 4ms/step - loss: 0.3539 - acc: 0.8715 - val_loss: 2.1562 - val_acc: 0.5667
Epoch 17/20
4800/4800 [==============================] - 22s 5ms/step - loss: 0.3081 - acc: 0.8869 - val_loss: 1.8828 - val_acc: 0.5667
Epoch 18/20
4800/4800 [==============================] - 17s 4ms/step - loss: 0.2920 - acc: 0.8865 - val_loss: 2.0210 - val_acc: 0.5500
Epoch 19/20
4800/4800 [==============================] - 17s 3ms/step - loss: 0.2882 - acc: 0.8985 - val_loss: 2.0693 - val_acc: 0.5333
Epoch 20/20
4800/4800 [==============================] - 17s 3ms/step - loss: 0.2591 - acc: 0.9033 - val_loss: 3.1138 - val_acc: 0.4917

該当のソースコード

python3
1import keras
2from keras.models import  Sequential
3from keras.layers import Dense,Dropout,Flatten
4from keras.layers import Conv2D,MaxPooling2D
5from keras.optimizers import  RMSprop
6
7import cnn_model
8import matplotlib.pyplot as plt
9import numpy as np
10from sklearn.model_selection import train_test_split
11
12from keras.preprocessing.image import ImageDataGenerator, array_to_img, img_to_array, load_img
13
14
15import pandas as pd
16import seaborn as sns
17from sklearn.metrics import confusion_matrix
18import matplotlib.pyplot as plt
19
20#cnnのモデルを定義する
21
22def def_model(in_shape,nb_classes):
23    model=Sequential()
24    model.add(Conv2D(32,kernel_size=(3,3),activation="relu",input_shape=in_shape))
25    model.add(Conv2D(32,(3,3),activation="relu"))
26    model.add(MaxPooling2D(pool_size=(2,2)))
27    model.add(Dropout(0.25))
28    
29    model.add(Conv2D(64,(3,3),activation="relu"))
30    model.add(Conv2D(64,(3,3),activation="relu",name="relu_conv2"))
31   #最後の畳み込み層の名称を後ほど使うので定義しておく。
32    model.add(MaxPooling2D(pool_size=(2,2)))
33    model.add(Dropout(0.25))
34    
35    model.add(Flatten())
36    model.add(Dense(512,activation="relu"))
37    model.add(Dropout(0.5))
38    model.add(Dense(nb_classes,activation="softmax"))
39    
40    return model
41
42#コンパイル済みのcnnのモデルを返す
43
44def get_model(in_shape,nb_classes):
45    model=def_model(in_shape,nb_classes)
46    model.compile(
47        loss="categorical_crossentropy",
48        optimizer=RMSprop(),
49        metrics=["accuracy"])
50    return model
51
52datagen = ImageDataGenerator(
53        rotation_range=45,
54        vertical_flip=True,
55        horizontal_flip=True,
56    fill_mode='nearest')
57
58def images_gen(x_list,y_list):
59    x_list_add=[]
60    y_list_add=[]
61    for x ,y in zip(x_list,y_list):#xは(3, 150, 150)で受け取る
62        x = x.reshape((1,) + x.shape)  #(1, 3, 150, 150)に変換する
63
64        batch_list=[]
65        i = 0
66        for batch in datagen.flow(x, batch_size=1):
67            batch=batch.astype(np.uint8)#データ型を揃える！！
68            batch=batch.reshape((32, 32, 3))
69            x_list_add.append(batch)
70            y_list_add.append(y)
71            i += 1
72            if i > 9:#１枚から10枚作る
73                break             
74    x_np_add=np.array(x_list_add)
75    y_np_add=np.array(y_list_add)
76
77            
78    return x_np_add,y_np_add
79
80
81#入力と出力を指定
82im_rows=32
83im_cols=32
84im_color=3
85in_shape=(im_rows,im_cols,im_color)
86nb_classes=3
87
88#写真データを読み込み
89photos=np.load("./photos_hanbetu1.npz")
90x=photos["x"]
91y=photos["y"]
92
93#読み込んだデータを三次元配列に変換
94x=x.reshape(-1,im_rows,im_cols,im_color)
95
96#学習用とテスト用に分ける
97x_train,x_test,y_train,y_test=train_test_split(x,y,train_size=0.8)
98
99#ここで水増しする！！
100x_train_add,y_train_add=images_gen(x_train,y_train)
101x_train_add=x_train_add.astype("float32")/255
102x_test=x_test.astype("float32")/255
103
104#ラベルデータをone-hotベクトルに直す
105y_train_add=keras.utils.np_utils.to_categorical(y_train_add.astype("int32"),nb_classes)
106y_test=keras.utils.np_utils.to_categorical(y_test.astype("int32"),nb_classes)
107
108#cnnモデルを取得
109model=cnn_model.get_model(in_shape,nb_classes)
110
111#モデルを評価
112
113
114def print_cmx(y_true, y_pred):
115    labels = sorted(list(set(y_true)))
116    cmx_data = confusion_matrix(y_true, y_pred, labels=labels)
117    labels= ["a","b","c"]#ラベルを付け加える
118    df_cmx = pd.DataFrame(cmx_data, index=labels, columns=labels)
119
120    plt.figure(figsize = (10,7))
121    sns.heatmap(df_cmx, annot=True)
122    plt.xlabel("Predict-labels")
123    plt.ylabel("True-labels")
124
125    plt.show()
126
127predict_classes = model.predict_classes(x_test, batch_size=32)
128true_classes = np.argmax(y_test,1)
129print_cmx(true_classes,predict_classes)