損失関数が下がらない

実現したいこと

y=sin(x)を[0,2π]の区間で学習する

前提

Pythonでy=sin(x)を学習するプログラムを作っています
データ数は20、それを正規化し±0.001%の誤差を加えて3倍のデータ拡張を行っています

発生している問題・エラーメッセージ

valの損失関数がどうしても下がらないです

該当のソースコード

Pyhon
1import numpy as np
2import matplotlib.pyplot as plt
3from tqdm.auto import tqdm
4from collections import OrderedDict
5
6class Adam:
7
8    def __init__(self, lr=0.01, beta1=0.9, beta2=0.999):
9        self.lr = lr
10        self.beta1 = beta1
11        self.beta2 = beta2
12        self.iter = 0
13        self.m = None
14        self.v = None
15        
16    def update(self, params, grads):
17        if self.m is None:
18            self.m, self.v = {}, {}
19            for key, val in params.items():
20                self.m[key] = np.zeros_like(val)
21                self.v[key] = np.zeros_like(val)
22        
23        self.iter += 1
24        lr_t  = self.lr * np.sqrt(1.0 - self.beta2**self.iter) / (1.0 - self.beta1**self.iter)         
25        
26        for key in params.keys():
27            #self.m[key] = self.beta1*self.m[key] + (1-self.beta1)*grads[key]
28            #self.v[key] = self.beta2*self.v[key] + (1-self.beta2)*(grads[key]**2)
29            self.m[key] += (1 - self.beta1) * (grads[key] - self.m[key])
30            self.v[key] += (1 - self.beta2) * (grads[key]**2 - self.v[key])
31            
32            params[key] -= lr_t * self.m[key] / (np.sqrt(self.v[key]) + 1e-7)
33
34            
35class Affine:
36    def __init__(self, W, b):
37        self.W = W
38        self.b = b
39        
40        self.x = None
41        self.original_x_shape = None
42        # 重み・バイアスパラメータの微分
43        self.dW = None
44        self.db = None
45
46    def forward(self, x):
47        # テンソル対応
48        self.original_x_shape = x.shape
49        x = x.reshape(x.shape[0], -1)
50        self.x = x
51        out = np.dot(self.x, self.W) + self.b
52
53        return out
54
55    def backward(self, dout):
56        dx = np.dot(dout, self.W.T)
57        self.dW = np.dot(self.x.T, dout)
58        self.db = np.sum(dout, axis=0)
59        dx = dx.reshape(*self.original_x_shape)  # 入力データの形状に戻す（テンソル対応）
60        
61        return dx
62
63class Relu:
64    def __init__(self):
65        self.mask = None
66
67    def forward(self, x):
68        self.mask = (x <= 0)
69        out = x.copy()
70        out[self.mask] = 0
71        
72        return out
73
74    def backward(self, dout):
75        dout[self.mask] = 0
76        dx = dout
77        
78        return dx
79    
80class Mseloss:
81    def __init__(self):
82        self.loss = None
83        self.y = None 
84        self.t = None #target
85
86    def forward(self, x, t):
87        self.t = t
88        self.y = x
89        self.loss = sum_squared_error(self.y, self.t)
90        
91        return self.loss
92
93    def backward(self, dout=1):
94        batch_size = self.t.shape[0]
95        if self.t.size == self.y.size: #targetデータがone-hot-vectorの場合
96            dx = (self.y - self.t) / batch_size
97        else:
98            dx = self.y.copy()
99            dx[np.arange(batch_size), self.t] -= 1
100            dx = dx / batch_size
101        
102        return dx
103    
104    
105class TwoLayerNet:
106
107    def __init__(self, input_size, hidden_size, output_size, weight_init_std = 0.01):
108        # 重みの初期化
109        self.params = {}
110        self.params['W1'] = weight_init_std * np.random.randn(input_size, hidden_size)
111        self.params['b1'] = np.zeros(hidden_size)
112        self.params['W2'] = weight_init_std * np.random.randn(hidden_size, hidden_size) 
113        self.params['b2'] = np.zeros(hidden_size)
114        self.params['W3'] = weight_init_std * np.random.randn(hidden_size, hidden_size)
115        self.params['b3'] = np.zeros(hidden_size)
116        self.params['W4'] = weight_init_std * np.random.randn(hidden_size, output_size)
117        self.params['b4'] = np.zeros(output_size)
118
119        # レイヤの生成
120        self.layers = OrderedDict()
121        self.layers['Affine1'] = Affine(self.params['W1'], self.params['b1'])
122        self.layers['activation function1'] = Relu()
123        self.layers['Affine2'] = Affine(self.params['W2'], self.params['b2'])
124        self.layers['activation function2'] = Relu()
125        self.layers['Affine3'] = Affine(self.params['W3'], self.params['b3'])
126        self.layers['activation function3'] = Relu()
127        self.layers['Affine4'] = Affine(self.params['W4'], self.params['b4'])
128        self.lastLayer = Mseloss()
129        
130    def predict(self, x):
131        for layer in self.layers.values():
132            x = layer.forward(x)
133        
134        return x
135        
136    # x:入力データ, t:target
137    def loss(self, x, t):
138        y = self.predict(x)
139        return self.lastLayer.forward(y, t)
140    
141
142    def gradient(self, x, t):
143        # forward
144        self.loss(x, t)
145
146        # backward
147        dout = 1
148        dout = self.lastLayer.backward(dout)
149        
150        layers = list(self.layers.values())
151        layers.reverse()
152        for layer in layers:
153            dout = layer.backward(dout)
154
155        # 設定
156        grads = {}
157        grads['W1'], grads['b1'] = self.layers['Affine1'].dW, self.layers['Affine1'].db
158        grads['W2'], grads['b2'] = self.layers['Affine2'].dW, self.layers['Affine2'].db
159        grads['W3'], grads['b3'] = self.layers['Affine3'].dW, self.layers['Affine3'].db
160        grads['W4'], grads['b4'] = self.layers['Affine4'].dW, self.layers['Affine4'].db
161
162        return grads
163
164def sum_squared_error(y, t):
165    return 0.5 * np.sum((y-t)**2)
166
167def min_max(x,x_min,x_max,axis=None):
168    x_normalized = (x-x_min)/(x_max-x_min)
169    return x_normalized
170
171def re_min_max(x_normalized,x_min,x_max,axis=None):
172    re_normalized = x_normalized*(x_max-x_min)+x_min
173    return re_normalized
174
175x = np.random.uniform(0,2*np.pi, 20)
176
177#データの分割
178x_train = x[0:int(len(x)*0.6)]
179x_val   = x[int(len(x)*0.6):int(len(x)*0.9)]
180x_test  = x[int(len(x)*0.9):int(len(x))]
181
182#最小・最大の獲得
183x_train_min = x_train.min()
184x_train_max = x_train.max()
185
186x_val_min = x_val.min()
187x_val_max = x_val.max()
188
189x_test_min = x_test.min()
190x_test_max = x_test.max()
191
192#教師データの作成
193t_train = np.sin(x_train)
194t_val = np.sin(x_val)
195t_test = np.sin(x_test)
196
197t_train_copy1 = np.copy(t_train)
198t_train_copy2 = np.copy(t_train)
199t_train = np.concatenate([t_train,t_train_copy1,t_train_copy2])
200
201t_val_copy1 = np.copy(t_val)
202t_val_copy2 = np.copy(t_val)
203t_val = np.concatenate([t_val,t_val_copy1,t_val_copy2])
204
205t_test_copy1 = np.copy(t_test)
206t_test_copy2 = np.copy(t_test)
207t_test = np.concatenate([t_test,t_test_copy1,t_test_copy2])
208
209#入力データの正規化
210x_train_normalized = min_max(x_train,x_train_min,x_train_max)
211x_val_normalized = min_max(x_val,x_val_min,x_val_max)
212x_test_normalized = min_max(x_test,x_test_min,x_test_max)
213
214#誤差を加える(""変えるとこ"")
215gosaritsu = 0.001
216for i in range(x_train.shape[0]):
217    z = np.random.uniform([0],[gosaritsu],[1])
218    x_train_plus_error = x_train_normalized+z
219    x_train_minus_error = x_train_normalized-z
220    
221    z = np.random.uniform([0],[gosaritsu],[1])
222    x_val_plus_error = x_val_normalized+z
223    x_val_minus_error = x_val_normalized-z
224    
225    z = np.random.uniform([0],[gosaritsu],[1])
226    x_test_plus_error = x_test_normalized+z
227    x_test_minus_error = x_test_normalized-z
228    
229#入力データの連結
230x_train = np.concatenate([x_train_normalized,x_train_plus_error,x_train_minus_error])
231
232x_val = np.concatenate([x_val_normalized,x_val_plus_error,x_val_minus_error])
233
234x_test = np.concatenate([x_test_normalized,x_test_plus_error,x_test_minus_error])
235
236train_size = x_train.shape[0]
237val_size = x_val.shape[0]
238test_size = x_test.shape[0]
239
240train_loss_list = []
241val_loss_list = []
242test_loss_list = []
243
244network = TwoLayerNet(input_size=1, hidden_size=40, output_size=1)
245iters_num = 6000 # 繰り返しの回数を適宜設定する
246data_divs = 5
247batch_size = int(len(x_train) / data_divs) # 実際は2^nにするのが普通
248optimizer = Adam()
249
250for i in tqdm(range(iters_num)):
251  
252    batch_mask_train = np.random.choice(train_size, batch_size)
253    batch_mask_val = np.random.choice(val_size, batch_size)
254    batch_mask_test = np.random.choice(test_size, batch_size)
255    
256    x_batch_train = x_train[batch_mask_train]
257    x_batch_val = x_val[batch_mask_val]
258    x_batch_test = x_test[batch_mask_test]
259    
260    x_batch_train = np.reshape(x_batch_train, (batch_size,1))
261    x_batch_val = np.reshape(x_batch_val, (batch_size,1))
262    x_batch_test = np.reshape(x_batch_test, (batch_size,1))
263    
264    
265    t_batch_train = t_train[batch_mask_train]
266    t_batch_train = np.reshape(t_batch_train, (batch_size,1))
267    
268    t_batch_val = t_val[batch_mask_val]
269    t_batch_val = np.reshape(t_batch_val, (batch_size,1))
270    
271    t_batch_test = t_test[batch_mask_test]
272    t_batch_test = np.reshape(t_batch_test, (batch_size,1))
273    
274    # 勾配の計算
275    grad = network.gradient(x_batch_train, t_batch_train)
276    
277    # パラメータの更新
278    params = network.params
279    optimizer.update(params, grad)
280    
281    loss_train = network.loss(x_batch_train, t_batch_train)
282    loss_val = network.loss(x_batch_val, t_batch_val)
283    loss_test = network.loss(x_batch_test, t_batch_test)
284    
285    train_loss_list.append(loss_train)
286    val_loss_list.append(loss_val)
287    test_loss_list.append(loss_test)
288    
289pre_train = network.predict(x_train)
290pre_val = network.predict(x_val)
291pre_test= network.predict(x_test)
292
293re_x_train = re_min_max(x_train,x_train_min,x_train_max)
294re_x_val = re_min_max(x_val,x_val_min,x_val_max)
295re_x_test = re_min_max(x_test,x_test_min,x_test_max)
296
297X = np.arange(0,2*np.pi, 0.01)
298Y = np.sin(X)
299
300plt.plot(re_x_train,pre_train,marker='.',ls='',color="red", label='pre_train')
301plt.plot(re_x_val,pre_val,marker='.',ls='',color="green", label='pre_val')
302plt.plot(X,Y,marker='',ls='-',color="blue", label='x=^2')
303plt.grid()
304plt.xlabel('x',fontsize='16')
305plt.ylabel('y_pred',fontsize='16')
306plt.legend(edgecolor='k',fontsize='16')
307plt.show()
308plt.close()
309
310plt.plot(train_loss_list,color="blue",label="train_loss")
311plt.plot(val_loss_list,color="red",label="val_loss")
312plt.yscale('log')
313plt.grid()
314plt.legend(edgecolor='k',fontsize='16')
315plt.show()
316plt.close()