teratail
回答率
85.35%
teratail header banner
teratail header banner
質問するログイン新規登録
トップに関する質問ValueError: Error when checking model input: expected convolution2d_input_1 to have shape (None, 3,

編集履歴

質問編集履歴

3

最終的なコード

2021/12/09 06:11

投稿

smart-material
smart-material

スコア12

test CHANGED
File without changes
test CHANGED
@@ -6,166 +6,658 @@
6
6
 
7
7
 
8
8
 
9
+
10
+
9
11
  ### 発生している問題・エラーメッセージ
10
12
 
11
13
 
12
14
 
13
15
  ```
14
16
 
15
-
16
-
17
-
18
-
19
- I tensorflow/core/common_runtime/bfc_allocator.cc:702] Stats:
20
-
21
- Limit: 7582672487
22
-
23
- InUse: 6971735552
24
-
25
- MaxInUse: 7541746432
26
-
27
- NumAllocs: 443
28
-
29
- MaxAllocSize: 3043936256
30
-
31
-
32
-
33
- W tensorflow/core/common_runtime/bfc_allocator.cc:274] **************************_**___****************************************************************xxxx
34
-
35
- W tensorflow/core/common_runtime/bfc_allocator.cc:275] Ran out of memory trying to allocate 392.00MiB. See logs for memory state.
36
-
37
- W tensorflow/core/framework/op_kernel.cc:993] Resource exhausted: OOM when allocating tensor with shape[100352,1024]
38
-
39
- Traceback (most recent call last):
40
-
41
- File "/home/a/libraries/anaconda3/envs/main/lib/python3.6/site-packages/tensorflow/python/client/session.py", line 1022, in _do_call
42
-
43
- return fn(*args)
44
-
45
- File "/home/a/libraries/anaconda3/envs/main/lib/python3.6/site-packages/tensorflow/python/client/session.py", line 1004, in _run_fn
46
-
47
- status, run_metadata)
48
-
49
- File "/home/a/libraries/anaconda3/envs/main/lib/python3.6/contextlib.py", line 88, in __exit__
50
-
51
- next(self.gen)
52
-
53
- File "/home/a/libraries/anaconda3/envs/main/lib/python3.6/site-packages/tensorflow/python/framework/errors_impl.py", line 469, in raise_exception_on_not_ok_status
54
-
55
- pywrap_tensorflow.TF_GetCode(status))
56
-
57
- tensorflow.python.framework.errors_impl.ResourceExhaustedError: OOM when allocating tensor with shape[100352,1024]
17
+ Error when checking model input: expected convolution2d_input_1 to have shape (None, 3, 32, 32) but got array with shape (500, 32, 32, 3)
58
-
59
- [[Node: sub_28 = Sub[T=DT_FLOAT, _device="/job:localhost/replica:0/task:0/gpu:0"](add_14, mul_60)]]
60
-
61
-
62
-
63
- During handling of the above exception, another exception occurred:
64
-
65
-
66
-
67
- Traceback (most recent call last):
68
-
69
- File "n.py", line 299, in <module>
70
-
71
- run_train('9_Layer_CNN')
72
-
73
- File "n.py", line 208, in run_train
74
-
75
- callbacks=[cp])
76
-
77
- File "/home/a/libraries/anaconda3/envs/main/lib/python3.6/site-packages/keras/models.py", line 870, in fit
78
-
79
- initial_epoch=initial_epoch)
80
-
81
- File "/home/a/libraries/anaconda3/envs/main/lib/python3.6/site-packages/keras/engine/training.py", line 1507, in fit
82
-
83
- initial_epoch=initial_epoch)
84
-
85
- File "/home/a/libraries/anaconda3/envs/main/lib/python3.6/site-packages/keras/engine/training.py", line 1156, in _fit_loop
86
-
87
- outs = f(ins_batch)
88
-
89
- File "/home/a/libraries/anaconda3/envs/main/lib/python3.6/site-packages/keras/backend/tensorflow_backend.py", line 2269, in __call__
90
-
91
- **self.session_kwargs)
92
-
93
- File "/home/a/libraries/anaconda3/envs/main/lib/python3.6/site-packages/tensorflow/python/client/session.py", line 767, in run
94
-
95
- run_metadata_ptr)
96
-
97
- File "/home/a/libraries/anaconda3/envs/main/lib/python3.6/site-packages/tensorflow/python/client/session.py", line 965, in _run
98
-
99
- feed_dict_string, options, run_metadata)
100
-
101
- File "/home/a/libraries/anaconda3/envs/main/lib/python3.6/site-packages/tensorflow/python/client/session.py", line 1015, in _do_run
102
-
103
- target_list, options, run_metadata)
104
-
105
- File "/home/a/libraries/anaconda3/envs/main/lib/python3.6/site-packages/tensorflow/python/client/session.py", line 1035, in _do_call
106
-
107
- raise type(e)(node_def, op, message)
108
-
109
- tensorflow.python.framework.errors_impl.ResourceExhaustedError: OOM when allocating tensor with shape[100352,1024]
110
-
111
- [[Node: sub_28 = Sub[T=DT_FLOAT, _device="/job:localhost/replica:0/task:0/gpu:0"](add_14, mul_60)]]
112
-
113
-
114
-
115
- Caused by op 'sub_28', defined at:
116
-
117
- File "n.py", line 299, in <module>
118
-
119
- run_train('9_Layer_CNN')
120
-
121
- File "n.py", line 208, in run_train
122
-
123
- callbacks=[cp])
124
-
125
- File "/home/a/libraries/anaconda3/envs/main/lib/python3.6/site-packages/keras/models.py", line 870, in fit
126
-
127
- initial_epoch=initial_epoch)
128
-
129
- File "/home/a/libraries/anaconda3/envs/main/lib/python3.6/site-packages/keras/engine/training.py", line 1490, in fit
130
-
131
- self._make_train_function()
132
-
133
- File "/home/a/libraries/anaconda3/envs/main/lib/python3.6/site-packages/keras/engine/training.py", line 1014, in _make_train_function
134
-
135
- self.total_loss)
136
-
137
- File "/home/a/libraries/anaconda3/envs/main/lib/python3.6/site-packages/keras/optimizers.py", line 169, in get_updates
138
-
139
- new_p = p + self.momentum * v - lr * g
140
-
141
- File "/home/a/libraries/anaconda3/envs/main/lib/python3.6/site-packages/tensorflow/python/ops/math_ops.py", line 884, in binary_op_wrapper
142
-
143
- return func(x, y, name=name)
144
-
145
- File "/home/a/libraries/anaconda3/envs/main/lib/python3.6/site-packages/tensorflow/python/ops/gen_math_ops.py", line 2775, in _sub
146
-
147
- result = _op_def_lib.apply_op("Sub", x=x, y=y, name=name)
148
-
149
- File "/home/a/libraries/anaconda3/envs/main/lib/python3.6/site-packages/tensorflow/python/framework/op_def_library.py", line 763, in apply_op
150
-
151
- op_def=op_def)
152
-
153
- File "/home/a/libraries/anaconda3/envs/main/lib/python3.6/site-packages/tensorflow/python/framework/ops.py", line 2395, in create_op
154
-
155
- original_op=self._default_original_op, op_def=op_def)
156
-
157
- File "/home/a/libraries/anaconda3/envs/main/lib/python3.6/site-packages/tensorflow/python/framework/ops.py", line 1264, in __init__
158
-
159
- self._traceback = _extract_stack()
160
-
161
-
162
-
163
- ResourceExhaustedError (see above for traceback): OOM when allocating tensor with shape[100352,1024]
164
-
165
- [[Node: sub_28 = Sub[T=DT_FLOAT, _device="/job:localhost/replica:0/task:0/gpu:0"](add_14, mul_60)]]
166
-
167
-
168
-
169
-
170
18
 
171
19
  ```
20
+
21
+
22
+
23
+ ### 該当のソースコード
24
+
25
+
26
+
27
+ ```
28
+
29
+ import numpy as np
30
+
31
+ from numpy.random import permutation
32
+
33
+
34
+
35
+ import os, glob, cv2, math, sys
36
+
37
+ import pandas as pd
38
+
39
+
40
+
41
+ from keras.models import Sequential, model_from_json
42
+
43
+ from keras.layers.core import Dense, Dropout, Flatten
44
+
45
+ from keras.layers.convolutional import Convolution2D, MaxPooling2D
46
+
47
+ from keras.layers.advanced_activations import LeakyReLU
48
+
49
+ from keras.callbacks import ModelCheckpoint
50
+
51
+ from keras.optimizers import SGD
52
+
53
+ from keras.utils import np_utils
54
+
55
+
56
+
57
+ # seed値
58
+
59
+ np.random.seed(1)
60
+
61
+
62
+
63
+ # 使用する画像サイズ
64
+
65
+ img_rows, img_cols = 224, 224
66
+
67
+
68
+
69
+ # 画像データ 1枚の読み込みとリサイズを行う
70
+
71
+ def get_im(path):
72
+
73
+
74
+
75
+ img = cv2.imread(path)
76
+
77
+ resized = cv2.resize(img, (img_cols, img_rows))
78
+
79
+
80
+
81
+ return resized
82
+
83
+
84
+
85
+
86
+
87
+ # データの読み込み、正規化、シャッフルを行う
88
+
89
+ def read_train_data(ho=0, kind='train'):
90
+
91
+
92
+
93
+ train_data = []
94
+
95
+ train_target = []
96
+
97
+
98
+
99
+ # 学習用データ読み込み
100
+
101
+ for j in range(0, 6): # 0～5まで
102
+
103
+
104
+
105
+ path = '../../data/Caltech-101/'
106
+
107
+ path += '%s/%i/*/%i/*.jpg'%(kind, ho, j)
108
+
109
+
110
+
111
+ files = sorted(glob.glob(path))
112
+
113
+
114
+
115
+ for fl in files:
116
+
117
+
118
+
119
+ flbase = os.path.basename(fl)
120
+
121
+
122
+
123
+ # 画像 1枚 読み込み
124
+
125
+ img = get_im(fl)
126
+
127
+ img = np.array(img, dtype=np.float32)
128
+
129
+
130
+
131
+ # 正規化(GCN)実行
132
+
133
+ img -= np.mean(img)
134
+
135
+ img /= np.std(img)
136
+
137
+
138
+
139
+ train_data.append(img)
140
+
141
+ train_target.append(j)
142
+
143
+
144
+
145
+ # 読み込んだデータを numpy の array に変換
146
+
147
+ train_data = np.array(train_data, dtype=np.float32)
148
+
149
+ train_target = np.array(train_target, dtype=np.uint8)
150
+
151
+
152
+
153
+ # (レコード数,縦,横,channel数) を (レコード数,channel数,縦,横) に変換
154
+
155
+ #train_data = train_data.transpose((0, 3, 1, 2))
156
+
157
+
158
+
159
+ # target を 6次元のデータに変換。
160
+
161
+ # ex) 1 -> 0,1,0,0,0,0 2 -> 0,0,1,0,0,0
162
+
163
+ train_target = np_utils.to_categorical(train_target, 6)
164
+
165
+
166
+
167
+ # データをシャッフル
168
+
169
+ perm = permutation(len(train_target))
170
+
171
+ train_data = train_data[perm]
172
+
173
+ train_target = train_target[perm]
174
+
175
+
176
+
177
+ return train_data, train_target
178
+
179
+
180
+
181
+
182
+
183
+ # テストデータ読み込み
184
+
185
+ def load_test(test_class, aug_i):
186
+
187
+
188
+
189
+ path = '../../data/Caltech-101/test/%i/%i/*.jpg'%(aug_i, test_class)
190
+
191
+
192
+
193
+ files = sorted(glob.glob(path))
194
+
195
+ X_test = []
196
+
197
+ X_test_id = []
198
+
199
+
200
+
201
+ for fl in files:
202
+
203
+ flbase = os.path.basename(fl)
204
+
205
+
206
+
207
+ img = get_im(fl)
208
+
209
+ img = np.array(img, dtype=np.float32)
210
+
211
+
212
+
213
+ # 正規化(GCN)実行
214
+
215
+ img -= np.mean(img)
216
+
217
+ img /= np.std(img)
218
+
219
+
220
+
221
+ X_test.append(img)
222
+
223
+ X_test_id.append(flbase)
224
+
225
+
226
+
227
+ # 読み込んだデータを numpy の array に変換
228
+
229
+ test_data = np.array(X_test, dtype=np.float32)
230
+
231
+
232
+
233
+ # (レコード数,縦,横,channel数) を (レコード数,channel数,縦,横) に変換
234
+
235
+ #test_data = test_data.transpose((0, 3, 1, 2))
236
+
237
+
238
+
239
+ return test_data, X_test_id
240
+
241
+
242
+
243
+
244
+
245
+ # 9層 CNNモデル 作成
246
+
247
+ def layer_9_model():
248
+
249
+
250
+
251
+ # KerasのSequentialをモデルの元として使用 ---①
252
+
253
+ model = Sequential()
254
+
255
+
256
+
257
+ # 畳み込み層(Convolution)をモデルに追加 ---②
258
+
259
+ model.add(Convolution2D(32, 3, 3, border_mode='same', activation='linear',
260
+
261
+ input_shape=(img_rows, img_cols, 3)))
262
+
263
+ model.add(LeakyReLU(alpha=0.3))
264
+
265
+
266
+
267
+ model.add(Convolution2D(32, 3, 3, border_mode='same', activation='linear'))
268
+
269
+ model.add(LeakyReLU(alpha=0.3))
270
+
271
+
272
+
273
+ # プーリング層(MaxPooling)をモデルに追加 ---③
274
+
275
+ model.add(MaxPooling2D((2, 2), strides=(2, 2)))
276
+
277
+
278
+
279
+ model.add(Convolution2D(64, 3, 3, border_mode='same', activation='linear'))
280
+
281
+ model.add(LeakyReLU(alpha=0.3))
282
+
283
+ model.add(Convolution2D(64, 3, 3, border_mode='same', activation='linear'))
284
+
285
+ model.add(LeakyReLU(alpha=0.3))
286
+
287
+ model.add(MaxPooling2D((2, 2), strides=(2, 2)))
288
+
289
+
290
+
291
+ model.add(Convolution2D(128, 3, 3, border_mode='same', activation='linear'))
292
+
293
+ model.add(LeakyReLU(alpha=0.3))
294
+
295
+ model.add(Convolution2D(128, 3, 3, border_mode='same', activation='linear'))
296
+
297
+ model.add(LeakyReLU(alpha=0.3))
298
+
299
+ model.add(MaxPooling2D((2, 2), strides=(2, 2)))
300
+
301
+
302
+
303
+ # Flatten層をモデルに追加 -- ④
304
+
305
+ model.add(Flatten())
306
+
307
+ # 全接続層(Dense)をモデルに追加 --- ⑤
308
+
309
+ model.add(Dense(1024, activation='linear'))
310
+
311
+ model.add(LeakyReLU(alpha=0.3))
312
+
313
+ # Dropout層をモデルに追加 --- ⑥
314
+
315
+ model.add(Dropout(0.5))
316
+
317
+ model.add(Dense(1024, activation='linear'))
318
+
319
+ model.add(LeakyReLU(alpha=0.3))
320
+
321
+ model.add(Dropout(0.5))
322
+
323
+ # 最終的なアウトプットを作成。 --- ⑦
324
+
325
+ model.add(Dense(6, activation='softmax'))
326
+
327
+
328
+
329
+ # ロス計算や勾配計算に使用する式を定義する。 --- ⑧
330
+
331
+ sgd = SGD(lr=1e-3, decay=1e-6, momentum=0.9, nesterov=True)
332
+
333
+ model.compile(optimizer=sgd,
334
+
335
+ loss='categorical_crossentropy', metrics=["accuracy"])
336
+
337
+ return model
338
+
339
+
340
+
341
+
342
+
343
+ # モデルの構成と重みを読み込む
344
+
345
+ def read_model(ho, modelStr='', epoch='00'):
346
+
347
+ # モデル構成のファイル名
348
+
349
+ json_name = 'architecture_%s_%i.json'%(modelStr, ho)
350
+
351
+ # モデル重みのファイル名
352
+
353
+ weight_name = 'model_weights_%s_%i_%s.h5'%(modelStr, ho, epoch)
354
+
355
+
356
+
357
+ # モデルの構成を読込み、jsonからモデルオブジェクトへ変換
358
+
359
+ model = model_from_json(open(os.path.join('cache', json_name)).read())
360
+
361
+ # モデルオブジェクトへ重みを読み込む
362
+
363
+ model.load_weights(os.path.join('cache', weight_name))
364
+
365
+
366
+
367
+ return model
368
+
369
+
370
+
371
+
372
+
373
+ # モデルの構成を保存
374
+
375
+ def save_model(model, ho, modelStr=''):
376
+
377
+ # モデルオブジェクトをjson形式に変換
378
+
379
+ json_string = model.to_json()
380
+
381
+ # カレントディレクトリにcacheディレクトリがなければ作成
382
+
383
+ if not os.path.isdir('cache'):
384
+
385
+ os.mkdir('cache')
386
+
387
+ # モデルの構成を保存するためのファイル名
388
+
389
+ json_name = 'architecture_%s_%i.json'%(modelStr, ho)
390
+
391
+ # モデル構成を保存
392
+
393
+ open(os.path.join('cache', json_name), 'w').write(json_string)
394
+
395
+
396
+
397
+
398
+
399
+ def run_train(modelStr=''):
400
+
401
+
402
+
403
+ # HoldOut 2回行う
404
+
405
+ for ho in range(2):
406
+
407
+
408
+
409
+ # モデルの作成
410
+
411
+ model = layer_9_model()
412
+
413
+
414
+
415
+ # trainデータ読み込み
416
+
417
+ t_data, t_target = read_train_data(ho, 'train')
418
+
419
+ v_data, v_target = read_train_data(ho, 'valid')
420
+
421
+
422
+
423
+ # CheckPointを設定。エポック毎にweightsを保存する。
424
+
425
+ cp = ModelCheckpoint('./cache/model_weights_%s_%i_{epoch:02d}.h5'%(modelStr, ho),
426
+
427
+ monitor='val_loss', save_best_only=False)
428
+
429
+
430
+
431
+ # train実行
432
+
433
+ print(t_data.shape)
434
+
435
+ print(t_data.dtype)
436
+
437
+ print(v_data.shape)
438
+
439
+ print(v_data.dtype)
440
+
441
+ print(t_target.shape)
442
+
443
+ print(t_target.dtype)
444
+
445
+ print(v_target.shape)
446
+
447
+ print(v_target.dtype)
448
+
449
+ model.fit(t_data, t_target, batch_size=64,
450
+
451
+ nb_epoch=40,
452
+
453
+ verbose=1,
454
+
455
+ validation_data=(v_data, v_target),
456
+
457
+ shuffle=True,
458
+
459
+ callbacks=[cp])
460
+
461
+
462
+
463
+
464
+
465
+ # モデルの構成を保存
466
+
467
+ save_model(model, ho, modelStr)
468
+
469
+
470
+
471
+
472
+
473
+
474
+
475
+ # テストデータのクラスを推測
476
+
477
+ def run_test(modelStr, epoch1, epoch2):
478
+
479
+
480
+
481
+ # クラス名取得
482
+
483
+ columns = []
484
+
485
+ for line in open("../../data/Caltech-101/label.csv", 'r'):
486
+
487
+ sp = line.split(',')
488
+
489
+ for column in sp:
490
+
491
+ columns.append(column.split(":")[1])
492
+
493
+
494
+
495
+ # テストデータが各クラスに分かれているので、
496
+
497
+ # 1クラスずつ読み込んで推測を行う。
498
+
499
+ for test_class in range(0, 6):
500
+
501
+
502
+
503
+ yfull_test = []
504
+
505
+
506
+
507
+ # データ拡張した画像を読み込むために5回繰り返す
508
+
509
+ for aug_i in range(0,5):
510
+
511
+
512
+
513
+ # テストデータを読み込む
514
+
515
+ test_data, test_id = load_test(test_class, aug_i)
516
+
517
+
518
+
519
+ # HoldOut 2回繰り返す
520
+
521
+ for ho in range(2):
522
+
523
+
524
+
525
+ if ho == 0:
526
+
527
+ epoch_n = epoch1
528
+
529
+ else:
530
+
531
+ epoch_n = epoch2
532
+
533
+
534
+
535
+ # 学習済みモデルの読み込み
536
+
537
+ model = read_model(ho, modelStr, epoch_n)
538
+
539
+
540
+
541
+ # 推測の実行
542
+
543
+ test_p = model.predict(test_data, batch_size=128, verbose=1)
544
+
545
+
546
+
547
+ yfull_test.append(test_p)
548
+
549
+
550
+
551
+ # 推測結果の平均化
552
+
553
+ test_res = np.array(yfull_test[0])
554
+
555
+ for i in range(1,10):
556
+
557
+ test_res += np.array(yfull_test[i])
558
+
559
+ test_res /= 10
560
+
561
+
562
+
563
+ # 推測結果とクラス名、画像名を合わせる
564
+
565
+ result1 = pd.DataFrame(test_res, columns=columns)
566
+
567
+ result1.loc[:, 'img'] = pd.Series(test_id, index=result1.index)
568
+
569
+
570
+
571
+ # 順番入れ替え
572
+
573
+ result1 = result1.ix[:,[6, 0, 1, 2, 3, 4, 5]]
574
+
575
+
576
+
577
+ if not os.path.isdir('subm'):
578
+
579
+ os.mkdir('subm')
580
+
581
+ sub_file = './subm/result_%s_%i.csv'%(modelStr, test_class)
582
+
583
+
584
+
585
+ # 最終推測結果を出力する
586
+
587
+ result1.to_csv(sub_file, index=False)
588
+
589
+
590
+
591
+ # 推測の精度を測定する。
592
+
593
+ # 一番大きい値が入っているカラムがtest_classであるレコードを探す
594
+
595
+ one_column = np.where(np.argmax(test_res, axis=1)==test_class)
596
+
597
+ print ("正解数　　" + str(len(one_column[0])))
598
+
599
+ print ("不正解数　" + str(test_res.shape[0] - len(one_column[0])))
600
+
601
+
602
+
603
+
604
+
605
+
606
+
607
+
608
+
609
+ # 実行した際に呼ばれる
610
+
611
+ if __name__ == '__main__':
612
+
613
+
614
+
615
+ # 引数を取得
616
+
617
+ # [1] = train or test
618
+
619
+ # [2] = test時のみ、使用Epoch数 1
620
+
621
+ # [3] = test時のみ、使用Epoch数 2
622
+
623
+ param = sys.argv
624
+
625
+
626
+
627
+ if len(param) < 2:
628
+
629
+ sys.exit ("Usage: python 9_Layer_CNN.py [train, test] [1] [2]")
630
+
631
+
632
+
633
+ # train or test
634
+
635
+ run_type = param[1]
636
+
637
+
638
+
639
+ if run_type == 'train':
640
+
641
+ run_train('9_Layer_CNN')
642
+
643
+ elif run_type == 'test':
644
+
645
+ # testの場合、使用するエポック数を引数から取得する
646
+
647
+ if len(param) == 4:
648
+
649
+ epoch1 = "%02d"%(int(param[2])-1)
650
+
651
+ epoch2 = "%02d"%(int(param[3])-1)
652
+
653
+ run_test('9_Layer_CNN', epoch1, epoch2)
654
+
655
+ else:
656
+
657
+ sys.exit ("Usage: python 9_Layer_CNN.py [train, test] [1] [2]")
658
+
659
+ else:
660
+
661
+ sys.exit ("Usage: python 9_Layer_CNN.py [train, test] [1] [2]")
662
+
663
+ ```

2

質問事項の追加をしました。

2021/12/09 06:11

投稿

smart-material
smart-material

スコア12

test CHANGED
File without changes
test CHANGED
@@ -12,650 +12,160 @@
12
12
 
13
13
  ```
14
14
 
15
+
16
+
17
+
18
+
19
+ I tensorflow/core/common_runtime/bfc_allocator.cc:702] Stats:
20
+
21
+ Limit: 7582672487
22
+
23
+ InUse: 6971735552
24
+
25
+ MaxInUse: 7541746432
26
+
27
+ NumAllocs: 443
28
+
29
+ MaxAllocSize: 3043936256
30
+
31
+
32
+
33
+ W tensorflow/core/common_runtime/bfc_allocator.cc:274] **************************_**___****************************************************************xxxx
34
+
35
+ W tensorflow/core/common_runtime/bfc_allocator.cc:275] Ran out of memory trying to allocate 392.00MiB. See logs for memory state.
36
+
37
+ W tensorflow/core/framework/op_kernel.cc:993] Resource exhausted: OOM when allocating tensor with shape[100352,1024]
38
+
15
39
  Traceback (most recent call last):
16
40
 
17
-
41
+ File "/home/a/libraries/anaconda3/envs/main/lib/python3.6/site-packages/tensorflow/python/client/session.py", line 1022, in _do_call
18
42
 
19
-
43
+ return fn(*args)
20
44
 
21
- File "/home/a/libraries/anaconda3/envs/main/lib/python3.6/site-packages/tensorflow/python/framework/common_shapes.py", line 610, in call_cpp_shape_fn
45
+ File "/home/a/libraries/anaconda3/envs/main/lib/python3.6/site-packages/tensorflow/python/client/session.py", line 1004, in _run_fn
22
46
 
23
- debug_python_shape_fn, require_shape_fn)
47
+ status, run_metadata)
24
48
 
25
- File "/home/a/libraries/anaconda3/envs/main/lib/python3.6/site-packages/tensorflow/python/framework/common_shapes.py", line 675, in _call_cpp_shape_fn_impl
49
+ File "/home/a/libraries/anaconda3/envs/main/lib/python3.6/contextlib.py", line 88, in __exit__
26
50
 
27
- raise ValueError(err.message)
51
+ next(self.gen)
28
52
 
53
+ File "/home/a/libraries/anaconda3/envs/main/lib/python3.6/site-packages/tensorflow/python/framework/errors_impl.py", line 469, in raise_exception_on_not_ok_status
54
+
55
+ pywrap_tensorflow.TF_GetCode(status))
56
+
57
+ tensorflow.python.framework.errors_impl.ResourceExhaustedError: OOM when allocating tensor with shape[100352,1024]
58
+
59
+ [[Node: sub_28 = Sub[T=DT_FLOAT, _device="/job:localhost/replica:0/task:0/gpu:0"](add_14, mul_60)]]
60
+
61
+
62
+
63
+ During handling of the above exception, another exception occurred:
64
+
65
+
66
+
67
+ Traceback (most recent call last):
68
+
69
+ File "n.py", line 299, in <module>
70
+
71
+ run_train('9_Layer_CNN')
72
+
73
+ File "n.py", line 208, in run_train
74
+
75
+ callbacks=[cp])
76
+
29
- ValueError: Negative dimension size caused by subtracting 2 from 1 for 'max_pooling2d_2/MaxPool' (op: 'MaxPool') with input shapes: [?,112,1,64].
77
+ File "/home/a/libraries/anaconda3/envs/main/lib/python3.6/site-packages/keras/models.py", line 870, in fit
78
+
79
+ initial_epoch=initial_epoch)
80
+
81
+ File "/home/a/libraries/anaconda3/envs/main/lib/python3.6/site-packages/keras/engine/training.py", line 1507, in fit
82
+
83
+ initial_epoch=initial_epoch)
84
+
85
+ File "/home/a/libraries/anaconda3/envs/main/lib/python3.6/site-packages/keras/engine/training.py", line 1156, in _fit_loop
86
+
87
+ outs = f(ins_batch)
88
+
89
+ File "/home/a/libraries/anaconda3/envs/main/lib/python3.6/site-packages/keras/backend/tensorflow_backend.py", line 2269, in __call__
90
+
91
+ **self.session_kwargs)
92
+
93
+ File "/home/a/libraries/anaconda3/envs/main/lib/python3.6/site-packages/tensorflow/python/client/session.py", line 767, in run
94
+
95
+ run_metadata_ptr)
96
+
97
+ File "/home/a/libraries/anaconda3/envs/main/lib/python3.6/site-packages/tensorflow/python/client/session.py", line 965, in _run
98
+
99
+ feed_dict_string, options, run_metadata)
100
+
101
+ File "/home/a/libraries/anaconda3/envs/main/lib/python3.6/site-packages/tensorflow/python/client/session.py", line 1015, in _do_run
102
+
103
+ target_list, options, run_metadata)
104
+
105
+ File "/home/a/libraries/anaconda3/envs/main/lib/python3.6/site-packages/tensorflow/python/client/session.py", line 1035, in _do_call
106
+
107
+ raise type(e)(node_def, op, message)
108
+
109
+ tensorflow.python.framework.errors_impl.ResourceExhaustedError: OOM when allocating tensor with shape[100352,1024]
110
+
111
+ [[Node: sub_28 = Sub[T=DT_FLOAT, _device="/job:localhost/replica:0/task:0/gpu:0"](add_14, mul_60)]]
112
+
113
+
114
+
115
+ Caused by op 'sub_28', defined at:
116
+
117
+ File "n.py", line 299, in <module>
118
+
119
+ run_train('9_Layer_CNN')
120
+
121
+ File "n.py", line 208, in run_train
122
+
123
+ callbacks=[cp])
124
+
125
+ File "/home/a/libraries/anaconda3/envs/main/lib/python3.6/site-packages/keras/models.py", line 870, in fit
126
+
127
+ initial_epoch=initial_epoch)
128
+
129
+ File "/home/a/libraries/anaconda3/envs/main/lib/python3.6/site-packages/keras/engine/training.py", line 1490, in fit
130
+
131
+ self._make_train_function()
132
+
133
+ File "/home/a/libraries/anaconda3/envs/main/lib/python3.6/site-packages/keras/engine/training.py", line 1014, in _make_train_function
134
+
135
+ self.total_loss)
136
+
137
+ File "/home/a/libraries/anaconda3/envs/main/lib/python3.6/site-packages/keras/optimizers.py", line 169, in get_updates
138
+
139
+ new_p = p + self.momentum * v - lr * g
140
+
141
+ File "/home/a/libraries/anaconda3/envs/main/lib/python3.6/site-packages/tensorflow/python/ops/math_ops.py", line 884, in binary_op_wrapper
142
+
143
+ return func(x, y, name=name)
144
+
145
+ File "/home/a/libraries/anaconda3/envs/main/lib/python3.6/site-packages/tensorflow/python/ops/gen_math_ops.py", line 2775, in _sub
146
+
147
+ result = _op_def_lib.apply_op("Sub", x=x, y=y, name=name)
148
+
149
+ File "/home/a/libraries/anaconda3/envs/main/lib/python3.6/site-packages/tensorflow/python/framework/op_def_library.py", line 763, in apply_op
150
+
151
+ op_def=op_def)
152
+
153
+ File "/home/a/libraries/anaconda3/envs/main/lib/python3.6/site-packages/tensorflow/python/framework/ops.py", line 2395, in create_op
154
+
155
+ original_op=self._default_original_op, op_def=op_def)
156
+
157
+ File "/home/a/libraries/anaconda3/envs/main/lib/python3.6/site-packages/tensorflow/python/framework/ops.py", line 1264, in __init__
158
+
159
+ self._traceback = _extract_stack()
160
+
161
+
162
+
163
+ ResourceExhaustedError (see above for traceback): OOM when allocating tensor with shape[100352,1024]
164
+
165
+ [[Node: sub_28 = Sub[T=DT_FLOAT, _device="/job:localhost/replica:0/task:0/gpu:0"](add_14, mul_60)]]
30
166
 
31
167
 
32
168
 
33
169
 
34
170
 
35
171
  ```
36
-
37
- ```該当ソースコード
38
-
39
- import numpy as np
40
-
41
- from numpy.random import permutation
42
-
43
-
44
-
45
- import os, glob, cv2, math, sys
46
-
47
- import pandas as pd
48
-
49
-
50
-
51
- from keras.models import Sequential, model_from_json
52
-
53
- from keras.layers.core import Dense, Dropout, Flatten
54
-
55
- from keras.layers.convolutional import Convolution2D, MaxPooling2D
56
-
57
- from keras.layers.advanced_activations import LeakyReLU
58
-
59
- from keras.callbacks import ModelCheckpoint
60
-
61
- from keras.optimizers import SGD
62
-
63
- from keras.utils import np_utils
64
-
65
-
66
-
67
- # seed値
68
-
69
- np.random.seed(1)
70
-
71
-
72
-
73
- # 使用する画像サイズ
74
-
75
- img_rows, img_cols = 224, 224
76
-
77
-
78
-
79
- # 画像データ 1枚の読み込みとリサイズを行う
80
-
81
- def get_im(path):
82
-
83
-
84
-
85
- img = cv2.imread(path)
86
-
87
- resized = cv2.resize(img, (img_cols, img_rows))
88
-
89
-
90
-
91
- return resized
92
-
93
-
94
-
95
-
96
-
97
- # データの読み込み、正規化、シャッフルを行う
98
-
99
- def read_train_data(ho=0, kind='train'):
100
-
101
-
102
-
103
- train_data = []
104
-
105
- train_target = []
106
-
107
-
108
-
109
- # 学習用データ読み込み
110
-
111
- for j in range(0, 6): # 0～5まで
112
-
113
-
114
-
115
- path = '../../data/Caltech-101/'
116
-
117
- path += '%s/%i/*/%i/*.jpg'%(kind, ho, j)
118
-
119
-
120
-
121
- files = sorted(glob.glob(path))
122
-
123
-
124
-
125
- for fl in files:
126
-
127
-
128
-
129
- flbase = os.path.basename(fl)
130
-
131
-
132
-
133
- # 画像 1枚 読み込み
134
-
135
- img = get_im(fl)
136
-
137
- img = np.array(img, dtype=np.float32)
138
-
139
-
140
-
141
- # 正規化(GCN)実行
142
-
143
- img -= np.mean(img)
144
-
145
- img /= np.std(img)
146
-
147
-
148
-
149
- train_data.append(img)
150
-
151
- train_target.append(j)
152
-
153
-
154
-
155
- # 読み込んだデータを numpy の array に変換
156
-
157
- train_data = np.array(train_data, dtype=np.float32)
158
-
159
- train_target = np.array(train_target, dtype=np.uint8)
160
-
161
-
162
-
163
- # (レコード数,縦,横,channel数) を (レコード数,channel数,縦,横) に変換
164
-
165
- #train_data = train_data.transpose((0, 3, 1, 2))
166
-
167
-
168
-
169
- # target を 6次元のデータに変換。
170
-
171
- # ex) 1 -> 0,1,0,0,0,0 2 -> 0,0,1,0,0,0
172
-
173
- train_target = np_utils.to_categorical(train_target, 6)
174
-
175
-
176
-
177
- # データをシャッフル
178
-
179
- perm = permutation(len(train_target))
180
-
181
- train_data = train_data[perm]
182
-
183
- train_target = train_target[perm]
184
-
185
-
186
-
187
- return train_data, train_target
188
-
189
-
190
-
191
-
192
-
193
- # テストデータ読み込み
194
-
195
- def load_test(test_class, aug_i):
196
-
197
-
198
-
199
- path = '../../data/Caltech-101/test/%i/%i/*.jpg'%(aug_i, test_class)
200
-
201
-
202
-
203
- files = sorted(glob.glob(path))
204
-
205
- X_test = []
206
-
207
- X_test_id = []
208
-
209
-
210
-
211
- for fl in files:
212
-
213
- flbase = os.path.basename(fl)
214
-
215
-
216
-
217
- img = get_im(fl)
218
-
219
- img = np.array(img, dtype=np.float32)
220
-
221
-
222
-
223
- # 正規化(GCN)実行
224
-
225
- img -= np.mean(img)
226
-
227
- img /= np.std(img)
228
-
229
-
230
-
231
- X_test.append(img)
232
-
233
- X_test_id.append(flbase)
234
-
235
-
236
-
237
- # 読み込んだデータを numpy の array に変換
238
-
239
- test_data = np.array(X_test, dtype=np.float32)
240
-
241
-
242
-
243
- # (レコード数,縦,横,channel数) を (レコード数,channel数,縦,横) に変換
244
-
245
- #test_data = test_data.transpose((0, 3, 1, 2))
246
-
247
-
248
-
249
- return test_data, X_test_id
250
-
251
-
252
-
253
-
254
-
255
- # 9層 CNNモデル 作成
256
-
257
- def layer_9_model():
258
-
259
-
260
-
261
- # KerasのSequentialをモデルの元として使用 ---①
262
-
263
- model = Sequential()
264
-
265
-
266
-
267
- # 畳み込み層(Convolution)をモデルに追加 ---②
268
-
269
- model.add(Convolution2D(32, 3, 3, border_mode='same', activation='linear',
270
-
271
- input_shape=(img_rows, img_cols, 3)))
272
-
273
- model.add(LeakyReLU(alpha=0.3))
274
-
275
-
276
-
277
- model.add(Convolution2D(32, 3, 3, border_mode='same', activation='linear'))
278
-
279
- model.add(LeakyReLU(alpha=0.3))
280
-
281
-
282
-
283
- # プーリング層(MaxPooling)をモデルに追加 ---③
284
-
285
- model.add(MaxPooling2D((2, 2), strides=(2, 2)))
286
-
287
-
288
-
289
- model.add(Convolution2D(64, 3, 3, border_mode='same', activation='linear'))
290
-
291
- model.add(LeakyReLU(alpha=0.3))
292
-
293
- model.add(Convolution2D(64, 3, 3, border_mode='same', activation='linear'))
294
-
295
- model.add(LeakyReLU(alpha=0.3))
296
-
297
- model.add(MaxPooling2D((2, 2), strides=(2, 2)))
298
-
299
-
300
-
301
- model.add(Convolution2D(128, 3, 3, border_mode='same', activation='linear'))
302
-
303
- model.add(LeakyReLU(alpha=0.3))
304
-
305
- model.add(Convolution2D(128, 3, 3, border_mode='same', activation='linear'))
306
-
307
- model.add(LeakyReLU(alpha=0.3))
308
-
309
- model.add(MaxPooling2D((2, 2), strides=(2, 2)))
310
-
311
-
312
-
313
- # Flatten層をモデルに追加 -- ④
314
-
315
- model.add(Flatten())
316
-
317
- # 全接続層(Dense)をモデルに追加 --- ⑤
318
-
319
- model.add(Dense(1024, activation='linear'))
320
-
321
- model.add(LeakyReLU(alpha=0.3))
322
-
323
- # Dropout層をモデルに追加 --- ⑥
324
-
325
- model.add(Dropout(0.5))
326
-
327
- model.add(Dense(1024, activation='linear'))
328
-
329
- model.add(LeakyReLU(alpha=0.3))
330
-
331
- model.add(Dropout(0.5))
332
-
333
- # 最終的なアウトプットを作成。 --- ⑦
334
-
335
- model.add(Dense(6, activation='softmax'))
336
-
337
-
338
-
339
- # ロス計算や勾配計算に使用する式を定義する。 --- ⑧
340
-
341
- sgd = SGD(lr=1e-3, decay=1e-6, momentum=0.9, nesterov=True)
342
-
343
- model.compile(optimizer=sgd,
344
-
345
- loss='categorical_crossentropy', metrics=["accuracy"])
346
-
347
- return model
348
-
349
-
350
-
351
-
352
-
353
- # モデルの構成と重みを読み込む
354
-
355
- def read_model(ho, modelStr='', epoch='00'):
356
-
357
- # モデル構成のファイル名
358
-
359
- json_name = 'architecture_%s_%i.json'%(modelStr, ho)
360
-
361
- # モデル重みのファイル名
362
-
363
- weight_name = 'model_weights_%s_%i_%s.h5'%(modelStr, ho, epoch)
364
-
365
-
366
-
367
- # モデルの構成を読込み、jsonからモデルオブジェクトへ変換
368
-
369
- model = model_from_json(open(os.path.join('cache', json_name)).read())
370
-
371
- # モデルオブジェクトへ重みを読み込む
372
-
373
- model.load_weights(os.path.join('cache', weight_name))
374
-
375
-
376
-
377
- return model
378
-
379
-
380
-
381
-
382
-
383
- # モデルの構成を保存
384
-
385
- def save_model(model, ho, modelStr=''):
386
-
387
- # モデルオブジェクトをjson形式に変換
388
-
389
- json_string = model.to_json()
390
-
391
- # カレントディレクトリにcacheディレクトリがなければ作成
392
-
393
- if not os.path.isdir('cache'):
394
-
395
- os.mkdir('cache')
396
-
397
- # モデルの構成を保存するためのファイル名
398
-
399
- json_name = 'architecture_%s_%i.json'%(modelStr, ho)
400
-
401
- # モデル構成を保存
402
-
403
- open(os.path.join('cache', json_name), 'w').write(json_string)
404
-
405
-
406
-
407
-
408
-
409
- def run_train(modelStr=''):
410
-
411
-
412
-
413
- # HoldOut 2回行う
414
-
415
- for ho in range(2):
416
-
417
-
418
-
419
- # モデルの作成
420
-
421
- model = layer_9_model()
422
-
423
-
424
-
425
- # trainデータ読み込み
426
-
427
- t_data, t_target = read_train_data(ho, 'train')
428
-
429
- v_data, v_target = read_train_data(ho, 'valid')
430
-
431
-
432
-
433
- # CheckPointを設定。エポック毎にweightsを保存する。
434
-
435
- cp = ModelCheckpoint('./cache/model_weights_%s_%i_{epoch:02d}.h5'%(modelStr, ho),
436
-
437
- monitor='val_loss', save_best_only=False)
438
-
439
-
440
-
441
- # train実行
442
-
443
- model.fit(t_data, t_target, batch_size=64,
444
-
445
- nb_epoch=40,
446
-
447
- verbose=1,
448
-
449
- validation_data=(v_data, v_target),
450
-
451
- shuffle=True,
452
-
453
- callbacks=[cp])
454
-
455
-
456
-
457
-
458
-
459
- # モデルの構成を保存
460
-
461
- save_model(model, ho, modelStr)
462
-
463
-
464
-
465
-
466
-
467
-
468
-
469
- # テストデータのクラスを推測
470
-
471
- def run_test(modelStr, epoch1, epoch2):
472
-
473
-
474
-
475
- # クラス名取得
476
-
477
- columns = []
478
-
479
- for line in open("../../data/Caltech-101/label.csv", 'r'):
480
-
481
- sp = line.split(',')
482
-
483
- for column in sp:
484
-
485
- columns.append(column.split(":")[1])
486
-
487
-
488
-
489
- # テストデータが各クラスに分かれているので、
490
-
491
- # 1クラスずつ読み込んで推測を行う。
492
-
493
- for test_class in range(0, 6):
494
-
495
-
496
-
497
- yfull_test = []
498
-
499
-
500
-
501
- # データ拡張した画像を読み込むために5回繰り返す
502
-
503
- for aug_i in range(0,5):
504
-
505
-
506
-
507
- # テストデータを読み込む
508
-
509
- test_data, test_id = load_test(test_class, aug_i)
510
-
511
-
512
-
513
- # HoldOut 2回繰り返す
514
-
515
- for ho in range(2):
516
-
517
-
518
-
519
- if ho == 0:
520
-
521
- epoch_n = epoch1
522
-
523
- else:
524
-
525
- epoch_n = epoch2
526
-
527
-
528
-
529
- # 学習済みモデルの読み込み
530
-
531
- model = read_model(ho, modelStr, epoch_n)
532
-
533
-
534
-
535
- # 推測の実行
536
-
537
- test_p = model.predict(test_data, batch_size=128, verbose=1)
538
-
539
-
540
-
541
- yfull_test.append(test_p)
542
-
543
-
544
-
545
- # 推測結果の平均化
546
-
547
- test_res = np.array(yfull_test[0])
548
-
549
- for i in range(1,10):
550
-
551
- test_res += np.array(yfull_test[i])
552
-
553
- test_res /= 10
554
-
555
-
556
-
557
- # 推測結果とクラス名、画像名を合わせる
558
-
559
- result1 = pd.DataFrame(test_res, columns=columns)
560
-
561
- result1.loc[:, 'img'] = pd.Series(test_id, index=result1.index)
562
-
563
-
564
-
565
- # 順番入れ替え
566
-
567
- result1 = result1.ix[:,[6, 0, 1, 2, 3, 4, 5]]
568
-
569
-
570
-
571
- if not os.path.isdir('subm'):
572
-
573
- os.mkdir('subm')
574
-
575
- sub_file = './subm/result_%s_%i.csv'%(modelStr, test_class)
576
-
577
-
578
-
579
- # 最終推測結果を出力する
580
-
581
- result1.to_csv(sub_file, index=False)
582
-
583
-
584
-
585
- # 推測の精度を測定する。
586
-
587
- # 一番大きい値が入っているカラムがtest_classであるレコードを探す
588
-
589
- one_column = np.where(np.argmax(test_res, axis=1)==test_class)
590
-
591
- print ("正解数　　" + str(len(one_column[0])))
592
-
593
- print ("不正解数　" + str(test_res.shape[0] - len(one_column[0])))
594
-
595
-
596
-
597
-
598
-
599
-
600
-
601
-
602
-
603
- # 実行した際に呼ばれる
604
-
605
- if __name__ == '__main__':
606
-
607
-
608
-
609
- # 引数を取得
610
-
611
- # [1] = train or test
612
-
613
- # [2] = test時のみ、使用Epoch数 1
614
-
615
- # [3] = test時のみ、使用Epoch数 2
616
-
617
- param = sys.argv
618
-
619
-
620
-
621
- if len(param) < 2:
622
-
623
- sys.exit ("Usage: python 9_Layer_CNN.py [train, test] [1] [2]")
624
-
625
-
626
-
627
- # train or test
628
-
629
- run_type = param[1]
630
-
631
-
632
-
633
- if run_type == 'train':
634
-
635
- run_train('9_Layer_CNN')
636
-
637
- elif run_type == 'test':
638
-
639
- # testの場合、使用するエポック数を引数から取得する
640
-
641
- if len(param) == 4:
642
-
643
- epoch1 = "%02d"%(int(param[2])-1)
644
-
645
- epoch2 = "%02d"%(int(param[3])-1)
646
-
647
- run_test('9_Layer_CNN', epoch1, epoch2)
648
-
649
- else:
650
-
651
- sys.exit ("Usage: python 9_Layer_CNN.py [train, test] [1] [2]")
652
-
653
- else:
654
-
655
- sys.exit ("Usage: python 9_Layer_CNN.py [train, test] [1] [2]")
656
-
657
-
658
-
659
-
660
-
661
- ```

1

コード２

2021/12/09 05:25

投稿

smart-material
smart-material

スコア12

test CHANGED
File without changes
test CHANGED
@@ -12,39 +12,23 @@
12
12
 
13
13
  ```
14
14
 
15
- Using TensorFlow backend.
16
-
17
-
18
-
19
-
20
-
21
15
  Traceback (most recent call last):
22
16
 
23
- File "9_Layer_CNN.py", line 301, in <module>
17
+
24
-
25
- run_train('9_Layer_CNN')
18
+
26
-
27
- File "9_Layer_CNN.py", line 210, in run_train
19
+
28
-
29
- callbacks=[cp])
20
+
30
-
31
- File "/home/a/libraries/anaconda3/envs/main/lib/python3.6/site-packages/keras/models.py", line 672, in fit
32
-
33
- initial_epoch=initial_epoch)
34
-
35
- File "/home/a/libraries/anaconda3/envs/main/lib/python3.6/site-packages/keras/engine/training.py", line 1116, in fit
36
-
37
- batch_size=batch_size)
38
-
39
- File "/home/a/libraries/anaconda3/envs/main/lib/python3.6/site-packages/keras/engine/training.py", line 1029, in _standardize_user_data
21
+ File "/home/a/libraries/anaconda3/envs/main/lib/python3.6/site-packages/tensorflow/python/framework/common_shapes.py", line 610, in call_cpp_shape_fn
40
-
22
+
41
- exception_prefix='model input')
23
+ debug_python_shape_fn, require_shape_fn)
42
-
24
+
43
- File "/home/a/libraries/anaconda3/envs/main/lib/python3.6/site-packages/keras/engine/training.py", line 124, in standardize_input_data
25
+ File "/home/a/libraries/anaconda3/envs/main/lib/python3.6/site-packages/tensorflow/python/framework/common_shapes.py", line 675, in _call_cpp_shape_fn_impl
44
-
26
+
45
- str(array.shape))
27
+ raise ValueError(err.message)
46
-
28
+
47
- ValueError: Error when checking model input: expected convolution2d_input_1 to have shape (None, 3, 224, 224) but got array with shape (1305, 224, 224, 3)
29
+ ValueError: Negative dimension size caused by subtracting 2 from 1 for 'max_pooling2d_2/MaxPool' (op: 'MaxPool') with input shapes: [?,112,1,64].
30
+
31
+
48
32
 
49
33
 
50
34
 
@@ -284,7 +268,7 @@
284
268
 
285
269
  model.add(Convolution2D(32, 3, 3, border_mode='same', activation='linear',
286
270
 
287
- input_shape=(3, img_rows, img_cols)))
271
+ input_shape=(img_rows, img_cols, 3)))
288
272
 
289
273
  model.add(LeakyReLU(alpha=0.3))
290
274