python 強制終了の原因

Question

前提・実現したいこと

pythonで処理中にエラーメッセージなしで強制終了してしまう原因を突き止めたい
numpyの処理付近が原因かもしれません。

ご教示いただけると幸いです。

該当のソースコード

#おそらく強制終了が出る原因があるコード
img_path = 'dataset6/test/train'
img_list = os.listdir(img_path)

x_train_img = []

for i in img_list:
    abs_path = os.path.join(img_path, i)
    temp = load_img(abs_path)
    temp = img_to_array(temp)
    x_train_img.append(temp)

x_train_img = np.array(x_train_img)


batch_size = 1

step = int(x_train_img.shape[0] / batch_size)
print(step)


y_train_img_dummy = []

for i in range(step):
    
    x_temp = x_train_img[batch_size*i:batch_size*(i+1)]
    
    x_temp = x_temp / 255.
   
    temp = teacher_model.predict(x_temp)
    
    y_train_img_dummy.extend(temp)


y_train_img_dummy = np.array(y_train_img_dummy)
print(y_train_img_dummy)

threhold = 0.75
y_train_img_dummy_th =  y_train_img_dummy[np.max(y_train_img_dummy, axis=1) > threhold]
x_train_img_th = x_train_img[np.max(y_train_img_dummy, axis=1) > threhold]

#コード全文
from keras import layers
from keras import models
from keras import optimizers
from keras.layers import GlobalAveragePooling2D
import numpy as np
import pandas as pd
import os
import glob
import matplotlib.pyplot as plt
import seaborn as sns
from PIL import Image
from efficientnet.keras import EfficientNetB7
#from efficientnet.keras import EfficientNetL2
from keras.utils import to_categorical
from keras.layers import Input, Dense, Reshape
from keras.models import Model
from keras.callbacks import ModelCheckpoint
from keras.models import load_model
from keras.preprocessing.image import ImageDataGenerator
from keras.preprocessing.image import load_img, save_img, img_to_array, array_to_img

inputs1 = Input(shape=(200, 200, 3))
efficient_netB7 = EfficientNetB7(weights='noisy-student',include_top=False,input_tensor = inputs1,pooling='max')
for layer in efficient_netB7.layers:
    layer.trainable = True
x = efficient_netB7.output
x = layers.Dense(32, activation='relu')(x)
output1 = layers.Dense(1, activation='sigmoid')(x)

teacher_model = Model(inputs=inputs1, outputs=output1)

teacher_model.compile(loss='categorical_crossentropy',optimizer=optimizers.RMSprop(lr=1e-4), metrics=['acc'])

teacher_model.summary()

inputs2 = Input(shape=(200, 200, 3))
efficient_netL2 = EfficientNetB7(weights='noisy-student',include_top=False,input_tensor = inputs2,pooling='max')
for layer in efficient_netL2.layers:
    layer.trainable = True
x = efficient_netL2.output
x = layers.Dense(32, activation='relu')(x)
output2 = layers.Dense(1, activation='sigmoid')(x)

student_model = Model(inputs=inputs2, outputs=output2)

student_model.compile(loss='categorical_crossentropy',optimizer=optimizers.RMSprop(lr=1e-4), metrics=['acc'])

student_model.summary()

train_dir = 'dataset6/train'
validation_dir = 'dataset6/validation'

# rescaled by 1./255
train_datagen = ImageDataGenerator(rescale=1./255)
test_datagen = ImageDataGenerator(rescale=1./255)

train_generator = train_datagen.flow_from_directory(train_dir,target_size=(200, 200),batch_size=10,class_mode='categorical')

validation_generator = test_datagen.flow_from_directory(validation_dir,target_size=(200, 200),batch_size=10,class_mode='categorical')

#history1 = teacher_model.fit_generator(train_generator,steps_per_epoch=100,epochs=1,validation_data=validation_generator,validation_steps=30)

teacher_model.save('teacher_model.h5')

#history2 = student_model.fit_generator(train_generator,steps_per_epoch=100,epochs=1,validation_data=validation_generator,validation_steps=30)

student_model.save('teacher_model.h5')

img_path = 'dataset6/test/train'
img_list = os.listdir(img_path)

x_train_img = []

for i in img_list:
    abs_path = os.path.join(img_path, i)
    temp = load_img(abs_path)
    temp = img_to_array(temp)
    x_train_img.append(temp)

x_train_img = np.array(x_train_img)

#バッチサイズの設定
batch_size = 1
#何ステップfor文を回すか
step = int(x_train_img.shape[0] / batch_size)
print(step)

#疑似ラベルのための空リスト
y_train_img_dummy = []

for i in range(step):
    #バッチサイズ分画像データを取り出す
    x_temp = x_train_img[batch_size*i:batch_size*(i+1)]
    #正規化
    x_temp = x_temp / 255.
    #推論
    temp = teacher_model.predict(x_temp)
    #空リストへ追加
    y_train_img_dummy.extend(temp)

#リストをnumpy配列へ
y_train_img_dummy = np.array(y_train_img_dummy)
print(y_train_img_dummy)
#閾値決め
threhold = 0.75
y_train_img_dummy_th =  y_train_img_dummy[np.max(y_train_img_dummy, axis=1) > threhold]
x_train_img_th = x_train_img[np.max(y_train_img_dummy, axis=1) > threhold]

#onehot vectorから分類のインデックスにする
y_student_all_dummy_label = np.argmax(y_train_img_dummy_th, axis=1)

#疑似ラベルの各クラスの数をカウント
u, counts = np.unique(y_student_all_dummy_label, return_counts=True)
print(u, counts)

#カウント数の最大値を計算
student_label_max =  max(counts)

#numpy配列を各ラベル毎に分ける
y_student_per_label = []
y_student_per_img_path = []

for i in range(10):
    temp_l = y_train_img_dummy_th[y_student_all_dummy_label == i]
    print(i, ":", temp_l.shape)
    y_student_per_label.append(temp_l)
    temp_i = x_train_img_th[y_student_all_dummy_label == i]
    print(i, ":", temp_i.shape)
    y_student_per_img_path.append(temp_i)

#それぞれのラベルで最大のカウント数になるようにデータをコピー
y_student_per_label_add = []
y_student_per_img_add = []

for i in range(10):
    num = y_student_per_label[i].shape[0]
    temp_l = y_student_per_label[i]
    temp_i = y_student_per_img_path[i]
    add_num = student_label_max - num
    q, mod = divmod(add_num, num)
    print(q, mod)
    temp_l_tile = np.tile(temp_l, (q+1, 1))
    temp_i_tile = np.tile(temp_i, (q+1, 1, 1, 1))
    temp_l_add = temp_l[:mod]
    temp_i_add = temp_i[:mod]
    y_student_per_label_add.append(np.concatenate([temp_l_tile, temp_l_add], axis=0))
    y_student_per_img_add.append(np.concatenate([temp_i_tile, temp_i_add], axis=0))

#それぞれラベルのカウント数を確認
print([len(i) for i in y_student_per_label_add])

#ラベルごとのデータを合体させる
student_train_img = np.concatenate(y_student_per_img_add, axis=0)
student_train_label = np.concatenate(y_student_per_label_add, axis=0)

#元のcifar10のnumpy配列と合体
x_train_student = np.concatenate([x_train_10, student_train_img], axis=0)
y_train_student = np.concatenate([y_train_10, student_train_label], axis=0)

気になること

メモリ不足の可能性あるかもしれません。

Comments

[sin471]

メモリ不足の可能性があるのなら、タスクマネージャーを開いて確認してみては？

[meg_]

> メモリ不足の可能性あるかもしれません。
それが疑われます。ご自身の環境でしか確かめられないので確認してください。

Answer 1

私は以前メモリ不足のときにでました