from tqdm import tqdm import concurrent.futures import numpy as np class PredictData: def __init__(self,X_train_trans,X_test_trans,y_train,y_test): self.X_train_trans=X_train_trans self.X_test_trans=X_test_trans self.y_train=y_train self.y_test=y_test print(self.X_train_trans.shape) print(self.X_test_trans.shape) def forcount(self,random_st): print('start') param=[_ for _ in range(5)] for i in tqdm(range(1,3)): for j in tqdm(range(1,2)): for k in tqdm(np.arange(.01,.02,.01)): forest=GradientBoostingClassifier(random_state=random_st,n_estimators=i,max_depth=j,learning_rate=k) forest=forest.fit(self.X_train_trans,self.y_train) if forest.score(self.X_test_trans,self.y_test) > param[0]: param[0]=forest.score(self.X_test_trans,self.y_test) param[1]=forest.score(self.X_train_trans,self.y_train) param[2]=i param[3]=j param[4]=k return param def GraForestCl_predict_data(self): print(self.forcount(0)) print('並行') with concurrent.futures.ProcessPoolExecutor(2) as executor: print('開始') futures=[executor.submit(self.forcount,i) for i in tqdm(range(2))] print('途中') for future in tqdm(concurrent.futures.as_completed(futures)): # test_score,train_score,n_es,max,rate=future.result() print(future.result()) print() if __name__ =='__main__': prd=PredictData(X_train_trans,X_test_trans,y_train,y_test) prd.GraForestCl_predict_data() ''' (8210, 4140) (2733, 4140) start 100%|███████████████████████████████████████████████████████████████████████████| 1/1 [00:03<00:00, 3.15s/it] 100%|███████████████████████████████████████████████████████████████████████████| 1/1 [00:03<00:00, 3.15s/it] 100%|███████████████████████████████████████████████████████████████████████████| 1/1 [00:01<00:00, 1.26s/it] 100%|███████████████████████████████████████████████████████████████████████████| 1/1 [00:01<00:00, 1.26s/it] 100%|███████████████████████████████████████████████████████████████████████████| 2/2 [00:04<00:00, 2.22s/it] [0.7939992682034395, 0.7884287454323995, 1, 1, 0.01] 並行 開始 100%|███████████████████████████████████████████████████████████████████████████| 2/2 [00:00<00:00, 48.65it/s] 途中 0it [00:00, ?it/s] MemoryError exception: no description File "data_predicting.py", line 67, in GraForestCl_predict_data print(future.result()) File "data_predicting.py", line 78, in <module> prd.GraForestCl_predict_data() '''
forcount関数単体で与えたデータ基に処理すると正常に動きますが、並行処理しようとするとうまくいきません。print(future.result())でMemoryErrorを吐いています。これは自分の使っているPCの性能の問題なのでしょうか?データを前処理することで解決できますか?あいまいな表現で申し訳ありません。
あなたの回答
tips
プレビュー