scikit-learn,lightGBM使用時のエラー解消

Question

前提

kaggleのHouse Pricesをやっています。
lightGBM, scikit-learnでエラーが出てしまったのですが、どのように解消したらよいかわかりません。

モデルを関数化し、動かせるようにしています。この関数はデータの前処理前は正常に動いたのですが、前処理後にもう一度動かすとこのようなエラーが出るようになりました。
原因の一つに、前処理の中でデータフレーム内のデータの数を削減（外れ値を除外）したことがあるのではないかと考えています。

よろしくお願いします。

実現したいこと

エラーを解消する

発生している問題・エラーメッセージ

---

KeyError Traceback (most recent call last)
Input In [99], in <cell line: 1>()
----> 1 train_cv(train_X, train_Y, train_df_id, params, n_splits=5)

Input In [31], in train_cv(input_x, input_y, input_id, params, n_splits)
27 print('idx_tr', idx_tr.shape)
28 print('idx_va', idx_va.shape)
---> 29 x_tr, y_tr = input_x.loc[idx_tr], input_y.loc[idx_tr]
30 x_va, y_va = input_x.loc[idx_va], input_y.loc[idx_va]
31 print(x_tr.shape, y_tr.shape)

File ~\anaconda3\lib\site-packages\pandas\core\indexing.py:967, in _LocationIndexer.getitem(self, key)
964 axis = self.axis or 0
966 maybe_callable = com.apply_if_callable(key, self.obj)
--> 967 return self._getitem_axis(maybe_callable, axis=axis)

File ~\anaconda3\lib\site-packages\pandas\core\indexing.py:1191, in _LocIndexer._getitem_axis(self, key, axis)
1188 if hasattr(key, "ndim") and key.ndim > 1:
1189 raise ValueError("Cannot index with multidimensional key")
-> 1191 return self._getitem_iterable(key, axis=axis)
1193 # nested tuple slicing
1194 if is_nested_tuple(key, labels):

File ~\anaconda3\lib\site-packages\pandas\core\indexing.py:1132, in _LocIndexer._getitem_iterable(self, key, axis)
1129 self._validate_key(key, axis)
1131 # A collection of keys
-> 1132 keyarr, indexer = self._get_listlike_indexer(key, axis)
1133 return self.obj._reindex_with_indexers(
1134 {axis: [keyarr, indexer]}, copy=True, allow_dups=True
1135 )

File ~\anaconda3\lib\site-packages\pandas\core\indexing.py:1327, in _LocIndexer._get_listlike_indexer(self, key, axis)
1324 ax = self.obj._get_axis(axis)
1325 axis_name = self.obj._get_axis_name(axis)
-> 1327 keyarr, indexer = ax._get_indexer_strict(key, axis_name)
1329 return keyarr, indexer

File ~\anaconda3\lib\site-packages\pandas\core\indexes\base.py:5782, in Index._get_indexer_strict(self, key, axis_name)
5779 else:
5780 keyarr, indexer, new_indexer = self._reindex_non_unique(keyarr)
-> 5782 self._raise_if_missing(keyarr, indexer, axis_name)
5784 keyarr = self.take(indexer)
5785 if isinstance(key, Index):
5786 # GH 42790 - Preserve name from an Index

File ~\anaconda3\lib\site-packages\pandas\core\indexes\base.py:5845, in Index._raise_if_missing(self, key, indexer, axis_name)
5842 raise KeyError(f"None of [{key}] are in the [{axis_name}]")
5844 not_found = list(ensure_index(key)[missing_mask.nonzero()[0]].unique())
-> 5845 raise KeyError(f"{not_found} not in index")

KeyError: '[313, 332, 335, 440, 496, 706, 898, 1024, 1044, 1182, 1298] not in index'

エラーメッセージ
KeyError: '[313, 332, 335, 440, 496, 706, 898, 1024, 1044, 1182, 1298] not in index'

該当のソースコード

``python

params = {
    'boosting_type': 'gbdt',
    'objective': 'regression', 
    'metric': 'rmse',
    'learning_rate': 0.1,
    'num_leaves': 16,
    'n_estimators': 100000,
    "random_state": 123,
    "importance_type": "gain",
}

n_splits = 5

from sklearn.metrics import mean_squared_error
rmses = []
models = []
oof = np.zeros(len(train_X))

def train_cv(input_x, input_y, input_id, params, n_splits=5):
    imp = pd.DataFrame()
    print(imp)
    cv = list(KFold(n_splits=n_splits, shuffle=True, random_state=123).split(input_x, input_y))
    for nfold in np.arange(n_splits):
        print("-"*20, nfold, "-"*20)
        print(cv[nfold])
        idx_tr, idx_va = cv[nfold][0], cv[nfold][1]
        print('idx_tr', idx_tr.shape)
        print('idx_va', idx_va.shape)
        x_tr, y_tr = input_x.loc[idx_tr], input_y.loc[idx_tr]
        x_va, y_va = input_x.loc[idx_va], input_y.loc[idx_va]
        print(x_tr.shape, y_tr.shape)
        print(x_va.shape, y_va.shape)
        
        model = lgb.LGBMRegressor(**params)
        models.append(model)
        model.fit(x_tr, y_tr, eval_set=[(x_tr,y_tr), (x_va,y_va)], early_stopping_rounds=1000, verbose=100)
        
        y_tr_pred = model.predict(x_tr)
        print(y_tr_pred)
        y_va_pred = model.predict(x_va)
        print(y_va_pred)   
        tmp_rmse = np.sqrt(mean_squared_error(np.log(y_va), np.log(y_va_pred)))
        print('tmp_rmse', tmp_rmse)
        rmses.append(tmp_rmse)
        oof[idx_va] = y_va_pred
        
        _imp = pd.DataFrame({"col":input_x.columns, "imp":model.feature_importances_, "nfold":nfold})
        imp = pd.concat([imp, _imp], axis=0, ignore_index=True)

        imp = imp.groupby("col")["imp"].agg(["mean", "std"])
        imp.columns = ["imp", "imp_std"]
        imp = imp.reset_index(drop=False)
            
    return imp

params = {
    'boosting_type': 'gbdt',
    'objective': 'regression', 
    'metric': 'rmse',
    'learning_rate': 0.1,
    'num_leaves': 16,
    'n_estimators': 100000,
    "random_state": 123,
    "importance_type": "gain",
}

n_splits = 5

from sklearn.metrics import mean_squared_error
rmses = []
models = []
oof = np.zeros(len(train_X))

train_cv(train_X, train_Y, train_df_id, params, n_splits=5)

Answer 1

input_xとinput_yのindexをresetすると解決できます

Comments

[chapchap1234]

解決しました！ありがとうございます！