Chainer ValueError: Out shape is mismatched

Question

https://github.com/cocon-ai-group/turf-tipster
上記のtrain.pyを動かした際にOut shape is mismatchedのエラーが出ます
なんどもググって見ましたが発生原因が検討もつきません。

環境はUbuntu16.04 chainer3.5.0 です
エラー発生箇所はソースコードの"＃ゲート分のデータを作る"の部分です。
勉強不足ですいませんが、エラーの正体を教えていただけると大変助かります
よろしくお願いいたします。

発生している問題・エラーメッセージ

Exception in main training loop: Out shape is mismatched
Traceback (most recent call last):
File "/home/gutsmans/.pyenv/versions/3.6.5/lib/python3.6/site-packages/chainer/training/trainer.py", line 299, in run
update()
File "/home/gutsmans/.pyenv/versions/3.6.5/lib/python3.6/site-packages/chainer/training/updater.py", line 223, in update
self.update_core()
File "/home/gutsmans/.pyenv/versions/3.6.5/lib/python3.6/site-packages/chainer/training/updater.py", line 234, in update_core
optimizer.update(loss_func, *in_arrays)
File "/home/gutsmans/.pyenv/versions/3.6.5/lib/python3.6/site-packages/chainer/optimizer.py", line 541, in update
loss = lossfun(*args, **kwds)
File "train.py", line 305, in call
r = super(An_Classifier, self).call(*args, **kwargs)
File "/home/gutsmans/.pyenv/versions/3.6.5/lib/python3.6/site-packages/chainer/links/model/classifier.py", line 114, in call
self.y = self.predictor(*args, **kwargs)
File "/home/gutsmans/works/turf-tipster/rnn_network.py", line 67, in call
l_h1[i] = l
File "cupy/core/core.pyx", line 1553, in cupy.core.core.ndarray.setitem
File "cupy/core/core.pyx", line 3207, in cupy.core.core._scatter_op
File "cupy/core/elementwise.pxi", line 807, in cupy.core.core.ufunc.call
File "cupy/core/elementwise.pxi", line 391, in cupy.core.core._get_out_args
Will finalize trainer extensions and updater before reraising the exception.
Traceback (most recent call last):
File "train.py", line 399, in <module>
main()
File "train.py", line 391, in main
trainer.run()
File "/home/gutsmans/.pyenv/versions/3.6.5/lib/python3.6/site-packages/chainer/training/trainer.py", line 313, in run
six.reraise(*sys.exc_info())
File "/home/gutsmans/.pyenv/versions/3.6.5/lib/python3.6/site-packages/six.py", line 693, in reraise
raise value
File "/home/gutsmans/.pyenv/versions/3.6.5/lib/python3.6/site-packages/chainer/training/trainer.py", line 299, in run
update()
File "/home/gutsmans/.pyenv/versions/3.6.5/lib/python3.6/site-packages/chainer/training/updater.py", line 223, in update
self.update_core()
File "/home/gutsmans/.pyenv/versions/3.6.5/lib/python3.6/site-packages/chainer/training/updater.py", line 234, in update_core
optimizer.update(loss_func, *in_arrays)
File "/home/gutsmans/.pyenv/versions/3.6.5/lib/python3.6/site-packages/chainer/optimizer.py", line 541, in update
loss = lossfun(*args, **kwds)
File "train.py", line 305, in call
r = super(An_Classifier, self).call(*args, **kwargs)
File "/home/gutsmans/.pyenv/versions/3.6.5/lib/python3.6/site-packages/chainer/links/model/classifier.py", line 114, in call
self.y = self.predictor(*args, **kwargs)
File "/home/gutsmans/works/turf-tipster/rnn_network.py", line 67, in call
l_h1[i] = l
File "cupy/core/core.pyx", line 1553, in cupy.core.core.ndarray.setitem
File "cupy/core/core.pyx", line 3207, in cupy.core.core._scatter_op
File "cupy/core/elementwise.pxi", line 807, in cupy.core.core.ufunc.call
File "cupy/core/elementwise.pxi", line 391, in cupy.core.core._get_out_args
ValueError: Out shape is mismatched

該当のソースコード

# -*- coding: utf-8 -*-
import chainer
import chainer.functions as F
import chainer.links as L
from chainer import training, datasets, iterators, optimizers
from chainer.training import extensions
from chainer.functions.evaluation import accuracy
from chainer.links import caffe as C
from chainer import reporter
import numpy as np

nn_nodes = (1500, 350, 45, 300, 150)

# ニューラルネットワークの定義をするクラス
class Turf_Tipster_NN(chainer.Chain):

	def __init__(self, n_horses, n_jockeys):
		n_units_h, n_units_v, n_units_j, n_units_r, n_units_d = nn_nodes
		super(Turf_Tipster_NN, self).__init__()
		with self.init_scope():
			# 馬モデルのステータス（モデルを保存できるようにParameterで保持しておく）
			self.vc = chainer.Parameter(np.zeros((n_horses, n_units_v), dtype=np.float32))
			self.vh = chainer.Parameter(np.zeros((n_horses, n_units_v), dtype=np.float32))
			# 馬モデルのレイヤー
			self.le = L.EmbedID(n_horses, n_horses)
			self.l1 = L.Linear(n_horses, n_units_h)
			self.l2 = L.LSTM(n_units_h, n_units_v)
			# 騎手モデルのレイヤー
			self.re = L.EmbedID(n_jockeys, n_jockeys)
			self.r1 = L.Linear(n_jockeys, n_units_j)
			# レースモデルのレイヤー
			self.m1 = L.EmbedID(11, 11)
			self.m2 = L.EmbedID(8, 8)
			self.m3 = L.EmbedID(4, 4)
			self.m4 = L.EmbedID(4, 4)
			self.j1 = L.Linear(n_units_v + n_units_j + 11 + 8 + 4 + 4, n_units_r)
			self.j2 = L.Linear(n_units_r, n_units_r)
			self.j3 = L.Linear(n_units_r, n_units_d)

	# 引数は(レースメタ情報, グリッド情報)で着順になっている
	def __call__(self, t, train=True):
		grid = t[0]
		num_gates = len(grid)
		cp = self.xp
		# 馬モデルのステータス
		l_vc = cp.zeros((num_gates, self.vc.data.shape[1]), dtype=cp.float32)
		l_vh = cp.zeros((num_gates, self.vh.data.shape[1]), dtype=cp.float32)
		# ゲート分のステータスを作る
		for i in range(num_gates):
			l, r, m1, m2, m3, m4 = grid[i] # l, rは(馬番号, 騎手番号)
			_l = l
			if cp != np:
				l = chainer.cuda.to_cpu(l)
			l = int(l[0])
			l_vc[i][:] = self.vc.data[l]
			l_vh[i][:] = self.vh.data[l]
		l_h1 = cp.zeros((num_gates), dtype=cp.int32)
		r_h1 = cp.zeros((num_gates), dtype=cp.int32)
		t_m1 = cp.zeros((num_gates), dtype=cp.int32)
		t_m2 = cp.zeros((num_gates), dtype=cp.int32)
		t_m3 = cp.zeros((num_gates), dtype=cp.int32)
		t_m4 = cp.zeros((num_gates), dtype=cp.int32)
		# ゲート分のデータを作る
		for i in range(num_gates):
			l, r, m1, m2, m3, m4 = grid[i]
			l_h1[i] = l
			r_h1[i] = r
			t_m1[i] = m1
			t_m2[i] = m2
			t_m3[i] = m3
			t_m4[i] = m4
		# 馬モデルを計算
		self.l2.set_state(chainer.Variable(l_vc), chainer.Variable(l_vh))
		l_l1 = F.tanh(self.le(l_h1))
		l_l2 = F.tanh(self.l1(l_l1))
		l_l3 = F.tanh(self.l2(l_l2))
		# 馬モデルのステータスを保存
		if train == True:
			l_wc = cp.copy(self.vc.data)
			l_wh = cp.copy(self.vh.data)
			for i in range(num_gates):
				l, r, m1, m2, m3, m4 = grid[i] # l, rは(馬番号, 騎手番号)
				if cp != np:
					l = chainer.cuda.to_cpu(l)
				l = int(l[0])
				l_wc[l][:] = self.l2.c.data[i]
				l_wh[l][:] = self.l2.h.data[i]
			self.vc.copydata(chainer.Variable(l_wc))
			self.vh.copydata(chainer.Variable(l_wh))
		# レースモデルを計算
		r_h1 = F.tanh(self.re(r_h1))
		r_h2 = F.tanh(self.r1(r_h1))
		j_m1 = self.m1(t_m1)
		j_m2 = self.m2(t_m2)
		j_m3 = self.m3(t_m3)
		j_m4 = self.m4(t_m4)
		j_h1 = F.concat([l_l3, r_h2, j_m1, j_m2, j_m3, j_m4], axis=1)
		j_h2 = F.dropout(F.sigmoid(self.j1(j_h1)))
		j_h3 = F.sigmoid(self.j2(j_h2))
		return F.sigmoid(j_h3)

	def reset_state(self):
		self.l2.reset_state()
		self.vc.copydata(chainer.Variable(cp.zeros(self.vc.data.shape, dtype=cp.float32)))
		self.vh.copydata(chainer.Variable(cp.zeros(self.vh.data.shape, dtype=cp.float32)))

試したこと

ipythonでembed()を入れ試して見ましたがどうもなりませんでした

Answer 1

Ubuntu16.04 chainer4.1で同様のエラーが出ましたが、chainerを3.1までダウングレードしたところ解消しました。

pip install chainer==3.1.0

cupyもダウングレードされ、2.1.0.1が入りました。

Comments

[YukiSato]

ありがとうございます。
おっしゃる通り、chainerとcupyのバージョンの問題でした。
chainerのバージョンを3.5.0に下げcupyもpip install cupy==2.4.0 --no-cache-dir で入れ直したところエラー解消しました！