現在、SSDを用いた物体検出を実施しようとこのサイトを参考に学習を進めています。
上記サイトのコードをコピーして実行したところ、google colaboratoryでは学習できましたが、
azure上で実行するとtrainer.run()時に下記エラーが出ます。
なにとぞご教授よろしくお願いします。
python
1Exception in main training loop: merge_sort: failed to synchronize: unspecified launch failure 2Traceback (most recent call last): 3 File "/usr/local/lib/python3.6/dist-packages/chainer/training/trainer.py", line 316, in run 4 update() 5 File "/usr/local/lib/python3.6/dist-packages/chainer/training/updaters/standard_updater.py", line 175, in update 6 self.update_core() 7 File "/usr/local/lib/python3.6/dist-packages/chainer/training/updaters/standard_updater.py", line 187, in update_core 8 optimizer.update(loss_func, *in_arrays) 9 File "/usr/local/lib/python3.6/dist-packages/chainer/optimizer.py", line 825, in update 10 loss = lossfun(*args, **kwds) 11 File "/usr/local/lib/python3.6/dist-packages/chainer/link.py", line 294, in __call__ 12 out = forward(*args, **kwargs) 13 File "<ipython-input-7-393c695393be>", line 18, in forward 14 mb_locs, mb_confs, gt_mb_locs, gt_mb_labels, self.k) 15 File "/usr/local/lib/python3.6/dist-packages/chainercv/links/model/ssd/multibox_loss.py", line 91, in multibox_loss 16 hard_negative = _hard_negative(conf_loss.array, positive, k) 17 File "/usr/local/lib/python3.6/dist-packages/chainercv/links/model/ssd/multibox_loss.py", line 19, in _hard_negative 18 rank = (x * (positive - 1)).argsort(axis=1).argsort(axis=1) 19 File "cupy/core/core.pyx", line 627, in cupy.core.core.ndarray.argsort 20 File "cupy/core/core.pyx", line 644, in cupy.core.core.ndarray.argsort 21 File "cupy/core/_routines_sorting.pyx", line 101, in cupy.core._routines_sorting._ndarray_argsort 22 File "cupy/cuda/thrust.pyx", line 135, in cupy.cuda.thrust.argsort 23Will finalize trainer extensions and updater before reraising the exception. 24 25RuntimeErrorTraceback (most recent call last) 26<ipython-input-17-041e2033e90a> in <module> 27----> 1 trainer.run() 28 29/usr/local/lib/python3.6/dist-packages/chainer/training/trainer.py in run(self, show_loop_exception_msg) 30 347 f.write('Traceback (most recent call last):\n') 31 348 traceback.print_tb(sys.exc_info()[2]) 32--> 349 six.reraise(*exc_info) 33 350 finally: 34 351 for _, entry in extensions: 35 36/usr/local/lib/python3.6/dist-packages/six.py in reraise(tp, value, tb) 37 691 if value.__traceback__ is not tb: 38 692 raise value.with_traceback(tb) 39--> 693 raise value 40 694 finally: 41 695 value = None 42 43/usr/local/lib/python3.6/dist-packages/chainer/training/trainer.py in run(self, show_loop_exception_msg) 44 314 self.observation = {} 45 315 with reporter.scope(self.observation): 46--> 316 update() 47 317 for name, entry in extensions: 48 318 if entry.trigger(self): 49 50/usr/local/lib/python3.6/dist-packages/chainer/training/updaters/standard_updater.py in update(self) 51 173 52 174 """ 53--> 175 self.update_core() 54 176 self.iteration += 1 55 177 56 57/usr/local/lib/python3.6/dist-packages/chainer/training/updaters/standard_updater.py in update_core(self) 58 185 59 186 if isinstance(in_arrays, tuple): 60--> 187 optimizer.update(loss_func, *in_arrays) 61 188 elif isinstance(in_arrays, dict): 62 189 optimizer.update(loss_func, **in_arrays) 63 64/usr/local/lib/python3.6/dist-packages/chainer/optimizer.py in update(self, lossfun, *args, **kwds) 65 823 if lossfun is not None: 66 824 use_cleargrads = getattr(self, '_use_cleargrads', True) 67--> 825 loss = lossfun(*args, **kwds) 68 826 if use_cleargrads: 69 827 self.target.cleargrads() 70 71/usr/local/lib/python3.6/dist-packages/chainer/link.py in __call__(self, *args, **kwargs) 72 292 # forward is implemented in the child classes 73 293 forward = self.forward # type: ignore 74--> 294 out = forward(*args, **kwargs) 75 295 76 296 # Call forward_postprocess hook 77 78<ipython-input-7-393c695393be> in forward(self, imgs, gt_mb_locs, gt_mb_labels) 79 16 mb_locs, mb_confs = self.model(imgs) 80 17 loc_loss, conf_loss = multibox_loss( 81---> 18 mb_locs, mb_confs, gt_mb_locs, gt_mb_labels, self.k) 82 19 loss = loc_loss * self.alpha + conf_loss 83 20 84 85/usr/local/lib/python3.6/dist-packages/chainercv/links/model/ssd/multibox_loss.py in multibox_loss(mb_locs, mb_confs, gt_mb_locs, gt_mb_labels, k, comm) 86 89 87 90 conf_loss = _elementwise_softmax_cross_entropy(mb_confs, gt_mb_labels) 88---> 91 hard_negative = _hard_negative(conf_loss.array, positive, k) 89 92 conf_loss *= xp.logical_or( 90 93 positive, hard_negative).astype(conf_loss.dtype) 91 92/usr/local/lib/python3.6/dist-packages/chainercv/links/model/ssd/multibox_loss.py in _hard_negative(x, positive, k) 93 17 94 18 def _hard_negative(x, positive, k): 95---> 19 rank = (x * (positive - 1)).argsort(axis=1).argsort(axis=1) 96 20 hard_negative = rank < (positive.sum(axis=1) * k)[:, np.newaxis] 97 21 return hard_negative 98 99cupy/core/core.pyx in cupy.core.core.ndarray.argsort() 100 101cupy/core/core.pyx in cupy.core.core.ndarray.argsort() 102 103cupy/core/_routines_sorting.pyx in cupy.core._routines_sorting._ndarray_argsort() 104 105cupy/cuda/thrust.pyx in cupy.cuda.thrust.argsort() 106 107RuntimeError: merge_sort: failed to synchronize: unspecified launch failure

あなたの回答
tips
プレビュー