常連

### 前提・実現したいこと
teratailで初めて質問させていただきます。

Unityではじめての自作ゲーム(3Dballに似た何か)を作り、ML-Agentsで学習させようとして、いつものようにanaconda powershell promptにmlagents-learn config/ppo/3B.yaml --run-id=3B1223jと入力したところ、unityのロゴまでは出たのですが、再生ボタンを押すと、以下のようなエラーメッセージが表示されました。

**__特に最後の行で、Please add an entry in the configuration file for 3BBrain, or set default_settings.
とあるのは具体的に何をすればよいのでしょうか。__**


Traceback (most recent call last):
  File "C:\Users\daisuke\anaconda3\envs\mlagents\Scripts\mlagents-learn-script.py", line 33, in <module>
    sys.exit(load_entry_point('mlagents', 'console_scripts', 'mlagents-learn')())
  File "c:\users\daisuke\desktop\ml-agents\ml-agents\mlagents\trainers\learn.py", line 250, in main
    run_cli(parse_command_line())
  File "c:\users\daisuke\desktop\ml-agents\ml-agents\mlagents\trainers\learn.py", line 246, in run_cli
    run_training(run_seed, options)
  File "c:\users\daisuke\desktop\ml-agents\ml-agents\mlagents\trainers\learn.py", line 125, in run_training
    tc.start_learning(env_manager)
  File "c:\users\daisuke\desktop\ml-agents\ml-agents-envs\mlagents_envs\timers.py", line 305, in wrapped
    return func(*args, **kwargs)
  File "c:\users\daisuke\desktop\ml-agents\ml-agents\mlagents\trainers\trainer_controller.py", line 173, in start_learning
    self._reset_env(env_manager)
  File "c:\users\daisuke\desktop\ml-agents\ml-agents-envs\mlagents_envs\timers.py", line 305, in wrapped
    return func(*args, **kwargs)
  File "c:\users\daisuke\desktop\ml-agents\ml-agents\mlagents\trainers\trainer_controller.py", line 107, in _reset_env
    self._register_new_behaviors(env_manager, env_manager.first_step_infos)
  File "c:\users\daisuke\desktop\ml-agents\ml-agents\mlagents\trainers\trainer_controller.py", line 268, in _register_new_behaviors
    self._create_trainers_and_managers(env_manager, new_behavior_ids)
  File "c:\users\daisuke\desktop\ml-agents\ml-agents\mlagents\trainers\trainer_controller.py", line 166, in _create_trainers_and_managers
    self._create_trainer_and_manager(env_manager, behavior_id)
  File "c:\users\daisuke\desktop\ml-agents\ml-agents\mlagents\trainers\trainer_controller.py", line 125, in _create_trainer_and_manager
    trainer = self.trainer_factory.generate(brain_name)
  File "c:\users\daisuke\desktop\ml-agents\ml-agents\mlagents\trainers\trainer\trainer_factory.py", line 59, in generate
    trainer_settings = self.trainer_config[behavior_name]
  File "c:\users\daisuke\desktop\ml-agents\ml-agents\mlagents\trainers\settings.py", line 732, in __missing__
    f"The behavior name {key} has not been specified in the trainer configuration. "
mlagents.trainers.exception.TrainerConfigError: The behavior name 3BBrain has not been specified in the trainer configuration. Please add an entry in the configuration file for 3BBrain, or set default_settings.


### 試したこと

エラーコードを検索にかけてみましたが思うようにヒットしませんでした。
ご回答お願い致します。

![](86ffaae278ac7313cb165b52ba93bfb6.png)

Unity ML-Agents エラーコードの意味が分かりませんm(_ _m)

以下のプログラムは途中までのものですが、google colabだと動いたのですが、会社のGPUを積んだマシンだとエラーがでました。エラーが出た原因が知りたいです。
動かしたプログラム
#####
```ここに言語を入力
import gym
from creversi.gym_reversi.envs import ReversiVecEnv
from creversi import *

import os
import datetime
import math
import random
import numpy as np
from collections import namedtuple
from itertools import count
from tqdm import tqdm_notebook as tqdm
import matplotlib.pyplot as plt

import torch
import torch.nn as nn
import torch.optim as optim
import torch.nn.functional as F

BATCH_SIZE = 256

vecenv = ReversiVecEnv(BATCH_SIZE)

# if gpu is to be used
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

######################################################################
# Replay Memory

Transition = namedtuple('Transition',
                        ('state', 'action', 'next_state', 'next_actions', 'reward'))


class ReplayMemory(object):

    def __init__(self, capacity):
        self.capacity = capacity
        self.memory = []
        self.position = 0

    def push(self, *args):
        """Saves a transition."""
        if len(self.memory) < self.capacity:
            self.memory.append(None)
        self.memory[self.position] = Transition(*args)
        self.position = (self.position + 1) % self.capacity

    def sample(self, batch_size):
        return random.sample(self.memory, batch_size)

    def __len__(self):
        return len(self.memory)

######################################################################
# DQN

k = 192
fcl_units = 256
class DQN(nn.Module):

    def __init__(self):
        super(DQN, self).__init__()
        self.conv1 = nn.Conv2d(2, k, kernel_size=3, padding=1)
        self.bn1 = nn.BatchNorm2d(k)
        self.conv2 = nn.Conv2d(k, k, kernel_size=3, padding=1)
        self.bn2 = nn.BatchNorm2d(k)
        self.conv3 = nn.Conv2d(k, k, kernel_size=3, padding=1)
        self.bn3 = nn.BatchNorm2d(k)
        self.conv4 = nn.Conv2d(k, k, kernel_size=3, padding=1)
        self.bn4 = nn.BatchNorm2d(k)
        self.conv5 = nn.Conv2d(k, k, kernel_size=3, padding=1)
        self.bn5 = nn.BatchNorm2d(k)
        self.conv6 = nn.Conv2d(k, k, kernel_size=3, padding=1)
        self.bn6 = nn.BatchNorm2d(k)
        self.conv7 = nn.Conv2d(k, k, kernel_size=3, padding=1)
        self.bn7 = nn.BatchNorm2d(k)
        self.conv8 = nn.Conv2d(k, k, kernel_size=3, padding=1)
        self.bn8 = nn.BatchNorm2d(k)
        self.conv9 = nn.Conv2d(k, k, kernel_size=3, padding=1)
        self.bn9 = nn.BatchNorm2d(k)
        self.conv10 = nn.Conv2d(k, k, kernel_size=3, padding=1)
        self.bn10 = nn.BatchNorm2d(k)
        self.fcl1 = nn.Linear(k * 64, fcl_units)
        self.fcl2 = nn.Linear(fcl_units, 65)

    def forward(self, x):
        x = F.relu(self.bn1(self.conv1(x)))
        x = F.relu(self.bn2(self.conv2(x)))
        x = F.relu(self.bn3(self.conv3(x)))
        x = F.relu(self.bn4(self.conv4(x)))
        x = F.relu(self.bn5(self.conv5(x)))
        x = F.relu(self.bn6(self.conv6(x)))
        x = F.relu(self.bn7(self.conv7(x)))
        x = F.relu(self.bn8(self.conv8(x)))
        x = F.relu(self.bn9(self.conv9(x)))
        x = F.relu(self.bn10(self.conv10(x)))
        x = F.relu(self.fcl1(x.view(-1, k * 64)))
        x = self.fcl2(x)
        return x.tanh()

def get_states(envs):
    features_vec = np.zeros((BATCH_SIZE, 2, 8, 8), dtype=np.float32)
    for i, env in enumerate(envs):
        env.board.piece_planes(features_vec[i])
    return torch.from_numpy(features_vec).to(device)
```
この最後のreturn torch.from_numpy(features_vec).to(device)のところで、、google colabでは
エラーが出ないのですが、会社のGPUを積んだLinuxだと
RuntimeError: CUDA error: out of memory
CUDA kernel errors might be asynchronously reported at some other API call,so the stacktrace below might be incorrect.
For debugging consider passing CUDA_LAUNCH_BLOCKING=1.
というエラーがでます。ググったりして調べたのですが、原因が分かりませんでした。
会社のGPUの環境は、下記の通りです。
-bash-4.2$ lspci | grep -i nvidia
3b:00.0 VGA compatible controller: NVIDIA Corporation TU102 [GeForce RTX 2080 Ti] (rev a1)
3b:00.1 Audio device: NVIDIA Corporation TU102 High Definition Audio Controller (rev a1)
3b:00.2 USB controller: NVIDIA Corporation TU102 USB 3.1 Host Controller (rev a1)
3b:00.3 Serial bus controller [0c80]: NVIDIA Corporation TU102 USB Type-C UCSI Controller (rev a1)
af:00.0 VGA compatible controller: NVIDIA Corporation GP102 [GeForce GTX 1080 Ti] (rev a1)
af:00.1 Audio device: NVIDIA Corporation GP102 HDMI Audio Controller (rev a1)

ご教示いただけますと幸いです。
何卒、よろしくお願い申し上げます。

強化学習でRuntimeError: CUDA error: out of memoryが出る

## 強化学習に関する質問

方策ベースのアルゴリズムを実装するとき、例えばエージェントの行動数が5次元だとすると、一つの方策(確率)だけが1になりあとの方策(確率)は全て０になることはあるのでしょうか。
## 状況

使用している言語:python
フレームワーク:pytorch
方策はpytorchのF.softmaxで出力させています


強化学習の方策の出力結果が1か0だけになる

### 前提・実現したいこと

unityでの強化学習を自分が作ったエージェントや環境で行いたく、コードを作成したがエラーが出て進めなくなってしまった。


### 発生している問題・エラーメッセージ

```
Assets\CarAgent.cs(37,26): error CS0115: 'CarAgent.CollectObservations()': no suitable method found to override
Assets\CarAgent.cs(46,26): error CS0115: 'CarAgent.OnActionReceived(float[], string)': no suitable method found to override
Assets\CarAgent.cs(28,26): error CS0115: 'CarAgent.AgentReset()': no suitable method found to override
```

### 該当のソースコード

```ここに言語名を入力
using System.Collections;
using System.Collections.Generic;
using UnityEngine;
using Unity.MLAgents;
using Unity.MLAgents.Sensors;
using Unity.MLAgents.Actuators;

public class CarAgent : Agent
{
    
    private RayPerceptionSensor rayPer;
    private Rigidbody rigidbody;

    private Vector3 initPosition;
    private Quaternion initRotation;
    private bool crush;

    public override void OnEpisodeBegin()
    {

        this.rayPer = GetComponent<RayPerceptionSensor>();
        this.rigidbody = GetComponent<Rigidbody>();

        this.initPosition = this.transform.position;
        this.initRotation = this.transform.rotation;
    }

    public override void AgentReset()
    {
        this.transform.position = this.initPosition;
        this.transform.rotation = this.initRotation;
        rigidbody.velocity = new Vector3(0, 0, 0);
        rigidbody.anglarVelocity = new Vector3(0, 0, 0);
        this.crush = false;
    }

    public override void CollectObservations()
    {
        float rayDistance = 50.0f;
        float[] rayAngles = { 0f, 45f, 90f, 135f, 180f, 110f, 70f };
        string[] detectableObjects;
        detectableObjects = new string[] { "car", "wall" };
        addvectoerObs(rayPer.Perceive(rayDistance, rayAngles, detectableObjects, 1f, 0f));
    }

    public override void OnActionReceived(float[]vectorAction,string textAction)
    {
        float handle = Mathf.Clamp(vectorAction[0], -1.0f, 1.0f) * 1.5f;

        this.gameObject.transform.Rotate(0, handle, 0);
        this.rigidbody.velocity = this.gameObject.transform.rotation * new Vector3(0, 0, 20);

        AddReward(0.001f);

        if (this.crush) Done();
    }

    void OnCollisionEnter(Collision collision)
    {
        this.crush = true;
    }

}

```

### 試したこと

Agentの基底クラスにOnCollisionEnterがそんざいしないということなのか。
なにかほかの部分でのコードが間違っているのか。

### 補足情報（FW/ツールのバージョンなど）

ここにより詳細な情報を記載してください。

no suitable method found to overrideの解決（unity　ml-agents）

### 実現したいこと

numyとthreadingでOpen AI gym のcartpole-v1を学習させたい

### 前提
https://qiita.com/s05319ss/items/2fe9bfe562fea1707e79

上の記事を参考にA3CでOpen AI gym のcartpole-v1を学習させているが、うまく学習ができない

### 発生している問題・エラーメッセージ
獲得報酬が20ほどであり、最大報酬である300(私が報酬をいじっています)を獲得できるよううまく学習ができていない


### 該当のソースコード

```Python
import numpy as np
import gym
import matplotlib.pylab as plt
import random
import threading
import copy

class Environment:
    def __init__(self,batchsize=1):
        self.batchsize=batchsize
        self.state=np.zeros((batchsize, 4))
        self.step=np.zeros(batchsize)
        self.score=np.zeros(batchsize)
        self.flag=np.ones(batchsize).astype(bool)
        self.environment=gym.make('CartPole-v1', render_mode='rgb_array')
        initial_state, info = self.environment.reset()
        for i in range(4):
            self.state[0][i] = initial_state[i]
            
    def transition(self,action):
        a = np.copy(action)
        for i in range(self.batchsize):
            next_state, reward, terminated, truncated, info = self.environment.step(int(a[i]))
            self.flag[i] = not terminated
            if self.flag[i]==True:
                self.score[i]+= reward
                self.step[i]+=1
                if self.step[i]>=200:
                    self.flag[i]=False
                    self.score[i]+= 100
            else:
                self.score[i]+= (-1)
            for j in range(4):
                self.state[i][j] = next_state[j]
        

    def initialize(self, flag=None):
        if flag is None:
            flag = np.ones(self.state.shape[0], dtype=bool)
        self.state[flag]*=0
        self.step[flag]*=0
        self.score[flag]*=0
        self.flag[flag]=np.ones(np.sum(flag)).astype(bool)
        initial_state, info = self.environment.reset()
        for i in range(4):
            self.state[0][i] = initial_state[i]


# Agent to play the above RL task with finite, discrete action space.
class Agent_Ex1_Discrete:
    def __init__(self):
        self.sharedbody=SharedBody()
        self.vhead=Affine()
        self.pihead=Affine()
        self.V=None
        self.probdist=None
        self.a=None
        self.action_kinds=None
        self.H=None
        self.action_number = 2
        
    def action(self,state,params):
        s=copy.deepcopy(state)
        latentvect=self.sharedbody.forward(s,params['sharedbody'])
        self.V=self.vhead.forward(latentvect,params['vhead'])
        preprobdist=self.pihead.forward(latentvect,params['pihead'])
        self.probdist=softmax(preprobdist)
        self.action_kinds = np.zeros(self.probdist.shape[0])
        self.a=np.zeros([self.probdist.shape[0],self.action_number])
        for i in range(self.action_kinds.shape[0]):
            self.action_kinds[i] = np.random.choice(a=self.action_number,p=self.probdist[i, :])
            self.a[i][int(self.action_kinds[i])] = 1
        self.H=-np.sum(self.probdist*np.log(self.probdist),axis=1).reshape(-1,1)
        return self.a
  #def gradsは元の記事と同じ
　def initialparams(self):
        params={}
        params['sharedbody']={}
        params['vhead']={}
        params['pihead']={}
        params['sharedbody']['W1']=np.random.randn(4,10)* 0.01
        params['sharedbody']['b1']=np.zeros(10)
        params['sharedbody']['W2']=np.random.randn(10,10)/np.sqrt(10)
        params['sharedbody']['b2']=np.zeros(10)
        params['vhead']['W']=np.random.randn(10,1)/np.sqrt(10)
        params['vhead']['b']=np.zeros(1)
        params['pihead']['W']=np.random.randn(10,2)/np.sqrt(10)
        params['pihead']['b']=np.zeros(2)
        return params

#def softmax,class SharedBody,class Affine, class Sigmoid, class Tanhは元の記事と同じ

    
# Global computation in the A3C algorithm.
class A3C:
    def __init__(self,t_max,T_max,agents,environments,params=None,gamma=0.99,alpha=0.99,learningrate=7e-4,vweight=0.5,entropyweight=0.01,epsilon=0.1,epoch=10):
        self.t_max=t_max
        self.T_max=T_max
        self.N=len(agents)
        self.gamma=gamma
        self.lr=learningrate
        self.kappa=vweight
        self.beta=entropyweight
        self.epsilon=epsilon
        self.c=epoch
        self.scorehistory=np.array([])
        if params==None:
            self.params=agents[0].initialparams()
        else:
            self.params=params.copy
        self.g={}
        for key1 in self.params.keys():
            self.g[key1]={}
            for key2 in self.params[key1].keys():
                self.g[key1][key2]=np.zeros_like(self.params[key1][key2])
        self.alpha=alpha
        self.T=0
        self.threadinstances={}
        for i in range(self.N):
            self.threadinstances[i]=A3Cthread(agents[i],environments[i])
        self.accumulatedscores=0.0
        self.numberofsamples=0
    def update(self,threadinstance):
        while self.T<self.T_max:
            eta=-self.T*self.lr/float(self.T_max)+self.lr
            threadgrads,cnt,ifterminated,threadscore=threadinstance.grads(self.params,self.gamma,self.t_max,self.kappa,self.beta)
            for key1 in self.params.keys():
                for key2 in self.params[key1].keys():
                    self.g[key1][key2]*=self.alpha
                    self.g[key1][key2]+=(1-self.alpha)*threadgrads[key1][key2]**2
                    self.params[key1][key2]-=eta*threadgrads[key1][key2]/np.sqrt(self.g[key1][key2]+self.epsilon)
            self.T+=cnt
            self.accumulatedscores+=ifterminated*threadscore
            self.numberofsamples+=ifterminated
            if self.numberofsamples>self.c:
                self.scorehistory=np.append(self.scorehistory,self.accumulatedscores/float(max(1,self.numberofsamples)))
                self.accumulatedscores*=0
                self.numberofsamples*=0
                print(str(self.T)+" steps done with average score:")
                print(self.scorehistory[-1])
    def train(self):
        threads=[]
        for i in range(self.N):
            threads.append(threading.Thread(target=self.update,args=(self.threadinstances[i],)))
        for i in range(self.N):
            threads[i].start()
        for i in range(self.N):
            threads[i].join()
        return self.params

# Local computation for each A3C thread.
class A3Cthread:
    def __init__(self,agent,environment):
        self.agent=agent
        self.environment=environment
        self.environment.initialize()
        self.t=0
        self.vlast=None
        self.alast=None
        
    def grads(self,globalparams,gamma,t_max,kappa=0.5,beta=0.01):
        params={}
        grads={}
        for key1 in globalparams.keys():
            params[key1]={}
            grads[key1]={}
            for key2 in globalparams[key1].keys():
                params[key1][key2]=globalparams[key1][key2].copy()
                grads[key1][key2]=np.zeros_like(globalparams[key1][key2])
        gradstoaccumulate={}
        rewards={}
        vvalues={}
        t_start=self.t
        while bool((self.t-t_start<t_max)*(np.prod(self.environment.flag))):
            rewards[self.t]=-self.environment.score.copy()
            if bool((self.t>0)*(self.t==t_start)):
                a=self.alast.copy()
                vvalues[self.t]=self.vlast.copy()
            else:
                a=self.agent.action(self.environment.state,params)
                vvalues[self.t]=self.agent.V.copy()
            gradstoaccumulate[self.t]=self.agent.grads(1,2*kappa,beta)
            self.environment.transition(self.agent.action_kinds)
            rewards[self.t]+=self.environment.score.copy()
            self.t+=1
        self.alast=self.agent.action(self.environment.state,params)
        self.vlast=self.agent.V.copy()
        R=self.vlast
        for i in range(self.t-t_start):
            R=gamma*R+rewards[self.t-i-1]
            for key1 in globalparams.keys():
                for key2 in globalparams[key1].keys():
                    grads[key1][key2]-=np.sum(R-vvalues[self.t-i-1])*gradstoaccumulate[self.t-i-1][0][key1][key2]+gradstoaccumulate[self.t-i-1][1][key1][key2]
        threadscore=np.sum(self.environment.score.copy())
        ifterminated=not(bool(np.prod(self.environment.flag)))
        if ifterminated:
            self.environment.initialize()
        return grads,self.t-t_start,ifterminated,threadscore

# Showing how the agent with finite, discrete action space performs before training.
testagent=Agent_Ex1_Discrete()
untrainedparams=testagent.initialparams()
testenvironment=Environment()
while np.sum(testenvironment.flag)>0:
    a=testagent.action(testenvironment.state,untrainedparams)
    testenvironment.transition(testagent.action_kinds)
print("Average performance of the agent with finite, discrete action space before training:")
print(np.sum(testenvironment.score))
# Training the agent.
agents=[Agent_Ex1_Discrete()]
environments=[Environment()]
for i in range(7):
    agents.append(Agent_Ex1_Discrete())
    environments.append(Environment())
trainer=A3C(t_max=5,T_max=90000,agents=agents,environments=environments)
print("Training the agent by the A3C algorithm:")
trainedparams=trainer.train()
scorehistory_discrete=trainer.scorehistory.copy()
# Showing how the A3C-trained agent with finite, discrete action space performs.
testagent=Agent_Ex1_Discrete()
testenvironment=Environment()
while np.sum(testenvironment.flag)>0:
    a=testagent.action(testenvironment.state,trainedparams)
    testenvironment.transition(testagent.action_kinds)
print("Average performance of the A3C-trained agent with finite, discrete action space:")
print(np.sum(testenvironment.score))
```

### 試したこと

T_maxを大きめに取ったりはしていますがうまくいってないです



A3Cでcarttpole-v1の学習がうまくいっていない件

### 実現したいこと


UnityのML-Agentsを使って強化学習がしたい

### 前提



mlagents-learn config\RollerBall.yaml --run-id=firstRunを実行し、Unityの実行ボタンを押すがすぐに実行が止まり、学習が出来ずに以下のメッセージが出る
（同様の質問をされている方について確認しましたがエラーの内容が少し異なり、解決できなかったため質問させていただいております）

また、MarkupSafeが2.1.1より新しくないと駄目だけど私のは2.0.1だから駄目とあるように見えますが
もしこれが原因でしたら推奨される更新方法をご教示いただければ幸いです。何卒宜しくお願いいたします。


### 発生している問題・エラーメッセージ

```
エラーメッセージ

(env) C:\Users\coke8\Downloads\ml-agents-release_19>mlagents-learn ./config/sample/RollerBall.yaml --run-id=RollerBall-1

Traceback (most recent call last):
  File "c:\users\coke8\helloworld\env\lib\site-packages\pkg_resources\__init__.py", line 629, in _build_master
    ws.require(__requires__)
  File "c:\users\coke8\helloworld\env\lib\site-packages\pkg_resources\__init__.py", line 966, in require
    needed = self.resolve(parse_requirements(requirements))
  File "c:\users\coke8\helloworld\env\lib\site-packages\pkg_resources\__init__.py", line 827, in resolve
    dist = self._resolve_dist(
  File "c:\users\coke8\helloworld\env\lib\site-packages\pkg_resources\__init__.py", line 873, in _resolve_dist
    raise VersionConflict(dist, req).with_context(dependent_req)
pkg_resources.ContextualVersionConflict: (MarkupSafe 2.0.1 (c:\users\coke8\helloworld\env\lib\site-packages), Requirement.parse('MarkupSafe>=2.1.1'), {'werkzeug'})

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "C:\Users\coke8\helloworld\env\Scripts\mlagents-learn-script.py", line 33, in <module>
    sys.exit(load_entry_point('mlagents', 'console_scripts', 'mlagents-learn')())
  File "C:\Users\coke8\helloworld\env\Scripts\mlagents-learn-script.py", line 25, in importlib_load_entry_point
    return next(matches).load()
  File "C:\Program Files\WindowsApps\PythonSoftwareFoundation.Python.3.8_3.8.2800.0_x64__qbz5n2kfra8p0\lib\importlib\metadata.py", line 77, in load
    module = import_module(match.group('module'))
  File "C:\Program Files\WindowsApps\PythonSoftwareFoundation.Python.3.8_3.8.2800.0_x64__qbz5n2kfra8p0\lib\importlib\__init__.py", line 127, in import_module
    return _bootstrap._gcd_import(name[level:], package, level)
  File "<frozen importlib._bootstrap>", line 1014, in _gcd_import
  File "<frozen importlib._bootstrap>", line 991, in _find_and_load
  File "<frozen importlib._bootstrap>", line 975, in _find_and_load_unlocked
  File "<frozen importlib._bootstrap>", line 671, in _load_unlocked
  File "<frozen importlib._bootstrap_external>", line 848, in exec_module
  File "<frozen importlib._bootstrap>", line 219, in _call_with_frames_removed
  File "c:\users\coke8\downloads\ml-agents-release_19\ml-agents\mlagents\trainers\learn.py", line 2, in <module>
    from mlagents import torch_utils
  File "c:\users\coke8\downloads\ml-agents-release_19\ml-agents\mlagents\torch_utils\__init__.py", line 1, in <module>
    from mlagents.torch_utils.torch import torch as torch  # noqa
  File "c:\users\coke8\downloads\ml-agents-release_19\ml-agents\mlagents\torch_utils\torch.py", line 4, in <module>
    import pkg_resources
  File "c:\users\coke8\helloworld\env\lib\site-packages\pkg_resources\__init__.py", line 3324, in <module>
    def _initialize_master_working_set():
  File "c:\users\coke8\helloworld\env\lib\site-packages\pkg_resources\__init__.py", line 3298, in _call_aside
    f(*args, **kwargs)
  File "c:\users\coke8\helloworld\env\lib\site-packages\pkg_resources\__init__.py", line 3336, in _initialize_master_working_set
    working_set = WorkingSet._build_master()
  File "c:\users\coke8\helloworld\env\lib\site-packages\pkg_resources\__init__.py", line 631, in _build_master
    return cls._build_from_requirements(__requires__)
  File "c:\users\coke8\helloworld\env\lib\site-packages\pkg_resources\__init__.py", line 644, in _build_from_requirements
    dists = ws.resolve(reqs, Environment())
  File "c:\users\coke8\helloworld\env\lib\site-packages\pkg_resources\__init__.py", line 827, in resolve
    dist = self._resolve_dist(
  File "c:\users\coke8\helloworld\env\lib\site-packages\pkg_resources\__init__.py", line 868, in _resolve_dist
    raise DistributionNotFound(req, requirers)
pkg_resources.DistributionNotFound: The 'MarkupSafe>=2.1.1' distribution was not found and is required by werkzeug
```



### 該当のソースコード

```yamlファイルの中身は以下の通りです(入門書通りのつもりではあります)
behaviors:
 RollerBall:
	
	trainer_type: ppo

	
	max_steps: 500000
	time_horizon: 64
	summary_freq: 1000
	keep_checkpoints: 5


	hyperparameters:
		
		batch_size: 10
		buffer_size: 100
		learning_rate: 0.0003
		learning_rate_schedule: linear


		beta: 0.005
		epsilon: 0.2
		lambd: 0.95
		num_epoch: 3

	
	network_settings:
		normalize: true
		hidden_units: 128
		num_layers: 2

	
	reward_signals:
		
		extrinsic:
			gamma: 0.99
			strength: 1.0
```

### 試したこと
メッセージをそのまま調べる
Unity ML-Agentsの入門書を買う(https://www.borndigital.co.jp/book/19053.html)



### 補足情報（FW/ツールのバージョンなど）
absl-py                              1.4.0
attrs                                22.2.0
mlagents                          0.28.0
mlagents-envs                  0.28.0
wheel                                0.40.0
zipp                                 3.15.0

MLagentのダウンロードサイト
https://github.com/Unity-Technologies/ml-agents/releases/tag/release_19



UnityのMLAgentで機械学習(mlagents実行で困っております)

### 前提・実現したいこと
こんにちは。１－２か月前pythonのプログラミングを始めた超初心者です。UnityのML-Agentsのサンプル集を見て強化学習に興味を持ったので、とりあえずサンプルを使って学習をやってみよう！と思ったのですが、うまくいっておりません。
最初の方は３０個くらいエラーがあったのですが、パッケージマネージャーからInput systemなるものを導入することで５個まで減りました。
コメントいただけたらすぐにでも返信いたしますので、どうぞよろしくお願いいたします。
問題のスクリプトの内容も追記しておきますね。
### 発生している問題・エラーメッセージ
Assets\ML-Agents\Examples\PushBlockWithInput\Scripts\PushBlockWithInputPlayerController.cs(1,33): error CS0234: The type or namespace name 'Input' does not exist in the namespace 'Unity.MLAgents.Extensions' (are you missing an assembly reference?)

Assets\ML-Agents\Examples\PushBlockWithInput\Scripts\PushBlockWithInputPlayerController.cs(21,66): error CS0246: The type or namespace name 'IInputActionAssetProvider' could not be found (are you missing a using directive or an assembly reference?)

Assets\ML-Agents\Examples\PushBlockWithInput\Scripts\PushBlockActions.cs(18,42): error CS0246: The type or namespace name 'IInputActionCollection2' could not be found (are you missing a using directive or an assembly reference?)

Assets\ML-Agents\Examples\PushBlockWithInput\Scripts\PushBlockWithInputPlayerController.cs(109,31): error CS0246: The type or namespace name 'IInputActionCollection2' could not be found (are you missing a using directive or an assembly reference?)

Assets\ML-Agents\Examples\PushBlockWithInput\Scripts\PushBlockActions.cs(237,17): error CS0540: 'PushBlockActions.IEnumerable.GetEnumerator()': containing type does not implement interface 'IEnumerable'

###スクリプトの内容
（PushBlockWithInputPlayerController）

using Unity.MLAgents.Extensions.Input;
using UnityEngine;
using UnityEngine.InputSystem;

/// <summary>
/// This class handles the input for the PushBlock Cube character in the PushBlock scene.
/// Note that the only ML-Agents code here is the implementation of the <see cref="IInputActionAssetProvider"/>.
/// The <see cref="InputActuatorComponent"/> looks for a component that implements that interface in order to
/// rebind actions to virtual controllers when training agents or running inference.  This means that you can
/// keep your input handling code separate from ML-Agents, and have your agent's action space defined by the
/// actions defined in your project's <see cref="GetInputActionAsset"/>.
///
/// If you don't implement <see cref="IInputActionAssetProvider"/> the <see cref="InputActuatorComponent"/> will
/// look for a <see cref="PlayerInput"/> component on the GameObject it live on.  It will rebind the actions of that
/// instance of the asset.
///
/// It is important to note that if you have multiple components on the same GameObject handling input, you will
/// need to share the instance of the generated C# <see cref="IInputActionCollection2"/> (named <see cref="m_PushBlockActions"/>
/// here) in order to ensure that all of your actions are bound correctly for ml-agents training and inference.
/// </summary>
public class PushBlockWithInputPlayerController : MonoBehaviour, IInputActionAssetProvider
{

    PushBlockWithInputSettings m_PushBlockSettings;
    public float JumpTime = 0.5f;
    float m_JumpTimeRemaining;
    Rigidbody m_PlayerRb; //cached on initialization
    PushBlockActions m_PushBlockActions;
    float m_JumpCoolDownStart;

    void Awake()
    {
        m_PushBlockSettings = FindObjectOfType<PushBlockWithInputSettings>();
        LazyInitializeActions();

        // Cache the agent rigidbody
        m_PlayerRb = GetComponent<Rigidbody>();
    }

    void LazyInitializeActions()
    {
        if (m_PushBlockActions != null)
        {
            return;
        }

        m_PushBlockActions = new PushBlockActions();
        m_PushBlockActions.Enable();

        // You can listen to C# events.
        m_PushBlockActions.Movement.jump.performed += JumpOnperformed;
    }

    void JumpOnperformed(InputAction.CallbackContext callbackContext)
    {
        InnerJump(gameObject.transform);
    }

    void FixedUpdate()
    {
        // Or you can poll the action itself like we do here.
        InnerMove(gameObject.transform, m_PushBlockActions.Movement.movement.ReadValue<Vector2>());
        if (m_JumpTimeRemaining < 0)
        {
            m_PlayerRb.AddForce(-transform.up * (m_PushBlockSettings.agentJumpForce * 3), ForceMode.Acceleration);
        }

        m_JumpTimeRemaining -= Time.fixedDeltaTime;
    }

    void InnerJump(Transform t)
    {
        if (Time.realtimeSinceStartup - m_JumpCoolDownStart > m_PushBlockSettings.agentJumpCoolDown)
        {
            m_JumpTimeRemaining = JumpTime;
            m_PlayerRb.AddForce(t.up * m_PushBlockSettings.agentJumpForce, ForceMode.VelocityChange);
            m_JumpCoolDownStart = Time.realtimeSinceStartup;
        }
    }

    void InnerMove(Transform t, Vector2 v)
    {
        var forward = CreateForwardVector(v);
        var up = CreateUpVector(v);
        var dirToGo = t.forward * forward;
        var rotateDir = t.up * up;
        t.Rotate(rotateDir, Time.deltaTime * 200f);
        m_PlayerRb.AddForce(dirToGo * m_PushBlockSettings.agentRunSpeed,
            ForceMode.VelocityChange);
    }

    static float CreateUpVector(Vector2 move)
    {
        return Mathf.Abs(move.x) > Mathf.Abs(move.y) ? move.x : 0f;
    }

    static float CreateForwardVector(Vector2 move)
    {
        return Mathf.Abs(move.y) > Mathf.Abs(move.x) ? move.y : 0f;
    }

    /// <summary>
    /// This is the implementation of the <see cref="IInputActionAssetProvider"/> for this class.  We need
    /// both the <see cref="GetInputActionAsset"/> and the <see cref="IInputActionCollection2"/> if you are
    /// listening to C# events, Unity Events, or receiving Messages from the Input System Package as those callbacks
    /// are set up through the generated <see cref="IInputActionCollection2"/>.
    /// </summary>
    /// <returns></returns>
    public (InputActionAsset, IInputActionCollection2) GetInputActionAsset()
    {
        LazyInitializeActions();
        return (m_PushBlockActions.asset, m_PushBlockActions);
    }
}

### 試したこと

参考にさせていただいておりますサイトはこちらです。
https://www.suzu6.net/posts/270-isntall-unity-ml-agents/
一度上記の方法でサンプルを使った強化学習までは行うことができました。その際resultsにonnxファイルが生成されたのですが、それをAgentsに貼り付けても動作せず、いろいろ調べたところML-Agentsのバージョンが悪い？といった情報が出てきたので、pythonやunity、ml-agentsのバージョンをできる限り最新にした上で新たに挑戦したとところ、行き詰った次第です。

### 補足情報（FW/ツールのバージョンなど）(どこまで書いたらよいのかわからないので、必要な情報があればすぐに追記いたします。）
Windows10
Unity 2021.1.19

使用しているファイル
ml-agents-release_18

パッケージマネージャー
ML Agents　バージョン 2.1.0-exp.1
ML Agents Extensions　バージョン 0.5.0-preview
Input System　バージョン 1.0.2 - January 21, 2021
JetBrains Rider Editor　バージョン 3.0.7 - May 21, 2021
Test Framework　バージョン 1.1.29 - August 17, 2021
TextMeshPro　バージョン 3.0.6 - April 22, 2021
Timeline　バージョン 1.5.6 - July 08, 2021
Unity UI　バージョン 1.0.0 - August 24, 2021
Version Control　バージョン 1.9.0 - August 05, 2021
Visual Scripting　バージョン 1.6.1 - April 22, 2021
Visual Studio Code Editor　バージョン 1.2.3 - October 28, 2020
Visual Studio Editor　バージョン 2.0.11 - July 08, 2021

ML-Agentsで強化学習を始めたい

### 前提・実現したいこと
UnityのMl-agentsを用い強化学習及び、模倣学習をしたい

### 発生している問題・エラーメッセージ
Ml-agentsの最新バージョンである 0.11.0を使用しているが参考文献が少なすぎて解決できない。

```
(base) C:\Users\user\Documents\ml-agents-master\ml-agents>mlagents-learn ../config/trainer_config.yaml --run-id=firstRun --train
2019-11-14 14:45:59.431992: W tensorflow/stream_executor/platform/default/dso_loader.cc:55] Could not load dynamic library 'cudart64_100.dll'; dlerror: cudart64_100.dll not found
2019-11-14 14:45:59.436532: I tensorflow/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine.
WARNING:tensorflow:
The TensorFlow contrib module will not be included in TensorFlow 2.0.
For more information, please see:
  * https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md
  * https://github.com/tensorflow/addons
  * https://github.com/tensorflow/io (for I/O related ops)
If you depend on functionality not listed there, please file an issue.



                        ▄▄▄▓▓▓▓
                   ╓▓▓▓▓▓▓█▓▓▓▓▓
              ,▄▄▄m▀▀▀'  ,▓▓▓▀▓▓▄                           ▓▓▓  ▓▓▌
            ▄▓▓▓▀'      ▄▓▓▀  ▓▓▓      ▄▄     ▄▄ ,▄▄ ▄▄▄▄   ,▄▄ ▄▓▓▌▄ ▄▄▄    ,▄▄
          ▄▓▓▓▀        ▄▓▓▀   ▐▓▓▌     ▓▓▌   ▐▓▓ ▐▓▓▓▀▀▀▓▓▌ ▓▓▓ ▀▓▓▌▀ ^▓▓▌  ╒▓▓▌
        ▄▓▓▓▓▓▄▄▄▄▄▄▄▄▓▓▓      ▓▀      ▓▓▌   ▐▓▓ ▐▓▓    ▓▓▓ ▓▓▓  ▓▓▌   ▐▓▓▄ ▓▓▌
        ▀▓▓▓▓▀▀▀▀▀▀▀▀▀▀▓▓▄     ▓▓      ▓▓▌   ▐▓▓ ▐▓▓    ▓▓▓ ▓▓▓  ▓▓▌    ▐▓▓▐▓▓
          ^█▓▓▓        ▀▓▓▄   ▐▓▓▌     ▓▓▓▓▄▓▓▓▓ ▐▓▓    ▓▓▓ ▓▓▓  ▓▓▓▄    ▓▓▓▓`
            '▀▓▓▓▄      ^▓▓▓  ▓▓▓       └▀▀▀▀ ▀▀ ^▀▀    `▀▀ `▀▀   '▀▀    ▐▓▓▌
               ▀▀▀▀▓▄▄▄   ▓▓▓▓▓▓,                                      ▓▓▓▓▀
                   `▀█▓▓▓▓▓▓▓▓▓▌
                        ¬`▀▀▀█▓


INFO:mlagents.trainers:CommandLineOptions(debug=False, num_runs=1, seed=-1, env_path=None, run_id='firstRun', load_model=False, train_model=True, save_freq=50000, keep_checkpoints=5, base_port=5005, num_envs=1, curriculum_folder=None, lesson=0, slow=False, no_graphics=False, multi_gpu=False, trainer_config_path='../config/trainer_config.yaml', sampler_file_path=None, docker_target_name=None, env_args=None, cpu=False)
INFO:mlagents.envs:Start training by pressing the Play button in the Unity Editor.
```

### 該当のソースコード
上記のメッセージが出たのちUnity上でplayを押すと以下のメッセージが出る。
```
Process Process-1:
Traceback (most recent call last):
  File "c:\users\user\anaconda3\lib\multiprocessing\process.py", line 297, in _bootstrap
    self.run()
  File "c:\users\user\anaconda3\lib\multiprocessing\process.py", line 99, in run
    self._target(*self._args, **self._kwargs)
  File "c:\users\user\anaconda3\lib\site-packages\mlagents\envs\subprocess_env_manager.py", line 82, in worker
    env = env_factory(worker_id)
  File "c:\users\user\anaconda3\lib\site-packages\mlagents\trainers\learn.py", line 359, in create_unity_environment
    args=env_args,
  File "c:\users\user\anaconda3\lib\site-packages\mlagents\envs\environment.py", line 105, in __init__
    aca_output = self.send_academy_parameters(rl_init_parameters_in)
  File "c:\users\user\anaconda3\lib\site-packages\mlagents\envs\environment.py", line 689, in send_academy_parameters
    return self.communicator.initialize(inputs)
  File "c:\users\user\anaconda3\lib\site-packages\mlagents\envs\rpc_communicator.py", line 88, in initialize
    "The Unity environment took too long to respond. Make sure that :\n"
mlagents.envs.exception.UnityTimeOutException: The Unity environment took too long to respond. Make sure that :
         The environment does not need user interaction to launch
         The Agents are linked to the appropriate Brains
         The environment and the Python interface have compatible versions.
Traceback (most recent call last):
  File "c:\users\user\anaconda3\lib\multiprocessing\connection.py", line 312, in _recv_bytes
    nread, err = ov.GetOverlappedResult(True)
BrokenPipeError: [WinError 109] パイプは終了しました。

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "c:\users\user\anaconda3\lib\site-packages\mlagents\envs\subprocess_env_manager.py", line 59, in recv
    response: EnvironmentResponse = self.conn.recv()
  File "c:\users\user\anaconda3\lib\multiprocessing\connection.py", line 250, in recv
    buf = self._recv_bytes()
  File "c:\users\user\anaconda3\lib\multiprocessing\connection.py", line 321, in _recv_bytes
    raise EOFError
EOFError

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "c:\users\user\anaconda3\lib\runpy.py", line 193, in _run_module_as_main
    "__main__", mod_spec)
  File "c:\users\user\anaconda3\lib\runpy.py", line 85, in _run_code
    exec(code, run_globals)
  File "C:\Users\user\Anaconda3\Scripts\mlagents-learn.exe\__main__.py", line 9, in <module>
  File "c:\users\user\anaconda3\lib\site-packages\mlagents\trainers\learn.py", line 408, in main
    run_training(0, run_seed, options, Queue())
  File "c:\users\user\anaconda3\lib\site-packages\mlagents\trainers\learn.py", line 222, in run_training
    options.sampler_file_path, env.reset_parameters, run_seed
  File "c:\users\user\anaconda3\lib\site-packages\mlagents\envs\subprocess_env_manager.py", line 225, in reset_parameters
    return self.env_workers[0].recv().payload
  File "c:\users\user\anaconda3\lib\site-packages\mlagents\envs\subprocess_env_manager.py", line 62, in recv
    raise UnityCommunicationException("UnityEnvironment worker: recv failed.")
mlagents.envs.exception.UnityCommunicationException: UnityEnvironment worker: recv failed.
```

### 試したこと
再インストール、webサイトの閲覧

### 補足情報（FW/ツールのバージョンなど）

TensorFlow 1.15