kerasでfitができない。

実現したいこと

model.fitが実行できるようにする。

発生している問題・分からないこと

fit属性を持っていないとエラーが出る。

エラーメッセージ

error
1AttributeError                            Traceback (most recent call last)
2Cell In[3], line 1
3----> 1 tameshi=Trainer(episodes=10)
4      2 tameshi.train()
5
6Cell In[2], line 6, in Trainer.__init__(self, episodes, model, epsilon)
7      5 def __init__(self, episodes: int, model = None, epsilon: float = 0.2):
8----> 6     self.agent = ValueFunctionAgent(model=model, epsilon=epsilon)
9      7     self.episodes = episodes
10
11File ~/プログラミング/Mercurial用/人工知能/ポケモン/ポケモンAI/simulator/value_function_agent.py:156, in ValueFunctionAgent.__init__(self, model, epsilon)
12    154 fake_state = np.array([np.zeros((7 * 6) * 2)])
13    155 fake_estimation = np.array([np.zeros(10)])  # change * 6, moves * 4
14--> 156 self.model.fit(fake_state, fake_estimation,epochs=1,verbose=0)
15
16AttributeError: 'QNetwork' object has no attribute 'fit'

該当のソースコード

python
1import logging
2
3GAMMA = 0.9
4
5# パラメータの準備 #追加
6NUM_EPISODES = 500 # エピソード数
7MAX_STEPS = 200 # 最大ステップ数
8GAMMA = 0.99 # 時間割引率
9WARMUP = 10 # 無操作ステップ数
10
11# 探索パラメータ
12E_START = 1.0 # εの初期値
13E_STOP = 0.01 # εの最終値
14E_DECAY_RATE = 0.001 # εの減衰率
15
16# メモリパラメータ
17MEMORY_SIZE = 10000 # 経験メモリのサイズ
18BATCH_SIZE = 32 # バッチサイズ
19
20# 行動価値関数の定義 #追加
21class QNetwork:
22    # 初期化
23    def __init__(self, state_size, action_size):
24        # モデルの作成
25        self.model = Sequential()
26        self.model.add(Dense(16, activation='relu', input_dim=state_size))
27        self.model.add(Dense(16, activation='relu'))
28        self.model.add(Dense(16, activation='relu'))
29        self.model.add(Dense(action_size, activation='linear'))
30        
31        # モデルのコンパイル
32        self.model.compile(optimizer=Adam(learning_rate=0.001),loss=Huber())
33
34class NeuralNetworkPlayer(Player):
35    model: QNetwork #全結合
36    epsilon: float
37
38    def __init__(
39        self,
40        pokemons: list[p.Pokemon],
41        model: QNetwork,
42        epsilon: float,
43    ):
44        self.pokemons = pokemons
45        self.model = model ##モデル決める
46        self.epsilon = epsilon
47
48    def choose_action(self, opponent: Player) -> Action:
49        available_pokemons = self.get_available_pokemons_for_change()
50        if random() < self.epsilon: #一部ランダムにする
51            if len(available_pokemons) == 0 or random() < 0.5:
52                return self.pick_random_move_action()
53            else:
54                return Action.change_to(self.get_random_living_pokemon_index_to_replace())
55        predicts = self.model.predict([[*self.to_array(), *opponent.to_array()]])[0]
56        for index, _ in enumerate(predicts):
57            if index < 6 and (
58                self.pokemons[index].actual_hp <= 0 or index == self.active_pokemon_index
59            ):
60                predicts[index] = predicts.min() - 1
61        logging.info(f"predictions:\n{predicts}")
62        return Action(predicts.argmax())
63
64    def choose_action_on_pokemon_dead(self, opponent: Player) -> Action:
65        if random() < self.epsilon:
66            return Action.change_to(self.get_random_living_pokemon_index_to_replace())
67        else:
68            predicts_arr = self.model.predict([[*self.to_array(), *opponent.to_array()]])
69            predicts = predicts_arr[0][:6]  # [:6] removes skill moves
70            for index in range(0, len(self.pokemons)):
71                if self.pokemons[index].actual_hp <= 0 or index == self.active_pokemon_index:
72                    predicts[index] = predicts.min() - 1
73            logging.info(f"predictions:\n{predicts}")
74            return Action(predicts.argmax())
75
76class ValueFunctionAgent:
77    # model: Pipeline
78    model: QNetwork
79    battle: Battle
80    learner: Player
81    opponent: Player
82    epsilon: float
83
84    def __init__(
85        self,
86        model  = None,
87        epsilon: float = 0.2,
88    ):
89        if model:
90            self.model = model
91        else:
92            self.model = QNetwork( #状態数、行動数
93                state_size=5,
94                action_size=4
95                
96            )
97        self.epsilon = epsilon
98        # pokemon(id + is_active + actual_hp + 4 moves) * 8, 2 players
99        fake_state = np.array([np.zeros((7 * 6) * 2)])
100        fake_estimation = np.array([np.zeros(10)])  # change * 6, moves * 4
101        self.model.fit(fake_state, fake_estimation,epochs=1,verbose=0)
102
103    def reset(self, max_episodes: int, episodes: int):
104     #   self.learner = NeuralNetworkPlayer(
105      #      build_random_team(),
106       #     self.model,
107       #     self.epsilon,
108        #)
109        #バトルするプレイヤーの中身
110        self.learner=NeuralNetworkPlayer(
111            [
112                p.Jolteon(
113                    [
114                        m.BodySlam(),
115                        m.DoubleKick(),
116                        m.PinMissle(),
117                        m.Thunderbolt(),
118                    ]
119                )
120            ],
121            self.model,
122            self.epsilon
123        )
124
125    def update(self, experiences: list[Experience]):
126        states = np.array([e.state for e in experiences])
127        next_states = np.array([e.next_state for e in experiences])
128
129        #価値を計算
130        y = self.model.predict(states)
131        future = self.model.predict(next_states)
132
133        for i, experience in enumerate(experiences):
134            reward = experience.reward
135            if not experience.done:
136                reward += GAMMA * np.max(future[i])
137            y[i][experience.action.value] = reward
138
139        self.model.fit(states, y,epochs=1,verbose=0)

python
1class Trainer:
2    step = 0
3    experiences = deque(maxlen=1024)
4
5    def __init__(self, episodes: int, model = None, epsilon: float = 0.2):
6        self.agent = ValueFunctionAgent(model=model, epsilon=epsilon)
7        self.episodes = episodes
8    def train(self):
9        win_count_total = 0
10        win_count_of_100 = 0
11        for episode in range(0, self.episodes):
12            self.agent.reset(self.episodes, episode)
13            print("agent",self.agent.learner)
14            battle = Battle(self.agent.learner, self.agent.opponent)
15            battle.validate()
16            logging.info(battle)
17            #print(battle)
18            while True:
19                logging.info(f"=== turn {battle.turn} ===")
20                current_state = battle.to_array()
21
22                current_learner_pokemon_count = len(battle.player1.get_available_pokemons())
23                current_opponent_pokemon_count = len(battle.player2.get_available_pokemons())
24
25                action= battle.forward_step()
26                winner = battle.get_winner()
27
28                if action is not None:
29                    reward = 0
30                    if winner == self.agent.learner:
31                        logging.info(f"learner won the battle!")
32                        print("優勝はlearner")
33                        reward = 1
34                    elif winner == self.agent.opponent:
35                        logging.info(f"learner lost...")
36                        print("優勝はopponent")
37                        reward = -1
38                    elif battle.turn > 500:
39                        logging.info("battle is too long")
40                        print("ターンが長い")
41                        reward = -0.1
42                    else:
43                        print("バトル中")
44
45                    self.experiences.append(
46                        Experience(
47                            state=current_state,
48                            action=action,
49                            reward=reward,
50                            next_state=battle.to_array(),
51                            done=winner is not None,
52                        )
53                    )
54
55                next_learner_pokemon_count = len(battle.player1.get_available_pokemons())
56                next_opponent_pokemon_count = len(battle.player2.get_available_pokemons())
57
58                self.step += 1
59                logging.info(battle)
60                if winner is not None or battle.turn > 500:
61                    if winner == self.agent.learner:
62                        win_count_total += 1
63                        win_count_of_100 += 1
64                    break
65
66            if len(self.experiences) > 64:
67                self.agent.update(sample(self.experiences, 64))
68            if episode > 0 and episode % 1 == 0:
69                print(f"=============")
70                print(f"episode {episode}")
71                print(f"Win Rate of 100: {win_count_of_100 / 100}")
72                print(f"Total Win Rate: {win_count_total / episode}")
73                win_count_of_100 = 0
74        print(f"=======================================")
75        print(f"Total Win Count: {win_count_total}")
76        print(f"Total Win Rate: {win_count_total / self.episodes}")
77        sheipu=(current_state,action,reward,battle.to_array(),winner is not None)
78        print("経験のシェイプは",len(sheipu),"です")
79
80tameshi=Trainer(episodes=10)
81tameshi.train()

試したこと・調べたこと

teratailやGoogle等で検索した
ソースコードを自分なりに変更した
知人に聞いた
その他

上記の詳細・結果

HuberをHuber()に変更したら解決したとあったので変更しました。

補足

jupyter notebook
mac M1
参考サイト
https://acro5piano.com/post/2022-09-07-pokemon-reinforcement-python/

miraimirai

2024/03/21 09:13

文字数制限のためimport文を省略したので追記します from random import random from typing import Optional import numpy as np from sklearn.neural_network import MLPRegressor import pokedex as p from experience import Experience from utils import build_random_team from battle import Battle from player import Action, Player from sample_players import JustAttackPlayer, StupidRandomPlayer import numpy as np from tensorflow.keras.models import Sequential from tensorflow.keras.layers import Dense from tensorflow.keras.optimizers.legacy import Adam from tensorflow.keras.losses import Huber # パッケージのインポート import logging import moves as m import pokedex as p from battle import Battle from player import Action, Player from random import sample from collections import deque from value_function_agent import ValueFunctionAgent from experience import Experience