強化学習にて同じデータを使っているのにトレーニングとテストで報酬に大幅は差異が出る

tensorflowでトレーニングしたモデルをテストしたいのですがうまくいきません。
どのように改善すればいいのでしょうか？

後、出来ればでいいのですが、アクションの選択が大幅に偏る原因がわかれば教えていただきたいです。

いかがコードです（問題に関係するとこだけで失礼します）

python
1class Agent:
2    def __init__(self, path, window_size, skip, save=False, saver_path=None, restore=False, action_prior="normal", reparameterize=True, noise=True):
3        self.save = save
4        self.saver = tf.train.Saver()
5        self.saver_path = saver_path
6          
7        if restore == True:
8          tf.reset_default_graph()
9          self.saver.restore(self.sess, self.saver_path)
10          
11    def discount_rewards(self, r):
12        running_add = 0
13        for t in reversed(range(0, len(r))):
14            running_add = running_add * self.GAMMA + r[t]
15        return running_add
16
17    def _preproc(self):
18        df = pd.read_csv(self.path)
19        X = df[["Close"]]
20        X = np.asanyarray(X)
21
22        gen = tf.keras.preprocessing.sequence.TimeseriesGenerator(X, np.asanyarray(df[["Open"]])[-len(X)::], self.window_size)
23        x = []
24        y = []
25        for i in gen:
26            x.extend(i[0].tolist())
27            y.extend(i[1].tolist())
28        x = np.asanyarray(x)
29        y = np.asanyarray(y)
30
31        self.df = x[-28000::]
32        self.trend = y[-28000::]
33
34    def _assign(self, from_name, to_name):
35        from_w = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope=from_name)
36        to_w = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope=to_name)
37        for i in range(len(from_w)):
38            assign_op = to_w[i].assign(from_w[i])
39            self.sess.run(assign_op)
40
41    def _select_action(self, state):
42        prediction = self.sess.run(self.actor.logits, feed_dict={self.actor.X:[state]})[0]
43        action = exploration(prediction, self.OUTPUT_SIZE)
44        return action
45    
46    def get_state(self, t):
47        return self.df[t]
48
49    def buy(self, spread, pip_cost, sl):
50        position = 3
51        pip = []
52        spread = spread / pip_cost
53        loscut = False
54
55        for t in  range(0, len(self.trend), self.skip):
56                state = self.get_state(t)
57                action = self._select_action(state)
58              
59                if action == 0:
60                    if position == 3:
61                        states = [self.trend[t] + spread]
62                        position = 1
63                    elif position == 1:
64                      sub = 0
65                      p = [(self.trend[t] - s) * pip_cost for s in states]
66                      for b in range(0,len(p)):
67                        r = [np.asanyarray([-40.0]), True] if p[b] <= -40 else [p[b], False]
68                        if r[1]:
69                          pip.extend(r[0].tolist())
70                          states.pop(b-sub)
71                          sub += 1
72                        states.append(self.trend[t] + spread)
73                        position = 1
74                    elif position == 2:
75                      p = [(s - self.trend[t]) * pip_cost for s in states]
76                      for b in p:
77                        b = np.asanyarray([-40.0]) if b <= -40 else b
78                        pip.extend(b.tolist())
79                      self.pip = np.asanyarray(pip)
80                      total_pip = np.sum(self.pip)
81                      states = [self.trend[t] + spread]
82                      position = 1
83
84
85                elif action == 1:
86                    if position == 3:
87                        states = [self.trend[t] - spread]
88                        position = 2
89                    elif position == 2:
90                      sub = 0
91                      p = [(s - self.trend[t]) * pip_cost for s in states]
92                      for b in range(0,len(p)):
93                        r = [np.asanyarray([-40.0]), True] if p[b] <= -40 else [p[b], False]
94                        if r[1]:
95                          pip.extend(r[0].tolist())
96                          states.pop(b-sub)
97                          sub += 1
98                          
99                      states.append(self.trend[t] - spread)
100                      position = 2
101                    elif position == 1:
102                        p = [(self.trend[t] - s) * pip_cost for s in states]
103                        for b in p:
104                          b = np.asanyarray([-40.0]) if b <= -40 else b
105                          pip.extend(b.tolist())
106                        self.pip = np.asanyarray(pip)
107                        total_pip = np.sum(self.pip)
108                        states = [self.trend[t] - spread]
109                        position = 2
110
111              # st ate = next_state
112        return total_pip, self.pip
113
114
115    def train(self, iterations, checkpoint, spread, pip_cost, n=4):
116        copy = 0
117        for i in range(iterations):
118            position = 3
119            total_pip = 0.00
120            max_pip = 0
121            pip = []
122            spread = spread / pip_cost
123            done = False
124            state = self.get_state(0)
125            states = []
126            p = []
127            h_s = []
128            h_r = []
129            tau = 0
130            old_reword = 0.0
131            self.history = []
132            if (copy + 1) % 4 == 0: 
133              self._assign('actor-original', 'actor-target')
134              self._assign('critic-original', 'critic-target')
135
136            for t in  range(0, len(self.trend)-1, self.skip):
137                h_s.append(state)
138                action = self._select_action(state)
139                self.history.append(action)
140                next_state = self.get_state(t + 1)
141                
142                if action == 0:
143                    if position == 3:
144                        states = [self.trend[t] + spread]
145                        position = 1
146                    elif position == 1:
147                      sub = 0
148                      p = [(self.trend[t] - s) * pip_cost for s in states]
149                      for b in range(0,len(p)):
150                        r = [np.asanyarray([-40.0]), True] if p[b] <= -40 else [p[b], False]
151                        if r[1]:
152                          pip.extend(r[0].tolist())
153                          states.pop(b-sub)
154                          sub += 1
155                        states.append(self.trend[t] + spread)
156                        position = 1
157                    elif position == 2:
158                      p = [(s - self.trend[t]) * pip_cost for s in states]
159                      for b in p:
160                        b = np.asanyarray([-40.0]) if b <= -40 else b
161                        pip.extend(b.tolist())
162                      self.pip = np.asanyarray(pip)
163                      total_pip = np.sum(self.pip)
164                      states = [self.trend[t] + spread]
165                      position = 1
166
167
168                elif action == 1:
169                    if position == 3:
170                        states = [self.trend[t] - spread]
171                        position = 2
172                    elif position == 2:
173                      sub = 0
174                      p = [(s - self.trend[t]) * pip_cost for s in states]
175                      for b in range(0,len(p)):
176                        r = [np.asanyarray([-40.0]), True] if p[b] <= -40 else [p[b], False]
177                        if r[1]:
178                          pip.extend(r[0].tolist())
179                          states.pop(b-sub)
180                          sub += 1
181                          
182                      states.append(self.trend[t] - spread)
183                      position = 2
184                    elif position == 1:
185                        p = [(self.trend[t] - s) * pip_cost for s in states]
186                        for b in p:
187                          b = np.asanyarray([-40.0]) if b <= -40 else b
188                          pip.extend(b.tolist())
189                        self.pip = np.asanyarray(pip)
190                        total_pip = np.sum(self.pip)
191                        states = [self.trend[t] - spread]
192                        position = 2
193                  
194                mean_pip = total_pip / (t + 1)
195                reward = mean_pip - old_reword
196                old_reword = mean_pip
197                h_r.append(reward)
198                
199                tau = t - n + 1
200                if tau >= 0:
201                  reward = self.discount_rewards([r for r in h_r[tau+1:tau+n]])
202                  experience = h_s[tau], self.history[tau], reward, next_state, done
203                  self.memory.store(experience)
204                  
205                state = next_state
206                self.EPSILON = self.MIN_EPSILON + (1.0 - self.MIN_EPSILON) * np.exp(-self.DECAY_RATE * i)
207
208            self.total_pip = total_pip
209            trade_accuracy = np.mean(self.pip > 0)
210            self.trade = trade_accuracy
211            mean_pip *= 1440
212            
213            batch_size = self.BATCH_SIZE
214            self.tree_idx, batch = self.memory.sample(batch_size)
215            cost = self._construct_memories_and_train(batch)
216            prob = self.prob()
217            copy += 1
218            try:
219              saved_path = self.saver.save(self.sess, self.saver_path)
220            except:
221              print("do not save")
222
223            print('action probability = ', prob)
224            print('trade accuracy = ', trade_accuracy)
225            print('epoch: %d, total rewards: %f, cost: %f, mean rewards: %f' % (i + 1, total_pip, cost, mean_pip))