Keras-rlで画像を扱ったときの形状があわない

Keras-RLを用いた深層強化学習コト始め
https://qiita.com/panchovie/items/af9bac0e8ad8945bdf09 という記事のプログラムを再現したい思っているのですが、エラーの中にでてくる(1, 1, 3)が何なのかがよくわからずうまく動かすことができません。できればお教えしていただきたいです。

Python
1img_size = 128
2channel = 3
3
4class PendulumProcessor(Processor):
5    def __init__(self):
6        self.rgb_state = np.zeros((img_size, img_size, channel))
7
8    # 状態（x,y座標）から対応画像を描画する関数
9    def _get_rgb_state(self, state):
10
11        h_size = img_size/2.0
12
13        img = Image.new("RGB", (img_size, img_size), (255, 255, 255))
14        dr = ImageDraw.Draw(img)
15
16        # 棒の長さ
17        l = img_size/4.0 * 3.0/ 2.0
18
19        # 棒のラインの描写
20        dr.line(((h_size - l * state[1], h_size - l * state[0]), (h_size, h_size)), (0, 0, 0), 1)
21
22        # 棒の中心の円を描写（それっぽくしてみた）
23        buff = img_size/32.0
24        dr.ellipse(((h_size - buff, h_size - buff), (h_size + buff, h_size + buff)), 
25                   outline=(0, 0, 0), fill=(255, 0, 0))
26
27        # 画像の一次元化（GrayScale化）とarrayへの変換
28        pilImg = img.convert("L")
29        img_arr = np.asarray(pilImg)
30
31        # 画像の規格化
32        img_arr = img_arr/255.0
33
34        return img_arr
35
36
37    # Gym環境の出力と、Duel-DQNアルゴリズムへの入力との違いを吸収
38    def process_step(self, observation, reward, done, info):
39
40        old_rgb_state = self.rgb_state.copy()
41
42        # アルゴリズムの状態入力として、画像を用いる（過去３フレームを入力する）
43        # 直近の状態に対応する画像を作成
44        self.rgb_state[:, :, 0] = self.self._get_rgb_state(observation)
45        # 過去２フレームも保持
46        for i in range(1, channel):
47            self.rgb_state[:, :, i] = old_rgb_state[:, :, i-1] # shift old state
48
49        # アルゴリズムへの報酬として、設定課題に沿った報酬を用いる（上記通り）
50        reward = self.process_reward(reward)
51
52        return self.rgb_state, reward, done, info
53
54    # Duel-DQNの出力と、Gym環境の入力の違いを吸収
55    def process_action(self, action):
56        return ACT_ID_TO_VALUE[action]
57
58    # Gym環境の報酬の出力と、Duel-DQNの報酬の入力との違いを吸収
59    def process_reward(self, reward):
60        if reward > -0.2:
61            return 1
62        elif reward > -1.0:
63            return 0
64        else:
65            return 0   
66
67processor = PendulumProcessor()  
68print(processor.rgb_state.shape) #(128,128,3)

Python
1
2# 画像の特徴量抽出ネットワークのパラメタ
3n_filters = 32
4kernel = (3, 3)
5strides = (2, 2)
6
7model = Sequential()
8# 畳込み層による画像の特徴量抽出ネットワーク
9model.add(Reshape((img_size, img_size, channel), input_shape=(1, img_size, img_size, channel)))
10model.add(Conv2D(n_filters, kernel, strides=strides, padding="same", activation="relu"))
11model.add(Conv2D(n_filters, kernel, strides=strides, padding="same", activation="relu"))
12model.add(MaxPooling2D(pool_size=(2, 2)))
13model.add(Conv2D(n_filters, kernel, strides=strides, padding="same", activation="relu"))
14model.add(Conv2D(n_filters, kernel, strides=strides, padding="same", activation="relu"))
15model.add(MaxPooling2D(pool_size=(2, 2)))
16model.add(Flatten())
17# 以前と同様の2層FCのQ関数ネットワーク
18model.add(Dense(16, activation="relu"))
19model.add(Dense(16, activation="relu"))
20model.add(Dense(nb_actions, activation="linear"))

Python
1memory = SequentialMemory(limit=50000, window_length=1)
2policy = BoltzmannQPolicy()
3
4# Duel-DQNのAgentクラスオブジェクトの準備 （上記processorやmodelを元に）
5dqn = DQNAgent(model=model, nb_actions=nb_actions, memory=memory, nb_steps_warmup=10,
6               enable_dueling_network=True, dueling_type="avg", target_model_update=1e-2, policy=policy,
7               processor=processor)
8print(dqn.model.summary())
9dqn.compile(Adam(lr=1e-3), metrics=["mae"])
10history = dqn.fit(env, nb_steps=1000, visualize=False, verbose=2) #エラーがでているところ

Python
1---------------------------------------------------------------------------
2ValueError                                Traceback (most recent call last)
3<ipython-input-19-f4b86796f9d8> in <module>()
4     14 print(dqn.model.summary())
5     15 dqn.compile(Adam(lr=1e-3), metrics=["mae"])
6---> 16 history = dqn.fit(env, nb_steps=1000, visualize=False, verbose=2)
7
86 frames
9/usr/local/lib/python3.6/dist-packages/rl/core.py in fit(self, env, nb_steps, action_repetition, callbacks, verbose, visualize, nb_max_start_steps, start_step_policy, log_interval, nb_max_episode_steps)
10    167                 # This is were all of the work happens. We first perceive and compute the action
11    168                 # (forward step) and then use the reward to improve (backward step).
12--> 169                 action = self.forward(observation)
13    170                 if self.processor is not None:
14    171                     action = self.processor.process_action(action)
15
16/usr/local/lib/python3.6/dist-packages/rl/agents/dqn.py in forward(self, observation)
17    227         # Select an action.
18    228         state = self.memory.get_recent_state(observation)
19--> 229         q_values = self.compute_q_values(state)
20    230         if self.training:
21    231             action = self.policy.select_action(q_values=q_values)
22
23/usr/local/lib/python3.6/dist-packages/rl/agents/dqn.py in compute_q_values(self, state)
24     68 
25     69     def compute_q_values(self, state):
26---> 70         q_values = self.compute_batch_q_values([state]).flatten()
27     71         assert q_values.shape == (self.nb_actions,)
28     72         return q_values
29
30/usr/local/lib/python3.6/dist-packages/rl/agents/dqn.py in compute_batch_q_values(self, state_batch)
31     63     def compute_batch_q_values(self, state_batch):
32     64         batch = self.process_state_batch(state_batch)
33---> 65         q_values = self.model.predict_on_batch(batch)
34     66         assert q_values.shape == (len(state_batch), self.nb_actions)
35     67         return q_values
36
37/usr/local/lib/python3.6/dist-packages/keras/engine/training.py in predict_on_batch(self, x)
38   1572             Numpy array(s) of predictions.
39   1573         
40-> 1574         x, _, _ = self._standardize_user_data(x)
41   1575         if self._uses_dynamic_learning_phase():
42   1576             ins = x + [0]
43
44/usr/local/lib/python3.6/dist-packages/keras/engine/training.py in _standardize_user_data(self, x, y, sample_weight, class_weight, check_array_lengths, batch_size)
45    577             feed_input_shapes,
46    578             check_batch_axis=False,  # Don't enforce the batch size.
47--> 579             exception_prefix='input')
48    580 
49    581         if y is not None:
50
51/usr/local/lib/python3.6/dist-packages/keras/engine/training_utils.py in standardize_input_data(data, names, shapes, check_batch_axis, exception_prefix)
52    133                         ': expected ' + names[i] + ' to have ' +
53    134                         str(len(shape)) + ' dimensions, but got array '
54--> 135                         'with shape ' + str(data_shape))
55    136                 if not check_batch_axis:
56    137                     data_shape = data_shape[1:]
57
58ValueError: Error when checking input: expected reshape_5_input to have 5 dimensions, but got array with shape (1, 1, 3)　#ここのエラーの意味がよく分かっていない