コード

C#
1using System;
2using System.Linq;
3using System.Collections.Generic;
4using UnityEngine;
5
6public class NewBehaviourScript : MonoBehaviour
7{
8    int now_episode = 1;
9    int now_step = 1;
10    float[][][] table = new float[9][][];
11    float xbefore = 0;
12    float zbefore = 0;
13    int n_posi = 0;
14    int n_rota = 0;
15    int posi = 0;
16    int rota = 0;
17    int state = 0;
18    void Start()
19    {
20        Transform tf = this.transform;
21        Vector3 posi = tf.position;
22        posi.x = -1.6666f;
23        posi.y = 1.1f;
24        posi.z = -1.6666f;
25        tf.position = posi;
26
27        for (int i = 0; i <= 9; i++)
28        {
29            table[i] = new float[9][];
30            for (int i2 = 0; i <= 9; i++)
31            {
32                table[i][i2] = new float[9] { 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f };
33            }
34        }
35    }
36    void Update()
37    {
38        //run
39        int action = Get_action(posi, rota, now_episode);
40        int reward = Get_reward();
41        Rigidbody rb = GetComponent<Rigidbody>();
42        float x = 0.0f;
43        float z = 0.0f;
44        if (action == 3 | action == 4 | action == 5)
45        {
46            x = 5.0f;
47        }
48        if (action == 6 | action == 7 | action == 8)
49        {
50            x = -5.0f;
51        }
52        if (action == 1 | action == 4 | action == 7)
53        {
54            z = 5.0f;
55        }
56        if (action == 2 | action == 5 | action == 8)
57        {
58            z = -5.0f;
59        }
60        Vector3 force = new Vector3(x, 0.0f, z);
61        rb.AddForce(force, ForceMode.Force);
62        State_update();
63        if (rota == 0)
64        {
65            reward += 5;
66        }
67        if (state == 1)
68        {
69            init();
70            reward -= 5;
71            now_episode += 1;
72            now_step = 0;
73        }
74        Q_table(action, reward, posi, rota, n_posi, n_rota);
75        posi = n_posi;
76        rota = n_rota;
77        if (now_step == 100000)
78        {
79            now_episode += 1;
80            now_step = 0;
81            init();
82        }
83        now_step += 1;
84    }
85    int Get_reward()
86    {
87        int reward = 0;
88        Transform tf = this.transform;
89        Quaternion rota = tf.rotation;
90        float x = rota.eulerAngles.x;
91        float z = rota.eulerAngles.z;
92        float x_before = System.Math.Abs(xbefore);
93        float z_before = System.Math.Abs(zbefore);
94        float x_ = System.Math.Abs(x);
95        float z_ = System.Math.Abs(z);
96
97        if (x_before >= x_)
98        {
99            reward += 5;
100        }
101        else
102        {
103            reward -= 5;
104        }
105        if (z_before >= z_)
106        {
107            reward += 5;
108        }
109        else
110        {
111            reward -= 5;
112        }
113        xbefore = x;
114        zbefore = z;
115        return reward;
116    }
117    void State_update()
118    {
119        Transform tf = this.transform;
120        Vector3 posi = tf.position;
121        if (posi.z <= -1.6666f)
122        {
123            if (posi.x <= -1.6666f)
124            {
125                n_posi = 0;
126            }
127            else if (posi.x < 1.6666f)
128            {
129                n_posi = 1;
130            }
131            else if (posi.x >= 1.6666f)
132            {
133                n_posi = 2;
134            }
135        }
136        else if (posi.z < 1.6666f)
137        {
138            if (posi.x <= -1.6666f)
139            {
140                n_posi = 3;
141            }
142            else if (posi.x < 1.6666f)
143            {
144                n_posi = 4;
145            }
146            else if (posi.x >= 1.6666f)
147            {
148                n_posi = 5;
149            }
150        }
151        else if (posi.z >= 1.6666f)
152        {
153            if (posi.x <= -1.6666f)
154            {
155                n_posi = 6;
156            }
157            else if (posi.x < 1.6666f)
158            {
159                n_posi = 7;
160            }
161            else if (posi.x >= 1.6666f)
162            {
163                n_posi = 8;
164            }
165        }
166
167        Quaternion rota = tf.rotation;
168        float x = rota.eulerAngles.x;
169        float z = rota.eulerAngles.z;
170        if (x <= 1.0f & x >= -1.0f)
171        {
172            if (z <= 1.0f & z >= -1.0f)
173            {
174                n_rota = 0;
175            }
176            if (z > 1.0f)
177            {
178                n_rota = 1;
179            }
180            if (z < -1.0f)
181            {
182                n_rota = 2;
183            }
184        }
185        else if (x > 1.0f)
186        {
187            if (z <= 1.0f & z >= -1.0f)
188            {
189                n_rota = 3;
190            }
191            if (z > 1.0f)
192            {
193                n_rota = 4;
194            }
195            if (z < -1.0f)
196            {
197                n_rota = 5;
198            }
199        }
200        else if (x < -1.0f)
201        {
202            if (z <= 1.0f & z >= -1.0f)
203            {
204                n_rota = 6;
205            }
206            if (z > 1.0f)
207            {
208                n_rota = 7;
209            }
210            if (z < -1.0f)
211            {
212                n_rota = 8;
213            }
214        }
215
216        if (System.Math.Abs(x) > 50.0f | System.Math.Abs(z) > 50.0f)
217        {
218            state = 1;
219        }
220    }
221    int Get_action(int next_posi, int next_rota,int episode)
222    {
223        System.Random ra = new System.Random();
224        int action = 0;
225        var index = new List<int>();
226        if (0.5 * (1 / (episode + 1)) <= ra.NextDouble())
227        {
228            float m_ta = table[next_posi][next_rota].Max();
229            for (int i = 0; i < table[next_posi][next_rota].Length; i++)
230            {
231                if (table[next_posi][next_rota][i] == m_ta)
232                {
233                    index.Add(i);
234                }
235            }
236            action = index[ra.Next(0, index.Count - 1)];
237        }
238        else
239        {
240            action = ra.Next(0, 8);
241        }
242        return action;
243    }
244    void Q_table(int action, int reward, int posi, int rota, int next_posi, int next_rota)
245    {
246        float gamma = 0.9f;
247        float alpha = 0.5f;
248        float n_max = table[next_posi][next_rota].Max();
249
250        table[posi][rota][action] = (1 - alpha) * table[posi][rota][action] + alpha * (reward + gamma * n_max);
251    }
252    void init()
253    {
254        System.Random ra = new System.Random();
255        var rd = ra.Next(1, 4);
256        Transform tf = this.transform;
257        tf.rotation = Quaternion.Euler(0.0f, 0.0f, 0.0f);
258        Vector3 posi = tf.position;
259        if (rd == 1)
260        {
261            posi.x = -16.6666f;
262            posi.z = -16.6666f;
263            posi.y = 1.1f;
264            tf.position = posi;
265            n_posi = 0;
266            n_rota = 0;
267        }
268        if (rd == 2)
269        {
270            posi.x = 16.6666f;
271            posi.z = -16.6666f;
272            posi.y = 1.1f;
273            tf.position = posi;
274            n_posi = 2;
275            n_rota = 0;
276        }
277        if (rd == 3)
278        {
279            posi.x = -16.6666f;
280            posi.z = 16.6666f;
281            posi.y = 1.1f;
282            tf.position = posi;
283            n_posi = 6;
284            n_rota = 0;
285        }
286        if (rd == 4)
287        {
288            posi.x = 16.6666f;
289            posi.z = 16.6666f;
290            posi.y = 1.1f;
291            tf.position = posi;
292            n_posi = 8;
293            n_rota = 0;
294        }
295    }
296}