回答編集履歴
1
コードを追記
answer
CHANGED
|
@@ -6,4 +6,33 @@
|
|
|
6
6
|
|
|
7
7
|
return tf.math.rsqrt(self.d_model) * tf.math.minimum(arg1, arg2)
|
|
8
8
|
```
|
|
9
|
-
[tf.math.rsqrt](https://www.tensorflow.org/api_docs/python/tf/math/rsqrt#args)
|
|
9
|
+
[tf.math.rsqrt](https://www.tensorflow.org/api_docs/python/tf/math/rsqrt#args)
|
|
10
|
+
|
|
11
|
+
---
|
|
12
|
+
Google Colaboratory(cpu)で実際に本日試したコードを載せます。
|
|
13
|
+
|
|
14
|
+
```Python
|
|
15
|
+
import tensorflow as tf
|
|
16
|
+
|
|
17
|
+
class CustomSchedule(tf.keras.optimizers.schedules.LearningRateSchedule):
|
|
18
|
+
def __init__(self, d_model, warmup_steps=4000):
|
|
19
|
+
super(CustomSchedule, self).__init__()
|
|
20
|
+
|
|
21
|
+
self.d_model = tf.cast(d_model, tf.float32)
|
|
22
|
+
self.warmup_steps = warmup_steps
|
|
23
|
+
|
|
24
|
+
def __call__(self, step):
|
|
25
|
+
step = tf.cast(step, tf.float32)
|
|
26
|
+
arg1 = tf.math.rsqrt(step)
|
|
27
|
+
arg2 = step * (self.warmup_steps ** -1.5)
|
|
28
|
+
|
|
29
|
+
return tf.math.rsqrt(self.d_model) * tf.math.minimum(arg1, arg2)
|
|
30
|
+
|
|
31
|
+
learning_rate = CustomSchedule(128)
|
|
32
|
+
print(f'learning_rate: {learning_rate}')
|
|
33
|
+
optimizer = tf.keras.optimizers.Adam(learning_rate, beta_1=0.9, beta_2=0.98, epsilon=1e-9)
|
|
34
|
+
print('optimizer is ready')
|
|
35
|
+
|
|
36
|
+
# learning_rate: <__main__.CustomSchedule object at 0x7bfec8bdb130>
|
|
37
|
+
# optimizer is ready
|
|
38
|
+
```
|