質問編集履歴
1
内容の加筆
title
CHANGED
|
File without changes
|
body
CHANGED
|
@@ -2,42 +2,211 @@
|
|
|
2
2
|
|
|
3
3
|
現在、Grad-CAMというものが画像を生成する時に変え合わせる数値を確認していたのですがこれを３Dグラフ化して確認したいと考えています。数値の出力はうまくいったのですが、その値（行列式）を用いてグラフ化する方法が調べてもよくわかりません。
|
|
4
4
|
下記コードの”heatmap”という関数をグラフ化し、x軸、z軸を１から２２４、y軸を計算される値にしたいです。
|
|
5
|
+
[参考にしているコードの説明](https://qiita.com/MuAuan/items/cbd739808c64501a1024)
|
|
6
|
+
[コードDLリンク](https://github.com/MuAuan/cheating_DL)
|
|
7
|
+
grad-cam_5category.pyというプログラムです。
|
|
8
|
+
３Dマップで出したいのは下記のコードのheatmap部分です
|
|
9
|
+
一番下に全体のコードを載せておきます。
|
|
10
|
+
printした際、普通にheatmapを出力した時は１列２２４行でしか出なかったのでfor文で224回繰り返すことで224列分出力しています。
|
|
11
|
+
この２２４行２２４列の行列式を３Dプロットしたいです。
|
|
12
|
+
|
|
13
|
+
###該当コード
|
|
14
|
+
```python
|
|
15
|
+
def grad_cam(input_model, image, category_index, layer_name):
|
|
16
|
+
nb_classes = 1000
|
|
17
|
+
target_layer = lambda x: target_category_loss(x, category_index, nb_classes)
|
|
18
|
+
x = Lambda(target_layer, output_shape = target_category_loss_output_shape)(input_model.output)
|
|
19
|
+
model = Model(inputs=input_model.input, outputs=x)
|
|
20
|
+
#model.summary()
|
|
21
|
+
loss = K.sum(model.output)
|
|
22
|
+
conv_output = [l for l in model.layers if l.name == layer_name][0].output #is
|
|
23
|
+
grads = normalize(_compute_gradients(loss, [conv_output])[0])
|
|
24
|
+
gradient_function = K.function([model.input], [conv_output, grads])
|
|
5
25
|
|
|
26
|
+
output, grads_val = gradient_function([image])
|
|
27
|
+
output, grads_val = output[0, :], grads_val[0, :, :, :]
|
|
6
28
|
|
|
29
|
+
weights = np.mean(grads_val, axis = (0, 1))
|
|
30
|
+
cam = np.ones(output.shape[0 : 2], dtype = np.float32)
|
|
7
31
|
|
|
32
|
+
for i, w in enumerate(weights):
|
|
8
|
-
|
|
33
|
+
cam += w * output[:, :, i]
|
|
9
34
|
|
|
35
|
+
cam = cv2.resize(cam, (224,224)) #299,299)) #224, 224))
|
|
36
|
+
cam = np.maximum(cam, 0)
|
|
37
|
+
heatmap = cam / np.max(cam)
|
|
38
|
+
for x in range(224):
|
|
39
|
+
print (heatmap[x]) #３マップで出力したい部分
|
|
40
|
+
|
|
41
|
+
#Return to BGR [0..255] from the preprocessed image
|
|
42
|
+
image = image[0, :]
|
|
43
|
+
image -= np.min(image)
|
|
44
|
+
image = np.minimum(image, 255)
|
|
45
|
+
|
|
46
|
+
cam = cv2.applyColorMap(np.uint8(255*heatmap), cv2.COLORMAP_JET)
|
|
47
|
+
cam = np.float32(cam) + np.float32(image)
|
|
48
|
+
cam = 255 * cam / np.max(cam)
|
|
49
|
+
return np.uint8(cam), heatmap
|
|
50
|
+
```
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
###全体のコード
|
|
10
54
|
```python
|
|
55
|
+
from keras.applications.vgg16 import (VGG16, preprocess_input, decode_predictions)
|
|
56
|
+
from keras.models import Model
|
|
57
|
+
from keras.preprocessing import image
|
|
58
|
+
from keras.layers.core import Lambda
|
|
59
|
+
from keras.models import Sequential
|
|
60
|
+
from tensorflow.python.framework import ops
|
|
61
|
+
import keras.backend as K
|
|
62
|
+
import tensorflow as tf
|
|
63
|
+
import numpy as np
|
|
64
|
+
import keras
|
|
65
|
+
import sys
|
|
66
|
+
import cv2
|
|
67
|
+
#from keras.applications.resnet50 import ResNet50, preprocess_input, decode_predictions
|
|
68
|
+
#from keras.applications.vgg19 import VGG19, preprocess_input, decode_predictions
|
|
69
|
+
#from keras.applications.inception_v3 import InceptionV3, preprocess_input, decode_predictions
|
|
70
|
+
|
|
71
|
+
def target_category_loss(x, category_index, nb_classes):
|
|
72
|
+
return tf.multiply(x, K.one_hot([category_index], nb_classes))
|
|
73
|
+
|
|
74
|
+
def target_category_loss_output_shape(input_shape):
|
|
75
|
+
return input_shape
|
|
76
|
+
|
|
11
|
-
|
|
77
|
+
def normalize(x):
|
|
78
|
+
# utility function to normalize a tensor by its L2 norm
|
|
79
|
+
return x / (K.sqrt(K.mean(K.square(x))) + 1e-5)
|
|
80
|
+
|
|
81
|
+
def load_image(path):
|
|
82
|
+
img_path = sys.argv[1]
|
|
83
|
+
img = image.load_img(img_path, target_size=(224,224)) #299,299)) #224, 224))
|
|
84
|
+
x = image.img_to_array(img)
|
|
85
|
+
x = np.expand_dims(x, axis=0)
|
|
86
|
+
x = preprocess_input(x)
|
|
12
|
-
|
|
87
|
+
return x
|
|
88
|
+
|
|
89
|
+
def register_gradient():
|
|
90
|
+
if "GuidedBackProp" not in ops._gradient_registry._registry:
|
|
13
|
-
|
|
91
|
+
@ops.RegisterGradient("GuidedBackProp")
|
|
14
|
-
|
|
92
|
+
def _GuidedBackProp(op, grad):
|
|
15
|
-
print(predictions.argsort()[0][::-1][i],predictions[0][::-1][i])
|
|
16
|
-
|
|
93
|
+
dtype = op.inputs[0].dtype
|
|
17
|
-
|
|
94
|
+
return grad * tf.cast(grad > 0., dtype) * \
|
|
95
|
+
tf.cast(op.inputs[0] > 0., dtype)
|
|
96
|
+
|
|
97
|
+
def compile_saliency_function(model, activation_layer='block5_conv3'): #mixed10 'activation_49' add_16 add_32 activation_98
|
|
98
|
+
input_img = model.input
|
|
18
|
-
|
|
99
|
+
layer_dict = dict([(layer.name, layer) for layer in model.layers[1:]])
|
|
19
|
-
#print(
|
|
100
|
+
#print(layer_dict)
|
|
101
|
+
layer_output = layer_dict[activation_layer].output
|
|
102
|
+
max_output = K.max(layer_output, axis=3)
|
|
103
|
+
saliency = K.gradients(K.sum(max_output), input_img)[0]
|
|
104
|
+
return K.function([input_img, K.learning_phase()], [saliency])
|
|
105
|
+
|
|
106
|
+
def modify_backprop(model, name):
|
|
107
|
+
g = tf.get_default_graph()
|
|
108
|
+
with g.gradient_override_map({'Relu': name}):
|
|
109
|
+
|
|
110
|
+
# get layers that have an activation
|
|
111
|
+
layer_dict = [layer for layer in model.layers[1:]
|
|
112
|
+
if hasattr(layer, 'activation')]
|
|
113
|
+
|
|
114
|
+
# replace relu activation
|
|
115
|
+
for layer in layer_dict:
|
|
116
|
+
if layer.activation == keras.activations.relu:
|
|
117
|
+
layer.activation = tf.nn.relu
|
|
118
|
+
|
|
119
|
+
# re-instanciate a new model
|
|
120
|
+
new_model = VGG16(weights='imagenet')
|
|
121
|
+
#new_model = ResNet50(weights='imagenet')
|
|
122
|
+
new_model.summary()
|
|
123
|
+
return new_model
|
|
124
|
+
|
|
125
|
+
def deprocess_image(x):
|
|
126
|
+
'''
|
|
127
|
+
Same normalization as in:
|
|
128
|
+
https://github.com/fchollet/keras/blob/master/examples/conv_filter_visualization.py
|
|
129
|
+
'''
|
|
130
|
+
if np.ndim(x) > 3:
|
|
131
|
+
x = np.squeeze(x)
|
|
132
|
+
# normalize tensor: center on 0., ensure std is 0.1
|
|
133
|
+
x -= x.mean()
|
|
134
|
+
x /= (x.std() + 1e-5)
|
|
135
|
+
x *= 0.1
|
|
136
|
+
|
|
137
|
+
# clip to [0, 1]
|
|
138
|
+
x += 0.5
|
|
139
|
+
x = np.clip(x, 0, 1)
|
|
140
|
+
|
|
141
|
+
# convert to RGB array
|
|
142
|
+
x *= 255
|
|
143
|
+
if K.image_dim_ordering() == 'th':
|
|
144
|
+
x = x.transpose((1, 2, 0))
|
|
145
|
+
x = np.clip(x, 0, 255).astype('uint8')
|
|
146
|
+
return x
|
|
147
|
+
|
|
148
|
+
def _compute_gradients(tensor, var_list):
|
|
149
|
+
grads = tf.gradients(tensor, var_list)
|
|
150
|
+
return [grad if grad is not None else tf.zeros_like(var) for var, grad in zip(var_list, grads)]
|
|
151
|
+
|
|
152
|
+
def grad_cam(input_model, image, category_index, layer_name):
|
|
153
|
+
nb_classes = 1000
|
|
154
|
+
target_layer = lambda x: target_category_loss(x, category_index, nb_classes)
|
|
155
|
+
x = Lambda(target_layer, output_shape = target_category_loss_output_shape)(input_model.output)
|
|
156
|
+
model = Model(inputs=input_model.input, outputs=x)
|
|
157
|
+
#model.summary()
|
|
158
|
+
loss = K.sum(model.output)
|
|
159
|
+
conv_output = [l for l in model.layers if l.name == layer_name][0].output #is
|
|
160
|
+
grads = normalize(_compute_gradients(loss, [conv_output])[0])
|
|
161
|
+
gradient_function = K.function([model.input], [conv_output, grads])
|
|
162
|
+
|
|
163
|
+
output, grads_val = gradient_function([image])
|
|
164
|
+
output, grads_val = output[0, :], grads_val[0, :, :, :]
|
|
165
|
+
|
|
166
|
+
weights = np.mean(grads_val, axis = (0, 1))
|
|
167
|
+
cam = np.ones(output.shape[0 : 2], dtype = np.float32)
|
|
168
|
+
|
|
169
|
+
for i, w in enumerate(weights):
|
|
170
|
+
cam += w * output[:, :, i]
|
|
171
|
+
|
|
172
|
+
cam = cv2.resize(cam, (224,224)) #299,299)) #224, 224))
|
|
173
|
+
cam = np.maximum(cam, 0)
|
|
174
|
+
heatmap = cam / np.max(cam)
|
|
20
175
|
for x in range(224):
|
|
21
176
|
print (heatmap[x])
|
|
22
|
-
#print("---------------------")
|
|
23
|
-
#print ("\n".join([str(x) for x in heatmap[x]]))
|
|
24
|
-
|
|
25
|
-
import matplotlib.pyplot as plt
|
|
26
|
-
from mpl_toolkits.mplot3d import Axes3D
|
|
27
|
-
# 3D散布図でプロットするデータを生成する為にnumpyを使用
|
|
28
|
-
X = np.array([heatmap for heatmap in range(224)]) # 自然数の配列
|
|
29
|
-
Y = np.sin(X) # 特に意味のない正弦
|
|
30
|
-
Z = np.sin(Y) # 特に意味のない正弦
|
|
31
177
|
|
|
178
|
+
#Return to BGR [0..255] from the preprocessed image
|
|
32
|
-
|
|
179
|
+
image = image[0, :]
|
|
33
|
-
|
|
180
|
+
image -= np.min(image)
|
|
34
|
-
|
|
181
|
+
image = np.minimum(image, 255)
|
|
35
182
|
|
|
183
|
+
cam = cv2.applyColorMap(np.uint8(255*heatmap), cv2.COLORMAP_JET)
|
|
36
|
-
|
|
184
|
+
cam = np.float32(cam) + np.float32(image)
|
|
37
|
-
|
|
185
|
+
cam = 255 * cam / np.max(cam)
|
|
38
|
-
|
|
186
|
+
return np.uint8(cam), heatmap
|
|
39
|
-
ax.set_zlabel("Z")
|
|
40
187
|
|
|
41
|
-
|
|
188
|
+
preprocessed_input = load_image(sys.argv[1])
|
|
189
|
+
model = VGG16(weights='imagenet')
|
|
190
|
+
#model = VGG19(weights='imagenet')
|
|
42
|
-
|
|
191
|
+
#model = InceptionV3(weights='imagenet')
|
|
192
|
+
#model = ResNet50(weights = 'imagenet')
|
|
193
|
+
#model.summary()
|
|
194
|
+
target_layer = 'block5_conv3' #'activation_49' add_16 "block5_conv3"
|
|
195
|
+
|
|
196
|
+
predictions = model.predict(preprocessed_input)
|
|
197
|
+
register_gradient()
|
|
198
|
+
guided_model = modify_backprop(model, 'GuidedBackProp')
|
|
199
|
+
guided_model.summary()
|
|
200
|
+
for i in range(5):
|
|
201
|
+
top_1 = decode_predictions(predictions)[0][i]
|
|
202
|
+
print(predictions.argsort()[0][::-1][i])
|
|
203
|
+
print('Predicted class:')
|
|
204
|
+
print('%s (%s) with probability %.2f' % (top_1[1], top_1[0], top_1[2]))
|
|
205
|
+
predicted_class = predictions.argsort()[0][::-1][i] #np.argmax(predictions)
|
|
206
|
+
cam, heatmap = grad_cam(model, preprocessed_input, predicted_class, target_layer)
|
|
207
|
+
cv2.imwrite("gradcam"+str(top_1[1])+".jpg", cam)
|
|
208
|
+
saliency_fn = compile_saliency_function(guided_model)
|
|
209
|
+
saliency = saliency_fn([preprocessed_input, 0])
|
|
210
|
+
gradcam = saliency[0] * heatmap[..., np.newaxis]
|
|
211
|
+
cv2.imwrite("guided_gradcam"+str(top_1[1])+".jpg", deprocess_image(gradcam))
|
|
43
212
|
```
|