teratail
回答率
85.48%
質問するログイン新規登録
トップに関する質問Apache Spark(Java) + Spring Boot構成で、Sparkアクション実行時にDIできない

編集履歴

質問編集履歴

1

例外スタックトレース追記、サンプルコード追記

2016/07/03 01:55

投稿

guzzle
guzzle

スコア43

test CHANGED
File without changes
test CHANGED
@@ -12,11 +12,79 @@
12
12
 
13
13
 
14
14
 
15
+ **サンプルコードをgithubにUPしました。**Java8が入ってれば動作すると思います。
16
+
17
+ [https://github.com/guzzle0527/spark-spring-sample](https://github.com/guzzle0527/spark-spring-sample)
18
+
19
+
20
+
15
21
  ###発生している問題・エラーメッセージ
16
22
 
17
23
  ```
18
24
 
19
25
  Caused by: org.apache.spark.SparkException: Task not serializable
26
+
27
+ at org.apache.spark.util.ClosureCleaner$.ensureSerializable(ClosureCleaner.scala:304) ~[spark-core_2.10-1.6.0.jar:1.6.0]
28
+
29
+ at org.apache.spark.util.ClosureCleaner$.org$apache$spark$util$ClosureCleaner$$clean(ClosureCleaner.scala:294) ~[spark-core_2.10-1.6.0.jar:1.6.0]
30
+
31
+ at org.apache.spark.util.ClosureCleaner$.clean(ClosureCleaner.scala:122) ~[spark-core_2.10-1.6.0.jar:1.6.0]
32
+
33
+ at org.apache.spark.SparkContext.clean(SparkContext.scala:2055) ~[spark-core_2.10-1.6.0.jar:1.6.0]
34
+
35
+ at org.apache.spark.rdd.RDD$$anonfun$foreach$1.apply(RDD.scala:911) ~[spark-core_2.10-1.6.0.jar:1.6.0]
36
+
37
+ at org.apache.spark.rdd.RDD$$anonfun$foreach$1.apply(RDD.scala:910) ~[spark-core_2.10-1.6.0.jar:1.6.0]
38
+
39
+ at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:150) ~[spark-core_2.10-1.6.0.jar:1.6.0]
40
+
41
+ at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:111) ~[spark-core_2.10-1.6.0.jar:1.6.0]
42
+
43
+ at org.apache.spark.rdd.RDD.withScope(RDD.scala:316) ~[spark-core_2.10-1.6.0.jar:1.6.0]
44
+
45
+ at org.apache.spark.rdd.RDD.foreach(RDD.scala:910) ~[spark-core_2.10-1.6.0.jar:1.6.0]
46
+
47
+ at org.apache.spark.api.java.JavaRDDLike$class.foreach(JavaRDDLike.scala:332) ~[spark-core_2.10-1.6.0.jar:1.6.0]
48
+
49
+ at org.apache.spark.api.java.AbstractJavaRDDLike.foreach(JavaRDDLike.scala:46) ~[spark-core_2.10-1.6.0.jar:1.6.0]
50
+
51
+ at sample.HelloWorld.run(HelloWorld.java:29) [classes/:na]
52
+
53
+ at org.springframework.boot.SpringApplication.callRunner(SpringApplication.java:804) [spring-boot-1.3.1.RELEASE.jar:1.3.1.RELEASE]
54
+
55
+ ... 7 common frames omitted
56
+
57
+ Caused by: java.io.NotSerializableException: sample.HelloWorld$$EnhancerBySpringCGLIB$$5e382b74
58
+
59
+ Serialization stack:
60
+
61
+ - object not serializable (class: sample.HelloWorld$$EnhancerBySpringCGLIB$$5e382b74, value: sample.HelloWorld$$EnhancerBySpringCGLIB$$5e382b74@2a225dd7)
62
+
63
+ - element of array (index: 0)
64
+
65
+ - array (class [Ljava.lang.Object;, size 1)
66
+
67
+ - field (class: java.lang.invoke.SerializedLambda, name: capturedArgs, type: class [Ljava.lang.Object;)
68
+
69
+ - object (class java.lang.invoke.SerializedLambda, SerializedLambda[capturingClass=class sample.HelloWorld, functionalInterfaceMethod=org/apache/spark/api/java/function/VoidFunction.call:(Ljava/lang/Object;)V, implementation=invokeSpecial sample/HelloWorld.lambda$0:(Ljava/lang/String;)V, instantiatedMethodType=(Ljava/lang/String;)V, numCaptured=1])
70
+
71
+ - writeReplace data (class: java.lang.invoke.SerializedLambda)
72
+
73
+ - object (class sample.HelloWorld$$Lambda$2/1447521302, sample.HelloWorld$$Lambda$2/1447521302@3bb5ceb)
74
+
75
+ - field (class: org.apache.spark.api.java.JavaRDDLike$$anonfun$foreach$1, name: f$14, type: interface org.apache.spark.api.java.function.VoidFunction)
76
+
77
+ - object (class org.apache.spark.api.java.JavaRDDLike$$anonfun$foreach$1, <function1>)
78
+
79
+ at org.apache.spark.serializer.SerializationDebugger$.improveException(SerializationDebugger.scala:40) ~[spark-core_2.10-1.6.0.jar:1.6.0]
80
+
81
+ at org.apache.spark.serializer.JavaSerializationStream.writeObject(JavaSerializer.scala:47) ~[spark-core_2.10-1.6.0.jar:1.6.0]
82
+
83
+ at org.apache.spark.serializer.JavaSerializerInstance.serialize(JavaSerializer.scala:101) ~[spark-core_2.10-1.6.0.jar:1.6.0]
84
+
85
+ at org.apache.spark.util.ClosureCleaner$.ensureSerializable(ClosureCleaner.scala:301) ~[spark-core_2.10-1.6.0.jar:1.6.0]
86
+
87
+ ... 20 common frames omitted
20
88
 
21
89
  ```
22
90
 
@@ -26,17 +94,19 @@
26
94
 
27
95
  ```Java
28
96
 
29
- @Component
97
+ @Service
30
98
 
31
99
  @Scope("prototype")
32
100
 
33
- public class TestImpl impliments Test {
101
+ public class TestServiceImpl implements TestService, Serializable {
34
102
 
35
- public void xxx(Model model) {
103
+ @Override
36
104
 
37
- // 何かロジック
105
+ public void test(String data) {
38
106
 
107
+ System.out.println(data);
108
+
39
- }
109
+ }
40
110
 
41
111
  }
42
112
 
@@ -44,39 +114,53 @@
44
114
 
45
115
  ```Java
46
116
 
47
- @Service
117
+ @SpringBootApplication
48
118
 
49
- @Scope("prototype")
50
-
51
- public class Sample {
119
+ public class HelloWorld implements CommandLineRunner {
52
-
53
- @Autowired
54
-
55
- private Test test;
56
-
57
- @Autowired
58
-
59
- private ApplicationContext context;
60
120
 
61
121
 
62
122
 
63
- public void test() {
123
+ @Autowired
64
124
 
65
- JavaRdd<Model> rdd = XXX;
125
+ private TestService testService;
66
-
67
- rdd.foreach(f -> {
68
-
69
- test.xxx(f); // Task not serializableエラー
70
-
71
- Test t1 = context.getBean(Test.class); // これもTask not serializableエラー
72
-
73
- Test t2 = new TestImpl(); // これは動くけど。。。
74
-
75
- });
76
-
77
- }
78
126
 
79
127
 
128
+
129
+ @Autowired
130
+
131
+ private ApplicationContext context;
132
+
133
+
134
+
135
+ public static void main(String[] args) {
136
+
137
+ SpringApplication.run(HelloWorld.class, args);
138
+
139
+ }
140
+
141
+
142
+
143
+ @Override
144
+
145
+ public void run(String... args) throws Exception {
146
+
147
+ JavaSparkContext sc = new JavaSparkContext("local[*]", "Hello");
148
+
149
+ JavaRDD<String> rdd = sc.parallelize(Arrays.asList("a", "b", "c", "d", "e"));
150
+
151
+
152
+
153
+ rdd.foreach(data -> {
154
+
155
+ testService.test(data); // Task not serializableエラー
156
+
157
+ TestService t1 = context.getBean(TestService.class); // これもTask not serializableエラー
158
+
159
+ TestService t2 = new TestServiceImpl(); // これは動作する
160
+
161
+ });
162
+
163
+ }
80
164
 
81
165
  }
82
166