Pythonに不慣れなうちは標準ライブラリで並列化したほうが良いですよ。
PythonにはJavaみたいなThreadPoolExecutor
と、それをマルチプロセスでやってしまおうというProcessPoolExecutor
があります。
詳細は以下のコードとその実行結果、ドキュメントを参照してください。
python
1from concurrent.futures import ProcessPoolExecutor
2from concurrent.futures.process import BrokenProcessPool
3from concurrent.futures import ThreadPoolExecutor
4from os import getpid
5from time import sleep
6from threading import current_thread
7
8
9def func_result_as_str(n):
10 sleep(1) # あまりにはやいと並列化されないのでsleep
11 return "n=%d, Process ID=%d, Thread ID=%d" %(
12 n, getpid(), current_thread().ident)
13
14
15def func_result_as_generator(n):
16 yield from func_result_as_str(n)
17
18
19def func_result_as_tuple(n):
20 return tuple(func_result_as_generator(n))
21
22
23def main():
24 N_jobs = 10
25
26 # スレッドベースの並列化
27 # メリット - 基本的になんでも簡単に並列化できる
28 # デメリット - GILの制限から解放されない
29 with ThreadPoolExecutor() as pool:
30 print("*" * 10, "func_result_as_strの戻り値たち: Thread IDがばらばら")
31 for result in pool.map(func_result_as_str, range(N_jobs)):
32 print(result)
33
34 print("*" * 10, "func_result_as_generatorの戻り値たち: generatorが返ってくる、"
35 "でも結局generator生成だけが並列化されてそれを回すところが並列されない")
36 for result in pool.map(func_result_as_generator, range(N_jobs)):
37 print(result, "".join(result))
38
39 # プロセスベースの並列化
40 # メリット - GILの制限から解放される
41 # デメリット - プロセス間でpicklable(シリアライズ可能)なオブジェクトしかやり取りできない
42 with ProcessPoolExecutor() as pool:
43 print("*" * 10, "strはpicklable(シリアライズ可能)なのでProcessPoolExecutor"
44 "でも並列化OK、Process IDがばらばら")
45 for result in pool.map(func_result_as_str, range(N_jobs)):
46 print(result)
47
48 print("*" * 10, "generatorはpicklable(シリアライズ可能)ではないのでなので"
49 "ProcessPoolExecutorでは並列化NG")
50 try:
51 for result in pool.map(func_result_as_generator, range(N_jobs)):
52 print(result, "".join(result))
53 except BrokenProcessPool as e:
54 print(e)
55
56 with ProcessPoolExecutor() as pool:
57 print("*" * 10, "tupleはpicklable(シリアライズ可能)なのでなので"
58 "ProcessPoolExecutorでも並列化OK")
59 for result in pool.map(func_result_as_tuple, range(N_jobs)):
60 print(result, "".join(result))
61
62
63if __name__ == "__main__": # マルチプロセス化には必須
64 main()
********** func_result_as_strの戻り値たち: Thread IDがばらばら
n=0, Process ID=13768, Thread ID=7692
n=1, Process ID=13768, Thread ID=15164
n=2, Process ID=13768, Thread ID=13164
n=3, Process ID=13768, Thread ID=1312
n=4, Process ID=13768, Thread ID=2292
n=5, Process ID=13768, Thread ID=7260
n=6, Process ID=13768, Thread ID=6204
n=7, Process ID=13768, Thread ID=1296
n=8, Process ID=13768, Thread ID=16284
n=9, Process ID=13768, Thread ID=14668
********** func_result_as_generatorの戻り値たち: generatorが返ってくる、でも結局generator生成だけが並列化されてそれを回すところが並列されない
<generator object func_result_as_generator at 0x0000019061330410> n=0, Process ID=13768, Thread ID=12788
<generator object func_result_as_generator at 0x0000019061330468> n=1, Process ID=13768, Thread ID=12788
<generator object func_result_as_generator at 0x0000019061330570> n=2, Process ID=13768, Thread ID=12788
<generator object func_result_as_generator at 0x00000190613305C8> n=3, Process ID=13768, Thread ID=12788
<generator object func_result_as_generator at 0x00000190613304C0> n=4, Process ID=13768, Thread ID=12788
<generator object func_result_as_generator at 0x0000019061330620> n=5, Process ID=13768, Thread ID=12788
<generator object func_result_as_generator at 0x0000019061330518> n=6, Process ID=13768, Thread ID=12788
<generator object func_result_as_generator at 0x0000019061330780> n=7, Process ID=13768, Thread ID=12788
<generator object func_result_as_generator at 0x0000019061330728> n=8, Process ID=13768, Thread ID=12788
<generator object func_result_as_generator at 0x00000190613306D0> n=9, Process ID=13768, Thread ID=12788
********** strはpicklable(シリアライズ可能)なのでProcessPoolExecutorでも並列化OK、Process IDがばらばら
n=0, Process ID=8180, Thread ID=8664
n=1, Process ID=3628, Thread ID=15880
n=2, Process ID=8920, Thread ID=12636
n=3, Process ID=6400, Thread ID=908
n=4, Process ID=8180, Thread ID=8664
n=5, Process ID=3628, Thread ID=15880
n=6, Process ID=8920, Thread ID=12636
n=7, Process ID=6400, Thread ID=908
n=8, Process ID=8180, Thread ID=8664
n=9, Process ID=3628, Thread ID=15880
********** generatorはpicklable(シリアライズ可能)ではないのでなのでProcessPoolExecutorでは並列化NG
Process Process-4:
Process Process-3:
Process Process-1:
Process Process-2:
Traceback (most recent call last):
File "C:\Python36\lib\multiprocessing\process.py", line 249, in _bootstrap
self.run()
File "C:\Python36\lib\multiprocessing\process.py", line 93, in run
self._target(*self._args, **self._kwargs)
File "C:\Python36\lib\concurrent\futures\process.py", line 181, in _process_worker
result=r))
File "C:\Python36\lib\multiprocessing\queues.py", line 349, in put
obj = _ForkingPickler.dumps(obj)
File "C:\Python36\lib\multiprocessing\reduction.py", line 51, in dumps
cls(buf, protocol).dump(obj)
TypeError: can't pickle generator objects
Traceback (most recent call last):
File "C:\Python36\lib\multiprocessing\process.py", line 249, in _bootstrap
self.run()
File "C:\Python36\lib\multiprocessing\process.py", line 93, in run
self._target(*self._args, **self._kwargs)
File "C:\Python36\lib\concurrent\futures\process.py", line 181, in _process_worker
result=r))
File "C:\Python36\lib\multiprocessing\queues.py", line 349, in put
obj = _ForkingPickler.dumps(obj)
File "C:\Python36\lib\multiprocessing\reduction.py", line 51, in dumps
cls(buf, protocol).dump(obj)
TypeError: can't pickle generator objects
Traceback (most recent call last):
File "C:\Python36\lib\multiprocessing\process.py", line 249, in _bootstrap
self.run()
File "C:\Python36\lib\multiprocessing\process.py", line 93, in run
self._target(*self._args, **self._kwargs)
File "C:\Python36\lib\concurrent\futures\process.py", line 181, in _process_worker
result=r))
File "C:\Python36\lib\multiprocessing\queues.py", line 349, in put
obj = _ForkingPickler.dumps(obj)
File "C:\Python36\lib\multiprocessing\reduction.py", line 51, in dumps
cls(buf, protocol).dump(obj)
Traceback (most recent call last):
TypeError: can't pickle generator objects
File "C:\Python36\lib\multiprocessing\process.py", line 249, in _bootstrap
self.run()
File "C:\Python36\lib\multiprocessing\process.py", line 93, in run
self._target(*self._args, **self._kwargs)
File "C:\Python36\lib\concurrent\futures\process.py", line 181, in _process_worker
result=r))
File "C:\Python36\lib\multiprocessing\queues.py", line 349, in put
obj = _ForkingPickler.dumps(obj)
File "C:\Python36\lib\multiprocessing\reduction.py", line 51, in dumps
cls(buf, protocol).dump(obj)
TypeError: can't pickle generator objects
A process in the process pool was terminated abruptly while the future was running or pending.
********** tupleはpicklable(シリアライズ可能)なのでなのでProcessPoolExecutorでも並列化OK
('n', '=', '0', ',', ' ', 'P', 'r', 'o', 'c', 'e', 's', 's', ' ', 'I', 'D', '=', '1', '2', '5', '1', '2', ',', ' ', 'T', 'h', 'r', 'e', 'a', 'd', ' ', 'I', 'D', '=', '9', '9', '2', '4') n=0, Process ID=12512, Thread ID=9924
('n', '=', '1', ',', ' ', 'P', 'r', 'o', 'c', 'e', 's', 's', ' ', 'I', 'D', '=', '1', '3', '8', '0', '8', ',', ' ', 'T', 'h', 'r', 'e', 'a', 'd', ' ', 'I', 'D', '=', '1', '5', '5', '9', '6') n=1, Process ID=13808, Thread ID=15596
('n', '=', '2', ',', ' ', 'P', 'r', 'o', 'c', 'e', 's', 's', ' ', 'I', 'D', '=', '9', '7', '8', '8', ',', ' ', 'T', 'h', 'r', 'e', 'a', 'd', ' ', 'I', 'D', '=', '1', '1', '2', '7', '6') n=2, Process ID=9788, Thread ID=11276
('n', '=', '3', ',', ' ', 'P', 'r', 'o', 'c', 'e', 's', 's', ' ', 'I', 'D', '=', '4', '9', '9', '6', ',', ' ', 'T', 'h', 'r', 'e', 'a', 'd', ' ', 'I', 'D', '=', '1', '3', '5', '2', '4') n=3, Process ID=4996, Thread ID=13524
('n', '=', '4', ',', ' ', 'P', 'r', 'o', 'c', 'e', 's', 's', ' ', 'I', 'D', '=', '1', '2', '5', '1', '2', ',', ' ', 'T', 'h', 'r', 'e', 'a', 'd', ' ', 'I', 'D', '=', '9', '9', '2', '4') n=4, Process ID=12512, Thread ID=9924
('n', '=', '5', ',', ' ', 'P', 'r', 'o', 'c', 'e', 's', 's', ' ', 'I', 'D', '=', '1', '3', '8', '0', '8', ',', ' ', 'T', 'h', 'r', 'e', 'a', 'd', ' ', 'I', 'D', '=', '1', '5', '5', '9', '6') n=5, Process ID=13808, Thread ID=15596
('n', '=', '6', ',', ' ', 'P', 'r', 'o', 'c', 'e', 's', 's', ' ', 'I', 'D', '=', '9', '7', '8', '8', ',', ' ', 'T', 'h', 'r', 'e', 'a', 'd', ' ', 'I', 'D', '=', '1', '1', '2', '7', '6') n=6, Process ID=9788, Thread ID=11276
('n', '=', '7', ',', ' ', 'P', 'r', 'o', 'c', 'e', 's', 's', ' ', 'I', 'D', '=', '4', '9', '9', '6', ',', ' ', 'T', 'h', 'r', 'e', 'a', 'd', ' ', 'I', 'D', '=', '1', '3', '5', '2', '4') n=7, Process ID=4996, Thread ID=13524
('n', '=', '8', ',', ' ', 'P', 'r', 'o', 'c', 'e', 's', 's', ' ', 'I', 'D', '=', '1', '2', '5', '1', '2', ',', ' ', 'T', 'h', 'r', 'e', 'a', 'd', ' ', 'I', 'D', '=', '9', '9', '2', '4') n=8, Process ID=12512, Thread ID=9924
('n', '=', '9', ',', ' ', 'P', 'r', 'o', 'c', 'e', 's', 's', ' ', 'I', 'D', '=', '1', '3', '8', '0', '8', ',', ' ', 'T', 'h', 'r', 'e', 'a', 'd', ' ', 'I', 'D', '=', '1', '5', '5', '9', '6') n=9, Process ID=13808, Thread ID=15596
ちなみに多重度の初期値はProcessPoolExecutorよりもThreadPoolExecutorのほうが多くなっています。
バッドをするには、ログインかつ
こちらの条件を満たす必要があります。
2017/11/12 06:22
2017/11/12 06:47
2017/11/12 09:04
2017/11/12 10:22
2017/11/12 23:30
2017/11/13 00:06