お世話になっております。
multiprocessingを使って、class内の関数を並列に処理したいと考えており、こちらのブロク様にあったMyPoolを使ってClass内の関数でPoolしたいと考えて言います。
#!/usr/bin/env python # coding: utf-8 import sys,re import subprocess import json import functools from mypool import MyPool from multiprocessing import Pool from multiprocessing import Process class Aws_connect(): def command(self, profile, cmd): if cmd in {"get_instance"}: instances_cmd = ('/usr/local/python/bin/aws rds describe-db-instances \ --query \'DBInstances[].{{DBInstanceIdentifier:DBInstanceIdentifier,\ DBInstanceStatus:DBInstanceStatus,\ DBInstanceClass:DBInstanceClass,AllocatedStorage:AllocatedStorage,\ InstanceCreateTime:InstanceCreateTime,Engine:Engine,\ Endpoint:Endpoint,EngineVersion:EngineVersioni,\ DBClusterIdentifier:DBClusterIdentifier}}\' \ --profile {in_profile}').format(in_profile=profile) instances = subprocess.check_output(instances_cmd,shell=True) return instances elif cmd in {"get_snapshot"}: snapshots_cmd = ('/usr/local/python/bin/aws rds describe-db-snapshots \ --query \'reverse(sort_by(DBSnapshots,&InstanceCreateTime))[].\ {{SnapshotCreateTime:SnapshotCreateTime,\ DBInstanceIdentifier:DBInstanceIdentifier,\ DBSnapshotIdentifier:DBSnapshotIdentifier,\ Engine:Engine,\ EngineVersion:EngineVersion,\ AllocatedStorage:AllocatedStorage}}\' \ --profile {in_profile}').format(in_profile=profile) snapshots = subprocess.check_output(snapshots_cmd,shell=True) return snapshots elif cmd in {"get_cluster_instance"}: clu_instances_cmd = ('/usr/local/python/bin/aws rds describe-db-clusters --query \ \'DBClusters[].{{DBClusterMembers:DBClusterMembers[].DBInstanceIdentifier,\ AllocatedStorage:AllocatedStorage,\ Engine:Engine,\ DBClusterIdentifier:DBClusterIdentifier,\ Endpoint:Endpoint,\ EngineVersion:EngineVersion}}\' \ --profile {in_profile}').format(in_profile=profile) clu_instances = subprocess.check_output(clu_instances_cmd,shell=True) return clu_instances elif cmd in {"get_cluster_snapshot"}: clu_snapshots_cmd = ('/usr/local/python/bin/aws rds describe-db-cluster-snapshots \ --query \'DBClusterSnapshots[].\ {{Engine:Engine,\ DBClusterIdentifier:DBClusterIdentifier,\ SnapshotCreateTime:SnapshotCreateTime,\ AllocatedStorage:AllocatedStorage,\ ClusterCreateTime:ClusterCreateTime,\ VpcId:VpcId, \ DBClusterSnapshotIdentifier:DBClusterSnapshotIdentifier}}\' \ --profile {in_profile}').format(in_profile=profile) clu_snapshots = subprocess.check_output(clu_snapshots_cmd,shell=True) return clu_snapshots # main def execute(self, profile): self.profile = profile cmd = ['get_instance','get_snapshot','get_cluster_snapshot','get_cluster_instance'] p = MyPool(4) result = p.map(functools.partial(self.command , profile=self.profile), cmd) print result aws = Aws_connect() aws.execute('system')
これを実行すると、
Process Process-1: Process Process-2: Traceback (most recent call last): Traceback (most recent call last): File "/usr/local/python/lib/python2.7/multiprocessing/process.py", line 258, in _bootstrap File "/usr/local/python/lib/python2.7/multiprocessing/process.py", line 258, in _bootstrap self.run() File "/usr/local/python/lib/python2.7/multiprocessing/process.py", line 114, in run self._target(*self._args, **self._kwargs) File "/root/work/matsumoto/mypool.py", line 21, in pipefunc self.run() File "/usr/local/python/lib/python2.7/multiprocessing/process.py", line 114, in run conn.send(func(arg)) self._target(*self._args, **self._kwargs) File "/root/work/matsumoto/mypool.py", line 21, in pipefunc Process Process-3: TypeError: command() got multiple values for keyword argument 'profile' Traceback (most recent call last): conn.send(func(arg)) File "/usr/local/python/lib/python2.7/multiprocessing/process.py", line 258, in _bootstrap TypeError: command() got multiple values for keyword argument 'profile' self.run() File "/usr/local/python/lib/python2.7/multiprocessing/process.py", line 114, in run self._target(*self._args, **self._kwargs) File "/root/work/matsumoto/mypool.py", line 21, in pipefunc Process Process-4: conn.send(func(arg)) Traceback (most recent call last): TypeError: command() got multiple values for keyword argument 'profile' File "/usr/local/python/lib/python2.7/multiprocessing/process.py", line 258, in _bootstrap self.run() File "/usr/local/python/lib/python2.7/multiprocessing/process.py", line 114, in run self._target(*self._args, **self._kwargs) File "/root/work/matsumoto/mypool.py", line 21, in pipefunc conn.send(func(arg)) TypeError: command() got multiple values for keyword argument 'profile' ^CTraceback (most recent call last): File "aws_connect.py", line 75, in <module> aws.execute('system') File "aws_connect.py", line 71, in execute result = p.map(functools.partial(self.command , profile=self.profile), cmd) File "/root/work/matsumoto/mypool.py", line 36, in map ret.append(conn.recv())
とエラーが発生して動きません。
どこを直したら良いのかご教示頂けますでしょうか。
回答1件
あなたの回答
tips
プレビュー
バッドをするには、ログインかつ
こちらの条件を満たす必要があります。