前提・実現したいこと
TwitterのAPIを通して、ツイートデータを取得し、そのデータをcsvに保存することが目的です。
1週間ほど前まで、問題なくできていたのですが、ここ最近、急にTimeoutエラーが表示され、途中で取得することができなくなってしまいました。
発生している問題・エラーメッセージ
TimeoutError Traceback (most recent call last)
~/opt/anaconda3/lib/python3.8/site-packages/urllib3/connection.py in _new_conn(self)
158 try:
--> 159 conn = connection.create_connection(
160 (self._dns_host, self.port), self.timeout, **extra_kw
~/opt/anaconda3/lib/python3.8/site-packages/urllib3/util/connection.py in create_connection(address, timeout, source_address, socket_options)
83 if err is not None:
---> 84 raise err
85
~/opt/anaconda3/lib/python3.8/site-packages/urllib3/util/connection.py in create_connection(address, timeout, source_address, socket_options)
73 sock.bind(source_address)
---> 74 sock.connect(sa)
75 return sock
TimeoutError: [Errno 60] Operation timed out
During handling of the above exception, another exception occurred:
NewConnectionError Traceback (most recent call last)
~/opt/anaconda3/lib/python3.8/site-packages/urllib3/connectionpool.py in urlopen(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, **response_kw)
669 # Make the request on the httplib connection object.
--> 670 httplib_response = self._make_request(
671 conn,
~/opt/anaconda3/lib/python3.8/site-packages/urllib3/connectionpool.py in _make_request(self, conn, method, url, timeout, chunked, **httplib_request_kw)
380 try:
--> 381 self._validate_conn(conn)
382 except (SocketTimeout, BaseSSLError) as e:
~/opt/anaconda3/lib/python3.8/site-packages/urllib3/connectionpool.py in _validate_conn(self, conn)
977 if not getattr(conn, "sock", None): # AppEngine might not have .sock
--> 978 conn.connect()
979
~/opt/anaconda3/lib/python3.8/site-packages/urllib3/connection.py in connect(self)
308 # Add certificate verification
--> 309 conn = self._new_conn()
310 hostname = self.host
~/opt/anaconda3/lib/python3.8/site-packages/urllib3/connection.py in _new_conn(self)
170 except SocketError as e:
--> 171 raise NewConnectionError(
172 self, "Failed to establish a new connection: %s" % e
NewConnectionError: <urllib3.connection.HTTPSConnection object at 0x7fb7560f1d60>: Failed to establish a new connection: [Errno 60] Operation timed out
During handling of the above exception, another exception occurred:
MaxRetryError Traceback (most recent call last)
~/opt/anaconda3/lib/python3.8/site-packages/requests/adapters.py in send(self, request, stream, timeout, verify, cert, proxies)
438 if not chunked:
--> 439 resp = conn.urlopen(
440 method=request.method,
~/opt/anaconda3/lib/python3.8/site-packages/urllib3/connectionpool.py in urlopen(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, **response_kw)
725
--> 726 retries = retries.increment(
727 method, url, error=e, _pool=self, _stacktrace=sys.exc_info()[2]
~/opt/anaconda3/lib/python3.8/site-packages/urllib3/util/retry.py in increment(self, method, url, response, error, _pool, _stacktrace)
445 if new_retry.is_exhausted():
--> 446 raise MaxRetryError(_pool, url, error or ResponseError(cause))
447
MaxRetryError: HTTPSConnectionPool(host='api.twitter.com', port=443): Max retries exceeded with url: /1.1/application/rate_limit_status.json (Caused by NewConnectionError('<urllib3.connection.HTTPSConnection object at 0x7fb7560f1d60>: Failed to establish a new connection: [Errno 60] Operation timed out'))
During handling of the above exception, another exception occurred:
ConnectionError Traceback (most recent call last)
<ipython-input-26-85fd4aeb3d5c> in <module>
1 for i1 in range(5000):
2 #API使用制限回数チェック
----> 3 limit, remaining, reset_minute = rate_limit_status()
4 print(remaining)
5 #API使用制限解除待機
<ipython-input-12-011298f97dcf> in rate_limit_status()
5 twitter = OAuth1Session(CONSUMER_KEY, CONSUMER_SECRE, ACCESS_TOKEN, ACCESS_SECRET)
6 url = "https://api.twitter.com/1.1/application/rate_limit_status.json"
----> 7 req = twitter.get(url)
8 if req.status_code == 200:
9 res = json.loads(req.text)
~/opt/anaconda3/lib/python3.8/site-packages/requests/sessions.py in get(self, url, **kwargs)
553
554 kwargs.setdefault('allow_redirects', True)
--> 555 return self.request('GET', url, **kwargs)
556
557 def options(self, url, **kwargs):
~/opt/anaconda3/lib/python3.8/site-packages/requests/sessions.py in request(self, method, url, params, data, headers, cookies, files, auth, timeout, allow_redirects, proxies, hooks, stream, verify, cert, json)
540 }
541 send_kwargs.update(settings)
--> 542 resp = self.send(prep, **send_kwargs)
543
544 return resp
~/opt/anaconda3/lib/python3.8/site-packages/requests/sessions.py in send(self, request, **kwargs)
653
654 # Send the request
--> 655 r = adapter.send(request, **kwargs)
656
657 # Total elapsed time of the request (approximately)
~/opt/anaconda3/lib/python3.8/site-packages/requests/adapters.py in send(self, request, stream, timeout, verify, cert, proxies)
514 raise SSLError(e, request=request)
515
--> 516 raise ConnectionError(e, request=request)
517
518 except ClosedPoolError as e:
ConnectionError: HTTPSConnectionPool(host='api.twitter.com', port=443): Max retries exceeded with url: /1.1/application/rate_limit_status.json (Caused by NewConnectionError('<urllib3.connection.HTTPSConnection object at 0x7fb7560f1d60>: Failed to establish a new connection: [Errno 60] Operation timed out'))
該当のソースコード
Python
1for i1 in range(5000): 2 #API使用制限回数チェック 3 limit, remaining, reset_minute = rate_limit_status() 4 print(remaining) 5 #API使用制限解除待機 6 if remaining == 0: 7 print("** Waiting for limit release **") 8 time.sleep(60 * (int(reset_minute) + 1)) 9 time.sleep(1) 10 twitter, params, req = getParam() 11 if req.status_code == 200: 12 res = json.loads(req.text) 13 i2 = 0 14 i3 = 0 15 for line in res['statuses']: 16 i3 += 1 17 if i3 <= 3: 18 continue 19 t = str_to_date_jp(line["created_at"]) 20 maxid = line['id'] 21 maxid -= 1 22 #Nameの文字列を整形 23 csvlinename = line["user"]["name"].strip() 24 csvlinename = csvlinename.replace(",", " ") 25 csvlines = csvlinename.splitlines() 26 csvlinename = ''.join(csvlines) 27 #Screen Nameの文字列を整形 28 csvlinescreen_name = line["user"]["screen_name"].strip() 29 csvlinescreen_name = csvlinescreen_name.replace(",", " ") 30 csvlines = csvlinescreen_name.splitlines() 31 csvlinescreen_name = ''.join(csvlines) 32 #textの文字列を整形 33 csvlinetext = line["text"].strip() 34 csvlinetext = csvlinetext.replace(",", " ") 35 csvlines = csvlinetext.splitlines() 36 csvlinetext = ''.join(csvlines) 37 #descriptionの文字列を整形 38 csvlinedescription = line["user"]["description"].replace(",", " ") 39 csvlinedescription = csvlinedescription.strip() 40 csvlines = csvlinedescription.splitlines() 41 csvlinedescription = ''.join(csvlines) 42 #locationの文字列を整形 43 csvlinelocation = line["user"]["location"].replace(",", " ") 44 csvlinelocation = csvlinelocation.strip() 45 csvlines = csvlinelocation.splitlines() 46 csvlinelocation = ''.join(csvlines) 47 csvline1 = "%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s" % (t,line["id"],line["user"]["id"],csvlinename,csvlinescreen_name,csvlinetext,line["user"]["followers_count"],line["user"]["friends_count"],line["favorite_count"],line["retweet_count"],csvlinedescription,csvlinelocation) 48 #末尾に改行を追加 49 csvline1 = format(csvline1) + '\n' 50 with open("twitter_data_%s_%s.csv" % (keyword,today), "a") as f: 51 f.write(csvline1) 52 i2 += 1 53 else: 54 print("Failed: %d" % req.status_code) 55f.close() 56print(maxid) 57print("** Finished **")
試したこと
最初はネットワーク上のエラーだと思い、インターネットの接続を見ましたが、何も異常がありませんでした。また、URLも確認したのですが、変化はおきませんでした。サーバーに負荷がかからないようにTime関数を用いましたが、だめでした。
そこでAnacondaNavigatorのアップデートを同時並行でやっていたからだと思い、試してみましたが、必ず途中でこのエラーが出てきてしまいます。
もう一度書きますが、先週までツイートデータを問題なく取得できていました。
補足情報(FW/ツールのバージョンなど)
環境
AnacondaNavigator1.10.0
Python3.8
JupyterNotebook
回答2件
あなたの回答
tips
プレビュー
バッドをするには、ログインかつ
こちらの条件を満たす必要があります。
2020/12/04 14:55
2020/12/04 23:28
2020/12/09 01:49