前提
https://qiita.com/kazumatsukazu/items/5b7141c85fd85c7d1c0d
https://blowup-bbs.com/gcp-cloud-functions-python3/
上記サイトを参考に、ローカルでのWebスクレイピング作業をGCPに移行させたい。
実現したいこと
Cloud Functions上でheadress-chromeをPythonで動かし、ランダムなwikiの値を返す。
現状
※デプロイはできているっぽいが、関数のテストを実施するとエラーになる。
発生している問題・エラーメッセージ
handler g7q08av14yd0 Error occurred during loading data. Trying to use cache server http://d2g6u4gh6d9rq0.cloudfront.net/browsers/fake_useragent_0.1.10.json Traceback (most recent call last): File "/workspace/fake_useragent/utils.py", line 154, in load for item in get_browsers(verify_ssl=verify_ssl): File "/workspace/fake_useragent/utils.py", line 99, in get_browsers html = html.split('<table class="w3-table-all notranslate">')[1] IndexError: list index out of range
handlerg7q08av14yd0 Exception on / [GET] Traceback (most recent call last): File "/workspace/fake_useragent/utils.py", line 154, in load for item in get_browsers(verify_ssl=verify_ssl): File "/workspace/fake_useragent/utils.py", line 99, in get_browsers html = html.split('<table class="w3-table-all notranslate">')[1] IndexError: list index out of range During handling of the above exception, another exception occurred: Traceback (most recent call last): File "/layers/google.python.runtime/python/lib/python3.7/urllib/request.py", line 1350, in do_open encode_chunked=req.has_header('Transfer-encoding')) File "/layers/google.python.runtime/python/lib/python3.7/http/client.py", line 1281, in request self._send_request(method, url, body, headers, encode_chunked) File "/layers/google.python.runtime/python/lib/python3.7/http/client.py", line 1327, in _send_request self.endheaders(body, encode_chunked=encode_chunked) File "/layers/google.python.runtime/python/lib/python3.7/http/client.py", line 1276, in endheaders self._send_output(message_body, encode_chunked=encode_chunked) File "/layers/google.python.runtime/python/lib/python3.7/http/client.py", line 1036, in _send_output self.send(msg) File "/layers/google.python.runtime/python/lib/python3.7/http/client.py", line 976, in send self.connect() File "/layers/google.python.runtime/python/lib/python3.7/http/client.py", line 948, in connect (self.host,self.port), self.timeout, self.source_address) File "/layers/google.python.runtime/python/lib/python3.7/socket.py", line 707, in create_connection for res in getaddrinfo(host, port, 0, SOCK_STREAM): File "/layers/google.python.runtime/python/lib/python3.7/socket.py", line 752, in getaddrinfo for res in _socket.getaddrinfo(host, port, family, type, proto, flags): socket.gaierror: [Errno -3] Temporary failure in name resolution During handling of the above exception, another exception occurred: Traceback (most recent call last): File "/workspace/fake_useragent/utils.py", line 67, in get context=context, File "/layers/google.python.runtime/python/lib/python3.7/urllib/request.py", line 222, in urlopen return opener.open(url, data, timeout) File "/layers/google.python.runtime/python/lib/python3.7/urllib/request.py", line 525, in open response = self._open(req, data) File "/layers/google.python.runtime/python/lib/python3.7/urllib/request.py", line 543, in _open '_open', req) File "/layers/google.python.runtime/python/lib/python3.7/urllib/request.py", line 503, in _call_chain result = func(*args) File "/layers/google.python.runtime/python/lib/python3.7/urllib/request.py", line 1378, in http_open return self.do_open(http.client.HTTPConnection, req) File "/layers/google.python.runtime/python/lib/python3.7/urllib/request.py", line 1352, in do_open raise URLError(err) urllib.error.URLError: <urlopen error [Errno -3] Temporary failure in name resolution> During handling of the above exception, another exception occurred: Traceback (most recent call last): File "/layers/google.python.pip/pip/lib/python3.7/site-packages/flask/app.py", line 2073, in wsgi_app response = self.full_dispatch_request() File "/layers/google.python.pip/pip/lib/python3.7/site-packages/flask/app.py", line 1518, in full_dispatch_request rv = self.handle_user_exception(e) File "/layers/google.python.pip/pip/lib/python3.7/site-packages/flask/app.py", line 1516, in full_dispatch_request rv = self.dispatch_request() File "/layers/google.python.pip/pip/lib/python3.7/site-packages/flask/app.py", line 1502, in dispatch_request return self.ensure_sync(self.view_functions[rule.endpoint])(**req.view_args) File "/layers/google.python.pip/pip/lib/python3.7/site-packages/functions_framework/__init__.py", line 99, in view_func return function(request._get_current_object()) File "/workspace/main.py", line 18, in handler chrome_options.add_argument('user-agent='+UserAgent().random) File "/workspace/fake_useragent/fake.py", line 69, in __init__ self.load() File "/workspace/fake_useragent/fake.py", line 78, in load verify_ssl=self.verify_ssl, File "/workspace/fake_useragent/utils.py", line 250, in load_cached update(path, use_cache_server=use_cache_server, verify_ssl=verify_ssl) File "/workspace/fake_useragent/utils.py", line 245, in update write(path, load(use_cache_server=use_cache_server, verify_ssl=verify_ssl)) File "/workspace/fake_useragent/utils.py", line 189, in load verify_ssl=verify_ssl, File "/workspace/fake_useragent/utils.py", line 84, in get raise FakeUserAgentError('Maximum amount of retries reached') fake_useragent.errors.FakeUserAgentError: Maximum amount of retries reached
該当のソースコード
Python
1import os 2from selenium import webdriver 3from fake_useragent import UserAgent 4 5def handler(request): 6 chrome_options = webdriver.ChromeOptions() 7 8 chrome_options.add_argument('--headless') 9 chrome_options.add_argument('--disable-gpu') 10 chrome_options.add_argument('--window-size=1280x1696') 11 chrome_options.add_argument('--no-sandbox') 12 chrome_options.add_argument('--hide-scrollbars') 13 chrome_options.add_argument('--enable-logging') 14 chrome_options.add_argument('--log-level=0') 15 chrome_options.add_argument('--v=99') 16 chrome_options.add_argument('--single-process') 17 chrome_options.add_argument('--ignore-certificate-errors') 18 chrome_options.add_argument('user-agent='+UserAgent().random) 19 20 chrome_options.binary_location = os.getcwd() + "/headless-chromium" 21 driver = webdriver.Chrome(os.getcwd() + "/chromedriver",chrome_options=chrome_options) 22 driver.get('https://en.wikipedia.org/wiki/Special:Random') 23 line = driver.find_element_by_class_name('firstHeading').text 24 print(line) 25 driver.quit() 26 return line 27
試したこと
再クローンの実施

バッドをするには、ログインかつ
こちらの条件を満たす必要があります。