Q&A
herokuでseleniumを用いたアプリケーションを作ろうと思い、以下のコードを書きました。
python
1import responder 2from scrape import Scraper 3import os 4 5api = responder.API() 6searcher = Scraper(os.environ["CHROME_DRIVER_PATH"], os.environ["CHROME_PATH"]) 7 8 9@api.route("/") 10def index(req, resp): 11 resp.text = f"{searcher.scrape('')}" 12 13 14if __name__ == "__main__": 15 api.run()
from selenium import webdriver from selenium.webdriver.support import ui, expected_conditions from bs4 import BeautifulSoup class Scraper: def __init__(self, driver_path: str, chrome_path) -> None: self.url = "https://google.com/" self.driver_path = driver_path self.chrome_path = chrome_path def _setup(self) -> None: options = webdriver.ChromeOptions() options.add_argument("--headless") options.add_argument("--no-sandbox") options.add_argument("--disable--dev-shim-usage") options.binary_locations = self.chrome_path self.driver = webdriver.Chrome(executable_path=self.driver_path, options=options) def scrape(self, query: str) -> tuple: self._setup() self.driver.get(self.url + query) return self._construct(self.driver.page_source.encode("utf-8")) def _construct(self, foo) -> tuple: return 1, 2, 3, 4 # 実際の処理をやる
このアプリケーションをherokuにデプロイした後、"/"にアクセスすると
2021-03-03T08:47:29.750985+00:00 app[web.1]: Traceback (most recent call last): 2021-03-03T08:47:29.751002+00:00 app[web.1]: File "/app/.heroku/python/lib/python3.9/site-packages/uvicorn/protocols/http/httptools_impl.py", line 396, in run _asgi 2021-03-03T08:47:29.751003+00:00 app[web.1]: result = await app(self.scope, self.receive, self.send) 2021-03-03T08:47:29.751004+00:00 app[web.1]: File "/app/.heroku/python/lib/python3.9/site-packages/uvicorn/middleware/proxy_headers.py", line 45, in __call__ 2021-03-03T08:47:29.751004+00:00 app[web.1]: return await self.app(scope, receive, send) 2021-03-03T08:47:29.751005+00:00 app[web.1]: File "/app/.heroku/python/lib/python3.9/site-packages/responder/api.py", line 361, in __call__ 2021-03-03T08:47:29.751005+00:00 app[web.1]: await self.app(scope, receive, send) 2021-03-03T08:47:29.751006+00:00 app[web.1]: File "/app/.heroku/python/lib/python3.9/site-packages/starlette/middleware/sessions.py", line 75, in __call__ 2021-03-03T08:47:29.751006+00:00 app[web.1]: await self.app(scope, receive, send_wrapper) 2021-03-03T08:47:29.751006+00:00 app[web.1]: File "/app/.heroku/python/lib/python3.9/site-packages/starlette/middleware/errors.py", line 181, in __call__ 2021-03-03T08:47:29.751007+00:00 app[web.1]: raise exc from None 2021-03-03T08:47:29.751007+00:00 app[web.1]: File "/app/.heroku/python/lib/python3.9/site-packages/starlette/middleware/errors.py", line 159, in __call__ 2021-03-03T08:47:29.751008+00:00 app[web.1]: await self.app(scope, receive, _send) 2021-03-03T08:47:29.751008+00:00 app[web.1]: File "/app/.heroku/python/lib/python3.9/site-packages/starlette/middleware/trustedhost.py", line 34, in __call__ 2021-03-03T08:47:29.751009+00:00 app[web.1]: await self.app(scope, receive, send) 2021-03-03T08:47:29.751009+00:00 app[web.1]: File "/app/.heroku/python/lib/python3.9/site-packages/starlette/middleware/gzip.py", line 18, in __call__ 2021-03-03T08:47:29.751009+00:00 app[web.1]: await responder(scope, receive, send) 2021-03-03T08:47:29.751010+00:00 app[web.1]: File "/app/.heroku/python/lib/python3.9/site-packages/starlette/middleware/gzip.py", line 35, in __call__ 2021-03-03T08:47:29.751010+00:00 app[web.1]: await self.app(scope, receive, self.send_with_gzip) 2021-03-03T08:47:29.751011+00:00 app[web.1]: File "/app/.heroku/python/lib/python3.9/site-packages/starlette/exceptions.py", line 82, in __call__ 2021-03-03T08:47:29.751011+00:00 app[web.1]: raise exc from None 2021-03-03T08:47:29.751011+00:00 app[web.1]: File "/app/.heroku/python/lib/python3.9/site-packages/starlette/exceptions.py", line 71, in __call__ 2021-03-03T08:47:29.751012+00:00 app[web.1]: await self.app(scope, receive, sender) 2021-03-03T08:47:29.751012+00:00 app[web.1]: File "/app/.heroku/python/lib/python3.9/site-packages/responder/routes.py", line 342, in __call__ 2021-03-03T08:47:29.751013+00:00 app[web.1]: await route(scope, receive, send) 2021-03-03T08:47:29.751016+00:00 app[web.1]: File "/app/.heroku/python/lib/python3.9/site-packages/responder/routes.py", line 138, in __call__ 2021-03-03T08:47:29.751016+00:00 app[web.1]: await run_in_threadpool(view, request, response, **path_params) 2021-03-03T08:47:29.751017+00:00 app[web.1]: File "/app/.heroku/python/lib/python3.9/site-packages/starlette/concurrency.py", line 34, in run_in_threadpool 2021-03-03T08:47:29.751017+00:00 app[web.1]: return await loop.run_in_executor(None, func, *args) 2021-03-03T08:47:29.751018+00:00 app[web.1]: File "/app/.heroku/python/lib/python3.9/concurrent/futures/thread.py", line 52, in run 2021-03-03T08:47:29.751018+00:00 app[web.1]: result = self.fn(*self.args, **self.kwargs) 2021-03-03T08:47:29.751019+00:00 app[web.1]: File "/app/run.py", line 11, in index 2021-03-03T08:47:29.751019+00:00 app[web.1]: resp.text = f"{searcher.scrape('')}" 2021-03-03T08:47:29.751019+00:00 app[web.1]: File "/app/scrape.py", line 22, in scrape 2021-03-03T08:47:29.751020+00:00 app[web.1]: self._setup() 2021-03-03T08:47:29.751020+00:00 app[web.1]: File "/app/scrape.py", line 18, in _setup 2021-03-03T08:47:29.751021+00:00 app[web.1]: self.driver = webdriver.Chrome(executable_path=self.driver_path, 2021-03-03T08:47:29.751021+00:00 app[web.1]: File "/app/.heroku/python/lib/python3.9/site-packages/selenium/webdriver/chrome/webdriver.py", line 76, in __init__ 2021-03-03T08:47:29.751021+00:00 app[web.1]: RemoteWebDriver.__init__( 2021-03-03T08:47:29.751022+00:00 app[web.1]: File "/app/.heroku/python/lib/python3.9/site-packages/selenium/webdriver/remote/webdriver.py", line 157, in __init__ 2021-03-03T08:47:29.751022+00:00 app[web.1]: self.start_session(capabilities, browser_profile) 2021-03-03T08:47:29.751022+00:00 app[web.1]: File "/app/.heroku/python/lib/python3.9/site-packages/selenium/webdriver/remote/webdriver.py", line 252, in start_session 2021-03-03T08:47:29.751023+00:00 app[web.1]: response = self.execute(Command.NEW_SESSION, parameters) 2021-03-03T08:47:29.751023+00:00 app[web.1]: File "/app/.heroku/python/lib/python3.9/site-packages/selenium/webdriver/remote/webdriver.py", line 321, in execute 2021-03-03T08:47:29.751023+00:00 app[web.1]: self.error_handler.check_response(response) 2021-03-03T08:47:29.751024+00:00 app[web.1]: File "/app/.heroku/python/lib/python3.9/site-packages/selenium/webdriver/remote/errorhandler.py", line 242, in check_response 2021-03-03T08:47:29.751024+00:00 app[web.1]: raise exception_class(message, screen, stacktrace) 2021-03-03T08:47:29.751024+00:00 app[web.1]: selenium.common.exceptions.WebDriverException: Message: unknown error: Chrome failed to start: exited abnormally. 2021-03-03T08:47:29.751025+00:00 app[web.1]: (unknown error: DevToolsActivePort file doesn't exist) 2021-03-03T08:47:29.751026+00:00 app[web.1]: (The process started from chrome location /app/.apt/usr/bin/google-chrome is no longer running, so ChromeDriver is assuming that Chrome has crashed.)
環境変数は以下を設定しています。
CHROME_DRIVER_PATH: /app/.chromedriver/bin/chromedriver CHROME_PATH: /app/.apt/usr/bin/google-chrome
エラーのログを見るとchromeが起動していなそうなのでheroku run bashで以下を実行しました。
bash
1$ which google-chrome 2> /app/.apt/usr/bin/google-chrome 3 4$ /app/.apt/usr/bin/google-chrome --version 5> /app/.apt/opt/google/chrome/chrome: error while loading shared libraries: libxshmfence.so.1: cannot open shared object file: No such file or directory
現状発生している問題
共有ライブラリが見つからないため、chromeが実行できない
詳しい人がいらっしゃいましたらご教示いただければ幸いです。
実行環境
[[source]] url = "https://pypi.org/simple" verify_ssl = true name = "pypi" [packages] responder = "*" gunicorn = "*" selenium = "*" beautifulsoup4 = "*" [dev-packages] [requires] python_version = "3.9" ↪ $ heroku --version (test-selen) [ 18:09 ] heroku/7.49.1 linux-x64 node-v15.10.0 ~ $ $CHROME_DRIVER_PATH --version ChromeDriver 88.0.4324.96 (68dba2d8a0b149a1d3afac56fa74648032bcf46b-refs/branch-heads/4324@{#1784})
補足情報
githubのissueを確認していたところ以下を見つけましたが、閲覧時にはまだ解決策は投稿されていませんでした。
https://github.com/heroku/heroku-buildpack-google-chrome/issues/105
自己解決
バッドをするには、ログインかつ
こちらの条件を満たす必要があります。