Q&A
分かりました、別の方法を模索しようと思います。
ありがとうございました。
前提・実現したいこと
最近まで使えていたコードが急にエラーを出すようになったので、エラーを改善したい。
発生している問題・エラーメッセージ
--------------------------------------------------------------------------- RemoteDataError Traceback (most recent call last) <ipython-input-4-a59ab8bb6171> in <module> ----> 1 df = web.DataReader(Ticker[0],'yahoo',start,end) ~/anaconda3/lib/python3.7/site-packages/pandas/util/_decorators.py in wrapper(*args, **kwargs) 186 else: 187 kwargs[new_arg_name] = new_arg_value --> 188 return func(*args, **kwargs) 189 return wrapper 190 return _deprecate_kwarg ~/anaconda3/lib/python3.7/site-packages/pandas_datareader/data.py in DataReader(name, data_source, start, end, retry_count, pause, session, api_key) 382 retry_count=retry_count, 383 pause=pause, --> 384 session=session, 385 ).read() 386 ~/anaconda3/lib/python3.7/site-packages/pandas_datareader/base.py in read(self) 251 # If a single symbol, (e.g., 'GOOG') 252 if isinstance(self.symbols, (string_types, int)): --> 253 df = self._read_one_data(self.url, params=self._get_params(self.symbols)) 254 # Or multiple symbols, (e.g., ['GOOG', 'AAPL', 'MSFT']) 255 elif isinstance(self.symbols, DataFrame): ~/anaconda3/lib/python3.7/site-packages/pandas_datareader/yahoo/daily.py in _read_one_data(self, url, params) 151 url = url.format(symbol) 152 --> 153 resp = self._get_response(url, params=params) 154 ptrn = r"root.App.main = (.*?);\n}(this));" 155 try: ~/anaconda3/lib/python3.7/site-packages/pandas_datareader/base.py in _get_response(self, url, params, headers) 179 msg += "\nResponse Text:\n{0}".format(last_response_text) 180 --> 181 raise RemoteDataError(msg) 182 183 def _get_crumb(self, *args): RemoteDataError: Unable to read URL: https://finance.yahoo.com/quote/GOOG/history?period1=1539077816&period2=1625543999&interval=1d&frequency=1d&filter=history Response Text: b'<!DOCTYPE html>\n <html lang="en-us"><head>\n <meta http-equiv="content-type" content="text/html; charset=UTF-8">\n <meta charset="utf-8">\n <title>Yahoo</title>\n <meta name="viewport" content="width=device-width,initial-scale=1,minimal-ui">\n <meta http-equiv="X-UA-Compatible" content="IE=edge,chrome=1">\n <style>\n html {\n height: 100%;\n }\n body {\n background: #fafafc url(https://s.yimg.com/nn/img/sad-panda-201402200631.png) 50% 50%;\n background-size: cover;\n height: 100%;\n text-align: center;\n font: 300 18px "helvetica neue", helvetica, verdana, tahoma, arial, sans-serif;\n }\n table {\n height: 100%;\n width: 100%;\n table-layout: fixed;\n border-collapse: collapse;\n border-spacing: 0;\n border: none;\n }\n h1 {\n font-size: 42px;\n font-weight: 400;\n color: #400090;\n }\n p {\n color: #1A1A1A;\n }\n #message-1 {\n font-weight: bold;\n margin: 0;\n }\n #message-2 {\n display: inline-block;\n *display: inline;\n zoom: 1;\n max-width: 17em;\n _width: 17em;\n }\n </style>\n <script>\n document.write(\'<img src="//geo.yahoo.com/b?s=1197757129&t=\'+new Date().getTime()+\'&src=aws&err_url=\'+encodeURIComponent(document.URL)+\'&err=%<pssc>&test=\'+encodeURIComponent(\'%<{Bucket}cqh[:200]>\')+\'" width="0px" height="0px"/>\');var beacon = new Image();beacon.src="//bcn.fp.yahoo.com/p?s=1197757129&t="+new Date().getTime()+"&src=aws&err_url="+encodeURIComponent(document.URL)+"&err=%<pssc>&test="+encodeURIComponent(\'%<{Bucket}cqh[:200]>\');\n </script>\n </head>\n <body>\n <!-- status code : 404 -->\n <!-- Not Found on Server -->\n <table>\n <tbody><tr>\n <td>\n <img src="https://s.yimg.com/rz/p/yahoo_frontpage_en-US_s_f_p_205x58_frontpage.png" alt="Yahoo Logo">\n <h1 style="margin-top:20px;">Will be right back...</h1>\n <p id="message-1">Thank you for your patience.</p>\n <p id="message-2">Our engineers are working quickly to resolve the issue.</p>\n </td>\n </tr>\n </tbody></table>\n </body></html>'
該当のソースコード
pytyon
1import datetime as dt 2from datetime import timedelta as td 3import pandas as pd 4import pandas_datareader.data as web 5import math 6import numpy as np 7import talib as ta 8from tqdm import tqdm 9 10today = dt.datetime.today() 11end = today - dt.timedelta(days=0) 12start = end - td(days=1000) 13 14Ticker = ['GOOG','AAPL','MSFT','AMZN'] 15 16df = web.DataReader(Ticker[0],'yahoo',start,end) 17 18df['close'] = df['Close'] 19df['sma03'] = df['Close'].rolling(window=200).mean() 20df['cci'] = ta.CCI(df['High'], df['Low'], df['Close'], timeperiod=14) 21df['tenkan'] = (df['High'].rolling(window=9, min_periods=1).max()+df['Low'].rolling(window=9, min_periods=1).min())/2 22df['base'] = (df['High'].rolling(window=26, min_periods=1).max()+df['Low'].rolling(window=26, min_periods=1).min())/2 23df['senkou1'] = ((df['tenkan']+df['base'])/2).iloc[:-26] 24df['senkou2'] = ((df['High'].rolling(window=52, min_periods=1).max()+df['Low'].rolling(window=52, min_periods=1).min())/2).iloc[:-26] 25df['close_diff'] = df['Close'].diff() 26 27df[Ticker[0]] = df['close'] 28df[Ticker[0]] = df['sma03'] 29df[Ticker[0]] = df['cci'] 30df[Ticker[0]] = df['tenkan'] 31df[Ticker[0]] = df['base'] 32df[Ticker[0]] = df['senkou1'] 33df[Ticker[0]] = df['senkou2'] 34df[Ticker[0]] = df['close_diff'] 35 36da1 = pd.DataFrame(df[Ticker[0]].dropna(how='any')) 37da2 = pd.DataFrame(df[Ticker[0]].dropna(how='any')) 38da3 = pd.DataFrame(df[Ticker[0]].dropna(how='any')) 39da4 = pd.DataFrame(df[Ticker[0]].dropna(how='any')) 40da5 = pd.DataFrame(df[Ticker[0]].dropna(how='any')) 41da6 = pd.DataFrame(df[Ticker[0]].dropna(how='any')) 42da7 = pd.DataFrame(df[Ticker[0]].dropna(how='any')) 43da8 = pd.DataFrame(df[Ticker[0]].dropna(how='any')) 44 45for i in tqdm(range(1,len(Ticker))): 46 try: 47 globals()['df'+str(i)] = web.DataReader(Ticker[i],'yahoo',start,end) 48 globals()['df'+str(i)]['close'] = globals()['df'+str(i)]['Close'] 49 globals()['df'+str(i)]['sma03'] = globals()['df'+str(i)]['Close'].rolling(window=200).mean() 50 globals()['df'+str(i)]['cci'] = ta.CCI(globals()['df'+str(i)]['High'], globals()['df'+str(i)]['Low'], globals()['df'+str(i)]['Close'], timeperiod=14) 51 globals()['df'+str(i)]['tenkan'] = (globals()['df'+str(i)]['High'].rolling(window=9, min_periods=1).max()+globals()['df'+str(i)]['Low'].rolling(window=9, min_periods=1).min())/2 52 globals()['df'+str(i)]['base'] = (globals()['df'+str(i)]['High'].rolling(window=26, min_periods=1).max()+globals()['df'+str(i)]['Low'].rolling(window=26, min_periods=1).min())/2 53 globals()['df'+str(i)]['senkou1'] = ((globals()['df'+str(i)]['tenkan']+globals()['df'+str(i)]['base'])/2).iloc[:-26] 54 globals()['df'+str(i)]['senkou2'] = ((globals()['df'+str(i)]['High'].rolling(window=52, min_periods=1).max()+globals()['df'+str(i)]['Low'].rolling(window=52, min_periods=1).min())/2).iloc[:-26] 55 globals()['df'+str(i)]['close_diff'] = globals()['df'+str(i)]['Close'].diff() 56 57 58 da1[Ticker[i]] = globals()['df'+str(i)]['close'].dropna(how='any') 59 da2[Ticker[i]] = globals()['df'+str(i)]['sma03'].dropna(how='any') 60 da3[Ticker[i]] = globals()['df'+str(i)]['cci'].dropna(how='any') 61 da4[Ticker[i]] = globals()['df'+str(i)]['tenkan'].dropna(how='any') 62 da5[Ticker[i]] = globals()['df'+str(i)]['base'].dropna(how='any') 63 da6[Ticker[i]] = globals()['df'+str(i)]['senkou1'].dropna(how='any') 64 da7[Ticker[i]] = globals()['df'+str(i)]['senkou2'].dropna(how='any') 65 da8[Ticker[i]] = globals()['df'+str(i)]['close_diff'].dropna(how='any') 66 except: 67 pass 68 69pd.options.display.precision = 2 70print(pd.concat([da1.tail(78), da2.tail(103), da3.tail(52), da4.tail(52), da5.tail(52), da6.tail(79), da7.tail(79)]).transpose())
試したこと
各種バージョンのアップグレード
df = web.DataReader(Ticker[0],'yahoo',start,end)の
「yahoo」部分を「google」「fred」に変えましたが結果は同じでした。
補足情報（FW/ツールのバージョンなど）
使用モジュール
numpy 1.21.0 pandas 1.3.0 pandas-datareader 0.9.0 TA-Lib 0.4.19 tqdm 4.31.1
回答2件
0
エラーメッセージをちゃんと見ましょう。
404 Not Found
のエラーです。つまり該当ページが無い。
投稿2021/07/05 12:44
総合スコア85901
0
ベストアンサー
yahoo financeのスクレイピングは規約違反、
質問文のコードを実行した場合おそらく短時間に大量のリクエストを送りつける行為となり、サーバーに負担をかける立派な威力業務妨害行為になりえます。
このようなことを可能にするpandas-datareader自体使えなくなるように「対策された」と考えるのが自然だとおもいます。
諦めた方がいいんじゃないですかね。
投稿2021/07/05 12:36
退会済みユーザー
総合スコア0
あなたの回答
tips
プレビュー
バッドをするには、ログインかつ
こちらの条件を満たす必要があります。