リスト型についてのプログラム

前提・実現したいこと

Tripadvisorのレビューをスクレイピングで抽出しているのですが、抽出する際に観光地のURLをプログラム内で入力しております。
その際に、現在は1つのURLを入力する形の為、複数のURLを入力して取得したい為リスト型で組んだのですがプログラムが動かないので実現したいです。

発生している問題・エラーメッセージ

  File "run_get_review.py", line 294, in <module>
    url_en = url_jp.replace("https://www.tripadvisor.jp/", "https://www.tripadvisor.com/")
AttributeError: 'list' object has no attribute 'replace'

該当のソースコード

python3.
1# -*- coding:utf-8 -*-
2#レビュー抽出 実行ファイル
3
4
5from bs4 import BeautifulSoup
6import urllib.request
7import urllib.parse
8import random
9from time import sleep
10import os
11import codecs
12
13import ssl
14ssl._create_default_https_context = ssl._create_unverified_context
15
16### "次へ"のページのurl取得 1
17### レビューのページ数分のurl(1，2,…最後 までのurl)
18def Get_nextpage_url_jp(url):
19    links = []
20    opener = urllib.request.build_opener()
21    opener.addheaders = [('User-agent', 'Mozilla/5.0')]
22    html = opener.open(url)
23    soup = BeautifulSoup(html,"lxml")
24
25    spotname = soup.find("h1",{"class":"ui_header h1"}).get_text().strip() # タグ指定
26    links.append(url)
27    #次のページのurl(最後の口コミのとこまで)
28    for url2 in links:
29        html = opener.open(url2)
30        soup = BeautifulSoup(html,"lxml")
31        try:
32            link = soup.find("a",{"class":"nav next taLnk "}).get("href") # タグ指定
33            links.append("https://www.tripadvisor.jp" + link)
34        except:
35            try:
36                link = soup.find("a",{"class":"nav next taLnk ui_button primary"}).get("href") # タグ指定
37                links.append("https://www.tripadvisor.jp" + link)
38            except:
39                print("Not next")
40    return links, spotname
41
42### レビューの詳細ページ取得 2
43def Get_review_url_jp(url):
44    links = []
45    opener = urllib.request.build_opener()
46    opener.addheaders = [('User-agent', 'Mozilla/5.0')]
47    html = opener.open(url)
48    soup = BeautifulSoup(html,"lxml")
49
50    #url取得
51    for div in soup.find_all("div",{"class":"ui_column is-9"}):
52        try:
53            links.append("https://www.tripadvisor.jp" + div.a.get("href"))
54        except:
55            print("not url")
56    return links
57
58### 英語 1
59def Get_review_url_en(url):
60    links = []
61    opener = urllib.request.build_opener()
62    opener.addheaders = [('User-agent', 'Mozilla/5.0')]
63    html = opener.open(url)
64    soup = BeautifulSoup(html,"lxml")
65
66    #url取得
67    for div in soup.find_all("div",{"class":"ui_column is-9"}):
68        try:
69            links.append("https://www.tripadvisor.com" + div.a.get("href"))
70        except:
71            print("not url")
72    return links
73
74### 英語 2
75def Get_nextpage_url_en(url):
76    links = []
77    opener = urllib.request.build_opener()
78    opener.addheaders = [('User-agent', 'Mozilla/5.0')]
79    html = opener.open(url)
80    soup = BeautifulSoup(html,"lxml")
81
82    spotname = soup.find("h1",{"class":"ui_header h1"}).get_text().strip() # タグ指定
83    links.append(url)
84    #次のページのurl(最後の口コミのとこまで)
85    for url2 in links:
86        html = opener.open(url2)
87        soup = BeautifulSoup(html,"lxml")
88        try:
89            link = soup.find("a",{"class":"nav next taLnk "}).get("href") # タグ指定
90            links.append("https://www.tripadvisor.com" + link)
91        except:
92            try:
93                link = soup.find("a",{"class":"nav next taLnk ui_button primary"}).get("href") # タグ指定
94                links.append("https://www.tripadvisor.com" + link)
95            except:
96                print("Not next")
97    return links, spotname
98
99
100
101### review取得 3
102def Get_review_data(url):
103    try:
104        title = [] #タイトル
105        review = [] #レビュー
106        opener = urllib.request.build_opener()
107        opener.addheaders = [('User-agent', 'Mozilla/5.0')]
108        html = opener.open(url)
109        soup = BeautifulSoup(html, "lxml")
110
111        #タイトル抽出1つ
112        try:
113            t = soup.find("h1",{"class":"title"})
114            title = t.get_text().replace("\n","").replace("\r","").replace(",", "").strip()
115        except:
116            t = soup.find("span",{"class":"noQuotes"})
117            title = t.get_text().replace("\n","").replace("\r","").replace(",", "").strip()
118#        for url in soup.find_all("span",{"class":"noQuotes"}): # 必要なタグ指定
119
120        #レビュー抽出1つ
121        try:
122            r = soup.find("span",{"class":"fullText "})
123            review = r.get_text().replace(",", "").replace("\n", "").replace("\r\n", "").replace("\r","").strip()
124        except:
125            r = soup.find("p",{"class":"partial_entry"})
126            review = r.get_text().replace(",", "").replace("\n", "").replace("\r\n", "").replace("\r","").strip()
127    except:
128        import traceback
129        traceback.print_exc()
130
131    return title, review # [文字列, 文字列]
132
133
134### 取得してきたもの全てをcsvファイルに保存 4
135def Save_path(name):
136    path = os.getcwd() # このプログラムの場所
137    savdir = "\review\" # "review"を保存するディレクトリ
138    if os.path.isdir(path + savdir) == False: # "review"というフォルダが無ければ作成
139        os.mkdir(path + savdir)
140    filename = name + ".csv"
141    filepath = path + savdir + filename
142    return filepath
143
144def Write_file(filepath, data):
145    #ファイルに書き込み w:上書き a:追記
146    file_object= codecs.open(filepath, "a", "cp932", "ignore")
147    file_object.write(str(data) + "\n")
148    file_object.close()
149
150
151
152### 実行部分_jp
153def Start_extract_review_jp(start_url):
154    #各値 初期化
155    title = []
156    review = []
157    links = []
158    urls = []
159
160    print("get_nextpage_url:  Start")
161    try:
162        links, spotname = Get_nextpage_url_jp(start_url)
163    except:
164        print("get_nextpage_url is error")
165    print("Total Next url =  " + str(len(links)))
166    print("get_review_url:  Start")
167    #1つずつurlを
168    for link in links:
169        try:
170            urls.extend(Get_review_url_jp(link)) #必要
171        except:
172            print("get_review_url is error")
173    print("Total review =  " + str(len(urls)))
174    #urls = review数
175    print("Get_review_data:  Start")
176    debug = 0
177
178    #filename指定してcsvに保存
179    try:
180        filepath = Save_path(spotname)
181    except:
182        print("Save_path is error")
183
184    #file_open
185    file_object= codecs.open(filepath, "a", "cp932", "ignore")
186    for url in urls:
187        debug += 1
188        print("review = " + str(debug))
189
190        interval = 5 + random.uniform(-3.0, 3.0)
191        print("interval 1: begin")
192        sleep(interval)
193        print("interval 1: end")
194
195        try:
196            title, review = Get_review_data(url)
197        except:
198            print("Get_review_data is error")
199
200        file_object.write("{}\s{}\n".format(str(title), str(review)))
201
202    file_object.close()
203    print("save: Complete")
204
205### 実行部分_jp
206def Start_extract_review_en(start_url):
207    #各値 初期化
208    title = []
209    review = []
210    links = []
211    urls = []
212
213    print("get_nextpage_url:  Start")
214    try:
215        links, spotname = Get_nextpage_url_en(start_url)
216    except:
217        print("get_nextpage_url is error")
218    print("Total Next url =  " + str(len(links)))
219    print("get_review_url:  Start")
220    #1つずつurlを
221    for link in links:
222        try:
223            urls.extend(Get_review_url_en(link)) #必要
224        except:
225            print("get_review_url is error")
226    print("Total review =  " + str(len(urls)))
227    #urls = review数
228    print("Get_review_data:  Start")
229    debug = 0
230
231    #filename指定してcsvに保存
232    try:
233        filepath = Save_path(spotname)
234    except:
235        print("Save_path is error")
236
237    #file_open
238    file_object= codecs.open(filepath, "a", "cp932", "ignore")
239    for url in urls:
240        debug += 1
241        print("review = " + str(debug))
242
243        interval = 5 + random.uniform(-3.0, 3.0)
244        print("interval 1: begin")
245        sleep(interval)
246        print("interval 1: end")
247
248        try:
249            title, review = Get_review_data(url)
250        except:
251            print("Get_review_data is error")
252
253        file_object.write("{}\s{}\n".format(str(title), str(review)))
254
255    file_object.close()
256    print("save: Complete")
257
258
259if __name__ =='__main__':
260
261    """
262    #debug Get_next_url
263    #"次へ"のページのurl取得 1
264    #url = "https://www.tripadvisor.jp/Attraction_Review-g1022838-d1548549-Reviews-Takosenbei_no_Sato-Awaji_Awaji_shima_Hyogo_Prefecture_Kinki.html" #最初のページ
265    url = "https://www.tripadvisor.com/Attraction_Review-g1022838-d1548549-Reviews-Takosenbei_no_Sato-Awaji_Awaji_shima_Hyogo_Prefecture_Kinki.html"
266    links = Get_nextpage_url_jp(url)
267    print(links)
268    """
269
270    """
271    #debug Get_review_url
272    # レビューの詳細ページ取得 2
273    #url = "https://www.tripadvisor.jp/Attraction_Review-g298562-d1384635-Reviews-Suma_Rikyu_Park-Kobe_Hyogo_Prefecture_Kinki.html"
274    url = "https://www.tripadvisor.com/Attraction_Review-g1022838-d1548549-Reviews-Takosenbei_no_Sato-Awaji_Awaji_shima_Hyogo_Prefecture_Kinki.html"
275    links = Get_review_url(url)
276    print(links)
277    """
278
279    """
280    #debug Get_review_data
281    # review 取得 3
282    #url = "https://www.tripadvisor.jp/ShowUserReviews-g298562-d1384635-r631487360-Suma_Rikyu_Park-Kobe_Hyogo_Prefecture_Kinki.html"
283    #url = "https://www.tripadvisor.jp/ShowUserReviews-g1022838-d1548549-r631449250-Takosenbei_no_Sato-Awaji_Awaji_shima_Hyogo_Prefecture_Kinki.html"
284    url = "https://www.tripadvisor.com/ShowUserReviews-g1022838-d1548549-r536077279-Takosenbei_no_Sato-Awaji_Awaji_shima_Hyogo_Prefecture_Kinki.html"
285    title, review = Get_review_data(url)
286    print("{}\s{}".format(title,review))
287    """
288
289    #"""
290    ###手打ち###
291    url_jp = ["https://www.tripadvisor.jp/Attraction_Review-g1022818-d1724619-Reviews-Uontana_Shopping_Street-Akashi_Hyogo_Prefecture_Kinki.html","https://www.tripadvisor.jp/Attraction_Review-g298192-d1604489-Reviews-Aeon_Mall_Itami-Itami_Hyogo_Prefecture_Kinki.html"]
292    ############
293
294    url_en = url_jp.replace("https://www.tripadvisor.jp/", "https://www.tripadvisor.com/")
295
296    Start_extract_review_jp(url_jp)
297    Start_extract_review_en(url_en)
298    #"""

行動規範の内容に同意します

回答2件

python
1    url_en = [url.replace("https://www.tripadvisor.jp/", "https://www.tripadvisor.com/") for url in url_jp]
2

で質問文に記載されたエラーは消えますが、残念ながら他の関数（Start_extract_review_jp()など、すべて）がURLのlistを受け取る仕様になっていないので、動かないでしょう。

期待通りの動作かどうかはわかりませんが、こんな感じでURLごとに個別に動かせばとりあえずちゃんと動き始めはするようです（時間かかるので最後まで回していません）。

python
1    for url in url_jp:
2        Start_extract_review_jp(url)
3
4    for url in url_en:
5        Start_extract_review_en(url)

投稿2018/11/12 07:44

編集2018/11/12 07:45

hayataka2049

総合スコア30933

処理詳細は見ていませんが、こういうことでしょうか？

Python
1    url_jp = ["https://www.tripadvisor.jp/Attraction_Review-g1022818-d1724619-Reviews-Uontana_Shopping_Street-Akashi_Hyogo_Prefecture_Kinki.html","https://www.tripadvisor.jp/Attraction_Review-g298192-d1604489-Reviews-Aeon_Mall_Itami-Itami_Hyogo_Prefecture_Kinki.html"]
2
3    url_en = []
4    for url in url_jp:
5        url_en.append(url.replace("https://www.tripadvisor.jp/", "https://www.tripadvisor.com/"))