編集履歴

質問編集履歴

いろいろ

2018/11/12 08:14

投稿

スコア0

test CHANGED Viewed

	@@ -1 +1 @@
1	- 英語の単語数カウント
1	+ nltkを用いた英語の単語数カウント

test CHANGED Viewed

@@ -16,259 +16,65 @@
 ```python3.
-# -*- coding:utf-8 -*-
+# -*- coding: utf-8 -*-
-#レビュー抽出 実行ファイル
+# 単語の出現頻度をExcelに
-from bs4 import BeautifulSoup
-import urllib.request
-import urllib.parse
-import random
+import openpyxl
-from time import sleep
 import os
+import juman
 import codecs
+import glob
+from collections import Counter
-import ssl
+def Juman_for_Frequency(spotname):
+    try:
+        word = [] # 単語
+        try:
+            fr = codecs.open(spotname, "rb", "sjis", "ignore") #読み込み
+            texts = fr.read().split("\n") #ファイルを1行ずつリストとしてtextに
+            fr.close()
-ssl._create_default_https_context = ssl._create_unverified_context
+            texts = list(filter(lambda a: a != "", texts)) #空リスト削除
+            debug = 0
+            for text in texts:
+                debug +=1
+                print("open_file = " + spotname + "lines = " + str(debug))
+                word = word + juman.keitaiso(text) #分かち書きしたワード
+            counter = Counter(word) #同じ文字列をカウントしてcounterに
-### "次へ"のページのurl取得 1
+        except Exception:
-### レビューのページ数分のurl(1，2,…最後 までのurl)
+            import traceback
-def Get_nextpage_url_jp(url):
-    links = []
-    opener = urllib.request.build_opener()
-    opener.addheaders = [('User-agent', 'Mozilla/5.0')]
-    html = opener.open(url)
+            traceback.print_exc()
-    soup = BeautifulSoup(html,"lxml")
-    spotname = soup.find("h1",{"class":"ui_header h1"}).get_text().strip() # タグ指定
-    links.append(url)
-    #次のページのurl(最後の口コミのとこまで)
-    for url2 in links:
-        html = opener.open(url2)
-        soup = BeautifulSoup(html,"lxml")
-        try:
-            link = soup.find("a",{"class":"nav next taLnk "}).get("href") # タグ指定
-            links.append("https://www.tripadvisor.jp" + link)
-        except:
+    except Exception:
-            try:
-                link = soup.find("a",{"class":"nav next taLnk ui_button primary"}).get("href") # タグ指定
-                links.append("https://www.tripadvisor.jp" + link)
-            except:
-                print("Not next")
-    return links, spotname
-### レビューの詳細ページ取得 2
-def Get_review_url_jp(url):
-    links = []
-    opener = urllib.request.build_opener()
-    opener.addheaders = [('User-agent', 'Mozilla/5.0')]
-    html = opener.open(url)
-    soup = BeautifulSoup(html,"lxml")
-    #url取得
-    for div in soup.find_all("div",{"class":"ui_column is-9"}):
-        try:
-            links.append("https://www.tripadvisor.jp" + div.a.get("href"))
-        except:
-            print("not url")
-    return links
-### 英語 1
-def Get_review_url_en(url):
-    links = []
-    opener = urllib.request.build_opener()
-    opener.addheaders = [('User-agent', 'Mozilla/5.0')]
-    html = opener.open(url)
-    soup = BeautifulSoup(html,"lxml")
-    #url取得
-    for div in soup.find_all("div",{"class":"ui_column is-9"}):
-        try:
-            links.append("https://www.tripadvisor.com" + div.a.get("href"))
-        except:
-            print("not url")
-    return links
-### 英語 2
-def Get_nextpage_url_en(url):
-    links = []
-    opener = urllib.request.build_opener()
-    opener.addheaders = [('User-agent', 'Mozilla/5.0')]
-    html = opener.open(url)
-    soup = BeautifulSoup(html,"lxml")
-    spotname = soup.find("h1",{"class":"ui_header h1"}).get_text().strip() # タグ指定
-    links.append(url)
-    #次のページのurl(最後の口コミのとこまで)
-    for url2 in links:
-        html = opener.open(url2)
-        soup = BeautifulSoup(html,"lxml")
-        try:
-            link = soup.find("a",{"class":"nav next taLnk "}).get("href") # タグ指定
-            links.append("https://www.tripadvisor.com" + link)
-        except:
-            try:
-                link = soup.find("a",{"class":"nav next taLnk ui_button primary"}).get("href") # タグ指定
-                links.append("https://www.tripadvisor.com" + link)
-            except:
-                print("Not next")
-    return links, spotname
-### review取得 3
-def Get_review_data(url):
-    try:
-        title = [] #タイトル
-        review = [] #レビュー
-        opener = urllib.request.build_opener()
-        opener.addheaders = [('User-agent', 'Mozilla/5.0')]
-        html = opener.open(url)
-        soup = BeautifulSoup(html, "lxml")
-        #タイトル抽出1つ
-        try:
-            t = soup.find("h1",{"class":"title"})
-            title = t.get_text().replace("\n","").replace("\r","").replace(",", "").strip()
-        except:
-            t = soup.find("span",{"class":"noQuotes"})
-            title = t.get_text().replace("\n","").replace("\r","").replace(",", "").strip()
-#        for url in soup.find_all("span",{"class":"noQuotes"}): # 必要なタグ指定
-        #レビュー抽出1つ
-        try:
-            r = soup.find("span",{"class":"fullText "})
-            review = r.get_text().replace(",", "").replace("\n", "").replace("\r\n", "").replace("\r","").strip()
-        except:
-            r = soup.find("p",{"class":"partial_entry"})
-            review = r.get_text().replace(",", "").replace("\n", "").replace("\r\n", "").replace("\r","").strip()
-    except:
         import traceback
@@ -276,341 +82,75 @@
-    return title, review # [文字列, 文字列]
+    return counter
-### 取得してきたもの全てをcsvファイルに保存 4
+def Save_File(data, save_directory, save_filename):
-def Save_path(name):
+    #ファイルに書き込み
-    path = os.getcwd() # このプログラムの場所
+    wb = openpyxl.Workbook()
-    savdir = "\review\" # "review"を保存するディレクトリ
-    if os.path.isdir(path + savdir) == False: # "review"というフォルダが無ければ作成
-        os.mkdir(path + savdir)
+    ws = wb.active
-    filename = name + ".csv"
-    filepath = path + savdir + filename
-    return filepath
-def Write_file(filepath, data):
+    num = 1
-    #ファイルに書き込み w:上書き a:追記
+    for k,v in sorted(data.items(),key=lambda x:x[1],reverse=True):
-    file_object= codecs.open(filepath, "a", "cp932", "ignore")
+        key = []
-    file_object.write(str(data) + "\n")
+        key = k.split(",")
+        ws.cell(column=1, row=num).value=key[0] # 単語
+        ws.cell(column=2, row=num).value=key[1] # 品詞
+        ws.cell(column=3, row=num).value=v # 出現回数
+        num += 1
+    wb.save(save_directory + save_filename)
-    file_object.close()
+    print("Save_Complete!")
+if __name__ == '__main__':
+    path = os.getcwd() # このプログラムの場所
-### 実行部分_jp
+    textrank_dir = "\textrank\" # "textrank"を保存するディレクトリ
-def Start_extract_review_jp(start_url):
+    review_dir = "\review\" # "review"ディレクトリ
-    #各値 初期化
+    if os.path.isdir(path + textrank_dir) == False: # "textrank"というフォルダが無ければ作成
-    title = []
+        os.mkdir(path + textrank_dir)
-    review = []
+    os.chdir("review/")
-    links = []
-    urls = []
-    print("get_nextpage_url:  Start")
+    csv_files = glob.glob("*.csv")
-    try:
+    save_directory = path + textrank_dir
-        links, spotname = Get_nextpage_url_jp(start_url)
+    for spotname in csv_files:
-    except:
+        print(spotname)
-        print("get_nextpage_url is error")
+        get_data = Juman_for_Frequency(spotname)
-    print("Total Next url =  " + str(len(links)))
-    print("get_review_url:  Start")
-    #1つずつurlを
-    for link in links:
-        try:
-            urls.extend(Get_review_url_jp(link)) #必要
-        except:
-            print("get_review_url is error")
-    print("Total review =  " + str(len(urls)))
-    #urls = review数
-    print("Get_review_data:  Start")
-    debug = 0
-    #filename指定してcsvに保存
-    try:
-        filepath = Save_path(spotname)
+        Save_File(get_data, save_directory, spotname)
-    except:
-        print("Save_path is error")
-    #file_open
-    file_object= codecs.open(filepath, "a", "cp932", "ignore")
-    for url in urls:
-        debug += 1
-        print("review = " + str(debug))
-        interval = 5 + random.uniform(-3.0, 3.0)
-        print("interval 1: begin")
-        sleep(interval)
-        print("interval 1: end")
-        try:
-            title, review = Get_review_data(url)
-        except:
-            print("Get_review_data is error")
-        file_object.write("{}\s{}\n".format(str(title), str(review)))
-    file_object.close()
-    print("save: Complete")
-### 実行部分_jp
-def Start_extract_review_en(start_url):
-    #各値 初期化
-    title = []
-    review = []
-    links = []
-    urls = []
-    print("get_nextpage_url:  Start")
-    try:
-        links, spotname = Get_nextpage_url_en(start_url)
-    except:
-        print("get_nextpage_url is error")
-    print("Total Next url =  " + str(len(links)))
-    print("get_review_url:  Start")
-    #1つずつurlを
-    for link in links:
-        try:
-            urls.extend(Get_review_url_en(link)) #必要
-        except:
-            print("get_review_url is error")
-    print("Total review =  " + str(len(urls)))
-    #urls = review数
-    print("Get_review_data:  Start")
-    debug = 0
-    #filename指定してcsvに保存
-    try:
-        filepath = Save_path(spotname)
-    except:
-        print("Save_path is error")
-    #file_open
-    file_object= codecs.open(filepath, "a", "cp932", "ignore")
-    for url in urls:
-        debug += 1
-        print("review = " + str(debug))
-        interval = 5 + random.uniform(-3.0, 3.0)
-        print("interval 1: begin")
-        sleep(interval)
-        print("interval 1: end")
-        try:
-            title, review = Get_review_data(url)
-        except:
-            print("Get_review_data is error")
-        file_object.write("{}\s{}\n".format(str(title), str(review)))
-    file_object.close()
-    print("save: Complete")
-if __name__ =='__main__':
-    """
-    #debug Get_next_url
-    #"次へ"のページのurl取得 1
-    #url = "https://www.tripadvisor.jp/Attraction_Review-g1022838-d1548549-Reviews-Takosenbei_no_Sato-Awaji_Awaji_shima_Hyogo_Prefecture_Kinki.html" #最初のページ
-    url = "https://www.tripadvisor.com/Attraction_Review-g1022838-d1548549-Reviews-Takosenbei_no_Sato-Awaji_Awaji_shima_Hyogo_Prefecture_Kinki.html"
-    links = Get_nextpage_url_jp(url)
-    print(links)
-    """
-    """
-    #debug Get_review_url
-    # レビューの詳細ページ取得 2
-    #url = "https://www.tripadvisor.jp/Attraction_Review-g298562-d1384635-Reviews-Suma_Rikyu_Park-Kobe_Hyogo_Prefecture_Kinki.html"
-    url = "https://www.tripadvisor.com/Attraction_Review-g1022838-d1548549-Reviews-Takosenbei_no_Sato-Awaji_Awaji_shima_Hyogo_Prefecture_Kinki.html"
-    links = Get_review_url(url)
-    print(links)
-    """
-    """
-    #debug Get_review_data
-    # review 取得 3
-    #url = "https://www.tripadvisor.jp/ShowUserReviews-g298562-d1384635-r631487360-Suma_Rikyu_Park-Kobe_Hyogo_Prefecture_Kinki.html"
-    #url = "https://www.tripadvisor.jp/ShowUserReviews-g1022838-d1548549-r631449250-Takosenbei_no_Sato-Awaji_Awaji_shima_Hyogo_Prefecture_Kinki.html"
-    url = "https://www.tripadvisor.com/ShowUserReviews-g1022838-d1548549-r536077279-Takosenbei_no_Sato-Awaji_Awaji_shima_Hyogo_Prefecture_Kinki.html"
-    title, review = Get_review_data(url)
-    print("{}\s{}".format(title,review))
-    """
-    #"""
-    ###手打ち###
-    url_jp = "https://www.tripadvisor.jp/Attraction_Review-g1121309-d5017432-Reviews-Amarube_Railroad_Bridge_Sorano_Eki-Kami_cho_Mikata_gun_Hyogo_Prefecture_Kinki.html"
-    ############
-    url_en = url_jp.replace("https://www.tripadvisor.jp/", "https://www.tripadvisor.com/")
-    Start_extract_review_jp(url_jp)
-    Start_extract_review_en(url_en)
-    #"""