ストップワードを除去しきれない

Question

リアルタイムのツイート群から抽出した名詞を対象に、辞書を構築したいと考えています。
ストップワードを除去してコードを実行したところ、下記のValueErrorが出てしまいました。
これに関して、解決策を教えていただけませんでしょうか？
宜しくお願いします。

エラー

Traceback (most recent call last):
  File "final.py", line 117, in <module>
    stream.sample()
  File "/Users/macuser/Workspaces/jxpress/trendword/.direnv/python-3.7.3/lib/python3.7/site-packages/tweepy/streaming.py", line 449, in sample
    self._start(is_async)
  File "/Users/macuser/Workspaces/jxpress/trendword/.direnv/python-3.7.3/lib/python3.7/site-packages/tweepy/streaming.py", line 389, in _start
    self._run()
  File "/Users/macuser/Workspaces/jxpress/trendword/.direnv/python-3.7.3/lib/python3.7/site-packages/tweepy/streaming.py", line 320, in _run
    six.reraise(*exc_info)
  File "/Users/macuser/Workspaces/jxpress/trendword/.direnv/python-3.7.3/lib/python3.7/site-packages/six.py", line 693, in reraise
    raise value
  File "/Users/macuser/Workspaces/jxpress/trendword/.direnv/python-3.7.3/lib/python3.7/site-packages/tweepy/streaming.py", line 289, in _run
    self._read_loop(resp)
  File "/Users/macuser/Workspaces/jxpress/trendword/.direnv/python-3.7.3/lib/python3.7/site-packages/tweepy/streaming.py", line 351, in _read_loop
    self._data(next_status_obj)
  File "/Users/macuser/Workspaces/jxpress/trendword/.direnv/python-3.7.3/lib/python3.7/site-packages/tweepy/streaming.py", line 323, in _data
    if self.listener.on_data(data) is False:
  File "/Users/macuser/Workspaces/jxpress/trendword/.direnv/python-3.7.3/lib/python3.7/site-packages/tweepy/streaming.py", line 54, in on_data
    if self.on_status(status) is False:
  File "final.py", line 99, in on_status
    tfidf = vectorizer.fit_transform(corpus)
  File "/Users/macuser/Workspaces/jxpress/trendword/.direnv/python-3.7.3/lib/python3.7/site-packages/sklearn/feature_extraction/text.py", line 1652, in fit_transform
    X = super().fit_transform(raw_documents)
  File "/Users/macuser/Workspaces/jxpress/trendword/.direnv/python-3.7.3/lib/python3.7/site-packages/sklearn/feature_extraction/text.py", line 1058, in fit_transform
    self.fixed_vocabulary_)
  File "/Users/macuser/Workspaces/jxpress/trendword/.direnv/python-3.7.3/lib/python3.7/site-packages/sklearn/feature_extraction/text.py", line 989, in _count_vocab
    raise ValueError("empty vocabulary; perhaps the documents only"
ValueError: empty vocabulary; perhaps the documents only contain stop words

該当のソースコード

import os
import tweepy
import redis
import math
from collections import Counter
from collections import defaultdict
import re
from natto import MeCab
import codecs
import sys
from sklearn.feature_extraction.text import TfidfVectorizer
import glob
import numpy as np
import urllib.request
from gensim import corpora
from itertools import chain

#frequency = defaultdict(int)

#r = redis.Redis(host='localhost', port=6379, db=0)

TWITTER_CLIENT_ID = os.environ['TWITTER_CLIENT_ID']
TWITTER_CLIENT_SECRET = os.environ['TWITTER_CLIENT_SECRET']

TWITTER_OAUTH_TOKEN = os.environ['TWITTER_OAUTH_TOKEN']
TWITTER_OAUTH_TOKEN_SECRET = os.environ['TWITTER_OAUTH_TOKEN_SECRET']

auth = tweepy.OAuthHandler(TWITTER_CLIENT_ID,TWITTER_CLIENT_SECRET)
auth.set_access_token(TWITTER_OAUTH_TOKEN,TWITTER_OAUTH_TOKEN_SECRET)

class StreamListener(tweepy.StreamListener):
    def __init__(self):
        super().__init__()
        self.count = 0 # 取得したtweet数

    def on_status(self, status):
        text = str(status.text)
        #https://www.pytry3g.com/entry/master-Preprocessing#正規表現を使う
        text = re.sub(r"http\S+", "", text)
        text = re.sub(r"@(\w+) ", "", text)
        text = re.sub(r"#(\w+)", "", text)
        #リツイートは消えていない
        text = re.sub(r"(^RT.*)", "", text, flags=re.MULTILINE | re.DOTALL)
        #絵文字も消したい
        emoji_pattern = re.compile("["
        u"\U0001F600-\U0001F64F"
        u"\U0001F300-\U0001F5FF"
        u"\U0001F680-\U0001F6FF"
        u"\U0001F1E0-\U0001F1FF"
        "]+", flags=re.UNICODE)
        text = emoji_pattern.sub("", text)

        #日本語ツイートをファイルに書き込んでいる + ツイートの数を表示
        if status.lang == "ja":
            self.count += 1
            print(self.count, text)
            with open("test45.txt", "a", encoding="utf-8") as f:
                f.write(text)
            with codecs.open("test45.txt", "r", "utf-8") as f:
                corpus = f.read().split("\n")

            mecab = MeCab('-d /usr/local/lib/mecab/dic/mecab-ipadic-neologd')

            #if tagger.lang == 'ja':
            #名詞の抽出と整形
            rm_list = ["RT","https","co","さん","フォロー","本日","応募","今日","プレゼント","お金","FGO","無料","本人","投稿","動画","ツイート","リツイート","Twitter","ローソン","Peing","http","Amazonギフト券","bot","発売中","Youtube","www","WWW","質問箱","コラボ","フォロワー","DM","いいね","ＲＴ","lawson","://","！","peing","youtube","抽選","jp","リプ","キャンペーン","チケット","期間限定","DHC","日本","amp","人間","チャンネル","配信中","YouTube","WEB","楽しみ","イラスト","くじ","@","__"]

            stop_words = []
            path = 'stop_words.txt'
            with open(path) as g:
                stop_words = g.readlines()

            docs = []
            for txt in corpus:
                words = mecab.parse(txt, as_nodes=True)
                doc = []

                for w in words:
                    if w.feature.split(",")[0] == "名詞":
                        if len(w.surface) >= 3:
                            if not any(rm in w.surface for rm in rm_list):
                                doc.extend(str(w.surface))

                docs.extend(doc)
            corpus = docs

            #辞書の構築
            dictionary = corpora.Dictionary([corpus])
            dictionary.filter_extremes(no_below=20, no_above=0.5)
            dictionary.save_as_text('test40-dic8.txt')

            #単語の頻出度の抽出
            counter = Counter(chain.from_iterable(corpus))
            for word, count in counter.most_common():
                if count > 19:
                    print(f"{word}: {count}")

　　　  #tf-idf計算
            vectorizer = TfidfVectorizer(min_df=0.02)
            tfidf = vectorizer.fit_transform(corpus)

            #スコアの表示
            print(tfidf.toarray())
            # テキストの数、出現した単語の数
            print(tfidf.shape)

            #並べ替え
            feature_names = np.array(vectorizer.get_feature_names())
            for vec in tfidf:
                index = np.argsort(vec.toarray(), axis=1)[:,::-1]
                feature_words = feature_names[index]
                print(feature_words[:,:10])

    def on_error(self, status_code):
        return False

stream = tweepy.Stream(auth=auth, listener=StreamListener())
stream.sample()

補足情報（FW/ツールのバージョンなど）

iOS 10.12.6, Python 3.7.3, Atom

Answer 1

ストップワードどうこうではなくて、データの作り方が間違っているように思います。doc.extend(str(w.surface))ではなくdoc.append(w.surface)に、docs.extend(doc)ではなくdocs.append(doc)にしてください。TfidfVectorizerの側ではTfidfVectorizer(min_df=0.02, analyzer=lambda x:x)とするなどして調整してください。

あと、if not any(rm in w.surface for rm in rm_list):の部分も意図がよくわかりません。w.surface not in rm_listとかでいいのでは。

Comments

[farinelli]

上手くいきました。有難うございます！