自然言語処理：エラー箇所と改善方法を教えてください

Question

発生している問題・エラーメッセージ

Traceback (most recent call last):
  File "test2.py", line 59, in <module>
    train_vecs = vectorizer.fit_transform(train_data)
  File "C:\Users\Anaconda3\lib\site-packages\sklearn\feature_extraction\text.py", line 1381, in fit_transform
    X = super(TfidfVectorizer, self).fit_transform(raw_documents)
  File "C:\Users\Anaconda3\lib\site-packages\sklearn\feature_extraction\text.py", line 869, in fit_transform
    self.fixed_vocabulary_)
  File "C:\Users\Anaconda3\lib\site-packages\sklearn\feature_extraction\text.py", line 811, in _count_vocab
    raise ValueError("empty vocabulary; perhaps the documents only"
ValueError: empty vocabulary; perhaps the documents only contain stop words

該当のソースコード

import glob
import time
from sklearn.model_selection import train_test_split
from janome.tokenizer import Tokenizer
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.naive_bayes import MultinomialNB
from sklearn.ensemble import RandomForestClassifier
import random

    for c_name, c_id in category.items():
        files = glob.glob("./text/{c_name}/{c_name}*.txt".format(c_name=c_name))

        text = ''
        for file in files:
            with open(file, 'r') as f:
                lines = f.read().splitlines()
                url = lines[0]
                datetime = lines[1]
                subject = lines[2]
                body = "".join(lines[3:])
                text = subject + body

            docs.append(text)
            labels.append(c_id)

    return docs, labels

docs, labels = load_livedoor_news_corpus()



random.seed()
indices = list(range(len(docs)))
random.shuffle(indices)

train_data   = [docs[i] for i in indices[0:7000]]
train_labels = [labels[i] for i in indices[0:7000]]
test_data    = [docs[i] for i in indices[7000:]]
test_labels  = [labels[i] for i in indices[7000:]]

vectorizer = TfidfVectorizer()
train_vecs = vectorizer.fit_transform(train_data)
test_vecs = vectorizer.transforn(test_data)

clf = MultinomiaINB
model = clf.fit(train_vecs,train_labels)
print(clf.score(train_vecs,train_labels))
print(clf.score(test_vecs,test_labels))

clf = RandomForestClassifier
model = clf.fit(train_vecs,train_labels)
print(clf.score(train_vecs,train_labels))
print(clf.score(test_vecs,test_labels))

def tokenize(text):
    tokens = t.tokenize(','.join(text))
    noun = []
    for token in tokens:
        partOfSpeech = token.part_of_speech.split(',')[0]

        if partOfSpeech == "名詞":
            noun.append(token.surface)
        if partOfSpeech == "動詞":
            noun.append(token.surface)
        if partOfSpeech == "形容詞":
            noun.append(token.surface)
        if partOfSpeech == "形容動詞":
            noun.append(token.surface)

    return noun

empty vocabulary; perhaps the documents only contain stop words
空の語彙。 おそらく文書にはストップワードしか含まれていない

Answer 1

エラー部分を解決することができました