Document Classification

Natural Language Processing with Python

Chapter 6.1

由于nltk.FreqDist的排序问题，获取电影文本特征词的代码有些微改动。

import nltk
from nltk.corpus import movie_reviews as mr   
        
def document_features(document,words_features):
    document_words=set(document)
    features={}
    for word in words_features:
        features['has(%s)' %word] = (word in document_words)
    return features   

def test_doc_classification():
    documents=[(list(mr.words(fileid)),category)
                for category in mr.categories()
                for fileid in mr.fileids(categories=category)]
    all_words_dist=nltk.FreqDist(w.lower() for w in mr.words())
    words_freq =sorted(all_words_dist.items(), key=lambda x: (-1*x[1], x[0]))[:2000]
    words_features=[word[0] for word in words_freq]
    
    featuresets=[(document_features(doc,words_features),c) for (doc,c) in
                    documents]
                    
    train_set, test_set= featuresets[100:],featuresets[:100]
    classifier=nltk.NaiveBayesClassifier.train(train_set)
    
    print nltk.classify.accuracy(classifier,test_set)

    classifier.show_most_informative_features(5)

结果如下，accuracy为0.86：

0.86
Most Informative Features
has(outstanding) = True pos : neg = 10.4 : 1.0
has(seagal) = True neg : pos = 8.7 : 1.0
has(mulan) = True pos : neg = 8.1 : 1.0
has(wonderfully) = True pos : neg = 6.3 : 1.0
has(damon) = True pos : neg = 5.7 : 1.0