学习笔记CB002:词干提取、词性标注、中文切词、文档分类

# coding:utf-8

import sys

import importlib

importlib.reload(sys)

import nltk

for word in nltk.corpus.sinica_treebank.tagged_words():

    print(word[0], word[1])

# coding:utf-8

import sys

import importlib

importlib.reload(sys)

import nltk

default_tagger = nltk.DefaultTagger('NN')

raw = '我 好 想 你'

tokens = nltk.word_tokenize(raw)

tags = default_tagger.tag(tokens)

print(tags)

# coding:utf-8

import sys

import importlib

importlib.reload(sys)

import nltk

pattern = [(r'.*们$','PRO')]

tagger = nltk.RegexpTagger(pattern)

print(tagger.tag(nltk.word_tokenize('我们 一起  去 你们 和 他们 去过 的 地方')))

# coding:utf-8

import sys

import importlib

importlib.reload(sys)

import nltk

tagged_sents = [[(u'我', u'PRO'), (u'小兔', u'NN')]]

unigram_tagger = nltk.UnigramTagger(tagged_sents)

sents = [[u'我', u'你', u'小兔']]

# brown_tagged_sents = nltk.corpus.brown.tagged_sents(categories='news')

# unigram_tagger = nltk.UnigramTagger(brown_tagged_sents)

# sents = nltk.corpus.brown.sents(categories='news')

tags = unigram_tagger.tag(sents[0])

print(tags)

# coding:utf-8

import sys

import importlib

importlib.reload(sys)

import nltk

train_sents = [[(u'我', u'PRO'), (u'小兔', u'NN')]]

t0 = nltk.DefaultTagger('NN')

t1 = nltk.UnigramTagger(train_sents, backoff=t0)

t2 = nltk.BigramTagger(train_sents, backoff=t1)

sents = [[u'我', u'你', u'小兔']]

tags = t2.tag(sents[0])

print(tags)

from pickle import dump

print(t2)

output = open('t2.pkl', 'wb')

dump(t2, output, -1)

output.close()

from pickle import load

input = open('t2.pkl', 'rb')

tagger = load(input)

input.close()

print(tagger)

# coding:utf-8

import sys

import importlib

importlib.reload(sys)

import nltk

my_train_set = [

        ({'feature1':u'a'},'1'),

        ({'feature1':u'a'},'2'),

        ({'feature1':u'a'},'3'),

        ({'feature1':u'a'},'3'),

        ({'feature1':u'b'},'2'),

        ({'feature1':u'b'},'2'),

        ({'feature1':u'b'},'2'),

        ({'feature1':u'b'},'2'),

        ({'feature1':u'b'},'2'),

        ({'feature1':u'b'},'2'),

        ]

classifier = nltk.NaiveBayesClassifier.train(my_train_set)

print(classifier.classify({'feature1':u'a'}))

print(classifier.classify({'feature1':u'b'}))

# coding:utf-8

import sys

import importlib

importlib.reload(sys)

import nltk

from nltk.corpus import movie_reviews

import random

documents =[(list(movie_reviews.words(fileid)),category)for category in movie_reviews.categories()for fileid in movie_reviews.fileids(category)]

random.shuffle(documents)

all_words = nltk.FreqDist(w.lower() for w in movie_reviews.words())

word_features = [word for (word, freq) in all_words.most_common(2000)]

def document_features(document):

        document_words = set(document)

        features = {}

        for word in word_features:

                features['contains(%s)' % word] = (word in document_words)

        return features

featuresets = [(document_features(d), c) for (d,c) in documents]

# classifier = nltk.NaiveBayesClassifier.train(featuresets)

# classifier.classify(document_features(d))

train_set, test_set = featuresets[100:], featuresets[:100]

classifier = nltk.NaiveBayesClassifier.train(train_set)

print(nltk.classify.accuracy(classifier, test_set))

classifier.show_most_informative_features(5)