Python NLTK解读

pythonCopy codepip install nltk

pythonCopy codeimport nltk

sentence = "NLTK is a powerful tool for natural language processing."
tokens = nltk.word_tokenize(sentence)

print(tokens)

pythonCopy codefrom nltk.corpus import stopwords

nltk.download('stopwords')

sentence = "NLTK is a powerful tool for natural language processing."
tokens = nltk.word_tokenize(sentence)

filtered_tokens = [word for word in tokens if word.lower() not in stopwords.words('english')]

print(filtered_tokens)

pythonCopy codefrom nltk.stem import PorterStemmer

porter = PorterStemmer()

words = ["running", "jumps", "played"]
stemmed_words = [porter.stem(word) for word in words]

print(stemmed_words)

pythonCopy codenltk.download()

pythonCopy codefrom nltk import FreqDist
import matplotlib.pyplot as plt
from wordcloud import WordCloud

text = "NLTK is a powerful tool for natural language processing. NLTK provides easy-to-use interfaces to over 50 corpora and lexical resources such as WordNet."

tokens = nltk.word_tokenize(text)
fdist = FreqDist(tokens)

# 绘制词频分布图
fdist.plot(30, cumulative=False)
plt.show()

# 生成词云
wordcloud = WordCloud().generate(text)
plt.imshow(wordcloud, interpolation='bilinear')
plt.axis("off")
plt.show()

pythonCopy codefrom nltk import CFG, ChartParser

# 定义语法规则
grammar = CFG.fromstring("""
    S -> NP VP
    NP -> Det N | 'I'
    VP -> V NP
    Det -> 'an' | 'the'
    N -> 'elephant' | 'pajamas'
    V -> 'saw' | 'ate'
""")

# 创建分析器
parser = ChartParser(grammar)

# 句子
sentence = "I saw an elephant"

# 分析句子
for tree in parser.parse(sentence.split()):
    tree.pretty_print()

pythonCopy codefrom nltk import ne_chunk

sentence = "Barack Obama was born in Hawaii."

# 分词
tokens = nltk.word_tokenize(sentence)

# 执行命名实体识别
entities = ne_chunk(nltk.pos_tag(tokens))

print(entities)

pythonCopy codefrom nltk.corpus import movie_reviews
from nltk.classify import NaiveBayesClassifier
from nltk.classify.util import accuracy

# 构建特征提取器
def extract_features(words):
    return dict([(word, True) for word in words])

# 获取影评数据
positive_reviews = [(extract_features(movie_reviews.words(fileids=[f])), 'Positive') for f in movie_reviews.fileids('pos')]
negative_reviews = [(extract_features(movie_reviews.words(fileids=[f])), 'Negative') for f in movie_reviews.fileids('neg')]

# 划分数据集
split = int(len(positive_reviews) * 0.8)
train_set = positive_reviews[:split] + negative_reviews[:split]
test_set = positive_reviews[split:] + negative_reviews[split:]

# 训练朴素贝叶斯分类器
classifier = NaiveBayesClassifier.train(train_set)

# 评估分类器
accuracy_score = accuracy(classifier, test_set)
print("Accuracy:", accuracy_score)

pythonCopy codefrom nltk.corpus import wordnet

# 查找单词的同义词
synonyms = []
for syn in wordnet.synsets("happy"):
    for lemma in syn.lemmas():
        synonyms.append(lemma.name())

print(set(synonyms))

pythonCopy codefrom nltk.corpus import stopwords
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics.pairwise import cosine_similarity

# 文本
text1 = "Natural Language Processing is a field of study in artificial intelligence."
text2 = "NLP is a subfield of artificial intelligence that focuses on the interaction between computers and humans using natural language."

# 停用词
stop_words = set(stopwords.words('english'))

# TF-IDF向量化
vectorizer = TfidfVectorizer(stop_words=stop_words)
tfidf_matrix = vectorizer.fit_transform([text1, text2])

# 计算余弦相似度
similarity_score = cosine_similarity(tfidf_matrix[0], tfidf_matrix[1])[0][0]

print("Cosine Similarity:", similarity_score)

pythonCopy codefrom nltk import Text

# 大型文本
corpus = Text("Large text corpus goes here...")

# 并行搜索
concordance_results = corpus.concordance("search term", width=50, lines=10, num_procs=4)

print(concordance_results)