我想从python中的两个嵌入式文档中获取语义相似的单词列表。

import nltk
from nltk.corpus import wordnet
from nltk.tokenize import word_tokenize
from nltk.stem import WordNetLemmatizer

def preprocess_text(text):
    tokens = word_tokenize(text)  # 分词
    lemmatizer = WordNetLemmatizer()  # 词形还原
    lemmatized_tokens = [lemmatizer.lemmatize(token) for token in tokens]
    return lemmatized_tokens

document1 = "Your first document here."
document2 = "Your second document here."

tokens1 = preprocess_text(document1)
tokens2 = preprocess_text(document2)

def get_semantic_similar_words(word):
    synsets = wordnet.synsets(word)  # 获取单词的同义词集合
    similar_words = []
    for synset in synsets:
        for lemma in synset.lemmas():
            similar_words.append(lemma.name())  # 获取同义词
    return similar_words

similar_words = []
for token in tokens1:
    similar_words.extend(get_semantic_similar_words(token))

similar_words = list(set(similar_words))  # 去重