Gensim Word2Vec或FastText从频率构建词汇

pip install gensim

from collections import Counter

# 示例文本数据
texts = [
    ["hello", "world", "hello"],
    ["machine", "learning", "is", "fun"],
    ["hello", "machine", "learning"]
]

# 计算词频
word_freq = Counter(word for text in texts for word in text)
print(word_freq)

from gensim.models import Word2Vec

# 构建 Word2Vec 模型
model = Word2Vec(sentences=texts, vector_size=100, window=5, min_count=1, sg=0)

# 打印词汇表
print(model.wv.key_to_index)

from gensim.models import FastText

# 构建 FastText 模型
model = FastText(sentences=texts, vector_size=100, window=5, min_count=1, sg=0)

# 打印词汇表
print(model.wv.key_to_index)

from gensim.models import Word2Vec, FastText

# 手动构建词汇表
vocab = {word: freq for word, freq in word_freq.items() if freq >= 1}

# 创建一个包含词汇表的句子列表
filtered_texts = [[word for word in text if word in vocab] for text in texts]

# 构建 Word2Vec 模型
word2vec_model = Word2Vec(sentences=filtered_texts, vector_size=100, window=5, min_count=1, sg=0)

# 构建 FastText 模型
fasttext_model = FastText(sentences=filtered_texts, vector_size=100, window=5, min_count=1, sg=0)

# 打印词汇表
print(word2vec_model.wv.key_to_index)
print(fasttext_model.wv.key_to_index)