自然语言处理库gensim之Word2vec

glove和word2vec是目前最常用的两个训练词向量的模型，两者训练出来的文件都以文本格式呈现，区别在于word2vec包含向量的数量及其维度。

1.gensim加载glove训练的词向量

import numpy as np

import matplotlib.pyplot as plt
plt.style.use('ggplot')

from sklearn.decomposition import PCA
#加载Glove向量
from gensim.test.utils import datapath, get_tmpfile
from gensim.models import KeyedVectors
#加载word2vec向量
from gensim.scripts.glove2word2vec import glove2word2vec


glove_file = datapath('F:/python_DemoCode/PytorchEx/.vector_cache/glove.6B.100d.txt')                         #输入文件
word2vec_glove_file = get_tmpfile("F:/python_DemoCode/PytorchEx/.vector_cache/glove.6B.100d.word2vec.txt")    #输出文件
glove2word2vec(glove_file, word2vec_glove_file)                   #转换

model = KeyedVectors.load_word2vec_format(word2vec_glove_file)    #加载转化后的文件

测试一下：

print(model['banana'].shape)               #(100,)

print(model.most_similar('banana'))
print(model.most_similar(negative='banana'))

#计算词语相似度
result = model.most_similar(positive=['woman', 'king'], negative=['man'])
print("{}: {:.4f}".format(*result[0]))                    #queen: 0.7699


def analogy(x1, x2, y1):
    result = model.most_similar(positive=[y1, x2], negative=[x1])
    return result[0][0]

print(analogy('man', 'king', 'woman'))                    #queen
print(analogy('japan', 'japanese', 'australia'))          #australian
print(analogy('tall', 'tallest', 'long'))                 #longest
print(analogy('good', 'fantastic', 'bad'))                #terrible

print(model.doesnt_match("breakfast cereal dinner lunch".split()))    #cereal

散点图演示一下：

def display_pca_scatterplot(model, words=None, sample=0):
    if words == None:
        if sample > 0:
            words = np.random.choice(list(model.vocab.keys()), sample)
        else:
            words = [ word for word in model.vocab ]           #words里面存储了单词集，len(model.vocab))=400000
        
    word_vectors = np.array([model[w] for w in words])         #word_vectors里面存储了单词集对应的嵌入向量

    twodim = PCA().fit_transform(word_vectors)[:,:2]           #降维，取前两个维度
    
    plt.figure(figsize=(6,6))
    plt.scatter(twodim[:,0], twodim[:,1], edgecolors='k', c='r')
    for word, (x,y) in zip(words, twodim):
        plt.text(x+0.05, y+0.05, word)
        

display_pca_scatterplot(model, 
                        ['coffee', 'tea', 'beer', 'wine', 'brandy', 'rum', 'champagne', 'water',
                         'spaghetti', 'borscht', 'hamburger', 'pizza', 'falafel', 'sushi', 'meatballs',
                         'dog', 'horse', 'cat', 'monkey', 'parrot', 'koala', 'lizard',
                         'frog', 'toad', 'monkey', 'ape', 'kangaroo', 'wombat', 'wolf',
                         'france', 'germany', 'hungary', 'luxembourg', 'australia', 'fiji', 'china',
                         'homework', 'assignment', 'problem', 'exam', 'test', 'class',
                         'school', 'college', 'university', 'institute'])

由图可知，相关性较大的词语会靠的近一些。