NLP相似度之tf-idf计算

当然，在学习过程中也是参考了很多其他的资料，代码都是一行一行敲出来的。

一、将多个文件合并成一个文件，避免频繁的打开和关闭

import sys

for line in sys.stdin:
    ss = line.strip().split('	')
    file_name = ss[0].strip()
    file_context = ss[1].strip()
    word_list = file_context.split(' ')

    word_set = set()
    for word in word_list:
        word_set.add(word)

    for word in word_set:
        print '	'.join([word, '1'])

执行命令：就可以得到合并后的文件啦！！！

python convert.py input_tfidf_dir/ > merge_files.data 

tf-idf计算流程图：

二 、计算IDF的值：

map阶段：读取每一行

import sys

for line in sys.stdin:
    ss = line.strip().split('	')
    file_name = ss[0].strip()
    file_context = ss[1].strip()
    word_list = file_context.split(' ')

    word_set = set()
    for word in word_list:
        word_set.add(word)

    for word in word_set:
        print '	'.join([word, '1'])

reduce阶段：

import sys
import math

current_word = None
doc_cnt = 508
count_pool = []
sum = 0

for line in sys.stdin:
    ss = line.strip().split('	')
    if len(ss) != 2:
        continue

    word, val = ss
    if current_word == None:
        current_word = word
    if current_word != word:
        for count in count_pool:
            sum += count

        idf_score = math.log(float(doc_cnt) / (float(sum) + 1))
        print '	'.join([current_word, str(idf_score)])

        current_word = word
        count_pool = []
        sum = 0

    count_pool.append((int(val)))

for count in count_pool:
    sum += count

idf_score = math.log(float(doc_cnt) / (float(sum) + 1))
print '	'.join([current_word, str(idf_score)])

三、计算TF的值：

# 计算tf
# 读取合并后的数据
# 执行命令 cat merge_files.data | python map_tf.py mapper_func idf.data

import sys

word_dict = {}
idf_dict = {}

# 读取计算的idf数据文件
def read_idf_file_func(idf_file_fd):
    with open() as fd:
        for line in fd:
            ss = line.strip().split('	')
            if len(ss) != 2:
                continue
            token = ss[0].strip()
            idf_score = ss[1].strip()
            idf_dict[token] = float(idf_score)
    return idf_dict

# cat merge_files.data | python map_tf.py mapper_func
def mapper_func(idf_file_fd):
    idf_dict = read_idf_file_func(idf_file_fd)
    # 标准输入
    for line in sys.stdin:
        ss = line.strip().split('	')
        file_name = ss[0].strip()
        file_context = ss[1].strip()
        word_list = file_context.split(' ')

        for word in word_list:
            if word not in word_dict:
                word_dict[word] = 1
            else:
                word_dict[word] += 1

        for k,v in word_dict.item():
            if k not in idf_dict:
                continue
            print(file_name,k,v,idf_file_fd[k])
            print(k,v)

if __name__ == "__main__":
    module = sys.modules[__name__]
    func = getattr(module, sys.argv[1])
    args = None
    if len(sys.argv) > 1:
        args = sys.argv[2:]
    func(*args)