• 第四章--基于概率论的分类方法:朴素贝叶斯--新闻分类(三)


      1 import os
      2 import jieba
      3 import random
      4 """
      5 函数说明:中文文本处理
      6 Parameters:
      7     folder_path - 文本存放的路径
      8     test_size - 测试集占比,默认占所有数据集的百分之20
      9 Returns:
     10     all_words_list - 按词频降序排序的训练集列表
     11     train_data_list - 训练集列表
     12     test_data_list - 测试集列表
     13     train_class_list - 训练集标签列表
     14     test_class_list - 测试集标签列表
     15 """
     16 def TextProcessing(folder_path):
     17     folder_list = os.listdir(folder_path)                        #查看folder_path下的文件
     18     data_list = []                                                #训练集
     19     class_list = []
     20 
     21     #遍历每个子文件夹
     22     for folder in folder_list:
     23         new_folder_path = os.path.join(folder_path, folder)        #根据子文件夹,生成新的路径
     24         files = os.listdir(new_folder_path)                        #存放子文件夹下的txt文件的列表
     25 
     26         j = 1
     27         #遍历每个txt文件
     28         for file in files:
     29             if j > 100:                                            #每类txt样本数最多100个
     30                 break
     31             with open(os.path.join(new_folder_path, file), 'r', encoding = 'utf-8') as f:    #打开txt文件
     32                 raw = f.read()
     33 
     34             word_cut = jieba.cut(raw, cut_all = False)            #精简模式,返回一个可迭代的generator
     35             word_list = list(word_cut)                            #generator转换为list
     36 
     37             data_list.append(word_list)
     38             class_list.append(folder)
     39             j += 1
     40             data_class_list = list(zip(data_list, class_list))  # zip压缩合并,将数据与标签对应压缩
     41             random.shuffle(data_class_list)  # 将data_class_list乱序
     42             index = int(len(data_class_list) * test_size) + 1  # 训练集和测试集切分的索引值
     43             train_list = data_class_list[index:]  # 训练集
     44             test_list = data_class_list[:index]  # 测试集
     45             train_data_list, train_class_list = zip(*train_list)  # 训练集解压缩
     46             test_data_list, test_class_list = zip(*test_list)  # 测试集解压缩
     47 
     48             all_words_dict = {}  # 统计训练集词频
     49             for word_list in train_data_list:
     50                 for word in word_list:
     51                     if word in all_words_dict.keys():
     52                         all_words_dict[word] += 1
     53                     else:
     54                         all_words_dict[word] = 1
     55 
     56             # 根据键的值倒序排序
     57             all_words_tuple_list = sorted(all_words_dict.items(), key=lambda f: f[1], reverse=True)
     58             all_words_list, all_words_nums = zip(*all_words_tuple_list)  # 解压缩
     59             all_words_list = list(all_words_list)  # 转换成列表
     60             return all_words_list, train_data_list, test_data_list, train_class_list, test_class_list
     61 
     62 if __name__ == '__main__':
     63     # 文本预处理
     64     folder_path = './SogouC/Sample'  # 训练集存放地址
     65     all_words_list, train_data_list, test_data_list, train_class_list, test_class_list = TextProcessing(
     66            folder_path, test_size=0.2)
     67     print(all_words_list)
     68 """
     69 函数说明:读取文件里的内容,并去重
     70 Parameters:
     71      words_file - 文件路径
     72 Returns:
     73      words_set - 读取的内容的set集合
     74 """
     75 def MakeWordsSet(words_file):
     76      words_set = set()  # 创建set集合
     77      with open(words_file, 'r', encoding='utf-8') as f:  # 打开文件
     78           for line in f.readlines():  # 一行一行读取
     79               word = line.strip()  # 去回车
     80               if len(word) > 0:  # 有文本,则添加到words_set中
     81                   words_set.add(word)
     82      return words_set  # 返回处理结果
     83 """
     84 函数说明:文本特征选取
     85 Parameters:
     86     all_words_list - 训练集所有文本列表
     87     deleteN - 删除词频最高的deleteN个词
     88     stopwords_set - 指定的结束语
     89 Returns:
     90     feature_words - 特征集
     91 """
     92 def words_dict(all_words_list, deleteN, stopwords_set=set()):
     93        feature_words = []  # 特征列表
     94        n = 1
     95        for t in range(deleteN, len(all_words_list), 1):
     96             if n > 1000:  # feature_words的维度为1000
     97                break
     98        # 如果这个词不是数字,并且不是指定的结束语,并且单词长度大于1小于5,那么这个词就可以作为特征词
     99        if not all_words_list[t].isdigit() and all_words_list[t] not in stopwords_set and 1 < len(
    100                all_words_list[t]) < 5:
    101                feature_words.append(all_words_list[t])
    102            n += 1
    103        return feature_words
    104 
    105 if __name__ == '__main__':
    106             # 文本预处理
    107      folder_path = './SogouC/Sample'  # 训练集存放地址
    108      all_words_list, train_data_list, test_data_list, train_class_list, test_class_list = TextProcessing(
    109           folder_path, test_size=0.2)
    110 
    111 # 生成stopwords_set
    112      stopwords_file = './stopwords_cn.txt'
    113      stopwords_set = MakeWordsSet(stopwords_file)
    114      feature_words = words_dict(all_words_list, 100, stopwords_set)
    115      print(feature_words)
    116 
    117         # print(data_list)
    118         # print(class_list)
    119 if __name__ == '__main__':
    120     #文本预处理
    121     folder_path = './SogouC/Sample'                #训练集存放地址
    122     TextProcessing(folder_path)
    123 '''
    124 将所有文本分成训练集和测试集,并对训练集中的所有单词进行词频统计,并按降序排序。
    125 使用Sklearn构建朴素贝叶斯分类器
    126 在scikit-learn中,一共有3个朴素贝叶斯的分类算法类。
    127 分别是GaussianNB,MultinomialNB和BernoulliNB。
    128 其中GaussianNB就是先验为高斯分布的朴素贝叶斯,
    129 MultinomialNB就是先验为多项式分布的朴素贝叶斯,
    130 而BernoulliNB就是先验为伯努利分布的朴素贝叶斯。
    131 '''
    132 from sklearn.naive_bayes import MultinomialNB
    133 import matplotlib.pyplot as plt
    134 """
    135 函数说明:根据feature_words将文本向量化
    136 Parameters:
    137     train_data_list - 训练集
    138     test_data_list - 测试集
    139     feature_words - 特征集
    140 Returns:
    141     train_feature_list - 训练集向量化列表
    142     test_feature_list - 测试集向量化列表
    143 """
    144 def TextFeatures(train_data_list, test_data_list, feature_words):
    145     def text_features(text, feature_words):                        #出现在特征集中,则置1
    146         text_words = set(text)
    147         features = [1 if word in text_words else 0 for word in feature_words]
    148         return features
    149     train_feature_list = [text_features(text, feature_words) for text in train_data_list]
    150     test_feature_list = [text_features(text, feature_words) for text in test_data_list]
    151     return train_feature_list, test_feature_list                #返回结果
    152 
    153 """
    154 函数说明:新闻分类器
    155 
    156 Parameters:
    157     train_feature_list - 训练集向量化的特征文本
    158     test_feature_list - 测试集向量化的特征文本
    159     train_class_list - 训练集分类标签
    160     test_class_list - 测试集分类标签
    161 Returns:
    162     test_accuracy - 分类器精度
    163 """
    164 def TextClassifier(train_feature_list, test_feature_list, train_class_list, test_class_list):
    165     classifier = MultinomialNB().fit(train_feature_list, train_class_list)
    166     test_accuracy = classifier.score(test_feature_list, test_class_list)
    167     return test_accuracy
    168 
    169 # if __name__ == '__main__':
    170 #     #文本预处理
    171 #     folder_path = './SogouC/Sample'                #训练集存放地址
    172 #     all_words_list, train_data_list, test_data_list, train_class_list, test_class_list = TextProcessing(folder_path, test_size=0.2)
    173 #
    174 #     # 生成stopwords_set
    175 #     stopwords_file = './stopwords_cn.txt'
    176 #     stopwords_set = MakeWordsSet(stopwords_file)
    177 
    178     test_accuracy_list = []
    179     deleteNs = range(0, 1000, 20)                #0 20 40 60 ... 980
    180     for deleteN in deleteNs:
    181         feature_words = words_dict(all_words_list, deleteN, stopwords_set)
    182         train_feature_list, test_feature_list = TextFeatures(train_data_list, test_data_list, feature_words)
    183         test_accuracy = TextClassifier(train_feature_list, test_feature_list, train_class_list, test_class_list)
    184         test_accuracy_list.append(test_accuracy)
    185 
    186     plt.figure()
    187     plt.plot(deleteNs, test_accuracy_list)
    188     plt.title('Relationship of deleteNs and test_accuracy')
    189     plt.xlabel('deleteNs')
    190     plt.ylabel('test_accuracy')
    191     plt.show()
    192     if __name__ == '__main__':
    193         # 文本预处理
    194         folder_path = './SogouC/Sample'  # 训练集存放地址
    195         all_words_list, train_data_list, test_data_list, train_class_list, test_class_list = TextProcessing(folder_path,
    196                                                                                                             test_size=0.2)
    197 
    198         # 生成stopwords_set
    199         stopwords_file = './stopwords_cn.txt'
    200         stopwords_set = MakeWordsSet(stopwords_file)
    201 
    202         test_accuracy_list = []
    203         feature_words = words_dict(all_words_list, 450, stopwords_set)
    204         train_feature_list, test_feature_list = TextFeatures(train_data_list, test_data_list, feature_words)
    205         test_accuracy = TextClassifier(train_feature_list, test_feature_list, train_class_list, test_class_list)
    206         test_accuracy_list.append(test_accuracy)
    207         ave = lambda c: sum(c) / len(c)
  • 相关阅读:
    Alpha 冲刺 (10/10)
    Alpha 冲刺 (9/10)
    Alpha 冲刺 (8/10)
    Alpha 冲刺 (7/10)
    Alpha 冲刺 (6/10)
    Alpha 冲刺 (5/10)
    18软工实践-团队现场编程实战(抽奖系统)
    Alpha 冲刺 (4/10)
    BETA(4)
    BETA(3)
  • 原文地址:https://www.cnblogs.com/fd-682012/p/11660071.html
Copyright © 2020-2023  润新知