第四次作业：完整的中英文词频统计

（一）英文歌：

f = open('lemon tree.txt', 'r', encoding='utf-8')
# 通过文件读取字符串 str
str = f.read()
f.close()
print(str)

#预处理标点符号
str = str.replace('.','')
print(str)

#预处理特殊字符
sep = '.,:'';?!-_'
for ch in sep:
    str = str.replace(ch,'')
    print(str)

#分解提取单词 list
strlist = str.split()
print(len(strlist),strlist)

#单词计数set
strSet  = set(strlist)
print(len(strSet),strSet)

#单词计数dict
strDict={}
for word in strSet:
    strDict[word]=strlist.count(word)
    print(len(strDict),strDict)
# 词频排序list.sort(key=)
    Dict = dict(strDict)
    DictList = list(Dict.items())
    print(DictList)
    DictList.sort(key=lambda x: x[1], reverse=True)
    print(DictList)

# 排除语法型词汇，代词、冠词、连词等无语义词
strSet = set(strSet)
exclude = {'a', 'and', 'the', 'in', 'you'}
exset = strSet - exclude
print(len(exset), exset)
# 输出TOP(20)
for i in range(20):
     print(DictList[i])

运行结果：

（二）中文小说：

import jieba
f=open('白夜行.txt','r',encoding='utf-8')
lines=f.read()
f.close()

sep = '，。？！；：“”‘’-——<_/>'
for en in sep:
    lines=lines.replace(en, '')

lines = list(jieba.cut_for_search(lines))

strSet = set(lines)
    #print(len(strSet), strSet)

strDict = dict()
for word in strSet:
    strDict[word] = lines.count(word)
        #print(len(strDict), strDict)

wcList = list(strDict.items())
#print(wcList)
wcList.sort(key=lambda x: x[1], reverse=True)
#print(wcList)

for i in range(20):
    print(wcList[i])

 运行结果：