爬取“我们是冠军”的评论

微博热搜第一名；B站人气超过3.5亿，满屏弹幕；腾讯视频超过600万人观看；央视新闻也发微博祝贺EDG；今天用python来爬下B站“我们是冠军”这个视频的评论并做些可视化。获取呐喊的正确姿势。

评论爬取代码：

import csv
import pprint
import random
import time
import requests
import openpyxl as opx
import json

f = open('我们是冠军.csv', mode='a', encoding='utf-8-sig', newline='')
csvWriter = csv.DictWriter(f, fieldnames=[
    '评论人',
    '性别',
    '点赞数',
    '评论时间',
    '评论内容',
])
csvWriter.writeheader() # 写入头
startStampTime = int(time.time() * 1000)
# # 新建excel文档
# wb = opx.Workbook()
# ws = wb.create_sheet(index=0)
#
# # 先写入表头
# ws.cell(row=1, column=1, value='评论人')
# ws.cell(row=1, column=2, value='性别')
# ws.cell(row=1, column=3, value='点赞数')
# ws.cell(row=1, column=4, value='评论时间')
# ws.cell(row=1, column=5, value='评论内容')

headers = {
    "cookie": "_uuid=BE35640F-EB4E-F87D-53F2-7A8FD5D50E3330964infoc; buvid3=D0213B95-F001-4A46-BE4F-E921AE18EB67167647infoc; CURRENT_BLACKGAP=1; CURRENT_QUALITY=0; rpdid=|(u))ku~m)kJ0J'uYJuRRRYmk; CURRENT_FNVAL=976; video_page_version=v_old_home_17; blackside_state=1; LIVE_BUVID=AUTO1516364619569495; sid=bqyo86kv; innersign=1; PVID=2",
    "referer": "https://www.xxx.com/video/BV12R4y1E7kn?spm_id_from=333.999.0.0",
    "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.69 Safari/537.36",
}
# # 初始计数
# count = 2
for page in range(1, 100 + 1):
    print(f'====================正在爬取第{page}页的数据====================')
    time.sleep(random.randint(2,5)) # 随机休眠
    nextStampTime = int(time.time() * 1000)
    # 请求的网址
    url = f'https://api.xxx.com/x/v2/reply/main?callback=jQuery172046940903221511165_{startStampTime}&jsonp=jsonp&next={page}&type=1&oid=336587753&mode=3&plat=1&_={nextStampTime}'
    # 开始请求数据
    response = requests.get(url=url, headers=headers)
    # print(response.text)
    json_data = json.loads(response.text[42:-1])
    # 提取我们要的数据
    data = json_data['data']['replies']
    print(f'第{page}页包含:' + str(len(data)))
    for item in data:
        # pprint.pprint(item)
        name = item['member']['uname']
        sex = item['member']['sex']
        like = item['like']
        ctime = item.get('ctime')
        # print(ctime)
        commenttime = time.strftime('%Y-%m-%d %H:%M', time.localtime(ctime))
        content = item['content']['message']
        # print(name, sex, like, commenttime, content, sep=' | ')
        dit = {
            '评论人':name,
            '性别':sex,
            '点赞数':like,
            '评论时间':commenttime,
            '评论内容':content,
        }
        print(dit)
        csvWriter.writerow(dit)
    break
print('数据采集完毕！')

下面来做个词云：

import re
import jieba
import matplotlib.pyplot as plt
import pandas as pd
from wordcloud import WordCloud
# 读取
df = pd.read_csv('我们是冠军.csv')

# 删除重复记录和NA值
df_new = df.drop_duplicates() # 去重
df_new = df_new.dropna() # 删除缺失值
# print(df_new)
STOPWORDS = {"回复", "@", "我", "她", "你", "他", "了", "的", "吧", "吗", "在", "啊", "不", "也", "还", "是",
             "说", "都", "就", "没", "做", "人", "赵薇", "被", "不是", "现在", "什么", "这", "呢", "知道", "邓", "我们", "他们", "和", "有", "", "",
            "要", "就是", "但是", "而", "为", "自己", "中", "问题", "一个", "没有", "到", "这个", "并", "对", "点赞", "热词", "系列", "热词系列"}

# 取出评论区的词进行分词
textList = df_new['评论内容'].value_counts().sort_values().index.tolist()
# print(textList)
# 将列表转换成字符串
strText = ' '.join(textList)
# 正则替换一下指定次
newTxt = re.sub("A-Z0-9-a-z!\%[]\,。", "", strText)
# print(newTxt)
words = jieba.lcut(newTxt)

# 制作词云
wordcloudword = WordCloud(
    background_color='white',
    width = 1080,
    height = 960,
    # font_path = "../文悦新青年.otf",
    font_path = 'C:/Windows/Fonts/simhei.ttf',
    max_words = 150,
    scale = 10, #清晰度
    max_font_size = 100,
    stopwords=STOPWORDS,
    # mask = img_array, # 可以设置背景图像
    collocations=False).generate(newTxt)

plt.imshow(wordcloudword)
plt.axis('off')
plt.show()
wordcloudword.to_file('wc.png')