JAVA日报

双人结对作业疫情可视化（j数据爬取）

# -*- coding:utf-8 -*-
import requests
import json
import re  # 内库, 标准库
import MysqlUtil


# url = 'https://view.inews.qq.com/g2/getOnsInfo?name=disease_h5'
# def __init__(self):
# self.session = requests.Session()  # 下载器

def get_data(url):  # 主逻辑
        """下载内容"""
        """下载网页首页面html"""
        #  index_html = self.download(url, encoding='utf-8')
        response = requests.get(url)  # unicode
        print(url)
        # print(response.text)
        d = json.loads(response.text)  # dict
        print(type(d["data"]))
        print(d.keys())
        data_all = json.loads(d["data"])
        print(data_all.keys())
        print("_______________________________________________________________________________________________")
        print("lastUpdateTime")
        print(type(data_all["lastUpdateTime"]))
        print(data_all["lastUpdateTime"])  # 更新时间
        print("_______________________________________________________________________________________________")
        print("chinaTotal:")
        print(type(data_all["chinaTotal"]))
        print(data_all["chinaTotal"].keys())
        print(data_all["chinaTotal"])
        print("_______________________________________________________________________________________________")
        print("chinaAdd:")
        print(type(data_all["chinaAdd"]))
        print(data_all["chinaAdd"].keys())
        print(data_all["chinaAdd"])
        print("_______________________________________________________________________________________________")
        print("isShowAdd")
        print(type(data_all["isShowAdd"]))
        print(data_all["isShowAdd"])
        print("_______________________________________________________________________________________________")
        print("showAddSwitch")
        print(type(data_all["showAddSwitch"]))
        print(data_all["showAddSwitch"])
        print("_______________________________________________________________________________________________")
        print("areaTree")
        print(type(data_all["areaTree"]))
        print(type(data_all["areaTree"][0]))
        print(data_all["areaTree"][0].keys())
        print("_______________________________________________________________________________________________")
        print(data_all["areaTree"][0]["name"])
        print("_______________________________________________________________________________________________")
        print(data_all["areaTree"][0]["today"])
        print("_______________________________________________________________________________________________")
        print(data_all["areaTree"][0]["total"])
        print("_______________________________________________________________________________________________")
        print(len(data_all["areaTree"][0]["children"]))
        # print(data_all["areaTree"][0]["children"])
        print(type(data_all["areaTree"][0]["children"]))
        print(data_all["areaTree"][0]["children"][0].keys())
        print(type(data_all["areaTree"][0]["children"]))
        for i in data_all["areaTree"][0]["children"]:
            print(i["name"])
            print(i["total"])
            print(i["today"])
            for k in i["children"]:
                print(k["name"])
                print(k["total"])
                print(k["today"])
        print("_______________________________________________________________________________________________")
        country = {}  # history
        time = data_all["lastUpdateTime"]
        confirm = data_all["chinaTotal"]["confirm"]
        heal = data_all["chinaTotal"]["heal"]
        suspect = data_all["chinaTotal"]["suspect"]
        dead = data_all["chinaTotal"]["dead"]
        country[time] = {"confirm": confirm, "heal": heal, "suspect": suspect, "dead": dead}
        confirmAdd = data_all["chinaAdd"]["confirm"]
        healAdd = data_all["chinaAdd"]["heal"]
        suspectAdd = data_all["chinaAdd"]["suspect"]
        deadAdd = data_all["chinaAdd"]["dead"]
        country[time].update({"confirmAdd": confirmAdd, "healAdd": healAdd, "suspectAdd": suspectAdd, "deadAdd": deadAdd})
        # print(country)

        province =[]
        area = []
        for i in data_all["areaTree"][0]["children"]:
            ptime = data_all["lastUpdateTime"]
            pname = i["name"]
            pconfirm = i["total"]["confirm"]
            pheal = i["total"]["heal"]
            psuspect = i["total"]["suspect"]
            pdead = i["total"]["dead"]
            pconfirmAdd = i["today"]["confirm"]
            province.append([ptime, pname, pconfirm, pheal, psuspect, pdead, pconfirmAdd])
            for k in i["children"]:
                cprovince = i["name"]
                ctime = data_all["lastUpdateTime"]
                cname = k["name"]
                cconfirm = k["total"]["confirm"]
                cheal = k["total"]["heal"]
                csuspect = k["total"]["suspect"]
                cdead = k["total"]["dead"]
                cconfirmAdd = k["today"]["confirm"]
                area.append([ctime, cprovince, cname, cconfirm, cheal, csuspect, cdead, cconfirmAdd])

        # print(province)
        # print(city)
        return country, province, area


if __name__ == '__main__':
    #  url = 'https://voice.baidu.com/act/newpneumonia/newpneumonia'  # 网页html
    url = 'https://view.inews.qq.com/g2/getOnsInfo?name=disease_h5'
    get_data(url);
    #  spider = Spider()  # 实例化
    #  MysqlUtil.insert_province()
    # MysqlUtil.update_area()
    # RES = MysqlUtil.get_province()
    # print(RES)
    # qwe = []
    # for tup in RES:
    #     qwe.append({"name": tup[0], "value": tup[1]})
    # print(type(RES))
    # print(qwe)
    # MysqlUtil.insert_area()
    # RES = MysqlUtil.get_area("北京")
    # print(RES)