分享一个简单的python+mysql网络数据抓取一(百度热点信息抓取)

　　最近学习python网络爬虫，所以自己写了一个简单的程序练练手（呵呵。。）。我使用的环境是python3.6和mysql8.0，抓取目标网站为百度热点（http://top.baidu.com/）。我只抓取了实时热点内容，其他栏目应该类似。代码中有两个变量SECONDS_PER_CRAWL和CRAWL_PER_UPDATE_TO_DB，前者为抓取频率，后者为抓取多少次写一次数据库，可自由设置。我抓取的数据内容是热点信息，链接，关注人数和时间。其在内存中存放的结构为dict{tuple(热点信息，链接):list[tuple(关注人数，时间)...]...}，数据库存储是将热点信息和链接放到hotnews表中，关注人数和对应时间放到直接以热点信息为表名的表内（因为热点信息可能长时间存在但是随时间变化关注度会有变化）。下面可以看下数据库存放样例

代码比较简单我就暂时没有上传github直接贴到下面供大家参考:

# -*- coding: UTF-8 -*-

from bs4 import BeautifulSoup
import threading
import requests
import pymysql
import string
import time
import sys
import re


#爬取频率，单位：秒
SECONDS_PER_CRAWL = 10

#更新到数据库频率，单位：爬取次数
CRAWL_PER_UPDATE_TO_DB = 1

#爬取目标网站url
CRAWL_TARGET_URL = 'http://top.baidu.com/'

class DataProducer(threading.Thread):

    #临时存放爬取结果
    NewsDict = {}

    def __init__(self):
        threading.Thread.__init__(self)
        db = pymysql.connect(host="localhost", user="root", port=3306, passwd=None, db="crawler", charset="utf8")
        cursor = db.cursor()
        sql = """CREATE TABLE IF NOT EXISTS HOTNEWS (
              INFORMATION VARCHAR(64) NOT NULL,
              HYPERLINK      VARCHAR(256),
              PRIMARY KEY (INFORMATION) );"""
        cursor.execute(sql)
        db.close()

    def run(self):
        print("DataProducer Thread start!")
        crawl_data(self.NewsDict)
        print("DataProducer Thread exit!")


def crawl_data(nd):
    count = 0;
    while 1:
        req = requests.get(url=CRAWL_TARGET_URL)
        req.encoding = req.apparent_encoding
        bf = BeautifulSoup(req.text, "html.parser")
        texts = bf.find_all('ul', id="hot-list", class_="list")
        bfs = BeautifulSoup(str(texts), "html.parser")
        spans = bfs.find_all('span', class_ = re.compile("icon-fall|icon-rise|icon-fair"))
        lis = bfs.find_all('a', class_ = "list-title")
        for i in range(10):
            vtup = (spans[i].get_text(), time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()))
            ktup = (lis[i].get('title'), lis[i].get('href'))
            if ktup in nd.keys():
                nd[ktup].append(vtup)
            else:
                nd[ktup] = [vtup]
        count = count+1
        if count%CRAWL_PER_UPDATE_TO_DB == 0:
            update_to_db(nd)
            nd.clear()
        time.sleep(SECONDS_PER_CRAWL)

def update_to_db(nd):
    db = pymysql.connect(host="localhost", user="root", port=3306, passwd=None, db="crawler", charset="utf8")
    cursor = db.cursor()
    for k in nd.keys():
        #将热点插入至主表
        sql1 = "REPLACE INTO HOTNEWS (INFORMATION, HYPERLINK) VALUES ( '%s', '%s');"
        #热点创建其分表
        sql2 = "CREATE TABLE IF NOT EXISTS `%s`( NUMBEROFPEOPLE INT NOT NULL, OCCURTIME DATETIME, PRIMARY KEY (OCCURTIME) );"
        try:
            cursor.execute(sql1%(k[0], k[1]))
            cursor.execute(sql2%(k[0]))
            db.commit()
        except:
            db.rollback()
        #将每个热点数据插入到对应热点分表
        for e in nd[k]:
            insert_sql = "INSERT INTO %s (NUMBEROFPEOPLE, OCCURTIME) VALUES (%d, '%s');"
            try:
                cursor.execute(insert_sql%(k[0], int(e[0]), e[1]))
            except:
                db.rollback()
    db.close()

if __name__ == '__main__':
    t = DataProducer()
    t.start()
    t.join()