python获取小说内容

在使用前要安装python的第3方库，BeautifulSoup，pymysql

 代码里面用了mysql数据库

 代码里面获取小说网站地址是：http://www.kbiquge.com

 mysql里面的表结构：

CREATE TABLE `story` (
  `id` varchar(200) NOT NULL DEFAULT '',
  `name` varchar(200) DEFAULT NULL COMMENT '名称',
  `start` varchar(20) DEFAULT NULL COMMENT '状态',
  `end_start` varchar(200) DEFAULT NULL COMMENT '更新时间',
  `author` varchar(200) DEFAULT NULL COMMENT '作者',
  PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;


CREATE TABLE `chapter` (
  `chapter_id` varchar(200) NOT NULL DEFAULT '0' COMMENT '章节ID',
  `story_id` varchar(200) DEFAULT NULL COMMENT '小说ID',
  `chapter_name` varchar(200) DEFAULT NULL COMMENT '章节名称',
  `chapter_content` mediumtext COMMENT '内容',
  `chapter_href` varchar(2000) DEFAULT NULL COMMENT 'URL',
  PRIMARY KEY (`chapter_id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;

以下是源码：

#coding=utf-8
import pymysql
import time
import datetime
import uuid


from urllib import request
from bs4 import BeautifulSoup


#数据存入章节表中 批量提价数据， usersvalues[] 包含chapter_id，story_id，chapter_name，chapter_content,chapter_href
def Write_info(usersvalues):
    db = pymysql.connect("localhost","root","123456","python" )
    cursor = db.cursor()
    try:
        sql = "INSERT  INTO chapter(chapter_id,story_id,chapter_name,chapter_content,chapter_href) 
          VALUES(%s,%s,%s,%s,%s)"
        # 执行sql语句 批量插入数据
        cursor.executemany(sql, usersvalues)
        db.commit()
    except ZeroDivisionError:
        print ("Error: unable to fetch data")
        db.rollback()
    db.close()

#小说名称 story_name
def Story_name(story_name):
    db = pymysql.connect("localhost","root","123456","python" )
    uuids=str(uuid.uuid1()).replace('-','')
    cursor = db.cursor()
    try:
        cursor.execute("select id from story  where name='"+story_name+"'")
        fname=""
        results = cursor.fetchall()
        for row in results:
            fname= row[0]
        if cursor.rowcount!=1:
            sql = """INSERT INTO STORY(id,name, start, end_start,author) 
             VALUES ('"""+uuids+"""', '"""+story_name+"""', '1', '1', 'wangyh')"""
            cursor.execute(sql)
            db.commit()
            return uuids
        else:
            return fname
    except ZeroDivisionError:
        print ("Error: unable to fetch data")
        db.rollback()
    db.close()


if __name__ == '__main__':
    # 目录页
    url_xs='http://www.kbiquge.com'
    url = url_xs+'/86_86683/'
    head = {}
    head['User-Agent'] = 'Mozilla/5.0 (Linux; Android 4.1.1; Nexus 7 Build/JRO03D) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.166  Safari/535.19'
    req = request.Request(url, headers = head)
    response = request.urlopen(req)
    html = response.read()
    # 解析目录页
    soup = BeautifulSoup(html, 'lxml')
    #小说名称 id="info"
    story_name = soup.find('div', id = 'info').find("h1").text
    #查询是否存入 story表中 story_id 小说ID
    story_id= Story_name(story_name)
    print("story_id:"+story_id)
    # find_next找到第二个<div> 小说目录
    soup_texts = soup.find('div', id = 'list')
    usersvalues=[]
    # 遍历ol的子节点，打印出章节标题和对应的链接地址
    for link in soup_texts.dl.children:
        if link != '
':
            print('start')
            list_tmp=link.find_all('a')
            for a in list_tmp:
                #0.5秒
                time.sleep(0.5)
                download_url = url_xs+a.get('href')
                download_req = request.Request(download_url, headers = head)
                download_response = request.urlopen(download_req)
                download_html = download_response.read()
                download_soup = BeautifulSoup(download_html, 'lxml')
                download_soup_texts = download_soup.find('div', id = 'content')
                download_soup_texts = download_soup_texts.text
                download_soup_texts= download_soup_texts.replace(u'xa0', u' ')
                uuids="w"+str(int(round(time.time() * 1000)))
                data=(uuids,story_id,a.text,download_soup_texts,download_url)
                usersvalues.append(data)
    Write_info(usersvalues)