爬取笔下wenxue小说

import urllib.request
from bs4 import BeautifulSoup
import  re

def gethtml(url):
    page=urllib.request.urlopen(url)
    html=page.read().decode('gbk')
    soup=BeautifulSoup(html,"html.parser")
    #print(soup)
    return soup


def getcontent(soup,load):                            #获取章节内容以及章节名称
    content1=""
    content=re.findall(r'<div id="content"><div id="adright"></div>(.*?)</div>',str(soup))
    for i in range(0,len(content)):
        content1+=content[i]
    content2 = re.sub("</?w+[^>]*>", "", content1)
    content3=content2.replace('。','。

')
    #以上获取章节内容
    zjname = re.findall(r'<div id="title">(.*?)</div>', str(soup))
    #获取章节名称

    with open(load, 'a', encoding='utf-8') as f:
        f.write("-----------------------------------------------------------"+zjname[0]+"------------------------------------------------------

"+content3)

def book(soup):
    bookurl=re.findall(r'<td class="odd"><a href="(.*?)">',str(soup))      #get every book url

    for i in range(0,len(bookurl)):
        #找到“[点击阅读]按钮”的链接,以及书名
        print(bookurl[i])
        soup1=gethtml(bookurl[i])
        allcontent=re.findall(r'</a>xa0xa0xa0xa0<a href="(.*?)">',str(soup1))
        bookname=re.findall(r'<strong>(.*?)全集下载</strong>', str(soup1))

        soup2=gethtml(allcontent[0])
        #打开点击阅读的按钮链接，找到第一章的链接
        firsturl1 = re.findall(r'<dd><a href="(.*?)">.*?</a></dd>', str(soup2))
        headurl=bookurl[i][0:-4].replace("binfo","b")
        firsturl2=headurl+"/"+firsturl1[0]
        print(firsturl2)
        
        #打开链接，开始爬取内容,同时获取下一章内容，并判断是否到最后一章
        soup3=gethtml(firsturl2)
        k=0
        load="d:/77/%s.txt" % bookname[0]
        try:
            
            while True:
                nexturl = re.findall(r'<li><a href="(.*?)">下一页', str(soup3))
                getcontent(soup3,load)
                soup3=gethtml(headurl+"/"+nexturl[0])
                k+=1
                print("第%d章下载完成" % int(k))
        except:
            
            print("-------------------第%d本书下载完成---------------" % int(i+1))

url="http://www.bxwx9.org/modules/article/toplist.php?sort=dayvisit"
soup=gethtml(url)
load=book(soup)

保存到txt文件中

2017-05-28

22:58:35