python3+beautifulSoup4.6抓取某网站小说（四）多线程抓取

上一篇多文章，是二级目录，根目录“小说”，二级目录“作品名称”，之后就是小说文件。

本篇改造了部分代码，将目录设置为根目录->作者目录->作品目录->作品章节.txt.

但这并不是本章内容当重点，重点是使用这个爬虫程序抓取当时候，经常会因为网络丢包等原因导致程序中断，

本来想着是循环获取网站状态，然后重新发起请求，结果好像也没什么用。然后在虫师讲selenium的书中看到了多线程，正好就实验下，结果发现，速度很快，cool！

以下代码基本摘自虫师的selenium2

多线程的引用

import threading

方法调用：threading.Thread(target=music, args=('music方法参数1',music方法参数2) )

from time import sleep,ctime
import threading

def music(func,loop):
    for i in range(loop):
        print('music',func,ctime())
        sleep(2)

def movie(func,loop):
    for i in range(loop):
        print('movie',func,ctime())
        sleep(4)

def testOne():
    music('简单的歌', 2)
    movie('两杆大烟枪', 2)
    print('all end', ctime())

def testTwo():
    threads = []
    t1 = threading.Thread(target=music, args=('喜欢的人',2) )
    threads.append(t1)

    t2 = threading.Thread(target=movie, args=('搏击俱乐部',2) )
    threads.append(t2)

    t3= threading.Thread(target=music, args=('喜欢的人2', 2))
    threads.append(t3)

    for t in threads:
        t.start()

    for t in threads:
        t.join()

    print('all end', ctime())

if __name__ == '__main__':
    testOne()
    #testTwo()
    #testThree()
    #threadsRun()

t.join方法用来串联线程，可以保证all end 语句在最后打印出来。

　　

创建线程管理类

创建类名时就引入Thread：class MyThread(threading.Thread)

class MyThread(threading.Thread):

    def __init__(self, func, args, name):
        threading.Thread.__init__(self)
        self.func = func
        self.args = args
        self.name = name

    def run(self):
        self.func(*self.args)

　self：类实例，默认参数

　func：调用方法名

   args：参数

   name：方法+".__name__"

完整代码：

class MyThread(threading.Thread):

    def __init__(self, func, args, name):
        threading.Thread.__init__(self)
        self.func = func
        self.args = args
        self.name = name

    def run(self):
        self.func(*self.args)

def super_play(file_,time):
    for i in range(3):
        print('play', file_, ctime())
        sleep(time)


def time(args):
    pass


def testThree():
    threads = []
    lists = {'气球.mp3': 3, '电影.rmvb': 4, 'last.avg' : 2}
    for file_, time_ in lists.items():
        t = MyThread(super_play, (file_, time_), super_play.__name__)
        threads.append(t)

    files = range(len(lists))

    for f in files:
        threads[f].start()
    for f in files:
        threads[f].join()

    print('all end', ctime())

改造小说爬虫

好了，多线程说完了，怎么调用咱们写的小说类呢，很简单

首先，改造pageOne

    def readPageOneByThread(self,page,time_):
        page_url = str(self.two_page_url)
        new_page_url = page_url.replace("?", page)
        print('第', page, '页---', new_page_url)
        path = self.folder_path              
        self.readPageTwo(new_page_url, path)
        sleep(time_)
    # end readPageOneByThread  ---------------------------------------

　init方法中，self.two_page_url = "http://www.cuiweijuxs.com/jingpinxiaoshuo/5_?.html"

接下来，编写添加线程的方法：

    def threadsRun(self):

        #self.readPageOne(122)

        for i in range(1,123):
            page = str(i)
            t = MyThread( self.readPageOneByThread, (page,2) , self.readPageOneByThread.__name__)
            #t = threading.Thread(target=self.testRun, args=( str(i) ))
            self.threads.append(t)

        for t in self.threads:
            t.start()
        for t in self.threads:
            t.join()
            #t.join()

        print('all end: %s' % ctime())


class MyThread(threading.Thread):

    def __init__(self, func, args, name):
        threading.Thread.__init__(self)
        self.func = func
        self.args = args
        self.name = name

    def run(self):
        self.func(*self.args)

　　这里偷了个懒，直接写了总页数，其实也可以使用原来的pageone方法读取last的div获取页数

下面是完整代码：

# -*- coding: UTF-8 -*-
from urllib import request
from bs4 import BeautifulSoup
from time import sleep,ctime
import os
import threading
import re
import random

'''
使用BeautifulSoup抓取网页
version:0.5 更新为本地缓存链接
author:yaowei
date:2018-03-23
'''


class Capture():

    def __init__(self):
        self.index_page_url = 'http://www.cuiweijuxs.com/'
        self.one_page_url = 'http://www.cuiweijuxs.com/jingpinxiaoshuo/'
        self.two_page_url = "http://www.cuiweijuxs.com/jingpinxiaoshuo/5_?.html"
        self.folder_path = '绯色/'
        self.href_list = []
        self.head = {}
        self.threads = []
        # 写入User Agent信息
        self.head[
            'User-Agent'] = 'Mozilla/5.0 (Linux; Android 4.1.1; Nexus 7 Build/JRO03D) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.166  Safari/535.19'

    # end __init__ ---------------------------------------

    # 获取BeautifulSoup
    def getSoup(self,query_url):
        req = request.Request(query_url, headers=self.head)
        webpage = request.urlopen(req)
        html = webpage.read()
        soup = BeautifulSoup(html, 'html.parser')
        return soup
        # soup = BeautifulSoup(html, 'html5lib')

    # 读取分版页面，打开分页链接
    def readPageOne(self,count,time_):

        print('count=====',count)

        # 总页数
        if count :
            item_size = count
        else :
            # 读取页面
            soup = self.getSoup(self.one_page_url)
            last = soup.find("a", 'last')
            item_size = int(last.string)

        print('item_size=====',item_size)
        page_url = str(self.two_page_url)

        # 循环打开分页链接，读取分页页面
        for item in range(item_size):
            page = str(item + 1)
            new_page_url = page_url.replace("?", page)
            print('第', page, '页---', new_page_url)
            path = self.folder_path
            self.readPageTwo(new_page_url, path)

        sleep(time_)
    # end readPageOne  ---------------------------------------

    def readPageOneByThread(self,page,time_):
        page_url = str(self.two_page_url)
        new_page_url = page_url.replace("?", page)
        print('第', page, '页---', new_page_url)
        path = self.folder_path              
        self.readPageTwo(new_page_url, path)
        sleep(time_)
    # end readPageOneByThread  ---------------------------------------

    # 读取分页页面
    def readPageTwo(self, page_url, path):
        soup = self.getSoup(page_url)
        # first div[id="newscontent"]->div[class="l"]
        con_div = soup.find('div', {'id': 'newscontent'}).find('div', {'class': 'l'})
        # first div[id="newscontent"]->div[class="l"]->all spann[class="s2"]
        span_list = con_div.find_all('span', {'class': 's2'})

        # 遍历span
        for span in span_list:
            # 找到父节点下的span[class="s5"]，以作者为文件夹名字
            author = span.parent.find('span', {'class': 's5'}).get_text()

            # span[class="s2"]->a
            a_href = span.find('a')
            href = a_href.get('href')  # 单部作品链接
            folder_name = a_href.get_text()  # 作品名字
            print('a_href', href, '---folder_name', folder_name)
            new_path = path + '/' + author + '/' + folder_name
            self.createFolder(new_path)  # 创建文件夹

            self.readPageThree(href, new_path)  # 读取单部作品

            # t = threading.Thread(target=self.readPageThree, args={href, new_path})
            # self.threads.append(t)
            # end for

    # end readPage  ---------------------------------------

    # 打开作品链接，遍历单章
    def readPageThree(self, page_url, path):
        soup = self.getSoup(page_url)  # 作品页面
        print('readPageThree--', page_url)
        a_list = soup.find('div', {'id': 'list'}).find_all('a')
        idx = 0  # 序号
        for a_href in a_list:
            idx = idx + 1
            href = self.index_page_url + a_href.get('href')
            file_path = path + '/' + str(idx) + '_' + a_href.get_text() + '.txt'
            print('file_a_href', href, '---file_path', file_path)

            '''
            new_path = self.isTxt(file_path)
            if new_path:
                print(new_path)
                file_object = open('网页链接//hrefs.txt', 'w', encoding='utf-8')
                file_object.write(href+','+new_path)
                file_object.close()
             '''
            self.readPageFour(href, file_path)

            #self.href_list.append({'href': href, 'file_path': file_path})

            # 多线程
            #t = threading.Thread(target=self.readPageFour, args={href, file_path})
            #t.start()
            #t.join(15)

    # end readPageThree  ---------------------------------------

    # 读取单章内容并写入
    def readPageFour(self, page_url, path):
        new_path = self.isTxt(path)  # 是否存在，存在则返回'',没创建则返回合法文件名
        if new_path:
            soup = self.getSoup(page_url)
            con_div = soup.find('div', {'id': 'content'})  # 读取文本内容
            content = con_div.get_text().replace('<br/>', '
').replace('&nbsp;', ' ')
            # content = content.replace('&amp;','').replace('amp;','').replace('rdquo;','').replace('ldquo;','')
            # content = content.rstrip("& amp;rdquo;amp;& amp;ldquo;")

            self.writeTxt(new_path, content)  # 写入文件

    # end readPageFour  ---------------------------------------

    def readPageHtml(self, page_url, path):
        soup = self.getSoup(page_url)
        con_div = soup.find('div', {'id': 'content'})
        content = con_div.get_text().replace('<br/>', '
').replace('&nbsp;', ' ')

    def createFolder(self, path):
        path = path.strip()
        # 去除尾部  符号
        path = path.rstrip("\")
        rstr = r"[:*?"<>|]"  # '/  : * ? " < > |'
        new_path = re.sub(rstr, "_", path)  # 替换为下划线
        is_exists = os.path.exists(new_path)
        # 不存在则创建
        if not is_exists:
            os.makedirs(new_path)
            print('目录:', new_path + ' create')
        else:
            print(new_path + ' 目录已存在')

    # end createFolder  ---------------------------------------

    def isTxt(self, path):
        path = path.strip()
        # 去除尾部  符号
        path = path.rstrip("\")
        rstr = r"[:*?"<>|]"  # '/  : * ? " < > |'
        new_path = re.sub(rstr, "_", path)  # 替换为下划线
        isExists = os.path.exists(new_path)
        if isExists:
            print(new_path, '已存在')
            return ''
        else:
            return new_path

    # end createTxt ---------------------------------------

    def writeTxt(self, file_name, content):
        isExists = os.path.exists(file_name)
        if isExists:
            print(file_name, '已存在')
        else:
            file_object = open(file_name, 'w', encoding='utf-8')
            file_object.write(content)
            file_object.close()

    # end writeTxt ------------------------------------------

    def run(self):
        try:
            self.readPageOne()
        except BaseException as error:
            print('error--', error)

    def runTest(self):
        try:
            page_url = 'http://www.cuiweijuxs.com/4_4508/'
            path = '小说/runTest'
            self.readPageThree(page_url, path)
        except BaseException as error:
            print('error--', error)

    def testRun(self,num,time_):
        for i in range(3):
            print('num=',num,ctime())
            sleep(time_)
            
    def threadsRun(self):

        #self.readPageOne(122)

        for i in range(1,123):
            page = str(i)
            t = MyThread( self.readPageOneByThread, (page,2) , self.readPageOneByThread.__name__)
            #t = threading.Thread(target=self.testRun, args=( str(i) ))
            self.threads.append(t)

        for t in self.threads:
            t.start()
        for t in self.threads:
            t.join()
            #t.join()

        print('all end: %s' % ctime())


class MyThread(threading.Thread):

    def __init__(self, func, args, name):
        threading.Thread.__init__(self)
        self.func = func
        self.args = args
        self.name = name

    def run(self):
        self.func(*self.args)


Capture().threadsRun()