• 使用beautifulsoup和pyquery爬小说


    
    

    1. 

    # -*- coding:UTF-8 -*-
    from bs4 import BeautifulSoup    #BeautifulSoup就是处理字符串的工具
    import requests, sys
    
    """
    类说明:下载《笔趣看》网小说《一念永恒》
    Parameters:
        无
    Returns:
        无
    Modify:
        2017-09-13
    代码出处
        https://zhuanlan.zhihu.com/p/29809609 
    """
    class downloader(object):
    
        def __init__(self):
            self.server = 'http://www.biqukan.com/'
            self.target = 'http://www.biqukan.com/1_1094/'
            self.names = []            #存放章节名
            self.urls = []            #存放章节链接
            self.nums = 0            #章节数
    
        """
        函数说明:获取下载链接
        Parameters:
            无
        Returns:
            无
        Modify:
            2017-09-13
        """
        def get_download_url(self):
            req = requests.get(url = self.target)
            html = req.text
            div_bf = BeautifulSoup(html)
            div = div_bf.find_all('div', class_ = 'listmain')
            a_bf = BeautifulSoup(str(div[0]))
            a = a_bf.find_all('a')
            self.nums = len(a[15:])                                #剔除不必要的章节,并统计章节数
            for each in a[15:]:
                self.names.append(each.string)
                self.urls.append(self.server + each.get('href'))
    
        """
        函数说明:获取章节内容
        Parameters:
            target - 下载连接(string)
        Returns:
            texts - 章节内容(string)
        Modify:
            2017-09-13
        """
        def get_contents(self, target):
            req = requests.get(url = target)
            html = req.text
            bf = BeautifulSoup(html)
            texts = bf.find_all('div', class_ = 'showtxt')
            texts = texts[0].text.replace('xa0'*8,'
    
    ')  # .text去掉非文字的符号,是bs4的标签属性 .text属性包含了标签内的子标签的内容
            return texts
    
        """
        函数说明:将爬取的文章内容写入文件
        Parameters:
            name - 章节名称(string)
            path - 当前路径下,小说保存名称(string)
            text - 章节内容(string)
        Returns:
            无
        Modify:
            2017-09-13
        """
        def writer(self, name, path, text):
            write_flag = True
            with open(path, 'a', encoding='utf-8') as f:
                f.write(name + '
    ')
                f.writelines(text) #file.writelines(sequence)的参数是序列,比如列表,它会迭代帮你写入文件。
                f.write('
    
    ')  #file.write(str)的参数是一个字符串,就是你要写入文件的内容
    
    if __name__ == "__main__":
        dl = downloader()
        dl.get_download_url()
        print('《一年永恒》开始下载:')
        for i in range(dl.nums):
            dl.writer(dl.names[i], '一念永恒.txt', dl.get_contents(dl.urls[i]))
            sys.stdout.write("  已下载:%.3f%%" %  float(i/dl.nums) + '
    ')    #其中%.3f表示精确到3位, 后面的2个%表示字符”%“,在3位小数后面会显示出来一个%   '
    '表示换行符
            sys.stdout.flush()    #刷新stdout缓存,这样会实时的显示输出。https://zhidao.baidu.com/question/575650032.html
        print('《一年永恒》下载完成')

    2. 用pquery改写代码如下,增加了可以定量下载小说的功能,并且在动态输出进程改写为 类似 50%的格式,而不是0.50 %,

    # 爬取笔趣阁小说
    # encoding: utf-8
    from pyquery import PyQuery as pq
    import sys
    
    class downloader():
    
    
        def __init__(self):
            self.base_server = 'http://www.biqukan.com'
            self.base_url = 'http://www.biqukan.com/1_1094/'
            self.url_list = []
            self.full_urls = []
            self.nums = 0
            self.wanted_pages_num = 0
    
    
        def get_url(self): 
            doc = pq(self.base_url)
            target_a = doc('dd a').items()
            for each_a in target_a:
                each_url = each_a.attr.href
                self.url_list.append(each_url)
            self.nums= len(self.url_list)
            for each in self.url_list[15:]:
                each_full_url = self.base_server + each
                self.full_urls.append(each_full_url)
    
    
    
        def get_content(self,target):
            content = pq(target)
            text = content('#content').text()
            head = content('h1').text()
            full_text = (head+'
    '*2+text)
            return full_text
    
    
    
        
        def writer(self,path,text):
            with open(path, 'a') as f:
                f.write(text+'
    ')
    
    
    
        def choose_page(self):
            begin = input('please input your starting page: ')
            end = input('please input your ending page: ')
            self.wanted_pages_num = int(end)-int(begin)
            return self.wanted_pages_num
            
    
    
    
    
    if __name__ == "__main__":
        dl = downloader()
        dl.get_url()
        print("总章节数: ", dl.nums)
    
        number = dl.choose_page()
        for i in range(number1):
            each_url = dl.full_urls[i]
            each_text = dl.get_content(each_url)
            dl.writer('321.txt',each_text)
            
    
            x = (i / number * 100)
            sys.stdout.write("已下载:%.2f %% " % x + '
    ')
            sys.stdout.flush()
  • 相关阅读:
    让程序的性能提升10倍
    以正确的姿势实现一棵JavaScript菜单树
    利用php数组函数进行函数式编程
    yum安装下的nginx,如何添加模块,和添加第三方模块
    前端面试题 vue
    前端常用面试题目及答案-HTML&CSS篇
    使用after伪类,配合IE的zoom或者overflow清除浮动
    函数式编程
    在CentOS 7上安装Node.js
    node中__dirname、__filename、process.cwd()、process.chdir()表示的路径
  • 原文地址:https://www.cnblogs.com/regit/p/8529222.html
Copyright © 2020-2023  润新知