爬虫爬取博客园文章的文字【练手】

注意：只是文字，其他都行，自己实现吧。

import requests
from lxml import etree
from urllib.request import urlopen, Request

import time
class blog():

    def __init__(self,url):
        self.base_url = url
        self.headers = {"User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.86 Safari/537.36"}#模拟浏览器

    def get_html(self,url):
        response = requests.get(url, self.headers, timeout = 5)#因为请求方式位get所以，用get请求
        if response.status_code == 200:#访问正常状态码为200
            response.encoding = response.apparent_encoding
            return response.text
        return None
    def get_url1(self,html):#获取每个分页内的文章标题和url
        url_1 = []
        name = []
        x_html = etree.HTML(html)
        #print(x_html)
        url_1 = x_html.xpath('//a[@class="postTitle2"]/@href')
        name = x_html.xpath('//a[@class="postTitle2"]/text()')
        names = []

        for i in name:
            names.append(i.strip())#去除空格
        if names:
            return names, url_1
        else:
            return None
    def get_url2(self,s):

        url = self.base_url + "default.html?page=" + s
        return url
    def get_text(self,html):
        x_html = etree.HTML(html)
        txt = []

        txt = x_html.xpath('//div[@id="cnblogs_post_body"]//p//text()')
        return txt
    def save_text(self,name,txt):#保存成txt文件，txt是文章内容
        print(name + "loading...")
        with open("C:/Users/25766/Desktop/sa/" + name + '.txt','w',encoding='utf-8') as f:#文件路径
            for i in txt:
                f.write(i)
                f.write('
')
            f.close()
        print("finsh")
surl = input("输入爬取博客的首页url:")
c = blog(surl)
for i in range(1,200):#获取其各个分页，通过规律发现
    url = c.get_url2(str(i))
    ls = c.get_html(url)
    #print(ls)
    time.sleep(2)
    names, urls = c.get_url1(ls)
    for name, url in zip(names,urls):
        #print(name,":", url)
        html = c.get_html(url)
        txt = c.get_text(html)
        c.save_text(name[:5],txt)#以前5个字符命名