Xpath数据解析

好段子网内容爬取

from lxml import etree
import requests
url='http://www.haoduanzi.com'
headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.139 Safari/537.36',
    }
url_content=requests.get(url=url,headers=headers).text
tree=etree.HTML(url_content)
#xpath返回值是一个列表
div_list=tree.xpath('//div[@id="main"]/div')[2:-2]
ur_list=[]
for div in div_list:
    img_url=div.xpath('./div/img/@src')[0]
    ur_list.append(img_url)
print(ur_list)

煎蛋网图片爬取

from lxml import etree
import requests
import base64
url="http://jandan.net/ooxx"
headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.139 Safari/537.36',
    }
url_content=requests.get(url=url,headers=headers).text
# url_content
tree=etree.HTML(url_content)
img_list=tree.xpath('//span[@class="img-hash"]/text()')
img_so_list=[]
for imgcode in img_list:
    img_url="http:"+base64.b64decode(imgcode).decode()
    img_so_list.append(img_url)
img_so_list

站长素材中进行免费简历模板的下载

from lxml import etree
import requests
import random
url = 'http://sc.chinaz.com/jianli/free_%d.html'
headers = {
    'User-Agent':'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36'
    'Connection':'close'
}

for pageNum in range(1,3):
    pageUrl = format(url%pageNum)
    if pageNum == 1:
        pageUrl = 'http://sc.chinaz.com/jianli/free.html'
    #page_text = requests.get(url=pageUrl,headers=headers).text
    response = requests.get(url=pageUrl,headers=headers)
    #将请求到的页面源码数据的编码格式进行指定的修改
    response.encoding = 'utf-8'
    page_text = response.text
    #解析：简历的名称 详情页的url
    tree = etree.HTML(page_text)
    div_list = tree.xpath('//div[@id="container"]/div')
    for div in div_list:
        #存储谋一份建立的12个下载通道
        download_list = []
        
        detail_url = div.xpath('./p/a/@href')[0]
        title = div.xpath('./p/a/text()')[0]
        
        #获取详情页面的页面数据
        detail_text = requests.get(url=detail_url,headers=headers).text
        #一个etree对象只可以加载一个页面源码数据
        tree = etree.HTML(detail_text)
        #12个下载通道的url
        li_list =  tree.xpath('//div[@class="clearfix mt20 downlist"]/ul[@class="clearfix"]/li')
        for li in li_list:
            download_url = li.xpath('./a/@href')[0]
            download_list.append(download_url)
        #从12个下载通道中随机选择某一个下载通道    
        download_data_url = random.choice(download_list)
        #简历模板的下载
        data = requests.get(url=download_data_url,headers=headers).content
        data_path = title+'.rar'
        with open(data_path,'wb') as fp:
            fp.write(data)
            print(data_path+'下载成功')