Python+Requests+Xpath（解析）爬取某站点简历图片（数据分析三）

1、环境安装

pip install lxml

2、解析原理

• 使用通用爬虫爬取网页数据

• 实例化etree对象，且将页面数据加载到该对象中

• 使用xpath函数结合xpath表达式进行标签定位和指定数据提取

3、实战案例

- 项目需求：解析房天下新房的相关数据

import requests
import os
from lxml import etree
import json
import csv

if __name__ == '__main__':
    url = 'https://huizhou.newhouse.fang.com/house/s/'
    headers = {
        'User-Agent':'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.111 Safari/537.36'
    }
    if not os.path.exists('./fangtianxiaLibs'):
        os.makedirs('./fangtianxiaLibs')
    response = requests.get(url=url,headers=headers)
    # 手动设置响应数据的编码格式
    response.encoding = 'utf-8'
    page_text = response.text
    tree = etree.HTML(page_text)
    li_list = tree.xpath('//div[@id = "newhouse_loupai_list"]/ul/li')
    # 爬取的数据信息放到列表里面
    datas = []
    for li in li_list:
        # 解析标题
        try:
            detail_url = li.xpath('.//div[@class="nlcd_name"]/a/@href')[0]
            if detail_url != []:
                detail_url = 'https:'+detail_url
            detail_text = requests.get(url=detail_url,headers=headers).text
            # 字符串替换Url后缀
            detail_url_new = detail_url.replace('.htm','/housedetail.htm')
            tree = etree.HTML(detail_text)
            # 解析二级页面的描述和价格（均价）
            title = tree.xpath('//div[@class="information"]//div[@class="tit"]/h1/strong/text()')[0]
            price = "".join(tree.xpath('//div[@class="information_li mb5"]/div[@class="inf_left fl mr10"]/h3/text() | //div[@class="information_li mb5"]/div[@class="inf_left fl mr10"]/span/text() | //div[@class="information_li mb5"]/div[@class="inf_left fl mr10"]/text()')).strip('
 	')
            # 二级页面再次发起请求
            detail_text_new = requests.get(url=detail_url_new,headers=headers).text
            tree_new = etree.HTML(detail_text_new)
            # 解析详情页信息
            tree_list = tree_new.xpath('//div[@id="Configuration"]')
            # print(tree_list[0].xpath('./h3/text()'))

            for index in tree_list:
                zhoubian = "".join(index.xpath('./h3/text()')).strip('
 	 
 ')
                jiaotong = "".join(index.xpath('./ul[@class="sheshi_zb"]/li/span/text()|./ul[@class="sheshi_zb"]/li[@class="jiaotong_color"]/text()')).strip('
 	 
 ')
                qita = "".join(index.xpath('./ul[@class="sheshi_zb"]/li/span/text()|./ul[@class="sheshi_zb"]/li/text()')).strip('
 	 
 ')
            desc = zhoubian+":"+jiaotong+':'+qita+'
'
            dic = {
                'title':title,
                'desc':desc,
                'price':price
            }
            datas.append(dic)
        except Exception as msg:
            pass
            # print('报错原因:{}'.format(msg))
    fileName = './fangtianxiaLibs/'+title+'.txt'
    print(datas)
    title_header = ['title','desc','price']
    with open(fileName,'a',encoding='utf-8') as fp:
        writer = csv.DictWriter(fp,title_header)
        writer.writeheader()
        writer.writerows(datas)

- 项目需求：解析图片数据：http://pic.netbian.com/4kmeinv/

import requests
from lxml import etree 
url = 'http://pic.netbian.com/4kmeinv/'
headers = {
    'User-Agent':'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36'
}
response = requests.get(url=url,headers=headers)
#获取页面原始编码格式
print(response.encoding)
page_text = response.text
tree = etree.HTML(page_text)
li_list = tree.xpath('//div[@class="slist"]/ul/li')
for li in li_list:
    img_url = 'http://pic.netbian.com'+li.xpath('./a/img/@src')[0]
    img_name = li.xpath('./a/img/@alt')[0]
    img_name = img_name.encode('iso-8859-1').decode('gbk')
    print(img_url,img_name)

- 项目需求：解析出所有城市名称https://www.aqistudy.cn/historydata/

import requests
from lxml import etree 
url = 'https://www.aqistudy.cn/historydata/'
headers = {
    'User-Agent':'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36'
}
response = requests.get(url=url,headers=headers)
#获取页面原始编码格式
print(response.encoding)
page_text = response.text
tree = etree.HTML(page_text)
li_list = tree.xpath('//div[@class="bottom"]/ul/li | //div[@class="bottom"]/ul//li')
for li in li_list:
    city_name = li.xpath('./a/text()')[0]
    city_url = 'https://www.aqistudy.cn/historydata/'+li.xpath('./a/@href')[0]
    print(city_name,city_url)

- 项目需求：下载网站站点简历中的图片数据：https://sc.chinaz.com/

import requests
from lxml import etree
import os

# 新建文件夹
if not os.path.exists('./jianliLibs'):
    os.makedirs('./jianliLibs')

# 站点第一层 进入简历门户站点
url = 'https://sc.chinaz.com/'
headers = {
    'User-Agent':'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.111 Safari/537.36'
}
response_text = requests.get(url=url,headers=headers).text
# 解析获取模板信息
tree = etree.HTML(response_text)
# 解析出简历模板Url
def page_index(latest):
    for index in range(1,latest):
        if index == 1:
            muban_url = 'https://sc.chinaz.com' + tree.xpath('//div[@class="nav"]//li[@class="nos no3"]/a/@href')[3]
            # print("one",muban_url)
        else:
            muban_url = 'https://sc.chinaz.com' + tree.xpath('//div[@class="nav"]//li[@class="nos no3"]/a/@href')[3] + 'index_{}.html'.format(index)
            # print("two",muban_url)
        # 模板简历站点获取每个简历的信息
        response = requests.get(muban_url,headers=headers)
         # 手动设置响应数据的编码格式
        response.encoding = 'utf-8'
        muban_text = response.text
        # print(muban_text)
        # 解析获取简历信息
        jianli_tree = etree.HTML(muban_text)
        # 解析出简历信息的Url
        jianli_url_list = jianli_tree.xpath('//div[@class="main_list jl_main"]//a/@href')
        # print(jianli_url_list)
        for jianli_url in jianli_url_list:
            jianli_url = "https:"+jianli_url
            # print(jianli_url)
            # 第三层获取简历信息
            jianli_detail = requests.get(jianli_url,headers=headers).text
            detail_tree = etree.HTML(jianli_detail)
            img_src_list = detail_tree.xpath('//div[@class="show_warp jl_warp clearfix"]//img/@src')
            for img_src in img_src_list:
                img_src = 'https:'+img_src
                # print(img_src)
                img_src_content = requests.get(img_src,headers=headers).content
                # print(img_src_content)
                # 生成图片的名称
                imgName = img_src.split('/')[-2]
                # print(imgName)
                # 图片路径
                imgPath = './jianliLibs/'+imgName+'.jpg'
                # 持久化存储
                with open(imgPath, 'wb') as fp:
                    fp.write(img_src_content)
                print('简历:'+imgName, '下载成功!!!')

if __name__ == '__main__':
    while True:
        try:
            values = int(input('请输入站点页分页数:'))
            page_index(values)
        except Exception as msg:
            print('输入错误,错误信息为{}'.format(msg))
        finally:
            break