xpath练习(链家二手房案例,百度贴吧图片抓取案例)

链家二手房案例(xpath)

实现步骤

　　1.确定是否为静态

　　　　打开二手房页面 -> 查看网页源码 -> 搜索关键字

　　2.xpath表达式

1、基准xpath表达式(匹配每个房源信息节点列表)
  //ul[@class="sellListContent"]/li[@class="clear LOGVIEWDATA LOGCLICKDATA"]

2、依次遍历后每个房源信息xpath表达式
   * 名称: './/a[@data-el="region"]/text()'
   
   # 户型+面积+方位+是否精装
   info_list = './/div[@class="houseInfo"]/text()'  [0].strip().split('|')
   * 户型(model): info_list[1]
   * 面积(area): info_list[2]
   * 方位(direction): info_list[3]
   * 精装(perfect): info_list[4]
   

   * 楼层(floor): './/div[@class="positionInfo"]/text()'
   * 区域(address): './/div[@class="positionInfo"]/a/text()'
   * 总价(total_price): './/div[@class="totalPrice"]/span/text()'
   * 单价(unit_price): './/div[@class="unitPrice"]/span/text()'

　　3.实现代码

import requests
from lxml import etree
import time
import random

class LianjiaSpider(object):
  def __init__(self):
    self.url = 'https://bj.lianjia.com/ershoufang/pg{}/'
    self.headers = {'User-Agent' : 'Mozilla/5.0'}

  def get_page(self,url):
    try:
        # 设定超时时间,超时后抛出异常,被except捕捉,继续执行此函数再次请求
        res = requests.get(url,headers=self.headers,timeout=5)
        res.encoding = 'utf-8'
        html = res.text
        self.parse_page(html)
    except Exception as e:
        self.get_page(url)

  def parse_page(self,html):
    parse_html = etree.HTML(html)
    # 基准xpath,匹配每个房源信息的节点对象
    li_list = parse_html.xpath('//ul[@class="sellListContent"]/li[@class="clear LOGVIEWDATA LOGCLICKDATA"]')
    # 定义空字典,用来存储抓取的最终数据
    house_dict = {}
    # 遍历依次匹配每个房源信息,获取所有所需数据
    for li in li_list:
      # 房源名称
      name_list = li.xpath('.//a[@data-el="region"]/text()')
      house_dict['house_name'] = [ name_list[0] if name_list else None ][0]

      # 列表：户型+面积+方位+是否精装
      info_list = li.xpath('.//div[@class="houseInfo"]/text()')
      house_info = [ info_list[0].strip().split('|') if info_list else None ][0]
      if house_info:
          # 户型
          house_dict['house_model'] = house_info[1]
          # 面积
          house_dict['area'] = house_info[2]
          # 方位
          house_dict['direction'] = house_info[3]
          # 是否精装
          house_dict['hardcover'] = house_info[4]
      ###########################################
      # 楼层
      floor_list = li.xpath('.//div[@class="positionInfo"]/text()')
      house_dict['floor'] = [ floor_list[0].strip()[:-2] if floor_list else None ][0]
      # 区域
      address_list = li.xpath('.//div[@class="positionInfo"]/a/text()')
      house_dict['address'] = [ address_list[0].strip() if address_list else None ][0]
      # 总价
      total_list = li.xpath('.//div[@class="totalPrice"]/span/text()')
      house_dict['total_price'] = [ total_list[0].strip() if total_list else None ][0]
      # 单价
      unit_list = li.xpath('.//div[@class="unitPrice"]/span/text()')
      house_dict['unit_price'] = [ unit_list[0].strip() if unit_list else None ][0]

      print(house_dict)

  def main(self):
    for pg in range(1,11):
      url = self.url.format(str(pg))
      self.get_page(url)
      print('第%d页爬取成功' % pg)
      time.sleep(random.randint(1,3))

if __name__ == '__main__':
  start = time.time()
  spider = LianjiaSpider()
  spider.main()
  end = time.time()
  print('执行时间:%.2f' % (end-start))

百度贴吧图片抓取

目标:抓取指定贴吧所有图片

思路:

1、获取贴吧主页URL,下一页,找到不同页的URL规律
2、获取1页中所有帖子URL地址: [帖子链接1,帖子链接2,...]
3、对每个帖子链接发请求,获取图片URL
4、向图片的URL发请求,以wb方式写入本地文件

实现步骤:

　　1.贴吧url规律

http://tieba.baidu.com/f?kw=??&pn=50

　　2.xpath表达式

1、帖子链接xpath
   //div[@class="t_con cleafix"]/div/div/div/a/@href
    
2、图片链接xpath
   //div[@class="d_post_content j_d_post_content  clearfix"]/img[@class="BDE_Image"]/@src
    
3、视频链接xpath
   //div[@class="video_src_wrapper"]/embed/@data-video
   # 注意: 此处视频链接前端对响应内容做了处理,需要查看网页源代码来查看，复制HTML代码在线格式化

　　3.代码实现

import requests
from urllib import parse
from lxml import etree
import time 
import random

class BaiduImgSpider(object):
  def __init__(self):
    self.url = 'http://tieba.baidu.com/f?{}'
    self.headers = {'User-Agent':'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; InfoPath.3)'}

  # 获取html函数
  def get_html(self,url):
      try:
          res = requests.get(url=url,headers=self.headers)
          res.encoding = 'utf-8'
          html = res.text

          return html
      except Exception as e:
          self.get_html(url)

  # 解析html函数
  def xpath_func(self,xpath_bds,html):
      parse_html = etree.HTML(html)
      r_list = parse_html.xpath(xpath_bds)

      return r_list


  # 一级页面:获取帖子链接,最终搞定所有图片下载
  # 还记得吗？多级页面抓取所有数据都在一级页面中搞定！！！
  def get_tlink(self,url):
    html = self.get_html(url)
    xpath_bds = '//div[@class="t_con cleafix"]/div/div/div/a/@href'
    # tlink_list: ['/p/23234','/p/9032323']
    tlink_list = self.xpath_func(xpath_bds,html)
    # 依次遍历每个帖子链接,搞定所有的图片下载
    if tlink_list:
        for tlink in tlink_list:
          t_url = 'http://tieba.baidu.com' + tlink
          # 提取图片链接并保存
          self.get_image(t_url)
          time.sleep(random.randint(1,3))
    else:
        print('No Data')

  # 获取图片链接
  def get_image(self,t_url):
    html = self.get_html(t_url)
    # 提取图片链接
    xpath_bds = '//*[@class="d_post_content j_d_post_content  clearfix"]/img/@src'
    imglink_list = self.xpath_func(xpath_bds,html)

    for imglink in imglink_list:
      html_content = requests.get(imglink,headers=self.headers).content
      filename = imglink[-10:]
      with open(filename,'wb') as f:
          f.write(html_content)
          print('%s下载成功' % filename)

  # 指定贴吧名称,起始页和终止页,爬取图片
  def main(self):
    name = input('请输入贴吧名:')
    begin = int(input('请输入起始页:'))
    end = int(input('请输入终止页:'))
    for page in range(begin,end+1):
      # 查询参数编码
      params = {
        'kw' : name,
        'pn' : str( (page-1)*50 )
      }
      params = parse.urlencode(params)
      url = self.url.format(params)

      # 开始获取图片
      self.get_tlink(url)

if __name__ == '__main__':
  spider = BaiduImgSpider()
  spider.main()