scrapy框架下爬虫实现详情页抓取

以爬取阳光阳光热线问政平台网站为例，进行详情页的爬取。

# -*- coding: utf-8 -*-
import scrapy
from yanguang.items import YanguangItem

class SunSpider(scrapy.Spider):
    name = 'sun'
    allowed_domains = ['sun0769.com']
    start_urls = ['http://wz.sun0769.com/index.php/question/questionType?type=4']

    def parse(self, response):
        tr_list=response.xpath("//div[@class='greyframe']/table[2]/tr/td/table/tr")
        for tr in tr_list:
            item=YanguangItem()
            item['title']=tr.xpath("./td[2]/a[@class='news14']/@title").extract_first()
            item["href"]=tr.xpath("./td[2]/a[@class='news14']/@href").extract_first()
            item["publish_date"]=tr.xpath("./td[last()]/text()").extract_first()

            yield scrapy.Request(
                item["href"],
                callback=self.parse_detail,
                meta={"item":item},
            )
        #翻页
        next_url=response.xpath(".//a[text()='>']/@href").extract_first()
        if next_url is not None:
            yield scrapy.Request(
                next_url,
                callback=self.parse()
            )


    def parse_detail(self,response): #处理详情页
        item=response.meta["item"]
        item["content"]=response.xpath("//div[@class='c1 text14_2']//text()").extract()
        item["content_img"] = response.xpath("//div[@class='c1 text14_2']//img/@src").extract()
        item["content_img"] =["http://wz.sun0769.com"+i for i in item["content_img"]]
        yield item

下面为pipelines.py文件中对爬取的数据处理操作。

import re
class YanguangPipeline(object):
    def process_item(self, item, spider):
        item["content"]=self.process_content(item["content"])
        print(item)
        return item

    def process_content(self,content):#文本内容的处理
        content=[re.sub(r"xa0|s","",i)for i in content]
        content=[i for i in content if len(i)>0]#去除列表中的空字符串
        return content

在settings.py文件中修改USER_AGENT的内容是对方服务器无法一眼看出我们的请求是爬虫。

默认settings.py文件中的USER_AGENT为：

# Crawl responsibly by identifying yourself (and your website) on the user-agent
USER_AGENT = 'tencent (+http://www.yourdomain.com)'

将settings.py文件中的USER_AGENT修改为：

# Crawl responsibly by identifying yourself (and your website) on the user-agent
USER_AGENT = 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36'