day96 scrapy

1．scrapy安装 　　ubuntu下安装　

1.先安装ｐｉｐ
    
            sudo apt install python3-pip

2.安装依耐
    
            sudo apt install python-dev

3.安装非Python的依赖 sudo apt-get install python-dev python-pip libxml2-dev libxslt1-dev zlib1g-dev libffi-dev libssl-dev

4.用ｐｉｐ安装ｓｃｒａｐｙ
            
            sudo pip install scrapy

 2.创建爬虫项目　

scrapy startproject mySpider

3.在当前目录下输入命令，将在mySpider/spider目录下创建一个名为itcast的爬虫，并指定爬取域的

scrapy genspider itcast 'itcast.cn'

4.items 创建类　主要用来存数据的　有点像ｍｏｄｅ里面的ｏｒｍ

import scrapy


class MyspiderItem(scrapy.Item):
    # define the fields for your item here like:
    # name = scrapy.Field()

    name = scrapy.Field()
    title = scrapy.Field()
    info= scrapy.Field()

5.spiders 文件里面刚才创建的爬虫

# -*- coding: utf-8 -*-
import scrapy
from mySpider.items import MyspiderItem


class ChuanzhiSpider(scrapy.Spider):
    name = 'chuanzhi'
    allowed_domains = ['itcast.cn']
    start_urls = ['http://www.itcast.cn/channel/teacher.shtml#apython']

    def parse(self, response):
        # with open("teacher.html", "w") as f:
        #    f.write(response.body)
        # 通过scrapy自带的xpath匹配出所有老师的根节点列表集合
        teacher_list = response.xpath('//div[@class="li_txt"]')

        # 所有老师信息的列表集合
        teacherItem = []
        # 遍历根节点集合
        for each in teacher_list:
            # Item对象用来保存数据的
            item = MyspiderItem()
            # name, extract() 将匹配出来的结果转换为Unicode字符串
            # 不加extract() 结果为xpath匹配对象
            #extract_first 直接提取第一个不用加下标取值
            name = each.xpath('./h3/text()').extract_first()
            # title
            title = each.xpath('./h4/text()').extract()
            # info
            info = each.xpath('./p/text()').extract()

            item['name'] = name
            # item['title'] = title[0]
            # item['info'] = info[0]

            # item['name'] = name[0]
            # item['title'] = title[0]
            # item['info'] = info[0]

            #yield item


            teacherItem.append(item)

            # print name[0]
            # print title[0]
            # print info[0]

        return teacherItem

注意：运行用命令 scrapy  crawl  itcast   --nolog　参数--nolog不显示执行中的其它信息　可以不加　　　 导出的命令  scrapy crawl itcast -o test.json/test.csv　　

实例：

　　items文件

import scrapy

class cqitItme(scrapy.Item):
    pageText = scrapy.Field()
    hrefText = scrapy.Field()

爬虫文件

import scrapy
from scrapy.selector import Selector, XmlXPathSelector
from scrapy.http import Request
from ..items import cqitItme


class CqitSpider(scrapy.Spider):
    name = 'cqit'
    allowed_domains = ['cqsq.com'] #注意这个域名范围　　不能写成　'www.cqsq.com' 或者'cqsq.com/'
    start_urls = ['https://www.cqsq.com/list/7']
    set_href = []

    def parse(self, response):
        a = Selector(response=response).xpath("//div[@class= 'pages']/a")
　　#实例化items里面创建的类　以便下面赋值并返回数据到pipelines文件中去做数据处理
        itme = cqitItme()
        for i in a:

            i1 = i.xpath('.//@href').extract_first()
            itext = i.xpath('.//text()').extract_first()

            if i1 in self.set_href or i1 == "//www.cqsq.com/list/7/50":
                pass
            else:

                self.set_href.append(i1)
                url1 = 'https:' + i1
                itme['pageText'] =itext
                itme['hrefText'] =url1

                print(url1)
        　　＃这里是返回实例数据并提交到pipelines文件中去处理
                yield itme
                #发送新的ｕｒｌ到控制的阵列里去等去等待下次扏行　　注意dont_filter去重参数　true是代表不去重　默认为false
                yield Request(url=url1, callback=self.parse, dont_filter=True)

pipelines文件

class dyspiderPipeline(object):　　＃注意在settings文件中去设置该类的权重

    def __init__(self):
        self.f = open('test1','w')
    def process_item(self, item, spider):＃这里的ｉｔｍ就是爬虫文件k yield回来的数据
        import json
        content = json.dumps(dict(item),ensure_ascii=False)+'
'
        self.f.write(content)
        return item

 setting 配置文件　

ITEM_PIPELINES = {
   'mySpider.pipelines.MyspiderPipeline': 300,
   'mySpider.pipelines.dyspiderPipeline': 300, #PIPELINENS　注册的位置　300代表的是权重
}

DEPTH_LIMIT = 2 #深度为2　如果为0的话就是代表没有深度

DUPEFILTER_CLASS = "mySpider.spiders.repeatfilter.Repeat"　#自定义去重的路径

 自定义去重　重写yield Rquest(url = url ,callback = self.parse ,dont_filter = False ) #false 代表的就是去重　默认调用的是  #from scrapy.dupefilters import RFPDupeFilter   

重构代码：　新建一个repeatfilter 的python文件　　导入方法看上面setting配置文件里面的设置

class Repeat(object):
    def __init__(self):
　　　　　　　'''第二步执行　构造函数'''
        self.visited_set = []
    @classmethod
    def from_settings(cls, settings):
　　　　　　'''这里是第一步执行　相当于是obj = Repeat.from_settings'''
        return cls()

    def request_seen(self, request):
　　　　'''第里是第四步执行　主要是判断罗辑的'''

        if request.url in self.visited_set or request.url == "https://www.cqsq.com/list/7/50":

            return True
        else:
            self.visited_set.append(request.url)

            return False



    def open(self):  # can return deferred
        # print('open')　第三步执行
        pass

    def close(self, reason):  # can return a deferred
        # print('close')　第５步执行
        pass
    def log(self, request, spider):  # log that a request has been filtered
        # print('log....')　最后执行的日志
        pass

pipeline补充

from scrapy.exceptions import DropItem

class Day96Pipeline(object):

    def __init__(self,conn_str):
        '''第二步'''
        self.conn_str = conn_str

    @classmethod
    def from_crawler(cls, crawler):
        """
        初始化时候，用于创建pipeline对象
        :param crawler:
        :return:
        第一步
        """
        conn_str = crawler.settings.get('DB')
        return cls(conn_str)

    def open_spider(self,spider):
        """
        第三步
        爬虫开始执行时，调用
        :param spider:
        :return:
        """
        self.conn = open(self.conn_str, 'a')

    def close_spider(self,spider):
        """
        第五步
        爬虫关闭时，被调用
        :param spider:
        :return:
        """
        self.conn.close()

    def process_item(self, item, spider):
        """
        第四步
        每当数据需要持久化时，就会被调用
        :param item:
        :param spider:
        :return:
        """
        # if spider.name == 'chouti'
        tpl = "%s
%s

" %(item['title'],item['href'])
        self.conn.write(tpl)
        # 交给下一个pipeline处理
        return item
        # 丢弃item，不交给
        # raise DropItem()

class Day97Pipeline(object):

    def __init__(self,conn_str):
        self.conn_str = conn_str

    @classmethod
    def from_crawler(cls, crawler):
        """
        初始化时候，用于创建pipeline对象
        :param crawler:
        :return:
        """
        conn_str = crawler.settings.get('DB')
        return cls(conn_str)

    def open_spider(self,spider):
        """
        爬虫开始执行时，调用
        :param spider:
        :return:
        """
        self.conn = open(self.conn_str, 'a')

    def close_spider(self,spider):
        """
        爬虫关闭时，被调用
        :param spider:
        :return:
        """
        self.conn.close()

    def process_item(self, item, spider):
        """
        每当数据需要持久化时，就会被调用
        :param item:
        :param spider:
        :return:
        """
        # if spider.name == 'chouti'
        tpl = "%s
%s

" %(item['title'],item['href'])
        self.conn.write(tpl)

        #这几个方法的套路和上面自定义去重是一样的执行套路　　注意process_itme里面return itme　代表交给下个权重执行　如果是return　Dropｉtme()　则代表抛异常　不交给下个执行