Scrapy之定制命令与自由发挥的信号

定制命令

在项目目录下创建一个脚本

　　

#!/usr/bin/env python
# -*- coding:utf-8 -*-

import sys
from scrapy.cmdline import execute

if __name__ == '__main__':
    # execute(['scrapy', 'crawl', 'chouti', '--nolog'])
    execute(['scrapy', 'crawlall', '--nolog'])

在spiders同级目录创建文件夹commands

#!/usr/bin/env python
# -*- coding:utf-8 -*-

from scrapy.commands import ScrapyCommand
from scrapy.utils.project import get_project_settings


class Command(ScrapyCommand):
    requires_project = True

    def syntax(self):
        return '[options]'

    def short_desc(self):
        return 'Runs all of the spiders'

    def run(self, args, opts):
        spider_list = self.crawler_process.spiders.list()
        print(spider_list)
        for name in spider_list:
            self.crawler_process.crawl(name, **opts.__dict__)
        self.crawler_process.start()

配置文件

# 自定制命令
COMMANDS_MODULE =  "redisdepth.commands"

 信号开发框架的大佬给我们预留的位置自由发挥

#!/usr/bin/env python
# -*- coding:utf-8 -*-

from scrapy import signals


class MyExtension(object):
    def __init__(self):
        # self.value = value
        pass

    @classmethod
    def from_crawler(cls, crawler):
        # val = crawler.settings.getint('MMMM')
        self = cls()

        crawler.signals.connect(self.spider_opened, signal=signals.spider_opened)
        crawler.signals.connect(self.spider_closed, signal=signals.spider_closed)

        return self

    def spider_opened(self, spider):
        print('open')

    def spider_closed(self, spider):
        print('close')

 配置

EXTENSIONS = {
   # 'scrapy.extensions.telnet.TelnetConsole': None,
    'redisdepth.ext.MyExtension': 666,
}

 内置信号

"""
Scrapy signals

These signals are documented in docs/topics/signals.rst. Please don't add new
signals here without documenting them there.
"""

engine_started = object()
engine_stopped = object()
spider_opened = object()
spider_idle = object()
spider_closed = object()
spider_error = object()
request_scheduled = object()
request_dropped = object()
request_reached_downloader = object()
response_received = object()
response_downloaded = object()
item_scraped = object()
item_dropped = object()
item_error = object()

# for backward compatibility
stats_spider_opened = spider_opened
stats_spider_closing = spider_closed
stats_spider_closed = spider_closed

item_passed = item_scraped

request_received = request_scheduled