二十七、miniscrapy,scrapy源码初解

基本使用

from twisted.web.client import getPage, defer
from twisted.internet import reactor


# 基本使用

def all_done(contents):
    # 所有爬虫执行完毕后，循环终止
    reactor.stop()


def callback(contents):
    # 每一个爬虫获取结果后，自动执行
    print(contents)


deferred_list = list()

url_list = ['http://www.bing.com', 'http://www.baidu.com']
for url in url_list:
    deferred = getPage(bytes(url, encoding='utf8'))
    deferred.addCallback(callback)
    deferred_list.append(deferred)

dlist = defer.DeferredList(deferred_list)
dlist.addBoth(all_done)

reactor.run()

基于装饰器1

from twisted.web.client import getPage, defer
from twisted.internet import reactor

# 基于装饰器1
def all_done(arg):
    reactor.stop()


def onedone(response):
    print(response)

# 三要素：装饰器，deferred对象，yield
@defer.inlineCallbacks
def task(url):
    deferred = getPage(bytes(url, encoding='utf8'))
    deferred.addCallback(onedone)
    yield deferred


deferred_list = list()

url_list = ['http://www.bing.com', 'http://www.baidu.com']

for url in url_list:
    deferred = task(url)
    deferred_list.append(deferred)

dlist = defer.DeferredList(deferred_list)
dlist.addBoth(all_done)

reactor.run()

基于装饰器2

from twisted.web.client import getPage, defer
from twisted.internet import reactor


# 基于装饰器2

def all_done(arg):
    reactor.stop()


def onedone(response):
    print(response)


@defer.inlineCallbacks
def task():
    deferred_1 = getPage(bytes('http://www.baidu.com', encoding='utf8'))
    deferred_1.addCallback(onedone)
    yield deferred_1

    deferred_2 = getPage(bytes('http://www.bing.com', encoding='utf8'))
    deferred_2.addCallback(onedone)
    yield deferred_2


ret = task()
ret.addBoth(all_done)

reactor.run()

基于装饰器3

from twisted.web.client import getPage, defer
from twisted.internet import reactor


# 基于装饰器3，永恒循环
def all_done(arg):
    reactor.stop()


def onedone(response):
    print(response)


@defer.inlineCallbacks
def task():
    deferred_1 = getPage(bytes('http://www.bing.com', encoding='utf8'))
    deferred_1.addCallback(onedone)
    yield deferred_1

    stop_deferred = defer.Deferred()  # 永远不能完成的任务
    # stop_deferred.callback(None)  # 自定义callback对象，通过回调终止操作
    yield stop_deferred


ret = task()
ret.addBoth(all_done)

reactor.run()  # run是事件循环

基于装饰器4

from twisted.web.client import defer, getPage
from twisted.internet import reactor

# 基于装饰器，执行完毕后停止事件循环

running_list = list()
stop_deferred = None


def all_done(arg):
    reactor.stop()


def onedone(response, url):
    print(response)
    running_list.remove(url)


def check_empty(response):
    if not running_list:
        stop_deferred.callback(None)


@defer.inlineCallbacks
def task(url):
    deferred = getPage(bytes(url, encoding='utf8'))
    deferred.addCallback(onedone, url)
    deferred.addCallabck(check_empty)
    yield deferred

    global stop_deferred
    stop_deferred = defer.Deferred()
    yield stop_deferred


running_list.append('http://www.baidu.com')
ret = task('http://www.baidu.com')
ret.addBoth(all_done)

reactor.run()

基于装饰器5

from twisted.web.client import getPage, defer
from twisted.internet import reactor


class ExecutionEngine(object):

    def __init__(self):
        self.stop_deferred = None
        self.running_list = list()

    def one_done(self, response, url):
        print(response)
        self.running_list.remove(url)

    def check_empty(self, url):
        if not self.running_list:
            self.stop_deferred.callback(None)

    @defer.inlineCallbacks
    def open_spider(self, url):
        deferred = getPage(bytes(url, encoding='utf8'))
        deferred.addCallback(self.one_done, url)
        deferred.addCallback(self.check_empty)
        yield deferred

    @defer.inlineCallbacks
    def close_spider(self, url):
        self.stop_deferred = defer.Deferred()
        yield self.stop_deferred


@defer.inlineCallbacks
def task(url):
    engine = ExecutionEngine()
    engine.running_list.append(url)
    yield engine.open_spider(url)
    yield engine.close_spider(url)


def all_done(arg):
    reactor.stop()


if __name__ == "__main__":
    ret = task("http://www.bing.com")
    ret.addBoth(all_done)
    reactor.run()

Miniscrapy,scrapy源码初解

from twisted.web.client import getPage, defer
from twisted.internet import reactor
import queue


class Request(object):

    def __init(self, url, callback):
        self.url = url
        self.callback = callback


class Response(object):

    def __init__(self, body, request):
        self.body = body
        self.request = request
        self.url = reqeust.url

    @property
    def text(self):
        return self.body.decode('utf8')


class Scheduler(object):

    def __init(self, engine):
        self.q = queue.Queue()
        self.engine = engine

    def enqueue_request(self, request):
        self.q.put(request)

    def next_request(self):
        try:
            req = self.q.get(block=False)
        except Exception as e:
            req = None
        return req

    def size(self):
        return self.q.qsize()


class ExecutionEngine(object):

    def __init(self):
        self._closewait = None
        self.runing = True
        self.start_requests = None
        self.scheduler = Scheduler(self)
        self.inprogress = set()

    def check_empty(self, response):
        if not self.runing:
            self._closewait.callback(None)

    def _next_request(self):
        while self.start_requests:
            try:
                request = next(self.start_requests)
            except StopIteration:
                self.start_requests = None
            else:
                self.scheduler.enqueue_request(request)
        while len(self.inprogress) < 5 and self.scheduler.size() > 0:  # 最大并发数
            request = self.scheduler.next_request()
            if not request:
                break
            self.inprogress.add(request)
            d = getPage(bytes(request.url, encoding='utf8'))
            d.addBoth(self._handle_downloader_output, request)
            d.addBoth(lambda x, req: self.inprogress.remove(req), request)
            d.addBoth(lambda x: self._next_request())
        if len(self.inprogress) == 0 and self.scheduler.size() == 0:
            self._closewait.callback(None)

    def _handle_downloader_output(self, body, request):
        # 获取内容，执行回调函数，并且把回调函数中的返回值获取，并添加到队列中
        import types
        response = Response(body, request)
        func = request.callback or self.spider.parse
        gen = func(response)
        if isinstance(gen, types.GeneratorType):
            for req in gen:
                self.scheduler.enqueue_request(req)

    @defer.inlineCallbacks
    def start(self):
        self._closewait = defer_Deferred()
        yield self._closewait

    @defer.inlineCallbacks
    def open_spider(self, spider, start_requests):
        self.start_requests = start_requests
        self.spider = spider
        yield None
        reactor.callLater(0, self._next_request)


class Crawler(object):

    def __init__(self, spider_cls):
        self.spider_cls = spider_cls
        self.spider = None
        self.engine = None

    @defer.inlineCallbacks
    def crawl(self):
        self.engine = ExecutionEngine()
        self.spider = self.spider_cls()
        start_requests = iter(self.spider.start_requests())
        yield self.engine.open_spider(self.spider, start_requests)
        yield self.engine.start()


class CrawlerProcess(object):

    def __init__(self):
        self._active = set()
        self.crawlers = set()

    def crawl(self, spider_cls, *args, **kwargs):
        crawler = Crawler(spider_cls)

        self.crawlers.add(crawler)
        d = crawler.crawl(*args, **kwargs)
        self._active.add(d)
        return d

    def start(self):
        d = defer.DeferredList(self._active)
        d.addBoth(self._stop_reactor)
        reactor.run()

    def _stop_reactor(self, _=None):
        reactor.stop()


class Spider(object):

    def start_requests(self):
        for url in self.start_urls:
            yield Request(url)


class BaiduSpider(spider):
    name = 'baidu'
    start_urls = [
        'http://www.baidu.com'
    ]

    def parse(self, response):
        print(response.text)


class BingSpider(spider):
    pass


if __name__ == "__main__":
    spider_cls_list = [BaiduSpider, BingSpider]
    crawler_process = CrawlerProcess()
    for spider_cls in spider_cls_list:
        crawler_process.crawl(spider_cls)
    crawler_process.start()