爬虫常用设置

1. setting.py

#!/usr/bin/python
# -*- coding: utf-8 -*-
"""
@author: yugengde
@contact: yugengde@163.com
@file : settings.py
@time: 2017/11/22 15:41
"""

BOT_NAME = 'pro'

SPIDER_MODULES = ['pro.spiders']
NEWSPIDER_MODULE = 'pro.spiders'

ROBOTSTXT_OBEY = False

DOWNLOAD_DELAY = 3
COOKIES_ENABLED = False

DOWNLOADER_MIDDLEWARES = {
    'pro.middlewares.PhantomJSMiddleware': 301,
    'pro.middlewares.UserAgentMiddleware': 300,
}

ITEM_PIPELINES = {
    'scrapy_redis.pipelines.RedisPipeline': 301,
    'pro.pipelines.DuplicatesPipeline': 300,
}

LOG_ENABLED = True
LOG_ENCODING = 'utf-8'
LOG_FILE = 'pro.log'
LOG_LEVEL = 'DEBUG'
# LOG_STDOUT =

SCHEDULER = "scrapy_redis.scheduler.Scheduler"
DUPEFILTER_CLASS = "scrapy_redis.dupefilter.RFPDupeFilter"
REDIS_URL = 'redis://root:password@localhost:6379'

2. middlewares.py

class PhantomJSMiddleware(object):
    @classmethod
    def process_request(cls, request, spider):
        from selenium import webdriver
        from scrapy.http import HtmlResponse
        driver = webdriver.PhantomJS(r'C:InstallFilePhantomjsinphantomjs.exe')
        driver.get(request.url)
        content = driver.page_source.encode('utf-8')
        driver.quit()

        return HtmlResponse(request.url, encoding='utf-8', body=content, request=request)


class UserAgentMiddleware(object):
    @classmethod
    def process_request(cls, request, spider):
        import random
        user_agents = [ 
 20             "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:21.0) Gecko/20130331 Firefox/21.0",
            "Mozilla/5.0 (Windows x86; rv:19.0) Gecko/20100101 Firefox/19.0", # 可以使用 UserAgent()函数生成
    　　　　 ]
        request.headers.setdefault('UserAgent',random.choice(user_agents))

 3. pipelines.py

#!/usr/bin/python
# -*-coding:utf-8-*-

from scrapy.exceptions import DropItem


# 数据的去重
class DuplicatesPipeline(object):
    def __init__(self):
        self.ids_seen = set()

    def process_item(self, item, spider):
        if not item['title']:
            raise DropItem("Missing title in %s " % item)

        if item['item_id'] in self.ids_seen:
            raise DropItem("Duplicate item found: %s" % item)
        else:
            self.ids_seen.add(item['item_id'])
            yield item