封装IP池和用户代理相应的类（python3）

一、middlewares.py源代码：

# -*- coding: utf-8 -*-
# 导入随机模块
import random
# 导入有关IP池有关的模块
from scrapy.contrib.downloadermiddleware.httpproxy import HttpProxyMiddleware
# 导入有关用户代理有关的模块
from scrapy.contrib.downloadermiddleware.useragent import UserAgentMiddleware

# IP池
class HTTPPROXY(HttpProxyMiddleware):
    # 初始化 注意一定是 ip=''
    def __init__(self, ip=''):
        self.ip = ip

    def process_request(self, request, spider):
        item = random.choice(IPPOOL)
        print("当前的IP是："+item["ipaddr"])
        request.meta["proxy"] = "http://"+item["ipaddr"]


# 设置IP池
IPPOOL = [
    {"ipaddr": "182.117.102.10:8118"},
    {"ipaddr": "121.31.102.215:8123"},
    {"ipaddr": "1222.94.128.49:8118"}
]


# 用户代理
class USERAGENT(UserAgentMiddleware):
    #初始化 注意一定是 user_agent=''
    def __init__(self, user_agent=''):
        self.user_agent = user_agent

    def process_request(self, request, spider):
        item = random.choice(UPPOOL)
        try:
            print("当前的User-Agent是："+item)
            request.headers.setdefault('User-Agent', item)
        except Exception as e:
            print(e)
            pass


# 设置用户代理池
UPPOOL = [
    "Mozilla/5.0 (Windows NT 10.0; WOW64; rv:52.0) Gecko/20100101 Firefox/52.0", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.115 Safari/537.36", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.79 Safari/537.36 Edge/14.14393"
]

二、settings.py文件设置，添加下面代码：

#==============================================

# 禁止Cookie
COOKIES_ENABLED = False

# 下载中间件指向
DOWNLOADER_MIDDLEWARES = {
    # 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware':123,
    # '工程名.middlewares.HTTPPROXY' : 125,
    'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware': 2,
    '工程名.middlewares.USERAGENT': 1
}

# 管道指向
ITEM_PIPELINES = {
    '工程名.pipelines.管道中对应的类名': 300,
}

# 注意将Obey robots关闭（上面已经开启了，所以需要找到并设置为False）
ROBOTSTXT_OBEY = False

#==============================================

三、总结一些需要的东西

1.在引入（有关IP池有关的和关用户代理的）模块是注意这个

# 导入有关IP池有关的模块
from scrapy.contrib.downloadermiddleware.httpproxy import HttpProxyMiddleware
# 导入有关用户代理有关的模块
from scrapy.contrib.downloadermiddleware.useragent import UserAgentMiddleware

"contrib"不能丢了

2.配置下载中间件的连接信息时，注意文件指向，也就是调用类的指向

3.“.”表示这一级目录，".."表示上一级目录