custom_setting

custom_setting

一、定义

二、配置

　　1.middlewares

# ----- SeleniumMiddlerware中间件，不添加全局 -----
from selenium import webdriver
from selenium.common.exceptions import TimeoutException
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as  EC
from selenium.webdriver.common.keys import Keys
from scrapy.http import HtmlResponse
from logging import getLogger
import time
class SeleniumMiddleware(object):
    # Middleware中会传递进来一个spider，这就是我们的spider对象，从中可以获取__init__时的chrome相关元素
    @classmethod
    def process_request(self, request, spider):
        """
        用chrome抓取页面
        :param request:Request请求对象
        :param spider: Spider对象
        :return: HtmlResponse响应
        """
        print(f"chrome is getting page")
        # 依靠meta中的标记，决定是否启用selenium来爬取
        usedSelenium = request.meta.get('usedSelenium',False)
        print("usedSelenium开始执行")
        if usedSelenium:
            try:
                spider.browser.get(request.url)
                # 搜索框是否出现
                input = spider.wait.until(
                    EC.presence_of_element_located((By.XPATH, "//div[@class='nav-search-field ']/input"))
                )
                time.sleep(2)
                input.clear()
                input.send_keys("iphone 7s")
                # 按enter键，进行搜索
                input.send_keys(Keys.RETURN)
                # 查看搜索结果是否出现
                searchRes = spider.wait.until(
                    EC.presence_of_all_elements_located((By.XPATH, "//div[@id='resultsCol']"))
                )
            except Exception as err:
                print(f"chrome getting page error,Exception = {err}")
                return HtmlResponse(url=request.url, status=500, request=request)
            finally:
                time.sleep(3)
                # 页面爬取成功，构造一个成功的Response对象(HtmlResponse是它的子类)
                return HtmlResponse(url=request.url,
                                    body=spider.browser.page_source,
                                    request=request,
                                    # 指定网页编码
                                    encoding = "UTF-8",
                                    status=200,
                                    )

　　2.settings/在settings同目录下新建custom_settings.py

# -*- coding: utf-8 -*-
custom_settings_for_spider1 = {
    'LOG_LEVEL': 'INFO',
    'DOWNLOAD_DELAY': 0,
    'COOKIES_ENABLED': False,  # enabled by default
    'DOWNLOADER_MIDDLEWARES': {
        # 代理中间件
        'video_spider.middlewares.ProxiesMiddleware': 400,
        # SeleniumMiddleware中间件
        'video_spider.middlewares.SeleniumMiddleware': 543,
        # 将scrapy默认的user-agent中间件关闭
        'scrapy.downloadmiddlewares.useragent.UserAgentMiddleware': None,
    },

    }

　　3.在spider文件中引入custom_settings

import scrapy
from scrapy import Request
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
# scrapy 信号相关库
from scrapy.utils.project import get_project_settings
from scrapy import signals
from pydispatch import dispatcher
# setting
from ..custom_settings import *

class ShanbaySpider(scrapy.Spider):
    name = 'shanbay'
    allowed_domains = ['shanbay.com']
   # start_urls = ['http://shanbay.com/']
    custom_settings = custom_settings_for_spider1
    # 将Chrome初始化放到spider中，成为spider中的元素
    def __init__(self, timeout=30, isLoadImage=True, windowHeight=None, windowWidth=None):
        # 从settings.py中获取设置参数
        print("浏览器开始执行")
        self.mySetting = get_project_settings()
        self.timeout = self.mySetting['SELENIUM_TIMEOUT']
        self.isLoadImage = self.mySetting['LOAD_IMAGE']
        self.windowHeight = self.mySetting['WINDOW_HEIGHT']
        self.windowWidth = self.mySetting['windowWidth']
        # 初始化Chrome对象
        self.browser = webdriver.Chrome()
        print("六拉你去")
        if self.windowHeight and self.windowHeight:
            self.browser.set_window_size(900, 900)
        self.browser.set_page_load_timeout(self.timeout)
        self.wait = WebDriverWait(self.browser, 25)
        # 初始化父类，方便不同爬虫文件执行不同执行方式
        super(ShanbaySpider, self).__init__()
        # 设置信号量，当收到spider_closed信号时，调用mySpiderCloseHandle方法，关闭chrome
        dispatcher.connect(receiver=self.CloseHandle,
                           signal=signals.spider_closed
                           )

    # 信号量处理函数：关闭chrome浏览器
    def CloseHandle(self, spider):
        print(f"CloseHandle：enter")
        self.browser.quit()

    # ------------ spider 开始执行 --------------
    # --- 网络请求 ---
    def start_requests(self):
        for i in range(29):
            page = 540709 + i * 3
            url_base = 'https://www.shanbay.com/wordlist/187711/' + str(page) + '/?page={}'
            for x in range(10):
                url = url_base.format(x+1)
                yield Request(
                    url,
                    meta={'usedSelenium': True, 'dont_redirect': True},
                    callback=self.parse,
                    errback=self.error,
                    )
    def error(self, response):
        pass
    def parse(self, response):
        from ..items import ShanbaySpiderItem
        html_contenxs = response.xpath('/html/body/div[3]/div/div[1]/div[2]/div/table/tbody/tr//*/text()')
        item = ShanbaySpiderItem()

        for result in html_contenxs:
            item['Chinese'] = result.extract()
            print(item['Chinese'])
            yield item