为了爬取拉钩,今天学习了selenum的使用技巧.
from scrapy.http import HtmlResponse
class JSPageMiddleware(object):
def process_request(self, request, spider):
if spider.name == "zhihu":
browser = webdriver.Firefox(executable_path="C:/codeapp/seleniumDriver/firefox/geckodriver.exe")
browser.get(request.url)
time.sleep(3)
print("访问:{0}".format(request.url))
# 请求已完成,所以不用再发送到下载器。用HtmlResponse 之后就不会发送到downlaoder,而是直接返回给spider
return HtmlResponse(url=browser.current_url, body=browser.page_source, encoding="utf-8", request=request)
这样有个不好的地方就是每次来一个请求时都会启动一次浏览器,浏览器启动是很慢的,可以在类中初始化一个浏览器,
class JSPageMiddleware(object):
def __init__(self):
self.browser=webdriver.Firefox(executable_path="C:/codeapp/seleniumDriver/firefox/geckodriver.exe")
super(JSPageMiddleware,self).__init__()
def process_request(self, request, spider):
if spider.name == "zhihu":
self.browser.get(request.url)
time.sleep(3)
print("访问:{0}".format(request.url))
# 请求已完成,所以不用再发送到下载器。用HtmlResponse 之后就不会发送到downlaoder,而是直接返回给spider
return HtmlResponse(url=self.browser.current_url, body=self.browser.page_source, encoding="utf-8", request=request)
用上述方法,爬虫自动关闭时,浏览器不会关闭,并且每个spider会共用一个打开的浏览器,这样也不方便调试,可以把这个初始化的工作放在各个spider中
# -*- coding: utf-8 -*-
import scrapy
from selenium import webdriver
from scrapy import signals
from scrapy.xlib.pydispatch import dispatcher
from scrapy.http import HtmlResponse
class ZhihuSpider(scrapy.Spider):
name = 'zhihu'
allowed_domains = ['oschina.net/blog']
start_urls = ['https://www.oschina.net/blog']
def __init__(self):
self.browser = webdriver.Firefox(executable_path="C:/codeapp/seleniumDriver/firefox/geckodriver.exe")
super(ZhihuSpider, self).__init__()
#传递信息,也就是当爬虫关闭时scrapy会发出一个spider_closed的信息,当这个信号发出时就调用closeSpider函数关闭这个浏览器.
dispatcher.connect(self.closeSpider, signals.spider_closed)
def closeSpider(self, spider):
print("spider closed")
# 当爬虫退出的时关闭浏览器
self.browser.quit()
def parse(self, response):
# data=response.css(".SignFlow-accountInpu input[]").extract()
pass
Middleware 中的类
class JSPageMiddleware(object):
def process_request(self, request, spider):
if spider.name == "zhihu":
spider.browser.get(request.url)
time.sleep(3)
print("访问:{0}".format(request.url))
return HtmlResponse(url=spider.browser.current_url, body=spider.browser.page_source, encoding="utf-8",request=request)
注意 上面三种方法都必须在settings中把JSPageMiddleware加入
DOWNLOADER_MIDDLEWARES = {
# 'outlook.middlewares.MyCustomDownloaderMiddleware': 543,
'outlook.middlewares.JSPageMiddleware': 1,