python爬虫学习：从数据库读取目标爬虫站点及爬虫规则，批量爬取目标站点指定数据（scrapy框架）

1. 数据库databaseConfig.py

from urllib.parse import quote_plus
from pymongo import MongoClient
import settings


class DB:
    def __init__(self):
        # 从配置文件总获取数据库连接的参数
        host = settings.MONGODB_HOST
        port = settings.MONGODB_PORT
        dbname = settings.MONGODB_DBNAME
        user_name = settings.MONGODB_USERNAME
        password = settings.MONGODB_PASSWORD

        # 存放爬取数据的表名
        self.spider_result_sheet_name = settings.MONGODB_SAVE_SPIDER_RESULT_SHEET_NAME
        # 存放爬虫目标网站信息
        self.spider_station_sheet_name = settings.MONGODB_SPIDER_STATION_SHEET_NAME

        # 创建MONGODB数据库链接
        uri = "mongodb://%s:%s@%s:%s" % (quote_plus(user_name), quote_plus(password),
                                         quote_plus(host), quote_plus(port))
        client = MongoClient(uri)
        # 指定数据库
        self.collection = client[dbname]

2. 修改 scrapy 框架的 pipelines.py 文件，添加爬虫数据保存到数据库的方法

# -*- coding: utf-8 -*-
# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html


# useful for handling different item types with a single interface
import codecs
import json
import os
from MySpider.databaseConfig import DB


class MyScrapyPipeline:
    def process_item(self, item, spider):
        return item

# # 以json文件保存
# class JsonPipeline(object):
#     def process_item(self, item, spider):
#         # base_dir = os.getcwd()
#         # filename = base_dir + '/spiderData.json'
#         filename = 'D:/development/datas' + '/spiderData.json'
#         # 打开json文件，向里面以dumps的方式吸入数据
#         # 注意需要有一个参数ensure_ascii=False ，不然数据会直接为utf编码的方式存入比如
#         # :“/xe15”
#         with codecs.open(filename, 'a', encoding='utf-8') as f:
#             line = json.dumps(dict(item), ensure_ascii=False) + '
'
#             f.write(line)
#         return item


# 保存到mongodb数据库
class SpiderMongoPipeline(object):
    def process_item(self, item, spider):
        data = dict(item)
        db = DB()
        db.collection[db.spider_result_sheet_name].insert(data)
        return item

3. 编辑items.py 对应数据库字段

# Define here the models for your scraped items
#
# See documentation in:
# https://docs.scrapy.org/en/latest/topics/items.html
from scrapy.item import Item, Field


class MyDataItem(Item):
    # define the fields for your item here like:
    # name = scrapy.Field()
    title = Field()
    author = Field()
    release_time = Field()
    url = Field()
    create_time = Field()
    # pass

4. 核心爬虫方法mySpider.py

# coding=utf-8

import time
import scrapy
from scrapy.selector import Selector
from mySpider.databaseConfig import DB
from mySpider.items import MyDataItem


class MySpider(scrapy.Spider):
    name = 'mySpider'  # 爬虫的唯一标识，不能重复，启动爬虫的时候要用

    # 重写Scrapy的start_requests方法
    def start_requests(self):
        # 数据库连接
        collection = DB().collection[DB().spider_station_sheet_name]
        items = collection.find() # 从数据库中查询所有需要爬取的站点信息
        for item in items:
            station_url = item["station_url"] # 目标站点url
            yield scrapy.Request(url=station_url, meta=item, callback=self.parse_station)


     # 站点爬虫方法
    def parse_station(self, response):
        meta = response.meta  # 从请求上获取手动传入的meta参数
        articles = Selector(response).xpath(meta["table_xpath"]) # 获取到文章列表
        for article in articles:
            article_detail_url = meta["station_root_url"] + article.xpath(meta["article_detail_xpath"]).extract()[0]
            # dont_filter=True 表示不过滤，不然会导致parse_detail只执行一次
            yield scrapy.Request(url=article_detail_url, meta=meta, callback=self.parse_detail, dont_filter=True) 

    # 爬取详情页
    def parse_detail(self, response):
        items = MyDataItem()
        current_time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())

        meta = response.meta
        selector = Selector(response)
        items['title'] = selector.xpath(meta["title_xpath"]).extract()[0]
        items['author'] = meta["station_name"] if meta["author_xpath"] == "" else selector.xpath(meta["author_xpath"]).extract()[0]
        items['release_time'] = selector.xpath(meta["release_time_xpath"]).extract()[0]
        items['url'] = response.url
        items['create_time'] = current_time
        yield items # 提交爬虫信息（到pipelines.py）

dont_filter=True 表示不过滤，不然会导致parse_detail只执行一次，这是一个坑点，前期由于查询资料的方向和关键字不对，导致卡壳很久。最后搜到scrapy - Request 中的回调函数不执行或者只执行一次这篇文章才得以解决

5. settings.py 修改（以下为settings.py的部分配置内容）

BOT_NAME = 'mySpider'

SPIDER_MODULES = ['myScrapy.spiders'] # 爬虫核心方法所在的项目文件路径（从项目根开始）
NEWSPIDER_MODULE = 'myScrapy.spiders'

# Obey robots.txt rules
ROBOTSTXT_OBEY = True
# LOG_LEVEL = 'ERROR'

# Override the default request headers:
DEFAULT_REQUEST_HEADERS = {
    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
    'Accept-Language': 'en',
}

# 优先级
ITEM_PIPELINES = {
    'myScrapy.pipelines.SpiderMongoPipeline': 200 
}

# 以下mongodb数据库配置信息省略

6. 启动类main.py

# 方法一：执行单一爬虫模块，并根据pipeline.py的配置保存
from scrapy import cmdline
cmdline.execute("scrapy crawl recruit".split())

# 方法二：执行单一爬虫模块，以文件形式保存（在当前项目根）
# cmdline.execute("scrapy crawl recruit -o rsj.json".split())


# 方法三： 批量制定执行爬虫模块
# 批量方法1
# cmdline.execute("scrapy crawlProcess rsj cqgsdx".split())
# 批量方法2
# cmdline.execute(['scrapy', 'crawl', 'recruit'])

from scrapy.crawler import CrawlerProcess
from scrapy.utils.project import get_project_settings

# 方法四：批量运行spider
# process = CrawlerProcess(get_project_settings())
# didntWorkSpider = ['rsj', 'cqgsdx'] # 不需要执行的spider模块

# process_spider_list = process.spiders.list() # 取spiders路径下所有的spider模块
# for the_spider_name in process_spider_list:
#     if the_spider_name in didntWorkSpider:
#         continue
#     print("Running spider %s" % (the_spider_name))
#     process.crawl(the_spider_name)

# process.start()