爬虫———通过pipeline以及items 将数据以json类型存储

1、books.py

import scrapy
from scrapy.selector.unified import SelectorList

class BooksSpider(scrapy.Spider):
    name = 'books'
    allowed_domains = ['books.toscrape.com']
    start_urls = ['http://books.toscrape.com/catalogue/page-1.html']

    def parse(self, response):
        # print('*')
        # print(type(response))    #  <class 'scrapy.http.response.html.HtmlResponse'>
        # print('*')

        Books_lis = response.xpath("//div/ol[@class='row']/li")
        # print('*')
        # print(type(Books_lis))
        # print('*')
        for Books_li in Books_lis:
            book_name = Books_li.xpath(".//h3/a/@title").get()
            book_price = Books_li.xpath(".//div[@class='product_price']/p[@class='price_color']/text()").getall()

            book_price = "".join(book_price)
            books_all = {"book_name":book_name,"book_price":book_price}
            yield books_all

2、pipeline .py

import json


class BooksPipeline(object):
    def __init__(self):
        self.file = open("books_all.json", "w", encoding="utf-8")

    def open_spider(self, spider):
        # self.file = open('items.jl', 'w')
        print("爬虫开始了。。。")

    def close_spider(self, spider):
        self.file.close()
        print("爬虫结束了。。。")

    def process_item(self, item, spider):
        # line = json.dumps(dict(item)) + "
"
        # self.file.write(line)
        item_json = json.dumps(item,ensure_ascii= False)
        self.file.write(item_json+'
')
        return item

3、结果

 3、犯的错误

 

 4、专业化

设置item.py文件

from scrapy import Item,Field


class BooksItem(Item):
    # define the fields for your item here like:
    # name = scrapy.Field()
    book_name = Field()
    book_price = Field()