scrapy框架

几个scrapy框架的指令：
　　scrapy startproject xxxx
　　scrapy genspider xxx  www.ooo.com
　　scrapy crawl xxx
基于管道的持久化存储：
　　1.数据解析
　　2.在item类中定义相关的属性
　　3.在parse方法中实例化一个item类型的对象
　　4.将解析到的数据存储到item类型的对象中
　　5.使用yield item将item对象提交给管道
　　6.在process_item 这里面 接收数据 并进行持久化存储
　　7.在配置文件里面开启管道

# -*- coding: utf-8 -*-
import scrapy
from qiubai01.items import Qiubai01Item


class QiubaiSpider(scrapy.Spider):
    name = 'qiubai'
    # allowed_domains = ['www.xxx.com']
    start_urls = ['https://www.qiushibaike.com/text/']

    # def parse(self, response):
    #     # response 是请求返回对象
    #     div_list = response.xpath("//*[@id="content-left"]/div")
    #     for div in div_list:
    #         # author = div.xpath("./div[1]/a[2]/h2/text()")[0].extract()
    #         # 这里面换一种写法也可以
    #         author = div.xpath("./div[1]/a[2]/h2/text()").extract_first()
    #         content = div.xpath('./a/div/span//text()').extract()
    #         content = "".join(content)
    #         print(author)
    #         print()
    #         print(content)

    # 基于终端指令存储  scrapy crawl -o qiushi.csv
    # def parse(self, response):
    #     # response 是请求返回对象
    #     div_list = response.xpath("//*[@id="content-left"]/div")
    #     all_data_list = []
    #     for div in div_list:
    #         # author = div.xpath("./div[1]/a[2]/h2/text()")[0].extract()
    #         # 这里面换一种写法也可以
    #         author = div.xpath("./div[1]/a[2]/h2/text()").extract_first()
    #         content = div.xpath('./a/div/span//text()').extract()
    #         content = "".join(content)
    #         dic = {}
    #         dic['author'] = author
    #         dic['content'] = content
    #         all_data_list.append(dic)
    #     return all_data_list

    # 基于管道的持久化存储

    def parse(self, response):
        """
        1.数据解析
        2.在item类中定义相关的属性
        3.在parse方法中实例化一个item类型的对象
        4.将解析到的数据存储到item类型的对象中
        5.使用yield item将item对象提交给管道
        6.在process_item 这里面 接收数据 并进行持久化存储
        7.在配置文件里面开启管道
        """
        # response 是请求返回对象
        div_list = response.xpath("//*[@id="content-left"]/div")

        for div in div_list:
            # author = div.xpath("./div[1]/a[2]/h2/text()")[0].extract()
            # 这里面换一种写法也可以
            author = div.xpath("./div[1]/a[2]/h2/text()").extract_first()
            content = div.xpath('./a/div/span//text()').extract()
            content = "".join(content)
            item = Qiubai01Item()
            item['author'] = author
            item['content'] = content
            # 向管道提交item
            yield item

 全栈数据爬取：

1.解析数据页面

# -*- coding: utf-8 -*-
import scrapy
from choutipro.items import ChoutiproItem


class ChoutiSpider(scrapy.Spider):
    name = 'chouti'
    # allowed_domains = ['https://dig.chouti.com/']
    start_urls = ['https://dig.chouti.com/all/hot/recent/1']

    pageNum = 1
    url = "https://dig.chouti.com/all/hot/recent/%s"

    def parse(self, response):
        div_list = response.xpath('//*[@id="content-list"]/div')
        for div in div_list:
            # //*[@id="newsContent26168656"]/div[1]/a[1]
            # //*[@id="newsContent26168656"]/div[1]
            # //*[@id="newsContent26168656"]
            content = div.xpath('./div[4]/div[1]/a[1]/text()').extract_first()
            author = div.xpath('./div[4]/div[2]/a[4]/b/text()').extract_first()
            item = ChoutiproItem()
            item['author'] = author
            item['content'] = content
            yield item
        if self.pageNum <= 120:
            self.pageNum += 1
            new_url = self.url % str(self.pageNum)
            print(new_url)
            yield scrapy.Request(url=new_url, callback=self.parse)

2.item类定义

import scrapy


class ChoutiproItem(scrapy.Item):
    # define the fields for your item here like:
    author = scrapy.Field()
    content = scrapy.Field()

3.管道文件

class ChoutiproPipeline(object):
    file = None

    def open_spider(self, spider):
        self.file = open("./test.txt", "a", encoding='utf-8')

    def process_item(self, item, spider):
        # author = item['author']
        content = item['content'].strip()
        try:
            self.file.write( content+'
')
        except:
            pass
        return item

    def close_spider(self, spider):
        self.file.close()

# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html

import pymysql


class Qiubai01Pipeline(object):
    fp = None

    def open_spider(self, spider):
        self.fp = open('./qiubai.txt', 'w', encoding='utf-8')

    def process_item(self, item, spider):
        author = item['author']
        content = item['content']
        print(type(author), type(content))
        try:
            self.fp.write(author + ":" + content)
        except:
            pass
        return item

    def close_soider(self, spider):
        self.fp.close()


class MySql01Pipeline(object):
    conn = None
    cursor = None

    def open_spider(self, item):
        self.conn = pymysql.Connect(host='127.0.0.1', port=3306, user='', password='', db='spider')

    def process_item(self, item, spider):
        self.cursor = self.conn.cursor()
        try:
            self.cursor.execute('insert into qiubqi values ("%s","%s")' % (item['author'], item['content']))
            self.conn.commit()
        except:
            self.conn.rollback()

    def close_spider(self, spider):
        self.cursor.close()
        self.conn.close()

 4.日志等级处理

　　在setting里面设置：LOG_LEVEL = 'ERROR'  或者LOG_FILE = 'log.txt' 都可以。

5.请求传参：当解析的页面不在同一个页面的时候需要使用请求传参，主要是回调函数里面的参数需要重新写。