Scrapy 实现爬取多页数据 多层url数据爬取 meta传递 链接为相对路径

提取电影网站的片名，导演，影片播放地址。

item.py

import scrapy


class MovieItem(scrapy.Item):
    # define the fields for your item here like:
    name = scrapy.Field() #没啥好说的，定义三个返回变量
    actor = scrapy.Field()
    link = scrapy.Field()

spider,py

import scrapy

from movie.items import MovieItem

class MovieproSpider(scrapy.Spider):
    name = 'moviePro'
    allowed_domains = ['4567tv.tv']
    start_urls = ['https://www.4567tv.tv/frim/index1.html']
    page = 1
    page_url = 'https://www.4567tv.tv/frim/index1-%s.html'


    def parse(self, response):
        li_list = response.xpath('//li[@class="col-md-6 col-sm-4 col-xs-3"]')
        for li in li_list:
            item = MovieItem()
            name = li.xpath('./div/div/h4/a/text()').extract_first()
            detail_url = 'https://www.4567tv.tv' + li.xpath('./div/div/h4/a/@href').extract_first()
            item['name'] = name
            yield scrapy.Request(url = detail_url, callback = self.parse_detail, meta ={'item':item})

        if self.page <= 10:
            self.page += 1
            new_page_url = self.page_url % self.page
            yield scrapy.Request(url = new_page_url, callback = (self.parse))

    def parse_detail(self, response):
        item = response.meta['item'] #注意这里实例化的是meta的，是parse函数传递过来的第二层内容
        actor = response.xpath('/html/body/div[1]/div/div/div/div[2]/p[3]/a/text()').extract_first() 
        link = response.xpath('/html/body/div[1]/div/div/div/div[1]/a/@href').extract_first()
        item['actor'] = actor
        item['link'] = 'https://www.4567tv.tv' + link #这个连接是相对路径，重新构造地址
        yield item

settings.py

 FEED_EXPORT_ENCODING ='utf-8' #插入这两行，解码成中文，不然出现的是乱码

2 FEED_EXPORT_ENCODING = 'gb18030' 

运行

scrapy crawl moviePro -o mov.csv

结果