利用Scrapy框架对4567电影爬取

1.创建一个爬虫文件Movie:--scrapy genspider Movie

2.在爬虫文件中编写:

　　

# -*- coding: utf-8 -*-
import scrapy
from dianying.items import DianyingItem


class MovieSpider(scrapy.Spider):
    name = 'Movie'
    #allowed_domains = ['www.xxx.com']
    start_urls = ['https://www.4567kan.com/index.php/vod/show/class/爱情/id/7.html']
    url = 'https://www.4567kan.com/index.php/vod/show/class/爱情/id/7/page/%d.html'
    pageNumber = 2 # 爬取的页码

    def parse(self, response):
        li_list = response.xpath('/html/body/div[1]/div/div/div/div[2]/ul/li')
        for li in li_list:
            name = li.xpath('./div/a/@title')[0].extract()
            detai_url = 'https://www.4567kan.com'+li.xpath('./div/a/@href').extract_first()
            item = DianyingItem()
            item['name'] = name
            # 利用dedail_url对每个详情页面进行爬取
            yield scrapy.Request(detai_url,callback=self.parse_detai,meta={'item':item})  # meta参数的作用,给回调函数
        if self.pageNumber < 5: #爬取前五页的代码数据
            new_url = format(self.url%self.pageNumber)
            yield scrapy.Request(new_url,callback=self.parse)

    def parse_detai(self,response):
        item = response.meta['item']
        desc = response.xpath('/html/body/div[1]/div/div/div/div[2]/p[5]/span[2]/text()').extract_first()
        item['desc'] = desc
        yield item  # 把item传给管道

3.在item.py中编写

import scrapy


class DianyingItem(scrapy.Item):
    # define the fields for your item here like:
    name = scrapy.Field()
    desc = scrapy.Field()


4.在管道中编写

# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html
import pymysql

class DianyingPipeline:
    conn = None
    c = None
    def open_spider(self,spider):
        self.conn = pymysql.Connect(user='root', password='123456', host='localhost'
                               , port=3306, database='xuezhijun',charset = 'utf8')
        self.c = self.conn.cursor()

    def process_item(self, item, spider):
        name = item['name']
        desc = item['desc']
        try:
            self.c.execute('insert into DY values (%s,%s)',
                      (name, desc))
        except Exception as  e:
            print(e)
            self.conn.rollback()
        self.conn.commit()
        return item
    def close_spider(self,spider):
        self.c.close()
        self.conn.close()

5.设置一下配置文件

ITEM_PIPELINES = {
   'dianying.pipelines.DianyingPipeline': 300,
}

ROBOTSTXT_OBEY = False
LOG_LEVEL = 'ERROR'

USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.132 Safari/537.36'