scrapy--doutu

　　年轻人都爱斗图,可是有时候斗图的数量比较少.就想办法收藏其他的人图片,然而只要能在doutula网页里爬取图片,是一件很棒的的事,看别人写爬斗图的爬虫程序有点麻烦,自己也来动动手,简单,实用.给大家分享一下。先给大家看看成果,喜欢的话,就开始吧!!!

1.doutu.py

# -*- coding: utf-8 -*-
import scrapy
from doutu.items import DoutuItem
from scrapy.linkextractors import LinkExtractor
import pdb

class DoutuSpider(scrapy.Spider):
    name = 'Doutu'
    #allowed_domains = ['www.doutula.com']
    start_urls = ['http://www.doutula.com/']

    def parse(self, response):
        le = LinkExtractor(restrict_css='div.col-sm-9')
        links = le.extract_links(response)
        for link in links[1:4]:
            yield scrapy.Request(link.url,callback=self.parse_pager)

        le1 = LinkExtractor(restrict_css='ul.pagination')
        links1 = le1.extract_links(response)
        for link1 in links1:
            yield scrapy.Request(link1.url,callback=self.parse)

    def parse_pager(self,response):
        le2 = LinkExtractor(restrict_css='div.pic-content')
        links2 = le2.extract_links(response)
        for link2 in links2:
            yield scrapy.Request(link2.url,callback=self.parse_img)

    def parse_img(self,response):
        doutu = DoutuItem()
        doutu['image_urls'] = response.xpath('//div[@class="swiper-slide"]//img/@src').extract()[0]items.py

2.items.py

import scrapy

class DoutuItem(scrapy.Item):
    image_urls = scrapy.Field()
    images = scrapy.Field()

3.pipelines.py

import pdb
import scrapy
from scrapy.pipelines.images import ImagesPipeline

class DoutuPipeline(ImagesPipeline):
    def get_media_requests(self, item, info):

        yield scrapy.Request(item['image_urls'])

    def item_completed(self, results, item, info):

        image_paths = [x['path'] for ok, x in results if ok]  # ok判断是否下载成功

        if not image_paths:
            raise DropItem("Item contains no images")
        return item

4.settings.py

IMAGES_STORE = r'C:Desktopdoutula'     #图片存储文件名

USER_AGENT ={       #设置浏览器的User_agent
    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/22.0.1207.1 Safari/537.1",
    "Mozilla/5.0 (X11; CrOS i686 2268.111.0) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.57 Safari/536.11",
    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1092.0 Safari/536.6",
    "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1090.0 Safari/536.6",
    "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/19.77.34.5 Safari/537.1",
    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.9 Safari/536.5",
    "Mozilla/5.0 (Windows NT 6.0) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.36 Safari/536.5",
    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3",
    "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_0) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3",
    "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1062.0 Safari/536.3",
    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1062.0 Safari/536.3",
    "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3",
    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3",
    "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3",
    "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.0 Safari/536.3",
    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.24 (KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24",
    "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/535.24 (KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24"
}

CONCURRENT_REQUESTS = 16    #同时来16个请求
DOWNLOAD_DELAY = 0.2        #0.2s后开启处理第一个请求

IMAGES_THUMBS = {#缩略图的尺寸，设置这个值就会产生缩略图
    'small': (50, 50),      #full/small
    'big': (200, 200),      #full/big
}

ROBOTSTXT_OBEY = False      #不遵守robot.txt条约

COOKIES_ENABLED = False     #禁用cookies

ITEM_PIPELINES = {
    'doutu.pipelines.DoutuPipeline': 1,     #设置优先级1-1000
}

有遇到问题的小伙伴,可以在下面留言.欢迎