scrapy爬取极客学院全部课程

# -*- coding: utf-8 -*-
# scrapy爬取极客学院全部课程
import scrapy
from pyquery import PyQuery as pq
from jike.items import JikeItem

class JikespiderSpider(scrapy.Spider):
    name = "jikespider"
    allowed_domains = ["www.jikexueyuan.com"]
    base_url = 'http://www.jikexueyuan.com/course/?pageNum='

    def start_requests(self):
        for page_num in range(1,96):
            url = self.base_url + str(page_num)
            yield scrapy.Request(url, callback=self.parse_index)

    def parse_index(self, response):
        doc = pq(response.text)
        lis = doc('.lesson-list .cf li').items()
        # pyquery心得, 以为pyquery有点问题而导致无法遍历数据结构,
        # 研究发现是'http:' + item('.lessonimg-box a').attr('href')
        # 的问题, href是相对路径没有得到一个有效的请求链接
        for item in lis:
            detail_url = 'http:' + item('.lessonimg-box a').attr('href')
            yield scrapy.Request(url=detail_url,callback=self.parse_detail)

    def parse_detail(self, response):
        item = JikeItem()
        doc = pq(response.text)
        item['title'] = doc('.lesson-teacher .bc-box h2').text()
        item['time'] = doc('.lesson-teacher .bc-box .timebox').text()
        item['content'] = doc('.lesson-teacher .infor-content').text()

        yield item