初探爬虫小练习，爬取知乎圆桌内容标题

先上代码：

# -*- coding: utf-8 -*-
import urllib
import urllib2
import re
import json


class Spider:

    def __init__(self):
        # 记录爬取每页的开始
        self.offset = 0
        self.title = []

    def spider_page(self):
        url = 'https://www.zhihu.com/r/roundtables?offset={offset}'
        headers = {
                      "Host": "www.zhihu.com",
                      "User-Agent": "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:48.0) Gecko/20100101 Firefox/48.0",
                      "Accept": "*/*",
                      "Accept-Language": "zh-CN;en-US,en;q=0.5",
                      "X-Requested-With": "XMLHttpRequest",
                      "Referer": "https://www.zhihu.com/roundtable",
                      "Connection": "keep-alive",
                      "Accept-Charset": "utf-8"
        }

        url = url.format(offset=str(self.offset))
        req = urllib2.Request(url, headers=headers)
        html = urllib2.urlopen(req).read().decode('utf-8').encode('utf-8')
        html = json.loads(html)

        before = self.offset
        try:
            next_offset = re.findall(r'offset=(.*)', html['paging']['next'])
            self.offset = int(next_offset[0])
        except:
            self.offset = 0

        titles = html['htmls']
        temp = ""
        for title in titles:
            temp += title

        return {"before": before , "temp": temp}

    def find_title(self, titles):
        titles = titles[:]
        title = re.findall(r'<span.*?class="name">(.*?)</span></span></a>', titles)

        for item in title:
            self.title.append(item)


if __name__ == '__main__':
    s = Spider()
    for i in range(6):
        results = s.spider_page()
        s.find_title(results["temp"])

    for item in s.title:
        print item

总结：

    总体来说，这段代码还是比较好理解的，有以下几点需要注意：

1.     输出乱码，看返回报文首部有Content-Encoding:"gzip"，那么你就要注意你的请求首部信息Accept-Ecoding的内容是否能解压，或者直接告知服务器直接不加密传输。
2.     在就是对返回的json对象调用json.loads进行解码成json数据类型，否则数据是以字符串的形式存在的。