python抓取伯乐在线的全部文章，对标题分词后存入mongodb中

依赖包：

1.pymongo

2.jieba

# -*- coding: utf-8 -*-

"""
@author: jiangfuqiang
"""
from HTMLParser import HTMLParser
import urllib2
import sys
import pymongo
import time
import jieba
import traceback

default_encoding = 'utf-8'
if sys.getdefaultencoding() != default_encoding:
    reload(sys)
    sys.setdefaultencoding(default_encoding)
class FetchJobble(HTMLParser):

    def __init__(self):
        HTMLParser.__init__(self)
        self.isPostThumb = False
        self.isPostMeta = False
        self.isMetaTitle = False
        self.isCategoryTag = False
        self.isComment = False
        self.isexcerpt = False
        self.isReadMore = False
        self.isPicture = False
        self.data = {}
        self.result = []

    def handle_starttag(self,tag,attrs):
        if tag == 'div':
            for key,value in attrs:
                if key == 'class':
                    if value == 'post-thumb':
                        self.isPostThumb = True
                    elif value == 'meta-title':
                        self.isMetaTitle = True
        elif tag == 'a' and self.isPostThumb == True:

            for key, value in attrs:
                if self.isReadMore:
                    if key == 'href':
                        self.data['redmoreLink'] = value
                        self.data['keyword'] = ",".join(jieba.cut(self.data['title']))
                        self.result.append(self.data)
                        self.isPostThumb = False
                        self.isMetaTitle = False
                        self.isReadMore = False
                        self.isCategoryTag = False
                        self.isComment = False
                        self.isexcerpt = False
                        self.isPicture = False

                        self.data = {}
                else:
                    if key == 'class':
                        if value == 'meta-title':
                            self.isMetaTitle = True
                    elif key == 'rel':
                        if value == 'category tag':
                            self.isCategoryTag = True
                    elif key =='href':
                        if value.find('#respond') > 0:
                            self.isComment = True
        elif tag == 'span' and self.isComment == True:
            for key, value in attrs:
                if key == 'class' and value == 'excerpt':
                    self.isexcerpt = True
                elif key == 'class' and value == 'read-more':
                    self.isReadMore = True
        elif tag == 'img' and self.isPostThumb and self.isPostMeta == False:
            for key, value in attrs:
                if key == 'src':
                    self.data['imgSrc'] = value

    def handle_endtag(self,tag):

        pass

    def handle_data(self,data):
         if self.isMetaTitle:
            self.data['title'] = data
            self.isMetaTitle = False
         elif self.isCategoryTag:
             ct = ''
             if 'tag' in self.data.keys() :
                 ct = self.data['tag'] + "," + data
             else:
                 ct = data
             self.data['tag'] = ct
             self.isCategoryTag = False
         elif self.isComment and 'comment' not in self.data.keys():
             self.data['comment'] = data.split(" ")[0]
         elif self.isexcerpt:
             self.data['desc'] = data
             self.isexcerpt = False


    def getResult(self):
        return self.result

if __name__ == "__main__":
    con = pymongo.Connection('localhost', 27017)
    db = con.blog
  
    fetchblog = db.fetch_blog

    url = "http://blog.jobbole.com/all-posts/page/%d"
    count = 1
    flag = False
    headers={
             'User-Agent':'Mozilla/5.0 （Windows； U； Windows NT 6.1； en-US； rv：1.9.1.6） Gecko/20091201 Firefox/3.5.6'}
    while flag == False:
        try:
            req = urllib2.Request(url%count,headers=headers)
            request = urllib2.urlopen(req)
            data = request.read()
            fj = FetchJobble()
            fj.feed(data)
            result = fj.getResult()
            if len(result) < 1:
                flag = True
            else:
                for doc in result:
                    fetchblog.insert(doc)
                print "page is %d"%count
                count += 1

                time.sleep(5)
        except Exception, e:
            traceback.print_exc()
            print "parse error",e