beautifulsoup爬取糗事百科

# _*_ coding:utf-8 _*_
import urllib2
from bs4 import BeautifulSoup

user_agent = "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.221 Safari/537.36 SE 2.X MetaSr 1.0"
headers = {'User-Agent':user_agent}
url = "https://www.qiushibaike.com"
# 爬取article链接的content内容
def getContent(article_url, headers):
    request = urllib2.Request(article_url,data=None,headers=headers)
    response = urllib2.urlopen(request,timeout=60)
    html = response.read().decode('utf-8')
    soup = BeautifulSoup(html,'html.parser')
    contents= soup.select('.content')[0].strings
    print u"内容："
    for content in contents:
        print u"%s" % content.strip()
    print '
'
# ----------------------------
# 获取"https://www.qiushibaike.com/hot/page/1/"页面的作者、好笑、评论，文章地址信息
def getData(url, headers,pages=1):
    for page in range(1,pages+1):
        page_url = url + "/hot/page/" + str(page)
        print "正在爬取第 %s 页+++" % page
        request = urllib2.Request(page_url,data=None,headers=headers)
        response = urllib2.urlopen(request,timeout=60)
        html = response.read().decode('utf-8')
        # print html
        soup = BeautifulSoup(html,'html.parser')

        authors = soup.select('h2')
        smile_nums = soup.select('.stats-vote > .number')
        comment_nums = soup.select('.stats-comments > .qiushi_comments > .number')
        article_urls = soup.select('.contentHerf')
        for i in range(25):
            print "正在爬取第 %s 页的第 %s 条数据---" % (page,i+1)
            author = authors[i].string.strip()
            print u"作者： %s" % author
            funny_num = smile_nums[i].string
            comment_num = comment_nums[i].string
            print u"好笑： %s" % funny_num
            print u"评论： %s"% comment_num
            article_url = article_urls[i]['href']

            article_url = url + article_url
            # print article_url
            getContent(article_url, headers)
# ---------------------------------
getData(url,headers,pages=10)