python之中国大学爬虫

#!/usr/bin/env python3
#-*- coding:utf-8 -*-
############################
#File Name: zuihaodaxuepaiming.py
#Author: frank
#Mail: frank0903@aliyun.com
#Created Time:2018-05-15 21:45:55
############################

import requests
from bs4 import BeautifulSoup


#1. 获取html页面的内容
def getHTMLText(url):
    try:
        r = requests.get(url)
        r.raise_for_status()
        #r.encoding = r.apparent_encoding
        r.encoding = 'utf-8'
        return r.text

    except:
        print("爬取异常")
        return ""
    
#2. 从html页面内容中爬取大学排名信息, (排名，学校名称，总分)
def getUnivInfo(uLst, html):
    soup = BeautifulSoup(html, "lxml")
    #获取排名中的大学的数量
    print(len(soup.tbody.find_all("tr", recursive=False)))
    #列出tbody的子节点
    #for tag in soup.tbody.find_all('tr', recursive=False):
    #    #print("name:{},type:{}".format(tag.name, type(tag)))
    #    print(tag.td.string)
    for tr in soup.tbody.find_all('tr', recursive=False,limit=1):
        tds = tr('td')
        #print(tr.prettify())
        #print(tds)
        #print(tds[0].string,tds[1].string, tds[3].string)
        uLst.append([tds[0].string,tds[1].string, tds[3].string])

#3. 显示排名信息
#def printUnivInfo(uLst, num):
#    print("{0:^10}	{1:{3}^10}	{2:^10}".format("排名","学校名称","总分",chr(12288)))
#    for uni in uLst[0:num]:
#        print("{0:^10}	{1:{3}^10}	{2:^10}".format(uni[0],uni[1],uni[2],chr(12288)))

def printUnivInfo(uLst, num):
    ptstr = "{0:^10}	{1:{3}^10}	{2:^10}"
    print(ptstr.format("排名","学校名称","总分",chr(12288)))
    for uni in uLst[0:num]:
        print(ptstr.format(uni[0],uni[1],uni[2],chr(12288))) #中文字符的空格填充 chr(12288)

def main():
    url = "http://www.zuihaodaxue.cn/zuihaodaxuepaiming2016.html"
    uLst = []
    html = getHTMLText(url)
    #print(html)
    print("getHTMLText")
    getUnivInfo(uLst, html)
    print("getUnivInfo")
    printUnivInfo(uLst, 10)

main()

该实例主要是针对 requests 和 BeautifulSoup的应用。