python-网络爬虫

1.用requests库和BeautifulSoup4库，爬取校园新闻列表的时间、标题、链接、来源。

import requests   #HTTP库 从html或xml中提取数据
from bs4 import BeautifulSoup #爬虫库BeautifulSoup4
url = requests.get("http://news.gzcc.cn/html/xiaoyuanxinwen/")
url.encoding = "utf-8"
soup = BeautifulSoup(url.text,'html.parser')
#print(soup.head.title.text)
 
 8 #找出含有特定标签的html元素：‘ID’前缀‘#’；‘class’前缀‘.’,其它直接soup.select('p')
for news in soup.select('li'):
    if len(news.select('.news-list-title'))>0:
        #print(news.select('.news-list-title'))          #1.爬出的资料生成一个大列表
        #print(news.select('.news-list-title')[0])       #2.取出了列表中的第一个元素
        #print(news.select('.news-list-title')[0].text)  #3.Bs4库text()负责取出文本

        time = news.select('.news-list-info')[0].contents[0].text #时间
        title = news.select('.news-list-title')[0].text           #标题
        href = news.select('a')[0]['href']                        #链接

        href_text = requests.get(href)                            #链接内容
        href_text.encoding = "utf-8"
        href_soup = BeautifulSoup(href_text.text,'html.parser')
        href_text_body = href_soup.select('.show-content')[0].text
        print(time,title,href,href_text_body)     

2.选一个自己感兴趣的主题，做类似的操作，为“爬取网络数据并进行文本分析”做准备。

import requests  
from bs4 import BeautifulSoup
url= requests.get('https://www.bilibili.com')
url.encoding="utf-8"
soup=BeautifulSoup(url.text,'html.parser')#'html.parser'是HTML的解析器
#print(soup)
for titles in soup.select('.nav-name'):
#    if len(titles.select('.nav-name'))>0:
#        title=titles.select('.nav-name')[0].text
    print(titles.text)
douga = soup.select('#bili_douga')
for rank in donga.select('.rank-list hot-list'):
    

    if len(rank.select('.rank-list hot-list'))>0:
        ranks=rank.select('.rank-item highlight')[0].text
    print(ranks)

 

 并不能爬到“动画”里这个“排行”的url。