爬取全部的校园新闻

作业要求来源：https://edu.cnblogs.com/campus/gzcc/GZCC-16SE2/homework/2941

要求：

1、从新闻url获取新闻详情

2、从列表页的url获取新闻url

3、生成所页列表页的url并获取全部新闻

4、设置合理的爬取间隔

5、用pandas做简单的数据处理并保存成csv和sql文件

源代码：

import requests
from bs4 import BeautifulSoup
from datetime import datetime
import re
import pandas as pd
import time
import random
import sqlite3

newsUrl = 'http://news.gzcc.cn/html/2005/xiaoyuanxinwen_0710/4.html'
listUrl = 'http://news.gzcc.cn/html/xiaoyuanxinwen/'


def click(url):
    id = re.findall('(d{1,5})', url)[-1]
    clickUrl = 'http://oa.gzcc.cn/api.php?op=count&id={}&modelid=80'.format(id)
    resClick = requests.get(clickUrl)
    newsClick = int(resClick.text.split('.html')[-1].lstrip("('").rstrip("');"))
    return newsClick


def newsdt(showinfo):
    newsDate = showinfo.split()[0].split(':')[1]
    newsTime = showinfo.split()[1]
    newsDT = newsDate + ' ' + newsTime
    dt = datetime.strptime(newsDT, '%Y-%m-%d %H:%M:%S')
    return dt


def anews(url):#从新闻url获取新闻详情： 字典,anews
    newsDetail = {}
    res = requests.get(url)
    res.encoding = 'utf-8'
    soup = BeautifulSoup(res.text, 'html.parser')
    newsDetail['newsTitle'] = soup.select('.show-title')[0].text
    showinfo = soup.select('.show-info')[0].text
    newsDetail['newsDT'] = newsdt(showinfo)
    newsDetail['newsClick'] = click(newsUrl)
    return newsDetail


def alist(url):#从列表页的url获取新闻url：列表append(字典) alist
    res = requests.get(listUrl)
    res.encoding = 'utf-8'
    soup = BeautifulSoup(res.text, 'html.parser')
    newsList = []
    for news in soup.select('li'):
        if len(news.select('.news-list-title')) > 0:
            newsUrl = news.select('a')[0]['href']
            newsDesc = news.select('.news-list-description')[0].text
            newsDict = anews(newsUrl)
            newsDict['description'] = newsDesc
            newsList.append(newsDict)
    return newsList


alist(listUrl)

alist(newsUrl)
res = requests.get('http://news.gzcc.cn/html/xiaoyuanxinwen/')
res.encoding = 'utf-8'
soup = BeautifulSoup(res.text, 'html.parser')

for news in soup.select('li'):
    if len(news.select('.news-list-title')) > 0:
        newsUrl = news.select('a')[0]['href']
        print(anews(newsUrl))

allnews = []
for i in range(97, 107):#爬取学号尾数开始的10个列表页
    listUrl = 'http://news.gzcc.cn/html/xiaoyuanxinwen/{}.html'.format(i)
    allnews.extend(alist(listUrl))

print("allnewsLength={}".format(len(allnews)))
print(allnews)

res = requests.get('http://news.gzcc.cn/html/xiaoyuanxinwen/')
res.encoding = 'utf-8'
soup = BeautifulSoup(res.text, 'html.parser')
for news in soup.select('li'):
    if len(news.select('.news-list-title')) > 0:
        newsUrl = news.select('a')[0]['href']
        print(anews(newsUrl))

s1 = pd.Series([100, 23, 'bugingcode'])
print(s1)
pd.Series(anews)
newsdf = pd.DataFrame(allnews)
for i in range(5):
    print(i)
    time.sleep(random.random() * 3)#设置爬取的时间间隔
    print(newsdf)

newsdf.to_csv(r'D:py_filegzcc.csv',encoding='utf_8_sig')#保存成csv格式，为避免乱码，设置编码格式为utf_8_sig

with sqlite3.connect(r'D:py_filegzccnewsdb.sqlite') as db:#保存文件为sql
    newsdf.to_sql('gzccnewsdb',db)

结果：

1、新闻详情：

2、新闻列表：

3、保存成csv文件：

4、保存成为sql文件