爬虫————以表格的形式存储

代码参考：https://aistudio.baidu.com/aistudio/projectdetail/467668

                  https://www.cnblogs.com/feifeifeisir/p/10627474.html

本次爬虫主要是通过正则表达式获取想要的信息，以表格的形式存储

参考代码的过程中，加入了自己的一些代码形式，运行过程中有过错误

re.S的运用

import urllib
from urllib import request, error
import re
from bs4 import BeautifulSoup
import xlwt
import time

def askURL(url):
    headers = {
    "User-Agent":"Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.120 Safari/537.36"
              }
    request = urllib.request.Request(url,headers = headers)#发送请求
    try:
        response = urllib.request.urlopen(request)#取得响应
        html= response.read()#获取网页内容
        # print (html)
    except urllib.error.URLError as e:
        if hasattr(e,"code"):
            print (e.code)
        if hasattr(e,"reason"):
            print (e.reason)
    return html



def getData(baseurl):
    remove = re.compile(r'                            |
|</br>|.*')
    datalist = []

    for i in range(0, 10):
        url = baseurl + str(i * 25)
        html = askURL(url)
        soup = BeautifulSoup(html, "html.parser")
        for item in soup.find_all('div', class_='item'):  # 找到每一个影片项
            data = []
            item = str(item)  # 转换成字符串
            # 影片详情链接
            link = re.findall('<a href="(.*?)">', item)[0]
            data.append(link)  # 添加详情链接
            imgSrc = re.findall('<img.*src="(.*?)"', item, re.S)[0]
            data.append(imgSrc)  # 添加图片链接
            titles = re.findall('<span class="title">(.*)</span>', item)
            # 片名可能只有一个中文名，没有外国名
            if (len(titles) == 2):
                ctitle = titles[0]
                data.append(ctitle)  # 添加中文片名
                otitle = titles[1].replace("/", "")  # 去掉无关符号
                data.append(otitle)  # 添加外国片名
            else:
                data.append(titles[0])  # 添加中文片名
                data.append(' ')  # 留空

            rating = re.findall('<span class="rating_num" property="v:average">(.*)</span>', item)[0]
            data.append(rating)  # 添加评分
            judgeNum = re.findall('<span>(d*)人评价</span>', item)[0]
            data.append(judgeNum)  # 添加评论人数
            inq = re.findall('<span class="inq">(.*)</span>', item)

            # 可能没有概况
            if len(inq) != 0:
                inq = inq[0].replace("。", "")  # 去掉句号
                data.append(inq)  # 添加概况
            else:
                data.append(' ')  # 留空
            bd = re.findall('<p class="">(.*?)</p>', item, re.S)[0]
            bd = re.sub(remove, "", bd)
            bd = re.sub('<br(s+)?/?>(s+)?', " ", bd)  # 去掉<br >
            bd = re.sub('/', " ", bd)  # 替换/
            data.append(bd.strip())
            datalist.append(data)

    time.sleep(5)
    return datalist

# 将相关数据写入excel表格中
def saveData(datalist, savepath):
    book = xlwt.Workbook(encoding='utf-8', style_compression=0)
    sheet = book.add_sheet('豆瓣电影Top250', cell_overwrite_ok=True)
    col = ('电影详情链接', '图片链接', '影片中文名', '影片外国名', '评分', '评价数', '概况', '相关信息')
    for i in range(0, 8):
        sheet.write(0, i, col[i])
    for i in range(0, 250):
        data = datalist[i]  # 列名
        for j in range(0, 8):
            sheet.write(i + 1, j, data[j])  # 数据
    book.save(savepath)


# savepath = u'/home/cxf/ML_1/豆瓣电影Top250.xlsx'
# saveData(datalist,savepath)


def main():
    print("爬虫开始了........................")
    baseurl = 'https://movie.douban.com/top250?start='
    datalist = getData(baseurl)
    savepath = '1豆瓣电影Top250_2.xlsx'
    saveData(datalist, savepath)


if __name__ == "__main__":
    main()
    print("爬取完成，请查看.xlsx文件")