def main():
baseurl = "https://movie.douban.com/top250?start="
#1.爬取网页
datalist = getData(baseurl)
savepath = ".\豆瓣电影Top250.xls"
#3.保存数据
#savapath(savepath)
#askURL("https://movie.douban.com/top250?start=")
#爬取网页
def getData(baseurl):
datalist = []
for i in range(0, 10): #调用获取页面信息的函数 10次
url = baseurl + str(i*25)
html = askURL(url) #保存获取到的网页源码
# 2.逐一解析数据
#????
return datalist
#得到指定一个url的网页内容
def askURL(url):
head = { #模拟浏览器头部信息,向豆瓣服务器发送消息
"User-Agent": "Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.106 Mobile Safari/537.36"
} #用户代理 . 表示高速豆瓣服务器,我们是什么类型的机器/浏览器(本质上是告诉浏览器,我们可以接受什么水品的文件内容
request = urllib.request.Request(url,headers=head)
html = ""
try:
request = urllib.request.urlopen(request)
html = response.read().decode("utf-8")
print(html)
except urllib.error.URLError as e:
if hasattr(e,"code"):
print(e,code)
if hasattr(e,"reason")
print(e,reason)
return html
#保存数据
data saveData(savepath)
print("save.......")