Python爬虫第一个成功版

爬取http://www.mzitu.com/all里面的图片

import urllib.request
import re
import os

url = 'http://www.mzitu.com/all/' # 爬虫入口
req = urllib.request.Request(url)
req.add_header('Referer','http://www.mzitu.com/all/')
req.add_header('User-Agent','Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36')

html = urllib.request.urlopen(req).read().decode('utf-8') # 得到入口页面的HTML

reg = re.compile(r'<a.+?href="(.+?)"')

hrlist = re.findall(reg,html) # 得到所有图片的页面链接
print(hrlist)

def getImgUrl(url): # 根据图片页面的链接得到图片链接
    html = getHtml(url).decode('utf-8')
    reg = re.compile(r'<img.+?src="(.+?.jpg)"')
    imgUrl = re.findall(reg,html)
    return imgUrl # 返回得到的图片链接

def getHtml(url): # 得到HTML页面信息
    req = urllib.request.Request(url)
    req.add_header('Referer','http://www.mzitu.com/all/')
    req.add_header('User-Agent','Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36')

    res = urllib.request.urlopen(req)
    html = res.read() # 得到入口页面的HTML

    return html # 返回得到的字符串形式的HTML页面

# 根据图片的链接下载图片
def download(url):
    filename = url.split('/')[-1]
    with open(filename,'wb') as f:
        img = getHtml(url)
        f.write(img)

# 保存所有图片
def save_all(folder='mm'):
    os.mkdir(folder)
    os.chdir(folder)
    for each in hrlist:
        imgUrl_list = getImgUrl(each)
        for imgUrl in imgUrl_list:
            download(imgUrl)
        
    

if __name__=='__main__':
    save_all()