爬取图片（一）

源码：

import requests
from lxml import etree
from urllib import request
import os

# 获取页面图集链接
def get_url_list(page,headers):
    url = 'http://www.meizitu.com/a/more_{}.html'.format(page)
    response = requests.get(url,headers=headers)
    response.encoding = 'gbk'
    # print(response.text)
    html_ele = etree.HTML(response.text)
    ele_list = html_ele.xpath('//ul[@class="wp-list clearfix"]/li')
    page_list = []
    for ele in ele_list:
        url = ele.xpath('./div/div/a/@href')
        # print(url)
        page_list.append(url[0])
    return page_list

# 保存图片
def get_pictures(url,headers):
    response = requests.get(url,headers=headers)
    response.encoding = 'gbk'
    html_ele = etree.HTML(response.text)
    dir = html_ele.xpath('//div[@class="metaRight"]/h2/a')[0].text
    dir_name = '妹子图/' + dir
    if not os.path.exists(dir_name):
        os.makedirs(dir_name)
    url_list = html_ele.xpath('//div[@id="picture"]/p/img/@src')
    for url in url_list:
        try:
            name = url.split('/')[-1]
            filename = dir_name + '/' + name
            if not os.path.exists(filename):
                request.urlretrieve(url,filename)
                print(filename)
        except:
            pass


if __name__ == '__main__':
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36'
    }
    for page in range(1,73):
        page_list = get_url_list(page,headers)
        for url in page_list:
            get_pictures(url,headers)