爬虫_百度图片下载

帮别的院的同学批量下载点图片，并进行简单筛选

import requests
import re
import os
from lxml import etree
import json



def get_html(url, param):
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36'
    }
    response = requests.get(url, params=param, headers=headers)
    response.encoding = response.apparent_encoding
    # 返回json数据,str格式
    json_str = response.text
    response.encoding = 'utf-8'
    return json_str


def parse_page(html):
    data = json.loads(html, strict=False)
    objs = data['data'][:-1]
    urls = []
    for obj in objs:
        url = obj['middleURL']
        key = obj['fromPageTitleEnc']
        # if '纹枯病' in key:
        if '全蚀病' in key:
            urls.append(url)
        # if '叶锈病' in key:
        #     urls.append(url)
        # elif '条锈病' in key:
        #     urls.append(url)
        else:
            print('该标题被筛选掉: '+key)
    print(len(urls))
    a = len(urls)
    return urls, a
    # return url_list


def run(keyword, path):
    url = "https://image.baidu.com/search/acjson"
    # https://image.baidu.com/search/acjson?ipn=rj&tn=resultjson_com&word=小麦纹枯病矢量图大图&pn=30
    i = 0
    sum_pic = 0
    for j in range(30, 1800, 30):
        params = {
                    "ipn": "rj", 
                    "tn": "resultjson_com", 
                    "word": keyword, 
                    "pn": str(j)
                }
        html = get_html(url, params)
        lists, num_pic = parse_page(html)
        sum_pic += num_pic
  
        for item in lists:
            try:
                img_data = requests.get(item, timeout=10).content
                with open(path + "/" + str(i) + ".jpg", "wb") as f:
                    f.write(img_data)
                    f.close()
                i = i+1
            except requests.exceptions.ConnectionError:
                print('can not download')
                continue

def make_dir(keyword):
    path = "images/"
    path = path+keyword
    is_exists = os.path.exists(path)
    if not is_exists:
        os.makedirs(path)
        return path
    else:
        print(path + '目录已存在')
        return path


def main():
    # keyword = '小麦纹枯病矢量图大图'
    keyword = '小麦全蚀病'
    path = make_dir(keyword)
    run(keyword, path)



if __name__ == '__main__':
    main()