本文转自 https://blog.csdn.net/baidu_35085676/article/details/68958267
文中的代码，我自己跑了一遍，主要的解析的方式用的是 BeautifulSoup 但是代码跑起来可能会出现一些问题 TimeoutError: [WinError 10060] 由于连接方在一段时间后没有正确答复或连接的主机没有反应，连接尝试失败。个人觉得应该是网站的反爬虫机制的问题，可以尝试一下，变换ip地址。
1 import requests 2 from bs4 import BeautifulSoup 3 import os 4 import time 5 6 all_url = 'http://www.mzitu.com' 7 #http请求头 8 Hostreferer = { 9 'User-Agent': 'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)', 10 'Referer': 'http://www.mzitu.com' 11 } 12 Picreferer = { 13 'User-Agent': 'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)', 14 'Referer': 'http://i.meizitu.net' 15 } 16 #此请求头破解盗链 17 #保存地址 18 path = 'E:/pythonFile/meititu/mei/' 19 #记录文件 20 data = 'E:/pythonFile/meititu/mei/.data' 21 #读取保存记录 22 def get_log(file): 23 page = 1 24 line = 0 25 try: 26 with open(file, 'r') as f: 27 l = f.readline() 28 page, line = [int(i) for i in l.split('|')] 29 except Exception as e: 30 print(e) 31 print('读取记录失败，从初始开始') 32 return page, line 33 34 #保存记录 35 def put_log(file, page, line): 36 try: 37 with open(file, "w") as f: 38 f.write('{}|{}'.format(page, line)) 39 except Exception as e: 40 print('保存记录失败：[{}]'.format(e)) 41 42 #找寻最大页数 43 def find_max_page(): 44 start_html = requests.get(all_url, headers=Hostreferer) 45 soup = BeautifulSoup(start_html.text, "html.parser") 46 page = soup.find_all('a', class_='page-numbers') 47 max_page = page[-2].text 48 max_page = int(max_page) 49 return max_page 50 51 if __name__ == "__main__": 52 same_url = 'http://www.mzitu.com/page/' 53 max_page = find_max_page() 54 page, line = get_log(data) 55 print('从{}页，{}行开始缓存'.format(page, line)) 56 for n in range(page, int(max_page)+1): 57 ul = same_url+str(n) 58 start_html = requests.get(ul, headers=Hostreferer) 59 soup = BeautifulSoup(start_html.text, "html.parser") 60 all_a = soup.find('div', class_='postlist').find_all('a', target='_blank') 61 for lines in range(line, len(all_a)): 62 a = all_a[lines] 63 title = a.get_text() #提取文本 64 if(title != ''): 65 print("准备扒取："+title) 66 #win不能创建带？的目录 67 if(os.path.exists(path+title.strip().replace('?',''))): 68 #print('目录已存在') 69 flag = 1 70 else: 71 os.makedirs(path+title.strip().replace('?','')) 72 flag = 0 73 os.chdir(path + title.strip().replace('?', '')) 74 href = a['href'] 75 html = requests.get(href, headers=Hostreferer) 76 mess = BeautifulSoup(html.text, "html.parser") 77 # 最大也在class='pagenavi'div中的第6个span 78 pic_max = mess.find("div", class_='pagenavi').find_all('span') 79 print(pic_max) 80 print(len(pic_max)) #确定最大页数在第几个span标签，网页可能会变动 81 pic_max = pic_max[6].text #最大页数 82 print(pic_max) 83 if(flag == 1 and len(os.listdir(path+title.strip().replace('?',''))) >= int(pic_max)): 84 print('已经保存完毕，跳过') 85 continue 86 for num in range(1, int(pic_max)+1): 87 while True: 88 pic = href+'/'+str(num) 89 html = requests.get(pic, headers=Hostreferer) 90 mess = BeautifulSoup(html.text, "html.parser") 91 pic_url = mess.find('img', alt=title) 92 if(pic_url): 93 break 94 # print(pic_url['src']) 95 html = requests.get(pic_url['src'], headers=Picreferer) 96 file_name = pic_url['src'].split(r'/')[-1] 97 f = open(file_name, 'wb') 98 f.write(html.content) 99 f.close() 100 put_log(data, n, lines) 101 time.sleep(0.5) 102 print('第',n,'页完成') 103 line = 0 104 time.sleep(10)