结果演示
代码:
#encoding:utf-8
# __author__ = 'donghao'
# __time__ = 2018/12/24 15:20
import requests
import threading
import urllib.request
import urllib3
import os
import re
import time
from lxml import etree
from queue import Queue
#负责解析图片
class Producer(threading.Thread):
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64; rv:57.0) Gecko/20100101 Firefox/57.0'
}
def __init__(self,img_queue,page_queue,*args,**kwargs):
super(Producer, self).__init__(*args,**kwargs)
self.img_queue = img_queue
self.page_queue = page_queue
def run(self):
while True:
if self.page_queue.empty():
break
url = self.page_queue.get()
self.parse_page(url)
def parse_page(self,url):
resp = requests.get(url=url,headers=self.headers)
text = resp.text
html = etree.HTML(text)
imgs = html.xpath("//div[@class='page-content text-center']//img[@class!='gif']")
for img in imgs:
#获取图片url
img_url = img.get('data-original')
#获取图片Url的后缀名
end = os.path.splitext(img_url)[1]
#替换掉url中特殊字符
end = re.sub(r'[,。??,/\·]','',end)
# 获取图片描述,并加上后缀
name = img.get('alt')+end
#存储图片url和文件名队列
self.img_queue.put((img_url,name))
#负责下载图片
class Consumer(threading.Thread):
def __init__(self,img_queue,page_queue,*args,**kwargs):
super(Consumer, self).__init__(*args,**kwargs)
self.img_queue = img_queue
self.page_queue = page_queue
def run(self):
while True:
if self.img_queue.empty() and self.page_queue.empty():
break
img_url,filename = self.img_queue.get()
urllib.request.urlretrieve(img_url, 'images/' + filename)
print(filename+'张图片下载完成')
def main():
#爬取10页
start = time.time()
image_queue = Queue(1000)
page_queue = Queue(100)
tsk = []
for x in range(1,10):
url = 'http://www.doutula.com/photo/list/?page=%d'%x
#存储页面信息
page_queue.put(url)
for x in range(5):
t = Producer(image_queue,page_queue)
t.start()
tsk.append(t)
for x in range(5):
t = Consumer(image_queue,page_queue)
t.start()
tsk.append(t)
#终止运行,统计时间
for t in tsk:
t.join()
end = time.time()
print('耗时:%0.002fs' % (end - start))
if __name__ == '__main__':
main()