没有用线程池进行的视频爬取代码如下:
1 from lxml import etree 2 from uuid import uuid4 3 from urllib import request 4 import requests 5 import re 6 7 url = 'https://www.pearvideo.com/category_3' 8 headers = { 9 'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36' 10 } 11 page_text = requests.get(url=url, headers=headers).text 12 # 解析：详情url 视频的标题 13 tree = etree.HTML(page_text) 14 li_list = tree.xpath('//ul[@id="listvideoListUl"]/li') 15 for li in li_list: 16 filename = uuid4() 17 detail_url = 'https://www.pearvideo.com/' + li.xpath('./div/a/@href')[0] 18 detail_text = requests.get(url=detail_url, headers=headers).text 19 # 解析：视频的url 20 exp ='srcUrl="(.*?)"' 21 video_url = re.findall(exp, detail_text, re.S)[0] 22 request.urlretrieve(url=video_url, filename=f'{filename}.mp4')
基于线程池的视频爬取,爬取梨视频中的视频数据
使用并发机制进行多线程的数据下载：数据下载和io存储中使用并发机制
1 from lxml import etree 2 from uuid import uuid4 3 from urllib import request 4 from multiprocessing.dummy import Pool 5 import requests 6 import re 7 8 # 实例化一个线程池对象, 参数表示的是开启线程的个 9 pool = Pool(10) 10 url = 'https://www.pearvideo.com/category_3' 11 headers = { 12 'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36' 13 } 14 page_text = requests.get(url=url, headers=headers).text 15 # 解析：详情url 视频的标题 16 tree = etree.HTML(page_text) 17 li_list = tree.xpath('//ul[@id="listvideoListUl"]/li') 18 video_url_list = [] 19 20 for li in li_list: 21 filename = uuid4() 22 23 detail_url = 'https://www.pearvideo.com/' + li.xpath('./div/a/@href')[0] 24 detail_text = requests.get(url=detail_url, headers=headers).text 25 # 解析：视频的url 26 exp = 'srcUrl="(.*?)"' 27 video_url = re.findall(exp, detail_text, re.S)[0] 28 # 将解析到的视频的url存储到video_url_list列表中 29 video_url_list.append(video_url) 30 # 使用线程池进行视频数据的并发下载 31 pool.map(lambda u: request.urlretrieve(url=u, filename=f'{filename}.mp4'), 32 video_url_list) 33 pool.close() 34 pool.join()