• 梨视频,进程池、线程池爬取


    进程池

    import requests, re, time
    from multiprocessing.dummy import Pool
    import random
    
    IpPool = [{'http': '183.147.230.104: 8118'}, {'http': '60.217.64.237: 31923'},
              {'http': '221.193.50.166: 8118'}]
    
    
    url = 'https://www.pearvideo.com/category_loading.jsp?reqType=5&categoryId=9&start=0'
    
    start = time.time()
    
    video_list=[]
    
    ret = requests.get(url)
    reg = '<a href="(.*?)" class="vervideo-lilink actplay">'
    video_urls = re.findall(reg, ret.text)
    print(video_urls)
    for url in video_urls:
        proxy = random.choice(IpPool)
        ret_detail = requests.get('https://www.pearvideo.com/' + url, proxies=proxy)
        print(proxy)
        reg = 'srcUrl="(.*?)",vdoUrl=srcUrl'
        mp4_url = re.findall(reg, ret_detail.text)[0]  # type:str
    
        video_name = mp4_url.rsplit('/', 1)[-1]
    
        dic = {
            'v_name': video_name,
            'v_url': mp4_url
        }
        video_list.append(dic)
    
    print(video_list)
    def get_video(dic):
        url = dic['v_url']
        name = dic['v_name']
        print(f'开始下载{name}')
        video_data = requests.get(url=url)
        print(url)
        with open(name, 'wb') as f:
            for line in video_data.iter_content():
                f.write(line)
            print(f'{name}下载完成')
    
        end = time.time()
        ctime = end - start
        print(ctime)
    
    pools = Pool(12)
    pools.map(get_video, video_list)
    pools.close()
    pools.join()
    

    线程池

    import requests
    import re
    import random
    from concurrent.futures import ThreadPoolExecutor
    
    import time
    start = time.time()
    
    pool = ThreadPoolExecutor(12)
    
    IpPool = [{'http': '183.147.230.104: 8118'}, {'http': '60.217.64.237: 31923'},
              {'http': '221.193.50.166: 8118'}]
    
    url = 'https://www.pearvideo.com/category_loading.jsp?reqType=5&categoryId=9&start=0'
    
    
    video_list=[]
    
    ret = requests.get(url)
    reg = '<a href="(.*?)" class="vervideo-lilink actplay">'
    video_urls = re.findall(reg, ret.text)
    print(video_urls)
    for url in video_urls:
        proxy = random.choice(IpPool)
        ret_detail = requests.get('https://www.pearvideo.com/' + url, proxies=proxy)
        print(proxy)
        reg = 'srcUrl="(.*?)",vdoUrl=srcUrl'
        mp4_url = re.findall(reg, ret_detail.text)[0]  # type:str
    
        video_name = mp4_url.rsplit('/', 1)[-1]
    
        dic = {
            'v_name': video_name,
            'v_url': mp4_url
        }
        video_list.append(dic)
    
    def get_video(dic):
        url = dic['v_url']
        name = dic['v_name']
        print(f'开始下载{name}')
        video_data = requests.get(url=url)
        print(url)
        with open(name, 'wb') as f:
            for line in video_data.iter_content():
                f.write(line)
            print(f'{name}下载完成')
            end = time.time()
            ctime = end - start
            print(ctime)
    
    print(video_list)
    def main():
        for url in video_list:
            done = pool.submit(get_video, url)
    
    
    if __name__ == '__main__':
        main()
        pool.shutdown(wait=True)
    
  • 相关阅读:
    BestCoder Round #67 (div.2) N bulbs(hdu 5600)
    蓝桥杯 基础练习 特殊回文数
    蓝桥杯 基础练习 十进制转十六进制
    蓝桥杯 基础练习 十六进制转十进制
    蓝桥杯 基础练习 十六进制转八进制
    蓝桥杯 入门训练 Fibonacci数列
    Codeforces 600B Queries about less or equal elements(二分查找)
    Codeforces Round #332 (Div. 二) B. Spongebob and Joke
    Codeforces Wilbur and Array
    UVA 10129-Play on Words(欧拉通路)
  • 原文地址:https://www.cnblogs.com/kai-/p/12658461.html
Copyright © 2020-2023  润新知