05_多协程爬取斗鱼高颜值美女图片-2.0版本

import requests
import re
# from bs4 import BeautifulSoup
from urllib import request
# import threading
import gevent
from gevent import monkey

monkey.patch_all()

def get_html_text(url):
    try:
        hd = {'User-Agent':'Mozilla/5.0'} #添加伪装浏览器头部信息
        r = requests.get(url, timeout=10, headers = hd)
        r.raise_for_status()
        r.encoding = r.apparent_encoding
        print(len(r.text))
        return r.text
    except Exception as result:
        print('错误类型：', result)


def html_text_parser(img_list, html):

# 下面是修改的重点部分，采用了正则表达式，没有采用bs4
# 技术大神可以研究一下斗鱼这个网页的源代码，真正的图片信息都存储在后面，不是传统的# # html,我还没怎么接触过前端的知识，不知怎么使用bs4,所以使用了正则表达式。
    
    img_pat = r'"rsw+":"(.*?g)"'  
    links = re.compile(img_pat, re.S).findall(html)
    print(len(links))
    print(links)
    for link in links:
        if link:
            img_list.append(link)
    return img_list




def get_douyu_img(Img_list):
    for i,j in enumerate(Img_list):
        # name = j.split('.')[-1]
        try: #异常捕获，如果链接不能访问，退出当前一次循环，进入下一次循环
            r = request.urlopen(j)
            ima_content = r.read()
            path = str(i)
            with open(path, 'wb') as f:
                f.write(ima_content)
        except:
            continue
def main():
    url = 'https://www.douyu.com/g_yz'
    html = get_html_text(url)
    img_list = list()
    Img_list = html_text_parser(img_list, html)
    # print(Img_list)
    #t1 = threading.Thread(target=get_html_text, args=(url,))
    #t2 = threading.Thread(target=html_text_parser, args=(img_list,html))
    #t3 = threading.Thread(target=get_douyu_img, args=(Img_list,))
    #t1.start()
    #t2.start()
    #t3.start()
    gevent.joinall([
        gevent.spawn(get_html_text, url),
        gevent.spawn(html_text_parser, img_list, html),
        gevent.spawn(get_douyu_img, Img_list)
    ])


if __name__ == '__main__':
    main()