1、协程(greenlet模块):在单线程中切换状态得模块 。greenlet模块封装在gevent内部。
import time from greenlet import greenlet def eat1(): print('吃鸡腿1') g2.switch() #线程切换到其他任务 time.sleep(2) print('吃鸡腿2') g2.switch() def eat2(): print('吃饺子1') g1.switch() print('吃饺子2') g1=greenlet(eat1) g2=greenlet(eat2) g1.switch()
2、gevent:在实现切换线程得基础上又实现了规避IO
from gevent import monkey;monkey.patch_all() import time # time socket urllib requests import gevent # greenlet gevent在切换程序的基础上又实现了规避IO from threading import current_thread def func1(): print(current_thread().name) print(123) time.sleep(1) print(456) def func2(): print(current_thread().name) # dummythread print('hahaha') time.sleep(1) print('10jq') g1 = gevent.spawn(func1) # 遇见他认识的io会自动切换的模块 g2 = gevent.spawn(func2) # g1.join() #分别加入和joinall是一样的 # g2.join() gevent.joinall([g1,g2])
3、爬取网页的例子
from gevent import monkey;monkey.patch_all() import time import gevent import requests # 爬取网页 # 10个网页 # 协程函数去发起10个网页的爬取任务 def get_url(url): res = requests.get(url) print(url,res.status_code,len(res.text)) url_lst =[ 'http://www.sohu.com', 'http://www.baidu.com', 'http://www.qq.com', 'http://www.python.org', 'http://www.cnblogs.com', 'http://www.mi.com', 'http://www.apache.org', 'https://www.taobao.com', 'http://www.360.com', 'http://www.7daysinn.cn/' ] start = time.time() for url in url_lst: get_url(url) print(time.time() - start)
from gevent import monkey;monkey.patch_all() import time import gevent import requests # 爬取网页 # 10个网页 # 协程函数去发起10个网页的爬取任务 def get_url(url): res = requests.get(url) print(url,res.status_code,len(res.text)) url_lst =[ 'http://www.sohu.com', 'http://www.baidu.com', 'http://www.qq.com', 'http://www.python.org', 'http://www.cnblogs.com', 'http://www.mi.com', 'http://www.apache.org', 'https://www.taobao.com', 'http://www.360.com', 'http://www.7daysinn.cn/' ] g_lst = [] start = time.time() for url in url_lst: g = gevent.spawn(get_url,url) g_lst.append(g) gevent.joinall(g_lst) print(time.time() - start)