1：asynio 基本介绍与使用
# async 函数创建 # async函数创建与运行
async def funcC(): print("A方法需要等待C方法执行完毕") time.sleep(2) print("C方法完毕") # 如果在执行A里面想异步执行其他任务，就异步执行其他任务 asyncio.run(funcC()) async def funcA(): print("AAAAAAA") asyncio.run(funcC()) # A方法需要等待C方法执行完毕 在等待C方法执行的时间 先执行B方法 print("A发送完毕") async def funcB(): print("BBBBBBB") print("B发送完毕") # 当你要异步执行某个函数 这个函数必须是async函数 asyncio.run(funcB()) # 创建任务池， 进行异步执行任务池任务 async def run(): # 创建任务池 tasks_list = [ asyncio.create_task(funcA()), asyncio.create_task(funcB()) ] # 异步执行任务池里面的任务 done,p = await asyncio.wait(tasks_list) print(done,p) # 执行run函数 asyncio.run(run())
2：当第三方库不支持异步时，使用协程异步来创建异步效果，例子：
async def download_image(ur1) : # 发送网络请求，下载图片（(遇到网络下载图片的Io请求，自动化切换到其他任务) loop = asyncio.get_event_loop() future = loop.run_in_executor(None,requests.get,ur1) # run_in_executor创建线程池来执行函数 response = await future print('下载完成') #图片保存到本地文件 file_name = ur1.rsplit('_')[-1] with open(file_name[-6:], mode='wb' ) as file_object: file_object.write(response.content) async def download_image(ur1) : # 发送网络请求，下载图片（(遇到网络下载图片的Io请求，自动化切换到其他任务) loop = asyncio.get_event_loop() future = loop.run_in_executor(None,requests.get,ur1) # run_in_executor创建线程池来执行函数 response = await future print('下载完成') #图片保存到本地文件 file_name = ur1.rsplit('_')[-1] with open(file_name[-6:], mode='wb' ) as file_object: file_object.write(response.content) # 请求路由池 ur1_list = [ "http://img10.360buyimg.com/seckillcms/s250x250_jfs/t1/138825/10/19608/59466/5fe29cfaE90f6af35/d055957f818b9efd.jpg", "http://img11.360buyimg.com/seckillcms/s250x250_jfs/t1/158318/34/17929/182997/606ff641Ee3ba3345/eadceab9bacba3aa.jpg", "http://img13.360buyimg.com/seckillcms/s250x250_jfs/t1/174834/11/2912/180533/607017ecEaf816e5f/b213be4e188cf9af.jpg" ] # 添加到任务池d tasks = [] for url in ur1_list: tasks.append(download_image(url))
# 执行任务 loop = asyncio.get_event_loop() loop.run_until_complete(asyncio.wait(tasks))
3：asynio 异步redis 与异步mysql
3-1 ：基本使用例子，
import redis import asyncio # 必须安装的模块 import aioredis async def execute(address,password): print("开始执行") # 网络Io操作:创建redis连接 如果有密码后面跟上create_redis(address,password) redis = await aioredis.create_redis(address) #网络Io操作:在redis中设置值car await redis.set('car',"ssss") #网络工o操作:去redis中获取值 result = await redis.get('car', encoding="utf-8") print(result) redis.close() #网络Io操作:关闭redis连接await redis.wait_closedo print("结束") asyncio.run(execute("redis://127.0.0.1:6379","") )
3-2 ：连接两台不同服务器的redis执行操作
import redis import asyncio import aioredis async def execute_main(address,password): print("开始执行") # 网络Io操作:先去连接传入的第一台，遇到IO则切换任务，去连接第二台 | create_redis(address,password) redis = await aioredis.create_redis_pool(address) #网络Io操作:在redis中设置值car await redis.set('car',"ssss") #网络工o操作:去redis中获取值 result = await redis.get('car', encoding="utf-8") print(result) redis.close() #网络Io操作:关闭redis连接await redis.wait_closedo print("结束") # 创建连接池 redis_list = [ execute_main("redis://127.0.0.1:6379",""), execute_main("redis://127.0.0.2:6379","") ] # 执行异步操作 asyncio.run(asyncio.wait(redis_list))
3-3 ：连接两台不同服务器的mysql执行操作
import asyncio import aiomysql async def execute(host,password): print("开始"，host) #网络IO操作︰先去连接47.93.40.197，遇到Io则自动切换任务，去连接47.93.40.198:6379 conn = await aiomysq1.connect(host=host，port=3306，user=' root'， password=password,db='mysql') #网络Io操作:遇到IO会自动切换任务 cur = await conn.cursor() #网络Io操作:遇到Io会自动切换任务 await cur.execute("SELECT Host,user FROM user") #网络Io操作:遇到Io会自动切换任务 result = await cur.fetcha11oprint(result) #网络Io操作:遇到Io会自动切换任务await cur.close( conn.close( print("结束"，host) task_list = [ execute('47.93.41.197',"root!2345"), execute('47.93.40.196',"root!2345") ] asyncio.run(asyncio.wait(task_1ist))
3：异步爬虫 aiohttp
import aiohttp import asyncio # 内部事件循环自动化会变成uvloop import uvloop asyncio.set_event_loop_policy(uvloop.EventLoopPolicy()) import random # 随机UA的请求头 def head(): user_agent = ["Mozilla/5.0 (Windows NT 10.0; WOW64)", 'Mozilla/5.0 (Windows NT 6.3; WOW64)', 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11', 'Mozilla/5.0 (Windows NT 6.3; WOW64; Trident/7.0; rv:11.0) like Gecko', 'Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1500.95 Safari/537.36', 'Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; rv:11.0) like Gecko)', 'Mozilla/5.0 (Windows; U; Windows NT 5.2) Gecko/2008070208 Firefox/3.0.1', 'Mozilla/5.0 (Windows; U; Windows NT 5.1) Gecko/20070309 Firefox/2.0.0.3', 'Mozilla/5.0 (Windows; U; Windows NT 5.1) Gecko/20070803 Firefox/1.5.0.12', 'Opera/9.27 (Windows NT 5.2; U; zh-cn)', 'Mozilla/5.0 (Macintosh; PPC Mac OS X; U; en) Opera 8.0', 'Opera/8.0 (Macintosh; PPC Mac OS X; U; en)', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.12) Gecko/20080219 Firefox/2.0.0.12 Navigator/9.0.0.6', 'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; Win64; x64; Trident/4.0)', 'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; Trident/4.0)', 'Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; WOW64; Trident/6.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; InfoPath.2; .NET4.0C; .NET4.0E)', 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Maxthon/4.0.6.2000 Chrome/26.0.1410.43 Safari/537.1 ', 'Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; WOW64; Trident/6.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; InfoPath.2; .NET4.0C; .NET4.0E; QQBrowser/7.3.9825.400)', 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:21.0) Gecko/20100101 Firefox/21.0 ', 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.92 Safari/537.1 LBBROWSER', 'Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; WOW64; Trident/6.0; BIDUBrowser 2.x)', 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.11 TaoBrowser/3.0 Safari/536.11', 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36', ] user_agent = random.choice(user_agent) headers = { 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3', 'Accept-Encoding': 'gzip, deflate, br', 'Accept-Language': 'en-US,en;q=0.9', 'User-Agent': user_agent, # 设置随机请求头 'Connection': 'keep-alive', 'Upgrade-Insecure-Requests': '1', 'Sec-Fetch-Mode': 'navigate', 'Sec-Fetch-Site': 'none', } return headers # 执行异步请求页面 async def fetch(session,url): # 上下文交接： 路由 SSL证书FLASE 请求头 async with session.get(url,verify_ssl=False,headers=head()) as response: #图片保存到本地文件 file_name = url.rsplit('_')[-1] with open(file_name[-6:], mode='wb') as file_object: file_object.write(response.content) print ("下载完成:",ur1) # 创建异步任务池，进行IO执行 async def main(): # 连接池执行任务 async with aiohttp.ClientSession() as session: print("开始") ur1_list = [ "http://img10.360buyimg.com/seckillcms/s250x250_jfs/t1/138825/10/19608/59466/5fe29cfaE90f6af35/d055957f818b9efd.jpg", "http://img11.360buyimg.com/seckillcms/s250x250_jfs/t1/158318/34/17929/182997/606ff641Ee3ba3345/eadceab9bacba3aa.jpg", "http://img13.360buyimg.com/seckillcms/s250x250_jfs/t1/174834/11/2912/180533/607017ecEaf816e5f/b213be4e188cf9af.jpg" ] # 创建执行任务 tasks = [] for url_l in ur1_list: print(url_l) # 请求连接池 请求路由 tasks.append( asyncio.create_task( fetch(session,url_l))) # 执行异步任务 done, pending = await asyncio.wait(tasks) # jupyter 已经启动循环事件 无需自己再run Pycahrm里面则可使用asyncio.run( main() ) await main()
