https://www.bilibili.com/video/BV1g7411k7MD?from=search&seid=13649975876676293013
import asyncio import random #需求:有一个crontab调度器,每隔1秒,拉起1个job,要求这些job可以并发爬取网页 async def cron_scheduler(): page = 1 while True: url = "{}/{}".format("https://baidu.com",page) job = cron_job(url) #必须将新携程分离出去,让它和当前携程并发 asyncio.create_task(job) #注册到事件循环 await asyncio.sleep(0) #主动让渡线程 page = page+1 async def cron_job(url): n = random.randint(1,3) #模拟下载延迟 await asyncio.sleep(n) print("下载结束:",url) asyncio.run(cron_scheduler())
下载结束: https://baidu.com/145032
下载结束: https://baidu.com/144970
下载结束: https://baidu.com/144972
下载结束: https://baidu.com/145096
下载结束: https://baidu.com/145097
下载结束: https://baidu.com/145042
下载结束: https://baidu.com/145168
下载结束: https://baidu.com/144960
下载结束: https://baidu.com/145037