python3 多进程、队列、进程池、协程

多进程

import multiprocessing
import time


def demo1():
    while True:
        print("demo1")
        time.sleep(1)


def demo2():
    while True:
        print("demo2")
        time.sleep(1)


def main():
    p1 = multiprocessing.Process(target=demo1)
    p2 = multiprocessing.Process(target=demo2)
    p1.start()
    p2.start()


if __name__ == '__main__':
    main()

互斥锁

import threading
import time

g_num = 0


def demo1(num):
    global g_num
    # 上锁，如果之前没有被上锁，则上锁成功
    # 如果上锁之前，已经被锁上，那么会堵塞在这里，直到这个锁被解开为止
    mutex.acquire()
    for i in range(num):
        g_num += 1
    # 解锁
    mutex.release()
    print("%s" % g_num)


def demo2(num):
    global g_num
    mutex.acquire()
    for i in range(num):
        g_num += 1
    # 解锁
    mutex.release()
    print("%s" % g_num)


# 创建一个互斥锁，默认没有上锁
mutex = threading.Lock()


def main():
    t1 = threading.Thread(target=demo1, args=(1000000,))
    t2 = threading.Thread(target=demo2, args=(1000000,))

    t1.start()
    t2.start()

    time.sleep(3)
    print("%s" % g_num)


if __name__ == '__main__':
    main()

队列

import multiprocessing


def download_from_web(q):
    # 模拟网上下载的数据
    data = [1, 2, 3, 4, 5, 6]
    for temp in data:
        q.put(temp)


def analysis_data(q):
    # 数据处理
    waiting = list()
    while True:
        data = q.get()
        waiting.append(data)
        if q.empty():
            break

    print(waiting)


def main():
    # 创建队列
    q = multiprocessing.Queue()
    p1 = multiprocessing.Process(target=download_from_web, args=(q,))
    p2 = multiprocessing.Process(target=analysis_data, args=(q,))

    p1.start()
    p2.start()


if __name__ == '__main__':
    main()

进程池

from multiprocessing import Pool
import time
import random
import os


def worker(msg):
    t_start = time.time()
    print("%s 开始执行,进程号是 %d" % (msg, os.getpid()))
    time.sleep(random.random() * 2)
    t_stop = time.time()
    print(msg, "执行完毕，耗时%.2f秒" % (t_stop - t_start))


if __name__ == '__main__':
    # 定义1个进程池，最大进程数为3
    po = Pool(3)
    for i in range(8):
        # 调用目标，传递给目标的参数元组
        po.apply_async(worker, (i,))
    print("-- start --")
    # 关闭进程池，不再接收新的请求
    po.close()
    # 等待po中所有子进程执行完成，必须放在close语句后面
    po.join()
    print("-- end --")

协程

from gevent import monkey
import gevent
import time

monkey.patch_all()


def f1(n):
    for i in range(n):
        print(gevent.getcurrent(), i)
        time.sleep(0.5)


def f2(n):
    for i in range(n):
        print(gevent.getcurrent(), i)
        time.sleep(0.5)


def f3(n):
    for i in range(n):
        print(gevent.getcurrent(), i)
        time.sleep(0.5)


if __name__ == '__main__':
    gevent.joinall([
        gevent.spawn(f1, 3),
        gevent.spawn(f2, 3),
        gevent.spawn(f3, 3)
    ])

 使用协程下载图片

import gevent
import time
from urllib import request
from gevent import monkey
from numba import jit

monkey.patch_all()


@jit
def cbk(a, b, c):
    """ 回调函数
    @a:已经下载的数据块
    @b:数据块的大小
    @c:远程文件的大小
    """
    per = 100.0 * a * b / c
    if per > 100:
        per = 100
    print('
下载已经完成 %.2f%%' % per, end="")


imgurl = ["https://ww3.sinaimg.cn/mw600/006XNEY7gy1fwj4w6ttb6j30hs0k4tc9.jpg",
          "https://ww3.sinaimg.cn/mw600/0073tLPGgy1fx6pxmpwmuj30qo0zkn2x.jpg"]

headers = [
    ("User-agent",
     "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36")]


@jit
def downloader(jpg_name, url):
    opener = request.build_opener()
    opener.addheaders = headers
    request.install_opener(opener)
    request.urlretrieve(url, jpg_name, cbk)
    time.sleep(0.1)


@jit
def main():
    time1 = time.time()
    gevent.joinall([
        gevent.spawn(downloader, "1.jpg", imgurl[0]),
        gevent.spawn(downloader, "2.jpg", imgurl[1]),
    ])
    time2 = time.time()
    print(time2 - time1)


if __name__ == '__main__':
    main()