• Python多进程与单进程效率对比


    运行环境:Python3 in win10
    先生成200个测试文件

    # generate.py
    i = 0
    while i < 200:
        o = open("test\" + str(i) + ".py", "w")
        content = str(i)
        o.write(content)
        o.close()
        i += 1
    

    多进程拷贝文件

    # multi-pool-copy.py
    from multiprocessing import Pool, Manager
    import time
    import os
    import shutil
    import random
    
    start = time.time()
    
    def copyFile(file_name, old_folder_name, new_folder_name, q):
        time.sleep(random.random())
        shutil.copyfile(old_folder_name + '\' + file_name, new_folder_name + '\' + file_name,)
        q.put(file_name)  # put item into the queue
    
    def main():
        pool = Pool(5)
        q = Manager().Queue()
    
        old_folder_name = input("Please input the folder name you want to copy: ")
    
        new_folder_name = old_folder_name + "-copy"
        os.mkdir(new_folder_name)
    
        file_name_list = os.listdir(old_folder_name)
    
        for file in file_name_list:
            pool.apply_async(copyFile, args=(file, old_folder_name, new_folder_name, q))
        
        cnt = 0
        allLength = len(file_name_list)
        while cnt < allLength:
            message = q.get()
            cnt += 1
            print("
    Copying %s, Process Bar is:%d%%" % (message, (cnt / allLength) * 100), end="")
        print("Copy Done!")
    
    if __name__ == "__main__":
        main()
        end = time.time()
        print("Time-consuming: %#.2fs" % (end-start))
    

    在使用单进程拷贝文件之前,需要手动删除test-copy文件夹

    # single-pool-copy.py
    import time 
    import os
    import shutil
    import random
    
    start = time.time()
    
    def copyFile(file_name, old_folder_name, new_folder_name):
        time.sleep(random.random())
        shutil.copyfile(old_folder_name + '\' + file_name, new_folder_name + '\' + file_name, )
    
    
    def main():
        old_folder_name = input("Please input the folder name you want to copy: ")
    
        new_folder_name = old_folder_name + "-copy"
        os.mkdir(new_folder_name)
    
        file_name_list = os.listdir(old_folder_name)
    
        cnt = 0
        allLength = len(file_name_list)
    
        for file in file_name_list:
            copyFile(file, old_folder_name, new_folder_name)
            cnt += 1
            print("
    Copying %s, Process Bar is:%d%%" % (file, (cnt / allLength) * 100), end="")        
        print("Copy Done!")
    
    if __name__ == "__main__":
        main()
        end = time.time()
        print("Time-consuming: %#.2fs" % (end-start))
    
  • 相关阅读:
    最全负载均衡:算法、实现、亿级负载解决方案详解
    淘宝分布式架构演变案例详解
    分布式一致性协议实现原理
    ReentrantReadWriteLock的使用
    线程之单例
    线程的优先级
    java线程的6种状态
    mybatis <foreach> 标签
    java多线程 上下文切换
    docker的复制和挂载
  • 原文地址:https://www.cnblogs.com/sayiqiu/p/10675485.html
Copyright © 2020-2023  润新知