• python引入线程池进行多线程操作


    import sys
    from glob import glob
    from os import path
    import random
    from redis import StrictRedis
    from BICDetection import biccheck 
    from concurrent.futures import ProcessPoolExecutor  #  引入进程池 然后将进程处理过的结果存入大Redis中,这样可以在多台服务器上直接跑数据。
    
    def get_direct_fps(fp):
        with open(fp) as fd:
            fps = [path.join(root_3421, i[:8], i.split()[0]) for i in fd]
            fps.sort()
        return fps
    
    def get_fps(fp):
        with open(fp) as fd: 
            return [i.strip() for i in fd]
    
    folder = 'PWD'
    fps = glob(path.join(folder, '*.wav'))
    
    redis_conn = StrictRedis(host="IP", db=0)
    
    def bic_check_wrapper(args):
        fp, idx, label = args
        if not path.exists(fp):
            print(fp)
            return
        filename = path.basename(fp)
        if redis_conn.hget(label, filename) is not None:
            return
        r = biccheck([fp])
        flag = r['flag'].iloc[0]
        redis_conn.hset(label, filename, flag)
        return flag
    
    def cu_flag_detect(suffix, nproc=30, select_num=None):
        passed_fps = get_direct_fps('cu.log')
        new_args = [(fp, idx, 'passed_{}'.format(suffix)) for idx, fp in enumerate(passed_fps)]
        unpassed_fps = get_fps('cu_71.log')
        new_args += [(fp, idx, 'unpassed_{}'.format(suffix)) for idx, fp in enumerate(unpassed_fps)]
        random.seed(1)
        random.shuffle(new_args)
        if select_num is None:
            selected = new_args
        with ProcessPoolExecutor(int(nproc)) as executor:
            executor.map(bic_check_wrapper, selected)
    
    
    关注公众号 海量干货等你
  • 相关阅读:
    给 Advice 传递参数
    jenkins 部署问题
    Linux里的2>&1究竟是什么
    记一次 Spring 事务配置踩坑记
    Netty
    springboot logback 集成
    SpringBoot-服务端参数验证-JSR-303验证框架
    mysql 数据库 简单存储过程游标使用
    SQL Case when 的使用方法
    解决Unsupported major.minor version 51.0错误
  • 原文地址:https://www.cnblogs.com/sowhat1412/p/12734312.html
Copyright © 2020-2023  润新知