Python3用多线程替代for循环提升程序运行速度

[本文出自天外归云的博客园]

优化前后新老代码如下：

from git_tools.git_tool import get_collect_projects, QQNews_Git
from threading import Thread, Lock
import datetime

base_url = "http://git.xx.com"
project_members_commits_lang_info = {}
lock = Lock()
threads = []

'''
Author:zenkilan
'''


def count_time(func):
    def took_up_time(*args, **kwargs):
        start_time = datetime.datetime.now()
        ret = func(*args, **kwargs)
        end_time = datetime.datetime.now()
        took_up_time = (end_time - start_time).total_seconds()
        print(f"{func.__name__} execution took up time:{took_up_time}")
        return ret

    return took_up_time


def get_project_member_lang_code_lines(git, member, begin_date, end_date):
    global project_members_commits_lang_info
    global lock
    member_name = member["username"]
    r = git.get_user_info(member_name)
    if not r["id"]:
        return
    user_commits_lang_info = git.get_commits_user_lang_diff_between(r["id"], begin_date, end_date)
    if len(user_commits_lang_info) == 0:
        return
    lock.acquire()
    project_members_commits_lang_info.setdefault(git.project, dict())
    project_members_commits_lang_info[git.project][member_name] = user_commits_lang_info
    lock.release()


def get_project_lang_code_lines(project, begin_date, end_date):
    global threads
    git = QQNews_Git(project[1], base_url, project[0])
    project_members = git.get_project_members()
    if len(project_members) == 0:
        return
    for member in project_members:
        thread = Thread(target=get_project_member_lang_code_lines, args=(git, member, begin_date, end_date))
        threads.append(thread)
        thread.start()


@count_time
def get_projects_lang_code_lines(begin_date, end_date):
    """
    获取项目代码行语言相关统计——新方法（提升效率）
    应用多线程替代for循环
    并发访问共享外部资源
    :return:
    """
    global project_members_commits_lang_info
    global threads
    for project in get_collect_projects():
        thread = Thread(target=get_project_lang_code_lines, args=(project, begin_date, end_date))
        threads.append(thread)
        thread.start()


@count_time
def get_projects_lang_code_lines_old(begin_date, end_date):
    """
    获取项目代码行语言相关统计——老方法（耗时严重）
    使用最基本的思路进行编程
    双层for循环嵌套并且每层都包含耗时操作
    :return:
    """
    project_members_commits_lang_info = {}
    for project in get_collect_projects():
        git = QQNews_Git(project[1], base_url, project[0])
        project_members = git.get_project_members()
        user_commits_lang_info_dict = {}
        if len(project_members) == 0:
            continue
        for member in project_members:
            member_name = member["username"]
            r = git.get_user_info(member_name, debug=False)
            if not r["id"]:
                continue
            try:
                user_commits_lang_info = git.get_commits_user_lang_diff_between(r["id"], begin_date, end_date)
                if len(user_commits_lang_info) == 0:
                    continue
                user_commits_lang_info_dict[member_name] = user_commits_lang_info
                project_members_commits_lang_info[git.project] = user_commits_lang_info_dict
            except:
                pass
    return project_members_commits_lang_info


def test_results_equal(resultA, resultB):
    """
    测试方法
    :param resultA:
    :param resultB:
    :return:
    """
    print(resultA)
    print(resultB)
    assert len(str(resultA)) == len(str(resultB))


if __name__ == '__main__':
    from git_tools.config import begin_date, end_date

    get_projects_lang_code_lines(begin_date, end_date)
    for t in threads:
        t.join()
    old_result = get_projects_lang_code_lines_old(begin_date, end_date)
    test_results_equal(old_result, project_members_commits_lang_info)

老方法里外层for循环和内层for循环里均存在耗时操作：

1）git.get_project_members()

2）git.get_user_info(member_name, debug=False)

分两步来优化，先里后外或先外后里都行。用多线程替换for循环，并发共享外部资源，加锁避免写冲突。

测试结果通过，函数运行时间装饰器显示（单位秒）：

get_projects_lang_code_lines execution took up time:1.85294

get_projects_lang_code_lines_old execution took up time:108.604177

速度提升了约58倍