celery的使用

配置celery

celery文档

英文文档：https://docs.celeryproject.org/en/stable/userguide/configuration.html?highlight=beat_schedule#beat-settings-celery-beat

中文文档：https://www.celerycn.io/ru-men/celery-jin-jie-shi-yong

安装包

pip3 install Django==2.2
pip3 install celery==4.4.7
pip3 install redis==3.5.3

项目配置

在项目同级简历celery_task文件夹专门来存储celery任务

目录如下

创建main.py来作为celery的启动文件，并配置celery

# -*- coding: utf-8 -*-
from celery import Celery
import os,sys
import django
from datetime import timedelta
# # 1.添加django项目根路径
CELERY_BASE_DIR = os.path.dirname(os.path.abspath(__file__))
sys.path.insert(0, os.path.join(CELERY_BASE_DIR, '../opwf'))

# 2.添加django环境
os.environ.setdefault("DJANGO_SETTINGS_MODULE","opwf.settings")
django.setup() # 读取配置


# 3.celery基本配置
app = Celery('mycelery',
             broker='redis://localhost:6379/14',
             backend='redis://localhost:6379/15',
             include=['celery_task.tasks',
                      'celery_task.tasks2',
                      ])

# 4.实例化时可以添加下面这个属性
app.conf.update(
   result_expires=3600,        #执行结果放到redis里，一个小时没人取就丢弃
)

# 5.配置定时任务：每5秒钟执行 调用一次celery_pro下tasks.py文件中的add函数
app.conf.beat_schedule = {
    'add-every-5-seconds': {
        'task': 'tasks.test_task_crontab',
        'schedule': timedelta(seconds=5),
        'args': (16, 16)
    },
}

# 6.添加时区配置
app.conf.timezone = 'UTC'

if __name__ == '__main__':
   app.start()

创建tasks.py来定义异步任务

# -*- coding:utf8 -*-
from .main import app       #从当前目录导入app
import os,sys
from .main import CELERY_BASE_DIR


# 1.test_task_crontab测试定时任务
@app.task
def test_task_crontab(x,y):
    # 添加django项目路径
    # sys.path.insert(0, os.path.join(CELERY_BASE_DIR, '../opwf'))

    from utils.rl_sms import test_crontab

    res = test_crontab(x, y)

    print(x+y)
    return x + y


# 2.测试异步发送邮件
@app.task(bind=True)
def send_sms_code(self, mobile, datas):
    sys.path.insert(0, os.path.join(CELERY_BASE_DIR, '../opwf'))
    # 在方法中导包
    from utils.rl_sms import send_message
    # time.sleep(5)
    try:
        # 用 res 接收发送结果, 成功是:０，　失败是：－１
        res = send_message(mobile, datas)
    except Exception as e:
        res = '-1'

    if res == '-1':
        # 如果发送结果是 -1  就重试.
        self.retry(countdown=5, max_retries=3, exc=Exception('短信发送失败'))

utils的rl_sms.py封装的公用方法

# -*- coding: utf-8 -*-
# utils/rl_sms.py
from ronglian_sms_sdk import SmsSDK
from user.models import User


accId = '8a216da8747ac98201749c0de38723b7'
accToken = '86072b540b4648229b27400414150ef2'
appId = '8a216da8747ac98201749c0de45123be'


def send_message(phone, datas):
    user = User.objects.all()[0]
    print(user.username, '%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%')
    sdk = SmsSDK(accId, accToken, appId)
    tid = '1'  # 测试模板id为: 1. 内容为: 【云通讯】您的验证码是{1}，请于{2}分钟内正确输入。
    # mobile = '13303479527'
    # datas = ('666777', '5')  # 模板中的参数按照位置传递
    # resp = sdk.sendMessage(tid, phone, datas)
    print("##########################################")
    print('执行了这个方法 send_message')
    return ''

from workorder.models import SubOrder



def test_crontab(x,y):
    # need_approve = SubOrder.objects.filter(suborder_status='1')

    print('############### 执行test_crontab测试任务 #############')
    print(x+y)
    print('############### 邮件审批超时提醒 #############')

在views.py里使用异步任务来处理函数

import sys,os
from opwf.settings import BASE_DIR
from django.http.response import JsonResponse
# Create your views here.


def test(request):
    print(BASE_DIR)
    #添加总文件opwf_project文件夹为新的导包路径，将celery_task中的文件导入
    sys.path.insert(0, os.path.join(BASE_DIR, '../../opwf_project'))
    print(sys.path)
    
    #导入celery_task文件中的异步任务
    from celery_task.tasks import test_task_crontab

    #异步任务的调用
    test_task_crontab.delay(10,20)
    
    return JsonResponse({"code":200})

管理celery任务

1.celery管理

celery -A celery_task.main worker -l INFO               # 单线程
celery multi start w1 w2 -A celery_task.main -l info     #一次性启动w1,w2两个worker
celery -A celery_task.main status                        #查看当前有哪些worker在运行
celery multi stop w1 w2 -A celery_task.main              #停止w1,w2两个worker

# 1.项目中启动celery worker
celery  multi start celery_task -A celery_task -l debug --autoscale=50,10      # celery并发数：最多50个，最少5个
# 2.在项目中关闭celery worker
ps auxww|grep "celery worker"|grep -v grep|awk '{print $2}'|xargs kill -9      # 关闭所有celery进程

2.django_celery_beat管理

# 1.普通测试启动celery beat
celery -A celery_task.main beat -l info
# 2.在项目中后台启动celery beat
celery -A celery_task.main beat -l debug >> /aaa/Scheduler.log 2>&1 & 
# 3.停止celery beat
ps -ef | grep -E "celery -A celery_test beat" | grep -v grep| awk '{print $2}' | xargs kill -TERM &> /dev/null  # 杀死心跳所有进程

3.启动celery的sh脚本

#!/bin/bash
export C_FORCE_ROOT="true"
base_dir=`pwd`


celery_pid() {
    echo `ps -ef | grep -E "celery -A celery_task worker" | grep -v grep| awk '{print $2}'`
}
start() {
    celery  multi start celery_task -A celery_task -l debug --autoscale=50,5 --logfile=$base_dir/var/celery-%I.log --pidfile=celery_task.pid
}
restart() {
    celery  multi restart celery_task -A celery_task -l debug
}
stop() {
    celery  multi stop celery_task -A celery_task -l debug
}
#restart(){
#    stop
#    start
#}


# See how we were called.
case "$1" in
  start)
        start
        ;;
  restart)
        restart
        ;;
  stop)
        stop
        ;;
  *)
        echo $"Usage: $0 {start|stop|restart}"
        exit 2
esac

#nohup celery -A celery_task worker -l debug --concurrency=10 --autoreload  & >>celery.log