python实现RabbitMQ同步跟异步消费模型

1，消息推送类

import pika


# 同步消息推送类
class RabbitPublisher(object):

    # 传入RabbitMQ的ip，用户名，密码，实例化一个管道
    def __init__(self, host, user, password):
        self.host = host
        self.user = user
        self.password = password
        self.connection = pika.BlockingConnection(pika.ConnectionParameters(host=self.host, credentials=pika.PlainCredentials(self.user, self.password)))
        self.channel = self.connection.channel()

    # 发送消息在队列中
    def send(self, queue_name, body):
        self.channel.queue_declare(queue=queue_name, durable=True)  # 声明一个持久化队列
        self.channel.basic_publish(exchange='',
                                   routing_key=queue_name,  # 队列名字
                                   body=body,  # 消息内容
                                   properties=pika.BasicProperties(
                                       delivery_mode=2,  # 消息持久化
                                   ))

    # 清除指定队列的所有的消息
    def purge(self, queue_name):
        self.channel.queue_purge(queue_name)

    # 删除指定队列
    def delete(self, queue_name, if_unused=False, if_empty=False):
        self.channel.queue_delete(queue_name, if_unused=if_unused, if_empty=if_empty)

    # 断开连接
    def stop(self):
        self.connection.close()

2.消息消费类

（1）同步消息消费

 在同步消息消费的时候可能会出现pika库断开的情况，原因是因为pika客户端没有及时发送心跳，连接就被server端断开了。解决方案就是做一个心跳线程来维护连接。

心跳线程类

class Heartbeat(threading.Thread):

    def __init__(self, connection):
        super(Heartbeat, self).__init__()
        self.lock = threading.Lock()  # 线程锁
        self.connection = connection  # rabbit连接
        self.quitflag = False  # 退出标志
        self.stopflag = True  # 暂停标志
        self.setDaemon(True)  # 设置为守护线程，当消息处理完，自动清除

    # 间隔10s发送心跳
    def run(self):
        while not self.quitflag:
            time.sleep(10)  # 睡10s发一次心跳
            self.lock.acquire()  # 加线程锁
            if self.stopflag:
                self.lock.release()
                continue
            try:
                self.connection.process_data_events()  # 一直等待服务段发来的消息
            except Exception as e:
                print "Error format: %s" % (str(e))
                self.lock.release()
                return
            self.lock.release()

    # 开启心跳保护
    def startheartbeat(self):
        self.lock.acquire()
        if self.quitflag:
            self.lock.release()
            return
        self.stopflag = False
        self.lock.release()

消息消费类

# 同步消息消费类
class RabbitConsumer(object):

    # 传入RabbitMQ的ip，用户名，密码，实例化一个管道
    def __init__(self, host, user, password):
        self.host = host
        self.user = user
        self.password = password
        self.connection = pika.BlockingConnection(pika.ConnectionParameters(host=self.host, credentials=pika.PlainCredentials(self.user, self.password)))
        self.channel = self.connection.channel()

    # 进行消费
    def receive(self, queue_name, callback_worker, prefetch_count=1):  # callback_worker为消费的回调函数
        self.channel.queue_declare(queue=queue_name, durable=True)
        self.channel.basic_qos(prefetch_count=prefetch_count)  # 设置预取的数量，如果为0则不预取，消费者处理越快，可以将这个这设置的越高
        self.channel.basic_consume(callback_worker, queue=queue_name)  # callback_worker为消费的回调函数
        heartbeat = Heartbeat(self.connection)  # 实例化一个心跳类
        heartbeat.start()  # 开启一个心跳线程，不传target的值默认运行run函数
        heartbeat.startheartbeat()  # 开启心跳保护
        self.channel.start_consuming()  # 开始消费

调用方法

# 消费回调函数
def callback(ch, method, properties, body):
    print(" [x] Received %r" % body)
    ch.basic_ack(delivery_tag=method.delivery_tag)  # 告诉生产者处理完成

consumer = RabbitConsumer(host="12.12.12.12", user="test", password="123")
consumer.receive(queue_name="queue1", callback_worker=callback)  

（2）异步消息消费（推荐）

pika提供了支持异步发送模式的selectconnection方法支持异步发送接收（通过回调的方式）

在连接的时候stop_ioloop_on_close=False需要低版本的pika，比如0.13.1，安装方式 pip install pika==0.13.1

connectioon建立时回调建立channel， channel建立时一次回调各种declare方法，declare建立时依次回调publish。

同使用blockconnection方法相比，通过wireshark抓包来看，使用 异步的方式会对发包进行一些优化，会将几个包合并成一个大包，然后做一次ack应答从而提高效率，与之相反使用blockconnection时将会做至少两次ack，head一次content一次等

因此再试用异步的方式时会获得一定的优化 

异步消息消费类

# 异步消息消费类
class RabbitConsumerAsync(object):
    EXCHANGE = 'amq.direct'
    EXCHANGE_TYPE = 'direct'

    def __init__(self, host, user, password, queue_name="fish_test", callback_worker=None, prefetch_count=1):
        self.host = host
        self.user = user
        self.password = password
        self._connection = None
        self._channel = None
        self._closing = False
        self._consumer_tag = None
        self.QUEUE = queue_name
        self.callbackworker = callback_worker
        self.prefetch_count = prefetch_count

    def connect(self):
        return pika.SelectConnection(pika.ConnectionParameters(host=self.host, credentials=pika.PlainCredentials(self.user, self.password)), self.on_connection_open,
                                     stop_ioloop_on_close=False)

    def on_connection_open(self, unused_connection):
        self.add_on_connection_close_callback()
        self.open_channel()

    def add_on_connection_close_callback(self):
        self._connection.add_on_close_callback(self.on_connection_closed)

    def on_connection_closed(self, connection, reply_code, reply_text):
        self._channel = None
        if self._closing:
            self._connection.ioloop.stop()
        else:
            self._connection.add_timeout(5, self.reconnect)

    def reconnect(self):
        self._connection.ioloop.stop()
        if not self._closing:
            self._connection = self.connect()
            self._connection.ioloop.start()

    def open_channel(self):
        self._connection.channel(on_open_callback=self.on_channel_open)

    def on_channel_open(self, channel):
        self._channel = channel
        self._channel.basic_qos(prefetch_count=self.prefetch_count)
        self.add_on_channel_close_callback()
        self.setup_exchange(self.EXCHANGE)

    def add_on_channel_close_callback(self):
        self._channel.add_on_close_callback(self.on_channel_closed)

    def on_channel_closed(self, channel, reply_code, reply_text):
        print reply_text
        self._connection.close()

    def setup_exchange(self, exchange_name):
        self._channel.exchange_declare(self.on_exchange_declareok, exchange_name, self.EXCHANGE_TYPE, durable=True)

    def on_exchange_declareok(self, unused_frame):
        self.setup_queue()

    def setup_queue(self):
        self._channel.queue_declare(self.on_queue_declareok, self.QUEUE, durable=True)

    def on_queue_declareok(self, method_frame):
        self._channel.queue_bind(self.on_bindok, self.QUEUE, self.EXCHANGE, self.QUEUE)

    def on_bindok(self, unused_frame):
        self.start_consuming()

    def start_consuming(self):
        self.add_on_cancel_callback()
        self._consumer_tag = self._channel.basic_consume(self.on_message, self.QUEUE)

    def add_on_cancel_callback(self):
        self._channel.add_on_cancel_callback(self.on_consumer_cancelled)

    def on_consumer_cancelled(self, method_frame):
        if self._channel:
            self._channel.close()

    def on_message(self, unused_channel, basic_deliver, properties, body):
        self.callbackworker(body)
        self.acknowledge_message(basic_deliver.delivery_tag)

    def acknowledge_message(self, delivery_tag):
        self._channel.basic_ack(delivery_tag)

    def stop_consuming(self):
        if self._channel:
            self._channel.basic_cancel(self.on_cancelok, self._consumer_tag)

    def on_cancelok(self, unused_frame):
        self.close_channel()

    def close_channel(self):
        self._channel.close()

    def run(self):
        self._connection = self.connect()
        self._connection.ioloop.start()

    def stop(self):
        self._closing = True
        self.stop_consuming()
        self._connection.ioloop.start()

    def close_connection(self):
        self._connection.close()

调用方法

# 消费回调函数
def callback(body):
    print(" [x] Received %r" % body)

consumer = RabbitConsumerAsync(host="12.12.12.12", user="test", password="123", queue_name="fish_test", callback_worker=callback, prefetch_count=2)
consumer.run()

 （后面这两个可不加入）守护进程类（保证消费运行）

class CDaemon(object):
    """
    a generic daemon class.
    usage: subclass the CDaemon class and override the run() method
    stderr  表示错误日志文件绝对路径, 收集启动过程中的错误日志
    verbose 表示将启动运行过程中的异常错误信息打印到终端,便于调试,建议非调试模式下关闭, 默认为1, 表示开启
    save_path 表示守护进程pid文件的绝对路径
    """

    def __init__(self, save_path, stdin=os.devnull, stdout=os.devnull, stderr=os.devnull, home_dir='.', umask=022, verbose=1):
        self.stdin = stdin
        self.stdout = stdout
        self.stderr = stderr
        self.pidfile = save_path  # pid文件绝对路径
        self.home_dir = home_dir
        self.verbose = verbose  # 调试开关
        self.umask = umask
        self.daemon_alive = True

    def daemonize(self):
        try:
            pid = os.fork()
            if pid > 0:
                sys.exit(0)
        except OSError, e:
            sys.stderr.write('fork #1 failed: %d (%s)
' % (e.errno, e.strerror))
            sys.exit(1)

        os.chdir(self.home_dir)
        os.setsid()
        os.umask(self.umask)

        try:
            pid = os.fork()
            if pid > 0:
                sys.exit(0)
        except OSError, e:
            sys.stderr.write('fork #2 failed: %d (%s)
' % (e.errno, e.strerror))
            sys.exit(1)

        sys.stdout.flush()
        sys.stderr.flush()

        si = file(self.stdin, 'r')
        so = file(self.stdout, 'a+')
        if self.stderr:
            se = file(self.stderr, 'a+', 0)
        else:
            se = so

        os.dup2(si.fileno(), sys.stdin.fileno())
        os.dup2(so.fileno(), sys.stdout.fileno())
        os.dup2(se.fileno(), sys.stderr.fileno())

        def sig_handler(signum, frame):
            self.daemon_alive = False

        signal.signal(signal.SIGTERM, sig_handler)
        signal.signal(signal.SIGINT, sig_handler)

        if self.verbose >= 1:
            print 'daemon process started ...'

        atexit.register(self.del_pid)
        pid = str(os.getpid())
        file(self.pidfile, 'w+').write('%s
' % pid)

    def get_pid(self):
        try:
            pf = file(self.pidfile, 'r')
            pid = int(pf.read().strip())
            pf.close()
        except IOError:
            pid = None
        except SystemExit:
            pid = None
        return pid

    def del_pid(self):
        if os.path.exists(self.pidfile):
            os.remove(self.pidfile)

    def start(self, *args, **kwargs):
        if self.verbose >= 1:
            print 'ready to starting ......'
        # check for a pid file to see if the daemon already runs
        pid = self.get_pid()
        if pid:
            msg = 'pid file %s already exists, is it already running?
'
            sys.stderr.write(msg % self.pidfile)
            sys.exit(0)
        # start the daemon
        self.daemonize()
        self.run(*args, **kwargs)

    def stop(self):
        if self.verbose >= 1:
            print 'stopping ...'
        pid = self.get_pid()
        if not pid:
            msg = 'pid file [%s] does not exist. Not running?
' % self.pidfile
            sys.stderr.write(msg)
            if os.path.exists(self.pidfile):
                os.remove(self.pidfile)
            return
        # try to kill the daemon process
        try:
            i = 0
            while 1:
                os.kill(pid, signal.SIGTERM)
                time.sleep(0.1)
                i = i + 1
                if i % 10 == 0:
                    os.kill(pid, signal.SIGHUP)
        except OSError, err:
            err = str(err)
            if err.find('No such process') > 0:
                if os.path.exists(self.pidfile):
                    os.remove(self.pidfile)
            else:
                print str(err)
                sys.exit(1)
            if self.verbose >= 1:
                print 'Stopped!'

    def restart(self, *args, **kwargs):
        self.stop()
        self.start(*args, **kwargs)

    def is_running(self):
        pid = self.get_pid()
        # print(pid)
        return pid and os.path.exists('/proc/%d' % pid)

    def run(self, *args, **kwargs):
        # NOTE: override the method in subclass
        print 'base class run()'

调用

class RabbitDaemon(CDaemon):
    def __init__(self, name, save_path, stdin=os.devnull, stdout=os.devnull, stderr=os.devnull, home_dir='.', umask=022, verbose=1):
        CDaemon.__init__(self, save_path, stdin, stdout, stderr, home_dir, umask, verbose)
        self.name = name  # 派生守护进程类的名称

    def run(self, **kwargs):
        # 新建一个队列链接
        rab_con = RabbitConsumerAysnc(queuename="test", callbackworker=liando_sf_consumer, prefetch_count=2)
        # 开启消费者进程
        rab_con.run()


p_name = 'test'  # 守护进程名称
pid_fn = '/www/rabbit/test.pid'  # 守护进程pid文件的绝对路径
err_fn = '/www/rabbit/test_err.log'  # 守护进程启动过程中的错误日志,内部出错能从这里看到
cD = RabbitDaemon(p_name, pid_fn, stderr=err_fn, verbose=1)

参考链接

https://pika.readthedocs.io/en/0.10.0/examples.html

https://www.jianshu.com/p/a4671c59351a

源码下载地址：https://github.com/sy159/RabbiyMQ.git