Python 之RabbitMQ使用

1. IO 多路复用

# select 模拟socket server
# server 端
import select
import socket
import sys
import queue

server = socket.socket()
server.setblocking(False)

server_addr = ('localhost', 10000)

print('starting up on %s port %s' % server_addr)
server.bind(server_addr)

server.listen(5)

inputs = [server, ]
outputs = []

message_queues = {}

while True:
    print('waiting for next event...')

    readable, writeable, exeptional = select.select(inputs, outputs, inputs)

    for s in readable:  # 每个 s 就是一个socket
        if s in server:
            conn, client_addr = s.accept()
            print('new connection from', client_addr)
            conn.setblocking(False)
            inputs.append(conn)

            message_queues[conn] = queue.Queue()

        else:
            data = s.recv(1024)
            if data:
                print('收到来自[%s]的数据:' % s.getpeername()[0], data)
                message_queues[s].put[data]
                if s not in outputs:
                    outputs.append(s)

            else:
                print('客户端断开了', s)

                if s in outputs:
                    outputs.remove(s)

                inputs.remove(s)

                del message_queues[s]

    for s in writeable:
        try:
            next_msg = message_queues[s].get_nowait()

        except queue.Empty:
            print('client [%s]' % s.getpeername()[0], 'queue is empty...')
            outputs.remove(s)

        else:
            print('sending msg to [%s]' % s.getpeername()[0], next_msg)
            s.send(next_msg.upper())

    for s in exceptional:
        print('handling exception for', s.getpeername())
        inputs.remove(s)
        if s in outputs:
            outputs.remove(s)
        s.close()

        del message_queues[s]


# client 端
import socket
import sys

messages = [b'This is the message ',
           b'It will be sent',
           b'in parts']

server_address = ('localhost', 10000)

socks = [socket.socket(socket.AF_INET, socket.SOCK_STREAM),
         socket.socket(socket.AF_INET, socket.SOCK_STREAM),]

print('connecting to %s port %s' % server_address)
for s in socks:
    s.connect(server_address)

for message in messages:
    for s in socks:
        print('%s: sending "%s"' % (s.getsockname(), message))
        s.send(message)

    for s in socks:
        data = s.recv(1024)
        print('%s: received "%s"' % (s.getsockname(), data))
        if not data:
            print(sys.stderr, 'closing socket', s.getsockname())

2. RabbitMQ

2.1 RabbitMQ 准备工作(以mac为例)

• 安装RabbitMQ: brew install rabbitmq
• 安装pika: pip3 install pika
• 启动RabbitMQ: /usr/local/Cellar/rabbitmq/3.7.4/sbin/rabbitmq-server

# 示例:
# 发送端
import pika

connection = pika.BlockingConnection(pika.ConnectionParameters('localhost'))
channel = connection.channel()     # 声明一个管道

# 声明queue
channel.queue_declare(queue='hello')

# 需要 exchange 作为中转站
channel.basic_publish(exchange='',
                     routing_key='hello',   # queue 名字
                     body='Hello World!')

print("' [x] Sent 'Hello World!'")
connection.close()


# 接收端
import pika

connection = pika.BlockingConnection(pika.ConnectionParameters('localhost'))
channel = connection.channel()

channel.queue_declare(queue='hello')

def callback(ch, method, properties, body):
    print(" [x] Received %r" % body)

channel.basic_consume(callback,     # 如果收到消息,就调用 CALLBACK 函数来处理消息
                      queue='hello',
                      no_ack=True)      # no acknowledgement 不确认

print(' [*] Waiting fo messages. To exit press CTRL+C')
channel.start_consuming()

2.2 RabbitMQ 消息分发轮询

• 先启动消息生产者,然后再分别启动3个消费者,通过生产者多发送几条消息,这几条消息会依次分配到各个消费者身上;

2.3 RabbitMQ 消息持久化(发送端)

# 示例:
`channel.queue_declare(queue='hello', durable=True)`: 队列持久化;

# 示例二: 消息持久化
channel.basic_publish(exchange='',
                      routing_key='hello',
                      body='Hello World!',
                      properties=pika.BasicProperties(delivery_mode=2,)  # 将消息持久化
                      )

2.4 RabbitMQ fanout广播模式

• 如果Rabbit只管按顺序把消息发到各个消费者身上,不考虑消费者负载的话,很可能出现,一个配置不高的机器的消费者那里
  堆积了很多消息处理不完,同时配置高的消费者却一直很轻松;
• 为解决上述问题,可以在各个消费者端,配置perfetch=1,意思是告诉RabbitMQ,消费者当前消息还没处理完的时候,就不
  要再向该消费者发送新消息了。

＃ 示例:
# 发送端:
import pika
import sys

connection = pika.BlockingConnection(pika.ConnectionParameters(host='localhost'))
channel = connection.channel()

channel.queue_declare(queue='hello', durable=True)

message='Hello World!'
channel.basic_publish(exchange='',
                      routing_key='hello',
                      body=message,
                      properties=pika.BasicProperties(delivery_mode=2,))

print(' [x] Sent %r' % message)
connection.close()

# 消费者端
import pika
import time

connection = pika.BlockingConnection(pika.ConnectionParameters(host='localhost'))
channel = connection.channel()

channel.queue_declare(queue='hello', durable=True)

def callback(ch, method, properties, body):
    print(' [x] Received %r' % body)
    time.sleep(body.count(b'.'))
    print(' [x] Done')
    ch.basic_ack(delivery_tag = method.delivery_tag)

channel.basic_qos(prefetch_count=1)     
channel.basic_consume(callback,
                      queue='hello')

channel.start_consuming()

2.5 RabbitMQ (发布/订阅模式)

• 发布/订阅模式,需要使用Exchange;
• Exchange在定义的时候是有类型的,以决定到底是哪些Queue符合条件,可以接收消息:
  • fanout: 所有bind到此exchange的queue都可以接收消息;
  • direct: 通过routingKey和exchange决定的那个唯一的queue可以接收消息;
  • topic: 所有符合routingKey(此时可以是一个表达式)的routingKey所bind的queue可以接收消息;
  • headers: 通过headers来决定把消息发给哪些queue;

# 表达式符号说明: # 代表一个或多个字符, * 代表任何字符
# 例: #.a 会匹配 a.a, aa.a, aaa.a 等
#     *.a 会匹配 a.a, b.a, c.a 等

# 示例: fanout模式,一方发送,多方同时接收
# 发送端
import pika

connection = pika.BlockingConnection(pika.ConnectionParameters(host='localhost'))
channel = connection.channel()

channel.exchange_declare(exchange='logs',
                         exchange_type='fanout')

message = 'Hello World!'
channel.basic_publish(exchange='logs',
                      routing_key='',
                      body=message)
print(' [x] Sent %r' % message)
connection.close()

# 接收端
import pika

connection = pika.BlockingConnection(pika.ConnectionParameters(host='localhost'))
channel = connection.channel()

channel.exchange_declare(exchange='logs',
                         exchange_type='fanout')

result = channel.queue_declare(exclusive=True) # 不指定queue名字,rabbit会随机分配一个名字, exclusive=True
                                               # 会在使用此queue的消费者断开后,自动将queue删除
queue_name = result.method.queue

channel.queue_bind(exchange='logs',
                   queue=queue_name)

print(' [*]Waiting for logs. To exit press CTRL+C')

def callback(ch, method, properties, body):
    print(' [x] %r' % body)

channel.basic_consume(callback,
                      queue=queue_name,
                      no_ack=True)

channel.start_consuming()


# 示例二: direct 模式(有选择的接收消息)
# 发送端(server.py)
import pika
import sys

connection = pika.BlockingConnection(pika.ConnectionParameters(host='localhost'))
channel = connection.channel()

channel.exchange_declare(exchange='direct_logs',
                        exchange_type='direct')

severity = sys.argv[1] if len(sys.argv) > 1 else 'info'
message = ' '.join(sys.argv[2:]) or 'Hello World!'
channel.basic_publish(exchange='direct_logs',
                      routing_key=severity,
                      body=message)
print(' [x] Sent %r:%r' % (severity, message))
connection.close()

# 接收端(client.py)
import pika
import sys

connection = pika.BlockingConnection(pika.ConnectionParameters(host='localhost'))
channel = connection.channel()

channel.exchange_declare(exchange='direct_logs',
                         exchange_type='direct')

result = channel.queue_declare(exclusive=True)
queue_name = result.method.queue

severities = sys.argv[1:]
if not severities:
    sys.stderr.write('Usage: %s [info] [warning] [error]
' % sys.argv[0])
    sys.exit(1)

for severity in severities:
    channel.queue_bind(exchange='direct_logs',
                       queue=queue_name,
                       routing_key=severity)

print(' [*]Waiting for logs. To exit press CTRL+C')

def callback(ch, method, properties, body):
    print(' [x] %r:%r' % (method.routing_key, body))

channel.basic_consume(callback,
                      queue=queue_name,
                      no_ack=True)

channel.start_consuming()


# 说明:
#   以上操作在terminal启动: python3 client.py info  (info级别的接收方)
#                         python3 server.py info  (info级别的发送方)
#                         python3 client.py error (error级别的接收方)
#                         python3 server.py error (error级别的发送方)


# 示例三: 更细致的消息过滤(topic 模式)
# 划分为不同应用程序(例如mysql,python等),不同级别(error, info, debug)
# 发送端
import pika
import sys

connection = pika.BlockingConnection(pika.ConnectionParameters(host='localhost'))
channel = connection.channel()

channel.exchange_declare(exchange='topic_logs',
                         exchange_type='topic')

routing_key = sys.argv[1] if len(sys.argv) > 1 else 'anonymous.info'
message = ' '.join(sys.argv[2:]) or 'Hello World!'
channel.basic_publish(exchange='topic_logs',
                      routing_key=routing_key,
                      body=message)
print(' [x] Sent %r:%r' % (routing_key, message))
connection.close()


# 接收端
import pika
import sys

connection = pika.BlockingConnection(pika.ConnectionParameters(host='localhost'))
channel = connection.channel()

channel.exchange_declare(exchange='topic_logs',
                         exchange_type='topic')

result = channel.queue_declare(exclusive=True)
queue_name = result.method.queue

binding_keys = sys.argv[1:]
if not binding_keys:
    sys.stderr.write('Usage: %s [binding_key]...
' % sys.argv[0])
    sys.exit(1)

for binding_key in binding_keys:
    channel.queue_bind(exchange='topic_logs',
                       queue=queue_name,
                       routing_key=binding_key)

print(' [*] Waiting for logs. To exit press CTRL+C')

def callback(ch, method, properties, body):
    print(' [x] %r:%r' % (method.routing_key, body))

channel.basic_consume(callback,
                      queue=queue_name,
                      no_ack=True)

channel.start_consuming()

# 说明:
#   以上操作在terminal启动: python3 client.py mysql.*  (接收任何以mysql开头的消息)
#                         python3 server.py mysql.error  (发送mysql的报错日志)

2.6 RabbitMQ rpc实现

# 示例:
# RPC server
import pika
import time
connection = pika.BlockingConnection(pika.ConnectionParameters(host='localhost'))

channel = connection.channel()
channel.queue_declare(queue='rpc_queue')

def fib(n):
    if n == 0:
        return 0
    elif n == 1:
        return 1
    else:
        return fib(n-1) + fib(n-2)

def on_request(ch, method, props, body):
    n = int(body)

    print(' [.] fib(%s)' % n)
    response = fib(n)

    ch.basic_publish(exchange='',
                     routing_key=props.reply_to,
                     properties=pika.BasicProperties(correlation_id = 
                                                        props.correlation_id),
                      body=str(response))
    ch.basic_ack(delivery_tag = method.delivery_tag)

channel.basic_qos(prefetch_count=1)
channel.basic_consume(on_request, queue='rpc_queue')

print(' [x] Awaiting RPC requests')
channel.start_consuming()


# RPC client
import pika
import uuid

class FibonacciRpcClient(object):
    def __init__(self):
        self.connection = pika.BlockingConnection(pika.ConnectionParameters(host='localhost'))
        self.channel = self.connection.channel()

        result = self.channel.queue_declare(exclusive=True)
        self.callback_queue = result.method.queue   # 生成随机Queue

        self.channel.basic_consume(self.on_response,    # 只要一收到消息,就调用 on_response
                                   no_ack=True,
                                   queue=self.callback_queue)

    def on_response(self, ch, method, props, body):
        if self.corr_id == props.correlation_id:
            self.response = body

    def call(self, n):
        self.response = None
        self.corr_id = str(uuid.uuid4())
        self.channel.basic_publish(exchange='',
                                   routing_key='rpc_queue',
                                   properties=pika.BasicProperties(
                                        reply_to = self.callback_queue,
                                        correlation_id = self.corr_id,
                                        ),
                                   body=str(n))
        while self.response is None:
            self.connection.process_data_events()   # 非阻塞版的start_consuming
            print('no message')
        return int(self.response)

fibonacci_rpc = FibonacciRpcClient()

print(' [x] Requesting fib(30)')
response = fibonacci_rpc.call(30)
print(' [.] Got %r' % response)

参考资料:

• Rabbit MQ
• Python 全栈
• Python 之 IO 模型
• Mac 下安装 RabbitMQ