kafka

上个章节我们讲了kafka的环境安装（这里），现在主要来了解下Kafka使用，基于.net实现kafka的消息队列应用，本文用的是Confluent.Kafka，版本0.11.6

1、安装：

在NuGet程序包中搜索“Confluent.Kafka”下载安装即可

2、producer发送消息：

using System;
using System.Collections.Generic;
using System.Text;
using Confluent.Kafka;
using Confluent.Kafka.Serialization;

namespace KafKa
{
    /// <summary>
    /// Kafka消息生产者
    /// </summary>
    public sealed class KafkaProducer
    {
        /// <summary>
        /// 生产消息并发送消息
        /// </summary>
        /// <param name="broker">kafka的服务器地址</param>
        /// <param name="topic">kafka的消息主题名称</param>
        /// <param name="partion">分区</param>
        /// <param name="message">需要传送的消息</param>
        public bool Produce(string broker, string topic, int partion, string message)
        {
            bool result = false;
            if (string.IsNullOrEmpty(broker) || string.IsNullOrWhiteSpace(broker) || broker.Length <= 0)
            {
                throw new ArgumentNullException("Kafka消息服务器地址不能为空！");
            }

            if (string.IsNullOrEmpty(topic) || string.IsNullOrWhiteSpace(topic) || topic.Length <= 0)
            {
                throw new ArgumentNullException("消息所属的主题不能为空！");
            }

            if (string.IsNullOrEmpty(message) || string.IsNullOrWhiteSpace(message) || message.Length <= 0)
            {
                throw new ArgumentNullException("消息内容不能为空！");
            }

            var config = new Dictionary<string, object>
            {
                { "bootstrap.servers", broker }
            };
            using (var producer = new Producer<Null, string>(config, null, new StringSerializer(Encoding.UTF8)))
            {
                var deliveryReport = producer.ProduceAsync(topic, null, message, partion);
                deliveryReport.ContinueWith(task =>
                {
                    if (task.Result.Error.Code == ErrorCode.NoError)
                    {
                        result = true;
                    }
                    //可以在控制台使用以下语句
                    //Console.WriteLine("Producer：" + producer.Name + "
Topic：" + topic + "
Partition：" + task.Result.Partition + "
Offset：" + task.Result.Offset + "
Message：" + task.Result.Value);
                });

                producer.Flush(TimeSpan.FromSeconds(10));
            }
            return result;
        }
    }
}

3、consumer接收消息：

using System;
using System.Collections.Generic;
using System.Text;
using System.Threading;
using Confluent.Kafka;
using Confluent.Kafka.Serialization;

namespace KafKa
{
    /// <summary>
    /// Kafka消息消费者
    /// </summary>
    public sealed class KafkaConsumer
    {
        #region 私有字段

        private bool isCancelled;

        #endregion

        #region 构造函数

        /// <summary>
        /// 构造函数，初始化IsCancelled属性
        /// </summary>
        public KafkaConsumer()
        {
            isCancelled = false;
        }

        #endregion

        #region 属性

        /// <summary>
        /// 是否应该取消继续消费Kafka的消息，默认值是false，继续消费消息
        /// </summary>
        public bool IsCancelled
        {
            get { return isCancelled; }
            set { isCancelled = value; }
        }

        #endregion

        #region 同步版本

        /// <summary>
        /// 指定的组别的消费者开始消费指定主题的消息
        /// </summary>
        /// <param name="broker">Kafka消息服务器的地址</param>
        /// <param name="topic">Kafka消息所属的主题</param>
        /// <param name="groupID">Kafka消费者所属的组别</param>
        /// <param name="action">可以对已经消费的消息进行相关处理</param>
        public void Consume(string broker, string topic, string groupID, Action<ConsumerResult> action = null)
        {
            if (string.IsNullOrEmpty(broker) || string.IsNullOrWhiteSpace(broker) || broker.Length <= 0)
            {
                throw new ArgumentNullException("Kafka消息服务器的地址不能为空！");
            }

            if (string.IsNullOrEmpty(topic) || string.IsNullOrWhiteSpace(topic) || topic.Length <= 0)
            {
                throw new ArgumentNullException("消息所属的主题不能为空！");
            }

            if (string.IsNullOrEmpty(groupID) || string.IsNullOrWhiteSpace(groupID) || groupID.Length <= 0)
            {
                throw new ArgumentNullException("用户分组ID不能为空！");
            }

            var config = new Dictionary<string, object>
                {
                    { "bootstrap.servers", broker },
                    { "group.id", groupID },
                    { "enable.auto.commit", true },  // this is the default
                    { "auto.commit.interval.ms", 5000 },
                    { "statistics.interval.ms", 60000 },
                    { "session.timeout.ms", 6000 },
                    { "auto.offset.reset", "smallest" }
                };


            using (var consumer = new Consumer<Ignore, string>(config, null, new StringDeserializer(Encoding.UTF8)))
            {
                if (action != null)
                {
                    consumer.OnMessage += (_, message) => {
                        ConsumerResult messageResult = new ConsumerResult();
                        messageResult.Broker = broker;
                        messageResult.Topic = message.Topic;
                        messageResult.Partition = message.Partition;
                        messageResult.Offset = message.Offset.Value;
                        messageResult.Message = message.Value;

                        //执行外界自定义的方法
                        action(messageResult);
                    };
                }

                consumer.OnPartitionEOF += (_, end) => Console.WriteLine("Reached end of topic " + end.Topic + " partition " + end.Partition + ", next message will be at offset " + end.Offset);

                consumer.OnError += (_, error) => Console.WriteLine("Error：" + error);

                //引发反序列化错误或消费消息出现错误！= NoError。
                consumer.OnConsumeError += (_, message) => Console.WriteLine("Error consuming from topic/partition/offset " + message.Topic + "/" + message.Partition + "/" + message.Offset + ": " + message.Error);

                consumer.OnOffsetsCommitted += (_, commit) => Console.WriteLine(commit.Error ? "Failed to commit offsets：" + commit.Error : "Successfully committed offsets：" + commit.Offsets);

                // 当消费者被分配一组新的分区时引发。
                consumer.OnPartitionsAssigned += (_, partitions) =>
                {
                    Console.WriteLine("Assigned Partitions：" + partitions + ", Member ID：" + consumer.MemberId);
                    //如果您未向OnPartitionsAssigned事件添加处理程序，则会自动执行以下.Assign调用。 如果你为它添加了事件处理程序，你必须明确地调用.Assign以便消费者开始消费消息。
                    consumer.Assign(partitions);
                };

                // Raised when the consumer's current assignment set has been revoked.
                //当消费者的当前任务集已被撤销时引发。
                consumer.OnPartitionsRevoked += (_, partitions) =>
                {
                    Console.WriteLine("Revoked Partitions：" + partitions);
                    // If you don't add a handler to the OnPartitionsRevoked event,the below .Unassign call happens automatically. If you do, you must call .Unassign explicitly in order for the consumer to stop consuming messages from it's previously assigned partitions.
                    //如果您未向OnPartitionsRevoked事件添加处理程序，则下面的.Unassign调用会自动发生。 如果你为它增加了事件处理程序，你必须明确地调用.Usessign以便消费者停止从它先前分配的分区中消费消息。
                    consumer.Unassign();
                };

                //consumer.OnStatistics += (_, json) => Console.WriteLine("Statistics: " + json);

                consumer.Subscribe(topic);

                //Console.WriteLine("Subscribed to:" + consumer.Subscription);

                while (!IsCancelled)
                {
                    consumer.Poll(TimeSpan.FromMilliseconds(100));
                }
            }
        }

        #endregion

        #region 异步版本

        /// <summary>
        /// 指定的组别的消费者开始消费指定主题的消息
        /// </summary>
        /// <param name="broker">Kafka消息服务器的地址</param>
        /// <param name="topic">Kafka消息所属的主题</param>
        /// <param name="groupID">Kafka消费者所属的组别</param>
        /// <param name="action">可以对已经消费的消息进行相关处理</param>
        public void ConsumeAsync(string broker, string topic, string groupID, Action<ConsumerResult> action = null)
        {
            if (string.IsNullOrEmpty(broker) || string.IsNullOrWhiteSpace(broker) || broker.Length <= 0)
            {
                throw new ArgumentNullException("Kafka消息服务器的地址不能为空！");
            }

            if (string.IsNullOrEmpty(topic) || string.IsNullOrWhiteSpace(topic) || topic.Length <= 0)
            {
                throw new ArgumentNullException("消息所属的主题不能为空！");
            }

            if (string.IsNullOrEmpty(groupID) || string.IsNullOrWhiteSpace(groupID) || groupID.Length <= 0)
            {
                throw new ArgumentNullException("用户分组ID不能为空！");
            }

            ThreadPool.QueueUserWorkItem(KafkaAutoCommittedOffsets, new ConsumerSetting() { Broker = broker, Topic = topic, GroupID = groupID, Action = action });
        }

        #endregion

        #region 两种提交Offsets的版本

        /// <summary>
        /// Kafka消息队列服务器自动提交offset
        /// </summary>
        /// <param name="state">消息消费者信息</param>
        private void KafkaAutoCommittedOffsets(object state)
        {
            ConsumerSetting setting = state as ConsumerSetting;

            var config = new Dictionary<string, object>
                {
                    { "bootstrap.servers", setting.Broker },
                    { "group.id", setting.GroupID },
                    { "enable.auto.commit", true },  // this is the default
                    { "auto.commit.interval.ms", 5000 },
                    { "statistics.interval.ms", 60000 },
                    { "session.timeout.ms", 6000 },
                    { "auto.offset.reset", "smallest" }
                };

            using (var consumer = new Consumer<Ignore, string>(config, null, new StringDeserializer(Encoding.UTF8)))
            {
                if (setting.Action != null)
                {
                    consumer.OnMessage += (_, message) =>
                    {
                        ConsumerResult messageResult = new ConsumerResult();
                        messageResult.Broker = setting.Broker;
                        messageResult.Topic = message.Topic;
                        messageResult.Partition = message.Partition;
                        messageResult.Offset = message.Offset.Value;
                        messageResult.Message = message.Value;

                        //执行外界自定义的方法
                        setting.Action(messageResult);
                    };
                }

                //consumer.OnStatistics += (_, json)=> Console.WriteLine("Statistics: {json}");

                //可以写日志
                //consumer.OnError += (_, error)=> Console.WriteLine("Error："+error);

                //可以写日志
                //consumer.OnConsumeError += (_, msg) => Console.WriteLine("Error consuming from topic/partition/offset {msg.Topic}/{msg.Partition}/{msg.Offset}: {msg.Error}");

                consumer.Subscribe(setting.Topic);

                while (!IsCancelled)
                {
                    consumer.Poll(TimeSpan.FromMilliseconds(100));
                }
            }
        }

        /// <summary>
        /// Kafka消息队列服务器手动提交offset
        /// </summary>
        /// <param name="state">消息消费者信息</param>
        private void KafkaManuallyCommittedOffsets(object state)
        {
            ConsumerSetting setting = state as ConsumerSetting;

            var config = new Dictionary<string, object>
                {
                    { "bootstrap.servers", setting.Broker },
                    { "group.id", setting.GroupID },
                    { "enable.auto.commit", false },//不是自动提交的
                    { "auto.commit.interval.ms", 5000 },
                    { "statistics.interval.ms", 60000 },
                    { "session.timeout.ms", 6000 },
                    { "auto.offset.reset", "smallest" }
                };

            using (var consumer = new Consumer<Ignore, string>(config, null, new StringDeserializer(Encoding.UTF8)))
            {
                //可以写日志
                //consumer.OnError += (_, error) => Console.WriteLine("Error："+error);

                //可以写日志
                // Raised on deserialization errors or when a consumed message has an error != NoError.
                //consumer.OnConsumeError += (_, error)=> Console.WriteLine("Consume error："+error);

                consumer.Subscribe(setting.Topic);

                Message<Ignore, string> message = null;

                while (!isCancelled)
                {
                    if (!consumer.Consume(out message, TimeSpan.FromMilliseconds(100)))
                    {
                        continue;
                    }

                    if (setting.Action != null)
                    {
                        ConsumerResult messageResult = new ConsumerResult();
                        messageResult.Broker = setting.Broker;
                        messageResult.Topic = message.Topic;
                        messageResult.Partition = message.Partition;
                        messageResult.Offset = message.Offset.Value;
                        messageResult.Message = message.Value;

                        //执行外界自定义的方法
                        setting.Action(messageResult);
                    }

                    if (message.Offset % 5 == 0)
                    {
                        var committedOffsets = consumer.CommitAsync(message).Result;
                        //Console.WriteLine("Committed offset："+committedOffsets);
                    }
                }
            }
        }

        #endregion
    }
}

4、新建Producer控制台发送消息

using System;
using KafKa;

namespace ConsoleProducer
{
    class Program
    {
        static void Main(string[] args)
        {
            while (true)
            {
                var message = Console.ReadLine();
                var producer = new KafkaProducer();
                producer.Produce("localhost:9092", "test", 0, message);
            }

            Console.ReadKey();
        }
    }
}

  

5、新建Consumer控制台接收消息

using System;
using System.Collections.Generic;
using KafKa;

namespace ConsoleConsumer
{
    class Program
    {
        static void Main(string[] args)
        {
            var dts = new List<TimeSpan>();

            var consumer = new KafkaConsumer();
            consumer.ConsumeAsync("localhost:9092", "test", "0", result =>
              {
                  Console.WriteLine(result.Message);
              });

            Console.ReadKey();
        }
    }
}

通过以上步骤运行producer控制台，发送消息回车，在consumer控制台就可以接收到消息了。

那么我们如何通过多个consumer来消费消息呢，kafka默认采用的是range分配方法，即平均分配分区。首先注意在创建topic的命令行时创建多个分区（--partitions 5），这里我们创建了5个分区，在发送消息时选择不同的分区发送（0-5），打开5个consumer控制台（注意要同一个分组），我们会发现5个consumer会分别消费对应分区的消息