由于apache 官网的 docker image 是单点，要实现集群方式部署。
rocketmq 分为 nameserver 和 broker ， 对于之间调用频繁的服务，会增加网络压力， 所以 考虑又网络通信 变成 进程间通信， 那么1个pod 中包含两个container。
关于rocketmq 下载编译， https://www.cnblogs.com/fengjian2016/p/10150394.html
拿到 编译后的 apache-rocketmq , 需要修改一些配置， 例如：
1. rm -rf 删除 rocketmq/bin/*.cmd
2 .把runserver.sh HEAP_OPTS 变成环境变量， 把runbroker.sh HEAP_OPTS 变成环境变量
3. 指定消息存储路径
[root@rocketmq1 conf]#mkdir -p /data/rocketmq/store [root@rocketmq1 conf]#mkdir -p /data/rocketmq/store/commitlog [root@rocketmq1 conf]#mkdir -p /data/rocketmq/logs
4. 指定日志输出路径
/data/rocketmq/conf/ 中的xml文件
sed -i 's#${user.home}#/data/rocketmq/logs#g' *.xml
5. 删除多余配置文件 2m-2s 2master 等等， 启动的时候k8s command 直接指向文件
/data/rocketmq/conf/ 目录下新建 broker-a.conf broker-b.conf 2个文件
6. 配置修改
(1) rocketmqHome 修改
(2) namesrvAddr 修改，强调 通一个pod 必须写localhost 或者 127.0.0.1 要不然调不通
[root@harbor conf]# cat broker-a.conf rocketmqHome=/data/rocketmq namesrvAddr=localhost:9876;rocketmq2:9876 brokerName=broker-a brokerClusterName=DefaultCluster brokerId=0 brokerPermission=6 #在发送消息时，自动创建服务器不存在的topic，默认创建的队列数 defaultTopicQueueNums=4 #是否允许 Broker 自动创建Topic，建议线下开启，线上关闭 autoCreateTopicEnable=true clusterTopicEnable=true brokerTopicEnable=true #是否允许 Broker 自动创建订阅组，建议线下开启，线上关闭 autoCreateSubscriptionGroup=true sendMessageThreadPoolNums=1 pullMessageThreadPoolNums=20 queryMessageThreadPoolNums=10 adminBrokerThreadPoolNums=16 clientManageThreadPoolNums=32 consumerManageThreadPoolNums=32 heartbeatThreadPoolNums=2 endTransactionThreadPoolNums=12 flushConsumerOffsetInterval=5000 flushConsumerOffsetHistoryInterval=60000 rejectTransactionMessage=false fetchNamesrvAddrByAddressServer=false sendThreadPoolQueueCapacity=10000 pullThreadPoolQueueCapacity=100000 queryThreadPoolQueueCapacity=20000 clientManagerThreadPoolQueueCapacity=1000000 consumerManagerThreadPoolQueueCapacity=1000000 heartbeatThreadPoolQueueCapacity=50000 endTransactionPoolQueueCapacity=100000 filterServerNums=0 longPollingEnable=true shortPollingTimeMills=1000 notifyConsumerIdsChangedEnable=true highSpeedMode=false commercialEnable=true commercialTimerCount=1 commercialTransCount=1 commercialBigCount=1 commercialBaseCount=1 transferMsgByHeap=true maxDelayTime=40 regionId=DefaultRegion registerBrokerTimeoutMills=6000 slaveReadEnable=false disableConsumeIfConsumerReadSlowly=false consumerFallbehindThreshold=17179869184 brokerFastFailureEnable=true waitTimeMillsInSendQueue=200 waitTimeMillsInPullQueue=5000 waitTimeMillsInHeartbeatQueue=31000 waitTimeMillsInTransactionQueue=3000 startAcceptSendRequestTimeStamp=0 traceOn=true enableCalcFilterBitMap=false expectConsumerNumUseFilter=32 maxErrorRateOfBloomFilter=20 filterDataCleanTimeSpan=86400000 filterSupportRetry=false enablePropertyFilter=false compressedRegister=false forceRegister=true registerNameServerPeriod=30000 transactionTimeOut=6000 transactionCheckMax=15 transactionCheckInterval=60000 #Broker 对外服务的监听端口 listenPort=10911 serverWorkerThreads=8 serverCallbackExecutorThreads=0 serverSelectorThreads=3 serverOnewaySemaphoreValue=256 serverAsyncSemaphoreValue=64 serverChannelMaxIdleTimeSeconds=120 serverSocketSndBufSize=131072 serverSocketRcvBufSize=131072 serverPooledByteBufAllocatorEnable=true useEpollNativeSelector=false clientWorkerThreads=4 clientCallbackExecutorThreads=2 clientOnewaySemaphoreValue=65535 clientAsyncSemaphoreValue=65535 connectTimeoutMillis=3000 channelNotActiveInterval=60000 clientChannelMaxIdleTimeSeconds=120 clientSocketSndBufSize=131072 clientSocketRcvBufSize=131072 clientPooledByteBufAllocatorEnable=false clientCloseSocketIfTimeout=false useTLS=false storePathRootDir=/data/rocketmq/store storePathCommitLog=/root/rocketmq/store/commitlog #commitLog每个文件的大小默认1G mapedFileSizeCommitLog=1073741824 #ConsumeQueue每个文件默认存60W条，根据业务情况调整, mapedFileSizeConsumeQueue=3000000 enableConsumeQueueExt=false mappedFileSizeConsumeQueueExt=50331648 bitMapLengthConsumeQueueExt=64 flushIntervalCommitLog=500 commitIntervalCommitLog=200 useReentrantLockWhenPutMessage=false flushCommitLogTimed=false flushIntervalConsumeQueue=1000 cleanResourceInterval=10000 deleteCommitLogFilesInterval=100 deleteConsumeQueueFilesInterval=100 destroyMapedFileIntervalForcibly=120000 redeleteHangedFileInterval=120000 #删除文件时间点，默认凌晨 2点 deleteWhen=02 #检测物理文件磁盘空间 diskMaxUsedSpaceRatio=75 #文件保留时间，默认 72 小时 fileReservedTime=128 putMsgIndexHightWater=600000 #限制的消息大小 maxMessageSize=65536 checkCRCOnRecover=true flushCommitLogLeastPages=4 commitCommitLogLeastPages=4 flushLeastPagesWhenWarmMapedFile=4096 flushConsumeQueueLeastPages=2 flushCommitLogThoroughInterval=10000 commitCommitLogThoroughInterval=200 flushConsumeQueueThoroughInterval=60000 maxTransferBytesOnMessageInMemory=262144 maxTransferCountOnMessageInMemory=32 maxTransferBytesOnMessageInDisk=65536 maxTransferCountOnMessageInDisk=8 accessMessageInMemoryMaxRatio=40 messageIndexEnable=true maxHashSlotNum=5000000 maxIndexNum=20000000 maxMsgsNumBatch=64 messageIndexSafe=false haListenPort=10912 haSendHeartbeatInterval=5000 haHousekeepingInterval=20000 haTransferBatchSize=32768 haMasterAddress= haSlaveFallbehindMax=268435456 brokerRole=ASYNC_MASTER flushDiskType=ASYNC_FLUSH syncFlushTimeout=5000 messageDelayLevel=1s 5s 10s 30s 1m 2m 3m 4m 5m 6m 7m 8m 9m 10m 20m 30m 1h 2h flushDelayOffsetInterval=10000 cleanFileForciblyEnable=true warmMapedFileEnable=false offsetCheckInSlave=false debugLockEnable=false duplicationEnable=false diskFallRecorded=true osPageCacheBusyTimeOutMills=1000 defaultQueryMaxNum=32 transientStorePoolEnable=false transientStorePoolSize=5 fastFailIfNoBufferInStorePool=false
[root@harbor conf]# cat broker-b.conf rocketmqHome=/data/rocketmq namesrvAddr=rocketmq1:9876;localhost:9876 brokerName=broker-b brokerClusterName=DefaultCluster brokerId=0 brokerPermission=6 #在发送消息时，自动创建服务器不存在的topic，默认创建的队列数 defaultTopicQueueNums=4 #是否允许 Broker 自动创建Topic，建议线下开启，线上关闭 autoCreateTopicEnable=true clusterTopicEnable=true brokerTopicEnable=true #是否允许 Broker 自动创建订阅组，建议线下开启，线上关闭 autoCreateSubscriptionGroup=true sendMessageThreadPoolNums=1 pullMessageThreadPoolNums=20 queryMessageThreadPoolNums=10 adminBrokerThreadPoolNums=16 clientManageThreadPoolNums=32 consumerManageThreadPoolNums=32 heartbeatThreadPoolNums=2 endTransactionThreadPoolNums=12 flushConsumerOffsetInterval=5000 flushConsumerOffsetHistoryInterval=60000 rejectTransactionMessage=false fetchNamesrvAddrByAddressServer=false sendThreadPoolQueueCapacity=10000 pullThreadPoolQueueCapacity=100000 queryThreadPoolQueueCapacity=20000 clientManagerThreadPoolQueueCapacity=1000000 consumerManagerThreadPoolQueueCapacity=1000000 heartbeatThreadPoolQueueCapacity=50000 endTransactionPoolQueueCapacity=100000 filterServerNums=0 longPollingEnable=true shortPollingTimeMills=1000 notifyConsumerIdsChangedEnable=true highSpeedMode=false commercialEnable=true commercialTimerCount=1 commercialTransCount=1 commercialBigCount=1 commercialBaseCount=1 transferMsgByHeap=true maxDelayTime=40 regionId=DefaultRegion registerBrokerTimeoutMills=6000 slaveReadEnable=false disableConsumeIfConsumerReadSlowly=false consumerFallbehindThreshold=17179869184 brokerFastFailureEnable=true waitTimeMillsInSendQueue=200 waitTimeMillsInPullQueue=5000 waitTimeMillsInHeartbeatQueue=31000 waitTimeMillsInTransactionQueue=3000 startAcceptSendRequestTimeStamp=0 traceOn=true enableCalcFilterBitMap=false expectConsumerNumUseFilter=32 maxErrorRateOfBloomFilter=20 filterDataCleanTimeSpan=86400000 filterSupportRetry=false enablePropertyFilter=false compressedRegister=false forceRegister=true registerNameServerPeriod=30000 transactionTimeOut=6000 transactionCheckMax=15 transactionCheckInterval=60000 #Broker 对外服务的监听端口 listenPort=10911 serverWorkerThreads=8 serverCallbackExecutorThreads=0 serverSelectorThreads=3 serverOnewaySemaphoreValue=256 serverAsyncSemaphoreValue=64 serverChannelMaxIdleTimeSeconds=120 serverSocketSndBufSize=131072 serverSocketRcvBufSize=131072 serverPooledByteBufAllocatorEnable=true useEpollNativeSelector=false clientWorkerThreads=4 clientCallbackExecutorThreads=2 clientOnewaySemaphoreValue=65535 clientAsyncSemaphoreValue=65535 connectTimeoutMillis=3000 channelNotActiveInterval=60000 clientChannelMaxIdleTimeSeconds=120 clientSocketSndBufSize=131072 clientSocketRcvBufSize=131072 clientPooledByteBufAllocatorEnable=false clientCloseSocketIfTimeout=false useTLS=false storePathRootDir=/data/rocketmq/store storePathCommitLog=/root/rocketmq/store/commitlog #commitLog每个文件的大小默认1G mapedFileSizeCommitLog=1073741824 #ConsumeQueue每个文件默认存60W条，根据业务情况调整, mapedFileSizeConsumeQueue=3000000 enableConsumeQueueExt=false mappedFileSizeConsumeQueueExt=50331648 bitMapLengthConsumeQueueExt=64 flushIntervalCommitLog=500 commitIntervalCommitLog=200 useReentrantLockWhenPutMessage=false flushCommitLogTimed=false flushIntervalConsumeQueue=1000 cleanResourceInterval=10000 deleteCommitLogFilesInterval=100 deleteConsumeQueueFilesInterval=100 destroyMapedFileIntervalForcibly=120000 redeleteHangedFileInterval=120000 #删除文件时间点，默认凌晨 2点 deleteWhen=02 #检测物理文件磁盘空间 diskMaxUsedSpaceRatio=75 #文件保留时间，默认 72 小时 fileReservedTime=128 putMsgIndexHightWater=600000 #限制的消息大小 maxMessageSize=65536 checkCRCOnRecover=true flushCommitLogLeastPages=4 commitCommitLogLeastPages=4 flushLeastPagesWhenWarmMapedFile=4096 flushConsumeQueueLeastPages=2 flushCommitLogThoroughInterval=10000 commitCommitLogThoroughInterval=200 flushConsumeQueueThoroughInterval=60000 maxTransferBytesOnMessageInMemory=262144 maxTransferCountOnMessageInMemory=32 maxTransferBytesOnMessageInDisk=65536 maxTransferCountOnMessageInDisk=8 accessMessageInMemoryMaxRatio=40 messageIndexEnable=true maxHashSlotNum=5000000 maxIndexNum=20000000 maxMsgsNumBatch=64 messageIndexSafe=false haListenPort=10912 haSendHeartbeatInterval=5000 haHousekeepingInterval=20000 haTransferBatchSize=32768 haMasterAddress= haSlaveFallbehindMax=268435456 brokerRole=ASYNC_MASTER flushDiskType=ASYNC_FLUSH syncFlushTimeout=5000 messageDelayLevel=1s 5s 10s 30s 1m 2m 3m 4m 5m 6m 7m 8m 9m 10m 20m 30m 1h 2h flushDelayOffsetInterval=10000 cleanFileForciblyEnable=true warmMapedFileEnable=false offsetCheckInSlave=false debugLockEnable=false duplicationEnable=false diskFallRecorded=true osPageCacheBusyTimeOutMills=1000 defaultQueryMaxNum=32 transientStorePoolEnable=false transientStorePoolSize=5 fastFailIfNoBufferInStorePool=false
broker Dockerfile
[root@harbor rocketmq-broker]# cat Dockerfile FROM centos MAINTAINER fengjian <fengjian@senyint.com> ENV TZ "Asia/Shanghai" ENV TERM xterm ENV MALLOC_ARENA_MAX=1 ENV JAVA_HOME /data/jdk ENV CLASSPATH $JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar ENV CATALINA_HOME /data/tomcat ENV CATALINA_BASE /data/tomcat ENV PATH $PATH:$JAVA_HOME/bin:$CATALINA_HOME/lib:$CATALINA_HOME/bin ENV ROCKETMQ_HOME /data/rocketmq RUN rm /etc/yum.repos.d/* -rf ADD Centos-7.repo /etc/yum.repos.d/ ADD epel.repo /etc/yum.repos.d/ RUN yum clean all RUN yum -y install net-tools bind-utils sysstat ADD localtime /etc/ ADD jdk.tar.gz /data ADD profile /etc ADD sysctl.conf /etc ADD 20-nproc.conf /etc/security/limits.d/ ADD limits.conf /etc/security/ ADD rocketmq.tar.gz /data/ WORKDIR ${ROCKETMQ_HOME}/bin EXPOSE 10909 10911
nameserver Dockerfile
FROM centos MAINTAINER fengjian <fengjian@senyint.com> ENV TZ "Asia/Shanghai" ENV TERM xterm ENV MALLOC_ARENA_MAX=1 ENV JAVA_HOME /data/jdk ENV CLASSPATH $JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar ENV CATALINA_HOME /data/tomcat ENV CATALINA_BASE /data/tomcat ENV PATH $PATH:$JAVA_HOME/bin:$CATALINA_HOME/lib:$CATALINA_HOME/bin ENV ROCKETMQ_HOME /data/rocketmq RUN rm /etc/yum.repos.d/* -rf ADD Centos-7.repo /etc/yum.repos.d/ ADD epel.repo /etc/yum.repos.d/ RUN yum clean all RUN yum -y install net-tools bind-utils sysstat ADD localtime /etc/ ADD jdk.tar.gz /data ADD profile /etc ADD sysctl.conf /etc ADD 20-nproc.conf /etc/security/limits.d/ ADD limits.conf /etc/security/ ADD rocketmq.tar.gz /data/ WORKDIR ${ROCKETMQ_HOME}/bin EXPOSE 9876 ENTRYPOINT ["sh","mqnamesrv"]
cat ceph-rocketmq1-pvc.yaml
kind: PersistentVolumeClaim apiVersion: v1 metadata: name: rocketmq1-brokerstore-pvc annotations: volume.beta.kubernetes.io/storage-class: ceph-rbd-rocketmq spec: accessModes: - ReadWriteOnce #必须写ReadWriteOnce,否则报错。 resources: requests: storage: 20Gi --- kind: PersistentVolumeClaim apiVersion: v1 metadata: name: rocketmq1-brokerlog-pvc annotations: volume.beta.kubernetes.io/storage-class: ceph-rbd-rocketmq spec: accessModes: - ReadWriteOnce #必须写ReadWriteOnce,否则报错。 resources: requests: storage: 20Gi --- kind: PersistentVolumeClaim apiVersion: v1 metadata: name: rocketmq1-namesrvlog-pvc annotations: volume.beta.kubernetes.io/storage-class: ceph-rbd-rocketmq spec: accessModes: - ReadWriteOnce #必须写ReadWriteOnce,否则报错。 resources: requests: storage: 20Gi
rocketmq1 yaml文件
apiVersion: extensions/v1beta1 kind: Deployment metadata: name: rocketmq1 spec: replicas: 1 template: metadata: labels: app: rocketmq1 spec: containers: - name: namesrv image: 192.168.200.10/fengjian/rocketmq-nameserver:4.3 imagePullPolicy: Always ports: - containerPort: 9876 env: - name: ROCKETMQ_HEAP_OPTS value : "-Xms1g -Xmx1g -Xmn512m" volumeMounts: - name: rocketmq1namesrvlog mountPath: /data/rocketmq/logs - name: broker image: 192.168.200.10/fengjian/rocketmq-broker:4.3 imagePullPolicy: Always ports: - name: brockerserver containerPort: 10909 - name: brockerslave containerPort: 10911 command: ["/data/rocketmq/bin/mqbroker"] args: ["-c", "/data/rocketmq/conf/broker-a.conf"] env: - name: ROCKETMQ_HEAP_OPTS value : "-Xms2g -Xmx2g -Xmn1g" volumeMounts: - name: brokerlogs mountPath: /data/rocketmq/logs - name: brokerstore mountPath: /data/rocketmq/store volumes: - name: rocketmq1namesrvlog persistentVolumeClaim: claimName: rocketmq1-namesrvlog-pvc - name: brokerlogs persistentVolumeClaim: claimName: rocketmq1-brokerlog-pvc - name: brokerstore persistentVolumeClaim: claimName: rocketmq1-brokerstore-pvc --- apiVersion: v1 kind: Service metadata: name: rocketmq1 spec: ports: - name: brokerslave port: 10909 protocol: TCP targetPort: 10909 - name: brokerserver port: 10911 protocol: TCP targetPort: 10911 - name: namesrvport port: 9876 protocol: TCP targetPort: 9876 selector: app: rocketmq1
还要提前定义pvc 卷
vim rocketmq2.yaml
apiVersion: extensions/v1beta1 kind: Deployment metadata: name: rocketmq1 spec: replicas: 1 template: metadata: labels: app: rocketmq1 spec: containers: - name: namesrv image: 192.168.200.10/fengjian/rocketmq-nameserver:4.3 imagePullPolicy: Always ports: - containerPort: 9876 env: - name: ROCKETMQ_HEAP_OPTS value : "-Xms1g -Xmx1g -Xmn512m" volumeMounts: - name: rocketmq1namesrvlog mountPath: /data/rocketmq/logs - name: broker image: 192.168.200.10/fengjian/rocketmq-broker:4.3 imagePullPolicy: Always ports: - name: brockerserver containerPort: 10909 - name: brockerslave containerPort: 10911 command: ["/data/rocketmq/bin/mqbroker"] args: ["-c", "/data/rocketmq/conf/broker-b.conf"] env: - name: ROCKETMQ_HEAP_OPTS value : "-Xms2g -Xmx2g -Xmn1g" volumeMounts: - name: brokerlogs mountPath: /data/rocketmq/logs - name: brokerstore mountPath: /data/rocketmq/store volumes: - name: rocketmq1namesrvlog persistentVolumeClaim: claimName: rocketmq1-namesrvlog-pvc - name: brokerlogs persistentVolumeClaim: claimName: rocketmq1-brokerlog-pvc - name: brokerstore persistentVolumeClaim: claimName: rocketmq1-brokerstore-pvc --- apiVersion: v1 kind: Service metadata: name: rocketmq1 spec: ports: - name: brokerslave port: 10909 protocol: TCP targetPort: 10909 - name: brokerserver port: 10911 protocol: TCP targetPort: 10911 - name: namesrvport port: 9876 protocol: TCP targetPort: 9876 selector: app: rocketmq1