EMQ集群搭建实现高可用和负载均衡(百万级设备连接)

一．EMQ集群搭建实现高可用和负载均衡

架构服务器规划

服务器IP	部署业务	作用
192.168.81.13	EMQTTD	EMQ集群
192.168.81.22	EMQTTD	EMQ集群
192.168.81.23	EMQTTD	EMQ集群
192.168.81.12(VIP:192.168.81.101)	haproxy、keepalived	HA和LB
192.168.81.21(VIP:192.168.81.101)	haproxy、keepalived	HA和LB

二．架构图

                                                                                                           

三．EMQ集群搭建

192.168.81.13 , 192.168.81.22 , 192.168.81.23 三台服务器作为emq集成服务器，三台都部署emqttd服务

3.1 环境安装

https://www.erlang-solutions.com/resources/download.html;https://github.com/rabbitmq/erlang-rpm/releases  （erlang下载）

yum -y remove erlang

rpm -qa | grep erlang | xargs -I {} rpm -e {}

rpm -ivh erlang-21.3.8.1-1.el7.x86_64.rpm

socat安装

yum -y install socat

nginx安装

yum -y install nginx

3.2  SSL证书准备

mkdir –p /etc/nginx/ssl

mkdir –p /etc/nginx/ssl/ca

mkdir –p /etc/nginx/ssl/server

mkdir –p /etc/nginx/ssl/client

mkdir –p /etc/nginx/ssl/certs

mkdir –p /etc/nginx/ssl/crl

touch /etc/nginx/ssl/index.txt

cat <<EOF >/etc/nginx/ssl/serial

0

EOF

cat <<EOF >/etc/nginx/ssl/crlnumber

01

EOF

mkdir -p /etc/nginx/ssl/newcerts/server

生成ca证书:

openssl genrsa -out /etc/nginx/ssl/ca/ca.key 1024

openssl req -out /etc/nginx/ssl/ca/ca.req -key /etc/nginx/ssl/ca/ca.key -new -subj "/C=CN/ST=GuangDong/L=ShenZhen/O=Xxxxxx/OU=Xxxxxx/CN=xxxxxx/email=info@xxxxxx.com"

openssl x509 -req -in /etc/nginx/ssl/ca/ca.req -out /etc/nginx/ssl/ca/ca.crt -sha1 -days 5000 -signkey /etc/nginx/ssl/ca/ca.key

rm -f /etc/nginx/ssl/ca/ca.req

生成server服务端证书:

openssl genrsa -out /etc/nginx/ssl/server/dev.xxxxxx.com.key 1024

openssl req -out /etc/nginx/ssl/server/dev.xxxxxx.com.req -key /etc/nginx/ssl/server/dev.xxxxxx.com.key -new -subj "/C=CN/ST=GuangDong/L=ShenZhen/O=Xxxxxx/OU=Medc IoT/CN=dev.xxxxxx.com/Email=info@xxxxxx.com"

openssl x509 -req -in /etc/nginx/ssl/server/dev.xxxxxx.com.req -out /etc/nginx/ssl/server/dev.xxxxxx.com.crt -sha1 -CAcreateserial -days 5000  -CA /etc/nginx/ssl/ca/ca.crt -CAkey /etc/nginx/ssl/ca/ca.key

rm -f /etc/nginx/ssl/server/dev.xxxxxx.com.req

生成client客户端证书（若有先吊销然后再生成 ）:

openssl genrsa -out /etc/nginx/ssl/client/client.key 1024

openssl req -out /etc/nginx/ssl/client/client.req -key /etc/nginx/ssl/client/client.key -new -subj "/C=CN/ST=GuangDong/L=ShenZhen/O=Xxxxxx/OU=Medc IoT/Email=info@xxxxxx.com"

openssl x509 -req -in /etc/nginx/ssl/client/client.req -out /etc/nginx/ssl/client/client.crt -sha1 -CAcreateserial -days 5000  -CA /etc/nginx/ssl/ca/ca.crt -CAkey /etc/nginx/ssl/ca/ca.key

rm -f /etc/nginx/ssl/client/client.req

3.3 修改openssl配置文件(centos7)

sed -i "s//etc/pki/CA//etc/nginx/ssl/g" /etc/pki/tls/openssl.cnf

sed -i "s/cacert.pem/ca/ca.crt/g" /etc/pki/tls/openssl.cnf

sed -i "s/private/cakey.pem/ca/ca.key/g" /etc/pki/tls/openssl.cnf

sed -i "s/private/.rand/ca/.rand/g" /etc/pki/tls/openssl.cnf

3.4 准备nginx的配置文件

配置nginx

cat <<EOF >/etc/nginx/nginx.conf

# For more information on configuration, see:

#   * Official English Documentation: http://nginx.org/en/docs/

#   * Official Russian Documentation: http://nginx.org/ru/docs/

user nginx;

worker_processes auto;

error_log /var/log/nginx/error.log;

pid /run/nginx.pid;

# Load dynamic modules. See /usr/share/nginx/README.dynamic.

include /usr/share/nginx/modules/*.conf;

events {

    worker_connections 1024;

}

http {

    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '

                      '$status $body_bytes_sent "$http_referer" '

                      '"$http_user_agent" "$http_x_forwarded_for"';

    access_log  /var/log/nginx/access.log  main;

    sendfile            on;

    tcp_nopush          on;

    tcp_nodelay         on;

    keepalive_timeout   65;

    types_hash_max_size 2048;

    include             /etc/nginx/mime.types;

    default_type        application/octet-stream;

    include /etc/nginx/conf.d/http/*.conf;

}

stream {

    include /etc/nginx/conf.d/tcp/*.conf;

}

EOF

新建相关文件夹

mkdir -p /etc/nginx/conf.d/{http,tcp}

cat <<EOF >/etc/nginx/conf.d/http/80.conf

upstream xxxxxx{

    ip_hash;

    server ${gateway_ip_address}:8800; #网关微服务地址和端口号

}

server {

    error_log /var/log/nginx/80_error.log;

    listen 80;

    server_name xxxxxx.com;

    location / {

        index index.html index.htm;

        root         /usr/share/nginx/html/dist;

        try_files $uri $uri/ /index.html;

    }

    location /api/ {

                add_header 'Access-Control-Allow-Origin' '*';

                add_header 'Access-Control-Allow-Credentials' 'true';

                add_header 'Access-Control-Allow-Methods' '*';

                add_header 'Access-Control-Max-Age' '18000L';

                add_header 'Access-Control-Allow-Headers' '*';

        client_max_body_size    1000m;

        proxy_http_version 1.1;

        proxy_set_header Upgrade $http_upgrade;

        proxy_set_header Connection "upgrade";

        proxy_set_header Host $host;

        proxy_set_header X-Real-IP $remote_addr;

        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;

        proxy_pass http://xxxxxx/;

        }

    error_page 500 502 503 504 /50x.html;

    location = /50x.html {

        root html;

    }

}

EOF

cat <<EOF >/etc/nginx/conf.d/tcp/1882.conf

server {

    error_log /var/log/nginx/1882_error.log;

    listen 1882;

    proxy_connect_timeout 600s;

    proxy_pass xxxxxx;

}

EOF

cat <<EOF >/etc/nginx/conf.d/tcp/1883.conf

upstream xxxxxx{

    server ${emq_ip_address}:1885; #这个配置无论写哪个tcp的配置文件里都可以，emq的端口必须写无鉴权的

}

server {

    error_log /var/log/nginx/1883_error.log;

listen 1883 ssl;

    proxy_connect_timeout 600s;

    #proxy_timeout 3s;

    proxy_pass xxxxxx;

    ssl_certificate       /etc/nginx/ssl/server/dev.xxxxxx.com.crt;

    ssl_certificate_key   /etc/nginx/ssl/server/dev.xxxxxx.com.key;

    ssl_protocols         SSLv3 TLSv1 TLSv1.1 TLSv1.2;

    ssl_ciphers           HIGH:!aNULL:!MD5;

    ssl_session_cache     shared:SSL:20m;

    ssl_session_timeout   4h;

    ssl_handshake_timeout 30s;

}

EOF

cat <<EOF >/etc/nginx/conf.d/tcp/1884.conf

server {

    error_log /var/log/nginx/1884_error.log;

    listen 1884 ssl;

    proxy_connect_timeout 600s;

    #proxy_timeout 3s;

    proxy_pass xxxxxx;

    ssl_verify_client on;

    ssl_client_certificate /etc/nginx/ssl/ca/ca.crt;

    ssl_certificate       /etc/nginx/ssl/server/dev.xxxxxx.com.crt;

    ssl_certificate_key   /etc/nginx/ssl/server/dev.xxxxxx.com.key;

    ssl_protocols         SSLv3 TLSv1 TLSv1.1 TLSv1.2;

    ssl_ciphers           HIGH:!aNULL:!MD5;

    ssl_session_cache     shared:SSL:20m;

    ssl_session_timeout   4h;

    ssl_handshake_timeout 30s;

}

EOF

3.5 修改emq的配置文件

修改插件配置文件，更改域名地址

sed -i "s/test.xxxxxx.com/dev.xxxxxx.com/g" /usr/local/src/emqx-rel/deps/emqx_plugin_template/src/emqx_plugin_template.erl

cd /usr/local/src/emqx-rel/

重新编译，此操作会重新生成配置文件

make -C /usr/local/src/emqx-rel

修改emq配置文件，开启双向认证

sed  -i  "/^node.name = emqx@/cnode.name = emqx@192.168.81.XX" /usr/local/src/emqx-rel/_rel/emqx/etc/emqx.conf

 配置端口号和证书路径

sed  -i  "/^listener.tcp.external =/clistener.tcp.external = 0.0.0.0:1885" /usr/local/src/emqx-rel/_rel/emqx/etc/emqx.conf

sed  -i  "/^{deny, all, subscribe, /c%%{deny, all, subscribe, ["$SYS/#", {eq, "#"}]}." /usr/local/src/emqx-rel/_rel/emqx/etc/acl.conf

sed -i"/^cluster.discovery=/ccluster.discovery=static"/usr/local/src/emqx-rel/emqx/etc/emqx.conf

sed-i"/^##cluster.static.seeds=/ccluster.static.seeds=emqx@192.168.81.13,emqx@192.168.81.22,emqx@192.168.81.23" /usr/local/src/emqx-rel/_rel/emqx/etc/emqx.conf

启动emq

/usr/local/src/emqx-rel/_rel/emqx/bin/emqx start

启动nginx

/usr/sbin/nginx -c /etc/nginx/nginx.conf

查看集群状态：

[root@dev-13 bin]# ./emqx_ctl status

Node 'emqx@192.168.81.13' is started

emqx a2a8e428 is running

[root@dev-13 bin]# ./emqx_ctl cluster status

Cluster status: [{running_nodes,['emqx@192.168.81.22','emqx@192.168.81.23',

                                 'emqx@192.168.81.13']}]

3.6  Liunx和Erlang虚拟机调优

Linux 系统参数优化

修改系统所有进程可打开的文件数量:

vim  /etc/sysctl.conf

fs.file-max = 2097152

fs.nr_open = 2097152

### backlog - Socket 监听队列长度:

net.core.somaxconn=32768

net.ipv4.tcp_max_syn_backlog=16384

net.core.netdev_max_backlog=16384

## 可用知名端口范围:

net.ipv4.ip_local_port_range='1000 65535'

## TCP Socket 读写 Buffer 设置:

net.core.rmem_default=262144

net.core.wmem_default=262144

net.core.rmem_max=16777216

net.core.wmem_max=16777216

net.core.optmem_max=16777216

#sysctl -w net.ipv4.tcp_mem='16777216 16777216 16777216'

net.ipv4.tcp_rmem='1024 4096 16777216'

net.ipv4.tcp_wmem='1024 4096 16777216'

## TCP 连接追踪设置:

net.nf_conntrack_max=1000000

net.netfilter.nf_conntrack_max=1000000

net.netfilter.nf_conntrack_tcp_timeout_time_wait=30

## FIN-WAIT-2 Socket 超时设置:

net.ipv4.tcp_fin_timeout = 15

## TIME-WAIT Socket 最大数量、回收与重用设置:

net.ipv4.tcp_max_tw_buckets=1048576

# 注意: 不建议开启該设置，NAT模式下可能引起连接RST

# net.ipv4.tcp_tw_recycle = 1

# net.ipv4.tcp_tw_reuse = 1

设置服务最大文件句柄数:

vim /etc/systemd/system.conf

DefaultLimitNOFILE=1048576

持久化设置允许用户/进程打开文件句柄数:

vim /etc/security/limits.conf

* soft nofile 1048576

* hard nofile 1048576

Erlang 虚拟机参数:

vim /usr/local/emqttd/etc/emq.conf

## Erlang Process Limit

node.process_limit = 2097152

## Sets the maximum number of simultaneously existing ports for this system

node.max_ports = 1048576

## EMQ 最大允许连接数

## Size of acceptor pool

listener.tcp.external.acceptors = 64

## Maximum number of concurrent clients(以1G内存比5W进行配置)

listener.tcp.external.max_clients = 1000000

四．haproxy部署

192.168.81.12和192.168.81.21服务器部署和配置一样

4.1  haproxy安装

yum install -y pcre-devel  bzip2-devel  gcc gcc-c++ make

tar   –zxvf   haproxy-1.8.26.tar.gz

cd   haproxy-1.8.26

make TARGET=linux2628 PREFIX=/usr/local/haproxy

make install PREFIX=/usr/local/haproxy

/usr/local/haproxy/sbin/haproxy -v

HA-Proxy version 1.8.26 2020/08/03

Copyright 2000-2020 Willy Tarreau willy@haproxy.org

mkdir   /etc/haproxy

groupadd   haproxy

useradd -s /sbin/nologin -M -g haproxy haproxy //添加haproxy运行haproxy账号并设置及属主与属组

cp examples/haproxy.init   /etc/init.d/haproxy

chmod 755  /etc/init.d/haproxy

chkconfig   --add haproxy

cp  /usr/local/haproxy/sbin/haproxy  /usr/sbin/

4.2 增加配置文件

           cat <<EOF>>/etc/haproxy/haproxy.cfg

#---------------------------------------------------------------------

# common defaults that all the 'listen' and 'backend' sections will

# use if not designated in their block

#---------------------------------------------------------------------

defaults

    log                     global

    option                  dontlognull

    option http-server-close

    # option forwardfor

    retries                 3

    timeout http-request    10s

    timeout queue           1m

    timeout connect         60s

    timeout client          2m

    timeout server          2m

    timeout http-keep-alive 10s

    timeout check           10s

frontend emqtt-front

    bind *:1885

    maxconn     1000000

    mode tcp

    default_backend emqtt-backend

backend emqtt-backend

    balance roundrobin

    # balance source

    server emq1 192.168.81.13:1885 check inter 100000 fall 2 rise 5 weight 1

    server emq2 192.168.81.22:1885 check inter 100000 fall 2 rise 5 weight 1

    server emq3 192.168.81.23:1885 check inter 100000 fall 2 rise 5 weight 1

    # source 0.0.0.0 usesrc clientip

frontend emqtt-admin-front

    bind *:18083

    mode http

    default_backend emqtt-admin-backend

backend emqtt-admin-backend

    mode http

    balance roundrobin

    server emq1 192.168.81.13:18083 check

    server emq2 192.168.81.22:18083 check

    server emq3 192.168.81.23:18083 check

listen admin_stats

        stats   enable

        bind    *:8080

        mode    http

        option  httplog

        log     global

        maxconn 10

        stats   refresh 30s

        stats   uri /admin

        stats   realm haproxy

        stats   auth admin:admin

        stats   hide-version

        stats   admin if TRUE

4.3 启动haproxy

systemctl start haproxy

chkconfig  haproxy  on

五．keepalived部署

192.168.81.12和192.168.81.21服务器部署和配置略有差别

5.1  keepalived安装

yum  –y  Install  keepalived

5.2 增加配置文件

  ! Configuration File for keepalived

global_defs {

   router_id mqtt81

   vrrp_skip_check_adv_addr

   vrrp_garp_interval 0

   vrrp_gna_interval 0

}

vrrp_instance VI_1 {

    state BACKUP

    interface ens192

    virtual_router_id 81

    nopreempt

    priority 100

    advert_int 1

    authentication {

        auth_type PASS

        auth_pass 1111

    }

    mcast_src_ip 192.168.81.12

    virtual_ipaddress {

        192.168.81.101/24

    }

}

5.3 启动keepalived并设置开机自启动

  systemctl   start  keepalived

  systemctl   enable  keepalived

六．架构测试

6.1  keepalived架构测试

关闭192.168.81.12或者192.168.81.21其中一台，查看服务是否还可以正常访问

6.2  haproxy负载均衡

客户端工具发起pub，看是否按照轮询方式发送到后端3台EMQ服务上面。

七．结果展示

7.1  emq集群展示

                                                                                                      

                  

7.2  客户端发起6个连接测试

                                                                                                       

7.3  haproxy统计显示