Elasticsearch 7.10 集群部署 + 负载(haproxy 1.5) + cerebro(0.9.2) （烨哥提供）

Elasticsearch 7.10 集群部署 + 负载(haproxy 1.5) + cerebro(0.9.2)

环境说明(关闭firewalld、iptables、selinux)

系统版本	ES版本	JAVA版本	节点IP	角色
CentOS 7.9	7.10	1.8.0_181	10.3.232.82	node1
CentOS 7.9	7.10	1.8.0_181	10.3.232.77	node2
CentOS 7.9	7.10	1.8.0_181	10.3.232.76	node3
CentOS 7.9	7.10	1.8.0_181	10.3.232.18	node4
CentOS 7.9	7.10	1.8.0_181	10.3.232.11	node5

ES 集群部署

部署JAVA 环境变量

# 解压到/usr/local目录

tar fzx /tmp/jdk-8u181-linux-x64.tar.gz -C /usr/local/

# 配置环境变量

cat /etc/profile

export JAVA_HOME=/usr/local/jdk1.8.0_181
export JRE_HOME=/usr/local/jdk1.8.0_181/jre
export PATH=$JAVA_HOME/bin:$JRE_HOME/bin:$PATH

# 环境变量生效

source /etc/profile

java -version

java version "1.8.0_181"
Java(TM) SE Runtime Environment (build 1.8.0_181-b13)
Java HotSpot(TM) 64-Bit Server VM (build 25.181-b13, mixed mode)

Elasticsearch 软件安装

下载Elasticsearch 7.10版本

# 安装

yum localinstall elasticsearch-7.10.0-x86_64.rpm -y

# 配置限制

cat /etc/security/limits.conf

elasticsearch    -       nproc           300000
elasticsearch    -       nofile          1000000

Elasticsearch 配置

# 添加host(或者配置DNS解析)

cat /etc/hosts
10.3.232.82     node1
10.3.232.77     node2
10.3.232.76     node3
10.3.232.18     node4
10.3.232.11     node5

修改配置文件

配置文件 /etc/elasticsearch/elasticsearch.yml

# node5

cluster.name: ES-SSO
node.name: node5
path.data: /var/lib/elasticsearch
path.logs: /var/log/elasticsearch
bootstrap.memory_lock: false
bootstrap.system_call_filter: false
network.host: 10.3.232.11
http.port: 9200
discovery.seed_hosts: ["node1","node2","node3","node4","node5"]
xpack.security.enabled: true
xpack.license.self_generated.type: basic

# 开始设置密码的时候将下面的注释，设置完密码，生成证书完成后在开启--- 可以添加使用安全方式传输，也可以不使用
#xpack.security.transport.ssl.enabled: true
#xpack.security.transport.ssl.verification_mode: certificate
#xpack.security.transport.ssl.keystore.path: certs/elastic-certificates.p12
#xpack.security.transport.ssl.truststore.path: certs/elastic-certificates.p12
http.cors.enabled: true
http.cors.allow-origin: "*"
http.cors.allow-headers: "Authorization"

# node 4

cluster.name: ES-SSO
node.name: node4
path.data: /var/lib/elasticsearch
path.logs: /var/log/elasticsearch
bootstrap.memory_lock: false
bootstrap.system_call_filter: false
network.host: 10.3.232.18
http.port: 9200
discovery.seed_hosts: ["node1","node2","node3","node4","node5"]
xpack.security.enabled: true
xpack.license.self_generated.type: basic
#xpack.security.transport.ssl.enabled: true
#xpack.security.transport.ssl.verification_mode: certificate
#xpack.security.transport.ssl.keystore.path: certs/elastic-certificates.p12
#xpack.security.transport.ssl.truststore.path: certs/elastic-certificates.p12
http.cors.enabled: true
http.cors.allow-origin: "*"
http.cors.allow-headers: "Authorization"

# node3

cluster.name: ES-SSO
node.name: node3
path.data: /var/lib/elasticsearch
path.logs: /var/log/elasticsearch
bootstrap.memory_lock: false
bootstrap.system_call_filter: false
network.host: 10.3.232.76
http.port: 9200
discovery.seed_hosts: ["node1","node2","node3","node4","node5"]
xpack.security.enabled: true
xpack.license.self_generated.type: basic
#xpack.security.transport.ssl.enabled: true
#xpack.security.transport.ssl.verification_mode: certificate
#xpack.security.transport.ssl.keystore.path: certs/elastic-certificates.p12
#xpack.security.transport.ssl.truststore.path: certs/elastic-certificates.p12
http.cors.enabled: true
http.cors.allow-origin: "*"
http.cors.allow-headers: "Authorization"

# node2

cluster.name: ES-SSO
node.name: node2
path.data: /var/lib/elasticsearch
path.logs: /var/log/elasticsearch
bootstrap.memory_lock: false
bootstrap.system_call_filter: false
network.host: 10.3.232.77
http.port: 9200
discovery.seed_hosts: ["node1","node2","node3","node4","node5"]
xpack.security.enabled: true
xpack.license.self_generated.type: basic
#xpack.security.transport.ssl.enabled: true
#xpack.security.transport.ssl.verification_mode: certificate
#xpack.security.transport.ssl.keystore.path: certs/elastic-certificates.p12
#xpack.security.transport.ssl.truststore.path: certs/elastic-certificates.p12
http.cors.enabled: true
http.cors.allow-origin: "*"
http.cors.allow-headers: "Authorization"

# node1

cluster.name: ES-SSO
node.name: node1
path.data: /var/lib/elasticsearch
path.logs: /var/log/elasticsearch
bootstrap.memory_lock: false
bootstrap.system_call_filter: false
network.host: 10.3.232.82
http.port: 9200
discovery.seed_hosts: ["node1","node2","node3","node4","node5"]
cluster.initial_master_nodes: ["node1","node2"]
xpack.security.enabled: true
xpack.license.self_generated.type: basic
#xpack.security.transport.ssl.enabled: true
#xpack.security.transport.ssl.verification_mode: certificate
#xpack.security.transport.ssl.keystore.path: certs/elastic-certificates.p12
#xpack.security.transport.ssl.truststore.path: certs/elastic-certificates.p12
http.cors.enabled: true
http.cors.allow-origin: "*"
http.cors.allow-headers: "Authorization"

生成证书(在任意一节点上执行，这里在node4上执行)，如果不使用安全传输，这一步可以省略

cd /usr/share/elasticsearch

bin/elasticsearch-certutil ca # 一顿回车(不设置密码)

bin/elasticsearch-certutil cert --ca elastic-stack-ca.p12 # 一顿回车操作(不设置密码)

mkdir /etc/elasticsearch/certs

#将elastic-certificates.p12  elastic-stack-ca.p12 两个证书拷贝到每个节点的/etc/elasticsearch/certs

ls -l /etc/elasticsearch/certs/
total 8
-rw-rw---- 1 root elasticsearch 3443 Nov 24 16:20 elastic-certificates.p12
-rw-rw---- 1 root elasticsearch 2527 Nov 24 16:20 elastic-stack-ca.p12

启动Elasticsearch 服务

systemctl start elasticsearch
systemctl enable elasticsearch

创建登录密码(任意节点执行，同样在node4上执行)

cd /usr/share/elasticsearch

bin/elasticsearch-setup-passwords interactive # 会生成elastic，logstash，kibana等密码，我这边都是统一的一个密码，各个用户权限不一样

其他命令使用

# 添加用户以及权限

/usr/share/elasticsearch/bin/elasticsearch-users useradd tempuser -p 123456 -r superuser

# 通过curl 修改elastic密码

curl -XPUT -u tempuser:123456 http://ip:9200/_xpack/security/user/elastic/_password -H "Content-Type: application/json" -d '
{
"password": "xxx"
}'{} #执行成功

查看集群状态

curl -XGET --user elastic:123456 'http://10.3.232.11:9200/_cat/nodes?v'

ip          heap.percent ram.percent cpu load_1m load_5m load_15m node.role  master name
10.3.232.11           21          80   2    0.00    0.01     0.05 cdhilmrstw -      node5
10.3.232.76           36          80   0    0.00    0.01     0.05 cdhilmrstw -      node3
10.3.232.82           13          86   0    0.00    0.01     0.05 cdhilmrstw *      node1
10.3.232.77           29          77   0    0.00    0.01     0.05 cdhilmrstw -      node2
10.3.232.18           33          77   0    0.00    0.01     0.05 cdhilmrstw -      node4

Cerebro 安装部署

软件下载

wget https://github.com/lmenezes/cerebro/releases/download/v0.9.2/cerebro-0.9.2-1.noarch.rpm

安装

yum localinstall cerebro-0.9.2-1.noarch.rpm -y

配置

cat /etc/cerebro/application.conf 
# Secret will be used to sign session cookies, CSRF tokens and for other encryption utilities.
# It is highly recommended to change this value before running cerebro in production.
secret = "ki:s:[[@=Ag?QI`W2jMwkY:eqvrJ]JqoJyi2axj3ZvOv^/KavOT4ViJSv?6YY4[N"

# Application base path
basePath = "/"

# Defaults to RUNNING_PID at the root directory of the app.
# To avoid creating a PID file set this value to /dev/null
pidfile.path = "/var/run/cerebro/cerebro.pid"  # 可以修改pid文件位置，默认/dev/null

# Rest request history max size per user
rest.history.size = 50 // defaults to 50 if not specified

# Path of local database file
data.path: "/var/lib/cerebro/cerebro.db"
#data.path = "./cerebro.db"

play {
  # Cerebro port, by default it's 9000 (play's default)
  server.http.port = ${?CEREBRO_PORT}
}

es = {
  gzip = true
}

# Authentication
auth = {
  # either basic or ldap
  type: basic		# 修改认证类型(支持基本认证和ldap两种)
  settings {
    # LDAP
    url = ${?LDAP_URL}
    # OpenLDAP might be something like "ou=People,dc=domain,dc=com"
    base-dn = ${?LDAP_BASE_DN}
    # Usually method should  be "simple" otherwise, set it to the SASL mechanisms to try
    method = ${?LDAP_METHOD}
    # user-template executes a string.format() operation where
    # username is passed in first, followed by base-dn. Some examples
    #  - %s => leave user untouched
    #  - %s@domain.com => append "@domain.com" to username
    #  - uid=%s,%s => usual case of OpenLDAP
    user-template = ${?LDAP_USER_TEMPLATE}
    // User identifier that can perform searches
    bind-dn = ${?LDAP_BIND_DN}
    bind-pw = ${?LDAP_BIND_PWD}
    group-search {
      // If left unset parent's base-dn will be used
      base-dn = ${?LDAP_GROUP_BASE_DN}
      // Attribute that represent the user, for example uid or mail
      user-attr = ${?LDAP_USER_ATTR}
      // Define a separate template for user-attr
      // If left unset parent's user-template will be used
      user-attr-template = ${?LDAP_USER_ATTR_TEMPLATE}
      // Filter that tests membership of the group. If this property is empty then there is no group membership check
      // AD example => memberOf=CN=mygroup,ou=ouofthegroup,DC=domain,DC=com
      // OpenLDAP example => CN=mygroup
      group = ${?LDAP_GROUP}
    }

    # Basic auth
    username = ${?BASIC_AUTH_USER}
    password = ${?BASIC_AUTH_PWD}
  }
}

# A list of known hosts
hosts = [
  {
    host = "http://node1:9200"
    name = "ES-SSO"
    auth = {
      username = "elastic"
      password = "Lo6ahhie"
    }
  }
]

# 查看环境变量文件位置

cat /usr/lib/systemd/system/cerebro.service |grep EnvironmentFile
EnvironmentFile=/etc/default/cerebro 

# 添加账号密码和JAVA环境变量 

cat /etc/default/cerebro

JAVA_HOME=/usr/local/jdk1.8.0_181
BASIC_AUTH_USER="Admin"
BASIC_AUTH_PWD="Lo6ahhie"

启动，并登录

systemctl start cerebro && systemctl enable cerebro

# 默认端口是9000，可以修改 https://github.com/lmenezes/cerebro
http://10.3.232.11:9000 输入账号密码 

Haproxy 安装部署

# 安装软件,保证安装了epel源
yum install epel-release -y
yum install haproxy -y

# 查看配置文件

cat /etc/haproxy/haproxy.cfg
#---------------------------------------------------------------------
# Example configuration for a possible web application.  See the
# full configuration options online.
#
#   http://haproxy.1wt.eu/download/1.4/doc/configuration.txt
#
#---------------------------------------------------------------------

#---------------------------------------------------------------------
# Global settings
#---------------------------------------------------------------------
global
    log         127.0.0.1 local2
    chroot      /var/lib/haproxy
    pidfile     /var/run/haproxy.pid
    maxconn     4000
    user        haproxy
    group       haproxy
    daemon

    # turn on stats unix socket
    stats socket /var/lib/haproxy/stats

#---------------------------------------------------------------------
# common defaults that all the 'listen' and 'backend' sections will
# use if not designated in their block
#---------------------------------------------------------------------
defaults
    mode                    tcp
    log                     global
    option                  tcplog
    option                  dontlognull
    #option http-server-close
    #option forwardfor       except 127.0.0.0/8
    #option                  redispatch
    retries                 3
    timeout http-request    10s
    timeout queue           1m
    timeout connect         10s
    timeout client          1m
    timeout server          1m
    timeout http-keep-alive 10s
    timeout check           10s
    maxconn                 3000

#---------------------------------------------------------------------
# main frontend which proxys to the backends
#---------------------------------------------------------------------
frontend  main *:8000
    default_backend            es-cluster

#---------------------------------------------------------------------
# round robin balancing between the various backends
#---------------------------------------------------------------------
backend es-cluster
    balance source
    server  es1 10.3.232.82:9200 check
    server  es2 10.3.232.77:9200 check
    server  es3 10.3.232.76:9200 check
    server  es4 10.3.232.18:9200 check
    server  es5 10.3.232.11:9200 check
   
#--------------------------------------------------------------------
# monitor
#-------------------------------------------------------------------
listen admin_stats
    bind 0.0.0.0:8088
    mode http
    log 127.0.0.1 local2 notice
    stats refresh 30s
    stats uri /haproxy-status
    stats realm welcome login\ Haproxy
    stats auth admin:nicai!
    stats hide-version
    stats admin if TRUE

haproxy -f /etc/haproxy/haproxy.cfg -c
systemctl start haproxy
systemctl enable haproxy

http://10.2.232.11:8088 输入用户名 admin  密码 nicai!  可以正常登录

Nginx 代理安装部署(补充)

# 配置 nginx repo源

cat /etc/yum.repos.d/nginx.repo 
[nginx-stable]
name=nginx stable repo
baseurl=http://nginx.org/packages/centos/$releasever/$basearch/
gpgcheck=1
enabled=1
gpgkey=https://nginx.org/keys/nginx_signing.key
module_hotfixes=true

[nginx-mainline]
name=nginx mainline repo
baseurl=http://nginx.org/packages/mainline/centos/$releasever/$basearch/
gpgcheck=1
enabled=0
gpgkey=https://nginx.org/keys/nginx_signing.key
module_hotfixes=true

# 安装nginx
yum install nginx -y

# 配置

mkdir /etc/nginx/stream.d

cat /etc/nginx/nginx.conf

user  nginx;
worker_processes  auto;

error_log  /var/log/nginx/error.log warn;
pid        /var/run/nginx.pid;


events {
    worker_connections  1024;
}


http {
    include       /etc/nginx/mime.types;
    default_type  application/octet-stream;

    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
                      '$status $body_bytes_sent "$http_referer" '
                      '"$http_user_agent" "$http_x_forwarded_for"';

    access_log  /var/log/nginx/access.log  main;

    sendfile        on;
    tcp_nopush     on;
    keepalive_timeout   65;
    types_hash_max_size 2048;
    client_max_body_size 100M;

    # open file
    open_file_cache max=204800 inactive=20s;
    open_file_cache_min_uses 1;
    open_file_cache_valid 30s;


    # gzip
    gzip on;
    gzip_vary on;
    gzip_proxied any;
    gzip_comp_level 6;
    gzip_types text/plain text/css text/xml application/json application/javascript application/rss+xml;

    # proxy
    proxy_cache_valid  200 206 304 301 302 2h;
    proxy_cache_key $uri$request_body;
    proxy_cache_methods GET POST;
    proxy_ignore_headers Cache-Control;
    proxy_redirect  off;
    proxy_connect_timeout 300s;
    proxy_read_timeout 300s;
    proxy_send_timeout 300s;
    proxy_buffer_size	64k;
    proxy_buffers    4 128k;
    proxy_busy_buffers_size 256k;

    include /etc/nginx/conf.d/*.conf;
}

stream {

    include /etc/nginx/stream.d/*.conf;

}

# 7层 状态配置
cat /etc/nginx/conf.d/default.conf

server {
    listen       80;
    server_name  localhost;

    #charset koi8-r;
    #access_log  /var/log/nginx/host.access.log  main;

    location =/basic_status {
	auth_basic "Nginx Basic Authorization";
	auth_basic_user_file pwd/htpasswd;
	stub_status;
	allow 10.0.0.0/8;
	deny all;
    }

    error_page  404              /404.html;

    # redirect server error pages to the static page /50x.html
    #
    error_page   500 502 503 504  /50x.html;
    location = /50x.html {
        root   /usr/share/nginx/html;
    }

}

#4层代理设置 

cat /etc/nginx/stream.d/sso-es.conf 
upstream es-cluster {
    hash $remote_addr consistent;
    server 10.3.232.82:9200 weight=10;
    server 10.3.232.77:9200 weight=10;
    server 10.3.232.76:9200 weight=10;
    server 10.3.232.18:9200 weight=10;
    server 127.0.0.1:9200 weight=6;
}


server {
    listen 8080;
    proxy_pass es-cluster;
    
}

systemctl start  nginx && systemctl enable nginx

http://10.3.232.11:80/basic_status  输入用户名和密码 

# 用户名和密码创建 可以使用 htpasswd 或者 openssl

yum install httpd-tools -y

htpasswd -c /etc/nginx/pwd/htpasswd admin # 回车 输入两次密码即可

echo "admin:$(openssl passwd -crypt 123456)" >> /etc/nginx/pwd/htpasswd