4 搭建etcd

2 安装etcd

1 生成证书

安装部署主控节点服务 -- etcd

部署master节点服务

部署etcd集群

集群规划
主机名 角色 ip

rstx-203.host.com	ectc follow	192.168.1.203
rstx-204.host.com	ectc follow	192.168.1.204
rstx-205.host.com	ectc lead	192.168.1.205

注释：这里部署文档以rstx-202.host.com主机为例，另外两台安装部署方法类似

在rstx-53上创建基于根证书的config配置文件 -- 此文件夹内有，直接上传，不要粘贴复制

[root@rstx-53 ~]# vi /opt/certs/ca-config.json
{
    "signing": {
        "default": {
            "expiry": "175200h"
        },
        "profiles": {
            "server": {
                "expiry": "175200h",
                "usages": [
                    "signing",
                    "key encipherment",
                    "server auth"
                ]
            },
            "client": {
                "expiry": "175200h",
                "usages": [
                    "signing",
                    "key encipherment",
                    "client auth"
                ]
            },
            "peer": {
                "expiry": "175200h",
                "usages": [
                    "signing",
                    "key encipherment",
                    "server auth",
                    "client auth"
                ]
            }
        }
    }
}

server: server启动的时候需要证书
client: client找server需要证书 server找client不需要证书
peer:server找client需要证书 client找server也需要证书

此文档IP地址必须在文档内更改好再粘贴复制进去，IP地址为有可能装ETCD的主机，多一个IP为预备，-- 此文件夹内有，直接上传，不要粘贴复制

[root@rstx-53 ~]# vi /opt/certs/etcd-peer-csr.json
{
    "CN": "k8s-etcd",
    "hosts": [
        "192.168.1.201",
        "192.168.1.202",
        "192.168.1.203",
        "192.168.1.204",
        "192.168.1.205"
    ],
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [
        {
            "C": "CN",
            "ST": "beijing",
            "L": "beijing",
            "O": "od",
            "OU": "ops"
        }
    ]
}

hosts段：etcd部署的主机ip

[root@rstx-53 certs]# cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=peer etcd-peer-csr.json |cfssl-json -bare etcd-peer
[root@rstx-53 certs]# ll
总用量 36
-rw-r--r-- 1 root root  836 12月 10 16:29 ca-config.json
-rw-r--r-- 1 root root  993 12月 10 11:54 ca.csr
-rw-r--r-- 1 root root  328 12月 10 11:53 ca-csr.json
-rw------- 1 root root 1679 12月 10 11:54 ca-key.pem
-rw-r--r-- 1 root root 1346 12月 10 11:54 ca.pem
-rw-r--r-- 1 root root 1062 12月 10 16:31 etcd-peer.csr
-rw-r--r-- 1 root root  383 12月 10 16:31 etcd-peer-csr.json
-rw------- 1 root root 1679 12月 10 16:31 etcd-peer-key.pem
-rw-r--r-- 1 root root 1428 12月 10 16:31 etcd-peer.pem

在etcd主机上创建etcd用户

useradd -s /sbin/nologin -M etcd
id etcd
mkdir /opt/src
cd /opt/src/

下载etcd软件，建议用不超3.3的版本

# ls
etcd-v3.1.20-linux-amd64.tar.gz
# tar xfv etcd-v3.1.20-linux-amd64.tar.gz -C /opt/
# cd ..
# mv etcd-v3.1.20-linux-amd64 etcd-v3.1.20

创建软链接方便以后更新版本

[root@rstx-202 opt]# ln -s /opt/etcd-v3.1.20 /opt/etcd
# ll
总用量 0
lrwxrwxrwx 1 root   root   17 12月 10 16:45 etcd -> /opt/etcd-v3.1.20
drwxr-xr-x 3 478493 89939 123 10月 11 2018 etcd-v3.1.20
drwxr-xr-x 2 root   root   45 12月 10 16:41 src

创建目录，拷贝证书、私钥

# mkdir -p /opt/etcd/certs /data/etcd /data/logs/etcd-server

将运维主机上生成的ca.pem etc-peer-key.pem etc-peer.pem 拷贝到/opt/etcd/certs目录中，私钥文件权限为600
etcd节点执行

cd /opt/etcd/certs

scp 192.168.1.53:/opt/certs/{ca.pem,etcd-peer-key.pem,etcd-peer.pem}  .

#scp rstx-53.rongbiz.cn:/opt/certs/ca.pem .
#scp rstx-53.rongbiz.cn:/opt/certs/etcd-peer-key.pem .
#scp rstx-53.rongbiz.cn:/opt/certs/etcd-peer.pem .

更改属主属组

# cd /opt/etcd/certs
# chown -R etcd.etcd /opt/etcd/certs /data/etcd /data/logs/etcd-server
# ll
-rw-r--r-- 1 etcd etcd 1346 12月 10 16:52 ca.pem
-rw------- 1 etcd etcd 1679 12月 10 16:53 etcd-peer-key.pem
-rw-r--r-- 1 etcd etcd 1428 12月 10 16:53 etcd-peer.pem

创建etcd服务启动脚本IP地址改成本机IP -- 此文件本目录中有，上传修改即可，不要粘贴复制,格式容易出错

[root@rstx-202 certs]# cat > /opt/etcd/etcd-server-startup.sh<<EOF
#!/bin/sh
./etcd --name etcd-server-202 \
       --data-dir /data/etcd/etcd-server \
       --listen-peer-urls https://192.168.1.202:2380 \
       --listen-client-urls https://192.168.1.202:2379,http://127.0.0.1:2379 \
       --quota-backend-bytes 8000000000 \
       --initial-advertise-peer-urls https://192.168.1.202:2380 \
       --advertise-client-urls https://192.168.1.202:2379,http://127.0.0.1:2379 \
       --initial-cluster  etcd-server-202=https://192.168.1.202:2380,etcd-server-203=https://192.168.1.203:2380,etcd-server-204=https://192.168.1.204:2380 \
       --ca-file ./certs/ca.pem \
       --cert-file ./certs/etcd-peer.pem \
       --key-file ./certs/etcd-peer-key.pem \
       --client-cert-auth  \
       --trusted-ca-file ./certs/ca.pem \
       --peer-ca-file ./certs/ca.pem \
       --peer-cert-file ./certs/etcd-peer.pem \
       --peer-key-file ./certs/etcd-peer-key.pem \
       --peer-client-cert-auth \
       --peer-trusted-ca-file ./certs/ca.pem \
       --log-output stdout
EOF

赋予执行权限

[root@rstx-202 certs]#  chmod +x /opt/etcd/etcd-server-startup.sh

更改属主属组

[root@rstx-202 certs]# chown -R etcd.etcd /opt/etcd-v3.1.20/ /data/etcd /data/logs/etcd-server

使etcd后端运行

[root@rstx-202 logs]# yum install supervisor -y
[root@rstx-202 logs]# systemctl start supervisord
[root@rstx-202 logs]# systemctl enable supervisord

更改supervisord的配置文件：[program:etcd-server-202]名字需要根据实际更改

[root@rstx-202 logs]# cat >/etc/supervisord.d/etcd-server.ini<<EOF
[program:etcd-server-202]
command=/opt/etcd/etcd-server-startup.sh                        ; the program (relative uses PATH, can take args)
numprocs=1                                                      ; number of processes copies to start (def 1)
directory=/opt/etcd                                             ; directory to cwd to before exec (def no cwd)
autostart=true                                                  ; start at supervisord start (default: true)
autorestart=true                                                ; retstart at unexpected quit (default: true)
startsecs=30                                                    ; number of secs prog must stay running (def. 1)
startretries=3                                                  ; max # of serial start failures (default 3)
exitcodes=0,2                                                   ; 'expected' exit codes for process (default 0,2)
stopsignal=QUIT                                                 ; signal used to kill process (default TERM)
stopwaitsecs=10                                                 ; max num secs to wait b4 SIGKILL (default 10)
user=etcd                                                       ; setuid to this UNIX account to run the program
redirect_stderr=true                                            ; redirect proc stderr to stdout (default false)
stdout_logfile=/data/logs/etcd-server/etcd.stdout.log           ; stdout log path, NONE for none; default AUTO
stdout_logfile_maxbytes=64MB                                    ; max # logfile bytes b4 rotation (default 50MB)
stdout_logfile_backups=4                                        ; # of stdout logfile backups (default 10)
stdout_capture_maxbytes=1MB                                     ; number of bytes in 'capturemode' (default 0)
stdout_events_enabled=false                                     ; emit events on stdout writes (default false)
killasgroup=true
stopasgroup=true
EOF

etcd02

[root@rstx-203 etcd]# cat >/opt/etcd/etcd-server-startup.sh <<EOF
#!/bin/sh
./etcd --name etcd-server-203 \
       --data-dir /data/etcd/etcd-server \
       --listen-peer-urls https://192.168.1.203:2380 \
       --listen-client-urls https://192.168.1.203:2379,http://127.0.0.1:2379 \
       --quota-backend-bytes 8000000000 \
       --initial-advertise-peer-urls https://192.168.1.203:2380 \
       --advertise-client-urls https://192.168.1.203:2379,http://127.0.0.1:2379 \
       --initial-cluster  etcd-server-202=https://192.168.1.202:2380,etcd-server-203=https://192.168.1.203:2380,etcd-server-204=https://192.168.1.204:2380 \
       --ca-file ./certs/ca.pem \
       --cert-file ./certs/etcd-peer.pem \
       --key-file ./certs/etcd-peer-key.pem \
       --client-cert-auth  \
       --trusted-ca-file ./certs/ca.pem \
       --peer-ca-file ./certs/ca.pem \
       --peer-cert-file ./certs/etcd-peer.pem \
       --peer-key-file ./certs/etcd-peer-key.pem \
       --peer-client-cert-auth \
       --peer-trusted-ca-file ./certs/ca.pem \
       --log-output stdout
EOF

创建etcd的supervisor配置文件

[root@rstx-203 etcd]# cat >/etc/supervisord.d/etcd-server.ini<<EOF
[program:etcd-server-203]
command=/opt/etcd/etcd-server-startup.sh                        ; the program (relative uses PATH, can take args)	
numprocs=1                                                      ; number of processes copies to start (def 1)
directory=/opt/etcd                                             ; directory to cwd to before exec (def no cwd)
autostart=true                                                  ; start at supervisord start (default: true)
autorestart=true                                                ; retstart at unexpected quit (default: true)
startsecs=30                                                    ; number of secs prog must stay running (def. 1)
startretries=3                                                  ; max # of serial start failures (default 3)
exitcodes=0,2                                                   ; 'expected' exit codes for process (default 0,2)
stopsignal=QUIT                                                 ; signal used to kill process (default TERM)
stopwaitsecs=10                                                 ; max num secs to wait b4 SIGKILL (default 10)
user=etcd                                                       ; setuid to this UNIX account to run the program
redirect_stderr=true                                            ; redirect proc stderr to stdout (default false)
stdout_logfile=/data/logs/etcd-server/etcd.stdout.log           ; stdout log path, NONE for none; default AUTO
stdout_logfile_maxbytes=64MB                                    ; max # logfile bytes b4 rotation (default 50MB)
stdout_logfile_backups=4                                        ; # of stdout logfile backups (default 10)
stdout_capture_maxbytes=1MB                                     ; number of bytes in 'capturemode' (default 0)
stdout_events_enabled=false                                     ; emit events on stdout writes (default false)
killasgroup=true
stopasgroup=true
EOF

etcd03

cat >/opt/etcd/etcd-server-startup.sh <<EOF
#!/bin/sh
./etcd --name etcd-server-204 \
       --data-dir /data/etcd/etcd-server \
       --listen-peer-urls https://192.168.1.204:2380 \
       --listen-client-urls https://192.168.1.204:2379,http://127.0.0.1:2379 \
       --quota-backend-bytes 8000000000 \
       --initial-advertise-peer-urls https://192.168.1.204:2380 \
       --advertise-client-urls https://192.168.1.204:2379,http://127.0.0.1:2379 \
       --initial-cluster  etcd-server-202=https://192.168.1.202:2380,etcd-server-203=https://192.168.1.203:2380,etcd-server-204=https://192.168.1.204:2380 \
       --ca-file ./certs/ca.pem \
       --cert-file ./certs/etcd-peer.pem \
       --key-file ./certs/etcd-peer-key.pem \
       --client-cert-auth  \
       --trusted-ca-file ./certs/ca.pem \
       --peer-ca-file ./certs/ca.pem \
       --peer-cert-file ./certs/etcd-peer.pem \
       --peer-key-file ./certs/etcd-peer-key.pem \
       --peer-client-cert-auth \
       --peer-trusted-ca-file ./certs/ca.pem \
       --log-output stdout
EOF

cat >/etc/supervisord.d/etcd-server.ini<<EOF
[program:etcd-server-204]
command=/opt/etcd/etcd-server-startup.sh                        ; the program (relative uses PATH, can take args)	
numprocs=1                                                      ; number of processes copies to start (def 1)
directory=/opt/etcd                                             ; directory to cwd to before exec (def no cwd)
autostart=true                                                  ; start at supervisord start (default: true)
autorestart=true                                                ; retstart at unexpected quit (default: true)
startsecs=30                                                    ; number of secs prog must stay running (def. 1)
startretries=3                                                  ; max # of serial start failures (default 3)
exitcodes=0,2                                                   ; 'expected' exit codes for process (default 0,2)
stopsignal=QUIT                                                 ; signal used to kill process (default TERM)
stopwaitsecs=10                                                 ; max num secs to wait b4 SIGKILL (default 10)
user=etcd                                                       ; setuid to this UNIX account to run the program
redirect_stderr=true                                            ; redirect proc stderr to stdout (default false)
stdout_logfile=/data/logs/etcd-server/etcd.stdout.log           ; stdout log path, NONE for none; default AUTO
stdout_logfile_maxbytes=64MB                                    ; max # logfile bytes b4 rotation (default 50MB)
stdout_logfile_backups=4                                        ; # of stdout logfile backups (default 10)
stdout_capture_maxbytes=1MB                                     ; number of bytes in 'capturemode' (default 0)
stdout_events_enabled=false                                     ; emit events on stdout writes (default false)
killasgroup=true
stopasgroup=true
EOF

创建后端启动etcd

[root@rstx-202 logs]# supervisorctl update
etcd-server-7-12: added process group

[root@rstx-202 logs]# supervisorctl status
etcd-server-7-12                 STARTING 

[root@rstx-202 logs]# netstat -luntp|grep etcd
tcp        0      0 192.168.153.12:2379     0.0.0.0:*               LISTEN      19395/./etcd        
tcp        0      0 127.0.0.1:2379          0.0.0.0:*               LISTEN      19395/./etcd        
tcp        0      0 192.168.153.12:2380     0.0.0.0:*               LISTEN      19395/./etcd  

查看日志

[root@rstx-202 logs]# tail -fn 200 /data/logs/etcd-server/etcd.stdout.log

检查状态

[root@rstx-202 etcd]# ./etcdctl cluster-health
member 1221562a073d0c1a is healthy: got healthy result from http://127.0.0.1:2379
member 89fe55dc5058e687 is healthy: got healthy result from http://127.0.0.1:2379
member af26e2bc543fbfba is healthy: got healthy result from http://127.0.0.1:2379
cluster is healthy
[root@rstx-202 etcd]# ./etcdctl member list
1221562a073d0c1a: name=etcd-server-203 peerURLs=https://192.168.1.203:2380 clientURLs=http://127.0.0.1:2379,https://192.168.1.203:2379 isLeader=false
89fe55dc5058e687: name=etcd-server-204 peerURLs=https://192.168.1.204:2380 clientURLs=http://127.0.0.1:2379,https://192.168.1.204:2379 isLeader=false
af26e2bc543fbfba: name=etcd-server-202 peerURLs=https://192.168.1.202:2380 clientURLs=http://127.0.0.1:2379,https://192.168.1.202:2379 isLeader=true