etcd 集群部署

etcd 是coreos团队开发的分布式服务发现键值存储仓库，github地址： https://github.com/coreos/etcd

三个etcd节点：

• node01 172.16.65.181
• node02 172.16.65.182
• node03 172.16.65.183

1、三个node节点hosts记录

cat <<EOF > /etc/hosts
127.0.0.1   localhost localhost.localdomain localhost4 localhost4.localdomain4
::1         localhost localhost.localdomain localhost6 localhost6.localdomain6
172.16.65.181 node01
172.16.65.182 node02
172.16.65.183 node03
EOF

2、在node01上配置SSH无密码访问

ssh-keygen  #一路回车即可
ssh-copy-id  node02
ssh-copy-id  node03

3、创建etcd证书

3.1 设置cfssl环境 (cloud flare推出开源的PKI工具箱CFSSL)

wget https://pkg.cfssl.org/R1.2/cfssl_linux-amd64
wget https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64
wget https://pkg.cfssl.org/R1.2/cfssl-certinfo_linux-amd64
chmod +x cfssl_linux-amd64
mv cfssl_linux-amd64 /usr/local/bin/cfssl
chmod +x cfssljson_linux-amd64
mv cfssljson_linux-amd64 /usr/local/bin/cfssljson
chmod +x cfssl-certinfo_linux-amd64
mv cfssl-certinfo_linux-amd64 /usr/local/bin/cfssl-certinfo
export PATH=/usr/local/bin:$PATH

3.2 创建CA配置文件

mkdir /root/ssl
cd /root/ssl
cat >  ca-config.json <<EOF
{
"signing": {
"default": {
  "expiry": "8760h"
},
"profiles": {
  "kubernetes-Soulmate": {
    "usages": [
        "signing",
        "key encipherment",
        "server auth",
        "client auth"
    ],
    "expiry": "8760h"
  }
}
}
}
EOF

cat >  ca-csr.json <<EOF
{
"CN": "kubernetes-Soulmate",
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
  "C": "CN",
  "ST": "shanghai",
  "L": "shanghai",
  "O": "k8s",
  "OU": "System"
}
]
}
EOF

cfssl gencert -initca ca-csr.json | cfssljson -bare ca

cat > etcd-csr.json <<EOF
{
  "CN": "etcd",
  "hosts": [
    "127.0.0.1",
    "172.16.65.181",
    "172.16.65.182",
    "172.16.65.183"
  ],
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "ST": "shanghai",
      "L": "shanghai",
      "O": "k8s",
      "OU": "System"
    }
  ]
}
EOF

cfssl gencert -ca=ca.pem 
  -ca-key=ca-key.pem 
  -config=ca-config.json 
  -profile=kubernetes-Soulmate etcd-csr.json | cfssljson -bare etcd

3.3 node01分发etcd证书到node02、node03上面

mkdir -p /etc/etcd/ssl
cp etcd.pem etcd-key.pem ca.pem /etc/etcd/ssl/
ssh -n node02 "mkdir -p /etc/etcd/ssl && exit"
ssh -n node03 "mkdir -p /etc/etcd/ssl && exit"
scp -r /etc/etcd/ssl/*.pem node02:/etc/etcd/ssl/
scp -r /etc/etcd/ssl/*.pem node03:/etc/etcd/ssl/

4、安装etcd

yum install etcd -y
mkdir -p /var/lib/etcd

5、编辑etcd配置文件

node01配置文件etcd.service

cat <<EOF >/etc/systemd/system/etcd.service
[Unit]
Description=Etcd Server
After=network.target
After=network-online.target
Wants=network-online.target
Documentation=https://github.com/coreos

[Service]
Type=notify
WorkingDirectory=/var/lib/etcd/
ExecStart=/usr/bin/etcd 
  --name node01 
  --cert-file=/etc/etcd/ssl/etcd.pem 
  --key-file=/etc/etcd/ssl/etcd-key.pem 
  --peer-cert-file=/etc/etcd/ssl/etcd.pem 
  --peer-key-file=/etc/etcd/ssl/etcd-key.pem 
  --trusted-ca-file=/etc/etcd/ssl/ca.pem 
  --peer-trusted-ca-file=/etc/etcd/ssl/ca.pem 
  --initial-advertise-peer-urls https://172.16.65.181:2380 
  --listen-peer-urls https://172.16.65.181:2380 
  --listen-client-urls https://172.16.65.181:2379,http://127.0.0.1:2379 
  --advertise-client-urls https://172.16.65.181:2379 
  --initial-cluster-token etcd-cluster-0 
  --initial-cluster node01=https://172.16.65.181:2380,node02=https://172.16.65.182:2380,node03=https://172.16.65.183:2380 
  --initial-cluster-state new 
  --data-dir=/var/lib/etcd
Restart=on-failure
RestartSec=5
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target
EOF

node02配置文件etcd.service

cat <<EOF >/etc/systemd/system/etcd.service
[Unit]
Description=Etcd Server
After=network.target
After=network-online.target
Wants=network-online.target
Documentation=https://github.com/coreos

[Service]
Type=notify
WorkingDirectory=/var/lib/etcd/
ExecStart=/usr/bin/etcd 
  --name node02 
  --cert-file=/etc/etcd/ssl/etcd.pem 
  --key-file=/etc/etcd/ssl/etcd-key.pem 
  --peer-cert-file=/etc/etcd/ssl/etcd.pem 
  --peer-key-file=/etc/etcd/ssl/etcd-key.pem 
  --trusted-ca-file=/etc/etcd/ssl/ca.pem 
  --peer-trusted-ca-file=/etc/etcd/ssl/ca.pem 
  --initial-advertise-peer-urls https://172.16.65.182:2380 
  --listen-peer-urls https://172.16.65.182:2380 
  --listen-client-urls https://172.16.65.182:2379,http://127.0.0.1:2379 
  --advertise-client-urls https://172.16.65.182:2379 
  --initial-cluster-token etcd-cluster-0 
  --initial-cluster node01=https://172.16.65.181:2380,node02=https://172.16.65.182:2380,node03=https://172.16.65.183:2380 
  --initial-cluster-state new 
  --data-dir=/var/lib/etcd
Restart=on-failure
RestartSec=5
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target
EOF

node03配置文件etcd.service

cat <<EOF >/etc/systemd/system/etcd.service
[Unit]
Description=Etcd Server
After=network.target
After=network-online.target
Wants=network-online.target
Documentation=https://github.com/coreos

[Service]
Type=notify
WorkingDirectory=/var/lib/etcd/
ExecStart=/usr/bin/etcd 
  --name node03 
  --cert-file=/etc/etcd/ssl/etcd.pem 
  --key-file=/etc/etcd/ssl/etcd-key.pem 
  --peer-cert-file=/etc/etcd/ssl/etcd.pem 
  --peer-key-file=/etc/etcd/ssl/etcd-key.pem 
  --trusted-ca-file=/etc/etcd/ssl/ca.pem 
  --peer-trusted-ca-file=/etc/etcd/ssl/ca.pem 
  --initial-advertise-peer-urls https://172.16.65.183:2380 
  --listen-peer-urls https://172.16.65.183:2380 
  --listen-client-urls https://172.16.65.183:2379,http://127.0.0.1:2379 
  --advertise-client-urls https://172.16.65.183:2379 
  --initial-cluster-token etcd-cluster-0 
--initial-cluster node01=https://172.16.65.181:2380,node02=https://172.16.65.182:2380,node03=https://172.16.65.183:2380 
  --initial-cluster-state new 
  --data-dir=/var/lib/etcd
Restart=on-failure
RestartSec=5
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target
EOF

6、启动etcd集群

etc集群最少2个节点才能启动，启动报错看/var/log/mesages日志

在三个节点上运行

mv /etc/systemd/system/etcd.service /usr/lib/systemd/system/

systemctl daemon-reload

systemctl enable etcd

systemctl start etcd

systemctl status etcd

在三个节点上运行，检查etcd cluster状态

[root@node01 ~]# etcdctl --endpoints=https://172.16.65.181:2379,https://172.16.65.182:2379,https://172.16.65.183:2379
> --ca-file=/etc/etcd/ssl/ca.pem
> --cert-file=/etc/etcd/ssl/etcd.pem
> --key-file=/etc/etcd/ssl/etcd-key.pem cluster-health
member 5a8035d253973b is healthy: got healthy result from https://172.16.65.181:2379
member 8514f670b8a71207 is healthy: got healthy result from https://172.16.65.183:2379
member c9b41f79b970ff94 is healthy: got healthy result from https://172.16.65.182:2379
cluster is healthy