kubernetes operator 监控etcd集群（二进制安装的k8s）

1）建立一个ServiceMonitor对象，用于Prometheus添加监控项
查看etcd引用的证书文件
[root@k8s-master diy]# find / -name etcd.service
/sys/fs/cgroup/devices/system.slice/etcd.service
/sys/fs/cgroup/pids/system.slice/etcd.service
/sys/fs/cgroup/memory/system.slice/etcd.service
/sys/fs/cgroup/blkio/system.slice/etcd.service
/sys/fs/cgroup/cpu,cpuacct/system.slice/etcd.service
/sys/fs/cgroup/systemd/system.slice/etcd.service
/etc/systemd/system/multi-user.target.wants/etcd.service
/usr/lib/systemd/system/etcd.service

[root@k8s-master diy]# cat /usr/lib/systemd/system/etcd.service
[Unit]
Description=Etcd Server
After=network.target
After=network-online.target
Wants=network-online.target
[Service]
Type=notify
EnvironmentFile=/opt/etcd/cfg/etcd.conf
ExecStart=/opt/etcd/bin/etcd --cert-file=/opt/etcd/ssl/server.pem --key-file=/opt/etcd/ssl/server-key.pem --peer-cert-file=/opt/etcd/ssl/server.pem --peer-key-file=/opt/etcd/ssl/server-key.pem --trusted-ca-file=/opt/etcd/ssl/ca.pem --peer-trusted-ca-file=/opt/etcd/ssl/ca.pem --logger=zap
Restart=on-failure
LimitNOFILE=65536
[Install]
WantedBy=multi-user.target

主要用到这3个证书
--cert-file=/opt/etcd/ssl/server.pem
--key-file=/opt/etcd/ssl/server-key.pem
--trusted-ca-file=/opt/etcd/ssl/ca.pem

创建一个secret，让prometheus pod节点挂载
kubectl -n monitoring create secret generic etcd-certs
--from-file=/opt/etcd/ssl/server.pem
--from-file=/opt/etcd/ssl/server-key.pem
--from-file=/opt/etcd/ssl/ca.pem

2）为ServiceMonitor对象关联metrics数据接口的一个Service对象
vim prometheus-prometheus.yaml
apiVersion: monitoring.coreos.com/v1
kind: Prometheus
metadata:
labels:
prometheus: k8s
name: k8s
namespace: monitoring
spec:
alerting:
alertmanagers:
- name: alertmanager-main
namespace: monitoring
port: web
image: quay.io/prometheus/prometheus:v2.19.2
nodeSelector:
kubernetes.io/os: linux
podMonitorNamespaceSelector: {}
podMonitorSelector: {}
replicas: 2
resources:
requests:
memory: 400Mi
ruleSelector:
matchLabels:
prometheus: k8s
role: alert-rules
secrets: ##此次增加,目的：使prometheus使用etcd证书。
- etcd-certs ##此次增加，目的：使prometheus使用etcd证书。
securityContext:
fsGroup: 2000
runAsNonRoot: true
runAsUser: 1000
serviceAccountName: prometheus-k8s
serviceMonitorNamespaceSelector: {}
serviceMonitorSelector: {}
version: v2.19.2

kubectl apply -f prometheus-prometheus.yaml
#等到pod重启后,进入pod查看是否可以看到证书
kubectl exec -it -n monitoring prometheus-k8s-0 -- /bin/sh

/prometheus $ ls -l /etc/prometheus/secrets/etcd-certs/
total 0
lrwxrwxrwx 1 root root 13 Jul 20 04:01 ca.pem -> ..data/ca.pem
lrwxrwxrwx 1 root root 21 Jul 20 04:01 server-key.pem -> ..data/server-key.pem
lrwxrwxrwx 1 root root 17 Jul 20 04:01 server.pem -> ..data/server.pem

创建 ServiceMonitor
vim prometheus-serviceMonitorEtcd.yaml
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
name: etcd-k8s
namespace: monitoring
labels:
k8s-app: etcd-k8s
spec:
jobLabel: k8s-app
endpoints:
- port: port
interval: 30s
scheme: https
tlsConfig:
caFile: /etc/prometheus/secrets/etcd-certs/ca.pem
certFile: /etc/prometheus/secrets/etcd-certs/server.pem
keyFile: /etc/prometheus/secrets/etcd-certs/server-key.pem
insecureSkipVerify: true
selector:
matchLabels:
k8s-app: etcd
namespaceSelector:
matchNames:
- kube-system
注：此文件内的证书路径为prometheus-k8s-0 里的etcd证书路径！！！

kubectl apply -f prometheus-serviceMonitorEtcd.yaml

创建 Service
vim prometheus-etcdService.yaml
apiVersion: v1
kind: Service
metadata:
name: etcd-k8s
namespace: kube-system
labels:
k8s-app: etcd
spec:
type: ClusterIP
clusterIP: None
ports:
- name: port
port: 2379
protocol: TCP
---
apiVersion: v1
kind: Endpoints
metadata:
name: etcd-k8s
namespace: kube-system
labels:
k8s-app: etcd
subsets:
- addresses:
- ip: 10.1.9.170
nodeName: k8s-master
- ip: 10.1.9.171
nodeName: k8s-node1
- ip: 10.1.9.172
nodeName: k8s-node2
ports:
- name: port
port: 2379
protocol: TCP

kubectl apply -f prometheus-etcdService.yaml

完