Prometheus监控拓展

Prometheus监控拓展

一、Prometheus采集Tomcat数据

tomcat_exporter地址：https://github.com/nlighten/tomcat_exporter

1）下载相关jar包

2）构建基础镜像

# 上传jar包至服务器
[root@k8s-master1 tomcat]# ll
total 124
-rw-r--r-- 1 root root   340 Jul 12 14:25 Dockerfile
-rw-r--r-- 1 root root  3405 Jul 12 14:21 metrics.war
-rw-r--r-- 1 root root 59477 Jul 12 14:21 simpleclient-0.8.0.jar
-rw-r--r-- 1 root root 21767 Jul 12 14:21 simpleclient_hotspot-0.8.0.jar
-rw-r--r-- 1 root root  7104 Jul 12 14:21 simpleclient_servlet-0.8.0.jar
-rw-r--r-- 1 root root 19582 Jul 12 14:21 tomcat_exporter_client-0.0.12.jar
-rw-r--r-- 1 root root  3405 Jul 12 14:21 tomcat_exporter_servlet-0.0.12.war

# 编写Dockerfile
[root@k8s-master1 tomcat]# cat Dockerfile 
FROM tomcat
ADD metrics.war /usr/local/tomcat/webapps/
ADD simpleclient-0.8.0.jar  /usr/local/tomcat/lib/
ADD simpleclient_common-0.8.0.jar /usr/local/tomcat/lib/
ADD simpleclient_hotspot-0.8.0.jar /usr/local/tomcat/lib/
ADD simpleclient_servlet-0.8.0.jar /usr/local/tomcat/lib/
ADD tomcat_exporter_client-0.0.12.jar /usr/local/tomcat/lib/

# 构建镜像
[root@k8s-master1 tomcat]# docker build -t='hujinzhong/tomcat_prometheus:v1' .

# 推送至dockerhub
[root@k8s-master1 tomcat]# docker login
[root@k8s-master1 tomcat]# docker push hujinzhong/tomcat_prometheus:v1 #上传镜像到hub仓库
[root@k8s-master1 tomcat]# docker pull hujinzhong/tomcat_prometheus:v1  #在k8s的node节点拉取镜像拉取镜像

4）基于上面的镜像创建一个tomcat实例

[root@k8s-master1 tomcat]# vim deploy.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: tomcat-deployment
  namespace: default
spec:
  selector: 
    matchLabels: 
     app: tomcat
  replicas: 2 # tells deployment to run 2 pods matching the template
  template: # create pods using pod definition in this template
    metadata:
      labels:
        app: tomcat
      annotations:
        prometheus.io/scrape: 'true'
    spec:
      containers:
      - name: tomcat
        image: hujinzhong/tomcat_prometheus:v1
        ports:
        - containerPort: 8080
        securityContext: 
          privileged: true
          
[root@k8s-master1 tomcat]# kubectl apply -f deploy.yaml
[root@k8s-master1 tomcat]# kubectl get pods
NAME                                 READY   STATUS    RESTARTS   AGE
tomcat-deployment-75dbbf85b4-hmtq5   1/1     Running   0          5s
tomcat-deployment-75dbbf85b4-r8kxn   1/1     Running   0          5s

5）创建一个service

[root@k8s-master1 tomcat]# cat tomcat-service.yaml
kind: Service
apiVersion: v1
metadata:
  annotations:
    prometheus.io/scrape: 'true'
  name: tomcat-service
spec:
  selector:
    app: tomcat
  ports:
  - nodePort: 31360
    port: 80
    protocol: TCP
    targetPort: 8080
  type: NodePort
  
[root@k8s-master1 tomcat]# kubectl apply -f tomcat-service.yaml 
[root@k8s-master1 tomcat]# kubectl apply -f tomcat-service.yaml 
service/tomcat-service created
[root@k8s-master1 tomcat]# kubectl get svc
NAME             TYPE           CLUSTER-IP       EXTERNAL-IP   PORT(S)        AGE
kubernetes       ClusterIP      10.96.0.1        <none>        443/TCP        3d18h
tomcat-service   NodePort       10.110.178.105   <none>        80:31360/TCP   22s

二、Promethues采集redis监控数据

1）配置一个Redis的exporter，我们通过redis进行暴露监控

# redis这个Pod中包含了两个容器，一个就是 redis 本身的主应用，另外一个容器就是 redis_exporter
# Redis服务的metrics接口在redis-exporter 9121上，添加prometheus.io/port=9121的annotation,在prometheus就会自动发现redis了
[root@k8s-master1 redis]# vim redis.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: redis
  namespace: kube-system
spec:
  replicas: 1
  selector:
    matchLabels:
      app: redis
  template:
    metadata:
      labels:
        app: redis
    spec:
      containers:
      - name: redis
        image: redis:4
        resources:
          requests:
            cpu: 100m
            memory: 100Mi
        ports:
        - containerPort: 6379
      - name: redis-exporter
        image: oliver006/redis_exporter:latest
        resources:
          requests:
            cpu: 100m
            memory: 100Mi
        ports:
        - containerPort: 9121
---
kind: Service
apiVersion: v1
metadata:
  name: redis
  namespace: kube-system
  annotations:
    prometheus.io/scrape: "true"
    prometheus.io/port: "9121"
spec:
  selector:
    app: redis
  ports:
  - name: redis
    port: 6379
    targetPort: 6379
  - name: prom
    port: 9121
    targetPort: 9121
    
[root@k8s-master1 redis]# kubectl apply -f  redis.yaml
deployment.apps/redis created
service/redis created
[root@k8s-master1 redis]# kubectl get pods -n kube-system |grep redis
redis-7fb8ff6779-2klp5                     2/2     Running   0          2m55s

2）导入监控redis的模板

三、Prometheus采集MySQL数据

1）安装mysql及mysql_exporter

[root@k8s-master1 ~]# yum install mariadb-server -y
[root@k8s-master1 ~]# systemctl start mariadb

# 上传mysqld_exporter-0.10.0.linux-amd64.tar.gz
[root@k8s-master1 mysql]# tar -xvf mysqld_exporter-0.10.0.linux-amd64.tar.gz
[root@k8s-master1 mysql]# cd mysqld_exporter-0.10.0.linux-amd64
[root@k8s-master1 mysqld_exporter-0.10.0.linux-amd64]# cp -ar mysqld_exporter /usr/local/bin/
[root@k8s-master1 mysqld_exporter-0.10.0.linux-amd64]# chmod +x /usr/local/bin/mysqld_exporter

2）登陆mysql为mysql_exporter创建账号并授权

MariaDB [(none)]> CREATE USER 'mysql_exporter'@'localhost' IDENTIFIED BY 'Abcdef123!.';
Query OK, 0 rows affected (0.00 sec)

MariaDB [(none)]> GRANT PROCESS, REPLICATION CLIENT, SELECT ON *.* TO 'mysql_exporter'@'localhost';
Query OK, 0 rows affected (0.00 sec)

MariaDB [(none)]> q
Bye

3）创建mysql配置文件、运行时可免密码连接数据库

[root@k8s-master1 mysqld_exporter-0.10.0.linux-amd64]# pwd
/root/prometheus/mysql/mysqld_exporter-0.10.0.linux-amd64
[root@k8s-master1 mysqld_exporter-0.10.0.linux-amd64]# vim my.cnf
[client]
user=mysql_exporter
password=Abcdef123!.

4）启动mysql_exporter客户端

[root@k8s-master1 mysqld_exporter-0.10.0.linux-amd64]# nohup ./mysqld_exporter --config.my-cnf=./my.cnf &
[root@k8s-master1 mysqld_exporter-0.10.0.linux-amd64]# netstat -lntp|grep 9104
tcp6       0      0 :::9104                 :::*                    LISTEN      21388/./mysqld_expo

6）修改prometheus-alertmanager-cfg.yaml文件，添加target

[root@k8s-master1 prometheus]# vim prometheus-alertmanager-cfg.yaml
    - job_name: 'mysql'
      static_configs:
      - targets: ['192.168.40.180:9104']
      
[root@k8s-master1 prometheus]# kubectl apply -f prometheus-alertmanager-cfg.yaml
[root@k8s-master1 prometheus]# kubectl delete -f prometheus-alertmanager-deploy.yaml
[root@k8s-master1 prometheus]# kubectl apply -f prometheus-alertmanager-deploy.yaml

7）导入监控模板mysql-overview_rev5.json

四、Prometheus采集Nginx数据

1）下载nginx-module-vts模块并解压

[root@k8s-master1 nginx]# ll
total 400
-rw-r--r-- 1 root root 407765 Jun 28 17:19 nginx-module-vts-master.zip
[root@k8s-master1 nginx]# unzip nginx-module-vts-master.zip
[root@k8s-master1 nginx]# mv nginx-module-vts-master /usr/local/

2）安装nginx

[root@k8s-master1 nginx]# tar -xf nginx-1.15.7.tar.gz 
[root@k8s-master1 nginx]# cd nginx-1.15.7
[root@k8s-master1 nginx-1.15.7]# ./configure  --prefix=/usr/local/nginx --with-http_gzip_static_module --with-http_stub_status_module --with-http_ssl_module --with-pcre --with-file-aio --with-http_realip_module --add-module=/usr/local/nginx-module-vts-master
[root@k8s-master1 nginx-1.15.7]# make && make install

3）修改nginx配置

[root@k8s-master1 nginx-1.15.7]# vim /usr/local/nginx/conf/nginx.conf
# server下添加如下：
location /status {
	vhost_traffic_status_display;
	vhost_traffic_status_display_format html;
}
#http中添加如下：
vhost_traffic_status_zone;

# 测试nginx配置文件是否正确：
[root@k8s-master1 nginx-1.15.7]# /usr/local/nginx/sbin/nginx -t
# 如果正确没问题，启动nginx
[root@k8s-master1 nginx-1.15.7]# /usr/local/nginx/sbin/nginx

# 访问192.168.40.180/status可以看到nginx监控数据

4）安装nginx-vts-exporter

[root@k8s-master1 nginx]# unzip  nginx-vts-exporter-0.5.zip
[root@k8s-master1 nginx]# mv nginx-vts-exporter-0.5  /usr/local/
[root@k8s-master1 nginx]# chmod +x /usr/local/nginx-vts-exporter-0.5/bin/nginx-vts-exporter
[root@k8s-master1 nginx]# cd /usr/local/nginx-vts-exporter-0.5/bin
[root@k8s-master1 bin]# nohup ./nginx-vts-exporter -nginx.scrape_uri http://192.168.40.180/status/format/json &
[root@k8s-master1 bin]# netstat -lntp|grep 9913
tcp6       0      0 :::9913                 :::*                    LISTEN      27929/./nginx-vts-e

5）修改prometheus-alertmanager-cfg.yaml文件

[root@k8s-master1 prometheus]# vim prometheus-alertmanager-cfg.yaml
    - job_name: 'nginx'
      scrape_interval: 5s
      static_configs:
      - targets: ['192.168.40.180:9913']
      
# 更新
[root@k8s-master1 prometheus]# kubectl apply -f prometheus-alertmanager-cfg.yaml
[root@k8s-master1 prometheus]# kubectl delete -f prometheus-alertmanager-deploy.yaml 
[root@k8s-master1 prometheus]# kubectl apply -f prometheus-alertmanager-deploy.yaml 
[root@k8s-master1 prometheus]# kubectl get pods -n monitor-sa 
NAME                                 READY   STATUS    RESTARTS   AGE
node-exporter-nl5qz                  1/1     Running   1          23h
node-exporter-nxwkf                  1/1     Running   1          23h
node-exporter-x494t                  1/1     Running   1          23h
prometheus-server-85598946c7-9d5sf   2/2     Running   0          7s

五、Prometheus采集MongoDB数据

https://cloud.tencent.com/document/product/1416/56037

1）下载mongodb和mongodb_exporter镜像

# 不要使用eses/mongodb_exporter这个镜像，启动不起来
[root@k8s-master1 mongo]# docker pull mongo
[root@k8s-master1 mongo]# docker pull ssheehy/mongodb-exporter

2）启动mongodb

[root@k8s-master1 mongo]# mkdir -p /data/db
[root@k8s-master1 mongo]# docker run -d --name mongodb -p 27017:27017 -v /data/db:/data/db mongo
ed33aab7e19fc0f26a41ab8a90912350d9f4ee356a65997b0f52f7c9ebcca593
[root@k8s-master1 mongo]# docker ps -a|grep mongo
ed33aab7e19f   mongo                                               "docker-entrypoint.s…"   5 seconds ago   Up 5 seconds               0.0.0.0:27017->27017/tcp, :::27017->27017/tcp   mongodb

# 登录到容器
docker exec -it ed33aab7e19fc0f26a41ab8a90912350d9f4ee356a65997b0f52f7c9ebcca593 mongo admin
#设置密码
use admin
db.createUser({ user: 'admin', pwd: 'admin111111', roles: [ { role: "userAdminAnyDatabase", db: "admin" } ] })

3）启动mongo_exporter

[root@k8s-master1 mongo]# docker run -d --name mongo-explorer -p 30056:9216 ssheehy/mongodb-exporter --mongodb.uri mongodb://admin:admin111111@192.168.40.180:27017
892a5e1d62f33f6159f1ed978edfd53e5b9351ffeaa1240d5c19d7d911bb8f0a
[root@k8s-master1 mongo]# docker ps -a|grep mongo
892a5e1d62f3   ssheehy/mongodb-exporter                            "/bin/mongodb_export…"   6 seconds ago        Up 5 seconds               0.0.0.0:30056->9216/tcp, :::30056->9216/tcp     mongo-explorer
ed33aab7e19f   mongo                                               "docker-entrypoint.s…"   About a minute ago   Up About a minute          0.0.0.0:27017->27017/tcp, :::27017->27017/tcp   mongodb

4）修改prometheus-alertmanager-cfg.yaml文件

[root@k8s-master1 prometheus]# vim prometheus-alertmanager-cfg.yaml
    - job_name: 'mongodb'
      scrape_interval: 5s
      static_configs:
      - targets: ['192.168.40.180:30056']
      
# 更新
[root@k8s-master1 prometheus]# kubectl apply -f prometheus-alertmanager-cfg.yaml
[root@k8s-master1 prometheus]# kubectl delete -f prometheus-alertmanager-deploy.yaml 
[root@k8s-master1 prometheus]# kubectl apply -f prometheus-alertmanager-deploy.yaml 
[root@k8s-master1 prometheus]# kubectl get pods -n monitor-sa 
NAME                                 READY   STATUS    RESTARTS   AGE
node-exporter-nl5qz                  1/1     Running   1          24h
node-exporter-nxwkf                  1/1     Running   1          24h
node-exporter-x494t                  1/1     Running   1          24h
prometheus-server-85598946c7-dntwf   2/2     Running   0          7s