实践，可用
1、确保prometheus采集了alertmanager的metrics
- job_name: "alertmanager"
static_configs:
- targets: ["localhost:9093"]
2、插入dashboard：
Grafana Dashboard: 9741号, 地址: https://grafana.com/grafana/dashboards/9741
3、修改dashboard的prometheus数据源
4、刷新页面
---------------以下内容配置失败，仅参考---------------
参考：
https://jishuin.proginn.com/p/763bfbd3c114
https://www.cszhi.com/2019/10/18/Prometheus%E5%91%8A%E8%AD%A6%E5%A4%A7%E5%B1%8F/
数据源插件(grafana-prometheus-alertmanager-datasource): https://github.com/camptocamp/grafana-prometheus-alertmanager-datasource/
Grafana Dashboard: 12947号, 地址: https://grafana.com/grafana/dashboards/12947
安装：
grafana-cli plugins install camptocamp-prometheus-alertmanager-datasource
导入：
{ "annotations": { "list": [ { "builtIn": 1, "datasource": "-- Grafana --", "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "showIn": 0, "type": "dashboard" } ] }, "description": "Dashboard to visualize only Open(LIVE) Alerts of AlertManager instead of looking in Slack, Teams, and Emails.", "editable": true, "gnetId": 12947, "graphTooltip": 0, "id": null, "iteration": 1599147643921, "links": [], "panels": [ { "cacheTimeout": null, "datasource": "$alertmanager", "description": "", "fieldConfig": { "defaults": { "custom": {}, "displayName": "", "mappings": [ { "id": 0, "op": "=", "text": "0", "type": 1, "value": "null" } ], "nullValueMode": "connected", "thresholds": { "mode": "absolute", "steps": [ { "color": "red", "value": null } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 4, "w": 3, "x": 0, "y": 0 }, "id": 8, "links": [], "options": { "colorMode": "background", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": true }, "textMode": "auto" }, "pluginVersion": "7.1.4", "targets": [ { "annotations": false, "expr": "region=~\"$region\", severity=~\"$severity\",alertname=~\".*\", team=~\"$team\",env=~\"$env\"", "labelSelector": "*", "legendFormat": "", "refId": "A", "target": "Query", "type": "single" } ], "title": "Total", "type": "stat" }, { "cacheTimeout": null, "datasource": "$alertmanager", "description": "", "fieldConfig": { "defaults": { "custom": {}, "displayName": "", "mappings": [ { "id": 0, "op": "=", "text": "0", "type": 1, "value": "null" } ], "nullValueMode": "connected", "thresholds": { "mode": "absolute", "steps": [ { "color": "red", "value": null } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 4, "w": 3, "x": 3, "y": 0 }, "id": 20, "links": [], "options": { "colorMode": "background", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": true }, "textMode": "auto" }, "pluginVersion": "7.1.4", "targets": [ { "annotations": false, "expr": "region=~\"$region\", severity=~\"critical\",alertname=~\".*\", team=~\"$team\",env=~\"$env\"", "labelSelector": "*", "legendFormat": "", "refId": "A", "target": "Query", "type": "single" } ], "title": "Total Critical", "type": "stat" }, { "cacheTimeout": null, "datasource": "$alertmanager", "description": "", "fieldConfig": { "defaults": { "custom": {}, "displayName": "", "mappings": [ { "id": 0, "op": "=", "text": "0", "type": 1, "value": "null" } ], "nullValueMode": "connected", "thresholds": { "mode": "absolute", "steps": [ { "color": "orange", "value": null } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 4, "w": 3, "x": 6, "y": 0 }, "id": 21, "links": [], "options": { "colorMode": "background", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": true }, "textMode": "auto" }, "pluginVersion": "7.1.4", "targets": [ { "annotations": false, "expr": "region=~\"$region\", severity=~\"warning\",alertname=~\".*\", team=~\"$team\",env=~\"$env\"", "labelSelector": "*", "legendFormat": "", "refId": "A", "target": "Query", "type": "single" } ], "title": "Total Warning", "type": "stat" }, { "cacheTimeout": null, "datasource": "$alertmanager", "description": "", "fieldConfig": { "defaults": { "custom": {}, "displayName": "", "mappings": [ { "id": 0, "op": "=", "text": "0", "type": 1, "value": "null" } ], "nullValueMode": "connected", "thresholds": { "mode": "absolute", "steps": [ { "color": "red", "value": null } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 4, "w": 3, "x": 9, "y": 0 }, "id": 18, "links": [], "options": { "colorMode": "background", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": true }, "textMode": "auto" }, "pluginVersion": "7.1.4", "targets": [ { "annotations": false, "expr": "region=~\"$region\", severity=~\"$severity\",alertname=~\"InstanceDown\", team=~\"$team\",env=~\"$env\"", "labelSelector": "*", "legendFormat": "", "refId": "A", "target": "Query", "type": "single" } ], "title": "Instance Down", "type": "stat" }, { "cacheTimeout": null, "datasource": "$alertmanager", "description": "", "fieldConfig": { "defaults": { "custom": {}, "mappings": [ { "id": 0, "op": "=", "text": "0", "type": 1, "value": "null" } ], "nullValueMode": "connected", "thresholds": { "mode": "absolute", "steps": [ { "color": "red", "value": null } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 4, "w": 3, "x": 12, "y": 0 }, "id": 15, "links": [], "options": { "colorMode": "background", "graphMode": "area", "justifyMode": "auto", "orientation": "horizontal", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": true }, "textMode": "auto" }, "pluginVersion": "7.1.4", "targets": [ { "annotations": false, "expr": "region=~\"$region\", severity=~\"$severity\",alertname=~\"ServiceDown\", team=~\"$team\",env=~\"$env\"", "labelSelector": "*", "legendFormat": "", "refId": "A", "target": "Query", "type": "single" } ], "title": "Service Down", "type": "stat" }, { "cacheTimeout": null, "datasource": "$alertmanager", "description": "", "fieldConfig": { "defaults": { "custom": {}, "mappings": [ { "id": 0, "op": "=", "text": "0", "type": 1, "value": "null" } ], "nullValueMode": "connected", "thresholds": { "mode": "absolute", "steps": [ { "color": "orange", "value": null } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 4, "w": 3, "x": 15, "y": 0 }, "id": 16, "links": [], "options": { "colorMode": "background", "graphMode": "area", "justifyMode": "auto", "orientation": "horizontal", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": true }, "textMode": "auto" }, "pluginVersion": "7.1.4", "targets": [ { "annotations": false, "expr": "region=~\"$region\", severity=~\"$severity\",alertname=~\".*MemoryUtilization-WARN\", team=~\"$team\",env=~\"$env\"", "labelSelector": "*", "legendFormat": "", "refId": "A", "target": "Query", "type": "single" } ], "title": "MemoryUtilization-WARN", "type": "stat" }, { "cacheTimeout": null, "datasource": "$alertmanager", "description": "", "fieldConfig": { "defaults": { "custom": {}, "mappings": [ { "id": 0, "op": "=", "text": "0", "type": 1, "value": "null" } ], "nullValueMode": "connected", "thresholds": { "mode": "absolute", "steps": [ { "color": "orange", "value": null } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 4, "w": 3, "x": 18, "y": 0 }, "id": 17, "links": [], "options": { "colorMode": "background", "graphMode": "area", "justifyMode": "auto", "orientation": "horizontal", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": true }, "textMode": "auto" }, "pluginVersion": "7.1.4", "targets": [ { "annotations": false, "expr": "region=~\"$region\", severity=~\"$severity\",alertname=~\".*CpuUtilization-WARN\", team=~\"$team\",env=~\"$env\"", "labelSelector": "*", "legendFormat": "", "refId": "A", "target": "Query", "type": "single" } ], "title": "CpuUtilization-WARN", "type": "stat" }, { "cacheTimeout": null, "datasource": "$alertmanager", "description": "", "fieldConfig": { "defaults": { "custom": {}, "mappings": [ { "id": 0, "op": "=", "text": "0", "type": 1, "value": "null" } ], "nullValueMode": "connected", "thresholds": { "mode": "absolute", "steps": [ { "color": "orange", "value": null } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 4, "w": 3, "x": 21, "y": 0 }, "id": 19, "links": [], "options": { "colorMode": "background", "graphMode": "area", "justifyMode": "auto", "orientation": "horizontal", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": true }, "textMode": "auto" }, "pluginVersion": "7.1.4", "targets": [ { "annotations": false, "expr": "region=~\"$region\", severity=~\"$severity\",alertname=~\".*DiskUtilization - WARN\", team=~\"$team\",env=~\"$env\"", "labelSelector": "*", "legendFormat": "", "refId": "A", "target": "Query", "type": "single" } ], "title": "DiskUtilization - WARN", "type": "stat" }, { "columns": [], "datasource": "$alertmanager", "description": "MAKE SURE TO SCROLL TO BOTTOM AND CHECK FOR MORE ALERTS", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fontSize": "100%", "gridPos": { "h": 98, "w": 24, "x": 0, "y": 4 }, "id": 14, "pageSize": null, "showHeader": true, "sort": { "col": 0, "desc": true }, "styles": [ { "alias": "TIME", "align": "", "dateFormat": "YYYY-MM-DD HH:mm:ss", "link": false, "pattern": "Time", "type": "date" }, { "alias": "ALERT", "align": "auto", "colorMode": null, "colors": [ "rgba(245, 54, 54, 0.9)", "rgba(237, 129, 40, 0.89)", "rgba(50, 172, 45, 0.97)" ], "dateFormat": "YYYY-MM-DD HH:mm:ss", "decimals": 2, "mappingType": 1, "pattern": "alertname", "preserveFormat": false, "thresholds": [], "type": "string", "unit": "short" }, { "alias": "DESCRIPTION", "align": "auto", "colorMode": null, "colors": [ "rgba(245, 54, 54, 0.9)", "rgba(237, 129, 40, 0.89)", "rgba(50, 172, 45, 0.97)" ], "dateFormat": "YYYY-MM-DD HH:mm:ss", "decimals": 2, "link": true, "linkTargetBlank": true, "linkTooltip": "Click to view detail metrics", "linkUrl": "/d/LINK_TO_YOUR_NODE_SPECIFIC_DASHBOARD_GOES_HERE?var-host=${__cell_2}", "mappingType": 1, "pattern": "data", "preserveFormat": false, "sanitize": false, "thresholds": [], "type": "string", "unit": "short" }, { "alias": "SEVERITY", "align": "auto", "colorMode": "row", "colors": [ "rgba(50, 172, 45, 0.97)", "rgba(237, 129, 40, 0.89)", "rgba(245, 54, 54, 0.9)" ], "dateFormat": "YYYY-MM-DD HH:mm:ss", "decimals": 0, "mappingType": 1, "pattern": "severity", "preserveFormat": false, "sanitize": false, "thresholds": [ "2", "4" ], "type": "string", "unit": "short", "valueMaps": [ { "text": "CRITICAL", "value": "4" }, { "text": "WARNING", "value": "2" } ] }, { "alias": "REGION", "align": "auto", "colorMode": null, "colors": [ "rgba(245, 54, 54, 0.9)", "rgba(237, 129, 40, 0.89)", "rgba(50, 172, 45, 0.97)" ], "dateFormat": "YYYY-MM-DD HH:mm:ss", "decimals": 2, "mappingType": 1, "pattern": "region", "thresholds": [], "type": "number", "unit": "short" }, { "alias": "ENV", "align": "auto", "colorMode": null, "colors": [ "rgba(245, 54, 54, 0.9)", "rgba(237, 129, 40, 0.89)", "rgba(50, 172, 45, 0.97)" ], "dateFormat": "YYYY-MM-DD HH:mm:ss", "decimals": 2, "mappingType": 1, "pattern": "env", "thresholds": [], "type": "number", "unit": "short" }, { "alias": "HOST", "align": "auto", "colorMode": null, "colors": [ "rgba(245, 54, 54, 0.9)", "rgba(237, 129, 40, 0.89)", "rgba(50, 172, 45, 0.97)" ], "dateFormat": "YYYY-MM-DD HH:mm:ss", "decimals": 2, "mappingType": 1, "pattern": "host", "thresholds": [], "type": "number", "unit": "short" }, { "alias": "", "align": "right", "colorMode": null, "colors": [ "rgba(245, 54, 54, 0.9)", "rgba(237, 129, 40, 0.89)", "rgba(50, 172, 45, 0.97)" ], "decimals": 2, "pattern": "/.*/", "thresholds": [], "type": "hidden", "unit": "short" } ], "targets": [ { "annotations": false, "expr": "region=~\"$region\", severity=~\"$severity\",alertname=~\"$alertname\", team=~\"$team\",env=~\"$env\"", "labelSelector": "*", "legendFormat": "{{ msg }}", "refId": "A", "target": "Query", "type": "table" } ], "timeFrom": null, "timeShift": null, "title": "OPEN ALERTS", "transform": "table", "transformations": [ { "id": "organize", "options": { "excludeByName": {}, "indexByName": { "Time": 0, "alertname": 1, "availability_zone": 3, "env": 4, "container_name": 18, "data": 17, "device": 5, "fstype": 6, "host": 2, "image_name": 19, "instance": 7, "instance_type": 8, "ipaddress": 9, "job": 10, "mountpoint": 11, "name": 23, "platform_version": 12, "prometheus": 13, "region": 14, "service_name": 20, "severity": 15, "slack_channel": 21, "state": 24, "tag_name": 22, "team": 16, "type": 25 }, "renameByName": {} } } ], "type": "table-old" } ], "refresh": "30s", "schemaVersion": 26, "style": "dark", "tags": [ "alertmanager", "open-alerts", "shubhamc183" ], "templating": { "list": [ { "current": { "selected": true, "text": "PROD Alertmanager", "value": "PROD Alertmanager" }, "hide": 0, "includeAll": false, "label": "Alertmanager", "multi": false, "name": "alertmanager", "options": [], "query": "camptocamp-prometheus-alertmanager-datasource", "queryValue": "", "refresh": 1, "regex": "", "skipUrlSync": false, "type": "datasource" }, { "allValue": ".*", "current": { "selected": true, "text": [ "All" ], "value": [ "$__all" ] }, "datasource": "$alertmanager", "definition": "label_values(region)", "hide": 0, "includeAll": true, "label": "Region", "multi": true, "name": "region", "options": [], "query": "label_values(region)", "refresh": 1, "regex": "", "skipUrlSync": false, "sort": 0, "tagValuesQuery": "", "tags": [], "tagsQuery": "", "type": "query", "useTags": false }, { "allValue": ".*", "current": { "selected": true, "tags": [], "text": [ "All" ], "value": [ "$__all" ] }, "datasource": "$alertmanager", "definition": "label_values(region=~\"$region\")", "hide": 0, "includeAll": true, "label": "Severity", "multi": true, "name": "severity", "options": [], "query": "label_values(region=~\"$region\")", "refresh": 2, "regex": "/\"severity\":\"(.*?)\"/", "skipUrlSync": false, "sort": 0, "tagValuesQuery": "", "tags": [], "tagsQuery": "", "type": "query", "useTags": false }, { "allValue": ".*", "current": { "selected": true, "text": [ "All" ], "value": [ "$__all" ] }, "datasource": "$alertmanager", "definition": "label_values(region=~\"$region\", severity=~\"$severity\")", "hide": 0, "includeAll": true, "label": "Alert", "multi": true, "name": "alertname", "options": [], "query": "label_values(region=~\"$region\", severity=~\"$severity\")", "refresh": 2, "regex": "/\"alertname\":\"(.*?)\"/", "skipUrlSync": false, "sort": 0, "tagValuesQuery": "", "tags": [], "tagsQuery": "", "type": "query", "useTags": false }, { "allValue": ".*", "current": { "selected": true, "text": [ "All" ], "value": [ "$__all" ] }, "datasource": "$alertmanager", "definition": "label_values(region=~\"$region\", severity=~\"$severity\", alertname=~\"$alertname\")", "hide": 0, "includeAll": true, "label": "Product", "multi": true, "name": "team", "options": [], "query": "label_values(region=~\"$region\", severity=~\"$severity\", alertname=~\"$alertname\")", "refresh": 2, "regex": "/\"team\":\"(.*?)\"/", "skipUrlSync": false, "sort": 0, "tagValuesQuery": "", "tags": [], "tagsQuery": "", "type": "query", "useTags": false }, { "allValue": ".*", "current": { "selected": true, "text": [ "All" ], "value": [ "$__all" ] }, "datasource": "$alertmanager", "definition": "label_values(region=~\"$region\", severity=~\"$severity\", alertname=~\"$alertname\", team=~\"$team\")", "hide": 0, "includeAll": true, "label": "ENV", "multi": true, "name": "env", "options": [], "query": "label_values(region=~\"$region\", severity=~\"$severity\", alertname=~\"$alertname\", team=~\"$team\")", "refresh": 2, "regex": "/\"env\":\"(.*?)\"/", "skipUrlSync": false, "sort": 0, "tagValuesQuery": "", "tags": [], "tagsQuery": "", "type": "query", "useTags": false }, { "allValue": ".*", "current": { "selected": true, "text": [ "All" ], "value": [ "$__all" ] }, "datasource": "$alertmanager", "definition": "label_values(region=~\"$region\", severity=~\"$severity\", alertname=~\"$alertname\", team=~\"$team\", env=~\"$env\")", "hide": 2, "includeAll": true, "label": "Node", "multi": true, "name": "host", "options": [], "query": "label_values(region=~\"$region\", severity=~\"$severity\", alertname=~\"$alertname\", team=~\"$team\", env=~\"$env\")", "refresh": 2, "regex": "/\"host\":\"(.*?)\"/", "skipUrlSync": false, "sort": 0, "tagValuesQuery": "", "tags": [], "tagsQuery": "", "type": "query", "useTags": false } ] }, "time": { "from": "now-6h", "to": "now" }, "timepicker": { "refresh_intervals": [ "5s", "10s", "30s", "1m", "5m", "15m", "30m", "1h", "2h", "1d" ], "time_options": [ "5m", "15m", "1h", "6h", "12h", "24h", "2d", "7d", "30d" ] }, "timezone": "browser", "title": "OPEN ALERTS OF ALERTMANAGER", "uid": "WojOgXTmk", "version": 1 }