mkdir -p /home/prom/{prometheus,prometheus/data,alertmanager,grafana}chmod 777 /home/prom/{prometheus/data,grafana}cd /home/promtree .. ├── alertmanager │ ├── alertmanager.yml │ └── config.yml ├── docker-compose.yml ├── grafana └── prometheus├── alert-rules.yml├── data└── prometheus.yml4 directories, 5 filesvim /home/prom/prometheus/alert-rules.yml groups:- name: node-alertrules:- alert: NodeDownexpr: up{job="node"} == 0for: 5mlabels:severity: criticalinstance: "{{ $labels.instance }}"annotations:summary: "instance: {{ $labels.instance }} down"description: "Instance: {{ $labels.instance }} 已經宕機 5分鐘"value: "{{ $value }}"- alert: NodeCpuHighexpr: (1 - avg by (instance) (irate(node_cpu_seconds_total{job="node",mode="idle"}[5m]))) * 100 > 80for: 5mlabels:severity: warninginstance: "{{ $labels.instance }}"annotations:summary: "instance: {{ $labels.instance }} cpu使用率過高"description: "CPU 使用率超過 80%"value: "{{ $value }}"- alert: NodeCpuIowaitHighexpr: avg by (instance) (irate(node_cpu_seconds_total{job="node",mode="iowait"}[5m])) * 100 > 50for: 5mlabels:severity: warninginstance: "{{ $labels.instance }}"annotations:summary: "instance: {{ $labels.instance }} cpu iowait 使用率過高"description: "CPU iowait 使用率超過 50%"value: "{{ $value }}"- alert: NodeLoad5Highexpr: node_load5 > (count by (instance) (node_cpu_seconds_total{job="node",mode='system'})) * 1.2for: 5mlabels:severity: warninginstance: "{{ $labels.instance }}"annotations:summary: "instance: {{ $labels.instance }} load(5m) 過高"description: "Load(5m) 過高，超出cpu核數 1.2倍"value: "{{ $value }}"- alert: NodeMemoryHighexpr: (1 - node_memory_MemAvailable_bytes{job="node"} / node_memory_MemTotal_bytes{job="node"}) * 100 > 90for: 5mlabels:severity: warninginstance: "{{ $labels.instance }}"annotations:summary: "instance: {{ $labels.instance }} memory 使用率過高"description: "Memory 使用率超過 90%"value: "{{ $value }}"- alert: NodeDiskRootHighexpr: (1 - node_filesystem_avail_bytes{job="node",fstype=~"ext.*|xfs",mountpoint ="/"} / node_filesystem_size_bytes{job="node",fstype=~"ext.*|xfs",mountpoint ="/"}) * 100 > 90for: 10mlabels:severity: warninginstance: "{{ $labels.instance }}"annotations:summary: "instance: {{ $labels.instance }} disk(/ 分區) 使用率過高"description: "Disk(/ 分區) 使用率超過 90%"value: "{{ $value }}"- alert: NodeDiskBootHighexpr: (1 - node_filesystem_avail_bytes{job="node",fstype=~"ext.*|xfs",mountpoint ="/boot"} / node_filesystem_size_bytes{job="node",fstype=~"ext.*|xfs",mountpoint ="/boot"}) * 100 > 80for: 10mlabels:severity: warninginstance: "{{ $labels.instance }}"annotations:summary: "instance: {{ $labels.instance }} disk(/boot 分區) 使用率過高"description: "Disk(/boot 分區) 使用率超過 80%"value: "{{ $value }}"- alert: NodeDiskReadHighexpr: irate(node_disk_read_bytes_total{job="node"}[5m]) > 20 * (1024 ^ 2)for: 5mlabels:severity: warninginstance: "{{ $labels.instance }}"annotations:summary: "instance: {{ $labels.instance }} disk 讀取字節數 速率過高"description: "Disk 讀取字節數 速率超過 20 MB/s"value: "{{ $value }}"- alert: NodeDiskWriteHighexpr: irate(node_disk_written_bytes_total{job="node"}[5m]) > 20 * (1024 ^ 2)for: 5mlabels:severity: warninginstance: "{{ $labels.instance }}"annotations:summary: "instance: {{ $labels.instance }} disk 寫入字節數 速率過高"description: "Disk 寫入字節數 速率超過 20 MB/s"value: "{{ $value }}"- alert: NodeDiskReadRateCountHighexpr: irate(node_disk_reads_completed_total{job="node"}[5m]) > 3000for: 5mlabels:severity: warninginstance: "{{ $labels.instance }}"annotations:summary: "instance: {{ $labels.instance }} disk iops 每秒讀取速率過高"description: "Disk iops 每秒讀取速率超過 3000 iops"value: "{{ $value }}"- alert: NodeDiskWriteRateCountHighexpr: irate(node_disk_writes_completed_total{job="node"}[5m]) > 3000for: 5mlabels:severity: warninginstance: "{{ $labels.instance }}"annotations:summary: "instance: {{ $labels.instance }} disk iops 每秒寫入速率過高"description: "Disk iops 每秒寫入速率超過 3000 iops"value: "{{ $value }}"- alert: NodeInodeRootUsedPercentHighexpr: (1 - node_filesystem_files_free{job="node",fstype=~"ext4|xfs",mountpoint="/"} / node_filesystem_files{job="node",fstype=~"ext4|xfs",mountpoint="/"}) * 100 > 80for: 10mlabels:severity: warninginstance: "{{ $labels.instance }}"annotations:summary: "instance: {{ $labels.instance }} disk(/ 分區) inode 使用率過高"description: "Disk (/ 分區) inode 使用率超過 80%"value: "{{ $value }}"- alert: NodeInodeBootUsedPercentHighexpr: (1 - node_filesystem_files_free{job="node",fstype=~"ext4|xfs",mountpoint="/boot"} / node_filesystem_files{job="node",fstype=~"ext4|xfs",mountpoint="/boot"}) * 100 > 80for: 10mlabels:severity: warninginstance: "{{ $labels.instance }}"annotations:summary: "instance: {{ $labels.instance }} disk(/boot 分區) inode 使用率過高"description: "Disk (/boot 分區) inode 使用率超過 80%"value: "{{ $value }}"- alert: NodeFilefdAllocatedPercentHighexpr: node_filefd_allocated{job="node"} / node_filefd_maximum{job="node"} * 100 > 80for: 10mlabels:severity: warninginstance: "{{ $labels.instance }}"annotations:summary: "instance: {{ $labels.instance }} filefd 打開百分比過高"description: "Filefd 打開百分比 超過 80%"value: "{{ $value }}"- alert: NodeNetworkNetinBitRateHighexpr: avg by (instance) (irate(node_network_receive_bytes_total{device=~"eth0|eth1|ens33|ens37"}[1m]) * 8) > 20 * (1024 ^ 2) * 8for: 3mlabels:severity: warninginstance: "{{ $labels.instance }}"annotations:summary: "instance: {{ $labels.instance }} network 接收比特數 速率過高"description: "Network 接收比特數 速率超過 20MB/s"value: "{{ $value }}"- alert: NodeNetworkNetoutBitRateHighexpr: avg by (instance) (irate(node_network_transmit_bytes_total{device=~"eth0|eth1|ens33|ens37"}[1m]) * 8) > 20 * (1024 ^ 2) * 8for: 3mlabels:severity: warninginstance: "{{ $labels.instance }}"annotations:summary: "instance: {{ $labels.instance }} network 發送比特數 速率過高"description: "Network 發送比特數 速率超過 20MB/s"value: "{{ $value }}"- alert: NodeNetworkNetinPacketErrorRateHighexpr: avg by (instance) (irate(node_network_receive_errs_total{device=~"eth0|eth1|ens33|ens37"}[1m])) > 15for: 3mlabels:severity: warninginstance: "{{ $labels.instance }}"annotations:summary: "instance: {{ $labels.instance }} 接收錯誤包 速率過高"description: "Network 接收錯誤包 速率超過 15個/秒"value: "{{ $value }}"- alert: NodeNetworkNetoutPacketErrorRateHighexpr: avg by (instance) (irate(node_network_transmit_packets_total{device=~"eth0|eth1|ens33|ens37"}[1m])) > 15for: 3mlabels:severity: warninginstance: "{{ $labels.instance }}"annotations:summary: "instance: {{ $labels.instance }} 發送錯誤包 速率過高"description: "Network 發送錯誤包 速率超過 15個/秒"value: "{{ $value }}"- alert: NodeProcessBlockedHighexpr: node_procs_blocked{job="node"} > 10for: 10mlabels:severity: warninginstance: "{{ $labels.instance }}"annotations:summary: "instance: {{ $labels.instance }} 當前被阻塞的任務的數量過多"description: "Process 當前被阻塞的任務的數量超過 10個"value: "{{ $value }}"- alert: NodeTimeOffsetHighexpr: abs(node_timex_offset_seconds{job="node"}) > 3 * 60for: 2mlabels:severity: infoinstance: "{{ $labels.instance }}"annotations:summary: "instance: {{ $labels.instance }} 時間偏差過大"description: "Time 節點的時間偏差超過 3m"value: "{{ $value }}"vim /home/prom/prometheus/prometheus.yml global:scrape_interval: 15sevaluation_interval: 15salerting:alertmanagers:- static_configs:- targets:- alertmanager:9093rule_files:- "*rules.yml"scrape_configs:- job_name: 'prometheus'static_configs:- targets: ['prometheus:9090']- job_name: 'node'static_configs:- targets: ['node-exporter:9100','node2:9100','node2:9100']- job_name: 'alertmanager'static_configs:- targets: ['alertmanager:9093']vim /home/prom/alertmanager/config.yml targets:webhook:url: https://oapi.dingtalk.com/robot/send?access_token=xxxxxxxxxxxx #修改為釘釘機器人的webhookmention:all: truevim /home/prom/alertmanager/alertmanager.yml global:resolve_timeout: 5msmtp_smarthost: 'smtp.163.com:465' #郵箱smtp服務器代理，啟用SSL發信, 端口一般是465smtp_from: 'alert@163.com' #發送郵箱名稱smtp_auth_username: 'alert@163.com' #郵箱名稱smtp_auth_password: 'password' #郵箱密碼或授權碼smtp_require_tls: falseroute:receiver: 'default'group_wait: 10sgroup_interval: 1mrepeat_interval: 1hgroup_by: ['alertname']inhibit_rules: - source_match:severity: 'critical'target_match:severity: 'warning'equal: ['alertname', 'instance']receivers: - name: 'default'email_configs:- to: 'receiver@163.com'send_resolved: truewebhook_configs:- url: 'http://dingtalk:8060/dingtalk/webhook/send'send_resolved: truevim /home/prom/docker-compose.yml version: '3.7'services:node-exporter:image: prom/node-exporter:latestports:- "9100:9100"networks:- promdingtalk:image: timonwong/prometheus-webhook-dingtalk:latestvolumes:- type: bindsource: ./alertmanager/config.ymltarget: /etc/prometheus-webhook-dingtalk/config.ymlread_only: trueports:- "8060:8060"networks:- promalertmanager:depends_on:- dingtalkimage: prom/alertmanager:latestvolumes:- type: bindsource: ./alertmanager/alertmanager.ymltarget: /etc/alertmanager/alertmanager.ymlread_only: trueports:- "9093:9093"- "9094:9094"networks:- promprometheus:depends_on:- alertmanagerimage: prom/prometheus:latestvolumes:- type: bindsource: ./prometheus/prometheus.ymltarget: /etc/prometheus/prometheus.ymlread_only: true- type: bindsource: ./prometheus/alert-rules.ymltarget: /etc/prometheus/alert-rules.ymlread_only: true- type: volumesource: prometheustarget: /prometheusports:- "9090:9090"networks:- promgrafana:depends_on:- prometheusimage: grafana/grafana:latestvolumes:- type: volumesource: grafanatarget: /var/lib/grafanaports:- "3000:3000"networks:- promvolumes:prometheus:driver: localdriver_opts:type: noneo: binddevice: /home/prom/prometheus/datagrafana:driver: localdriver_opts:type: noneo: binddevice: /home/prom/grafananetworks:prom:driver: bridgedocker-compose up -ddocker-compose psName Command State Ports -------------------------------------------------------------------------------------------------------------- prom_alertmanager_1 /bin/alertmanager --config ... Up 0.0.0.0:9093->9093/tcp, 0.0.0.0:9094->9094/tcp prom_dingtalk_1 /bin/prometheus-webhook-di ... Up 0.0.0.0:8060->8060/tcp prom_grafana_1 /run.sh Up 0.0.0.0:3000->3000/tcp prom_node-exporter_1 /bin/node_exporter Up 0.0.0.0:9100->9100/tcp prom_prometheus_1 /bin/prometheus --config.f ... Up 0.0.0.0:9090->9090/tcpdocker psCONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES c1ec4cc9c41f grafana/grafana:latest "/run.sh" 36 seconds ago Up 36 seconds 0.0.0.0:3000->3000/tcp prom_grafana_1 8cd521c327d8 prom/prometheus:latest "/bin/prometheus --c…" 37 seconds ago Up 36 seconds 0.0.0.0:9090->9090/tcp prom_prometheus_1 ef93c8c06ca0 prom/alertmanager:latest "/bin/alertmanager -…" 37 seconds ago Up 37 seconds 0.0.0.0:9093-9094->9093-9094/tcp prom_alertmanager_1 d358a2a39b8d timonwong/prometheus-webhook-dingtalk:latest "/bin/prometheus-web…" 38 seconds ago Up 37 seconds 0.0.0.0:8060->8060/tcp prom_dingtalk_1 366ff81e7a65 prom/node-exporter:latest "/bin/node_exporter" 38 seconds ago Up 37 seconds 0.0.0.0:9100->9100/tcp prom_node-exporter_1

參考鏈接：

https://blog.csdn.net/miss1181248983/article/details/107496421

總結

以上是生活随笔為你收集整理的【脚本整理】docker-compose 部署prometheus + grafana的全部內容，希望文章能夠幫你解決所遇到的問題。

如果覺得生活随笔網站內容還不錯，歡迎將生活随笔推薦給好友。