dingtalk使用自定义告警模板

朱雀 2022-10-26 03:54 431阅读 0赞

本文内容基于 [k8s部署prometheus + grafana][k8s_prometheus _ grafana]，只针对 dingtalk 部分进行修改。

--------------------

vim dingtalk/config.yaml

apiVersion: v1
    kind: ConfigMap
    metadata:
      name: dingtalk-config
      namespace: monitoring
    data:
      config.yml: |-
        templates:
          - /etc/prometheus-webhook-dingtalk/default.tmpl
        targets:
          webhook:
            url: https://oapi.dingtalk.com/robot/send?access_token=xxxxxxxxxxxx             #修改为钉钉机器人的webhook
    # message:
    # title: '{ { template "legacy.title" . }}' #使用 Legacy（旧版） 模板
    # text: '{ { template "legacy.content" . }}'
            mention:
              all: true #@所有人
    
      default.tmpl: | { { define "__subject" }}[{ { .Status | toUpper }}{ { if eq .Status "firing" }}:{ { .Alerts.Firing | len }}{ { end }}] { { .GroupLabels.SortedPairs.Values | join " " }} { { if gt (len .CommonLabels) (len .GroupLabels) }}({ { with .CommonLabels.Remove .GroupLabels.Names }}{ { .Values | join " " }}{ { end }}){ { end }}{ { end }} { { define "__alertmanagerURL" }}{ { .ExternalURL }}/d/9CWBz0bik/k8sji-qun-shou-ye?orgId=1 { { end }}
        
        { {  define "__text_alert_list" }}{ {  range . }}
        **Labels**
        { {  range .Labels.SortedPairs }}> - { {  .Name }}: { {  .Value | markdown | html }}
        { {  end }}
        **Annotations**
        { {  range .Annotations.SortedPairs }}> - { {  .Name }}: { {  .Value | markdown | html }}
        { {  end }}
        **Source:** [{ {  .GeneratorURL }}]({ {  .GeneratorURL }})
        { {  end }}{ {  end }}
        
        { {  define "default.__text_alert_list" }}{ {  range . }}
        ---
        **告警级别:** { {  .Labels.severity | upper }}
        
        **触发时间:** { {  dateInZone "2006.01.02 15:04:05" (.StartsAt) "Asia/Shanghai" }}
        
        **事件信息:**
        { {  range .Annotations.SortedPairs }}> - { {  .Name }}: { {  .Value | markdown | html }}    
    
        { {  end }}{ {  end }}
        { {  end }}
    
        { {  define "default.__text_alertresolve_list" }}{ {  range . }}
        ---
        **告警级别:** { {  .Labels.severity | upper }}
        
        **触发时间:** { {  dateInZone "2006.01.02 15:04:05" (.StartsAt) "Asia/Shanghai" }}
        
        **结束时间:** { {  dateInZone "2006.01.02 15:04:05" (.EndsAt) "Asia/Shanghai" }}
        
        **事件信息:**
        { {  range .Annotations.SortedPairs }}> - { {  .Name }}: { {  .Value | markdown | html }}    
    
        { {  end }}{ {  end }}
        { {  end }}
        
        { { /* Default */}}
        { {  define "default.title" }}{ {  template "__subject" . }}{ {  end }}
        { {  define "default.content" }}#### \[{ { .Status | toUpper }}{ { if eq .Status "firing" }}:{ { .Alerts.Firing | len }}{ { end }}\] **[{ { index .GroupLabels "alertname" }}]({ { template "__alertmanagerURL" . }})**
    
        { {  if gt (len .Alerts.Firing) 0 -}} 
        ![警报 图标](http://wx4.sinaimg.cn/large/006APoFYly1g0ggpt1aq3j306o06o3yt.jpg)
    
        **=====侦测到故障=====**
        { {  template "default.__text_alert_list" .Alerts.Firing }} 
        { { - end }}
        
        { {  if gt (len .Alerts.Resolved) 0 -}}
        ![警报 图标](https://i01piccdn.sogoucdn.com/2dbf8a9d48e921e7)
    
        **=====故障已修复=====**
        { {  template "default.__text_alertresolve_list" .Alerts.Resolved }}
        { { - end }}
        { { - end }}
        
        { { /* Legacy */}}
        { {  define "legacy.title" }}{ {  template "__subject" . }}{ {  end }}
        { {  define "legacy.content" }}#### \[{ { .Status | toUpper }}{ { if eq .Status "firing" }}:{ { .Alerts.Firing | len }}{ { end }}\] **[{ { index .GroupLabels "alertname" }}]({ { template "__alertmanagerURL" . }})**
        { {  template "__text_alert_list" .Alerts.Firing }}
        { { - end }}
        
        { { /* Following names for compatibility */}}
        { {  define "ding.link.title" }}{ {  template "default.title" . }}{ {  end }}
        { {  define "ding.link.content" }}{ {  template "default.content" . }}{ {  end }}

vim dingtalk/dingtalk.yaml

apiVersion: extensions/v1beta1
    kind: Ingress
    metadata:
      name: dingtalk
      namespace: monitoring
    spec:
      rules:
      - host: dingtalk.lzxlinux.cn
        http:
          paths:
          - path: /
            backend:
              serviceName: dingtalk
              servicePort: 8060
    
    ---
    apiVersion: v1
    kind: Service
    metadata:
      name: dingtalk
      namespace: monitoring
      labels:
        app: dingtalk
      annotations:
        prometheus.io/scrape: 'false'
    spec:
      selector:
        app: dingtalk
      ports:
      - name: dingtalk
        port: 8060
        protocol: TCP
        targetPort: 8060
      
    ---
    apiVersion: apps/v1
    kind: Deployment
    metadata:
      name: dingtalk
      namespace: monitoring
    spec:
      replicas: 1
      selector:
        matchLabels:
          app: dingtalk
      template:
        metadata:
          name: dingtalk
          labels:
            app: dingtalk
        spec:
          containers:
          - name: dingtalk
            image: timonwong/prometheus-webhook-dingtalk:latest
            imagePullPolicy: IfNotPresent
            args:
            - --web.listen-address=:8060
            - --config.file=/etc/prometheus-webhook-dingtalk/config.yml
            - --web.enable-ui             #启动ui访问，进行模板验证
            ports:
            - containerPort: 8060
            volumeMounts:
            - name: config
              mountPath: /etc/prometheus-webhook-dingtalk
          securityContext:
            runAsUser: 0
            fsGroup: 0
          volumes:
          - name: config
            configMap:
              name: dingtalk-config

kubectl apply -f dingtalk/

本地添加hosts，访问 `https://dingtalk.lzxlinux.cn/ui`，进行模板验证，

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L21pc3MxMTgxMjQ4OTgz_size_16_color_FFFFFF_t_70_pic_center]

--------------------

*  测试告警：

模拟CPU使用率为80%，

yum install -y stress-ng
    
    stress-ng -c 0 -l 80                # -c 指定压力源进程的数量，以匹配在线CPU的数量，0表示加载每个cpu；-l 指定CPU使用率；Ctrl + C 退出

收到故障告警：

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L21pc3MxMTgxMjQ4OTgz_size_16_color_FFFFFF_t_70_pic_center 1]

收到恢复告警：

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L21pc3MxMTgxMjQ4OTgz_size_16_color_FFFFFF_t_70_pic_center 2]

需要注意的是，default.tmpl 中包含了 `/d/9CWBz0bik/k8sji-qun-shou-ye?orgId=1`，这正是 grafana 的 url。因为 alertmanager 的 `--web.external-url` 参数已经被更改为 grafana 地址 `http://grafana.lzxlinux.cn`，最终点击告警消息中的链接就能够跳转到 `http://grafana.lzxlinux.cn/d/9CWBz0bik/k8sji-qun-shou-ye?orgId=1`。

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L21pc3MxMTgxMjQ4OTgz_size_16_color_FFFFFF_t_70_pic_center 3]

default.tmpl 中包含的 url 请按自己的情况设置，dingtalk 使用自定义告警模板配置至此完成。具体还可参考：[prometheus-webhook-dingtalk][]

--------------------

[k8s_prometheus _ grafana]: https://blog.csdn.net/miss1181248983/article/details/107673128
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L21pc3MxMTgxMjQ4OTgz_size_16_color_FFFFFF_t_70_pic_center]: /images/20221024/84a63f04ae04483499d18723d2fbc546.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L21pc3MxMTgxMjQ4OTgz_size_16_color_FFFFFF_t_70_pic_center 1]: /images/20221024/c7bed48f9c574f209f1e62efd4b97900.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L21pc3MxMTgxMjQ4OTgz_size_16_color_FFFFFF_t_70_pic_center 2]: /images/20221024/71d5c5fc06504590ab3ff89f65b4e61b.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L21pc3MxMTgxMjQ4OTgz_size_16_color_FFFFFF_t_70_pic_center 3]: /images/20221024/8eb7ca5018db4922935b15d8cb9d347e.png
[prometheus-webhook-dingtalk]: https://github.com/timonwong/prometheus-webhook-dingtalk/blob/master/docs/FAQ_zh.md