path: root/tools/lma/ansible-server/roles/monitoring/files/prometheus/prometheus-config.yaml

# Copyright 2020 Aditya Srivastava.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

apiVersion: v1
kind: ConfigMap
metadata:
  name: prometheus-config
  namespace: monitoring
data:
  alert.rules: |-
    groups:
    - name: targets
      rules:
      - alert: MonitorServiceDown
        expr: up == 0
        for: 30s
        labels:
          severity: critical
        annotations:
          summary: "Monitor service non-operational"
          description: "Service {{ $labels.instance }} is down."
      - alert: HighCpuLoad
        expr: node_load1 > 1.9
        for: 15s
        labels:
          severity: critical
        annotations:
          summary: "Service under high load"
          description: "Docker host is under high load, the avg load 1m is at {{ $value}}. Reported by instance {{ $labels.instance }} of job {{ $labels.job }}."

    - name: host and hardware
      rules:
      - alert: HostHighCpuLoad
        expr: 100 - (avg by(instance) (irate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 80
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Host high CPU load (instance {{ $labels.instance }})"
          description: "CPU load is > 80%\n  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

      - alert: HostSwapIsFillingUp
        expr: (1 - (node_memory_SwapFree_bytes / node_memory_SwapTotal_bytes)) * 100 > 80
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Host swap is filling up (instance {{ $labels.instance }})"
          description: "Swap is filling up (>80%)\n  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

      - alert: HighMemoryLoad
        expr: (sum(node_memory_MemTotal_bytes) - sum(node_memory_MemFree_bytes + node_memory_Buffers_bytes + node_memory_Cached_bytes) ) / sum(node_memory_MemTotal_bytes) * 100 > 85
        for: 30s
        labels:
          severity: warning
        annotations:
          summary: "Server memory is almost full"
          description: "Docker host memory usage is {{ humanize $value}}%. Reported by instance {{ $labels.instance }} of job {{ $labels.job }}."

      - alert: HighStorageLoad
        expr: (node_filesystem_size_bytes{fstype="aufs"} - node_filesystem_free_bytes{fstype="aufs"}) / node_filesystem_size_bytes{fstype="aufs"}  * 100 > 85
        for: 30s
        labels:
          severity: warning
        annotations:
          summary: "Server storage is almost full"
          description: "Docker host storage usage is {{ humanize $value}}%. Reported by instance {{ $labels.instance }} of job {{ $labels.job }}."

      - alert: HostNetworkTransmitErrors
        expr: increase(node_network_transmit_errs_total[5m]) > 0
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Host Network Transmit Errors (instance {{ $labels.instance }})"
          description: "{{ $labels.instance }} interface {{ $labels.device }} has encountered {{ printf \"%.0f\" $value }} transmit errors in the last five minutes.\n  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

      - alert: HostOutOfMemory
        expr: node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes * 100 < 10
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Host out of memory (instance {{ $labels.instance }})"
          description: "Node memory is filling up (< 10% left)\n  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

      - alert: HostMemoryUnderMemoryPressure
        expr: rate(node_vmstat_pgmajfault[1m]) > 1000
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Host memory under memory pressure (instance {{ $labels.instance }})"
          description: "The node is under heavy memory pressure. High rate of major page faults\n  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

      - alert: HostUnusualNetworkThroughputIn
        expr: sum by (instance) (irate(node_network_receive_bytes_total[2m])) / 1024 / 1024 > 100
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Host unusual network throughput in (instance {{ $labels.instance }})"
          description: "Host network interfaces are probably receiving too much data (> 100 MB/s)\n  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

      - alert: HostUnusualNetworkThroughputOut
        expr: sum by (instance) (irate(node_network_transmit_bytes_total[2m])) / 1024 / 1024 > 100
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Host unusual network throughput out (instance {{ $labels.instance }})"
          description: "Host network interfaces are probably sending too much data (> 100 MB/s)\n  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

      - alert: HostUnusualDiskRateRead
        expr: sum by (instance) (irate(node_disk_read_bytes_total[2m])) / 1024 / 1024 > 50
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Host unusual disk read rate (instance {{ $labels.instance }})"
          description: "Disk is probably reading too much data (> 50 MB/s)\n  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

      - alert: HostUnusualDiskRateWrite
        expr: sum by (instance) (irate(node_disk_written_bytes_total[2m])) / 1024 / 1024 > 50
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Host unusual disk write rate (instance {{ $labels.instance }})"
          description: "Disk is probably writing too much data (> 50 MB/s)\n  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

      - alert: HostOutOfDiskSpace
        expr: (node_filesystem_avail_bytes{mountpoint="/rootfs"}  * 100) / node_filesystem_size_bytes{mountpoint="/rootfs"} < 10
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Host out of disk space (instance {{ $labels.instance }})"
          description: "Disk is almost full (< 10% left)\n  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

      - alert: HostDiskWillFillIn4Hours
        expr: predict_linear(node_filesystem_free_bytes{fstype!~"tmpfs"}[1h], 4 * 3600) < 0
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Host disk will fill in 4 hours (instance {{ $labels.instance }})"
          description: "Disk will fill in 4 hours at current write rate\n  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

      - alert: HostPhysicalComponentTooHot
        expr: node_hwmon_temp_celsius > 75
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Host physical component too hot (instance {{ $labels.instance }})"
          description: "Physical hardware component too hot\n  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

      - alert: HostNodeOvertemperatureAlarm
        expr: node_hwmon_temp_alarm == 1
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "Host node overtemperature alarm (instance {{ $labels.instance }})"
          description: "Physical node temperature alarm triggered\n  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

      - alert: HostKernelVersionDeviations
        expr: count(sum(label_replace(node_uname_info, "kernel", "$1", "release", "([0-9]+.[0-9]+.[0-9]+).*")) by (kernel)) > 1
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Host kernel version deviations (instance {{ $labels.instance }})"
          description: "Different kernel versions are running\n  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

      - alert: HostOomKillDetected
        expr: increase(node_vmstat_oom_kill[5m]) > 0
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Host OOM kill detected (instance {{ $labels.instance }})"
          description: "OOM kill detected\n  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

      - alert: HostEdacCorrectableErrorsDetected
        expr: increase(node_edac_correctable_errors_total[5m]) > 0
        for: 5m
        labels:
          severity: info
        annotations:
          summary: "Host EDAC Correctable Errors detected (instance {{ $labels.instance }})"
          description: "{{ $labels.instance }} has had {{ printf \"%.0f\" $value }} correctable memory errors reported by EDAC in the last 5 minutes.\n  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

      - alert: HostEdacUncorrectableErrorsDetected
        expr: node_edac_uncorrectable_errors_total > 0
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Host EDAC Uncorrectable Errors detected (instance {{ $labels.instance }})"
          description: "{{ $labels.instance }} has had {{ printf \"%.0f\" $value }} uncorrectable memory errors reported by EDAC in the last 5 minutes.\n  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

      - alert: HostNetworkReceiveErrors
        expr: increase(node_network_receive_errs_total[5m]) > 0
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Host Network Receive Errors (instance {{ $labels.instance }})"
          description: "{{ $labels.instance }} interface {{ $labels.device }} has encountered {{ printf \"%.0f\" $value }} receive errors in the last five minutes.\n  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

      - alert: HostNetworkTransmitErrors
        expr: increase(node_network_transmit_errs_total[5m]) > 0
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Host Network Transmit Errors (instance {{ $labels.instance }})"
          description: "{{ $labels.instance }} interface {{ $labels.device }} has encountered {{ printf \"%.0f\" $value }} transmit errors in the last five minutes.\n  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

    - name: container
      rules:
      - alert: ContainerKilled
        expr: time() - container_last_seen > 60
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Container killed (instance {{ $labels.instance }})"
          description: "A container has disappeared\n  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

      - alert: ContainerCpuUsage
        expr: sum by(instance, name)  (rate(container_cpu_usage_seconds_total[3m]) * 100 > 80)
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Container CPU usage (instance {{ $labels.instance }})"
          description: "Container CPU usage is above 80%\n  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

      - alert: ContainerMemoryUsage
        expr: (sum(container_memory_usage_bytes) BY (instance, name) / sum(container_spec_memory_limit_bytes > 0) BY (instance, name) * 100) > 125
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Container Memory usage (instance {{ $labels.instance }})"
          description: "Container Memory usage is above 80%\n  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

      - alert: ContainerVolumeUsage
        expr: (1 - (sum(container_fs_inodes_free) BY (instance) / sum(container_fs_inodes_total) BY (instance)) * 100) > 80
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Container Volume usage (instance {{ $labels.instance }})"
          description: "Container Volume usage is above 80%\n  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

      - alert: ContainerVolumeIoUsage
        expr: (sum(container_fs_io_current) BY (instance, name) * 100) > 80
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Container Volume IO usage (instance {{ $labels.instance }})"
          description: "Container Volume IO usage is above 80%\n  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

      - alert: ContainerHighThrottleRate
        expr: rate(container_cpu_cfs_throttled_seconds_total[3m]) > 1
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Container high throttle rate (instance {{ $labels.instance }})"
          description: "Container is being throttled\n  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

    - name: kubernetes
      rules:
      - alert: KubernetesNodeReady
        expr: kube_node_status_condition{condition="Ready",status="true"} == 0
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "Kubernetes Node ready (instance {{ $labels.instance }})"
          description: "Node {{ $labels.node }} has been unready for a long time\n  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

      - alert: KubernetesMemoryPressure
        expr: kube_node_status_condition{condition="MemoryPressure",status="true"} == 1
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "Kubernetes memory pressure (instance {{ $labels.instance }})"
          description: "{{ $labels.node }} has MemoryPressure condition\n  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

      - alert: KubernetesDiskPressure
        expr: kube_node_status_condition{condition="DiskPressure",status="true"} == 1
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "Kubernetes disk pressure (instance {{ $labels.instance }})"
          description: "{{ $labels.node }} has DiskPressure condition\n  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

      - alert: KubernetesOutOfDisk
        expr: kube_node_status_condition{condition="OutOfDisk",status="true"} == 1
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "Kubernetes out of disk (instance {{ $labels.instance }})"
          description: "{{ $labels.node }} has OutOfDisk condition\n  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

      - alert: KubernetesJobFailed
        expr: kube_job_status_failed > 0
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Kubernetes Job failed (instance {{ $labels.instance }})"
          description: "Job {{$labels.namespace}}/{{$labels.exported_job}} failed to complete\n  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

      - alert: KubernetesCronjobSuspended
        expr: kube_cronjob_spec_suspend != 0
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Kubernetes CronJob suspended (instance {{ $labels.instance }})"
          description: "CronJob {{ $labels.namespace }}/{{ $labels.cronjob }} is suspended\n  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

      - alert: KubernetesPersistentvolumeclaimPending
        expr: kube_persistentvolumeclaim_status_phase{phase="Pending"} == 1
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Kubernetes PersistentVolumeClaim pending (instance {{ $labels.instance }})"
          description: "PersistentVolumeClaim {{ $labels.namespace }}/{{ $labels.persistentvolumeclaim }} is pending\n  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

      - alert: KubernetesVolumeOutOfDiskSpace
        expr: kubelet_volume_stats_available_bytes / kubelet_volume_stats_capacity_bytes * 100 < 10
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Kubernetes Volume out of disk space (instance {{ $labels.instance }})"
          description: "Volume is almost full (< 10% left)\n  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

      - alert: KubernetesVolumeFullInFourDays
        expr: predict_linear(kubelet_volume_stats_available_bytes[6h], 4 * 24 * 3600) < 0
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "Kubernetes Volume full in four days (instance {{ $labels.instance }})"
          description: "{{ $labels.namespace }}/{{ $labels.persistentvolumeclaim }} is expected to fill up within four days. Currently {{ $value | humanize }}% is available.\n  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

      - alert: KubernetesPersistentvolumeError
        expr: kube_persistentvolume_status_phase{phase=~"Failed|Pending",job="kube-state-metrics"} > 0
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "Kubernetes PersistentVolume error (instance {{ $labels.instance }})"
          description: "Persistent volume is in bad state\n  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

      - alert: KubernetesStatefulsetDown
        expr: (kube_statefulset_status_replicas_ready / kube_statefulset_status_replicas_current) != 1
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "Kubernetes StatefulSet down (instance {{ $labels.instance }})"
          description: "A StatefulSet went down\n  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

      - alert: KubernetesHpaScalingAbility
        expr: kube_hpa_status_condition{condition="false", status="AbleToScale"} == 1
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Kubernetes HPA scaling ability (instance {{ $labels.instance }})"
          description: "Pod is unable to scale\n  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

      - alert: KubernetesHpaMetricAvailability
        expr: kube_hpa_status_condition{condition="false", status="ScalingActive"} == 1
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Kubernetes HPA metric availability (instance {{ $labels.instance }})"
          description: "HPA is not able to colelct metrics\n  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

      - alert: KubernetesHpaScaleCapability
        expr: kube_hpa_status_desired_replicas >= kube_hpa_spec_max_replicas
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Kubernetes HPA scale capability (instance {{ $labels.instance }})"
          description: "The maximum number of desired Pods has been hit\n  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

      - alert: KubernetesPodNotHealthy
        expr: min_over_time(sum by (namespace, pod) (kube_pod_status_phase{phase=~"Pending|Unknown|Failed"})[1h:]) > 0
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "Kubernetes Pod not healthy (instance {{ $labels.instance }})"
          description: "Pod has been in a non-ready state for longer than an hour.\n  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

      - alert: KubernetesPodCrashLooping
        expr: rate(kube_pod_container_status_restarts_total[15m]) * 60 * 5 > 5
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Kubernetes pod crash looping (instance {{ $labels.instance }})"
          description: "Pod {{ $labels.pod }} is crash looping\n  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

      - alert: KubernetesReplicassetMismatch
        expr: kube_replicaset_spec_replicas != kube_replicaset_status_ready_replicas
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Kubernetes ReplicasSet mismatch (instance {{ $labels.instance }})"
          description: "Deployment Replicas mismatch\n  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

      - alert: KubernetesDeploymentReplicasMismatch
        expr: kube_deployment_spec_replicas != kube_deployment_status_replicas_available
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Kubernetes Deployment replicas mismatch (instance {{ $labels.instance }})"
          description: "Deployment Replicas mismatch\n  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

      - alert: KubernetesStatefulsetReplicasMismatch
        expr: kube_statefulset_status_replicas_ready != kube_statefulset_status_replicas
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Kubernetes StatefulSet replicas mismatch (instance {{ $labels.instance }})"
          description: "A StatefulSet has not matched the expected number of replicas for longer than 15 minutes.\n  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

      - alert: KubernetesDeploymentGenerationMismatch
        expr: kube_deployment_status_observed_generation != kube_deployment_metadata_generation
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "Kubernetes Deployment generation mismatch (instance {{ $labels.instance }})"
          description: "A Deployment has failed but has not been rolled back.\n  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

      - alert: KubernetesStatefulsetGenerationMismatch
        expr: kube_statefulset_status_observed_generation != kube_statefulset_metadata_generation
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "Kubernetes StatefulSet generation mismatch (instance {{ $labels.instance }})"
          description: "A StatefulSet has failed but has not been rolled back.\n  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

      - alert: KubernetesStatefulsetUpdateNotRolledOut
        expr: max without (revision) (kube_statefulset_status_current_revision unless kube_statefulset_status_update_revision) * (kube_statefulset_replicas != kube_statefulset_status_replicas_updated)
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "Kubernetes StatefulSet update not rolled out (instance {{ $labels.instance }})"
          description: "StatefulSet update has not been rolled out.\n  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

      - alert: KubernetesDaemonsetRolloutStuck
        expr: kube_daemonset_status_number_ready / kube_daemonset_status_desired_number_scheduled * 100 < 100 or kube_daemonset_status_desired_number_scheduled - kube_daemonset_status_current_number_scheduled > 0
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "Kubernetes DaemonSet rollout stuck (instance {{ $labels.instance }})"
          description: "Some Pods of DaemonSet are not scheduled or not ready\n  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

      - alert: KubernetesDaemonsetMisscheduled
        expr: kube_daemonset_status_number_misscheduled > 0
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "Kubernetes DaemonSet misscheduled (instance {{ $labels.instance }})"
          description: "Some DaemonSet Pods are running where they are not supposed to run\n  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

      - alert: KubernetesCronjobTooLong
        expr: time() - kube_cronjob_next_schedule_time > 3600
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Kubernetes CronJob too long (instance {{ $labels.instance }})"
          description: "CronJob {{ $labels.namespace }}/{{ $labels.cronjob }} is taking more than 1h to complete.\n  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

      - alert: KubernetesJobCompletion
        expr: kube_job_spec_completions - kube_job_status_succeeded > 0 or kube_job_status_failed > 0
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "Kubernetes job completion (instance {{ $labels.instance }})"
          description: "Kubernetes Job failed to complete\n  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

      - alert: KubernetesApiServerErrors
        expr: sum(rate(apiserver_request_count{job="apiserver",code=~"^(?:5..)$"}[2m])) / sum(rate(apiserver_request_count{job="apiserver"}[2m])) * 100 > 3
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "Kubernetes API server errors (instance {{ $labels.instance }})"
          description: "Kubernetes API server is experiencing high error rate\n  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

      - alert: KubernetesApiClientErrors
        expr: (sum(rate(rest_client_requests_total{code=~"(4|5).."}[2m])) by (instance, job) / sum(rate(rest_client_requests_total[2m])) by (instance, job)) * 100 > 1
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "Kubernetes API client errors (instance {{ $labels.instance }})"
          description: "Kubernetes API client is experiencing high error rate\n  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

      - alert: KubernetesClientCertificateExpiresNextWeek
        expr: apiserver_client_certificate_expiration_seconds_count{job="apiserver"} > 0 and histogram_quantile(0.01, sum by (job, le) (rate(apiserver_client_certificate_expiration_seconds_bucket{job="apiserver"}[5m]))) < 7*24*60*60
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Kubernetes client certificate expires next week (instance {{ $labels.instance }})"
          description: "A client certificate used to authenticate to the apiserver is expiring next week.\n  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

      - alert: KubernetesClientCertificateExpiresSoon
        expr: apiserver_client_certificate_expiration_seconds_count{job="apiserver"} > 0 and histogram_quantile(0.01, sum by (job, le) (rate(apiserver_client_certificate_expiration_seconds_bucket{job="apiserver"}[5m]))) < 24*60*60
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "Kubernetes client certificate expires soon (instance {{ $labels.instance }})"
          description: "A client certificate used to authenticate to the apiserver is expiring in less than 24.0 hours.\n  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"

      - alert: KubernetesApiServerLatency
        expr: histogram_quantile(0.99, sum(apiserver_request_latencies_bucket{verb!~"CONNECT|WATCHLIST|WATCH|PROXY"}) WITHOUT (instance, resource)) / 1e+06 > 1
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Kubernetes API server latency (instance {{ $labels.instance }})"
          description: "Kubernetes API server has a 99th percentile latency of {{ $value }} seconds for {{ $labels.verb }} {{ $labels.resource }}.\n  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"


  prometheus.yml: |-
    global:
      scrape_interval:     15s
      evaluation_interval: 15s

    rule_files:
      - "/etc/prometheus/alert.rules"

    scrape_configs:
      - job_name: 'collectd-exporter'
        scrape_interval: 5s
        static_configs:
          - targets: ['collectd-exporter:9103']

      - job_name: 'cadvisor'
        scrape_interval: 5s
        static_configs:
          - targets: ['cadvisor:8080']

      - job_name: 'node-exporter'
        scrape_interval: 5s
        static_configs:
          - targets: ['node-exporter:9100']

      - job_name: 'prometheus'
        scrape_interval: 10s
        static_configs:
          - targets: ['localhost:9090']

      - job_name: 'kube-state-metrics'
        scrape_interval: 10s
        static_configs:
          - targets: ['kube-state-metrics.kube-system.svc.cluster.local:8080']

    alerting:
      alertmanagers:
      - scheme: http
        static_configs:
        - targets: ['alertmanager:9093', 'alertmanager1:9093']