diff options
Diffstat (limited to 'tools/lma/ansible-server/roles/monitoring/files/prometheus/prometheus-config.yaml')
| -rw-r--r-- | tools/lma/ansible-server/roles/monitoring/files/prometheus/prometheus-config.yaml | 609 |
1 files changed, 609 insertions, 0 deletions
diff --git a/tools/lma/ansible-server/roles/monitoring/files/prometheus/prometheus-config.yaml b/tools/lma/ansible-server/roles/monitoring/files/prometheus/prometheus-config.yaml new file mode 100644 index 00000000..917f978f --- /dev/null +++ b/tools/lma/ansible-server/roles/monitoring/files/prometheus/prometheus-config.yaml @@ -0,0 +1,609 @@ +# Copyright 2020 Aditya Srivastava. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: ConfigMap +metadata: + name: prometheus-config + namespace: monitoring +data: + alert.rules: |- + groups: + - name: targets + rules: + - alert: MonitorServiceDown + expr: up == 0 + for: 30s + labels: + severity: critical + annotations: + summary: "Monitor service non-operational" + description: "Service {{ $labels.instance }} is down." + - alert: HighCpuLoad + expr: node_load1 > 1.9 + for: 15s + labels: + severity: critical + annotations: + summary: "Service under high load" + description: "Docker host is under high load, the avg load 1m is at {{ $value}}. Reported by instance {{ $labels.instance }} of job {{ $labels.job }}." + + - name: host and hardware + rules: + - alert: HostHighCpuLoad + expr: 100 - (avg by(instance) (irate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 80 + for: 5m + labels: + severity: warning + annotations: + summary: "Host high CPU load (instance {{ $labels.instance }})" + description: "CPU load is > 80%\n VALUE = {{ $value }}\n LABELS: {{ $labels }}" + + - alert: HostSwapIsFillingUp + expr: (1 - (node_memory_SwapFree_bytes / node_memory_SwapTotal_bytes)) * 100 > 80 + for: 5m + labels: + severity: warning + annotations: + summary: "Host swap is filling up (instance {{ $labels.instance }})" + description: "Swap is filling up (>80%)\n VALUE = {{ $value }}\n LABELS: {{ $labels }}" + + - alert: HighMemoryLoad + expr: (sum(node_memory_MemTotal_bytes) - sum(node_memory_MemFree_bytes + node_memory_Buffers_bytes + node_memory_Cached_bytes) ) / sum(node_memory_MemTotal_bytes) * 100 > 85 + for: 30s + labels: + severity: warning + annotations: + summary: "Server memory is almost full" + description: "Docker host memory usage is {{ humanize $value}}%. Reported by instance {{ $labels.instance }} of job {{ $labels.job }}." + + - alert: HighStorageLoad + expr: (node_filesystem_size_bytes{fstype="aufs"} - node_filesystem_free_bytes{fstype="aufs"}) / node_filesystem_size_bytes{fstype="aufs"} * 100 > 85 + for: 30s + labels: + severity: warning + annotations: + summary: "Server storage is almost full" + description: "Docker host storage usage is {{ humanize $value}}%. Reported by instance {{ $labels.instance }} of job {{ $labels.job }}." + + - alert: HostNetworkTransmitErrors + expr: increase(node_network_transmit_errs_total[5m]) > 0 + for: 5m + labels: + severity: warning + annotations: + summary: "Host Network Transmit Errors (instance {{ $labels.instance }})" + description: "{{ $labels.instance }} interface {{ $labels.device }} has encountered {{ printf \"%.0f\" $value }} transmit errors in the last five minutes.\n VALUE = {{ $value }}\n LABELS: {{ $labels }}" + + - alert: HostOutOfMemory + expr: node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes * 100 < 10 + for: 5m + labels: + severity: warning + annotations: + summary: "Host out of memory (instance {{ $labels.instance }})" + description: "Node memory is filling up (< 10% left)\n VALUE = {{ $value }}\n LABELS: {{ $labels }}" + + - alert: HostMemoryUnderMemoryPressure + expr: rate(node_vmstat_pgmajfault[1m]) > 1000 + for: 5m + labels: + severity: warning + annotations: + summary: "Host memory under memory pressure (instance {{ $labels.instance }})" + description: "The node is under heavy memory pressure. High rate of major page faults\n VALUE = {{ $value }}\n LABELS: {{ $labels }}" + + - alert: HostUnusualNetworkThroughputIn + expr: sum by (instance) (irate(node_network_receive_bytes_total[2m])) / 1024 / 1024 > 100 + for: 5m + labels: + severity: warning + annotations: + summary: "Host unusual network throughput in (instance {{ $labels.instance }})" + description: "Host network interfaces are probably receiving too much data (> 100 MB/s)\n VALUE = {{ $value }}\n LABELS: {{ $labels }}" + + - alert: HostUnusualNetworkThroughputOut + expr: sum by (instance) (irate(node_network_transmit_bytes_total[2m])) / 1024 / 1024 > 100 + for: 5m + labels: + severity: warning + annotations: + summary: "Host unusual network throughput out (instance {{ $labels.instance }})" + description: "Host network interfaces are probably sending too much data (> 100 MB/s)\n VALUE = {{ $value }}\n LABELS: {{ $labels }}" + + - alert: HostUnusualDiskRateRead + expr: sum by (instance) (irate(node_disk_read_bytes_total[2m])) / 1024 / 1024 > 50 + for: 5m + labels: + severity: warning + annotations: + summary: "Host unusual disk read rate (instance {{ $labels.instance }})" + description: "Disk is probably reading too much data (> 50 MB/s)\n VALUE = {{ $value }}\n LABELS: {{ $labels }}" + + - alert: HostUnusualDiskRateWrite + expr: sum by (instance) (irate(node_disk_written_bytes_total[2m])) / 1024 / 1024 > 50 + for: 5m + labels: + severity: warning + annotations: + summary: "Host unusual disk write rate (instance {{ $labels.instance }})" + description: "Disk is probably writing too much data (> 50 MB/s)\n VALUE = {{ $value }}\n LABELS: {{ $labels }}" + + - alert: HostOutOfDiskSpace + expr: (node_filesystem_avail_bytes{mountpoint="/rootfs"} * 100) / node_filesystem_size_bytes{mountpoint="/rootfs"} < 10 + for: 5m + labels: + severity: warning + annotations: + summary: "Host out of disk space (instance {{ $labels.instance }})" + description: "Disk is almost full (< 10% left)\n VALUE = {{ $value }}\n LABELS: {{ $labels }}" + + - alert: HostDiskWillFillIn4Hours + expr: predict_linear(node_filesystem_free_bytes{fstype!~"tmpfs"}[1h], 4 * 3600) < 0 + for: 5m + labels: + severity: warning + annotations: + summary: "Host disk will fill in 4 hours (instance {{ $labels.instance }})" + description: "Disk will fill in 4 hours at current write rate\n VALUE = {{ $value }}\n LABELS: {{ $labels }}" + + - alert: HostPhysicalComponentTooHot + expr: node_hwmon_temp_celsius > 75 + for: 5m + labels: + severity: warning + annotations: + summary: "Host physical component too hot (instance {{ $labels.instance }})" + description: "Physical hardware component too hot\n VALUE = {{ $value }}\n LABELS: {{ $labels }}" + + - alert: HostNodeOvertemperatureAlarm + expr: node_hwmon_temp_alarm == 1 + for: 5m + labels: + severity: critical + annotations: + summary: "Host node overtemperature alarm (instance {{ $labels.instance }})" + description: "Physical node temperature alarm triggered\n VALUE = {{ $value }}\n LABELS: {{ $labels }}" + + - alert: HostKernelVersionDeviations + expr: count(sum(label_replace(node_uname_info, "kernel", "$1", "release", "([0-9]+.[0-9]+.[0-9]+).*")) by (kernel)) > 1 + for: 5m + labels: + severity: warning + annotations: + summary: "Host kernel version deviations (instance {{ $labels.instance }})" + description: "Different kernel versions are running\n VALUE = {{ $value }}\n LABELS: {{ $labels }}" + + - alert: HostOomKillDetected + expr: increase(node_vmstat_oom_kill[5m]) > 0 + for: 5m + labels: + severity: warning + annotations: + summary: "Host OOM kill detected (instance {{ $labels.instance }})" + description: "OOM kill detected\n VALUE = {{ $value }}\n LABELS: {{ $labels }}" + + - alert: HostEdacCorrectableErrorsDetected + expr: increase(node_edac_correctable_errors_total[5m]) > 0 + for: 5m + labels: + severity: info + annotations: + summary: "Host EDAC Correctable Errors detected (instance {{ $labels.instance }})" + description: "{{ $labels.instance }} has had {{ printf \"%.0f\" $value }} correctable memory errors reported by EDAC in the last 5 minutes.\n VALUE = {{ $value }}\n LABELS: {{ $labels }}" + + - alert: HostEdacUncorrectableErrorsDetected + expr: node_edac_uncorrectable_errors_total > 0 + for: 5m + labels: + severity: warning + annotations: + summary: "Host EDAC Uncorrectable Errors detected (instance {{ $labels.instance }})" + description: "{{ $labels.instance }} has had {{ printf \"%.0f\" $value }} uncorrectable memory errors reported by EDAC in the last 5 minutes.\n VALUE = {{ $value }}\n LABELS: {{ $labels }}" + + - alert: HostNetworkReceiveErrors + expr: increase(node_network_receive_errs_total[5m]) > 0 + for: 5m + labels: + severity: warning + annotations: + summary: "Host Network Receive Errors (instance {{ $labels.instance }})" + description: "{{ $labels.instance }} interface {{ $labels.device }} has encountered {{ printf \"%.0f\" $value }} receive errors in the last five minutes.\n VALUE = {{ $value }}\n LABELS: {{ $labels }}" + + - alert: HostNetworkTransmitErrors + expr: increase(node_network_transmit_errs_total[5m]) > 0 + for: 5m + labels: + severity: warning + annotations: + summary: "Host Network Transmit Errors (instance {{ $labels.instance }})" + description: "{{ $labels.instance }} interface {{ $labels.device }} has encountered {{ printf \"%.0f\" $value }} transmit errors in the last five minutes.\n VALUE = {{ $value }}\n LABELS: {{ $labels }}" + + - name: container + rules: + - alert: ContainerKilled + expr: time() - container_last_seen > 60 + for: 5m + labels: + severity: warning + annotations: + summary: "Container killed (instance {{ $labels.instance }})" + description: "A container has disappeared\n VALUE = {{ $value }}\n LABELS: {{ $labels }}" + + - alert: ContainerCpuUsage + expr: sum by(instance, name) (rate(container_cpu_usage_seconds_total[3m]) * 100 > 80) + for: 5m + labels: + severity: warning + annotations: + summary: "Container CPU usage (instance {{ $labels.instance }})" + description: "Container CPU usage is above 80%\n VALUE = {{ $value }}\n LABELS: {{ $labels }}" + + - alert: ContainerMemoryUsage + expr: (sum(container_memory_usage_bytes) BY (instance, name) / sum(container_spec_memory_limit_bytes > 0) BY (instance, name) * 100) > 125 + for: 5m + labels: + severity: warning + annotations: + summary: "Container Memory usage (instance {{ $labels.instance }})" + description: "Container Memory usage is above 80%\n VALUE = {{ $value }}\n LABELS: {{ $labels }}" + + - alert: ContainerVolumeUsage + expr: (1 - (sum(container_fs_inodes_free) BY (instance) / sum(container_fs_inodes_total) BY (instance)) * 100) > 80 + for: 5m + labels: + severity: warning + annotations: + summary: "Container Volume usage (instance {{ $labels.instance }})" + description: "Container Volume usage is above 80%\n VALUE = {{ $value }}\n LABELS: {{ $labels }}" + + - alert: ContainerVolumeIoUsage + expr: (sum(container_fs_io_current) BY (instance, name) * 100) > 80 + for: 5m + labels: + severity: warning + annotations: + summary: "Container Volume IO usage (instance {{ $labels.instance }})" + description: "Container Volume IO usage is above 80%\n VALUE = {{ $value }}\n LABELS: {{ $labels }}" + + - alert: ContainerHighThrottleRate + expr: rate(container_cpu_cfs_throttled_seconds_total[3m]) > 1 + for: 5m + labels: + severity: warning + annotations: + summary: "Container high throttle rate (instance {{ $labels.instance }})" + description: "Container is being throttled\n VALUE = {{ $value }}\n LABELS: {{ $labels }}" + + - name: kubernetes + rules: + - alert: KubernetesNodeReady + expr: kube_node_status_condition{condition="Ready",status="true"} == 0 + for: 5m + labels: + severity: critical + annotations: + summary: "Kubernetes Node ready (instance {{ $labels.instance }})" + description: "Node {{ $labels.node }} has been unready for a long time\n VALUE = {{ $value }}\n LABELS: {{ $labels }}" + + - alert: KubernetesMemoryPressure + expr: kube_node_status_condition{condition="MemoryPressure",status="true"} == 1 + for: 5m + labels: + severity: critical + annotations: + summary: "Kubernetes memory pressure (instance {{ $labels.instance }})" + description: "{{ $labels.node }} has MemoryPressure condition\n VALUE = {{ $value }}\n LABELS: {{ $labels }}" + + - alert: KubernetesDiskPressure + expr: kube_node_status_condition{condition="DiskPressure",status="true"} == 1 + for: 5m + labels: + severity: critical + annotations: + summary: "Kubernetes disk pressure (instance {{ $labels.instance }})" + description: "{{ $labels.node }} has DiskPressure condition\n VALUE = {{ $value }}\n LABELS: {{ $labels }}" + + - alert: KubernetesOutOfDisk + expr: kube_node_status_condition{condition="OutOfDisk",status="true"} == 1 + for: 5m + labels: + severity: critical + annotations: + summary: "Kubernetes out of disk (instance {{ $labels.instance }})" + description: "{{ $labels.node }} has OutOfDisk condition\n VALUE = {{ $value }}\n LABELS: {{ $labels }}" + + - alert: KubernetesJobFailed + expr: kube_job_status_failed > 0 + for: 5m + labels: + severity: warning + annotations: + summary: "Kubernetes Job failed (instance {{ $labels.instance }})" + description: "Job {{$labels.namespace}}/{{$labels.exported_job}} failed to complete\n VALUE = {{ $value }}\n LABELS: {{ $labels }}" + + - alert: KubernetesCronjobSuspended + expr: kube_cronjob_spec_suspend != 0 + for: 5m + labels: + severity: warning + annotations: + summary: "Kubernetes CronJob suspended (instance {{ $labels.instance }})" + description: "CronJob {{ $labels.namespace }}/{{ $labels.cronjob }} is suspended\n VALUE = {{ $value }}\n LABELS: {{ $labels }}" + + - alert: KubernetesPersistentvolumeclaimPending + expr: kube_persistentvolumeclaim_status_phase{phase="Pending"} == 1 + for: 5m + labels: + severity: warning + annotations: + summary: "Kubernetes PersistentVolumeClaim pending (instance {{ $labels.instance }})" + description: "PersistentVolumeClaim {{ $labels.namespace }}/{{ $labels.persistentvolumeclaim }} is pending\n VALUE = {{ $value }}\n LABELS: {{ $labels }}" + + - alert: KubernetesVolumeOutOfDiskSpace + expr: kubelet_volume_stats_available_bytes / kubelet_volume_stats_capacity_bytes * 100 < 10 + for: 5m + labels: + severity: warning + annotations: + summary: "Kubernetes Volume out of disk space (instance {{ $labels.instance }})" + description: "Volume is almost full (< 10% left)\n VALUE = {{ $value }}\n LABELS: {{ $labels }}" + + - alert: KubernetesVolumeFullInFourDays + expr: predict_linear(kubelet_volume_stats_available_bytes[6h], 4 * 24 * 3600) < 0 + for: 5m + labels: + severity: critical + annotations: + summary: "Kubernetes Volume full in four days (instance {{ $labels.instance }})" + description: "{{ $labels.namespace }}/{{ $labels.persistentvolumeclaim }} is expected to fill up within four days. Currently {{ $value | humanize }}% is available.\n VALUE = {{ $value }}\n LABELS: {{ $labels }}" + + - alert: KubernetesPersistentvolumeError + expr: kube_persistentvolume_status_phase{phase=~"Failed|Pending",job="kube-state-metrics"} > 0 + for: 5m + labels: + severity: critical + annotations: + summary: "Kubernetes PersistentVolume error (instance {{ $labels.instance }})" + description: "Persistent volume is in bad state\n VALUE = {{ $value }}\n LABELS: {{ $labels }}" + + - alert: KubernetesStatefulsetDown + expr: (kube_statefulset_status_replicas_ready / kube_statefulset_status_replicas_current) != 1 + for: 5m + labels: + severity: critical + annotations: + summary: "Kubernetes StatefulSet down (instance {{ $labels.instance }})" + description: "A StatefulSet went down\n VALUE = {{ $value }}\n LABELS: {{ $labels }}" + + - alert: KubernetesHpaScalingAbility + expr: kube_hpa_status_condition{condition="false", status="AbleToScale"} == 1 + for: 5m + labels: + severity: warning + annotations: + summary: "Kubernetes HPA scaling ability (instance {{ $labels.instance }})" + description: "Pod is unable to scale\n VALUE = {{ $value }}\n LABELS: {{ $labels }}" + + - alert: KubernetesHpaMetricAvailability + expr: kube_hpa_status_condition{condition="false", status="ScalingActive"} == 1 + for: 5m + labels: + severity: warning + annotations: + summary: "Kubernetes HPA metric availability (instance {{ $labels.instance }})" + description: "HPA is not able to colelct metrics\n VALUE = {{ $value }}\n LABELS: {{ $labels }}" + + - alert: KubernetesHpaScaleCapability + expr: kube_hpa_status_desired_replicas >= kube_hpa_spec_max_replicas + for: 5m + labels: + severity: warning + annotations: + summary: "Kubernetes HPA scale capability (instance {{ $labels.instance }})" + description: "The maximum number of desired Pods has been hit\n VALUE = {{ $value }}\n LABELS: {{ $labels }}" + + - alert: KubernetesPodNotHealthy + expr: min_over_time(sum by (namespace, pod) (kube_pod_status_phase{phase=~"Pending|Unknown|Failed"})[1h:]) > 0 + for: 5m + labels: + severity: critical + annotations: + summary: "Kubernetes Pod not healthy (instance {{ $labels.instance }})" + description: "Pod has been in a non-ready state for longer than an hour.\n VALUE = {{ $value }}\n LABELS: {{ $labels }}" + + - alert: KubernetesPodCrashLooping + expr: rate(kube_pod_container_status_restarts_total[15m]) * 60 * 5 > 5 + for: 5m + labels: + severity: warning + annotations: + summary: "Kubernetes pod crash looping (instance {{ $labels.instance }})" + description: "Pod {{ $labels.pod }} is crash looping\n VALUE = {{ $value }}\n LABELS: {{ $labels }}" + + - alert: KubernetesReplicassetMismatch + expr: kube_replicaset_spec_replicas != kube_replicaset_status_ready_replicas + for: 5m + labels: + severity: warning + annotations: + summary: "Kubernetes ReplicasSet mismatch (instance {{ $labels.instance }})" + description: "Deployment Replicas mismatch\n VALUE = {{ $value }}\n LABELS: {{ $labels }}" + + - alert: KubernetesDeploymentReplicasMismatch + expr: kube_deployment_spec_replicas != kube_deployment_status_replicas_available + for: 5m + labels: + severity: warning + annotations: + summary: "Kubernetes Deployment replicas mismatch (instance {{ $labels.instance }})" + description: "Deployment Replicas mismatch\n VALUE = {{ $value }}\n LABELS: {{ $labels }}" + + - alert: KubernetesStatefulsetReplicasMismatch + expr: kube_statefulset_status_replicas_ready != kube_statefulset_status_replicas + for: 5m + labels: + severity: warning + annotations: + summary: "Kubernetes StatefulSet replicas mismatch (instance {{ $labels.instance }})" + description: "A StatefulSet has not matched the expected number of replicas for longer than 15 minutes.\n VALUE = {{ $value }}\n LABELS: {{ $labels }}" + + - alert: KubernetesDeploymentGenerationMismatch + expr: kube_deployment_status_observed_generation != kube_deployment_metadata_generation + for: 5m + labels: + severity: critical + annotations: + summary: "Kubernetes Deployment generation mismatch (instance {{ $labels.instance }})" + description: "A Deployment has failed but has not been rolled back.\n VALUE = {{ $value }}\n LABELS: {{ $labels }}" + + - alert: KubernetesStatefulsetGenerationMismatch + expr: kube_statefulset_status_observed_generation != kube_statefulset_metadata_generation + for: 5m + labels: + severity: critical + annotations: + summary: "Kubernetes StatefulSet generation mismatch (instance {{ $labels.instance }})" + description: "A StatefulSet has failed but has not been rolled back.\n VALUE = {{ $value }}\n LABELS: {{ $labels }}" + + - alert: KubernetesStatefulsetUpdateNotRolledOut + expr: max without (revision) (kube_statefulset_status_current_revision unless kube_statefulset_status_update_revision) * (kube_statefulset_replicas != kube_statefulset_status_replicas_updated) + for: 5m + labels: + severity: critical + annotations: + summary: "Kubernetes StatefulSet update not rolled out (instance {{ $labels.instance }})" + description: "StatefulSet update has not been rolled out.\n VALUE = {{ $value }}\n LABELS: {{ $labels }}" + + - alert: KubernetesDaemonsetRolloutStuck + expr: kube_daemonset_status_number_ready / kube_daemonset_status_desired_number_scheduled * 100 < 100 or kube_daemonset_status_desired_number_scheduled - kube_daemonset_status_current_number_scheduled > 0 + for: 5m + labels: + severity: critical + annotations: + summary: "Kubernetes DaemonSet rollout stuck (instance {{ $labels.instance }})" + description: "Some Pods of DaemonSet are not scheduled or not ready\n VALUE = {{ $value }}\n LABELS: {{ $labels }}" + + - alert: KubernetesDaemonsetMisscheduled + expr: kube_daemonset_status_number_misscheduled > 0 + for: 5m + labels: + severity: critical + annotations: + summary: "Kubernetes DaemonSet misscheduled (instance {{ $labels.instance }})" + description: "Some DaemonSet Pods are running where they are not supposed to run\n VALUE = {{ $value }}\n LABELS: {{ $labels }}" + + - alert: KubernetesCronjobTooLong + expr: time() - kube_cronjob_next_schedule_time > 3600 + for: 5m + labels: + severity: warning + annotations: + summary: "Kubernetes CronJob too long (instance {{ $labels.instance }})" + description: "CronJob {{ $labels.namespace }}/{{ $labels.cronjob }} is taking more than 1h to complete.\n VALUE = {{ $value }}\n LABELS: {{ $labels }}" + + - alert: KubernetesJobCompletion + expr: kube_job_spec_completions - kube_job_status_succeeded > 0 or kube_job_status_failed > 0 + for: 5m + labels: + severity: critical + annotations: + summary: "Kubernetes job completion (instance {{ $labels.instance }})" + description: "Kubernetes Job failed to complete\n VALUE = {{ $value }}\n LABELS: {{ $labels }}" + + - alert: KubernetesApiServerErrors + expr: sum(rate(apiserver_request_count{job="apiserver",code=~"^(?:5..)$"}[2m])) / sum(rate(apiserver_request_count{job="apiserver"}[2m])) * 100 > 3 + for: 5m + labels: + severity: critical + annotations: + summary: "Kubernetes API server errors (instance {{ $labels.instance }})" + description: "Kubernetes API server is experiencing high error rate\n VALUE = {{ $value }}\n LABELS: {{ $labels }}" + + - alert: KubernetesApiClientErrors + expr: (sum(rate(rest_client_requests_total{code=~"(4|5).."}[2m])) by (instance, job) / sum(rate(rest_client_requests_total[2m])) by (instance, job)) * 100 > 1 + for: 5m + labels: + severity: critical + annotations: + summary: "Kubernetes API client errors (instance {{ $labels.instance }})" + description: "Kubernetes API client is experiencing high error rate\n VALUE = {{ $value }}\n LABELS: {{ $labels }}" + + - alert: KubernetesClientCertificateExpiresNextWeek + expr: apiserver_client_certificate_expiration_seconds_count{job="apiserver"} > 0 and histogram_quantile(0.01, sum by (job, le) (rate(apiserver_client_certificate_expiration_seconds_bucket{job="apiserver"}[5m]))) < 7*24*60*60 + for: 5m + labels: + severity: warning + annotations: + summary: "Kubernetes client certificate expires next week (instance {{ $labels.instance }})" + description: "A client certificate used to authenticate to the apiserver is expiring next week.\n VALUE = {{ $value }}\n LABELS: {{ $labels }}" + + - alert: KubernetesClientCertificateExpiresSoon + expr: apiserver_client_certificate_expiration_seconds_count{job="apiserver"} > 0 and histogram_quantile(0.01, sum by (job, le) (rate(apiserver_client_certificate_expiration_seconds_bucket{job="apiserver"}[5m]))) < 24*60*60 + for: 5m + labels: + severity: critical + annotations: + summary: "Kubernetes client certificate expires soon (instance {{ $labels.instance }})" + description: "A client certificate used to authenticate to the apiserver is expiring in less than 24.0 hours.\n VALUE = {{ $value }}\n LABELS: {{ $labels }}" + + - alert: KubernetesApiServerLatency + expr: histogram_quantile(0.99, sum(apiserver_request_latencies_bucket{verb!~"CONNECT|WATCHLIST|WATCH|PROXY"}) WITHOUT (instance, resource)) / 1e+06 > 1 + for: 5m + labels: + severity: warning + annotations: + summary: "Kubernetes API server latency (instance {{ $labels.instance }})" + description: "Kubernetes API server has a 99th percentile latency of {{ $value }} seconds for {{ $labels.verb }} {{ $labels.resource }}.\n VALUE = {{ $value }}\n LABELS: {{ $labels }}" + + + prometheus.yml: |- + global: + scrape_interval: 15s + evaluation_interval: 15s + + rule_files: + - "/etc/prometheus/alert.rules" + + scrape_configs: + - job_name: 'collectd-exporter' + scrape_interval: 5s + static_configs: + - targets: ['collectd-exporter:9103'] + + - job_name: 'cadvisor' + scrape_interval: 5s + static_configs: + - targets: ['cadvisor:8080'] + + - job_name: 'node-exporter' + scrape_interval: 5s + static_configs: + - targets: ['node-exporter:9100'] + + - job_name: 'prometheus' + scrape_interval: 10s + static_configs: + - targets: ['localhost:9090'] + + - job_name: 'kube-state-metrics' + scrape_interval: 10s + static_configs: + - targets: ['kube-state-metrics.kube-system.svc.cluster.local:8080'] + + alerting: + alertmanagers: + - scheme: http + static_configs: + - targets: ['alertmanager:9093', 'alertmanager1:9093'] |