diff options
Diffstat (limited to 'tools/lma/ansible-server/roles/logging/files/elastalert')
4 files changed, 324 insertions, 0 deletions
diff --git a/tools/lma/ansible-server/roles/logging/files/elastalert/ealert-conf-cm.yaml b/tools/lma/ansible-server/roles/logging/files/elastalert/ealert-conf-cm.yaml new file mode 100644 index 00000000..a320ef75 --- /dev/null +++ b/tools/lma/ansible-server/roles/logging/files/elastalert/ealert-conf-cm.yaml @@ -0,0 +1,48 @@ +# Copyright 2020 Adarsh yadav +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +apiVersion: v1 +kind: ConfigMap +metadata: + name: elastalert-config +data: + elastalert.yaml: | + # This is the folder that contains the rule yaml files + # Any .yaml file will be loaded as a rule + rules_folder: rules + # How often ElastAlert will query Elasticsearch + # The unit can be anything from weeks to seconds + run_every: + minutes: 1 + # ElastAlert will buffer results from the most recent + # period of time, in case some log sources are not in real time + buffer_time: + minutes: 15 + + scan_subdirectories: false + + # The Elasticsearch hostname for metadata writeback + # Note that every rule can have its own Elasticsearch host + es_host: logging-es-http + es_port: 9200 + es_username: ${ES_USERNAME} + es_password: ${ES_PASSWORD} + es_conn_timeout: 120 + verify_certs: False + use_ssl: True + client_cert: '/opt/elastalert/key/elastalert.pem' + client_key: '/opt/elastalert/key/elastalert.key' + writeback_index: elastalert_status + writeback_alias: elastalert_alerts + alert_time_limit: + days: 2 diff --git a/tools/lma/ansible-server/roles/logging/files/elastalert/ealert-key-cm.yaml b/tools/lma/ansible-server/roles/logging/files/elastalert/ealert-key-cm.yaml new file mode 100644 index 00000000..0c606a9c --- /dev/null +++ b/tools/lma/ansible-server/roles/logging/files/elastalert/ealert-key-cm.yaml @@ -0,0 +1,68 @@ +# Copyright 2020 Adarsh yadav +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +apiVersion: v1 +kind: ConfigMap +metadata: + name: elastalert-key +data: + elastalert.key: | + -----BEGIN PRIVATE KEY----- + MIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQC0uQ+B0gy3VB4w + 5CeWOx575lqSUuYvrGW3ILpV1gmj0ZZCMZUGvt4UvaCEaNPIAqNaHPmaslQqJb5C + PJH9pMN7vUVp3DACzmYrS4HdROHamn5gjebXs4hq43heLaIB1Kb+4F+7sEY88irK + xOevadcN35y5ld7lVUGRsj6JYcweaAeh/YZ/HaBT5RfdGF+x07NDus+mFqT8j3PD + rs2+JtEvEoWtjcxwFgloc9GkHsWZoV1AQHgyAWjmDXZtZeV0HQSkl7hWFG9vxTni + DvdrdhX0g+D+u8jWnlR4Za4jd64KbTp9C9trSHyMSRIvN5obm/H8O5MQ+sZ+NQ0X + PdK92MjbAgMBAAECggEASbRPxrpLxVjhFz91haeGvzErLxHwHvFIam9Gj0tDkzQe + +9AM3ztohzzvAhFejevFgzLd+WFRQf8yoQDi6XcQ4p5GeO38Bqj2siGRTRSSp/zq + HabBxqbJtA4hQQeLUwPPN5N6d6lke+an3RqBAuE/e8D+whGFXjJvE2SGbLEd9if2 + uzHj37sPsVi8kRvgZBDOozmt7YFzQVO/1V+4Lw6nz48M3t+hOHaUXY0Yd8nsk5A6 + kgoDQ4CGUHjtWfSrccZrYNk51Zows9/sX8axfJ94wKJSImWJcuW9PXIQhzT4exnH + sPOwY6Noy3nXRk9gcchT60fKpp+tsJZk3ezkwSpgwQKBgQDvsaYcbnIVdFZpaNKF + Tmt/w60CmfGeNozRygfi84ot7edUf93cB6WSKChcAE8fbq9Ji5USPNtfbnZfFXsI + IyTr2KHW3RkHuDEyu+Lan9JuReEH3QOG83vvN/oYA3J3hqUTCjEGkPjqnoFtdk8L + f7WH1jZvXYEMo0C48SXo+yGohQKBgQDBBGkzL928j1QB9NfiNFk70EalDsF8Im2W + n8bQ54KYspUybKD/Hmw0jIV7kdu2vhgGC4RYkn9c5qATtulbYJUgUBelaSi0vhXT + gfAuO+JIIZ50P+mkkxH/KIUyu1xWUB2jtMulqLLomdoBvfp/u51qCY6fT3WMCB+R + ouWLr2oZ3wKBgQCAuas4AaiLFRuDKKRGq0LYLsIvb3VvPmSKFjH+FETVPbrKipEf + pYup3p8uKYxUmSDSIoBAdyZpLe2sSuD0Ecu2TXU86yiSGL1zPawrNUHRrv2XN365 + bvHUGv/Y/aDvyAPHIeYKXLkRZ2ai3rK8vi1Dcitxy4mOu+36ZKezY4tD8QKBgQCd + hakJUj4nPd20fwqUnF5a1z5gRGuZkEtZiunp4ZaOYegrL8YwjraGKExjrYTfXcIj + ZNDMrDpvKfRoQnWt0mPB7DtwDiNfZmZPqBLI2Kxya6VygBqA6lncoEgcQBY6hsW5 + rbopZ0UjWTQ3CcFe71GnkUcpMuLetl51L7kgR7dShwKBgQC+vqjhe/h081JGLTo1 + tKeRUCaDA/V3VHjFKgM5g+S3/KzgU/EaB1rq3Qja1quGv0zHveca3zibdNQi1ENm + KSutWh2zQXzzvmycPmVcthhOxaKzRXDjG0mXiA0bnSgK3F2o9t4196RYhIiiSvAH + shVjZMTK04h8ciTLIqK/GtZr+g== + -----END PRIVATE KEY----- + elastalert.pem: | + -----BEGIN CERTIFICATE----- + MIIDVzCCAj+gAwIBAgIJAORgkR7Y0Nk9MA0GCSqGSIb3DQEBCwUAMEIxCzAJBgNV + BAYTAlhYMRUwEwYDVQQHDAxEZWZhdWx0IENpdHkxHDAaBgNVBAoME0RlZmF1bHQg + Q29tcGFueSBMdGQwHhcNMjAwNjI4MTM1NjAwWhcNMjEwNjI4MTM1NjAwWjBCMQsw + CQYDVQQGEwJYWDEVMBMGA1UEBwwMRGVmYXVsdCBDaXR5MRwwGgYDVQQKDBNEZWZh + dWx0IENvbXBhbnkgTHRkMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA + tLkPgdIMt1QeMOQnljsee+ZaklLmL6xltyC6VdYJo9GWQjGVBr7eFL2ghGjTyAKj + Whz5mrJUKiW+QjyR/aTDe71FadwwAs5mK0uB3UTh2pp+YI3m17OIauN4Xi2iAdSm + /uBfu7BGPPIqysTnr2nXDd+cuZXe5VVBkbI+iWHMHmgHof2Gfx2gU+UX3RhfsdOz + Q7rPphak/I9zw67NvibRLxKFrY3McBYJaHPRpB7FmaFdQEB4MgFo5g12bWXldB0E + pJe4VhRvb8U54g73a3YV9IPg/rvI1p5UeGWuI3euCm06fQvba0h8jEkSLzeaG5vx + /DuTEPrGfjUNFz3SvdjI2wIDAQABo1AwTjAdBgNVHQ4EFgQUFAvjohHTavHmbRbj + Yq2h3cq7UMEwHwYDVR0jBBgwFoAUFAvjohHTavHmbRbjYq2h3cq7UMEwDAYDVR0T + BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEAB9oDASl4OfF/D49i3KtVzjzge4up + WssBPYKVwASh3cXfLLe3NdY9ihdCXFd/8Rus0hBGaRPIyR06sZoHRDEfJ2xrRD6g + pr4iHRfaoEWqols7+iW0cgQehvw5efEpFL1vg9zK9kOwruS4ZUhDrak6GcO/O8Jh + 6lSGmidHSHrQmfqFeTotaezwylV/uHvRZHPvk2JhQfC+vFjn5/iN/0wCeQCwYvOC + rePq2ZFdYg/0bS9BYwKsT2w1Z/AU/wIMLmbNB1af+fTBBEQlxb4rAeDb+J9EoSQ5 + MVP7jm3BVnHQCs6CA4LV4yRQNF2K6GkWem1oUg/H3S2SG8TAUlKpX/1XRw== + -----END CERTIFICATE----- diff --git a/tools/lma/ansible-server/roles/logging/files/elastalert/ealert-rule-cm.yaml b/tools/lma/ansible-server/roles/logging/files/elastalert/ealert-rule-cm.yaml new file mode 100644 index 00000000..af28b6f6 --- /dev/null +++ b/tools/lma/ansible-server/roles/logging/files/elastalert/ealert-rule-cm.yaml @@ -0,0 +1,132 @@ +# Copyright 2020 Adarsh yadav +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +apiVersion: v1 +kind: ConfigMap +metadata: + name: elastalert-rule +data: + rule-node4-vswitch.yaml: | + name: vswitch-3-sec + type: any + index: node4* + filter: + - range: + time_vswitchd: + gt: 3 #Greater than + + realert: + minutes: 0 + + alert: post + http_post_url: "http://10.10.120.211:31000/alerts" + http_post_static_payload: + type: threshold + label: vswitchd start time > 3 sec + http_post_payload: + index: _index + log: msg + log_path: log_path + time_vswitchd: time_vswitchd + num_hits: num_hits + num_matches: num_matches + + rule-node1-vswitch.yaml: | + name: vswitch-3-sec + type: any + index: node1* + filter: + - range: + time_vswitchd: + gt: 3 #Greater than + + realert: + minutes: 0 + + alert: post + http_post_url: "http://10.10.120.211:31000/alerts" + http_post_static_payload: + type: threshold + label: vswitchd start time > 3 sec + http_post_payload: + index: _index + log: msg + log_path: log_path + time_vswitchd: time_vswitchd + num_hits: num_hits + num_matches: num_matches + + rule-node4-blacklist.yaml: | + name: error-finder-node4 + type: blacklist + compare_key: alert + index: node4* + blacklist: + - "Failed to run test" + - "Failed to execute in '30' seconds" + - "('Result', 'Failed')" + - "could not open socket: connection refused" + - "Input/output error" + - "dpdk|ERR|EAL: Error - exiting with code: 1" + - "Failed to execute in '30' seconds" + - "dpdk|ERR|EAL: Driver cannot attach the device" + - "dpdk|EMER|Cannot create lock on" + - "device not found" + + realert: + minutes: 0 + + alert: post + http_post_url: "http://10.10.120.211:31000/alerts" + http_post_static_payload: + type: pattern-match + label: failed + http_post_payload: + index: _index + log: msg + log_path: log_path + reason: alert + num_hits: num_hits + num_matches: num_matches + rule-node1-blacklist.yaml: | + name: error-finder-node1 + type: blacklist + compare_key: alert + index: node1* + blacklist: + - "Failed to run test" + - "Failed to execute in '30' seconds" + - "('Result', 'Failed')" + - "could not open socket: connection refused" + - "Input/output error" + - "dpdk|ERR|EAL: Error - exiting with code: 1" + - "Failed to execute in '30' seconds" + - "dpdk|ERR|EAL: Driver cannot attach the device" + - "dpdk|EMER|Cannot create lock on" + - "device not found" + + realert: + minutes: 0 + + alert: post + http_post_url: "http://10.10.120.211:31000/alerts" + http_post_static_payload: + type: pattern-match + label: failed + http_post_payload: + index: _index + log: msg + log_path: log_path + reason: alert + num_hits: num_hits + num_matches: num_matches diff --git a/tools/lma/ansible-server/roles/logging/files/elastalert/elastalert.yaml b/tools/lma/ansible-server/roles/logging/files/elastalert/elastalert.yaml new file mode 100644 index 00000000..9e32e2b7 --- /dev/null +++ b/tools/lma/ansible-server/roles/logging/files/elastalert/elastalert.yaml @@ -0,0 +1,76 @@ +# Copyright 2020 Adarsh yadav +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: elastalert +spec: + replicas: 1 + selector: + matchLabels: + run: elastalert + template: + metadata: + labels: + run: elastalert + spec: + volumes: + - name: econfig + configMap: + name: elastalert-config + items: + - key: elastalert.yaml + path: elastalert.yaml + - name: erule + configMap: + name: elastalert-rule + items: + - key: rule-node4-vswitch.yaml + path: rule-node4-vswitch.yaml + - key: rule-node4-blacklist.yaml + path: rule-node4-blacklist.yaml + - key: rule-node1-blacklist.yaml + path: rule-node1-blacklist.yaml + - name: ekey + configMap: + name: elastalert-key + items: + - key: elastalert.key + path: elastalert.key + - key: elastalert.pem + path: elastalert.pem + initContainers: + - name: init-myservice + image: busybox:1.28 + command: ['sh', '-c', 'until nslookup logging-es-http; do echo "waiting for myservice"; sleep 2; done;'] + containers: + - name: elastalert + image: adi0509/elastalert:latest + env: + - name: ES_USERNAME + value: "elastic" + - name: ES_PASSWORD + valueFrom: + secretKeyRef: + name: logging-es-elastic-user + key: elastic + command: [ "sh", "-c"] + args: ["elastalert-create-index --config /opt/elastalert/elastalert.yaml; python -m elastalert.elastalert --config /opt/elastalert/elastalert.yaml"] + volumeMounts: + - mountPath: /opt/elastalert/ + name: econfig + - mountPath: /opt/elastalert/rules/ + name: erule + - mountPath: /opt/elastalert/key + name: ekey |