Merge "LMA: Deployment of LMA solution.(ansible)"

4 files changed, 324 insertions, 0 deletions

diff --git a/tools/lma/ansible-server/roles/logging/files/elastalert/ealert-conf-cm.yaml b/tools/lma/ansible-server/roles/logging/files/elastalert/ealert-conf-cm.yaml
new file mode 100644
index 00000000..a320ef75
--- /dev/null
+++ b/tools/lma/ansible-server/roles/logging/files/elastalert/ealert-conf-cm.yaml
@@ -0,0 +1,48 @@
+# Copyright 2020 Adarsh yadav
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: elastalert-config
+data:
+  elastalert.yaml: |
+      # This is the folder that contains the rule yaml files
+      # Any .yaml file will be loaded as a rule
+      rules_folder: rules
+      # How often ElastAlert will query Elasticsearch
+      # The unit can be anything from weeks to seconds
+      run_every:
+        minutes: 1
+      # ElastAlert will buffer results from the most recent
+      # period of time, in case some log sources are not in real time
+      buffer_time:
+        minutes: 15
+
+      scan_subdirectories: false
+
+      # The Elasticsearch hostname for metadata writeback
+      # Note that every rule can have its own Elasticsearch host
+      es_host: logging-es-http
+      es_port: 9200
+      es_username: ${ES_USERNAME}
+      es_password: ${ES_PASSWORD}
+      es_conn_timeout: 120
+      verify_certs: False
+      use_ssl: True
+      client_cert: '/opt/elastalert/key/elastalert.pem'
+      client_key: '/opt/elastalert/key/elastalert.key'
+      writeback_index: elastalert_status
+      writeback_alias: elastalert_alerts
+      alert_time_limit:
+        days: 2
diff --git a/tools/lma/ansible-server/roles/logging/files/elastalert/ealert-key-cm.yaml b/tools/lma/ansible-server/roles/logging/files/elastalert/ealert-key-cm.yaml
new file mode 100644
index 00000000..0c606a9c
--- /dev/null
+++ b/tools/lma/ansible-server/roles/logging/files/elastalert/ealert-key-cm.yaml
@@ -0,0 +1,68 @@
+# Copyright 2020 Adarsh yadav
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: elastalert-key
+data:
+  elastalert.key: |
+      -----BEGIN PRIVATE KEY-----
+      MIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQC0uQ+B0gy3VB4w
+      5CeWOx575lqSUuYvrGW3ILpV1gmj0ZZCMZUGvt4UvaCEaNPIAqNaHPmaslQqJb5C
+      PJH9pMN7vUVp3DACzmYrS4HdROHamn5gjebXs4hq43heLaIB1Kb+4F+7sEY88irK
+      xOevadcN35y5ld7lVUGRsj6JYcweaAeh/YZ/HaBT5RfdGF+x07NDus+mFqT8j3PD
+      rs2+JtEvEoWtjcxwFgloc9GkHsWZoV1AQHgyAWjmDXZtZeV0HQSkl7hWFG9vxTni
+      DvdrdhX0g+D+u8jWnlR4Za4jd64KbTp9C9trSHyMSRIvN5obm/H8O5MQ+sZ+NQ0X
+      PdK92MjbAgMBAAECggEASbRPxrpLxVjhFz91haeGvzErLxHwHvFIam9Gj0tDkzQe
+      +9AM3ztohzzvAhFejevFgzLd+WFRQf8yoQDi6XcQ4p5GeO38Bqj2siGRTRSSp/zq
+      HabBxqbJtA4hQQeLUwPPN5N6d6lke+an3RqBAuE/e8D+whGFXjJvE2SGbLEd9if2
+      uzHj37sPsVi8kRvgZBDOozmt7YFzQVO/1V+4Lw6nz48M3t+hOHaUXY0Yd8nsk5A6
+      kgoDQ4CGUHjtWfSrccZrYNk51Zows9/sX8axfJ94wKJSImWJcuW9PXIQhzT4exnH
+      sPOwY6Noy3nXRk9gcchT60fKpp+tsJZk3ezkwSpgwQKBgQDvsaYcbnIVdFZpaNKF
+      Tmt/w60CmfGeNozRygfi84ot7edUf93cB6WSKChcAE8fbq9Ji5USPNtfbnZfFXsI
+      IyTr2KHW3RkHuDEyu+Lan9JuReEH3QOG83vvN/oYA3J3hqUTCjEGkPjqnoFtdk8L
+      f7WH1jZvXYEMo0C48SXo+yGohQKBgQDBBGkzL928j1QB9NfiNFk70EalDsF8Im2W
+      n8bQ54KYspUybKD/Hmw0jIV7kdu2vhgGC4RYkn9c5qATtulbYJUgUBelaSi0vhXT
+      gfAuO+JIIZ50P+mkkxH/KIUyu1xWUB2jtMulqLLomdoBvfp/u51qCY6fT3WMCB+R
+      ouWLr2oZ3wKBgQCAuas4AaiLFRuDKKRGq0LYLsIvb3VvPmSKFjH+FETVPbrKipEf
+      pYup3p8uKYxUmSDSIoBAdyZpLe2sSuD0Ecu2TXU86yiSGL1zPawrNUHRrv2XN365
+      bvHUGv/Y/aDvyAPHIeYKXLkRZ2ai3rK8vi1Dcitxy4mOu+36ZKezY4tD8QKBgQCd
+      hakJUj4nPd20fwqUnF5a1z5gRGuZkEtZiunp4ZaOYegrL8YwjraGKExjrYTfXcIj
+      ZNDMrDpvKfRoQnWt0mPB7DtwDiNfZmZPqBLI2Kxya6VygBqA6lncoEgcQBY6hsW5
+      rbopZ0UjWTQ3CcFe71GnkUcpMuLetl51L7kgR7dShwKBgQC+vqjhe/h081JGLTo1
+      tKeRUCaDA/V3VHjFKgM5g+S3/KzgU/EaB1rq3Qja1quGv0zHveca3zibdNQi1ENm
+      KSutWh2zQXzzvmycPmVcthhOxaKzRXDjG0mXiA0bnSgK3F2o9t4196RYhIiiSvAH
+      shVjZMTK04h8ciTLIqK/GtZr+g==
+      -----END PRIVATE KEY-----
+  elastalert.pem: |
+      -----BEGIN CERTIFICATE-----
+      MIIDVzCCAj+gAwIBAgIJAORgkR7Y0Nk9MA0GCSqGSIb3DQEBCwUAMEIxCzAJBgNV
+      BAYTAlhYMRUwEwYDVQQHDAxEZWZhdWx0IENpdHkxHDAaBgNVBAoME0RlZmF1bHQg
+      Q29tcGFueSBMdGQwHhcNMjAwNjI4MTM1NjAwWhcNMjEwNjI4MTM1NjAwWjBCMQsw
+      CQYDVQQGEwJYWDEVMBMGA1UEBwwMRGVmYXVsdCBDaXR5MRwwGgYDVQQKDBNEZWZh
+      dWx0IENvbXBhbnkgTHRkMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA
+      tLkPgdIMt1QeMOQnljsee+ZaklLmL6xltyC6VdYJo9GWQjGVBr7eFL2ghGjTyAKj
+      Whz5mrJUKiW+QjyR/aTDe71FadwwAs5mK0uB3UTh2pp+YI3m17OIauN4Xi2iAdSm
+      /uBfu7BGPPIqysTnr2nXDd+cuZXe5VVBkbI+iWHMHmgHof2Gfx2gU+UX3RhfsdOz
+      Q7rPphak/I9zw67NvibRLxKFrY3McBYJaHPRpB7FmaFdQEB4MgFo5g12bWXldB0E
+      pJe4VhRvb8U54g73a3YV9IPg/rvI1p5UeGWuI3euCm06fQvba0h8jEkSLzeaG5vx
+      /DuTEPrGfjUNFz3SvdjI2wIDAQABo1AwTjAdBgNVHQ4EFgQUFAvjohHTavHmbRbj
+      Yq2h3cq7UMEwHwYDVR0jBBgwFoAUFAvjohHTavHmbRbjYq2h3cq7UMEwDAYDVR0T
+      BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEAB9oDASl4OfF/D49i3KtVzjzge4up
+      WssBPYKVwASh3cXfLLe3NdY9ihdCXFd/8Rus0hBGaRPIyR06sZoHRDEfJ2xrRD6g
+      pr4iHRfaoEWqols7+iW0cgQehvw5efEpFL1vg9zK9kOwruS4ZUhDrak6GcO/O8Jh
+      6lSGmidHSHrQmfqFeTotaezwylV/uHvRZHPvk2JhQfC+vFjn5/iN/0wCeQCwYvOC
+      rePq2ZFdYg/0bS9BYwKsT2w1Z/AU/wIMLmbNB1af+fTBBEQlxb4rAeDb+J9EoSQ5
+      MVP7jm3BVnHQCs6CA4LV4yRQNF2K6GkWem1oUg/H3S2SG8TAUlKpX/1XRw==
+      -----END CERTIFICATE-----
diff --git a/tools/lma/ansible-server/roles/logging/files/elastalert/ealert-rule-cm.yaml b/tools/lma/ansible-server/roles/logging/files/elastalert/ealert-rule-cm.yaml
new file mode 100644
index 00000000..af28b6f6
--- /dev/null
+++ b/tools/lma/ansible-server/roles/logging/files/elastalert/ealert-rule-cm.yaml
@@ -0,0 +1,132 @@
+# Copyright 2020 Adarsh yadav
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: elastalert-rule
+data:
+  rule-node4-vswitch.yaml: |
+    name: vswitch-3-sec
+    type: any
+    index: node4*
+    filter:
+    - range:
+        time_vswitchd:
+          gt: 3    #Greater than
+
+    realert:
+      minutes: 0
+
+    alert: post
+    http_post_url: "http://10.10.120.211:31000/alerts"
+    http_post_static_payload:
+      type: threshold
+      label: vswitchd start time > 3 sec
+    http_post_payload:
+       index: _index
+       log: msg
+       log_path: log_path
+       time_vswitchd: time_vswitchd
+       num_hits: num_hits
+       num_matches: num_matches
+
+  rule-node1-vswitch.yaml: |
+    name: vswitch-3-sec
+    type: any
+    index: node1*
+    filter:
+    - range:
+        time_vswitchd:
+          gt: 3    #Greater than
+
+    realert:
+      minutes: 0
+
+    alert: post
+    http_post_url: "http://10.10.120.211:31000/alerts"
+    http_post_static_payload:
+      type: threshold
+      label: vswitchd start time > 3 sec
+    http_post_payload:
+       index: _index
+       log: msg
+       log_path: log_path
+       time_vswitchd: time_vswitchd
+       num_hits: num_hits
+       num_matches: num_matches
+
+  rule-node4-blacklist.yaml: |
+      name: error-finder-node4
+      type: blacklist
+      compare_key: alert
+      index: node4*
+      blacklist:
+        - "Failed to run test"
+        - "Failed to execute in '30' seconds"
+        - "('Result', 'Failed')"
+        - "could not open socket: connection refused"
+        - "Input/output error"
+        - "dpdk|ERR|EAL: Error - exiting with code: 1"
+        - "Failed to execute in '30' seconds"
+        - "dpdk|ERR|EAL: Driver cannot attach the device"
+        - "dpdk|EMER|Cannot create lock on"
+        - "device not found"
+
+      realert:
+        minutes: 0
+
+      alert: post
+      http_post_url: "http://10.10.120.211:31000/alerts"
+      http_post_static_payload:
+        type: pattern-match
+        label: failed
+      http_post_payload:
+         index: _index
+         log: msg
+         log_path: log_path
+         reason: alert
+         num_hits: num_hits
+         num_matches: num_matches
+  rule-node1-blacklist.yaml: |
+      name: error-finder-node1
+      type: blacklist
+      compare_key: alert
+      index: node1*
+      blacklist:
+        - "Failed to run test"
+        - "Failed to execute in '30' seconds"
+        - "('Result', 'Failed')"
+        - "could not open socket: connection refused"
+        - "Input/output error"
+        - "dpdk|ERR|EAL: Error - exiting with code: 1"
+        - "Failed to execute in '30' seconds"
+        - "dpdk|ERR|EAL: Driver cannot attach the device"
+        - "dpdk|EMER|Cannot create lock on"
+        - "device not found"
+
+      realert:
+        minutes: 0
+
+      alert: post
+      http_post_url: "http://10.10.120.211:31000/alerts"
+      http_post_static_payload:
+        type: pattern-match
+        label: failed
+      http_post_payload:
+         index: _index
+         log: msg
+         log_path: log_path
+         reason: alert
+         num_hits: num_hits
+         num_matches: num_matches
diff --git a/tools/lma/ansible-server/roles/logging/files/elastalert/elastalert.yaml b/tools/lma/ansible-server/roles/logging/files/elastalert/elastalert.yaml
new file mode 100644
index 00000000..9e32e2b7
--- /dev/null
+++ b/tools/lma/ansible-server/roles/logging/files/elastalert/elastalert.yaml
@@ -0,0 +1,76 @@
+# Copyright 2020 Adarsh yadav
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: elastalert
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      run: elastalert
+  template:
+    metadata:
+      labels:
+        run: elastalert
+    spec:
+      volumes:
+      - name: econfig
+        configMap:
+          name: elastalert-config
+          items:
+          - key: elastalert.yaml
+            path: elastalert.yaml
+      - name: erule
+        configMap:
+          name: elastalert-rule
+          items:
+          - key: rule-node4-vswitch.yaml
+            path: rule-node4-vswitch.yaml
+          - key: rule-node4-blacklist.yaml
+            path: rule-node4-blacklist.yaml
+          - key: rule-node1-blacklist.yaml
+            path: rule-node1-blacklist.yaml
+      - name: ekey
+        configMap:
+          name: elastalert-key
+          items:
+          - key: elastalert.key
+            path: elastalert.key
+          - key: elastalert.pem
+            path: elastalert.pem
+      initContainers:
+      - name: init-myservice
+        image: busybox:1.28
+        command: ['sh', '-c', 'until nslookup logging-es-http; do echo "waiting for myservice"; sleep 2; done;']
+      containers:
+      - name: elastalert
+        image: adi0509/elastalert:latest
+        env:
+        - name: ES_USERNAME
+          value: "elastic"
+        - name: ES_PASSWORD
+          valueFrom:
+            secretKeyRef:
+              name: logging-es-elastic-user
+              key: elastic
+        command: [ "sh", "-c"]
+        args: ["elastalert-create-index --config /opt/elastalert/elastalert.yaml; python -m elastalert.elastalert --config /opt/elastalert/elastalert.yaml"]
+        volumeMounts:
+        - mountPath: /opt/elastalert/
+          name: econfig
+        - mountPath: /opt/elastalert/rules/
+          name: erule
+        - mountPath: /opt/elastalert/key
+          name: ekey