ElasticSearch improvement

Added elasticseach manifest in cntt type for common configurations tailored
for opnfv intel pods, including limiting the memory and disk requests and
limits, and tightened the log curation policy.

Signed-off-by: James Gu <james.gu@att.com>
Change-Id: Ic926b7d0625125ce047ee1fb3cf91df2ecefa77a

5 files changed, 115 insertions, 208 deletions

diff --git a/site/intel-pod10/software/charts/osh-infra/elasticsearch.yaml b/site/intel-pod10/software/charts/osh-infra/elasticsearch.yaml
deleted file mode 100644
index ef0a42e..0000000
--- a/site/intel-pod10/software/charts/osh-infra/elasticsearch.yaml
+++ /dev/null
@@ -1,34 +0,0 @@
----
-schema: armada/Chart/v1
-metadata:
-  schema: metadata/Document/v1
-  name: elasticsearch
-  labels:
-    name: elasticsearch-type
-  layeringDefinition:
-    abstract: false
-    layer: site
-    parentSelector:
-      hosttype: elasticsearch-global
-    actions:
-      - method: merge
-        path: .
-  storagePolicy: cleartext
-data:
-  values:
-    pod:
-      replicas:
-        master: 2
-        data: 1
-        client: 2
-    storage:
-      requests:
-        storage: 20Gi
-    conf:
-      elasticsearch:
-        env:
-          java_opts:
-            client: "-Xms2048m -Xmx2048m"
-            data: "-Xms2048m -Xmx2048m"
-            master: "-Xms2048m -Xmx2048m"
-...
diff --git a/site/intel-pod15/software/charts/osh-infra/elasticsearch.yaml b/site/intel-pod15/software/charts/osh-infra/elasticsearch.yaml
deleted file mode 100644
index ef0a42e..0000000
--- a/site/intel-pod15/software/charts/osh-infra/elasticsearch.yaml
+++ /dev/null
@@ -1,34 +0,0 @@
----
-schema: armada/Chart/v1
-metadata:
-  schema: metadata/Document/v1
-  name: elasticsearch
-  labels:
-    name: elasticsearch-type
-  layeringDefinition:
-    abstract: false
-    layer: site
-    parentSelector:
-      hosttype: elasticsearch-global
-    actions:
-      - method: merge
-        path: .
-  storagePolicy: cleartext
-data:
-  values:
-    pod:
-      replicas:
-        master: 2
-        data: 1
-        client: 2
-    storage:
-      requests:
-        storage: 20Gi
-    conf:
-      elasticsearch:
-        env:
-          java_opts:
-            client: "-Xms2048m -Xmx2048m"
-            data: "-Xms2048m -Xmx2048m"
-            master: "-Xms2048m -Xmx2048m"
-...
diff --git a/site/intel-pod17/software/charts/osh-infra/elasticsearch.yaml b/site/intel-pod17/software/charts/osh-infra/elasticsearch.yaml
deleted file mode 100644
index 2f7b3c1..0000000
--- a/site/intel-pod17/software/charts/osh-infra/elasticsearch.yaml
+++ /dev/null
@@ -1,70 +0,0 @@
----
-schema: armada/Chart/v1
-metadata:
-  schema: metadata/Document/v1
-  name: elasticsearch
-  labels:
-    name: elasticsearch-type
-  layeringDefinition:
-    abstract: false
-    layer: site
-    parentSelector:
-      hosttype: elasticsearch-global
-    actions:
-      - method: merge
-        path: .
-  storagePolicy: cleartext
-data:
-  values:
-    pod:
-      replicas:
-        client: 3
-      resources:
-        enabled: true
-        apache_proxy:
-          limits:
-            memory: "1024Mi"
-            cpu: "2000m"
-          requests:
-            memory: "0"
-            cpu: "0"
-        client:
-          requests:
-            memory: "4Gi"
-            cpu: "1000m"
-          limits:
-            memory: "8Gi"
-            cpu: "2000m"
-        master:
-          requests:
-            memory: "4Gi"
-            cpu: "1000m"
-          limits:
-            memory: "8Gi"
-            cpu: "2000m"
-        data:
-          requests:
-            memory: "4Gi"
-            cpu: "1000m"
-          limits:
-            memory: "8Gi"
-            cpu: "2000m"
-        prometheus_elasticsearch_exporter:
-          requests:
-            memory: "0"
-            cpu: "0"
-          limits:
-            memory: "1024Mi"
-            cpu: "2000m"
-
-    storage:
-      requests:
-        storage: 10Gi
-    conf:
-      elasticsearch:
-        env:
-          java_opts:
-            client: "-Xms2048m -Xmx2048m"
-            data: "-Xms2048m -Xmx2048m"
-            master: "-Xms2048m -Xmx2048m"
-...
diff --git a/site/intel-pod18/software/charts/osh-infra/elasticsearch.yaml b/site/intel-pod18/software/charts/osh-infra/elasticsearch.yaml
deleted file mode 100644
index 2f7b3c1..0000000
--- a/site/intel-pod18/software/charts/osh-infra/elasticsearch.yaml
+++ /dev/null
@@ -1,70 +0,0 @@
----
-schema: armada/Chart/v1
-metadata:
-  schema: metadata/Document/v1
-  name: elasticsearch
-  labels:
-    name: elasticsearch-type
-  layeringDefinition:
-    abstract: false
-    layer: site
-    parentSelector:
-      hosttype: elasticsearch-global
-    actions:
-      - method: merge
-        path: .
-  storagePolicy: cleartext
-data:
-  values:
-    pod:
-      replicas:
-        client: 3
-      resources:
-        enabled: true
-        apache_proxy:
-          limits:
-            memory: "1024Mi"
-            cpu: "2000m"
-          requests:
-            memory: "0"
-            cpu: "0"
-        client:
-          requests:
-            memory: "4Gi"
-            cpu: "1000m"
-          limits:
-            memory: "8Gi"
-            cpu: "2000m"
-        master:
-          requests:
-            memory: "4Gi"
-            cpu: "1000m"
-          limits:
-            memory: "8Gi"
-            cpu: "2000m"
-        data:
-          requests:
-            memory: "4Gi"
-            cpu: "1000m"
-          limits:
-            memory: "8Gi"
-            cpu: "2000m"
-        prometheus_elasticsearch_exporter:
-          requests:
-            memory: "0"
-            cpu: "0"
-          limits:
-            memory: "1024Mi"
-            cpu: "2000m"
-
-    storage:
-      requests:
-        storage: 10Gi
-    conf:
-      elasticsearch:
-        env:
-          java_opts:
-            client: "-Xms2048m -Xmx2048m"
-            data: "-Xms2048m -Xmx2048m"
-            master: "-Xms2048m -Xmx2048m"
-...
diff --git a/type/cntt/software/charts/osh-infra/elasticsearch.yaml b/type/cntt/software/charts/osh-infra/elasticsearch.yaml
new file mode 100644
index 0000000..f69a0a5
--- /dev/null
+++ b/type/cntt/software/charts/osh-infra/elasticsearch.yaml
@@ -0,0 +1,115 @@
+---
+schema: armada/Chart/v1
+metadata:
+  schema: metadata/Document/v1
+  replacement: true
+  name: elasticsearch
+  labels:
+    name: elasticsearch-type
+  layeringDefinition:
+    abstract: false
+    layer: type
+    parentSelector:
+      hosttype: elasticsearch-global
+    actions:
+      - method: merge
+        path: .
+  storagePolicy: cleartext
+data:
+  values:
+    pod:
+      replicas:
+        master: 3
+        data: 3
+        client: 3
+      resources:
+        enabled: true
+        apache_proxy:
+          limits:
+            memory: "1024Mi"
+            cpu: "2000m"
+          requests:
+            memory: "0"
+            cpu: "0"
+        client:
+          requests:
+            memory: "4Gi"
+            cpu: "1000m"
+          limits:
+            memory: "8Gi"
+            cpu: "2000m"
+        master:
+          requests:
+            memory: "4Gi"
+            cpu: "1000m"
+          limits:
+            memory: "8Gi"
+            cpu: "2000m"
+        data:
+          requests:
+            memory: "4Gi"
+            cpu: "1000m"
+          limits:
+            memory: "8Gi"
+            cpu: "2000m"
+        prometheus_elasticsearch_exporter:
+          requests:
+            memory: "0"
+            cpu: "0"
+          limits:
+            memory: "1024Mi"
+            cpu: "2000m"
+
+    storage:
+      requests:
+        storage: 50Gi
+    conf:
+      elasticsearch:
+        env:
+          java_opts:
+            client: "-Xms2048m -Xmx2048m"
+            data: "-Xms2048m -Xmx2048m"
+            master: "-Xms2048m -Xmx2048m"
+      curator:
+        #run every 6th hour
+        schedule:  "0 */6 * * *"
+        action_file:
+          actions:
+            1:
+              action: delete_indices
+              description: >-
+                "Delete indices older than 1 days"
+              options:
+                timeout_override:
+                continue_if_exception: False
+                ignore_empty_list: True
+                disable_action: False
+              filters:
+              - filtertype: pattern
+                kind: prefix
+                value: logstash-
+              - filtertype: age
+                source: name
+                direction: older
+                timestring: '%Y.%m.%d'
+                unit: days
+                unit_count: 1
+            2:
+              action: delete_indices
+              description: >-
+                "Delete indices by age if available disk space is
+                 less than 80% total disk"
+              options:
+                timeout_override: 600
+                continue_if_exception: False
+                ignore_empty_list: True
+                disable_action: False
+              filters:
+              - filtertype: pattern
+                kind: prefix
+                value: logstash-
+              - filtertype: space
+                source: creation_date
+                use_age: True
+                disk_space: 120
+...