Initial commit for logging installation and validataion

- install fluentd with elastic stack
- validate the installation

JIRA: CLOVER-5
Change-Id: I181a7277bc332ceac549d384cf2c3817a182b06e
Signed-off-by: Yujun Zhang <zhang.yujunz@zte.com.cn>

3 files changed, 301 insertions, 0 deletions

diff --git a/clover/logging/install/fluentd-istio.yaml b/clover/logging/install/fluentd-istio.yaml
new file mode 100644
index 0000000..1853831
--- /dev/null
+++ b/clover/logging/install/fluentd-istio.yaml
@@ -0,0 +1,40 @@
+# Configuration for logentry instances
+apiVersion: "config.istio.io/v1alpha2"
+kind: logentry
+metadata:
+  name: newlog
+  namespace: istio-system
+spec:
+  severity: '"info"'
+  timestamp: request.time
+  variables:
+    source: source.labels["app"] | source.service | "unknown"
+    user: source.user | "unknown"
+    destination: destination.labels["app"] | destination.service | "unknown"
+    responseCode: response.code | 0
+    responseSize: response.size | 0
+    latency: response.duration | "0ms"
+  monitored_resource_type: '"UNSPECIFIED"'
+---
+# Configuration for a fluentd handler
+apiVersion: "config.istio.io/v1alpha2"
+kind: fluentd
+metadata:
+  name: handler
+  namespace: istio-system
+spec:
+  address: "fluentd-es.logging:24224"
+---
+# Rule to send logentry instances to the fluentd handler
+apiVersion: "config.istio.io/v1alpha2"
+kind: rule
+metadata:
+  name: newlogtofluentd
+  namespace: istio-system
+spec:
+  match: "true" # match for all requests
+  actions:
+   - handler: handler.fluentd
+     instances:
+     - newlog.logentry
+---
diff --git a/clover/logging/install/logging-stack.yaml b/clover/logging/install/logging-stack.yaml
new file mode 100644
index 0000000..9542496
--- /dev/null
+++ b/clover/logging/install/logging-stack.yaml
@@ -0,0 +1,205 @@
+# Logging Namespace. All below are a part of this namespace.
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: logging
+---
+# Elasticsearch Service
+apiVersion: v1
+kind: Service
+metadata:
+  name: elasticsearch
+  namespace: logging
+  labels:
+    app: elasticsearch
+spec:
+  ports:
+  - port: 9200
+    protocol: TCP
+    targetPort: db
+  selector:
+    app: elasticsearch
+  type: NodePort
+---
+# Elasticsearch Deployment
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  name: elasticsearch
+  namespace: logging
+  labels:
+    app: elasticsearch
+  annotations:
+    sidecar.istio.io/inject: "false"
+spec:
+  template:
+    metadata:
+      labels:
+        app: elasticsearch
+    spec:
+      containers:
+      - image: docker.elastic.co/elasticsearch/elasticsearch-oss:6.1.1
+        name: elasticsearch
+        resources:
+          # need more cpu upon initialization, therefore burstable class
+          limits:
+            cpu: 1000m
+          requests:
+            cpu: 100m
+        env:
+          - name: discovery.type
+            value: single-node
+        ports:
+        - containerPort: 9200
+          name: db
+          protocol: TCP
+        - containerPort: 9300
+          name: transport
+          protocol: TCP
+        volumeMounts:
+        - name: elasticsearch
+          mountPath: /data
+      volumes:
+      - name: elasticsearch
+        emptyDir: {}
+---
+# Fluentd Service
+apiVersion: v1
+kind: Service
+metadata:
+  name: fluentd-es
+  namespace: logging
+  labels:
+    app: fluentd-es
+spec:
+  ports:
+  - name: fluentd-tcp
+    port: 24224
+    protocol: TCP
+    targetPort: 24224
+  - name: fluentd-udp
+    port: 24224
+    protocol: UDP
+    targetPort: 24224
+  selector:
+    app: fluentd-es
+---
+# Fluentd Deployment
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  name: fluentd-es
+  namespace: logging
+  labels:
+    app: fluentd-es
+  annotations:
+    sidecar.istio.io/inject: "false"
+spec:
+  template:
+    metadata:
+      labels:
+        app: fluentd-es
+    spec:
+      containers:
+      - name: fluentd-es
+        image: gcr.io/google-containers/fluentd-elasticsearch:v2.0.1
+        env:
+        - name: FLUENTD_ARGS
+          value: --no-supervisor -q
+        resources:
+          limits:
+            memory: 500Mi
+          requests:
+            cpu: 100m
+            memory: 200Mi
+        volumeMounts:
+        - name: config-volume
+          mountPath: /etc/fluent/config.d
+      terminationGracePeriodSeconds: 30
+      volumes:
+      - name: config-volume
+        configMap:
+          name: fluentd-es-config
+---
+# Fluentd ConfigMap, contains config files.
+kind: ConfigMap
+apiVersion: v1
+data:
+  forward.input.conf: |-
+    # Takes the messages sent over TCP
+    <source>
+      type forward
+    </source>
+  output.conf: |-
+    <match **>
+       type elasticsearch
+       log_level info
+       include_tag_key true
+       host elasticsearch
+       port 9200
+       logstash_format true
+       # Set the chunk limits.
+       buffer_chunk_limit 2M
+       buffer_queue_limit 8
+       flush_interval 5s
+       # Never wait longer than 5 minutes between retries.
+       max_retry_wait 30
+       # Disable the limit on the number of retries (retry forever).
+       disable_retry_limit
+       # Use multiple threads for processing.
+       num_threads 2
+    </match>
+metadata:
+  name: fluentd-es-config
+  namespace: logging
+---
+# Kibana Service
+apiVersion: v1
+kind: Service
+metadata:
+  name: kibana
+  namespace: logging
+  labels:
+    app: kibana
+spec:
+  ports:
+  - port: 5601
+    protocol: TCP
+    targetPort: ui
+  selector:
+    app: kibana
+  type: NodePort
+---
+# Kibana Deployment
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  name: kibana
+  namespace: logging
+  labels:
+    app: kibana
+  annotations:
+    sidecar.istio.io/inject: "false"
+spec:
+  template:
+    metadata:
+      labels:
+        app: kibana
+    spec:
+      containers:
+      - name: kibana
+        image: docker.elastic.co/kibana/kibana-oss:6.1.1
+        resources:
+          # need more cpu upon initialization, therefore burstable class
+          limits:
+            cpu: 1000m
+          requests:
+            cpu: 100m
+        env:
+          - name: ELASTICSEARCH_URL
+            value: http://elasticsearch:9200
+        ports:
+        - containerPort: 5601
+          name: ui
+          protocol: TCP
+---
diff --git a/clover/logging/validate.py b/clover/logging/validate.py
new file mode 100644
index 0000000..821f912
--- /dev/null
+++ b/clover/logging/validate.py
@@ -0,0 +1,56 @@
+# Copyright (c) Authors of Clover
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+
+from kubernetes import client, config
+from kubernetes.stream import stream
+import sh
+import re
+
+FLUENTD_NAMESPACE = 'logging'
+FLUENTD_PATTERN = 'fluentd-.*'
+FLUENTD_LABELS = 'app=fluentd-es'
+FLUENTD_INPUT = """<source>
+  type forward
+</source>"""
+
+def main():
+    # Load config from default location.
+    config.load_kube_config()
+
+    v1 = client.CoreV1Api()
+
+    fluentd_pod_name = None
+
+    # find by name
+    print("Find fluentd pod by name '{}'".format(FLUENTD_PATTERN))
+    fluentd_regex = re.compile(FLUENTD_PATTERN)
+    resp = v1.list_namespaced_pod(FLUENTD_NAMESPACE)
+    for i in resp.items:
+        if fluentd_regex.search(i.metadata.name) is not None:
+            print(i.metadata.name)
+
+    # find by label selector
+    print("Find fluentd pod by label selector '{}'".format(FLUENTD_LABELS))
+    resp = v1.list_namespaced_pod(FLUENTD_NAMESPACE, label_selector=FLUENTD_LABELS)
+    for i in resp.items:
+        print(i.metadata.name)
+        fluentd_pod_name = i.metadata.name
+
+    # check fluentd configuration
+    # NOTE: exec in Python librarry does not work well, use shell command as a workaround
+    # See https://github.com/kubernetes-client/python/issues/485
+    result = sh.kubectl((
+        'exec -n logging ' +
+        fluentd_pod_name +
+        ' cat /etc/fluent/config.d/forward.input.conf').split())
+    if FLUENTD_INPUT in result:
+        print("fluentd input configured correctly")
+    else:
+        print("fluentd input not configured\n{}".format(FLUENTD_INPUT))
+
+if __name__ == '__main__':
+    main()