From e20d8ba804539203664a542521db5b64826ba05c Mon Sep 17 00:00:00 2001 From: adi0509 Date: Fri, 21 Aug 2020 23:24:08 +0530 Subject: LMA: Deployment of LMA solution.(ansible) Ansible playbooks for LMA deployment Signed-off-by: Adarsh Yadav Change-Id: I3ab71e139bc1668577506fb996105a834a9c3c65 --- .../roles/logging/files/fluentd/fluent-cm.yaml | 525 +++++++++++++++++++++ .../logging/files/fluentd/fluent-service.yaml | 34 ++ .../roles/logging/files/fluentd/fluent.yaml | 65 +++ 3 files changed, 624 insertions(+) create mode 100644 tools/lma/ansible-server/roles/logging/files/fluentd/fluent-cm.yaml create mode 100644 tools/lma/ansible-server/roles/logging/files/fluentd/fluent-service.yaml create mode 100644 tools/lma/ansible-server/roles/logging/files/fluentd/fluent.yaml (limited to 'tools/lma/ansible-server/roles/logging/files/fluentd') diff --git a/tools/lma/ansible-server/roles/logging/files/fluentd/fluent-cm.yaml b/tools/lma/ansible-server/roles/logging/files/fluentd/fluent-cm.yaml new file mode 100644 index 00000000..36ff80d6 --- /dev/null +++ b/tools/lma/ansible-server/roles/logging/files/fluentd/fluent-cm.yaml @@ -0,0 +1,525 @@ +# Copyright 2020 Adarsh yadav +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +apiVersion: v1 +kind: ConfigMap +metadata: + name: fluentd-config +data: + index_template.json: | + { + "index_patterns": [ + "node*" + ], + "settings": { + "index.lifecycle.name": "delete_policy", + "number_of_replicas": 1 + } + } + fluent.conf: | + + @type forward + port 24224 + bind 0.0.0.0 + tag log + + + #tag the .dat file + + @type rewrite_tag_filter + #Trex data + + key log_path + pattern /\/tmp\/result.*\/.*counts.dat/ + tag countdat.${tag} + + + key log_path + pattern /\/tmp\/result.*\/.*errors.dat/ + tag errordat.${tag} + + #Spirent data + + key log_path + pattern /\/tmp\/result.*\/stc-liveresults.dat.tx/ + tag stcdattx.${tag} + + + key log_path + pattern /\/tmp\/result.*\/stc-liveresults.dat.rx/ + tag stcdatrx.${tag} + + #Ixia data + + key log_path + pattern /\/tmp\/result.*\/.*Statistics.csv/ + tag ixia.${tag} + + #log files + + key log_path + pattern /vsperf-overall/ + tag vsperf.${tag} + + + key log_path + pattern /vswitchd/ + tag vswitchd.${tag} + + + key log_path + pattern /\/var\/log\/userspace/ + tag userspace.${tag} + + + key log_path + pattern /\/var\/log\/sriovdp/ + tag sriovdp.${tag} + + + key log_path + pattern /\/var\/log\/pods/ + tag pods.${tag} + + + + #to find error + @include error.conf + + #to parse time-series data + @include time-series.conf + + #to calculate time analysis + @include time-analysis.conf + + #give tag 'node1' if host is worker and tag 'node4' if host is pod12-node4 + + @type rewrite_tag_filter + + key host + pattern /pod12-node4/ + tag node4 + + + key host + pattern /worker/ + tag node1 + + + + + + @type elasticsearch_genid + hash_id_key _hash1 + + + #send the node1 log to node1 index in elasticsearch + + @type copy + + @type elasticsearch + host logging-es-http + port 9200 + scheme https + ssl_verify false + user "#{ENV['FLUENT_ELASTICSEARCH_USER']}" + password "#{ENV['FLUENT_ELASTICSEARCH_PASSWORD']}" + logstash_format true + logstash_prefix node1 + logstash_dateformat %Y%m%d + flush_interval 1s + id_key _hash1 + remove_keys _hash1 + + enable_ilm true + application_name ${tag} + index_date_pattern "" + ilm_policy_id delete_policy + template_name delpol-test + template_file /fluentd/etc/index_template.json + ilm_policy { + "policy": { + "phases": { + "delete": { + "min_age": "3m", + "actions": { + "delete": {} + } + } + } + } + } + + + @type stdout + + + + + @type elasticsearch_genid + hash_id_key _hash4 + + + #send the node4 log to node4 index in elasticsearch + + @type copy + + @type elasticsearch + host logging-es-http + port 9200 + scheme https + ssl_verify false + user "#{ENV['FLUENT_ELASTICSEARCH_USER']}" + password "#{ENV['FLUENT_ELASTICSEARCH_PASSWORD']}" + logstash_format true + logstash_prefix node4 + logstash_dateformat %Y%m%d + flush_interval 1s + id_key _hash4 + remove_keys _hash4 + + enable_ilm true + application_name ${tag} + index_date_pattern "" + ilm_policy_id delete_policy + template_name delpol-test + template_file /fluentd/etc/index_template.json + ilm_policy { + "policy": { + "phases": { + "delete": { + "min_age": "3m", + "actions": { + "delete": {} + } + } + } + } + } + + + @type stdout + + + error.conf: | + + @type parser + reserve_data true + key_name msg + emit_invalid_record_to_error false + + @type regexp + expression /(?\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3}).*(?Failed to run test|Failed to execute in '30' seconds|\('Result', 'Failed'\)|could not open socket: connection refused|Input\/output error)/ + + + + + @type parser + reserve_data true + key_name msg + emit_invalid_record_to_error false + + @type regexp + expression /(?\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z).*(?dpdk\|ERR\|EAL: Error - exiting with code: 1|Failed to execute in '30' seconds|dpdk\|ERR\|EAL: Driver cannot attach the device|dpdk\|EMER\|Cannot create lock on)/ + + + + @type parser + reserve_data true + key_name msg + emit_invalid_record_to_error false + + @type regexp + expression /(?\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z).*dpdk\|ERR\|VHOST_CONFIG:.*(?device not found)/ + + + time-series.conf: | + #parse *counts.dat + + @type parser + key_name msg + reserve_data true + emit_invalid_record_to_error false + + @type regexp + expression /^(?[\.\d]*),(?\d*),(?\d*),(?[\.\d]*),(?[\.\d]*),(?[\.\d]*),(?[\.\d]*),(?[\.\d]*),(?[\.\d]*),(?[\.\d]*),(?[\.\d]*)$/ + types rx_port:integer,tx_port:integer,rx_pkts:float,tx_pkts:float,rx_pps:float,tx_pps:float,rx_bps_num:float,rx_bps_den:float,tx_bps_num:float,tx_bps_den:float + + + + #parse *errors.dat + + @type parser + key_name msg + reserve_data true + emit_invalid_record_to_error false + + @type regexp + expression /^(?[\.\d]*),(?[\.\d]*),(?[\.\d]*),(?[\.\d]*),(?[\.\d]*),(?[\.\d]*)$/ + types ts:integer,dropped:integer,ooo:integer,dup:integer,seq_too_high:integer,seq_too_low:integer + + + + #parse stc-liveresults.dat.tx + + @type parser + key_name msg + reserve_data true + emit_invalid_record_to_error false + + @type regexp + expression /^(?[\.\d]*),(?[\.\d]*),(?[\.\d]*),(?[\.\d]*),(?[\.\d]*),(?[\.\d]*),(?[\.\d]*),(?[\.\d]*),(?[\.\d]*),(?[\.\d]*)$/ + types ts:integer,StrId:integer,BlkId:integer,FrCnt:integer,FrRate:integer,ERxFrCnt:integer,OctCnt:integer,OctRate:integer,bitCnt:integer,bitRate:integer + + + + #parse stc-liveresults.dat.rx + + @type parser + key_name msg + reserve_data true + emit_invalid_record_to_error false + + @type regexp + expression /^(?[\.\d]*),(.*, |)(?.*),(?[\.\d]*),(?[\.\d]*),(?.*),(?[\.\d]*),(?[\.\d]*),(?[\.\d]*),(?[\.\d]*),(?[\.\d]*),(?[\.\d]*),(?[\.\d]*)$/ + types ts:integer,DrpFrCnt:integer,SeqRnLen:integer,FrCnt:integer,FrRate:integer,MaxLat:integer,MinLat:integer,OctCnt:integer,OctRate:integer + + + time-analysis.conf: | + # 1. Test Duration - Duration Between: first line and last line. + # 2. Setup Duration - Duration Between: Creating result directory TO Class found --- + # 3. Traffic Duration - Duration between From Starting traffic at 0.1 Gbps speed TO Traffic Results + # 4. Iteration Durations -- Example: Duration between - Starting traffic at 10.0 Gbps TO Starting traffic at 5.0 Gbps speed + # 5. Reporting Duration - Duration between From Traffic Results TO Write results to file + # 6. Vswitchd start Duration- Duration between From Starting vswitchd... TO send_traffic with + + + @type rewrite_tag_filter + + key msg + pattern /Creating result directory:/ + tag firstline.${tag} + + + key msg + pattern /Write results to file/ + tag lastline.${tag} + + + + key msg + pattern /Class found/ + tag setupend.${tag} + + + key msg + pattern /Starting traffic at 0.1 Gbps speed/ + tag trafficstart.${tag} + + + key msg + pattern /Traffic Results/ + tag trafficend.${tag} + + + key msg + pattern /Starting traffic at 10.0 Gbps/ + tag iterationstart.${tag} + + + key msg + pattern /Starting traffic at 5.0 Gbps speed/ + tag iterationend.${tag} + + + key msg + pattern /Starting vswitchd/ + tag vswitchstart.${tag} + + + key msg + pattern /send_traffic/ + tag vswitch.${tag} + + + key msg + pattern ^.*$ + tag logs.${tag} + + + + ############################################################################################# + #save the starting log and append that log in ending log + ############################################################################################# + + @type record_transformer + enable_ruby true + + msg ${$vswitch_start="";$reportstart="";$firstline="";$traffic_start="";$iteration_start="";$firstline = record["msg"];return record["msg"];} + + + + @type record_transformer + enable_ruby true + + newmsg ${record["msg"]+" | "+$firstline + " | "+ $reportstart} + + + + + @type record_transformer + enable_ruby true + + newmsg ${record["msg"]+" "+$firstline} + + + + + @type record_transformer + enable_ruby true + + msg ${if $traffic_start.eql?("");$traffic_start=record["msg"];end;return record["msg"];} + + + + @type record_transformer + enable_ruby true + + newmsg ${if $reportstart.eql?("");$reportstart=record["msg"];end;return record["msg"]+" "+$traffic_start;} + + + + + @type record_transformer + enable_ruby true + + msg ${if $iteration_start.eql?("");$iteration_start=record["msg"];end;return record["msg"];} + + + + @type record_transformer + enable_ruby true + + newmsg ${record["msg"]+" "+$iteration_start} + + + + + @type record_transformer + enable_ruby true + + msg ${$vswitch_start=record["msg"];return record["msg"];} + + + + @type record_transformer + enable_ruby true + + newmsg ${record["msg"]+" "+$vswitch_start} + + + ############################################################################################# + #parse time from the log + ############################################################################################# + + @type parser + key_name newmsg + reserve_data true + remove_key_name_field true + + @type regexp + expression /^(?.*) : Class found: Trex. (?.*) : .*$/ + + + + @type parser + key_name newmsg + reserve_data true + remove_key_name_field true + + @type regexp + expression /^(?.*) : Starting traffic at 5.0 Gbps speed (?.*) : Starting traffic at 10.0 Gbps speed$/ + + + + @type parser + key_name newmsg + reserve_data true + remove_key_name_field true + + @type regexp + expression /^(?.*) : send_traffic with <.*> (?.*) : Starting vswitchd...$/ + + + + @type parser + key_name newmsg + reserve_data true + remove_key_name_field true + + @type regexp + expression /^(?.*) : Traffic Results: (?.*) : Starting traffic at 0.1 Gbps speed/ + + + + @type parser + key_name newmsg + reserve_data true + remove_key_name_field true + + @type regexp + expression /^(?.*) : Write results to file: .* \| (?.*) : Creating result directory: .* \| (?.*) : Traffic Results:$/ + + + ############################################################################################# + #calculate time + ############################################################################################# + + @type record_transformer + enable_ruby + + setup_duration ${ require 'time';Time.parse(record["setupend"])-Time.parse(record["setupstart"]); } + + + + @type record_transformer + enable_ruby + + iteration_duration ${ require 'time';Time.parse(record["iterationend"])-Time.parse(record["iterationstart"]); } + + + + @type record_transformer + enable_ruby + + vswitch_duration ${ require 'time';Time.parse(record["vswitch"])-Time.parse(record["vswitchstart"]); } + + + + @type record_transformer + enable_ruby + + traffic_duration ${ require 'time';Time.parse(record["trafficend"])-Time.parse(record["trafficstart"]); } + + + + @type record_transformer + enable_ruby + + test_duration ${ require 'time';Time.parse(record["lastline"])-Time.parse(record["firstline"]); } + + + report_duration ${ require 'time';Time.parse(record["lastline"])-Time.parse(record["reportstart"]); } + + + ############################################################################################# diff --git a/tools/lma/ansible-server/roles/logging/files/fluentd/fluent-service.yaml b/tools/lma/ansible-server/roles/logging/files/fluentd/fluent-service.yaml new file mode 100644 index 00000000..9a43b82f --- /dev/null +++ b/tools/lma/ansible-server/roles/logging/files/fluentd/fluent-service.yaml @@ -0,0 +1,34 @@ +# Copyright 2020 Adarsh yadav +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +apiVersion: v1 +kind: Service +metadata: + name: fluentd + labels: + run: fluentd +spec: + type: NodePort + ports: + - name: tcp + port: 32224 + targetPort: 24224 + protocol: TCP + nodePort: 32224 + - name: udp + port: 32224 + targetPort: 24224 + protocol: UDP + nodePort: 32224 + selector: + run: fluentd diff --git a/tools/lma/ansible-server/roles/logging/files/fluentd/fluent.yaml b/tools/lma/ansible-server/roles/logging/files/fluentd/fluent.yaml new file mode 100644 index 00000000..3830f682 --- /dev/null +++ b/tools/lma/ansible-server/roles/logging/files/fluentd/fluent.yaml @@ -0,0 +1,65 @@ +# Copyright 2020 Adarsh yadav +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: fluentd +spec: + replicas: 2 + selector: + matchLabels: + run: fluentd + template: + metadata: + labels: + run: fluentd + spec: + volumes: + - name: fconfig + configMap: + name: fluentd-config + items: + - key: fluent.conf + path: fluent.conf + - key: error.conf + path: error.conf + - key: time-series.conf + path: time-series.conf + - key: time-analysis.conf + path: time-analysis.conf + - key: index_template.json + path: index_template.json + initContainers: + - name: init-myservice + image: busybox:1.28 + command: ['sh', '-c', 'until nslookup logging-es-http; do echo "waiting for myservice"; sleep 2; done;'] + containers: + - name: fluentd + image: adi0509/fluentd:latest + env: + - name: FLUENT_ELASTICSEARCH_USER + value: "elastic" + - name: FLUENT_ELASTICSEARCH_PASSWORD + valueFrom: + secretKeyRef: + name: logging-es-elastic-user + key: elastic + ports: + - containerPort: 24224 + protocol: TCP + - containerPort: 24224 + protocol: UDP + volumeMounts: + - name: fconfig + mountPath: /fluentd/etc/ -- cgit 1.2.3-korg