Merge "Adapt monitoring install to apex"

6 files changed, 188 insertions, 149 deletions

diff --git a/monitor/dashboard/jump_server_status.json b/monitor/dashboard/jump_server_status.json
index 308e99c0..823caba8 100644
--- a/monitor/dashboard/jump_server_status.json
+++ b/monitor/dashboard/jump_server_status.json
@@ -28,7 +28,7 @@
   "gnetId": 159,
   "graphTooltip": 1,
   "hideControls": false,
-  "id": 2,
+  "id": null,
   "links": [],
   "refresh": false,
   "rows": [
diff --git a/monitor/dispatch/automate_barometer_client.py b/monitor/dispatch/automate_barometer_client.py
deleted file mode 100644
index 3a246028..00000000
--- a/monitor/dispatch/automate_barometer_client.py
+++ /dev/null
@@ -1,51 +0,0 @@
-##############################################################################
-# Copyright (c) 2018 Huawei Tech and others.
-#
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-
-import logging
-import yaml
-import utils.infra_setup.passwordless_SSH.ssh as ssh
-
-logger = logging.getLogger(__name__)
-barometer_client_install_sh =\
-    "/home/opnfv/bottlenecks/monitor/dispatch/install_barometer_client.sh"
-barometer_client_install_conf =\
-    "/home/opnfv/bottlenecks/monitor/config/barometer_client.conf"
-
-with open('/tmp/pod.yaml') as f:
-    dataMap = yaml.safe_load(f)
-    for x in dataMap:
-        for y in dataMap[x]:
-            if (y['role'] == 'Controller') or (y['role'] == 'Compute'):
-                ip = str(y['ip'])
-                user = str(y['user'])
-                pwd = str(y['password'])
-                ssh_d = ssh.SSH(user, host=ip, password=pwd)
-                status, stdout, stderr = ssh_d.execute(
-                    "cd /etc && mkdir barometer_config"
-                )
-                if status:
-                    print Exception(
-                        "Command: \"mkdir barometer_config\" failed.")
-                    logger.info(stdout.splitlines())
-                with open(barometer_client_install_conf) as stdin_file:
-                    ssh_d.run("cat > /etc/barometer_config/\
-barometer_client.conf",
-                              stdin=stdin_file)
-                with open(barometer_client_install_sh) as stdin_file:
-                    ssh_d.run("cat > /etc/barometer_config/install.sh",
-                              stdin=stdin_file)
-
-                status, stdout, stderr = ssh_d.execute(
-                    "sudo apt-get install -y docker.io"
-                )
-                if status:
-                    raise Exception("Command for installing docker failed.")
-                    logger.info(stdout.splitlines())
-
-                ssh_d.run("cd /etc/barometer_config/ && bash ./install.sh")
diff --git a/monitor/dispatch/automate_cadvisor_client.py b/monitor/dispatch/automate_cadvisor_client.py
deleted file mode 100644
index 3a65ff1d..00000000
--- a/monitor/dispatch/automate_cadvisor_client.py
+++ /dev/null
@@ -1,43 +0,0 @@
-##############################################################################
-# Copyright (c) 2017 Huawei Tech and others.
-#
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-
-import logging
-import yaml
-import utils.infra_setup.passwordless_SSH.ssh as ssh
-
-logger = logging.getLogger(__name__)
-cadvisor_client_install_sh =\
-    "/home/opnfv/bottlenecks/monitor/dispatch/install_cadvisor_client.sh"
-
-with open('/tmp/pod.yaml') as f:
-    dataMap = yaml.safe_load(f)
-    for x in dataMap:
-        for y in dataMap[x]:
-            if (y['role'] == 'Controller') or (y['role'] == 'Compute'):
-                ip = str(y['ip'])
-                user = str(y['user'])
-                pwd = str(y['password'])
-                ssh_d = ssh.SSH(user, host=ip, password=pwd)
-                status, stdout, stderr = ssh_d.execute(
-                    "cd /etc && mkdir cadvisor_config"
-                )
-                if status:
-                    print Exception(
-                        "Command: \"mkdir cadvisor_config\" failed.")
-                    logger.info(stdout.splitlines())
-                with open(cadvisor_client_install_sh) as stdin_file:
-                    ssh_d.run("cat > /etc/cadvisor_config/install.sh",
-                              stdin=stdin_file)
-                status, stdout, stderr = ssh_d.execute(
-                    "sudo apt-get install -y docker.io"
-                )
-                if status:
-                    raise Exception("Command for installing docker failed.")
-                    logger.info(stdout.splitlines())
-                ssh_d.run("cd /etc/cadvisor_config/ && bash ./install.sh")
diff --git a/monitor/dispatch/automate_collectd_client.py b/monitor/dispatch/automate_collectd_client.py
deleted file mode 100644
index c4346e51..00000000
--- a/monitor/dispatch/automate_collectd_client.py
+++ /dev/null
@@ -1,50 +0,0 @@
-##############################################################################
-# Copyright (c) 2017 Huawei Tech and others.
-#
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-
-import logging
-import yaml
-import utils.infra_setup.passwordless_SSH.ssh as ssh
-
-logger = logging.getLogger(__name__)
-collectd_client_install_sh =\
-    "/home/opnfv/bottlenecks/monitor/dispatch/install_collectd_client.sh"
-collectd_client_install_conf =\
-    "/home/opnfv/bottlenecks/monitor/config/collectd_client.conf"
-
-with open('/tmp/pod.yaml') as f:
-    dataMap = yaml.safe_load(f)
-    for x in dataMap:
-        for y in dataMap[x]:
-            if (y['role'] == 'Controller') or (y['role'] == 'Compute'):
-                ip = str(y['ip'])
-                user = str(y['user'])
-                pwd = str(y['password'])
-                ssh_d = ssh.SSH(user, host=ip, password=pwd)
-                status, stdout, stderr = ssh_d.execute(
-                    "cd /etc && mkdir collectd_config"
-                )
-                if status:
-                    print Exception(
-                        "Command: \"mkdir collectd_config\" failed.")
-                    logger.info(stdout.splitlines())
-                with open(collectd_client_install_sh) as stdin_file:
-                    ssh_d.run("cat > /etc/collectd_config/install.sh",
-                              stdin=stdin_file)
-                with open(collectd_client_install_conf) as stdin_file:
-                    ssh_d.run(
-                        "cat > /etc/collectd_config/collectd_client.conf",
-                        stdin=stdin_file
-                    )
-                status, stdout, stderr = ssh_d.execute(
-                    "sudo apt-get install -y docker.io"
-                )
-                if status:
-                    raise Exception("Command for installing docker failed.")
-                    logger.info(stdout.splitlines())
-                ssh_d.run("cd /etc/collectd_config/ && bash ./install.sh")
diff --git a/monitor/dispatch/install_clients.py b/monitor/dispatch/install_clients.py
new file mode 100644
index 00000000..ab7d0f5a
--- /dev/null
+++ b/monitor/dispatch/install_clients.py
@@ -0,0 +1,118 @@
+##############################################################################
+# Copyright (c) 2018 Huawei Tech and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+'''
+Currently supported installers are Apex, Compass...
+Supported monitoring tools are Cadvisor, Collectd, Barometer...
+Carefully, do not change the path or name of the configuration files which
+are hard coded for docker volume mapping.
+'''
+import os
+import logging
+import yaml
+import utils.infra_setup.passwordless_SSH.ssh as ssh
+import argparse
+
+logger = logging.getLogger(__name__)
+
+parser = argparse.ArgumentParser(description='Monitoring Clients Dispatcher')
+parser.add_argument("-i", "--INSTALLER_TYPE",
+                    help="The installer for the system under monitoring")
+# Barometer config and installation files
+# /home/opnfv/bottlenecks/monitor/dispatch/install_barometer_client.sh
+# /home/opnfv/bottlenecks/monitor/config/barometer_client.conf
+# Cadvisor installation file
+# /home/opnfv/bottlenecks/monitor/dispatch/install_cadvisor_client.sh
+# Collectd config and installation files
+# /home/opnfv/bottlenecks/monitor/dispatch/install_collectd_client.sh
+# /home/opnfv/bottlenecks/monitor/config/collectd_client.conf
+parser.add_argument("-s", "--INSTALlATION_SCRIPT",
+                    help="The path of the script to install monitoring script")
+parser.add_argument("-c", "--CLIENT_CONFIG", default="",
+                    help="The path of the config of monitoring client")
+parser.add_argument("-p", "--POD_DISCRIPTOR", default="/tmp/pod.yaml",
+                    help="The path of pod discrition file")
+args = parser.parse_args()
+
+INSTALLERS = ['apex', 'compass']
+if args.INSTALLER_TYPE not in INSTALLERS:
+    raise Exception("The installer is not supported.")
+if not args.INSTALlATION_SCRIPT:
+    raise Exception("Must specify the client installation script path!")
+
+if "barometer" in args.INSTALlATION_SCRIPT.lower():
+    CONFIG_FILE = "/etc/barometer_config/barometer_client.conf"
+    CONFIG_DIR = "barometer_config"
+    INSTALlATION_SCRIPT = "/etc/barometer_config/install.sh"
+elif "collectd" in args.INSTALlATION_SCRIPT.lower():
+    CONFIG_FILE = "/etc/collectd_config/collectd_client.conf"
+    CONFIG_DIR = "collectd_config"
+    INSTALlATION_SCRIPT = "/etc/collectd_config/install.sh"
+elif "cadvisor" in args.INSTALlATION_SCRIPT.lower():
+    CONFIG_DIR = "cadvisor_config"
+    INSTALlATION_SCRIPT = "/etc/cadvisor_config/install.sh"
+else:
+    raise Exception("The monitor client is not supported")
+
+
+def main():
+    with open(args.POD_DISCRIPTOR) as f:
+        dataMap = yaml.safe_load(f)
+        for x in dataMap:
+            for y in dataMap[x]:
+                if (y['role'].lower() == 'controller') or (
+                        y['role'].lower() == 'compute'):
+                    ip = str(y['ip'])
+                    user = str(y['user'])
+                    pwd = str(y['password'])
+                    idkey = str(y['key_filename'])
+
+                    if pwd:
+                        ssh_d = ssh.SSH(user, host=ip, password=pwd)
+                    elif idkey:
+                        idkey = "/tmp/id_rsa"
+                        ssh_d = ssh.SSH(user, host=ip, key_filename=idkey)
+                    status, stdout, stderr = ssh_d.execute(
+                        "cd /etc && mkdir " + CONFIG_DIR
+                    )
+                    if status:
+                        print Exception(
+                            "Command: \"mkdir {}\".format(CONFIG_DIR) failed.")
+                        logger.info(stdout.splitlines())
+                    if args.CLIENT_CONFIG:
+                        with open(args.CLIENT_CONFIG) as stdin_file:
+                            ssh_d.run("cat > " + CONFIG_FILE,
+                                      stdin=stdin_file)
+                    with open(args.INSTALlATION_SCRIPT) as stdin_file:
+                        ssh_d.run("cat > " + INSTALlATION_SCRIPT,
+                                  stdin=stdin_file)
+
+                    for u in os.uname():
+                        if 'ubuntu' in u.lower():
+                            NODE_OS = 'ubuntu'
+                            break
+                    if NODE_OS == 'ubuntu':
+                        status, stdout, stderr = ssh_d.execute(
+                            "sudo apt-get install -y docker.io"
+                        )
+                    else:
+                        status, stdout, stderr = ssh_d.execute(
+                            "sudo yum install -y docker-io"
+                        )
+                    if status:
+                        raise Exception(
+                            "Command for installing docker failed.")
+                        logger.info(stdout.splitlines())
+
+                    ssh_d.run(
+                        "cd /etc/{}/ && bash ./install.sh".format(CONFIG_DIR)
+                    )
+
+
+if __name__ == '__main__':
+    main()
diff --git a/monitor/monitoring.sh b/monitor/monitoring.sh
index 0a450b7f..89750614 100644
--- a/monitor/monitoring.sh
+++ b/monitor/monitoring.sh
@@ -9,6 +9,69 @@
 ##############################################################################
 MONITOR_CONFIG="/home/opnfv/bottlenecks/monitor/config"
 DISPATCH="/home/opnfv/bottlenecks/monitor/dispatch"
+OPENSTACK_ENV=${MONITOR_CONFIG}/openstack_exporter.conf
+
+usage="Script to run the tests in Bottlenecks.
+
+usage:
+    bash $(basename "$0") [-h|--help] [-i|--installer <installer typer>] [-o|--openstack-env <openstack env>]
+
+where:
+    -h|--help         show the help text
+    -i|--installer    specify the installer for the system to be monitored
+      <installer type>
+                      one of the following:
+                              (apex, compass)
+    -o|--opentack-env specify the openstack env file for openstack monitoring
+                      defalt value is \"${MONITOR_CONFIG}/openstack_exporter.conf\"
+
+examples:
+    $(basename "$0") -i compass"
+
+
+info () {
+    logger -s -t "BOTTLENECKS INFO" "$*"
+}
+
+error () {
+    logger -s -t "BOTTLENECKS ERROR" "$*"
+    exit 1
+}
+
+# Process input variables
+while [[ $# > 0 ]]
+    do
+    key="$1"
+    case $key in
+        -h|--help)
+            echo "$usage"
+            exit 0
+            shift
+        ;;
+        -i|--installer)
+            INSTALLER_TYPE="$2"
+            shift
+        ;;
+        -i|--openstack-env)
+            OPENSTACK_ENV="$2"
+            shift
+        ;;
+        *)
+            error "unkown input options $1 $2"
+            exit 1
+        ;;
+     esac
+     shift
+done
+
+
+barometer_client_install_sh="/home/opnfv/bottlenecks/monitor/dispatch/install_barometer_client.sh"
+barometer_client_install_conf="/home/opnfv/bottlenecks/monitor/config/barometer_client.conf"
+
+cadvisor_client_install_sh="/home/opnfv/bottlenecks/monitor/dispatch/install_cadvisor_client.sh"
+
+collectd_client_install_sh="/home/opnfv/bottlenecks/monitor/dispatch/install_collectd_client.sh"
+collectd_client_install_conf="/home/opnfv/bottlenecks/monitor/config/collectd_client.conf"
 
 # INSTALL GRAFANA + PROMETHEUS + CADVISOR + BAROMETER on the JUMPERSERVER
 # # Node-Exporter
@@ -50,7 +113,7 @@ sudo docker run --name bottlenecks-node-exporter \
 # Openstack-Exporter
 sudo docker run --name bottlenecks-openstack-exporter \
   -v /tmp:/tmp \
-  -p 9104:9104 --env-file ${MONITOR_CONFIG}/openstack_exporter.conf \
+  -p 9104:9104 --env-file ${OPENSTACK_ENV} \
   -d gabrielyuyang/att-prometheus-openstack-exporter:latest
 
 # Grafana
@@ -99,9 +162,10 @@ set -e
 # Configure IP Address in barometer client configuration
 python ${DISPATCH}/client_ip_configure.py ${MONITOR_CONFIG}/barometer_client.conf
 
-
 # Automate Barometer client installation
-python ${DISPATCH}/automate_barometer_client.py
+python ${DISPATCH}/install_clients.py \
+  -i ${INSTALLER_TYPE} -s ${barometer_client_install_sh} \
+  -c ${barometer_client_install_conf}
 
 # # Configure IP Address in collectd client configuration
 # python ${DISPATCH}/client_ip_configure.py ${MONITOR_CONFIG}/collectd_client.conf
@@ -109,6 +173,7 @@ python ${DISPATCH}/automate_barometer_client.py
 # python ${DISPATCH}/automate_collectd_client.py
 
 # Automate Cadvisor Client
-python ${DISPATCH}/automate_cadvisor_client.py
+python ${DISPATCH}/install_clients.py \
+  -i ${INSTALLER_TYPE} -s ${cadvisor_client_install_sh}
 
 echo == installation of monitoring module is finished ==