Installer adapters

This tool might be used by any project in OPNFV
to retrieve information about the deployed
OpenStack nodes.
It is python based and using paramiko.

It can:
  - get info about deployment
  - get the info about the nodes (ip, mac, ...)
  - stablish ssh connection even with ProxyCommand option
  - run remote commands
  - scp to/from nodes (i.e. fetch credentials from controller)

Added FuelAdapter as an example.

JIRA: RELENG-149
JIRA: RELENG-150

Change-Id: I49d8be96d754e0950e337aa2f88172341446fdd4
Signed-off-by: jose.lausuch <jose.lausuch@ericsson.com>

9 files changed, 606 insertions, 0 deletions

diff --git a/utils/installer-adapter/ApexAdapter.py b/utils/installer-adapter/ApexAdapter.py
new file mode 100644
index 000000000..bf451f3d2
--- /dev/null
+++ b/utils/installer-adapter/ApexAdapter.py
@@ -0,0 +1,35 @@
+##############################################################################
+# Copyright (c) 2016 Ericsson AB and others.
+# Author: Jose Lausuch (jose.lausuch@ericsson.com)
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+
+from SSHUtils import SSH_Connection
+
+
+class ApexAdapter:
+
+    def __init__(self, installer_ip):
+        self.installer_ip = installer_ip
+
+    def get_deployment_info(self):
+        pass
+
+    def get_nodes(self):
+        pass
+
+    def get_controller_ips(self):
+        pass
+
+    def get_compute_ips(self):
+        pass
+
+    def get_file_from_installer(self, origin, target, options=None):
+        pass
+
+    def get_file_from_controller(self, origin, target, ip=None, options=None):
+        pass
\ No newline at end of file
diff --git a/utils/installer-adapter/CompassAdapter.py b/utils/installer-adapter/CompassAdapter.py
new file mode 100644
index 000000000..b40a8d788
--- /dev/null
+++ b/utils/installer-adapter/CompassAdapter.py
@@ -0,0 +1,35 @@
+##############################################################################
+# Copyright (c) 2016 Ericsson AB and others.
+# Author: Jose Lausuch (jose.lausuch@ericsson.com)
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+
+from SSHUtils import SSH_Connection
+
+
+class CompassAdapter:
+
+    def __init__(self, installer_ip):
+        self.installer_ip = installer_ip
+
+    def get_deployment_info(self):
+        pass
+
+    def get_nodes(self):
+        pass
+
+    def get_controller_ips(self):
+        pass
+
+    def get_compute_ips(self):
+        pass
+
+    def get_file_from_installer(self, origin, target, options=None):
+        pass
+
+    def get_file_from_controller(self, origin, target, ip=None, options=None):
+        pass
\ No newline at end of file
diff --git a/utils/installer-adapter/FuelAdapter.py b/utils/installer-adapter/FuelAdapter.py
new file mode 100644
index 000000000..15f0e929f
--- /dev/null
+++ b/utils/installer-adapter/FuelAdapter.py
@@ -0,0 +1,219 @@
+##############################################################################
+# Copyright (c) 2016 Ericsson AB and others.
+# Author: Jose Lausuch (jose.lausuch@ericsson.com)
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+
+from SSHUtils import SSH_Connection
+import RelengLogger as rl
+
+
+class FuelAdapter:
+
+    def __init__(self, installer_ip, user="root", password="r00tme"):
+        self.installer_ip = installer_ip
+        self.user = user
+        self.password = password
+        self.connection = SSH_Connection(
+            installer_ip, self.user, self.password, use_system_keys=False)
+        self.logger = rl.Logger("Handler").getLogger()
+
+    def runcmd_fuel_nodes(self):
+        output, error = self.connection.run_remote_cmd('fuel nodes')
+        if len(error) > 0:
+            self.logger.error("error %s" % error)
+            return error
+        return output
+
+    def runcmd_fuel_env(self):
+        output, error = self.connection.run_remote_cmd('fuel env')
+        if len(error) > 0:
+            self.logger.error("error %s" % error)
+            return error
+        return output
+
+    def get_clusters(self):
+        environments = []
+        output = self.runcmd_fuel_env()
+        lines = output.rsplit('\n')
+        if len(lines) < 2:
+            self.logger.infp("No environments found in the deployment.")
+            return None
+        else:
+            fields = lines[0].rsplit(' | ')
+
+            index_id = -1
+            index_status = -1
+            index_name = -1
+            index_release_id = -1
+
+            for i in range(0, len(fields) - 1):
+                if "id" in fields[i]:
+                    index_id = i
+                elif "status" in fields[i]:
+                    index_status = i
+                elif "name" in fields[i]:
+                    index_name = i
+                elif "release_id" in fields[i]:
+                    index_release_id = i
+
+            # order env info
+            for i in range(2, len(lines) - 1):
+                fields = lines[i].rsplit(' | ')
+                dict = {"id": fields[index_id].strip(),
+                        "status": fields[index_status].strip(),
+                        "name": fields[index_name].strip(),
+                        "release_id": fields[index_release_id].strip()}
+                environments.append(dict)
+
+        return environments
+
+    def get_nodes(self, options=None):
+        nodes = []
+        output = self.runcmd_fuel_nodes()
+        lines = output.rsplit('\n')
+        if len(lines) < 2:
+            self.logger.info("No nodes found in the deployment.")
+            return None
+        else:
+            # get fields indexes
+            fields = lines[0].rsplit(' | ')
+
+            index_id = -1
+            index_status = -1
+            index_name = -1
+            index_cluster = -1
+            index_ip = -1
+            index_mac = -1
+            index_roles = -1
+            index_online = -1
+
+            for i in range(0, len(fields) - 1):
+                if "id" in fields[i]:
+                    index_id = i
+                elif "status" in fields[i]:
+                    index_status = i
+                elif "name" in fields[i]:
+                    index_name = i
+                elif "cluster" in fields[i]:
+                    index_cluster = i
+                elif "ip" in fields[i]:
+                    index_ip = i
+                elif "mac" in fields[i]:
+                    index_mac = i
+                elif "roles " in fields[i]:
+                    index_roles = i
+                elif "online" in fields[i]:
+                    index_online = i
+
+            # order nodes info
+            for i in range(2, len(lines) - 1):
+                fields = lines[i].rsplit(' | ')
+                dict = {"id": fields[index_id].strip(),
+                        "status": fields[index_status].strip(),
+                        "name": fields[index_name].strip(),
+                        "cluster": fields[index_cluster].strip(),
+                        "ip": fields[index_ip].strip(),
+                        "mac": fields[index_mac].strip(),
+                        "roles": fields[index_roles].strip(),
+                        "online": fields[index_online].strip()}
+                if options and options['cluster']:
+                    if fields[index_cluster].strip() == options['cluster']:
+                        nodes.append(dict)
+                else:
+                    nodes.append(dict)
+
+        return nodes
+
+    def get_controller_ips(self, options):
+        nodes = self.get_nodes(options=options)
+        controllers = []
+        for node in nodes:
+            if "controller" in node["roles"]:
+                controllers.append(node['ip'])
+        return controllers
+
+    def get_compute_ips(self, options=None):
+        nodes = self.get_nodes(options=options)
+        computes = []
+        for node in nodes:
+            if "compute" in node["roles"]:
+                computes.append(node['ip'])
+        return computes
+
+    def get_deployment_info(self):
+        str = "Deployment details:\n"
+        str += "\tInstaller:  Fuel\n"
+        str += "\tScenario:   Unknown\n"
+        sdn = "None"
+        clusters = self.get_clusters()
+        str += "\tN.Clusters: %s\n" % len(clusters)
+        for cluster in clusters:
+            cluster_dic = {'cluster': cluster['id']}
+            str += "\tCluster info:\n"
+            str += "\t   ID:          %s\n" % cluster['id']
+            str += "\t   NAME:        %s\n" % cluster['name']
+            str += "\t   STATUS:      %s\n" % cluster['status']
+            nodes = self.get_nodes(options=cluster_dic)
+            num_nodes = len(nodes)
+            for node in nodes:
+                if "opendaylight" in node['roles']:
+                    sdn = "OpenDaylight"
+                elif "onos" in node['roles']:
+                    sdn = "ONOS"
+            num_controllers = len(
+                self.get_controller_ips(options=cluster_dic))
+            num_computes = len(self.get_compute_ips(options=cluster_dic))
+            ha = False
+            if num_controllers > 1:
+                ha = True
+
+            str += "\t   HA:          %s\n" % ha
+            str += "\t   NUM.NODES:   %s\n" % num_nodes
+            str += "\t   CONTROLLERS: %s\n" % num_controllers
+            str += "\t   COMPUTES:    %s\n" % num_computes
+            str += "\t   SDN CONTR.:  %s\n\n" % sdn
+        str += self.runcmd_fuel_nodes()
+        return str
+
+    def get_file_from_installer(self, remote_path, local_path, options=None):
+        self.logger.debug("Fetching %s from %s" %
+                          (remote_path, self.installer_ip))
+        if self.connection.scp_get(local_path, remote_path) != 0:
+            self.logger.error("SCP failed to retrieve the file.")
+            return 1
+        self.logger.info("%s successfully copied from Fuel to %s" %
+                         (remote_path, local_path))
+
+    def get_file_from_controller(self,
+                                 remote_path,
+                                 local_path,
+                                 ip=None,
+                                 options=None):
+        if ip is None:
+            controllers = self.get_controller_ips(options=options)
+            if len(controllers) == 0:
+                self.logger.info("No controllers found in the deployment.")
+                return 1
+            else:
+                target_ip = controllers[0]
+        else:
+            target_ip = ip
+
+        fuel_dir = '/root/scp/'
+        cmd = 'mkdir -p %s;rsync -Rav %s:%s %s' % (
+            fuel_dir, target_ip, remote_path, fuel_dir)
+        self.logger.info("Copying %s from %s to Fuel..." %
+                         (remote_path, target_ip))
+        output, error = self.connection.run_remote_cmd(cmd)
+        self.logger.debug("Copying files from Fuel to %s..." % local_path)
+        self.get_file_from_installer(
+            fuel_dir + remote_path, local_path, options)
+        cmd = 'rm -r %s' % fuel_dir
+        output, error = self.connection.run_remote_cmd(cmd)
+        self.logger.info("%s successfully copied from %s to %s" %
+                         (remote_path, target_ip, local_path))
diff --git a/utils/installer-adapter/InstallerHandler.py b/utils/installer-adapter/InstallerHandler.py
new file mode 100644
index 000000000..b81b806ca
--- /dev/null
+++ b/utils/installer-adapter/InstallerHandler.py
@@ -0,0 +1,78 @@
+##############################################################################
+# Copyright (c) 2015 Ericsson AB and others.
+# Author: Jose Lausuch (jose.lausuch@ericsson.com)
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+from FuelAdapter import FuelAdapter
+from ApexAdapter import ApexAdapter
+from CompassAdapter import CompassAdapter
+from JoidAdapter import JoidAdapter
+
+
+INSTALLERS = ["fuel", "apex", "compass", "joid"]
+
+
+class InstallerHandler:
+
+    def __init__(self,
+                 installer,
+                 installer_ip,
+                 installer_user,
+                 installer_pwd=None):
+        self.installer = installer.lower()
+        self.installer_ip = installer_ip
+        self.installer_user = installer_user
+        self.installer_pwd = installer_pwd
+
+        if self.installer == INSTALLERS[0]:
+            self.InstallerAdapter = FuelAdapter(self.installer_ip,
+                                                self.installer_user,
+                                                self.installer_pwd)
+        elif self.installer == INSTALLERS[1]:
+            self.InstallerAdapter = ApexAdapter(self.installer_ip)
+        elif self.installer == INSTALLERS[2]:
+            self.InstallerAdapter = CompassAdapter(self.installer_ip)
+        elif self.installer == INSTALLERS[3]:
+            self.InstallerAdapter = JoidAdapter(self.installer_ip)
+        else:
+            print("Installer %s is  not valid. "
+                  "Please use one of the followings: %s"
+                  % (self.installer, INSTALLERS))
+            exit(1)
+
+    def get_deployment_info(self):
+        return self.InstallerAdapter.get_deployment_info()
+
+    def get_nodes(self, options=None):
+        return self.InstallerAdapter.get_nodes(options=options)
+
+    def get_controller_ips(self, options=None):
+        return self.InstallerAdapter.get_controller_ips(options=options)
+
+    def get_compute_ips(self, options=None):
+        return self.InstallerAdapter.get_compute_ips(options=options)
+
+    def get_file_from_installer(self,
+                                remote_path,
+                                local_path,
+                                options=None):
+        return self.InstallerAdapter.get_file_from_installer(remote_path,
+                                                             local_path,
+                                                             options=options)
+
+    def get_file_from_controller(self,
+                                 remote_path,
+                                 local_path,
+                                 ip=None,
+                                 options=None):
+        return self.InstallerAdapter.get_file_from_controller(remote_path,
+                                                              local_path,
+                                                              ip=ip,
+                                                              options=options)
+
+    def get_all(self):
+        pass
diff --git a/utils/installer-adapter/JoidAdapter.py b/utils/installer-adapter/JoidAdapter.py
new file mode 100644
index 000000000..e78ca0fae
--- /dev/null
+++ b/utils/installer-adapter/JoidAdapter.py
@@ -0,0 +1,35 @@
+##############################################################################
+# Copyright (c) 2016 Ericsson AB and others.
+# Author: Jose Lausuch (jose.lausuch@ericsson.com)
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+
+from SSHUtils import SSH_Connection
+
+
+class JoidAdapter:
+
+    def __init__(self, installer_ip):
+        self.installer_ip = installer_ip
+
+    def get_deployment_info(self):
+        pass
+
+    def get_nodes(self):
+        pass
+
+    def get_controller_ips(self):
+        pass
+
+    def get_compute_ips(self):
+        pass
+
+    def get_file_from_installer(self, origin, target, options=None):
+        pass
+
+    def get_file_from_controller(self, origin, target, ip=None, options=None):
+        pass
\ No newline at end of file
diff --git a/utils/installer-adapter/RelengLogger.py b/utils/installer-adapter/RelengLogger.py
new file mode 100644
index 000000000..b38e78095
--- /dev/null
+++ b/utils/installer-adapter/RelengLogger.py
@@ -0,0 +1,52 @@
+#!/usr/bin/env python
+#
+# jose.lausuch@ericsson.com
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Logging levels:
+#  Level     Numeric value
+#  CRITICAL  50
+#  ERROR     40
+#  WARNING   30
+#  INFO      20
+#  DEBUG     10
+#  NOTSET    0
+#
+# Usage:
+#  import RelengLogger as rl
+#  logger = fl.Logger("script_name").getLogger()
+#  logger.info("message to be shown with - INFO - ")
+#  logger.debug("message to be shown with - DEBUG -")
+
+import logging
+import os
+
+
+class Logger:
+
+    def __init__(self, logger_name, level="INFO"):
+
+        self.logger = logging.getLogger(logger_name)
+        self.logger.propagate = 0
+        self.logger.setLevel(logging.DEBUG)
+
+        ch = logging.StreamHandler()
+        formatter = logging.Formatter('%(asctime)s - %(name)s - '
+                                      '%(levelname)s - %(message)s')
+        ch.setFormatter(formatter)
+        if level.lower() == "debug":
+            ch.setLevel(logging.DEBUG)
+        else:
+            ch.setLevel(logging.INFO)
+        self.logger.addHandler(ch)
+
+        hdlr = logging.FileHandler('/tmp/releng.log')
+        hdlr.setFormatter(formatter)
+        hdlr.setLevel(logging.DEBUG)
+        self.logger.addHandler(hdlr)
+
+    def getLogger(self):
+        return self.logger
diff --git a/utils/installer-adapter/SSHUtils.py b/utils/installer-adapter/SSHUtils.py
new file mode 100644
index 000000000..9c92a3be1
--- /dev/null
+++ b/utils/installer-adapter/SSHUtils.py
@@ -0,0 +1,130 @@
+##############################################################################
+# Copyright (c) 2015 Ericsson AB and others.
+# Author: Jose Lausuch (jose.lausuch@ericsson.com)
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+
+import paramiko
+from scp import SCPClient
+import time
+import RelengLogger as rl
+
+
+class SSH_Connection:
+
+    def __init__(self,
+                 host,
+                 user,
+                 password,
+                 use_system_keys=True,
+                 private_key=None,
+                 use_proxy=False,
+                 proxy_host=None,
+                 proxy_user=None,
+                 proxy_password=None,
+                 timeout=10):
+        self.host = host
+        self.user = user
+        self.password = password
+        self.use_system_keys = use_system_keys
+        self.private_key = private_key
+        self.use_proxy = use_proxy
+        self.proxy_host = proxy_host
+        self.proxy_user = proxy_user
+        self.proxy_password = proxy_password
+        self.timeout = timeout
+        paramiko.util.log_to_file("paramiko.log")
+        self.logger = rl.Logger("SSHUtils").getLogger()
+
+    def connect(self):
+        client = paramiko.SSHClient()
+        if self.use_system_keys:
+            client.load_system_host_keys()
+        elif self.private_key:
+            client.load_host_keys(self.private_key)
+        else:
+            client.load_host_keys('/dev/null')
+
+        client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+
+        t = self.timeout
+        proxy = None
+        if self.use_proxy:
+            proxy_command = 'ssh -o UserKnownHostsFile=/dev/null '
+            '-o StrictHostKeyChecking=no %s@%s -W %s:%s' % (self.proxy_user,
+                                                            self.proxy_host,
+                                                            self.host, 22)
+            proxy = paramiko.ProxyCommand(proxy_command)
+            self.logger.debug("Proxy command: %s" % proxy_command)
+        while t > 0:
+            try:
+                self.logger.debug(
+                    "Trying to stablish ssh connection to %s..." % self.host)
+                client.connect(self.host,
+                               username=self.user,
+                               password=self.password,
+                               look_for_keys=True,
+                               sock=proxy,
+                               pkey=self.private_key,
+                               timeout=self.timeout)
+                self.logger.debug("Successfully connected to %s!" % self.host)
+                return client
+            except:
+                time.sleep(1)
+                t -= 1
+
+        if t == 0:
+            return None
+
+    def scp_put(self, local_path, remote_path):
+        client = self.connect()
+        if client:
+            scp = SCPClient(client.get_transport())
+            try:
+                scp.put(local_path, remote_path)
+                client.close()
+                return 0
+            except Exception, e:
+                self.logger.error(e)
+                client.close()
+                return 1
+        else:
+            self.logger.error("Cannot stablish ssh connection.")
+
+    def scp_get(self, local_path, remote_path):
+        client = self.connect()
+        if client:
+            scp = SCPClient(client.get_transport())
+            try:
+                scp.get(remote_path, local_path)
+                client.close()
+                return 0
+            except Exception, e:
+                self.logger.error(e)
+                client.close()
+                return 1
+        else:
+            self.logger.error("Cannot stablish ssh connection.")
+            return 1
+
+    def run_remote_cmd(self, command):
+        client = self.connect()
+        if client:
+            try:
+                stdin, stdout, stderr = client.exec_command(command)
+                out = ''
+                for line in stdout.readlines():
+                    out += line
+                err = stderr.readlines()
+                client.close()
+                return out, err
+            except:
+                client.close()
+                return 1
+        else:
+            self.logger.error("Cannot stablish ssh connection.")
+            return 1
diff --git a/utils/installer-adapter/__init__.py b/utils/installer-adapter/__init__.py
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/utils/installer-adapter/__init__.py
diff --git a/utils/installer-adapter/example.py b/utils/installer-adapter/example.py
new file mode 100644
index 000000000..804d79c3d
--- /dev/null
+++ b/utils/installer-adapter/example.py
@@ -0,0 +1,22 @@
+# This is an example of usage of this Tool
+# Author: Jose Lausuch (jose.lausuch@ericsson.com)
+
+from InstallerHandler import InstallerHandler
+
+fuel_handler = InstallerHandler(installer='fuel',
+                                installer_ip='10.20.0.2',
+                                installer_user='root',
+                                installer_pwd='r00tme')
+print("Nodes in cluster 1:\n%s\n" %
+      fuel_handler.get_nodes(options={'cluster': '1'}))
+print("Nodes in cluster 2:\n%s\n" %
+      fuel_handler.get_nodes(options={'cluster': '2'}))
+print("Nodes:\n%s\n" % fuel_handler.get_nodes())
+print("Controller nodes:\n%s\n" % fuel_handler.get_controller_ips())
+print("Compute nodes:\n%s\n" % fuel_handler.get_compute_ips())
+print("\n%s\n" % fuel_handler.get_deployment_info())
+fuel_handler.get_file_from_installer('/root/deploy/dea.yaml', './dea.yaml')
+fuel_handler.get_file_from_controller(
+    '/etc/neutron/neutron.conf', './neutron.conf')
+fuel_handler.get_file_from_controller(
+    '/root/openrc', './openrc')