Migrates Apex to Python

Removes all bash libraries and converts almost all of the code to a
mixture of Python and Ansible.  utils.sh and clean.sh still exist.
clean.sh will be migrated fully to clean.py in another patch.

The Apex Python package is now built into the opnfv-apex-common RPM.  To
install locally do 'pip3 install .'.  To deploy:

opnfv-deploy -d <file> -n <file> --image-dir /root/apex/.build  -v --debug

Non-python files (THT yaml, settings files, ansible playbooks) are all
installed into /usr/share/opnfv-apex/.  The RPM will copy settings files
into /etc/opnfv-apex/.

JIRA: APEX-317

Change-Id: I3232f0329bcd13bce5a28da6a8c9c84d0b048024
Signed-off-by: Tim Rozet <trozet@redhat.com>

5 files changed, 238 insertions, 0 deletions

diff --git a/apex/common/__init__.py b/apex/common/__init__.py
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/apex/common/__init__.py
diff --git a/apex/common/constants.py b/apex/common/constants.py
new file mode 100644
index 00000000..0df71526
--- /dev/null
+++ b/apex/common/constants.py
@@ -0,0 +1,46 @@
+##############################################################################
+# Copyright (c) 2016 Tim Rozet (trozet@redhat.com) and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+import os
+
+ADMIN_NETWORK = 'admin'
+TENANT_NETWORK = 'tenant'
+EXTERNAL_NETWORK = 'external'
+STORAGE_NETWORK = 'storage'
+API_NETWORK = 'api'
+CONTROLLER = 'controller'
+COMPUTE = 'compute'
+
+OPNFV_NETWORK_TYPES = [ADMIN_NETWORK, TENANT_NETWORK, EXTERNAL_NETWORK,
+                       STORAGE_NETWORK, API_NETWORK]
+DNS_SERVERS = ["8.8.8.8", "8.8.4.4"]
+NTP_SERVER = ["pool.ntp.org"]
+COMPUTE = 'compute'
+CONTROLLER = 'controller'
+ROLES = [COMPUTE, CONTROLLER]
+DOMAIN_NAME = 'localdomain.com'
+COMPUTE_PRE = "OS::TripleO::ComputeExtraConfigPre"
+CONTROLLER_PRE = "OS::TripleO::ControllerExtraConfigPre"
+PRE_CONFIG_DIR = "/usr/share/openstack-tripleo-heat-templates/puppet/" \
+                 "extraconfig/pre_deploy/"
+DEFAULT_ROOT_DEV = 'sda'
+LIBVIRT_VOLUME_PATH = '/var/lib/libvirt/images'
+
+VIRT_UPLOAD = '--upload'
+VIRT_INSTALL = '-install'
+VIRT_RUN_CMD = '--run-command'
+VIRT_PW = '--root-password'
+
+THT_DIR = '/usr/share/openstack-tripleo-heat-templates'
+THT_ENV_DIR = os.path.join(THT_DIR, 'environments')
+
+DEFAULT_ODL_VERSION = 'carbon'
+DEBUG_OVERCLOUD_PW = 'opnfvapex'
+NET_ENV_FILE = 'network-environment.yaml'
+DEPLOY_TIMEOUT = 90
diff --git a/apex/common/exceptions.py b/apex/common/exceptions.py
new file mode 100644
index 00000000..c660213f
--- /dev/null
+++ b/apex/common/exceptions.py
@@ -0,0 +1,12 @@
+##############################################################################
+# Copyright (c) 2017 Tim Rozet (trozet@redhat.com) and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+
+class ApexDeployException(Exception):
+    pass
diff --git a/apex/common/parsers.py b/apex/common/parsers.py
new file mode 100644
index 00000000..8744c862
--- /dev/null
+++ b/apex/common/parsers.py
@@ -0,0 +1,73 @@
+##############################################################################
+# Copyright (c) 2017 Tim Rozet (trozet@redhat.com) and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+import json
+import logging
+import pprint
+import os
+import re
+
+from apex.common.exceptions import ApexDeployException
+
+"""Parser functions for overcloud/openstack output"""
+
+
+def parse_nova_output(in_file):
+    """
+    Parses nova list output into a dictionary format for node name and ip
+    :param in_file: json format from openstack server list
+    :return: dictionary format for {"node name": "node ip"}
+    """
+    if not os.path.isfile(in_file):
+        raise FileNotFoundError(in_file)
+    node_dict = dict()
+    with open(in_file, 'r') as fh:
+        nova_list = json.load(fh)
+
+    for server in nova_list:
+        ip_match = re.search('([0-9]+\.){3}[0-9]+', server['Networks'])
+        if ip_match is None:
+            logging.error("Unable to find IP in nova output "
+                          "{}".format(pprint.pformat(server, indent=4)))
+            raise ApexDeployException("Unable to parse IP from nova output")
+        else:
+            node_dict[server['Name']] = ip_match.group(0)
+
+    if not node_dict:
+        raise ApexDeployException("No overcloud nodes found in: {}".format(
+            in_file))
+    return node_dict
+
+
+def parse_overcloudrc(in_file):
+    """
+    Parses overcloudrc into a dictionary format for key and value
+    :param in_file:
+    :return: dictionary format for {"variable": "value"}
+    """
+    logging.debug("Parsing overcloudrc file {}".format(in_file))
+    if not os.path.isfile(in_file):
+        raise FileNotFoundError(in_file)
+    creds = {}
+    with open(in_file, 'r') as fh:
+        lines = fh.readlines()
+    kv_pattern = re.compile('^export\s+([^\s]+)=([^\s]+)$')
+    for line in lines:
+        if 'export' not in line:
+            continue
+        else:
+            res = re.search(kv_pattern, line.strip())
+            if res:
+                creds[res.group(1)] = res.group(2)
+                logging.debug("os cred found: {}, {}".format(res.group(1),
+                                                             res.group(2)))
+            else:
+                logging.debug("os cred not found in: {}".format(line))
+
+    return creds
diff --git a/apex/common/utils.py b/apex/common/utils.py
new file mode 100644
index 00000000..848f2644
--- /dev/null
+++ b/apex/common/utils.py
@@ -0,0 +1,107 @@
+##############################################################################
+# Copyright (c) 2016 Tim Rozet (trozet@redhat.com) and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+import json
+import logging
+import os
+import pprint
+import subprocess
+import yaml
+
+
+def str2bool(var):
+    if isinstance(var, bool):
+        return var
+    else:
+        return var.lower() in ("true", "yes")
+
+
+def parse_yaml(yaml_file):
+    with open(yaml_file) as f:
+        parsed_dict = yaml.safe_load(f)
+        return parsed_dict
+
+
+def dump_yaml(data, file):
+    """
+    Dumps data to a file as yaml
+    :param data: yaml to be written to file
+    :param file: filename to write to
+    :return:
+    """
+    logging.debug("Writing file {} with "
+                  "yaml data:\n{}".format(file, yaml.safe_dump(data)))
+    with open(file, "w") as fh:
+        yaml.safe_dump(data, fh, default_flow_style=False)
+
+
+def dict_objects_to_str(dictionary):
+        if isinstance(dictionary, list):
+            tmp_list = []
+            for element in dictionary:
+                if isinstance(element, dict):
+                    tmp_list.append(dict_objects_to_str(element))
+                else:
+                    tmp_list.append(str(element))
+            return tmp_list
+        elif not isinstance(dictionary, dict):
+            if not isinstance(dictionary, bool):
+                return str(dictionary)
+            else:
+                return dictionary
+        return dict((k, dict_objects_to_str(v)) for
+                    k, v in dictionary.items())
+
+
+def run_ansible(ansible_vars, playbook, host='localhost', user='root',
+                tmp_dir=None, dry_run=False):
+    """
+    Executes ansible playbook and checks for errors
+    :param ansible_vars: dictionary of variables to inject into ansible run
+    :param playbook: playbook to execute
+    :param tmp_dir: temp directory to store ansible command
+    :param dry_run: Do not actually apply changes
+    :return: None
+    """
+    logging.info("Executing ansible playbook: {}".format(playbook))
+    inv_host = "{},".format(host)
+    if host == 'localhost':
+        conn_type = 'local'
+    else:
+        conn_type = 'smart'
+    ansible_command = ['ansible-playbook', '--become', '-i', inv_host,
+                       '-u', user, '-c', conn_type, playbook, '-vvv']
+    if dry_run:
+        ansible_command.append('--check')
+
+    if isinstance(ansible_vars, dict) and ansible_vars:
+        logging.debug("Ansible variables to be set:\n{}".format(
+            pprint.pformat(ansible_vars)))
+        ansible_command.append('--extra-vars')
+        ansible_command.append(json.dumps(ansible_vars))
+        if tmp_dir:
+            ansible_tmp = os.path.join(tmp_dir,
+                                       os.path.basename(playbook) + '.rerun')
+            # FIXME(trozet): extra vars are printed without single quotes
+            # so a dev has to add them manually to the command to rerun
+            # the playbook.  Need to test if we can just add the single quotes
+            # to the json dumps to the ansible command and see if that works
+            with open(ansible_tmp, 'w') as fh:
+                fh.write("ANSIBLE_HOST_KEY_CHECKING=FALSE {}".format(
+                    ' '.join(ansible_command)))
+    try:
+        my_env = os.environ.copy()
+        my_env['ANSIBLE_HOST_KEY_CHECKING'] = 'False'
+        logging.info("Executing playbook...this may take some time")
+        logging.debug(subprocess.check_output(ansible_command, env=my_env,
+                      stderr=subprocess.STDOUT).decode('utf-8'))
+    except subprocess.CalledProcessError as e:
+        logging.error("Error executing ansible: {}".format(
+            pprint.pformat(e.output.decode('utf-8'))))
+        raise