Migrates Apex to Python

Removes all bash libraries and converts almost all of the code to a
mixture of Python and Ansible.  utils.sh and clean.sh still exist.
clean.sh will be migrated fully to clean.py in another patch.

The Apex Python package is now built into the opnfv-apex-common RPM.  To
install locally do 'pip3 install .'.  To deploy:

opnfv-deploy -d <file> -n <file> --image-dir /root/apex/.build  -v --debug

Non-python files (THT yaml, settings files, ansible playbooks) are all
installed into /usr/share/opnfv-apex/.  The RPM will copy settings files
into /etc/opnfv-apex/.

JIRA: APEX-317

Change-Id: I3232f0329bcd13bce5a28da6a8c9c84d0b048024
Signed-off-by: Tim Rozet <trozet@redhat.com>

3 files changed, 515 insertions, 0 deletions

diff --git a/apex/settings/__init__.py b/apex/settings/__init__.py
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/apex/settings/__init__.py
diff --git a/apex/settings/deploy_settings.py b/apex/settings/deploy_settings.py
new file mode 100644
index 00000000..c8e347b7
--- /dev/null
+++ b/apex/settings/deploy_settings.py
@@ -0,0 +1,188 @@
+##############################################################################
+# Copyright (c) 2016 Michael Chapman (michapma@redhat.com) and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+
+import yaml
+
+from apex.common import utils
+from apex.common import constants
+
+REQ_DEPLOY_SETTINGS = ['sdn_controller',
+                       'odl_version',
+                       'tacker',
+                       'congress',
+                       'dataplane',
+                       'sfc',
+                       'vpn',
+                       'vpp',
+                       'ceph',
+                       'gluon',
+                       'rt_kvm']
+
+OPT_DEPLOY_SETTINGS = ['performance',
+                       'vsperf',
+                       'ceph_device',
+                       'yardstick',
+                       'dovetail',
+                       'odl_vpp_routing_node',
+                       'odl_vpp_netvirt',
+                       'barometer']
+
+VALID_ROLES = ['Controller', 'Compute', 'ObjectStorage']
+VALID_PERF_OPTS = ['kernel', 'nova', 'vpp', 'ovs']
+VALID_DATAPLANES = ['ovs', 'ovs_dpdk', 'fdio']
+VALID_ODL_VERSIONS = ['carbon', 'nitrogen', 'master']
+
+
+class DeploySettings(dict):
+    """
+    This class parses a APEX deploy settings yaml file into an object
+
+    Currently the parsed object is dumped into a bash global definition file
+    for deploy.sh consumption. This object will later be used directly as
+    deployment script move to python.
+    """
+    def __init__(self, filename):
+        if isinstance(filename, str):
+            with open(filename, 'r') as deploy_settings_file:
+                init_dict = yaml.safe_load(deploy_settings_file)
+        else:
+            # assume input is a dict to build from
+            init_dict = filename
+
+        super().__init__(init_dict)
+        self._validate_settings()
+
+    def _validate_settings(self):
+        """
+        Validates the deploy settings file provided
+
+        DeploySettingsException will be raised if validation fails.
+        """
+
+        if 'deploy_options' not in self:
+            raise DeploySettingsException("No deploy options provided in"
+                                          " deploy settings file")
+        if 'global_params' not in self:
+            raise DeploySettingsException("No global options provided in"
+                                          " deploy settings file")
+
+        deploy_options = self['deploy_options']
+        if not isinstance(deploy_options, dict):
+            raise DeploySettingsException("deploy_options should be a list")
+
+        if ('gluon' in self['deploy_options'] and
+           'vpn' in self['deploy_options']):
+                if (self['deploy_options']['gluon'] is True and
+                   self['deploy_options']['vpn'] is False):
+                        raise DeploySettingsException(
+                            "Invalid deployment configuration: "
+                            "If gluon is enabled, "
+                            "vpn also needs to be enabled")
+
+        for setting, value in deploy_options.items():
+            if setting not in REQ_DEPLOY_SETTINGS + OPT_DEPLOY_SETTINGS:
+                raise DeploySettingsException("Invalid deploy_option {} "
+                                              "specified".format(setting))
+            if setting == 'dataplane':
+                if value not in VALID_DATAPLANES:
+                    planes = ' '.join(VALID_DATAPLANES)
+                    raise DeploySettingsException(
+                        "Invalid dataplane {} specified. Valid dataplanes:"
+                        " {}".format(value, planes))
+
+        for req_set in REQ_DEPLOY_SETTINGS:
+            if req_set not in deploy_options:
+                if req_set == 'dataplane':
+                    self['deploy_options'][req_set] = 'ovs'
+                elif req_set == 'ceph':
+                    self['deploy_options'][req_set] = True
+                elif req_set == 'odl_version':
+                    self['deploy_options'][req_set] = \
+                        constants.DEFAULT_ODL_VERSION
+                else:
+                    self['deploy_options'][req_set] = False
+            elif req_set == 'odl_version' and self['deploy_options'][
+                    'odl_version'] not in VALID_ODL_VERSIONS:
+                raise DeploySettingsException(
+                    "Invalid ODL version: {}".format(self[deploy_options][
+                        'odl_version']))
+
+        if 'performance' in deploy_options:
+            if not isinstance(deploy_options['performance'], dict):
+                raise DeploySettingsException("Performance deploy_option"
+                                              "must be a dictionary.")
+            for role, role_perf_sets in deploy_options['performance'].items():
+                if role not in VALID_ROLES:
+                    raise DeploySettingsException("Performance role {}"
+                                                  "is not valid, choose"
+                                                  "from {}".format(
+                                                      role,
+                                                      " ".join(VALID_ROLES)
+                                                  ))
+
+                for key in role_perf_sets:
+                    if key not in VALID_PERF_OPTS:
+                        raise DeploySettingsException("Performance option {} "
+                                                      "is not valid, choose"
+                                                      "from {}".format(
+                                                          key,
+                                                          " ".join(
+                                                              VALID_PERF_OPTS)
+                                                      ))
+
+    def _dump_performance(self):
+        """
+        Creates performance settings string for bash consumption.
+
+        Output will be in the form of a list that can be iterated over in
+        bash, with each string being the direct input to the performance
+        setting script in the form <role> <category> <key> <value> to
+        facilitate modification of the correct image.
+        """
+        bash_str = 'performance_options=(\n'
+        deploy_options = self['deploy_options']
+        for role, settings in deploy_options['performance'].items():
+            for category, options in settings.items():
+                for key, value in options.items():
+                    bash_str += "\"{} {} {} {}\"\n".format(role,
+                                                           category,
+                                                           key,
+                                                           value)
+        bash_str += ')\n'
+        bash_str += '\n'
+        bash_str += 'performance_roles=(\n'
+        for role in self['deploy_options']['performance']:
+            bash_str += role + '\n'
+        bash_str += ')\n'
+        bash_str += '\n'
+
+        return bash_str
+
+    def _dump_deploy_options_array(self):
+        """
+        Creates deploy settings array in bash syntax.
+        """
+        bash_str = ''
+        for key, value in self['deploy_options'].items():
+            if not isinstance(value, bool):
+                bash_str += "deploy_options_array[{}]=\"{}\"\n".format(key,
+                                                                       value)
+            else:
+                bash_str += "deploy_options_array[{}]={}\n".format(key,
+                                                                   value)
+        return bash_str
+
+
+class DeploySettingsException(Exception):
+    def __init__(self, value):
+        self.value = value
+
+    def __str__(self):
+        return self.value
diff --git a/apex/settings/network_settings.py b/apex/settings/network_settings.py
new file mode 100644
index 00000000..14870078
--- /dev/null
+++ b/apex/settings/network_settings.py
@@ -0,0 +1,327 @@
+##############################################################################
+# Copyright (c) 2016 Feng Pan (fpan@redhat.com) and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+import ipaddress
+import logging
+from copy import copy
+
+import yaml
+
+from apex.common import utils
+from apex.common.constants import (
+    CONTROLLER,
+    COMPUTE,
+    ROLES,
+    DOMAIN_NAME,
+    DNS_SERVERS,
+    NTP_SERVER,
+    ADMIN_NETWORK,
+    EXTERNAL_NETWORK,
+    OPNFV_NETWORK_TYPES,
+)
+from apex.network import ip_utils
+
+
+class NetworkSettings(dict):
+    """
+    This class parses APEX network settings yaml file into an object. It
+    generates or detects all missing fields for deployment.
+
+    The resulting object will be used later to generate network environment
+    file as well as configuring post deployment networks.
+
+    Currently the parsed object is dumped into a bash global definition file
+    for deploy.sh consumption. This object will later be used directly as
+    deployment script move to python.
+    """
+    def __init__(self, filename):
+        init_dict = {}
+        if isinstance(filename, str):
+            with open(filename, 'r') as network_settings_file:
+                init_dict = yaml.safe_load(network_settings_file)
+        else:
+            # assume input is a dict to build from
+            init_dict = filename
+        super().__init__(init_dict)
+
+        if 'apex' in self:
+            # merge two dicts Non-destructively
+            def merge(pri, sec):
+                for key, val in sec.items():
+                    if key in pri:
+                        if isinstance(val, dict):
+                            merge(pri[key], val)
+                        # else
+                        # do not overwrite what's already there
+                    else:
+                        pri[key] = val
+            # merge the apex specific config into the first class settings
+            merge(self, copy(self['apex']))
+
+        self.enabled_network_list = []
+        self.nics = {COMPUTE: {}, CONTROLLER: {}}
+        self.nics_specified = {COMPUTE: False, CONTROLLER: False}
+        self._validate_input()
+
+    def get_network(self, network):
+        if network == EXTERNAL_NETWORK and self['networks'][network]:
+            for net in self['networks'][network]:
+                if 'public' in net:
+                    return net
+
+            raise NetworkSettingsException("The external network, "
+                                           "'public', should be defined "
+                                           "when external networks are "
+                                           "enabled")
+        else:
+            return self['networks'][network]
+
+    def _validate_input(self):
+        """
+        Validates the network settings file and populates all fields.
+
+        NetworkSettingsException will be raised if validation fails.
+        """
+        if not self['networks'].get(ADMIN_NETWORK, {}).get('enabled', False):
+            raise NetworkSettingsException("You must enable admin network "
+                                           "and configure it explicitly or "
+                                           "use auto-detection")
+
+        for network in OPNFV_NETWORK_TYPES:
+            if network in self['networks']:
+                _network = self.get_network(network)
+                if _network.get('enabled', True):
+                    logging.info("{} enabled".format(network))
+                    self._config_required_settings(network)
+                    nicmap = _network['nic_mapping']
+                    self._validate_overcloud_nic_order(network)
+                    iface = nicmap[CONTROLLER]['members'][0]
+                    self._config_ip_range(network=network,
+                                          interface=iface,
+                                          ip_range='overcloud_ip_range',
+                                          start_offset=21, end_offset=21)
+                    self.enabled_network_list.append(network)
+                    # TODO self._config_optional_settings(network)
+                else:
+                    logging.info("{} disabled, will collapse with "
+                                 "admin network".format(network))
+            else:
+                logging.info("{} is not in specified, will collapse with "
+                             "admin network".format(network))
+
+        if 'dns-domain' not in self:
+            self['domain_name'] = DOMAIN_NAME
+        else:
+            self['domain_name'] = self['dns-domain']
+        self['dns_servers'] = self.get('dns_nameservers', DNS_SERVERS)
+        self['ntp_servers'] = self.get('ntp', NTP_SERVER)
+
+    def _validate_overcloud_nic_order(self, network):
+        """
+        Detects if nic order is specified per profile (compute/controller)
+        for network
+
+        If nic order is specified in a network for a profile, it should be
+        specified for every network with that profile other than admin network
+
+        Duplicate nic names are also not allowed across different networks
+
+        :param network: network to detect if nic order present
+        :return: None
+        """
+        for role in ROLES:
+            _network = self.get_network(network)
+            _nicmap = _network.get('nic_mapping', {})
+            _role = _nicmap.get(role, {})
+            interfaces = _role.get('members', [])
+
+            if interfaces:
+                interface = interfaces[0]
+                if not isinstance(_role.get('vlan', 'native'), int) and \
+                   any(y == interface for x, y in self.nics[role].items()):
+                    raise NetworkSettingsException(
+                        "Duplicate {} already specified for "
+                        "another network".format(interface))
+                self.nics[role][network] = interface
+                self.nics_specified[role] = True
+                logging.info("{} nic order specified for network {"
+                             "}".format(role, network))
+            else:
+                raise NetworkSettingsException(
+                    "Interface members are not supplied for {} network "
+                    "for the {} role. Please add nic assignments"
+                    "".format(network, role))
+
+    def _config_required_settings(self, network):
+        """
+        Configures either CIDR or bridged_interface setting
+
+        cidr takes precedence if both cidr and bridged_interface are specified
+        for a given network.
+
+        When using bridged_interface, we will detect network setting on the
+        given NIC in the system. The resulting config in settings object will
+        be an ipaddress.network object, replacing the NIC name.
+        """
+        _network = self.get_network(network)
+        # if vlan not defined then default it to native
+        if network is not ADMIN_NETWORK:
+            for role in ROLES:
+                if 'vlan' not in _network['nic_mapping'][role]:
+                    _network['nic_mapping'][role]['vlan'] = 'native'
+
+        cidr = _network.get('cidr')
+
+        if cidr:
+            cidr = ipaddress.ip_network(_network['cidr'])
+            _network['cidr'] = cidr
+            logging.info("{}_cidr: {}".format(network, cidr))
+        elif 'installer_vm' in _network:
+            ucloud_if_list = _network['installer_vm']['members']
+            # If cidr is not specified, we need to know if we should find
+            # IPv6 or IPv4 address on the interface
+            ip = ipaddress.ip_address(_network['installer_vm']['ip'])
+            nic_if = ip_utils.get_interface(ucloud_if_list[0], ip.version)
+            if nic_if:
+                logging.info("{}_bridged_interface: {}".
+                             format(network, nic_if))
+            else:
+                raise NetworkSettingsException(
+                    "Auto detection failed for {}: Unable to find valid "
+                    "ip for interface {}".format(network, ucloud_if_list[0]))
+
+        else:
+            raise NetworkSettingsException(
+                "Auto detection failed for {}: either installer_vm "
+                "members or cidr must be specified".format(network))
+
+        # undercloud settings
+        if network == ADMIN_NETWORK:
+            provisioner_ip = _network['installer_vm']['ip']
+            iface = _network['installer_vm']['members'][0]
+            if not provisioner_ip:
+                _network['installer_vm']['ip'] = self._gen_ip(network, 1)
+            self._config_ip_range(network=network, interface=iface,
+                                  ip_range='dhcp_range',
+                                  start_offset=2, count=9)
+            self._config_ip_range(network=network, interface=iface,
+                                  ip_range='introspection_range',
+                                  start_offset=11, count=9)
+        elif network == EXTERNAL_NETWORK:
+            provisioner_ip = _network['installer_vm']['ip']
+            iface = _network['installer_vm']['members'][0]
+            if not provisioner_ip:
+                _network['installer_vm']['ip'] = self._gen_ip(network, 1)
+            self._config_ip_range(network=network, interface=iface,
+                                  ip_range='floating_ip_range',
+                                  end_offset=2, count=20)
+
+            gateway = _network['gateway']
+            interface = _network['installer_vm']['ip']
+            self._config_gateway(network, gateway, interface)
+
+    def _config_ip_range(self, network, ip_range, interface=None,
+                         start_offset=None, end_offset=None, count=None):
+        """
+        Configures IP range for a given setting.
+        If the setting is already specified, no change will be made.
+        The spec for start_offset, end_offset and count are identical to
+        ip_utils.get_ip_range.
+        """
+        _network = self.get_network(network)
+        if ip_range not in _network:
+            cidr = _network.get('cidr')
+            _ip_range = ip_utils.get_ip_range(start_offset=start_offset,
+                                              end_offset=end_offset,
+                                              count=count,
+                                              cidr=cidr,
+                                              interface=interface)
+            _network[ip_range] = _ip_range.split(',')
+
+        logging.info("Config IP Range: {} {}".format(network, ip_range))
+
+    def _gen_ip(self, network, offset):
+        """
+        Generate and ip offset within the given network
+        """
+        _network = self.get_network(network)
+        cidr = _network.get('cidr')
+        ip = ip_utils.get_ip(offset, cidr)
+        logging.info("Config IP: {} {}".format(network, ip))
+        return ip
+
+    def _config_optional_settings(self, network):
+        """
+        Configures optional settings:
+        - admin_network:
+            - provisioner_ip
+            - dhcp_range
+            - introspection_range
+        - public_network:
+            - provisioner_ip
+            - floating_ip_range
+            - gateway
+        """
+        if network == ADMIN_NETWORK:
+            # FIXME: _config_ip  function does not exist!
+            self._config_ip(network, None, 'provisioner_ip', 1)
+            self._config_ip_range(network=network,
+                                  ip_range='dhcp_range',
+                                  start_offset=2, count=9)
+            self._config_ip_range(network=network,
+                                  ip_range='introspection_range',
+                                  start_offset=11, count=9)
+        elif network == EXTERNAL_NETWORK:
+            # FIXME: _config_ip  function does not exist!
+            self._config_ip(network, None, 'provisioner_ip', 1)
+            self._config_ip_range(network=network,
+                                  ip_range='floating_ip_range',
+                                  end_offset=2, count=20)
+            self._config_gateway(network)
+
+    def _config_gateway(self, network, gateway, interface):
+        """
+        Configures gateway setting for a given network.
+
+        If cidr is specified, we always use the first address in the address
+        space for gateway. Otherwise, we detect the system gateway.
+        """
+        _network = self.get_network(network)
+        if not gateway:
+            cidr = _network.get('cidr')
+            if cidr:
+                _gateway = ip_utils.get_ip(1, cidr)
+            else:
+                _gateway = ip_utils.find_gateway(interface)
+
+            if _gateway:
+                _network['gateway'] = _gateway
+            else:
+                raise NetworkSettingsException("Failed to set gateway")
+
+        logging.info("Config Gateway: {} {}".format(network, gateway))
+
+    def get_ip_addr_family(self,):
+        """
+        Returns IP address family for current deployment.
+
+        If any enabled network has IPv6 CIDR, the deployment is classified as
+        IPv6.
+        """
+        return max([
+            ipaddress.ip_network(self.get_network(n)['cidr']).version
+            for n in self.enabled_network_list])
+
+
+class NetworkSettingsException(Exception):
+    def __init__(self, value):
+        self.value = value
+
+    def __str__(self):
+            return self.value