Merge "Migrates network environment parsing to python"

10 files changed, 232 insertions, 79 deletions

diff --git a/build/opnfv-apex-common.spec b/build/opnfv-apex-common.spec
index 6915b795..464bae91 100644
--- a/build/opnfv-apex-common.spec
+++ b/build/opnfv-apex-common.spec
@@ -47,10 +47,7 @@ install lib/common-functions.sh %{buildroot}%{_var}/opt/opnfv/lib/
 install lib/utility-functions.sh %{buildroot}%{_var}/opt/opnfv/lib/
 install lib/python/apex-python-utils.py %{buildroot}%{_var}/opt/opnfv/lib/python/
 mkdir -p %{buildroot}%{python3_sitelib}/apex/
-install lib/python/apex/__init__.py %{buildroot}%{python3_sitelib}/apex/
-install lib/python/apex/ip_utils.py %{buildroot}%{python3_sitelib}/apex/
-install lib/python/apex/net_env.py %{buildroot}%{python3_sitelib}/apex/
-install lib/python/apex/deploy_env.py %{buildroot}%{python3_sitelib}/apex/
+install lib/python/apex/ %{buildroot}%{python3_sitelib}/apex/
 mkdir -p %{buildroot}%{_var}/opt/opnfv/lib/installer/onos/
 install lib/installer/onos/onos_gw_mac_update.sh %{buildroot}%{_var}/opt/opnfv/lib/installer/onos/
 
diff --git a/ci/deploy.sh b/ci/deploy.sh
index bb4c1f81..a40b92a2 100755
--- a/ci/deploy.sh
+++ b/ci/deploy.sh
@@ -115,7 +115,7 @@ parse_setting_value() {
 ##parses network settings yaml into globals
 parse_network_settings() {
   local output
-  if output=$(python3.4 -B $LIB/python/apex-python-utils.py parse-net-settings -f $NETSETS -i $net_isolation_enabled); then
+  if output=$(python3.4 -B $LIB/python/apex-python-utils.py parse-net-settings -s $NETSETS -i $net_isolation_enabled -e $CONFIG/network-environment.yaml); then
       echo -e "${blue}${output}${reset}"
       eval "$output"
   else
@@ -609,53 +609,6 @@ function define_vm () {
                                               --baremetal-interface $4
 }
 
-##Set network-environment settings
-##params: network-environment file to edit
-function configure_network_environment {
-  local tht_dir
-  tht_dir=/usr/share/openstack-tripleo-heat-templates/network
-
-  sed -i '/ControlPlaneSubnetCidr/c\\  ControlPlaneSubnetCidr: "'${admin_network_cidr##*/}'"' $1
-  sed -i '/ControlPlaneDefaultRoute/c\\  ControlPlaneDefaultRoute: '${admin_network_provisioner_ip}'' $1
-  sed -i '/ExternalNetCidr/c\\  ExternalNetCidr: '${public_network_cidr}'' $1
-  sed -i "/ExternalAllocationPools/c\\  ExternalAllocationPools: [{'start': '${public_network_usable_ip_range%%,*}', 'end': '${public_network_usable_ip_range##*,}'}]" $1
-  sed -i '/ExternalInterfaceDefaultRoute/c\\  ExternalInterfaceDefaultRoute: '${public_network_gateway}'' $1
-  sed -i '/EC2MetadataIp/c\\  EC2MetadataIp: '${admin_network_provisioner_ip}'' $1
-
-  # check for private network
-  if [[ ! -z "$private_network_enabled" && "$private_network_enabled" == "True" ]]; then
-      sed -i 's#^.*Network::Tenant.*$#  OS::TripleO::Network::Tenant: '${tht_dir}'/tenant.yaml#' $1
-      sed -i 's#^.*Controller::Ports::TenantPort:.*$#  OS::TripleO::Controller::Ports::TenantPort: '${tht_dir}'/ports/tenant.yaml#' $1
-      sed -i 's#^.*Compute::Ports::TenantPort:.*$#  OS::TripleO::Compute::Ports::TenantPort: '${tht_dir}'/ports/tenant.yaml#' $1
-      sed -i "/TenantAllocationPools/c\\  TenantAllocationPools: [{'start': '${private_network_usable_ip_range%%,*}', 'end': '${private_network_usable_ip_range##*,}'}]" $1
-      sed -i '/TenantNetCidr/c\\  TenantNetCidr: '${private_network_cidr}'' $1
-  else
-      sed -i 's#^.*Network::Tenant.*$#  OS::TripleO::Network::Tenant: '${tht_dir}'/noop.yaml#' $1
-      sed -i 's#^.*Controller::Ports::TenantPort:.*$#  OS::TripleO::Controller::Ports::TenantPort: '${tht_dir}'/ports/noop.yaml#' $1
-      sed -i 's#^.*Compute::Ports::TenantPort:.*$#  OS::TripleO::Compute::Ports::TenantPort: '${tht_dir}'/ports/noop.yaml#' $1
-  fi
-
-  # check for storage network
-  if [[ ! -z "$storage_network_enabled" && "$storage_network_enabled" == "True" ]]; then
-      sed -i 's#^.*Network::Storage:.*$#  OS::TripleO::Network::Storage: '${tht_dir}'/storage.yaml#' $1
-      sed -i 's#^.*Network::Ports::StorageVipPort:.*$#  OS::TripleO::Network::Ports::StorageVipPort: '${tht_dir}'/ports/storage.yaml#' $1
-      sed -i 's#^.*Controller::Ports::StoragePort:.*$#  OS::TripleO::Controller::Ports::StoragePort: '${tht_dir}'/ports/storage.yaml#' $1
-      sed -i 's#^.*Compute::Ports::StoragePort:.*$#  OS::TripleO::Compute::Ports::StoragePort: '${tht_dir}'/ports/storage.yaml#' $1
-      sed -i "/StorageAllocationPools/c\\  StorageAllocationPools: [{'start': '${storage_network_usable_ip_range%%,*}', 'end': '${storage_network_usable_ip_range##*,}'}]" $1
-      sed -i '/StorageNetCidr/c\\  StorageNetCidr: '${storage_network_cidr}'' $1
-  else
-      sed -i 's#^.*Network::Storage:.*$#  OS::TripleO::Network::Storage: '${tht_dir}'/noop.yaml#' $1
-      sed -i 's#^.*Network::Ports::StorageVipPort:.*$#  OS::TripleO::Network::Ports::StorageVipPort: '${tht_dir}'/ports/noop.yaml#' $1
-      sed -i 's#^.*Controller::Ports::StoragePort:.*$#  OS::TripleO::Controller::Ports::StoragePort: '${tht_dir}'/ports/noop.yaml#' $1
-      sed -i 's#^.*Compute::Ports::StoragePort:.*$#  OS::TripleO::Compute::Ports::StoragePort: '${tht_dir}'/ports/noop.yaml#' $1
-  fi
-
-  # check for ODL L3
-  if [ "${deploy_options_array['sdn_l3']}" == 'True' ]; then
-      ext_net_type=br-ex
-  fi
-
-}
 ##Copy over the glance images and instackenv json file
 ##params: none
 function configure_undercloud {
@@ -663,10 +616,15 @@ function configure_undercloud {
   echo
   echo "Copying configuration files to Undercloud"
   if [[ "$net_isolation_enabled" == "TRUE" ]]; then
-    configure_network_environment $CONFIG/network-environment.yaml
     echo -e "${blue}Network Environment set for Deployment: ${reset}"
-    cat $CONFIG/network-environment.yaml
-    scp ${SSH_OPTIONS[@]} $CONFIG/network-environment.yaml "stack@$UNDERCLOUD":
+    cat /tmp/network-environment.yaml
+    scp ${SSH_OPTIONS[@]} /tmp/network-environment.yaml "stack@$UNDERCLOUD":
+
+    # check for ODL L3/ONOS
+    if [ "${deploy_options_array['sdn_l3']}" == 'True' ]; then
+      ext_net_type=br-ex
+    fi
+
     if ! controller_nic_template=$(python3.4 -B $LIB/python/apex-python-utils.py nic-template -t $CONFIG/nics-controller.yaml.jinja2 -n "$enabled_network_list" -e $ext_net_type -af $ip_addr_family); then
       echo -e "${red}ERROR: Failed to generate controller NIC heat template ${reset}"
       exit 1
diff --git a/lib/python/apex-python-utils.py b/lib/python/apex-python-utils.py
index 4eee4f75..6c76cb13 100755
--- a/lib/python/apex-python-utils.py
+++ b/lib/python/apex-python-utils.py
@@ -12,6 +12,7 @@ import sys
 import apex
 import logging
 import os
+import yaml
 from jinja2 import Environment, FileSystemLoader
 
 
@@ -26,10 +27,24 @@ def parse_net_settings(args):
     - network_isolation: bool
       enable or disable network_isolation
     """
-    settings = apex.NetworkSettings(args.file,
+    settings = apex.NetworkSettings(args.net_settings_file,
                                     args.network_isolation)
+    net_env = apex.NetworkEnvironment(settings, args.net_env_file)
+    dump_yaml(net_env.get_netenv_settings(), '/tmp/network-environment.yaml')
     settings.dump_bash()
 
+
+def dump_yaml(data, file):
+    """
+    Dumps data to a file as yaml
+    :param data: yaml to be written to file
+    :param file: filename to write to
+    :return:
+    """
+    with open(file, "w") as fh:
+        yaml.dump(data, fh, default_flow_style=False)
+
+
 def parse_deploy_settings(args):
     settings = apex.DeploySettings(args.file)
     settings.dump_bash()
@@ -84,11 +99,17 @@ def parse_args():
 
     net_settings = subparsers.add_parser('parse-net-settings',
                                          help='Parse network settings file')
-    net_settings.add_argument('-f', '--file', default='network-settings.yaml',
+    net_settings.add_argument('-s', '--net-settings-file',
+                              default='network-settings.yaml',
+                              dest='net_settings_file',
                               help='path to network settings file')
     net_settings.add_argument('-i', '--network-isolation', type=bool,
                               default=True, dest='network_isolation',
                               help='network isolation')
+    net_settings.add_argument('-e', '--net-env-file',
+                              default="network-environment.yaml",
+                              dest='net_env_file',
+                              help='path to network environment file')
     net_settings.set_defaults(func=parse_net_settings)
 
     get_int_ip = subparsers.add_parser('find-ip',
diff --git a/lib/python/apex/__init__.py b/lib/python/apex/__init__.py
index 2efc64f4..5b158501 100644
--- a/lib/python/apex/__init__.py
+++ b/lib/python/apex/__init__.py
@@ -8,5 +8,6 @@
 ##############################################################################
 
 
-from .net_env import NetworkSettings
+from .network_settings import NetworkSettings
 from .deploy_env import DeploySettings
+from .network_environment import NetworkEnvironment
diff --git a/lib/python/apex/common/__init__.py b/lib/python/apex/common/__init__.py
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/lib/python/apex/common/__init__.py
diff --git a/lib/python/apex/common/constants.py b/lib/python/apex/common/constants.py
new file mode 100644
index 00000000..506b7b2b
--- /dev/null
+++ b/lib/python/apex/common/constants.py
@@ -0,0 +1,16 @@
+##############################################################################
+# Copyright (c) 2016 Tim Rozet (trozet@redhat.com) and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+ADMIN_NETWORK = 'admin_network'
+PRIVATE_NETWORK = 'private_network'
+PUBLIC_NETWORK = 'public_network'
+STORAGE_NETWORK = 'storage_network'
+API_NETWORK = 'api_network'
+OPNFV_NETWORK_TYPES = [ADMIN_NETWORK, PRIVATE_NETWORK, PUBLIC_NETWORK,
+                       STORAGE_NETWORK, API_NETWORK]
diff --git a/lib/python/apex/common/utils.py b/lib/python/apex/common/utils.py
new file mode 100644
index 00000000..b7678a20
--- /dev/null
+++ b/lib/python/apex/common/utils.py
@@ -0,0 +1,15 @@
+##############################################################################
+# Copyright (c) 2016 Tim Rozet (trozet@redhat.com) and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+
+def str2bool(var):
+    if isinstance(var, bool):
+        return var
+    else:
+        return var.lower() in ("true", "yes")
diff --git a/lib/python/apex/network_environment.py b/lib/python/apex/network_environment.py
new file mode 100644
index 00000000..e6f0135a
--- /dev/null
+++ b/lib/python/apex/network_environment.py
@@ -0,0 +1,135 @@
+##############################################################################
+# Copyright (c) 2016 Tim Rozet (trozet@redhat.com) and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+import yaml
+import re
+from .common import constants
+
+PORTS = '/ports'
+# Resources defined by <resource name>: <prefix>
+TENANT_RESOURCES = {'OS::TripleO::Network::Tenant': None,
+                    'OS::TripleO::Controller::Ports::TenantPort': PORTS,
+                    'OS::TripleO::Compute::Ports::TenantPort': PORTS}
+STORAGE_RESOURCES = {'OS::TripleO::Network::Storage': None,
+                     'OS::TripleO::Network::Ports::StorageVipPort': PORTS,
+                     'OS::TripleO::Controller::Ports::StoragePort': PORTS,
+                     'OS::TripleO::Compute::Ports::StoragePort': PORTS}
+
+
+class NetworkEnvironment:
+    """
+    This class creates a Network Environment to be used in TripleO Heat
+    Templates.
+
+    The class builds upon an existing network-environment file and modifies
+    based on a NetworkSettings object.
+    """
+    def __init__(self, net_settings, filename):
+        with open(filename, 'r') as net_env_fh:
+            self.netenv_obj = yaml.load(net_env_fh)
+            if net_settings:
+                settings_obj = net_settings.get_network_settings()
+                enabled_networks = net_settings.get_enabled_networks()
+                self.netenv_obj = \
+                    self._update_net_environment(settings_obj,
+                                                 enabled_networks)
+            else:
+                raise NetworkEnvException("Network Settings does not exist")
+
+    def _update_net_environment(self, net_settings, enabled_networks):
+        """
+        Updates Network Environment according to Network Settings
+        :param: network settings dictionary
+        :param: enabled network list
+        :return:  None
+        """
+        param_def = 'parameter_defaults'
+        reg = 'resource_registry'
+        for key, prefix in TENANT_RESOURCES.items():
+            if prefix is None:
+                prefix = ''
+            m = re.split('%s/\w+\.yaml' % prefix, self.netenv_obj[reg][key])
+            if m is not None:
+                tht_dir = m[0]
+                break
+        if not tht_dir:
+            raise NetworkEnvException('Unable to parse THT Directory')
+        admin_cidr = net_settings[constants.ADMIN_NETWORK]['cidr']
+        admin_prefix = str(admin_cidr.prefixlen)
+        self.netenv_obj[param_def]['ControlPlaneSubnetCidr'] = admin_prefix
+        self.netenv_obj[param_def]['ControlPlaneDefaultRoute'] = \
+            net_settings[constants.ADMIN_NETWORK]['provisioner_ip']
+        public_cidr = net_settings[constants.PUBLIC_NETWORK]['cidr']
+        self.netenv_obj[param_def]['ExternalNetCidr'] = str(public_cidr)
+        public_range = net_settings[constants.PUBLIC_NETWORK][
+                                         'usable_ip_range'].split(',')
+        self.netenv_obj[param_def]['ExternalAllocationPools'] = \
+            [{'start':
+              public_range[0],
+              'end': public_range[1]
+              }]
+        self.netenv_obj[param_def]['ExternalInterfaceDefaultRoute'] = \
+            net_settings[constants.PUBLIC_NETWORK]['gateway']
+        self.netenv_obj[param_def]['EC2MetadataIp'] = \
+            net_settings[constants.ADMIN_NETWORK]['provisioner_ip']
+
+        if constants.PRIVATE_NETWORK in enabled_networks:
+            priv_range = net_settings[constants.PRIVATE_NETWORK][
+                'usable_ip_range'].split(',')
+            self.netenv_obj[param_def]['TenantAllocationPools'] = \
+                [{'start':
+                  priv_range[0],
+                  'end': priv_range[1]
+                  }]
+            priv_cidr = net_settings[constants.PRIVATE_NETWORK]['cidr']
+            self.netenv_obj[param_def]['TenantNetCidr'] = str(priv_cidr)
+            postfix = '/tenant.yaml'
+        else:
+            postfix = '/noop.yaml'
+
+        for key, prefix in TENANT_RESOURCES.items():
+            if prefix is None:
+                prefix = ''
+            self.netenv_obj[reg][key] = tht_dir + prefix + postfix
+
+        if constants.STORAGE_NETWORK in enabled_networks:
+            storage_range = net_settings[constants.STORAGE_NETWORK][
+                'usable_ip_range'].split(',')
+            self.netenv_obj[param_def]['StorageAllocationPools'] = \
+                [{'start':
+                  storage_range[0],
+                  'end':
+                  storage_range[1]
+                  }]
+            storage_cidr = net_settings[constants.STORAGE_NETWORK]['cidr']
+            self.netenv_obj[param_def]['StorageNetCidr'] = str(storage_cidr)
+            postfix = '/storage.yaml'
+        else:
+            postfix = '/noop.yaml'
+
+        for key, prefix in STORAGE_RESOURCES.items():
+            if prefix is None:
+                prefix = ''
+            self.netenv_obj[reg][key] = tht_dir + prefix + postfix
+        return self.netenv_obj
+
+    def get_netenv_settings(self):
+        """
+        Getter for netenv settings
+        :return: Dictionary of network environment settings
+        """
+        return self.netenv_obj
+
+
+class NetworkEnvException(Exception):
+    def __init__(self, value):
+        self.value = value
+
+    def __str__(self):
+            return self.value
diff --git a/lib/python/apex/net_env.py b/lib/python/apex/network_settings.py
index 3ca28f8a..ea7f4ca9 100644
--- a/lib/python/apex/net_env.py
+++ b/lib/python/apex/network_settings.py
@@ -7,20 +7,11 @@
 # http://www.apache.org/licenses/LICENSE-2.0
 ##############################################################################
 
-
 import yaml
 import logging
 import ipaddress
 from . import ip_utils
-
-
-ADMIN_NETWORK = 'admin_network'
-PRIVATE_NETWORK = 'private_network'
-PUBLIC_NETWORK = 'public_network'
-STORAGE_NETWORK = 'storage_network'
-API_NETWORK = 'api_network'
-OPNFV_NETWORK_TYPES = [ADMIN_NETWORK, PRIVATE_NETWORK, PUBLIC_NETWORK,
-                       STORAGE_NETWORK, API_NETWORK]
+from .common import constants, utils
 
 
 class NetworkSettings:
@@ -48,21 +39,23 @@ class NetworkSettings:
 
         NetworkSettingsException will be raised if validation fails.
         """
-        if ADMIN_NETWORK not in self.settings_obj or \
-                self.settings_obj[ADMIN_NETWORK].get('enabled') != True:
+        if constants.ADMIN_NETWORK not in self.settings_obj or \
+            not utils.str2bool(self.settings_obj[constants.ADMIN_NETWORK].get(
+                    'enabled')):
             raise NetworkSettingsException("You must enable admin_network "
                                            "and configure it explicitly or "
                                            "use auto-detection")
         if self.network_isolation and \
-            (PUBLIC_NETWORK not in self.settings_obj or
-                self.settings_obj[PUBLIC_NETWORK].get('enabled') != True):
+            (constants.PUBLIC_NETWORK not in self.settings_obj or not
+                utils.str2bool(self.settings_obj[constants.PUBLIC_NETWORK].get(
+                    'enabled'))):
             raise NetworkSettingsException("You must enable public_network "
                                            "and configure it explicitly or "
                                            "use auto-detection")
 
-        for network in OPNFV_NETWORK_TYPES:
+        for network in constants.OPNFV_NETWORK_TYPES:
             if network in self.settings_obj:
-                if self.settings_obj[network].get('enabled') == True:
+                if utils.str2bool(self.settings_obj[network].get('enabled')):
                     logging.info("{} enabled".format(network))
                     self._config_required_settings(network)
                     self._config_ip_range(network=network,
@@ -99,7 +92,7 @@ class NetworkSettings:
         elif nic_name:
             # If cidr is not specified, we need to know if we should find
             # IPv6 or IPv4 address on the interface
-            if self.settings_obj[network].get('ipv6') == True:
+            if utils.str2bool(self.settings_obj[network].get('ipv6')):
                 address_family = 6
             else:
                 address_family = 4
@@ -175,14 +168,14 @@ class NetworkSettings:
             - floating_ip
             - gateway
         """
-        if network == ADMIN_NETWORK:
+        if network == constants.ADMIN_NETWORK:
             self._config_ip(network, 'provisioner_ip', 1)
             self._config_ip_range(network=network, setting='dhcp_range',
                                   start_offset=2, count=9)
             self._config_ip_range(network=network,
                                   setting='introspection_range',
                                   start_offset=11, count=9)
-        elif network == PUBLIC_NETWORK:
+        elif network == constants.PUBLIC_NETWORK:
             self._config_ip(network, 'provisioner_ip', 1)
             self._config_ip_range(network=network,
                                   setting='floating_ip',
@@ -213,7 +206,6 @@ class NetworkSettings:
 
         logging.info("{}_gateway: {}".format(network, gateway))
 
-
     def dump_bash(self, path=None):
         """
         Prints settings for bash consumption.
@@ -248,9 +240,27 @@ class NetworkSettings:
 
         return 4
 
+    def get_network_settings(self):
+        """
+        Getter for network settings
+        :return: network settings dictionary
+        """
+        return self.settings_obj
+
+    def get_enabled_networks(self):
+        """
+        Getter for enabled network list
+        :return: list of enabled networks
+        """
+        return self.enabled_network_list
+
+
 class NetworkSettingsException(Exception):
     def __init__(self, value):
         self.value = value
 
     def __str__(self):
             return self.value
+
+
+
diff --git a/tests/python-coverage.sh b/tests/python-coverage.sh
index 2814363e..ee471446 100755
--- a/tests/python-coverage.sh
+++ b/tests/python-coverage.sh
@@ -7,7 +7,7 @@ coverage-3.4 run ../lib/python/apex-python-utils.py -l /dev/null > /dev/null
 
 # exercise parse-net-settings
 # throw debug on the first to exercise it
-coverage-3.4 run -a ../lib/python/apex-python-utils.py --debug parse-net-settings -f ../config/network/network_settings.yaml -i True > /dev/null
+coverage-3.4 run -a ../lib/python/apex-python-utils.py --debug parse-net-settings -s ../config/network/network_settings.yaml -i True -e ../build/network-environment.yaml> /dev/null
 
 # exercise proper nic-template runs
 coverage-3.4 run -a ../lib/python/apex-python-utils.py -l /dev/null nic-template -t ../config/network/network_settings.yaml -n "$all_networks" -e interface -af 4 > /dev/null