Change NIC template format

Convert compute and controller NIC templates to jinja2 format.
Added support for IPv6 for those NIC templates. Existing IPv4
template generation behavior does not change.

The .template files are kept in tree for now, they will be
removed after VLAN changes are made.

Change-Id: I2ffc7c403af3659db780ece2bd9195cd62748f0c
Signed-off-by: Feng Pan <fpan@redhat.com>

6 files changed, 380 insertions, 7 deletions

diff --git a/build/nics-compute.yaml.jinja2 b/build/nics-compute.yaml.jinja2
new file mode 100644
index 00000000..12d90596
--- /dev/null
+++ b/build/nics-compute.yaml.jinja2
@@ -0,0 +1,170 @@
+heat_template_version: 2015-04-30
+
+description: >
+  Software Config to drive os-net-config to configure multiple interfaces
+  for the compute role.
+
+parameters:
+  ControlPlaneIp:
+    default: ''
+    description: IP address/subnet on the ctlplane network
+    type: string
+  ExternalIpSubnet:
+    default: ''
+    description: IP address/subnet on the external network
+    type: string
+  InternalApiIpSubnet:
+    default: ''
+    description: IP address/subnet on the internal API network
+    type: string
+  StorageIpSubnet:
+    default: ''
+    description: IP address/subnet on the storage network
+    type: string
+  StorageMgmtIpSubnet:
+    default: ''
+    description: IP address/subnet on the storage mgmt network
+    type: string
+  StorageMgmtNetworkVlanID:
+    default: 40
+    description: Vlan ID for the storage mgmt network traffic.
+    type: number
+  TenantIpSubnet:
+    default: ''
+    description: IP address/subnet on the tenant network
+    type: string
+  ManagementIpSubnet: # Only populated when including environments/network-management.yaml
+    default: ''
+    description: IP address/subnet on the management network
+    type: string
+  ExternalNetworkVlanID:
+    default: 10
+    description: Vlan ID for the external network traffic.
+    type: number
+  InternalApiNetworkVlanID:
+    default: 20
+    description: Vlan ID for the internal_api network traffic.
+    type: number
+  StorageNetworkVlanID:
+    default: 30
+    description: Vlan ID for the storage network traffic.
+    type: number
+  TenantNetworkVlanID:
+    default: 50
+    description: Vlan ID for the tenant network traffic.
+    type: number
+  ManagementNetworkVlanID:
+    default: 60
+    description: Vlan ID for the management network traffic.
+    type: number
+  ExternalInterfaceDefaultRoute:
+    default: '10.0.0.1'
+    description: default route for the external network
+    type: string
+  ControlPlaneSubnetCidr: # Override this via parameter_defaults
+    default: '24'
+    description: The subnet CIDR of the control plane network.
+    type: string
+  ControlPlaneDefaultRoute: # Override this via parameter_defaults
+    description: The default route of the control plane network.
+    type: string
+  DnsServers: # Override this via parameter_defaults
+    default: []
+    description: A list of DNS servers (2 max for some implementations) that will be added to resolv.conf.
+    type: comma_delimited_list
+  EC2MetadataIp: # Override this via parameter_defaults
+    description: The IP address of the EC2 metadata server.
+    type: string
+
+resources:
+  OsNetConfigImpl:
+    type: OS::Heat::StructuredConfig
+    properties:
+      group: os-apply-config
+      config:
+        os_net_config:
+          network_config:
+            -
+              type: interface
+              name: nic1
+              use_dhcp: false
+              dns_servers: {get_param: DnsServers}
+              addresses:
+                -
+                  ip_netmask:
+                    list_join:
+                      - '/'
+                      - - {get_param: ControlPlaneIp}
+                        - {get_param: ControlPlaneSubnetCidr}
+              routes:
+                -
+                  ip_netmask: 169.254.169.254/32
+                  next_hop: {get_param: EC2MetadataIp}
+                -
+                  default: true
+                  next_hop: {get_param: ControlPlaneDefaultRoute}
+            {%- set nic_index = 2 %}
+            {%- if 'private_network' in enabled_networks %}
+            -
+              # Create a bridge which can also be used for VLAN-mode bridge mapping
+              type: ovs_bridge
+              name: br-tenant
+              use_dhcp: false
+              addresses:
+                -
+                  ip_netmask: {get_param: TenantIpSubnet}
+              members:
+                -
+                  type: interface
+                  name: nic{{ nic_index }}{% set nic_index = nic_index + 1 %}
+                  use_dhcp: false
+                  # force the MAC address of the bridge to this interface
+                  primary: true
+            {%- endif %}
+            {%- if external_net_type == "interface" %}
+            -
+              type: interface
+              name: nic{{ nic_index }}{% set nic_index = nic_index + 1 %}
+              use_dhcp: false
+              addresses:
+                -
+                  ip_netmask: {get_param: ExternalIpSubnet}
+              routes:
+                -
+                  ip_netmask: 0.0.0.0/0
+                  next_hop: {get_param: ExternalInterfaceDefaultRoute}
+            {%- else %}
+            -
+              type: ovs_bridge
+              name: {get_input: bridge_name}
+              use_dhcp: false
+              members:
+                -
+                  type: interface
+                  name: nic{{ nic_index }}{% set nic_index = nic_index + 1 %}
+                  # force the MAC address of the bridge to this interface
+                  primary: true
+            {%- endif %}
+            {%- if 'storage_network' in enabled_networks %}
+            -
+              type: interface
+              name: nic{{ nic_index }}{% set nic_index = nic_index + 1 %}
+              use_dhcp: false
+              addresses:
+                -
+                  ip_netmask: {get_param: StorageIpSubnet}
+            {%- endif %}
+            {%- if 'api_network' in enabled_networks %}
+            -
+              type: interface
+              name: nic{{ nic_index }}{% set nic_index = nic_index + 1 %}
+              use_dhcp: false
+              addresses:
+                -
+                  ip_netmask: {get_param: InternalApiIpSubnet}
+            {%- endif %}
+
+outputs:
+  OS::stack_id:
+    description: The OsNetConfigImpl resource.
+    value: {get_resource: OsNetConfigImpl}
diff --git a/build/nics-controller.yaml.jinja2 b/build/nics-controller.yaml.jinja2
new file mode 100644
index 00000000..d135e2e5
--- /dev/null
+++ b/build/nics-controller.yaml.jinja2
@@ -0,0 +1,162 @@
+heat_template_version: 2015-04-30
+
+description: >
+  Software Config to drive os-net-config to configure multiple interfaces
+  for the controller role.
+
+parameters:
+  ControlPlaneIp:
+    default: ''
+    description: IP address/subnet on the ctlplane network
+    type: string
+  ExternalIpSubnet:
+    default: ''
+    description: IP address/subnet on the external network
+    type: string
+  InternalApiIpSubnet:
+    default: ''
+    description: IP address/subnet on the internal API network
+    type: string
+  StorageIpSubnet:
+    default: ''
+    description: IP address/subnet on the storage network
+    type: string
+  StorageMgmtIpSubnet:
+    default: ''
+    description: IP address/subnet on the storage mgmt network
+    type: string
+  TenantIpSubnet:
+    default: ''
+    description: IP address/subnet on the tenant network
+    type: string
+  ManagementIpSubnet: # Only populated when including environments/network-management.yaml
+    default: ''
+    description: IP address/subnet on the management network
+    type: string
+  ExternalNetworkVlanID:
+    default: 10
+    description: Vlan ID for the external network traffic.
+    type: number
+  InternalApiNetworkVlanID:
+    default: 20
+    description: Vlan ID for the internal_api network traffic.
+    type: number
+  StorageNetworkVlanID:
+    default: 30
+    description: Vlan ID for the storage network traffic.
+    type: number
+  StorageMgmtNetworkVlanID:
+    default: 40
+    description: Vlan ID for the storage mgmt network traffic.
+    type: number
+  TenantNetworkVlanID:
+    default: 50
+    description: Vlan ID for the tenant network traffic.
+    type: number
+  ExternalInterfaceDefaultRoute:
+    default: '10.0.0.1'
+    description: default route for the external network
+    type: string
+  ControlPlaneSubnetCidr: # Override this via parameter_defaults
+    default: '24'
+    description: The subnet CIDR of the control plane network.
+    type: string
+  ControlPlaneDefaultRoute: # Override this via parameter_defaults
+    description: The subnet CIDR of the control plane network.
+    type: string
+  DnsServers: # Override this via parameter_defaults
+    default: []
+    description: A list of DNS servers (2 max for some implementations) that will be added to resolv.conf.
+    type: comma_delimited_list
+  EC2MetadataIp: # Override this via parameter_defaults
+    description: The IP address of the EC2 metadata server.
+    type: string
+
+resources:
+  OsNetConfigImpl:
+    type: OS::Heat::StructuredConfig
+    properties:
+      group: os-apply-config
+      config:
+        os_net_config:
+          network_config:
+            -
+              type: interface
+              name: nic1
+              use_dhcp: false
+              dns_servers: {get_param: DnsServers}
+              addresses:
+                -
+                  ip_netmask:
+                    list_join:
+                      - '/'
+                      - - {get_param: ControlPlaneIp}
+                        - {get_param: ControlPlaneSubnetCidr}
+              routes:
+                -
+                  ip_netmask: 169.254.169.254/32
+                  next_hop: {get_param: EC2MetadataIp}
+                {%- if external_net_af == 6 %}
+                -
+                  default: true
+                  next_hop: {get_param: ControlPlaneDefaultRoute}
+                {%- endif %}
+            {%- set nic_index = 2 %}
+            {%- if 'private_network' in enabled_networks %}
+            -
+              # Create a bridge which can also be used for VLAN-mode bridge mapping
+              type: ovs_bridge
+              name: br-tenant
+              use_dhcp: false
+              addresses:
+                -
+                  ip_netmask: {get_param: TenantIpSubnet}
+              members:
+                -
+                  type: interface
+                  name: nic{{ nic_index }}{% set nic_index = nic_index + 1 %}
+                  use_dhcp: false
+                  # force the MAC address of the bridge to this interface
+                  primary: true
+            {%- endif %}
+            -
+              type: ovs_bridge
+              name: {get_input: bridge_name}
+              dns_servers: {get_param: DnsServers}
+              use_dhcp: false
+              addresses:
+                -
+                  ip_netmask: {get_param: ExternalIpSubnet}
+              routes:
+                -
+                  default: true
+                  next_hop: {get_param: ExternalInterfaceDefaultRoute}
+              members:
+                -
+                  type: interface
+                  name: nic{{ nic_index }}{% set nic_index = nic_index + 1 %}
+                  # force the MAC address of the bridge to this interface
+                  primary: true
+            {%- if 'storage_network' in enabled_networks %}
+            -
+              type: interface
+              name: nic{{ nic_index }}{% set nic_index = nic_index + 1 %}
+              use_dhcp: false
+              addresses:
+                -
+                  ip_netmask: {get_param: StorageIpSubnet}
+            {%- endif %}
+            {%- if 'api_network' in enabled_networks %}
+            -
+              type: interface
+              name: nic{{ nic_index }}{% set nic_index = nic_index + 1 %}
+              use_dhcp: false
+              addresses:
+                -
+                  ip_netmask: {get_param: InternalApiIpSubnet}
+            {%- endif %}
+
+outputs:
+  OS::stack_id:
+    description: The OsNetConfigImpl resource.
+    value: {get_resource: OsNetConfigImpl}
diff --git a/build/opnfv-apex-common.spec b/build/opnfv-apex-common.spec
index 56f0f305..8048dea8 100644
--- a/build/opnfv-apex-common.spec
+++ b/build/opnfv-apex-common.spec
@@ -11,7 +11,7 @@ Source0:	opnfv-apex-common.tar.gz
 BuildArch:	noarch
 BuildRequires:	python-docutils python34-devel
 Requires:	openstack-tripleo opnfv-apex-sdn opnfv-apex-undercloud openvswitch qemu-kvm bridge-utils libguestfs-tools
-Requires:	initscripts net-tools iputils iproute iptables python34 python34-yaml
+Requires:	initscripts net-tools iputils iproute iptables python34 python34-yaml python34-setuptools
 
 %description
 Scripts for OPNFV deployment using RDO Manager
@@ -90,6 +90,8 @@ install config/inventory/pod_example_settings.yaml %{buildroot}%{_docdir}/opnfv/
 %doc %{_docdir}/opnfv/inventory.yaml.example
 
 %changelog
+* Sun May 8 2016 Feng Pan <fpan@redhat.com> - 3.0-4
+- Adds dependency for python34-setuptools
 * Fri Apr 22 2016 Feng Pan <fpan@redhat.com> - 3.0-3
 - Adds python network setting parsing lib.
 * Fri Apr 15 2016 Feng Pan <fpan@redhat.com> - 3.0-2
diff --git a/ci/deploy.sh b/ci/deploy.sh
index cd76558c..f096e4d6 100755
--- a/ci/deploy.sh
+++ b/ci/deploy.sh
@@ -29,7 +29,6 @@ ntp_server="pool.ntp.org"
 net_isolation_enabled="TRUE"
 post_config="TRUE"
 debug="FALSE"
-nics_cfg=''
 
 declare -i CNT
 declare UNDERCLOUD
@@ -46,6 +45,8 @@ NET_MAP['admin_network']="br-admin"
 NET_MAP['private_network']="br-private"
 NET_MAP['public_network']="br-public"
 NET_MAP['storage_network']="br-storage"
+ext_net_type="interface"
+ip_address_family=4
 
 ##FUNCTIONS
 ##translates yaml into variables
@@ -579,7 +580,6 @@ function configure_network_environment {
       sed -i 's#^.*Compute::Ports::TenantPort:.*$#  OS::TripleO::Compute::Ports::TenantPort: '${tht_dir}'/ports/tenant.yaml#' $1
       sed -i "/TenantAllocationPools/c\\  TenantAllocationPools: [{'start': '${private_network_usable_ip_range%%,*}', 'end': '${private_network_usable_ip_range##*,}'}]" $1
       sed -i '/TenantNetCidr/c\\  TenantNetCidr: '${private_network_cidr}'' $1
-      nics_cfg+=_private
   else
       sed -i 's#^.*Network::Tenant.*$#  OS::TripleO::Network::Tenant: '${tht_dir}'/noop.yaml#' $1
       sed -i 's#^.*Controller::Ports::TenantPort:.*$#  OS::TripleO::Controller::Ports::TenantPort: '${tht_dir}'/ports/noop.yaml#' $1
@@ -594,7 +594,6 @@ function configure_network_environment {
       sed -i 's#^.*Compute::Ports::StoragePort:.*$#  OS::TripleO::Compute::Ports::StoragePort: '${tht_dir}'/ports/storage.yaml#' $1
       sed -i "/StorageAllocationPools/c\\  StorageAllocationPools: [{'start': '${storage_network_usable_ip_range%%,*}', 'end': '${storage_network_usable_ip_range##*,}'}]" $1
       sed -i '/StorageNetCidr/c\\  StorageNetCidr: '${storage_network_cidr}'' $1
-      nics_cfg+=_storage
   else
       sed -i 's#^.*Network::Storage:.*$#  OS::TripleO::Network::Storage: '${tht_dir}'/noop.yaml#' $1
       sed -i 's#^.*Network::Ports::StorageVipPort:.*$#  OS::TripleO::Network::Ports::StorageVipPort: '${tht_dir}'/ports/noop.yaml#' $1
@@ -604,7 +603,7 @@ function configure_network_environment {
 
   # check for ODL L3
   if [ "${deploy_options_array['sdn_l3']}" == 'true' ]; then
-      nics_cfg+=_br-ex_no-public-ip
+      ext_net_type=br-ex
   fi
 
 }
@@ -622,10 +621,10 @@ function configure_undercloud {
     ssh -T ${SSH_OPTIONS[@]} "stack@$UNDERCLOUD" << EOI
 mkdir nics/
 cat > nics/controller.yaml << EOF
-$(nics_cfg=$nics_cfg sh $CONFIG/nics-controller.yaml.template)
+$(python3.4 -B $CONFIG/lib/python/apex-python-utils.py nic_template -d $CONFIG -f nics-controller.yaml.jinja2 -n "$enabled_network_list" -e $ext_net_type -af $ip_addr_family)
 EOF
 cat > nics/compute.yaml << EOF
-$(nics_cfg=$nics_cfg sh $CONFIG/nics-compute.yaml.template)
+$(python3.4 -B $CONFIG/lib/python/apex-python-utils.py nic_template -d $CONFIG -f nics-compute.yaml.jinja2 -n "$enabled_network_list" -e $ext_net_type -af $ip_addr_family)
 EOF
 EOI
   fi
@@ -1126,6 +1125,8 @@ parse_cmdline() {
 ##END FUNCTIONS
 
 main() {
+  # Make sure jinja2 is installed
+  easy_install-3.4 jinja2 > /dev/null
   parse_cmdline "$@"
   echo -e "${blue}INFO: Parsing network settings file...${reset}"
   parse_network_settings
diff --git a/lib/python/apex-python-utils.py b/lib/python/apex-python-utils.py
index 802e8571..1d5b4a7c 100755
--- a/lib/python/apex-python-utils.py
+++ b/lib/python/apex-python-utils.py
@@ -13,6 +13,8 @@ import sys
 import apex
 import logging
 import os
+from jinja2 import Environment, FileSystemLoader
+
 
 def parse_net_settings(settings_args):
     settings = apex.NetworkSettings(settings_args.path,
@@ -27,6 +29,14 @@ def find_ip(int_args):
         print(interface.ip)
 
 
+def build_nic_template(nic_args):
+    env = Environment(loader=FileSystemLoader(nic_args.template_directory))
+    template = env.get_template(nic_args.template_filename)
+    print(template.render(enabled_networks=nic_args.enabled_networks,
+                          external_net_type=nic_args.ext_net_type,
+                          external_net_af=nic_args.address_family))
+
+
 parser = argparse.ArgumentParser()
 parser.add_argument('--DEBUG', action='store_true', default=False,
                     help="Turn on debug messages")
@@ -49,6 +59,20 @@ get_int_ip.add_argument('-af', '--address_family', default=4, type=int,
                         help='IP Address family')
 get_int_ip.set_defaults(func=find_ip)
 
+nic_template = subparsers.add_parser('nic_template', help='Build NIC templates')
+nic_template.add_argument('-d', '--template_directory', required=True,
+                          help='Template file directory')
+nic_template.add_argument('-f', '--template_filename', required=True,
+                          help='Template file to process')
+nic_template.add_argument('-n', '--enabled_networks', required=True,
+                          help='enabled network list')
+nic_template.add_argument('-e', '--ext_net_type', default='interface',
+                          choices=['interface', 'br-ex'],
+                          help='External network type')
+nic_template.add_argument('-af', '--address_family', type=int, default=4,
+                          help='IP address family')
+nic_template.set_defaults(func=build_nic_template)
+
 args = parser.parse_args(sys.argv[1:])
 if args.DEBUG:
     logging.basicConfig(level=logging.DEBUG)
diff --git a/lib/python/apex/net_env.py b/lib/python/apex/net_env.py
index ec46fe28..3ca28f8a 100644
--- a/lib/python/apex/net_env.py
+++ b/lib/python/apex/net_env.py
@@ -227,12 +227,26 @@ class NetworkSettings:
                 bash_str += "{}_{}={}\n".format(network, key, value)
         bash_str += "enabled_network_list='{}'\n" \
             .format(' '.join(self.enabled_network_list))
+        bash_str += "ip_addr_family={}\n".format(self.get_ip_addr_family())
         if path:
             with open(path, 'w') as file:
                 file.write(bash_str)
         else:
             print(bash_str)
 
+    def get_ip_addr_family(self):
+        """
+        Returns IP address family for current deployment.
+
+        If any enabled network has IPv6 CIDR, the deployment is classified as
+        IPv6.
+        """
+        for network in self.enabled_network_list:
+            cidr = ipaddress.ip_network(self.settings_obj[network]['cidr'])
+            if cidr.version == 6:
+                return 6
+
+        return 4
 
 class NetworkSettingsException(Exception):
     def __init__(self, value):