Add support for fdio scenarios

Add support for the following scenarios:
- os-odl-fdio-ha
- os-odl-fdio-noha
- os-nosdn-fdio-noha
- os-nosdn-fdio-ha

apex-tripleo-heat-templates: If11092e6581445a70e63c8f6c48518698b3cc8fc
apex-puppet-tripleo: If498c41d706c8f14a5b0bbee64cb4d26cd78c2d0
apex-os-net-config: If7a2c6119bf613f1fc8846237b077cd8f0e26015

Change-Id: Id3fdd09f6e0d2b7666d08c0dc7802165797eefdc
Signed-off-by: Feng Pan <fpan@redhat.com>

7 files changed, 122 insertions, 67 deletions

diff --git a/build/opnfv-environment.yaml b/build/opnfv-environment.yaml
index 79b9542e..2a97dceb 100644
--- a/build/opnfv-environment.yaml
+++ b/build/opnfv-environment.yaml
@@ -122,11 +122,11 @@ parameter_defaults:
     - OS::TripleO::Services::Etcd
 #    - OS::TripleO::Services::Gluon
     - OS::TripleO::Services::Tacker
-#    - OS::TripleO::Services::NeutronHoneycombAgent
+    - OS::TripleO::Services::NeutronHoneycombAgent
     - OS::TripleO::Services::Congress
-#    - OS::TripleO::Services::NeutronVppAgent
+    - OS::TripleO::Services::NeutronVppAgent
     - OS::TripleO::Services::OVNDBs
-#    - OS::TripleO::Services::Vpp
+    - OS::TripleO::Services::Vpp
   ComputeServices:
     - OS::TripleO::Services::CACerts
     - OS::TripleO::Services::CephClient
@@ -150,6 +150,6 @@ parameter_defaults:
 #    - OS::TripleO::Services::ONOSOvs
     - OS::TripleO::Services::SensuClient
     - OS::TripleO::Services::FluentdClient
-#    - OS::TripleO::Services::NeutronHoneycombAgent
-#    - OS::TripleO::Services::NeutronVppAgent
-#    - OS::TripleO::Services::Vpp
+    - OS::TripleO::Services::NeutronHoneycombAgent
+    - OS::TripleO::Services::NeutronVppAgent
+    - OS::TripleO::Services::Vpp
diff --git a/build/overcloud-full.sh b/build/overcloud-full.sh
index cb1b94bc..40ffea33 100755
--- a/build/overcloud-full.sh
+++ b/build/overcloud-full.sh
@@ -55,9 +55,6 @@ LIBGUESTFS_BACKEND=direct virt-customize \
     --run-command "if ! rpm -qa | grep python-redis; then yum install -y python-redis; fi" \
     --run-command "sed -i 's/^#UseDNS.*$/UseDNS no/' /etc/ssh/sshd_config" \
     --run-command "sed -i 's/^GSSAPIAuthentication.*$/GSSAPIAuthentication no/' /etc/ssh/sshd_config" \
-    --run-command "yum install -y etcd" \
-    --install python-etcd \
-    --run-command "puppet module install cristifalcas/etcd" \
     --run-command "rm -f /etc/sysctl.d/80-vpp.conf" \
     --install unzip \
     --upload ${BUILD_DIR}/vsperf.tar.gz:/var/opt \
@@ -93,6 +90,8 @@ done
 rm -rf puppet-fdio
 git clone https://git.fd.io/puppet-fdio
 pushd puppet-fdio > /dev/null
+#TODO: Remove this when we update to 17.07
+git revert a6e575c8f0af17e62990653bcf4a12c688c21aad --no-edit
 git archive --format=tar.gz --prefix=fdio/ HEAD > ${BUILD_DIR}/puppet-fdio.tar.gz
 popd > /dev/null
 
@@ -132,13 +131,17 @@ LIBGUESTFS_BACKEND=direct virt-customize \
     --upload ${BUILD_DIR}/noarch/$netvpp_pkg:/root/fdio \
     $fdio_pkg_str \
     --run-command "yum install -y /root/fdio/*.rpm" \
-    --upload ${BUILD_DIR}/puppet-fdio.tar.gz:/etc/puppet/modules \
-    --run-command "cd /etc/puppet/modules && tar xzf puppet-fdio.tar.gz" \
     --run-command "curl -f https://copr.fedorainfracloud.org/coprs/leifmadsen/ovs-master/repo/epel-7/leifmadsen-ovs-master-epel-7.repo > /etc/yum.repos.d/leifmadsen-ovs-master-epel-7.repo" \
     --run-command "mkdir /root/ovs27" \
     --run-command "yumdownloader --destdir=/root/ovs27 openvswitch*2.7* python-openvswitch-2.7*" \
     --upload ${CACHE_DIR}/$kvmfornfv_kernel_rpm:/root/ \
     --install python2-networking-sfc \
+    --install python-etcd,puppet-etcd \
+    --install patch \
+    --upload ${BUILD_ROOT}/patches/puppet-neutron-ml2-ip-version-fix.patch:/usr/share/openstack-puppet/modules/neutron/ \
+    --run-command "cd /usr/share/openstack-puppet/modules/neutron && patch -p1 < puppet-neutron-ml2-ip-version-fix.patch" \
+    --upload ${BUILD_ROOT}/patches/puppet-neutron-vpp-ml2-type_drivers-setting.patch:/usr/share/openstack-puppet/modules/neutron/ \
+    --run-command "cd /usr/share/openstack-puppet/modules/neutron && patch -p1 < puppet-neutron-vpp-ml2-type_drivers-setting.patch" \
     -a overcloud-full_build.qcow2
 fi
 
diff --git a/build/overcloud-opendaylight.sh b/build/overcloud-opendaylight.sh
index 12b47f1e..3b4e02ae 100755
--- a/build/overcloud-opendaylight.sh
+++ b/build/overcloud-opendaylight.sh
@@ -49,6 +49,9 @@ popd > /dev/null
 # Download ODL netvirt for VPP
 populate_cache http://artifacts.opnfv.org/apex/danube/fdio_netvirt/opendaylight-7.0.0-0.1.20170531snap665.el7.noarch.rpm
 
+# Download ODL for fdio scenarios
+populate_cache http://artifacts.opnfv.org/apex/danube/fdio_odls/fdio_odl_carbon.tar.gz
+
 # install ODL packages
 # Patch in OPNFV custom puppet-tripleO
 # install Honeycomb
@@ -70,6 +73,7 @@ LIBGUESTFS_BACKEND=direct virt-customize \
     --install capnproto-libs,capnproto \
     --upload ${BUILD_ROOT}/patches/neutron-patch-NSDriver.patch:/usr/lib/python2.7/site-packages/ \
     --upload ${CACHE_DIR}/opendaylight-7.0.0-0.1.20170531snap665.el7.noarch.rpm:/root/ \
+    --upload ${CACHE_DIR}/fdio_odl_carbon.tar.gz:/root/ \
     -a overcloud-full-opendaylight_build.qcow2
 
 # Arch dependent on x86
@@ -78,11 +82,11 @@ if [ "$(uname -i)" == 'x86_64' ]; then
 # Download quagga/zrpc rpms
 populate_cache http://artifacts.opnfv.org/apex/danube/quagga/quagga-3.tar.gz
 # Download Honeycomb
-populate_cache https://nexus.fd.io/content/repositories/fd.io.stable.1704.centos7/io/fd/hc2vpp/honeycomb/1.17.04-2048.noarch/honeycomb-1.17.04-2048.noarch.rpm
+populate_cache $honeycomb_pkg
 
 LIBGUESTFS_BACKEND=direct virt-customize \
-    --upload ${CACHE_DIR}/honeycomb-1.17.04-2048.noarch.rpm:/root/fdio/ \
-    --run-command "yum install -y /root/fdio/honeycomb-1.17.04-2048.noarch.rpm" \
+    --upload ${CACHE_DIR}/${honeycomb_pkg##*/}:/root/fdio/ \
+    --run-command "yum install -y /root/fdio/${honeycomb_pkg##*/}" \
     --install zeromq-4.1.4 \
     --upload ${CACHE_DIR}/quagga-3.tar.gz:/root/ \
     --run-command "cd /root/ && tar xzf quagga-3.tar.gz" \
diff --git a/build/patches/neutron-patch-NSDriver.patch b/build/patches/neutron-patch-NSDriver.patch
index d5889e38..f01d0318 100644
--- a/build/patches/neutron-patch-NSDriver.patch
+++ b/build/patches/neutron-patch-NSDriver.patch
@@ -1,16 +1,16 @@
-From ff4e918d21970a81604a0aaa2af888141f93cdac Mon Sep 17 00:00:00 2001
+From d51e6ba77c3f40c7c04c97b1de06bf9344c95929 Mon Sep 17 00:00:00 2001
 From: Feng Pan <fpan@redhat.com>
-Date: Sun, 5 Feb 2017 21:34:19 -0500
+Date: Thu, 20 Jul 2017 16:12:45 -0400
 Subject: [PATCH] Add NSDriver
 
 ---
  neutron/agent/l3/namespaces.py   |  6 ++--
- neutron/agent/l3/router_info.py  | 14 ++++----
+ neutron/agent/l3/router_info.py  | 12 ++++---
  neutron/agent/linux/interface.py | 76 +++++++++++++++++++++++++++++++++++++++-
- 3 files changed, 87 insertions(+), 9 deletions(-)
+ 3 files changed, 86 insertions(+), 8 deletions(-)
 
 diff --git a/neutron/agent/l3/namespaces.py b/neutron/agent/l3/namespaces.py
-index e70d7bb..3c932a8 100644
+index f65c706..e9fc4b7 100644
 --- a/neutron/agent/l3/namespaces.py
 +++ b/neutron/agent/l3/namespaces.py
 @@ -18,6 +18,7 @@ import functools
@@ -32,7 +32,7 @@ index e70d7bb..3c932a8 100644
          name = self._get_ns_name(router_id)
          super(RouterNamespace, self).__init__(
              name, agent_conf, driver, use_ipv6)
-@@ -131,7 +133,7 @@ class RouterNamespace(Namespace):
+@@ -132,7 +134,7 @@ class RouterNamespace(Namespace):
              elif d.name.startswith(ROUTER_2_FIP_DEV_PREFIX):
                  ns_ip.del_veth(d.name)
              elif d.name.startswith(EXTERNAL_DEV_PREFIX):
@@ -42,10 +42,10 @@ index e70d7bb..3c932a8 100644
                      bridge=self.agent_conf.external_network_bridge,
                      namespace=self.name,
 diff --git a/neutron/agent/l3/router_info.py b/neutron/agent/l3/router_info.py
-index 3fd3934..b33fb7e 100644
+index 46db6a5..6775882 100644
 --- a/neutron/agent/l3/router_info.py
 +++ b/neutron/agent/l3/router_info.py
-@@ -27,6 +27,7 @@ from neutron.common import exceptions as n_exc
+@@ -30,6 +30,7 @@ from neutron.common import exceptions as n_exc
  from neutron.common import ipv6_utils
  from neutron.common import utils as common_utils
  from neutron.ipam import utils as ipam_utils
@@ -53,15 +53,15 @@ index 3fd3934..b33fb7e 100644
  
  LOG = logging.getLogger(__name__)
  INTERNAL_DEV_PREFIX = namespaces.INTERNAL_DEV_PREFIX
-@@ -47,6 +48,7 @@ class RouterInfo(object):
-                  agent_conf,
+@@ -52,6 +53,7 @@ class RouterInfo(object):
                   interface_driver,
                   use_ipv6=False):
+         self.agent = agent
 +        self.ovs_driver = OVSInterfaceDriver(agent_conf)
          self.router_id = router_id
          self.ex_gw_port = None
          self._snat_enabled = None
-@@ -57,7 +59,7 @@ class RouterInfo(object):
+@@ -63,7 +65,7 @@ class RouterInfo(object):
          self.router = router
          self.use_ipv6 = use_ipv6
          ns = self.create_router_namespace_object(
@@ -70,7 +70,7 @@ index 3fd3934..b33fb7e 100644
          self.router_namespace = ns
          self.ns_name = ns.name
          self.available_mark_ids = set(range(ADDRESS_SCOPE_MARK_ID_MIN,
-@@ -94,9 +96,9 @@ class RouterInfo(object):
+@@ -100,9 +102,9 @@ class RouterInfo(object):
          self.router_namespace.create()
  
      def create_router_namespace_object(
@@ -82,7 +82,7 @@ index 3fd3934..b33fb7e 100644
  
      @property
      def router(self):
-@@ -583,7 +585,7 @@ class RouterInfo(object):
+@@ -609,7 +611,7 @@ class RouterInfo(object):
                  for ip in floating_ips]
  
      def _plug_external_gateway(self, ex_gw_port, interface_name, ns_name):
@@ -91,7 +91,7 @@ index 3fd3934..b33fb7e 100644
                           ex_gw_port['id'],
                           interface_name,
                           ex_gw_port['mac_address'],
-@@ -641,7 +643,7 @@ class RouterInfo(object):
+@@ -679,7 +681,7 @@ class RouterInfo(object):
  
          self._add_route_to_gw(ex_gw_port, device_name=interface_name,
                                namespace=ns_name, preserve_ips=preserve_ips)
@@ -100,17 +100,8 @@ index 3fd3934..b33fb7e 100644
              interface_name,
              ip_cidrs,
              namespace=ns_name,
-@@ -735,7 +737,7 @@ class RouterInfo(object):
-         for stale_dev in stale_devs:
-             LOG.debug('Deleting stale external router device: %s', stale_dev)
-             pd.remove_gw_interface(self.router['id'])
--            self.driver.unplug(stale_dev,
-+            self.ovs_driver.unplug(stale_dev,
-                                bridge=self.agent_conf.external_network_bridge,
-                                namespace=self.ns_name,
-                                prefix=EXTERNAL_DEV_PREFIX)
 diff --git a/neutron/agent/linux/interface.py b/neutron/agent/linux/interface.py
-index c2eb06e..80da16f 100644
+index 1f1ed39..b7781e2 100644
 --- a/neutron/agent/linux/interface.py
 +++ b/neutron/agent/linux/interface.py
 @@ -15,7 +15,7 @@
@@ -122,7 +113,7 @@ index c2eb06e..80da16f 100644
  import netaddr
  from neutron_lib import constants
  from oslo_config import cfg
-@@ -288,6 +288,80 @@ class NullDriver(LinuxInterfaceDriver):
+@@ -317,6 +317,80 @@ class NullDriver(LinuxInterfaceDriver):
      def unplug(self, device_name, bridge=None, namespace=None, prefix=None):
          pass
  
@@ -203,29 +194,6 @@ index c2eb06e..80da16f 100644
  
  class OVSInterfaceDriver(LinuxInterfaceDriver):
      """Driver for creating an internal interface on an OVS bridge."""
-diff --git a/neutron/agent/l3/ha_router.py b/usr/lib/python2.7/site-packages/neutron/agent/l3/ha_router.py
-index ca0e50d..2c4fdad 100644
---- a/neutron/agent/l3/ha_router.py
-+++ b/usr/lib/python2.7/site-packages/neutron/agent/l3/ha_router.py
-@@ -52,15 +52,14 @@ class HaRouterNamespace(namespaces.RouterNamespace):
- class HaRouter(router.RouterInfo):
-     def __init__(self, state_change_callback, *args, **kwargs):
-         super(HaRouter, self).__init__(*args, **kwargs)
--
-         self.ha_port = None
-         self.keepalived_manager = None
-         self.state_change_callback = state_change_callback
- 
-     def create_router_namespace_object(
--            self, router_id, agent_conf, iface_driver, use_ipv6):
-+            self, router_id, agent_conf, iface_driver, use_ipv6, ovs_driver):
-         return HaRouterNamespace(
--            router_id, agent_conf, iface_driver, use_ipv6)
-+            router_id, agent_conf, iface_driver, use_ipv6, ovs_driver)
- 
-     @property
-     def ha_priority(self):
-
 -- 
 2.9.3
 
diff --git a/build/patches/puppet-neutron-ml2-ip-version-fix.patch b/build/patches/puppet-neutron-ml2-ip-version-fix.patch
new file mode 100644
index 00000000..3cbb3a8c
--- /dev/null
+++ b/build/patches/puppet-neutron-ml2-ip-version-fix.patch
@@ -0,0 +1,28 @@
+From e3e4a9cf57d5d7da6914b31527188eff7e290238 Mon Sep 17 00:00:00 2001
+From: Feng Pan <fpan@redhat.com>
+Date: Wed, 17 May 2017 11:39:31 -0400
+Subject: [PATCH] Make sure overlay_ip_version is treated as string
+
+puppet throws error if overlay_ip_version is passed in through hiera
+as an integer. We should make sure this variable is treated as string
+for the 'in' operator.
+
+Closes-Bug: #1691502
+
+Change-Id: I1c034e8018c959672b106e2d57992cf93b945d8a
+Signed-off-by: Feng Pan <fpan@redhat.com>
+---
+
+diff --git a/manifests/plugins/ml2.pp b/manifests/plugins/ml2.pp
+index d931e72..b943593 100644
+--- a/manifests/plugins/ml2.pp
++++ b/manifests/plugins/ml2.pp
+@@ -171,7 +171,7 @@
+     warning ('supported_pci_vendor_devs is deprecated, has no effect and will be removed in a future release.')
+   }
+ 
+-  if !is_service_default($overlay_ip_version) and !($overlay_ip_version in [4, 6]) {
++  if !is_service_default($overlay_ip_version) and !("${overlay_ip_version}" in ['4', '6']) {
+     fail('Invalid IP version for overlay_ip_version')
+   }
+ 
diff --git a/build/patches/puppet-neutron-vpp-ml2-type_drivers-setting.patch b/build/patches/puppet-neutron-vpp-ml2-type_drivers-setting.patch
new file mode 100644
index 00000000..a84ab83f
--- /dev/null
+++ b/build/patches/puppet-neutron-vpp-ml2-type_drivers-setting.patch
@@ -0,0 +1,52 @@
+From 8e37e0fae6195ec177828a9e9d36c6ee009cd372 Mon Sep 17 00:00:00 2001
+From: Feng Pan <fpan@redhat.com>
+Date: Thu, 18 May 2017 17:39:42 -0400
+Subject: [PATCH] Add ml2 type_drivers setting
+
+Change-Id: Ie47a1ace6302d7eccd3ead676c4e1cde7e82c5d2
+---
+ manifests/agents/ml2/vpp.pp | 10 ++++++++--
+ 1 file changed, 8 insertions(+), 2 deletions(-)
+
+diff --git a/manifests/agents/ml2/vpp.pp b/manifests/agents/ml2/vpp.pp
+index f17d67e8..08427833 100644
+--- a/manifests/agents/ml2/vpp.pp
++++ b/manifests/agents/ml2/vpp.pp
+@@ -42,6 +42,11 @@
+ #   in the vpp config.
+ #   Defaults to false.
+ #
++# [*type_drivers*]
++#   (optional) List of network type driver entrypoints to be loaded
++#   Could be an array that can contain flat, vlan or vxlan
++#   Defaults to $::os_service_default.
++#
+ class neutron::agents::ml2::vpp (
+   $package_ensure   = 'present',
+   $enabled          = true,
+@@ -51,6 +56,7 @@ class neutron::agents::ml2::vpp (
+   $etcd_port        = $::os_service_default,
+   $etcd_user        = $::os_service_default,
+   $etcd_pass        = $::os_service_default,
++  $type_drivers     = $::os_service_default,
+   $purge_config     = false,
+ ) {
+   include ::neutron::deps
+@@ -61,12 +67,12 @@ class neutron::agents::ml2::vpp (
+   }
+ 
+   neutron_agent_vpp {
+-    'ml2_vpp/physnets': value  => $physnets;
++    'ml2_vpp/physnets':  value => $physnets;
+     'ml2_vpp/etcd_host': value => $etcd_host;
+     'ml2_vpp/etcd_port': value => $etcd_port;
+     'ml2_vpp/etcd_user': value => $etcd_user;
+     'ml2_vpp/etcd_pass': value => $etcd_pass;
+-    'DEFAULT/host': value      => $::hostname;
++    'ml2/type_drivers':  value => join(any2array($type_drivers), ',');
+   }
+ 
+   package { 'neutron-vpp-agent':
+-- 
+2.13.3
+
diff --git a/build/variables.sh b/build/variables.sh
index 145b582f..d37be0d0 100644
--- a/build/variables.sh
+++ b/build/variables.sh
@@ -46,10 +46,10 @@ netvpp_pkg=python-networking-vpp-0.0.1-1.git${NETVPP_COMMIT}$(rpm -E %dist).noar
 gluon_rpm=gluon-0.0.1-1_20170302.noarch.rpm
 
 fdio_pkgs=(
-'https://nexus.fd.io/content/repositories/fd.io.centos7/io/fd/vpp/vpp/17.04-release.x86_64/vpp-17.04-release.x86_64.rpm'
-'https://nexus.fd.io/content/repositories/fd.io.centos7/io/fd/vpp/vpp-api-python/17.04-release.x86_64/vpp-api-python-17.04-release.x86_64.rpm'
-'https://nexus.fd.io/content/repositories/fd.io.centos7/io/fd/vpp/vpp-lib/17.04-release.x86_64/vpp-lib-17.04-release.x86_64.rpm'
-'https://nexus.fd.io/content/repositories/fd.io.centos7/io/fd/vpp/vpp-plugins/17.04-release.x86_64/vpp-plugins-17.04-release.x86_64.rpm'
+'http://artifacts.opnfv.org/apex/danube/fdio_common_rpms/vpp-17.04.1-3~ge3b7ad7~b72.x86_64.rpm'
+'http://artifacts.opnfv.org/apex/danube/fdio_common_rpms/vpp-api-python-17.04.1-3~ge3b7ad7~b72.x86_64.rpm'
+'http://artifacts.opnfv.org/apex/danube/fdio_common_rpms/vpp-lib-17.04.1-3~ge3b7ad7~b72.x86_64.rpm'
+'http://artifacts.opnfv.org/apex/danube/fdio_common_rpms/vpp-plugins-17.04.1-3~ge3b7ad7~b72.x86_64.rpm'
 )
 
-honeycomb_pkg='http://artifacts.opnfv.org/apex/danube/fdio_common_rpms/honeycomb-1.17.04-2048.noarch.rpm'
+honeycomb_pkg='http://artifacts.opnfv.org/apex/danube/fdio_common_rpms/honeycomb-1.17.04.1-2073.noarch.rpm'
\ No newline at end of file