A simple method to separate configuration for base fuel, plugins, PODs

and deployment/test scenarios

READY TO MERGE!

Replaces: https://gerrit.opnfv.org/gerrit/#/c/3995/

Abstract
--------
This deployment framework relies on a configuration structure,
providing base installer configuration, per POD specific configuration,
plugin configuration, and deployment scenario configuration.
- The base installer configuration resembles the least common denominator
  of all HW/POD environment and deployment scenarios (These configurations
  are normally carried by the the installer projects in this case (fuel@OPNFV).
- Per POD specific configuration specifies POD unique parameters, the POD
  parameter possible to alter is governed by the Fuel@OPNFV project.
- Plugin configuration - providing configuration of a specific plugin.
  these configurations maintain there own namespace and are normally maintained
  by collaborative projects building Fuel@OPNFV plugins
- Deployment scenario configuration - provides a high level, POD/HW environment
  independent scenario configuration for a specific deployment. It defines what
  features/plugins that shall be deployed - as well needed overrides of the
  plugin config as well as the base installer-, POD/HW environment-
  configurations. Objects allowed to override
  is governed by the Fuel@OPNFV project.

Executing a deployment
----------------------
deploy.sh must be executed locally at the target lab/pod/jumpserver
A lab configuration structure must be provided - see the section below.
It is straight forward to execute a deployment task - as an example:
sudo deploy.sh -b file:///home/jenkins/config -l ericsson-1 -p pod-2
-s os_odl-l2_no-ha -i file:///home/jenkins/MyIso.iso

-b and -i arguments should be expressed in URI style. The resources can thus be
local or remote.

Feedback
--------
Please give feed-back before I'm going to far on a wrong tangent

Implemented scenarios so far:
-----------------------------
- os_ha
- os_no-ha
- os_odl-l3_ha
- os_odl-l3_no-ha
- os_odl-l2_ha
- os_odl-l2_no-ha
- os_onos_ha
- os_onos_no-ha
- os_kvm_ha
- os_kvm_no-ha
- os_ovs_ha
- os_ovs_no-ha
- os_kvm_ovs_ha
- os_kvm_ovs_no-ha

VERIFIED
READY TO MERGE

JIRA: FUEL-35

Change-Id: I94a9b477d8ed4ee8057c16d8f20fe543f7ecc20d
Signed-off-by: Jonas Bjurel <jonas.bjurel@ericsson.com>
(cherry picked from commit 7fd2619e0df370fa832eeff00790bcaa52dc4ffc)

4 files changed, 631 insertions, 0 deletions

diff --git a/deploy/config/labs/devel-pipeline/default/fuel/config/dea-pod-override.yaml b/deploy/config/labs/devel-pipeline/default/fuel/config/dea-pod-override.yaml
new file mode 100644
index 000000000..2be68e25a
--- /dev/null
+++ b/deploy/config/labs/devel-pipeline/default/fuel/config/dea-pod-override.yaml
@@ -0,0 +1,278 @@
+dea-pod-override-config-metadata:
+  title: 'Deployment Environment Adapter POD override for Development Pipeline - Default version'
+  version: '0.1'
+  created: 'Thu 7 2016'
+  comment: 'None'
+environment:
+  name: opnfv_virt
+fuel:
+  ADMIN_NETWORK:
+    cidr: 10.20.0.0/24
+    dhcp_gateway: 10.20.0.2
+    dhcp_pool_end: 10.20.0.254
+    dhcp_pool_start: 10.20.0.3
+    ipaddress: 10.20.0.2
+    netmask: 255.255.0.0
+  DNS_DOMAIN: opnfvericsson.se
+  DNS_SEARCH: opnfvericsson.se
+  DNS_UPSTREAM: 10.118.32.193
+  FUEL_ACCESS:
+    password: admin
+    user: admin
+  HOSTNAME: opnfv_virt
+  NTP1: 81.170.152.66
+  NTP2: 83.168.200.199
+  NTP3: 78.108.60.252
+interfaces_1:
+  eth0:
+  - fuelweb_admin
+  - management
+  eth1:
+  - storage
+  eth2:
+  - private
+  eth3:
+  - public
+transformations_1:
+  transformations:
+  - action: add-br
+    name: br-fw-admin
+  - action: add-br
+    name: br-mgmt
+  - action: add-br
+    name: br-storage
+  - action: add-br
+    name: br-ex
+  - action: add-br
+    name: br-floating
+    provider: ovs
+  - action: add-patch
+    bridges:
+    - br-floating
+    - br-ex
+    mtu: 65000
+    provider: ovs
+  - action: add-br
+    name: br-mesh
+  - action: add-port
+    bridge: br-fw-admin
+    name: eth0
+  - action: add-port
+    bridge: br-mgmt
+    name: eth0.101
+  - action: add-port
+    bridge: br-storage
+    name: eth1.102
+  - action: add-port
+    bridge: br-mesh
+    name: eth2.103
+  - action: add-port
+    bridge: br-ex
+    name: eth3
+network:
+  management_vip: 192.168.0.2
+  management_vrouter_vip: 192.168.0.1
+  networking_parameters:
+    base_mac: fa:16:3e:00:00:00
+    configuration_template: null
+    dns_nameservers:
+    - 8.8.8.8
+    - 8.8.4.4
+    floating_ranges:
+    - - 172.16.0.130
+      - 172.16.0.254
+    gre_id_range:
+    - 2
+    - 65535
+    internal_cidr: 192.168.111.0/24
+    internal_gateway: 192.168.111.1
+    net_l23_provider: ovs
+    segmentation_type: tun
+    vlan_range:
+    - 1000
+    - 1030
+  networks:
+  - cidr: 172.16.0.0/24
+    gateway: 172.16.0.1
+    ip_ranges:
+    - - 172.16.0.2
+      - 172.16.0.126
+    meta:
+      cidr: 172.16.0.0/24
+      configurable: true
+      floating_range_var: floating_ranges
+      ip_range:
+      - 172.16.0.2
+      - 172.16.0.126
+      map_priority: 1
+      name: public
+      notation: ip_ranges
+      render_addr_mask: public
+      render_type: null
+      use_gateway: true
+      vips:
+      - haproxy
+      - vrouter
+      vlan_start: null
+    name: public
+    vlan_start: null
+  - cidr: 192.168.0.0/24
+    gateway: null
+    ip_ranges:
+    - - 192.168.0.1
+      - 192.168.0.254
+    meta:
+      cidr: 192.168.0.0/24
+      configurable: true
+      map_priority: 2
+      name: management
+      notation: cidr
+      render_addr_mask: internal
+      render_type: cidr
+      use_gateway: false
+      vips:
+      - haproxy
+      - vrouter
+      vlan_start: 101
+    name: management
+    vlan_start: 101
+  - cidr: 192.168.1.0/24
+    gateway: null
+    ip_ranges:
+    - - 192.168.1.1
+      - 192.168.1.254
+    meta:
+      cidr: 192.168.1.0/24
+      configurable: true
+      map_priority: 2
+      name: storage
+      notation: cidr
+      render_addr_mask: storage
+      render_type: cidr
+      use_gateway: false
+      vlan_start: 102
+    name: storage
+    vlan_start: 102
+  - cidr: 192.168.2.0/24
+    gateway: null
+    ip_ranges:
+    - - 192.168.2.1
+      - 192.168.2.254
+    meta:
+      cidr: 192.168.2.0/24
+      configurable: true
+      map_priority: 2
+      name: private
+      notation: cidr
+      render_addr_mask: null
+      render_type: cidr
+      seg_type: tun
+      use_gateway: false
+      vlan_start: 103
+    name: private
+    vlan_start: 103
+  - cidr: 10.20.0.0/16
+    gateway: 10.20.0.2
+    ip_ranges:
+    - - 10.20.0.3
+      - 10.20.0.254
+    meta:
+      configurable: false
+      map_priority: 0
+      notation: ip_ranges
+      render_addr_mask: null
+      render_type: null
+      unmovable: true
+      use_gateway: true
+    name: fuelweb_admin
+    vlan_start: null
+  public_vip: 172.16.0.3
+  public_vrouter_vip: 172.16.0.2
+  vips:
+    management:
+      ipaddr: 192.168.0.2
+      namespace: haproxy
+      network_role: mgmt/vip
+      node_roles:
+      - controller
+      - primary-controller
+    public:
+      ipaddr: 172.16.0.3
+      namespace: haproxy
+      network_role: public/vip
+      node_roles:
+      - controller
+      - primary-controller
+    vrouter:
+      ipaddr: 192.168.0.1
+      namespace: vrouter
+      network_role: mgmt/vip
+      node_roles:
+      - controller
+      - primary-controller
+    vrouter_pub:
+      ipaddr: 172.16.0.2
+      namespace: vrouter
+      network_role: public/vip
+      node_roles:
+      - controller
+      - primary-controller
+settings:
+  editable:
+    external_dns:
+      dns_list:
+        description: List of upstream DNS servers, separated by comma
+        label: DNS list
+        regex:
+          error: Invalid IP address list
+          source: ^\*$|^(?:\d|1?\d\d|2[0-4]\d|25[0-5])(?:\.(?:\d|1?\d\d|2[0-4]\d|25[0-5])){3}(?:\s*,\s*(?:\d|1?\d\d|2[0-4]\d|25[0-5])(?:\.(?:\d|1?\d\d|2[0-4]\d|25[0-5])){3})*$
+        type: text
+        value: 8.8.8.8,8.8.4.4
+        weight: 10
+      metadata:
+        label: Host OS DNS Servers
+        weight: 90
+    external_ntp:
+      metadata:
+        label: Host OS NTP Servers
+        weight: 100
+      ntp_list:
+        description: List of upstream NTP servers, separated by comma
+        label: NTP server list
+        regex:
+          error: Invalid NTP server list
+          source: ^\s*(?:(?:\w+(?:-+\w+)*\.)+[a-z]+|\d{1,3}(?:\.\d{1,3}){3})\s*(?:,\s*(?:(?:\w+(?:-+\w+)*\.)+[a-z]+|\d{1,3}(\.\d{1,3}){3})\s*)*$
+        type: text
+        value: 81.170.152.66,83.168.200.199,78.108.60.252
+        weight: 10
+    syslog:
+      metadata:
+        label: Syslog
+        weight: 50
+      syslog_port:
+        description: Remote syslog port
+        label: Port
+        regex:
+          error: Invalid syslog port
+          source: ^([1-9][0-9]{0,3}|[1-5][0-9]{4}|6[0-4][0-9]{3}|65[0-4][0-9]{2}|655[0-2][0-9]|6553[0-5])$
+        type: text
+        value: '514'
+        weight: 20
+      syslog_server:
+        description: Remote syslog hostname
+        label: Hostname
+        type: text
+        value: ''
+        weight: 10
+      syslog_transport:
+        label: Syslog transport protocol
+        type: radio
+        value: tcp
+        values:
+        - data: udp
+          description: ''
+          label: UDP
+        - data: tcp
+          description: ''
+          label: TCP
+        weight: 30
diff --git a/deploy/config/labs/devel-pipeline/default/fuel/config/dha.yaml b/deploy/config/labs/devel-pipeline/default/fuel/config/dha.yaml
new file mode 100644
index 000000000..26c4d2b16
--- /dev/null
+++ b/deploy/config/labs/devel-pipeline/default/fuel/config/dha.yaml
@@ -0,0 +1,38 @@
+dha-pod-config-metadata:
+  title: Deployment Hardware Adapter (DHA) for fuel development pipeline - default version
+  version: 0.0.1
+  created: Jan 7 2016
+  comment: Default version
+
+# Adapter to use for this definition
+adapter: libvirt
+
+# Node list.
+# Mandatory property is id, all other properties are adapter specific.
+
+nodes:
+- id: 1
+  libvirtName: controller1
+  libvirtTemplate: templates/virtual_environment/vms/controller.xml
+- id: 2
+  libvirtName: compute1
+  libvirtTemplate: templates/virtual_environment/vms/compute.xml
+- id: 3
+  libvirtName: compute2
+  libvirtTemplate: templates/virtual_environment/vms/compute.xml
+- id: 4
+  libvirtName: compute3
+  libvirtTemplate: templates/virtual_environment/vms/compute.xml
+- id: 5
+  libvirtName: fuel-master
+  libvirtTemplate: templates/virtual_environment/vms/fuel.xml
+  isFuel: yes
+  username: root
+  password: r00tme
+
+virtNetConfDir: templates/virtual_environment/networks
+
+disks:
+  fuel: 50G
+  controller: 100G
+  compute: 50G
diff --git a/deploy/config/labs/devel-pipeline/elx/fuel/config/dea-pod-override.yaml b/deploy/config/labs/devel-pipeline/elx/fuel/config/dea-pod-override.yaml
new file mode 100644
index 000000000..7fc429b84
--- /dev/null
+++ b/deploy/config/labs/devel-pipeline/elx/fuel/config/dea-pod-override.yaml
@@ -0,0 +1,277 @@
+dea-pod-override-config-metadata:
+  title: 'Deployment Environment Adapter POD override for Development Pipeline - Ericsson ELX version'
+  version: '0.1'
+  created: 'Mon Dec 30 2015'
+  comment: 'None'
+environment:
+  name: opnfv_virt
+interfaces_1:
+  eth0:
+  - fuelweb_admin
+  - management
+  eth1:
+  - storage
+  eth2:
+  - private
+  eth3:
+  - public
+transformations_1:
+  transformations:
+  - action: add-br
+    name: br-fw-admin
+  - action: add-br
+    name: br-mgmt
+  - action: add-br
+    name: br-storage
+  - action: add-br
+    name: br-ex
+  - action: add-br
+    name: br-floating
+    provider: ovs
+  - action: add-patch
+    bridges:
+    - br-floating
+    - br-ex
+    mtu: 65000
+    provider: ovs
+  - action: add-br
+    name: br-mesh
+  - action: add-port
+    bridge: br-fw-admin
+    name: eth0
+  - action: add-port
+    bridge: br-mgmt
+    name: eth0.101
+  - action: add-port
+    bridge: br-storage
+    name: eth1.102
+  - action: add-port
+    bridge: br-mesh
+    name: eth2.103
+  - action: add-port
+    bridge: br-ex
+    name: eth3
+fuel:
+  ADMIN_NETWORK:
+    cidr: 10.20.0.0/24
+    dhcp_gateway: 10.20.0.2
+    dhcp_pool_end: 10.20.0.254
+    dhcp_pool_start: 10.20.0.3
+    ipaddress: 10.20.0.2
+    netmask: 255.255.0.0
+  DNS_DOMAIN: opnfvericsson.se
+  DNS_SEARCH: opnfvericsson.se
+  DNS_UPSTREAM: 147.214.6.234
+  FUEL_ACCESS:
+    password: admin
+    user: admin
+  HOSTNAME: opnfv_virt
+  NTP1: 193.181.14.10
+  NTP2: 193.181.14.11
+  NTP3: null
+network:
+  management_vip: 192.168.0.2
+  management_vrouter_vip: 192.168.0.1
+  networking_parameters:
+    base_mac: fa:16:3e:00:00:00
+    configuration_template: null
+    dns_nameservers:
+    - 147.214.6.234
+    floating_ranges:
+    - - 172.16.0.130
+      - 172.16.0.254
+    gre_id_range:
+    - 2
+    - 65535
+    internal_cidr: 192.168.111.0/24
+    internal_gateway: 192.168.111.1
+    net_l23_provider: ovs
+    segmentation_type: tun
+    vlan_range:
+    - 1000
+    - 1030
+  networks:
+  - cidr: 172.16.0.0/24
+    gateway: 172.16.0.1
+    ip_ranges:
+    - - 172.16.0.2
+      - 172.16.0.126
+    meta:
+      cidr: 172.16.0.0/24
+      configurable: true
+      floating_range_var: floating_ranges
+      ip_range:
+      - 172.16.0.2
+      - 172.16.0.126
+      map_priority: 1
+      name: public
+      notation: ip_ranges
+      render_addr_mask: public
+      render_type: null
+      use_gateway: true
+      vips:
+      - haproxy
+      - vrouter
+      vlan_start: null
+    name: public
+    vlan_start: null
+  - cidr: 192.168.0.0/24
+    gateway: null
+    ip_ranges:
+    - - 192.168.0.1
+      - 192.168.0.254
+    meta:
+      cidr: 192.168.0.0/24
+      configurable: true
+      map_priority: 2
+      name: management
+      notation: cidr
+      render_addr_mask: internal
+      render_type: cidr
+      use_gateway: false
+      vips:
+      - haproxy
+      - vrouter
+      vlan_start: 101
+    name: management
+    vlan_start: 101
+  - cidr: 192.168.1.0/24
+    gateway: null
+    ip_ranges:
+    - - 192.168.1.1
+      - 192.168.1.254
+    meta:
+      cidr: 192.168.1.0/24
+      configurable: true
+      map_priority: 2
+      name: storage
+      notation: cidr
+      render_addr_mask: storage
+      render_type: cidr
+      use_gateway: false
+      vlan_start: 102
+    name: storage
+    vlan_start: 102
+  - cidr: 192.168.2.0/24
+    gateway: null
+    ip_ranges:
+    - - 192.168.2.1
+      - 192.168.2.254
+    meta:
+      cidr: 192.168.2.0/24
+      configurable: true
+      map_priority: 2
+      name: private
+      notation: cidr
+      render_addr_mask: null
+      render_type: cidr
+      seg_type: tun
+      use_gateway: false
+      vlan_start: 103
+    name: private
+    vlan_start: 103
+  - cidr: 10.20.0.0/16
+    gateway: 10.20.0.2
+    ip_ranges:
+    - - 10.20.0.3
+      - 10.20.0.254
+    meta:
+      configurable: false
+      map_priority: 0
+      notation: ip_ranges
+      render_addr_mask: null
+      render_type: null
+      unmovable: true
+      use_gateway: true
+    name: fuelweb_admin
+    vlan_start: null
+  public_vip: 172.16.0.3
+  public_vrouter_vip: 172.16.0.2
+  vips:
+    management:
+      ipaddr: 192.168.0.2
+      namespace: haproxy
+      network_role: mgmt/vip
+      node_roles:
+      - controller
+      - primary-controller
+    public:
+      ipaddr: 172.16.0.3
+      namespace: haproxy
+      network_role: public/vip
+      node_roles:
+      - controller
+      - primary-controller
+    vrouter:
+      ipaddr: 192.168.0.1
+      namespace: vrouter
+      network_role: mgmt/vip
+      node_roles:
+      - controller
+      - primary-controller
+    vrouter_pub:
+      ipaddr: 172.16.0.2
+      namespace: vrouter
+      network_role: public/vip
+      node_roles:
+      - controller
+      - primary-controller
+settings:
+  editable:
+    external_dns:
+      dns_list:
+        description: List of upstream DNS servers, separated by comma
+        label: DNS list
+        regex:
+          error: Invalid IP address list
+          source: ^\*$|^(?:\d|1?\d\d|2[0-4]\d|25[0-5])(?:\.(?:\d|1?\d\d|2[0-4]\d|25[0-5])){3}(?:\s*,\s*(?:\d|1?\d\d|2[0-4]\d|25[0-5])(?:\.(?:\d|1?\d\d|2[0-4]\d|25[0-5])){3})*$
+        type: text
+        value: 147.214.6.234
+        weight: 10
+      metadata:
+        label: Host OS DNS Servers
+        weight: 90
+    external_ntp:
+      metadata:
+        label: Host OS NTP Servers
+        weight: 100
+      ntp_list:
+        description: List of upstream NTP servers, separated by comma
+        label: NTP server list
+        regex:
+          error: Invalid NTP server list
+          source: ^\s*(?:(?:\w+(?:-+\w+)*\.)+[a-z]+|\d{1,3}(?:\.\d{1,3}){3})\s*(?:,\s*(?:(?:\w+(?:-+\w+)*\.)+[a-z]+|\d{1,3}(\.\d{1,3}){3})\s*)*$
+        type: text
+        value: 193.181.14.10, 193.181.14.11
+        weight: 10
+    syslog:
+      metadata:
+        label: Syslog
+        weight: 50
+      syslog_port:
+        description: Remote syslog port
+        label: Port
+        regex:
+          error: Invalid syslog port
+          source: ^([1-9][0-9]{0,3}|[1-5][0-9]{4}|6[0-4][0-9]{3}|65[0-4][0-9]{2}|655[0-2][0-9]|6553[0-5])$
+        type: text
+        value: '514'
+        weight: 20
+      syslog_server:
+        description: Remote syslog hostname
+        label: Hostname
+        type: text
+        value: ''
+        weight: 10
+      syslog_transport:
+        label: Syslog transport protocol
+        type: radio
+        value: tcp
+        values:
+        - data: udp
+          description: ''
+          label: UDP
+        - data: tcp
+          description: ''
+          label: TCP
+        weight: 30
\ No newline at end of file
diff --git a/deploy/config/labs/devel-pipeline/elx/fuel/config/dha.yaml b/deploy/config/labs/devel-pipeline/elx/fuel/config/dha.yaml
new file mode 100644
index 000000000..fc0e93a4f
--- /dev/null
+++ b/deploy/config/labs/devel-pipeline/elx/fuel/config/dha.yaml
@@ -0,0 +1,38 @@
+dha-pod-config-metadata:
+  title: Deployment Hardware Adapter (DHA) for fuel development pipeline - Ericsson ELX version
+  version: 0.0.1
+  created: Dececember 30 2015
+  comment: ELX version
+
+# Adapter to use for this definition
+adapter: libvirt
+
+# Node list.
+# Mandatory property is id, all other properties are adapter specific.
+
+nodes:
+- id: 1
+  libvirtName: controller1
+  libvirtTemplate: templates/virtual_environment/vms/controller.xml
+- id: 2
+  libvirtName: compute1
+  libvirtTemplate: templates/virtual_environment/vms/compute.xml
+- id: 3
+  libvirtName: compute2
+  libvirtTemplate: templates/virtual_environment/vms/compute.xml
+- id: 4
+  libvirtName: compute3
+  libvirtTemplate: templates/virtual_environment/vms/compute.xml
+- id: 5
+  libvirtName: fuel-master
+  libvirtTemplate: templates/virtual_environment/vms/fuel.xml
+  isFuel: yes
+  username: root
+  password: r00tme
+
+virtNetConfDir: templates/virtual_environment/networks
+
+disks:
+  fuel: 50G
+  controller: 100G
+  compute: 50G