diff options
| -rw-r--r-- | build/Makefile | 15 | ||||
| -rw-r--r-- | build/bootstrap_admin_node.sh.patch | 159 | ||||
| -rw-r--r-- | build/config.mk | 17 | ||||
| -rw-r--r-- | build/f_isoroot/f_bgpvpn-pluginbuild/config.mk | 2 | ||||
| -rw-r--r-- | build/f_isoroot/f_collectd-ceilometer-pluginbuild/config.mk | 2 | ||||
| -rw-r--r-- | build/f_isoroot/f_kvm-pluginbuild/config.mk | 2 | ||||
| -rwxr-xr-x | build/f_isoroot/f_repobuild/select_ubuntu_repo.sh | 2 | ||||
| -rw-r--r-- | build/f_isoroot/f_yardstick-pluginbuild/config.mk | 2 | ||||
| -rw-r--r-- | build/fuel-main_docker_version.patch | 20 | ||||
| -rw-r--r-- | build/isolinux.cfg.patch | 43 | ||||
| -rw-r--r-- | build/repo-mirror-cz.patch | 19 | ||||
| -rw-r--r-- | deploy/README.templater | 371 | ||||
| -rw-r--r-- | deploy/cloud/configure_nodes.py | 54 | ||||
| -rw-r--r-- | deploy/config/plugins/fuel-nfvkvm_0.9.0.yaml | 34 | ||||
| -rw-r--r-- | deploy/deploy-config.py | 250 | ||||
| -rw-r--r-- | deploy/reap.py | 36 | ||||
| -rw-r--r-- | deploy/scenario/ha_nfv-kvm_heat_ceilometer_scenario.yaml | 8 | ||||
| -rwxr-xr-x | deploy/templater.py | 185 | ||||
| -rw-r--r-- | deploy/templates/templater/base_dea_template.yaml | 24 |
19 files changed, 964 insertions, 281 deletions
diff --git a/build/Makefile b/build/Makefile index 956183cc6..f49947b88 100644 --- a/build/Makefile +++ b/build/Makefile @@ -109,9 +109,7 @@ $(ISOCACHE): cd /tmp && git clone $(FUEL_MAIN_REPO); \ fi cd /tmp/fuel-main && git checkout $(FUEL_MAIN_TAG) - @echo "fuel" `git -C /tmp/fuel-main show | grep commit | head -1 | cut -d " " -f2` >> $(VERSION_FILE) - # Patch for using the Czech Fuel mirror - cd /tmp/fuel-main && patch -p0 < $(TOPDIR)/repo-mirror-cz.patch + @echo "fuel" `git -C /tmp/fuel-main rev-parse HEAD` >> $(VERSION_FILE) # Remove Docker optimizations, otherwise multistrap will fail during # Fuel build. sudo rm -f /etc/apt/apt.conf.d/docker* @@ -135,13 +133,10 @@ $(ISOCACHE): # OPNFV patches at Fuel build time # Need to be commited in order for them to be considered by the Fuel # build system - cd /tmp/fuel-main/iso && git config user.name "Fuel OPNFV" - cd /tmp/fuel-main/iso && git config user.email "fuel@opnfv.org" - cd /tmp/fuel-main/iso && patch -p0 < $(TOPDIR)/bootstrap_admin_node.sh.patch - cd /tmp/fuel-main/iso && git add -u bootstrap_admin_node.sh - cd /tmp/fuel-main/iso/isolinux && patch -p0 < $(TOPDIR)/isolinux.cfg.patch - cd /tmp/fuel-main/iso/isolinux && git add -u isolinux.cfg - cd /tmp/fuel-main/iso && git commit -m "Added OPNFV patches" + cd /tmp/fuel-main && git config user.name "Fuel OPNFV" + cd /tmp/fuel-main && git config user.email "fuel@opnfv.org" + cd /tmp/fuel-main && git am $(TOPDIR)/bootstrap_admin_node.sh.patch + cd /tmp/fuel-main && git am $(TOPDIR)/isolinux.cfg.patch # Repeat build up to three times sudo -E ./fuel_build_loop cp /tmp/fuel-main/build/artifacts/fuel*.iso . diff --git a/build/bootstrap_admin_node.sh.patch b/build/bootstrap_admin_node.sh.patch index e137b740d..129339abd 100644 --- a/build/bootstrap_admin_node.sh.patch +++ b/build/bootstrap_admin_node.sh.patch @@ -1,95 +1,72 @@ -*** bootstrap_admin_node.sh.orig Mon May 30 06:31:38 2016 ---- bootstrap_admin_node.sh Mon May 30 06:35:11 2016 -*************** -*** 339,346 **** - set +x - echo "Done!" - - if [[ "$showmenu" == "yes" || "$showmenu" == "YES" ]]; then -! fuelmenu || fail - else - # Give user 15 seconds to enter fuelmenu or else continue - echo ---- 339,360 ---- - set +x - echo "Done!" - -+ ### OPNFV addition BEGIN -+ shopt -s nullglob -+ for script in /opt/opnfv/bootstrap/pre.d/*.sh -+ do -+ echo "Pre script: $script" >> /root/pre.log 2>&1 -+ $script >> /root/pre.log 2>&1 -+ done -+ shopt -u nullglob -+ ### OPNFV addition END -+ -+ # Enable sshd -+ systemctl enable sshd -+ systemctl start sshd -+ - if [[ "$showmenu" == "yes" || "$showmenu" == "YES" ]]; then -! fuelmenu - else - # Give user 15 seconds to enter fuelmenu or else continue - echo -*************** -*** 360,368 **** - fi - fi +From: Fuel OPNFV <fuel@opnfv.org> +Date: Mon, 13 Jun 2016 22:23:57 +0200 +Subject: OPNFV: Additions to bootstrap_admin_node.sh - # Enable online base MOS repos (security, updates) if we run an ISO installation -! [ -f /etc/fuel_build_id ] && \ -! yum-config-manager --enable mos${FUEL_RELEASE}-security mos${FUEL_RELEASE}-updates --save +--- +diff --git a/iso/bootstrap_admin_node.sh b/iso/bootstrap_admin_node.sh +index 3197c91..e035145 100755 +--- a/iso/bootstrap_admin_node.sh ++++ b/iso/bootstrap_admin_node.sh +@@ -339,8 +339,22 @@ fuelmenu --save-only --iface=$ADMIN_INTERFACE || fail + set +x + echo "Done!" - if [ ! -f "${ASTUTE_YAML}" ]; then - echo ${fuelmenu_fail_message} ---- 360,369 ---- - fi - fi ++### OPNFV addition BEGIN ++shopt -s nullglob ++for script in /opt/opnfv/bootstrap/pre.d/*.sh ++do ++ echo "Pre script: $script" >> /root/pre.log 2>&1 ++ $script >> /root/pre.log 2>&1 ++done ++shopt -u nullglob ++### OPNFV addition END ++ ++# Enable sshd ++systemctl enable sshd ++systemctl start sshd ++ + if [[ "$showmenu" == "yes" || "$showmenu" == "YES" ]]; then +- fuelmenu || fail ++ fuelmenu + else + # Give user 15 seconds to enter fuelmenu or else continue + echo +@@ -360,9 +374,10 @@ else + fi + fi -+ # OPNFV: Disabled to speedup installation in offline env. - # Enable online base MOS repos (security, updates) if we run an ISO installation -! #[ -f /etc/fuel_build_id ] && \ -! # yum-config-manager --enable mos${FUEL_RELEASE}-security mos${FUEL_RELEASE}-updates --save ++# OPNFV: Disabled to speedup installation in offline env. + # Enable online base MOS repos (security, updates) if we run an ISO installation +-[ -f /etc/fuel_build_id ] && \ +- yum-config-manager --enable mos${FUEL_RELEASE}-security mos${FUEL_RELEASE}-updates --save ++#[ -f /etc/fuel_build_id ] && \ ++# yum-config-manager --enable mos${FUEL_RELEASE}-security mos${FUEL_RELEASE}-updates --save - if [ ! -f "${ASTUTE_YAML}" ]; then - echo ${fuelmenu_fail_message} -*************** -*** 374,382 **** - [ ! -f /etc/fuel_build_id ] && \ - sed -i "s|127.0.0.1:8080/ubuntu/x86_64|mirror.fuel-infra.org/mos-repos/ubuntu/${FUEL_RELEASE}|g" "${ASTUTE_YAML}" - -! # Enable sshd -! systemctl enable sshd -! systemctl start sshd - - # Enable iptables - systemctl enable iptables.service ---- 388,394 ---- - [ ! -f /etc/fuel_build_id ] && \ - sed -i "s|127.0.0.1:8080/ubuntu/x86_64|mirror.fuel-infra.org/mos-repos/ubuntu/${FUEL_RELEASE}|g" "${ASTUTE_YAML}" - -! systemctl reload sshd - - # Enable iptables - systemctl enable iptables.service -*************** -*** 529,534 **** ---- 541,556 ---- - - bash /etc/rc.local - -+ ### OPNFV addition BEGIN -+ shopt -s nullglob -+ for script in /opt/opnfv/bootstrap/post.d/*.sh -+ do -+ echo "Post script: $script" >> /root/post.log 2>&1 -+ $script >> /root/post.log 2>&1 -+ done -+ shopt -u nullglob -+ ### OPNFV addition END -+ - if [ "`get_bootstrap_skip`" = "False" ]; then - build_ubuntu_bootstrap bs_status || true - else + if [ ! -f "${ASTUTE_YAML}" ]; then + echo ${fuelmenu_fail_message} +@@ -377,7 +392,5 @@ if [ ! -f /etc/fuel_build_id ]; then + +-# Enable sshd +-systemctl enable sshd +-systemctl start sshd ++systemctl reload sshd + + # Enable iptables + systemctl enable iptables.service +@@ -532,6 +545,16 @@ systemctl start ntpd + + bash /etc/rc.local + ++### OPNFV addition BEGIN ++shopt -s nullglob ++for script in /opt/opnfv/bootstrap/post.d/*.sh ++do ++ echo "Post script: $script" >> /root/post.log 2>&1 ++ $script >> /root/post.log 2>&1 ++done ++shopt -u nullglob ++### OPNFV addition END ++ + if [ "`get_bootstrap_skip`" = "False" ]; then + build_ubuntu_bootstrap bs_status || true + else diff --git a/build/config.mk b/build/config.mk index af63fcd8f..e3766a7ef 100644 --- a/build/config.mk +++ b/build/config.mk @@ -9,10 +9,25 @@ ############################################################################## FUEL_MAIN_REPO := https://github.com/openstack/fuel-main -FUEL_MAIN_TAG := 9.0 +FUEL_MAIN_TAG := 9.0.1 MOS_VERSION = 9.0 OPENSTACK_VERSION = mitaka-9.0 +# Pinning down exact Fuel repo versions for Fuel 9.0.1 +export FUELLIB_COMMIT?=e283b62750d9e26355981b3ad3be7c880944ae0f +export NAILGUN_COMMIT?=e2b85bafb68c348f25cb7cceda81edc668ba2e64 +export PYTHON_FUELCLIENT_COMMIT?=67d8c693a670d27c239d5d175f3ea2a0512c498c +export FUEL_AGENT_COMMIT?=7ffbf39caf5845bd82b8ce20a7766cf24aa803fb +export FUEL_NAILGUN_AGENT_COMMIT?=46fa0db0f8944f9e67699d281d462678aaf4db26 +export ASTUTE_COMMIT?=390b257240d49cc5e94ed5c4fcd940b5f2f6ec64 +export OSTF_COMMIT?=f09c98ff7cc71ee612b2450f68a19f2f9c64345a +export FUEL_MIRROR_COMMIT?=d1ef06b530ce2149230953bb3810a88ecaff870c +export FUELMENU_COMMIT?=0ed9e206ed1c6271121d3acf52a6bf757411286b +export SHOTGUN_COMMIT?=781a8cfa0b6eb290e730429fe2792f2b6f5e0c11 +export NETWORKCHECKER_COMMIT?=fcb47dd095a76288aacf924de574e39709e1f3ca +export FUELUPGRADE_COMMIT?=c1c4bac6a467145ac4fac73e4a7dd2b00380ecfb +export FUEL_UI_COMMIT?=90de7ef4477230cb7335453ed26ed4306ca6f04f + DOCKER_REPO := http://get.docker.com/builds/Linux/x86_64 DOCKER_TAG := docker-latest diff --git a/build/f_isoroot/f_bgpvpn-pluginbuild/config.mk b/build/f_isoroot/f_bgpvpn-pluginbuild/config.mk index 3d54e03b2..ba6fb73ff 100644 --- a/build/f_isoroot/f_bgpvpn-pluginbuild/config.mk +++ b/build/f_isoroot/f_bgpvpn-pluginbuild/config.mk @@ -9,4 +9,4 @@ BGPVPN_BRANCH?=master BGPVPN_REPO?="https://github.com/openstack/fuel-plugin-bgpvpn.git" -BGPVPN_CHANGE?=a2e5cabc2f1e04c948da7026f816549848c2bcd9 +BGPVPN_CHANGE?=3349842af5724be63a74a82c9060848d9d3d299e diff --git a/build/f_isoroot/f_collectd-ceilometer-pluginbuild/config.mk b/build/f_isoroot/f_collectd-ceilometer-pluginbuild/config.mk index 2baf67d18..4781466dd 100644 --- a/build/f_isoroot/f_collectd-ceilometer-pluginbuild/config.mk +++ b/build/f_isoroot/f_collectd-ceilometer-pluginbuild/config.mk @@ -8,6 +8,6 @@ # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## -COLLECTD_CEILOMETER_BRANCH?=6a0837238ff13d3294e5a7181fc3de40e3094625 +COLLECTD_CEILOMETER_BRANCH?=efa17094249282eb0726b3d30dff190b86a97938 COLLECTD_CEILOMETER_REPO?=http://gerrit.opnfv.org/gerrit/fastpathmetrics COLLECTD_CEILOMETER_CHANGE?= diff --git a/build/f_isoroot/f_kvm-pluginbuild/config.mk b/build/f_isoroot/f_kvm-pluginbuild/config.mk index b608adee0..416ed5e4f 100644 --- a/build/f_isoroot/f_kvm-pluginbuild/config.mk +++ b/build/f_isoroot/f_kvm-pluginbuild/config.mk @@ -7,6 +7,6 @@ # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## -KVMFORNFV_BRANCH=c715b6029fd5b4eaf323f5efde4ec5db5ba0a9b4 +KVMFORNFV_BRANCH=e9bb3d8bce40608c6890ea5298746668b8667462 KVMFORNFV_REPO=https://gerrit.opnfv.org/gerrit/kvmfornfv KVMFORNFV_CHANGE= diff --git a/build/f_isoroot/f_repobuild/select_ubuntu_repo.sh b/build/f_isoroot/f_repobuild/select_ubuntu_repo.sh index 540f7ebce..4bca6bd9b 100755 --- a/build/f_isoroot/f_repobuild/select_ubuntu_repo.sh +++ b/build/f_isoroot/f_repobuild/select_ubuntu_repo.sh @@ -21,7 +21,7 @@ RSYNC="rsync -4 --contimeout 5 --no-motd --list-only" # Some Ubuntu mirrors seem less reliable for this type of mirroring - # as they are discoved they can be added to the blacklist below in order # for them not to be considered. -BLACKLIST="mirrors.se.eu.kernel.org" +BLACKLIST="mirrors.se.eu.kernel.org mirror.its.dal.ca" return_url=0 diff --git a/build/f_isoroot/f_yardstick-pluginbuild/config.mk b/build/f_isoroot/f_yardstick-pluginbuild/config.mk index 3db4e9ce2..b87ab6f7c 100644 --- a/build/f_isoroot/f_yardstick-pluginbuild/config.mk +++ b/build/f_isoroot/f_yardstick-pluginbuild/config.mk @@ -8,6 +8,6 @@ # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## -YARDSTICK_BRANCH?=55e2e867e9a3f01b39f619251253470948e250d4 +YARDSTICK_BRANCH?=1e6ca0bfaaeadff7ce8ac9b1693b03d122e58a79 YARDSTICK_REPO?=https://gerrit.opnfv.org/gerrit/yardstick.git YARDSTICK_CHANGE?= diff --git a/build/fuel-main_docker_version.patch b/build/fuel-main_docker_version.patch deleted file mode 100644 index 22b58249b..000000000 --- a/build/fuel-main_docker_version.patch +++ /dev/null @@ -1,20 +0,0 @@ -*** prepare-build-env.sh.orig 2016-01-11 14:55:50.615286910 +0100 ---- prepare-build-env.sh 2016-01-11 14:59:44.775308422 +0100 -*************** -*** 102,108 **** - sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 36A1D7869245C8950F966E92D8576A8BA88D21E9 - # Install docker - sudo apt-get update -! sudo apt-get -y install lxc-docker-1.5.0 - fi - - # Install software ---- 102,109 ---- - sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 36A1D7869245C8950F966E92D8576A8BA88D21E9 - # Install docker - sudo apt-get update -! sudo sh -c 'echo DOCKER_OPTS=\"--bip 172.45.0.1/24\" > /etc/default/docker' -! sudo apt-get -y -o Dpkg::Options::="--force-confold" install lxc-docker-1.7.1 - fi - - # Install software diff --git a/build/isolinux.cfg.patch b/build/isolinux.cfg.patch index ebd991bd7..f09be2a7f 100644 --- a/build/isolinux.cfg.patch +++ b/build/isolinux.cfg.patch @@ -1,24 +1,21 @@ -*** isolinux.cfg.orig Tue May 10 10:13:21 2016 ---- isolinux.cfg Tue May 10 10:15:12 2016 -*************** -*** 19,27 **** - menu label ^1. Fuel Install (Static IP) - menu default - kernel vmlinuz -! append initrd=initrd.img net.ifnames=0 biosdevname=0 inst.repo=cdrom:LABEL=will_be_substituted_with_ISO_VOLUME_ID:/ inst.ks=cdrom:LABEL=will_be_substituted_with_ISO_VOLUME_ID:/ks.cfg ip=10.20.0.2::10.20.0.1:255.255.255.0:fuel.domain.tld:eth0:off::: nameserver=10.20.0.1 - - label nailgunifname - menu label ^2. Fuel Advanced Install (Static IP) - kernel vmlinuz -! append initrd=initrd.img inst.repo=cdrom:LABEL=will_be_substituted_with_ISO_VOLUME_ID:/ inst.ks=cdrom:LABEL=will_be_substituted_with_ISO_VOLUME_ID:/ks.cfg ip=10.20.0.2::10.20.0.1:255.255.255.0:fuel.domain.tld:adminif:off::: nameserver=10.20.0.1 ifname=adminif:XX:XX:XX:XX:XX:XX ---- 19,27 ---- - menu label ^1. Fuel Install (Static IP) - menu default - kernel vmlinuz -! append initrd=initrd.img net.ifnames=0 biosdevname=0 inst.repo=cdrom:LABEL=will_be_substituted_with_ISO_VOLUME_ID:/ inst.ks=cdrom:LABEL=will_be_substituted_with_ISO_VOLUME_ID:/ks.cfg ip=10.20.0.2::10.20.0.1:255.255.255.0:fuel.domain.tld:eth0:off::: nameserver=10.20.0.1 showmenu=yes - - label nailgunifname - menu label ^2. Fuel Advanced Install (Static IP) - kernel vmlinuz -! append initrd=initrd.img inst.repo=cdrom:LABEL=will_be_substituted_with_ISO_VOLUME_ID:/ inst.ks=cdrom:LABEL=will_be_substituted_with_ISO_VOLUME_ID:/ks.cfg ip=10.20.0.2::10.20.0.1:255.255.255.0:fuel.domain.tld:adminif:off::: nameserver=10.20.0.1 ifname=adminif:XX:XX:XX:XX:XX:XX showmenu=yes +From: Fuel OPNFV <fuel@opnfv.org> +Date: Mon, 13 Jun 2016 22:23:57 +0200 +Subject: OPNFV: showmenu=yes in isolinux.cfg +--- +diff --git a/iso/isolinux/isolinux.cfg b/iso/isolinux/isolinux.cfg +index c6b1ed9..77a4b18 100644 +--- a/iso/isolinux/isolinux.cfg ++++ b/iso/isolinux/isolinux.cfg +@@ -19,9 +19,9 @@ label nailgun + menu label ^1. Fuel Install (Static IP) + menu default + kernel vmlinuz +- append initrd=initrd.img net.ifnames=0 biosdevname=0 inst.repo=cdrom:LABEL=will_be_substituted_with_ISO_VOLUME_ID:/ inst.ks=cdrom:LABEL=will_be_substituted_with_ISO_VOLUME_ID:/ks.cfg ip=10.20.0.2::10.20.0.1:255.255.255.0:fuel.domain.tld:eth0:off::: nameserver=10.20.0.1 ++ append initrd=initrd.img net.ifnames=0 biosdevname=0 inst.repo=cdrom:LABEL=will_be_substituted_with_ISO_VOLUME_ID:/ inst.ks=cdrom:LABEL=will_be_substituted_with_ISO_VOLUME_ID:/ks.cfg ip=10.20.0.2::10.20.0.1:255.255.255.0:fuel.domain.tld:eth0:off::: nameserver=10.20.0.1 showmenu=yes + + label nailgunifname + menu label ^2. Fuel Advanced Install (Static IP) + kernel vmlinuz +- append initrd=initrd.img inst.repo=cdrom:LABEL=will_be_substituted_with_ISO_VOLUME_ID:/ inst.ks=cdrom:LABEL=will_be_substituted_with_ISO_VOLUME_ID:/ks.cfg ip=10.20.0.2::10.20.0.1:255.255.255.0:fuel.domain.tld:adminif:off::: nameserver=10.20.0.1 ifname=adminif:XX:XX:XX:XX:XX:XX ++ append initrd=initrd.img inst.repo=cdrom:LABEL=will_be_substituted_with_ISO_VOLUME_ID:/ inst.ks=cdrom:LABEL=will_be_substituted_with_ISO_VOLUME_ID:/ks.cfg ip=10.20.0.2::10.20.0.1:255.255.255.0:fuel.domain.tld:adminif:off::: nameserver=10.20.0.1 ifname=adminif:XX:XX:XX:XX:XX:XX showmenu=yes diff --git a/build/repo-mirror-cz.patch b/build/repo-mirror-cz.patch deleted file mode 100644 index aa8eaf9ad..000000000 --- a/build/repo-mirror-cz.patch +++ /dev/null @@ -1,19 +0,0 @@ -*** config.mk.orig Thu Jan 7 23:30:38 2016 ---- config.mk Thu Jan 7 23:32:36 2016 -*************** -*** 153,159 **** - # 'msk', 'srt', 'usa', 'hrk', 'cz' - # Setting any other value or removing of this variable will cause - # download of all the packages directly from internet -! USE_MIRROR?=ext - - ifeq ($(USE_MIRROR),ext) - MIRROR_FUEL?=http://mirror.fuel-infra.org/mos-repos/centos/$(PRODUCT_NAME)$(PRODUCT_VERSION)-centos$(CENTOS_MAJOR)-fuel/os/x86_64/ ---- 153,159 ---- - # 'msk', 'srt', 'usa', 'hrk', 'cz' - # Setting any other value or removing of this variable will cause - # download of all the packages directly from internet -! USE_MIRROR?=cz - - ifeq ($(USE_MIRROR),ext) - MIRROR_FUEL?=http://mirror.fuel-infra.org/mos-repos/centos/$(PRODUCT_NAME)$(PRODUCT_VERSION)-centos$(CENTOS_MAJOR)-fuel/os/x86_64/ diff --git a/deploy/README.templater b/deploy/README.templater new file mode 100644 index 000000000..b5d52ab9d --- /dev/null +++ b/deploy/README.templater @@ -0,0 +1,371 @@ +############################################################################## +# Copyright (c) 2016 Ericsson AB and others. +# peter.barabas@ericsson.com +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +======== TEMPLATING SUPPORT IN YAML CONFIGURATION FILES ======== + +deploy/templater.py makes it possible to use templates to generate configuration +files. It takes 2 input YAML files and an output file as arguments. One being +the dictionary (called the base file), which is used to look up values in; the +other file is the template, where the substitution will take place. Templater +will write the result to an output file, specified as the 3rd argument. + + +======== SYNTAX OF TEMPLATE FILES ======== + +A template file can contain any valid YAML data and template variables, whose +syntax is described below: + +1. Single value references + + %{title} + + %{environment/net_segment_type} + + Either a root element, or a path can be specified. + +2. YAML sections + + %{nodes} + + %{network/networking_parameters} + + Either a root element, or a path can be specified. + +3. Interface lookup for network + + %{interface(storage)} + + Specify a network type as argument to interface(). + +4. Interface lookup for network and role + + %{interface(public,compute)} + + Specify a network type and a role as arguments to interface(). + +5. File inclusion + + %{include(templates/interfaces.yaml)} + + Filename with absolute or relative path. + + +======== EXAMPLES ======== + +Base YAML file (excerpt): + +title: Deployment Environment Adapter (DEA) +version: 1.1 +created: Wed Mar 30 08:16:04 2016 +environment: + name: vCity + net_segment_type: tun +wanted_release: Liberty on Ubuntu 14.04 +nodes: +- id: 1 + interfaces: interfaces_1 + role: ceph-osd,compute + transformations: transformations_1 +- id: 2 + interfaces: interfaces_1 + role: ceph-osd,compute + transformations: transformations_1 +- id: 3 + interfaces: interfaces_1 + role: ceph-osd,compute + transformations: transformations_1 +- id: 4 + interfaces: interfaces_2 + role: controller,mongo + transformations: transformations_2 +- id: 5 + interfaces: interfaces_2 + role: controller,mongo + transformations: transformations_2 +- id: 6 + interfaces: interfaces_2 + role: controller,mongo + transformations: transformations_2 +interfaces_1: + ens3: + - fuelweb_admin + - management + ens4: + - storage + ens5: + - private + ens6: + - public +interfaces_2: + ens3: + - fuelweb_admin + - management + ens4: + - storage + - private + - public +network: + networks: + - cidr: 172.16.0.0/24 + gateway: 172.16.0.1 + ip_ranges: + - - 172.16.0.2 + - 172.16.0.126 + meta: + cidr: 172.16.0.0/24 + configurable: true + floating_range_var: floating_ranges + ip_range: + - 172.16.0.2 + - 172.16.0.126 + map_priority: 1 + name: public + notation: ip_ranges + render_addr_mask: public + render_type: null + use_gateway: true + vips: + - haproxy + - vrouter + vlan_start: null + name: public + vlan_start: null + - cidr: 192.168.1.0/24 + gateway: null + ip_ranges: + - - 192.168.1.1 + - 192.168.1.254 + meta: + cidr: 192.168.1.0/24 + configurable: true + map_priority: 2 + name: storage + notation: cidr + render_addr_mask: storage + render_type: cidr + use_gateway: false + vlan_start: 102 + name: storage + vlan_start: 102 + + +--- Example 1 --- + +Template file: + +deployment-scenario-metadata: + title: %{title} + version: 0.1 +dea-override-config: + environment: + net_segment_type: %{environment/net_segment_type} + nodes: + %{nodes} + + +Result: + +deployment-scenario-metadata: + title: Deployment Environment Adapter (DEA) + version: 0.1 +dea-override-config: + environment: + net_segment_type: tun + nodes: + - id: 1 + interfaces: interfaces_1 + role: ceph-osd,compute + transformations: transformations_1 + - id: 2 + interfaces: interfaces_1 + role: ceph-osd,compute + transformations: transformations_1 + - id: 3 + interfaces: interfaces_1 + role: ceph-osd,compute + transformations: transformations_1 + - id: 4 + interfaces: interfaces_2 + role: controller,mongo + transformations: transformations_2 + - id: 5 + interfaces: interfaces_2 + role: controller,mongo + transformations: transformations_2 + - id: 6 + interfaces: interfaces_2 + role: controller,mongo + transformations: transformations_2 + + +--- Example 2 --- + +Template file: + +dea-override-config: + network: + networks: + %{network/networks} + + +Result: + +dea-override-config: + network: + networks: + - cidr: 172.16.0.0/24 + gateway: 172.16.0.1 + ip_ranges: + - - 172.16.0.2 + - 172.16.0.126 + meta: + cidr: 172.16.0.0/24 + configurable: true + floating_range_var: floating_ranges + ip_range: + - 172.16.0.2 + - 172.16.0.126 + map_priority: 1 + name: public + notation: ip_ranges + render_addr_mask: public + render_type: null + use_gateway: true + vips: + - haproxy + - vrouter + vlan_start: null + name: public + vlan_start: null + - cidr: 192.168.1.0/24 + gateway: null + ip_ranges: + - - 192.168.1.1 + - 192.168.1.254 + meta: + cidr: 192.168.1.0/24 + configurable: true + map_priority: 2 + name: storage + notation: cidr + render_addr_mask: storage + render_type: cidr + use_gateway: false + vlan_start: 102 + name: storage + vlan_start: 102 + + +--- Example 3 --- + +Template file: + +storage_if: %{interface(storage)} +compute_private_if: %{interface(private,compute)} +# Management interface of a mongo node +mongo_mgmt_if: %{interface(management,mongo)} +controller_private_if: %{interface(private,controller)} + + +Result: + +storage_if: ens4 +compute_private_if: ens5 +# Management interface of a mongo node +mongo_mgmt_if: ens3 +controller_private_if: ens4 + + +--- Example 4 --- + +Template file: + +version: 1.1 +created: Mon Jun 13 19:39:35 2016 +comment: None +%{include(environment.yaml)} + + +environment.yaml: + +environment: + name: F9-NOSDN-NOFEATURE-VXLAN-BAREMETAL + net_segment_type: tun + + +Result: + +version: 1.1 +created: Mon Jun 13 19:39:35 2016 +comment: None +environment: + name: F9-NOSDN-NOFEATURE-VXLAN-BAREMETAL + net_segment_type: tun + + +--- Example 5 --- + +Template file (except): + +settings: + editable: + access: + email: + description: Email address for Administrator + label: Email + regex: + error: Invalid email + source: ^\S+@\S+$ + type: text + value: admin@localhost + weight: 40 +# ... +# lines omitted for brevity + %{include(templates/cgroups.yaml)} + + +cgroups.yaml: + + cgroups: + metadata: + always_editable: true + group: general + label: Cgroups conguration for services + restrictions: + - action: hide + condition: 'true' + weight: 90 + + +Result: + +settings: + editable: + access: + email: + description: Email address for Administrator + label: Email + regex: + error: Invalid email + source: ^\S+@\S+$ + type: text + value: admin@localhost + weight: 40 +# ... +# again, lines omitted for brevity + cgroups: + metadata: + always_editable: true + group: general + label: Cgroups conguration for services + restrictions: + - action: hide + condition: 'true' + weight: 90 + diff --git a/deploy/cloud/configure_nodes.py b/deploy/cloud/configure_nodes.py index b4875cc6a..20ecc1724 100644 --- a/deploy/cloud/configure_nodes.py +++ b/deploy/cloud/configure_nodes.py @@ -7,10 +7,12 @@ # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################### +import copy +import glob +import io +import six import yaml -import io -import glob from common import ( exec_cmd, @@ -47,6 +49,11 @@ class ConfigureNodes(object): # need it for the network config. self.download_deployment_config() for node_id, roles_blade in self.node_id_roles_dict.iteritems(): + # Modify node attributes + self.download_attributes(node_id) + self.modify_node_attributes(node_id, roles_blade) + self.upload_attributes(node_id) + # Modify interfaces configuration self.download_interface_config(node_id) self.modify_node_interface(node_id, roles_blade) self.modify_node_network_schemes(node_id, roles_blade) @@ -93,6 +100,35 @@ class ConfigureNodes(object): exec_cmd('fuel node --env %s --node %s --network --upload ' '--dir %s' % (self.env_id, node_id, self.yaml_config_dir)) + def download_attributes(self, node_id): + log('Download attributes for node %s' % node_id) + exec_cmd('fuel node --env %s --node %s --attributes --download ' + '--dir %s' % (self.env_id, node_id, self.yaml_config_dir)) + + def upload_attributes(self, node_id): + log('Upload attributes for node %s' % node_id) + exec_cmd('fuel node --env %s --node %s --attributes --upload ' + '--dir %s' % (self.env_id, node_id, self.yaml_config_dir)) + + def modify_node_attributes(self, node_id, roles_blade): + log('Modify attributes for node {0}'.format(node_id)) + dea_key = self.dea.get_node_property(roles_blade[1], 'attributes') + if not dea_key: + # Node attributes are not overridden. Nothing to do. + return + new_attributes = self.dea.get_property(dea_key) + attributes_yaml = ('%s/node_%s/attributes.yaml' + % (self.yaml_config_dir, node_id)) + check_file_exists(attributes_yaml) + backup('%s/node_%s' % (self.yaml_config_dir, node_id)) + + with open(attributes_yaml) as stream: + attributes = yaml.load(stream) + result_attributes = self._merge_dicts(attributes, new_attributes) + + with open(attributes_yaml, 'w') as stream: + yaml.dump(result_attributes, stream, default_flow_style=False) + def modify_node_interface(self, node_id, roles_blade): log('Modify interface config for node %s' % node_id) interface_yaml = ('%s/node_%s/interfaces.yaml' @@ -122,3 +158,17 @@ class ConfigureNodes(object): with io.open(interface_yaml, 'w') as stream: yaml.dump(interfaces, stream, default_flow_style=False) + + def _merge_dicts(self, dict1, dict2): + """Recursively merge dictionaries.""" + result = copy.deepcopy(dict1) + for k, v in six.iteritems(dict2): + if isinstance(result.get(k), list) and isinstance(v, list): + result[k].extend(v) + continue + if isinstance(result.get(k), dict) and isinstance(v, dict): + result[k] = self._merge_dicts(result[k], v) + continue + result[k] = copy.deepcopy(v) + return result + diff --git a/deploy/config/plugins/fuel-nfvkvm_0.9.0.yaml b/deploy/config/plugins/fuel-nfvkvm_0.9.0.yaml new file mode 100644 index 000000000..85f3221f7 --- /dev/null +++ b/deploy/config/plugins/fuel-nfvkvm_0.9.0.yaml @@ -0,0 +1,34 @@ +############################################################################## +# Copyright (c) 2015 Ericsson AB and others. +# jonas.bjurel@ericsson.com +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +plugin-config-metadata: + title: NFV KVM fuel Plugin configuration template + version: 0.9.0 + created: 27.07.2016 + comment: None + +fuel-plugin-kvm: + metadata: + #chosen_id: Assigned during installation + class: plugin + default: false + enabled: true + label: fuel-plugin-kvm + toggleable: true + versions: + - metadata: + always_editable: false + #plugin_id: 1 Assigned during installation + plugin_version: 0.9.0 + use_kvm: + label: 'EXPERIMENTAL: KVM enhancements for NFV' + type: checkbox + value: true + weight: 20 + weight: 70 diff --git a/deploy/deploy-config.py b/deploy/deploy-config.py index 65d51b228..d87103b6c 100644 --- a/deploy/deploy-config.py +++ b/deploy/deploy-config.py @@ -21,6 +21,7 @@ # 2) deployment-scenario dha-override-config section ############################################################################### + import os import yaml import sys @@ -44,24 +45,38 @@ from common import ( ArgParser, ) + def parse_arguments(): parser = ArgParser(prog='python %s' % __file__) parser.add_argument('-dha', dest='dha_uri', action='store', - default=False, help='dha configuration file FQDN URI', required=True) + default=False, + help='dha configuration file FQDN URI', + required=True) parser.add_argument('-deab', dest='dea_base_uri', action='store', - default=False, help='dea base configuration FQDN URI', required=True) - parser.add_argument('-deao', dest='dea_pod_override_uri', action='store', - default=False, help='dea POD override configuration FQDN URI', + default=False, + help='dea base configuration FQDN URI', required=True) - parser.add_argument('-scenario-base-uri', dest='scenario_base_uri', action='store', - default=False, help='Deploymen scenario base directory URI', + parser.add_argument('-deao', dest='dea_pod_override_uri', + action='store', + default=False, + help='dea POD override configuration FQDN URI', + required=True) + parser.add_argument('-scenario-base-uri', + dest='scenario_base_uri', + action='store', + default=False, + help='Deployment scenario base directory URI', required=True) parser.add_argument('-scenario', dest='scenario', action='store', - default=False, help='Deploymen scenario short-name (priority), or base file name (in the absense of a shortname defenition)', + default=False, + help=('Deployment scenario short-name (priority),' + 'or base file name (in the absense of a' + 'shortname defenition)'), required=True) parser.add_argument('-plugins', dest='plugins_uri', action='store', - default=False, help='Plugin configurations directory URI', + default=False, + help='Plugin configurations directory URI', required=True) parser.add_argument('-output', dest='output_path', action='store', default=False, @@ -78,6 +93,7 @@ def parse_arguments(): 'output_path': args.output_path} return kwargs + def warning(msg): red = '\033[0;31m' NC = '\033[0m' @@ -85,10 +101,12 @@ def warning(msg): 'msg': msg, 'NC': NC}) + def setup_yaml(): - represent_dict_order = lambda self, data: self.represent_mapping('tag:yaml.org,2002:map', data.items()) + represent_dict_order = lambda self, data: self.represent_mapping('tag:yaml.org,2002:map', data.items()) yaml.add_representer(collections.OrderedDict, represent_dict_order) + def sha_uri(uri): response = urllib2.urlopen(uri) data = response.read() @@ -96,34 +114,33 @@ def sha_uri(uri): sha1.update(data) return sha1.hexdigest() + def merge_fuel_plugin_version_list(list1, list2): final_list = [] # When the plugin version in not there in list1 it will # not be copied for e_l1 in list1: - plugin_version = e_l1.get('metadata', - {'plugin_version', None}).get('plugin_version') + plugin_version = e_l1.get('metadata', {}).get('plugin_version') plugin_version_found = False for e_l2 in list2: - if plugin_version == e_l2.get('metadata', - {'plugin_version', - None}).get('plugin_version'): - final_list.append(dict(mergedicts(e_l1, e_l2))) + if plugin_version == e_l2.get('metadata', {}).get('plugin_version'): + final_list.append(dict(merge_dicts(e_l1, e_l2))) plugin_version_found = True if not plugin_version_found: final_list.append(e_l1) return final_list + def merge_lists(list1, list2): if list1 and list2: if isinstance(list1[0], dict): if 'plugin_version' in list1[0].get('metadata', {}): return merge_fuel_plugin_version_list(list1, list2) else: - warning("Lists with dictionary inside are not merge able! " - "List2 will overwrite List1. " - "List1: %s; List2: %s" - % (list1, list2)) + warning("Lists with dictionary inside are not mergeable! " + "List2 will overwrite List1. " + "List1: %s\nList2: %s" + % (list1, list2)) return list2 else: return list2 @@ -132,11 +149,12 @@ def merge_lists(list1, list2): else: return list2 -def mergedicts(dict1, dict2): - for k in set(dict1.keys()).union(dict2.keys()): + +def merge_dicts(dict1, dict2): + for k in set(dict1).union(dict2): if k in dict1 and k in dict2: if isinstance(dict1[k], dict) and isinstance(dict2[k], dict): - yield (k, dict(mergedicts(dict1[k], dict2[k]))) + yield (k, dict(merge_dicts(dict1[k], dict2[k]))) elif isinstance(dict1[k], list) and isinstance(dict2[k], list): yield (k, merge_lists(dict1[k], dict2[k])) else: @@ -149,16 +167,17 @@ def mergedicts(dict1, dict2): else: yield (k, dict2[k]) + setup_yaml() kwargs = parse_arguments() # Generate final dea.yaml by merging following config files/fragments in revers priority order: # "dea-base", "dea-pod-override", "deplyment-scenario/module-config-override" # and "deployment-scenario/dea-override" -print 'Generating final dea.yaml configuration....' +print('Generating final dea.yaml configuration....') # Fetch dea-base, extract and purge meta-data -print 'Parsing dea-base from: ' + kwargs["dea_base_uri"] + "...." +print('Parsing dea-base from: ' + kwargs["dea_base_uri"] + "....") response = urllib2.urlopen(kwargs["dea_base_uri"]) dea_base_conf = yaml.load(response.read()) dea_base_title = dea_base_conf['dea-base-config-metadata']['title'] @@ -170,7 +189,7 @@ dea_base_conf.pop('dea-base-config-metadata') final_dea_conf = dea_base_conf # Fetch dea-pod-override, extract and purge meta-data, merge with previous dea data structure -print 'Parsing the dea-pod-override from: ' + kwargs["dea_pod_override_uri"] + "...." +print('Parsing the dea-pod-override from: ' + kwargs["dea_pod_override_uri"] + "....") response = urllib2.urlopen(kwargs["dea_pod_override_uri"]) dea_pod_override_conf = yaml.load(response.read()) if dea_pod_override_conf: @@ -179,19 +198,21 @@ if dea_pod_override_conf: dea_pod_creation = dea_pod_override_conf['dea-pod-override-config-metadata']['created'] dea_pod_sha = sha_uri(kwargs["dea_pod_override_uri"]) dea_pod_comment = dea_pod_override_conf['dea-pod-override-config-metadata']['comment'] - print 'Merging dea-base and dea-pod-override configuration ....' + print('Merging dea-base and dea-pod-override configuration ....') dea_pod_override_conf.pop('dea-pod-override-config-metadata') if dea_pod_override_conf: - final_dea_conf = dict(mergedicts(final_dea_conf, dea_pod_override_conf)) + final_dea_conf = dict(merge_dicts(final_dea_conf, dea_pod_override_conf)) # Fetch deployment-scenario, extract and purge meta-data, merge deployment-scenario/ # dea-override-configith previous dea data structure -print 'Parsing deployment-scenario from: ' + kwargs["scenario"] + "...." +print('Parsing deployment-scenario from: ' + kwargs["scenario"] + "....") response = urllib2.urlopen(kwargs["scenario_base_uri"] + "/scenario.yaml") scenario_short_translation_conf = yaml.load(response.read()) if kwargs["scenario"] in scenario_short_translation_conf: - scenario_uri = kwargs["scenario_base_uri"] + "/" + scenario_short_translation_conf[kwargs["scenario"]]['configfile'] + scenario_uri = (kwargs["scenario_base_uri"] + + "/" + + scenario_short_translation_conf[kwargs["scenario"]]['configfile']) else: scenario_uri = kwargs["scenario_base_uri"] + "/" + kwargs["scenario"] response = urllib2.urlopen(scenario_uri) @@ -205,14 +226,15 @@ if deploy_scenario_conf: deploy_scenario_comment = deploy_scenario_conf['deployment-scenario-metadata']['comment'] deploy_scenario_conf.pop('deployment-scenario-metadata') else: - print "Deployment scenario file not found or is empty" - print "Cannot continue, exiting ...." + print("Deployment scenario file not found or is empty") + print("Cannot continue, exiting ....") sys.exit(1) dea_scenario_override_conf = deploy_scenario_conf["dea-override-config"] if dea_scenario_override_conf: - print 'Merging dea-base-, dea-pod-override- and deployment-scenario configuration into final dea.yaml configuration....' - final_dea_conf = dict(mergedicts(final_dea_conf, dea_scenario_override_conf)) + print('Merging dea-base-, dea-pod-override- and deployment-scenario ' + 'configuration into final dea.yaml configuration....') + final_dea_conf = dict(merge_dicts(final_dea_conf, dea_scenario_override_conf)) # Fetch plugin-configuration configuration files, extract and purge meta-data, # merge/append with previous dea data structure, override plugin-configuration with @@ -226,15 +248,32 @@ module_shas = [] module_comments = [] if deploy_scenario_conf["stack-extensions"]: for module in deploy_scenario_conf["stack-extensions"]: - print 'Loading configuration for module: ' + module["module"] + ' and merging it to final dea.yaml configuration....' - response = urllib2.urlopen(kwargs["plugins_uri"] + '/' + module["module-config-name"] + '_' + module["module-config-version"] + '.yaml') + print('Loading configuration for module: ' + + module["module"] + + ' and merging it to final dea.yaml configuration....') + response = urllib2.urlopen(kwargs["plugins_uri"] + + '/' + + module["module-config-name"] + + '_' + + module["module-config-version"] + + '.yaml') module_conf = yaml.load(response.read()) modules.append(module["module"]) - module_uris.append(kwargs["plugins_uri"] + '/' + module["module-config-name"] + '_' + module["module-config-version"] + '.yaml') + module_uris.append(kwargs["plugins_uri"] + + '/' + + module["module-config-name"] + + '_' + + module["module-config-version"] + + '.yaml') module_titles.append(str(module_conf['plugin-config-metadata']['title'])) module_versions.append(str(module_conf['plugin-config-metadata']['version'])) module_creations.append(str(module_conf['plugin-config-metadata']['created'])) - module_shas.append(sha_uri(kwargs["plugins_uri"] + '/' + module["module-config-name"] + '_' + module["module-config-version"] + '.yaml')) + module_shas.append(sha_uri(kwargs["plugins_uri"] + + '/' + + module["module-config-name"] + + '_' + + module["module-config-version"] + + '.yaml')) module_comments.append(str(module_conf['plugin-config-metadata']['comment'])) module_conf.pop('plugin-config-metadata') final_dea_conf['settings']['editable'].update(module_conf) @@ -244,59 +283,62 @@ if deploy_scenario_conf["stack-extensions"]: dea_scenario_module_override_conf['settings'] = {} dea_scenario_module_override_conf['settings']['editable'] = {} dea_scenario_module_override_conf['settings']['editable'][module["module"]] = scenario_module_override_conf - final_dea_conf = dict(mergedicts(final_dea_conf, dea_scenario_module_override_conf)) + final_dea_conf = dict(merge_dicts(final_dea_conf, dea_scenario_module_override_conf)) # Dump final dea.yaml including configuration management meta-data to argument provided # directory if not os.path.exists(kwargs["output_path"]): os.makedirs(kwargs["output_path"]) -print 'Dumping final dea.yaml to ' + kwargs["output_path"] + '/dea.yaml....' +print('Dumping final dea.yaml to ' + kwargs["output_path"] + '/dea.yaml....') with open(kwargs["output_path"] + '/dea.yaml', "w") as f: - f.write("title: DEA.yaml file automatically generated from the configuration files stated in the \"configuration-files\" fragment below\n") - f.write("version: " + str(calendar.timegm(time.gmtime())) + "\n") - f.write("created: " + str(time.strftime("%d/%m/%Y")) + " " + str(time.strftime("%H:%M:%S")) + "\n") - f.write("comment: none\n") - - f.write("configuration-files:\n") - f.write(" dea-base:\n") - f.write(" uri: " + kwargs["dea_base_uri"] + "\n") - f.write(" title: " + str(dea_base_title) + "\n") - f.write(" version: " + str(dea_base_version) + "\n") - f.write(" created: " + str(dea_base_creation) + "\n") - f.write(" sha1: " + str(dea_base_sha) + "\n") - f.write(" comment: " + str(dea_base_comment) + "\n") - - f.write(" pod-override:\n") - f.write(" uri: " + kwargs["dea_pod_override_uri"] + "\n") - f.write(" title: " + str(dea_pod_title) + "\n") - f.write(" version: " + str(dea_pod_version) + "\n") - f.write(" created: " + str(dea_pod_creation) + "\n") - f.write(" sha1: " + str(dea_pod_sha) + "\n") - f.write(" comment: " + str(dea_pod_comment) + "\n") - - f.write(" deployment-scenario:\n") - f.write(" uri: " + str(scenario_uri) + "\n") - f.write(" title: " + str(deploy_scenario_title) + "\n") - f.write(" version: " + str(deploy_scenario_version) + "\n") - f.write(" created: " + str(deploy_scenario_creation) + "\n") - f.write(" sha1: " + str(deploy_scenario_sha) + "\n") - f.write(" comment: " + str(deploy_scenario_comment) + "\n") + f.write("\n".join([("title: DEA.yaml file automatically generated from the" + 'configuration files stated in the "configuration-files"' + "fragment below"), + "version: " + str(calendar.timegm(time.gmtime())), + "created: " + str(time.strftime("%d/%m/%Y")) + " " + + str(time.strftime("%H:%M:%S")), + "comment: none\n"])) + + f.write("\n".join(["configuration-files:", + " dea-base:", + " uri: " + kwargs["dea_base_uri"], + " title: " + str(dea_base_title), + " version: " + str(dea_base_version), + " created: " + str(dea_base_creation), + " sha1: " + str(dea_base_sha), + " comment: " + str(dea_base_comment) + "\n"])) + + f.write("\n".join([" pod-override:", + " uri: " + kwargs["dea_pod_override_uri"], + " title: " + str(dea_pod_title), + " version: " + str(dea_pod_version), + " created: " + str(dea_pod_creation), + " sha1: " + str(dea_pod_sha), + " comment: " + str(dea_pod_comment) + "\n"])) + + f.write("\n".join([" deployment-scenario:", + " uri: " + str(scenario_uri), + " title: " + str(deploy_scenario_title), + " version: " + str(deploy_scenario_version), + " created: " + str(deploy_scenario_creation), + " sha1: " + str(deploy_scenario_sha), + " comment: " + str(deploy_scenario_comment) + "\n"])) f.write(" plugin-modules:\n") - for k in range(0,len(modules)): - f.write(" - module: " + modules[k] + "\n") - f.write(" uri: " + module_uris[k] + "\n") - f.write(" title: " + module_titles[k] + "\n") - f.write(" version: " + module_versions[k] + "\n") - f.write(" created: " + module_creations[k] + "\n") - f.write(" sha-1: " + module_shas[k] + "\n") - f.write(" comment: " + module_comments[k] + "\n") + for k, _ in enumerate(modules): + f.write("\n".join([" - module: " + modules[k], + " uri: " + module_uris[k], + " title: " + module_titles[k], + " version: " + module_versions[k], + " created: " + module_creations[k], + " sha-1: " + module_shas[k], + " comment: " + module_comments[k] + "\n"])) yaml.dump(final_dea_conf, f, default_flow_style=False) # Load POD dha and override it with "deployment-scenario/dha-override-config" section -print 'Generating final dha.yaml configuration....' -print 'Parsing dha-pod yaml configuration....' +print('Generating final dha.yaml configuration....') +print('Parsing dha-pod yaml configuration....') response = urllib2.urlopen(kwargs["dha_uri"]) dha_pod_conf = yaml.load(response.read()) dha_pod_title = dha_pod_conf['dha-pod-config-metadata']['title'] @@ -312,33 +354,39 @@ dha_scenario_override_conf = deploy_scenario_conf["dha-override-config"] # is no way to programatically override a physical environment: # wireing, IPMI set-up, etc. # For Physical environments, dha.yaml overrides will be silently ignored -if dha_scenario_override_conf and (final_dha_conf['adapter'] == 'libvirt' or final_dha_conf['adapter'] == 'esxi' or final_dha_conf['adapter'] == 'vbox'): - print 'Merging dha-pod and deployment-scenario override information to final dha.yaml configuration....' - final_dha_conf = dict(mergedicts(final_dha_conf, dha_scenario_override_conf)) +if dha_scenario_override_conf and (final_dha_conf['adapter'] == 'libvirt' + or final_dha_conf['adapter'] == 'esxi' + or final_dha_conf['adapter'] == 'vbox'): + print('Merging dha-pod and deployment-scenario override information to final dha.yaml configuration....') + final_dha_conf = dict(merge_dicts(final_dha_conf, dha_scenario_override_conf)) # Dump final dha.yaml to argument provided directory -print 'Dumping final dha.yaml to ' + kwargs["output_path"] + '/dha.yaml....' +print('Dumping final dha.yaml to ' + kwargs["output_path"] + '/dha.yaml....') with open(kwargs["output_path"] + '/dha.yaml', "w") as f: - f.write("title: DHA.yaml file automatically generated from the configuration files stated in the \"configuration-files\" fragment below\n") - f.write("version: " + str(calendar.timegm(time.gmtime())) + "\n") - f.write("created: " + str(time.strftime("%d/%m/%Y")) + " " + str(time.strftime("%H:%M:%S")) + "\n") - f.write("comment: none\n") + f.write("\n".join([("title: DHA.yaml file automatically generated from" + "the configuration files stated in the" + '"configuration-files" fragment below'), + "version: " + str(calendar.timegm(time.gmtime())), + "created: " + str(time.strftime("%d/%m/%Y")) + " " + + str(time.strftime("%H:%M:%S")), + "comment: none\n"])) f.write("configuration-files:\n") - f.write(" dha-pod-configuration:\n") - f.write(" uri: " + kwargs["dha_uri"] + "\n") - f.write(" title: " + str(dha_pod_title) + "\n") - f.write(" version: " + str(dha_pod_version) + "\n") - f.write(" created: " + str(dha_pod_creation) + "\n") - f.write(" sha-1: " + str(dha_pod_sha) + "\n") - f.write(" comment: " + str(dha_pod_comment) + "\n") - - f.write(" deployment-scenario:\n") - f.write(" uri: " + str(scenario_uri) + "\n") - f.write(" title: " + str(deploy_scenario_title) + "\n") - f.write(" version: " + str(deploy_scenario_version) + "\n") - f.write(" created: " + str(deploy_scenario_creation) + "\n") - f.write(" sha-1: " + str(deploy_scenario_sha) + "\n") - f.write(" comment: " + str(deploy_scenario_comment) + "\n") + f.write("\n".join([" dha-pod-configuration:", + " uri: " + kwargs["dha_uri"], + " title: " + str(dha_pod_title), + " version: " + str(dha_pod_version), + " created: " + str(dha_pod_creation), + " sha-1: " + str(dha_pod_sha), + " comment: " + str(dha_pod_comment) + "\n"])) + + f.write("\n".join([" deployment-scenario:", + " uri: " + str(scenario_uri), + " title: " + str(deploy_scenario_title), + " version: " + str(deploy_scenario_version), + " created: " + str(deploy_scenario_creation), + " sha-1: " + str(deploy_scenario_sha), + " comment: " + str(deploy_scenario_comment) + "\n"])) + yaml.dump(final_dha_conf, f, default_flow_style=False) diff --git a/deploy/reap.py b/deploy/reap.py index ed5bc994e..7d996314b 100644 --- a/deploy/reap.py +++ b/deploy/reap.py @@ -1,7 +1,8 @@ #!/usr/bin/python ############################################################################### -# Copyright (c) 2015 Ericsson AB and others. +# Copyright (c) 2015, 2016 Ericsson AB and others. # szilard.cserey@ericsson.com +# peter.barabas@ericsson.com # All rights reserved. This program and the accompanying materials # are made available under the terms of the Apache License, Version 2.0 # which accompanies this distribution, and is available at @@ -76,6 +77,8 @@ DHA_2 = ''' # which may not be correct - please adjust as needed. ''' +TEMPLATER = 'templater.py' + DISKS = {'fuel': '100G', 'controller': '100G', 'compute': '100G'} @@ -83,10 +86,12 @@ DISKS = {'fuel': '100G', class Reap(object): - def __init__(self, dea_file, dha_file, comment): + def __init__(self, dea_file, dha_file, comment, base_dea, template): self.dea_file = dea_file self.dha_file = dha_file self.comment = comment + self.base_dea = base_dea + self.template = template self.temp_dir = None self.env = None self.env_id = None @@ -293,6 +298,7 @@ class Reap(object): def intro(self): delete(self.dea_file) delete(self.dha_file) + self.temp_dir = tempfile.mkdtemp() date = time.strftime('%c') self.write(self.dea_file, @@ -310,10 +316,16 @@ class Reap(object): self.download_config('settings') self.download_config('network') + def create_base_dea(self): + exec_cmd('python %s %s %s %s' + % (TEMPLATER, self.dea_file, self.template, self.base_dea)) + def finale(self): log('DEA file is available at %s' % self.dea_file) log('DHA file is available at %s (this is just a template)' % self.dha_file) + if self.base_dea: + log('DEA base file is available at %s' % self.base_dea) shutil.rmtree(self.temp_dir) def reap(self): @@ -323,6 +335,8 @@ class Reap(object): self.reap_fuel_settings() self.reap_network_settings() self.reap_settings() + if self.base_dea: + self.create_base_dea() self.finale() @@ -335,14 +349,26 @@ def parse_arguments(): default='dha.yaml', help='Deployment Hardware Adapter: dha.yaml') parser.add_argument('comment', nargs='?', action='store', help='Comment') + parser.add_argument('-base_dea', + dest='base_dea', + help='Create specified base DEA file from "dea_file"') + parser.add_argument('-template', + dest='template', + nargs='?', + default='base_dea_template.yaml', + help='Base DEA is generated from this template') args = parser.parse_args() - return (args.dea_file, args.dha_file, args.comment) + return (args.dea_file, + args.dha_file, + args.comment, + args.base_dea, + args.template) def main(): - dea_file, dha_file, comment = parse_arguments() + dea_file, dha_file, comment, base_dea, template = parse_arguments() - r = Reap(dea_file, dha_file, comment) + r = Reap(dea_file, dha_file, comment, base_dea, template) r.reap() diff --git a/deploy/scenario/ha_nfv-kvm_heat_ceilometer_scenario.yaml b/deploy/scenario/ha_nfv-kvm_heat_ceilometer_scenario.yaml index 2941a726e..f9863d11e 100644 --- a/deploy/scenario/ha_nfv-kvm_heat_ceilometer_scenario.yaml +++ b/deploy/scenario/ha_nfv-kvm_heat_ceilometer_scenario.yaml @@ -22,8 +22,8 @@ # deployment configuration meta-data deployment-scenario-metadata: title: NFV KVM HA deployment - version: 0.0.5 - created: Mar 18 2016 + version: 0.0.6 + created: 27.07.2016 comment: Ceph cannot be assigned to compute, ceph is collocated with one of the controllers ############################################################################## @@ -32,9 +32,9 @@ deployment-scenario-metadata: # <module-config-base-uri>/<module-config-name>_<module-config-version>.yaml # It does so by copying the config file to the local plugin config directory stack-extensions: - - module: fuel-plugin-qemu + - module: fuel-plugin-kvm module-config-name: fuel-nfvkvm - module-config-version: 0.0.2 + module-config-version: 0.9.0 module-config-override: # Module config overrides diff --git a/deploy/templater.py b/deploy/templater.py new file mode 100755 index 000000000..6b41e1f3c --- /dev/null +++ b/deploy/templater.py @@ -0,0 +1,185 @@ +#!/usr/bin/env python +############################################################################### +# Copyright (c) 2016 Ericsson AB and others. +# peter.barabas@ericsson.com +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################### + + +import io +import re +import yaml +from common import( + err, + ArgParser, +) + + +TAG_START = '%{' +TAG_END = '}' +DELIMITER = '/' + + +class Templater(object): + def __init__(self, base_file, template_file, output_file): + self.template_file = template_file + self.output_file = output_file + self.base = self.load_yaml(base_file) + + def load_yaml(self, filename): + try: + with io.open(filename) as yaml_file: + return yaml.load(yaml_file) + except Exception as error: + err('Error opening YAML file: %s' % error) + + def save_yaml(self, filename, content): + try: + with io.open(filename, 'w') as yaml_file: + yaml_file.write(content) + except Exception as error: + err('Error writing YAML file: %s' % error) + + def get_indent(self, line): + return len(line) - len(line.lstrip(' ')) + + def format_fragment(self, fragment, indent): + result = '' + is_first_line = True + + for line in fragment.splitlines(): + # Skip indenting the first line as it is already indented + if is_first_line: + line += '\n' + is_first_line = False + else: + line = ' ' * indent + line + '\n' + + result += line + + return result.rstrip('\n') + + def format_substitution(self, string): + if isinstance(string, basestring): + return string + else: + return yaml.dump(string, default_flow_style=False) + + def parse_interface_tag(self, tag): + # Remove 'interface(' prefix, trailing ')' and split arguments + args = tag[len('interface('):].rstrip(')').split(',') + + if len(args) == 1 and not args[0]: + err('No arguments for interface().') + elif len(args) == 2 and (not args[0] or not args[1]): + err('Empty argument for interface().') + elif len(args) > 2: + err('Too many arguments for interface().') + else: + return args + + def get_interface_from_network(self, interfaces, network): + nics = self.base[interfaces] + for nic in nics: + if network in nics[nic]: + return nic + + err('Network not found: %s' % network) + + def get_role_interfaces(self, role): + nodes = self.base['nodes'] + for node in nodes: + if role in node['role']: + return node['interfaces'] + + err('Role not found: %s' % role) + + def lookup_interface(self, args): + nodes = self.base['nodes'] + + if len(args) == 1: + interfaces = nodes[0]['interfaces'] + if len(args) == 2: + interfaces = self.get_role_interfaces(args[1]) + + return self.get_interface_from_network(interfaces, args[0]) + + def parse_include_tag(self, tag): + # Remove 'include(' prefix and trailing ')' + filename = tag[len('include('):].rstrip(')') + + if not filename: + err('No argument for include().') + + return filename + + def include_file(self, filename): + fragment = self.load_yaml(filename) + return yaml.dump(fragment, default_flow_style=False) + + def parse_tag(self, tag, indent): + fragment = '' + + if 'interface(' in tag: + args = self.parse_interface_tag(tag) + fragment = self.lookup_interface(args) + elif 'include(' in tag: + filename = self.parse_include_tag(tag) + fragment = self.include_file(filename) + else: + path = tag.split(DELIMITER) + fragment = self.base + for i in path: + if i in fragment: + fragment = fragment.get(i) + else: + err('Error: key "%s" does not exist in base YAML file' % i) + + fragment = self.format_substitution(fragment) + + return self.format_fragment(fragment, indent) + + def run(self): + result = '' + + regex = re.compile(re.escape(TAG_START) + r'([a-z].+)' + re.escape(TAG_END), + flags=re.IGNORECASE) + with io.open(self.template_file) as f: + for line in f: + indent = self.get_indent(line) + result += re.sub(regex, + lambda match: self.parse_tag(match.group(1), indent), + line) + + self.save_yaml(self.output_file, result) + + +def parse_arguments(): + description = '''Process 'template_file' using 'base_file' as source for +template variable substitution and write the results to 'output_file'.''' + + parser = ArgParser(prog='python %s' % __file__, + description=description) + parser.add_argument('base_file', + help='Base YAML filename') + parser.add_argument('template_file', + help='Fragment filename') + parser.add_argument('output_file', + help='Output filename') + + args = parser.parse_args() + return(args.base_file, args.template_file, args.output_file) + + +def main(): + base_file, template_file, output_file = parse_arguments() + + templater = Templater(base_file, template_file, output_file) + templater.run() + + +if __name__ == '__main__': + main() diff --git a/deploy/templates/templater/base_dea_template.yaml b/deploy/templates/templater/base_dea_template.yaml new file mode 100644 index 000000000..bdc1b2684 --- /dev/null +++ b/deploy/templates/templater/base_dea_template.yaml @@ -0,0 +1,24 @@ +############################################################################## +# Copyright (c) 2015, 2016 Ericsson AB and others. +# jonas.bjurel@ericsson.com +# peter.barabas@ericsson.com +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +dea-base-config-metadata: + title: 'Deployment Environment Adapter Base configuration' + # DEA API version supported + version: '0.4' + created: 'Fri Jun 10 2016' + comment: 'Rebased for Fuel 9' +environment: + net_segment_type: %{network/networking_parameters/segmentation_type} +fuel: + FUEL_ACCESS: + %{fuel/FUEL_ACCESS} +wanted_release: %{wanted_release} +settings: + %{settings} |