summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--build/Makefile2
-rwxr-xr-xbuild/cache.sh22
-rw-r--r--build/config.mk17
-rw-r--r--build/f_isoroot/f_bgpvpn-pluginbuild/config.mk2
-rw-r--r--build/f_isoroot/f_kvm-pluginbuild/config.mk2
-rw-r--r--build/f_isoroot/f_repobuild/Makefile8
-rw-r--r--build/f_isoroot/f_repobuild/config.mk13
-rwxr-xr-xbuild/f_isoroot/f_repobuild/select_ubuntu_repo.sh2
-rw-r--r--build/f_isoroot/f_yardstick-pluginbuild/config.mk2
-rw-r--r--build/repo-mirror-cz.patch19
-rw-r--r--deploy/README.templater277
-rwxr-xr-xdeploy/templater.py169
12 files changed, 504 insertions, 31 deletions
diff --git a/build/Makefile b/build/Makefile
index 956183cc6..44cee03be 100644
--- a/build/Makefile
+++ b/build/Makefile
@@ -110,8 +110,6 @@ $(ISOCACHE):
fi
cd /tmp/fuel-main && git checkout $(FUEL_MAIN_TAG)
@echo "fuel" `git -C /tmp/fuel-main show | grep commit | head -1 | cut -d " " -f2` >> $(VERSION_FILE)
- # Patch for using the Czech Fuel mirror
- cd /tmp/fuel-main && patch -p0 < $(TOPDIR)/repo-mirror-cz.patch
# Remove Docker optimizations, otherwise multistrap will fail during
# Fuel build.
sudo rm -f /etc/apt/apt.conf.d/docker*
diff --git a/build/cache.sh b/build/cache.sh
index 7a2a06385..d4b2c45dd 100755
--- a/build/cache.sh
+++ b/build/cache.sh
@@ -21,6 +21,7 @@ trap exit_trap EXIT
CACHETRANSPORT=${CACHETRANSPORT:-"curl --silent"}
CACHEMAXAGE=${CACHEMAXAGE:-$[14*24*3600]}
CACHEDEBUG=${CACHEDEBUG:-1}
+PLUGINS_MATCH="${BUILD_BASE}/f_isoroot/*/"
debugmsg () {
if [ "$CACHEDEBUG" -eq 1 ]; then
@@ -138,7 +139,22 @@ getcommitid() {
fi
}
+packages() {
+ local PLUGINS_SHA1=''
+ # globbing expansion is alphabetical
+ for plugin in $PLUGINS_MATCH ; do
+ if [ -f "${plugin}packages.yaml" ]
+ then
+ PLUGINS_SHA1+=$(sha1sum ${plugin}packages.yaml)
+ fi
+ done
+
+ if [ -n "${PLUGINS_SHA1}" ]
+ then
+ echo -n $PLUGINS_SHA1 | sha1sum
+ fi
+}
if [ -z "$CACHEBASE" ]; then
errorexit "CACHEBASE not set - exiting..."
@@ -176,6 +192,12 @@ case $1 in
$1 $2
exit $rc
;;
+ packages)
+ if [ $# -ne 1 ]; then
+ errorexit "No arguments can be given to packages!"
+ fi
+ packages
+ ;;
*)
errorexit "I only know about getcommitid, getid, check, get and put!"
esac
diff --git a/build/config.mk b/build/config.mk
index af63fcd8f..e3766a7ef 100644
--- a/build/config.mk
+++ b/build/config.mk
@@ -9,10 +9,25 @@
##############################################################################
FUEL_MAIN_REPO := https://github.com/openstack/fuel-main
-FUEL_MAIN_TAG := 9.0
+FUEL_MAIN_TAG := 9.0.1
MOS_VERSION = 9.0
OPENSTACK_VERSION = mitaka-9.0
+# Pinning down exact Fuel repo versions for Fuel 9.0.1
+export FUELLIB_COMMIT?=e283b62750d9e26355981b3ad3be7c880944ae0f
+export NAILGUN_COMMIT?=e2b85bafb68c348f25cb7cceda81edc668ba2e64
+export PYTHON_FUELCLIENT_COMMIT?=67d8c693a670d27c239d5d175f3ea2a0512c498c
+export FUEL_AGENT_COMMIT?=7ffbf39caf5845bd82b8ce20a7766cf24aa803fb
+export FUEL_NAILGUN_AGENT_COMMIT?=46fa0db0f8944f9e67699d281d462678aaf4db26
+export ASTUTE_COMMIT?=390b257240d49cc5e94ed5c4fcd940b5f2f6ec64
+export OSTF_COMMIT?=f09c98ff7cc71ee612b2450f68a19f2f9c64345a
+export FUEL_MIRROR_COMMIT?=d1ef06b530ce2149230953bb3810a88ecaff870c
+export FUELMENU_COMMIT?=0ed9e206ed1c6271121d3acf52a6bf757411286b
+export SHOTGUN_COMMIT?=781a8cfa0b6eb290e730429fe2792f2b6f5e0c11
+export NETWORKCHECKER_COMMIT?=fcb47dd095a76288aacf924de574e39709e1f3ca
+export FUELUPGRADE_COMMIT?=c1c4bac6a467145ac4fac73e4a7dd2b00380ecfb
+export FUEL_UI_COMMIT?=90de7ef4477230cb7335453ed26ed4306ca6f04f
+
DOCKER_REPO := http://get.docker.com/builds/Linux/x86_64
DOCKER_TAG := docker-latest
diff --git a/build/f_isoroot/f_bgpvpn-pluginbuild/config.mk b/build/f_isoroot/f_bgpvpn-pluginbuild/config.mk
index 3d54e03b2..ba6fb73ff 100644
--- a/build/f_isoroot/f_bgpvpn-pluginbuild/config.mk
+++ b/build/f_isoroot/f_bgpvpn-pluginbuild/config.mk
@@ -9,4 +9,4 @@
BGPVPN_BRANCH?=master
BGPVPN_REPO?="https://github.com/openstack/fuel-plugin-bgpvpn.git"
-BGPVPN_CHANGE?=a2e5cabc2f1e04c948da7026f816549848c2bcd9
+BGPVPN_CHANGE?=3349842af5724be63a74a82c9060848d9d3d299e
diff --git a/build/f_isoroot/f_kvm-pluginbuild/config.mk b/build/f_isoroot/f_kvm-pluginbuild/config.mk
index b608adee0..416ed5e4f 100644
--- a/build/f_isoroot/f_kvm-pluginbuild/config.mk
+++ b/build/f_isoroot/f_kvm-pluginbuild/config.mk
@@ -7,6 +7,6 @@
# http://www.apache.org/licenses/LICENSE-2.0
##############################################################################
-KVMFORNFV_BRANCH=c715b6029fd5b4eaf323f5efde4ec5db5ba0a9b4
+KVMFORNFV_BRANCH=e9bb3d8bce40608c6890ea5298746668b8667462
KVMFORNFV_REPO=https://gerrit.opnfv.org/gerrit/kvmfornfv
KVMFORNFV_CHANGE=
diff --git a/build/f_isoroot/f_repobuild/Makefile b/build/f_isoroot/f_repobuild/Makefile
index cbf81fe4e..8312f5e55 100644
--- a/build/f_isoroot/f_repobuild/Makefile
+++ b/build/f_isoroot/f_repobuild/Makefile
@@ -11,13 +11,9 @@
SHELL := /bin/bash
TOP := $(shell pwd)
TMP_ROOT_DIR := $(shell echo "$(MIRROR_UBUNTU_ROOT)" | cut -d "/" -f2)
-# Enable use of exact repo commit if defined, else use Fuel main branch
-FUEL_MIRROR_REPO?=https://github.com/openstack/fuel-mirror
-# Point to the commit where 302 redirects are handled
-# https://bugs.launchpad.net/fuel/mitaka/+bug/1593674
-FUEL_MIRROR_COMMIT?=192a3d9f8f993afb12c5108dd9339c6688c23e11 #$(FUEL_MAIN_TAG)
include ../../config.mk
+include config.mk
export MOS_VERSION
export OPENSTACK_VERSION
@@ -69,6 +65,8 @@ release:nailgun
.cacheid:
date +"Repocache %G%V" > .cachedata
sha1sum Makefile >> .cachedata
+ sha1sum config.mk >> .cachedata
+ $(CACHETOOL) packages >> .cachedata
cat .cachedata | $(CACHETOOL) getid > .cacheid
# Clean local data related to caching - called prior to ordinary build
diff --git a/build/f_isoroot/f_repobuild/config.mk b/build/f_isoroot/f_repobuild/config.mk
new file mode 100644
index 000000000..d81732849
--- /dev/null
+++ b/build/f_isoroot/f_repobuild/config.mk
@@ -0,0 +1,13 @@
+##############################################################################
+# Copyright (c) 2016 Ericsson AB and others.
+# mskalski@mirantis.com
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+FUEL_MIRROR_REPO?=https://github.com/openstack/fuel-mirror
+# Point to the commit where 302 redirects are handled
+# https://bugs.launchpad.net/fuel/mitaka/+bug/1593674
+FUEL_MIRROR_COMMIT?=192a3d9f8f993afb12c5108dd9339c6688c23e11
diff --git a/build/f_isoroot/f_repobuild/select_ubuntu_repo.sh b/build/f_isoroot/f_repobuild/select_ubuntu_repo.sh
index 540f7ebce..4bca6bd9b 100755
--- a/build/f_isoroot/f_repobuild/select_ubuntu_repo.sh
+++ b/build/f_isoroot/f_repobuild/select_ubuntu_repo.sh
@@ -21,7 +21,7 @@ RSYNC="rsync -4 --contimeout 5 --no-motd --list-only"
# Some Ubuntu mirrors seem less reliable for this type of mirroring -
# as they are discoved they can be added to the blacklist below in order
# for them not to be considered.
-BLACKLIST="mirrors.se.eu.kernel.org"
+BLACKLIST="mirrors.se.eu.kernel.org mirror.its.dal.ca"
return_url=0
diff --git a/build/f_isoroot/f_yardstick-pluginbuild/config.mk b/build/f_isoroot/f_yardstick-pluginbuild/config.mk
index 3db4e9ce2..b87ab6f7c 100644
--- a/build/f_isoroot/f_yardstick-pluginbuild/config.mk
+++ b/build/f_isoroot/f_yardstick-pluginbuild/config.mk
@@ -8,6 +8,6 @@
# http://www.apache.org/licenses/LICENSE-2.0
##############################################################################
-YARDSTICK_BRANCH?=55e2e867e9a3f01b39f619251253470948e250d4
+YARDSTICK_BRANCH?=1e6ca0bfaaeadff7ce8ac9b1693b03d122e58a79
YARDSTICK_REPO?=https://gerrit.opnfv.org/gerrit/yardstick.git
YARDSTICK_CHANGE?=
diff --git a/build/repo-mirror-cz.patch b/build/repo-mirror-cz.patch
deleted file mode 100644
index aa8eaf9ad..000000000
--- a/build/repo-mirror-cz.patch
+++ /dev/null
@@ -1,19 +0,0 @@
-*** config.mk.orig Thu Jan 7 23:30:38 2016
---- config.mk Thu Jan 7 23:32:36 2016
-***************
-*** 153,159 ****
- # 'msk', 'srt', 'usa', 'hrk', 'cz'
- # Setting any other value or removing of this variable will cause
- # download of all the packages directly from internet
-! USE_MIRROR?=ext
-
- ifeq ($(USE_MIRROR),ext)
- MIRROR_FUEL?=http://mirror.fuel-infra.org/mos-repos/centos/$(PRODUCT_NAME)$(PRODUCT_VERSION)-centos$(CENTOS_MAJOR)-fuel/os/x86_64/
---- 153,159 ----
- # 'msk', 'srt', 'usa', 'hrk', 'cz'
- # Setting any other value or removing of this variable will cause
- # download of all the packages directly from internet
-! USE_MIRROR?=cz
-
- ifeq ($(USE_MIRROR),ext)
- MIRROR_FUEL?=http://mirror.fuel-infra.org/mos-repos/centos/$(PRODUCT_NAME)$(PRODUCT_VERSION)-centos$(CENTOS_MAJOR)-fuel/os/x86_64/
diff --git a/deploy/README.templater b/deploy/README.templater
new file mode 100644
index 000000000..964872fb7
--- /dev/null
+++ b/deploy/README.templater
@@ -0,0 +1,277 @@
+##############################################################################
+# Copyright (c) 2016 Ericsson AB and others.
+# peter.barabas@ericsson.com
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+======== TEMPLATING SUPPORT IN YAML CONFIGURATION FILES ========
+
+deploy/templater.py makes it possible to use templates to generate configuration
+files. It takes 2 input YAML files and an output file as arguments. One being
+the dictionary (called the base file), which is used to look up values in; the
+other file is the template, where the substitution will take place. Templater
+will write the result to an output file, specified as the 3rd argument.
+
+
+======== SYNTAX OF TEMPLATE FILES ========
+
+A template file can contain any valid YAML data and template variables, whose
+syntax is described below:
+
+1. Single value references
+
+ %{title}
+
+ %{environment/net_segment_type}
+
+ Either a root element, or a path can be specified.
+
+2. YAML sections
+
+ %{nodes}
+
+ %{network/networking_parameters}
+
+ Either a root element, or a path can be specified.
+
+3. Interface lookup for network
+
+ %{interface(storage)}
+
+ Specify a network type as argument to interface().
+
+4. Interface lookup for network and role
+
+ %{interface(public,compute)}
+
+ Specify a network type and a role as arguments to interface().
+
+
+======== EXAMPLES ========
+
+Base YAML file (excerpt):
+
+title: Deployment Environment Adapter (DEA)
+version: 1.1
+created: Wed Mar 30 08:16:04 2016
+environment:
+ name: vCity
+ net_segment_type: tun
+wanted_release: Liberty on Ubuntu 14.04
+nodes:
+- id: 1
+ interfaces: interfaces_1
+ role: ceph-osd,compute
+ transformations: transformations_1
+- id: 2
+ interfaces: interfaces_1
+ role: ceph-osd,compute
+ transformations: transformations_1
+- id: 3
+ interfaces: interfaces_1
+ role: ceph-osd,compute
+ transformations: transformations_1
+- id: 4
+ interfaces: interfaces_2
+ role: controller,mongo
+ transformations: transformations_2
+- id: 5
+ interfaces: interfaces_2
+ role: controller,mongo
+ transformations: transformations_2
+- id: 6
+ interfaces: interfaces_2
+ role: controller,mongo
+ transformations: transformations_2
+interfaces_1:
+ ens3:
+ - fuelweb_admin
+ - management
+ ens4:
+ - storage
+ ens5:
+ - private
+ ens6:
+ - public
+interfaces_2:
+ ens3:
+ - fuelweb_admin
+ - management
+ ens4:
+ - storage
+ - private
+ - public
+network:
+ networks:
+ - cidr: 172.16.0.0/24
+ gateway: 172.16.0.1
+ ip_ranges:
+ - - 172.16.0.2
+ - 172.16.0.126
+ meta:
+ cidr: 172.16.0.0/24
+ configurable: true
+ floating_range_var: floating_ranges
+ ip_range:
+ - 172.16.0.2
+ - 172.16.0.126
+ map_priority: 1
+ name: public
+ notation: ip_ranges
+ render_addr_mask: public
+ render_type: null
+ use_gateway: true
+ vips:
+ - haproxy
+ - vrouter
+ vlan_start: null
+ name: public
+ vlan_start: null
+ - cidr: 192.168.1.0/24
+ gateway: null
+ ip_ranges:
+ - - 192.168.1.1
+ - 192.168.1.254
+ meta:
+ cidr: 192.168.1.0/24
+ configurable: true
+ map_priority: 2
+ name: storage
+ notation: cidr
+ render_addr_mask: storage
+ render_type: cidr
+ use_gateway: false
+ vlan_start: 102
+ name: storage
+ vlan_start: 102
+
+
+--- Example 1 ---
+
+Template file:
+
+deployment-scenario-metadata:
+ title: %{title}
+ version: 0.1
+dea-override-config:
+ environment:
+ net_segment_type: %{environment/net_segment_type}
+ nodes:
+ %{nodes}
+
+
+Result:
+
+deployment-scenario-metadata:
+ title: Deployment Environment Adapter (DEA)
+ version: 0.1
+dea-override-config:
+ environment:
+ net_segment_type: tun
+ nodes:
+ - id: 1
+ interfaces: interfaces_1
+ role: ceph-osd,compute
+ transformations: transformations_1
+ - id: 2
+ interfaces: interfaces_1
+ role: ceph-osd,compute
+ transformations: transformations_1
+ - id: 3
+ interfaces: interfaces_1
+ role: ceph-osd,compute
+ transformations: transformations_1
+ - id: 4
+ interfaces: interfaces_2
+ role: controller,mongo
+ transformations: transformations_2
+ - id: 5
+ interfaces: interfaces_2
+ role: controller,mongo
+ transformations: transformations_2
+ - id: 6
+ interfaces: interfaces_2
+ role: controller,mongo
+ transformations: transformations_2
+
+
+--- Example 2 ---
+
+Template file:
+
+dea-override-config:
+ network:
+ networks:
+ %{network/networks}
+
+
+Result:
+
+dea-override-config:
+ network:
+ networks:
+ - cidr: 172.16.0.0/24
+ gateway: 172.16.0.1
+ ip_ranges:
+ - - 172.16.0.2
+ - 172.16.0.126
+ meta:
+ cidr: 172.16.0.0/24
+ configurable: true
+ floating_range_var: floating_ranges
+ ip_range:
+ - 172.16.0.2
+ - 172.16.0.126
+ map_priority: 1
+ name: public
+ notation: ip_ranges
+ render_addr_mask: public
+ render_type: null
+ use_gateway: true
+ vips:
+ - haproxy
+ - vrouter
+ vlan_start: null
+ name: public
+ vlan_start: null
+ - cidr: 192.168.1.0/24
+ gateway: null
+ ip_ranges:
+ - - 192.168.1.1
+ - 192.168.1.254
+ meta:
+ cidr: 192.168.1.0/24
+ configurable: true
+ map_priority: 2
+ name: storage
+ notation: cidr
+ render_addr_mask: storage
+ render_type: cidr
+ use_gateway: false
+ vlan_start: 102
+ name: storage
+ vlan_start: 102
+
+
+--- Example 3 ---
+
+Template file:
+
+storage_if: %{interface(storage)}
+compute_private_if: %{interface(private,compute)}
+# Management interface of a mongo node
+mongo_mgmt_if: %{interface(management,mongo)}
+controller_private_if: %{interface(private,controller)}
+
+
+Result:
+
+storage_if: ens4
+compute_private_if: ens5
+# Management interface of a mongo node
+mongo_mgmt_if: ens3
+controller_private_if: ens4
+
diff --git a/deploy/templater.py b/deploy/templater.py
new file mode 100755
index 000000000..2ad6e05ba
--- /dev/null
+++ b/deploy/templater.py
@@ -0,0 +1,169 @@
+#!/usr/bin/env python
+###############################################################################
+# Copyright (c) 2016 Ericsson AB and others.
+# peter.barabas@ericsson.com
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+###############################################################################
+
+
+import io
+import re
+import yaml
+from common import(
+ err,
+ ArgParser,
+)
+
+
+TAG_START = '%{'
+TAG_END = '}'
+DELIMITER = '/'
+
+
+class Templater(object):
+ def __init__(self, base_file, template_file, output_file):
+ self.template_file = template_file
+ self.output_file = output_file
+ self.base = self.load_yaml(base_file)
+
+ def load_yaml(self, filename):
+ try:
+ with io.open(filename) as yaml_file:
+ return yaml.load(yaml_file)
+ except Exception as error:
+ err('Error opening YAML file: %s' % error)
+
+ def save_yaml(self, filename, content):
+ try:
+ with io.open(filename, 'w') as yaml_file:
+ yaml_file.write(content)
+ except Exception as error:
+ err('Error writing YAML file: %s' % error)
+
+ def get_indent(self, line):
+ return len(line) - len(line.lstrip(' '))
+
+ def format_fragment(self, fragment, indent):
+ result = ''
+ is_first_line = True
+
+ for line in fragment.splitlines():
+ # Skip indenting the first line as it is already indented
+ if is_first_line:
+ line += '\n'
+ is_first_line = False
+ else:
+ line = ' ' * indent + line + '\n'
+
+ result += line
+
+ return result.rstrip('\n')
+
+ def format_substitution(self, string):
+ if isinstance(string, basestring):
+ return string
+ else:
+ return yaml.dump(string, default_flow_style=False)
+
+ def parse_interface_tag(self, tag):
+ # Remove 'interface(' prefix, trailing ')' and split arguments
+ args = tag[len('interface('):].rstrip(')').split(',')
+
+ if len(args) == 1 and not args[0]:
+ err('No arguments for interface().')
+ elif len(args) == 2 and (not args[0] or not args[1]):
+ err('Empty argument for interface().')
+ elif len(args) > 2:
+ err('Too many arguments for interface().')
+ else:
+ return args
+
+ def get_interface_from_network(self, interfaces, network):
+ nics = self.base[interfaces]
+ for nic in nics:
+ if network in nics[nic]:
+ return nic
+
+ err('Network not found: %s' % network)
+
+ def get_role_interfaces(self, role):
+ nodes = self.base['nodes']
+ for node in nodes:
+ if role in node['role']:
+ return node['interfaces']
+
+ err('Role not found: %s' % role)
+
+ def lookup_interface(self, args):
+ nodes = self.base['nodes']
+
+ if len(args) == 1:
+ interfaces = nodes[0]['interfaces']
+ if len(args) == 2:
+ interfaces = self.get_role_interfaces(args[1])
+
+ return self.get_interface_from_network(interfaces, args[0])
+
+ def parse_tag(self, tag, indent):
+ fragment = ''
+
+ if 'interface(' in tag:
+ args = self.parse_interface_tag(tag)
+ fragment = self.lookup_interface(args)
+ else:
+ path = tag.split(DELIMITER)
+ fragment = self.base
+ for i in path:
+ if i in fragment:
+ fragment = fragment.get(i)
+ else:
+ err('Error: key "%s" does not exist in base YAML file' % i)
+
+ fragment = self.format_substitution(fragment)
+
+ return self.format_fragment(fragment, indent)
+
+ def run(self):
+ result = ''
+
+ regex = re.compile(re.escape(TAG_START) + r'([a-z].+)' + re.escape(TAG_END),
+ flags=re.IGNORECASE)
+ with io.open(self.template_file) as f:
+ for line in f:
+ indent = self.get_indent(line)
+ result += re.sub(regex,
+ lambda match: self.parse_tag(match.group(1), indent),
+ line)
+
+ self.save_yaml(self.output_file, result)
+
+
+def parse_arguments():
+ description = '''Process 'template_file' using 'base_file' as source for
+template variable substitution and write the results to 'output_file'.'''
+
+ parser = ArgParser(prog='python %s' % __file__,
+ description=description)
+ parser.add_argument('base_file',
+ help='Base YAML filename')
+ parser.add_argument('template_file',
+ help='Fragment filename')
+ parser.add_argument('output_file',
+ help='Output filename')
+
+ args = parser.parse_args()
+ return(args.base_file, args.template_file, args.output_file)
+
+
+def main():
+ base_file, template_file, output_file = parse_arguments()
+
+ templater = Templater(base_file, template_file, output_file)
+ templater.run()
+
+
+if __name__ == '__main__':
+ main()