diff options
| -rw-r--r-- | build/Makefile | 2 | ||||
| -rwxr-xr-x | build/cache.sh | 22 | ||||
| -rw-r--r-- | build/config.mk | 17 | ||||
| -rw-r--r-- | build/f_isoroot/f_bgpvpn-pluginbuild/config.mk | 2 | ||||
| -rw-r--r-- | build/f_isoroot/f_kvm-pluginbuild/config.mk | 2 | ||||
| -rw-r--r-- | build/f_isoroot/f_repobuild/Makefile | 8 | ||||
| -rw-r--r-- | build/f_isoroot/f_repobuild/config.mk | 13 | ||||
| -rwxr-xr-x | build/f_isoroot/f_repobuild/select_ubuntu_repo.sh | 2 | ||||
| -rw-r--r-- | build/repo-mirror-cz.patch | 19 | ||||
| -rw-r--r-- | deploy/README.templater | 277 | ||||
| -rwxr-xr-x | deploy/templater.py | 169 |
11 files changed, 503 insertions, 30 deletions
diff --git a/build/Makefile b/build/Makefile index 956183cc6..44cee03be 100644 --- a/build/Makefile +++ b/build/Makefile @@ -110,8 +110,6 @@ $(ISOCACHE): fi cd /tmp/fuel-main && git checkout $(FUEL_MAIN_TAG) @echo "fuel" `git -C /tmp/fuel-main show | grep commit | head -1 | cut -d " " -f2` >> $(VERSION_FILE) - # Patch for using the Czech Fuel mirror - cd /tmp/fuel-main && patch -p0 < $(TOPDIR)/repo-mirror-cz.patch # Remove Docker optimizations, otherwise multistrap will fail during # Fuel build. sudo rm -f /etc/apt/apt.conf.d/docker* diff --git a/build/cache.sh b/build/cache.sh index 7a2a06385..d4b2c45dd 100755 --- a/build/cache.sh +++ b/build/cache.sh @@ -21,6 +21,7 @@ trap exit_trap EXIT CACHETRANSPORT=${CACHETRANSPORT:-"curl --silent"} CACHEMAXAGE=${CACHEMAXAGE:-$[14*24*3600]} CACHEDEBUG=${CACHEDEBUG:-1} +PLUGINS_MATCH="${BUILD_BASE}/f_isoroot/*/" debugmsg () { if [ "$CACHEDEBUG" -eq 1 ]; then @@ -138,7 +139,22 @@ getcommitid() { fi } +packages() { + local PLUGINS_SHA1='' + # globbing expansion is alphabetical + for plugin in $PLUGINS_MATCH ; do + if [ -f "${plugin}packages.yaml" ] + then + PLUGINS_SHA1+=$(sha1sum ${plugin}packages.yaml) + fi + done + + if [ -n "${PLUGINS_SHA1}" ] + then + echo -n $PLUGINS_SHA1 | sha1sum + fi +} if [ -z "$CACHEBASE" ]; then errorexit "CACHEBASE not set - exiting..." @@ -176,6 +192,12 @@ case $1 in $1 $2 exit $rc ;; + packages) + if [ $# -ne 1 ]; then + errorexit "No arguments can be given to packages!" + fi + packages + ;; *) errorexit "I only know about getcommitid, getid, check, get and put!" esac diff --git a/build/config.mk b/build/config.mk index af63fcd8f..e3766a7ef 100644 --- a/build/config.mk +++ b/build/config.mk @@ -9,10 +9,25 @@ ############################################################################## FUEL_MAIN_REPO := https://github.com/openstack/fuel-main -FUEL_MAIN_TAG := 9.0 +FUEL_MAIN_TAG := 9.0.1 MOS_VERSION = 9.0 OPENSTACK_VERSION = mitaka-9.0 +# Pinning down exact Fuel repo versions for Fuel 9.0.1 +export FUELLIB_COMMIT?=e283b62750d9e26355981b3ad3be7c880944ae0f +export NAILGUN_COMMIT?=e2b85bafb68c348f25cb7cceda81edc668ba2e64 +export PYTHON_FUELCLIENT_COMMIT?=67d8c693a670d27c239d5d175f3ea2a0512c498c +export FUEL_AGENT_COMMIT?=7ffbf39caf5845bd82b8ce20a7766cf24aa803fb +export FUEL_NAILGUN_AGENT_COMMIT?=46fa0db0f8944f9e67699d281d462678aaf4db26 +export ASTUTE_COMMIT?=390b257240d49cc5e94ed5c4fcd940b5f2f6ec64 +export OSTF_COMMIT?=f09c98ff7cc71ee612b2450f68a19f2f9c64345a +export FUEL_MIRROR_COMMIT?=d1ef06b530ce2149230953bb3810a88ecaff870c +export FUELMENU_COMMIT?=0ed9e206ed1c6271121d3acf52a6bf757411286b +export SHOTGUN_COMMIT?=781a8cfa0b6eb290e730429fe2792f2b6f5e0c11 +export NETWORKCHECKER_COMMIT?=fcb47dd095a76288aacf924de574e39709e1f3ca +export FUELUPGRADE_COMMIT?=c1c4bac6a467145ac4fac73e4a7dd2b00380ecfb +export FUEL_UI_COMMIT?=90de7ef4477230cb7335453ed26ed4306ca6f04f + DOCKER_REPO := http://get.docker.com/builds/Linux/x86_64 DOCKER_TAG := docker-latest diff --git a/build/f_isoroot/f_bgpvpn-pluginbuild/config.mk b/build/f_isoroot/f_bgpvpn-pluginbuild/config.mk index 3d54e03b2..ba6fb73ff 100644 --- a/build/f_isoroot/f_bgpvpn-pluginbuild/config.mk +++ b/build/f_isoroot/f_bgpvpn-pluginbuild/config.mk @@ -9,4 +9,4 @@ BGPVPN_BRANCH?=master BGPVPN_REPO?="https://github.com/openstack/fuel-plugin-bgpvpn.git" -BGPVPN_CHANGE?=a2e5cabc2f1e04c948da7026f816549848c2bcd9 +BGPVPN_CHANGE?=3349842af5724be63a74a82c9060848d9d3d299e diff --git a/build/f_isoroot/f_kvm-pluginbuild/config.mk b/build/f_isoroot/f_kvm-pluginbuild/config.mk index b608adee0..416ed5e4f 100644 --- a/build/f_isoroot/f_kvm-pluginbuild/config.mk +++ b/build/f_isoroot/f_kvm-pluginbuild/config.mk @@ -7,6 +7,6 @@ # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## -KVMFORNFV_BRANCH=c715b6029fd5b4eaf323f5efde4ec5db5ba0a9b4 +KVMFORNFV_BRANCH=e9bb3d8bce40608c6890ea5298746668b8667462 KVMFORNFV_REPO=https://gerrit.opnfv.org/gerrit/kvmfornfv KVMFORNFV_CHANGE= diff --git a/build/f_isoroot/f_repobuild/Makefile b/build/f_isoroot/f_repobuild/Makefile index cbf81fe4e..8312f5e55 100644 --- a/build/f_isoroot/f_repobuild/Makefile +++ b/build/f_isoroot/f_repobuild/Makefile @@ -11,13 +11,9 @@ SHELL := /bin/bash TOP := $(shell pwd) TMP_ROOT_DIR := $(shell echo "$(MIRROR_UBUNTU_ROOT)" | cut -d "/" -f2) -# Enable use of exact repo commit if defined, else use Fuel main branch -FUEL_MIRROR_REPO?=https://github.com/openstack/fuel-mirror -# Point to the commit where 302 redirects are handled -# https://bugs.launchpad.net/fuel/mitaka/+bug/1593674 -FUEL_MIRROR_COMMIT?=192a3d9f8f993afb12c5108dd9339c6688c23e11 #$(FUEL_MAIN_TAG) include ../../config.mk +include config.mk export MOS_VERSION export OPENSTACK_VERSION @@ -69,6 +65,8 @@ release:nailgun .cacheid: date +"Repocache %G%V" > .cachedata sha1sum Makefile >> .cachedata + sha1sum config.mk >> .cachedata + $(CACHETOOL) packages >> .cachedata cat .cachedata | $(CACHETOOL) getid > .cacheid # Clean local data related to caching - called prior to ordinary build diff --git a/build/f_isoroot/f_repobuild/config.mk b/build/f_isoroot/f_repobuild/config.mk new file mode 100644 index 000000000..d81732849 --- /dev/null +++ b/build/f_isoroot/f_repobuild/config.mk @@ -0,0 +1,13 @@ +############################################################################## +# Copyright (c) 2016 Ericsson AB and others. +# mskalski@mirantis.com +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +FUEL_MIRROR_REPO?=https://github.com/openstack/fuel-mirror +# Point to the commit where 302 redirects are handled +# https://bugs.launchpad.net/fuel/mitaka/+bug/1593674 +FUEL_MIRROR_COMMIT?=192a3d9f8f993afb12c5108dd9339c6688c23e11 diff --git a/build/f_isoroot/f_repobuild/select_ubuntu_repo.sh b/build/f_isoroot/f_repobuild/select_ubuntu_repo.sh index 540f7ebce..4bca6bd9b 100755 --- a/build/f_isoroot/f_repobuild/select_ubuntu_repo.sh +++ b/build/f_isoroot/f_repobuild/select_ubuntu_repo.sh @@ -21,7 +21,7 @@ RSYNC="rsync -4 --contimeout 5 --no-motd --list-only" # Some Ubuntu mirrors seem less reliable for this type of mirroring - # as they are discoved they can be added to the blacklist below in order # for them not to be considered. -BLACKLIST="mirrors.se.eu.kernel.org" +BLACKLIST="mirrors.se.eu.kernel.org mirror.its.dal.ca" return_url=0 diff --git a/build/repo-mirror-cz.patch b/build/repo-mirror-cz.patch deleted file mode 100644 index aa8eaf9ad..000000000 --- a/build/repo-mirror-cz.patch +++ /dev/null @@ -1,19 +0,0 @@ -*** config.mk.orig Thu Jan 7 23:30:38 2016 ---- config.mk Thu Jan 7 23:32:36 2016 -*************** -*** 153,159 **** - # 'msk', 'srt', 'usa', 'hrk', 'cz' - # Setting any other value or removing of this variable will cause - # download of all the packages directly from internet -! USE_MIRROR?=ext - - ifeq ($(USE_MIRROR),ext) - MIRROR_FUEL?=http://mirror.fuel-infra.org/mos-repos/centos/$(PRODUCT_NAME)$(PRODUCT_VERSION)-centos$(CENTOS_MAJOR)-fuel/os/x86_64/ ---- 153,159 ---- - # 'msk', 'srt', 'usa', 'hrk', 'cz' - # Setting any other value or removing of this variable will cause - # download of all the packages directly from internet -! USE_MIRROR?=cz - - ifeq ($(USE_MIRROR),ext) - MIRROR_FUEL?=http://mirror.fuel-infra.org/mos-repos/centos/$(PRODUCT_NAME)$(PRODUCT_VERSION)-centos$(CENTOS_MAJOR)-fuel/os/x86_64/ diff --git a/deploy/README.templater b/deploy/README.templater new file mode 100644 index 000000000..964872fb7 --- /dev/null +++ b/deploy/README.templater @@ -0,0 +1,277 @@ +############################################################################## +# Copyright (c) 2016 Ericsson AB and others. +# peter.barabas@ericsson.com +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +======== TEMPLATING SUPPORT IN YAML CONFIGURATION FILES ======== + +deploy/templater.py makes it possible to use templates to generate configuration +files. It takes 2 input YAML files and an output file as arguments. One being +the dictionary (called the base file), which is used to look up values in; the +other file is the template, where the substitution will take place. Templater +will write the result to an output file, specified as the 3rd argument. + + +======== SYNTAX OF TEMPLATE FILES ======== + +A template file can contain any valid YAML data and template variables, whose +syntax is described below: + +1. Single value references + + %{title} + + %{environment/net_segment_type} + + Either a root element, or a path can be specified. + +2. YAML sections + + %{nodes} + + %{network/networking_parameters} + + Either a root element, or a path can be specified. + +3. Interface lookup for network + + %{interface(storage)} + + Specify a network type as argument to interface(). + +4. Interface lookup for network and role + + %{interface(public,compute)} + + Specify a network type and a role as arguments to interface(). + + +======== EXAMPLES ======== + +Base YAML file (excerpt): + +title: Deployment Environment Adapter (DEA) +version: 1.1 +created: Wed Mar 30 08:16:04 2016 +environment: + name: vCity + net_segment_type: tun +wanted_release: Liberty on Ubuntu 14.04 +nodes: +- id: 1 + interfaces: interfaces_1 + role: ceph-osd,compute + transformations: transformations_1 +- id: 2 + interfaces: interfaces_1 + role: ceph-osd,compute + transformations: transformations_1 +- id: 3 + interfaces: interfaces_1 + role: ceph-osd,compute + transformations: transformations_1 +- id: 4 + interfaces: interfaces_2 + role: controller,mongo + transformations: transformations_2 +- id: 5 + interfaces: interfaces_2 + role: controller,mongo + transformations: transformations_2 +- id: 6 + interfaces: interfaces_2 + role: controller,mongo + transformations: transformations_2 +interfaces_1: + ens3: + - fuelweb_admin + - management + ens4: + - storage + ens5: + - private + ens6: + - public +interfaces_2: + ens3: + - fuelweb_admin + - management + ens4: + - storage + - private + - public +network: + networks: + - cidr: 172.16.0.0/24 + gateway: 172.16.0.1 + ip_ranges: + - - 172.16.0.2 + - 172.16.0.126 + meta: + cidr: 172.16.0.0/24 + configurable: true + floating_range_var: floating_ranges + ip_range: + - 172.16.0.2 + - 172.16.0.126 + map_priority: 1 + name: public + notation: ip_ranges + render_addr_mask: public + render_type: null + use_gateway: true + vips: + - haproxy + - vrouter + vlan_start: null + name: public + vlan_start: null + - cidr: 192.168.1.0/24 + gateway: null + ip_ranges: + - - 192.168.1.1 + - 192.168.1.254 + meta: + cidr: 192.168.1.0/24 + configurable: true + map_priority: 2 + name: storage + notation: cidr + render_addr_mask: storage + render_type: cidr + use_gateway: false + vlan_start: 102 + name: storage + vlan_start: 102 + + +--- Example 1 --- + +Template file: + +deployment-scenario-metadata: + title: %{title} + version: 0.1 +dea-override-config: + environment: + net_segment_type: %{environment/net_segment_type} + nodes: + %{nodes} + + +Result: + +deployment-scenario-metadata: + title: Deployment Environment Adapter (DEA) + version: 0.1 +dea-override-config: + environment: + net_segment_type: tun + nodes: + - id: 1 + interfaces: interfaces_1 + role: ceph-osd,compute + transformations: transformations_1 + - id: 2 + interfaces: interfaces_1 + role: ceph-osd,compute + transformations: transformations_1 + - id: 3 + interfaces: interfaces_1 + role: ceph-osd,compute + transformations: transformations_1 + - id: 4 + interfaces: interfaces_2 + role: controller,mongo + transformations: transformations_2 + - id: 5 + interfaces: interfaces_2 + role: controller,mongo + transformations: transformations_2 + - id: 6 + interfaces: interfaces_2 + role: controller,mongo + transformations: transformations_2 + + +--- Example 2 --- + +Template file: + +dea-override-config: + network: + networks: + %{network/networks} + + +Result: + +dea-override-config: + network: + networks: + - cidr: 172.16.0.0/24 + gateway: 172.16.0.1 + ip_ranges: + - - 172.16.0.2 + - 172.16.0.126 + meta: + cidr: 172.16.0.0/24 + configurable: true + floating_range_var: floating_ranges + ip_range: + - 172.16.0.2 + - 172.16.0.126 + map_priority: 1 + name: public + notation: ip_ranges + render_addr_mask: public + render_type: null + use_gateway: true + vips: + - haproxy + - vrouter + vlan_start: null + name: public + vlan_start: null + - cidr: 192.168.1.0/24 + gateway: null + ip_ranges: + - - 192.168.1.1 + - 192.168.1.254 + meta: + cidr: 192.168.1.0/24 + configurable: true + map_priority: 2 + name: storage + notation: cidr + render_addr_mask: storage + render_type: cidr + use_gateway: false + vlan_start: 102 + name: storage + vlan_start: 102 + + +--- Example 3 --- + +Template file: + +storage_if: %{interface(storage)} +compute_private_if: %{interface(private,compute)} +# Management interface of a mongo node +mongo_mgmt_if: %{interface(management,mongo)} +controller_private_if: %{interface(private,controller)} + + +Result: + +storage_if: ens4 +compute_private_if: ens5 +# Management interface of a mongo node +mongo_mgmt_if: ens3 +controller_private_if: ens4 + diff --git a/deploy/templater.py b/deploy/templater.py new file mode 100755 index 000000000..2ad6e05ba --- /dev/null +++ b/deploy/templater.py @@ -0,0 +1,169 @@ +#!/usr/bin/env python +############################################################################### +# Copyright (c) 2016 Ericsson AB and others. +# peter.barabas@ericsson.com +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################### + + +import io +import re +import yaml +from common import( + err, + ArgParser, +) + + +TAG_START = '%{' +TAG_END = '}' +DELIMITER = '/' + + +class Templater(object): + def __init__(self, base_file, template_file, output_file): + self.template_file = template_file + self.output_file = output_file + self.base = self.load_yaml(base_file) + + def load_yaml(self, filename): + try: + with io.open(filename) as yaml_file: + return yaml.load(yaml_file) + except Exception as error: + err('Error opening YAML file: %s' % error) + + def save_yaml(self, filename, content): + try: + with io.open(filename, 'w') as yaml_file: + yaml_file.write(content) + except Exception as error: + err('Error writing YAML file: %s' % error) + + def get_indent(self, line): + return len(line) - len(line.lstrip(' ')) + + def format_fragment(self, fragment, indent): + result = '' + is_first_line = True + + for line in fragment.splitlines(): + # Skip indenting the first line as it is already indented + if is_first_line: + line += '\n' + is_first_line = False + else: + line = ' ' * indent + line + '\n' + + result += line + + return result.rstrip('\n') + + def format_substitution(self, string): + if isinstance(string, basestring): + return string + else: + return yaml.dump(string, default_flow_style=False) + + def parse_interface_tag(self, tag): + # Remove 'interface(' prefix, trailing ')' and split arguments + args = tag[len('interface('):].rstrip(')').split(',') + + if len(args) == 1 and not args[0]: + err('No arguments for interface().') + elif len(args) == 2 and (not args[0] or not args[1]): + err('Empty argument for interface().') + elif len(args) > 2: + err('Too many arguments for interface().') + else: + return args + + def get_interface_from_network(self, interfaces, network): + nics = self.base[interfaces] + for nic in nics: + if network in nics[nic]: + return nic + + err('Network not found: %s' % network) + + def get_role_interfaces(self, role): + nodes = self.base['nodes'] + for node in nodes: + if role in node['role']: + return node['interfaces'] + + err('Role not found: %s' % role) + + def lookup_interface(self, args): + nodes = self.base['nodes'] + + if len(args) == 1: + interfaces = nodes[0]['interfaces'] + if len(args) == 2: + interfaces = self.get_role_interfaces(args[1]) + + return self.get_interface_from_network(interfaces, args[0]) + + def parse_tag(self, tag, indent): + fragment = '' + + if 'interface(' in tag: + args = self.parse_interface_tag(tag) + fragment = self.lookup_interface(args) + else: + path = tag.split(DELIMITER) + fragment = self.base + for i in path: + if i in fragment: + fragment = fragment.get(i) + else: + err('Error: key "%s" does not exist in base YAML file' % i) + + fragment = self.format_substitution(fragment) + + return self.format_fragment(fragment, indent) + + def run(self): + result = '' + + regex = re.compile(re.escape(TAG_START) + r'([a-z].+)' + re.escape(TAG_END), + flags=re.IGNORECASE) + with io.open(self.template_file) as f: + for line in f: + indent = self.get_indent(line) + result += re.sub(regex, + lambda match: self.parse_tag(match.group(1), indent), + line) + + self.save_yaml(self.output_file, result) + + +def parse_arguments(): + description = '''Process 'template_file' using 'base_file' as source for +template variable substitution and write the results to 'output_file'.''' + + parser = ArgParser(prog='python %s' % __file__, + description=description) + parser.add_argument('base_file', + help='Base YAML filename') + parser.add_argument('template_file', + help='Fragment filename') + parser.add_argument('output_file', + help='Output filename') + + args = parser.parse_args() + return(args.base_file, args.template_file, args.output_file) + + +def main(): + base_file, template_file, output_file = parse_arguments() + + templater = Templater(base_file, template_file, output_file) + templater.run() + + +if __name__ == '__main__': + main() |