diff options
36 files changed, 420 insertions, 117 deletions
diff --git a/build/Makefile b/build/Makefile index 550f360f9..6c61714da 100644 --- a/build/Makefile +++ b/build/Makefile @@ -115,15 +115,16 @@ $(ISOCACHE): # cd /tmp/fuel-main && patch -f -p0 < $(TOPDIR)/fuel-main_docker_dns.patch # cd /tmp/fuel-main && ./prepare-build-env.sh + # Verify that Docker is alive + sudo docker info cd /tmp/fuel-main && make repos $(REPOINFO) -r /tmp/fuel-main > gitinfo_fuel.txt # Repeat build up to ten times sudo ./fuel_build_loop cp /tmp/fuel-main/build/artifacts/fuel*.iso . - - # Attempt to store ISO in cache to speed up the next - # build attempt. If cache is not enabled no harm is done. - $(MAKE) -f Makefile put-cache + # Store artifact in cache straight away if caching is enabled + # (no .cacheid will be present unless this is a cached build) + test -f .cacheid && $(MAKE) -f Makefile put-cache || exit 0 .PHONY: mount-origiso umount-origiso mount-origiso: $(ISOCACHE) diff --git a/build/cache.sh b/build/cache.sh index 07361d28a..7c2f2109c 100755 --- a/build/cache.sh +++ b/build/cache.sh @@ -19,7 +19,6 @@ exit_trap() { trap exit_trap EXIT CACHETRANSPORT=${CACHETRANSPORT:-"curl --silent"} -CACHEBASE=${CACHEBASE:-"file://${HOME}/cache"} CACHEMAXAGE=${CACHEMAXAGE:-$[14*24*3600]} CACHEDEBUG=${CACHEDEBUG:-1} @@ -132,6 +131,12 @@ getcommitid() { fi } + + +if [ -z "$CACHEBASE" ]; then + errorexit "CACHEBASE not set - exiting..." +fi + case $1 in getcommitid) if [ $# -ne 3 ]; then diff --git a/build/f_isoroot/Makefile b/build/f_isoroot/Makefile index ec0db48a3..babb5d68d 100644 --- a/build/f_isoroot/Makefile +++ b/build/f_isoroot/Makefile @@ -9,7 +9,8 @@ ############################################################################## # Add Fuel plugin build targets here -PLUGINS = f_odlpluginbuild f_qemupluginbuild f_ovs-nsh-dpdk-pluginbuild f_onosfwpluginbuild f_bgpvpn-pluginbuild f_ovsnfv-dpdk-pluginbuild f_vsperfpluginbuild +PLUGINS = f_odlpluginbuild +#NON_8-0_REBASED_PLUGINS = f_qemupluginbuild f_ovs-nsh-dpdk-pluginbuild f_onosfwpluginbuild f_bgpvpn-pluginbuild f_ovsnfv-dpdk-pluginbuild f_vsperfpluginbuild # If the BUILD_FUEL_PLUGINS environment variable is set, only build the plugins # indicated therein. diff --git a/build/f_isoroot/f_bgpvpn-pluginbuild/Makefile b/build/f_isoroot/f_bgpvpn-pluginbuild/Makefile index a99a426b6..71f2771a5 100644 --- a/build/f_isoroot/f_bgpvpn-pluginbuild/Makefile +++ b/build/f_isoroot/f_bgpvpn-pluginbuild/Makefile @@ -51,6 +51,9 @@ release:.bgpvpnbuild $(REPOINFO) -r . > gitinfo_bgpvpnplugin.txt rm -rf fuel-plugin-bgpvpn touch .bgpvpnbuild + # Store artifact in cache straight away if caching is enabled + # (no .cacheid will be present unless this is a cached build) + test -f .cacheid && $(MAKE) -f Makefile put-cache || exit 0 ############################################################################# # Cache operations - only used when building through ci/build.sh diff --git a/build/f_isoroot/f_bootstrap/bootstrap_admin_node.sh b/build/f_isoroot/f_bootstrap/bootstrap_admin_node.sh index 6318c5d39..35d107ab4 100755 --- a/build/f_isoroot/f_bootstrap/bootstrap_admin_node.sh +++ b/build/f_isoroot/f_bootstrap/bootstrap_admin_node.sh @@ -1,5 +1,56 @@ #!/bin/bash -FUEL_RELEASE=$(grep release: /etc/fuel/version.yaml | cut -d: -f2 | tr -d '" ') +mkdir -p /var/log/puppet +exec > >(tee -i /var/log/puppet/bootstrap_admin_node.log) +exec 2>&1 + +FUEL_RELEASE=$(cat /etc/fuel_release) +ASTUTE_YAML='/etc/fuel/astute.yaml' +BOOTSTRAP_NODE_CONFIG="/etc/fuel/bootstrap_admin_node.conf" +bs_build_log='/var/log/fuel-bootstrap-image-build.log' +bs_status=0 +# Backup network configs to this folder. Folder will be created only if +# backup process actually will be. +bup_folder="/var/bootstrap_admin_node_bup_$(date +%Y-%m-%d-%H-%M-%S)/" +### Long messages inside code makes them more complicated to read... +# bootstrap messages +# FIXME fix help links +bs_skip_message="WARNING: Ubuntu bootstrap build has been skipped. \ +Please build and activate bootstrap manually with CLI command \ +\`fuel-bootstrap build --activate\`. \ +While you don't activate any bootstrap - new nodes cannot be discovered \ +and added to cluster. \ +For more information please visit \ +https://docs.mirantis.com/openstack/fuel/fuel-master/" +bs_error_message="WARNING: Failed to build the bootstrap image, see $bs_build_log \ +for details. Perhaps your Internet connection is broken. Please fix the \ +problem and run \`fuel-bootstrap build --activate\`. \ +While you don\'t activate any bootstrap - new nodes cannot be discovered \ +and added to cluster. \ +For more information please visit \ +https://docs.mirantis.com/openstack/fuel/fuel-master/" +bs_progress_message="There is no active bootstrap. Bootstrap image building \ +is in progress. Usually it takes 15-20 minutes. It depends on your internet \ +connection and hardware performance. Please reboot failed to discover nodes \ +after bootstrap image become available." +bs_done_message="Default bootstrap image building done. Now you can boot new \ +nodes over PXE, they will be discovered and become available for installing \ +OpenStack on them" +bs_centos_message="WARNING: Deprecated Centos bootstrap has been chosen \ +and activated. Now you can boot new nodes over PXE, they will be discovered \ +and become available for installing OpenStack on them." +# Update issues messages +update_warn_message="There is an issue connecting to the Fuel update repository. \ +Please fix your connection prior to applying any updates. \ +Once the connection is fixed, we recommend reviewing and applying \ +Maintenance Updates for this release of Mirantis OpenStack: \ +https://docs.mirantis.com/openstack/fuel/fuel-${FUEL_RELEASE}/\ +release-notes.html#maintenance-updates" +update_done_message="We recommend reviewing and applying Maintenance Updates \ +for this release of Mirantis OpenStack: \ +https://docs.mirantis.com/openstack/fuel/fuel-${FUEL_RELEASE}/\ +release-notes.html#maintenance-updates" +fuelmenu_fail_message="Fuelmenu was not able to generate '/etc/fuel/astute.yaml' file! \ +Please, restart it manualy using 'fuelmenu' command." function countdown() { local i @@ -40,6 +91,10 @@ done shopt -u nullglob ### OPNFV addition END +# Enable sshd +systemctl enable sshd +systemctl start sshd + if [[ "$showmenu" == "yes" || "$showmenu" == "YES" ]]; then fuelmenu else @@ -61,6 +116,25 @@ if [[ "$showmenu" == "yes" || "$showmenu" == "YES" ]]; then fi fi +systemctl reload sshd + +# Enable iptables +systemctl enable iptables.service +systemctl start iptables.service + + +if [ ! -f "${ASTUTE_YAML}" ]; then + echo ${fuelmenu_fail_message} + fail +fi + +systemctl reload sshd + +# Enable iptables +systemctl enable iptables.service +systemctl start iptables.service + + if [ "$wait_for_external_config" == "yes" ]; then wait_timeout=3000 pidfile=/var/lock/wait_for_external_config @@ -101,11 +175,30 @@ make_ubuntu_bootstrap_stub () { } get_bootstrap_flavor () { - local ASTUTE_YAML='/etc/fuel/astute.yaml' python <<-EOF - from fuelmenu.fuelmenu import Settings - conf = Settings().read("$ASTUTE_YAML").get('BOOTSTRAP', {}) - print(conf.get('flavor', 'centos')) + from yaml import safe_load + with open("$ASTUTE_YAML", 'r') as f: + conf = safe_load(f).get('BOOTSTRAP', {}) + print(conf.get('flavor', 'centos').lower()) + EOF +} + +get_bootstrap_skip () { + python <<-EOF + from yaml import safe_load + with open("$ASTUTE_YAML", 'r') as f: + conf = safe_load(f).get('BOOTSTRAP', {}) + print(conf.get('skip_default_img_build', False)) + EOF +} + +set_ui_bootstrap_error () { + # This notify can't be closed or removed by user. + # For remove notify - send empty string. + local message=$1 + python <<-EOF + from fuel_bootstrap.utils import notifier + notifier.notify_webui('${message}') EOF } @@ -114,10 +207,7 @@ build_ubuntu_bootstrap () { local ret=1 echo ${bs_progress_message} >&2 set_ui_bootstrap_error "${bs_progress_message}" >&2 -# OPNFV modification to turn off biosdevname on the line below (extend-kopts) - if fuel-bootstrap -v --debug build --activate \ - --extend-kopts "biosdevname=0 net.ifnames=0 debug ignore_loglevel log_buf_len=10M print_fatal_signals=1 LOGLEVEL=8" \ - >>"$bs_build_log" 2>&1; then + if fuel-bootstrap -v --debug build --activate >>"$bs_build_log" 2>&1; then ret=0 fuel notify --topic "done" --send "${bs_done_message}" else diff --git a/build/f_isoroot/f_bootstrap/bootstrap_admin_node.sh.orig b/build/f_isoroot/f_bootstrap/bootstrap_admin_node.sh.orig index 1bc349314..e637c7b6d 100755 --- a/build/f_isoroot/f_bootstrap/bootstrap_admin_node.sh.orig +++ b/build/f_isoroot/f_bootstrap/bootstrap_admin_node.sh.orig @@ -1,5 +1,56 @@ #!/bin/bash -FUEL_RELEASE=$(grep release: /etc/fuel/version.yaml | cut -d: -f2 | tr -d '" ') +mkdir -p /var/log/puppet +exec > >(tee -i /var/log/puppet/bootstrap_admin_node.log) +exec 2>&1 + +FUEL_RELEASE=$(cat /etc/fuel_release) +ASTUTE_YAML='/etc/fuel/astute.yaml' +BOOTSTRAP_NODE_CONFIG="/etc/fuel/bootstrap_admin_node.conf" +bs_build_log='/var/log/fuel-bootstrap-image-build.log' +bs_status=0 +# Backup network configs to this folder. Folder will be created only if +# backup process actually will be. +bup_folder="/var/bootstrap_admin_node_bup_$(date +%Y-%m-%d-%H-%M-%S)/" +### Long messages inside code makes them more complicated to read... +# bootstrap messages +# FIXME fix help links +bs_skip_message="WARNING: Ubuntu bootstrap build has been skipped. \ +Please build and activate bootstrap manually with CLI command \ +\`fuel-bootstrap build --activate\`. \ +While you don't activate any bootstrap - new nodes cannot be discovered \ +and added to cluster. \ +For more information please visit \ +https://docs.mirantis.com/openstack/fuel/fuel-master/" +bs_error_message="WARNING: Failed to build the bootstrap image, see $bs_build_log \ +for details. Perhaps your Internet connection is broken. Please fix the \ +problem and run \`fuel-bootstrap build --activate\`. \ +While you don\'t activate any bootstrap - new nodes cannot be discovered \ +and added to cluster. \ +For more information please visit \ +https://docs.mirantis.com/openstack/fuel/fuel-master/" +bs_progress_message="There is no active bootstrap. Bootstrap image building \ +is in progress. Usually it takes 15-20 minutes. It depends on your internet \ +connection and hardware performance. Please reboot failed to discover nodes \ +after bootstrap image become available." +bs_done_message="Default bootstrap image building done. Now you can boot new \ +nodes over PXE, they will be discovered and become available for installing \ +OpenStack on them" +bs_centos_message="WARNING: Deprecated Centos bootstrap has been chosen \ +and activated. Now you can boot new nodes over PXE, they will be discovered \ +and become available for installing OpenStack on them." +# Update issues messages +update_warn_message="There is an issue connecting to the Fuel update repository. \ +Please fix your connection prior to applying any updates. \ +Once the connection is fixed, we recommend reviewing and applying \ +Maintenance Updates for this release of Mirantis OpenStack: \ +https://docs.mirantis.com/openstack/fuel/fuel-${FUEL_RELEASE}/\ +release-notes.html#maintenance-updates" +update_done_message="We recommend reviewing and applying Maintenance Updates \ +for this release of Mirantis OpenStack: \ +https://docs.mirantis.com/openstack/fuel/fuel-${FUEL_RELEASE}/\ +release-notes.html#maintenance-updates" +fuelmenu_fail_message="Fuelmenu was not able to generate '/etc/fuel/astute.yaml' file! \ +Please, restart it manualy using 'fuelmenu' command." function countdown() { local i @@ -51,6 +102,20 @@ if [[ "$showmenu" == "yes" || "$showmenu" == "YES" ]]; then fi fi +if [ ! -f "${ASTUTE_YAML}" ]; then + echo ${fuelmenu_fail_message} + fail +fi + +# Enable sshd +systemctl enable sshd +systemctl start sshd + +# Enable iptables +systemctl enable iptables.service +systemctl start iptables.service + + if [ "$wait_for_external_config" == "yes" ]; then wait_timeout=3000 pidfile=/var/lock/wait_for_external_config @@ -91,11 +156,30 @@ make_ubuntu_bootstrap_stub () { } get_bootstrap_flavor () { - local ASTUTE_YAML='/etc/fuel/astute.yaml' python <<-EOF - from fuelmenu.fuelmenu import Settings - conf = Settings().read("$ASTUTE_YAML").get('BOOTSTRAP', {}) - print(conf.get('flavor', 'centos')) + from yaml import safe_load + with open("$ASTUTE_YAML", 'r') as f: + conf = safe_load(f).get('BOOTSTRAP', {}) + print(conf.get('flavor', 'centos').lower()) + EOF +} + +get_bootstrap_skip () { + python <<-EOF + from yaml import safe_load + with open("$ASTUTE_YAML", 'r') as f: + conf = safe_load(f).get('BOOTSTRAP', {}) + print(conf.get('skip_default_img_build', False)) + EOF +} + +set_ui_bootstrap_error () { + # This notify can't be closed or removed by user. + # For remove notify - send empty string. + local message=$1 + python <<-EOF + from fuel_bootstrap.utils import notifier + notifier.notify_webui('${message}') EOF } diff --git a/build/f_isoroot/f_kscfg/ks.cfg b/build/f_isoroot/f_kscfg/ks.cfg index e16139a86..941bfdfd0 100644 --- a/build/f_isoroot/f_kscfg/ks.cfg +++ b/build/f_isoroot/f_kscfg/ks.cfg @@ -296,14 +296,49 @@ yum-plugin-priorities # POSTINSTALL SECTION # HERE ARE COMMANDS THAT WILL BE LAUNCHED JUST AFTER # INSTALLATION ITSELF COMPLETED -%post -echo -e "modprobe nf_conntrack_ipv4\nmodprobe nf_conntrack_ipv6\nmodprobe nf_conntrack_tftp\nmodprobe nf_nat_tftp" >> /etc/rc.modules -chmod +x /etc/rc.modules -echo -e "net.nf_conntrack_max=1048576" >> /etc/sysctl.conf -mkdir -p /var/log/coredump -echo -e "kernel.core_pattern=/var/log/coredump/core.%e.%p.%h.%t" >> /etc/sysctl.conf -chmod 777 /var/log/coredump -echo -e "* soft core unlimited\n* hard core unlimited" >> /etc/security/limits.conf + + +# Parse /proc/cmdline and save for next steps +%post --log=/root/anaconda-parse-cmdline.log +#!/bin/bash +set -x + +# Parse cmdline to alter keys which contains dot in their names +# Such keys can't be used as variables in bash, +# so every dot is replaced with double underscore. +# Double underscore needed to avoid possible naming collisions. +for item in $(cat /proc/cmdline); do + if [[ "${item}" =~ '=' ]]; then + key="${item%%=*}" + value="${item#*=}" + else + key="${item}" + value='yes' + fi + key="${key//\./__}" + value="${value:-'yes'}" + echo "${key}=${value}" >> /root/anaconda.cmdline.vars +done + +source /root/anaconda.cmdline.vars + +if [[ ! -z $ifname ]]; then + echo "adminif=$(udevadm info --query=property -p /sys/class/net/${ifname%%:*} | \ + awk -F\= '$1 == "ID_NET_NAME_ONBOARD" {s=$2; exit}; $1 == "ID_NET_NAME_SLOT" {s=$2; exit}; $1 == "ID_NET_NAME_PATH" {s=$2; next}; END {print s}')" >> /root/anaconda.cmdline.vars +fi + +# Add self entry to /etc/hosts +ipaddr=$(echo $ip | cut -d: -f1) +hostname=$(echo $ip | cut -d: -f5) +# Use default hostname if missing from parameters +[ -z "$hostname" ] && hostname="fuel.domain.tld" +short=$(echo $hostname | cut -d. -f1) +echo -e "${ipaddr} ${hostname} ${short}" >> /etc/hosts +%end + + + + # Mount installation media in chroot %post --nochroot --log=/mnt/sysimage/root/anaconda-post-before-chroot.log diff --git a/build/f_isoroot/f_kscfg/ks.cfg.orig b/build/f_isoroot/f_kscfg/ks.cfg.orig index 41a475bbe..e9c8ad588 100644 --- a/build/f_isoroot/f_kscfg/ks.cfg.orig +++ b/build/f_isoroot/f_kscfg/ks.cfg.orig @@ -296,14 +296,49 @@ yum-plugin-priorities # POSTINSTALL SECTION # HERE ARE COMMANDS THAT WILL BE LAUNCHED JUST AFTER # INSTALLATION ITSELF COMPLETED -%post -echo -e "modprobe nf_conntrack_ipv4\nmodprobe nf_conntrack_ipv6\nmodprobe nf_conntrack_tftp\nmodprobe nf_nat_tftp" >> /etc/rc.modules -chmod +x /etc/rc.modules -echo -e "net.nf_conntrack_max=1048576" >> /etc/sysctl.conf -mkdir -p /var/log/coredump -echo -e "kernel.core_pattern=/var/log/coredump/core.%e.%p.%h.%t" >> /etc/sysctl.conf -chmod 777 /var/log/coredump -echo -e "* soft core unlimited\n* hard core unlimited" >> /etc/security/limits.conf + + +# Parse /proc/cmdline and save for next steps +%post --log=/root/anaconda-parse-cmdline.log +#!/bin/bash +set -x + +# Parse cmdline to alter keys which contains dot in their names +# Such keys can't be used as variables in bash, +# so every dot is replaced with double underscore. +# Double underscore needed to avoid possible naming collisions. +for item in $(cat /proc/cmdline); do + if [[ "${item}" =~ '=' ]]; then + key="${item%%=*}" + value="${item#*=}" + else + key="${item}" + value='yes' + fi + key="${key//\./__}" + value="${value:-'yes'}" + echo "${key}=${value}" >> /root/anaconda.cmdline.vars +done + +source /root/anaconda.cmdline.vars + +if [[ ! -z $ifname ]]; then + echo "adminif=$(udevadm info --query=property -p /sys/class/net/${ifname%%:*} | \ + awk -F\= '$1 == "ID_NET_NAME_ONBOARD" {s=$2; exit}; $1 == "ID_NET_NAME_SLOT" {s=$2; exit}; $1 == "ID_NET_NAME_PATH" {s=$2; next}; END {print s}')" >> /root/anaconda.cmdline.vars +fi + +# Add self entry to /etc/hosts +ipaddr=$(echo $ip | cut -d: -f1) +hostname=$(echo $ip | cut -d: -f5) +# Use default hostname if missing from parameters +[ -z "$hostname" ] && hostname="fuel.domain.tld" +short=$(echo $hostname | cut -d. -f1) +echo -e "${ipaddr} ${hostname} ${short}" >> /etc/hosts +%end + + + + # Mount installation media in chroot %post --nochroot --log=/mnt/sysimage/root/anaconda-post-before-chroot.log diff --git a/build/f_isoroot/f_odlpluginbuild/Makefile b/build/f_isoroot/f_odlpluginbuild/Makefile index 18fbb9e60..024cd4a50 100644 --- a/build/f_isoroot/f_odlpluginbuild/Makefile +++ b/build/f_isoroot/f_odlpluginbuild/Makefile @@ -32,11 +32,15 @@ release:.odlbuild sudo gem install fpm sudo pip install fuel-plugin-builder git clone -b $(ODL_BRANCH) $(ODL_REPO) - fpb --debug --build fuel-plugin-opendaylight/ + # Temporary location of working Be snapshot + ODL_TARBALL_LOCATION=http://devel.opt.im/distribution-karaf-0.4.0-20160118.091622-3555.tar.gz fpb --debug --build fuel-plugin-opendaylight/ mv fuel-plugin-opendaylight/opendaylight*.rpm . $(REPOINFO) -r . > gitinfo_odlplugin.txt rm -rf fuel-plugin-opendaylight touch .odlbuild + # Store artifact in cache straight away if caching is enabled + # (no .cacheid will be present unless this is a cached build) + test -f .cacheid && $(MAKE) -f Makefile put-cache || exit 0 ############################################################################# # Cache operations - only used when building through ci/build.sh diff --git a/build/f_isoroot/f_onosfwpluginbuild/Makefile b/build/f_isoroot/f_onosfwpluginbuild/Makefile index 0cf290c1c..9516e3d46 100644 --- a/build/f_isoroot/f_onosfwpluginbuild/Makefile +++ b/build/f_isoroot/f_onosfwpluginbuild/Makefile @@ -49,6 +49,9 @@ release:.onosbuild $(REPOINFO) -r . > gitinfo_onos-plugin.txt @rm -rf fuel-plugin-onos @touch .onosbuild + # Store artifact in cache straight away if caching is enabled + # (no .cacheid will be present unless this is a cached build) + test -f .cacheid && $(MAKE) -f Makefile put-cache || exit 0 ############################################################################# # Cache operations - only used when building through ci/build.sh diff --git a/build/f_isoroot/f_ovs-nsh-dpdk-pluginbuild/Makefile b/build/f_isoroot/f_ovs-nsh-dpdk-pluginbuild/Makefile index 700a93c0d..f89f8d980 100644 --- a/build/f_isoroot/f_ovs-nsh-dpdk-pluginbuild/Makefile +++ b/build/f_isoroot/f_ovs-nsh-dpdk-pluginbuild/Makefile @@ -50,6 +50,9 @@ release:.ovsbuild $(REPOINFO) -r . > gitinfo_ovs-nsh-dpdk-plugin.txt @rm -rf fuel-plugin-ovs @touch .ovsbuild + # Store artifact in cache straight away if caching is enabled + # (no .cacheid will be present unless this is a cached build) + test -f .cacheid && $(MAKE) -f Makefile put-cache || exit 0 ############################################################################# # Cache operations - only used when building through ci/build.sh diff --git a/build/f_isoroot/f_ovsnfv-dpdk-pluginbuild/Makefile b/build/f_isoroot/f_ovsnfv-dpdk-pluginbuild/Makefile index 70fd22409..a74687307 100644 --- a/build/f_isoroot/f_ovsnfv-dpdk-pluginbuild/Makefile +++ b/build/f_isoroot/f_ovsnfv-dpdk-pluginbuild/Makefile @@ -50,6 +50,9 @@ release:.ovsnfvbuild $(REPOINFO) -r . > gitinfo_ovsnfv-dpdk-plugin.txt @rm -rf fuel-plugin-ovsnfv @touch .ovsnfvbuild + # Store artifact in cache straight away if caching is enabled + # (no .cacheid will be present unless this is a cached build) + test -f .cacheid && $(MAKE) -f Makefile put-cache || exit 0 ############################################################################# # Cache operations - only used when building through ci/build.sh diff --git a/build/f_isoroot/f_qemupluginbuild/Makefile b/build/f_isoroot/f_qemupluginbuild/Makefile index 126e20549..0f245aec3 100644 --- a/build/f_isoroot/f_qemupluginbuild/Makefile +++ b/build/f_isoroot/f_qemupluginbuild/Makefile @@ -50,6 +50,9 @@ release:.qemubuild $(REPOINFO) -r . > gitinfo_qemu-plugin.txt @rm -rf fuel-plugin-qemu @touch .qemubuild + # Store artifact in cache straight away if caching is enabled + # (no .cacheid will be present unless this is a cached build) + test -f .cacheid && $(MAKE) -f Makefile put-cache || exit 0 ############################################################################# # Cache operations - only used when building through ci/build.sh diff --git a/build/f_isoroot/f_repobuild/Makefile b/build/f_isoroot/f_repobuild/Makefile index 522722e66..1fc703031 100644 --- a/build/f_isoroot/f_repobuild/Makefile +++ b/build/f_isoroot/f_repobuild/Makefile @@ -45,6 +45,9 @@ nailgun: sudo su - -c /opt/fuel-createmirror-*/fuel-createmirror sudo chmod -R 755 /var/www/nailgun cp -Rp /var/www/nailgun . + # Store artifact in cache straight away if caching is enabled + # (no .cacheid will be present unless this is a cached build) + test -f .cacheid && $(MAKE) -f Makefile put-cache || exit 0 .PHONY: clean clean: diff --git a/build/f_isoroot/f_vsperfpluginbuild/Makefile b/build/f_isoroot/f_vsperfpluginbuild/Makefile index ccf596a14..c9653d785 100644 --- a/build/f_isoroot/f_vsperfpluginbuild/Makefile +++ b/build/f_isoroot/f_vsperfpluginbuild/Makefile @@ -50,6 +50,9 @@ release:.vsperfbuild $(REPOINFO) -r . > gitinfo_vsperf-plugin.txt @rm -rf fuel-plugin-vsperf @touch .vsperfbuild + # Store artifact in cache straight away if caching is enabled + # (no .cacheid will be present unless this is a cached build) + test -f .cacheid && $(MAKE) -f Makefile put-cache || exit 0 ############################################################################# # Cache operations - only used when building through ci/build.sh diff --git a/deploy/README b/deploy/README index 167078bf8..7a3e09dc9 100644 --- a/deploy/README +++ b/deploy/README @@ -5,7 +5,7 @@ the following dependencies and python modules are required to be installed: - for Ubuntu: -sudo apt-get install -y libvirt-bin qemu-kvm python-pip fuseiso mkisofs +sudo apt-get install -y libvirt-bin qemu-kvm python-pip fuseiso mkisofs genisoimage sudo apt-get install -y python-dev libz-dev libxml2-dev libxslt-dev sudo pip install pyyaml netaddr paramiko lxml scp pycrypto ecdsa diff --git a/deploy/cloud/configure_environment.py b/deploy/cloud/configure_environment.py index ed89fa600..1914d7e3e 100644 --- a/deploy/cloud/configure_environment.py +++ b/deploy/cloud/configure_environment.py @@ -46,14 +46,11 @@ class ConfigureEnvironment(object): delete(self.yaml_config_dir) create_dir_if_not_exists(self.yaml_config_dir) env_name = self.dea.get_env_name() - env_mode = self.dea.get_env_mode() env_net_segment_type = self.dea.get_env_net_segment_type() - log('Creating environment %s release %s, mode %s, network-mode neutron' - ', net-segment-type %s' - % (env_name, self.release_id, env_mode, env_net_segment_type)) - exec_cmd('fuel env create --name %s --release %s --mode %s ' - '--network-mode neutron --net-segment-type %s' - % (env_name, self.release_id, env_mode, env_net_segment_type)) + log('Creating environment %s release %s net-segment-type %s' + % (env_name, self.release_id, env_net_segment_type)) + exec_cmd('fuel env create --name %s --release %s --net-segment-type %s' + % (env_name, self.release_id, env_net_segment_type)) if not self.env_exists(env_name): err('Failed to create environment %s' % env_name) diff --git a/deploy/cloud/configure_settings.py b/deploy/cloud/configure_settings.py index 052634f99..b60a60fd1 100644 --- a/deploy/cloud/configure_settings.py +++ b/deploy/cloud/configure_settings.py @@ -54,12 +54,22 @@ class ConfigureSettings(object): backup(settings_yaml) settings = self.dea.get_property('settings') # Copy fuel defined plugin_id's to user defined settings + # From Fuel 8.0 chosen_id was added because it is now + # possible to install many version of the same plugin + # but we will install only one version for plugin in orig_dea['editable']: - if 'metadata' in orig_dea['editable'][plugin] and 'plugin_id' in orig_dea['editable'][plugin]['metadata']: - if not plugin in settings['editable']: - settings['editable'][plugin] = orig_dea['editable'][plugin] - else: - settings['editable'][plugin]["metadata"]["plugin_id"] = orig_dea['editable'][plugin]["metadata"]["plugin_id"] + if 'metadata' in orig_dea['editable'][plugin]: + if 'plugin_id' in orig_dea['editable'][plugin]['metadata']: + if not plugin in settings['editable']: + settings['editable'][plugin] = orig_dea['editable'][plugin] + else: + settings['editable'][plugin]["metadata"]["plugin_id"] = orig_dea['editable'][plugin]["metadata"]["plugin_id"] + elif 'chosen_id' in orig_dea['editable'][plugin]['metadata']: + if not plugin in settings['editable']: + settings['editable'][plugin] = orig_dea['editable'][plugin] + else: + settings['editable'][plugin]['metadata']['chosen_id'] = orig_dea['editable'][plugin]['metadata']['chosen_id'] + settings['editable'][plugin]['metadata']['versions'][0]['metadata']['plugin_id'] = orig_dea['editable'][plugin]['metadata']['versions'][0]['metadata']['plugin_id'] with io.open(settings_yaml, 'w') as stream: yaml.dump(settings, stream, default_flow_style=False) diff --git a/deploy/common.py b/deploy/common.py index bc2787617..4b0561cd4 100644 --- a/deploy/common.py +++ b/deploy/common.py @@ -18,9 +18,8 @@ import stat import errno N = {'id': 0, 'status': 1, 'name': 2, 'cluster': 3, 'ip': 4, 'mac': 5, - 'roles': 6, 'pending_roles': 7, 'online': 8} -E = {'id': 0, 'status': 1, 'name': 2, 'mode': 3, 'release_id': 4, - 'changes': 5, 'pending_release_id': 6} + 'roles': 6, 'pending_roles': 7, 'online': 8, 'group_id': 9} +E = {'id': 0, 'status': 1, 'name': 2, 'release_id': 3, 'pending_release_id': 5} R = {'id': 0, 'name': 1, 'state': 2, 'operating_system': 3, 'version': 4} RO = {'name': 0, 'conflicts': 1} CWD = os.getcwd() diff --git a/deploy/dea.py b/deploy/dea.py index 2d2b28ae8..b5b63f7cc 100644 --- a/deploy/dea.py +++ b/deploy/dea.py @@ -34,9 +34,6 @@ class DeploymentEnvironmentAdapter(object): def get_env_name(self): return self.get_property('environment')['name'] - def get_env_mode(self): - return self.get_property('environment')['mode'] - def get_env_net_segment_type(self): return self.get_property('environment')['net_segment_type'] diff --git a/deploy/deploy.py b/deploy/deploy.py index 8a4bfc40a..7635cedba 100755 --- a/deploy/deploy.py +++ b/deploy/deploy.py @@ -147,12 +147,15 @@ class AutoDeploy(object): log('isolinux.cfg after: %s' % exec_cmd('grep netmask %s' % isolinux)) + iso_label = self.parse_iso_volume_label(self.iso_file) + log('Volume label: %s' % iso_label) + iso_linux_bin = 'isolinux/isolinux.bin' exec_cmd('mkisofs -quiet -r -J -R -b %s ' '-no-emul-boot -boot-load-size 4 ' '-boot-info-table -hide-rr-moved ' - '-x "lost+found:" -o %s .' - % (iso_linux_bin, new_iso)) + '-x "lost+found:" -V %s -o %s .' + % (iso_linux_bin, iso_label, new_iso)) def update_fuel_isolinux(self, file): with io.open(file) as f: @@ -161,9 +164,34 @@ class AutoDeploy(object): pattern = r'%s=[^ ]\S+' % key replace = '%s=%s' % (key, val) data = re.sub(pattern, replace, data) + + netmask = self.fuel_conf['netmask'] + data = self.append_kernel_param(data, 'netmask=%s' % netmask) + with io.open(file, 'w') as f: f.write(data) + def append_kernel_param(self, data, kernel_param): + """Append the specified kernel parameter to a list of kernel + parameters. Do it only if it isn't already there. + """ + data_final = '' + key = re.match(r'(.+?=)', kernel_param).group() + + for line in data.splitlines(): + data_final += line + if (re.search(r'append ', line) and + not re.search(key, line)): + data_final += ' ' + kernel_param + data_final += '\n' + + return data_final + + def parse_iso_volume_label(self, iso_filename): + label_line = exec_cmd('isoinfo -d -i %s | grep -i "Volume id: "' % iso_filename) + # cut leading text: 'Volume id: ' + return label_line[11:] + def deploy_env(self): dep = CloudDeploy(self.dea, self.dha, self.fuel_conf['ip'], self.fuel_username, self.fuel_password, diff --git a/deploy/fuel_patch/ks.cfg.patch b/deploy/fuel_patch/ks.cfg.patch index 189695792..8c9cebc0f 100644 --- a/deploy/fuel_patch/ks.cfg.patch +++ b/deploy/fuel_patch/ks.cfg.patch @@ -1,19 +1,19 @@ -*** ks.cfg.orig Wed Apr 15 21:47:09 2015 ---- ks.cfg Wed Apr 15 21:47:24 2015 +*** ks.cfg.orig 2016-01-25 08:56:16.406241267 +0100 +--- ks.cfg 2016-01-25 14:31:11.417761494 +0100 *************** -*** 35,41 **** +*** 98,104 **** default_drive=`echo ${drives} ${removable_drives} | awk '{print $1}'` - installdrive="undefined" -! forceformat="no" - for I in `cat /proc/cmdline`; do case "$I" in *=*) eval $I;; esac ; done - - set ${drives} ${removable_drives} ---- 35,41 ---- + installdrive=${installdrive:-undefined} +! forceformat=${forceformat:-no} + for I in $(cat /proc/cmdline); do + case "$I" in + *=*) +--- 98,104 ---- default_drive=`echo ${drives} ${removable_drives} | awk '{print $1}'` - installdrive="undefined" -! forceformat="yes" - for I in `cat /proc/cmdline`; do case "$I" in *=*) eval $I;; esac ; done - - set ${drives} ${removable_drives} + installdrive=${installdrive:-undefined} +! forceformat=${forceformat:-yes} + for I in $(cat /proc/cmdline); do + case "$I" in + *=*) diff --git a/deploy/install_fuel_master.py b/deploy/install_fuel_master.py index e3a7327c0..4f6a0527b 100644 --- a/deploy/install_fuel_master.py +++ b/deploy/install_fuel_master.py @@ -87,7 +87,7 @@ class InstallFuelMaster(object): log('Found FUEL menu as PID %s, now killing it' % fuel_menu_pid) self.ssh_exec_cmd('kill %s' % fuel_menu_pid, False) - log('Wait until installation complete') + log('Wait until installation is complete') self.wait_until_installation_completed() log('Waiting for one minute for Fuel to stabilize') @@ -117,6 +117,7 @@ class InstallFuelMaster(object): plugin_files = [] with self.ssh as s: for plugin_location in [PLUGINS_DIR, LOCAL_PLUGIN_FOLDER]: + s.exec_cmd('mkdir -p %s' % plugin_location) r = s.exec_cmd('find %s -type f -name \'*.rpm\'' % plugin_location) plugin_files.extend(r.splitlines()) @@ -130,7 +131,7 @@ class InstallFuelMaster(object): 'failed: %s' % (f, e)) def wait_for_node_up(self): - WAIT_LOOP = 60 + WAIT_LOOP = 240 SLEEP_TIME = 10 success = False for i in range(WAIT_LOOP): diff --git a/deploy/reap.py b/deploy/reap.py index 72bfcee4c..dd5b242cf 100755 --- a/deploy/reap.py +++ b/deploy/reap.py @@ -75,8 +75,8 @@ DHA_2 = ''' ''' DISKS = {'fuel': '50G', - 'controller': '60G', - 'compute': '50G'} + 'controller': '100G', + 'compute': '100G'} class Reap(object): @@ -221,13 +221,8 @@ class Reap(object): % (self.temp_dir, self.env_id)) network = self.read_yaml(network_file) - # ha_compact not understood by Fuel when deploying...OD - if self.env[E['mode']] == 'ha_compact': - self.env[E['mode']] = 'ha' - env = {'environment': {'name': self.env[E['name']], - 'mode': self.env[E['mode']], 'net_segment_type': network['networking_parameters']['segmentation_type']}} self.write_yaml(self.dea_file, env) diff --git a/deploy/templates/ericsson/virtual_environment/noha/vms/compute.xml b/deploy/templates/ericsson/virtual_environment/noha/vms/compute.xml index fbef4bda7..063b23d98 100644 --- a/deploy/templates/ericsson/virtual_environment/noha/vms/compute.xml +++ b/deploy/templates/ericsson/virtual_environment/noha/vms/compute.xml @@ -57,19 +57,19 @@ <controller type='pci' index='0' model='pci-root'/> <interface type='network'> <source network='fuel1'/> - <model type='virtio'/> + <model type='e1000'/> </interface> <interface type='network'> <source network='fuel2'/> - <model type='virtio'/> + <model type='e1000'/> </interface> <interface type='network'> <source network='fuel3'/> - <model type='virtio'/> + <model type='e1000'/> </interface> <interface type='network'> <source network='fuel4'/> - <model type='virtio'/> + <model type='e1000'/> </interface> <serial type='pty'> <target port='0'/> diff --git a/deploy/templates/ericsson/virtual_environment/noha/vms/controller.xml b/deploy/templates/ericsson/virtual_environment/noha/vms/controller.xml index 3ff28218d..d30a95dcf 100644 --- a/deploy/templates/ericsson/virtual_environment/noha/vms/controller.xml +++ b/deploy/templates/ericsson/virtual_environment/noha/vms/controller.xml @@ -57,19 +57,19 @@ <controller type='pci' index='0' model='pci-root'/> <interface type='network'> <source network='fuel1'/> - <model type='virtio'/> + <model type='e1000'/> </interface> <interface type='network'> <source network='fuel2'/> - <model type='virtio'/> + <model type='e1000'/> </interface> <interface type='network'> <source network='fuel3'/> - <model type='virtio'/> + <model type='e1000'/> </interface> <interface type='network'> <source network='fuel4'/> - <model type='virtio'/> + <model type='e1000'/> </interface> <serial type='pty'> <target port='0'/> diff --git a/deploy/templates/ericsson/virtual_environment/noha/vms/fuel.xml b/deploy/templates/ericsson/virtual_environment/noha/vms/fuel.xml index 1a3286001..a693c96f9 100644 --- a/deploy/templates/ericsson/virtual_environment/noha/vms/fuel.xml +++ b/deploy/templates/ericsson/virtual_environment/noha/vms/fuel.xml @@ -67,7 +67,7 @@ <controller type='pci' index='0' model='pci-root'/> <interface type='network'> <source network='fuel1'/> - <model type='virtio'/> + <model type='e1000'/> </interface> <serial type='pty'> <target port='0'/> diff --git a/deploy/templates/intel/virtual_environment/noha/vms/compute.xml b/deploy/templates/intel/virtual_environment/noha/vms/compute.xml index fbef4bda7..063b23d98 100644 --- a/deploy/templates/intel/virtual_environment/noha/vms/compute.xml +++ b/deploy/templates/intel/virtual_environment/noha/vms/compute.xml @@ -57,19 +57,19 @@ <controller type='pci' index='0' model='pci-root'/> <interface type='network'> <source network='fuel1'/> - <model type='virtio'/> + <model type='e1000'/> </interface> <interface type='network'> <source network='fuel2'/> - <model type='virtio'/> + <model type='e1000'/> </interface> <interface type='network'> <source network='fuel3'/> - <model type='virtio'/> + <model type='e1000'/> </interface> <interface type='network'> <source network='fuel4'/> - <model type='virtio'/> + <model type='e1000'/> </interface> <serial type='pty'> <target port='0'/> diff --git a/deploy/templates/intel/virtual_environment/noha/vms/controller.xml b/deploy/templates/intel/virtual_environment/noha/vms/controller.xml index 3ff28218d..d30a95dcf 100644 --- a/deploy/templates/intel/virtual_environment/noha/vms/controller.xml +++ b/deploy/templates/intel/virtual_environment/noha/vms/controller.xml @@ -57,19 +57,19 @@ <controller type='pci' index='0' model='pci-root'/> <interface type='network'> <source network='fuel1'/> - <model type='virtio'/> + <model type='e1000'/> </interface> <interface type='network'> <source network='fuel2'/> - <model type='virtio'/> + <model type='e1000'/> </interface> <interface type='network'> <source network='fuel3'/> - <model type='virtio'/> + <model type='e1000'/> </interface> <interface type='network'> <source network='fuel4'/> - <model type='virtio'/> + <model type='e1000'/> </interface> <serial type='pty'> <target port='0'/> diff --git a/deploy/templates/intel/virtual_environment/noha/vms/fuel.xml b/deploy/templates/intel/virtual_environment/noha/vms/fuel.xml index 1a3286001..a693c96f9 100644 --- a/deploy/templates/intel/virtual_environment/noha/vms/fuel.xml +++ b/deploy/templates/intel/virtual_environment/noha/vms/fuel.xml @@ -67,7 +67,7 @@ <controller type='pci' index='0' model='pci-root'/> <interface type='network'> <source network='fuel1'/> - <model type='virtio'/> + <model type='e1000'/> </interface> <serial type='pty'> <target port='0'/> diff --git a/deploy/templates/virtual_environment/vms/compute.xml b/deploy/templates/virtual_environment/vms/compute.xml index cc60a2ad2..fa4ef5d8d 100644 --- a/deploy/templates/virtual_environment/vms/compute.xml +++ b/deploy/templates/virtual_environment/vms/compute.xml @@ -30,19 +30,19 @@ <controller type='pci' index='0' model='pci-root'/> <interface type='network'> <source network='fuel1'/> - <model type='virtio'/> + <model type='e1000'/> </interface> <interface type='network'> <source network='fuel2'/> - <model type='virtio'/> + <model type='e1000'/> </interface> <interface type='network'> <source network='fuel3'/> - <model type='virtio'/> + <model type='e1000'/> </interface> <interface type='network'> <source network='fuel4'/> - <model type='virtio'/> + <model type='e1000'/> </interface> <serial type='pty'> <target port='0'/> diff --git a/deploy/templates/virtual_environment/vms/controller.xml b/deploy/templates/virtual_environment/vms/controller.xml index 0a4b6937c..50950db17 100644 --- a/deploy/templates/virtual_environment/vms/controller.xml +++ b/deploy/templates/virtual_environment/vms/controller.xml @@ -30,19 +30,19 @@ <controller type='pci' index='0' model='pci-root'/> <interface type='network'> <source network='fuel1'/> - <model type='virtio'/> + <model type='e1000'/> </interface> <interface type='network'> <source network='fuel2'/> - <model type='virtio'/> + <model type='e1000'/> </interface> <interface type='network'> <source network='fuel3'/> - <model type='virtio'/> + <model type='e1000'/> </interface> <interface type='network'> <source network='fuel4'/> - <model type='virtio'/> + <model type='e1000'/> </interface> <serial type='pty'> <target port='0'/> diff --git a/deploy/templates/virtual_environment/vms/fuel.xml b/deploy/templates/virtual_environment/vms/fuel.xml index a9177f311..4e7c7fd2a 100644 --- a/deploy/templates/virtual_environment/vms/fuel.xml +++ b/deploy/templates/virtual_environment/vms/fuel.xml @@ -40,7 +40,7 @@ <controller type='pci' index='0' model='pci-root'/> <interface type='network'> <source network='fuel1'/> - <model type='virtio'/> + <model type='e1000'/> </interface> <serial type='pty'> <target port='0'/> diff --git a/deploy/templates/virtual_environment_noha/vms/compute.xml b/deploy/templates/virtual_environment_noha/vms/compute.xml index cc60a2ad2..fa4ef5d8d 100644 --- a/deploy/templates/virtual_environment_noha/vms/compute.xml +++ b/deploy/templates/virtual_environment_noha/vms/compute.xml @@ -30,19 +30,19 @@ <controller type='pci' index='0' model='pci-root'/> <interface type='network'> <source network='fuel1'/> - <model type='virtio'/> + <model type='e1000'/> </interface> <interface type='network'> <source network='fuel2'/> - <model type='virtio'/> + <model type='e1000'/> </interface> <interface type='network'> <source network='fuel3'/> - <model type='virtio'/> + <model type='e1000'/> </interface> <interface type='network'> <source network='fuel4'/> - <model type='virtio'/> + <model type='e1000'/> </interface> <serial type='pty'> <target port='0'/> diff --git a/deploy/templates/virtual_environment_noha/vms/controller.xml b/deploy/templates/virtual_environment_noha/vms/controller.xml index 0a4b6937c..50950db17 100644 --- a/deploy/templates/virtual_environment_noha/vms/controller.xml +++ b/deploy/templates/virtual_environment_noha/vms/controller.xml @@ -30,19 +30,19 @@ <controller type='pci' index='0' model='pci-root'/> <interface type='network'> <source network='fuel1'/> - <model type='virtio'/> + <model type='e1000'/> </interface> <interface type='network'> <source network='fuel2'/> - <model type='virtio'/> + <model type='e1000'/> </interface> <interface type='network'> <source network='fuel3'/> - <model type='virtio'/> + <model type='e1000'/> </interface> <interface type='network'> <source network='fuel4'/> - <model type='virtio'/> + <model type='e1000'/> </interface> <serial type='pty'> <target port='0'/> diff --git a/deploy/templates/virtual_environment_noha/vms/fuel.xml b/deploy/templates/virtual_environment_noha/vms/fuel.xml index a9177f311..4e7c7fd2a 100644 --- a/deploy/templates/virtual_environment_noha/vms/fuel.xml +++ b/deploy/templates/virtual_environment_noha/vms/fuel.xml @@ -40,7 +40,7 @@ <controller type='pci' index='0' model='pci-root'/> <interface type='network'> <source network='fuel1'/> - <model type='virtio'/> + <model type='e1000'/> </interface> <serial type='pty'> <target port='0'/> |