diff options
27 files changed, 520 insertions, 64 deletions
diff --git a/build/Makefile b/build/Makefile index c8fc79385..6ce6dbd9c 100644 --- a/build/Makefile +++ b/build/Makefile @@ -51,9 +51,13 @@ export REPOINFO := $(BUILD_BASE)/repo_info.sh # Use snapshots # Use nearby repositories -export MIRROR_UBUNTU_URL := $(shell ./f_isoroot/f_repobuild/select_ubuntu_repo.sh --url) -export MIRROR_UBUNTU := $(shell echo "$(MIRROR_UBUNTU_URL)" | cut -d'/' -f3 ) -export MIRROR_UBUNTU_ROOT := $(shell echo -n '/' ; echo "$(MIRROR_UBUNTU_URL)" | cut -d'/' -f4-) +export MIRROR_UBUNTU_URL ?= $(shell ./f_isoroot/f_repobuild/select_ubuntu_repo.sh) +export MIRROR_UBUNTU ?= $(shell echo "$(MIRROR_UBUNTU_URL)" | cut -d'/' -f3 ) +export MIRROR_UBUNTU_ROOT ?= $(shell echo -n '/' ; echo "$(MIRROR_UBUNTU_URL)" | cut -d'/' -f4-) +ifeq (,$(MIRROR_UBUNTU_URL)) +$(warning $(shell ./f_isoroot/f_repobuild/select_ubuntu_repo.sh -d)) +$(error No sane Ubuntu mirror available) +endif export LATEST_MIRROR_ID_URL := http://$(shell ./select_closest_fuel_mirror.py) @@ -180,7 +184,7 @@ deepclean: clean clean-cache docker rmi opnfv.org/ubuntu-builder:14.04 &>/dev/null || exit 0 docker rmi opnfv.org/ubuntu-builder:latest &>/dev/null || exit 0 docker rmi ubuntu:14.04 &>/dev/null || exit 0 - @if docker images | grep -q "ubuntu *14.04"; then \ + @if docker images | grep -q "^ubuntu *14.04"; then \ echo "Error: ubuntu:14.04 still present!"; \ exit 1; \ fi diff --git a/build/f_isoroot/Makefile b/build/f_isoroot/Makefile index e189e1697..c21546ad3 100644 --- a/build/f_isoroot/Makefile +++ b/build/f_isoroot/Makefile @@ -11,7 +11,7 @@ # Add Fuel plugin build targets here # Plugins disabled for the Fuel Newton uplift. Please re-enable your plugin as you have # verified it! -PLUGINS = f_ovs-nsh-dpdk-pluginbuild f_vsperfpluginbuild f_kvm-pluginbuild +PLUGINS ?= f_ovs-nsh-dpdk-pluginbuild f_vsperfpluginbuild f_kvm-pluginbuild f_yardstick-pluginbuild f_collectd-ceilometer-pluginbuild f_congress-pluginbuild export PLUGINS NON_10-0_REBASED_PLUGINS = f_odlpluginbuild f_onosfwpluginbuild f_ovsnfv-dpdk-pluginbuild f_vsperfpluginbuild f_bgpvpn-pluginbuild diff --git a/build/f_isoroot/f_collectd-ceilometer-pluginbuild/Makefile b/build/f_isoroot/f_collectd-ceilometer-pluginbuild/Makefile index 17bc36d91..bb4b19cc8 100644 --- a/build/f_isoroot/f_collectd-ceilometer-pluginbuild/Makefile +++ b/build/f_isoroot/f_collectd-ceilometer-pluginbuild/Makefile @@ -19,34 +19,35 @@ include config.mk ############################################################################# .PHONY: all -all: .fastpathmetrics +all: .barometer .PHONY: clean clean: - @rm -f .fastpathmetrics ../release/opnfv/fuel-plugin-collectd-ceilometer-*.rpm fuel-plugin-collectd-ceilometer-*.rpm + @rm -f .barometer ../release/opnfv/fuel-plugin-collectd-ceilometer-*.rpm fuel-plugin-collectd-ceilometer-*.rpm @rm -f $(BUILD_BASE)/gitinfo_collectd-ceilometer-plugin.txt gitinfo_collectd-ceilometer-plugin.txt .PHONY: release -release:.fastpathmetrics +release:.barometer @rm -f ../release/opnfv/fuel-plugin-collectd-ceilometer-*.rpm @mkdir -p ../release/opnfv @cp fuel-plugin-collectd-ceilometer*.rpm ../release/opnfv/ cp gitinfo_collectd-ceilometer-plugin.txt $(BUILD_BASE) -.fastpathmetrics: - @rm -rf fastpathmetrics +.barometer: + @rm -rf barometer + ../../docker/ubuntu-builder/install_docker.sh git clone $(COLLECTD_CEILOMETER_REPO) - cd fastpathmetrics; \ + cd barometer; \ git checkout $(COLLECTD_CEILOMETER_BRANCH); \ if [ ! -z $(COLLECTD_CEILOMETER_CHANGE) ]; then \ git fetch $(COLLECTD_CEILOMETER_REPO) $(COLLECTD_CEILOMETER_CHANGE); \ git checkout FETCH_HEAD; \ fi - fpb --debug --build fastpathmetrics/src/fuel-plugin - @mv fastpathmetrics/src/fuel-plugin/fuel-plugin-collectd-ceilometer*.rpm . + fpb --debug --build barometer/src/fuel-plugin + @mv barometer/src/fuel-plugin/fuel-plugin-collectd-ceilometer*.rpm . $(REPOINFO) -r . > gitinfo_collectd-ceilometer-plugin.txt - @rm -rf fastpathmetrics - @touch .fastpathmetrics + @rm -rf barometer + @touch .barometer # Store artifact in cache straight away if caching is enabled # (no .cacheid will be present unless this is a cached build) test -f .cacheid && $(MAKE) -f Makefile put-cache || exit 0 @@ -88,4 +89,4 @@ get-cache: .cacheid # Store cache if not already stored - called after ordinary build .PHONY: put-cache put-cache: .cacheid - @tar cf - .fastpathmetrics fuel-plugin-collectd-ceilometer*.rpm gitinfo_collectd-ceilometer-plugin.txt | $(CACHETOOL) put $(shell cat .cacheid) + @tar cf - .barometer fuel-plugin-collectd-ceilometer*.rpm gitinfo_collectd-ceilometer-plugin.txt | $(CACHETOOL) put $(shell cat .cacheid) diff --git a/build/f_isoroot/f_collectd-ceilometer-pluginbuild/config.mk b/build/f_isoroot/f_collectd-ceilometer-pluginbuild/config.mk index 0106213d3..5bf1368d1 100644 --- a/build/f_isoroot/f_collectd-ceilometer-pluginbuild/config.mk +++ b/build/f_isoroot/f_collectd-ceilometer-pluginbuild/config.mk @@ -8,6 +8,6 @@ # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## -COLLECTD_CEILOMETER_BRANCH?=3ddf95235ef62de5fb9a520b3aa8f120518b0077 -COLLECTD_CEILOMETER_REPO?=http://gerrit.opnfv.org/gerrit/fastpathmetrics -COLLECTD_CEILOMETER_CHANGE?= +COLLECTD_CEILOMETER_BRANCH?= +COLLECTD_CEILOMETER_REPO?=http://gerrit.opnfv.org/gerrit/barometer +COLLECTD_CEILOMETER_CHANGE?=refs/changes/13/26213/7 diff --git a/build/f_isoroot/f_congress-pluginbuild/Makefile b/build/f_isoroot/f_congress-pluginbuild/Makefile new file mode 100644 index 000000000..51167acd9 --- /dev/null +++ b/build/f_isoroot/f_congress-pluginbuild/Makefile @@ -0,0 +1,91 @@ +############################################################################## +# Copyright (c) 2016 Mirantis Inc and others. +# fzhadaev@mirantis.com +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +TOP := $(shell pwd) + +############################################################################ +# BEGIN of Include definitions +# +include config.mk +# +# END Include definitions +############################################################################# + + +.PHONY: all +all: .congressbuild + +.PHONY: clean +clean: + @rm -f .congressbuild ../release/opnfv/congress*.rpm congress*.rpm + @rm -f $(BUILD_BASE)/gitinfo_congressplugin.txt gitinfo_congressplugin.txt + +.PHONY: release +release:.congressbuild + @rm -f ../release/opnfv/congress*.rpm + @mkdir -p ../release/congress + @cp congress*.rpm ../release/opnfv/ + cp gitinfo_congressplugin.txt $(BUILD_BASE) + +.congressbuild: + rm -rf fuel-plugin-congress + git clone $(CONGRESS_REPO) + cd fuel-plugin-congress; \ + git checkout $(CONGRESS_BRANCH); \ + if [ ! -z $(CONGRESS_CHANGE) ]; then \ + git fetch $(CONGRESS_REPO) $(CONGRESS_CHANGE); \ + git checkout FETCH_HEAD; \ + fi + fpb --debug --build fuel-plugin-congress/ + @mv fuel-plugin-congress/congress*.rpm . + $(REPOINFO) -r . > gitinfo_congressplugin.txt + @rm -rf fuel-plugin-congress + @touch .congressbuild + # Store artifact in cache straight away if caching is enabled + # (no .cacheid will be present unless this is a cached build) + test -f .cacheid && $(MAKE) -f Makefile put-cache || exit 0 + +############################################################################# +# Cache operations - only used when building through ci/build.sh +############################################################################# + + +# Create a unique hash to be used for getting and putting cache, based on: +# - The SHA1 hash of the HEAD on the plugin repo's $(CONGRESS_BRANCH) +# - The contents of this Makefile +.cacheid: + @if [ ! -z $(CONGRESS_CHANGE) ]; then \ + $(CACHETOOL) getcommitid $(CONGRESS_REPO) $(CONGRESS_CHANGE) > .cachedata; \ + else \ + $(CACHETOOL) getcommitid $(CONGRESS_REPO) $(CONGRESS_BRANCH) > .cachedata; \ + fi + @sha1sum Makefile | awk {'print $$1'} >> .cachedata + @sha1sum config.mk | awk {'print $$1'} >> .cachedata + @echo -n $(UBUNTU_ARCH) | sha1sum | awk {'print $$1'} >> .cachedata + @cat .cachedata | $(CACHETOOL) getid > .cacheid + +# Clean local data related to caching - called prior to ordinary build +.PHONY: clean-cache +clean-cache: clean + @rm -f .cachedata .cacheid + +# Try to download cache - called prior to ordinary build +.PHONY: get-cache +get-cache: .cacheid + @if $(CACHETOOL) check $(shell cat .cacheid); then \ + $(CACHETOOL) get $(shell cat .cacheid) | tar xf -;\ + else \ + echo "No cache item found for $(shell cat .cacheid)" ;\ + exit 0;\ + fi + +# Store cache if not already stored - called after ordinary build +.PHONY: put-cache +put-cache: .cacheid + @tar cf - .congressbuild congress*.rpm gitinfo_congressplugin.txt | $(CACHETOOL) put $(shell cat .cacheid) diff --git a/build/f_isoroot/f_congress-pluginbuild/config.mk b/build/f_isoroot/f_congress-pluginbuild/config.mk new file mode 100644 index 000000000..5c241a127 --- /dev/null +++ b/build/f_isoroot/f_congress-pluginbuild/config.mk @@ -0,0 +1,12 @@ +############################################################################## +# Copyright (c) 2016 Mirantis Inc and others. +# fzhadaev@mirantis.com +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +CONGRESS_BRANCH?=master +CONGRESS_REPO?="https://github.com/openstack/fuel-plugin-congress" +CONGRESS_CHANGE?=adfa2db62988649219d64bd53746f2635d95aa43 diff --git a/build/f_isoroot/f_ovs-nsh-dpdk-pluginbuild/config.mk b/build/f_isoroot/f_ovs-nsh-dpdk-pluginbuild/config.mk index 8d267d249..49680f418 100644 --- a/build/f_isoroot/f_ovs-nsh-dpdk-pluginbuild/config.mk +++ b/build/f_isoroot/f_ovs-nsh-dpdk-pluginbuild/config.mk @@ -7,6 +7,6 @@ # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## -OVS_NSH_DPDK_BRANCH?=a8f78d202d502f0bc51d12d88b39905a8e8278e8 +OVS_NSH_DPDK_BRANCH?=d0487867be45c6e785f9fb241efd6adef0f0625f OVS_NSH_DPDK_REPO?=https://review.openstack.org/openstack/fuel-plugin-ovs OVS_NSH_DPDK_CHANGE?= diff --git a/build/f_isoroot/f_repobuild/select_ubuntu_repo.sh b/build/f_isoroot/f_repobuild/select_ubuntu_repo.sh index 24bd42234..5618eb79b 100755 --- a/build/f_isoroot/f_repobuild/select_ubuntu_repo.sh +++ b/build/f_isoroot/f_repobuild/select_ubuntu_repo.sh @@ -69,14 +69,14 @@ if [ "$1" == "-d" ]; then fi # Hardcode for testing purposes -DEBUG=1 +# DEBUG=1 TMPFILE=$(mktemp /tmp/mirrorsXXXXX)A trap cleanup exit # Generate a list of mirrors considered as "up" curl -s https://launchpad.net/ubuntu/+archivemirrors | \ - grep -P -B8 "statusUP|statusSIX" | \ + grep -P -B8 "statusUP|statusONE|statusSIX" | \ grep -o -P "(f|ht)tp.*\"" | \ sed 's/"$//' | sort | uniq > $TMPFILE diff --git a/build/f_isoroot/f_yardstick-pluginbuild/config.mk b/build/f_isoroot/f_yardstick-pluginbuild/config.mk index 88b1ea604..d47e6a8e0 100644 --- a/build/f_isoroot/f_yardstick-pluginbuild/config.mk +++ b/build/f_isoroot/f_yardstick-pluginbuild/config.mk @@ -8,6 +8,6 @@ # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## -YARDSTICK_BRANCH?=86b9fc4f434c72bd276e4bf1ce7d7f5a04b150b2 +YARDSTICK_BRANCH?=31f563b8aa2e6dcf652cb7e9dfdc3f6c7316dc1d YARDSTICK_REPO?=https://gerrit.opnfv.org/gerrit/yardstick.git YARDSTICK_CHANGE?= diff --git a/build/f_repos/patch/fuel-library/0001-Reduce-ceilometer-memory-usage.patch b/build/f_repos/patch/fuel-library/0001-Reduce-ceilometer-memory-usage.patch new file mode 100644 index 000000000..a1ee0364f --- /dev/null +++ b/build/f_repos/patch/fuel-library/0001-Reduce-ceilometer-memory-usage.patch @@ -0,0 +1,30 @@ +From 3c0a101279ef225c6d14b13a5424d0d190df1079 Mon Sep 17 00:00:00 2001 +From: Michael Polenchuk <mpolenchuk@mirantis.com> +Date: Mon, 19 Dec 2016 16:03:37 +0400 +Subject: [PATCH] Reduce ceilometer memory usage + +Change-Id: I11045c233eeb833108ffdee4a8952cd61b91c840 +--- + .../puppet/openstack_tasks/manifests/ceilometer/controller.pp | 7 +++++++ + 1 file changed, 7 insertions(+) + +diff --git a/deployment/puppet/openstack_tasks/manifests/ceilometer/controller.pp b/deployment/puppet/openstack_tasks/manifests/ceilometer/controller.pp +index f531b617d..7f811f276 100644 +--- a/deployment/puppet/openstack_tasks/manifests/ceilometer/controller.pp ++++ b/deployment/puppet/openstack_tasks/manifests/ceilometer/controller.pp +@@ -243,5 +243,12 @@ class openstack_tasks::ceilometer::controller { + 'DEFAULT/use_syslog_rfc_format': value => true; + } + } ++ ++ # Reduce memory usage ++ ceilometer_config { ++ 'DEFAULT/executor_thread_pool_size': value => ceiling($service_workers*0.5); ++ 'notification/pipeline_processing_queues': value => ceiling($service_workers*1.5); ++ 'oslo_messaging_rabbit/rabbit_qos_prefetch_count': value => 100; ++ } + } + } +-- +2.11.0 + diff --git a/build/f_repos/patch/fuel-library/0002-Rework-processorcount-limits-to-os_workers.patch b/build/f_repos/patch/fuel-library/0002-Rework-processorcount-limits-to-os_workers.patch new file mode 100644 index 000000000..6b495cbf1 --- /dev/null +++ b/build/f_repos/patch/fuel-library/0002-Rework-processorcount-limits-to-os_workers.patch @@ -0,0 +1,284 @@ +From 15284786b55e844fcdcc95d5865c91be03871487 Mon Sep 17 00:00:00 2001 +From: Bogdan Dobrelya <bdobrelia@mirantis.com> +Date: Fri, 30 Sep 2016 12:03:34 +0200 +Subject: [PATCH] Rework processorcount limits to os_workers + +* reduce workers_max default limits of a 16 to the $::os_workers, which is limited by [2, 8] interval. +* reduce workers/pool size/overflow limited by F($::processorcount) to F($::os_workers) + +Closes-bug: 1629238 + +Change-Id: I5cc4d70b902eeaa1c9cf42911606eba13dd84aa2 +Signed-off-by: Bogdan Dobrelya <bdobrelia@mirantis.com> + +diff --git a/deployment/puppet/openstack/manifests/cinder.pp b/deployment/puppet/openstack/manifests/cinder.pp +index 36747a61e..502bc94b8 100644 +--- a/deployment/puppet/openstack/manifests/cinder.pp ++++ b/deployment/puppet/openstack/manifests/cinder.pp +@@ -44,7 +44,7 @@ class openstack::cinder( + $keystone_user = 'cinder', + $region = 'RegionOne', + $notification_driver = undef, +- $service_workers = $::processorcount, ++ $service_workers = $::os_workers, + $vmware_host_ip = '10.10.10.10', + $vmware_host_username = 'administrator@vsphere.local', + $vmware_host_password = 'password', +diff --git a/deployment/puppet/openstack_tasks/manifests/ceilometer/controller.pp b/deployment/puppet/openstack_tasks/manifests/ceilometer/controller.pp +index 72fbe3271..e6fcf4f88 100644 +--- a/deployment/puppet/openstack_tasks/manifests/ceilometer/controller.pp ++++ b/deployment/puppet/openstack_tasks/manifests/ceilometer/controller.pp +@@ -35,7 +35,7 @@ class openstack_tasks::ceilometer::controller { + $service_endpoint = hiera('service_endpoint', $management_vip) + $ha_mode = pick($ceilometer_hash['ha_mode'], true) + $ssl_hash = hiera_hash('use_ssl', {}) +- $workers_max = hiera('workers_max', 16) ++ $workers_max = hiera('workers_max', $::os_workers) + $service_workers = pick($ceilometer_hash['workers'], + min(max($::processorcount, 2), $workers_max)) + +diff --git a/deployment/puppet/openstack_tasks/manifests/glance/glance.pp b/deployment/puppet/openstack_tasks/manifests/glance/glance.pp +index 4c951999d..f3a20c5a7 100644 +--- a/deployment/puppet/openstack_tasks/manifests/glance/glance.pp ++++ b/deployment/puppet/openstack_tasks/manifests/glance/glance.pp +@@ -21,7 +21,7 @@ class openstack_tasks::glance::glance { + $max_overflow = hiera('max_overflow') + $ceilometer_hash = hiera_hash('ceilometer', {}) + $region = hiera('region','RegionOne') +- $workers_max = hiera('workers_max', 16) ++ $workers_max = hiera('workers_max', $::os_workers) + $service_workers = pick($glance_hash['glance_workers'], + min(max($::processorcount, 2), $workers_max)) + $ironic_hash = hiera_hash('ironic', {}) +diff --git a/deployment/puppet/openstack_tasks/manifests/horizon/horizon.pp b/deployment/puppet/openstack_tasks/manifests/horizon/horizon.pp +index 1a70af711..824c700bb 100644 +--- a/deployment/puppet/openstack_tasks/manifests/horizon/horizon.pp ++++ b/deployment/puppet/openstack_tasks/manifests/horizon/horizon.pp +@@ -134,7 +134,7 @@ class openstack_tasks::horizon::horizon { + $wsgi_processes = 2 + $wsgi_threads = 9 + } else { +- $wsgi_processes = $::processorcount ++ $wsgi_processes = $::os_workers + $wsgi_threads = 15 + } + +diff --git a/deployment/puppet/openstack_tasks/manifests/ironic/ironic_compute.pp b/deployment/puppet/openstack_tasks/manifests/ironic/ironic_compute.pp +index 2d26e1f19..6e7d8ca2b 100644 +--- a/deployment/puppet/openstack_tasks/manifests/ironic/ironic_compute.pp ++++ b/deployment/puppet/openstack_tasks/manifests/ironic/ironic_compute.pp +@@ -35,8 +35,8 @@ class openstack_tasks::ironic::ironic_compute { + $db_name = pick($nova_hash['db_name'], 'nova') + $db_password = pick($nova_hash['db_password'], 'nova') + +- $max_pool_size = hiera('max_pool_size', min($::processorcount * 5 + 0, 30 + 0)) +- $max_overflow = hiera('max_overflow', min($::processorcount * 5 + 0, 60 + 0)) ++ $max_pool_size = hiera('max_pool_size', min($::os_workers * 5 + 0, 30 + 0)) ++ $max_overflow = hiera('max_overflow', min($::os_workers * 5 + 0, 60 + 0)) + $idle_timeout = hiera('idle_timeout', '3600') + $max_retries = hiera('max_retries', '-1') + +diff --git a/deployment/puppet/openstack_tasks/manifests/keystone/keystone.pp b/deployment/puppet/openstack_tasks/manifests/keystone/keystone.pp +index 0d96b177a..0d3d33bff 100644 +--- a/deployment/puppet/openstack_tasks/manifests/keystone/keystone.pp ++++ b/deployment/puppet/openstack_tasks/manifests/keystone/keystone.pp +@@ -22,7 +22,7 @@ class openstack_tasks::keystone::keystone { + $syslog_log_facility = hiera('syslog_log_facility_keystone') + $rabbit_hash = hiera_hash('rabbit', {}) + $neutron_user_password = hiera('neutron_user_password', false) +- $workers_max = hiera('workers_max', 16) ++ $workers_max = hiera('workers_max', $::os_workers) + $service_workers = pick($keystone_hash['workers'], + min(max($::processorcount, 2), $workers_max)) + $default_log_levels = hiera_hash('default_log_levels') +diff --git a/deployment/puppet/openstack_tasks/manifests/openstack_cinder/openstack_cinder.pp b/deployment/puppet/openstack_tasks/manifests/openstack_cinder/openstack_cinder.pp +index 6eccfc37b..644dd2440 100644 +--- a/deployment/puppet/openstack_tasks/manifests/openstack_cinder/openstack_cinder.pp ++++ b/deployment/puppet/openstack_tasks/manifests/openstack_cinder/openstack_cinder.pp +@@ -13,7 +13,7 @@ class openstack_tasks::openstack_cinder::openstack_cinder { + $sahara_hash = hiera_hash('sahara', {}) + $rabbit_hash = hiera_hash('rabbit', {}) + $service_endpoint = hiera('service_endpoint') +- $workers_max = hiera('workers_max', 16) ++ $workers_max = hiera('workers_max', $::os_workers) + $service_workers = pick($cinder_hash['workers'], min(max($::processorcount, 2), $workers_max)) + $cinder_user_password = $cinder_hash[user_password] + $keystone_user = pick($cinder_hash['user'], 'cinder') +@@ -84,8 +84,8 @@ class openstack_tasks::openstack_cinder::openstack_cinder { + } + + # SQLAlchemy backend configuration +- $max_pool_size = min($::processorcount * 5 + 0, 30 + 0) +- $max_overflow = min($::processorcount * 5 + 0, 60 + 0) ++ $max_pool_size = min($::os_workers * 5 + 0, 30 + 0) ++ $max_overflow = min($::os_workers * 5 + 0, 60 + 0) + $max_retries = '-1' + $idle_timeout = '3600' + +diff --git a/deployment/puppet/openstack_tasks/manifests/openstack_controller/openstack_controller.pp b/deployment/puppet/openstack_tasks/manifests/openstack_controller/openstack_controller.pp +index 458c34c8f..89fcb20b3 100644 +--- a/deployment/puppet/openstack_tasks/manifests/openstack_controller/openstack_controller.pp ++++ b/deployment/puppet/openstack_tasks/manifests/openstack_controller/openstack_controller.pp +@@ -48,7 +48,7 @@ class openstack_tasks::openstack_controller::openstack_controller { + $keystone_user = pick($nova_hash['user'], 'nova') + $keystone_tenant = pick($nova_hash['tenant'], 'services') + $region_name = hiera('region', 'RegionOne') +- $workers_max = hiera('workers_max', 16) ++ $workers_max = hiera('workers_max', $::os_workers) + $service_workers = pick($nova_hash['workers'], + min(max($::processorcount, 2), $workers_max)) + $compute_nodes = get_nodes_hash_by_roles($network_metadata, ['compute']) +@@ -117,8 +117,8 @@ class openstack_tasks::openstack_controller::openstack_controller { + }) + + # SQLAlchemy backend configuration +- $max_pool_size = hiera('max_pool_size', min($::processorcount * 5 + 0, 30 + 0)) +- $max_overflow = hiera('max_overflow', min($::processorcount * 5 + 0, 60 + 0)) ++ $max_pool_size = hiera('max_pool_size', min($::os_workers * 5 + 0, 30 + 0)) ++ $max_overflow = hiera('max_overflow', min($::os_workers * 5 + 0, 60 + 0)) + $idle_timeout = hiera('idle_timeout', '3600') + $max_retries = hiera('max_retries', '-1') + +diff --git a/deployment/puppet/openstack_tasks/manifests/openstack_network/agents/metadata.pp b/deployment/puppet/openstack_tasks/manifests/openstack_network/agents/metadata.pp +index 9492493d1..240a378b7 100644 +--- a/deployment/puppet/openstack_tasks/manifests/openstack_network/agents/metadata.pp ++++ b/deployment/puppet/openstack_tasks/manifests/openstack_network/agents/metadata.pp +@@ -9,7 +9,7 @@ class openstack_tasks::openstack_network::agents::metadata { + $neutron_advanced_config = hiera_hash('neutron_advanced_configuration', { }) + $neutron_config = hiera_hash('neutron_config') + $dvr = pick($neutron_advanced_config['neutron_dvr'], false) +- $workers_max = hiera('workers_max', 16) ++ $workers_max = hiera('workers_max', $::os_workers) + + if $compute { + $metadata_workers = pick($neutron_config['workers'], +diff --git a/deployment/puppet/openstack_tasks/manifests/openstack_network/server_config.pp b/deployment/puppet/openstack_tasks/manifests/openstack_network/server_config.pp +index 1d5c2d0e1..822e41750 100644 +--- a/deployment/puppet/openstack_tasks/manifests/openstack_network/server_config.pp ++++ b/deployment/puppet/openstack_tasks/manifests/openstack_network/server_config.pp +@@ -64,7 +64,7 @@ class openstack_tasks::openstack_network::server_config { + $auth_url = "${internal_auth_protocol}://${internal_auth_endpoint}:35357/" + $nova_admin_auth_url = "${admin_auth_protocol}://${admin_auth_endpoint}:35357/" + +- $workers_max = hiera('workers_max', 16) ++ $workers_max = hiera('workers_max', $::os_workers) + $service_workers = pick($neutron_config['workers'], min(max($::processorcount, 1), $workers_max)) + + $neutron_advanced_config = hiera_hash('neutron_advanced_configuration', { }) +diff --git a/deployment/puppet/openstack_tasks/manifests/roles/cinder.pp b/deployment/puppet/openstack_tasks/manifests/roles/cinder.pp +index b36619437..b6958b64f 100644 +--- a/deployment/puppet/openstack_tasks/manifests/roles/cinder.pp ++++ b/deployment/puppet/openstack_tasks/manifests/roles/cinder.pp +@@ -81,8 +81,8 @@ class openstack_tasks::roles::cinder { + } + + # SQLAlchemy backend configuration +- $max_pool_size = min($::processorcount * 5 + 0, 30 + 0) +- $max_overflow = min($::processorcount * 5 + 0, 60 + 0) ++ $max_pool_size = min($::os_workers * 5 + 0, 30 + 0) ++ $max_overflow = min($::os_workers * 5 + 0, 60 + 0) + $max_retries = '-1' + $idle_timeout = '3600' + +diff --git a/deployment/puppet/openstack_tasks/manifests/sahara/sahara.pp b/deployment/puppet/openstack_tasks/manifests/sahara/sahara.pp +index 9d011ceb5..0366659f0 100644 +--- a/deployment/puppet/openstack_tasks/manifests/sahara/sahara.pp ++++ b/deployment/puppet/openstack_tasks/manifests/sahara/sahara.pp +@@ -51,8 +51,8 @@ class openstack_tasks::sahara::sahara { + $sahara_user = pick($sahara_hash['user'], 'sahara') + $sahara_password = pick($sahara_hash['user_password']) + $tenant = pick($sahara_hash['tenant'], 'services') +- $max_pool_size = min($::processorcount * 5 + 0, 30 + 0) +- $max_overflow = min($::processorcount * 5 + 0, 60 + 0) ++ $max_pool_size = min($::os_workers * 5 + 0, 30 + 0) ++ $max_overflow = min($::os_workers * 5 + 0, 60 + 0) + $max_retries = '-1' + $idle_timeout = '3600' + +diff --git a/deployment/puppet/openstack_tasks/manifests/swift/parts/proxy.pp b/deployment/puppet/openstack_tasks/manifests/swift/parts/proxy.pp +index 81d64a2d8..8a68acdb3 100644 +--- a/deployment/puppet/openstack_tasks/manifests/swift/parts/proxy.pp ++++ b/deployment/puppet/openstack_tasks/manifests/swift/parts/proxy.pp +@@ -26,7 +26,7 @@ class openstack_tasks::swift::parts::proxy ( + 'account_quotas', + 'slo', + 'proxy-server'], +- $proxy_workers = $::processorcount, ++ $proxy_workers = $::os_workers, + $proxy_port = '8080', + $proxy_allow_account_management = true, + $proxy_account_autocreate = true, +diff --git a/deployment/puppet/openstack_tasks/manifests/swift/proxy_storage.pp b/deployment/puppet/openstack_tasks/manifests/swift/proxy_storage.pp +index a221baf36..77f4ba1a0 100644 +--- a/deployment/puppet/openstack_tasks/manifests/swift/proxy_storage.pp ++++ b/deployment/puppet/openstack_tasks/manifests/swift/proxy_storage.pp +@@ -30,7 +30,7 @@ class openstack_tasks::swift::proxy_storage { + $keystone_user = pick($swift_hash['user'], 'swift') + $keystone_password = pick($swift_hash['user_password'], 'passsword') + $keystone_tenant = pick($swift_hash['tenant'], 'services') +- $workers_max = hiera('workers_max', 16) ++ $workers_max = hiera('workers_max', $::os_workers) + $service_workers = pick($swift_hash['workers'], min(max($::processorcount, 2), $workers_max)) + $ssl_hash = hiera_hash('use_ssl', {}) + $rabbit_hash = hiera_hash('rabbit') +diff --git a/deployment/puppet/osnailyfacter/lib/facter/os_workers.rb b/deployment/puppet/osnailyfacter/lib/facter/os_workers.rb +new file mode 100644 +index 000000000..404fb9018 +--- /dev/null ++++ b/deployment/puppet/osnailyfacter/lib/facter/os_workers.rb +@@ -0,0 +1,20 @@ ++# ++# We've found that using $::processorcount for workers/threads can lead to ++# unexpected memory or process counts for people deploying on baremetal or ++# if they have large number of cpus. This fact allows us to tweak the formula ++# used to determine number of workers in a single place but use it across all ++# modules. ++# ++# The value for os_workers is max between '(<# processors> / 4)' and '2' with ++# a cap of 8. ++# ++# This fact can be overloaded by an external fact from /etc/factor/facts.d if ++# a user would like to provide their own default value. ++# ++Facter.add(:os_workers) do ++ has_weight 100 ++ setcode do ++ processors = Facter.value('processorcount') ++ [ [ (processors.to_i / 4), 2 ].max, 8 ].min ++ end ++end +diff --git a/deployment/puppet/osnailyfacter/manifests/apache_mpm.pp b/deployment/puppet/osnailyfacter/manifests/apache_mpm.pp +index 51bc3274d..70a654ae2 100644 +--- a/deployment/puppet/osnailyfacter/manifests/apache_mpm.pp ++++ b/deployment/puppet/osnailyfacter/manifests/apache_mpm.pp +@@ -11,7 +11,7 @@ class osnailyfacter::apache_mpm inherits ::osnailyfacter::apache { + if ($::processorcount + 0) <= 2 { + $startservers = 2 + } else { +- $startservers = $::processorcount ++ $startservers = $::os_workers + } + + $maxrequestsperchild = 0 +diff --git a/deployment/puppet/osnailyfacter/manifests/globals/globals.pp b/deployment/puppet/osnailyfacter/manifests/globals/globals.pp +index 99e0fa4cd..2e38b10ee 100644 +--- a/deployment/puppet/osnailyfacter/manifests/globals/globals.pp ++++ b/deployment/puppet/osnailyfacter/manifests/globals/globals.pp +@@ -311,14 +311,14 @@ class osnailyfacter::globals::globals { + # The default value assumes there are 20 different types of workers limited by 100Mb each. + $workers_ratio = hiera('workers_ratio', 2000) + # Defines the maximum allowed number of workers for each service by RAM limits. Cannot exceed the value of 30. +- $workers_max = inline_template("<%= [(@memorysize_mb.to_i / @workers_ratio.to_i).floor + 1, 30].min %>") ++ $workers_max = inline_template("<%= [(@memorysize_mb.to_i / @workers_ratio.to_i).floor + 1, $::os_workers].min %>") + + $node_name_prefix_for_messaging = hiera('node_name_prefix_for_messaging', 'messaging-') + + # MySQL and SQLAlchemy backend configuration + $custom_mysql_setup_class = hiera('custom_mysql_setup_class', 'galera') +- $max_pool_size = hiera('max_pool_size', min($::processorcount * 5 + 0, 30 + 0)) +- $max_overflow = hiera('max_overflow', min($::processorcount * 5 + 0, 60 + 0)) ++ $max_pool_size = hiera('max_pool_size', min($::os_workers * 5 + 0, 30 + 0)) ++ $max_overflow = hiera('max_overflow', min($::os_workers * 5 + 0, 60 + 0)) + $max_retries = hiera('max_retries', '-1') + $idle_timeout = hiera('idle_timeout','3600') + $nova_db_password = $nova_hash['db_password'] diff --git a/deploy/cloud/configure_nodes.py b/deploy/cloud/configure_nodes.py index ea50307b1..eabec397c 100644 --- a/deploy/cloud/configure_nodes.py +++ b/deploy/cloud/configure_nodes.py @@ -8,7 +8,6 @@ ############################################################################### import copy -import glob import io import six @@ -143,7 +142,8 @@ class ConfigureNodes(object): # ens5: # - interface_properties: # dpdk: - # enabled: true + # enabled: + # value: true # - private # ens6: # - public @@ -170,19 +170,19 @@ class ConfigureNodes(object): if interface['name'] in interface_config: for prop in interface_config[interface['name']]: net = {} - #net name + # net name if isinstance(prop, six.string_types): net['id'] = net_name_id[prop] net['name'] = prop interface['assigned_networks'].append(net) - #network properties + # network properties elif isinstance(prop, dict): - if not 'interface_properties' in prop: - log('Interface configuration contain unknow dict: %s' % prop) + if 'interface_properties' not in prop: + log('Interface configuration contains unknown dict: %s' % prop) continue - interface['interface_properties'] = \ - self._merge_dicts(interface.get('interface_properties', {}), - prop.get('interface_properties', {})) + interface['attributes'] = self._merge_dicts( + interface.get('attributes', {}), + prop.get('interface_properties', {})) with io.open(interface_yaml, 'w') as stream: yaml.dump(interfaces, stream, default_flow_style=False) @@ -199,4 +199,3 @@ class ConfigureNodes(object): continue result[k] = copy.deepcopy(v) return result - diff --git a/deploy/cloud/deploy.py b/deploy/cloud/deploy.py index e00934bd9..dac2fe8dd 100644 --- a/deploy/cloud/deploy.py +++ b/deploy/cloud/deploy.py @@ -8,9 +8,6 @@ ############################################################################### -import yaml -import io - from dea import DeploymentEnvironmentAdapter from configure_environment import ConfigureEnvironment from deployment import Deployment diff --git a/deploy/cloud/deployment.py b/deploy/cloud/deployment.py index 5dd026318..28bcfdf1d 100644 --- a/deploy/cloud/deployment.py +++ b/deploy/cloud/deployment.py @@ -13,15 +13,10 @@ import json from common import ( N, - E, exec_cmd, - run_proc, - run_proc_wait_terminated, - run_proc_kill, parse, err, log, - delete, ) SEARCH_TEXT = '(err)' diff --git a/deploy/common.py b/deploy/common.py index dab9602c5..b3d552415 100644 --- a/deploy/common.py +++ b/deploy/common.py @@ -15,7 +15,6 @@ import os import logging import argparse import shutil -import stat import errno import time import shlex @@ -126,7 +125,7 @@ def clean(lines): return parsed if len(parsed_list) == 1 else parsed_list -def err(message, fun = None, *args): +def err(message, fun=None, *args): LOG.error('%s\n' % message) if fun: fun(*args) diff --git a/deploy/config/labs/devel-pipeline/default/fuel/config/dea-pod-override.yaml b/deploy/config/labs/devel-pipeline/default/fuel/config/dea-pod-override.yaml index 32b2cfb22..852b363d4 100644 --- a/deploy/config/labs/devel-pipeline/default/fuel/config/dea-pod-override.yaml +++ b/deploy/config/labs/devel-pipeline/default/fuel/config/dea-pod-override.yaml @@ -35,7 +35,8 @@ interfaces_dpdk: - private - interface_properties: dpdk: - enabled: true + enabled: + value: true ens6: - public interfaces_vlan: diff --git a/deploy/config/labs/devel-pipeline/elx/fuel/config/dea-pod-override.yaml b/deploy/config/labs/devel-pipeline/elx/fuel/config/dea-pod-override.yaml index cb80e3855..150d4ed06 100644 --- a/deploy/config/labs/devel-pipeline/elx/fuel/config/dea-pod-override.yaml +++ b/deploy/config/labs/devel-pipeline/elx/fuel/config/dea-pod-override.yaml @@ -35,7 +35,8 @@ interfaces_dpdk: - private - interface_properties: dpdk: - enabled: true + enabled: + value: true ens6: - public interfaces_vlan: diff --git a/deploy/config/labs/devel-pipeline/huawei-ch/fuel/config/dea-pod-override.yaml b/deploy/config/labs/devel-pipeline/huawei-ch/fuel/config/dea-pod-override.yaml index 92cc0e616..be80b9724 100644 --- a/deploy/config/labs/devel-pipeline/huawei-ch/fuel/config/dea-pod-override.yaml +++ b/deploy/config/labs/devel-pipeline/huawei-ch/fuel/config/dea-pod-override.yaml @@ -35,7 +35,8 @@ interfaces_dpdk: - private - interface_properties: dpdk: - enabled: true + enabled: + value: true ens6: - public interfaces_vlan: diff --git a/deploy/config/labs/devel-pipeline/intel-sc/fuel/config/dea-pod-override.yaml b/deploy/config/labs/devel-pipeline/intel-sc/fuel/config/dea-pod-override.yaml index ed7c63fac..35a2201cf 100644 --- a/deploy/config/labs/devel-pipeline/intel-sc/fuel/config/dea-pod-override.yaml +++ b/deploy/config/labs/devel-pipeline/intel-sc/fuel/config/dea-pod-override.yaml @@ -35,7 +35,8 @@ interfaces_dpdk: - private - interface_properties: dpdk: - enabled: true + enabled: + value: true ens6: - public interfaces_vlan: diff --git a/deploy/config/plugins/fuel-congress_1.0.0.yaml b/deploy/config/plugins/fuel-congress_1.0.0.yaml new file mode 100644 index 000000000..c7ba8a138 --- /dev/null +++ b/deploy/config/plugins/fuel-congress_1.0.0.yaml @@ -0,0 +1,49 @@ +############################################################################## +# Copyright (c) 2016 Mirantis Inc and others. +# fzhadaev@mirantis.com +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +plugin-config-metadata: + title: Congress fuel plugin configuration template + version: 0.1 + created: 03.10.2016 + comment: None +congress: + metadata: + #chosen_id: Assigned during installation + class: plugin + default: false + enabled: true + label: Fuel Congress plugin + toggleable: true + versions: + - enabled: + description: Service enable state for congress-server + label: Enable + type: checkbox + value: true + weight: 25 + metadata: + contains_legacy_tasks: false + db_password: + generator: 'password' + group: openstack_services + hot_pluggable: false + #plugin_id: Assigned during installation + plugin_version: 1.0.0 + port: 1789 + service: congress-server + user: congress + user_password: + generator: 'password' + policies: + description: Set of policies to configure for congress (in JSON format) + label: Policies + type: text + value: '{}' + weight: 50 + weight: 70
\ No newline at end of file diff --git a/deploy/deploy-config.py b/deploy/deploy-config.py index d2567d97b..5f5699cd7 100644 --- a/deploy/deploy-config.py +++ b/deploy/deploy-config.py @@ -31,17 +31,7 @@ import time import collections import hashlib -from functools import reduce -from operator import or_ from common import ( - log, - exec_cmd, - err, - warn, - check_file_exists, - create_dir_if_not_exists, - delete, - check_if_root, ArgParser, ) diff --git a/deploy/deploy.py b/deploy/deploy.py index f491929e0..057a0579c 100755 --- a/deploy/deploy.py +++ b/deploy/deploy.py @@ -14,7 +14,6 @@ import io import re import sys import yaml -import errno import signal import netaddr diff --git a/deploy/deploy_env.py b/deploy/deploy_env.py index 93dc3959b..8afaeb143 100644 --- a/deploy/deploy_env.py +++ b/deploy/deploy_env.py @@ -20,7 +20,6 @@ from ssh_client import SSHClient from common import ( err, log, - exec_cmd, parse, N, E, diff --git a/deploy/dha_adapters/zte_adapter.py b/deploy/dha_adapters/zte_adapter.py index da651ca15..5d7939aa2 100644 --- a/deploy/dha_adapters/zte_adapter.py +++ b/deploy/dha_adapters/zte_adapter.py @@ -8,13 +8,11 @@ ############################################################################### -import time from ipmi_adapter import IpmiAdapter from common import ( log, exec_cmd, - err, ) diff --git a/deploy/install_fuel_master.py b/deploy/install_fuel_master.py index a0e28b033..2da69e350 100644 --- a/deploy/install_fuel_master.py +++ b/deploy/install_fuel_master.py @@ -16,7 +16,6 @@ from dha_adapters.libvirt_adapter import LibvirtAdapter from common import ( log, err, - clean, delete, ) diff --git a/deploy/scenario/ha_heat_ceilometer_scenario.yaml b/deploy/scenario/ha_heat_ceilometer_scenario.yaml index 4051c478a..6b032bfdd 100644 --- a/deploy/scenario/ha_heat_ceilometer_scenario.yaml +++ b/deploy/scenario/ha_heat_ceilometer_scenario.yaml @@ -32,6 +32,9 @@ deployment-scenario-metadata: # <module-config-base-uri>/<module-config-name>_<module-config-version>.yaml # It does so by copying the config file to the local plugin config directory stack-extensions: + - module: congress + module-config-name: fuel-congress + module-config-version: 1.0.0 ############################################################################## # By editing the override-config sections below, you can override arbitrary @@ -40,7 +43,7 @@ dea-override-config: nodes: - id: 1 interfaces: interfaces_1 - role: controller + role: controller, congress - id: 2 interfaces: interfaces_1 role: mongo, controller diff --git a/deploy/scenario/no-ha_heat_ceilometer_scenario.yaml b/deploy/scenario/no-ha_heat_ceilometer_scenario.yaml index 3dd5f841e..801b9fca7 100644 --- a/deploy/scenario/no-ha_heat_ceilometer_scenario.yaml +++ b/deploy/scenario/no-ha_heat_ceilometer_scenario.yaml @@ -32,6 +32,9 @@ deployment-scenario-metadata: # <module-config-base-uri>/<module-config-name>_<module-config-version>.yaml # It does so by copying the config file to the local plugin config directory stack-extensions: + - module: congress + module-config-name: fuel-congress + module-config-version: 1.0.0 ############################################################################## # By editing the override-config sections below, you can override arbitrary @@ -40,7 +43,7 @@ dea-override-config: nodes: - id: 1 interfaces: interfaces_1 - role: mongo,controller + role: mongo,controller,congress - id: 2 interfaces: interfaces_1 role: ceph-osd,compute |