diff options
15 files changed, 92 insertions, 636 deletions
diff --git a/.gitignore b/.gitignore index 588243653..20bd388d8 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,7 @@ deploy/autodeploy.log /docs_build/ /docs_output/ /releng/ +mcp/scripts/images/mcp_*iso +mcp/scripts/images/mcp_*qcow2 +mcp/scripts/mcp.rsa* +mcp/scripts/user-data.sh diff --git a/.gitmodules b/.gitmodules index 65eefe0b8..7a41f7523 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,4 +1,4 @@ -[submodule "mcp/reclass/classes/system"] +[submodule "reclass-system-salt-model"] path = mcp/reclass/classes/system url = https://github.com/Mirantis/reclass-system-salt-model branch = master diff --git a/build/f_repos/patch/fuel-library/0001-Reduce-ceilometer-memory-usage.patch b/build/f_repos/patch/fuel-library/0001-Reduce-ceilometer-memory-usage.patch deleted file mode 100644 index 27303f916..000000000 --- a/build/f_repos/patch/fuel-library/0001-Reduce-ceilometer-memory-usage.patch +++ /dev/null @@ -1,26 +0,0 @@ -From: Michael Polenchuk <mpolenchuk@mirantis.com> -Date: Mon, 19 Dec 2016 16:03:37 +0400 -Subject: [PATCH] Reduce ceilometer memory usage - -Change-Id: I11045c233eeb833108ffdee4a8952cd61b91c840 ---- - .../puppet/openstack_tasks/manifests/ceilometer/controller.pp | 7 +++++++ - 1 file changed, 7 insertions(+) - -diff --git a/deployment/puppet/openstack_tasks/manifests/ceilometer/controller.pp b/deployment/puppet/openstack_tasks/manifests/ceilometer/controller.pp -index 6f6e3b9..72fbe32 100644 ---- a/deployment/puppet/openstack_tasks/manifests/ceilometer/controller.pp -+++ b/deployment/puppet/openstack_tasks/manifests/ceilometer/controller.pp -@@ -229,5 +229,12 @@ class openstack_tasks::ceilometer::controller { - 'DEFAULT/use_syslog_rfc_format': value => true; - } - } -+ -+ # Reduce memory usage -+ ceilometer_config { -+ 'DEFAULT/executor_thread_pool_size': value => ceiling($service_workers*0.5); -+ 'notification/pipeline_processing_queues': value => ceiling($service_workers*1.5); -+ 'oslo_messaging_rabbit/rabbit_qos_prefetch_count': value => 100; -+ } - } - } diff --git a/build/f_repos/patch/fuel-library/0002-Rework-processorcount-limits-to-os_workers.patch b/build/f_repos/patch/fuel-library/0002-Rework-processorcount-limits-to-os_workers.patch deleted file mode 100644 index b1f1ef432..000000000 --- a/build/f_repos/patch/fuel-library/0002-Rework-processorcount-limits-to-os_workers.patch +++ /dev/null @@ -1,302 +0,0 @@ -From: Bogdan Dobrelya <bdobrelia@mirantis.com> -Date: Fri, 30 Sep 2016 12:03:34 +0200 -Subject: [PATCH] Rework processorcount limits to os_workers - -* reduce workers_max default limits of a 16 to the $::os_workers, which is limited by [2, 8] interval. -* reduce workers/pool size/overflow limited by F($::processorcount) to F($::os_workers) - -Closes-bug: 1629238 - -Change-Id: I5cc4d70b902eeaa1c9cf42911606eba13dd84aa2 -Signed-off-by: Bogdan Dobrelya <bdobrelia@mirantis.com> ---- - deployment/puppet/openstack/manifests/cinder.pp | 2 +- - .../manifests/ceilometer/controller.pp | 2 +- - .../openstack_tasks/manifests/glance/glance.pp | 2 +- - .../openstack_tasks/manifests/horizon/horizon.pp | 2 +- - .../manifests/ironic/ironic_compute.pp | 4 ++-- - .../openstack_tasks/manifests/keystone/keystone.pp | 2 +- - .../manifests/openstack_cinder/openstack_cinder.pp | 6 +++--- - .../openstack_controller/openstack_controller.pp | 6 +++--- - .../manifests/openstack_network/agents/metadata.pp | 2 +- - .../manifests/openstack_network/server_config.pp | 2 +- - .../puppet/openstack_tasks/manifests/roles/cinder.pp | 4 ++-- - .../openstack_tasks/manifests/sahara/sahara.pp | 4 ++-- - .../openstack_tasks/manifests/swift/parts/proxy.pp | 2 +- - .../openstack_tasks/manifests/swift/proxy_storage.pp | 2 +- - .../puppet/osnailyfacter/lib/facter/os_workers.rb | 20 ++++++++++++++++++++ - .../puppet/osnailyfacter/manifests/apache_mpm.pp | 2 +- - .../osnailyfacter/manifests/globals/globals.pp | 6 +++--- - 17 files changed, 45 insertions(+), 25 deletions(-) - create mode 100644 deployment/puppet/osnailyfacter/lib/facter/os_workers.rb - -diff --git a/deployment/puppet/openstack/manifests/cinder.pp b/deployment/puppet/openstack/manifests/cinder.pp -index 36747a6..502bc94 100644 ---- a/deployment/puppet/openstack/manifests/cinder.pp -+++ b/deployment/puppet/openstack/manifests/cinder.pp -@@ -44,6 +44,6 @@ class openstack::cinder( - $keystone_user = 'cinder', - $region = 'RegionOne', - $notification_driver = undef, -- $service_workers = $::processorcount, -+ $service_workers = $::os_workers, - $rbd_pool = 'volumes', - $rbd_user = 'volumes', -diff --git a/deployment/puppet/openstack_tasks/manifests/ceilometer/controller.pp b/deployment/puppet/openstack_tasks/manifests/ceilometer/controller.pp -index 72fbe32..e6fcf4f 100644 ---- a/deployment/puppet/openstack_tasks/manifests/ceilometer/controller.pp -+++ b/deployment/puppet/openstack_tasks/manifests/ceilometer/controller.pp -@@ -35,7 +35,7 @@ class openstack_tasks::ceilometer::controller { - $service_endpoint = hiera('service_endpoint', $management_vip) - $ha_mode = pick($ceilometer_hash['ha_mode'], true) - $ssl_hash = hiera_hash('use_ssl', {}) -- $workers_max = hiera('workers_max', 16) -+ $workers_max = hiera('workers_max', $::os_workers) - $service_workers = pick($ceilometer_hash['workers'], - min(max($::processorcount, 2), $workers_max)) - -diff --git a/deployment/puppet/openstack_tasks/manifests/glance/glance.pp b/deployment/puppet/openstack_tasks/manifests/glance/glance.pp -index bbba7dd..9e9f57c 100644 ---- a/deployment/puppet/openstack_tasks/manifests/glance/glance.pp -+++ b/deployment/puppet/openstack_tasks/manifests/glance/glance.pp -@@ -21,7 +21,7 @@ class openstack_tasks::glance::glance { - $max_overflow = hiera('max_overflow') - $ceilometer_hash = hiera_hash('ceilometer', {}) - $region = hiera('region','RegionOne') -- $workers_max = hiera('workers_max', 16) -+ $workers_max = hiera('workers_max', $::os_workers) - $service_workers = pick($glance_hash['glance_workers'], - min(max($::processorcount, 2), $workers_max)) - $ironic_hash = hiera_hash('ironic', {}) -diff --git a/deployment/puppet/openstack_tasks/manifests/horizon/horizon.pp b/deployment/puppet/openstack_tasks/manifests/horizon/horizon.pp -index 1a70af7..824c700 100644 ---- a/deployment/puppet/openstack_tasks/manifests/horizon/horizon.pp -+++ b/deployment/puppet/openstack_tasks/manifests/horizon/horizon.pp -@@ -134,7 +134,7 @@ class openstack_tasks::horizon::horizon { - $wsgi_processes = 2 - $wsgi_threads = 9 - } else { -- $wsgi_processes = $::processorcount -+ $wsgi_processes = $::os_workers - $wsgi_threads = 15 - } - -diff --git a/deployment/puppet/openstack_tasks/manifests/ironic/ironic_compute.pp b/deployment/puppet/openstack_tasks/manifests/ironic/ironic_compute.pp -index 2d26e1f..6e7d8ca 100644 ---- a/deployment/puppet/openstack_tasks/manifests/ironic/ironic_compute.pp -+++ b/deployment/puppet/openstack_tasks/manifests/ironic/ironic_compute.pp -@@ -35,8 +35,8 @@ class openstack_tasks::ironic::ironic_compute { - $db_name = pick($nova_hash['db_name'], 'nova') - $db_password = pick($nova_hash['db_password'], 'nova') - -- $max_pool_size = hiera('max_pool_size', min($::processorcount * 5 + 0, 30 + 0)) -- $max_overflow = hiera('max_overflow', min($::processorcount * 5 + 0, 60 + 0)) -+ $max_pool_size = hiera('max_pool_size', min($::os_workers * 5 + 0, 30 + 0)) -+ $max_overflow = hiera('max_overflow', min($::os_workers * 5 + 0, 60 + 0)) - $idle_timeout = hiera('idle_timeout', '3600') - $max_retries = hiera('max_retries', '-1') - -diff --git a/deployment/puppet/openstack_tasks/manifests/keystone/keystone.pp b/deployment/puppet/openstack_tasks/manifests/keystone/keystone.pp -index 8535821..bef595d 100644 ---- a/deployment/puppet/openstack_tasks/manifests/keystone/keystone.pp -+++ b/deployment/puppet/openstack_tasks/manifests/keystone/keystone.pp -@@ -22,7 +22,7 @@ class openstack_tasks::keystone::keystone { - $syslog_log_facility = hiera('syslog_log_facility_keystone') - $rabbit_hash = hiera_hash('rabbit', {}) - $neutron_user_password = hiera('neutron_user_password', false) -- $workers_max = hiera('workers_max', 16) -+ $workers_max = hiera('workers_max', $::os_workers) - $service_workers = pick($keystone_hash['workers'], - min(max($::processorcount, 2), $workers_max)) - $default_log_levels = hiera_hash('default_log_levels') -diff --git a/deployment/puppet/openstack_tasks/manifests/openstack_cinder/openstack_cinder.pp b/deployment/puppet/openstack_tasks/manifests/openstack_cinder/openstack_cinder.pp -index ec60184..ce03683 100644 ---- a/deployment/puppet/openstack_tasks/manifests/openstack_cinder/openstack_cinder.pp -+++ b/deployment/puppet/openstack_tasks/manifests/openstack_cinder/openstack_cinder.pp -@@ -13,7 +13,7 @@ class openstack_tasks::openstack_cinder::openstack_cinder { - $sahara_hash = hiera_hash('sahara', {}) - $rabbit_hash = hiera_hash('rabbit', {}) - $service_endpoint = hiera('service_endpoint') -- $workers_max = hiera('workers_max', 16) -+ $workers_max = hiera('workers_max', $::os_workers) - $service_workers = pick($cinder_hash['workers'], min(max($::processorcount, 2), $workers_max)) - $cinder_user_password = $cinder_hash[user_password] - $keystone_user = pick($cinder_hash['user'], 'cinder') -@@ -86,8 +86,8 @@ class openstack_tasks::openstack_cinder::openstack_cinder { - } - - # SQLAlchemy backend configuration -- $max_pool_size = min($::processorcount * 5 + 0, 30 + 0) -- $max_overflow = min($::processorcount * 5 + 0, 60 + 0) -+ $max_pool_size = min($::os_workers * 5 + 0, 30 + 0) -+ $max_overflow = min($::os_workers * 5 + 0, 60 + 0) - $max_retries = '-1' - $idle_timeout = '3600' - -diff --git a/deployment/puppet/openstack_tasks/manifests/openstack_controller/openstack_controller.pp b/deployment/puppet/openstack_tasks/manifests/openstack_controller/openstack_controller.pp -index 5222a37..38aaa40 100644 ---- a/deployment/puppet/openstack_tasks/manifests/openstack_controller/openstack_controller.pp -+++ b/deployment/puppet/openstack_tasks/manifests/openstack_controller/openstack_controller.pp -@@ -48,7 +48,7 @@ class openstack_tasks::openstack_controller::openstack_controller { - $keystone_user = pick($nova_hash['user'], 'nova') - $keystone_tenant = pick($nova_hash['tenant'], 'services') - $region_name = hiera('region', 'RegionOne') -- $workers_max = hiera('workers_max', 16) -+ $workers_max = hiera('workers_max', $::os_workers) - $service_workers = pick($nova_hash['workers'], - min(max($::processorcount, 2), $workers_max)) - $compute_nodes = get_nodes_hash_by_roles($network_metadata, ['compute']) -@@ -119,8 +119,8 @@ class openstack_tasks::openstack_controller::openstack_controller { - $transport_url = hiera('transport_url','rabbit://guest:password@127.0.0.1:5672/') - - # SQLAlchemy backend configuration -- $max_pool_size = hiera('max_pool_size', min($::processorcount * 5 + 0, 30 + 0)) -- $max_overflow = hiera('max_overflow', min($::processorcount * 5 + 0, 60 + 0)) -+ $max_pool_size = hiera('max_pool_size', min($::os_workers * 5 + 0, 30 + 0)) -+ $max_overflow = hiera('max_overflow', min($::os_workers * 5 + 0, 60 + 0)) - $idle_timeout = hiera('idle_timeout', '3600') - $max_retries = hiera('max_retries', '-1') - -diff --git a/deployment/puppet/openstack_tasks/manifests/openstack_network/agents/metadata.pp b/deployment/puppet/openstack_tasks/manifests/openstack_network/agents/metadata.pp -index fe74bc6..881deda 100644 ---- a/deployment/puppet/openstack_tasks/manifests/openstack_network/agents/metadata.pp -+++ b/deployment/puppet/openstack_tasks/manifests/openstack_network/agents/metadata.pp -@@ -9,7 +9,7 @@ class openstack_tasks::openstack_network::agents::metadata { - $neutron_advanced_config = hiera_hash('neutron_advanced_configuration', { }) - $neutron_config = hiera_hash('neutron_config') - $dvr = pick($neutron_advanced_config['neutron_dvr'], false) -- $workers_max = hiera('workers_max', 16) -+ $workers_max = hiera('workers_max', $::os_workers) - - if $compute { - $metadata_workers = pick($neutron_config['workers'], -diff --git a/deployment/puppet/openstack_tasks/manifests/openstack_network/server_config.pp b/deployment/puppet/openstack_tasks/manifests/openstack_network/server_config.pp -index 133c49e..4dbbf05 100644 ---- a/deployment/puppet/openstack_tasks/manifests/openstack_network/server_config.pp -+++ b/deployment/puppet/openstack_tasks/manifests/openstack_network/server_config.pp -@@ -64,7 +64,7 @@ class openstack_tasks::openstack_network::server_config { - $auth_url = "${internal_auth_protocol}://${internal_auth_endpoint}:35357/" - $nova_admin_auth_url = "${admin_auth_protocol}://${admin_auth_endpoint}:35357/" - -- $workers_max = hiera('workers_max', 16) -+ $workers_max = hiera('workers_max', $::os_workers) - $service_workers = pick($neutron_config['workers'], min(max($::processorcount, 1), $workers_max)) - - $neutron_advanced_config = hiera_hash('neutron_advanced_configuration', { }) -diff --git a/deployment/puppet/openstack_tasks/manifests/roles/cinder.pp b/deployment/puppet/openstack_tasks/manifests/roles/cinder.pp -index da3c148..09c3896 100644 ---- a/deployment/puppet/openstack_tasks/manifests/roles/cinder.pp -+++ b/deployment/puppet/openstack_tasks/manifests/roles/cinder.pp -@@ -81,8 +81,8 @@ class openstack_tasks::roles::cinder { - } - - # SQLAlchemy backend configuration -- $max_pool_size = min($::processorcount * 5 + 0, 30 + 0) -- $max_overflow = min($::processorcount * 5 + 0, 60 + 0) -+ $max_pool_size = min($::os_workers * 5 + 0, 30 + 0) -+ $max_overflow = min($::os_workers * 5 + 0, 60 + 0) - $max_retries = '-1' - $idle_timeout = '3600' - -diff --git a/deployment/puppet/openstack_tasks/manifests/sahara/sahara.pp b/deployment/puppet/openstack_tasks/manifests/sahara/sahara.pp -index 9d011ce..0366659 100644 ---- a/deployment/puppet/openstack_tasks/manifests/sahara/sahara.pp -+++ b/deployment/puppet/openstack_tasks/manifests/sahara/sahara.pp -@@ -51,8 +51,8 @@ class openstack_tasks::sahara::sahara { - $sahara_user = pick($sahara_hash['user'], 'sahara') - $sahara_password = pick($sahara_hash['user_password']) - $tenant = pick($sahara_hash['tenant'], 'services') -- $max_pool_size = min($::processorcount * 5 + 0, 30 + 0) -- $max_overflow = min($::processorcount * 5 + 0, 60 + 0) -+ $max_pool_size = min($::os_workers * 5 + 0, 30 + 0) -+ $max_overflow = min($::os_workers * 5 + 0, 60 + 0) - $max_retries = '-1' - $idle_timeout = '3600' - -diff --git a/deployment/puppet/openstack_tasks/manifests/swift/parts/proxy.pp b/deployment/puppet/openstack_tasks/manifests/swift/parts/proxy.pp -index 81d64a2..8a68acd 100644 ---- a/deployment/puppet/openstack_tasks/manifests/swift/parts/proxy.pp -+++ b/deployment/puppet/openstack_tasks/manifests/swift/parts/proxy.pp -@@ -26,7 +26,7 @@ class openstack_tasks::swift::parts::proxy ( - 'account_quotas', - 'slo', - 'proxy-server'], -- $proxy_workers = $::processorcount, -+ $proxy_workers = $::os_workers, - $proxy_port = '8080', - $proxy_allow_account_management = true, - $proxy_account_autocreate = true, -diff --git a/deployment/puppet/openstack_tasks/manifests/swift/proxy_storage.pp b/deployment/puppet/openstack_tasks/manifests/swift/proxy_storage.pp -index a221baf..77f4ba1 100644 ---- a/deployment/puppet/openstack_tasks/manifests/swift/proxy_storage.pp -+++ b/deployment/puppet/openstack_tasks/manifests/swift/proxy_storage.pp -@@ -30,7 +30,7 @@ class openstack_tasks::swift::proxy_storage { - $keystone_user = pick($swift_hash['user'], 'swift') - $keystone_password = pick($swift_hash['user_password'], 'passsword') - $keystone_tenant = pick($swift_hash['tenant'], 'services') -- $workers_max = hiera('workers_max', 16) -+ $workers_max = hiera('workers_max', $::os_workers) - $service_workers = pick($swift_hash['workers'], min(max($::processorcount, 2), $workers_max)) - $ssl_hash = hiera_hash('use_ssl', {}) - $rabbit_hash = hiera_hash('rabbit') -diff --git a/deployment/puppet/osnailyfacter/lib/facter/os_workers.rb b/deployment/puppet/osnailyfacter/lib/facter/os_workers.rb -new file mode 100644 -index 0000000..404fb90 ---- /dev/null -+++ b/deployment/puppet/osnailyfacter/lib/facter/os_workers.rb -@@ -0,0 +1,20 @@ -+# -+# We've found that using $::processorcount for workers/threads can lead to -+# unexpected memory or process counts for people deploying on baremetal or -+# if they have large number of cpus. This fact allows us to tweak the formula -+# used to determine number of workers in a single place but use it across all -+# modules. -+# -+# The value for os_workers is max between '(<# processors> / 2)' and '2' with -+# a cap of 12. -+# -+# This fact can be overloaded by an external fact from /etc/factor/facts.d if -+# a user would like to provide their own default value. -+# -+Facter.add(:os_workers) do -+ has_weight 100 -+ setcode do -+ processors = Facter.value('processorcount') -+ [ [ (processors.to_i / 2), 2 ].max, 12 ].min -+ end -+end -diff --git a/deployment/puppet/osnailyfacter/manifests/apache_mpm.pp b/deployment/puppet/osnailyfacter/manifests/apache_mpm.pp -index 51bc327..70a654a 100644 ---- a/deployment/puppet/osnailyfacter/manifests/apache_mpm.pp -+++ b/deployment/puppet/osnailyfacter/manifests/apache_mpm.pp -@@ -11,7 +11,7 @@ class osnailyfacter::apache_mpm inherits ::osnailyfacter::apache { - if ($::processorcount + 0) <= 2 { - $startservers = 2 - } else { -- $startservers = $::processorcount -+ $startservers = $::os_workers - } - - $maxrequestsperchild = 0 -diff --git a/deployment/puppet/osnailyfacter/manifests/globals/globals.pp b/deployment/puppet/osnailyfacter/manifests/globals/globals.pp -index f9e8ccb..6eee21e 100644 ---- a/deployment/puppet/osnailyfacter/manifests/globals/globals.pp -+++ b/deployment/puppet/osnailyfacter/manifests/globals/globals.pp -@@ -320,14 +320,14 @@ class osnailyfacter::globals::globals { - # The default value assumes there are 20 different types of workers limited by 100Mb each. - $workers_ratio = hiera('workers_ratio', 2000) - # Defines the maximum allowed number of workers for each service by RAM limits. Cannot exceed the value of 30. -- $workers_max = inline_template("<%= [(@memorysize_mb.to_i / @workers_ratio.to_i).floor + 1, 30].min %>") -+ $workers_max = inline_template("<%= [(@memorysize_mb.to_i / @workers_ratio.to_i).floor + 1, $::os_workers].min %>") - - $node_name_prefix_for_messaging = hiera('node_name_prefix_for_messaging', 'messaging-') - - # MySQL and SQLAlchemy backend configuration - $custom_mysql_setup_class = hiera('custom_mysql_setup_class', 'galera') -- $max_pool_size = hiera('max_pool_size', min($::processorcount * 5 + 0, 30 + 0)) -- $max_overflow = hiera('max_overflow', min($::processorcount * 5 + 0, 60 + 0)) -+ $max_pool_size = hiera('max_pool_size', min($::os_workers * 5 + 0, 30 + 0)) -+ $max_overflow = hiera('max_overflow', min($::os_workers * 5 + 0, 60 + 0)) - $max_retries = hiera('max_retries', '-1') - $idle_timeout = hiera('idle_timeout','3600') - $nova_db_password = $nova_hash['db_password'] diff --git a/build/f_repos/patch/fuel-main/0001-Patches-for-OPNFV.patch b/build/f_repos/patch/fuel-main/0001-Patches-for-OPNFV.patch deleted file mode 100644 index bf0f331db..000000000 --- a/build/f_repos/patch/fuel-main/0001-Patches-for-OPNFV.patch +++ /dev/null @@ -1,64 +0,0 @@ -From: "Stefan K. Berg" <sfb@consultron.com> -Date: Fri, 4 Nov 2016 16:06:42 +0100 -Subject: [PATCH] Patches for OPNFV - ---- - iso/bootstrap_admin_node.sh | 30 +++++++++++++++++++++++++++--- - 1 file changed, 27 insertions(+), 3 deletions(-) - -diff --git a/iso/bootstrap_admin_node.sh b/iso/bootstrap_admin_node.sh -index dbcac5f..472e418 100755 ---- a/iso/bootstrap_admin_node.sh -+++ b/iso/bootstrap_admin_node.sh -@@ -347,8 +347,23 @@ fuelmenu --save-only --iface=$ADMIN_INTERFACE || fail - set +x - echo "Done!" - -+### OPNFV addition BEGIN -+shopt -s nullglob -+for script in /opt/opnfv/bootstrap/pre.d/*.sh -+do -+ echo "Pre script: $script" >> /root/pre.log 2>&1 -+ $script >> /root/pre.log 2>&1 -+done -+shopt -u nullglob -+ -+# Enable sshd -+systemctl enable sshd -+systemctl start sshd -+ -+### OPNFV addition END -+ - if [[ "$showmenu" == "yes" || "$showmenu" == "YES" ]]; then -- fuelmenu || fail -+ fuelmenu - else - # Give user 15 seconds to enter fuelmenu or else continue - echo -@@ -386,8 +401,7 @@ if [ ! -f /etc/fuel_build_id ]; then - fi - - # Enable sshd --systemctl enable sshd --systemctl start sshd -+systemctl restart sshd - - # Enable iptables - systemctl enable iptables.service -@@ -546,6 +560,16 @@ systemctl start ntpd - - bash /etc/rc.local - -+### OPNFV addition BEGIN -+shopt -s nullglob -+for script in /opt/opnfv/bootstrap/post.d/*.sh -+do -+ echo "Post script: $script" >> /root/post.log 2>&1 -+ $script >> /root/post.log 2>&1 -+done -+shopt -u nullglob -+### OPNFV addition END -+ - if [ "`get_bootstrap_skip`" = "False" ]; then - build_ubuntu_bootstrap bs_status || true - else diff --git a/build/f_repos/patch/fuel-main/0002-xorriso-fails-to-add-files-with-path-longer-then-240.patch b/build/f_repos/patch/fuel-main/0002-xorriso-fails-to-add-files-with-path-longer-then-240.patch deleted file mode 100644 index 179cbdbd6..000000000 --- a/build/f_repos/patch/fuel-main/0002-xorriso-fails-to-add-files-with-path-longer-then-240.patch +++ /dev/null @@ -1,24 +0,0 @@ -From: Artem Silenkov <asilenkov@mirantis.com> -Date: Tue, 22 Sep 2015 21:54:44 +0300 -Subject: [PATCH] xorriso fails to add files with path longer then 240 - -joliet-long option added to xorriso command line - -Change-Id: I97016cf3e6a554f44d839740ba7d210337435cdc -Related-Bug: #1498619 ---- - iso/module.mk | 1 + - 1 file changed, 1 insertion(+) - -diff --git a/iso/module.mk b/iso/module.mk -index ab7542b..755e706 100644 ---- a/iso/module.mk -+++ b/iso/module.mk -@@ -191,6 +191,7 @@ $(ISO_PATH): $(BUILD_DIR)/iso/isoroot.done - -V $(ISO_VOLUME_ID) -p $(ISO_VOLUME_PREP) \ - -J -R \ - -graft-points \ -+ -joliet-long \ - -b isolinux/isolinux.bin -no-emul-boot -boot-load-size 4 -boot-info-table \ - -isohybrid-mbr /usr/lib/syslinux/isohdpfx.bin \ - -eltorito-alt-boot -e images/efiboot.img -no-emul-boot \ diff --git a/build/f_repos/patch/fuel-main/0003-repo-mirror-Allow-multi-arch-local-mirrors.patch b/build/f_repos/patch/fuel-main/0003-repo-mirror-Allow-multi-arch-local-mirrors.patch deleted file mode 100644 index fb9e86917..000000000 --- a/build/f_repos/patch/fuel-main/0003-repo-mirror-Allow-multi-arch-local-mirrors.patch +++ /dev/null @@ -1,59 +0,0 @@ -From: Stanislaw Kardach <stanislaw.kardach@cavium.com> -Date: Thu, 25 Feb 2016 13:31:19 +0100 -Subject: [PATCH] repo mirror: Allow multi-arch local mirrors - -This patch allows specifying multiple architectures via UBUNTU_ARCH in -form of a list of space separated architectures. The first architecture -in the list is considered primary and will be used for building all the -deb packages by fuel-main. Additional architectures are added to allow -targets of other architectures to use the mirror. -NOTE: this imposes a requirement that all packages built are arch -independent (which is true so far). - -Signed-off-by: Stanislaw Kardach <stanislaw.kardach@cavium.com> -Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com> ---- - config.mk | 2 +- - mirror/ubuntu/module.mk | 2 +- - sandbox.mk | 2 +- - 3 files changed, 3 insertions(+), 3 deletions(-) - -diff --git a/config.mk b/config.mk -index e0b988b..593f105 100644 ---- a/config.mk -+++ b/config.mk -@@ -48,7 +48,7 @@ UBUNTU_MINOR?=04 - UBUNTU_RELEASE_NUMBER:=$(UBUNTU_MAJOR).$(UBUNTU_MINOR) - UBUNTU_KERNEL_FLAVOR?=lts-xenial - UBUNTU_NETBOOT_FLAVOR?=netboot --UBUNTU_ARCH:=amd64 -+UBUNTU_ARCH?=amd64 - UBUNTU_IMAGE_RELEASE:=$(UBUNTU_MAJOR)$(UBUNTU_MINOR) - SEPARATE_IMAGES?=/boot,ext2 /,ext4 - -diff --git a/mirror/ubuntu/module.mk b/mirror/ubuntu/module.mk -index 5b4d762..b4fa99c 100644 ---- a/mirror/ubuntu/module.mk -+++ b/mirror/ubuntu/module.mk -@@ -33,7 +33,7 @@ set -ex; ./debmirror --progress --checksums --nocleanup \ - --root=$(MIRROR_MOS_UBUNTU_ROOT) \ - --dist=$(MIRROR_MOS_UBUNTU_SUITE)$1 \ - --section=$(subst $(space),$(comma),$(MIRROR_MOS_UBUNTU_SECTION)) \ -- --arch=$(UBUNTU_ARCH) \ -+ --arch=$(shell echo $(UBUNTU_ARCH) | tr ' ' ',') \ - $(LOCAL_MIRROR_UBUNTU)/ - endef - -diff --git a/sandbox.mk b/sandbox.mk -index 9d34dc9..e787f43 100644 ---- a/sandbox.mk -+++ b/sandbox.mk -@@ -199,7 +199,7 @@ touch $(SANDBOX_UBUNTU)/etc/init.d/.legacy-bootordering - mkdir -p $(SANDBOX_UBUNTU)/usr/sbin - cp -a $(BUILD_DIR)/policy-rc.d $(SANDBOX_UBUNTU)/usr/sbin - echo "Running debootstrap" --sudo debootstrap --no-check-gpg --include=ca-certificates --arch=$(UBUNTU_ARCH) $(MIRROR_UBUNTU_SUITE) $(SANDBOX_UBUNTU) $(MIRROR_UBUNTU_METHOD)://$(MIRROR_UBUNTU)$(MIRROR_UBUNTU_ROOT) -+sudo debootstrap --no-check-gpg --include=ca-certificates --arch=$(word 1,$(UBUNTU_ARCH)) $(MIRROR_UBUNTU_SUITE) $(SANDBOX_UBUNTU) $(MIRROR_UBUNTU_METHOD)://$(MIRROR_UBUNTU)$(MIRROR_UBUNTU_ROOT) - if [ -e $(SANDBOX_UBUNTU)/etc/resolv.conf ]; then sudo cp -a $(SANDBOX_UBUNTU)/etc/resolv.conf $(SANDBOX_UBUNTU)/etc/resolv.conf.orig; fi - sudo cp /etc/resolv.conf $(SANDBOX_UBUNTU)/etc/resolv.conf - if [ -e $(SANDBOX_UBUNTU)/etc/hosts ]; then sudo cp -a $(SANDBOX_UBUNTU)/etc/hosts $(SANDBOX_UBUNTU)/etc/hosts.orig; fi diff --git a/build/f_repos/patch/fuel-menu/0001-settings-OPNFV-bootstrap-Default-to-local-mirror.patch b/build/f_repos/patch/fuel-menu/0001-settings-OPNFV-bootstrap-Default-to-local-mirror.patch deleted file mode 100644 index 456eea7b9..000000000 --- a/build/f_repos/patch/fuel-menu/0001-settings-OPNFV-bootstrap-Default-to-local-mirror.patch +++ /dev/null @@ -1,66 +0,0 @@ -From: Alexandru Avadanii <Alexandru.Avadanii@enea.com> -Date: Sun, 9 Apr 2017 13:49:28 +0200 -Subject: [PATCH] settings: OPNFV bootstrap: Default to local mirror - -JIRA: https://jira.opnfv.org/browse/FUEL-266 - -Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com> ---- - fuelmenu/settings.yaml | 36 +++--------------------------------- - 1 file changed, 3 insertions(+), 33 deletions(-) - -diff --git a/fuelmenu/settings.yaml b/fuelmenu/settings.yaml -index e93309a..aaf07cd 100644 ---- a/fuelmenu/settings.yaml -+++ b/fuelmenu/settings.yaml -@@ -13,47 +13,17 @@ BOOTSTRAP: - http_proxy: "" - https_proxy: "" - repos: -- - name: "ubuntu" -- section: "main universe multiverse" -- uri: "http://archive.ubuntu.com/ubuntu" -+ - name: "ubuntu-local" -+ section: "main" -+ uri: "http://127.0.0.1:8080/mirrors/ubuntu" - priority: null - suite: "${codename}" - type: "deb" -- - name: "ubuntu-updates" -- section: "main universe multiverse" -- uri: "http://archive.ubuntu.com/ubuntu" -- priority: null -- suite: "${codename}-updates" -- type: "deb" -- - name: "ubuntu-security" -- section: "main universe multiverse" -- uri: "http://archive.ubuntu.com/ubuntu" -- priority: null -- suite: "${codename}-security" -- type: "deb" - - name: "mos" - section: "main restricted" - uri: "http://127.0.0.1:8080/ubuntu/x86_64" - priority: 1050 - suite: "mos${mos_version}" - type: "deb" -- - name: "mos-updates" -- section: "main restricted" -- uri: "http://mirror.fuel-infra.org/mos-repos/ubuntu/${mos_version}" -- priority: 1050 -- suite: "mos${mos_version}-updates" -- type: "deb" -- - name: "mos-security" -- section: "main restricted" -- uri: "http://mirror.fuel-infra.org/mos-repos/ubuntu/${mos_version}" -- priority: 1050 -- suite: "mos${mos_version}-security" -- type: "deb" -- - name: "mos-holdback" -- section: "main restricted" -- uri: "http://mirror.fuel-infra.org/mos-repos/ubuntu/${mos_version}" -- priority: 1100 -- suite: "mos${mos_version}-holdback" -- type: "deb" - PRODUCTION: docker - FEATURE_GROUPS: [] diff --git a/ci/deploy.sh b/ci/deploy.sh index d081c0332..3ac7ff1ea 100755 --- a/ci/deploy.sh +++ b/ci/deploy.sh @@ -269,8 +269,10 @@ pushd "${DEPLOY_DIR}" > /dev/null # scenario, etc. # Install required packages -[ -n "$(command -v apt-get)" ] && apt-get install -y mkisofs curl virtinst cpu-checker qemu-kvm -[ -n "$(command -v yum)" ] && yum install -y genisoimage curl virt-install qemu-kvm +[ -n "$(command -v apt-get)" ] && apt-get install -y \ + git make rsync mkisofs curl virtinst cpu-checker qemu-kvm +[ -n "$(command -v yum)" ] && yum install -y \ + git make rsync genisoimage curl virt-install qemu-kvm # Check scenario file existence if [[ ! -f ../config/${DEPLOY_SCENARIO}.yaml ]]; then diff --git a/build/f_repos/Makefile b/mcp/patches/Makefile index 5dd1c1e7f..6a176db8d 100644 --- a/build/f_repos/Makefile +++ b/mcp/patches/Makefile @@ -1,5 +1,5 @@ ############################################################################## -# Copyright (c) 2015,2016 Ericsson AB, Enea AB and others. +# Copyright (c) 2015,2016,2017 Ericsson AB, Enea AB and others. # stefan.k.berg@ericsson.com # jonas.bjurel@ericsson.com # Alexandru.Avadanii@enea.com @@ -9,13 +9,11 @@ # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## +include config.mk + SHELL = /bin/sh -REPOINFO = $(shell readlink -e ../repo_info.sh) -FREPODIR = $(shell pwd) FPATCHES = $(shell find ${F_PATCH_DIR} -name '*.patch') -include ../config.mk - # NOTE: Mechanism overview is presented in ./README.md. # Submodule consistent states: @@ -25,7 +23,7 @@ include ../config.mk # - patched (local patches are applied); # In order to keep things sort of separate, we should only pass up (to main -# Makefile) the fully-pactched repos, and gather any fingerprinting info here. +# Makefile) the fully-patched repos, and gather any fingerprinting info here. # Fuel@OPNFV relies on upstream git repos (one per component) in 1 of 2 ways: # - pinned down to tag objects (e.g. "9.0.1") @@ -54,8 +52,7 @@ sub: .cachefuelinfo else \ git submodule update --init 2>/dev/null; \ fi - @rm -f $@ - @git submodule -q foreach '${REPOINFO} . >> ${FREPODIR}/$@' + @touch $@ # Generate patches from submodules .PHONY: patches-export @@ -111,56 +108,10 @@ clean: @rm -f .cachepatched .PHONY: deepclean -deepclean: clean clean-cache - @git submodule deinit -f . +deepclean: clean + @git submodule deinit -f ${F_GIT_ROOT} @rm -f .cache* .PHONY: release release: sub - # Store artifact in cache straight away if caching is enabled - # (no .cacheid will be present unless this is a cached build) - test -f .cacheid && $(MAKE) -f Makefile put-cache || exit 0 - # NOTE: Patches are not included in cache $(MAKE) -f Makefile patches-import - -############################################################################## -# Cache operations - only used when building through ci/build.sh -############################################################################## - -# NOTE: Current method of collecting submodule refs requires submodules to be -# NOT initialized <OR> NO patches applied. -# NOTE: Querying `git submodule status` from parent will show the patched tree. -# Create a unique hash to be used for getting and putting cache, based on: -# - git submodule SHAs, collected with `git submodule status` -# - The contents of this Makefile -.cacheid: - @$(MAKE) clean - sha1sum Makefile > .cachedata - git submodule status | cut -c2-41 >> .cachedata - cat .cachedata | $(CACHETOOL) getid > .cacheid - -# Clean local data related to caching - called prior to ordinary build -.PHONY: clean-cache -clean-cache: clean - rm -f .cachedata .cacheid - -# Try to download cache - called prior to ordinary build -.PHONY: get-cache -get-cache: .cacheid - @if $(CACHETOOL) check $(shell cat .cacheid); then \ - $(MAKE) clean && \ - $(CACHETOOL) get $(shell cat .cacheid) | \ - tar xf - -C ${F_GIT_DIR}; \ - else \ - echo "No cache item found for $(shell cat .cacheid)" ;\ - exit 0;\ - fi - -# Store cache if not already stored - called after ordinary build -.PHONY: put-cache -put-cache: .cacheid - @if ! $(CACHETOOL) check $(shell cat .cacheid); then \ - $(MAKE) clean && \ - tar cf - -C ${F_GIT_DIR} modules | \ - $(CACHETOOL) put $(shell cat .cacheid); \ - fi diff --git a/build/f_repos/README.md b/mcp/patches/README.md index 0a52fe040..eab4d64e3 100644 --- a/build/f_repos/README.md +++ b/mcp/patches/README.md @@ -2,7 +2,7 @@ Fuel@OPNFV submodule fetching and patching ========================================== This directory holds submodule fetching/patching scripts, intended for -working with upstream Fuel components (fuel-library, ... , fuel-ui) in +working with upstream Fuel/MCP components (e.g.: reclass-system-salt-model) in developing/applying OPNFV patches (backports, custom fixes etc.). The scripts should be friendly to the following 2 use-cases: @@ -65,37 +65,19 @@ The standard development workflow should look as follows: $ make deepclean -Workflow (ISO build) --------------------- -Parent build scripts require this mechanism to do some fingerprinting, -so here is the intended flow for all artifacts to be generated right: - -1. (Optional) Cached submodules might be fetched from build cache. - -2. Submodules are updated - We also dump each submodule's git info using repo_info.sh, since - we want to collect git refs before patching (i.e. upstream refs). - -3. Make target `release` is built - This will make sure the modules are in a clean state, put them in cache, - then apply the patches. - -4. fuel-main's `${FUEL_MAIN_TAG}-opnfv-root` tag is used to determine VERSION info - It will accommodate both bound tags and remote tracking references. - Sub-project maintenance ----------------------- 1. Adding a new submodule If you need to add another subproject, you can do it with `git submodule`. - Make sure that you specify branch (with `-b`), short name (with `--name`) - and point it to `upstream/*` directory, i.e.: + Make sure that you specify branch (with `-b`), short name (with `--name`): - $ git submodule -b stable/mitaka add --name fuel-web \ - https://github.com/openstack/fuel-web.git upstream/fuel-web + $ git submodule -b master add --name reclass-system-salt-model \ + https://github.com/Mirantis/reclass-system-salt-model \ + relative/path/to/submodule 2. Working with remote tracking for upgrading Fuel components Enable remote tracking as described above, which at `make sub` will update - ALL submodules (fuel-main, fuel-library, ...) to remote branch (set in + ALL submodules (e.g. reclass-system-salt-model) to remote branch (set in .gitmodules) HEAD. * If upstream has NOT already tagged a new version, we can still work on @@ -118,4 +100,4 @@ Sub-project maintenance - if submodule tags have been updated (relevant when remote tracking is disabled, i.e. we have a stable upstream baseline), add submodules: - $ make deepclean sub && git add -f sub/* + $ make deepclean sub && git add -f relative/path/to/submodule diff --git a/mcp/patches/config.mk b/mcp/patches/config.mk new file mode 100644 index 000000000..260cbf829 --- /dev/null +++ b/mcp/patches/config.mk @@ -0,0 +1,23 @@ +############################################################################## +# Copyright (c) 2015,2016,2017 Ericsson AB, Enea AB and others. +# stefan.k.berg@ericsson.com +# jonas.bjurel@ericsson.com +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +############################################################################## +# Components pinning / remote tracking +############################################################################## + +# git submodule & patch locations for Fuel components +F_GIT_ROOT := $(shell git rev-parse --show-toplevel) +F_GIT_DIR := $(shell git rev-parse --git-dir) +F_PATCH_DIR := $(shell pwd) +F_OPNFV_TAG := master-opnfv + +# for the patches applying purposes (empty git config in docker build container) +export GIT_COMMITTER_NAME?=Fuel OPNFV +export GIT_COMMITTER_EMAIL?=fuel@opnfv.org diff --git a/mcp/patches/patches.list b/mcp/patches/patches.list index 0a84657d2..0542433c7 100644 --- a/mcp/patches/patches.list +++ b/mcp/patches/patches.list @@ -1,2 +1 @@ /usr/share/salt-formulas/env: 0002-opendaylight-formula-neutron.patch -/srv/salt/reclass/classes/system: 0001-opendaylight-reclass-system.patch diff --git a/mcp/patches/0001-opendaylight-reclass-system.patch b/mcp/patches/reclass-system-salt-model/0001-Bring-in-opendaylight-support.patch index f8c986be6..15f23db34 100644 --- a/mcp/patches/0001-opendaylight-reclass-system.patch +++ b/mcp/patches/reclass-system-salt-model/0001-Bring-in-opendaylight-support.patch @@ -3,6 +3,18 @@ Date: Thu, 29 Jun 2017 12:22:42 +0400 Subject: [PATCH] Bring in opendaylight support Change-Id: I3efec9a8b586a6c75b1c1635ad2a7024d73d9ad2 +--- + neutron/control/opendaylight/cluster.yml | 19 +++++++++++++++++++ + neutron/control/opendaylight/single.yml | 16 ++++++++++++++++ + neutron/gateway/opendaylight/single.yml | 8 ++++++++ + opendaylight/server/single.yml | 2 ++ + .../storage/system/opendaylight_control_single.yml | 15 +++++++++++++++ + 5 files changed, 60 insertions(+) + create mode 100644 neutron/control/opendaylight/cluster.yml + create mode 100644 neutron/control/opendaylight/single.yml + create mode 100644 neutron/gateway/opendaylight/single.yml + create mode 100644 opendaylight/server/single.yml + create mode 100644 reclass/storage/system/opendaylight_control_single.yml diff --git a/neutron/control/opendaylight/cluster.yml b/neutron/control/opendaylight/cluster.yml new file mode 100644 diff --git a/mcp/scripts/salt.sh b/mcp/scripts/salt.sh index 4c6cbf782..3d0fb9117 100755 --- a/mcp/scripts/salt.sh +++ b/mcp/scripts/salt.sh @@ -1,34 +1,58 @@ #!/bin/bash +############################################################################## +# Copyright (c) 2017 Mirantis Inc., Enea AB and others. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## # # Deploy Salt Master # +F_GIT_ROOT=$(git rev-parse --show-toplevel) +SALT_MASTER_USER=${SALT_MASTER_USER:-ubuntu} +SSH_SALT="${SALT_MASTER_USER}@${SALT_MASTER}" +OPNFV_TMP_DIR="/home/${SALT_MASTER_USER}/fuel" +OPNFV_FUEL_DIR="/root/fuel" + +# patch reclass-system-salt-model locally before copying it over +make -C "${F_GIT_ROOT}/mcp/patches" patches-import + +# push to cfg01 current git repo first (including submodules), at ~ubuntu/fuel +# later we move it to ~root/fuel and delete the temporary clone +rsync -Erl --delete -e "ssh ${SSH_OPTS}" \ + --exclude-from="${F_GIT_ROOT}/.gitignore" \ + "${F_GIT_ROOT}/" "${SSH_SALT}:$(basename "${OPNFV_TMP_DIR}")/" + # ssh to cfg01 # shellcheck disable=SC2086,2087 -ssh ${SSH_OPTS} "ubuntu@${SALT_MASTER}" bash -s << SALT_INSTALL_END +ssh ${SSH_OPTS} "${SSH_SALT}" bash -s << SALT_INSTALL_END sudo -i echo -n 'Checking out cloud-init has finished running ...' while [ ! -f /var/lib/cloud/instance/boot-finished ]; do echo -n '.'; sleep 1; done echo ' done' - apt-get install -y git curl subversion + DEBIAN_FRONTEND=noninteractive apt-get install -y git curl subversion - svn export --force https://github.com/salt-formulas/salt-formulas/trunk/deploy/scripts /srv/salt/scripts - git clone --depth=1 --recurse-submodules https://git.opnfv.org/fuel - ln -s /root/fuel/mcp/reclass /srv/salt/reclass + svn export --force \ + https://github.com/salt-formulas/salt-formulas/trunk/deploy/scripts /srv/salt/scripts + mv ${OPNFV_TMP_DIR} ${OPNFV_FUEL_DIR} && chown -R root.root ${OPNFV_FUEL_DIR} + ln -s ${OPNFV_FUEL_DIR}/mcp/reclass /srv/salt/reclass mkdir -p /usr/share/salt-formulas/reclass - cp -r /root/fuel/mcp/metadata/service /usr/share/salt-formulas/reclass - cd /srv/salt/reclass/classes/service && ln -s /usr/share/salt-formulas/reclass/service/opendaylight - cd /root/fuel/mcp/patches && ./patch.sh patches.list reclass + cp -r ${OPNFV_FUEL_DIR}/mcp/metadata/service /usr/share/salt-formulas/reclass + cd /srv/salt/reclass/classes/service && \ + ln -s /usr/share/salt-formulas/reclass/service/opendaylight + cd ${OPNFV_FUEL_DIR}/mcp/patches && ./patch.sh patches.list reclass cd /srv/salt/scripts MASTER_HOSTNAME=cfg01.${CLUSTER_DOMAIN} DISTRIB_REVISION=nightly ./salt-master-init.sh salt-key -Ay - cp -r /root/fuel/mcp/salt-formulas/* /usr/share/salt-formulas/env - cd /root/fuel/mcp/patches && ./patch.sh patches.list formulas + cp -r ${OPNFV_FUEL_DIR}/mcp/salt-formulas/* /usr/share/salt-formulas/env + cd ${OPNFV_FUEL_DIR}/mcp/patches && ./patch.sh patches.list formulas salt-call state.apply salt salt '*' state.apply salt | fgrep -q 'No response' && salt '*' state.apply salt |