diff options
Diffstat (limited to 'patches/opnfv-fuel/upstream-backports')
3 files changed, 0 insertions, 2421 deletions
diff --git a/patches/opnfv-fuel/upstream-backports/0001-deploy.sh-Remove-check-for-root-rm-umask-0000.patch b/patches/opnfv-fuel/upstream-backports/0001-deploy.sh-Remove-check-for-root-rm-umask-0000.patch deleted file mode 100644 index dcb47a7d..00000000 --- a/patches/opnfv-fuel/upstream-backports/0001-deploy.sh-Remove-check-for-root-rm-umask-0000.patch +++ /dev/null @@ -1,102 +0,0 @@ -:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: -: Copyright (c) 2017 Enea AB and others. -: -: All rights reserved. This program and the accompanying materials -: are made available under the terms of the Apache License, Version 2.0 -: which accompanies this distribution, and is available at -: http://www.apache.org/licenses/LICENSE-2.0 -:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: -From: Alexandru Avadanii <Alexandru.Avadanii@enea.com> -Date: Fri, 6 May 2016 03:07:40 +0200 -Subject: [PATCH] deploy.sh: Remove check for root (rm umask 0000) - -Patch(es) by Josep, signed by Alex only for upstreaming. - -[Alexandru Avadanii] -Squashed two atomic commits into one change. - -Signed-off-by: Josep Puigdemont <josep.puigdemont@enea.com> -Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com> ---- - ci/deploy.sh | 11 ----------- - deploy/deploy-config.py | 1 - - deploy/deploy.py | 2 -- - deploy/environments/virtual_fuel.py | 2 -- - 4 files changed, 16 deletions(-) - -diff --git a/ci/deploy.sh b/ci/deploy.sh -index 8411714..081806c 100755 ---- a/ci/deploy.sh -+++ b/ci/deploy.sh -@@ -84,9 +84,6 @@ Input parameters to the build script is: - -i .iso image to be deployed (needs to be provided in a URI - style, it can be a local resource: file:// or a remote resource http(s)://) - --NOTE: Root priviledges are needed for this script to run -- -- - Examples: - sudo `basename $0` -b file:///home/jenkins/lab-config -l lf -p pod1 -s ha_odl-l3_heat_ceilometer -i file:///home/jenkins/myiso.iso - EOF -@@ -212,11 +209,6 @@ do - esac - done - --if [[ $EUID -ne 0 ]]; then -- echo "This script must be run as root" 1>&2 -- exit 1 --fi -- - if [ -z $BASE_CONFIG_URI ] || [ -z $TARGET_LAB ] || \ - [ -z $TARGET_POD ] || [ -z $DEPLOY_SCENARIO ] || \ - [ -z $ISO ]; then -@@ -231,9 +223,6 @@ fi - # Enable the automatic exit trap - trap do_exit SIGINT SIGTERM EXIT - --# Set no restrictive umask so that Jenkins can removeeee any residuals --umask 0000 -- - clean - - pushd ${DEPLOY_DIR} > /dev/null -diff --git a/deploy/deploy.py b/deploy/deploy.py -index fe213e5..08702d2 100755 ---- a/deploy/deploy.py -+++ b/deploy/deploy.py -@@ -34,7 +34,6 @@ from common import ( - check_dir_exists, - create_dir_if_not_exists, - delete, -- check_if_root, - ArgParser, - ) - -@@ -252,7 +251,6 @@ class AutoDeploy(object): - return 0 - - def run(self): -- check_if_root() - if self.cleanup_only: - self.cleanup_execution_environment() - else: -diff --git a/deploy/environments/virtual_fuel.py b/deploy/environments/virtual_fuel.py -index b1a76e4..4ff68f6 100644 ---- a/deploy/environments/virtual_fuel.py -+++ b/deploy/environments/virtual_fuel.py -@@ -18,7 +18,6 @@ import time - from common import ( - exec_cmd, - check_file_exists, -- check_if_root, - delete, - log, - ) -@@ -135,7 +134,6 @@ class VirtualFuel(ExecutionEnvironment): - vm_definition_overwrite) - - def setup_environment(self): -- check_if_root() - self.cleanup_environment() - self.create_vm() - diff --git a/patches/opnfv-fuel/upstream-backports/0004-Fuel-rework-for-multiple-libvirt.patch b/patches/opnfv-fuel/upstream-backports/0004-Fuel-rework-for-multiple-libvirt.patch deleted file mode 100644 index e9c8e2ce..00000000 --- a/patches/opnfv-fuel/upstream-backports/0004-Fuel-rework-for-multiple-libvirt.patch +++ /dev/null @@ -1,1533 +0,0 @@ -:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: -: Copyright (c) 2017 Enea AB and others. -: -: All rights reserved. This program and the accompanying materials -: are made available under the terms of the Apache License, Version 2.0 -: which accompanies this distribution, and is available at -: http://www.apache.org/licenses/LICENSE-2.0 -:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: -From: cristinapauna <cristina.pauna@enea.com> -Date: Fri, 25 Nov 2016 16:38:46 +0200 -Subject: [PATCH] Fuel rework for multiple libvirt - -The current fuel arhitecture assumes that all the VMs -(Jenkins slave, Fuel master and Target nodes) are all on -the same server. There is code that allows the Fuel master -to be on a different machine, but nothing for the Target nodes. -In Enea-Arm lab we have a different server for each. - -This commit also adds specific templates for a virtual pod in -the Enea-Arm lab. - -Therefore, the following changes are done: -- The default libvirt URI is the machine on which the Target nodes are -- The parameter libvirtUri was added in the dha.yaml file for fuel master -- All virsh calls have "-c" option to explicitly specify the uri; this parameter - is set only when libvirtUri is explicitly set in the yaml file -- The scenarios dha override now point to the arm template files -- The network templates are not needed for ARM, as the network infra is provided - by the server on which the Target node VMs reside; the folder is added here - just for consistency -- The target nodes are created with virsh rather than quemu when the nodes are - not on the same machine as Jenkins slave (similar logic to what is done for - remote Fuel master) - -Initially only ha scenarios were successfully deployed on virtual pods -The default configuration is for ha scenarios and those have 5 target nodes -while on the noha scenarios there are 4. The noha scenarios failed because -it was trying to parse one extra node. - -The dea and dha files need to be modified for each noha scenario so that the -nodes are properly configured. - -Signed-off-by: cristinapauna <cristina.pauna@enea.com> ---- - deploy/dha_adapters/hardware_adapter.py | 5 ++ - deploy/dha_adapters/libvirt_adapter.py | 42 ++++----- - deploy/environments/execution_environment.py | 19 +++-- - deploy/environments/libvirt_environment.py | 59 ++++++++++++- - deploy/environments/virtual_fuel.py | 8 +- - deploy/install_fuel_master.py | 2 +- - deploy/scenario/ha_heat_ceilometer_scenario.yaml | 13 +-- - .../ha_nfv-kvm_heat_ceilometer_scenario.yaml | 13 +-- - .../ha_nfv-ovs_heat_ceilometer_scenario.yaml | 13 +-- - deploy/scenario/ha_odl-bgpvpn_scenario.yaml | 13 +-- - .../ha_odl-l2_heat_ceilometer_scenario.yaml | 13 +-- - .../ha_odl-l2_sfc_heat_ceilometer_scenario.yaml | 13 +-- - .../ha_odl-l3_heat_ceilometer_scenario.yaml | 13 +-- - ...ha_onos_nofeature_heat_ceilometer_scenario.yaml | 13 +-- - .../ha_onos_sfc_heat_ceilometer_scenario.yaml | 13 +-- - .../scenario/ha_vlan_heat_ceilometer_scenario.yaml | 13 +-- - .../scenario/no-ha_heat_ceilometer_scenario.yaml | 25 ++++++ - .../no-ha_nfv-kvm_heat_ceilometer_scenario.yaml | 25 ++++++ - ...a_nfv-kvm_nfv-ovs_heat_ceilometer_scenario.yaml | 25 ++++++ - .../no-ha_nfv-ovs_heat_ceilometer_scenario.yaml | 26 +++++- - deploy/scenario/no-ha_odl-bgpvpn_scenario.yaml | 25 ++++++ - .../no-ha_odl-l2_heat_ceilometer_scenario.yaml | 25 ++++++ - .../no-ha_odl-l2_sfc_heat_ceilometer_scenario.yaml | 25 ++++++ - .../no-ha_odl-l3_heat_ceilometer_scenario.yaml | 25 ++++++ - ...ha_onos_nofeature_heat_ceilometer_scenario.yaml | 26 ++++++ - .../no-ha_onos_sfc_heat_ceilometer_scenario.yaml | 25 ++++++ - .../no-ha_vlan_heat_ceilometer_scenario.yaml | 25 ++++++ - .../arm/virtual_environment/networks/.gitkeep | 0 - .../arm/virtual_environment/vms/compute.xml | 57 +++++++++++++ - .../arm/virtual_environment/vms/controller.xml | 57 +++++++++++++ - .../templates/arm/virtual_environment/vms/fuel.xml | 99 ++++++++++++++++++++++ - 31 files changed, 658 insertions(+), 97 deletions(-) - create mode 100644 deploy/templates/arm/virtual_environment/networks/.gitkeep - create mode 100644 deploy/templates/arm/virtual_environment/vms/compute.xml - create mode 100644 deploy/templates/arm/virtual_environment/vms/controller.xml - create mode 100644 deploy/templates/arm/virtual_environment/vms/fuel.xml - -diff --git a/deploy/dha_adapters/hardware_adapter.py b/deploy/dha_adapters/hardware_adapter.py -index aa59581..cf3d77a 100644 ---- a/deploy/dha_adapters/hardware_adapter.py -+++ b/deploy/dha_adapters/hardware_adapter.py -@@ -63,3 +63,8 @@ class HardwareAdapter(object): - vm_definition = self.dha_struct.get('define_vms') - if vm_definition: - return vm_definition.get(role) -+ def get_node_uri(self, node_id): -+ vm_libvirt_uri = self.get_node_property(node_id, 'libvirtUri') -+ if vm_libvirt_uri: -+ return '-c %s' % vm_libvirt_uri -+ return '' -diff --git a/deploy/dha_adapters/libvirt_adapter.py b/deploy/dha_adapters/libvirt_adapter.py -index a00f091..34535d1 100644 ---- a/deploy/dha_adapters/libvirt_adapter.py -+++ b/deploy/dha_adapters/libvirt_adapter.py -@@ -43,21 +43,21 @@ class LibvirtAdapter(HardwareAdapter): - def node_power_off(self, node_id): - vm_name = self.get_node_property(node_id, 'libvirtName') - log('Power OFF Node %s' % vm_name) -- state = exec_cmd('virsh domstate %s' % vm_name) -+ state = exec_cmd('virsh %s domstate %s' % (self.get_node_uri(node_id), vm_name)) - if state == 'running': -- exec_cmd('virsh destroy %s' % vm_name, False) -+ exec_cmd('virsh %s destroy %s' % (self.get_node_uri(node_id), vm_name), False) - - def node_power_on(self, node_id): - vm_name = self.get_node_property(node_id, 'libvirtName') - log('Power ON Node %s' % vm_name) -- state = exec_cmd('virsh domstate %s' % vm_name) -+ state = exec_cmd('virsh %s domstate %s' % (self.get_node_uri(node_id), vm_name)) - if state == 'shut off': -- exec_cmd('virsh start %s' % vm_name) -+ exec_cmd('virsh %s start %s' % (self.get_node_uri(node_id), vm_name)) - - def node_reset(self, node_id): - vm_name = self.get_node_property(node_id, 'libvirtName') - log('Reset Node %s' % vm_name) -- exec_cmd('virsh reset %s' % vm_name) -+ exec_cmd('virsh %s reset %s' % (self.get_node_uri(node_id), vm_name)) - - def translate(self, boot_order_list): - translated = [] -@@ -73,7 +73,7 @@ class LibvirtAdapter(HardwareAdapter): - vm_name = self.get_node_property(node_id, 'libvirtName') - temp_dir = tempfile.mkdtemp() - log('Set boot order %s on Node %s' % (boot_order_list, vm_name)) -- resp = exec_cmd('virsh dumpxml %s' % vm_name) -+ resp = exec_cmd('virsh %s dumpxml %s' % (self.get_node_uri(node_id), vm_name)) - xml_dump = etree.fromstring(resp, self.parser) - os = xml_dump.xpath('/domain/os') - for o in os: -@@ -92,12 +92,12 @@ class LibvirtAdapter(HardwareAdapter): - xml_file = temp_dir + '/%s.xml' % vm_name - with open(xml_file, 'w') as f: - tree.write(f, pretty_print=True, xml_declaration=True) -- exec_cmd('virsh define %s' % xml_file) -+ exec_cmd('virsh %s define %s' % (self.get_node_uri(node_id), xml_file)) - delete(temp_dir) - - def node_zero_mbr(self, node_id): - vm_name = self.get_node_property(node_id, 'libvirtName') -- resp = exec_cmd('virsh dumpxml %s' % vm_name) -+ resp = exec_cmd('virsh %s dumpxml %s' % (self.get_node_uri(node_id), vm_name)) - xml_dump = etree.fromstring(resp) - disks = xml_dump.xpath('/domain/devices/disk') - for disk in disks: -@@ -114,20 +114,20 @@ class LibvirtAdapter(HardwareAdapter): - - def node_eject_iso(self, node_id): - vm_name = self.get_node_property(node_id, 'libvirtName') -- device = self.get_name_of_device(vm_name, 'cdrom') -- exec_cmd('virsh change-media %s --eject %s --config --live' -- % (vm_name, device), False) -+ device = self.get_name_of_device(vm_name, 'cdrom', node_id) -+ exec_cmd('virsh %s change-media %s --eject %s --config --live' -+ % (self.get_node_uri(node_id), vm_name, device), False) - - def node_insert_iso(self, node_id, iso_file): - vm_name = self.get_node_property(node_id, 'libvirtName') -- device = self.get_name_of_device(vm_name, 'cdrom') -- exec_cmd('virsh change-media %s --insert %s %s' -- % (vm_name, device, iso_file)) -+ device = self.get_name_of_device(vm_name, 'cdrom', node_id) -+ exec_cmd('virsh %s change-media %s --insert %s %s' -+ % (self.get_node_uri(node_id), vm_name, device, iso_file)) - - def get_node_pxe_mac(self, node_id): - mac_list = [] - vm_name = self.get_node_property(node_id, 'libvirtName') -- resp = exec_cmd('virsh dumpxml %s' % vm_name) -+ resp = exec_cmd('virsh %s dumpxml %s' % (self.get_node_uri(node_id), vm_name)) - xml_dump = etree.fromstring(resp) - interfaces = xml_dump.xpath('/domain/devices/interface') - for interface in interfaces: -@@ -136,8 +136,8 @@ class LibvirtAdapter(HardwareAdapter): - mac_list.append(mac.get('address').lower()) - return mac_list - -- def get_name_of_device(self, vm_name, device_type): -- resp = exec_cmd('virsh dumpxml %s' % vm_name) -+ def get_name_of_device(self, vm_name, device_type, node_id): -+ resp = exec_cmd('virsh %s dumpxml %s' % (self.get_node_uri(node_id), vm_name)) - xml_dump = etree.fromstring(resp) - disks = xml_dump.xpath('/domain/devices/disk') - for disk in disks: -@@ -151,7 +151,7 @@ class LibvirtAdapter(HardwareAdapter): - def get_virt_net_conf_dir(self): - return self.dha_struct['virtNetConfDir'] - -- def upload_iso(self, iso_file): -+ def upload_iso(self, node_id, iso_file): - size = os.path.getsize(iso_file) - vol_name = os.path.basename(iso_file) - vol_xml = VOL_XML_TEMPLATE.format(name=vol_name, unit='bytes', -@@ -162,10 +162,10 @@ class LibvirtAdapter(HardwareAdapter): - - log(vol_xml) - pool = DEFAULT_POOL # FIXME -- exec_cmd('virsh vol-create --pool %s %s' % (pool, fname)) -- vol_path = exec_cmd('virsh vol-path --pool %s %s' % (pool, vol_name)) -+ exec_cmd('virsh %s vol-create --pool %s %s' % (self.get_node_uri(node_id), pool, fname)) -+ vol_path = exec_cmd('virsh %s vol-path --pool %s %s' % (self.get_node_uri(node_id), pool, vol_name)) - -- exec_cmd('virsh vol-upload %s %s' % (vol_path, iso_file), -+ exec_cmd('virsh %s vol-upload %s %s' % (self.get_node_uri(node_id), vol_path, iso_file), - attempts=5, delay=10, verbose=True) - - delete(fname) -diff --git a/deploy/environments/execution_environment.py b/deploy/environments/execution_environment.py -index 7a0b474..db25229 100644 ---- a/deploy/environments/execution_environment.py -+++ b/deploy/environments/execution_environment.py -@@ -29,12 +29,12 @@ class ExecutionEnvironment(object): - - def delete_vm(self, node_id): - vm_name = self.dha.get_node_property(node_id, 'libvirtName') -- r, c = exec_cmd('virsh dumpxml %s' % vm_name, False) -+ r, c = exec_cmd('virsh %s dumpxml %s' % (self.dha.get_node_uri(node_id), vm_name), False) - if c: - return -- self.undefine_vm_delete_disk(r, vm_name) -+ self.undefine_vm_delete_disk(r, vm_name, node_id) - -- def undefine_vm_delete_disk(self, printout, vm_name): -+ def undefine_vm_delete_disk(self, printout, vm_name, node_id): - disk_files = [] - xml_dump = etree.fromstring(printout, self.parser) - disks = xml_dump.xpath('/domain/devices/disk') -@@ -45,8 +45,13 @@ class ExecutionEnvironment(object): - if source_file: - disk_files.append(source_file) - log('Deleting VM %s with disks %s' % (vm_name, disk_files)) -- exec_cmd('virsh destroy %s' % vm_name, False) -- exec_cmd('virsh undefine --managed-save --remove-all-storage %s' % vm_name, False) -+ exec_cmd('virsh %s destroy %s' % (self.dha.get_node_uri(node_id), vm_name), False) -+ r, c = exec_cmd('virsh %s undefine --managed-save --remove-all-storage %s' % -+ (self.dha.get_node_uri(node_id), vm_name), False) -+ if c: -+ exec_cmd('virsh %s undefine --managed-save --remove-all-storage --nvram %s' % -+ (self.dha.get_node_uri(node_id), vm_name), False) -+ - for file in disk_files: - delete(file) - -@@ -75,7 +80,7 @@ class ExecutionEnvironment(object): - self.overwrite_xml(xml_element, value) - - def define_vm(self, vm_name, temp_vm_file, disk_path, -- vm_definition_overwrite): -+ vm_definition_overwrite, node_id): - log('Creating VM %s with disks %s' % (vm_name, disk_path)) - with open(temp_vm_file) as f: - vm_xml = etree.parse(f) -@@ -99,4 +104,4 @@ class ExecutionEnvironment(object): - disk.append(source) - with open(temp_vm_file, 'w') as f: - vm_xml.write(f, pretty_print=True, xml_declaration=True) -- exec_cmd('virsh define %s' % temp_vm_file) -+ exec_cmd('virsh %s define %s' % (self.dha.get_node_uri(node_id), temp_vm_file)) -diff --git a/deploy/environments/libvirt_environment.py b/deploy/environments/libvirt_environment.py -index 07a47fd..08d669b 100644 ---- a/deploy/environments/libvirt_environment.py -+++ b/deploy/environments/libvirt_environment.py -@@ -12,6 +12,9 @@ from lxml import etree - import glob - from execution_environment import ExecutionEnvironment - import tempfile -+import os -+import re -+import time - - from common import ( - exec_cmd, -@@ -21,6 +24,25 @@ from common import ( - delete, - ) - -+VOL_XML_TEMPLATE = '''<volume type='file'> -+ <name>{name}</name> -+ <capacity unit='{unit}'>{size!s}</capacity> -+ <target> -+ <format type='{format_type}'/> -+ </target> -+</volume>''' -+ -+DEFAULT_POOL = 'jenkins' -+ -+def get_size_and_unit(s): -+ p = re.compile('^(\d+)\s*(\D+)') -+ m = p.match(s) -+ if m == None: -+ return None, None -+ size = m.groups()[0] -+ unit = m.groups()[1] -+ return size, unit -+ - - class LibvirtEnvironment(ExecutionEnvironment): - -@@ -33,10 +55,39 @@ class LibvirtEnvironment(ExecutionEnvironment): - self.node_ids = self.dha.get_all_node_ids() - self.net_names = self.collect_net_names() - -- def create_storage(self, node_id, disk_path, disk_sizes): -+ def create_volume(self, pool, name, su, node_id, temp_dir, img_type='raw'): -+ log('Creating image using Libvirt volumes in pool %s, name: %s, uri: %s' % -+ (pool, name, self.dha.get_node_uri(node_id))) -+ size, unit = get_size_and_unit(su) -+ if size == None: -+ err('Could not determine size and unit of %s' % s) -+ -+ vol_xml = VOL_XML_TEMPLATE.format(name=name, unit=unit, size=size, -+ format_type=img_type) -+ fname = os.path.join(temp_dir, '%s_vol.xml' % name) -+ with file(fname, 'w') as f: -+ f.write(vol_xml) -+ -+ exec_cmd('virsh %s vol-create --pool %s %s' % -+ (self.dha.get_node_uri(node_id), pool, fname)) -+ vol_path = exec_cmd('virsh %s vol-path --pool %s %s' % -+ (self.dha.get_node_uri(node_id), pool, name)) -+ -+ delete(fname) -+ return vol_path -+ -+ -+ def create_storage(self, node_id, disk_path, disk_sizes, temp_dir): - role = self.dea.get_node_main_role(node_id, self.fuel_node_id) - disk_size = disk_sizes[role] -- exec_cmd('qemu-img create -f raw %s %s' % (disk_path, disk_size)) -+ if os.environ.get('LIBVIRT_DEFAULT_URI') == None: -+ exec_cmd('qemu-img create -f raw %s %s' % (disk_path, disk_size)) -+ else: -+ pool = DEFAULT_POOL # FIXME -+ name = os.path.basename(disk_path) -+ disk_path = self.create_volume(pool, name, disk_size, node_id, temp_dir) -+ return disk_path -+ - - def create_vms(self): - temp_dir = tempfile.mkdtemp() -@@ -48,13 +99,13 @@ class LibvirtEnvironment(ExecutionEnvironment): - node_id, 'libvirtTemplate')) - check_file_exists(vm_template) - disk_path = '%s/%s.raw' % (self.storage_dir, vm_name) -- self.create_storage(node_id, disk_path, disk_sizes) -+ disk_path = self.create_storage(node_id, disk_path, disk_sizes, temp_dir) - temp_vm_file = '%s/%s' % (temp_dir, vm_name) - exec_cmd('cp %s %s' % (vm_template, temp_vm_file)) - vm_definition_overwrite = self.dha.get_vm_definition( - self.dea.get_node_main_role(node_id, self.fuel_node_id)) - self.define_vm(vm_name, temp_vm_file, disk_path, -- vm_definition_overwrite) -+ vm_definition_overwrite, node_id) - delete(temp_dir) - - def start_vms(self): -diff --git a/deploy/environments/virtual_fuel.py b/deploy/environments/virtual_fuel.py -index fcfa532..5f50f40 100644 ---- a/deploy/environments/virtual_fuel.py -+++ b/deploy/environments/virtual_fuel.py -@@ -99,8 +99,10 @@ class VirtualFuel(ExecutionEnvironment): - with file(fname, 'w') as f: - f.write(vol_xml) - -- exec_cmd('virsh vol-create --pool %s %s' % (pool, fname)) -- vol_path = exec_cmd('virsh vol-path --pool %s %s' % (pool, name)) -+ exec_cmd('virsh %s vol-create --pool %s %s' % -+ (self.dha.get_node_uri(self.fuel_node_id), pool, fname)) -+ vol_path = exec_cmd('virsh %s vol-path --pool %s %s' % -+ (self.dha.get_node_uri(self.fuel_node_id), pool, name)) - - delete(fname) - -@@ -131,7 +133,7 @@ class VirtualFuel(ExecutionEnvironment): - vm_definition_overwrite = self.dha.get_vm_definition('fuel') - - self.define_vm(self.vm_name, self.temp_vm_file, disk_path, -- vm_definition_overwrite) -+ vm_definition_overwrite, self.fuel_node_id) - - def setup_environment(self): - self.cleanup_environment() -diff --git a/deploy/install_fuel_master.py b/deploy/install_fuel_master.py -index 2d89c8e..2615818 100644 ---- a/deploy/install_fuel_master.py -+++ b/deploy/install_fuel_master.py -@@ -59,7 +59,7 @@ class InstallFuelMaster(object): - - if os.environ.get('LIBVIRT_DEFAULT_URI'): - log('Upload ISO to pool') -- self.iso_file = self.dha.upload_iso(self.iso_file) -+ self.iso_file = self.dha.upload_iso(self.fuel_node_id, self.iso_file) - else: - log('Zero the MBR') - self.dha.node_zero_mbr(self.fuel_node_id) -diff --git a/deploy/scenario/ha_heat_ceilometer_scenario.yaml b/deploy/scenario/ha_heat_ceilometer_scenario.yaml -index 4051c47..78fa80d 100644 ---- a/deploy/scenario/ha_heat_ceilometer_scenario.yaml -+++ b/deploy/scenario/ha_heat_ceilometer_scenario.yaml -@@ -58,22 +58,23 @@ dha-override-config: - nodes: - - id: 1 - libvirtName: controller1 -- libvirtTemplate: templates/virtual_environment/vms/controller.xml -+ libvirtTemplate: templates/arm/virtual_environment/vms/controller.xml - - id: 2 - libvirtName: controller2 -- libvirtTemplate: templates/virtual_environment/vms/controller.xml -+ libvirtTemplate: templates/arm/virtual_environment/vms/controller.xml - - id: 3 - libvirtName: controller3 -- libvirtTemplate: templates/virtual_environment/vms/controller.xml -+ libvirtTemplate: templates/arm/virtual_environment/vms/controller.xml - - id: 4 - libvirtName: compute1 -- libvirtTemplate: templates/virtual_environment/vms/compute.xml -+ libvirtTemplate: templates/arm/virtual_environment/vms/compute.xml - - id: 5 - libvirtName: compute2 -- libvirtTemplate: templates/virtual_environment/vms/compute.xml -+ libvirtTemplate: templates/arm/virtual_environment/vms/compute.xml - - id: 6 - libvirtName: fuel-master -- libvirtTemplate: templates/virtual_environment/vms/fuel.xml -+ libvirtTemplate: templates/arm/virtual_environment/vms/fuel.xml -+ libvirtUri: qemu+ssh://jenkins@10.0.2.5/system - isFuel: yes - username: root - password: r00tme -diff --git a/deploy/scenario/ha_nfv-kvm_heat_ceilometer_scenario.yaml b/deploy/scenario/ha_nfv-kvm_heat_ceilometer_scenario.yaml -index 1217f72..bcdda61 100644 ---- a/deploy/scenario/ha_nfv-kvm_heat_ceilometer_scenario.yaml -+++ b/deploy/scenario/ha_nfv-kvm_heat_ceilometer_scenario.yaml -@@ -145,22 +145,23 @@ dha-override-config: - nodes: - - id: 1 - libvirtName: controller1 -- libvirtTemplate: templates/virtual_environment/vms/controller.xml -+ libvirtTemplate: templates/arm/virtual_environment/vms/controller.xml - - id: 2 - libvirtName: controller2 -- libvirtTemplate: templates/virtual_environment/vms/controller.xml -+ libvirtTemplate: templates/arm/virtual_environment/vms/controller.xml - - id: 3 - libvirtName: controller3 -- libvirtTemplate: templates/virtual_environment/vms/controller.xml -+ libvirtTemplate: templates/arm/virtual_environment/vms/controller.xml - - id: 4 - libvirtName: compute1 -- libvirtTemplate: templates/virtual_environment/vms/compute.xml -+ libvirtTemplate: templates/arm/virtual_environment/vms/compute.xml - - id: 5 - libvirtName: compute2 -- libvirtTemplate: templates/virtual_environment/vms/compute.xml -+ libvirtTemplate: templates/arm/virtual_environment/vms/compute.xml - - id: 6 - libvirtName: fuel-master -- libvirtTemplate: templates/virtual_environment/vms/fuel.xml -+ libvirtTemplate: templates/arm/virtual_environment/vms/fuel.xml -+ libvirtUri: qemu+ssh://jenkins@10.0.2.5/system - isFuel: yes - username: root - password: r00tme -diff --git a/deploy/scenario/ha_nfv-ovs_heat_ceilometer_scenario.yaml b/deploy/scenario/ha_nfv-ovs_heat_ceilometer_scenario.yaml -index 51deb4e..bee2d1a 100644 ---- a/deploy/scenario/ha_nfv-ovs_heat_ceilometer_scenario.yaml -+++ b/deploy/scenario/ha_nfv-ovs_heat_ceilometer_scenario.yaml -@@ -107,22 +107,23 @@ dha-override-config: - nodes: - - id: 1 - libvirtName: controller1 -- libvirtTemplate: templates/virtual_environment/vms/controller.xml -+ libvirtTemplate: templates/arm/virtual_environment/vms/controller.xml - - id: 2 - libvirtName: controller2 -- libvirtTemplate: templates/virtual_environment/vms/controller.xml -+ libvirtTemplate: templates/arm/virtual_environment/vms/controller.xml - - id: 3 - libvirtName: controller3 -- libvirtTemplate: templates/virtual_environment/vms/controller.xml -+ libvirtTemplate: templates/arm/virtual_environment/vms/controller.xml - - id: 4 - libvirtName: compute1 -- libvirtTemplate: templates/virtual_environment/vms/compute.xml -+ libvirtTemplate: templates/arm/virtual_environment/vms/compute.xml - - id: 5 - libvirtName: compute2 -- libvirtTemplate: templates/virtual_environment/vms/compute.xml -+ libvirtTemplate: templates/arm/virtual_environment/vms/compute.xml - - id: 6 - libvirtName: fuel-master -- libvirtTemplate: templates/virtual_environment/vms/fuel.xml -+ libvirtTemplate: templates/arm/virtual_environment/vms/fuel.xml -+ libvirtUri: qemu+ssh://jenkins@10.0.2.5/system - isFuel: yes - username: root - password: r00tme -diff --git a/deploy/scenario/ha_odl-bgpvpn_scenario.yaml b/deploy/scenario/ha_odl-bgpvpn_scenario.yaml -index 7d52e77..46efc4b 100644 ---- a/deploy/scenario/ha_odl-bgpvpn_scenario.yaml -+++ b/deploy/scenario/ha_odl-bgpvpn_scenario.yaml -@@ -77,22 +77,23 @@ dha-override-config: - nodes: - - id: 1 - libvirtName: controller1 -- libvirtTemplate: templates/virtual_environment/vms/controller.xml -+ libvirtTemplate: templates/arm/virtual_environment/vms/controller.xml - - id: 2 - libvirtName: controller2 -- libvirtTemplate: templates/virtual_environment/vms/controller.xml -+ libvirtTemplate: templates/arm/virtual_environment/vms/controller.xml - - id: 3 - libvirtName: controller3 -- libvirtTemplate: templates/virtual_environment/vms/controller.xml -+ libvirtTemplate: templates/arm/virtual_environment/vms/controller.xml - - id: 4 - libvirtName: compute1 -- libvirtTemplate: templates/virtual_environment/vms/compute.xml -+ libvirtTemplate: templates/arm/virtual_environment/vms/compute.xml - - id: 5 - libvirtName: compute2 -- libvirtTemplate: templates/virtual_environment/vms/compute.xml -+ libvirtTemplate: templates/arm/virtual_environment/vms/compute.xml - - id: 6 - libvirtName: fuel-master -- libvirtTemplate: templates/virtual_environment/vms/fuel.xml -+ libvirtTemplate: templates/arm/virtual_environment/vms/fuel.xml -+ libvirtUri: qemu+ssh://jenkins@10.0.2.5/system - isFuel: yes - username: root - password: r00tme -diff --git a/deploy/scenario/ha_odl-l2_heat_ceilometer_scenario.yaml b/deploy/scenario/ha_odl-l2_heat_ceilometer_scenario.yaml -index 90c89ae..50dde30 100644 ---- a/deploy/scenario/ha_odl-l2_heat_ceilometer_scenario.yaml -+++ b/deploy/scenario/ha_odl-l2_heat_ceilometer_scenario.yaml -@@ -77,22 +77,23 @@ dha-override-config: - nodes: - - id: 1 - libvirtName: controller1 -- libvirtTemplate: templates/virtual_environment/vms/controller.xml -+ libvirtTemplate: templates/arm/virtual_environment/vms/controller.xml - - id: 2 - libvirtName: controller2 -- libvirtTemplate: templates/virtual_environment/vms/controller.xml -+ libvirtTemplate: templates/arm/virtual_environment/vms/controller.xml - - id: 3 - libvirtName: controller3 -- libvirtTemplate: templates/virtual_environment/vms/controller.xml -+ libvirtTemplate: templates/arm/virtual_environment/vms/controller.xml - - id: 4 - libvirtName: compute1 -- libvirtTemplate: templates/virtual_environment/vms/compute.xml -+ libvirtTemplate: templates/arm/virtual_environment/vms/compute.xml - - id: 5 - libvirtName: compute2 -- libvirtTemplate: templates/virtual_environment/vms/compute.xml -+ libvirtTemplate: templates/arm/virtual_environment/vms/compute.xml - - id: 6 - libvirtName: fuel-master -- libvirtTemplate: templates/virtual_environment/vms/fuel.xml -+ libvirtTemplate: templates/arm/virtual_environment/vms/fuel.xml -+ libvirtUri: qemu+ssh://jenkins@10.0.2.5/system - isFuel: yes - username: root - password: r00tme -diff --git a/deploy/scenario/ha_odl-l2_sfc_heat_ceilometer_scenario.yaml b/deploy/scenario/ha_odl-l2_sfc_heat_ceilometer_scenario.yaml -index c478948..e20ec1a 100644 ---- a/deploy/scenario/ha_odl-l2_sfc_heat_ceilometer_scenario.yaml -+++ b/deploy/scenario/ha_odl-l2_sfc_heat_ceilometer_scenario.yaml -@@ -95,22 +95,23 @@ dha-override-config: - nodes: - - id: 1 - libvirtName: controller1 -- libvirtTemplate: templates/virtual_environment/vms/controller.xml -+ libvirtTemplate: templates/arm/virtual_environment/vms/controller.xml - - id: 2 - libvirtName: controller2 -- libvirtTemplate: templates/virtual_environment/vms/controller.xml -+ libvirtTemplate: templates/arm/virtual_environment/vms/controller.xml - - id: 3 - libvirtName: controller3 -- libvirtTemplate: templates/virtual_environment/vms/controller.xml -+ libvirtTemplate: templates/arm/virtual_environment/vms/controller.xml - - id: 4 - libvirtName: compute1 -- libvirtTemplate: templates/virtual_environment/vms/compute.xml -+ libvirtTemplate: templates/arm/virtual_environment/vms/compute.xml - - id: 5 - libvirtName: compute2 -- libvirtTemplate: templates/virtual_environment/vms/compute.xml -+ libvirtTemplate: templates/arm/virtual_environment/vms/compute.xml - - id: 6 - libvirtName: fuel-master -- libvirtTemplate: templates/virtual_environment/vms/fuel.xml -+ libvirtTemplate: templates/arm/virtual_environment/vms/fuel.xml -+ libvirtUri: qemu+ssh://jenkins@10.0.2.5/system - isFuel: yes - username: root - password: r00tme -diff --git a/deploy/scenario/ha_odl-l3_heat_ceilometer_scenario.yaml b/deploy/scenario/ha_odl-l3_heat_ceilometer_scenario.yaml -index 967e7d2..322dd27 100644 ---- a/deploy/scenario/ha_odl-l3_heat_ceilometer_scenario.yaml -+++ b/deploy/scenario/ha_odl-l3_heat_ceilometer_scenario.yaml -@@ -93,22 +93,23 @@ dha-override-config: - nodes: - - id: 1 - libvirtName: controller1 -- libvirtTemplate: templates/virtual_environment/vms/controller.xml -+ libvirtTemplate: templates/arm/virtual_environment/vms/controller.xml - - id: 2 - libvirtName: controller2 -- libvirtTemplate: templates/virtual_environment/vms/controller.xml -+ libvirtTemplate: templates/arm/virtual_environment/vms/controller.xml - - id: 3 - libvirtName: controller3 -- libvirtTemplate: templates/virtual_environment/vms/controller.xml -+ libvirtTemplate: templates/arm/virtual_environment/vms/controller.xml - - id: 4 - libvirtName: compute1 -- libvirtTemplate: templates/virtual_environment/vms/compute.xml -+ libvirtTemplate: templates/arm/virtual_environment/vms/compute.xml - - id: 5 - libvirtName: compute2 -- libvirtTemplate: templates/virtual_environment/vms/compute.xml -+ libvirtTemplate: templates/arm/virtual_environment/vms/compute.xml - - id: 6 - libvirtName: fuel-master -- libvirtTemplate: templates/virtual_environment/vms/fuel.xml -+ libvirtTemplate: templates/arm/virtual_environment/vms/fuel.xml -+ libvirtUri: qemu+ssh://jenkins@10.0.2.5/system - isFuel: yes - username: root - password: r00tme -diff --git a/deploy/scenario/ha_onos_nofeature_heat_ceilometer_scenario.yaml b/deploy/scenario/ha_onos_nofeature_heat_ceilometer_scenario.yaml -index f813458..88b8f6b 100644 ---- a/deploy/scenario/ha_onos_nofeature_heat_ceilometer_scenario.yaml -+++ b/deploy/scenario/ha_onos_nofeature_heat_ceilometer_scenario.yaml -@@ -88,22 +88,23 @@ dha-override-config: - nodes: - - id: 1 - libvirtName: controller1 -- libvirtTemplate: templates/virtual_environment/vms/controller.xml -+ libvirtTemplate: templates/arm/virtual_environment/vms/controller.xml - - id: 2 - libvirtName: controller2 -- libvirtTemplate: templates/virtual_environment/vms/controller.xml -+ libvirtTemplate: templates/arm/virtual_environment/vms/controller.xml - - id: 3 - libvirtName: controller3 -- libvirtTemplate: templates/virtual_environment/vms/controller.xml -+ libvirtTemplate: templates/arm/virtual_environment/vms/controller.xml - - id: 4 - libvirtName: compute1 -- libvirtTemplate: templates/virtual_environment/vms/compute.xml -+ libvirtTemplate: templates/arm/virtual_environment/vms/compute.xml - - id: 5 - libvirtName: compute2 -- libvirtTemplate: templates/virtual_environment/vms/compute.xml -+ libvirtTemplate: templates/arm/virtual_environment/vms/compute.xml - - id: 6 - libvirtName: fuel-master -- libvirtTemplate: templates/virtual_environment/vms/fuel.xml -+ libvirtTemplate: templates/arm/virtual_environment/vms/fuel.xml -+ libvirtUri: qemu+ssh://jenkins@10.0.2.5/system - isFuel: yes - username: root - password: r00tme -diff --git a/deploy/scenario/ha_onos_sfc_heat_ceilometer_scenario.yaml b/deploy/scenario/ha_onos_sfc_heat_ceilometer_scenario.yaml -index 32fbfcf..c390172 100644 ---- a/deploy/scenario/ha_onos_sfc_heat_ceilometer_scenario.yaml -+++ b/deploy/scenario/ha_onos_sfc_heat_ceilometer_scenario.yaml -@@ -68,22 +68,23 @@ dha-override-config: - nodes: - - id: 1 - libvirtName: controller1 -- libvirtTemplate: templates/virtual_environment/vms/controller.xml -+ libvirtTemplate: templates/arm/virtual_environment/vms/controller.xml - - id: 2 - libvirtName: controller2 -- libvirtTemplate: templates/virtual_environment/vms/controller.xml -+ libvirtTemplate: templates/arm/virtual_environment/vms/controller.xml - - id: 3 - libvirtName: controller3 -- libvirtTemplate: templates/virtual_environment/vms/controller.xml -+ libvirtTemplate: templates/arm/virtual_environment/vms/controller.xml - - id: 4 - libvirtName: compute1 -- libvirtTemplate: templates/virtual_environment/vms/compute.xml -+ libvirtTemplate: templates/arm/virtual_environment/vms/compute.xml - - id: 5 - libvirtName: compute2 -- libvirtTemplate: templates/virtual_environment/vms/compute.xml -+ libvirtTemplate: templates/arm/virtual_environment/vms/compute.xml - - id: 6 - libvirtName: fuel-master -- libvirtTemplate: templates/virtual_environment/vms/fuel.xml -+ libvirtTemplate: templates/arm/virtual_environment/vms/fuel.xml -+ libvirtUri: qemu+ssh://jenkins@10.0.2.5/system - isFuel: yes - username: root - password: r00tme -diff --git a/deploy/scenario/ha_vlan_heat_ceilometer_scenario.yaml b/deploy/scenario/ha_vlan_heat_ceilometer_scenario.yaml -index d1d5191..65b8b20 100644 ---- a/deploy/scenario/ha_vlan_heat_ceilometer_scenario.yaml -+++ b/deploy/scenario/ha_vlan_heat_ceilometer_scenario.yaml -@@ -102,22 +102,23 @@ dha-override-config: - nodes: - - id: 1 - libvirtName: controller1 -- libvirtTemplate: templates/virtual_environment/vms/controller.xml -+ libvirtTemplate: templates/arm/virtual_environment/vms/controller.xml - - id: 2 - libvirtName: controller2 -- libvirtTemplate: templates/virtual_environment/vms/controller.xml -+ libvirtTemplate: templates/arm/virtual_environment/vms/controller.xml - - id: 3 - libvirtName: controller3 -- libvirtTemplate: templates/virtual_environment/vms/controller.xml -+ libvirtTemplate: templates/arm/virtual_environment/vms/controller.xml - - id: 4 - libvirtName: compute1 -- libvirtTemplate: templates/virtual_environment/vms/compute.xml -+ libvirtTemplate: templates/arm/virtual_environment/vms/compute.xml - - id: 5 - libvirtName: compute2 -- libvirtTemplate: templates/virtual_environment/vms/compute.xml -+ libvirtTemplate: templates/arm/virtual_environment/vms/compute.xml - - id: 6 - libvirtName: fuel-master -- libvirtTemplate: templates/virtual_environment/vms/fuel.xml -+ libvirtTemplate: templates/arm/virtual_environment/vms/fuel.xml -+ libvirtUri: qemu+ssh://jenkins@10.0.2.5/system - isFuel: yes - username: root - password: r00tme -diff --git a/deploy/scenario/no-ha_heat_ceilometer_scenario.yaml b/deploy/scenario/no-ha_heat_ceilometer_scenario.yaml -index 3dd5f84..d738154 100644 ---- a/deploy/scenario/no-ha_heat_ceilometer_scenario.yaml -+++ b/deploy/scenario/no-ha_heat_ceilometer_scenario.yaml -@@ -41,17 +41,42 @@ dea-override-config: - - id: 1 - interfaces: interfaces_1 - role: mongo,controller,congress -+ transformations: transformations_1 - - id: 2 - interfaces: interfaces_1 - role: ceph-osd,compute -+ transformations: transformations_2 - - id: 3 - interfaces: interfaces_1 - role: ceph-osd,compute -+ transformations: transformations_2 - - id: 4 - interfaces: interfaces_1 - role: ceph-osd,compute -+ transformations: transformations_2 - - dha-override-config: -+ nodes: -+ - id: 1 -+ libvirtName: controller1 -+ libvirtTemplate: templates/arm/virtual_environment/vms/controller.xml -+ - id: 2 -+ libvirtName: compute1 -+ libvirtTemplate: templates/arm/virtual_environment/vms/compute.xml -+ - id: 3 -+ libvirtName: compute2 -+ libvirtTemplate: templates/arm/virtual_environment/vms/compute.xml -+ - id: 4 -+ libvirtName: compute3 -+ libvirtTemplate: templates/arm/virtual_environment/vms/compute.xml -+ - id: 5 -+ libvirtName: fuel-master -+ libvirtTemplate: templates/arm/virtual_environment/vms/fuel.xml -+ libvirtUri: qemu+ssh://jenkins@10.0.2.5/system -+ isFuel: yes -+ username: root -+ password: r00tme -+ - # disks: - # contrail: 500G - -diff --git a/deploy/scenario/no-ha_nfv-kvm_heat_ceilometer_scenario.yaml b/deploy/scenario/no-ha_nfv-kvm_heat_ceilometer_scenario.yaml -index 4661a26..7ade095 100644 ---- a/deploy/scenario/no-ha_nfv-kvm_heat_ceilometer_scenario.yaml -+++ b/deploy/scenario/no-ha_nfv-kvm_heat_ceilometer_scenario.yaml -@@ -48,15 +48,19 @@ dea-override-config: - - id: 1 - interfaces: interfaces_1 - role: mongo,controller -+ transformations: transformations_1 - - id: 2 - interfaces: interfaces_1 - role: ceph-osd -+ transformations: transformations_2 - - id: 3 - interfaces: interfaces_1 - role: compute -+ transformations: transformations_2 - - id: 4 - interfaces: interfaces_1 - role: compute -+ transformations: transformations_2 - - settings: - editable: -@@ -141,6 +145,27 @@ dea-override-config: - weight: 10 - - dha-override-config: -+ nodes: -+ - id: 1 -+ libvirtName: controller1 -+ libvirtTemplate: templates/arm/virtual_environment/vms/controller.xml -+ - id: 2 -+ libvirtName: compute1 -+ libvirtTemplate: templates/arm/virtual_environment/vms/compute.xml -+ - id: 3 -+ libvirtName: compute2 -+ libvirtTemplate: templates/arm/virtual_environment/vms/compute.xml -+ - id: 4 -+ libvirtName: compute3 -+ libvirtTemplate: templates/arm/virtual_environment/vms/compute.xml -+ - id: 5 -+ libvirtName: fuel-master -+ libvirtTemplate: templates/arm/virtual_environment/vms/fuel.xml -+ libvirtUri: qemu+ssh://jenkins@10.0.2.5/system -+ isFuel: yes -+ username: root -+ password: r00tme -+ - # disks: - # contrail: 500G - -diff --git a/deploy/scenario/no-ha_nfv-kvm_nfv-ovs_heat_ceilometer_scenario.yaml b/deploy/scenario/no-ha_nfv-kvm_nfv-ovs_heat_ceilometer_scenario.yaml -index 87364e8..a3f0864 100644 ---- a/deploy/scenario/no-ha_nfv-kvm_nfv-ovs_heat_ceilometer_scenario.yaml -+++ b/deploy/scenario/no-ha_nfv-kvm_nfv-ovs_heat_ceilometer_scenario.yaml -@@ -55,15 +55,19 @@ dea-override-config: - - id: 1 - interfaces: interfaces_1 - role: mongo,controller -+ transformations: transformations_1 - - id: 2 - interfaces: interfaces_1 - role: ceph-osd,compute -+ transformations: transformations_2 - - id: 3 - interfaces: interfaces_1 - role: ceph-osd,compute -+ transformations: transformations_2 - - id: 4 - interfaces: interfaces_1 - role: ceph-osd,compute -+ transformations: transformations_2 - settings: - editable: - additional_components: -@@ -104,6 +108,27 @@ dea-override-config: - weight: 30 - - dha-override-config: -+ nodes: -+ - id: 1 -+ libvirtName: controller1 -+ libvirtTemplate: templates/arm/virtual_environment/vms/controller.xml -+ - id: 2 -+ libvirtName: compute1 -+ libvirtTemplate: templates/arm/virtual_environment/vms/compute.xml -+ - id: 3 -+ libvirtName: compute2 -+ libvirtTemplate: templates/arm/virtual_environment/vms/compute.xml -+ - id: 4 -+ libvirtName: compute3 -+ libvirtTemplate: templates/arm/virtual_environment/vms/compute.xml -+ - id: 5 -+ libvirtName: fuel-master -+ libvirtTemplate: templates/arm/virtual_environment/vms/fuel.xml -+ libvirtUri: qemu+ssh://jenkins@10.0.2.5/system -+ isFuel: yes -+ username: root -+ password: r00tme -+ - # disks: - # contrail: 500G - -diff --git a/deploy/scenario/no-ha_nfv-ovs_heat_ceilometer_scenario.yaml b/deploy/scenario/no-ha_nfv-ovs_heat_ceilometer_scenario.yaml -index 7ab76cb..f4e0adc 100644 ---- a/deploy/scenario/no-ha_nfv-ovs_heat_ceilometer_scenario.yaml -+++ b/deploy/scenario/no-ha_nfv-ovs_heat_ceilometer_scenario.yaml -@@ -56,19 +56,22 @@ dea-override-config: - - id: 1 - interfaces: interfaces_vlan - role: mongo,controller -+ transformations: transformations_1 - - id: 2 - interfaces: interfaces_dpdk - role: ceph-osd,compute - attributes: attributes_1 -+ transformations: transformations_2 - - id: 3 - interfaces: interfaces_dpdk - role: ceph-osd,compute - attributes: attributes_1 -+ transformations: transformations_2 - - id: 4 - interfaces: interfaces_dpdk - role: ceph-osd,compute - attributes: attributes_1 -- -+ transformations: transformations_2 - attributes_1: - hugepages: - dpdk: -@@ -102,6 +105,27 @@ dea-override-config: - vlan_start: null - - dha-override-config: -+ nodes: -+ - id: 1 -+ libvirtName: controller1 -+ libvirtTemplate: templates/arm/virtual_environment/vms/controller.xml -+ - id: 2 -+ libvirtName: compute1 -+ libvirtTemplate: templates/arm/virtual_environment/vms/compute.xml -+ - id: 3 -+ libvirtName: compute2 -+ libvirtTemplate: templates/arm/virtual_environment/vms/compute.xml -+ - id: 4 -+ libvirtName: compute3 -+ libvirtTemplate: templates/arm/virtual_environment/vms/compute.xml -+ - id: 5 -+ libvirtName: fuel-master -+ libvirtTemplate: templates/arm/virtual_environment/vms/fuel.xml -+ libvirtUri: qemu+ssh://jenkins@10.0.2.5/system -+ isFuel: yes -+ username: root -+ password: r00tme -+ - # disks: - # contrail: 500G - -diff --git a/deploy/scenario/no-ha_odl-bgpvpn_scenario.yaml b/deploy/scenario/no-ha_odl-bgpvpn_scenario.yaml -index 6f21ea2..e92a741 100644 ---- a/deploy/scenario/no-ha_odl-bgpvpn_scenario.yaml -+++ b/deploy/scenario/no-ha_odl-bgpvpn_scenario.yaml -@@ -60,22 +60,47 @@ dea-override-config: - - id: 1 - interfaces: interfaces_1 - role: mongo,controller -+ transformations: transformations_1 - - id: 2 - interfaces: interfaces_1 - role: ceph-osd,opendaylight -+ transformations: transformations_2 - - id: 3 - interfaces: interfaces_1 - role: ceph-osd,compute -+ transformations: transformations_2 - - id: 4 - interfaces: interfaces_1 - role: ceph-osd,compute -+ transformations: transformations_2 - settings: - editable: - public_network_assignment: - assign_to_all_nodes: - value: true - - dha-override-config: -+ nodes: -+ - id: 1 -+ libvirtName: controller1 -+ libvirtTemplate: templates/arm/virtual_environment/vms/controller.xml -+ - id: 2 -+ libvirtName: compute1 -+ libvirtTemplate: templates/arm/virtual_environment/vms/compute.xml -+ - id: 3 -+ libvirtName: compute2 -+ libvirtTemplate: templates/arm/virtual_environment/vms/compute.xml -+ - id: 4 -+ libvirtName: compute3 -+ libvirtTemplate: templates/arm/virtual_environment/vms/compute.xml -+ - id: 5 -+ libvirtName: fuel-master -+ libvirtTemplate: templates/arm/virtual_environment/vms/fuel.xml -+ libvirtUri: qemu+ssh://jenkins@10.0.2.5/system -+ isFuel: yes -+ username: root -+ password: r00tme -+ - # disks: - # contrail: 500G - -diff --git a/deploy/scenario/no-ha_odl-l2_heat_ceilometer_scenario.yaml b/deploy/scenario/no-ha_odl-l2_heat_ceilometer_scenario.yaml -index f8787d7..1efa89a 100644 ---- a/deploy/scenario/no-ha_odl-l2_heat_ceilometer_scenario.yaml -+++ b/deploy/scenario/no-ha_odl-l2_heat_ceilometer_scenario.yaml -@@ -61,17 +61,42 @@ dea-override-config: - - id: 1 - interfaces: interfaces_1 - role: mongo,controller -+ transformations: transformations_1 - - id: 2 - interfaces: interfaces_1 - role: ceph-osd,opendaylight -+ transformations: transformations_2 - - id: 3 - interfaces: interfaces_1 - role: ceph-osd,compute -+ transformations: transformations_2 - - id: 4 - interfaces: interfaces_1 - role: ceph-osd,compute -+ transformations: transformations_2 - - dha-override-config: -+ nodes: -+ - id: 1 -+ libvirtName: controller1 -+ libvirtTemplate: templates/arm/virtual_environment/vms/controller.xml -+ - id: 2 -+ libvirtName: compute1 -+ libvirtTemplate: templates/arm/virtual_environment/vms/compute.xml -+ - id: 3 -+ libvirtName: compute2 -+ libvirtTemplate: templates/arm/virtual_environment/vms/compute.xml -+ - id: 4 -+ libvirtName: compute3 -+ libvirtTemplate: templates/arm/virtual_environment/vms/compute.xml -+ - id: 5 -+ libvirtName: fuel-master -+ libvirtTemplate: templates/arm/virtual_environment/vms/fuel.xml -+ libvirtUri: qemu+ssh://jenkins@10.0.2.5/system -+ isFuel: yes -+ username: root -+ password: r00tme -+ - # disks: - # contrail: 500G - -diff --git a/deploy/scenario/no-ha_odl-l2_sfc_heat_ceilometer_scenario.yaml b/deploy/scenario/no-ha_odl-l2_sfc_heat_ceilometer_scenario.yaml -index 90a45d5..b1fe773 100644 ---- a/deploy/scenario/no-ha_odl-l2_sfc_heat_ceilometer_scenario.yaml -+++ b/deploy/scenario/no-ha_odl-l2_sfc_heat_ceilometer_scenario.yaml -@@ -80,17 +80,42 @@ dea-override-config: - - id: 1 - interfaces: interfaces_1 - role: mongo,controller,tacker -+ transformations: transformations_1 - - id: 2 - interfaces: interfaces_1 - role: ceph-osd,opendaylight -+ transformations: transformations_2 - - id: 3 - interfaces: interfaces_1 - role: ceph-osd,compute -+ transformations: transformations_2 - - id: 4 - interfaces: interfaces_1 - role: ceph-osd,compute -+ transformations: transformations_2 - - dha-override-config: -+ nodes: -+ - id: 1 -+ libvirtName: controller1 -+ libvirtTemplate: templates/arm/virtual_environment/vms/controller.xml -+ - id: 2 -+ libvirtName: compute1 -+ libvirtTemplate: templates/arm/virtual_environment/vms/compute.xml -+ - id: 3 -+ libvirtName: compute2 -+ libvirtTemplate: templates/arm/virtual_environment/vms/compute.xml -+ - id: 4 -+ libvirtName: compute3 -+ libvirtTemplate: templates/arm/virtual_environment/vms/compute.xml -+ - id: 5 -+ libvirtName: fuel-master -+ libvirtTemplate: templates/arm/virtual_environment/vms/fuel.xml -+ libvirtUri: qemu+ssh://jenkins@10.0.2.5/system -+ isFuel: yes -+ username: root -+ password: r00tme -+ - # disks: - # contrail: 500G - -diff --git a/deploy/scenario/no-ha_odl-l3_heat_ceilometer_scenario.yaml b/deploy/scenario/no-ha_odl-l3_heat_ceilometer_scenario.yaml -index 0c8415f..b59a049 100644 ---- a/deploy/scenario/no-ha_odl-l3_heat_ceilometer_scenario.yaml -+++ b/deploy/scenario/no-ha_odl-l3_heat_ceilometer_scenario.yaml -@@ -71,15 +71,19 @@ dea-override-config: - - id: 1 - interfaces: interfaces_1 - role: mongo,controller -+ transformations: transformations_1 - - id: 2 - interfaces: interfaces_1 - role: ceph-osd,opendaylight -+ transformations: transformations_2 - - id: 3 - interfaces: interfaces_1 - role: ceph-osd,compute -+ transformations: transformations_2 - - id: 4 - interfaces: interfaces_1 - role: ceph-osd,compute -+ transformations: transformations_2 - settings: - editable: - public_network_assignment: -@@ -87,6 +91,27 @@ dea-override-config: - value: true - - dha-override-config: -+ nodes: -+ - id: 1 -+ libvirtName: controller1 -+ libvirtTemplate: templates/arm/virtual_environment/vms/controller.xml -+ - id: 2 -+ libvirtName: compute1 -+ libvirtTemplate: templates/arm/virtual_environment/vms/compute.xml -+ - id: 3 -+ libvirtName: compute2 -+ libvirtTemplate: templates/arm/virtual_environment/vms/compute.xml -+ - id: 4 -+ libvirtName: compute3 -+ libvirtTemplate: templates/arm/virtual_environment/vms/compute.xml -+ - id: 5 -+ libvirtName: fuel-master -+ libvirtTemplate: templates/arm/virtual_environment/vms/fuel.xml -+ libvirtUri: qemu+ssh://jenkins@10.0.2.5/system -+ isFuel: yes -+ username: root -+ password: r00tme -+ - # disks: - # contrail: 500G - -diff --git a/deploy/scenario/no-ha_onos_nofeature_heat_ceilometer_scenario.yaml b/deploy/scenario/no-ha_onos_nofeature_heat_ceilometer_scenario.yaml -index f5c00f0..e51dd56 100644 ---- a/deploy/scenario/no-ha_onos_nofeature_heat_ceilometer_scenario.yaml -+++ b/deploy/scenario/no-ha_onos_nofeature_heat_ceilometer_scenario.yaml -@@ -68,21 +68,47 @@ dea-override-config: - - id: 1 - interfaces: interfaces_1 - role: mongo,controller,onos -+ transformations: transformations_1 - - id: 2 - interfaces: interfaces_1 - role: ceph-osd,compute -+ transformations: transformations_2 - - id: 3 - interfaces: interfaces_1 - role: ceph-osd,compute -+ transformations: transformations_2 - - id: 4 - interfaces: interfaces_1 - role: ceph-osd,compute -+ transformations: transformations_2 - settings: - editable: - public_network_assignment: - assign_to_all_nodes: - value: true -+ - dha-override-config: -+ nodes: -+ - id: 1 -+ libvirtName: controller1 -+ libvirtTemplate: templates/arm/virtual_environment/vms/controller.xml -+ - id: 2 -+ libvirtName: compute1 -+ libvirtTemplate: templates/arm/virtual_environment/vms/compute.xml -+ - id: 3 -+ libvirtName: compute2 -+ libvirtTemplate: templates/arm/virtual_environment/vms/compute.xml -+ - id: 4 -+ libvirtName: compute3 -+ libvirtTemplate: templates/arm/virtual_environment/vms/compute.xml -+ - id: 5 -+ libvirtName: fuel-master -+ libvirtTemplate: templates/arm/virtual_environment/vms/fuel.xml -+ libvirtUri: qemu+ssh://jenkins@10.0.2.5/system -+ isFuel: yes -+ username: root -+ password: r00tme -+ - # disks: - # contrail: 500G - -diff --git a/deploy/scenario/no-ha_onos_sfc_heat_ceilometer_scenario.yaml b/deploy/scenario/no-ha_onos_sfc_heat_ceilometer_scenario.yaml -index 587346e..21a2be2 100644 ---- a/deploy/scenario/no-ha_onos_sfc_heat_ceilometer_scenario.yaml -+++ b/deploy/scenario/no-ha_onos_sfc_heat_ceilometer_scenario.yaml -@@ -48,21 +48,46 @@ dea-override-config: - - id: 1 - interfaces: interfaces_1 - role: mongo,controller,onos -+ transformations: transformations_1 - - id: 2 - interfaces: interfaces_1 - role: ceph-osd,compute -+ transformations: transformations_2 - - id: 3 - interfaces: interfaces_1 - role: ceph-osd,compute -+ transformations: transformations_2 - - id: 4 - interfaces: interfaces_1 - role: ceph-osd,compute -+ transformations: transformations_2 - settings: - editable: - public_network_assignment: - assign_to_all_nodes: - value: true - dha-override-config: -+ nodes: -+ - id: 1 -+ libvirtName: controller1 -+ libvirtTemplate: templates/arm/virtual_environment/vms/controller.xml -+ - id: 2 -+ libvirtName: compute1 -+ libvirtTemplate: templates/arm/virtual_environment/vms/compute.xml -+ - id: 3 -+ libvirtName: compute2 -+ libvirtTemplate: templates/arm/virtual_environment/vms/compute.xml -+ - id: 4 -+ libvirtName: compute3 -+ libvirtTemplate: templates/arm/virtual_environment/vms/compute.xml -+ - id: 5 -+ libvirtName: fuel-master -+ libvirtTemplate: templates/arm/virtual_environment/vms/fuel.xml -+ libvirtUri: qemu+ssh://jenkins@10.0.2.5/system -+ isFuel: yes -+ username: root -+ password: r00tme -+ - # disks: - # contrail: 500G - -diff --git a/deploy/scenario/no-ha_vlan_heat_ceilometer_scenario.yaml b/deploy/scenario/no-ha_vlan_heat_ceilometer_scenario.yaml -index 567ea98..ada5b7e 100644 ---- a/deploy/scenario/no-ha_vlan_heat_ceilometer_scenario.yaml -+++ b/deploy/scenario/no-ha_vlan_heat_ceilometer_scenario.yaml -@@ -45,15 +45,19 @@ dea-override-config: - - id: 1 - interfaces: interfaces_1 - role: mongo,controller -+ transformations: transformations_1 - - id: 2 - interfaces: interfaces_1 - role: ceph-osd,compute -+ transformations: transformations_2 - - id: 3 - interfaces: interfaces_1 - role: ceph-osd,compute -+ transformations: transformations_2 - - id: 4 - interfaces: interfaces_1 - role: ceph-osd,compute -+ transformations: transformations_2 - settings: - editable: - additional_components: -@@ -94,6 +98,27 @@ dea-override-config: - weight: 30 - - dha-override-config: -+ nodes: -+ - id: 1 -+ libvirtName: controller1 -+ libvirtTemplate: templates/arm/virtual_environment/vms/controller.xml -+ - id: 2 -+ libvirtName: compute1 -+ libvirtTemplate: templates/arm/virtual_environment/vms/compute.xml -+ - id: 3 -+ libvirtName: compute2 -+ libvirtTemplate: templates/arm/virtual_environment/vms/compute.xml -+ - id: 4 -+ libvirtName: compute3 -+ libvirtTemplate: templates/arm/virtual_environment/vms/compute.xml -+ - id: 5 -+ libvirtName: fuel-master -+ libvirtTemplate: templates/arm/virtual_environment/vms/fuel.xml -+ libvirtUri: qemu+ssh://jenkins@10.0.2.5/system -+ isFuel: yes -+ username: root -+ password: r00tme -+ - # These overrides only take effect for virtual deployment scenarios - - ############################################################################## -diff --git a/deploy/templates/arm/virtual_environment/networks/.gitkeep b/deploy/templates/arm/virtual_environment/networks/.gitkeep -new file mode 100644 -index 0000000..e69de29 -diff --git a/deploy/templates/arm/virtual_environment/vms/compute.xml b/deploy/templates/arm/virtual_environment/vms/compute.xml -new file mode 100644 -index 0000000..db3ba29 ---- /dev/null -+++ b/deploy/templates/arm/virtual_environment/vms/compute.xml -@@ -0,0 +1,57 @@ -+<domain type='kvm'> -+ <name>compute</name> -+ <memory unit='KiB'>8392704</memory> -+ <currentMemory unit='KiB'>8392704</currentMemory> -+ <vcpu placement='static'>6</vcpu> -+ <os> -+ <type arch='aarch64' machine='virt-2.6'>hvm</type> -+ <loader readonly='yes' type='pflash'>/usr/share/AAVMF/AAVMF_CODE.fd</loader> -+ </os> -+ <features> -+ <gic version='3'/> -+ </features> -+ <cpu mode='host-model'> -+ <model fallback='allow'/> -+ </cpu> -+ <clock offset='utc'/> -+ <on_poweroff>destroy</on_poweroff> -+ <on_reboot>restart</on_reboot> -+ <on_crash>restart</on_crash> -+ <devices> -+ <emulator>/usr/bin/kvm</emulator> -+ <disk type='file' device='disk'> -+ <driver name='qemu' type='raw' cache='none' io='native'/> -+ <source file='disk.raw'/> -+ <target dev='vda' bus='virtio'/> -+ <boot order='2'/> -+ <address type='virtio-mmio'/> -+ </disk> -+ <controller type='scsi' index='0' model='virtio-scsi'> -+ <address type='virtio-mmio'/> -+ </controller> -+ <controller type='pci' index='0' model='pcie-root'/> -+ <interface type='bridge'> -+ <source bridge='admin8_br'/> -+ <model type='virtio'/> -+ <boot order='1'/> -+ <address type='virtio-mmio'/> -+ </interface> -+ <interface type='bridge'> -+ <source bridge='public8_br'/> -+ <model type='virtio'/> -+ <address type='virtio-mmio'/> -+ </interface> -+ <interface type='bridge'> -+ <source bridge='trunk8_br'/> -+ <model type='virtio'/> -+ <address type='virtio-mmio'/> -+ </interface> -+ <serial type='pty'> -+ <target port='0'/> -+ </serial> -+ <console type='pty'> -+ <target type='serial' port='0'/> -+ </console> -+ </devices> -+</domain> -+ -diff --git a/deploy/templates/arm/virtual_environment/vms/controller.xml b/deploy/templates/arm/virtual_environment/vms/controller.xml -new file mode 100644 -index 0000000..6bd0385 ---- /dev/null -+++ b/deploy/templates/arm/virtual_environment/vms/controller.xml -@@ -0,0 +1,57 @@ -+<domain type='kvm'> -+ <name>controller</name> -+ <memory unit='KiB'>8392704</memory> -+ <currentMemory unit='KiB'>8392704</currentMemory> -+ <vcpu placement='static'>6</vcpu> -+ <os> -+ <type arch='aarch64' machine='virt-2.6'>hvm</type> -+ <loader readonly='yes' type='pflash'>/usr/share/AAVMF/AAVMF_CODE.fd</loader> -+ </os> -+ <features> -+ <gic version='3'/> -+ </features> -+ <cpu mode='host-model'> -+ <model fallback='allow'/> -+ </cpu> -+ <clock offset='utc'/> -+ <on_poweroff>destroy</on_poweroff> -+ <on_reboot>restart</on_reboot> -+ <on_crash>restart</on_crash> -+ <devices> -+ <emulator>/usr/bin/kvm</emulator> -+ <disk type='file' device='disk'> -+ <driver name='qemu' type='raw' cache='none' io='native'/> -+ <source file='disk.raw'/> -+ <target dev='vda' bus='virtio'/> -+ <boot order='2'/> -+ <address type='virtio-mmio'/> -+ </disk> -+ <controller type='scsi' index='0' model='virtio-scsi'> -+ <address type='virtio-mmio'/> -+ </controller> -+ <controller type='pci' index='0' model='pcie-root'/> -+ <interface type='bridge'> -+ <source bridge='admin8_br'/> -+ <model type='virtio'/> -+ <boot order='1'/> -+ <address type='virtio-mmio'/> -+ </interface> -+ <interface type='bridge'> -+ <source bridge='public8_br'/> -+ <model type='virtio'/> -+ <address type='virtio-mmio'/> -+ </interface> -+ <interface type='bridge'> -+ <source bridge='trunk8_br'/> -+ <model type='virtio'/> -+ <address type='virtio-mmio'/> -+ </interface> -+ <serial type='pty'> -+ <target port='0'/> -+ </serial> -+ <console type='pty'> -+ <target type='serial' port='0'/> -+ </console> -+ </devices> -+</domain> -+ -diff --git a/deploy/templates/arm/virtual_environment/vms/fuel.xml b/deploy/templates/arm/virtual_environment/vms/fuel.xml -new file mode 100644 -index 0000000..ad7c67f ---- /dev/null -+++ b/deploy/templates/arm/virtual_environment/vms/fuel.xml -@@ -0,0 +1,99 @@ -+<domain type='kvm'> -+ <name>fuel</name> -+ <memory unit='KiB'>8290304</memory> -+ <currentMemory unit='KiB'>8290304</currentMemory> -+ <vcpu placement='static'>4</vcpu> -+ <resource> -+ <partition>/machine</partition> -+ </resource> -+ <os> -+ <type arch='x86_64' machine='pc-i440fx-rhel7.0.0'>hvm</type> -+ <boot dev='hd'/> -+ <boot dev='cdrom'/> -+ <bootmenu enable='no'/> -+ </os> -+ <features> -+ <acpi/> -+ <apic/> -+ <pae/> -+ </features> -+ <cpu mode='host-model'> -+ <model fallback='allow'/> -+ </cpu> -+ <clock offset='utc'> -+ <timer name='rtc' tickpolicy='catchup'/> -+ <timer name='pit' tickpolicy='delay'/> -+ <timer name='hpet' present='no'/> -+ </clock> -+ <on_poweroff>destroy</on_poweroff> -+ <on_reboot>restart</on_reboot> -+ <on_crash>restart</on_crash> -+ <pm> -+ <suspend-to-mem enabled='no'/> -+ <suspend-to-disk enabled='no'/> -+ </pm> -+ <devices> -+ <emulator>/usr/libexec/qemu-kvm</emulator> -+ <disk type='block' device='cdrom'> -+ <driver name='qemu' type='raw'/> -+ <target dev='hdc' bus='ide'/> -+ <readonly/> -+ </disk> -+ <disk type='file' device='disk'> -+ <driver name='qemu' type='raw' cache='none' io='native'/> -+ <source file='disk.raw'/> -+ <target dev='vda' bus='virtio'/> -+ <address type='pci' domain='0x0000' bus='0x00' slot='0x07' function='0x0'/> -+ </disk> -+ <controller type='usb' index='0' model='ich9-ehci1'> -+ <address type='pci' domain='0x0000' bus='0x00' slot='0x06' function='0x7'/> -+ </controller> -+ <controller type='usb' index='0' model='ich9-uhci1'> -+ <master startport='0'/> -+ <address type='pci' domain='0x0000' bus='0x00' slot='0x06' function='0x0' multifunction='on'/> -+ </controller> -+ <controller type='usb' index='0' model='ich9-uhci2'> -+ <master startport='2'/> -+ <address type='pci' domain='0x0000' bus='0x00' slot='0x06' function='0x1'/> -+ </controller> -+ <controller type='usb' index='0' model='ich9-uhci3'> -+ <master startport='4'/> -+ <address type='pci' domain='0x0000' bus='0x00' slot='0x06' function='0x2'/> -+ </controller> -+ <controller type='pci' index='0' model='pci-root'/> -+ <controller type='ide' index='0'> -+ <address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x1'/> -+ </controller> -+ <controller type='virtio-serial' index='0'> -+ <address type='pci' domain='0x0000' bus='0x00' slot='0x05' function='0x0'/> -+ </controller> -+ <interface type='bridge'> -+ <source bridge='admin8_br0'/> -+ <model type='virtio'/> -+ <address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x0'/> -+ </interface> -+ <interface type='bridge'> -+ <source bridge='public8_br0'/> -+ <model type='virtio'/> -+ <address type='pci' domain='0x0000' bus='0x00' slot='0x04' function='0x0'/> -+ </interface> -+ <serial type='pty'> -+ <target port='0'/> -+ </serial> -+ <console type='pty'> -+ <target type='serial' port='0'/> -+ </console> -+ <input type='mouse' bus='ps2'/> -+ <input type='keyboard' bus='ps2'/> -+ <graphics type='vnc' port='-1' autoport='yes' listen='127.0.0.1'> -+ <listen type='address' address='127.0.0.1'/> -+ </graphics> -+ <video> -+ <model type='vga' vram='16384' heads='1'/> -+ <address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x0'/> -+ </video> -+ <memballoon model='virtio'> -+ <address type='pci' domain='0x0000' bus='0x00' slot='0x08' function='0x0'/> -+ </memballoon> -+ </devices> -+</domain> diff --git a/patches/opnfv-fuel/upstream-backports/0005-CI-deploy-cache-Store-and-reuse-deploy-artifacts.patch b/patches/opnfv-fuel/upstream-backports/0005-CI-deploy-cache-Store-and-reuse-deploy-artifacts.patch deleted file mode 100644 index 9997dedf..00000000 --- a/patches/opnfv-fuel/upstream-backports/0005-CI-deploy-cache-Store-and-reuse-deploy-artifacts.patch +++ /dev/null @@ -1,786 +0,0 @@ -:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: -: Copyright (c) 2017 Enea AB and others. -: -: All rights reserved. This program and the accompanying materials -: are made available under the terms of the Apache License, Version 2.0 -: which accompanies this distribution, and is available at -: http://www.apache.org/licenses/LICENSE-2.0 -:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: -From: Alexandru Avadanii <Alexandru.Avadanii@enea.com> -Date: Thu, 24 Nov 2016 23:02:04 +0100 -Subject: [PATCH] CI: deploy-cache: Store and reuse deploy artifacts - -Add support for caching deploy artifacts, like bootstraps and -target images, which take a lot of time at each deploy to be built, -considering it requires a cross-debootstrap via qemu-user-static and -binfmt. - -For OPNFV CI, the cache will piggy back on the <iso_mount> mechanism, -and be located at: -/iso_mount/opnfv_ci/<branch>/deploy-cache - -TODO: Use dea interface adapter in target images fingerprinting. -TODO: remote fingerprinting -TODO: differentiate between bootstraps and targetimages, so we don't -end up trying to use one cache artifact type as the other. -TODO: implement sanity checks for bootstrap and target images; -TODO: switch `exec_cmd('mkdir ...')` to `create_dir_if_not_exists`; - -JIRA: ARMBAND-172 -JIRA: ARMBAND-242 - -Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com> ---- - ...p_admin_node.sh-deploy_cache-install-hook.patch | 90 ++++++ - ci/deploy.sh | 14 +- - deploy/cloud/configure_settings.py | 4 + - deploy/cloud/deployment.py | 12 + - deploy/deploy.py | 25 +- - deploy/deploy_cache.py | 314 +++++++++++++++++++++ - deploy/deploy_env.py | 13 +- - deploy/install_fuel_master.py | 9 +- - 8 files changed, 472 insertions(+), 9 deletions(-) - create mode 100644 build/f_repos/patch/fuel-main/0006-bootstrap_admin_node.sh-deploy_cache-install-hook.patch - create mode 100644 deploy/deploy_cache.py - -diff --git a/build/f_repos/patch/fuel-main/0006-bootstrap_admin_node.sh-deploy_cache-install-hook.patch b/build/f_repos/patch/fuel-main/0006-bootstrap_admin_node.sh-deploy_cache-install-hook.patch -new file mode 100644 -index 0000000..7acb746 ---- /dev/null -+++ b/build/f_repos/patch/fuel-main/0006-bootstrap_admin_node.sh-deploy_cache-install-hook.patch -@@ -0,0 +1,90 @@ -+From: Alexandru Avadanii <Alexandru.Avadanii@enea.com> -+Date: Mon, 28 Nov 2016 14:27:48 +0100 -+Subject: [PATCH] bootstrap_admin_node.sh: deploy_cache install hook -+ -+Tooling on the automatic deploy side was updated to support deploy -+caching of artifacts like bootstrap (and id_rsa keypair), target -+images etc. -+ -+Add installation hook that calls `fuel-bootstrap import` instead of -+`build` when a bootstrap tar is available in the agreed location, -+/var/lib/opnfv/cache/bootstraps/. -+ -+Temporary until Fuel@Openstack fixes Master key propagation to nodes' -+authorized_keys, use Mcollective remote shell execute to add it -+during deployment. -+This might duplicate the entry in authorized_keys during re-deploys. -+ -+JIRA: ARMBAND-172 -+JIRA: ARMBAND-242 -+ -+Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com> -+--- -+ iso/bootstrap_admin_node.sh | 35 ++++++++++++++++++++++++++++++++++- -+ 1 file changed, 34 insertions(+), 1 deletion(-) -+ -+diff --git a/iso/bootstrap_admin_node.sh b/iso/bootstrap_admin_node.sh -+index 4f5ce4e..4c79552 100755 -+--- a/iso/bootstrap_admin_node.sh -++++ b/iso/bootstrap_admin_node.sh -+@@ -64,6 +64,8 @@ wget \ -+ ASTUTE_YAML='/etc/fuel/astute.yaml' -+ BOOTSTRAP_NODE_CONFIG="/etc/fuel/bootstrap_admin_node.conf" -+ CUSTOM_REPOS="/root/default_deb_repos.yaml" -++OPNFV_CACHE_PATH="/var/cache/opnfv/bootstraps" -++OPNFV_CACHE_TAR="opnfv-bootstraps-cache.tar" -+ bs_build_log='/var/log/fuel-bootstrap-image-build.log' -+ bs_status=0 -+ # Backup network configs to this folder. Folder will be created only if -+@@ -97,6 +99,7 @@ image becomes available, reboot nodes that failed to be discovered." -+ bs_done_message="Default bootstrap image building done. Now you can boot new \ -+ nodes over PXE, they will be discovered and become available for installing \ -+ OpenStack on them" -++bs_cache_message="OPNFV deploy cache: bootstrap image injected." -+ # Update issues messages -+ update_warn_message="There is an issue connecting to update repository of \ -+ your distributions of OpenStack. \ -+@@ -509,12 +512,42 @@ set_ui_bootstrap_error () { -+ EOF -+ } -+ -++function inject_cached_ssh_key () { -++ # FIXME(armband): Propagate master ssh key to nodes' -++ # authorized_keys, until upstream fixes this for image build. -++ local moddir="/etc/puppet/${OPENSTACK_VERSION}/modules/osnailyfacter/modular" -++ cat >> "${moddir}/astute/generate_keys.sh" <<-EOF -++ mco rpc execute_shell_command execute \\ -++ cmd="echo $(cat /root/.ssh/id_rsa.pub) >> /root/.ssh/authorized_keys" -++ EOF -++} -++ -++function inject_cached_ubuntu_bootstrap () { -++ if [ -f "${OPNFV_CACHE_PATH}/${OPNFV_CACHE_TAR}" -a \ -++ -f "${OPNFV_CACHE_PATH}/id_rsa.pub" -a \ -++ -f "${OPNFV_CACHE_PATH}/id_rsa" ]; then -++ if cp "${OPNFV_CACHE_PATH}/id_rsa"* "/root/.ssh/" && \ -++ cp "/root/.ssh/id_rsa.pub" "/root/.ssh/authorized_keys" && \ -++ cp "/root/.ssh/id_rsa.pub" "/etc/cobbler/authorized_keys" && \ -++ sed -i -e "s|\"ssh-rsa .*\"|\"$(cat /root/.ssh/id_rsa.pub)\"|g" \ -++ /etc/nailgun/settings.yaml && \ -++ fuel-bootstrap -v --debug import --activate \ -++ "${OPNFV_CACHE_PATH}/${OPNFV_CACHE_TAR}" >>"$bs_build_log" 2>&1; then -++ inject_cached_ssh_key -++ fuel notify --topic "done" --send "${bs_cache_message}" -++ return 0 -++ fi -++ fi -++ return 1 -++} -++ -+ # Actually build the bootstrap image -+ build_ubuntu_bootstrap () { -+ local ret=1 -+ echo ${bs_progress_message} >&2 -+ set_ui_bootstrap_error "${bs_progress_message}" >&2 -+- if fuel-bootstrap -v --debug build --target_arch arm64 --activate >>"$bs_build_log" 2>&1; then -++ if inject_cached_ubuntu_bootstrap || fuel-bootstrap -v --debug \ -++ build --activate --target_arch arm64 >>"$bs_build_log" 2>&1; then -+ ret=0 -+ fuel notify --topic "done" --send "${bs_done_message}" -+ else -diff --git a/ci/deploy.sh b/ci/deploy.sh -index 081806c..4b1ae0e 100755 ---- a/ci/deploy.sh -+++ b/ci/deploy.sh -@@ -29,7 +29,7 @@ cat << EOF - xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx - `basename $0`: Deploys the Fuel@OPNFV stack - --usage: `basename $0` -b base-uri [-B PXE Bridge] [-f] [-F] [-H] -l lab-name -p pod-name -s deploy-scenario [-S image-dir] [-T timeout] -i iso -+usage: `basename $0` -b base-uri [-B PXE Bridge] [-f] [-F] [-H] -l lab-name -p pod-name -s deploy-scenario [-S image-dir] [-C deploy-cache-dir] [-T timeout] -i iso - -s deployment-scenario [-S optional Deploy-scenario path URI] - [-R optional local relen repo (containing deployment Scenarios] - -@@ -47,6 +47,7 @@ OPTIONS: - -p Pod-name - -s Deploy-scenario short-name/base-file-name - -S Storage dir for VM images -+ -C Deploy cache dir for storing image artifacts - -T Timeout, in minutes, for the deploy. - -i iso url - -@@ -79,6 +80,7 @@ Input parameters to the build script is: - or a deployment short-name as defined by scenario.yaml in the deployment - scenario path. - -S Storage dir for VM images, default is fuel/deploy/images -+-C Deploy cache dir for bootstrap and target image artifacts, optional - -T Timeout, in minutes, for the deploy. It defaults to using the DEPLOY_TIMEOUT - environment variable when defined, or to the default in deploy.py otherwise - -i .iso image to be deployed (needs to be provided in a URI -@@ -116,6 +118,7 @@ FUEL_CREATION_ONLY='' - NO_DEPLOY_ENVIRONMENT='' - STORAGE_DIR='' - DRY_RUN=0 -+DEPLOY_CACHE_DIR='' - if ! [ -z $DEPLOY_TIMEOUT ]; then - DEPLOY_TIMEOUT="-dt $DEPLOY_TIMEOUT" - else -@@ -128,7 +131,7 @@ fi - ############################################################################ - # BEGIN of main - # --while getopts "b:B:dfFHl:L:p:s:S:T:i:he" OPTION -+while getopts "b:B:dfFHl:L:p:s:S:C:T:i:he" OPTION - do - case $OPTION in - b) -@@ -179,6 +182,9 @@ do - STORAGE_DIR="-s ${OPTARG}" - fi - ;; -+ C) -+ DEPLOY_CACHE_DIR="-dc ${OPTARG}" -+ ;; - T) - DEPLOY_TIMEOUT="-dt ${OPTARG}" - ;; -@@ -243,8 +249,8 @@ if [ $DRY_RUN -eq 0 ]; then - ISO=${SCRIPT_PATH}/ISO/image.iso - fi - # Start deployment -- echo "python deploy.py $DEPLOY_LOG $STORAGE_DIR $PXE_BRIDGE $USE_EXISTING_FUEL $FUEL_CREATION_ONLY $NO_HEALTH_CHECK $NO_DEPLOY_ENVIRONMENT -dea ${SCRIPT_PATH}/config/dea.yaml -dha ${SCRIPT_PATH}/config/dha.yaml -iso $ISO $DEPLOY_TIMEOUT" -- python deploy.py $DEPLOY_LOG $STORAGE_DIR $PXE_BRIDGE $USE_EXISTING_FUEL $FUEL_CREATION_ONLY $NO_HEALTH_CHECK $NO_DEPLOY_ENVIRONMENT -dea ${SCRIPT_PATH}/config/dea.yaml -dha ${SCRIPT_PATH}/config/dha.yaml -iso $ISO $DEPLOY_TIMEOUT -+ echo "python deploy.py $DEPLOY_LOG $STORAGE_DIR $PXE_BRIDGE $USE_EXISTING_FUEL $FUEL_CREATION_ONLY $NO_HEALTH_CHECK $NO_DEPLOY_ENVIRONMENT -dea ${SCRIPT_PATH}/config/dea.yaml -dha ${SCRIPT_PATH}/config/dha.yaml -iso $ISO $DEPLOY_TIMEOUT $DEPLOY_CACHE_DIR" -+ python deploy.py $DEPLOY_LOG $STORAGE_DIR $PXE_BRIDGE $USE_EXISTING_FUEL $FUEL_CREATION_ONLY $NO_HEALTH_CHECK $NO_DEPLOY_ENVIRONMENT -dea ${SCRIPT_PATH}/config/dea.yaml -dha ${SCRIPT_PATH}/config/dha.yaml -iso $ISO $DEPLOY_TIMEOUT $DEPLOY_CACHE_DIR - fi - popd > /dev/null - -diff --git a/deploy/cloud/configure_settings.py b/deploy/cloud/configure_settings.py -index b60a60f..4e007e1 100644 ---- a/deploy/cloud/configure_settings.py -+++ b/deploy/cloud/configure_settings.py -@@ -71,5 +71,9 @@ class ConfigureSettings(object): - settings['editable'][plugin]['metadata']['chosen_id'] = orig_dea['editable'][plugin]['metadata']['chosen_id'] - settings['editable'][plugin]['metadata']['versions'][0]['metadata']['plugin_id'] = orig_dea['editable'][plugin]['metadata']['versions'][0]['metadata']['plugin_id'] - -+ # deploy-cache req: pass master id_rsa.pub as authorized key -+ with io.open('/root/.ssh/id_rsa.pub', 'r') as pkey: -+ settings['editable']['operator_user']['authkeys']['value'] = pkey.read() -+ - with io.open(settings_yaml, 'w') as stream: - yaml.dump(settings, stream, default_flow_style=False) -diff --git a/deploy/cloud/deployment.py b/deploy/cloud/deployment.py -index 4329a4c..a84d46c 100644 ---- a/deploy/cloud/deployment.py -+++ b/deploy/cloud/deployment.py -@@ -19,6 +19,8 @@ from common import ( - log, - ) - -+from deploy_cache import DeployCache -+ - SEARCH_TEXT = '(err)' - LOG_FILE = '/var/log/puppet.log' - GREP_LINES_OF_LEADING_CONTEXT = 100 -@@ -51,6 +53,14 @@ class Deployment(object): - self.pattern = re.compile( - '\d\d\d\d-\d\d-\d\d\s\d\d:\d\d:\d\d') - -+ def deploy_cache_install_targetimages(self): -+ log('Using target images from deploy cache') -+ DeployCache.install_targetimages_for_env(self.env_id) -+ -+ def deploy_cache_extract_targetimages(self): -+ log('Collecting Fuel target image files for deploy cache') -+ DeployCache.extract_targetimages_from_env(self.env_id) -+ - def collect_error_logs(self): - for node_id, roles_blade in self.node_id_roles_dict.iteritems(): - log_list = [] -@@ -112,6 +122,7 @@ class Deployment(object): - start = time.time() - - log('Starting deployment of environment %s' % self.env_id) -+ self.deploy_cache_install_targetimages() - deploy_id = None - ready = False - timeout = False -@@ -144,6 +155,7 @@ class Deployment(object): - err('Deployment timed out, environment %s is not operational, ' - 'snapshot will not be performed' - % self.env_id) -+ self.deploy_cache_extract_targetimages() - if ready: - log('Environment %s successfully deployed' - % self.env_id) -diff --git a/deploy/deploy.py b/deploy/deploy.py -index 7648baf..ee3cb7a 100755 ---- a/deploy/deploy.py -+++ b/deploy/deploy.py -@@ -22,6 +22,7 @@ from dea import DeploymentEnvironmentAdapter - from dha import DeploymentHardwareAdapter - from install_fuel_master import InstallFuelMaster - from deploy_env import CloudDeploy -+from deploy_cache import DeployCache - from execution_environment import ExecutionEnvironment - - from common import ( -@@ -61,7 +62,8 @@ class AutoDeploy(object): - def __init__(self, no_fuel, fuel_only, no_health_check, cleanup_only, - cleanup, storage_dir, pxe_bridge, iso_file, dea_file, - dha_file, fuel_plugins_dir, fuel_plugins_conf_dir, -- no_plugins, deploy_timeout, no_deploy_environment, deploy_log): -+ no_plugins, deploy_cache_dir, deploy_timeout, -+ no_deploy_environment, deploy_log): - self.no_fuel = no_fuel - self.fuel_only = fuel_only - self.no_health_check = no_health_check -@@ -75,6 +77,7 @@ class AutoDeploy(object): - self.fuel_plugins_dir = fuel_plugins_dir - self.fuel_plugins_conf_dir = fuel_plugins_conf_dir - self.no_plugins = no_plugins -+ self.deploy_cache_dir = deploy_cache_dir - self.deploy_timeout = deploy_timeout - self.no_deploy_environment = no_deploy_environment - self.deploy_log = deploy_log -@@ -116,7 +119,7 @@ class AutoDeploy(object): - self.fuel_username, self.fuel_password, - self.dea_file, self.fuel_plugins_conf_dir, - WORK_DIR, self.no_health_check, -- self.deploy_timeout, -+ self.deploy_cache_dir, self.deploy_timeout, - self.no_deploy_environment, self.deploy_log) - with old_dep.ssh: - old_dep.check_previous_installation() -@@ -128,6 +131,7 @@ class AutoDeploy(object): - self.fuel_conf['ip'], self.fuel_username, - self.fuel_password, self.fuel_node_id, - self.iso_file, WORK_DIR, -+ self.deploy_cache_dir, - self.fuel_plugins_dir, self.no_plugins) - fuel.install() - -@@ -136,6 +140,7 @@ class AutoDeploy(object): - tmp_new_dir = '%s/newiso' % self.tmp_dir - try: - self.copy(tmp_orig_dir, tmp_new_dir) -+ self.deploy_cache_fingerprints(tmp_new_dir) - self.patch(tmp_new_dir, new_iso) - except Exception as e: - exec_cmd('fusermount -u %s' % tmp_orig_dir, False) -@@ -156,6 +161,12 @@ class AutoDeploy(object): - delete(tmp_orig_dir) - exec_cmd('chmod -R 755 %s' % tmp_new_dir) - -+ def deploy_cache_fingerprints(self, tmp_new_dir): -+ if self.deploy_cache_dir: -+ log('Deploy cache: Collecting fingerprints...') -+ deploy_cache = DeployCache(self.deploy_cache_dir) -+ deploy_cache.do_fingerprints(tmp_new_dir, self.dea_file) -+ - def patch(self, tmp_new_dir, new_iso): - log('Patching...') - patch_dir = '%s/%s' % (CWD, PATCH_DIR) -@@ -218,7 +229,8 @@ class AutoDeploy(object): - dep = CloudDeploy(self.dea, self.dha, self.fuel_conf['ip'], - self.fuel_username, self.fuel_password, - self.dea_file, self.fuel_plugins_conf_dir, -- WORK_DIR, self.no_health_check, self.deploy_timeout, -+ WORK_DIR, self.no_health_check, -+ self.deploy_cache_dir, self.deploy_timeout, - self.no_deploy_environment, self.deploy_log) - return dep.deploy() - -@@ -343,6 +355,8 @@ def parse_arguments(): - help='Fuel Plugins Configuration directory') - parser.add_argument('-np', dest='no_plugins', action='store_true', - default=False, help='Do not install Fuel Plugins') -+ parser.add_argument('-dc', dest='deploy_cache_dir', action='store', -+ help='Deploy Cache Directory') - parser.add_argument('-dt', dest='deploy_timeout', action='store', - default=240, help='Deployment timeout (in minutes) ' - '[default: 240]') -@@ -376,6 +390,10 @@ def parse_arguments(): - for bridge in args.pxe_bridge: - check_bridge(bridge, args.dha_file) - -+ if args.deploy_cache_dir: -+ log('Using deploy cache directory: %s' % args.deploy_cache_dir) -+ create_dir_if_not_exists(args.deploy_cache_dir) -+ - - kwargs = {'no_fuel': args.no_fuel, 'fuel_only': args.fuel_only, - 'no_health_check': args.no_health_check, -@@ -386,6 +404,7 @@ def parse_arguments(): - 'fuel_plugins_dir': args.fuel_plugins_dir, - 'fuel_plugins_conf_dir': args.fuel_plugins_conf_dir, - 'no_plugins': args.no_plugins, -+ 'deploy_cache_dir': args.deploy_cache_dir, - 'deploy_timeout': args.deploy_timeout, - 'no_deploy_environment': args.no_deploy_environment, - 'deploy_log': args.deploy_log} -diff --git a/deploy/deploy_cache.py b/deploy/deploy_cache.py -new file mode 100644 -index 0000000..30bfe30 ---- /dev/null -+++ b/deploy/deploy_cache.py -@@ -0,0 +1,314 @@ -+############################################################################### -+# Copyright (c) 2016 Enea AB and others. -+# Alexandru.Avadanii@enea.com -+# All rights reserved. This program and the accompanying materials -+# are made available under the terms of the Apache License, Version 2.0 -+# which accompanies this distribution, and is available at -+# http://www.apache.org/licenses/LICENSE-2.0 -+############################################################################### -+ -+import glob -+import hashlib -+import io -+import json -+import os -+import shutil -+import yaml -+ -+from common import ( -+ exec_cmd, -+ log, -+) -+ -+############################################################################### -+# Deploy Cache Flow Overview -+############################################################################### -+# 1. do_fingerprints -+# Can be called as soon as a Fuel Master ISO chroot is available. -+# This will gather all required information for uniquely identifying the -+# objects in cache (bootstraps, targetimages). -+# 2. inject_cache -+# Can be called as soon as we have a steady SSH connection to the Fuel -+# Master node. It will inject cached artifacts over SSH, for later install. -+# 3. (external, async) install cached bootstrap instead of building a new one -+# /sbin/bootstrap_admin_node.sh will check for cached bootstrap images -+# (with id_rsa, id_rsa.pub attached) and will install those via -+# $ fuel-bootstrap import opfnv-bootstraps-cache.tar -+# 4. install_targetimages_for_env -+# Should be called before cloud deploy is started, to install env-generic -+# 'env_X_...' cached images for the current environment ID. -+# Static method, to be used on the remote Fuel Master node; does not require -+# access to the deploy cache, it only moves around some local files. -+# 5. extract_targetimages_from_env -+# Should be called at env deploy finish, to prepare artifacts for caching. -+# Static method, same observations as above apply. -+# 6. collect_artifacts -+# Call last, to collect all artifacts. -+############################################################################### -+ -+############################################################################### -+# Deploy cache artifacts: -+# - id_rsa -+# - bootstrap image (Ubuntu) -+# - environment target image (Ubuntu) -+############################################################################### -+# Cache fingerprint covers: -+# - bootstrap: -+# - local mirror contents -+# - FIXME(armband): [disabled] package list (old fuel_bootstrap_cli.yaml) -+# - target image: -+# - local mirror contents -+# - package list (determined from DEA) -+############################################################################### -+# WARN: Cache fingerprint does NOT yet cover: -+# - image_data (always assume the default /boot, /); -+# - output_dir (always assume the default /var/www/nailgun/targetimages; -+# - codename (always assume the default, currently 'trusty'); -+# - extra_dirs: /usr/share/fuel_bootstrap_cli/files/trusty -+# - root_ssh_authorized_file, inluding the contents of /root/.ssh/id_rsa.pub -+# - Auxiliary repo .../mitaka-9.0/ubuntu/auxiliary -+# If the above change without triggering a cache miss, try clearing the cache. -+############################################################################### -+# WARN: Bootstrap caching implies RSA keypair to be reused! -+############################################################################### -+ -+# Local mirrros will be used on Fuel Master for both bootstrap and target image -+# build, from `http://127.0.0.1:8080/...` or `http://10.20.0.2:8080/...`: -+# - MOS .../mitaka-9.0/ubuntu/x86_64 -+# - Ubuntu .../mirrors/ubuntu/ -+# All these reside on Fuel Master at local path: -+NAILGUN_PATH = '/var/www/nailgun/' -+ -+# Artifact names (corresponding to nailgun subdirs) -+MIRRORS = 'mirrors' -+BOOTSTRAPS = 'bootstraps' -+TARGETIMAGES = 'targetimages' -+ -+# Info for collecting RSA keypair -+RSA_KEYPAIR_PATH = '/root/.ssh' -+RSA_KEYPAIR_FILES = ['id_rsa', 'id_rsa.pub'] -+ -+# Relative path for collecting the active bootstrap image(s) after env deploy -+NAILGUN_ACT_BOOTSTRAP_SUBDIR = '%s/active_bootstrap' % BOOTSTRAPS -+ -+# Relative path for collecting target image(s) for deployed enviroment -+NAILGUN_TIMAGES_SUBDIR = TARGETIMAGES -+ -+# FIXME(armband): re-include package list (old fuel_bootstrap_cli.yaml) -+# ISO_BOOTSTRAP_CLI_YAML = '/opnfv/fuel_bootstrap_cli.yaml' -+ -+# OPNFV Deploy Cache path on Fuel Master, where artifacts will be injected -+REMOTE_CACHE_PATH = '/var/cache/opnfv' -+ -+# OPNFV Bootstrap Cache tar archive name, to be used by bootstrap_admin_node.sh -+BOOTSTRAP_ARCHIVE = 'opnfv-bootstraps-cache.tar' -+ -+# Env-ID indep prefix -+ENVX = 'env_X_' -+ -+class DeployCache(object): -+ """OPNFV Deploy Cache - managed storage for cacheable artifacts""" -+ -+ def __init__(self, cache_dir, -+ fingerprints_yaml='deploy_cache_fingerprints.yaml'): -+ self.cache_dir = cache_dir -+ self.fingerprints_yaml = fingerprints_yaml -+ self.fingerprints = {BOOTSTRAPS: None, -+ MIRRORS: None, -+ TARGETIMAGES: None} -+ -+ def __load_fingerprints(self): -+ """Load deploy cache yaml config holding fingerprints""" -+ if os.path.isfile(self.fingerprints_yaml): -+ cache_fingerprints = open(self.fingerprints_yaml).read() -+ self.fingerprints = yaml.load(cache_fingerprints) -+ -+ def __save_fingerprints(self): -+ """Update deploy cache yaml config holding fingerprints""" -+ with open(self.fingerprints_yaml, 'w') as outfile: -+ outfile.write(yaml.safe_dump(self.fingerprints, -+ default_flow_style=False)) -+ -+ def __fingerprint_mirrors(self, chroot_path): -+ """Collect repo mirror fingerprints""" -+ deb_packages = list() -+ # Scan ISO for deb files (MOS mirror + Ubuntu mirror, no plugins) -+ for repo_dir in ['ubuntu', 'opnfv/nailgun/mirrors/ubuntu']: -+ for _, _, files in os.walk(os.path.join(chroot_path, repo_dir)): -+ for fdeb in files: -+ if fdeb.endswith(".deb"): -+ deb_packages.append(fdeb) -+ sorted_debs = json.dumps(deb_packages, sort_keys=True) -+ self.fingerprints[MIRRORS] = hashlib.sha1(sorted_debs).hexdigest() -+ -+ def __fingerprint_bootstrap(self, chroot_path): -+ """Collect bootstrap image metadata fingerprints""" -+ # FIXME(armband): include 'extra_dirs' contents -+ sorted_data = '' -+ # FIXME(armband): re-include package list (old fuel_bootstrap_cli.yaml) -+ # cli_yaml_path = os.path.join(chroot_path, ISO_BOOTSTRAP_CLI_YAML[1:]) -+ # bootstrap_cli_yaml = open(cli_yaml_path).read() -+ # bootstrap_data = yaml.load(bootstrap_cli_yaml) -+ # sorted_data = json.dumps(bootstrap_data, sort_keys=True) -+ self.fingerprints[BOOTSTRAPS] = hashlib.sha1(sorted_data).hexdigest() -+ -+ def __fingerprint_target(self, dea_file): -+ """Collect target image metadata fingerprints""" -+ # FIXME(armband): include 'image_data', 'codename', 'output' -+ with io.open(dea_file) as stream: -+ dea = yaml.load(stream) -+ editable = dea['settings']['editable'] -+ target_data = {'packages': editable['provision']['packages'], -+ 'repos': editable['repo_setup']['repos']} -+ s_data = json.dumps(target_data, sort_keys=True) -+ self.fingerprints[TARGETIMAGES] = hashlib.sha1(s_data).hexdigest() -+ -+ def do_fingerprints(self, chroot_path, dea_file): -+ """Collect SHA1 fingerprints based on chroot contents, DEA settings""" -+ try: -+ self.__load_fingerprints() -+ self.__fingerprint_mirrors(chroot_path) -+ self.__fingerprint_bootstrap(chroot_path) -+ self.__fingerprint_target(dea_file) -+ self.__save_fingerprints() -+ except Exception as ex: -+ log('Failed to get cache fingerprint: %s' % str(ex)) -+ -+ def __lookup_cache(self, sha): -+ """Search for object in cache based on SHA fingerprint""" -+ cache_sha_dir = os.path.join(self.cache_dir, sha) -+ if not os.path.isdir(cache_sha_dir) or not os.listdir(cache_sha_dir): -+ return None -+ return cache_sha_dir -+ -+ def __inject_cache_dir(self, ssh, sha, artifact): -+ """Stage cached object (dir) in Fuel Master OPNFV local cache""" -+ local_path = self.__lookup_cache(sha) -+ if local_path: -+ remote_path = os.path.join(REMOTE_CACHE_PATH, artifact) -+ with ssh: -+ ssh.exec_cmd('mkdir -p %s' % remote_path) -+ for cachedfile in glob.glob('%s/*' % local_path): -+ ssh.scp_put(cachedfile, remote_path) -+ return local_path -+ -+ def __mix_fingerprints(self, f1, f2): -+ """Compute composite fingerprint""" -+ if self.fingerprints[f1] is None or self.fingerprints[f2] is None: -+ return None -+ return hashlib.sha1('%s%s' % -+ (self.fingerprints[f1], self.fingerprints[f2])).hexdigest() -+ -+ def inject_cache(self, ssh): -+ """Lookup artifacts in cache and inject them over SSH/SCP into Fuel""" -+ try: -+ self.__load_fingerprints() -+ for artifact in [BOOTSTRAPS, TARGETIMAGES]: -+ sha = self.__mix_fingerprints(MIRRORS, artifact) -+ if sha is None: -+ log('Missing fingerprint for: %s' % artifact) -+ continue -+ if not self.__inject_cache_dir(ssh, sha, artifact): -+ log('SHA1 not in cache: %s (%s)' % (str(sha), artifact)) -+ else: -+ log('SHA1 injected: %s (%s)' % (str(sha), artifact)) -+ except Exception as ex: -+ log('Failed to inject cached artifacts into Fuel: %s' % str(ex)) -+ -+ def __extract_bootstraps(self, ssh, cache_sha_dir): -+ """Collect bootstrap artifacts from Fuel over SSH/SCP""" -+ remote_tar = os.path.join(REMOTE_CACHE_PATH, BOOTSTRAP_ARCHIVE) -+ local_tar = os.path.join(cache_sha_dir, BOOTSTRAP_ARCHIVE) -+ with ssh: -+ for k in RSA_KEYPAIR_FILES: -+ ssh.scp_get(os.path.join(RSA_KEYPAIR_PATH, k), -+ local=os.path.join(cache_sha_dir, k)) -+ ssh.exec_cmd('mkdir -p %s && cd %s && tar cf %s *' % -+ (REMOTE_CACHE_PATH, -+ os.path.join(NAILGUN_PATH, NAILGUN_ACT_BOOTSTRAP_SUBDIR), -+ remote_tar)) -+ ssh.scp_get(remote_tar, local=local_tar) -+ ssh.exec_cmd('rm -f %s' % remote_tar) -+ -+ def __extract_targetimages(self, ssh, cache_sha_dir): -+ """Collect target image artifacts from Fuel over SSH/SCP""" -+ cti_path = os.path.join(REMOTE_CACHE_PATH, TARGETIMAGES) -+ with ssh: -+ ssh.scp_get('%s/%s*' % (cti_path, ENVX), local=cache_sha_dir) -+ -+ def collect_artifacts(self, ssh): -+ """Collect artifacts from Fuel over SSH/SCP and add them to cache""" -+ try: -+ self.__load_fingerprints() -+ for artifact, func in { -+ BOOTSTRAPS: self.__extract_bootstraps, -+ TARGETIMAGES: self.__extract_targetimages -+ }.iteritems(): -+ sha = self.__mix_fingerprints(MIRRORS, artifact) -+ if sha is None: -+ log('WARN: Skip caching, NO fingerprint: %s' % artifact) -+ continue -+ local_path = self.__lookup_cache(sha) -+ if local_path: -+ log('SHA1 already in cache: %s (%s)' % (str(sha), artifact)) -+ else: -+ log('New cache SHA1: %s (%s)' % (str(sha), artifact)) -+ cache_sha_dir = os.path.join(self.cache_dir, sha) -+ exec_cmd('mkdir -p %s' % cache_sha_dir) -+ func(ssh, cache_sha_dir) -+ except Exception as ex: -+ log('Failed to extract artifacts from Fuel: %s' % str(ex)) -+ -+ @staticmethod -+ def extract_targetimages_from_env(env_id): -+ """Prepare targetimages from env ID for storage in deploy cache -+ -+ NOTE: This method should be executed locally ON the Fuel Master node. -+ WARN: This method overwrites targetimages cache on Fuel Master node. -+ """ -+ env_n = 'env_%s_' % str(env_id) -+ cti_path = os.path.join(REMOTE_CACHE_PATH, TARGETIMAGES) -+ ti_path = os.path.join(NAILGUN_PATH, NAILGUN_TIMAGES_SUBDIR) -+ try: -+ exec_cmd('rm -rf %s && mkdir -p %s' % (cti_path, cti_path)) -+ for root, _, files in os.walk(ti_path): -+ for tif in files: -+ if tif.startswith(env_n): -+ src = os.path.join(root, tif) -+ dest = os.path.join(cti_path, tif.replace(env_n, ENVX)) -+ if tif.endswith('.yaml'): -+ shutil.copy(src, dest) -+ exec_cmd('sed -i "s|%s|%s|g" %s' % -+ (env_n, ENVX, dest)) -+ else: -+ os.link(src, dest) -+ except Exception as ex: -+ log('Failed to extract targetimages artifacts from env %s: %s' % -+ (str(env_id), str(ex))) -+ -+ @staticmethod -+ def install_targetimages_for_env(env_id): -+ """Install targetimages artifacts for a specific env ID -+ -+ NOTE: This method should be executed locally ON the Fuel Master node. -+ """ -+ env_n = 'env_%s_' % str(env_id) -+ cti_path = os.path.join(REMOTE_CACHE_PATH, TARGETIMAGES) -+ ti_path = os.path.join(NAILGUN_PATH, NAILGUN_TIMAGES_SUBDIR) -+ if not os.path.isdir(cti_path): -+ log('%s cache dir not found: %s' % (TARGETIMAGES, cti_path)) -+ else: -+ try: -+ for root, _, files in os.walk(cti_path): -+ for tif in files: -+ src = os.path.join(root, tif) -+ dest = os.path.join(ti_path, tif.replace(ENVX, env_n)) -+ if tif.endswith('.yaml'): -+ shutil.copy(src, dest) -+ exec_cmd('sed -i "s|%s|%s|g" %s' % -+ (ENVX, env_n, dest)) -+ else: -+ os.link(src, dest) -+ except Exception as ex: -+ log('Failed to install targetimages for env %s: %s' % -+ (str(env_id), str(ex))) -diff --git a/deploy/deploy_env.py b/deploy/deploy_env.py -index d374cce..445070a 100644 ---- a/deploy/deploy_env.py -+++ b/deploy/deploy_env.py -@@ -15,6 +15,7 @@ import glob - import time - import shutil - -+from deploy_cache import DeployCache - from ssh_client import SSHClient - - from common import ( -@@ -35,7 +36,8 @@ class CloudDeploy(object): - - def __init__(self, dea, dha, fuel_ip, fuel_username, fuel_password, - dea_file, fuel_plugins_conf_dir, work_dir, no_health_check, -- deploy_timeout, no_deploy_environment, deploy_log): -+ deploy_cache_dir, deploy_timeout, -+ no_deploy_environment, deploy_log): - self.dea = dea - self.dha = dha - self.fuel_ip = fuel_ip -@@ -49,6 +51,8 @@ class CloudDeploy(object): - self.fuel_plugins_conf_dir = fuel_plugins_conf_dir - self.work_dir = work_dir - self.no_health_check = no_health_check -+ self.deploy_cache = ( DeployCache(deploy_cache_dir) -+ if deploy_cache_dir else None ) - self.deploy_timeout = deploy_timeout - self.no_deploy_environment = no_deploy_environment - self.deploy_log = deploy_log -@@ -82,9 +86,14 @@ class CloudDeploy(object): - self.work_dir, os.path.basename(self.dea_file))) - s.scp_put('%s/common.py' % self.file_dir, self.work_dir) - s.scp_put('%s/dea.py' % self.file_dir, self.work_dir) -+ s.scp_put('%s/deploy_cache.py' % self.file_dir, self.work_dir) - for f in glob.glob('%s/cloud/*' % self.file_dir): - s.scp_put(f, self.work_dir) - -+ def deploy_cache_collect_artifacts(self): -+ if self.deploy_cache: -+ self.deploy_cache.collect_artifacts(self.ssh) -+ - def power_off_nodes(self): - for node_id in self.node_ids: - self.dha.node_power_off(node_id) -@@ -281,4 +290,6 @@ class CloudDeploy(object): - - self.get_put_deploy_log() - -+ self.deploy_cache_collect_artifacts() -+ - return rc -diff --git a/deploy/install_fuel_master.py b/deploy/install_fuel_master.py -index b731c6b..83d31fb 100644 ---- a/deploy/install_fuel_master.py -+++ b/deploy/install_fuel_master.py -@@ -10,6 +10,7 @@ - import time - import os - import glob -+from deploy_cache import DeployCache - from ssh_client import SSHClient - from dha_adapters.libvirt_adapter import LibvirtAdapter - -@@ -32,7 +33,7 @@ class InstallFuelMaster(object): - - def __init__(self, dea_file, dha_file, fuel_ip, fuel_username, - fuel_password, fuel_node_id, iso_file, work_dir, -- fuel_plugins_dir, no_plugins): -+ deploy_cache_dir, fuel_plugins_dir, no_plugins): - self.dea_file = dea_file - self.dha = LibvirtAdapter(dha_file) - self.fuel_ip = fuel_ip -@@ -42,6 +43,8 @@ class InstallFuelMaster(object): - self.iso_file = iso_file - self.iso_dir = os.path.dirname(self.iso_file) - self.work_dir = work_dir -+ self.deploy_cache = ( DeployCache(deploy_cache_dir) -+ if deploy_cache_dir else None ) - self.fuel_plugins_dir = fuel_plugins_dir - self.no_plugins = no_plugins - self.file_dir = os.path.dirname(os.path.realpath(__file__)) -@@ -83,6 +86,10 @@ class InstallFuelMaster(object): - log('Wait until Fuel menu is up') - fuel_menu_pid = self.wait_until_fuel_menu_up() - -+ if self.deploy_cache: -+ log('Deploy cache: Injecting bootstraps and targetimages') -+ self.deploy_cache.inject_cache(self.ssh) -+ - log('Inject our own astute.yaml and fuel_bootstrap_cli.yaml settings') - self.inject_own_astute_and_bootstrap_yaml() - |