Preparing the experimental branch for improved Danube CI/CD experimentsexperimental

Fast forwarded to commit:cf93e6ee11c96de090b04196cc96b4a6b0948928

Change-Id: I13d10d870e8ffc7317ab03f8810592d5b2205875
Signed-off-by: Jonas Bjurel <jonas.bjurel@ericsson.com>

7 files changed, 529 insertions, 0 deletions

diff --git a/deploy/dha_adapters/__init__.py b/deploy/dha_adapters/__init__.py
new file mode 100644
index 000000000..fb73157f9
--- /dev/null
+++ b/deploy/dha_adapters/__init__.py
@@ -0,0 +1,8 @@
+###############################################################################
+# Copyright (c) 2015 Ericsson AB and others.
+# szilard.cserey@ericsson.com
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+###############################################################################
diff --git a/deploy/dha_adapters/amt_adapter.py b/deploy/dha_adapters/amt_adapter.py
new file mode 100644
index 000000000..02025b9dd
--- /dev/null
+++ b/deploy/dha_adapters/amt_adapter.py
@@ -0,0 +1,100 @@
+###############################################################################
+# Copyright (c) 2015 Ericsson AB and others.
+# liyi.meng@ericsson.com
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+###############################################################################
+
+
+from hardware_adapter import HardwareAdapter
+
+from common import (
+    log,
+    exec_cmd,
+    err,
+)
+
+
+'''
+This is hardware adapter for Intel AMT based system. It use amttool to interact
+ with the targeting node. It dose not support vPro v9. if the targeting system
+ is v9 or later, we need to consider a new adpater with using lib
+ like https://github.com/sdague/amt
+'''
+class AmtAdapter(HardwareAdapter):
+
+    def __init__(self, yaml_path):
+        super(AmtAdapter, self).__init__(yaml_path)
+        #amttool dose not allow you change bios setting permanently.
+        # so we have to make a workaround to make it IPMI like.
+        self.boot_order = {}
+
+    def node_get_boot_dev(self, node_id):
+        if node_id in self.boot_order:
+            dev = self.boot_order[node_id][0]
+            if dev == 'pxe':
+                return 'PXE-boot'
+            elif dev == 'iso':
+                return 'cd-boot'
+            elif dev == 'disk':
+                return 'HD-boot'
+        else:
+            return 'HD-boot'
+
+    def get_access_info(self, node_id):
+        ip = self.get_node_property(node_id, 'amtIp')
+        username = self.get_node_property(node_id, 'amtUser')
+        password = self.get_node_property(node_id, 'amtPass')
+        return ip, username, password
+
+    def amt_cmd(self, node_id):
+        ip, username, password = self.get_access_info(node_id)
+        # We first Setup password for amttool, then use ping to wake up the node over LAN
+        cmd = 'export AMT_PASSWORD={0};' \
+              'ping {1} -W 5 -c 1 -q;' \
+              'yes | amttool {1}'.format(password, ip)
+        return cmd
+
+    def get_node_pxe_mac(self, node_id):
+        mac_list = []
+        mac_list.append(self.get_node_property(node_id, 'pxeMac').lower())
+        return mac_list
+
+    def node_power_on(self, node_id):
+        log('Power ON Node %s' % node_id)
+        cmd_prefix = self.amt_cmd(node_id)
+        resp, ret = exec_cmd('{0} info'.format(cmd_prefix), check=False)
+        if 'Powerstate:   S0' not in resp:
+            dev = self.node_get_boot_dev(node_id)
+            resp, ret = exec_cmd('{0} powerup {1}'.format(cmd_prefix, dev), check=False)
+            if 'pt_status: success' not in resp:
+                err('Could Not Power ON Node %s' % node_id)
+
+    def node_power_off(self, node_id):
+        log('Power OFF Node %s' % node_id)
+        cmd_prefix = self.amt_cmd(node_id)
+        resp, ret = exec_cmd('{0} info'.format(cmd_prefix), check=False)
+        if "Powerstate:   S0" in resp:
+            resp, ret = exec_cmd('{0} powerdown'.format(cmd_prefix), check=False)
+            if 'pt_status: success' not in resp:
+                err('Could Not Power OFF Node %s' % node_id)
+
+    def node_reset(self, node_id):
+        log('RESET Node %s' % node_id)
+        cmd_prefix = self.amt_cmd(node_id)
+        dev = self.node_get_boot_dev(node_id)
+        resp, ret = exec_cmd('{0} info'.format(cmd_prefix), check=False)
+        if 'Powerstate:   S0' in resp:
+            resp, ret = exec_cmd('{0} reset {1}'.format(cmd_prefix, dev), check=False)
+            if 'pt_status: success' not in resp:
+                err('Could Not RESET Node %s' % node_id)
+        else:
+            err('Cannot RESET Node %s because it\'s not Active, state: %s'
+                % (node_id, resp))
+
+    def node_set_boot_order(self, node_id, boot_order_list):
+        log('Set boot order %s on Node %s' % (boot_order_list, node_id))
+        self.boot_order[node_id] = boot_order_list
+
diff --git a/deploy/dha_adapters/hardware_adapter.py b/deploy/dha_adapters/hardware_adapter.py
new file mode 100644
index 000000000..aa59581ee
--- /dev/null
+++ b/deploy/dha_adapters/hardware_adapter.py
@@ -0,0 +1,65 @@
+###############################################################################
+# Copyright (c) 2015 Ericsson AB and others.
+# szilard.cserey@ericsson.com
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+###############################################################################
+
+
+import yaml
+import io
+
+
+class HardwareAdapter(object):
+
+    def __init__(self, yaml_path):
+        self.dha_struct = None
+        self.parse_yaml(yaml_path)
+
+    def parse_yaml(self, yaml_path):
+        with io.open(yaml_path) as yaml_file:
+            self.dha_struct = yaml.load(yaml_file)
+
+    def get_adapter_type(self):
+        return self.dha_struct['adapter']
+
+    def get_all_node_ids(self):
+        node_ids = []
+        for node in self.dha_struct['nodes']:
+            node_ids.append(node['id'])
+        node_ids.sort()
+        return node_ids
+
+    def get_fuel_node_id(self):
+        for node in self.dha_struct['nodes']:
+            if 'isFuel' in node and node['isFuel']:
+                return node['id']
+
+    def get_node_ids(self):
+        node_ids = []
+        fuel_node_id = self.get_fuel_node_id()
+        for node in self.dha_struct['nodes']:
+            if node['id'] != fuel_node_id:
+                node_ids.append(node['id'])
+        node_ids.sort()
+        return node_ids
+
+    def get_node_property(self, node_id, property_name):
+        for node in self.dha_struct['nodes']:
+            if node['id'] == node_id and property_name in node:
+                return node[property_name]
+
+    def get_fuel_access(self):
+        for node in self.dha_struct['nodes']:
+            if 'isFuel' in node and node['isFuel']:
+                return node['username'], node['password']
+
+    def get_disks(self):
+        return self.dha_struct['disks']
+
+    def get_vm_definition(self, role):
+        vm_definition = self.dha_struct.get('define_vms')
+        if vm_definition:
+            return vm_definition.get(role)
diff --git a/deploy/dha_adapters/hp_adapter.py b/deploy/dha_adapters/hp_adapter.py
new file mode 100644
index 000000000..6434da868
--- /dev/null
+++ b/deploy/dha_adapters/hp_adapter.py
@@ -0,0 +1,37 @@
+###############################################################################
+# Copyright (c) 2015 Ericsson AB and others.
+# szilard.cserey@ericsson.com
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+###############################################################################
+
+
+from ipmi_adapter import IpmiAdapter
+from ssh_client import SSHClient
+
+from common import (
+    log,
+)
+
+DEV = {'pxe': 'bootsource5',
+       'disk': 'bootsource3',
+       'iso': 'bootsource1'}
+
+ROOT = '/system1/bootconfig1'
+
+
+class HpAdapter(IpmiAdapter):
+
+    def __init__(self, yaml_path):
+        super(HpAdapter, self).__init__(yaml_path)
+
+    def node_set_boot_order(self, node_id, boot_order_list):
+        log('Set boot order %s on Node %s' % (boot_order_list, node_id))
+        ip, username, password, ipmiport = self.get_access_info(node_id)
+        ssh = SSHClient(ip, username, password)
+        with ssh as s:
+            for order, dev in enumerate(boot_order_list):
+                s.exec_cmd('set %s/%s bootorder=%s'
+                           % (ROOT, DEV[dev], order + 1))
diff --git a/deploy/dha_adapters/ipmi_adapter.py b/deploy/dha_adapters/ipmi_adapter.py
new file mode 100644
index 000000000..7cc930554
--- /dev/null
+++ b/deploy/dha_adapters/ipmi_adapter.py
@@ -0,0 +1,114 @@
+###############################################################################
+# Copyright (c) 2015 Ericsson AB and others.
+#           (c) 2016 Enea Software AB
+# szilard.cserey@ericsson.com
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+###############################################################################
+
+
+import time
+from hardware_adapter import HardwareAdapter
+
+from common import (
+    log,
+    exec_cmd,
+    err,
+)
+
+
+class IpmiAdapter(HardwareAdapter):
+
+    def __init__(self, yaml_path, attempts=20, delay=3):
+        super(IpmiAdapter, self).__init__(yaml_path)
+        self.attempts = attempts
+        self.delay = delay
+
+    def get_access_info(self, node_id):
+        ip = self.get_node_property(node_id, 'ipmiIp')
+        username = self.get_node_property(node_id, 'ipmiUser')
+        password = self.get_node_property(node_id, 'ipmiPass')
+        ipmiport = self.get_node_property(node_id, 'ipmiPort')
+        return ip, username, password, ipmiport
+
+    def ipmi_cmd(self, node_id):
+        ip, username, password, ipmiport = self.get_access_info(node_id)
+        cmd = 'ipmitool -I lanplus -A password'
+        cmd += ' -H %s -U %s -P %s' % (ip, username, password)
+        if ipmiport:
+            cmd += ' -p %d' % int(ipmiport)
+        return cmd
+
+    def get_node_pxe_mac(self, node_id):
+        mac_list = []
+        mac_list.append(self.get_node_property(node_id, 'pxeMac').lower())
+        return mac_list
+
+    def node_get_state(self, node_id):
+        state = exec_cmd('%s chassis power status' % self.ipmi_cmd(node_id),
+                         attempts=self.attempts, delay=self.delay,
+                         verbose=True,
+                         mask_args=[8,10])
+        return state
+
+    def _node_power_cmd(self, node_id, cmd):
+        expected = 'Chassis Power is %s' % cmd
+        if self.node_get_state(node_id) == expected:
+            return
+
+        pow_cmd = '%s chassis power %s' % (self.ipmi_cmd(node_id), cmd)
+        exec_cmd(pow_cmd, attempts=self.attempts, delay=self.delay,
+                 verbose=True,
+                 mask_args=[8,10])
+
+        attempts = self.attempts
+        while attempts:
+            time.sleep(self.delay)
+            state = self.node_get_state(node_id)
+            attempts -= 1
+            if state == expected:
+                return
+            elif attempts != 0:
+                # reinforce our will, but allow the command to fail,
+                # we know our message got across once already...
+                exec_cmd(pow_cmd, check=False, mask_args=[8,10])
+
+        err('Could not set chassis %s for node %s' % (cmd, node_id))
+
+    def node_power_on(self, node_id):
+        log('Power ON Node %s' % node_id)
+        self._node_power_cmd(node_id, 'on')
+
+    def node_power_off(self, node_id):
+        log('Power OFF Node %s' % node_id)
+        self._node_power_cmd(node_id, 'off')
+
+    def node_reset(self, node_id):
+        log('RESET Node %s' % node_id)
+        cmd = '%s chassis power reset' % self.ipmi_cmd(node_id)
+        exec_cmd(cmd, attempts=self.attempts, delay=self.delay,
+                 verbose=True,
+                 mask_args=[8,10])
+
+    def node_set_boot_order(self, node_id, boot_order_list):
+        log('Set boot order %s on Node %s' % (boot_order_list, node_id))
+        boot_order_list.reverse()
+        cmd_prefix = self.ipmi_cmd(node_id)
+        for dev in boot_order_list:
+            if dev == 'pxe':
+                exec_cmd('%s chassis bootdev pxe options=persistent'
+                         % cmd_prefix, attempts=self.attempts, delay=self.delay,
+                         verbose=True,
+                         mask_args=[8,10])
+            elif dev == 'iso':
+                exec_cmd('%s chassis bootdev cdrom' % cmd_prefix,
+                         attempts=self.attempts, delay=self.delay,
+                         verbose=True,
+                         mask_args=[8,10])
+            elif dev == 'disk':
+                exec_cmd('%s chassis bootdev disk options=persistent'
+                         % cmd_prefix, attempts=self.attempts, delay=self.delay,
+                         verbose=True,
+                         mask_args=[8,10])
diff --git a/deploy/dha_adapters/libvirt_adapter.py b/deploy/dha_adapters/libvirt_adapter.py
new file mode 100644
index 000000000..466f134ae
--- /dev/null
+++ b/deploy/dha_adapters/libvirt_adapter.py
@@ -0,0 +1,173 @@
+###############################################################################
+# Copyright (c) 2015 Ericsson AB and others.
+# szilard.cserey@ericsson.com
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+###############################################################################
+
+
+from lxml import etree
+from hardware_adapter import HardwareAdapter
+import tempfile
+import os
+
+from common import (
+    log,
+    exec_cmd,
+    err,
+    delete,
+)
+
+DEV = {'pxe': 'network',
+       'disk': 'hd',
+       'iso': 'cdrom'}
+
+VOL_XML_TEMPLATE = '''<volume type='file'>
+  <name>{name}</name>
+  <capacity unit='{unit}'>{size!s}</capacity>
+  <target>
+    <format type='{format_type}'/>
+  </target>
+</volume>'''
+
+DEFAULT_POOL = 'jenkins'
+
+class LibvirtAdapter(HardwareAdapter):
+
+    def __init__(self, yaml_path):
+        super(LibvirtAdapter, self).__init__(yaml_path)
+        self.parser = etree.XMLParser(remove_blank_text=True)
+
+    def node_power_off(self, node_id):
+        vm_name = self.get_node_property(node_id, 'libvirtName')
+        log('Power OFF Node %s' % vm_name)
+        state = exec_cmd('virsh domstate %s' % vm_name)
+        if state == 'running':
+            exec_cmd('virsh destroy %s' % vm_name, False)
+
+    def node_power_on(self, node_id):
+        vm_name = self.get_node_property(node_id, 'libvirtName')
+        log('Power ON Node %s' % vm_name)
+        state = exec_cmd('virsh domstate %s' % vm_name)
+        if state == 'shut off':
+            exec_cmd('virsh start %s' % vm_name)
+
+    def node_reset(self, node_id):
+        vm_name = self.get_node_property(node_id, 'libvirtName')
+        log('Reset Node %s' % vm_name)
+        exec_cmd('virsh reset %s' % vm_name)
+
+    def translate(self, boot_order_list):
+        translated = []
+        for boot_dev in boot_order_list:
+            if boot_dev in DEV:
+                translated.append(DEV[boot_dev])
+            else:
+                err('Boot device %s not recognized' % boot_dev)
+        return translated
+
+    def node_set_boot_order(self, node_id, boot_order_list):
+        boot_order_list = self.translate(boot_order_list)
+        vm_name = self.get_node_property(node_id, 'libvirtName')
+        temp_dir = tempfile.mkdtemp()
+        log('Set boot order %s on Node %s' % (boot_order_list, vm_name))
+        resp = exec_cmd('virsh dumpxml %s' % vm_name)
+        xml_dump = etree.fromstring(resp, self.parser)
+        os = xml_dump.xpath('/domain/os')
+        for o in os:
+            for bootelem in ['boot', 'bootmenu']:
+                boot = o.xpath(bootelem)
+                for b in boot:
+                    o.remove(b)
+            for dev in boot_order_list:
+                b = etree.Element('boot')
+                b.set('dev', dev)
+                o.append(b)
+            bmenu = etree.Element('bootmenu')
+            bmenu.set('enable', 'no')
+            o.append(bmenu)
+        tree = etree.ElementTree(xml_dump)
+        xml_file = temp_dir + '/%s.xml' % vm_name
+        with open(xml_file, 'w') as f:
+            tree.write(f, pretty_print=True, xml_declaration=True)
+        exec_cmd('virsh define %s' % xml_file)
+        delete(temp_dir)
+
+    def node_zero_mbr(self, node_id):
+        vm_name = self.get_node_property(node_id, 'libvirtName')
+        resp = exec_cmd('virsh dumpxml %s' % vm_name)
+        xml_dump = etree.fromstring(resp)
+        disks = xml_dump.xpath('/domain/devices/disk')
+        for disk in disks:
+            if disk.get('device') == 'disk':
+                sources = disk.xpath('source')
+                for source in sources:
+                    disk_file = source.get('file')
+                    disk_size = exec_cmd('qemu-img info '
+                                         '%s |grep \"virtual size:\"'
+                                         % disk_file).split()[2]
+                    delete(disk_file)
+                    exec_cmd('qemu-img create -f qcow2 %s %s' % (disk_file,
+                                                                 disk_size))
+
+    def node_eject_iso(self, node_id):
+        vm_name = self.get_node_property(node_id, 'libvirtName')
+        device = self.get_name_of_device(vm_name, 'cdrom')
+        exec_cmd('virsh change-media %s --eject %s --config --live'
+                 % (vm_name, device), False)
+
+    def node_insert_iso(self, node_id, iso_file):
+        vm_name = self.get_node_property(node_id, 'libvirtName')
+        device = self.get_name_of_device(vm_name, 'cdrom')
+        exec_cmd('virsh change-media %s --insert %s %s'
+                 % (vm_name, device, iso_file))
+
+    def get_node_pxe_mac(self, node_id):
+        mac_list = []
+        vm_name = self.get_node_property(node_id, 'libvirtName')
+        resp = exec_cmd('virsh dumpxml %s' % vm_name)
+        xml_dump = etree.fromstring(resp)
+        interfaces = xml_dump.xpath('/domain/devices/interface')
+        for interface in interfaces:
+            macs = interface.xpath('mac')
+            for mac in macs:
+                mac_list.append(mac.get('address').lower())
+        return mac_list
+
+    def get_name_of_device(self, vm_name, device_type):
+        resp = exec_cmd('virsh dumpxml %s' % vm_name)
+        xml_dump = etree.fromstring(resp)
+        disks = xml_dump.xpath('/domain/devices/disk')
+        for disk in disks:
+            if disk.get('device') == device_type:
+                targets = disk.xpath('target')
+                for target in targets:
+                    device = target.get('dev')
+                    if device:
+                        return device
+
+    def get_virt_net_conf_dir(self):
+        return self.dha_struct['virtNetConfDir']
+
+    def upload_iso(self, iso_file):
+        size = os.path.getsize(iso_file)
+        vol_name = os.path.basename(iso_file)
+        vol_xml = VOL_XML_TEMPLATE.format(name=vol_name, unit='bytes',
+                                          size=size, format_type='raw')
+        fd, fname = tempfile.mkstemp(text=True, suffix='deploy')
+        os.write(fd, vol_xml)
+        os.close(fd)
+
+        log(vol_xml)
+        pool = DEFAULT_POOL # FIXME
+        exec_cmd('virsh vol-create --pool %s %s' % (pool, fname))
+        vol_path = exec_cmd('virsh vol-path --pool %s %s' % (pool, vol_name))
+
+        exec_cmd('virsh vol-upload %s %s' % (vol_path, iso_file),
+                 attempts=5, delay=10, verbose=True)
+
+        delete(fname)
+
+        return vol_path
diff --git a/deploy/dha_adapters/zte_adapter.py b/deploy/dha_adapters/zte_adapter.py
new file mode 100644
index 000000000..da651ca15
--- /dev/null
+++ b/deploy/dha_adapters/zte_adapter.py
@@ -0,0 +1,32 @@
+###############################################################################
+# Copyright (c) 2016 Ericsson AB, ZTE and others.
+# szilard.cserey@ericsson.com
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+###############################################################################
+
+
+import time
+from ipmi_adapter import IpmiAdapter
+
+from common import (
+    log,
+    exec_cmd,
+    err,
+)
+
+
+class ZteAdapter(IpmiAdapter):
+
+    def __init__(self, yaml_path, attempts=100):
+        super(ZteAdapter, self).__init__(yaml_path, attempts)
+
+    def node_reset(self, node_id):
+        log('RESET Node %s' % node_id)
+        cmd = '%s chassis power cycle' % self.ipmi_cmd(node_id)
+        exec_cmd(cmd, attempts=self.attempts, delay=self.delay,
+                 verbose=True,
+                 mask_args=[8,10])
+