Merge "Adds Libvirt Handler"

3 files changed, 678 insertions, 0 deletions

diff --git a/laas-fog/source/api/libvirt_api.py b/laas-fog/source/api/libvirt_api.py
new file mode 100644
index 0000000..4e19736
--- /dev/null
+++ b/laas-fog/source/api/libvirt_api.py
@@ -0,0 +1,331 @@
+"""
+#############################################################################
+#Copyright 2017 Parker Berberian and others                                 #
+#                                                                           #
+#Licensed under the Apache License, Version 2.0 (the "License");            #
+#you may not use this file except in compliance with the License.           #
+#You may obtain a copy of the License at                                    #
+#                                                                           #
+#    http://www.apache.org/licenses/LICENSE-2.0                             #
+#                                                                           #
+#Unless required by applicable law or agreed to in writing, software        #
+#distributed under the License is distributed on an "AS IS" BASIS,          #
+#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.   #
+#See the License for the specific language governing permissions and        #
+#limitations under the License.                                             #
+#############################################################################
+"""
+
+import libvirt
+import time
+import xml.dom
+import xml.dom.minidom
+from domain import Domain
+from network import Network
+from utilities import Utilities
+
+
+class Libvirt:
+    """
+    This class talks to the Libvirt api.
+    Given a config file, this class should create all networks and
+    domains.
+
+    TODO: convert prints to logging and remove uneeded pass statements
+    """
+
+    def __init__(self, hostAddr, net_conf=None, dom_conf=None):
+        """
+        init function
+        hostAddr is the ip address of the host
+        net_conf and dom_conf are the paths
+        to the config files
+        """
+        self.host = hostAddr
+        self.URI = "qemu+ssh://root@"+str(hostAddr)+"/system"
+        self.hypervisor = None
+        self.domains = []
+        self.networks = []
+        self.net_conf = net_conf
+        self.dom_conf = dom_conf
+
+    def setLogger(self, log):
+        """
+        Saves the logger in self.log
+        """
+        self.log = log
+
+    def bootMaster(self):
+        """
+        starts the previously defined master node
+        """
+        for dom in self.domains:
+            if 'master' in dom.name():
+                try:
+                    dom.create()
+                except Exception:
+                    pass
+
+    def bootSlaves(self):
+        """
+        boots every defined vm with 'slave' in its name
+        """
+        for dom in self.domains:
+            if 'slave' in dom.name():
+                try:
+                    dom.create()
+                    self.log.info("Booting %s", dom.name())
+                except Exception:
+                    self.log.exception("%s", "failed to boot domain")
+                time.sleep(5)
+
+    def getMacs(self, domName):
+        """
+        returns a dictionary with a network name
+        mapped to the mac address of the domain on that net
+        """
+        try:
+            dom = self.hypervisor.lookupByName(domName)
+            xmlDesc = dom.XMLDesc(0)
+            parsedXML = xml.dom.minidom.parseString(xmlDesc)
+            interfacesXML = parsedXML.getElementsByTagName('interface')
+            netDict = {}
+            for iface in interfacesXML:
+                src = iface.getElementsByTagName('source')[0]
+                mac = iface.getElementsByTagName('mac')[0]
+                netDict[src] = mac
+            return netDict
+        except Exception:
+            self.log.exception("%s", "Domain not found")
+
+    def defineVM(self, xmlConfig):
+        """
+        Generic method to define a persistent vm with the
+        given config.
+        Assumes that self.hypervisor is already connected.
+        """
+        if self.checkForVM(xmlConfig):
+            vm = self.hypervisor.defineXML(xmlConfig)
+            if vm is None:
+                name = self.getName(xmlConfig)
+                self.log.error("Failed to define vm %s. exiting", name)
+                exit(1)
+            else:
+                self.log.info("Successfully created vm %s", vm.name())
+                pass
+            self.domains.append(vm)
+
+    def checkForVM(self, xmlConfig):
+        """
+        Checks if another vm with the same name exists
+        on the remote host already. If it does, it will
+        delete that vm
+        """
+        allGood = False
+        vms = self.hypervisor.listAllDomains(0)
+        names = []
+        for dom in vms:
+            names.append(dom.name())
+        vmName = Utilities.getName(xmlConfig)
+        if vmName in names:
+            self.log.warning("domain %s already exists", vmName)
+            self.log.warning("%s", "Atempting to delete it")
+            self.deleteVM(vmName)
+            allGood = True
+        else:
+            allGood = True
+        return allGood
+
+    def deleteVM(self, name):
+        """
+        removes the given vm from the remote host
+        """
+        try:
+            vm = self.hypervisor.lookupByName(name)
+        except:
+            return
+        active = vm.isActive()
+        persistent = vm.isPersistent()
+        if active:
+            try:
+                vm.destroy()
+            except:
+                self.log.exception("%s", "Failed to destroy vm")
+
+        if persistent:
+            try:
+                vm.undefine()
+            except:
+                self.log.exception("%s", "Failed to undefine domain")
+                pass
+
+    def openConnection(self):
+        """
+        opens a connection to the remote host
+        and stores it in self.hypervisor
+        """
+        self.log.info("Attempting to connect to libvirt at %s", self.host)
+        try:
+            hostHypervisor = libvirt.open(self.URI)
+        except:
+            self.log.warning(
+                    "Failed to connect to %s. Trying again", self.host
+                    )
+            time.sleep(5)
+            try:
+                hostHypervisor = libvirt.open(self.URI)
+            except:
+                self.log.exception("Cannot connect to %s. Exiting", self.host)
+                exit(1)
+
+        if hostHypervisor is None:
+            self.log.error("Failed to connect to %s. Exiting", self.host)
+            exit(1)
+        self.hypervisor = hostHypervisor
+
+    def restartVM(self, vm):
+        """
+        causes the given vm to reboot
+        """
+        dom = self.hypervisor.lookupByName(vm)
+        dom.destroy()
+        time.sleep(15)
+        dom.create()
+
+    def close(self):
+        """
+        Closes connection to remote hypervisor
+        """
+        self.log.info("Closing connection to the hypervisor %s", self.host)
+        self.hypervisor.close()
+
+    def defineAllDomains(self, path):
+        """
+        Defines a domain from all the xml files in a directory
+        """
+        files = Utilities.getXMLFiles(path)
+        definitions = []
+        for xml_desc in files:
+            definitions.append(xml_desc.read())
+
+        for definition in definitions:
+            self.defineVM(definition)
+
+    def createAllNetworks(self, path):
+        """
+        Creates a network from all xml files in a directory
+        """
+        files = Utilities.getXMLFiles(path)
+        definitions = []
+        for xml_desc in files:
+            definitions.append(Utilities.fileToString(xml_desc))
+
+        for definition in definitions:
+            self.createNet(definition)
+
+    def createNet(self, config):
+        """
+        creates the network on the remote host
+        config is the xml in string representation
+        that defines the network
+        """
+        if self.checkNet(config):
+            network = self.hypervisor.networkDefineXML(config)
+
+            if network is None:
+                name = self.getName(config)
+                self.log.warning("Failed to define network %s", name)
+            network.create()
+            if network.isActive() == 1:
+                net = network.name()
+                self.log.info("Successfully defined network %s", net)
+            self.networks.append(network)
+
+    def checkNet(self, config):
+        """
+        Checks if another net with the same name exists, and
+        deletes that network if one is found
+        """
+        allGood = False
+        netName = Utilities.getName(config)
+        if netName not in self.hypervisor.listNetworks():
+            return True
+        else:  # net name is already used
+            self.log.warning(
+                    "Network %s already exists. Trying to delete it", netName
+                    )
+            network = self.hypervisor.networkLookupByName(netName)
+            self.deleteNet(network)
+            allGood = True
+        return allGood
+
+    def deleteNet(self, net):
+        """
+        removes the given network from the host
+        """
+        active = net.isActive()
+        persistent = net.isPersistent()
+        if active:
+            try:
+                net.destroy()
+            except:
+                self.log.warning("%s", "Failed to destroy network")
+
+        if persistent:
+            try:
+                net.undefine()
+            except:
+                self.log.warning("%s", "Failed to undefine network")
+
+    def go(self):
+        """
+        This method does all the work of this class,
+        Parsing the net and vm config files and creating
+        all the requested nets/domains
+        returns a list of all networks and a list of all domains
+        as Network and Domain objects
+        """
+        nets = self.makeNetworks(self.net_conf)
+        doms = self.makeDomains(self.dom_conf)
+        return doms, nets
+
+    def makeNetworks(self, conf):
+        """
+        Given a path to a  config file, this method
+        parses the config and creates all requested networks,
+        and returns them in a list of Network objects
+        """
+        networks = []
+        definitions = Network.parseConfigFile(conf)
+        for definition in definitions:
+            network = Network(definition)
+            networks.append(network)
+            self.createNet(network.toXML())
+        return networks
+
+    def makeDomains(self, conf):
+        """
+        Given a path to a config file, this method
+        parses the config and creates all requested vm's,
+        and returns them in a list of Domain objects
+        """
+        domains = []
+        definitions = Domain.parseConfigFile(conf)
+        for definition in definitions:
+            domain = Domain(definition)
+            domains.append(domain)
+            self.defineVM(domain.toXML())
+        return domains
+
+    @staticmethod
+    def getName(xmlString):
+        """
+        given xml with a name tag, this returns the value of name
+        eg:
+            <name>Parker</name>
+        returns 'Parker'
+        """
+        xmlDoc = xml.dom.minidom.parseString(xmlString)
+        nameNode = xmlDoc.documentElement.getElementsByTagName('name')
+        name = str(nameNode[0].firstChild.nodeValue)
+        return name
diff --git a/laas-fog/source/domain.py b/laas-fog/source/domain.py
new file mode 100644
index 0000000..6f00239
--- /dev/null
+++ b/laas-fog/source/domain.py
@@ -0,0 +1,244 @@
+"""
+#############################################################################
+#Copyright 2017 Parker Berberian and others                                 #
+#                                                                           #
+#Licensed under the Apache License, Version 2.0 (the "License");            #
+#you may not use this file except in compliance with the License.           #
+#You may obtain a copy of the License at                                    #
+#                                                                           #
+#    http://www.apache.org/licenses/LICENSE-2.0                             #
+#                                                                           #
+#Unless required by applicable law or agreed to in writing, software        #
+#distributed under the License is distributed on an "AS IS" BASIS,          #
+#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.   #
+#See the License for the specific language governing permissions and        #
+#limitations under the License.                                             #
+#############################################################################
+"""
+
+import xml.dom
+import xml.dom.minidom
+import yaml
+
+
+class Domain:
+    """
+    This class defines a libvirt vm abstraction that can parse our simple
+    config file and add all necessary boiler plate and info to write a full xml
+    definition of itself for libvirt.
+    """
+
+    def __init__(self, propertiesDict):
+        """
+        init function.
+        properiesDict should be one of the dictionaries returned by the static
+        method parseConfigFile
+        """
+        self.name = propertiesDict['name']
+        self.memory = propertiesDict['memory']
+        self.vcpus = propertiesDict['vcpus']
+        self.disk = propertiesDict['disk']
+        self.iso = propertiesDict['iso']
+        # the vm will either boot from an iso or pxe
+        self.netBoot = not self.iso['used']
+        self.interfaces = propertiesDict['interfaces']
+
+    def toXML(self):
+        """
+        combines the given configuration with a lot of
+        boiler plate to create a valid libvirt xml
+        definition of a domain.
+        returns a string
+        """
+        definition = xml.dom.minidom.parseString("<domain>\n</domain>")
+        definition.documentElement.setAttribute('type', 'kvm')
+
+        nameElem = definition.createElement('name')
+        nameElem.appendChild(definition.createTextNode(self.name))
+        definition.documentElement.appendChild(nameElem)
+
+        memElem = definition.createElement('memory')
+        memElem.appendChild(definition.createTextNode(str(self.memory)))
+        definition.documentElement.appendChild(memElem)
+
+        curMemElem = definition.createElement('currentMemory')
+        curMemElem.appendChild(definition.createTextNode(str(self.memory)))
+        definition.documentElement.appendChild(curMemElem)
+
+        vcpuElem = definition.createElement('vcpu')
+        vcpuElem.appendChild(definition.createTextNode(str(self.vcpus)))
+        definition.documentElement.appendChild(vcpuElem)
+
+        osElem = definition.createElement('os')
+
+        typeElem = definition.createElement('type')
+        typeElem.setAttribute('arch', 'x86_64')
+        typeElem.appendChild(definition.createTextNode('hvm'))
+        osElem.appendChild(typeElem)
+
+        if self.netBoot:
+            bootElem = definition.createElement('boot')
+            bootElem.setAttribute('dev', 'network')
+            osElem.appendChild(bootElem)
+
+        bootElem = definition.createElement('boot')
+        bootElem.setAttribute('dev', 'hd')
+        osElem.appendChild(bootElem)
+
+        if self.iso['used']:
+            bootElem = definition.createElement('boot')
+            bootElem.setAttribute('dev', 'cdrom')
+            osElem.appendChild(bootElem)
+
+        definition.documentElement.appendChild(osElem)
+
+        featureElem = definition.createElement('feature')
+        featureElem.appendChild(definition.createElement('acpi'))
+        featureElem.appendChild(definition.createElement('apic'))
+
+        definition.documentElement.appendChild(featureElem)
+
+        cpuElem = definition.createElement('cpu')
+        cpuElem.setAttribute('mode', 'custom')
+        cpuElem.setAttribute('match', 'exact')
+        modelElem = definition.createElement('model')
+        modelElem.appendChild(definition.createTextNode('Broadwell'))
+        cpuElem.appendChild(modelElem)
+
+        definition.documentElement.appendChild(cpuElem)
+
+        clockElem = definition.createElement('clock')
+        clockElem.setAttribute('offset', 'utc')
+
+        timeElem = definition.createElement('timer')
+        timeElem.setAttribute('name', 'rtc')
+        timeElem.setAttribute('tickpolicy', 'catchup')
+        clockElem.appendChild(timeElem)
+
+        timeElem = definition.createElement('timer')
+        timeElem.setAttribute('name', 'pit')
+        timeElem.setAttribute('tickpolicy', 'delay')
+        clockElem.appendChild(timeElem)
+
+        timeElem = definition.createElement('timer')
+        timeElem.setAttribute('name', 'hpet')
+        timeElem.setAttribute('present', 'no')
+        clockElem.appendChild(timeElem)
+
+        definition.documentElement.appendChild(clockElem)
+
+        poweroffElem = definition.createElement('on_poweroff')
+        poweroffElem.appendChild(definition.createTextNode('destroy'))
+
+        definition.documentElement.appendChild(poweroffElem)
+
+        rebootElem = definition.createElement('on_reboot')
+        rebootElem.appendChild(definition.createTextNode('restart'))
+
+        definition.documentElement.appendChild(rebootElem)
+
+        crashElem = definition.createElement('on_reboot')
+        crashElem.appendChild(definition.createTextNode('restart'))
+
+        definition.documentElement.appendChild(crashElem)
+
+        pmElem = definition.createElement('pm')
+        memElem = definition.createElement('suspend-to-mem')
+        memElem.setAttribute('enabled', 'no')
+        pmElem.appendChild(memElem)
+        diskElem = definition.createElement('suspend-to-disk')
+        diskElem.setAttribute('enabled', 'no')
+        pmElem.appendChild(diskElem)
+
+        definition.documentElement.appendChild(pmElem)
+
+        deviceElem = definition.createElement('devices')
+
+        emuElem = definition.createElement('emulator')
+        emuElem.appendChild(definition.createTextNode('/usr/libexec/qemu-kvm'))
+        deviceElem.appendChild(emuElem)
+
+        diskElem = definition.createElement('disk')
+        diskElem.setAttribute('type', 'file')
+        diskElem.setAttribute('device', 'disk')
+
+        driverElem = definition.createElement('driver')
+        driverElem.setAttribute('name', 'qemu')
+        driverElem.setAttribute('type', 'qcow2')
+        diskElem.appendChild(driverElem)
+
+        sourceElem = definition.createElement('source')
+        sourceElem.setAttribute('file', self.disk)
+        diskElem.appendChild(sourceElem)
+
+        targetElem = definition.createElement('target')
+        targetElem.setAttribute('dev', 'hda')
+        targetElem.setAttribute('bus', 'ide')
+        diskElem.appendChild(targetElem)
+
+        deviceElem.appendChild(diskElem)
+
+        if self.iso['used']:
+            diskElem = definition.createElement('disk')
+            diskElem.setAttribute('type', 'file')
+            diskElem.setAttribute('device', 'cdrom')
+
+            driverElem = definition.createElement('driver')
+            driverElem.setAttribute('name', 'qemu')
+            driverElem.setAttribute('type', 'raw')
+            diskElem.appendChild(driverElem)
+
+            sourceElem = definition.createElement('source')
+            sourceElem.setAttribute('file', self.iso['location'])
+            diskElem.appendChild(sourceElem)
+
+            targetElem = definition.createElement('target')
+            targetElem.setAttribute('dev', 'hdb')
+            targetElem.setAttribute('bus', 'ide')
+            diskElem.appendChild(targetElem)
+
+            diskElem.appendChild(definition.createElement('readonly'))
+            deviceElem.appendChild(diskElem)
+
+        for iface in self.interfaces:
+            ifaceElem = definition.createElement('interface')
+            ifaceElem.setAttribute('type', iface['type'])
+            sourceElem = definition.createElement('source')
+            sourceElem.setAttribute(iface['type'], iface['name'])
+            modelElem = definition.createElement('model')
+            modelElem.setAttribute('type', 'e1000')
+            ifaceElem.appendChild(sourceElem)
+            ifaceElem.appendChild(modelElem)
+            deviceElem.appendChild(ifaceElem)
+
+        graphicElem = definition.createElement('graphics')
+        graphicElem.setAttribute('type', 'vnc')
+        graphicElem.setAttribute('port', '-1')
+        deviceElem.appendChild(graphicElem)
+
+        consoleElem = definition.createElement('console')
+        consoleElem.setAttribute('type', 'pty')
+        deviceElem.appendChild(consoleElem)
+
+        definition.documentElement.appendChild(deviceElem)
+        return definition.toprettyxml()
+
+    def writeXML(self, filePath):
+        """
+        writes this domain's xml definition to the given file.
+        """
+        f = open(filePath, 'w')
+        f.write(self.toXML())
+        f.close()
+
+    @staticmethod
+    def parseConfigFile(path):
+        """
+        parses the domains config file
+        """
+        configFile = open(path, 'r')
+        try:
+            config = yaml.safe_load(configFile)
+        except Exception:
+            print "Invalid domain configuration. exiting"
+        return config
diff --git a/laas-fog/source/network.py b/laas-fog/source/network.py
new file mode 100644
index 0000000..234ba22
--- /dev/null
+++ b/laas-fog/source/network.py
@@ -0,0 +1,103 @@
+"""
+#############################################################################
+#Copyright 2017 Parker Berberian and others                                 #
+#                                                                           #
+#Licensed under the Apache License, Version 2.0 (the "License");            #
+#you may not use this file except in compliance with the License.           #
+#You may obtain a copy of the License at                                    #
+#                                                                           #
+#    http://www.apache.org/licenses/LICENSE-2.0                             #
+#                                                                           #
+#Unless required by applicable law or agreed to in writing, software        #
+#distributed under the License is distributed on an "AS IS" BASIS,          #
+#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.   #
+#See the License for the specific language governing permissions and        #
+#limitations under the License.                                             #
+#############################################################################
+"""
+
+import sys
+import xml.dom
+import xml.dom.minidom
+import yaml
+
+
+class Network:
+    """
+    This class has a similar role as the Domain class.
+    This class will parse a config file and
+    write the xml definitions of those networks for libvirt.
+    """
+
+    def __init__(self, propertiesDict):
+        """
+        init. propertiesDict should be
+        one of the dictionaries returned by parseConfigFile
+        """
+        self.name = propertiesDict['name']
+        self.brName = propertiesDict['brName']
+        self.brAddr = propertiesDict['brAddr']
+        self.netmask = propertiesDict['netmask']
+        self.forward = propertiesDict['forward']
+        self.dhcp = propertiesDict['dhcp']
+        self.cidr = propertiesDict['cidr']
+
+    def toXML(self):
+        """
+        Takes the config of this network and writes a valid xml definition
+        for libvirt.
+        returns a string
+        """
+        definition = xml.dom.minidom.parseString("<network>\n</network>")
+        nameElem = definition.createElement('name')
+        nameElem.appendChild(definition.createTextNode(self.name))
+        definition.documentElement.appendChild(nameElem)
+
+        if self.forward['used']:
+            forwardElem = definition.createElement('forward')
+            forwardElem.setAttribute('mode', self.forward['type'])
+            definition.documentElement.appendChild(forwardElem)
+
+        bridgeElem = definition.createElement('bridge')
+        bridgeElem.setAttribute('name', self.brName)
+        bridgeElem.setAttribute('stp', 'on')
+        bridgeElem.setAttribute('delay', '5')
+        definition.documentElement.appendChild(bridgeElem)
+
+        ipElem = definition.createElement('ip')
+        ipElem.setAttribute('address', self.brAddr)
+        ipElem.setAttribute('netmask', self.netmask)
+        if self.dhcp['used']:
+            dhcpElem = definition.createElement('dhcp')
+            rangeElem = definition.createElement('range')
+            rangeElem.setAttribute('start', self.dhcp['rangeStart'])
+            rangeElem.setAttribute('end', self.dhcp['rangeEnd'])
+            dhcpElem.appendChild(rangeElem)
+            ipElem.appendChild(dhcpElem)
+
+        definition.documentElement.appendChild(ipElem)
+
+        self.xml = definition.toprettyxml()
+        return self.xml
+
+    def writeXML(self, filePath):
+        """
+        writes xml definition to given file
+        """
+        f = open(filePath, 'w')
+        f.write(self.toXML())
+        f.close()
+
+    @staticmethod
+    def parseConfigFile(path):
+        """
+        parses given config file
+        """
+        configFile = open(path, 'r')
+        try:
+            config = yaml.safe_load(configFile)
+        except Exception:
+            print "Bad network configuration file. exiting"
+            sys.exit(1)
+
+        return config