diff options
| author |   Jack Morgan <jack.morgan@intel.com> | 2018-01-09 00:12:25 +0000 |
|---|---|---|
| committer |   Gerrit Code Review <gerrit@opnfv.org> | 2018-01-09 00:12:25 +0000 |
| commit | 37fa445afe6d63d44a3a52c2a89e4e21ae67b5a6 (patch) | |
| tree | a053095a7347ca30245081b3fc44d5f6b7f645d7 /laas-fog/source | |
| parent | 0700062b04ad49431fbebcfd7c44d4013d654bfe (diff) | |
| parent | 30f389c70e8a0a8bd2ef27be09839eef243ab7f5 (diff) | |
Merge "Initial Commit for new LaaS Software"
Diffstat (limited to 'laas-fog/source')
| -rw-r--r-- | laas-fog/source/__init__.py | 17 | ||||
| -rw-r--r-- | laas-fog/source/api/__init__.py | 17 | ||||
| -rw-r--r-- | laas-fog/source/api/fog.py | 288 | ||||
| -rw-r--r-- | laas-fog/source/api/fuel_api.py | 306 | ||||
| -rw-r--r-- | laas-fog/source/api/libvirt_api.py | 331 | ||||
| -rw-r--r-- | laas-fog/source/api/vpn.py | 235 | ||||
| -rw-r--r-- | laas-fog/source/database.py | 296 | ||||
| -rwxr-xr-x | laas-fog/source/deploy.py | 82 | ||||
| -rw-r--r-- | laas-fog/source/deployment_manager.py | 108 | ||||
| -rw-r--r-- | laas-fog/source/domain.py | 244 | ||||
| -rw-r--r-- | laas-fog/source/installers/__init__.py | 17 | ||||
| -rw-r--r-- | laas-fog/source/installers/fuel.py | 268 | ||||
| -rw-r--r-- | laas-fog/source/installers/installer.py | 35 | ||||
| -rw-r--r-- | laas-fog/source/installers/joid.py | 40 | ||||
| -rwxr-xr-x | laas-fog/source/listen.py | 59 | ||||
| -rw-r--r-- | laas-fog/source/network.py | 103 | ||||
| -rwxr-xr-x | laas-fog/source/pharos.py | 217 | ||||
| -rwxr-xr-x | laas-fog/source/pod_manager.py | 144 | ||||
| -rwxr-xr-x | laas-fog/source/resetDataBase.py | 110 | ||||
| -rwxr-xr-x | laas-fog/source/stop.sh | 24 | ||||
| -rw-r--r-- | laas-fog/source/utilities.py | 346 |
21 files changed, 0 insertions, 3287 deletions
diff --git a/laas-fog/source/__init__.py b/laas-fog/source/__init__.py deleted file mode 100644 index 7bb515b..0000000 --- a/laas-fog/source/__init__.py +++ /dev/null @@ -1,17 +0,0 @@ -""" -############################################################################# -#Copyright 2017 Parker Berberian and others # -# # -#Licensed under the Apache License, Version 2.0 (the "License"); # -#you may not use this file except in compliance with the License. # -#You may obtain a copy of the License at # -# # -# http://www.apache.org/licenses/LICENSE-2.0 # -# # -#Unless required by applicable law or agreed to in writing, software # -#distributed under the License is distributed on an "AS IS" BASIS, # -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # -#See the License for the specific language governing permissions and # -#limitations under the License. # -############################################################################# -""" diff --git a/laas-fog/source/api/__init__.py b/laas-fog/source/api/__init__.py deleted file mode 100644 index 7bb515b..0000000 --- a/laas-fog/source/api/__init__.py +++ /dev/null @@ -1,17 +0,0 @@ -""" -############################################################################# -#Copyright 2017 Parker Berberian and others # -# # -#Licensed under the Apache License, Version 2.0 (the "License"); # -#you may not use this file except in compliance with the License. # -#You may obtain a copy of the License at # -# # -# http://www.apache.org/licenses/LICENSE-2.0 # -# # -#Unless required by applicable law or agreed to in writing, software # -#distributed under the License is distributed on an "AS IS" BASIS, # -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # -#See the License for the specific language governing permissions and # -#limitations under the License. # -############################################################################# -""" diff --git a/laas-fog/source/api/fog.py b/laas-fog/source/api/fog.py deleted file mode 100644 index 6287403..0000000 --- a/laas-fog/source/api/fog.py +++ /dev/null @@ -1,288 +0,0 @@ -""" -############################################################################# -#Copyright 2017 Parker Berberian and others # -# # -#Licensed under the Apache License, Version 2.0 (the "License"); # -#you may not use this file except in compliance with the License. # -#You may obtain a copy of the License at # -# # -# http://www.apache.org/licenses/LICENSE-2.0 # -# # -#Unless required by applicable law or agreed to in writing, software # -#distributed under the License is distributed on an "AS IS" BASIS, # -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # -#See the License for the specific language governing permissions and # -#limitations under the License. # -############################################################################# -""" - -import requests -import sys -import time - - -class FOG_Handler: - """ - This class talks with the REST web api for the FOG server. - - TODO: convert prints to logs and remove uneeded pass's - """ - - def __init__(self, baseURL, fogKey=None, userKey=None): - """ - init function - baseURL should be http://fog.ip.or.hostname/fog/ - fogKey and userKey can optionally be supplied here or later - They can be found in fog and provide authentication. - """ - self.baseURL = baseURL - self.fogKey = fogKey - self.userKey = userKey - self.header = {} - self.updateHeader() - - def setLogger(self, logger): - """ - saves the refference to the log object as - self.log - """ - self.log = logger - - def getUserKeyFromFile(self, path): - """ - reads the user api key from a file - """ - self.userKey = open(path).read() - self.updateHeader() - - def getFogKeyFromFile(self, path): - """ - reads the api key from a file - """ - self.fogKey = open(path).read() - self.updateHeader() - - def setUserKey(self, key): - """ - sets the user key - """ - self.userKey = key - self.updateHeader() - - def setFogKey(self, key): - """ - sets the fog key - """ - self.fogKey = key - self.updateHeader() - - def updateHeader(self): - """ - recreates the http header used to talk to the fog api - """ - self.header = {} - self.header['fog-api-token'] = self.fogKey - self.header['fog-user-token'] = self.userKey - - def setImage(self, host, imgNum): - """ - Sets the image to be used during ghosting to the image - with id imgNum. host can either be a hostname or number. - """ - try: - host = int(host) - except: - host = self.getHostNumber(host) - url = self.baseURL+"host/"+str(host) - host_conf = requests.get(url, headers=self.header).json() - host_conf['imageID'] = str(imgNum) - requests.put(url+"/edit", headers=self.header, json=host_conf) - - def delTask(self, hostNum): - """ - Tries to delete an existing task for the host - with hostNum as a host number - """ - try: - url = self.baseURL+'fog/host/'+str(hostNum)+'/cancel' - req = requests.delete(url, headers=self.header) - if req.status_code == 200: - self.log.info("%s", "successfully deleted image task") - except Exception: - self.log.exception("Failed to delete the imaging task!") - - def getHostMac(self, hostname): - """ - returns the primary mac address if the given host. - """ - try: - hostNum = int(self.getHostNumber(hostname)) - url = self.baseURL + "host/"+str(hostNum) - req = requests.get(url, headers=self.header) - macAddr = req.json()['primac'] - return macAddr - except Exception: - self.log.exception('%s', "Failed to connect to the FOG server") - - def getHostNumber(self, hostname): - """ - returns the host number of given host - """ - try: - req = requests.get(self.baseURL+"host", headers=self.header) - hostData = req.json() - if hostData is not None: - for hostDict in hostData['hosts']: - if hostname == hostDict['name']: - return hostDict['id'] - return -1 - except Exception: - self.log.exception('%s', "Failed to connect to the FOG server") - - def imageHost(self, hostName, recurse=False): - """ - Schedules an imaging task for the given host. - This automatically uses the "associated" disk image. - To support extra installers, I will need to create - a way to change what that image is before calling - this method. - """ - num = str(self.getHostNumber(hostName)) - url = self.baseURL+'host/'+num+'/task' - - try: - req = requests.post( - url, - headers=self.header, - json={"taskTypeID": 1} - ) - if req.status_code == 200: - self.log.info("%s", "Scheduled image task for host") - except Exception: - if recurse: # prevents infinite loop - self.log.exception("%s", "Failed to schedule task. Exiting") - sys.exit(1) - self.log.warning("%s", "Failed to schedule host imaging") - self.log.warning("%s", "Trying to delete existing image task") - self.delTask(num) - self.imageHost(num, recurse=True) - - def waitForHost(self, host): - """ - tracks the imaging task to completion. - """ - while True: - imageTask = self.getImagingTask(host) - if imageTask is None: - self.log.info("%s", "Imaging complete") - return - state = int(imageTask['stateID']) - if state == 1: - self.log.info("%s", "Waiting for host to check in") - self.waitForTaskToActive(host) - continue - if state == 3: - self.waitForTaskToStart(host) - self.waitForImaging(host) - continue - time.sleep(8) - - def waitForImaging(self, host): - """ - Once the host begins being imaged, this tracks progress. - """ - # print "Host has begun the imaging process\n" - while True: - task = self.getImagingTask(host) - if task is None: - return - per = str(task['percent']) - self.log.info("%s percent done imaging", per) - time.sleep(15) - - def waitForTaskToActive(self, host): - """ - Waits for the host to reboot and pxe boot - into FOG - """ - while True: - try: - task = self.getImagingTask(host) - except: - pass - state = int(task['stateID']) - if state == 1: - time.sleep(4) - else: - return - - def waitForTaskToStart(self, host): - """ - waits for the task to start and imaging to begin. - """ - while True: - try: - per = str(self.getImagingTask(host)['percent']) - except: - pass - if per.strip() == '': - time.sleep(1) - else: - return - - def getImagingTask(self, host): - """ - Sorts through all current tasks to find the image task - associated with the given host. - """ - try: - taskList = requests.get( - self.baseURL+'task/current', - headers=self.header) - taskList = taskList.json()['tasks'] - imageTask = None - for task in taskList: - hostname = str(task['host']['name']) - if hostname == host and int(task['typeID']) == 1: - imageTask = task - return imageTask - except Exception: - self.log.exception("%s", "Failed to talk to FOG server") - sys.exit(1) - - def getHosts(self): - """ - returns a list of all hosts - """ - req = requests.get(self.baseURL+"host", headers=self.header) - return req.json()['hosts'] - - def getHostsinGroup(self, groupName): - """ - returns a list of all hosts in groupName - """ - groupID = None - groups = requests.get(self.baseURL+"group", headers=self.header) - groups = groups.json()['groups'] - for group in groups: - if groupName.lower() in group['name'].lower(): - groupID = group['id'] - if groupID is None: - return - hostIDs = [] - associations = requests.get( - self.baseURL+"groupassociation", - headers=self.header - ) - associations = associations.json()['groupassociations'] - for association in associations: - if association['groupID'] == groupID: - hostIDs.append(association['hostID']) - - hosts = [] - for hostID in hostIDs: - hosts.append(requests.get( - self.baseURL+"host/"+str(hostID), - headers=self.header - ).json()) - return hosts diff --git a/laas-fog/source/api/fuel_api.py b/laas-fog/source/api/fuel_api.py deleted file mode 100644 index 0127800..0000000 --- a/laas-fog/source/api/fuel_api.py +++ /dev/null @@ -1,306 +0,0 @@ -""" -############################################################################# -#Copyright 2017 Parker Berberian and others # -# # -#Licensed under the Apache License, Version 2.0 (the "License"); # -#you may not use this file except in compliance with the License. # -#You may obtain a copy of the License at # -# # -# http://www.apache.org/licenses/LICENSE-2.0 # -# # -#Unless required by applicable law or agreed to in writing, software # -#distributed under the License is distributed on an "AS IS" BASIS, # -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # -#See the License for the specific language governing permissions and # -#limitations under the License. # -############################################################################# -""" - -import requests -import time -import sys - - -class Fuel_api: - - def __init__(self, url, logger, user="admin", password="admin"): - """ - url is the url of the fog api in the form - http://ip.or.host.name:8000/ - logger is a reference to the logger - the default creds for fuel is admin/admin - """ - self.logger = logger - self.base = url - self.user = user - self.password = password - self.header = {"Content-Type": "application/json"} - - def getKey(self): - """ - authenticates with the user and password - to get a keystone key, used in the headers - from here on to talk to fuel. - """ - url = self.base + 'keystone/v2.0/tokens/' - reqData = {"auth": { - "tenantName": self.user, - "passwordCredentials": { - "username": self.user, - "password": self.password - } - }} - self.logger.info("Retreiving keystone token from %s", url) - token = requests.post(url, headers=self.header, json=reqData) - self.logger.info("Received response code %d", token.status_code) - self.token = token.json()['access']['token']['id'] - self.header['X-Auth-Token'] = self.token - - def getNotifications(self): - """ - returns the fuel notifications - """ - url = self.base+'/api/notifications' - try: - req = requests.get(url, headers=self.header) - return req.json() - - except Exception: - self.logger.exception('%s', "Failed to talk to the Fuel api!") - sys.exit(1) - - def waitForBootstrap(self): - """ - Waits for the bootstrap image to build. - """ - while True: - time.sleep(30) - notes = self.getNotifications() - for note in notes: - if "bootstrap image building done" in note['message']: - return - - def getNodes(self): - """ - returns a list of all nodes booted into fuel - """ - url = self.base+'api/nodes' - try: - req = requests.get(url, headers=self.header) - return req.json() - except Exception: - self.logger.exception('%s', "Failed to talk to the Fuel api!") - sys.exit(1) - - def getID(self, mac): - """ - gets the fuel id of node with given mac - """ - for node in self.getNodes(): - if node['mac'] == mac: - return node['id'] - - def getNetID(self, name, osid): - """ - gets the id of the network with name - """ - url = self.base+'api/clusters/' - url += str(osid)+'/network_configuration/neutron' - try: - req = requests.get(url, headers=self.header) - nets = req.json()['networks'] - for net in nets: - if net['name'] == name: - return net['id'] - return -1 - - except Exception: - self.logger.exception('%s', "Failed to talk to the Fuel api!") - sys.exit(1) - - def createOpenstack(self): - """ - defines a new openstack environment in fuel. - """ - url = self.base+'api/clusters' - data = { - "nodes": [], - "tasks": [], - "name": "OpenStack", - "release_id": 2, - "net_segment_type": "vlan" - } - try: - req = requests.post(url, json=data, headers=self.header) - return req.json()['id'] - except Exception: - self.logger.exception('%s', "Failed to talk to the Fuel api!") - sys.exit(1) - - def simpleNetDict(self, osID): - """ - returns a simple dict of network names and id numbers - """ - nets = self.getNetworks(osID) - netDict = {} - targetNets = ['admin', 'public', 'storage', 'management'] - for net in nets['networks']: - for tarNet in targetNets: - if tarNet in net['name']: - netDict[tarNet] = net['id'] - return netDict - - def getNetworks(self, osID): - """ - Returns the pythonizezd json of the openstack networks - """ - url = self.base + 'api/clusters/' - url += str(osID)+'/network_configuration/neutron/' - try: - req = requests.get(url, headers=self.header) - return req.json() - except Exception: - self.logger.exception('%s', "Failed to talk to the Fuel api!") - sys.exit(1) - - def uploadNetworks(self, netJson, osID): - """ - configures the networks of the openstack - environment with id osID based on netJson - """ - url = self.base+'api/clusters/' - url += str(osID)+'/network_configuration/neutron' - try: - req = requests.put(url, headers=self.header, json=netJson) - return req.json() - except Exception: - self.logger.exception('%s', "Failed to talk to the Fuel api!") - sys.exit(1) - - def addNodes(self, clusterID, nodes): - """ - Adds the nodes into this openstack environment. - nodes is valid json - """ - url = self.base + 'api/clusters/'+str(clusterID)+'/assignment' - try: - req = requests.post(url, headers=self.header, json=nodes) - return req.json() - - except Exception: - self.logger.exception('%s', "Failed to talk to the Fuel api!") - sys.exit(1) - - def getIfaces(self, nodeID): - """ - returns the pythonized json describing the - interfaces of given node - """ - url = self.base + 'api/nodes/'+str(nodeID)+'/interfaces' - try: - req = requests.get(url, headers=self.header) - return req.json() - - except Exception: - self.logger.exception('%s', "Failed to talk to the Fuel api!") - sys.exit(1) - - def setIfaces(self, nodeID, ifaceJson): - """ - configures the interfaces of node with id nodeID - with ifaceJson - ifaceJson is valid json that fits fuel's schema for ifaces - """ - url = self.base+'/api/nodes/'+str(nodeID)+'/interfaces' - try: - req = requests.put(url, headers=self.header, json=ifaceJson) - return req.json() - - except Exception: - self.logger.exception('%s', "Failed to talk to the Fuel api!") - sys.exit(1) - - def getTasks(self): - """ - returns a list of all tasks - """ - url = self.base+"/api/tasks/" - try: - req = requests.get(url, headers=self.header) - return req.json() - except Exception: - self.logger.exception('%s', "Failed to talk to the Fuel api!") - sys.exit(1) - - def waitForTask(self, uuid): - """ - Tracks the progress of task with uuid and - returns once the task finishes - """ - progress = 0 - while progress < 100: - for task in self.getTasks(): - if task['uuid'] == uuid: - progress = task['progress'] - self.logger.info("Task is %s percent done", str(progress)) - time.sleep(20) - # Task may hang a minute at 100% without finishing - while True: - for task in self.getTasks(): - if task['uuid'] == uuid and not task['status'] == "ready": - time.sleep(10) - elif task['uuid'] == uuid and task['status'] == "ready": - return - - def getHorizonIP(self, osid): - """ - returns the ip address of the horizon dashboard. - Horizon always takes the first ip after the public router's - """ - url = self.base+'api/clusters/' - url += str(osid)+'/network_configuration/neutron/' - try: - req = requests.get(url, headers=self.header) - routerIP = req.json()['vips']['vrouter_pub']['ipaddr'].split('.') - routerIP[-1] = str(int(routerIP[-1])+1) - return '.'.join(routerIP) - except Exception: - self.logger.exception('%s', "Failed to talk to the Fuel api!") - sys.exit(1) - - def deployOpenstack(self, clusterID): - """ - Once openstack and the nodes are configured, - this method actually deploys openstack. - It takes a while. - """ - # First, we need to provision the cluster - url = self.base+'/api/clusters/'+str(clusterID)+'/provision' - req = requests.put(url, headers=self.header) - if req.status_code < 300: - self.logger.info('%s', "Sent provisioning task") - else: - err = "failed to provision Openstack Environment" - self.logger.error('%s', err) - sys.exit(1) - - taskUID = '' - tasks = self.getTasks() - for task in tasks: - if task['name'] == "provision" and task['cluster'] == clusterID: - taskUID = task['uuid'] - - self.waitForTask(taskUID) - - # Then, we deploy cluster - url = self.base + '/api/clusters/'+str(clusterID)+'/deploy' - req = requests.put(url, headers=self.header) - if req.status_code < 300: - self.logger.info('%s', "Sent deployment task") - taskUID = '' - tasks = self.getTasks() - for task in tasks: - if 'deploy' in task['name'] and task['cluster'] == clusterID: - taskUID = task['uuid'] - if len(taskUID) > 0: - self.waitForTask(taskUID) diff --git a/laas-fog/source/api/libvirt_api.py b/laas-fog/source/api/libvirt_api.py deleted file mode 100644 index 4e19736..0000000 --- a/laas-fog/source/api/libvirt_api.py +++ /dev/null @@ -1,331 +0,0 @@ -""" -############################################################################# -#Copyright 2017 Parker Berberian and others # -# # -#Licensed under the Apache License, Version 2.0 (the "License"); # -#you may not use this file except in compliance with the License. # -#You may obtain a copy of the License at # -# # -# http://www.apache.org/licenses/LICENSE-2.0 # -# # -#Unless required by applicable law or agreed to in writing, software # -#distributed under the License is distributed on an "AS IS" BASIS, # -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # -#See the License for the specific language governing permissions and # -#limitations under the License. # -############################################################################# -""" - -import libvirt -import time -import xml.dom -import xml.dom.minidom -from domain import Domain -from network import Network -from utilities import Utilities - - -class Libvirt: - """ - This class talks to the Libvirt api. - Given a config file, this class should create all networks and - domains. - - TODO: convert prints to logging and remove uneeded pass statements - """ - - def __init__(self, hostAddr, net_conf=None, dom_conf=None): - """ - init function - hostAddr is the ip address of the host - net_conf and dom_conf are the paths - to the config files - """ - self.host = hostAddr - self.URI = "qemu+ssh://root@"+str(hostAddr)+"/system" - self.hypervisor = None - self.domains = [] - self.networks = [] - self.net_conf = net_conf - self.dom_conf = dom_conf - - def setLogger(self, log): - """ - Saves the logger in self.log - """ - self.log = log - - def bootMaster(self): - """ - starts the previously defined master node - """ - for dom in self.domains: - if 'master' in dom.name(): - try: - dom.create() - except Exception: - pass - - def bootSlaves(self): - """ - boots every defined vm with 'slave' in its name - """ - for dom in self.domains: - if 'slave' in dom.name(): - try: - dom.create() - self.log.info("Booting %s", dom.name()) - except Exception: - self.log.exception("%s", "failed to boot domain") - time.sleep(5) - - def getMacs(self, domName): - """ - returns a dictionary with a network name - mapped to the mac address of the domain on that net - """ - try: - dom = self.hypervisor.lookupByName(domName) - xmlDesc = dom.XMLDesc(0) - parsedXML = xml.dom.minidom.parseString(xmlDesc) - interfacesXML = parsedXML.getElementsByTagName('interface') - netDict = {} - for iface in interfacesXML: - src = iface.getElementsByTagName('source')[0] - mac = iface.getElementsByTagName('mac')[0] - netDict[src] = mac - return netDict - except Exception: - self.log.exception("%s", "Domain not found") - - def defineVM(self, xmlConfig): - """ - Generic method to define a persistent vm with the - given config. - Assumes that self.hypervisor is already connected. - """ - if self.checkForVM(xmlConfig): - vm = self.hypervisor.defineXML(xmlConfig) - if vm is None: - name = self.getName(xmlConfig) - self.log.error("Failed to define vm %s. exiting", name) - exit(1) - else: - self.log.info("Successfully created vm %s", vm.name()) - pass - self.domains.append(vm) - - def checkForVM(self, xmlConfig): - """ - Checks if another vm with the same name exists - on the remote host already. If it does, it will - delete that vm - """ - allGood = False - vms = self.hypervisor.listAllDomains(0) - names = [] - for dom in vms: - names.append(dom.name()) - vmName = Utilities.getName(xmlConfig) - if vmName in names: - self.log.warning("domain %s already exists", vmName) - self.log.warning("%s", "Atempting to delete it") - self.deleteVM(vmName) - allGood = True - else: - allGood = True - return allGood - - def deleteVM(self, name): - """ - removes the given vm from the remote host - """ - try: - vm = self.hypervisor.lookupByName(name) - except: - return - active = vm.isActive() - persistent = vm.isPersistent() - if active: - try: - vm.destroy() - except: - self.log.exception("%s", "Failed to destroy vm") - - if persistent: - try: - vm.undefine() - except: - self.log.exception("%s", "Failed to undefine domain") - pass - - def openConnection(self): - """ - opens a connection to the remote host - and stores it in self.hypervisor - """ - self.log.info("Attempting to connect to libvirt at %s", self.host) - try: - hostHypervisor = libvirt.open(self.URI) - except: - self.log.warning( - "Failed to connect to %s. Trying again", self.host - ) - time.sleep(5) - try: - hostHypervisor = libvirt.open(self.URI) - except: - self.log.exception("Cannot connect to %s. Exiting", self.host) - exit(1) - - if hostHypervisor is None: - self.log.error("Failed to connect to %s. Exiting", self.host) - exit(1) - self.hypervisor = hostHypervisor - - def restartVM(self, vm): - """ - causes the given vm to reboot - """ - dom = self.hypervisor.lookupByName(vm) - dom.destroy() - time.sleep(15) - dom.create() - - def close(self): - """ - Closes connection to remote hypervisor - """ - self.log.info("Closing connection to the hypervisor %s", self.host) - self.hypervisor.close() - - def defineAllDomains(self, path): - """ - Defines a domain from all the xml files in a directory - """ - files = Utilities.getXMLFiles(path) - definitions = [] - for xml_desc in files: - definitions.append(xml_desc.read()) - - for definition in definitions: - self.defineVM(definition) - - def createAllNetworks(self, path): - """ - Creates a network from all xml files in a directory - """ - files = Utilities.getXMLFiles(path) - definitions = [] - for xml_desc in files: - definitions.append(Utilities.fileToString(xml_desc)) - - for definition in definitions: - self.createNet(definition) - - def createNet(self, config): - """ - creates the network on the remote host - config is the xml in string representation - that defines the network - """ - if self.checkNet(config): - network = self.hypervisor.networkDefineXML(config) - - if network is None: - name = self.getName(config) - self.log.warning("Failed to define network %s", name) - network.create() - if network.isActive() == 1: - net = network.name() - self.log.info("Successfully defined network %s", net) - self.networks.append(network) - - def checkNet(self, config): - """ - Checks if another net with the same name exists, and - deletes that network if one is found - """ - allGood = False - netName = Utilities.getName(config) - if netName not in self.hypervisor.listNetworks(): - return True - else: # net name is already used - self.log.warning( - "Network %s already exists. Trying to delete it", netName - ) - network = self.hypervisor.networkLookupByName(netName) - self.deleteNet(network) - allGood = True - return allGood - - def deleteNet(self, net): - """ - removes the given network from the host - """ - active = net.isActive() - persistent = net.isPersistent() - if active: - try: - net.destroy() - except: - self.log.warning("%s", "Failed to destroy network") - - if persistent: - try: - net.undefine() - except: - self.log.warning("%s", "Failed to undefine network") - - def go(self): - """ - This method does all the work of this class, - Parsing the net and vm config files and creating - all the requested nets/domains - returns a list of all networks and a list of all domains - as Network and Domain objects - """ - nets = self.makeNetworks(self.net_conf) - doms = self.makeDomains(self.dom_conf) - return doms, nets - - def makeNetworks(self, conf): - """ - Given a path to a config file, this method - parses the config and creates all requested networks, - and returns them in a list of Network objects - """ - networks = [] - definitions = Network.parseConfigFile(conf) - for definition in definitions: - network = Network(definition) - networks.append(network) - self.createNet(network.toXML()) - return networks - - def makeDomains(self, conf): - """ - Given a path to a config file, this method - parses the config and creates all requested vm's, - and returns them in a list of Domain objects - """ - domains = [] - definitions = Domain.parseConfigFile(conf) - for definition in definitions: - domain = Domain(definition) - domains.append(domain) - self.defineVM(domain.toXML()) - return domains - - @staticmethod - def getName(xmlString): - """ - given xml with a name tag, this returns the value of name - eg: - <name>Parker</name> - returns 'Parker' - """ - xmlDoc = xml.dom.minidom.parseString(xmlString) - nameNode = xmlDoc.documentElement.getElementsByTagName('name') - name = str(nameNode[0].firstChild.nodeValue) - return name diff --git a/laas-fog/source/api/vpn.py b/laas-fog/source/api/vpn.py deleted file mode 100644 index 336a681..0000000 --- a/laas-fog/source/api/vpn.py +++ /dev/null @@ -1,235 +0,0 @@ -from abc import ABCMeta, abstractmethod -import ldap -import os -import random -from base64 import b64encode -from database import BookingDataBase - - -class VPN_BaseClass: - """ - the vpn handler abstract class / interface - - """ - __metaclass__ = ABCMeta - - @abstractmethod - def __init__(self, config): - """ - config is the parsed vpn.yaml file - """ - pass - - @abstractmethod - def makeNewUser(self, user=None): - """ - This method is called when a vpn user is needed. - This method should create a vpn user in whatever - runs the vpn in our infrastructure. returns the - credentials for the vpn user and some uid - that will be associated with the booking in the - database. This uid is used to track the vpn user and - to delete the user when there are no bookings associated - with that uid. - """ - user = "username" - passwd = "password" - uid = "some way for you to identify this user in the database" - return user, passwd, uid - - @abstractmethod - def removeOldUsers(self): - """ - checks the list of all vpn users against a list of - vpn users associated with active bookings and removes - users who dont have an active booking - - If you want your vpn accounts to be persistent, - you can just ignore this - """ - pass - - -names = [ - 'frodo baggins', 'samwise gamgee', 'peregrin took', 'meriadoc brandybuck', - 'bilbo baggins', 'gandalf grey', 'aragorn dunadan', 'arwen evenstar', - 'saruman white', 'pippin took', 'merry brandybuck', 'legolas greenleaf', - 'gimli gloin', 'anakin skywalker', 'padme amidala', 'han solo', - 'jabba hut', 'mace windu', 'sount dooku', 'qui-gon jinn', - 'admiral ackbar', 'emperor palpatine' -] - - -class VPN: - """ - This class communicates with the ldap server to manage vpn users. - This class extends the above ABC, and implements the makeNewUser, - removeOldUser, and __init__ abstract functions you must override to - extend the VPN_BaseClass - """ - - def __init__(self, config): - """ - init takes the parsed vpn config file as an arguement. - automatically connects and authenticates on the ldap server - based on the configuration file - """ - self.config = config - server = config['server'] - self.uri = "ldap://"+server - - self.conn = None - user = config['authentication']['user'] - pswd = config['authentication']['pass'] - if os.path.isfile(pswd): - pswd = open(pswd).read() - self.connect(user, pswd) - - def connect(self, root_dn, root_pass): - """ - Opens a connection to the server in the config file - and authenticates as the given user - """ - self.conn = ldap.initialize(self.uri) - self.conn.simple_bind_s(root_dn, root_pass) - - def addUser(self, full_name, passwd): - """ - Adds a user to the ldap server. Creates the new user with the classes - and in the directory given in the config file. - full_name should be two tokens seperated by a space. The first token - will become the username - private helper function for the makeNewUser() - """ - first = full_name.split(' ')[0] - last = full_name.split(' ')[1] - user_dir = self.config['directory']['user'] - user_dir += ','+self.config['directory']['root'] - dn = "uid=" + first + ',' + user_dir - record = [ - ('objectclass', ['top', 'inetOrgPerson']), - ('uid', first), - ('cn', full_name), - ('sn', last), - ('userpassword', passwd), - ('ou', self.config['directory']['user'].split('=')[1]) - ] - self.conn.add_s(dn, record) - return dn - - def makeNewUser(self, name=None): - """ - creates a new user in the ldap database, with the given name - if supplied. If no name is given, we will try to select from the - pre-written list above, and will resort to generating a random string - as a username if the preconfigured names are all taken. - Returns the username and password the user needs to authenticate, and - the dn that we can use to manage the user. - """ - if name is None: - i = 0 - while not self.checkName(name): - i += 1 - if i == 20: - name = self.randoString(8) - name += ' '+self.randoString(8) - break # generates a random name to prevent infinite loop - name = self.genUserName() - passwd = self.randoString(15) - dn = self.addUser(name, passwd) - return name, passwd, dn - - def checkName(self, name): - """ - returns true if the name is available - """ - if name is None: - return False - uid = name.split(' ')[0] - base = self.config['directory']['user'] + ',' - base += self.config['directory']['root'] - filtr = '(uid=' + uid + ')' - timeout = 5 - ans = self.conn.search_st( - base, - ldap.SCOPE_SUBTREE, - filtr, - timeout=timeout - ) - return len(ans) < 1 - - @staticmethod - def randoString(n): - """ - uses /dev/urandom to generate a random string of length n - """ - n = int(n) - # defines valid characters - alpha = 'abcdefghijklmnopqrstuvwxyz' - alpha_num = alpha - alpha_num += alpha.upper() - alpha_num += "0123456789" - - # generates random string from /dev/urandom - rnd = b64encode(os.urandom(3*n)).decode('utf-8') - random_string = '' - for char in rnd: - if char in alpha_num: - random_string += char - return str(random_string[:n]) - - def genUserName(self): - """ - grabs a random name from the list above - """ - i = random.randint(0, len(names) - 1) - return names[i] - - def deleteUser(self, dn): - self.conn.delete(dn) - - def getAllUsers(self): - """ - returns all the user dn's in the ldap database in a list - """ - base = self.config['directory']['user'] + ',' - base += self.config['directory']['root'] - filtr = '(objectclass='+self.config['user']['objects'][-1]+')' - timeout = 10 - ans = self.conn.search_st( - base, - ldap.SCOPE_SUBTREE, - filtr, - timeout=timeout - ) - users = [] - for user in ans: - users.append(user[0]) # adds the dn of each user - return users - - def removeOldUsers(self): - """ - removes users from the ldap server who dont have any active bookings. - will not delete a user if their uid's are named in the config - file as permanent users. - """ - db = self.config['database'] - # the dn of all users who have an active booking - active_users = BookingDataBase(db).getVPN() - all_users = self.getAllUsers() - for user in all_users: - # checks if they are a permanent user - if self.is_permanent_user(user): - continue - # deletes the user if they dont have an active booking - if user not in active_users: - self.deleteUser(user) - - def is_permanent_user(self, dn): - for user in self.config['permanent_users']: - if (user in dn) or (dn in user): - return True - return False - - -VPN_BaseClass.register(VPN) diff --git a/laas-fog/source/database.py b/laas-fog/source/database.py deleted file mode 100644 index ca7e5c8..0000000 --- a/laas-fog/source/database.py +++ /dev/null @@ -1,296 +0,0 @@ -""" -############################################################################# -#Copyright 2017 Parker Berberian and others # -# # -#Licensed under the Apache License, Version 2.0 (the "License"); # -#you may not use this file except in compliance with the License. # -#You may obtain a copy of the License at # -# # -# http://www.apache.org/licenses/LICENSE-2.0 # -# # -#Unless required by applicable law or agreed to in writing, software # -#distributed under the License is distributed on an "AS IS" BASIS, # -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # -#See the License for the specific language governing permissions and # -#limitations under the License. # -############################################################################# -""" -import sqlite3 -import sys -import time - - -class HostDataBase: - """ - This class talks with a simple sqlite database and can select a free host - when one is needed. - The layout of the database is: - TABLE host: - name <hostname> status <status_code> book_start - <Unix timestamp> book_end <Unix timestamp> - status_codes: - 0 - idle - 1 - deploying - 2 - deployed, in use - 3 - expired, ready to be reset - """ - - def __init__(self, path): - """ - init function. Will create the file at the end of path - if it doesnt already exist - """ - self.database = sqlite3.connect(path) - self.cursor = self.database.cursor() - - def resetHosts(self, hosts): - """ - Recreates the host table in the database. - WILL ERASE ALL DATA. USE WITH CAUTION. - """ - try: - self.cursor.execute("DROP TABLE hosts") - self.createTable() - except: - pass - - for host in hosts: - self.addHost(host) - - def createTable(self): - """ - This method creates the table hosts with - a name and status field - """ - self.cursor.execute("CREATE TABLE hosts (name text, status integer)") - self.database.commit() - - def addHost(self, name): - """ - Adds a host with name to the available hosts. - When first added, the host is assumed to be idle. - """ - host = (name, ) - self.cursor.execute("INSERT INTO hosts VALUES (?, 0) ", host) - self.database.commit() - - def getHost(self, requested=None): - """ - Returns the name of an available host. - If a host is specifically requested, - that host is returned. - If the requested host is not available, - this method will throw an error. - If no host is specificaly requested, - the next available host is returned. - """ - self.cursor.execute("SELECT name FROM hosts WHERE status = 0") - hostList = self.cursor.fetchall() - if len(hostList) < 1: - # throw and exception - sys.exit(1) - host = None - if requested is not None: - if (requested, ) in hostList and self.hostIsIdle(requested): - host = requested # If requested, exists, and idle, return it - else: - sys.exit(1) - else: - host = hostList[0][0] - self.makeHostBusy(host) - return host - - def makeHostBusy(self, name): - """ - makes the status of host 'name' equal 1, - making it 'busy' - """ - host = (name, ) - self.cursor.execute("UPDATE hosts SET status = 1 WHERE name=?", host) - self.database.commit() - - def makeHostDeployed(self, name): - """ - makes the status of host 'name' equal 2, - making it 'deployed' and/or in use - """ - host = (name, ) - self.cursor.execute("UPDATE hosts SET status = 2 WHERE name=?", host) - self.database.commit() - - def makeHostExpired(self, name): - """ - makes the status of host 'name' equal 3, - meaning its booking has ended and needs to be cleaned. - """ - host = (name, ) - self.cursor.execute("UPDATE hosts SET status = 3 WHERE name=?", host) - self.database.commit() - - def getExpiredHosts(self): - """ - returns a list of all hosts with an expired booking that - need to be cleaned. - """ - self.cursor.execute("SELECT name FROM hosts where status = 3") - host_tuples = self.cursor.fetchall() - hosts = [] - for host in host_tuples: - hosts.append(host[0]) - return hosts # returns list of strings, not tuples - - def hostIsBusy(self, name): - """ - returns True if the host is not idle - """ - host = (name, ) - self.cursor.execute("SELECT status FROM hosts WHERE name=?", host) - stat = self.cursor.fetchone()[0] - if stat < 1: - return False - return True - - def hostIsIdle(self, name): - """ - returns True if the host is idle. - """ - return not self.hostIsBusy(name) - - def getAllHosts(self): - """ - returns the whole host database. - """ - self.cursor.execute("SELECT * FROM hosts") - return self.cursor.fetchall() - - def close(self): - """ - commits and closes connection to the database file. - """ - self.database.commit() - self.database.close() - - -class BookingDataBase: - """ - Database to hold all active bookings for our servers. - Database contains table bookings - can be same or different - db file as the host database - bookings contains a field for every json key from the pharos dashboard, - plus a "status" integer which is either - 0 - waiting to start - 1 - started - 2 - booking over - - As written, the pharos listener will immediately store all bookings that - are both for your dev pods and not - yet over, regardless of when the booking starts. Once the booking ends - and the dev pod is cleaned, the booking is deleted to save space and cpu. - """ - - def __init__(self, path): - """ - creates a BookingDataBase object with the database located - at path. if path does not yet exist, it will be created. - """ - self.database = sqlite3.connect(path) - self.cursor = self.database.cursor() - - def createTable(self): - """ - Creates table in the database to store booking information - """ - try: - self.cursor.execute("DROP TABLE bookings") - except: - pass - self.cursor.execute("""CREATE TABLE bookings - (id integer, resource_id integer, start double, end double, - installer_name text, scenario_name text, - purpose text, status integer, vpn text)""") - self.database.commit() - - def checkAddBooking(self, booking): - """ - This method accepts a JSON booking definition from the dashboard - api and adds it to the database if it does not already exist. - """ - # first, check if booking is already expired - if time.time() > booking['end']: - return - # check if booking is in database already - b_id = (booking['id'], ) - self.cursor.execute("SELECT * FROM bookings WHERE id=?", b_id) - if len(self.cursor.fetchall()) > 0: # booking already in the db - return - tup = ( - booking['id'], - booking['resource_id'], - booking['start'], - booking['end'], - booking['installer_name'], - booking['scenario_name'], - booking['purpose'], - 0, - '' - ) - self.cursor.execute( - "INSERT INTO bookings VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", tup) - self.database.commit() - - def removeBooking(self, idNum): - """ - deletes booking with given id from the database. - """ - booking_id = (idNum, ) - self.cursor.execute("DELETE FROM bookings WHERE id=?", booking_id) - - def getBookings(self): - """ - returns a list of all bookings. - """ - self.cursor.execute("SELECT * FROM bookings") - return self.cursor.fetchall() - - def setStatus(self, booking_id, status): - """ - sets the status of the booking with booking id booking_id. - as noted above, the status codes are: - 0 - not yet started - 1 - started, but not yet over - 2 - over, expired - """ - data = (status, booking_id) - self.cursor.execute("UPDATE bookings SET status=? WHERE id=?", data) - self.database.commit() - - def setVPN(self, resource, uid): - data = (uid, resource, 1) - self.cursor.execute( - "UPDATE bookings SET vpn=? WHERE resource_id=? AND status=?", - data - ) - self.database.commit() - - def getVPN(self): - """ - returns a list of all vpn users associated with current - bookings. - """ - self.cursor.execute("SELECT vpn FROM bookings WHERE status=1") - users_messy = self.cursor.fetchall() - users = [] - for user in users_messy: - user = user[0] # get string rather than tuple - user = user.strip() - if len(user) < 1: - continue - users.append(user) # a list of non-empty strings - return users - - def close(self): - """ - commits changes and closes connection to db file. - """ - self.database.commit() - self.database.close() diff --git a/laas-fog/source/deploy.py b/laas-fog/source/deploy.py deleted file mode 100755 index a9c5e04..0000000 --- a/laas-fog/source/deploy.py +++ /dev/null @@ -1,82 +0,0 @@ -#!/usr/bin/python -""" -############################################################################# -#Copyright 2017 Parker Berberian and others # -# # -#Licensed under the Apache License, Version 2.0 (the "License"); # -#you may not use this file except in compliance with the License. # -#You may obtain a copy of the License at # -# # -# http://www.apache.org/licenses/LICENSE-2.0 # -# # -#Unless required by applicable law or agreed to in writing, software # -#distributed under the License is distributed on an "AS IS" BASIS, # -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # -#See the License for the specific language governing permissions and # -#limitations under the License. # -############################################################################# -""" - -import sys -import yaml -from pod_manager import Pod_Manager - -""" -This file is the first executed when a booking begins. -""" - -usage = """ -./deploy [--config CONFIG_FILE] [--host HOSTNAME] [--reset] -""" - - -def main(config_path, host): - """ - starts the deployment with the given configuration. - """ - config = yaml.safe_load(open(config_path)) - - manager = Pod_Manager(config, requested_host=host) - manager.start_deploy() - - -def reset(config_path, host): - """ - Tells the Pod Manager to clean and reset the given host. - """ - config = yaml.safe_load(open(config_path)) - Pod_Manager(config, requested_host=host, reset=True) - - -if __name__ == "__main__": - # parse command line - host = None - - if "--help" in sys.argv: - print usage - sys.exit(0) - - if "--config" in sys.argv: - try: - conf = sys.argv[1+sys.argv.index("--config")] - open(conf) - except Exception: - print "bad config file" - sys.exit(1) - if "--host" in sys.argv: - try: - host = sys.argv[1+sys.argv.index("--host")] - except: - "host not provided. Exiting" - sys.exit(1) - - try: - config_file = yaml.safe_load(open(conf)) - except: - print "Failed to read from config file" - sys.exit(1) - # reset or deploy host - if "--reset" in sys.argv: - reset(conf, host) - else: - main(conf, host) diff --git a/laas-fog/source/deployment_manager.py b/laas-fog/source/deployment_manager.py deleted file mode 100644 index f680fa5..0000000 --- a/laas-fog/source/deployment_manager.py +++ /dev/null @@ -1,108 +0,0 @@ -""" -############################################################################# -#Copyright 2017 Parker Berberian and others # -# # -#Licensed under the Apache License, Version 2.0 (the "License"); # -#you may not use this file except in compliance with the License. # -#You may obtain a copy of the License at # -# # -# http://www.apache.org/licenses/LICENSE-2.0 # -# # -#Unless required by applicable law or agreed to in writing, software # -#distributed under the License is distributed on an "AS IS" BASIS, # -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # -#See the License for the specific language governing permissions and # -#limitations under the License. # -############################################################################# -""" - -import logging -from api.libvirt_api import Libvirt - - -class Deployment_Manager: - """ - This class manages the deployment of OPNFV on a booked host - if it was requested. If no OPNFV installer was requested, this class will - create the virtual machines and networks in the config files and exit. - """ - def __init__(self, installerType, scenario, utility): - """ - init function - """ - # installerType will either be the constructor for an installer or None - self.installer = installerType - self.virt = Libvirt( - utility.host, - net_conf=utility.conf['hypervisor_config']['networks'], - dom_conf=utility.conf['hypervisor_config']['vms'] - ) - self.host = utility.host - self.util = utility - - def getIso(self): - """ - checks if any of the domains expect an ISO file to exist - and retrieves it. - """ - isoDom = None - for dom in self.doms: - if dom.iso['used']: - isoDom = dom - break - if isoDom: - path = isoDom.iso['location'] - url = isoDom.iso['URL'] - self.util.sshExec(['wget', '-q', '-O', path, url]) - - def getDomMacs(self): - """ - assigns the 'macs' instance variable to the domains - so that they know the mac addresses of their interfaces. - """ - for dom in self.doms: - dom.macs = self.virt.getMacs(dom.name) - - def makeDisks(self): - """ - Creates the qcow2 disk files the domains expect on the remote host. - """ - disks = [] - for dom in self.doms: - disks.append(dom.disk) - self.util.execRemoteScript("mkDisks.sh", disks) - - def go(self): - """ - 'main' function. - creates virtual machines/networks and either passes control to the - OPNFV installer, or finishes up if an installer was not requested. - """ - log = logging.getLogger(self.util.hostname) - self.virt.setLogger(log) - log.info("%s", "Connecting to the host hypervisor") - self.virt.openConnection() - domains, networks = self.virt.go() - log.info("%s", "Created all networks and VM's on host") - self.doms = domains - self.nets = networks - if self.installer is None: - log.warning("%s", "No installer requested. Finishing deployment") - self.util.finishDeployment() - return - log.info("%s", "retrieving ISO") - self.getIso() - self.getDomMacs() - self.util.copyScripts() - self.makeDisks() - log.info("%s", "Beginning installation of OPNFV") - try: - installer = self.installer( - self.doms, - self.nets, - self.virt, - self.util - ) - installer.go() - except Exception: - log.exception('%s', "failed to install OPNFV") diff --git a/laas-fog/source/domain.py b/laas-fog/source/domain.py deleted file mode 100644 index 6f00239..0000000 --- a/laas-fog/source/domain.py +++ /dev/null @@ -1,244 +0,0 @@ -""" -############################################################################# -#Copyright 2017 Parker Berberian and others # -# # -#Licensed under the Apache License, Version 2.0 (the "License"); # -#you may not use this file except in compliance with the License. # -#You may obtain a copy of the License at # -# # -# http://www.apache.org/licenses/LICENSE-2.0 # -# # -#Unless required by applicable law or agreed to in writing, software # -#distributed under the License is distributed on an "AS IS" BASIS, # -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # -#See the License for the specific language governing permissions and # -#limitations under the License. # -############################################################################# -""" - -import xml.dom -import xml.dom.minidom -import yaml - - -class Domain: - """ - This class defines a libvirt vm abstraction that can parse our simple - config file and add all necessary boiler plate and info to write a full xml - definition of itself for libvirt. - """ - - def __init__(self, propertiesDict): - """ - init function. - properiesDict should be one of the dictionaries returned by the static - method parseConfigFile - """ - self.name = propertiesDict['name'] - self.memory = propertiesDict['memory'] - self.vcpus = propertiesDict['vcpus'] - self.disk = propertiesDict['disk'] - self.iso = propertiesDict['iso'] - # the vm will either boot from an iso or pxe - self.netBoot = not self.iso['used'] - self.interfaces = propertiesDict['interfaces'] - - def toXML(self): - """ - combines the given configuration with a lot of - boiler plate to create a valid libvirt xml - definition of a domain. - returns a string - """ - definition = xml.dom.minidom.parseString("<domain>\n</domain>") - definition.documentElement.setAttribute('type', 'kvm') - - nameElem = definition.createElement('name') - nameElem.appendChild(definition.createTextNode(self.name)) - definition.documentElement.appendChild(nameElem) - - memElem = definition.createElement('memory') - memElem.appendChild(definition.createTextNode(str(self.memory))) - definition.documentElement.appendChild(memElem) - - curMemElem = definition.createElement('currentMemory') - curMemElem.appendChild(definition.createTextNode(str(self.memory))) - definition.documentElement.appendChild(curMemElem) - - vcpuElem = definition.createElement('vcpu') - vcpuElem.appendChild(definition.createTextNode(str(self.vcpus))) - definition.documentElement.appendChild(vcpuElem) - - osElem = definition.createElement('os') - - typeElem = definition.createElement('type') - typeElem.setAttribute('arch', 'x86_64') - typeElem.appendChild(definition.createTextNode('hvm')) - osElem.appendChild(typeElem) - - if self.netBoot: - bootElem = definition.createElement('boot') - bootElem.setAttribute('dev', 'network') - osElem.appendChild(bootElem) - - bootElem = definition.createElement('boot') - bootElem.setAttribute('dev', 'hd') - osElem.appendChild(bootElem) - - if self.iso['used']: - bootElem = definition.createElement('boot') - bootElem.setAttribute('dev', 'cdrom') - osElem.appendChild(bootElem) - - definition.documentElement.appendChild(osElem) - - featureElem = definition.createElement('feature') - featureElem.appendChild(definition.createElement('acpi')) - featureElem.appendChild(definition.createElement('apic')) - - definition.documentElement.appendChild(featureElem) - - cpuElem = definition.createElement('cpu') - cpuElem.setAttribute('mode', 'custom') - cpuElem.setAttribute('match', 'exact') - modelElem = definition.createElement('model') - modelElem.appendChild(definition.createTextNode('Broadwell')) - cpuElem.appendChild(modelElem) - - definition.documentElement.appendChild(cpuElem) - - clockElem = definition.createElement('clock') - clockElem.setAttribute('offset', 'utc') - - timeElem = definition.createElement('timer') - timeElem.setAttribute('name', 'rtc') - timeElem.setAttribute('tickpolicy', 'catchup') - clockElem.appendChild(timeElem) - - timeElem = definition.createElement('timer') - timeElem.setAttribute('name', 'pit') - timeElem.setAttribute('tickpolicy', 'delay') - clockElem.appendChild(timeElem) - - timeElem = definition.createElement('timer') - timeElem.setAttribute('name', 'hpet') - timeElem.setAttribute('present', 'no') - clockElem.appendChild(timeElem) - - definition.documentElement.appendChild(clockElem) - - poweroffElem = definition.createElement('on_poweroff') - poweroffElem.appendChild(definition.createTextNode('destroy')) - - definition.documentElement.appendChild(poweroffElem) - - rebootElem = definition.createElement('on_reboot') - rebootElem.appendChild(definition.createTextNode('restart')) - - definition.documentElement.appendChild(rebootElem) - - crashElem = definition.createElement('on_reboot') - crashElem.appendChild(definition.createTextNode('restart')) - - definition.documentElement.appendChild(crashElem) - - pmElem = definition.createElement('pm') - memElem = definition.createElement('suspend-to-mem') - memElem.setAttribute('enabled', 'no') - pmElem.appendChild(memElem) - diskElem = definition.createElement('suspend-to-disk') - diskElem.setAttribute('enabled', 'no') - pmElem.appendChild(diskElem) - - definition.documentElement.appendChild(pmElem) - - deviceElem = definition.createElement('devices') - - emuElem = definition.createElement('emulator') - emuElem.appendChild(definition.createTextNode('/usr/libexec/qemu-kvm')) - deviceElem.appendChild(emuElem) - - diskElem = definition.createElement('disk') - diskElem.setAttribute('type', 'file') - diskElem.setAttribute('device', 'disk') - - driverElem = definition.createElement('driver') - driverElem.setAttribute('name', 'qemu') - driverElem.setAttribute('type', 'qcow2') - diskElem.appendChild(driverElem) - - sourceElem = definition.createElement('source') - sourceElem.setAttribute('file', self.disk) - diskElem.appendChild(sourceElem) - - targetElem = definition.createElement('target') - targetElem.setAttribute('dev', 'hda') - targetElem.setAttribute('bus', 'ide') - diskElem.appendChild(targetElem) - - deviceElem.appendChild(diskElem) - - if self.iso['used']: - diskElem = definition.createElement('disk') - diskElem.setAttribute('type', 'file') - diskElem.setAttribute('device', 'cdrom') - - driverElem = definition.createElement('driver') - driverElem.setAttribute('name', 'qemu') - driverElem.setAttribute('type', 'raw') - diskElem.appendChild(driverElem) - - sourceElem = definition.createElement('source') - sourceElem.setAttribute('file', self.iso['location']) - diskElem.appendChild(sourceElem) - - targetElem = definition.createElement('target') - targetElem.setAttribute('dev', 'hdb') - targetElem.setAttribute('bus', 'ide') - diskElem.appendChild(targetElem) - - diskElem.appendChild(definition.createElement('readonly')) - deviceElem.appendChild(diskElem) - - for iface in self.interfaces: - ifaceElem = definition.createElement('interface') - ifaceElem.setAttribute('type', iface['type']) - sourceElem = definition.createElement('source') - sourceElem.setAttribute(iface['type'], iface['name']) - modelElem = definition.createElement('model') - modelElem.setAttribute('type', 'e1000') - ifaceElem.appendChild(sourceElem) - ifaceElem.appendChild(modelElem) - deviceElem.appendChild(ifaceElem) - - graphicElem = definition.createElement('graphics') - graphicElem.setAttribute('type', 'vnc') - graphicElem.setAttribute('port', '-1') - deviceElem.appendChild(graphicElem) - - consoleElem = definition.createElement('console') - consoleElem.setAttribute('type', 'pty') - deviceElem.appendChild(consoleElem) - - definition.documentElement.appendChild(deviceElem) - return definition.toprettyxml() - - def writeXML(self, filePath): - """ - writes this domain's xml definition to the given file. - """ - f = open(filePath, 'w') - f.write(self.toXML()) - f.close() - - @staticmethod - def parseConfigFile(path): - """ - parses the domains config file - """ - configFile = open(path, 'r') - try: - config = yaml.safe_load(configFile) - except Exception: - print "Invalid domain configuration. exiting" - return config diff --git a/laas-fog/source/installers/__init__.py b/laas-fog/source/installers/__init__.py deleted file mode 100644 index 7bb515b..0000000 --- a/laas-fog/source/installers/__init__.py +++ /dev/null @@ -1,17 +0,0 @@ -""" -############################################################################# -#Copyright 2017 Parker Berberian and others # -# # -#Licensed under the Apache License, Version 2.0 (the "License"); # -#you may not use this file except in compliance with the License. # -#You may obtain a copy of the License at # -# # -# http://www.apache.org/licenses/LICENSE-2.0 # -# # -#Unless required by applicable law or agreed to in writing, software # -#distributed under the License is distributed on an "AS IS" BASIS, # -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # -#See the License for the specific language governing permissions and # -#limitations under the License. # -############################################################################# -""" diff --git a/laas-fog/source/installers/fuel.py b/laas-fog/source/installers/fuel.py deleted file mode 100644 index c5b647c..0000000 --- a/laas-fog/source/installers/fuel.py +++ /dev/null @@ -1,268 +0,0 @@ -""" -############################################################################# -#Copyright 2017 Parker Berberian and others # -# # -#Licensed under the Apache License, Version 2.0 (the "License"); # -#you may not use this file except in compliance with the License. # -#You may obtain a copy of the License at # -# # -# http://www.apache.org/licenses/LICENSE-2.0 # -# # -#Unless required by applicable law or agreed to in writing, software # -#distributed under the License is distributed on an "AS IS" BASIS, # -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # -#See the License for the specific language governing permissions and # -#limitations under the License. # -############################################################################# -""" - -import time -import sys -from installer import Installer -from api.fuel_api import Fuel_api - - -class Fuel_Installer(Installer): - """ - This class is the installer for any OPNFV scenarios which use Fuel as the - installer. This class uses the libvirt api handler - to create all the virtual hosts, - then installs fuel and uses the fuel api handler - to create and deploy an openstack environment - - This class will get much smarter and have less configuration hardcoded - as we grow support for more OPNFV scenarios - """ - - def __init__(self, doms, nets, libvirt_handler, util): - """ - init function - Calls the super constructor - """ - super(Fuel_Installer, self).__init__(doms, nets, libvirt_handler, util) - url = 'http://'+self.libvirt.host+':8000/' - self.handler = Fuel_api(url, self.log, 'admin', 'admin') - self.fuelNets = None - - def bootMaster(self): - """ - Boots the fuel master node and waits - for it to come up - """ - self.libvirt.bootMaster() - time.sleep(100) - - def bootNodes(self): - """ - Boots all the slave nodes - """ - self.libvirt.bootSlaves() - - def waitForNodes(self, numNodes): - """ - Waits for the nodes to pxe boot and be recognized by Fuel - """ - done = False - self.log.info("Waiting for %i nodes to boot into Fuel", numNodes) - discoveredNodes = 0 - while not done: - discoveredNodes = len(self.handler.getNodes()) - nodes = int(discoveredNodes) - self.log.info("found %d nodes", nodes) - - done = discoveredNodes == numNodes - - def installMaster(self): - """ - runs the fuelInstall script, which uses the fuel iso to - install fuel onto the master node - """ - self.util.execRemoteScript("ipnat.sh", [self.libvirt.host]) - self.util.execRemoteScript("fuelInstall.sh", [self.util.remoteDir]) - - def makeOpenstack(self): - """ - creates an openstack environment and saves - the openstack id - """ - self.osid = self.handler.createOpenstack() - - def addNodesToOpenstack(self): - """ - Adds the nodes to the openstack environment with - compute / controller + cinder roles - """ - nodesList = [ - {"id": 1, "roles": ["controller", "cinder"]}, - {"id": 2, "roles": ["controller", "cinder"]}, - {"id": 3, "roles": ["controller", "cinder"]}, - {"id": 4, "roles": ["compute"]}, - {"id": 5, "roles": ["compute"]} - ] - - self.handler.addNodes(self.osid, nodesList) - - def configNetworks(self): - """ - configures the openstack networks by calling the 3 helper - methods - """ - self.configPublicNet() - self.configStorageNet() - self.configManagementNet() - - def configPublicNet(self): - """ - sets the default public network - changes the cidr, gateway, and floating ranges - """ - networks = self.handler.getNetworks(self.osid) - for net in networks['networks']: - if net['name'] == "public": - net["ip_ranges"] = [["10.20.1.10", "10.20.1.126"]] - net['cidr'] = "10.20.1.0/24" - net['gateway'] = "10.20.1.1" - - # updates the floating ranges - rng = [["10.20.1.130", "10.20.1.254"]] - networks['networking_parameters']['floating_ranges'] = rng - self.handler.uploadNetworks(networks, self.osid) - - def configStorageNet(self): - """ - sets the default storage network to have the right - cidr and gateway, and no vlan - """ - networks = self.handler.getNetworks(self.osid) - for net in networks['networks']: - if net['name'] == "storage": - net["ip_ranges"] = [["10.20.3.5", "10.20.3.254"]] - net["cidr"] = "10.20.3.0/24" - net["meta"]["notation"] = "ip_ranges" - net["meta"]["use_gateway"] = True - net["gateway"] = "10.20.3.1" - net["vlan_start"] = None - self.handler.uploadNetworks(networks, self.osid) - - def configManagementNet(self): - """ - sets the default management net to have the right - cidr and gatewar and no vlan - """ - networks = self.handler.getNetworks(self.osid) - for net in networks['networks']: - if net['name'] == "management": - net["ip_ranges"] = [["10.20.2.5", "10.20.2.254"]] - net["cidr"] = "10.20.2.0/24" - net["meta"]["notation"] = "ip_ranges" - net["meta"]["use_gateway"] = True - net["gateway"] = "10.20.2.1" - net["vlan_start"] = None - self.handler.uploadNetworks(networks, self.osid) - - # TODO: make this method smarter. I am making too many assumptions about - # the order of interfaces and networks - def configIfaces(self): - """ - assigns the proper networks to each interface of the nodes - """ - for x in range(1, 6): - idNum = x - ifaceJson = self.handler.getIfaces(idNum) - - ifaceJson[0]['assigned_networks'] = [ - {"id": 1, "name": "fuelweb_admin"}, - {"id": 5, "name": "private"} - ] - ifaceJson[2]['assigned_networks'] = [ - {"id": 4, "name": "storage"} - ] - ifaceJson[3]['assigned_networks'] = [ - {"id": 3, "name": "management"} - ] - if idNum < 4: - ifaceJson[1]['assigned_networks'] = [{ - "id": 2, - "name": "pubic" - }] - - self.handler.setIfaces(idNum, ifaceJson) - - def clearAdminIface(self, ifaceJson, node): - """ - makes the admin interface have *only* the admin network - assigned to it - """ - for iface in ifaceJson: - if iface['mac'] == node.macs['admin']: - iface['assigned_networks'] = [{ - "id": 1, - "name": "fuelweb_admin" - }] - - def deployOpenstack(self): - """ - Once openstack is properly configured, this method - deploy OS and returns when OS is running - """ - self.log.info("%s", "Deploying Openstack environment.") - self.log.info("%s", "This may take a while") - self.handler.deployOpenstack(self.osid) - - def getKey(self): - """ - Retrieves authentication tokens for the api handler, - while allowing the first few attempts to fail to - allow Fuel time to "wake up" - """ - i = 0 - while i < 20: - i += 1 - try: - self.handler.getKey() - return - except Exception: - self.log.warning("%s", "Failed to talk to Fuel api") - self.log.warning("Exec try %d/20", i) - try: - self.handler.getKey() - except Exception: - self.logger.exception("%s", "Fuel api is unavailable") - sys.exit(1) - - def go(self): - """ - This method does all the work of this class. - It installs the master node, boots the slaves - into Fuel, creates and configures OS, and then - deploys it and uses NAT to make the horizon dashboard - reachable - """ - self.libvirt.openConnection() - self.log.info('%s', 'installing the Fuel master node.') - self.log.info('%s', 'This will take some time.') - self.installMaster() - time.sleep(60) - self.getKey() - self.log.info('%s', 'The master node is installed.') - self.log.info('%s', 'Waiting for bootstrap image to build') - self.handler.waitForBootstrap() - self.bootNodes() - self.waitForNodes(5) - self.log.info('%s', "Defining an openstack environment") - self.makeOpenstack() - self.addNodesToOpenstack() - self.log.info('%s', "configuring interfaces...") - self.configIfaces() - self.log.info('%s', "configuring networks...") - self.configNetworks() - self.deployOpenstack() - - horizon = self.handler.getHorizonIP(self.osid) - self.util.execRemoteScript( - '/horizonNat.sh', [self.libvirt.host, horizon]) - notice = "You may access the Openstack dashboard at %s/horizon" - self.log.info(notice, self.libvirt.host) - - self.libvirt.close() - self.util.finishDeployment() diff --git a/laas-fog/source/installers/installer.py b/laas-fog/source/installers/installer.py deleted file mode 100644 index d4c4889..0000000 --- a/laas-fog/source/installers/installer.py +++ /dev/null @@ -1,35 +0,0 @@ -""" -############################################################################# -#Copyright 2017 Parker Berberian and others # -# # -#Licensed under the Apache License, Version 2.0 (the "License"); # -#you may not use this file except in compliance with the License. # -#You may obtain a copy of the License at # -# # -# http://www.apache.org/licenses/LICENSE-2.0 # -# # -#Unless required by applicable law or agreed to in writing, software # -#distributed under the License is distributed on an "AS IS" BASIS, # -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # -#See the License for the specific language governing permissions and # -#limitations under the License. # -############################################################################# -""" - - -class Installer(object): - """ - This is a simple base class to define a single constructor - for all the different installer types. - I may move more functionality to this class as we add support for more - installers and there becomes common fucntions that would be nice to share - between installers. - """ - - def __init__(self, domList, netList, libvirt_handler, util): - self.doms = domList - self.nets = netList - self.libvirt = libvirt_handler - self.osid = 0 - self.util = util - self.log = util.createLogger(util.hostname) diff --git a/laas-fog/source/installers/joid.py b/laas-fog/source/installers/joid.py deleted file mode 100644 index a3f3bcf..0000000 --- a/laas-fog/source/installers/joid.py +++ /dev/null @@ -1,40 +0,0 @@ -""" -############################################################################# -#Copyright 2017 Parker Berberian and others # -# # -#Licensed under the Apache License, Version 2.0 (the "License"); # -#you may not use this file except in compliance with the License. # -#You may obtain a copy of the License at # -# # -# http://www.apache.org/licenses/LICENSE-2.0 # -# # -#Unless required by applicable law or agreed to in writing, software # -#distributed under the License is distributed on an "AS IS" BASIS, # -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # -#See the License for the specific language governing permissions and # -#limitations under the License. # -############################################################################# -""" - -""" -This class will install Joid onto the remote host. -Currently only supports joid's "default" configuration -""" - - -class Joid_Installer: - - def __init__(self, doms, nets, libvirt_handler, util): - """ - init function calls the super constructor - """ - super(Joid_Installer, self).__init__(doms, nets, libvirt_handler, util) - - def go(self): - """ - does all the work of this class. - Currently just runs the joidInstall script, which installs joid - onto the remote host - """ - self.logger.info("%s", "Executing joid virtual installation") - self.util.execRemoteScript("joidInstall.sh") diff --git a/laas-fog/source/listen.py b/laas-fog/source/listen.py deleted file mode 100755 index ed714c9..0000000 --- a/laas-fog/source/listen.py +++ /dev/null @@ -1,59 +0,0 @@ -#!/usr/bin/python -""" -############################################################################# -#Copyright 2017 Parker Berberian and others # -# # -#Licensed under the Apache License, Version 2.0 (the "License"); # -#you may not use this file except in compliance with the License. # -#You may obtain a copy of the License at # -# # -# http://www.apache.org/licenses/LICENSE-2.0 # -# # -#Unless required by applicable law or agreed to in writing, software # -#distributed under the License is distributed on an "AS IS" BASIS, # -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # -#See the License for the specific language governing permissions and # -#limitations under the License. # -############################################################################# -""" -import subprocess -import sys -import os -import yaml - -""" -This is the file that the user will execute to start the whole process. -This file will start the pharos api listener in a new process and then exit. -""" - - -def checkArgs(): - """ - error checks the cmd line args and gets the path - of the config file - """ - usage = "./listen.py --config <path_to_pharos_config>" - if "--help" in sys.argv: - print usage - sys.exit(0) - - if "--config" not in sys.argv: - print usage - sys.exit(1) - - try: - i = sys.argv.index("--config") - config_file = sys.argv[i+1] - # verifies that the file exists, is readable, and formatted correctly - yaml.safe_load(open(config_file)) - return config_file - except Exception: - print "Bad config file" - sys.exit(1) - - -# reads args and starts the pharos listener in the background -config = checkArgs() -source_dir = os.path.dirname(os.path.realpath(__file__)) -pharos_path = os.path.join(source_dir, "pharos.py") -subprocess.Popen(['/usr/bin/python', pharos_path, '--config', config]) diff --git a/laas-fog/source/network.py b/laas-fog/source/network.py deleted file mode 100644 index 234ba22..0000000 --- a/laas-fog/source/network.py +++ /dev/null @@ -1,103 +0,0 @@ -""" -############################################################################# -#Copyright 2017 Parker Berberian and others # -# # -#Licensed under the Apache License, Version 2.0 (the "License"); # -#you may not use this file except in compliance with the License. # -#You may obtain a copy of the License at # -# # -# http://www.apache.org/licenses/LICENSE-2.0 # -# # -#Unless required by applicable law or agreed to in writing, software # -#distributed under the License is distributed on an "AS IS" BASIS, # -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # -#See the License for the specific language governing permissions and # -#limitations under the License. # -############################################################################# -""" - -import sys -import xml.dom -import xml.dom.minidom -import yaml - - -class Network: - """ - This class has a similar role as the Domain class. - This class will parse a config file and - write the xml definitions of those networks for libvirt. - """ - - def __init__(self, propertiesDict): - """ - init. propertiesDict should be - one of the dictionaries returned by parseConfigFile - """ - self.name = propertiesDict['name'] - self.brName = propertiesDict['brName'] - self.brAddr = propertiesDict['brAddr'] - self.netmask = propertiesDict['netmask'] - self.forward = propertiesDict['forward'] - self.dhcp = propertiesDict['dhcp'] - self.cidr = propertiesDict['cidr'] - - def toXML(self): - """ - Takes the config of this network and writes a valid xml definition - for libvirt. - returns a string - """ - definition = xml.dom.minidom.parseString("<network>\n</network>") - nameElem = definition.createElement('name') - nameElem.appendChild(definition.createTextNode(self.name)) - definition.documentElement.appendChild(nameElem) - - if self.forward['used']: - forwardElem = definition.createElement('forward') - forwardElem.setAttribute('mode', self.forward['type']) - definition.documentElement.appendChild(forwardElem) - - bridgeElem = definition.createElement('bridge') - bridgeElem.setAttribute('name', self.brName) - bridgeElem.setAttribute('stp', 'on') - bridgeElem.setAttribute('delay', '5') - definition.documentElement.appendChild(bridgeElem) - - ipElem = definition.createElement('ip') - ipElem.setAttribute('address', self.brAddr) - ipElem.setAttribute('netmask', self.netmask) - if self.dhcp['used']: - dhcpElem = definition.createElement('dhcp') - rangeElem = definition.createElement('range') - rangeElem.setAttribute('start', self.dhcp['rangeStart']) - rangeElem.setAttribute('end', self.dhcp['rangeEnd']) - dhcpElem.appendChild(rangeElem) - ipElem.appendChild(dhcpElem) - - definition.documentElement.appendChild(ipElem) - - self.xml = definition.toprettyxml() - return self.xml - - def writeXML(self, filePath): - """ - writes xml definition to given file - """ - f = open(filePath, 'w') - f.write(self.toXML()) - f.close() - - @staticmethod - def parseConfigFile(path): - """ - parses given config file - """ - configFile = open(path, 'r') - try: - config = yaml.safe_load(configFile) - except Exception: - print "Bad network configuration file. exiting" - sys.exit(1) - - return config diff --git a/laas-fog/source/pharos.py b/laas-fog/source/pharos.py deleted file mode 100755 index d5a6e8a..0000000 --- a/laas-fog/source/pharos.py +++ /dev/null @@ -1,217 +0,0 @@ -#!/usr/bin/python -""" -############################################################################# -#Copyright 2017 Parker Berberian and others # -# # -#Licensed under the Apache License, Version 2.0 (the "License"); # -#you may not use this file except in compliance with the License. # -#You may obtain a copy of the License at # -# # -# http://www.apache.org/licenses/LICENSE-2.0 # -# # -#Unless required by applicable law or agreed to in writing, software # -#distributed under the License is distributed on an "AS IS" BASIS, # -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # -#See the License for the specific language governing permissions and # -#limitations under the License. # -############################################################################# -""" - -import requests -import time -import calendar -import subprocess -import sys -import yaml -import os -import logging -from utilities import Utilities -from database import BookingDataBase - - -class Pharos_api: - """ - This class listens to the dashboard and starts/stops bookings accordingly. - This class should run in the background indefinitely. - Do not execute this file directly - run ./listen.py instead - """ - def __init__(self, config): - """ - init function. - config is the already-parsed config file - """ - self.conf = config - self.servers = yaml.safe_load(open(config['inventory'])) - self.log = self.createLogger("pharos_api") - self.polling = 60 / int(config['polling']) - self.log.info( - "polling the dashboard once every %d seconds", self.polling) - self.dashboard = config['dashboard'] - self.log.info("connecting to dashboard at %s", self.dashboard) - if os.path.isfile(config['token']): - self.token = open(config['token']).read() - else: - self.token = config['token'] - self.updateHeader() - self.database = BookingDataBase(config['database']) - self.log.info("using database at %s", self.conf['database']) - self.deploy_path = os.path.join( - os.path.dirname(os.path.realpath(__file__)), "deploy.py") - if not os.path.isfile(self.deploy_path): - self.log.error( - "Cannot find the deployment script at %s", self.deploy_path) - - def setToken(self, token): - """ - Sets authentication token. Not yet needed. - """ - self.token = token - self.updateHeader() - - def setTokenFromFile(self, path): - """ - reads auth token from a file. Not yet needed. - """ - self.setToken(open(path).read()) - - def updateHeader(self): - """ - updates the http header used when talking to the dashboard - """ - self.header = {"Authorization": "Token " + self.token} - - def listen(self): - """ - this method will continuously poll the pharos dashboard. - If a booking is found on our server, - we will start a deployment in the background with the - proper config file for the requested - installer and scenario. - """ - self.log.info("%s", "Beginning polling of dashboard") - try: - while True: - time.sleep(self.polling) - url = self.dashboard+"/api/bookings/" - bookings = requests.get(url, headers=self.header).json() - for booking in bookings: - if booking['resource_id'] in self.servers.keys(): - self.convertTimes(booking) - self.database.checkAddBooking(booking) - self.checkBookings() - except Exception: - self.log.exception('%s', "failed to connect to dashboard") - - self.listen() - - def convertTimes(self, booking): - """ - this method will take the time reported by Pharos in the - format yyyy-mm-ddThh:mm:ssZ - and convert it into seconds since the epoch, - for easier management - """ - booking['start'] = self.pharosToEpoch(booking['start']) - booking['end'] = self.pharosToEpoch(booking['end']) - - def pharosToEpoch(self, timeStr): - """ - Converts the dates from the dashboard to epoch time. - """ - time_struct = time.strptime(timeStr, '%Y-%m-%dT%H:%M:%SZ') - epoch_time = calendar.timegm(time_struct) - return epoch_time - - def checkBookings(self): - """ - This method checks all the bookings in our database to see if any - action is required. - """ - # get all active bookings from database into a usable form - bookings = self.database.getBookings() - for booking in bookings: - # first, check if booking is over - if time.time() > booking[3]: - self.log.info("ending the booking with id %i", booking[0]) - self.endBooking(booking) - # Then check if booking has begun and the host is still idle - elif time.time() > booking[2] and booking[7] < 1: - self.log.info("starting the booking with id %i", booking[0]) - self.startBooking(booking) - - def startBooking(self, booking): - """ - Starts the scheduled booking on the requested host with - the correct config file. - The provisioning process gets spun up in a subproccess, - so the api listener is not interupted. - """ - try: - host = self.servers[booking[1]] - self.log.info("Detected a new booking started for host %s", host) - config_file = self.conf['default_configs']["None"] - try: - config_file = self.conf['default_configs'][booking[4]] - except KeyError: - self.log.warning( - "No installer detected in the booking request.") - self.log.info("New booking started for host %s", host) - self.database.setStatus(booking[0], 1) # mark booking started - if not os.path.isfile(self.deploy_path): - error = "Cannot find the deploment script at %s" - self.log.error(error, self.deploy_path) - subprocess.Popen([ - '/usr/bin/python', - self.deploy_path, - '--config', config_file, - '--host', host - ]) - except Exception: - self.log.exception("Failed to start booking for %s", host) - - def endBooking(self, booking): - """ - Resets a host once its booking has ended. - """ - try: - try: - config_file = self.conf['default_configs'][booking[4]] - except KeyError: - warn = "No installer detected in booking request" - self.log.warning("%s", warn) - config_file = self.conf['default_configs']["None"] - - host = self.servers[booking[1]] - log = logging.getLogger(host) - log.info('Lease expired. Resetting host %s', host) - self.database.setStatus(booking[0], 3) - if not os.path.isfile(self.deploy_path): - err = "Cannot find deployment script at %s" - self.log.error(err, self.deploy_path) - subprocess.Popen([ - '/usr/bin/python', - self.deploy_path, - '--config', config_file, - '--host', host, - '--reset' - ]) - self.database.removeBooking(booking[0]) - except Exception: - self.log.exception("Failed to end booking for %s", host) - - def createLogger(self, name): - return Utilities.createLogger(name, self.conf['logging_dir']) - - -if __name__ == "__main__": - if "--config" not in sys.argv: - print "Specify config file with --config option" - sys.exit(1) - config = None - try: - config_file = sys.argv[1+sys.argv.index('--config')] - config = yaml.safe_load(open(config_file)) - except Exception: - sys.exit(1) - api = Pharos_api(config) - api.listen() diff --git a/laas-fog/source/pod_manager.py b/laas-fog/source/pod_manager.py deleted file mode 100755 index 3e1caa8..0000000 --- a/laas-fog/source/pod_manager.py +++ /dev/null @@ -1,144 +0,0 @@ -#!/usr/bin/python -""" -############################################################################# -#Copyright 2017 Parker Berberian and others # -# # -#Licensed under the Apache License, Version 2.0 (the "License"); # -#you may not use this file except in compliance with the License. # -#You may obtain a copy of the License at # -# # -# http://www.apache.org/licenses/LICENSE-2.0 # -# # -#Unless required by applicable law or agreed to in writing, software # -#distributed under the License is distributed on an "AS IS" BASIS, # -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # -#See the License for the specific language governing permissions and # -#limitations under the License. # -############################################################################# -""" - -import time -import sys -import yaml -import os -from api.fog import FOG_Handler -from utilities import Utilities -from deployment_manager import Deployment_Manager -from database import HostDataBase -from installers import fuel -from installers import joid - - -class Pod_Manager: - """ - This is the 'main' class that chooses a host and provisions & deploys it. - this class can be run directly from the command line, - or it can be called from the pharos dashboard listener when - a deployment is requested. - Either way, this file should be called with: - ./pod_manager.py --config <CONFIG_FILE> - """ - # This dictionary allows me to map the supported installers to the - # respective installer classes, for easier parsing of the config file - INSTALLERS = { - "fuel": fuel.Fuel_Installer, - "joid": joid.Joid_Installer, - "none": None - } - - def __init__(self, conf, requested_host=None, reset=False): - """ - init function. - conf is the read and parsed config file for this deployment - requested_host is the optional hostname of the host you request - if reset, we just flash the host to a clean state and return. - """ - self.conf = conf - if self.conf['installer'] is not None: - inst = Pod_Manager.INSTALLERS[self.conf['installer'].lower()] - self.conf['installer'] = inst - self.fog = FOG_Handler(self.conf['fog']['server']) - # Sets the fog keys, either from the config file - # or the secrets file the config points to - if os.path.isfile(self.conf['fog']['api_key']): - self.fog.getFogKeyFromFile(self.conf['fog']['api_key']) - else: - self.fog.setFogKey(self.conf['fog']['api_key']) - - if os.path.isfile(self.conf['fog']['user_key']): - self.fog.getUserKeyFromFile(self.conf['fog']['user_key']) - else: - self.fog.setUserKey(self.conf['fog']['user_key']) - self.database = HostDataBase(self.conf['database']) - self.request = requested_host - if reset: - mac = self.fog.getHostMac(self.request) - log = self.conf['dhcp_log'] - dhcp_serv = self.conf['dhcp_server'] - ip = Utilities.getIPfromMAC(mac, log, remote=dhcp_serv) - self.flash_host(self.request, ip) - - def start_deploy(self): - """ - Ghosts the machine with the proper disk image and hands off - control to the deployment manager. - """ - try: - host = self.database.getHost(self.request) - hostMac = self.fog.getHostMac(host) - dhcp_log = self.conf['dhcp_log'] - dhcp_server = self.conf['dhcp_server'] - host_ip = Utilities.getIPfromMAC( - hostMac, dhcp_log, remote=dhcp_server - ) - util = Utilities(host_ip, host, self.conf) - util.resetKnownHosts() - log = Utilities.createLogger(host, self.conf['logging_dir']) - self.fog.setLogger(log) - log.info("Starting booking on host %s", host) - log.info("host is reachable at %s", host_ip) - log.info('ghosting host %s with clean image', host) - self.flash_host(host, host_ip, util) - log.info('Host %s imaging complete', host) - inst = self.conf['installer'] - scenario = self.conf['scenario'] - Deployment_Manager(inst, scenario, util).go() - except Exception: - log.exception("Encountered an unexpected error") - - def flash_host(self, host, host_ip, util=None): - """ - We do this using a FOG server, but you can use whatever fits into your - lab infrastructure. This method should put the host into a state as if - centos was just freshly installed, updated, - and needed virtualization software installed. - This is the 'clean' starting point we work from - """ - self.fog.setImage(host, self.conf['fog']['image_id']) - self.fog.imageHost(host) - Utilities.restartRemoteHost(host_ip) - self.fog.waitForHost(host) - # if util is not given, then we are just - # flashing to reset after a booking expires - if util is not None: - time.sleep(30) - util.waitForBoot() - util.checkHost() - time.sleep(15) - util.checkHost() - - -if __name__ == "__main__": - configFile = "" - host = "" - for i in range(len(sys.argv) - 1): - if "--config" in sys.argv[i]: - configFile = sys.argv[i+1] - elif "--host" in sys.argv[i]: - host = sys.argv[i+1] - if len(configFile) < 1: - print "No config file specified" - sys.exit(1) - configFile = yaml.safe_load(open(configFile)) - manager = Pod_Manager(configFile, requested_host=host) - manager.start_deploy() diff --git a/laas-fog/source/resetDataBase.py b/laas-fog/source/resetDataBase.py deleted file mode 100755 index ff141e5..0000000 --- a/laas-fog/source/resetDataBase.py +++ /dev/null @@ -1,110 +0,0 @@ -#!/usr/bin/python -""" -############################################################################# -#Copyright 2017 Parker Berberian and others # -# # -#Licensed under the Apache License, Version 2.0 (the "License"); # -#you may not use this file except in compliance with the License. # -#You may obtain a copy of the License at # -# # -# http://www.apache.org/licenses/LICENSE-2.0 # -# # -#Unless required by applicable law or agreed to in writing, software # -#distributed under the License is distributed on an "AS IS" BASIS, # -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # -#See the License for the specific language governing permissions and # -#limitations under the License. # -############################################################################# -""" - -import sys -import os -import yaml -from api.fog import FOG_Handler -from database import HostDataBase -from database import BookingDataBase - -""" -This file just resets the host database with -all the hosts in fog, with all of them -showing as available - -This file is just provided to make populating the host db easier. -If you wanted to do this yourself, you could do the following in -a python command prompt: - from database import HostDataBase - db = HostDataBase("/path/to/file") - db.addHost("host-name") - db.addHost("host-name") - db.addHost("host-name") - -""" -config = None -if "--config" in sys.argv: - i = sys.argv.index("--config") - if len(sys.argv) > i+1 and os.path.isfile(sys.argv[i+1]): - try: - config = yaml.safe_load(open(sys.argv[i+1])) - except Exception: - print "failed to read config file. exiting" - sys.exit(1) - else: - print "config file not found. exiting" - sys.exit(1) -else: - print "no config file given. Specify file with '--config <FILE_PATH>'" - sys.exit(1) - -host = False -if "--host" in sys.argv or "--both" in sys.argv: - host = True - -booking = False -if "--booking" in sys.argv or "--both" in sys.argv: - booking = True - - -if host: - - fog = FOG_Handler( - config['fog']['server'] - ) - if os.path.isfile(config['fog']['api_key']): - fog.getFogKeyFromFile(config['fog']['api_key']) - else: - fog.setFogKey(config['fog']['api_key']) - - if os.path.isfile(config['fog']['user_key']): - fog.getUserKeyFromFile(config['fog']['user_key']) - else: - fog.setUserKey(config['fog']['user_key']) - hosts = fog.getHostsinGroup("vm") - host_names = [] - for host in hosts: - host_names.append(host['name']) - - # creates the directory of the db, if it doesnt yet exist - dbDir = os.path.dirname(config['database']) - if not os.path.isdir(dbDir): - os.makedirs(dbDir) - - db = HostDataBase(config['database']) - - # check if the table already exists or not - try: - db.cursor.execute("SELECT * FROM hosts") - except Exception as err: - if "no such table" in str(err): - db.createTable() - - db.resetHosts(host_names) - -if booking: - db = BookingDataBase(config['database']) - db.createTable() - db.close() - -else: - print "you must specify the '--host', '--booking', or '--both' option" - print "depending on which database you wish to reset" - sys.exit(0) diff --git a/laas-fog/source/stop.sh b/laas-fog/source/stop.sh deleted file mode 100755 index e721482..0000000 --- a/laas-fog/source/stop.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash -############################################################################# -#Copyright 2017 Parker Berberian and others # -# # -#Licensed under the Apache License, Version 2.0 (the "License"); # -#you may not use this file except in compliance with the License. # -#You may obtain a copy of the License at # -# # -# http://www.apache.org/licenses/LICENSE-2.0 # -# # -#Unless required by applicable law or agreed to in writing, software # -#distributed under the License is distributed on an "AS IS" BASIS, # -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # -#See the License for the specific language governing permissions and # -#limitations under the License. # -############################################################################# - - -# This just finds all processes from this program and kills them. - - -PIDS=$(ps -ef | grep laas/source/ | grep python | awk '{print $2}') - -kill ${PIDS[*]} diff --git a/laas-fog/source/utilities.py b/laas-fog/source/utilities.py deleted file mode 100644 index bbe0946..0000000 --- a/laas-fog/source/utilities.py +++ /dev/null @@ -1,346 +0,0 @@ -""" -############################################################################# -#Copyright 2017 Parker Berberian and others # -# # -#Licensed under the Apache License, Version 2.0 (the "License"); # -#you may not use this file except in compliance with the License. # -#You may obtain a copy of the License at # -# # -# http://www.apache.org/licenses/LICENSE-2.0 # -# # -#Unless required by applicable law or agreed to in writing, software # -#distributed under the License is distributed on an "AS IS" BASIS, # -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # -#See the License for the specific language governing permissions and # -#limitations under the License. # -############################################################################# -""" - -import os -import logging -import string -import sys -import subprocess -import xml.dom -import xml.dom.minidom -import re -import random -import yaml -from database import HostDataBase, BookingDataBase -from api.vpn import VPN -LOGGING_DIR = "" - - -class Utilities: - """ - This class defines some useful functions that may be needed - throughout the provisioning and deployment stage. - The utility object is carried through most of the deployment process. - """ - def __init__(self, host_ip, hostname, conf): - """ - init function - host_ip is the ip of the target host - hostname is the FOG hostname of the host - conf is the parsed config file - """ - self.host = host_ip - self.hostname = hostname - root_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) - self.scripts = os.path.join(root_dir, "hostScripts/") - self.remoteDir = "/root/hostScripts/" - self.conf = conf - self.logger = logging.getLogger(hostname) - - def execRemoteScript(self, script, args=[]): - """ - executes the given script on the - remote host with the given args. - script must be found in laas/hostScripts - """ - cmd = [self.remoteDir+script] - for arg in args: - cmd.append(arg) - self.sshExec(cmd) - - def waitForBoot(self): - """ - Continually pings the host, waiting for it to boot - """ - i = 0 - while (not self.pingHost()) and i < 30: - i += 1 - if i == 30: - self.logger.error("Host %s has not booted", self.host) - sys.exit(1) - - def checkHost(self): - """ - returns true if the host responds to two pings. - Sometimes, while a host is pxe booting, a host will - respond to one ping but quickly go back offline. - """ - if self.pingHost() and self.pingHost(): - return True - return False - - def pingHost(self): - """ - returns true if the host responds to a ping - """ - i = 0 - response = 1 - cmd = "ping -c 1 "+self.host - cmd = cmd.split(' ') - nul = open(os.devnull, 'w') - while i < 10 and response != 0: - response = subprocess.call(cmd, stdout=nul, stderr=nul) - i = i + 1 - if response == 0: - return True - return False - - def copyDir(self, localDir, remoteDir): - """ - uses scp to copy localDir to remoteDir on the - remote host - """ - cmd = "mkdir -p "+remoteDir - self.sshExec(cmd.split(" ")) - cmd = "scp -o StrictHostKeyChecking=no -r " - cmd += localDir+" root@"+self.host+":/root" - cmd = cmd.split() - nul = open(os.devnull, 'w') - subprocess.call(cmd, stdout=nul, stderr=nul) - - def copyScripts(self): - """ - Copies the hostScrpts dir to the remote host. - """ - self.copyDir(self.scripts, self.remoteDir) - - def sshExec(self, args): - """ - executes args as an ssh - command on the remote host. - """ - cmd = ['ssh', 'root@'+self.host] - for arg in args: - cmd.append(arg) - nul = open(os.devnull, 'w') - return subprocess.call(cmd, stdout=nul, stderr=nul) - - def resetKnownHosts(self): - """ - edits your known hosts file to remove the previous entry of host - Sometimes, the flashing process gives the remote host a new - signature, and ssh complains about it. - """ - lines = [] - sshFile = open('/root/.ssh/known_hosts', 'r') - lines = sshFile.read() - sshFile.close() - lines = lines.split('\n') - sshFile = open('/root/.ssh/known_hosts', 'w') - for line in lines: - if self.host not in line: - sshFile.write(line+'\n') - sshFile.close() - - def restartHost(self): - """ - restarts the remote host - """ - cmd = ['shutdown', '-r', 'now'] - self.sshExec(cmd) - - @staticmethod - def randoString(length): - """ - this is an adapted version of the code found here: - https://stackoverflow.com/questions/2257441/ - random-string-generation-with-upper-case-letters-and-digits-in-python - generates a random alphanumeric string of length length. - """ - randStr = '' - chars = string.ascii_uppercase + string.digits - for x in range(length): - randStr += random.SystemRandom().choice(chars) - return randStr - - def changePassword(self): - """ - Sets the root password to a random string and returns it - """ - paswd = self.randoString(15) - command = "printf "+paswd+" | passwd --stdin root" - self.sshExec(command.split(' ')) - return paswd - - def markHostDeployed(self): - """ - Tells the database that this host has finished its deployment - """ - db = HostDataBase(self.conf['database']) - db.makeHostDeployed(self.hostname) - db.close() - - def make_vpn_user(self): - """ - Creates a vpn user and associates it with this booking - """ - config = yaml.safe_load(open(self.conf['vpn_config'])) - myVpn = VPN(config) - # name = dashboard.getUserName() - u, p, uid = myVpn.makeNewUser() # may pass name arg if wanted - self.logger.info("%s", "created new vpn user") - self.logger.info("username: %s", u) - self.logger.info("password: %s", p) - self.logger.info("vpn user uid: %s", uid) - self.add_vpn_user(uid) - - def add_vpn_user(self, uid): - """ - Adds the dn of the vpn user to the database - so that we can clean it once the booking ends - """ - db = BookingDataBase(self.conf['database']) - # converts from hostname to pharos resource id - inventory = yaml.safe_load(open(self.conf['inventory'])) - host_id = -1 - for resource_id in inventory.keys(): - if inventory[resource_id] == self.hostname: - host_id = resource_id - break - db.setVPN(host_id, uid) - - def finishDeployment(self): - """ - Last method call once a host is finished being deployed. - It notifies the database and changes the password to - a random string - """ - self.markHostDeployed() - self.make_vpn_user() - passwd = self.changePassword() - self.logger.info("host %s provisioning done", self.hostname) - self.logger.info("You may access the host at %s", self.host) - self.logger.info("The password is %s", passwd) - notice = "You should change all passwords for security" - self.logger.warning('%s', notice) - - @staticmethod - def restartRemoteHost(host_ip): - """ - This method assumes that you already have ssh access to the target - """ - nul = open(os.devnull, 'w') - ret_code = subprocess.call([ - 'ssh', '-o', 'StrictHostKeyChecking=no', - 'root@'+host_ip, - 'shutdown', '-r', 'now'], - stdout=nul, stderr=nul) - - return ret_code - - @staticmethod - def getName(xmlString): - """ - Gets the name value from xml. for example: - <name>Parker</name> returns Parker - """ - xmlDoc = xml.dom.minidom.parseString(xmlString) - nameNode = xmlDoc.documentElement.getElementsByTagName('name') - name = str(nameNode[0].firstChild.nodeValue) - return name - - @staticmethod - def getXMLFiles(directory): - """ - searches directory non-recursively and - returns a list of all xml files - """ - contents = os.listdir(directory) - fileContents = [] - for item in contents: - if os.path.isfile(os.path.join(directory, item)): - fileContents.append(os.path.join(directory, item)) - xmlFiles = [] - for item in fileContents: - if 'xml' in os.path.basename(item): - xmlFiles.append(item) - return xmlFiles - - @staticmethod - def createLogger(name, log_dir=LOGGING_DIR): - """ - Initializes the logger if it does not yet exist, and returns it. - Because of how python logging works, calling logging.getLogger() - with the same name always returns a reference to the same log file. - So we can call this method from anywhere with the hostname as - the name arguement and it will return the log file for that host. - The formatting includes the level of importance and the time stamp - """ - global LOGGING_DIR - if log_dir != LOGGING_DIR: - LOGGING_DIR = log_dir - log = logging.getLogger(name) - if len(log.handlers) > 0: # if this logger is already initialized - return log - log.setLevel(10) - han = logging.FileHandler(os.path.join(log_dir, name+".log")) - han.setLevel(10) - log_format = '[%(levelname)s] %(asctime)s [#] %(message)s' - formatter = logging.Formatter(fmt=log_format) - han.setFormatter(formatter) - log.addHandler(han) - return log - - @staticmethod - def getIPfromMAC(macAddr, logFile, remote=None): - """ - searches through the dhcp logs for the given mac - and returns the associated ip. Will retrieve the - logFile from a remote host if remote is given. - if given, remote should be an ip address or hostname that - we can ssh to. - """ - if remote is not None: - logFile = Utilities.retrieveFile(remote, logFile) - ip = Utilities.getIPfromLog(macAddr, logFile) - if remote is not None: - os.remove(logFile) - return ip - - @staticmethod - def retrieveFile(host, remote_loc, local_loc=os.getcwd()): - """ - Retrieves file from host and puts it in the current directory - unless local_loc is given. - """ - subprocess.call(['scp', 'root@'+host+':'+remote_loc, local_loc]) - return os.path.join(local_loc, os.path.basename(remote_loc)) - - @staticmethod - def getIPfromLog(macAddr, logFile): - """ - Helper method for getIPfromMAC. - uses regex to find the ip address in the - log - """ - try: - messagesFile = open(logFile, "r") - allLines = messagesFile.readlines() - except Exception: - sys.exit(1) - importantLines = [] - for line in allLines: - if macAddr in line and "DHCPACK" in line: - importantLines.append(line) - ipRegex = r'(\d+\.\d+\.\d+\.\d+)' - IPs = [] - for line in importantLines: - IPs.append(re.findall(ipRegex, line)) - if len(IPs) > 0 and len(IPs[-1]) > 0: - return IPs[-1][0] - return None |