From 4b269fba0ca273dfa3acf44c9f5490f01e0c3d87 Mon Sep 17 00:00:00 2001 From: Trevor Bramwell Date: Fri, 22 Sep 2017 12:23:36 -0700 Subject: Rename pharos-dashboard and pharos-validator As subdirectories of the pharos-tools repo, there is little need to keep the pharos prefix. Change-Id: Ica3d79411f409df638647300036c0664183c2725 Signed-off-by: Trevor Bramwell --- validator/src/validation_tool/src/__init__.py | 0 validator/src/validation_tool/src/config.py | 176 +++++++++++++++++++++ validator/src/validation_tool/src/const.py | 48 ++++++ validator/src/validation_tool/src/dhcp.py | 102 ++++++++++++ validator/src/validation_tool/src/ipmi.py | 63 ++++++++ validator/src/validation_tool/src/jenkins.py | 8 + validator/src/validation_tool/src/node.py | 85 ++++++++++ validator/src/validation_tool/src/receiver.py | 46 ++++++ validator/src/validation_tool/src/server.py | 111 +++++++++++++ validator/src/validation_tool/src/test/__init__.py | 0 validator/src/validation_tool/src/test/evaluate.py | 159 +++++++++++++++++++ validator/src/validation_tool/src/test/probe.py | 137 ++++++++++++++++ validator/src/validation_tool/src/util.py | 107 +++++++++++++ 13 files changed, 1042 insertions(+) create mode 100644 validator/src/validation_tool/src/__init__.py create mode 100644 validator/src/validation_tool/src/config.py create mode 100644 validator/src/validation_tool/src/const.py create mode 100644 validator/src/validation_tool/src/dhcp.py create mode 100644 validator/src/validation_tool/src/ipmi.py create mode 100644 validator/src/validation_tool/src/jenkins.py create mode 100644 validator/src/validation_tool/src/node.py create mode 100644 validator/src/validation_tool/src/receiver.py create mode 100644 validator/src/validation_tool/src/server.py create mode 100644 validator/src/validation_tool/src/test/__init__.py create mode 100644 validator/src/validation_tool/src/test/evaluate.py create mode 100644 validator/src/validation_tool/src/test/probe.py create mode 100644 validator/src/validation_tool/src/util.py (limited to 'validator/src/validation_tool/src') diff --git a/validator/src/validation_tool/src/__init__.py b/validator/src/validation_tool/src/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/validator/src/validation_tool/src/config.py b/validator/src/validation_tool/src/config.py new file mode 100644 index 0000000..443467e --- /dev/null +++ b/validator/src/validation_tool/src/config.py @@ -0,0 +1,176 @@ +############################################################################## +# Copyright (c) 2015 Todd Gaunt and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +import logging +import sys +import os +import yaml +import struct +import socket + +from pharosvalidator import util +from collections import namedtuple + +class Topology(): + """ + Topology: Class to store any number of Network classes + and metadata about them + """ + def __init__(self, yaml_config): + # Dictionary of available networks + self.logger = logging.getLogger(__name__) + self.networks = {} + self.external_networks = [] + + # Fill the above dictionaries + self.parse_yaml(yaml_config) + + def parse_yaml(self, yaml_config): + """ + parse_yaml: parses the yaml configuration file this program uses + for all the network and node information + """ + config = safe_yaml_read(yaml_config) + for network in config["networks"]: + self.logger.info("Reading network section {}".format(network)) + if network == "admin": + self.networks[network] = Network(config["networks"][network]) + #TODO + elif network == "external": + for external_network in config["networks"][network]: + self.external_networks.append(Network(external_network)) + +class Network(): + """ + Network: Class to store all information on a given network + """ + def __init__(self, network): + try: + self.logger = logging.getLogger(__name__) + + # Some generic settings + self.enabled = network["enabled"] + self.vlan = network["vlan"] + + # VM settings + self.installer_nic_type = network["installer_vm"]["nic_type"] + self.installer_members = network["installer_vm"]["members"] + self.installer_ip = network["installer_vm"]["ip"] + + # Tuple containing the minimum and maximum + self.usable_ip_range = self.parse_ip_range(network["usable_ip_range"]) + self.gateway = network["gateway"] + self.cidr = network["cidr"] + self.dhcp_range = network["dhcp_range"] + self.dns_domain = network["dns-domain"] + self.dns_search = network["dns-search"] + + subnet, netmask = self.split_cidr(network["cidr"]) + self.subnet = subnet + self.netmask = netmask + + # List of all dns servers + self.dns_upstream = network["dns-upstream"] + + self.nic_mapping = {} + except KeyError as e: + self.logger.error("Field {} not available in network configuration file".format(e)) + + def split_cidr(self, cidr): + """ + split_cidr: Split up cidr notation subnets into a subnet string and a + netmask string + + input: cidr notation of a subnet + + output: Subnet string; Netmask string + """ + split = cidr.split('/') + host_bits = int(split[1]) + netmask = self.cidr_to_netmask(host_bits) + subnet = split[0] + + return subnet, netmask + + def parse_ip_range(self, ip_range_string): + """ + parse_ip_range: Create a named tuple object that contains the lowest + ip address and the highest ip address from a configuration file + + input: String formatted like so "min, max" where min/max are ip addresses + + output: Named tuple object containing a minimum and maximum field + """ + rp = ip_range_string.split(",") + ip_range = namedtuple("ip_range", ['minimum', 'maximum'])(minimum=min(rp), maximum=max(rp)) + return ip_range + + def cidr_to_netmask(self, cidr): + bits = 0xffffffff ^ (1 << 32 - cidr) - 1 + netmask = socket.inet_ntoa(struct.pack('>I', bits)) + self.logger.debug("Netmask generated from cidr '{}': '{}'".format(cidr, netmask)) + return netmask + +class Inventory(): + """ + Inventory: Class to hold configuration file data + """ + def __init__(self, yaml_config): + # Create the class logger + self.logger = logging.getLogger(__name__) + + self.nodes = [] + + # Fill the above list + self.parse_yaml(yaml_config) + + def parse_yaml(self, yaml_config): + config = safe_yaml_read(yaml_config) + nodes = [] + for node in config["nodes"]: + self.nodes.append(Node(node)) + + def nodecount(self): + return len(self.nodes) + +class Node(): + """ + Node: Class to hold + """ + def __init__(self, node): + self.logger = logging.getLogger(__name__) + try: + self.name = node["name"] + self.tags = node["tags"] + self.arch = node["arch"] + self.mac_address = node["mac_address"] # ipmi mac address + self.cpus = node["cpus"] + self.memory = node["memory"] + self.disk = node["disk"] + except KeyError as e: + self.logger.error("Field {} not available in inventory file".format(e)) + + # Power sub section + if node["power"]["type"] == "ipmi": + try: + self.ipmi_addr = node["power"]["address"] + self.ipmi_user = node["power"]["user"] + self.ipmi_pass = node["power"]["pass"] + except KeyError as e: + self.logger.error("Field {} not available in inventory file".format(e)) + else: + pass + +def safe_yaml_read(yamlfile): + logger = logging.getLogger(__name__) + if os.path.isfile(yamlfile) == False: + logger.critical("Could not open find {}".format(yamlfile)) + quit(1) + with open(yamlfile, 'r') as fd: + return yaml.load(fd.read()) diff --git a/validator/src/validation_tool/src/const.py b/validator/src/validation_tool/src/const.py new file mode 100644 index 0000000..a204a96 --- /dev/null +++ b/validator/src/validation_tool/src/const.py @@ -0,0 +1,48 @@ +############################################################################## +# Copyright (c) 2015 Todd Gaunt and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +## Various constant strings used throughout program +HARDWARE_TEST="pharos-validator-node" + +## Pharos hardware specification +# memory +MIN_MEMSIZE = 32000000 # In Kb + +# cpu +MIN_CPUFREQ = 1800.000 # In Mhz +MIN_CORECOUNT = 4 + +# storage +MIN_DISKCOUNT = 3 +MIN_SSDCOUNT = 1 +MIN_HDDSIZE = 1000 # In Gb +MIN_SSDSIZE = 100 # In Gb +# Smallest possible disk size +MIN_DISKSIZE = min(MIN_HDDSIZE, MIN_SSDSIZE) + +# Virtual deployments +# Requirements are per node +APEX_REQ = {"cores": 2, \ + "ram": 8000000, \ + "disk": 40} + +# Requirements are per node +COMPASS_REQ = {"cores": 4, \ + "ram": 4000000, \ + "disk": 100} + +# Requirements are per node +JOID_REQ = {"cores": 4, \ + "ram": 4000000, \ + "disk": 100} + +# Requirements are per node +FUEL_REQ = {"cores": 4, \ + "ram": 4000000, \ + "disk": 100} diff --git a/validator/src/validation_tool/src/dhcp.py b/validator/src/validation_tool/src/dhcp.py new file mode 100644 index 0000000..26c42f8 --- /dev/null +++ b/validator/src/validation_tool/src/dhcp.py @@ -0,0 +1,102 @@ +############################################################################## +# Copyright (c) 2015 Todd Gaunt and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +import yaml +import netifaces +import subprocess +import copy +import re +import os +import logging + +from pharosvalidator.specification import * +from pharosvalidator import util + +init_cmd = ["systemctl", "start", "dhcpd.service"] + +def gen_dhcpd_file(dhcpdfile, nodes, network): + """Generates and associates incremental ip addresses to + MAC addresses according to restrictions placed by network + configuration file. Writes all of this out in dhcpd.conf format""" + logger = logging.getLogger(__name__) + logger.info("Generating dhcpfile...") + + header = "default-lease-time 86400;\n\ + max-lease-time 604800;\n\ + max-lease-time 604800;\n\ + \n\ + allow booting;\n\ + authoritative;\n\ + \n" + + # Skip this network if it is disabled + if network.enabled == False: + logger.info("Admin network is disabled, please change the configuration to \"enabled\" if you would like this test to run") + quit() + + # Not explicitly in the cofiguration file + broadcastaddr = "0.0.0.0" + next_server = "0.0.0.0" + + ip_range = util.gen_ip_range(network.cidr, [network.installer_ip], network.usable_ip_range.minimum, \ + network.usable_ip_range.maximum) + + tab = ' ' + subnetconf = "subnet {} netmask {} {{\n".format(network.subnet, network.netmask)\ + + tab+"range {} {};\n".format(network.usable_ip_range.minimum, network.usable_ip_range.maximum)\ + + tab+"option broadcast-address {};\n".format(broadcastaddr)\ + + tab+'filename "pxelinux.0";\n'\ + + tab+"next-server {};\n".format(next_server) + + # For now no static addresses are assigned + """ + static_addrs = [] + for node in nodes: + # Skip the node if it doesn't have a name or mac address specified + if not node.name or not node.mac_address: + continue + + if node.ipmi_addr in ip_range: + ip_range.remove(node.ipmi_addr) + + static_line = "host {node} {{ hardware ethernet {ipmi_mac}; fixed-address {ip_addr}; }}\n".format\ + (node=node.name, ipmi_mac=node.mac_address, ip_addr=ip_range[0]) + ip_range = ip_range[1::] # Remove the assigned ip address + static_addrs.append(static_line) + + # Now add all statically assigned ip addresses + for addr in static_addrs: + subnetconf += tab+addr + """ + + subnetconf += "}\n" # Just the closing bracket + + # The final text to be written out to a file + dhcpdtext = header + subnetconf + + with open(dhcpdfile, "w+") as fd: + logger.info("Writing out dhcpd file to {}".format(dhcpdfile)) + fd.write(dhcpdtext) + + return dhcpdtext + +def start_server(): + logger = logging.getLogger(__name__) + global init_cmd + cmd = init_cmd + with open(os.devnull, 'w') as fn: + status = subprocess.Popen(cmd, stdout=fn, stderr=fn).wait() + if int(status) != 0: + logger.error("Could not bring up dhcpd server") + else: + logger.info("Dhcp server brought up") + return status + +if __name__ == "__main__": + split("inventory.yaml", "eth0") diff --git a/validator/src/validation_tool/src/ipmi.py b/validator/src/validation_tool/src/ipmi.py new file mode 100644 index 0000000..44be207 --- /dev/null +++ b/validator/src/validation_tool/src/ipmi.py @@ -0,0 +1,63 @@ +############################################################################## +# Copyright (c) 2015 Todd Gaunt and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +import os +import subprocess +import logging + +def power_nodes(nodes, action): + """ Attempts to power on all nodes specified in a list, then returns a list + of the names of all failures. The list will be empty if no failures.""" + failed_nodes = [] + logger = logging.getLogger(__name__) + if not nodes: + logger.info("No nodes, is empty list") + for node in nodes: + # -I flag must be 'lanplus', 'lan' by itself doesn't work with + # the most recent idrac/ipmi version + if action == "on": + pass + elif action == "off": + pass + else: + logger.error("Invalid ipmi command") + + cmd = ["ipmitool", \ + "-I", "lanplus", \ + "-H ", "'"+node.ipmi_addr+"'", \ + "-U ", "'"+node.ipmi_user+"'", \ + "-P ", "'"+node.ipmi_pass+"'", \ + "power", action] + + logger.debug("Running: \"{}\"".format(' '.join(cmd))) + try: + with open(os.devnull, 'w') as fn: + status = subprocess.check_call(" ".join(cmd), \ + stdout=fn, stderr=fn, shell=True) + except subprocess.CalledProcessError as e: + status = e.returncode + logger.error("{} could not be accessed at {} (exit code {})".format(\ + node.name, node.ipmi_addr, status)) + failed_nodes.append(node.name) + if int(status) == 0: + logger.info("{} successfully powered {}".format(node.name, action)) + + return failed_nodes + +def status(node, ipaddr, username, passwd): + # -I flag must be 'lanplus', 'lan' by itself doesn't work with + # the most recent idrac/ipmi version + chkcmd = ["ipmitool", \ + "-I", "lanplus", \ + "-H", ipaddr, \ + "-U", username, \ + "-P", passwd, \ + "chassis", "status"] + print(chkcmd) + subprocess.Popen(chkcmd) diff --git a/validator/src/validation_tool/src/jenkins.py b/validator/src/validation_tool/src/jenkins.py new file mode 100644 index 0000000..443a615 --- /dev/null +++ b/validator/src/validation_tool/src/jenkins.py @@ -0,0 +1,8 @@ +############################################################################## +# Copyright (c) 2015 Todd Gaunt and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## diff --git a/validator/src/validation_tool/src/node.py b/validator/src/validation_tool/src/node.py new file mode 100644 index 0000000..280abb7 --- /dev/null +++ b/validator/src/validation_tool/src/node.py @@ -0,0 +1,85 @@ +############################################################################## +# Copyright (c) 2015 Todd Gaunt and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +import logging +import socket +import yaml +import os + +import pharosvalidator.test.probe as probe +import pharosvalidator.test.evaluate as evaluate +from pharosvalidator.util import send_msg + +def hardware_test(): + """ + hardware_test: Run hardware probing/testing functions + + input: None + + output: String in YAML format of the tests that were run + """ + logger = logging.getLogger(__name__) + logger.info("Beginning hardware test") + + # Run test scripts + results = [] + results.append(testinterpreter("CPU test", evaluate.cpu, probe.cpu())) + results.append(testinterpreter("Memory test", evaluate.memory, probe.memory())) + results.append(testinterpreter("Storage test", evaluate.storage, probe.storage())) + + # Start generating the yaml file + yamltext = "" + for result in results: + yamltext += yaml.dump(result, default_flow_style=False) + return yamltext + +def network_test(networkfile): + logger = logging.getLogger(__name__) + logger.info("Beginning network test") + logger.info("Ending network test") + pass + +def send_result(host, port, result): + """ + send_result: Send the final test result to the central test server + + input: Host address of target; Port of target; String to send to server + + output: None + """ + logger = logging.getLogger(__name__) + logger.info("Sending test result") + + # Format the results properly + linecount = 0 + for c in result: + if c == "\n": + linecount += 1 + + result = str(linecount) + "\n" + result + + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.connect((host, int(port))) + send_msg(sock, result) + +def testinterpreter(name, test, dataset): + """High level function for test functions within this module to print out + their results in an ordered function while also writing out logs, + expects a list of testresults objects""" + + # Start the yaml file contents + data = {name:[]} + + # test the dataset + results = test(dataset) + + for result in results: + data[name].append(result) + + return data diff --git a/validator/src/validation_tool/src/receiver.py b/validator/src/validation_tool/src/receiver.py new file mode 100644 index 0000000..07d968e --- /dev/null +++ b/validator/src/validation_tool/src/receiver.py @@ -0,0 +1,46 @@ +############################################################################## +# Copyright (c) 2015 Todd Gaunt and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +import socket +import threading +import logging + +from pharosvalidator.util import read_msg + +def start(nodecount, port, q): + """Start a server to retrieve the files from the nodes. Server will run + indefinetely until the parent process ends""" + logging.basicConfig(level=0) + logger = logging.getLogger(__name__) + + address = "" # Empty means act as a server on all available interfaces + + logger.info("Bringing up receiver server...") + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.bind((address, port)) + sock.listen(nodecount) # Max connections is the amount of nodes + + while True: + # Receive a descriptor for the client socket, cl stands for client + (clsock, claddress) = sock.accept() + logger.info("Received client connection...") + client_thread = threading.Thread(target=_server_accept_thread, \ + args=(clsock, claddress, q), daemon=True) + # Start a new thread to read the new client socket connection + client_thread.start() + + socket.close() + logger.info("Bringing down receiver server...") + +def _server_accept_thread(clsock, claddress, q): + """Read from the socket into the queue, then close the connection""" + logger = logging.getLogger(__name__) + q.put(read_msg(clsock)) + logger.info("Retreived message from socket") + clsock.close() diff --git a/validator/src/validation_tool/src/server.py b/validator/src/validation_tool/src/server.py new file mode 100644 index 0000000..91c9a4f --- /dev/null +++ b/validator/src/validation_tool/src/server.py @@ -0,0 +1,111 @@ +############################################################################## +# Copyright (c) 2015 Todd Gaunt and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +import logging +import os +import subprocess +import time + +# Constant definitions +from pharosvalidator.const import * + +def ssh_thread(remoteaddr, returnaddr, port, passes): + """ + ssh_thread: the main loop of a thread the server spawns to connect to a node + over ssh. + + input: remoteaddr, returnaddr, and port to forward to run_remote_test; + passes to specify how many attempts should be made + """ + for i in range(passes): + status = run_remote_test(remoteaddr, returnaddr, port) + time.sleep(1) + +def run_remote_test(remoteaddr, returnaddr, port): + """ + run_remote_tests: ssh to a give remote address, and run a test program + on the remote machine specifying the address and port of where the results + should be sent (usually back to the machine this program was run on) + + input: ip address of the ssh target; Adress of the test results target; + Port of the test results target + + output: 0 if the test ran over ssh perfectly, non-zero if the test faild + """ + #TODO add way to keep attempting to ssh until service is up and running aka ping part 2 + logger = logging.getLogger(__name__) + + cmd = ["ssh", "root@"+remoteaddr, HARDWARE_TEST, \ + "-p", port, "-H", returnaddr, "hardware"] + + logger.debug("Running: {}".format(" ".join(cmd))) + try: + with open(os.devnull, 'w') as fn: + status = subprocess.check_call(" ".join(cmd), stdout=fn, stderr=fn, shell=True) + except subprocess.CalledProcessError as e: + status = e.returncode + logger.error("ssh attempt to '{}' failed".format(remoteaddr)) + + return status + +def ping_network(ip_range_list, ipcnt, passes): + """ + ping_network: Ping a range of ips until the amount of successful pings + reaches a number n + + input: List of ip addresses to be pinged; Counter for threshold + of successful pings; Number of iterations to pass + + output: List of ip addresses that were found to be up + """ + logger = logging.getLogger("pharosvalidator") + assert isinstance(ip_range_list, list) + ips_found = 0 + alive_ips = [] + for t in range(passes): + for addr in list(ip_range_list): + cmd = [ \ + "ping", \ + "-c", "1", \ + "-w", "1", \ + addr] + logger.debug("Running: \"{}\"".format(' '.join(cmd))) + try: + with open(os.devnull, 'w') as fn: + status = subprocess.check_call(" ".join(cmd), \ + stdout=fn, stderr=fn, shell=True) + except subprocess.CalledProcessError as e: + status = e.returncode + logger.error("Ping at '{}' failed".format(addr)) + # If the ip address was pinged successfully, then remove it from future attempts + if status == 0: + ips_found += 1 + logger.info("{} is up, {} total nodes up".format(addr, ips_found)) + + # Remove the ip that was successfully pinged from being tested again + ip_range_list.remove(addr) + + # Add the successfully pinged node to a list of successful pings + alive_ips.append(addr) + + if ips_found >= ipcnt: + break + + if ips_found >= ipcnt: + break + + return alive_ips + +def bring_up_admin_ip(ipaddr): + """ + Assign the machine this test is running on an address according to the + configuration file + """ + cmd = [""] + subprocess.Popen(cmd) diff --git a/validator/src/validation_tool/src/test/__init__.py b/validator/src/validation_tool/src/test/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/validator/src/validation_tool/src/test/evaluate.py b/validator/src/validation_tool/src/test/evaluate.py new file mode 100644 index 0000000..81a837d --- /dev/null +++ b/validator/src/validation_tool/src/test/evaluate.py @@ -0,0 +1,159 @@ +############################################################################## +# Copyright (c) 2015 Todd Gaunt and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +import logging + +from pharosvalidator.util import approxsize + +# Constant macros +from pharosvalidator.const import * + +def cpu(cpudata): + """Compares system cpu against the pharos specification""" + results = [] + + # Architecture evaluation, a value of 63 or greater indicates at least a 64-bit OS + if cpudata["bitsize"] >= 63: + val = True + else: + val = False + result = {"architecture": { + "pass": val, + "description": str(cpudata["architecture"])}} + results.append(result) + + # Core evaluation + if cpudata["cores"] < MIN_CORECOUNT: + val = False + else: + val = True + desc = "Have {0}, Need at least {1}".format(cpudata["cores"], MIN_CORECOUNT) + result = {"corecount": { + "pass": val, + "description": desc}} + results.append(result) + + # Speed evaluation + i = 0 + for cpufreq in cpudata["frequency"]: + # Cpufrequency was not read if this is the result + if cpufreq == -1: + desc = "(Cpu freuency could not be read)" + else: + if approxsize(cpufreq, MIN_CPUFREQ, 5) or cpufreq > MIN_CPUFREQ: + val = True + else: + val = False + desc = "Have {:.2f}Mhz, Need at least ~= {:.2f}Mhz".format( \ + cpufreq, MIN_CPUFREQ) + result = {"cpu"+str(i): { + "pass": val, + "description": desc}} + results.append(result) + i += 1 + + return results + +def memory(memdata): + """Compares system meminfo object against the pharos specification""" + logger = logging.getLogger(__name__) + + results = [] + + logger.debug("required memory: {}, detected memory: {}".format(\ + MIN_MEMSIZE, memdata["size"])) + # Capacity evaluation + if approxsize(memdata["size"], MIN_MEMSIZE, 5) or memdata["size"] > MIN_MEMSIZE: + val = True + else: + val = False + + desc = "Have {:.2f}G, Need at least ~= {:.2f}G".format( \ + memdata["size"], MIN_MEMSIZE/1000000) + + result = {"memory capacity": { + "pass": val, + "description": desc}} + results.append(result) + + return results + +def storage(diskdata): + """Compares system storage against the Pharos specification""" + def sizecmp(a, b, unit): + if approxsize(a, b, 10) or a > b: + val = True + else: + val = False + desc = "capacity is {:.2f}{}, Need at least ~= {:.2f}{}".format(a, \ + unit, b, unit) + return (val,desc) + + results = [] + # Disk size evaluation (also counts the disks) + diskcount = {"ssd":0, "non-ssd":0} + for disk in diskdata["names"]: + if diskdata["rotational"][disk]: + disktype = "non-ssd" + diskcount["non-ssd"] += 1 + else: + disktype = "ssd" + diskcount["ssd"] += 1 + val, desc = sizecmp(diskdata["sizes"][disk], MIN_SSDSIZE, 'G') + data = diskdata["sizes"][disk] + result = {disk: { + "pass": val, + "description": "Disk type: disktype; " + desc}} + results.append(result) + + # Disk number evaluation + if sum(diskcount.values()) >= 3 and diskcount["ssd"] >= 1: + val = True + else: + val = False + desc = "Have {0} drives, Need at least {1} drives and {3} ssds".format( \ + sum(diskcount.values()), MIN_DISKCOUNT, \ + diskcount["ssd"], MIN_SSDCOUNT) + + data = diskcount + result = {"diskcount": { + "pass": val, + "description": desc}} + results.append(result) + return results + +""" +def netinterfaces(netfaces): + results = [] + for netface in netfaces: + if netface.status <= 0: + val = False + state = "down" + else: + val = True + state = "up" + try: + MACaddr = netface.MAC[0]["addr"] + except IndexError: + MACaddr = "no MAC" + if len(netface.addrs) > 0: + addrs = "" + for addr in netface.addrs: + if len(addrs) > 0: + addrs += ", " + addrs += addr['addr'] + addrs = "addresses: " + addrs + else: + addrs = "no address" + desc = "({0} is {1} with {2})".format(netface.name, state, addrs) + data = MACaddr + results.append(gen_yamltext(netface.name, val, desc, data)) + return results + """ + diff --git a/validator/src/validation_tool/src/test/probe.py b/validator/src/validation_tool/src/test/probe.py new file mode 100644 index 0000000..daeccbc --- /dev/null +++ b/validator/src/validation_tool/src/test/probe.py @@ -0,0 +1,137 @@ +############################################################################## +# Copyright (c) 2015 Todd Gaunt and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +import os +import re +import sys +import platform +import subprocess +import netifaces +import logging + +from pharosvalidator.util import cd # Contains the pharos specification values + +# Static vars +mempath="/proc/meminfo" +cpuinfopath="/proc/cpuinfo" +cpupath="/sys/devices/system/cpu/" +diskpath="/sys/block/" + +def readproc(path): + """Reads and parses /proc from [path] argument files + and returns a hashmap of values""" + logger = logging.getLogger(__name__) + # Fail if path does not exist + try: + hashmap = {} + with open(path) as fd: + logger.debug("Reading {}".format(path)) + for line in fd: + data = line.split(":") + if len(data) == 2: + # Strip trailing characters from hashmap names and entries + # for less junk + hashmap[data[0].strip()] = data[1].strip() + return hashmap + except IOError: + logger.error("Path to file does not exist: {}".format(path)) + quit(1) + +def cpu(): + logger = logging.getLogger(__name__) + cpudata = {} + cpuinfo = readproc(cpuinfopath) + cpudata["bitsize"] = sys.maxsize.bit_length() + cpudata["architecture"] = platform.architecture()[0] + cpudata["cores"] = int(cpuinfo["cpu cores"]) + cpudata["frequency"] = [] + for i in range(cpudata["cores"]): + freqpath = "{0}/cpu{1}/cpufreq/cpuinfo_max_freq".format(cpupath, \ + str(i)) + try: + with open(freqpath) as fd: + logger.debug("Opening {}".format(freqpath)) + cpufreq = (float(fd.read(-1)))/1000 + except IOError: + # Less accurate way of getting cpu information as + # this frequency may change during operation, + # if dynamic frequency scaling is enabled, + # however it is better than nothing. + logger.error("Path to file does not exist: {}".format(freqpath)) + logger.error("Reading cpu frequency from {} instead".format(freqpath)) + cpufreq = float(cpuinfo["cpu MHz"]) + + if cpufreq < 0: + cpudata["frequency"].append(0) + else: + cpudata["frequency"].append(cpufreq) + + return cpudata + +def memory(): + logger = logging.getLogger(__name__) + meminfo=readproc(mempath) + # Create the memory object to store memory information + memdata = {} + memdata["size"] = (int(meminfo["MemTotal"].split(' ')[0]))/1000000 + return memdata + +def storage(): + """Gather's disk information""" + logger = logging.getLogger(__name__) + diskdata = {"names":[],"rotational":{},"sizes":{}} + for disk in os.listdir(diskpath): + #sdX is the naming schema for IDE/SATA interfaces in Linux + if re.match(r"sd\w",disk): + logger.debug("Found disk {}".format(disk)) + diskdata["names"].append(disk) + sizepath = "{0}/{1}/size".format(diskpath, disk) + try: + with open(sizepath) as fd: + size = int(fd.read(-1)) + except IOError: + size = -1 + # If the read was successful + if size != -1: + # Converts the value to Gb + diskdata["sizes"][disk] = (size * 512)/1000000000 + + rotationalpath = "{0}/{1}/queue/rotational".format(diskpath, disk) + try: + with open(rotationalpath) as fd: + rotational = int(fd.read(-1)) + except IOError: + rotational = -1 + if rotational == 0: + diskdata["rotational"][disk] = False + else: + diskdata["rotational"][disk] = True + + return diskdata + +def netinterfaces(nodeinfo): + """Uses netifaces to probe the system for network interface information""" + netfaces = [] + for interface in netifaces.interfaces(): + netface = netdata() + netface.name = interface + tmp = netifaces.ifaddresses(interface) + # If the interface is up and has at least one ip address + if netifaces.AF_INET in tmp: + netface.status = 1 # 1 stands for "up" + netface.addrs = tmp[netifaces.AF_INET] + # If the interface is down + else: + netface.status = 0 # 0 stands for "down" + # The file /proc/net/arp may also be used to read MAC addresses + if netifaces.AF_LINK in tmp: + netface.MAC = tmp[netifaces.AF_LINK] + netfaces.append(netface) + + return netfaces diff --git a/validator/src/validation_tool/src/util.py b/validator/src/validation_tool/src/util.py new file mode 100644 index 0000000..67a75a5 --- /dev/null +++ b/validator/src/validation_tool/src/util.py @@ -0,0 +1,107 @@ +############################################################################## +# Copyright (c) 2015 Todd Gaunt and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +import ipaddress +import logging +import os + +class cd: + """Context manager for changing the current working directory""" + def __init__(self, new_path): + self.new_path = os.path.expanduser(new_path) + + def __enter__(self): + self.saved_path = os.getcwd() + os.chdir(self.new_path) + + def __exit__(self, etype, value, traceback): + os.chdir(self.saved_path) + +def approxsize(x, y, deviation): + """Approximately compares 'x' to 'y' with in % of 'deviation'""" + logger = logging.getLogger(__name__) + + dev = (y * .01 * deviation) + + if x >= round(y - dev, 0) and x <= round(y + dev, 0): + logger.debug("{} is approximately {}".format(x, y)) + return True + else: + logger.debug("{} is not approximately {}".format(x, y)) + return False + +def read_line(sock): + """Reads from a socket until a \n character or 512 bytes have been read, + whichever comes first""" + c = "" + recvline = "" + reads = 0 + while (c != "\n" and reads < 512): + # Decode bytes to str, sockets output bytes which aren't pretty + c = sock.recv(1).decode("utf-8") + #print("char: '" + c + "'") # Debugging code + recvline += c + reads += 1 + return recvline + +def read_msg(sock): + """Reads a message prefixed with a number and a newline char, eg. "20\n" + then reads x lines, where x is equal to the number in the first line.""" + # Read the socket once initially for the line count + buf = read_line(sock) + buf = buf[:-1] # Cut off the '\n' character + length = int(buf) + + lines = [] + for i in range(length): + lines.append(read_line(sock)) + return "".join(lines) + +def send_msg(sock, msg): + """Sends a message to a socket""" + # Encode python str to bytes beforehand, sockets only deal in bytes + msg = bytes(msg, "utf-8") + totalsent = 0 + while totalsent < len(msg): + sent = sock.send(msg[totalsent:]) + if sent == 0: + return -1 + totalsent = totalsent + sent + return totalsent + +def get_addr(interface): + """Get the address of the machine that this program is running on""" + return netifaces.ifaddresses(interface)[netifaces.AF_INET][0]["addr"] + +def gen_ip_range(cidr, excluded, minimum, maximum ): + """Takes a network cidr number, and then a min max value, and creates a list + of ip addresses avalable on [a,b]. Also removes "excluded" addresses + from the range""" + logger = logging.getLogger(__name__) + # Generate a list of available ip addresses for the dhcp server + ip_range = list(map(lambda x: x.exploded, ipaddress.ip_network(cidr).hosts())) + + for addr in excluded: + # Remove the value from the list, if it isn't in the list then whatever + try: + ip_range.remove(addr) + except ValueError: + logger.debug("{} not in ip_range, cannot remove".format(addr)) + + # Remove all values before the minimum usable value + for i in range(len(ip_range)): + if ip_range[i] == minimum: + ip_range = ip_range[i::] + break + # Remove all values after the maximum usable value + for i in range(len(ip_range)): + if ip_range[i] == maximum: + ip_range = ip_range[0:i+1] + break + return ip_range -- cgit 1.2.3-korg