diff options
author | Tim Rozet <trozet@redhat.com> | 2017-06-25 21:25:36 -0400 |
---|---|---|
committer | Tim Rozet <trozet@redhat.com> | 2017-08-23 08:59:54 -0400 |
commit | f4d388ea508ba00771e43a219ac64e0d430b73bd (patch) | |
tree | 4f61a89664474154c3d6f7adecfbb0396617199c /apex/settings | |
parent | 807fad268c90649f2901c5f5c4cdeb788a0308e0 (diff) |
Migrates Apex to Python
Removes all bash libraries and converts almost all of the code to a
mixture of Python and Ansible. utils.sh and clean.sh still exist.
clean.sh will be migrated fully to clean.py in another patch.
The Apex Python package is now built into the opnfv-apex-common RPM. To
install locally do 'pip3 install .'. To deploy:
opnfv-deploy -d <file> -n <file> --image-dir /root/apex/.build -v --debug
Non-python files (THT yaml, settings files, ansible playbooks) are all
installed into /usr/share/opnfv-apex/. The RPM will copy settings files
into /etc/opnfv-apex/.
JIRA: APEX-317
Change-Id: I3232f0329bcd13bce5a28da6a8c9c84d0b048024
Signed-off-by: Tim Rozet <trozet@redhat.com>
Diffstat (limited to 'apex/settings')
-rw-r--r-- | apex/settings/__init__.py | 0 | ||||
-rw-r--r-- | apex/settings/deploy_settings.py | 188 | ||||
-rw-r--r-- | apex/settings/network_settings.py | 327 |
3 files changed, 515 insertions, 0 deletions
diff --git a/apex/settings/__init__.py b/apex/settings/__init__.py new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/apex/settings/__init__.py diff --git a/apex/settings/deploy_settings.py b/apex/settings/deploy_settings.py new file mode 100644 index 00000000..c8e347b7 --- /dev/null +++ b/apex/settings/deploy_settings.py @@ -0,0 +1,188 @@ +############################################################################## +# Copyright (c) 2016 Michael Chapman (michapma@redhat.com) and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + + +import yaml + +from apex.common import utils +from apex.common import constants + +REQ_DEPLOY_SETTINGS = ['sdn_controller', + 'odl_version', + 'tacker', + 'congress', + 'dataplane', + 'sfc', + 'vpn', + 'vpp', + 'ceph', + 'gluon', + 'rt_kvm'] + +OPT_DEPLOY_SETTINGS = ['performance', + 'vsperf', + 'ceph_device', + 'yardstick', + 'dovetail', + 'odl_vpp_routing_node', + 'odl_vpp_netvirt', + 'barometer'] + +VALID_ROLES = ['Controller', 'Compute', 'ObjectStorage'] +VALID_PERF_OPTS = ['kernel', 'nova', 'vpp', 'ovs'] +VALID_DATAPLANES = ['ovs', 'ovs_dpdk', 'fdio'] +VALID_ODL_VERSIONS = ['carbon', 'nitrogen', 'master'] + + +class DeploySettings(dict): + """ + This class parses a APEX deploy settings yaml file into an object + + Currently the parsed object is dumped into a bash global definition file + for deploy.sh consumption. This object will later be used directly as + deployment script move to python. + """ + def __init__(self, filename): + if isinstance(filename, str): + with open(filename, 'r') as deploy_settings_file: + init_dict = yaml.safe_load(deploy_settings_file) + else: + # assume input is a dict to build from + init_dict = filename + + super().__init__(init_dict) + self._validate_settings() + + def _validate_settings(self): + """ + Validates the deploy settings file provided + + DeploySettingsException will be raised if validation fails. + """ + + if 'deploy_options' not in self: + raise DeploySettingsException("No deploy options provided in" + " deploy settings file") + if 'global_params' not in self: + raise DeploySettingsException("No global options provided in" + " deploy settings file") + + deploy_options = self['deploy_options'] + if not isinstance(deploy_options, dict): + raise DeploySettingsException("deploy_options should be a list") + + if ('gluon' in self['deploy_options'] and + 'vpn' in self['deploy_options']): + if (self['deploy_options']['gluon'] is True and + self['deploy_options']['vpn'] is False): + raise DeploySettingsException( + "Invalid deployment configuration: " + "If gluon is enabled, " + "vpn also needs to be enabled") + + for setting, value in deploy_options.items(): + if setting not in REQ_DEPLOY_SETTINGS + OPT_DEPLOY_SETTINGS: + raise DeploySettingsException("Invalid deploy_option {} " + "specified".format(setting)) + if setting == 'dataplane': + if value not in VALID_DATAPLANES: + planes = ' '.join(VALID_DATAPLANES) + raise DeploySettingsException( + "Invalid dataplane {} specified. Valid dataplanes:" + " {}".format(value, planes)) + + for req_set in REQ_DEPLOY_SETTINGS: + if req_set not in deploy_options: + if req_set == 'dataplane': + self['deploy_options'][req_set] = 'ovs' + elif req_set == 'ceph': + self['deploy_options'][req_set] = True + elif req_set == 'odl_version': + self['deploy_options'][req_set] = \ + constants.DEFAULT_ODL_VERSION + else: + self['deploy_options'][req_set] = False + elif req_set == 'odl_version' and self['deploy_options'][ + 'odl_version'] not in VALID_ODL_VERSIONS: + raise DeploySettingsException( + "Invalid ODL version: {}".format(self[deploy_options][ + 'odl_version'])) + + if 'performance' in deploy_options: + if not isinstance(deploy_options['performance'], dict): + raise DeploySettingsException("Performance deploy_option" + "must be a dictionary.") + for role, role_perf_sets in deploy_options['performance'].items(): + if role not in VALID_ROLES: + raise DeploySettingsException("Performance role {}" + "is not valid, choose" + "from {}".format( + role, + " ".join(VALID_ROLES) + )) + + for key in role_perf_sets: + if key not in VALID_PERF_OPTS: + raise DeploySettingsException("Performance option {} " + "is not valid, choose" + "from {}".format( + key, + " ".join( + VALID_PERF_OPTS) + )) + + def _dump_performance(self): + """ + Creates performance settings string for bash consumption. + + Output will be in the form of a list that can be iterated over in + bash, with each string being the direct input to the performance + setting script in the form <role> <category> <key> <value> to + facilitate modification of the correct image. + """ + bash_str = 'performance_options=(\n' + deploy_options = self['deploy_options'] + for role, settings in deploy_options['performance'].items(): + for category, options in settings.items(): + for key, value in options.items(): + bash_str += "\"{} {} {} {}\"\n".format(role, + category, + key, + value) + bash_str += ')\n' + bash_str += '\n' + bash_str += 'performance_roles=(\n' + for role in self['deploy_options']['performance']: + bash_str += role + '\n' + bash_str += ')\n' + bash_str += '\n' + + return bash_str + + def _dump_deploy_options_array(self): + """ + Creates deploy settings array in bash syntax. + """ + bash_str = '' + for key, value in self['deploy_options'].items(): + if not isinstance(value, bool): + bash_str += "deploy_options_array[{}]=\"{}\"\n".format(key, + value) + else: + bash_str += "deploy_options_array[{}]={}\n".format(key, + value) + return bash_str + + +class DeploySettingsException(Exception): + def __init__(self, value): + self.value = value + + def __str__(self): + return self.value diff --git a/apex/settings/network_settings.py b/apex/settings/network_settings.py new file mode 100644 index 00000000..14870078 --- /dev/null +++ b/apex/settings/network_settings.py @@ -0,0 +1,327 @@ +############################################################################## +# Copyright (c) 2016 Feng Pan (fpan@redhat.com) and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +import ipaddress +import logging +from copy import copy + +import yaml + +from apex.common import utils +from apex.common.constants import ( + CONTROLLER, + COMPUTE, + ROLES, + DOMAIN_NAME, + DNS_SERVERS, + NTP_SERVER, + ADMIN_NETWORK, + EXTERNAL_NETWORK, + OPNFV_NETWORK_TYPES, +) +from apex.network import ip_utils + + +class NetworkSettings(dict): + """ + This class parses APEX network settings yaml file into an object. It + generates or detects all missing fields for deployment. + + The resulting object will be used later to generate network environment + file as well as configuring post deployment networks. + + Currently the parsed object is dumped into a bash global definition file + for deploy.sh consumption. This object will later be used directly as + deployment script move to python. + """ + def __init__(self, filename): + init_dict = {} + if isinstance(filename, str): + with open(filename, 'r') as network_settings_file: + init_dict = yaml.safe_load(network_settings_file) + else: + # assume input is a dict to build from + init_dict = filename + super().__init__(init_dict) + + if 'apex' in self: + # merge two dicts Non-destructively + def merge(pri, sec): + for key, val in sec.items(): + if key in pri: + if isinstance(val, dict): + merge(pri[key], val) + # else + # do not overwrite what's already there + else: + pri[key] = val + # merge the apex specific config into the first class settings + merge(self, copy(self['apex'])) + + self.enabled_network_list = [] + self.nics = {COMPUTE: {}, CONTROLLER: {}} + self.nics_specified = {COMPUTE: False, CONTROLLER: False} + self._validate_input() + + def get_network(self, network): + if network == EXTERNAL_NETWORK and self['networks'][network]: + for net in self['networks'][network]: + if 'public' in net: + return net + + raise NetworkSettingsException("The external network, " + "'public', should be defined " + "when external networks are " + "enabled") + else: + return self['networks'][network] + + def _validate_input(self): + """ + Validates the network settings file and populates all fields. + + NetworkSettingsException will be raised if validation fails. + """ + if not self['networks'].get(ADMIN_NETWORK, {}).get('enabled', False): + raise NetworkSettingsException("You must enable admin network " + "and configure it explicitly or " + "use auto-detection") + + for network in OPNFV_NETWORK_TYPES: + if network in self['networks']: + _network = self.get_network(network) + if _network.get('enabled', True): + logging.info("{} enabled".format(network)) + self._config_required_settings(network) + nicmap = _network['nic_mapping'] + self._validate_overcloud_nic_order(network) + iface = nicmap[CONTROLLER]['members'][0] + self._config_ip_range(network=network, + interface=iface, + ip_range='overcloud_ip_range', + start_offset=21, end_offset=21) + self.enabled_network_list.append(network) + # TODO self._config_optional_settings(network) + else: + logging.info("{} disabled, will collapse with " + "admin network".format(network)) + else: + logging.info("{} is not in specified, will collapse with " + "admin network".format(network)) + + if 'dns-domain' not in self: + self['domain_name'] = DOMAIN_NAME + else: + self['domain_name'] = self['dns-domain'] + self['dns_servers'] = self.get('dns_nameservers', DNS_SERVERS) + self['ntp_servers'] = self.get('ntp', NTP_SERVER) + + def _validate_overcloud_nic_order(self, network): + """ + Detects if nic order is specified per profile (compute/controller) + for network + + If nic order is specified in a network for a profile, it should be + specified for every network with that profile other than admin network + + Duplicate nic names are also not allowed across different networks + + :param network: network to detect if nic order present + :return: None + """ + for role in ROLES: + _network = self.get_network(network) + _nicmap = _network.get('nic_mapping', {}) + _role = _nicmap.get(role, {}) + interfaces = _role.get('members', []) + + if interfaces: + interface = interfaces[0] + if not isinstance(_role.get('vlan', 'native'), int) and \ + any(y == interface for x, y in self.nics[role].items()): + raise NetworkSettingsException( + "Duplicate {} already specified for " + "another network".format(interface)) + self.nics[role][network] = interface + self.nics_specified[role] = True + logging.info("{} nic order specified for network {" + "}".format(role, network)) + else: + raise NetworkSettingsException( + "Interface members are not supplied for {} network " + "for the {} role. Please add nic assignments" + "".format(network, role)) + + def _config_required_settings(self, network): + """ + Configures either CIDR or bridged_interface setting + + cidr takes precedence if both cidr and bridged_interface are specified + for a given network. + + When using bridged_interface, we will detect network setting on the + given NIC in the system. The resulting config in settings object will + be an ipaddress.network object, replacing the NIC name. + """ + _network = self.get_network(network) + # if vlan not defined then default it to native + if network is not ADMIN_NETWORK: + for role in ROLES: + if 'vlan' not in _network['nic_mapping'][role]: + _network['nic_mapping'][role]['vlan'] = 'native' + + cidr = _network.get('cidr') + + if cidr: + cidr = ipaddress.ip_network(_network['cidr']) + _network['cidr'] = cidr + logging.info("{}_cidr: {}".format(network, cidr)) + elif 'installer_vm' in _network: + ucloud_if_list = _network['installer_vm']['members'] + # If cidr is not specified, we need to know if we should find + # IPv6 or IPv4 address on the interface + ip = ipaddress.ip_address(_network['installer_vm']['ip']) + nic_if = ip_utils.get_interface(ucloud_if_list[0], ip.version) + if nic_if: + logging.info("{}_bridged_interface: {}". + format(network, nic_if)) + else: + raise NetworkSettingsException( + "Auto detection failed for {}: Unable to find valid " + "ip for interface {}".format(network, ucloud_if_list[0])) + + else: + raise NetworkSettingsException( + "Auto detection failed for {}: either installer_vm " + "members or cidr must be specified".format(network)) + + # undercloud settings + if network == ADMIN_NETWORK: + provisioner_ip = _network['installer_vm']['ip'] + iface = _network['installer_vm']['members'][0] + if not provisioner_ip: + _network['installer_vm']['ip'] = self._gen_ip(network, 1) + self._config_ip_range(network=network, interface=iface, + ip_range='dhcp_range', + start_offset=2, count=9) + self._config_ip_range(network=network, interface=iface, + ip_range='introspection_range', + start_offset=11, count=9) + elif network == EXTERNAL_NETWORK: + provisioner_ip = _network['installer_vm']['ip'] + iface = _network['installer_vm']['members'][0] + if not provisioner_ip: + _network['installer_vm']['ip'] = self._gen_ip(network, 1) + self._config_ip_range(network=network, interface=iface, + ip_range='floating_ip_range', + end_offset=2, count=20) + + gateway = _network['gateway'] + interface = _network['installer_vm']['ip'] + self._config_gateway(network, gateway, interface) + + def _config_ip_range(self, network, ip_range, interface=None, + start_offset=None, end_offset=None, count=None): + """ + Configures IP range for a given setting. + If the setting is already specified, no change will be made. + The spec for start_offset, end_offset and count are identical to + ip_utils.get_ip_range. + """ + _network = self.get_network(network) + if ip_range not in _network: + cidr = _network.get('cidr') + _ip_range = ip_utils.get_ip_range(start_offset=start_offset, + end_offset=end_offset, + count=count, + cidr=cidr, + interface=interface) + _network[ip_range] = _ip_range.split(',') + + logging.info("Config IP Range: {} {}".format(network, ip_range)) + + def _gen_ip(self, network, offset): + """ + Generate and ip offset within the given network + """ + _network = self.get_network(network) + cidr = _network.get('cidr') + ip = ip_utils.get_ip(offset, cidr) + logging.info("Config IP: {} {}".format(network, ip)) + return ip + + def _config_optional_settings(self, network): + """ + Configures optional settings: + - admin_network: + - provisioner_ip + - dhcp_range + - introspection_range + - public_network: + - provisioner_ip + - floating_ip_range + - gateway + """ + if network == ADMIN_NETWORK: + # FIXME: _config_ip function does not exist! + self._config_ip(network, None, 'provisioner_ip', 1) + self._config_ip_range(network=network, + ip_range='dhcp_range', + start_offset=2, count=9) + self._config_ip_range(network=network, + ip_range='introspection_range', + start_offset=11, count=9) + elif network == EXTERNAL_NETWORK: + # FIXME: _config_ip function does not exist! + self._config_ip(network, None, 'provisioner_ip', 1) + self._config_ip_range(network=network, + ip_range='floating_ip_range', + end_offset=2, count=20) + self._config_gateway(network) + + def _config_gateway(self, network, gateway, interface): + """ + Configures gateway setting for a given network. + + If cidr is specified, we always use the first address in the address + space for gateway. Otherwise, we detect the system gateway. + """ + _network = self.get_network(network) + if not gateway: + cidr = _network.get('cidr') + if cidr: + _gateway = ip_utils.get_ip(1, cidr) + else: + _gateway = ip_utils.find_gateway(interface) + + if _gateway: + _network['gateway'] = _gateway + else: + raise NetworkSettingsException("Failed to set gateway") + + logging.info("Config Gateway: {} {}".format(network, gateway)) + + def get_ip_addr_family(self,): + """ + Returns IP address family for current deployment. + + If any enabled network has IPv6 CIDR, the deployment is classified as + IPv6. + """ + return max([ + ipaddress.ip_network(self.get_network(n)['cidr']).version + for n in self.enabled_network_list]) + + +class NetworkSettingsException(Exception): + def __init__(self, value): + self.value = value + + def __str__(self): + return self.value |