From f4d388ea508ba00771e43a219ac64e0d430b73bd Mon Sep 17 00:00:00 2001 From: Tim Rozet Date: Sun, 25 Jun 2017 21:25:36 -0400 Subject: Migrates Apex to Python Removes all bash libraries and converts almost all of the code to a mixture of Python and Ansible. utils.sh and clean.sh still exist. clean.sh will be migrated fully to clean.py in another patch. The Apex Python package is now built into the opnfv-apex-common RPM. To install locally do 'pip3 install .'. To deploy: opnfv-deploy -d -n --image-dir /root/apex/.build -v --debug Non-python files (THT yaml, settings files, ansible playbooks) are all installed into /usr/share/opnfv-apex/. The RPM will copy settings files into /etc/opnfv-apex/. JIRA: APEX-317 Change-Id: I3232f0329bcd13bce5a28da6a8c9c84d0b048024 Signed-off-by: Tim Rozet --- apex/common/__init__.py | 0 apex/common/constants.py | 46 ++++++++++++++++++++ apex/common/exceptions.py | 12 ++++++ apex/common/parsers.py | 73 +++++++++++++++++++++++++++++++ apex/common/utils.py | 107 ++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 238 insertions(+) create mode 100644 apex/common/__init__.py create mode 100644 apex/common/constants.py create mode 100644 apex/common/exceptions.py create mode 100644 apex/common/parsers.py create mode 100644 apex/common/utils.py (limited to 'apex/common') diff --git a/apex/common/__init__.py b/apex/common/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/apex/common/constants.py b/apex/common/constants.py new file mode 100644 index 00000000..0df71526 --- /dev/null +++ b/apex/common/constants.py @@ -0,0 +1,46 @@ +############################################################################## +# Copyright (c) 2016 Tim Rozet (trozet@redhat.com) and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +import os + +ADMIN_NETWORK = 'admin' +TENANT_NETWORK = 'tenant' +EXTERNAL_NETWORK = 'external' +STORAGE_NETWORK = 'storage' +API_NETWORK = 'api' +CONTROLLER = 'controller' +COMPUTE = 'compute' + +OPNFV_NETWORK_TYPES = [ADMIN_NETWORK, TENANT_NETWORK, EXTERNAL_NETWORK, + STORAGE_NETWORK, API_NETWORK] +DNS_SERVERS = ["8.8.8.8", "8.8.4.4"] +NTP_SERVER = ["pool.ntp.org"] +COMPUTE = 'compute' +CONTROLLER = 'controller' +ROLES = [COMPUTE, CONTROLLER] +DOMAIN_NAME = 'localdomain.com' +COMPUTE_PRE = "OS::TripleO::ComputeExtraConfigPre" +CONTROLLER_PRE = "OS::TripleO::ControllerExtraConfigPre" +PRE_CONFIG_DIR = "/usr/share/openstack-tripleo-heat-templates/puppet/" \ + "extraconfig/pre_deploy/" +DEFAULT_ROOT_DEV = 'sda' +LIBVIRT_VOLUME_PATH = '/var/lib/libvirt/images' + +VIRT_UPLOAD = '--upload' +VIRT_INSTALL = '-install' +VIRT_RUN_CMD = '--run-command' +VIRT_PW = '--root-password' + +THT_DIR = '/usr/share/openstack-tripleo-heat-templates' +THT_ENV_DIR = os.path.join(THT_DIR, 'environments') + +DEFAULT_ODL_VERSION = 'carbon' +DEBUG_OVERCLOUD_PW = 'opnfvapex' +NET_ENV_FILE = 'network-environment.yaml' +DEPLOY_TIMEOUT = 90 diff --git a/apex/common/exceptions.py b/apex/common/exceptions.py new file mode 100644 index 00000000..c660213f --- /dev/null +++ b/apex/common/exceptions.py @@ -0,0 +1,12 @@ +############################################################################## +# Copyright (c) 2017 Tim Rozet (trozet@redhat.com) and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + + +class ApexDeployException(Exception): + pass diff --git a/apex/common/parsers.py b/apex/common/parsers.py new file mode 100644 index 00000000..8744c862 --- /dev/null +++ b/apex/common/parsers.py @@ -0,0 +1,73 @@ +############################################################################## +# Copyright (c) 2017 Tim Rozet (trozet@redhat.com) and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +import json +import logging +import pprint +import os +import re + +from apex.common.exceptions import ApexDeployException + +"""Parser functions for overcloud/openstack output""" + + +def parse_nova_output(in_file): + """ + Parses nova list output into a dictionary format for node name and ip + :param in_file: json format from openstack server list + :return: dictionary format for {"node name": "node ip"} + """ + if not os.path.isfile(in_file): + raise FileNotFoundError(in_file) + node_dict = dict() + with open(in_file, 'r') as fh: + nova_list = json.load(fh) + + for server in nova_list: + ip_match = re.search('([0-9]+\.){3}[0-9]+', server['Networks']) + if ip_match is None: + logging.error("Unable to find IP in nova output " + "{}".format(pprint.pformat(server, indent=4))) + raise ApexDeployException("Unable to parse IP from nova output") + else: + node_dict[server['Name']] = ip_match.group(0) + + if not node_dict: + raise ApexDeployException("No overcloud nodes found in: {}".format( + in_file)) + return node_dict + + +def parse_overcloudrc(in_file): + """ + Parses overcloudrc into a dictionary format for key and value + :param in_file: + :return: dictionary format for {"variable": "value"} + """ + logging.debug("Parsing overcloudrc file {}".format(in_file)) + if not os.path.isfile(in_file): + raise FileNotFoundError(in_file) + creds = {} + with open(in_file, 'r') as fh: + lines = fh.readlines() + kv_pattern = re.compile('^export\s+([^\s]+)=([^\s]+)$') + for line in lines: + if 'export' not in line: + continue + else: + res = re.search(kv_pattern, line.strip()) + if res: + creds[res.group(1)] = res.group(2) + logging.debug("os cred found: {}, {}".format(res.group(1), + res.group(2))) + else: + logging.debug("os cred not found in: {}".format(line)) + + return creds diff --git a/apex/common/utils.py b/apex/common/utils.py new file mode 100644 index 00000000..848f2644 --- /dev/null +++ b/apex/common/utils.py @@ -0,0 +1,107 @@ +############################################################################## +# Copyright (c) 2016 Tim Rozet (trozet@redhat.com) and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +import json +import logging +import os +import pprint +import subprocess +import yaml + + +def str2bool(var): + if isinstance(var, bool): + return var + else: + return var.lower() in ("true", "yes") + + +def parse_yaml(yaml_file): + with open(yaml_file) as f: + parsed_dict = yaml.safe_load(f) + return parsed_dict + + +def dump_yaml(data, file): + """ + Dumps data to a file as yaml + :param data: yaml to be written to file + :param file: filename to write to + :return: + """ + logging.debug("Writing file {} with " + "yaml data:\n{}".format(file, yaml.safe_dump(data))) + with open(file, "w") as fh: + yaml.safe_dump(data, fh, default_flow_style=False) + + +def dict_objects_to_str(dictionary): + if isinstance(dictionary, list): + tmp_list = [] + for element in dictionary: + if isinstance(element, dict): + tmp_list.append(dict_objects_to_str(element)) + else: + tmp_list.append(str(element)) + return tmp_list + elif not isinstance(dictionary, dict): + if not isinstance(dictionary, bool): + return str(dictionary) + else: + return dictionary + return dict((k, dict_objects_to_str(v)) for + k, v in dictionary.items()) + + +def run_ansible(ansible_vars, playbook, host='localhost', user='root', + tmp_dir=None, dry_run=False): + """ + Executes ansible playbook and checks for errors + :param ansible_vars: dictionary of variables to inject into ansible run + :param playbook: playbook to execute + :param tmp_dir: temp directory to store ansible command + :param dry_run: Do not actually apply changes + :return: None + """ + logging.info("Executing ansible playbook: {}".format(playbook)) + inv_host = "{},".format(host) + if host == 'localhost': + conn_type = 'local' + else: + conn_type = 'smart' + ansible_command = ['ansible-playbook', '--become', '-i', inv_host, + '-u', user, '-c', conn_type, playbook, '-vvv'] + if dry_run: + ansible_command.append('--check') + + if isinstance(ansible_vars, dict) and ansible_vars: + logging.debug("Ansible variables to be set:\n{}".format( + pprint.pformat(ansible_vars))) + ansible_command.append('--extra-vars') + ansible_command.append(json.dumps(ansible_vars)) + if tmp_dir: + ansible_tmp = os.path.join(tmp_dir, + os.path.basename(playbook) + '.rerun') + # FIXME(trozet): extra vars are printed without single quotes + # so a dev has to add them manually to the command to rerun + # the playbook. Need to test if we can just add the single quotes + # to the json dumps to the ansible command and see if that works + with open(ansible_tmp, 'w') as fh: + fh.write("ANSIBLE_HOST_KEY_CHECKING=FALSE {}".format( + ' '.join(ansible_command))) + try: + my_env = os.environ.copy() + my_env['ANSIBLE_HOST_KEY_CHECKING'] = 'False' + logging.info("Executing playbook...this may take some time") + logging.debug(subprocess.check_output(ansible_command, env=my_env, + stderr=subprocess.STDOUT).decode('utf-8')) + except subprocess.CalledProcessError as e: + logging.error("Error executing ansible: {}".format( + pprint.pformat(e.output.decode('utf-8')))) + raise -- cgit 1.2.3-korg