#!/usr/bin/env python # # jose.lausuch@ericsson.com # valentin.boucher@orange.com # All rights reserved. This program and the accompanying materials # are made available under the terms of the Apache License, Version 2.0 # which accompanies this distribution, and is available at # http://www.apache.org/licenses/LICENSE-2.0 # import os import os.path import subprocess import sys import time from cinderclient import client as cinderclient import functest.utils.functest_logger as ft_logger import functest.utils.functest_utils as ft_utils from glanceclient import client as glanceclient from keystoneclient.v2_0 import client as keystoneclient from neutronclient.v2_0 import client as neutronclient from novaclient import client as novaclient logger = ft_logger.Logger("openstack_utils").getLogger() # ********************************************* # CREDENTIALS # ********************************************* def check_credentials(): """ Check if the OpenStack credentials (openrc) are sourced """ env_vars = ['OS_AUTH_URL', 'OS_USERNAME', 'OS_PASSWORD', 'OS_TENANT_NAME'] return all(map(lambda v: v in os.environ and os.environ[v], env_vars)) def get_credentials(service): """Returns a creds dictionary filled with the following keys: * username * password/api_key (depending on the service) * tenant_name/project_id (depending on the service) * auth_url :param service: a string indicating the name of the service requesting the credentials. """ creds = {} # Check that the env vars exists: envvars = ('OS_USERNAME', 'OS_PASSWORD', 'OS_AUTH_URL', 'OS_TENANT_NAME') for envvar in envvars: if os.getenv(envvar) is None: logger.error("'%s' is not exported as an env variable." % envvar) exit(-1) # Unfortunately, each of the OpenStack client will request slightly # different entries in their credentials dict. if service.lower() in ("nova", "cinder"): password = "api_key" tenant = "project_id" else: password = "password" tenant = "tenant_name" # The most common way to pass these info to the script is to do it through # environment variables. creds.update({ "username": os.environ.get("OS_USERNAME"), password: os.environ.get("OS_PASSWORD"), "auth_url": os.environ.get("OS_AUTH_URL"), tenant: os.environ.get("OS_TENANT_NAME") }) if os.getenv('OS_ENDPOINT_TYPE') is not None: creds.update({ "endpoint_type": os.environ.get("OS_ENDPOINT_TYPE") }) if os.getenv('OS_REGION_NAME') is not None: creds.update({ "region_name": os.environ.get("OS_REGION_NAME") }) cacert = os.environ.get("OS_CACERT") if cacert is not None: # each openstack client uses differnt kwargs for this creds.update({"cacert": cacert, "ca_cert": cacert, "https_ca_cert": cacert, "https_cacert": cacert, "ca_file": cacert}) creds.update({"insecure": "True", "https_insecure": "True"}) if not os.path.isfile(cacert): logger.info("WARNING: The 'OS_CACERT' environment variable is " "set to %s but the file does not exist." % cacert) return creds def source_credentials(rc_file): pipe = subprocess.Popen(". %s; env" % rc_file, stdout=subprocess.PIPE, shell=True) output = pipe.communicate()[0] env = dict((line.split("=", 1) for line in output.splitlines())) os.environ.update(env) return env def get_credentials_for_rally(): creds = get_credentials("keystone") admin_keys = ['username', 'tenant_name', 'password'] endpoint_types = [('internalURL', 'internal'), ('publicURL', 'public'), ('adminURL', 'admin')] if 'endpoint_type' in creds.keys(): for k, v in endpoint_types: if creds['endpoint_type'] == k: creds['endpoint_type'] = v rally_conf = {"type": "ExistingCloud", "admin": {}} for key in creds: if key in admin_keys: rally_conf['admin'][key] = creds[key] else: rally_conf[key] = creds[key] return rally_conf # ********************************************* # CLIENTS # ********************************************* def get_keystone_client(): creds_keystone = get_credentials("keystone") return keystoneclient.Client(**creds_keystone) def get_nova_client(): creds_nova = get_credentials("nova") return novaclient.Client('2', **creds_nova) def get_cinder_client(): creds_cinder = get_credentials("cinder") creds_cinder.update({ "service_type": "volume" }) return cinderclient.Client('2', **creds_cinder) def get_neutron_client(): creds_neutron = get_credentials("neutron") return neutronclient.Client(**creds_neutron) def get_glance_client(): keystone_client = get_keystone_client() glance_endpoint_type = 'publicURL' os_endpoint_type = os.getenv('OS_ENDPOINT_TYPE') if os_endpoint_type is not None: glance_endpoint_type = os_endpoint_type glance_endpoint = keystone_client.service_catalog.url_for( service_type='image', endpoint_type=glance_endpoint_type) return glanceclient.Client(1, glance_endpoint, token=keystone_client.auth_token) # ********************************************* # NOVA # ********************************************* def get_instances(nova_client): try: instances = nova_client.servers.list(search_opts={'all_tenants': 1}) return instances except Exception, e: logger.error("Error [get_instances(nova_client)]: %s" % e) return None def get_instance_status(nova_client, instance): try: instance = nova_client.servers.get(instance.id) return instance.status except Exception, e: logger.error("Error [get_instance_status(nova_client)]: %s" % e) return None def get_instance_by_name(nova_client, instance_name): try: instance = nova_client.servers.find(name=instance_name) return instance except Exception, e: logger.error("Error [get_instance_by_name(nova_client, '%s')]: %s" % (instance_name, e)) return None def get_flavor_id(nova_client, flavor_name): flavors = nova_client.flavors.list(detailed=True) id = '' for f in flavors: if f.name == flavor_name: id = f.id break return id def get_flavor_id_by_ram_range(nova_client, min_ram, max_ram): flavors = nova_client.flavors.list(detailed=True) id = '' for f in flavors: if min_ram <= f.ram and f.ram <= max_ram: id = f.id break return id def create_flavor(nova_client, flavor_name, ram, disk, vcpus): try: flavor = nova_client.flavors.create(flavor_name, ram, vcpus, disk) try: extra_specs = ft_utils.get_functest_config( 'general.flavor_extra_specs') flavor.set_keys(extra_specs) except ValueError: # flavor extra specs are not configured, therefore skip the update pass except Exception, e: logger.error("Error [create_flavor(nova_client, '%s', '%s', '%s', " "'%s')]: %s" % (flavor_name, ram, disk, vcpus, e)) return None return flavor.id def get_or_create_flavor(flavor_name, ram, disk, vcpus): flavor_exists = False nova_client = get_nova_client() flavor_id = get_flavor_id(nova_client, flavor_name) if flavor_id != '': logger.info("Using existing flavor '%s'..." % flavor_name) flavor_exists = True else: logger.info("Creating flavor '%s' with '%s' RAM, '%s' disk size, " "'%s' vcpus..." % (flavor_name, ram, disk, vcpus)) flavor_id = create_flavor(nova_client, flavor_name, ram, disk, vcpus) if not flavor_id: logger.error("Failed to create flavor '%s'..." % (flavor_name)) else: logger.debug("Flavor '%s' with ID=%s created successfully." % (flavor_name, flavor_id)) return flavor_exists, flavor_id def get_floating_ips(nova_client): try: floating_ips = nova_client.floating_ips.list() return floating_ips except Exception, e: logger.error("Error [get_floating_ips(nova_client)]: %s" % e) return None def get_hypervisors(nova_client): try: nodes = [] hypervisors = nova_client.hypervisors.list() for hypervisor in hypervisors: if hypervisor.state == "up": nodes.append(hypervisor.hypervisor_hostname) return nodes except Exception, e: logger.error("Error [get_hypervisors(nova_client)]: %s" % e) return None def create_instance(flavor_name, image_id, network_id, instance_name="functest-vm", confdrive=True, userdata=None, av_zone='', fixed_ip=None, files=None): nova_client = get_nova_client() try: flavor = nova_client.flavors.find(name=flavor_name) except: flavors = nova_client.flavors.list() logger.error("Error: Flavor '%s' not found. Available flavors are: " "\n%s" % (flavor_name, flavors)) return None if fixed_ip is not None: nics = {"net-id": network_id, "v4-fixed-ip": fixed_ip} else: nics = {"net-id": network_id} if userdata is None: instance = nova_client.servers.create( name=instance_name, flavor=flavor, image=image_id, nics=[nics], availability_zone=av_zone, files=files ) else: instance = nova_client.servers.create( name=instance_name, flavor=flavor, image=image_id, nics=[nics], config_drive=confdrive, userdata=userdata, availability_zone=av_zone, files=files ) return instance def create_instance_and_wait_for_active(flavor_name, image_id, network_id, instance_name="", config_drive=False, userdata="", av_zone='', fixed_ip=None, files=None): SLEEP = 3 VM_BOOT_TIMEOUT = 180 nova_client = get_nova_client() instance = create_instance(flavor_name, image_id, network_id, instance_name, config_drive, us
##############################################################################
# Copyright (c) 2017 ZTE Corporation and others.
#
# All rights reserved. This program and the accompanying materials
# are made available under the terms of the Apache License, Version 2.0
# which accompanies this distribution, and is available at
# http://www.apache.org/licenses/LICENSE-2.0
##############################################################################
import os
import random
import time

from doctor_tests.alarm import Alarm
from doctor_tests.common.constants import Host
from doctor_tests.common.utils import get_doctor_test_root_dir
from doctor_tests.common.utils import match_rep_in_file
from doctor_tests.common.utils import SSHClient
from doctor_tests.consumer import get_consumer
from doctor_tests.identity_auth import get_identity_auth
from doctor_tests.identity_auth import get_session
from doctor_tests.instance import Instance
from doctor_tests.inspector import get_inspector
from doctor_tests.monitor import get_monitor
from doctor_tests.network import Network
from doctor_tests.profiler_poc import main as profiler_main
from doctor_tests.os_clients import nova_client


LINK_DOWN_SCRIPT = """
#!/bin/bash -x
dev=$(sudo ip a | awk '/ {compute_ip}\//{{print $NF}}')
sleep 1
sudo ip link set $dev down
echo "doctor set link down at" $(date "+%s.%N")
sleep 30
sudo ip link set $dev up
sleep 1
"""


class FaultManagement(object):

    def __init__(self, conf, installer, user, log):
        self.conf = conf
        self.log = log
        self.user = user
        self.installer = installer
        auth = get_identity_auth(project=self.conf.doctor_project)
        self.nova = nova_client(self.conf.nova_version,
                                get_session(auth=auth))
        self.test_dir = get_doctor_test_root_dir()
        self.down_host = None
        self.GetLog = False
        self.disable_network_log = None
        self.network = Network(self.conf, log)
        self.instance = Instance(self.conf, log)
        self.alarm = Alarm(self.conf, log)
        self.inspector = get_inspector(self.conf, log)
        self.monitor = get_monitor(self.conf,
                                   self.inspector.get_inspector_url(),
                                   log)
        self.consumer = get_consumer(self.conf, log)

    def setup(self):
        self.log.info('fault management setup......')

        # user settings...
        self.user.update_quota()

        # creating VM...
        self.network.create()
        self.instance.create()
        self.instance.wait_for_vm_launch()

        # creating alarm...
        self.alarm.create()

        # starting doctor sample components...
        # tbd tojuvone: move inspector and consumer to common setup
        # when they support updating VMs via instance.create and
        # instance.delete alarm

        self.inspector.start()
        self.consumer.start()
        self.down_host = self.get_host_info_for_random_vm()
        self.monitor.start(self.down_host)

    def start(self):
        self.log.info('fault management start......')
        self._set_link_down(self.down_host.ip)
        self.log.info('fault management end......')

    def cleanup(self):
        self.log.info('fault management cleanup......')

        self.get_disable_network_log()
        self.unset_forced_down_hosts()
        self.inspector.stop()
        self.monitor.stop()
        self.consumer.stop()
        self.alarm.delete()
        self.instance.delete()
        self.network.delete()

    def get_host_info_for_random_vm(self):
        num = random.randint(0, self.conf.instance_count - 1)
        vm_name = "%s%d" % (self.conf.instance_basename, num)

        servers = {getattr(server, 'name'): server
                   for server in self.nova.servers.list()}
        server = servers.get(vm_name)
        if not server:
            raise Exception('Can not find instance: vm_name(%s)' % vm_name)
        host_name = server.__dict__.get('OS-EXT-SRV-ATTR:hypervisor_hostname')
        host_ip = self.installer.get_host_ip_from_hostname(host_name)

        self.log.info('Get host info(name:%s, ip:%s) which vm(%s) launched at'
                      % (host_name, host_ip, vm_name))
        return Host(host_name, host_ip)

    def unset_forced_down_hosts(self):
        if self.down_host:
            self.nova.services.force_down(self.down_host.name,
                                          'nova-compute', False)
            time.sleep(2)
            self.check_host_status('up')

    def check_host_status(self, state):
        service = self.nova.services.list(host=self.down_host.name,
                                          binary='nova-compute')
        host_state = service[0].__dict__.get('state')
        assert host_state == state

    def get_disable_network_log(self):
        if