diff options
Diffstat (limited to 'utils')
| -rw-r--r-- | utils/clean_openstack.py | 442 | ||||
| -rw-r--r-- | utils/functest_logger.py | 6 | ||||
| -rw-r--r-- | utils/generate_defaults.py | 185 | ||||
| -rw-r--r-- | utils/openstack_utils.py | 10 |
4 files changed, 640 insertions, 3 deletions
diff --git a/utils/clean_openstack.py b/utils/clean_openstack.py new file mode 100644 index 000000000..838927faa --- /dev/null +++ b/utils/clean_openstack.py @@ -0,0 +1,442 @@ +#!/usr/bin/env python +# +# Description: +# Cleans possible leftovers after running functest tests: +# - Nova instances +# - Glance images +# - Cinder volumes +# - Floating IPs +# - Neutron networks, subnets and ports +# - Routers +# - Users and tenants +# +# Author: +# jose.lausuch@ericsson.com +# +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +# + +import argparse +import logging +import os +import time +import yaml + +from novaclient import client as novaclient +from neutronclient.v2_0 import client as neutronclient +from keystoneclient.v2_0 import client as keystoneclient +from cinderclient import client as cinderclient + +import openstack_utils + +parser = argparse.ArgumentParser() +parser.add_argument("-d", "--debug", help="Debug mode", action="store_true") +args = parser.parse_args() + + +""" logging configuration """ +logger = logging.getLogger('clean_openstack') +logger.setLevel(logging.DEBUG) + +ch = logging.StreamHandler() +if args.debug: + ch.setLevel(logging.DEBUG) +else: + ch.setLevel(logging.INFO) + +formatter = logging.Formatter('%(asctime)s - %(name)s - ' + '%(levelname)s - %(message)s') +ch.setFormatter(formatter) +logger.addHandler(ch) + +REPO_PATH = os.environ['repos_dir'] + '/functest/' + +DEFAULTS_FILE = '/home/opnfv/functest/conf/os_defaults.yaml' + +try: + with open(DEFAULTS_FILE) as f: + defaults_yaml = yaml.safe_load(f) +except Exception, e: + logger.info("The file %s does not exist. Please run generate_defaults.py " + "to create the OpenStack defaults. " + "Aborting cleanup..." % DEFAULTS_FILE) + exit(0) + +default_images = defaults_yaml.get('images') +default_instances = defaults_yaml.get('instances') +default_volumes = defaults_yaml.get('volumes') +default_networks = defaults_yaml.get('networks') +default_routers = defaults_yaml.get('routers') +default_security_groups = defaults_yaml.get('secgroups') +default_floatingips = defaults_yaml.get('floatingips') +default_users = defaults_yaml.get('users') +default_tenants = defaults_yaml.get('tenants') + + +def separator(): + logger.info("-------------------------------------------") + + +def remove_instances(nova_client): + logger.info("Removing Nova instances...") + instances = openstack_utils.get_instances(nova_client) + if instances is None or len(instances) == 0: + logger.debug("No instances found.") + return + + for instance in instances: + instance_name = getattr(instance, 'name') + instance_id = getattr(instance, 'id') + logger.debug("Removing instance '%s', ID=%s ..." + % (instance_name, instance_id)) + if openstack_utils.delete_instance(nova_client, instance_id): + logger.debug(" > Done!") + else: + logger.error("There has been a problem removing the " + "instance %s..." % instance_id) + + timeout = 50 + while timeout > 0: + instances = openstack_utils.get_instances(nova_client) + if instances is None or len(instances) == 0: + break + else: + logger.debug("Waiting for instances to be terminated...") + timeout -= 1 + time.sleep(1) + + +def remove_images(nova_client): + logger.info("Removing Glance images...") + images = openstack_utils.get_images(nova_client) + if images is None or len(images) == 0: + logger.debug("No images found.") + return + + for image in images: + image_name = getattr(image, 'name') + image_id = getattr(image, 'id') + logger.debug("'%s', ID=%s " % (image_name, image_id)) + if image_id not in default_images: + logger.debug("Removing image '%s', ID=%s ..." + % (image_name, image_id)) + if openstack_utils.delete_glance_image(nova_client, image_id): + logger.debug(" > Done!") + else: + logger.error("There has been a problem removing the" + "image %s..." % image_id) + else: + logger.debug(" > this is a default image and will " + "NOT be deleted.") + + +def remove_volumes(cinder_client): + logger.info("Removing Cinder volumes...") + volumes = openstack_utils.get_volumes(cinder_client) + if volumes is None or len(volumes) == 0: + logger.debug("No volumes found.") + return + + for volume in volumes: + volume_id = getattr(volume, 'id') + volume_name = getattr(volume, 'display_name') + logger.debug("'%s', ID=%s " % (volume_name, volume_id)) + if volume_id not in default_volumes: + logger.debug("Removing cinder volume %s ..." % volume_id) + if openstack_utils.delete_volume(cinder_client, volume_id): + logger.debug(" > Done!") + else: + logger.debug("Trying forced removal...") + if openstack_utils.delete_volume(cinder_client, + volume_id, + forced=True): + logger.debug(" > Done!") + else: + logger.error("There has been a problem removing the " + "volume %s..." % volume_id) + else: + logger.debug(" > this is a default volume and will " + "NOT be deleted.") + + +def remove_floatingips(nova_client): + logger.info("Removing floating IPs...") + floatingips = openstack_utils.get_floating_ips(nova_client) + if floatingips is None or len(floatingips) == 0: + logger.debug("No floating IPs found.") + return + + init_len = len(floatingips) + deleted = 0 + for fip in floatingips: + fip_id = getattr(fip, 'id') + fip_ip = getattr(fip, 'ip') + logger.debug("'%s', ID=%s " % (fip_ip, fip_id)) + if fip_id not in default_floatingips: + logger.debug("Removing floating IP %s ..." % fip_id) + if openstack_utils.delete_floating_ip(nova_client, fip_id): + logger.debug(" > Done!") + deleted += 1 + else: + logger.error("There has been a problem removing the " + "floating IP %s..." % fip_id) + else: + logger.debug(" > this is a default floating IP and will " + "NOT be deleted.") + + timeout = 50 + while timeout > 0: + floatingips = openstack_utils.get_floating_ips(nova_client) + if floatingips is None or len(floatingips) == (init_len - deleted): + break + else: + logger.debug("Waiting for floating ips to be released...") + timeout -= 1 + time.sleep(1) + + +def remove_networks(neutron_client): + logger.info("Removing Neutron objects") + network_ids = [] + networks = openstack_utils.get_network_list(neutron_client) + if networks is None: + logger.debug("There are no networks in the deployment. ") + else: + logger.debug("Existing networks:") + for network in networks: + net_id = network['id'] + net_name = network['name'] + logger.debug(" '%s', ID=%s " % (net_name, net_id)) + if net_id in default_networks: + logger.debug(" > this is a default network and will " + "NOT be deleted.") + elif network['router:external'] is True: + logger.debug(" > this is an external network and will " + "NOT be deleted.") + else: + logger.debug(" > this network will be deleted.") + network_ids.append(net_id) + + # delete ports + ports = openstack_utils.get_port_list(neutron_client) + if ports is None: + logger.debug("There are no ports in the deployment. ") + else: + remove_ports(neutron_client, ports, network_ids) + + # remove routers + routers = openstack_utils.get_router_list(neutron_client) + if routers is None: + logger.debug("There are no routers in the deployment. ") + else: + remove_routers(neutron_client, routers) + + # remove networks + if network_ids is not None: + for net_id in network_ids: + logger.debug("Removing network %s ..." % net_id) + if openstack_utils.delete_neutron_net(neutron_client, net_id): + logger.debug(" > Done!") + else: + logger.error("There has been a problem removing the " + "network %s..." % net_id) + + +def remove_ports(neutron_client, ports, network_ids): + for port in ports: + if port['network_id'] in network_ids: + port_id = port['id'] + try: + subnet_id = port['fixed_ips'][0]['subnet_id'] + except: + logger.info(" > WARNING: Port %s does not contain 'fixed_ips'" + % port_id) + print port + router_id = port['device_id'] + if len(port['fixed_ips']) == 0 and router_id == '': + logger.debug("Removing port %s ..." % port_id) + if (openstack_utils.delete_neutron_port(neutron_client, + port_id)): + logger.debug(" > Done!") + else: + logger.error("There has been a problem removing the " + "port %s ..." % port_id) + force_remove_port(neutron_client, port_id) + + elif port['device_owner'] == 'network:router_interface': + logger.debug("Detaching port %s (subnet %s) from router %s ..." + % (port_id, subnet_id, router_id)) + if openstack_utils.remove_interface_router( + neutron_client, router_id, subnet_id): + time.sleep(5) # leave 5 seconds to detach + logger.debug(" > Done!") + else: + logger.error("There has been a problem removing the " + "interface %s from router %s..." + % (subnet_id, router_id)) + force_remove_port(neutron_client, port_id) + else: + force_remove_port(neutron_client, port_id) + + +def force_remove_port(neutron_client, port_id): + logger.debug("Clearing device_owner for port %s ..." % port_id) + openstack_utils.update_neutron_port(neutron_client, + port_id, + device_owner='clear') + logger.debug("Removing port %s ..." % port_id) + if openstack_utils.delete_neutron_port(neutron_client, port_id): + logger.debug(" > Done!") + else: + logger.error("There has been a problem removing the port %s..." + % port_id) + + +def remove_routers(neutron_client, routers): + for router in routers: + router_id = router['id'] + router_name = router['name'] + if router_id not in default_routers: + logger.debug("Checking '%s' with ID=(%s) ..." % (router_name, + router_id)) + if router['external_gateway_info'] is not None: + logger.debug("Router has gateway to external network." + "Removing link...") + if openstack_utils.remove_gateway_router(neutron_client, + router_id): + logger.debug(" > Done!") + else: + logger.error("There has been a problem removing " + "the gateway...") + else: + logger.debug("Router is not connected to anything." + "Ready to remove...") + logger.debug("Removing router %s(%s) ..." + % (router_name, router_id)) + if openstack_utils.delete_neutron_router(neutron_client, + router_id): + logger.debug(" > Done!") + else: + logger.error("There has been a problem removing the " + "router '%s'(%s)..." % (router_name, router_id)) + + +def remove_security_groups(neutron_client): + logger.info("Removing Security groups...") + secgroups = openstack_utils.get_security_groups(neutron_client) + if secgroups is None or len(secgroups) == 0: + logger.debug("No security groups found.") + return + + for secgroup in secgroups: + secgroup_name = secgroup['name'] + secgroup_id = secgroup['id'] + logger.debug("'%s', ID=%s " % (secgroup_name, secgroup_id)) + if secgroup_id not in default_security_groups: + logger.debug(" Removing '%s'..." % secgroup_name) + if openstack_utils.delete_security_group(neutron_client, + secgroup_id): + logger.debug(" > Done!") + else: + logger.error("There has been a problem removing the " + "security group %s..." % secgroup_id) + else: + logger.debug(" > this is a default security group and will NOT " + "be deleted.") + + +def remove_users(keystone_client): + logger.info("Removing Users...") + users = openstack_utils.get_users(keystone_client) + if users is None: + logger.debug("There are no users in the deployment. ") + return + + for user in users: + user_name = getattr(user, 'name') + user_id = getattr(user, 'id') + logger.debug("'%s', ID=%s " % (user_name, user_id)) + if user_id not in default_users: + logger.debug(" Removing '%s'..." % user_name) + if openstack_utils.delete_user(keystone_client, user_id): + logger.debug(" > Done!") + else: + logger.error("There has been a problem removing the " + "user '%s'(%s)..." % (user_name, user_id)) + else: + logger.debug(" > this is a default user and will " + "NOT be deleted.") + + +def remove_tenants(keystone_client): + logger.info("Removing Tenants...") + tenants = openstack_utils.get_tenants(keystone_client) + if tenants is None: + logger.debug("There are no tenants in the deployment. ") + return + + for tenant in tenants: + tenant_name = getattr(tenant, 'name') + tenant_id = getattr(tenant, 'id') + logger.debug("'%s', ID=%s " % (tenant_name, tenant_id)) + if tenant_id not in default_tenants: + logger.debug(" Removing '%s'..." % tenant_name) + if openstack_utils.delete_tenant(keystone_client, tenant_id): + logger.debug(" > Done!") + else: + logger.error("There has been a problem removing the " + "tenant '%s'(%s)..." % (tenant_name, tenant_id)) + else: + logger.debug(" > this is a default tenant and will " + "NOT be deleted.") + + +def main(): + creds_nova = openstack_utils.get_credentials("nova") + nova_client = novaclient.Client('2', **creds_nova) + + creds_neutron = openstack_utils.get_credentials("neutron") + neutron_client = neutronclient.Client(**creds_neutron) + + creds_keystone = openstack_utils.get_credentials("keystone") + keystone_client = keystoneclient.Client(**creds_keystone) + + creds_cinder = openstack_utils.get_credentials("cinder") + # cinder_client = cinderclient.Client(**creds_cinder) + cinder_client = cinderclient.Client('1', creds_cinder['username'], + creds_cinder['api_key'], + creds_cinder['project_id'], + creds_cinder['auth_url'], + service_type="volume") + + if not openstack_utils.check_credentials(): + logger.error("Please source the openrc credentials and run " + "the script again.") + exit(-1) + + remove_instances(nova_client) + separator() + remove_images(nova_client) + separator() + remove_volumes(cinder_client) + separator() + remove_floatingips(nova_client) + separator() + remove_networks(neutron_client) + separator() + remove_security_groups(neutron_client) + separator() + remove_users(keystone_client) + separator() + remove_tenants(keystone_client) + separator() + + exit(0) + + +if __name__ == '__main__': + main() diff --git a/utils/functest_logger.py b/utils/functest_logger.py index 5639ed809..b9b308710 100644 --- a/utils/functest_logger.py +++ b/utils/functest_logger.py @@ -38,10 +38,10 @@ class Logger: CI_DEBUG = os.getenv('CI_DEBUG') - if CI_DEBUG.lower() == "true": + ch.setLevel(logging.INFO) + + if CI_DEBUG is not None and CI_DEBUG.lower() == "true": ch.setLevel(logging.DEBUG) - else: - ch.setLevel(logging.INFO) self.logger.addHandler(ch) diff --git a/utils/generate_defaults.py b/utils/generate_defaults.py new file mode 100644 index 000000000..4c2065e4a --- /dev/null +++ b/utils/generate_defaults.py @@ -0,0 +1,185 @@ +#!/usr/bin/env python +# +# Description: +# Generates a list of the current Openstack objects in the deployment: +# - Nova instances +# - Glance images +# - Cinder volumes +# - Floating IPs +# - Neutron networks, subnets and ports +# - Routers +# - Users and tenants +# +# Author: +# jose.lausuch@ericsson.com +# +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +# + +import os +import yaml + +from novaclient import client as novaclient +from neutronclient.v2_0 import client as neutronclient +from keystoneclient.v2_0 import client as keystoneclient +from cinderclient import client as cinderclient + +import functest.utils.openstack_utils as openstack_utils +import functest.utils.functest_logger as ft_logger + +""" logging configuration """ +logger = ft_logger.Logger("generate_defaults").getLogger() + +REPO_PATH = os.environ['repos_dir'] + '/functest/' +if not os.path.exists(REPO_PATH): + logger.error("Functest repository directory not found '%s'" % REPO_PATH) + exit(-1) + + +DEFAULTS_FILE = '/home/opnfv/functest/conf/os_defaults.yaml' + + +def separator(): + logger.info("-------------------------------------------") + + +def get_instances(nova_client): + logger.debug("Getting instances...") + dic_instances = {} + instances = openstack_utils.get_instances(nova_client) + if not (instances is None or len(instances) == 0): + for instance in instances: + dic_instances.update({getattr(instance, 'id'): getattr(instance, + 'name')}) + return {'instances': dic_instances} + + +def get_images(nova_client): + logger.debug("Getting images...") + dic_images = {} + images = openstack_utils.get_images(nova_client) + if not (images is None or len(images) == 0): + for image in images: + dic_images.update({getattr(image, 'id'): getattr(image, 'name')}) + return {'images': dic_images} + + +def get_volumes(cinder_client): + logger.debug("Getting volumes...") + dic_volumes = {} + volumes = openstack_utils.get_volumes(cinder_client) + if volumes is not None: + for volume in volumes: + dic_volumes.update({volume.id: volume.display_name}) + return {'volumes': dic_volumes} + + +def get_networks(neutron_client): + logger.debug("Getting networks") + dic_networks = {} + networks = openstack_utils.get_network_list(neutron_client) + if networks is not None: + for network in networks: + dic_networks.update({network['id']: network['name']}) + return {'networks': dic_networks} + + +def get_routers(neutron_client): + logger.debug("Getting routers") + dic_routers = {} + routers = openstack_utils.get_router_list(neutron_client) + if routers is not None: + for router in routers: + dic_routers.update({router['id']: router['name']}) + return {'routers': dic_routers} + + +def get_security_groups(neutron_client): + logger.debug("Getting Security groups...") + dic_secgroups = {} + secgroups = openstack_utils.get_security_groups(neutron_client) + if not (secgroups is None or len(secgroups) == 0): + for secgroup in secgroups: + dic_secgroups.update({secgroup['id']: secgroup['name']}) + return {'secgroups': dic_secgroups} + + +def get_floatinips(nova_client): + logger.debug("Getting Floating IPs...") + dic_floatingips = {} + floatingips = openstack_utils.get_floating_ips(nova_client) + if not (floatingips is None or len(floatingips) == 0): + for floatingip in floatingips: + dic_floatingips.update({floatingip.id: floatingip.ip}) + return {'floatingips': dic_floatingips} + + +def get_users(keystone_client): + logger.debug("Getting users...") + dic_users = {} + users = openstack_utils.get_users(keystone_client) + if not (users is None or len(users) == 0): + for user in users: + dic_users.update({getattr(user, 'id'): getattr(user, 'name')}) + return {'users': dic_users} + + +def get_tenants(keystone_client): + logger.debug("Getting users...") + dic_tenants = {} + tenants = openstack_utils.get_tenants(keystone_client) + if not (tenants is None or len(tenants) == 0): + for tenant in tenants: + dic_tenants.update({getattr(tenant, 'id'): + getattr(tenant, 'name')}) + return {'tenants': dic_tenants} + + +def main(): + creds_nova = openstack_utils.get_credentials("nova") + nova_client = novaclient.Client('2', **creds_nova) + + creds_neutron = openstack_utils.get_credentials("neutron") + neutron_client = neutronclient.Client(**creds_neutron) + + creds_keystone = openstack_utils.get_credentials("keystone") + keystone_client = keystoneclient.Client(**creds_keystone) + + creds_cinder = openstack_utils.get_credentials("cinder") + cinder_client = cinderclient.Client('1', creds_cinder['username'], + creds_cinder['api_key'], + creds_cinder['project_id'], + creds_cinder['auth_url'], + service_type="volume") + + if not openstack_utils.check_credentials(): + logger.error("Please source the openrc credentials and run the" + + "script again.") + exit(-1) + + defaults = {} + defaults.update(get_instances(nova_client)) + defaults.update(get_images(nova_client)) + defaults.update(get_volumes(cinder_client)) + defaults.update(get_networks(neutron_client)) + defaults.update(get_routers(neutron_client)) + defaults.update(get_security_groups(neutron_client)) + defaults.update(get_floatinips(nova_client)) + defaults.update(get_users(keystone_client)) + defaults.update(get_tenants(keystone_client)) + + with open(DEFAULTS_FILE, 'w+') as yaml_file: + yaml_file.write(yaml.safe_dump(defaults, default_flow_style=False)) + yaml_file.seek(0) + logger.debug("Openstack Defaults found in the deployment:") + print yaml_file.read() + logger.debug("NOTE: These objects will NOT be deleted after " + + "running the tests.") + + +if __name__ == '__main__': + main() diff --git a/utils/openstack_utils.py b/utils/openstack_utils.py index ff3968d67..2ae2842d4 100644 --- a/utils/openstack_utils.py +++ b/utils/openstack_utils.py @@ -10,6 +10,7 @@ import os import os.path +import subprocess import sys # ---------------------------------------------------------- @@ -73,6 +74,15 @@ def get_credentials(service): return creds +def source_credentials(rc_file): + pipe = subprocess.Popen(". %s; env" % rc_file, stdout=subprocess.PIPE, + shell=True) + output = pipe.communicate()[0] + env = dict((line.split("=", 1) for line in output.splitlines())) + os.environ.update(env) + return env + + # ********************************************* # NOVA # ********************************************* |