From bbc47d487f06da2906116e5ade134e11c4221786 Mon Sep 17 00:00:00 2001 From: "jose.lausuch" Date: Thu, 30 Jun 2016 14:12:54 +0200 Subject: Change OpenStack clean behaviour JIRA: FUNCTEST-236 The openstack snapshot generation is now triggered before running a test case and removed from prepare_env Change-Id: I4d1bc95dedd7f59d4b1d5866f288e1c1a70ec69e Signed-off-by: jose.lausuch --- ci/check_os.sh | 2 +- ci/config_functest.yaml | 2 + ci/prepare_env.py | 12 +- ci/run_tests.py | 14 +- cli/commands/cli_os.py | 26 +-- docs/configguide/configguide.rst | 4 +- utils/clean_openstack.py | 426 --------------------------------------- utils/functest_utils.py | 33 ++- utils/generate_defaults.py | 185 ----------------- utils/openstack_clean.py | 426 +++++++++++++++++++++++++++++++++++++++ utils/openstack_snapshot.py | 188 +++++++++++++++++ 11 files changed, 675 insertions(+), 643 deletions(-) delete mode 100644 utils/clean_openstack.py delete mode 100644 utils/generate_defaults.py create mode 100644 utils/openstack_clean.py create mode 100644 utils/openstack_snapshot.py diff --git a/ci/check_os.sh b/ci/check_os.sh index c9e0b513..38fe32f5 100755 --- a/ci/check_os.sh +++ b/ci/check_os.sh @@ -26,7 +26,7 @@ fi echo "Checking OpenStack endpoints:" publicURL=$OS_AUTH_URL publicIP=$(echo $publicURL|sed 's/^.*http\:\/\///'|sed 's/.[^:]*$//') -publicPort=$(echo $publicURL|sed 's/^.*://'|sed 's/.[^\/]*$//') +publicPort=$(echo $publicURL|sed 's/^.*://'|sed 's/\/.*$//') echo ">>Verifying connectivity to the public endpoint $publicIP:$publicPort..." verify_connectivity $publicIP $publicPort RETVAL=$? diff --git a/ci/config_functest.yaml b/ci/config_functest.yaml index dc0919bb..ea502dac 100644 --- a/ci/config_functest.yaml +++ b/ci/config_functest.yaml @@ -29,6 +29,8 @@ general: dir_rally_inst: /home/opnfv/.rally openstack: + snapshot_file: /home/opnfv/functest/conf/openstack_snapshot.yaml + image_name: Cirros-0.3.4 image_file_name: cirros-0.3.4-x86_64-disk.img image_disk_format: qcow2 diff --git a/ci/prepare_env.py b/ci/prepare_env.py index 5754a62b..4f29260f 100644 --- a/ci/prepare_env.py +++ b/ci/prepare_env.py @@ -12,20 +12,19 @@ # http://www.apache.org/licenses/LICENSE-2.0 # + import argparse import os import re import subprocess import sys -import yaml import functest.utils.functest_logger as ft_logger import functest.utils.functest_utils as ft_utils -import functest.utils.generate_defaults as gen_def import functest.utils.openstack_utils as os_utils +import yaml -""" arguments """ actions = ['start', 'check'] parser = argparse.ArgumentParser() parser.add_argument("action", help="Possible actions are: " @@ -241,12 +240,6 @@ def install_rally(): "OpenStack flavors.")) -def generate_os_defaults(): - print_separator() - logger.info("Generating OpenStack defaults...") - gen_def.main() - - def check_environment(): msg_not_active = "The Functest environment is not installed." if not os.path.isfile(ENV_FILE): @@ -274,7 +267,6 @@ def main(): source_rc_file() verify_deployment() install_rally() - generate_os_defaults() with open(ENV_FILE, "w") as env_file: env_file.write("1") diff --git a/ci/run_tests.py b/ci/run_tests.py index 5dba1819..29b01d7f 100644 --- a/ci/run_tests.py +++ b/ci/run_tests.py @@ -14,13 +14,13 @@ import re import sys import functest.ci.tier_builder as tb -import functest.utils.clean_openstack as clean_os import functest.utils.functest_logger as ft_logger import functest.utils.functest_utils as ft_utils +import functest.utils.openstack_clean as os_clean +import functest.utils.openstack_snapshot as os_snapshot import functest.utils.openstack_utils as os_utils -""" arguments """ parser = argparse.ArgumentParser() parser.add_argument("-t", "--test", dest="test", action='store', help="Test case or tier (group of tests) to be executed. " @@ -61,8 +61,13 @@ def source_rc_file(): os_utils.source_credentials(rc_file) +def generate_os_snapshot(): + logger.debug("Generating OpenStack snapshot...") + os_snapshot.main() + + def cleanup(): - clean_os.main() + os_clean.main() def run_test(test): @@ -71,6 +76,9 @@ def run_test(test): logger.info("Running test case '%s'..." % test_name) print_separator("=") logger.debug("\n%s" % test) + + generate_os_snapshot() + flags = (" -t %s" % (test_name)) if REPORT_FLAG: flags += " -r" diff --git a/cli/commands/cli_os.py b/cli/commands/cli_os.py index 23b7beaa..b007842b 100644 --- a/cli/commands/cli_os.py +++ b/cli/commands/cli_os.py @@ -7,15 +7,15 @@ # http://www.apache.org/licenses/LICENSE-2.0 # -import click + import os +import click +import functest.utils.functest_utils as ft_utils +import functest.utils.openstack_clean as os_clean +import functest.utils.openstack_snapshot as os_snapshot import yaml -import functest.utils.clean_openstack as clean_os -import functest.utils.functest_utils as ft_utils -import functest.utils.generate_defaults as gen_def -""" global variables """ with open(os.environ["CONFIG_FUNCTEST_YAML"]) as f: functest_yaml = yaml.safe_load(f) @@ -24,10 +24,12 @@ FUNCTEST_REPO = ("%s/functest/" % REPOS_DIR) FUNCTEST_CONF_DIR = functest_yaml.get("general").get( "directories").get("dir_functest_conf") RC_FILE = os.getenv('creds') -OS_DEFAULTS_FILE = FUNCTEST_CONF_DIR + '/os_defaults.yaml' +OS_SNAPSHOT_FILE = ft_utils.get_parameter_from_yaml( + "general.openstack.snapshot_file") class CliOpenStack: + def __init__(self): self.os_auth_url = os.getenv('OS_AUTH_URL') self.endpoint_ip = None @@ -86,7 +88,7 @@ class CliOpenStack: def snapshot_create(self): self.ping_endpoint() - if os.path.isfile(OS_DEFAULTS_FILE): + if os.path.isfile(OS_SNAPSHOT_FILE): answer = raw_input("It seems there is already an OpenStack " "snapshot. Do you want to overwrite it with " "the current OpenStack status? [y|n]\n") @@ -99,22 +101,22 @@ class CliOpenStack: answer = raw_input("Invalid answer. Please type [y|n]\n") click.echo("Generating Openstack snapshot...") - gen_def.main() + os_snapshot.main() def snapshot_show(self): - if not os.path.isfile(OS_DEFAULTS_FILE): + if not os.path.isfile(OS_SNAPSHOT_FILE): click.echo("There is no OpenStack snapshot created. To create " "one run the command 'functest env os-create-snapshot'") return - with open(OS_DEFAULTS_FILE, 'r') as yaml_file: + with open(OS_SNAPSHOT_FILE, 'r') as yaml_file: click.echo("\n%s" % yaml_file.read()) def clean(self): self.ping_endpoint() - if not os.path.isfile(OS_DEFAULTS_FILE): + if not os.path.isfile(OS_SNAPSHOT_FILE): click.echo("Not possible to clean OpenStack without a snapshot. " "This could cause problems. " "Run first the command 'os-create-shapshot'.") return - clean_os.main() + os_clean.main() diff --git a/docs/configguide/configguide.rst b/docs/configguide/configguide.rst index 7c8ce8b8..9f669398 100644 --- a/docs/configguide/configguide.rst +++ b/docs/configguide/configguide.rst @@ -327,10 +327,10 @@ The structure under the **functest** repository can be described as follows:: | `-- vIMS `-- utils |-- __init__.py - |-- clean_openstack.py |-- functest_logger.py |-- functest_utils.py - |-- generate_defaults.py + |-- openstack_clean.py + |-- openstack_snapshot.py `-- openstack_utils.py (Note: All *.pyc files removed from above list for brevity...) diff --git a/utils/clean_openstack.py b/utils/clean_openstack.py deleted file mode 100644 index 70eced62..00000000 --- a/utils/clean_openstack.py +++ /dev/null @@ -1,426 +0,0 @@ -#!/usr/bin/env python -# -# Description: -# Cleans possible leftovers after running functest tests: -# - Nova instances -# - Glance images -# - Cinder volumes -# - Floating IPs -# - Neutron networks, subnets and ports -# - Routers -# - Users and tenants -# -# Author: -# jose.lausuch@ericsson.com -# -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -# - -import os -import time -import yaml - -from novaclient import client as novaclient -from neutronclient.v2_0 import client as neutronclient -from keystoneclient.v2_0 import client as keystoneclient -from cinderclient import client as cinderclient - -import functest.utils.functest_logger as ft_logger -import functest.utils.openstack_utils as os_utils - - -""" logging configuration """ -logger = ft_logger.Logger("clean_openstack").getLogger() - -REPO_PATH = os.environ['repos_dir'] + '/functest/' -DEFAULTS_FILE = '/home/opnfv/functest/conf/os_defaults.yaml' - - -def separator(): - logger.info("-------------------------------------------") - - -def remove_instances(nova_client, default_instances): - logger.info("Removing Nova instances...") - instances = os_utils.get_instances(nova_client) - if instances is None or len(instances) == 0: - logger.debug("No instances found.") - return - - for instance in instances: - instance_name = getattr(instance, 'name') - instance_id = getattr(instance, 'id') - logger.debug("'%s', ID=%s " % (instance_name, instance_id)) - if instance_id not in default_instances: - logger.debug("Removing instance '%s' ..." % instance_id) - if os_utils.delete_instance(nova_client, instance_id): - logger.debug(" > Request sent.") - else: - logger.error("There has been a problem removing the " - "instance %s..." % instance_id) - else: - logger.debug(" > this is a default instance and will " - "NOT be deleted.") - - timeout = 50 - while timeout > 0: - instances = os_utils.get_instances(nova_client) - for instance in instances: - instance_id = getattr(instance, 'id') - if instance_id not in default_instances: - logger.debug("Waiting for instances to be terminated...") - timeout -= 1 - time.sleep(1) - continue - break - - -def remove_images(nova_client, default_images): - logger.info("Removing Glance images...") - images = os_utils.get_images(nova_client) - if images is None or len(images) == 0: - logger.debug("No images found.") - return - - for image in images: - image_name = getattr(image, 'name') - image_id = getattr(image, 'id') - logger.debug("'%s', ID=%s " % (image_name, image_id)) - if image_id not in default_images: - logger.debug("Removing image '%s', ID=%s ..." - % (image_name, image_id)) - if os_utils.delete_glance_image(nova_client, image_id): - logger.debug(" > Done!") - else: - logger.error("There has been a problem removing the" - "image %s..." % image_id) - else: - logger.debug(" > this is a default image and will " - "NOT be deleted.") - - -def remove_volumes(cinder_client, default_volumes): - logger.info("Removing Cinder volumes...") - volumes = os_utils.get_volumes(cinder_client) - if volumes is None or len(volumes) == 0: - logger.debug("No volumes found.") - return - - for volume in volumes: - volume_id = getattr(volume, 'id') - volume_name = getattr(volume, 'display_name') - logger.debug("'%s', ID=%s " % (volume_name, volume_id)) - if volume_id not in default_volumes: - logger.debug("Removing cinder volume %s ..." % volume_id) - if os_utils.delete_volume(cinder_client, volume_id): - logger.debug(" > Done!") - else: - logger.debug("Trying forced removal...") - if os_utils.delete_volume(cinder_client, - volume_id, - forced=True): - logger.debug(" > Done!") - else: - logger.error("There has been a problem removing the " - "volume %s..." % volume_id) - else: - logger.debug(" > this is a default volume and will " - "NOT be deleted.") - - -def remove_floatingips(nova_client, default_floatingips): - logger.info("Removing floating IPs...") - floatingips = os_utils.get_floating_ips(nova_client) - if floatingips is None or len(floatingips) == 0: - logger.debug("No floating IPs found.") - return - - init_len = len(floatingips) - deleted = 0 - for fip in floatingips: - fip_id = getattr(fip, 'id') - fip_ip = getattr(fip, 'ip') - logger.debug("'%s', ID=%s " % (fip_ip, fip_id)) - if fip_id not in default_floatingips: - logger.debug("Removing floating IP %s ..." % fip_id) - if os_utils.delete_floating_ip(nova_client, fip_id): - logger.debug(" > Done!") - deleted += 1 - else: - logger.error("There has been a problem removing the " - "floating IP %s..." % fip_id) - else: - logger.debug(" > this is a default floating IP and will " - "NOT be deleted.") - - timeout = 50 - while timeout > 0: - floatingips = os_utils.get_floating_ips(nova_client) - if floatingips is None or len(floatingips) == (init_len - deleted): - break - else: - logger.debug("Waiting for floating ips to be released...") - timeout -= 1 - time.sleep(1) - - -def remove_networks(neutron_client, default_networks, default_routers): - logger.info("Removing Neutron objects") - network_ids = [] - networks = os_utils.get_network_list(neutron_client) - if networks is None: - logger.debug("There are no networks in the deployment. ") - else: - logger.debug("Existing networks:") - for network in networks: - net_id = network['id'] - net_name = network['name'] - logger.debug(" '%s', ID=%s " % (net_name, net_id)) - if net_id in default_networks: - logger.debug(" > this is a default network and will " - "NOT be deleted.") - elif network['router:external'] is True: - logger.debug(" > this is an external network and will " - "NOT be deleted.") - else: - logger.debug(" > this network will be deleted.") - network_ids.append(net_id) - - # delete ports - ports = os_utils.get_port_list(neutron_client) - if ports is None: - logger.debug("There are no ports in the deployment. ") - else: - remove_ports(neutron_client, ports, network_ids) - - # remove routers - routers = os_utils.get_router_list(neutron_client) - if routers is None: - logger.debug("There are no routers in the deployment. ") - else: - remove_routers(neutron_client, routers, default_routers) - - # remove networks - if network_ids is not None: - for net_id in network_ids: - logger.debug("Removing network %s ..." % net_id) - if os_utils.delete_neutron_net(neutron_client, net_id): - logger.debug(" > Done!") - else: - logger.error("There has been a problem removing the " - "network %s..." % net_id) - - -def remove_ports(neutron_client, ports, network_ids): - for port in ports: - if port['network_id'] in network_ids: - port_id = port['id'] - try: - subnet_id = port['fixed_ips'][0]['subnet_id'] - except: - logger.info(" > WARNING: Port %s does not contain 'fixed_ips'" - % port_id) - print port - router_id = port['device_id'] - if len(port['fixed_ips']) == 0 and router_id == '': - logger.debug("Removing port %s ..." % port_id) - if (os_utils.delete_neutron_port(neutron_client, port_id)): - logger.debug(" > Done!") - else: - logger.error("There has been a problem removing the " - "port %s ..." % port_id) - force_remove_port(neutron_client, port_id) - - elif port['device_owner'] == 'network:router_interface': - logger.debug("Detaching port %s (subnet %s) from router %s ..." - % (port_id, subnet_id, router_id)) - if os_utils.remove_interface_router( - neutron_client, router_id, subnet_id): - time.sleep(5) # leave 5 seconds to detach - logger.debug(" > Done!") - else: - logger.error("There has been a problem removing the " - "interface %s from router %s..." - % (subnet_id, router_id)) - force_remove_port(neutron_client, port_id) - else: - force_remove_port(neutron_client, port_id) - - -def force_remove_port(neutron_client, port_id): - logger.debug("Clearing device_owner for port %s ..." % port_id) - os_utils.update_neutron_port(neutron_client, port_id, - device_owner='clear') - logger.debug("Removing port %s ..." % port_id) - if os_utils.delete_neutron_port(neutron_client, port_id): - logger.debug(" > Done!") - else: - logger.error("There has been a problem removing the port %s..." - % port_id) - - -def remove_routers(neutron_client, routers, default_routers): - for router in routers: - router_id = router['id'] - router_name = router['name'] - if router_id not in default_routers: - logger.debug("Checking '%s' with ID=(%s) ..." % (router_name, - router_id)) - if router['external_gateway_info'] is not None: - logger.debug("Router has gateway to external network." - "Removing link...") - if os_utils.remove_gateway_router(neutron_client, router_id): - logger.debug(" > Done!") - else: - logger.error("There has been a problem removing " - "the gateway...") - else: - logger.debug("Router is not connected to anything." - "Ready to remove...") - logger.debug("Removing router %s(%s) ..." - % (router_name, router_id)) - if os_utils.delete_neutron_router(neutron_client, router_id): - logger.debug(" > Done!") - else: - logger.error("There has been a problem removing the " - "router '%s'(%s)..." % (router_name, router_id)) - - -def remove_security_groups(neutron_client, default_security_groups): - logger.info("Removing Security groups...") - secgroups = os_utils.get_security_groups(neutron_client) - if secgroups is None or len(secgroups) == 0: - logger.debug("No security groups found.") - return - - for secgroup in secgroups: - secgroup_name = secgroup['name'] - secgroup_id = secgroup['id'] - logger.debug("'%s', ID=%s " % (secgroup_name, secgroup_id)) - if secgroup_id not in default_security_groups: - logger.debug(" Removing '%s'..." % secgroup_name) - if os_utils.delete_security_group(neutron_client, secgroup_id): - logger.debug(" > Done!") - else: - logger.error("There has been a problem removing the " - "security group %s..." % secgroup_id) - else: - logger.debug(" > this is a default security group and will NOT " - "be deleted.") - - -def remove_users(keystone_client, default_users): - logger.info("Removing Users...") - users = os_utils.get_users(keystone_client) - if users is None: - logger.debug("There are no users in the deployment. ") - return - - for user in users: - user_name = getattr(user, 'name') - user_id = getattr(user, 'id') - logger.debug("'%s', ID=%s " % (user_name, user_id)) - if user_id not in default_users: - logger.debug(" Removing '%s'..." % user_name) - if os_utils.delete_user(keystone_client, user_id): - logger.debug(" > Done!") - else: - logger.error("There has been a problem removing the " - "user '%s'(%s)..." % (user_name, user_id)) - else: - logger.debug(" > this is a default user and will " - "NOT be deleted.") - - -def remove_tenants(keystone_client, default_tenants): - logger.info("Removing Tenants...") - tenants = os_utils.get_tenants(keystone_client) - if tenants is None: - logger.debug("There are no tenants in the deployment. ") - return - - for tenant in tenants: - tenant_name = getattr(tenant, 'name') - tenant_id = getattr(tenant, 'id') - logger.debug("'%s', ID=%s " % (tenant_name, tenant_id)) - if tenant_id not in default_tenants: - logger.debug(" Removing '%s'..." % tenant_name) - if os_utils.delete_tenant(keystone_client, tenant_id): - logger.debug(" > Done!") - else: - logger.error("There has been a problem removing the " - "tenant '%s'(%s)..." % (tenant_name, tenant_id)) - else: - logger.debug(" > this is a default tenant and will " - "NOT be deleted.") - - -def main(): - logger.info("+++++++++++++++++++++++++++++++") - logger.info("Cleaning OpenStack resources...") - logger.info("+++++++++++++++++++++++++++++++") - - try: - with open(DEFAULTS_FILE) as f: - defaults_yaml = yaml.safe_load(f) - except Exception: - logger.info("The file %s does not exist. The OpenStack snapshot must" - " be created first. Aborting cleanup." % DEFAULTS_FILE) - exit(0) - - default_images = defaults_yaml.get('images') - default_instances = defaults_yaml.get('instances') - default_volumes = defaults_yaml.get('volumes') - default_networks = defaults_yaml.get('networks') - default_routers = defaults_yaml.get('routers') - default_security_groups = defaults_yaml.get('secgroups') - default_floatingips = defaults_yaml.get('floatingips') - default_users = defaults_yaml.get('users') - default_tenants = defaults_yaml.get('tenants') - - creds_nova = os_utils.get_credentials("nova") - nova_client = novaclient.Client('2', **creds_nova) - - creds_neutron = os_utils.get_credentials("neutron") - neutron_client = neutronclient.Client(**creds_neutron) - - creds_keystone = os_utils.get_credentials("keystone") - keystone_client = keystoneclient.Client(**creds_keystone) - - creds_cinder = os_utils.get_credentials("cinder") - # cinder_client = cinderclient.Client(**creds_cinder) - cinder_client = cinderclient.Client('1', creds_cinder['username'], - creds_cinder['api_key'], - creds_cinder['project_id'], - creds_cinder['auth_url'], - service_type="volume") - - if not os_utils.check_credentials(): - logger.error("Please source the openrc credentials and run " - "the script again.") - exit(-1) - - remove_instances(nova_client, default_instances) - separator() - remove_images(nova_client, default_images) - separator() - remove_volumes(cinder_client, default_volumes) - separator() - remove_floatingips(nova_client, default_floatingips) - separator() - remove_networks(neutron_client, default_networks, default_routers) - separator() - remove_security_groups(neutron_client, default_security_groups) - separator() - remove_users(keystone_client, default_users) - separator() - remove_tenants(keystone_client, default_tenants) - separator() - - -if __name__ == '__main__': - main() diff --git a/utils/functest_utils.py b/utils/functest_utils.py index 2e7a1223..285f887c 100644 --- a/utils/functest_utils.py +++ b/utils/functest_utils.py @@ -8,23 +8,25 @@ # http://www.apache.org/licenses/LICENSE-2.0 # +""" global variables """ + from datetime import datetime as dt import json import os import os.path import re -import requests import shutil import socket import subprocess import sys import urllib2 -import yaml -from git import Repo + import functest.ci.tier_builder as tb +from git import Repo +import requests +import yaml -""" global variables """ REPOS_DIR = os.getenv('repos_dir') FUNCTEST_REPO = ("%s/functest/" % REPOS_DIR) @@ -300,3 +302,26 @@ def get_criteria_by_test(testname): criteria = test.get_criteria() return criteria + + +# ---------------------------------------------------------- +# +# YAML UTILS +# +# ----------------------------------------------------------- +def get_parameter_from_yaml(parameter): + """ + Returns the value of a given parameter in config_functest.yaml + parameter must be given in string format with dots + Example: general.openstack.image_name + """ + with open(os.environ["CONFIG_FUNCTEST_YAML"]) as f: + functest_yaml = yaml.safe_load(f) + f.close() + value = functest_yaml + for element in parameter.split("."): + value = value.get(element) + if value is None: + raise ValueError("The parameter %s is not defined in" + " config_functest.yaml" % parameter) + return value diff --git a/utils/generate_defaults.py b/utils/generate_defaults.py deleted file mode 100644 index 35b9ed30..00000000 --- a/utils/generate_defaults.py +++ /dev/null @@ -1,185 +0,0 @@ -#!/usr/bin/env python -# -# Description: -# Generates a list of the current Openstack objects in the deployment: -# - Nova instances -# - Glance images -# - Cinder volumes -# - Floating IPs -# - Neutron networks, subnets and ports -# - Routers -# - Users and tenants -# -# Author: -# jose.lausuch@ericsson.com -# -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -# - -import os -import yaml - -from novaclient import client as novaclient -from neutronclient.v2_0 import client as neutronclient -from keystoneclient.v2_0 import client as keystoneclient -from cinderclient import client as cinderclient - -import functest.utils.openstack_utils as os_utils -import functest.utils.functest_logger as ft_logger - -""" logging configuration """ -logger = ft_logger.Logger("generate_defaults").getLogger() - -REPO_PATH = os.environ['repos_dir'] + '/functest/' -if not os.path.exists(REPO_PATH): - logger.error("Functest repository directory not found '%s'" % REPO_PATH) - exit(-1) - - -DEFAULTS_FILE = '/home/opnfv/functest/conf/os_defaults.yaml' - - -def separator(): - logger.info("-------------------------------------------") - - -def get_instances(nova_client): - logger.debug("Getting instances...") - dic_instances = {} - instances = os_utils.get_instances(nova_client) - if not (instances is None or len(instances) == 0): - for instance in instances: - dic_instances.update({getattr(instance, 'id'): getattr(instance, - 'name')}) - return {'instances': dic_instances} - - -def get_images(nova_client): - logger.debug("Getting images...") - dic_images = {} - images = os_utils.get_images(nova_client) - if not (images is None or len(images) == 0): - for image in images: - dic_images.update({getattr(image, 'id'): getattr(image, 'name')}) - return {'images': dic_images} - - -def get_volumes(cinder_client): - logger.debug("Getting volumes...") - dic_volumes = {} - volumes = os_utils.get_volumes(cinder_client) - if volumes is not None: - for volume in volumes: - dic_volumes.update({volume.id: volume.display_name}) - return {'volumes': dic_volumes} - - -def get_networks(neutron_client): - logger.debug("Getting networks") - dic_networks = {} - networks = os_utils.get_network_list(neutron_client) - if networks is not None: - for network in networks: - dic_networks.update({network['id']: network['name']}) - return {'networks': dic_networks} - - -def get_routers(neutron_client): - logger.debug("Getting routers") - dic_routers = {} - routers = os_utils.get_router_list(neutron_client) - if routers is not None: - for router in routers: - dic_routers.update({router['id']: router['name']}) - return {'routers': dic_routers} - - -def get_security_groups(neutron_client): - logger.debug("Getting Security groups...") - dic_secgroups = {} - secgroups = os_utils.get_security_groups(neutron_client) - if not (secgroups is None or len(secgroups) == 0): - for secgroup in secgroups: - dic_secgroups.update({secgroup['id']: secgroup['name']}) - return {'secgroups': dic_secgroups} - - -def get_floatinips(nova_client): - logger.debug("Getting Floating IPs...") - dic_floatingips = {} - floatingips = os_utils.get_floating_ips(nova_client) - if not (floatingips is None or len(floatingips) == 0): - for floatingip in floatingips: - dic_floatingips.update({floatingip.id: floatingip.ip}) - return {'floatingips': dic_floatingips} - - -def get_users(keystone_client): - logger.debug("Getting users...") - dic_users = {} - users = os_utils.get_users(keystone_client) - if not (users is None or len(users) == 0): - for user in users: - dic_users.update({getattr(user, 'id'): getattr(user, 'name')}) - return {'users': dic_users} - - -def get_tenants(keystone_client): - logger.debug("Getting users...") - dic_tenants = {} - tenants = os_utils.get_tenants(keystone_client) - if not (tenants is None or len(tenants) == 0): - for tenant in tenants: - dic_tenants.update({getattr(tenant, 'id'): - getattr(tenant, 'name')}) - return {'tenants': dic_tenants} - - -def main(): - creds_nova = os_utils.get_credentials("nova") - nova_client = novaclient.Client('2', **creds_nova) - - creds_neutron = os_utils.get_credentials("neutron") - neutron_client = neutronclient.Client(**creds_neutron) - - creds_keystone = os_utils.get_credentials("keystone") - keystone_client = keystoneclient.Client(**creds_keystone) - - creds_cinder = os_utils.get_credentials("cinder") - cinder_client = cinderclient.Client('1', creds_cinder['username'], - creds_cinder['api_key'], - creds_cinder['project_id'], - creds_cinder['auth_url'], - service_type="volume") - - if not os_utils.check_credentials(): - logger.error("Please source the openrc credentials and run the" + - "script again.") - exit(-1) - - defaults = {} - defaults.update(get_instances(nova_client)) - defaults.update(get_images(nova_client)) - defaults.update(get_volumes(cinder_client)) - defaults.update(get_networks(neutron_client)) - defaults.update(get_routers(neutron_client)) - defaults.update(get_security_groups(neutron_client)) - defaults.update(get_floatinips(nova_client)) - defaults.update(get_users(keystone_client)) - defaults.update(get_tenants(keystone_client)) - - with open(DEFAULTS_FILE, 'w+') as yaml_file: - yaml_file.write(yaml.safe_dump(defaults, default_flow_style=False)) - yaml_file.seek(0) - logger.info("Openstack Defaults found in the deployment:\n%s" - % yaml_file.read()) - logger.debug("NOTE: These objects will NOT be deleted after " + - "running the tests.") - - -if __name__ == '__main__': - main() diff --git a/utils/openstack_clean.py b/utils/openstack_clean.py new file mode 100644 index 00000000..3eb39a26 --- /dev/null +++ b/utils/openstack_clean.py @@ -0,0 +1,426 @@ +#!/usr/bin/env python +# +# Description: +# Cleans possible leftovers after running functest tests: +# - Nova instances +# - Glance images +# - Cinder volumes +# - Floating IPs +# - Neutron networks, subnets and ports +# - Routers +# - Users and tenants +# +# Author: +# jose.lausuch@ericsson.com +# +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +# + +""" logging configuration """ + +import time + +from cinderclient import client as cinderclient +import functest.utils.functest_logger as ft_logger +import functest.utils.functest_utils as ft_utils +import functest.utils.openstack_utils as os_utils +from keystoneclient.v2_0 import client as keystoneclient +from neutronclient.v2_0 import client as neutronclient +from novaclient import client as novaclient +import yaml + + +logger = ft_logger.Logger("openstack_clean").getLogger() + +OS_SNAPSHOT_FILE = ft_utils.get_parameter_from_yaml( + "general.openstack.snapshot_file") + + +def separator(): + logger.info("-------------------------------------------") + + +def remove_instances(nova_client, default_instances): + logger.info("Removing Nova instances...") + instances = os_utils.get_instances(nova_client) + if instances is None or len(instances) == 0: + logger.debug("No instances found.") + return + + for instance in instances: + instance_name = getattr(instance, 'name') + instance_id = getattr(instance, 'id') + logger.debug("'%s', ID=%s " % (instance_name, instance_id)) + if instance_id not in default_instances: + logger.debug("Removing instance '%s' ..." % instance_id) + if os_utils.delete_instance(nova_client, instance_id): + logger.debug(" > Request sent.") + else: + logger.error("There has been a problem removing the " + "instance %s..." % instance_id) + else: + logger.debug(" > this is a default instance and will " + "NOT be deleted.") + + timeout = 50 + while timeout > 0: + instances = os_utils.get_instances(nova_client) + for instance in instances: + instance_id = getattr(instance, 'id') + if instance_id not in default_instances: + logger.debug("Waiting for instances to be terminated...") + timeout -= 1 + time.sleep(1) + continue + break + + +def remove_images(nova_client, default_images): + logger.info("Removing Glance images...") + images = os_utils.get_images(nova_client) + if images is None or len(images) == 0: + logger.debug("No images found.") + return + + for image in images: + image_name = getattr(image, 'name') + image_id = getattr(image, 'id') + logger.debug("'%s', ID=%s " % (image_name, image_id)) + if image_id not in default_images: + logger.debug("Removing image '%s', ID=%s ..." + % (image_name, image_id)) + if os_utils.delete_glance_image(nova_client, image_id): + logger.debug(" > Done!") + else: + logger.error("There has been a problem removing the" + "image %s..." % image_id) + else: + logger.debug(" > this is a default image and will " + "NOT be deleted.") + + +def remove_volumes(cinder_client, default_volumes): + logger.info("Removing Cinder volumes...") + volumes = os_utils.get_volumes(cinder_client) + if volumes is None or len(volumes) == 0: + logger.debug("No volumes found.") + return + + for volume in volumes: + volume_id = getattr(volume, 'id') + volume_name = getattr(volume, 'display_name') + logger.debug("'%s', ID=%s " % (volume_name, volume_id)) + if volume_id not in default_volumes: + logger.debug("Removing cinder volume %s ..." % volume_id) + if os_utils.delete_volume(cinder_client, volume_id): + logger.debug(" > Done!") + else: + logger.debug("Trying forced removal...") + if os_utils.delete_volume(cinder_client, + volume_id, + forced=True): + logger.debug(" > Done!") + else: + logger.error("There has been a problem removing the " + "volume %s..." % volume_id) + else: + logger.debug(" > this is a default volume and will " + "NOT be deleted.") + + +def remove_floatingips(nova_client, default_floatingips): + logger.info("Removing floating IPs...") + floatingips = os_utils.get_floating_ips(nova_client) + if floatingips is None or len(floatingips) == 0: + logger.debug("No floating IPs found.") + return + + init_len = len(floatingips) + deleted = 0 + for fip in floatingips: + fip_id = getattr(fip, 'id') + fip_ip = getattr(fip, 'ip') + logger.debug("'%s', ID=%s " % (fip_ip, fip_id)) + if fip_id not in default_floatingips: + logger.debug("Removing floating IP %s ..." % fip_id) + if os_utils.delete_floating_ip(nova_client, fip_id): + logger.debug(" > Done!") + deleted += 1 + else: + logger.error("There has been a problem removing the " + "floating IP %s..." % fip_id) + else: + logger.debug(" > this is a default floating IP and will " + "NOT be deleted.") + + timeout = 50 + while timeout > 0: + floatingips = os_utils.get_floating_ips(nova_client) + if floatingips is None or len(floatingips) == (init_len - deleted): + break + else: + logger.debug("Waiting for floating ips to be released...") + timeout -= 1 + time.sleep(1) + + +def remove_networks(neutron_client, default_networks, default_routers): + logger.info("Removing Neutron objects") + network_ids = [] + networks = os_utils.get_network_list(neutron_client) + if networks is None: + logger.debug("There are no networks in the deployment. ") + else: + logger.debug("Existing networks:") + for network in networks: + net_id = network['id'] + net_name = network['name'] + logger.debug(" '%s', ID=%s " % (net_name, net_id)) + if net_id in default_networks: + logger.debug(" > this is a default network and will " + "NOT be deleted.") + elif network['router:external'] is True: + logger.debug(" > this is an external network and will " + "NOT be deleted.") + else: + logger.debug(" > this network will be deleted.") + network_ids.append(net_id) + + # delete ports + ports = os_utils.get_port_list(neutron_client) + if ports is None: + logger.debug("There are no ports in the deployment. ") + else: + remove_ports(neutron_client, ports, network_ids) + + # remove routers + routers = os_utils.get_router_list(neutron_client) + if routers is None: + logger.debug("There are no routers in the deployment. ") + else: + remove_routers(neutron_client, routers, default_routers) + + # remove networks + if network_ids is not None: + for net_id in network_ids: + logger.debug("Removing network %s ..." % net_id) + if os_utils.delete_neutron_net(neutron_client, net_id): + logger.debug(" > Done!") + else: + logger.error("There has been a problem removing the " + "network %s..." % net_id) + + +def remove_ports(neutron_client, ports, network_ids): + for port in ports: + if port['network_id'] in network_ids: + port_id = port['id'] + try: + subnet_id = port['fixed_ips'][0]['subnet_id'] + except: + logger.info(" > WARNING: Port %s does not contain 'fixed_ips'" + % port_id) + print port + router_id = port['device_id'] + if len(port['fixed_ips']) == 0 and router_id == '': + logger.debug("Removing port %s ..." % port_id) + if (os_utils.delete_neutron_port(neutron_client, port_id)): + logger.debug(" > Done!") + else: + logger.error("There has been a problem removing the " + "port %s ..." % port_id) + force_remove_port(neutron_client, port_id) + + elif port['device_owner'] == 'network:router_interface': + logger.debug("Detaching port %s (subnet %s) from router %s ..." + % (port_id, subnet_id, router_id)) + if os_utils.remove_interface_router( + neutron_client, router_id, subnet_id): + time.sleep(5) # leave 5 seconds to detach + logger.debug(" > Done!") + else: + logger.error("There has been a problem removing the " + "interface %s from router %s..." + % (subnet_id, router_id)) + force_remove_port(neutron_client, port_id) + else: + force_remove_port(neutron_client, port_id) + + +def force_remove_port(neutron_client, port_id): + logger.debug("Clearing device_owner for port %s ..." % port_id) + os_utils.update_neutron_port(neutron_client, port_id, + device_owner='clear') + logger.debug("Removing port %s ..." % port_id) + if os_utils.delete_neutron_port(neutron_client, port_id): + logger.debug(" > Done!") + else: + logger.error("There has been a problem removing the port %s..." + % port_id) + + +def remove_routers(neutron_client, routers, default_routers): + for router in routers: + router_id = router['id'] + router_name = router['name'] + if router_id not in default_routers: + logger.debug("Checking '%s' with ID=(%s) ..." % (router_name, + router_id)) + if router['external_gateway_info'] is not None: + logger.debug("Router has gateway to external network." + "Removing link...") + if os_utils.remove_gateway_router(neutron_client, router_id): + logger.debug(" > Done!") + else: + logger.error("There has been a problem removing " + "the gateway...") + else: + logger.debug("Router is not connected to anything." + "Ready to remove...") + logger.debug("Removing router %s(%s) ..." + % (router_name, router_id)) + if os_utils.delete_neutron_router(neutron_client, router_id): + logger.debug(" > Done!") + else: + logger.error("There has been a problem removing the " + "router '%s'(%s)..." % (router_name, router_id)) + + +def remove_security_groups(neutron_client, default_security_groups): + logger.info("Removing Security groups...") + secgroups = os_utils.get_security_groups(neutron_client) + if secgroups is None or len(secgroups) == 0: + logger.debug("No security groups found.") + return + + for secgroup in secgroups: + secgroup_name = secgroup['name'] + secgroup_id = secgroup['id'] + logger.debug("'%s', ID=%s " % (secgroup_name, secgroup_id)) + if secgroup_id not in default_security_groups: + logger.debug(" Removing '%s'..." % secgroup_name) + if os_utils.delete_security_group(neutron_client, secgroup_id): + logger.debug(" > Done!") + else: + logger.error("There has been a problem removing the " + "security group %s..." % secgroup_id) + else: + logger.debug(" > this is a default security group and will NOT " + "be deleted.") + + +def remove_users(keystone_client, default_users): + logger.info("Removing Users...") + users = os_utils.get_users(keystone_client) + if users is None: + logger.debug("There are no users in the deployment. ") + return + + for user in users: + user_name = getattr(user, 'name') + user_id = getattr(user, 'id') + logger.debug("'%s', ID=%s " % (user_name, user_id)) + if user_id not in default_users: + logger.debug(" Removing '%s'..." % user_name) + if os_utils.delete_user(keystone_client, user_id): + logger.debug(" > Done!") + else: + logger.error("There has been a problem removing the " + "user '%s'(%s)..." % (user_name, user_id)) + else: + logger.debug(" > this is a default user and will " + "NOT be deleted.") + + +def remove_tenants(keystone_client, default_tenants): + logger.info("Removing Tenants...") + tenants = os_utils.get_tenants(keystone_client) + if tenants is None: + logger.debug("There are no tenants in the deployment. ") + return + + for tenant in tenants: + tenant_name = getattr(tenant, 'name') + tenant_id = getattr(tenant, 'id') + logger.debug("'%s', ID=%s " % (tenant_name, tenant_id)) + if tenant_id not in default_tenants: + logger.debug(" Removing '%s'..." % tenant_name) + if os_utils.delete_tenant(keystone_client, tenant_id): + logger.debug(" > Done!") + else: + logger.error("There has been a problem removing the " + "tenant '%s'(%s)..." % (tenant_name, tenant_id)) + else: + logger.debug(" > this is a default tenant and will " + "NOT be deleted.") + + +def main(): + logger.info("+++++++++++++++++++++++++++++++") + logger.info("Cleaning OpenStack resources...") + logger.info("+++++++++++++++++++++++++++++++") + + try: + with open(OS_SNAPSHOT_FILE) as f: + snapshot_yaml = yaml.safe_load(f) + except Exception: + logger.info("The file %s does not exist. The OpenStack snapshot must" + " be created first. Aborting cleanup." % OS_SNAPSHOT_FILE) + exit(0) + + default_images = snapshot_yaml.get('images') + default_instances = snapshot_yaml.get('instances') + default_volumes = snapshot_yaml.get('volumes') + default_networks = snapshot_yaml.get('networks') + default_routers = snapshot_yaml.get('routers') + default_security_groups = snapshot_yaml.get('secgroups') + default_floatingips = snapshot_yaml.get('floatingips') + default_users = snapshot_yaml.get('users') + default_tenants = snapshot_yaml.get('tenants') + + creds_nova = os_utils.get_credentials("nova") + nova_client = novaclient.Client('2', **creds_nova) + + creds_neutron = os_utils.get_credentials("neutron") + neutron_client = neutronclient.Client(**creds_neutron) + + creds_keystone = os_utils.get_credentials("keystone") + keystone_client = keystoneclient.Client(**creds_keystone) + + creds_cinder = os_utils.get_credentials("cinder") + # cinder_client = cinderclient.Client(**creds_cinder) + cinder_client = cinderclient.Client('1', creds_cinder['username'], + creds_cinder['api_key'], + creds_cinder['project_id'], + creds_cinder['auth_url'], + service_type="volume") + + if not os_utils.check_credentials(): + logger.error("Please source the openrc credentials and run " + "the script again.") + exit(-1) + + remove_instances(nova_client, default_instances) + separator() + remove_images(nova_client, default_images) + separator() + remove_volumes(cinder_client, default_volumes) + separator() + remove_floatingips(nova_client, default_floatingips) + separator() + remove_networks(neutron_client, default_networks, default_routers) + separator() + remove_security_groups(neutron_client, default_security_groups) + separator() + remove_users(keystone_client, default_users) + separator() + remove_tenants(keystone_client, default_tenants) + separator() + + +if __name__ == '__main__': + main() diff --git a/utils/openstack_snapshot.py b/utils/openstack_snapshot.py new file mode 100644 index 00000000..704ef505 --- /dev/null +++ b/utils/openstack_snapshot.py @@ -0,0 +1,188 @@ +#!/usr/bin/env python +# +# Description: +# Generates a list of the current Openstack objects in the deployment: +# - Nova instances +# - Glance images +# - Cinder volumes +# - Floating IPs +# - Neutron networks, subnets and ports +# - Routers +# - Users and tenants +# +# Author: +# jose.lausuch@ericsson.com +# +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +# + +""" logging configuration """ + +import os + +from cinderclient import client as cinderclient +import functest.utils.functest_logger as ft_logger +import functest.utils.functest_utils as ft_utils +import functest.utils.openstack_utils as os_utils +from keystoneclient.v2_0 import client as keystoneclient +from neutronclient.v2_0 import client as neutronclient +from novaclient import client as novaclient +import yaml + + +logger = ft_logger.Logger("openstack_snapshot").getLogger() + +REPO_PATH = os.environ['repos_dir'] + '/functest/' +if not os.path.exists(REPO_PATH): + logger.error("Functest repository directory not found '%s'" % REPO_PATH) + exit(-1) + + +OS_SNAPSHOT_FILE = ft_utils.get_parameter_from_yaml( + "general.openstack.snapshot_file") + + +def separator(): + logger.info("-------------------------------------------") + + +def get_instances(nova_client): + logger.debug("Getting instances...") + dic_instances = {} + instances = os_utils.get_instances(nova_client) + if not (instances is None or len(instances) == 0): + for instance in instances: + dic_instances.update({getattr(instance, 'id'): getattr(instance, + 'name')}) + return {'instances': dic_instances} + + +def get_images(nova_client): + logger.debug("Getting images...") + dic_images = {} + images = os_utils.get_images(nova_client) + if not (images is None or len(images) == 0): + for image in images: + dic_images.update({getattr(image, 'id'): getattr(image, 'name')}) + return {'images': dic_images} + + +def get_volumes(cinder_client): + logger.debug("Getting volumes...") + dic_volumes = {} + volumes = os_utils.get_volumes(cinder_client) + if volumes is not None: + for volume in volumes: + dic_volumes.update({volume.id: volume.display_name}) + return {'volumes': dic_volumes} + + +def get_networks(neutron_client): + logger.debug("Getting networks") + dic_networks = {} + networks = os_utils.get_network_list(neutron_client) + if networks is not None: + for network in networks: + dic_networks.update({network['id']: network['name']}) + return {'networks': dic_networks} + + +def get_routers(neutron_client): + logger.debug("Getting routers") + dic_routers = {} + routers = os_utils.get_router_list(neutron_client) + if routers is not None: + for router in routers: + dic_routers.update({router['id']: router['name']}) + return {'routers': dic_routers} + + +def get_security_groups(neutron_client): + logger.debug("Getting Security groups...") + dic_secgroups = {} + secgroups = os_utils.get_security_groups(neutron_client) + if not (secgroups is None or len(secgroups) == 0): + for secgroup in secgroups: + dic_secgroups.update({secgroup['id']: secgroup['name']}) + return {'secgroups': dic_secgroups} + + +def get_floatinips(nova_client): + logger.debug("Getting Floating IPs...") + dic_floatingips = {} + floatingips = os_utils.get_floating_ips(nova_client) + if not (floatingips is None or len(floatingips) == 0): + for floatingip in floatingips: + dic_floatingips.update({floatingip.id: floatingip.ip}) + return {'floatingips': dic_floatingips} + + +def get_users(keystone_client): + logger.debug("Getting users...") + dic_users = {} + users = os_utils.get_users(keystone_client) + if not (users is None or len(users) == 0): + for user in users: + dic_users.update({getattr(user, 'id'): getattr(user, 'name')}) + return {'users': dic_users} + + +def get_tenants(keystone_client): + logger.debug("Getting tenants...") + dic_tenants = {} + tenants = os_utils.get_tenants(keystone_client) + if not (tenants is None or len(tenants) == 0): + for tenant in tenants: + dic_tenants.update({getattr(tenant, 'id'): + getattr(tenant, 'name')}) + return {'tenants': dic_tenants} + + +def main(): + creds_nova = os_utils.get_credentials("nova") + nova_client = novaclient.Client('2', **creds_nova) + + creds_neutron = os_utils.get_credentials("neutron") + neutron_client = neutronclient.Client(**creds_neutron) + + creds_keystone = os_utils.get_credentials("keystone") + keystone_client = keystoneclient.Client(**creds_keystone) + + creds_cinder = os_utils.get_credentials("cinder") + cinder_client = cinderclient.Client('1', creds_cinder['username'], + creds_cinder['api_key'], + creds_cinder['project_id'], + creds_cinder['auth_url'], + service_type="volume") + + if not os_utils.check_credentials(): + logger.error("Please source the openrc credentials and run the" + + "script again.") + exit(-1) + + snapshot = {} + snapshot.update(get_instances(nova_client)) + snapshot.update(get_images(nova_client)) + snapshot.update(get_volumes(cinder_client)) + snapshot.update(get_networks(neutron_client)) + snapshot.update(get_routers(neutron_client)) + snapshot.update(get_security_groups(neutron_client)) + snapshot.update(get_floatinips(nova_client)) + snapshot.update(get_users(keystone_client)) + snapshot.update(get_tenants(keystone_client)) + + with open(OS_SNAPSHOT_FILE, 'w+') as yaml_file: + yaml_file.write(yaml.safe_dump(snapshot, default_flow_style=False)) + yaml_file.seek(0) + logger.debug("Openstack Snapshot found in the deployment:\n%s" + % yaml_file.read()) + logger.debug("NOTE: These objects will NOT be deleted after " + + "running the test.") + + +if __name__ == '__main__': + main() -- cgit 1.2.3-korg