Change OpenStack clean behaviour

JIRA: FUNCTEST-236

The openstack snapshot generation is now triggered before running
a test case and removed from prepare_env

Change-Id: I4d1bc95dedd7f59d4b1d5866f288e1c1a70ec69e
Signed-off-by: jose.lausuch <jose.lausuch@ericsson.com>

9 files changed, 112 insertions, 80 deletions

diff --git a/ci/check_os.sh b/ci/check_os.sh
index c9e0b5136..38fe32f58 100755
--- a/ci/check_os.sh
+++ b/ci/check_os.sh
@@ -26,7 +26,7 @@ fi
 echo "Checking OpenStack endpoints:"
 publicURL=$OS_AUTH_URL
 publicIP=$(echo $publicURL|sed 's/^.*http\:\/\///'|sed 's/.[^:]*$//')
-publicPort=$(echo $publicURL|sed 's/^.*://'|sed 's/.[^\/]*$//')
+publicPort=$(echo $publicURL|sed 's/^.*://'|sed 's/\/.*$//')
 echo ">>Verifying connectivity to the public endpoint $publicIP:$publicPort..."
 verify_connectivity $publicIP $publicPort
 RETVAL=$?
diff --git a/ci/config_functest.yaml b/ci/config_functest.yaml
index dc0919bbf..ea502dacb 100644
--- a/ci/config_functest.yaml
+++ b/ci/config_functest.yaml
@@ -29,6 +29,8 @@ general:
         dir_rally_inst:     /home/opnfv/.rally
 
     openstack:
+        snapshot_file: /home/opnfv/functest/conf/openstack_snapshot.yaml
+
         image_name: Cirros-0.3.4
         image_file_name:  cirros-0.3.4-x86_64-disk.img
         image_disk_format:  qcow2
diff --git a/ci/prepare_env.py b/ci/prepare_env.py
index 5754a62b2..4f29260f6 100644
--- a/ci/prepare_env.py
+++ b/ci/prepare_env.py
@@ -12,20 +12,19 @@
 # http://www.apache.org/licenses/LICENSE-2.0
 #
 
+
 import argparse
 import os
 import re
 import subprocess
 import sys
-import yaml
 
 import functest.utils.functest_logger as ft_logger
 import functest.utils.functest_utils as ft_utils
-import functest.utils.generate_defaults as gen_def
 import functest.utils.openstack_utils as os_utils
+import yaml
 
 
-""" arguments """
 actions = ['start', 'check']
 parser = argparse.ArgumentParser()
 parser.add_argument("action", help="Possible actions are: "
@@ -241,12 +240,6 @@ def install_rally():
                                         "OpenStack flavors."))
 
 
-def generate_os_defaults():
-    print_separator()
-    logger.info("Generating OpenStack defaults...")
-    gen_def.main()
-
-
 def check_environment():
     msg_not_active = "The Functest environment is not installed."
     if not os.path.isfile(ENV_FILE):
@@ -274,7 +267,6 @@ def main():
         source_rc_file()
         verify_deployment()
         install_rally()
-        generate_os_defaults()
 
         with open(ENV_FILE, "w") as env_file:
             env_file.write("1")
diff --git a/ci/run_tests.py b/ci/run_tests.py
index 5dba18190..29b01d7ff 100644
--- a/ci/run_tests.py
+++ b/ci/run_tests.py
@@ -14,13 +14,13 @@ import re
 import sys
 
 import functest.ci.tier_builder as tb
-import functest.utils.clean_openstack as clean_os
 import functest.utils.functest_logger as ft_logger
 import functest.utils.functest_utils as ft_utils
+import functest.utils.openstack_clean as os_clean
+import functest.utils.openstack_snapshot as os_snapshot
 import functest.utils.openstack_utils as os_utils
 
 
-""" arguments """
 parser = argparse.ArgumentParser()
 parser.add_argument("-t", "--test", dest="test", action='store',
                     help="Test case or tier (group of tests) to be executed. "
@@ -61,8 +61,13 @@ def source_rc_file():
     os_utils.source_credentials(rc_file)
 
 
+def generate_os_snapshot():
+    logger.debug("Generating OpenStack snapshot...")
+    os_snapshot.main()
+
+
 def cleanup():
-    clean_os.main()
+    os_clean.main()
 
 
 def run_test(test):
@@ -71,6 +76,9 @@ def run_test(test):
     logger.info("Running test case '%s'..." % test_name)
     print_separator("=")
     logger.debug("\n%s" % test)
+
+    generate_os_snapshot()
+
     flags = (" -t %s" % (test_name))
     if REPORT_FLAG:
         flags += " -r"
diff --git a/cli/commands/cli_os.py b/cli/commands/cli_os.py
index 23b7beaa3..b007842b3 100644
--- a/cli/commands/cli_os.py
+++ b/cli/commands/cli_os.py
@@ -7,15 +7,15 @@
 # http://www.apache.org/licenses/LICENSE-2.0
 #
 
-import click
+
 import os
+import click
+import functest.utils.functest_utils as ft_utils
+import functest.utils.openstack_clean as os_clean
+import functest.utils.openstack_snapshot as os_snapshot
 import yaml
 
-import functest.utils.clean_openstack as clean_os
-import functest.utils.functest_utils as ft_utils
-import functest.utils.generate_defaults as gen_def
 
-""" global variables """
 with open(os.environ["CONFIG_FUNCTEST_YAML"]) as f:
     functest_yaml = yaml.safe_load(f)
 
@@ -24,10 +24,12 @@ FUNCTEST_REPO = ("%s/functest/" % REPOS_DIR)
 FUNCTEST_CONF_DIR = functest_yaml.get("general").get(
     "directories").get("dir_functest_conf")
 RC_FILE = os.getenv('creds')
-OS_DEFAULTS_FILE = FUNCTEST_CONF_DIR + '/os_defaults.yaml'
+OS_SNAPSHOT_FILE = ft_utils.get_parameter_from_yaml(
+    "general.openstack.snapshot_file")
 
 
 class CliOpenStack:
+
     def __init__(self):
         self.os_auth_url = os.getenv('OS_AUTH_URL')
         self.endpoint_ip = None
@@ -86,7 +88,7 @@ class CliOpenStack:
 
     def snapshot_create(self):
         self.ping_endpoint()
-        if os.path.isfile(OS_DEFAULTS_FILE):
+        if os.path.isfile(OS_SNAPSHOT_FILE):
             answer = raw_input("It seems there is already an OpenStack "
                                "snapshot. Do you want to overwrite it with "
                                "the current OpenStack status? [y|n]\n")
@@ -99,22 +101,22 @@ class CliOpenStack:
                     answer = raw_input("Invalid answer. Please type [y|n]\n")
 
         click.echo("Generating Openstack snapshot...")
-        gen_def.main()
+        os_snapshot.main()
 
     def snapshot_show(self):
-        if not os.path.isfile(OS_DEFAULTS_FILE):
+        if not os.path.isfile(OS_SNAPSHOT_FILE):
             click.echo("There is no OpenStack snapshot created. To create "
                        "one run the command 'functest env os-create-snapshot'")
             return
-        with open(OS_DEFAULTS_FILE, 'r') as yaml_file:
+        with open(OS_SNAPSHOT_FILE, 'r') as yaml_file:
             click.echo("\n%s"
                        % yaml_file.read())
 
     def clean(self):
         self.ping_endpoint()
-        if not os.path.isfile(OS_DEFAULTS_FILE):
+        if not os.path.isfile(OS_SNAPSHOT_FILE):
             click.echo("Not possible to clean OpenStack without a snapshot. "
                        "This could cause problems. "
                        "Run first the command 'os-create-shapshot'.")
             return
-        clean_os.main()
+        os_clean.main()
diff --git a/docs/configguide/configguide.rst b/docs/configguide/configguide.rst
index 7c8ce8b8c..9f6693980 100644
--- a/docs/configguide/configguide.rst
+++ b/docs/configguide/configguide.rst
@@ -327,10 +327,10 @@ The structure under the **functest** repository can be described as follows::
     |   `-- vIMS
     `-- utils
         |-- __init__.py
-        |-- clean_openstack.py
         |-- functest_logger.py
         |-- functest_utils.py
-        |-- generate_defaults.py
+        |-- openstack_clean.py
+        |-- openstack_snapshot.py
         `-- openstack_utils.py
 
     (Note: All *.pyc files removed from above list for brevity...)
diff --git a/utils/functest_utils.py b/utils/functest_utils.py
index 2e7a12238..285f887cf 100644
--- a/utils/functest_utils.py
+++ b/utils/functest_utils.py
@@ -8,23 +8,25 @@
 # http://www.apache.org/licenses/LICENSE-2.0
 #
 
+""" global variables """
+
 from datetime import datetime as dt
 import json
 import os
 import os.path
 import re
-import requests
 import shutil
 import socket
 import subprocess
 import sys
 import urllib2
-import yaml
-from git import Repo
+
 import functest.ci.tier_builder as tb
+from git import Repo
+import requests
+import yaml
 
 
-""" global variables """
 REPOS_DIR = os.getenv('repos_dir')
 FUNCTEST_REPO = ("%s/functest/" % REPOS_DIR)
 
@@ -300,3 +302,26 @@ def get_criteria_by_test(testname):
                 criteria = test.get_criteria()
 
     return criteria
+
+
+# ----------------------------------------------------------
+#
+#               YAML UTILS
+#
+# -----------------------------------------------------------
+def get_parameter_from_yaml(parameter):
+    """
+    Returns the value of a given parameter in config_functest.yaml
+    parameter must be given in string format with dots
+    Example: general.openstack.image_name
+    """
+    with open(os.environ["CONFIG_FUNCTEST_YAML"]) as f:
+        functest_yaml = yaml.safe_load(f)
+    f.close()
+    value = functest_yaml
+    for element in parameter.split("."):
+        value = value.get(element)
+        if value is None:
+            raise ValueError("The parameter %s is not defined in"
+                             " config_functest.yaml" % parameter)
+    return value
diff --git a/utils/clean_openstack.py b/utils/openstack_clean.py
index 70eced62c..3eb39a260 100644
--- a/utils/clean_openstack.py
+++ b/utils/openstack_clean.py
@@ -20,24 +20,24 @@
 # http://www.apache.org/licenses/LICENSE-2.0
 #
 
-import os
+""" logging configuration """
+
 import time
-import yaml
 
-from novaclient import client as novaclient
-from neutronclient.v2_0 import client as neutronclient
-from keystoneclient.v2_0 import client as keystoneclient
 from cinderclient import client as cinderclient
-
 import functest.utils.functest_logger as ft_logger
+import functest.utils.functest_utils as ft_utils
 import functest.utils.openstack_utils as os_utils
+from keystoneclient.v2_0 import client as keystoneclient
+from neutronclient.v2_0 import client as neutronclient
+from novaclient import client as novaclient
+import yaml
 
 
-""" logging configuration """
-logger = ft_logger.Logger("clean_openstack").getLogger()
+logger = ft_logger.Logger("openstack_clean").getLogger()
 
-REPO_PATH = os.environ['repos_dir'] + '/functest/'
-DEFAULTS_FILE = '/home/opnfv/functest/conf/os_defaults.yaml'
+OS_SNAPSHOT_FILE = ft_utils.get_parameter_from_yaml(
+    "general.openstack.snapshot_file")
 
 
 def separator():
@@ -365,22 +365,22 @@ def main():
     logger.info("+++++++++++++++++++++++++++++++")
 
     try:
-        with open(DEFAULTS_FILE) as f:
-            defaults_yaml = yaml.safe_load(f)
+        with open(OS_SNAPSHOT_FILE) as f:
+            snapshot_yaml = yaml.safe_load(f)
     except Exception:
         logger.info("The file %s does not exist. The OpenStack snapshot must"
-                    " be created first. Aborting cleanup." % DEFAULTS_FILE)
+                    " be created first. Aborting cleanup." % OS_SNAPSHOT_FILE)
         exit(0)
 
-    default_images = defaults_yaml.get('images')
-    default_instances = defaults_yaml.get('instances')
-    default_volumes = defaults_yaml.get('volumes')
-    default_networks = defaults_yaml.get('networks')
-    default_routers = defaults_yaml.get('routers')
-    default_security_groups = defaults_yaml.get('secgroups')
-    default_floatingips = defaults_yaml.get('floatingips')
-    default_users = defaults_yaml.get('users')
-    default_tenants = defaults_yaml.get('tenants')
+    default_images = snapshot_yaml.get('images')
+    default_instances = snapshot_yaml.get('instances')
+    default_volumes = snapshot_yaml.get('volumes')
+    default_networks = snapshot_yaml.get('networks')
+    default_routers = snapshot_yaml.get('routers')
+    default_security_groups = snapshot_yaml.get('secgroups')
+    default_floatingips = snapshot_yaml.get('floatingips')
+    default_users = snapshot_yaml.get('users')
+    default_tenants = snapshot_yaml.get('tenants')
 
     creds_nova = os_utils.get_credentials("nova")
     nova_client = novaclient.Client('2', **creds_nova)
diff --git a/utils/generate_defaults.py b/utils/openstack_snapshot.py
index 35b9ed305..704ef5053 100644
--- a/utils/generate_defaults.py
+++ b/utils/openstack_snapshot.py
@@ -20,19 +20,21 @@
 # http://www.apache.org/licenses/LICENSE-2.0
 #
 
+""" logging configuration """
+
 import os
-import yaml
 
-from novaclient import client as novaclient
-from neutronclient.v2_0 import client as neutronclient
-from keystoneclient.v2_0 import client as keystoneclient
 from cinderclient import client as cinderclient
-
-import functest.utils.openstack_utils as os_utils
 import functest.utils.functest_logger as ft_logger
+import functest.utils.functest_utils as ft_utils
+import functest.utils.openstack_utils as os_utils
+from keystoneclient.v2_0 import client as keystoneclient
+from neutronclient.v2_0 import client as neutronclient
+from novaclient import client as novaclient
+import yaml
 
-""" logging configuration """
-logger = ft_logger.Logger("generate_defaults").getLogger()
+
+logger = ft_logger.Logger("openstack_snapshot").getLogger()
 
 REPO_PATH = os.environ['repos_dir'] + '/functest/'
 if not os.path.exists(REPO_PATH):
@@ -40,7 +42,8 @@ if not os.path.exists(REPO_PATH):
     exit(-1)
 
 
-DEFAULTS_FILE = '/home/opnfv/functest/conf/os_defaults.yaml'
+OS_SNAPSHOT_FILE = ft_utils.get_parameter_from_yaml(
+    "general.openstack.snapshot_file")
 
 
 def separator():
@@ -129,7 +132,7 @@ def get_users(keystone_client):
 
 
 def get_tenants(keystone_client):
-    logger.debug("Getting users...")
+    logger.debug("Getting tenants...")
     dic_tenants = {}
     tenants = os_utils.get_tenants(keystone_client)
     if not (tenants is None or len(tenants) == 0):
@@ -161,24 +164,24 @@ def main():
                      "script again.")
         exit(-1)
 
-    defaults = {}
-    defaults.update(get_instances(nova_client))
-    defaults.update(get_images(nova_client))
-    defaults.update(get_volumes(cinder_client))
-    defaults.update(get_networks(neutron_client))
-    defaults.update(get_routers(neutron_client))
-    defaults.update(get_security_groups(neutron_client))
-    defaults.update(get_floatinips(nova_client))
-    defaults.update(get_users(keystone_client))
-    defaults.update(get_tenants(keystone_client))
-
-    with open(DEFAULTS_FILE, 'w+') as yaml_file:
-        yaml_file.write(yaml.safe_dump(defaults, default_flow_style=False))
+    snapshot = {}
+    snapshot.update(get_instances(nova_client))
+    snapshot.update(get_images(nova_client))
+    snapshot.update(get_volumes(cinder_client))
+    snapshot.update(get_networks(neutron_client))
+    snapshot.update(get_routers(neutron_client))
+    snapshot.update(get_security_groups(neutron_client))
+    snapshot.update(get_floatinips(nova_client))
+    snapshot.update(get_users(keystone_client))
+    snapshot.update(get_tenants(keystone_client))
+
+    with open(OS_SNAPSHOT_FILE, 'w+') as yaml_file:
+        yaml_file.write(yaml.safe_dump(snapshot, default_flow_style=False))
         yaml_file.seek(0)
-        logger.info("Openstack Defaults found in the deployment:\n%s"
-                    % yaml_file.read())
+        logger.debug("Openstack Snapshot found in the deployment:\n%s"
+                     % yaml_file.read())
         logger.debug("NOTE: These objects will NOT be deleted after " +
-                     "running the tests.")
+                     "running the test.")
 
 
 if __name__ == '__main__':