Migrate to SNAPs openstack library

Functest is deprecating its openstack library which we were using extensively
We need then to move to an alternative and the best appears to be SNAPs

Change-Id: Icaa0b9f1ec580545b9d0faa88be2080a310deaf7
Signed-off-by: Manuel Buil<mbuil@suse.com>

4 files changed, 236 insertions, 234 deletions

diff --git a/sfc/lib/cleanup.py b/sfc/lib/cleanup.py
index 9c7b5484..32835fa8 100644
--- a/sfc/lib/cleanup.py
+++ b/sfc/lib/cleanup.py
@@ -1,97 +1,85 @@
 import sys
 import time
 import logging
-import functest.utils.openstack_utils as os_utils
-import sfc.lib.openstack_tacker as os_tacker
-import sfc.lib.utils as utils
+import sfc.lib.openstack_utils as os_sfc_utils
+import sfc.lib.odl_utils as odl_utils
 
 
 logger = logging.getLogger(__name__)
 
 
 def delete_odl_resources(odl_ip, odl_port, resource):
-    rsrc_list = utils.get_odl_resource_list(odl_ip, odl_port, resource)
-    elem_names = utils.odl_resource_list_names(resource, rsrc_list)
+    rsrc_list = odl_utils.get_odl_resource_list(odl_ip, odl_port, resource)
+    elem_names = odl_utils.odl_resource_list_names(resource, rsrc_list)
     for elem in elem_names:
         logger.info("Removing ODL resource: {0}/{1}".format(resource, elem))
-        utils.delete_odl_resource_elem(odl_ip, odl_port, resource, elem)
+        odl_utils.delete_odl_resource_elem(odl_ip, odl_port, resource, elem)
 
 
 def delete_odl_ietf_access_lists(odl_ip, odl_port):
-    acl_list = utils.get_odl_acl_list(odl_ip, odl_port)
-    acl_types_names = utils.odl_acl_types_names(acl_list)
+    acl_list = odl_utils.get_odl_acl_list(odl_ip, odl_port)
+    acl_types_names = odl_utils.odl_acl_types_names(acl_list)
     for acl_type, acl_name in acl_types_names:
-        utils.delete_odl_acl(odl_ip, odl_port, acl_type, acl_name)
+        odl_utils.delete_odl_acl(odl_ip, odl_port, acl_type, acl_name)
 
 
 def delete_vnfds():
-    t = os_tacker.get_tacker_client()
-    vnfds = os_tacker.list_vnfds(t)
+    t = os_sfc_utils.get_tacker_client()
+    vnfds = os_sfc_utils.list_vnfds(t)
     if vnfds is None:
         return
     for vnfd in vnfds:
         logger.info("Removing vnfd: {0}".format(vnfd))
-        os_tacker.delete_vnfd(t, vnfd_id=vnfd)
+        os_sfc_utils.delete_vnfd(t, vnfd_id=vnfd)
 
 
 def delete_vnfs():
-    t = os_tacker.get_tacker_client()
-    vnfs = os_tacker.list_vnfs(t)
+    t = os_sfc_utils.get_tacker_client()
+    vnfs = os_sfc_utils.list_vnfs(t)
     if vnfs is None:
         return
     for vnf in vnfs:
         logger.info("Removing vnf: {0}".format(vnf))
-        os_tacker.delete_vnf(t, vnf_id=vnf)
+        os_sfc_utils.delete_vnf(t, vnf_id=vnf)
 
 
 def delete_vnffgs():
-    t = os_tacker.get_tacker_client()
-    vnffgs = os_tacker.list_vnffgs(t)
+    t = os_sfc_utils.get_tacker_client()
+    vnffgs = os_sfc_utils.list_vnffgs(t)
     if vnffgs is None:
         return
     for vnffg in reversed(vnffgs):
         logger.info("Removing vnffg: {0}".format(vnffg))
-        os_tacker.delete_vnffg(t, vnffg_id=vnffg)
+        os_sfc_utils.delete_vnffg(t, vnffg_id=vnffg)
 
 
 def delete_vnffgds():
-    t = os_tacker.get_tacker_client()
-    vnffgds = os_tacker.list_vnffgds(t)
+    t = os_sfc_utils.get_tacker_client()
+    vnffgds = os_sfc_utils.list_vnffgds(t)
     if vnffgds is None:
         return
     for vnffgd in vnffgds:
         logger.info("Removing vnffgd: {0}".format(vnffgd))
-        os_tacker.delete_vnffgd(t, vnffgd_id=vnffgd)
+        os_sfc_utils.delete_vnffgd(t, vnffgd_id=vnffgd)
 
 
 def delete_vims():
-    t = os_tacker.get_tacker_client()
-    vims = os_tacker.list_vims(t)
+    t = os_sfc_utils.get_tacker_client()
+    vims = os_sfc_utils.list_vims(t)
     if vims is None:
         return
     for vim in vims:
         logger.info("Removing vim: {0}".format(vim))
-        os_tacker.delete_vim(t, vim_id=vim)
+        os_sfc_utils.delete_vim(t, vim_id=vim)
 
 
-def delete_floating_ips():
-    n = os_utils.get_nova_client()
-    fips = os_utils.get_floating_ips(n)
-    if fips is None:
-        return
-    for fip in fips:
-        logger.info("Removing floating ip: {0}".format(fip.ip))
-        os_utils.delete_floating_ip(n, fip.id)
-
-
-def delete_instances():
-    n = os_utils.get_nova_client()
-    instances = os_utils.get_instances(n)
-    if instances is None:
-        return
-    for inst in instances:
-        logger.info("Removing instance: {0}".format(inst.id))
-        os_utils.delete_instance(n, inst.id)
+# Creators is a list full of SNAPs objects
+def delete_openstack_objects(creators):
+    for creator in reversed(creators):
+        try:
+            creator.clean()
+        except Exception as e:
+            logger.error('Unexpected error cleaning - %s', e)
 
 
 def cleanup_odl(odl_ip, odl_port):
@@ -102,21 +90,29 @@ def cleanup_odl(odl_ip, odl_port):
     delete_odl_ietf_access_lists(odl_ip, odl_port)
 
 
-def cleanup(odl_ip=None, odl_port=None):
+def cleanup(creators, odl_ip=None, odl_port=None):
+    delete_vnffgs()
+    delete_vnffgds()
+    delete_vnfs()
+    time.sleep(20)
+    delete_vnfds()
+    delete_vims()
+    delete_openstack_objects(creators)
+    if odl_ip is not None and odl_port is not None:
+        cleanup_odl(odl_ip, odl_port)
+
+
+def cleanup_from_bash(odl_ip=None, odl_port=None):
     delete_vnffgs()
     delete_vnffgds()
     delete_vnfs()
     time.sleep(20)
     delete_vnfds()
     delete_vims()
-    delete_floating_ips()
-    delete_instances()
     if odl_ip is not None and odl_port is not None:
         cleanup_odl(odl_ip, odl_port)
 
 
 if __name__ == '__main__':
     if len(sys.argv) > 2:
-        cleanup(sys.argv[1], sys.argv[2])
-    else:
-        cleanup()
+        cleanup_from_bash(sys.argv[1], sys.argv[2])
diff --git a/sfc/lib/config.py b/sfc/lib/config.py
index fe95e983..bc955d8b 100644
--- a/sfc/lib/config.py
+++ b/sfc/lib/config.py
@@ -13,7 +13,7 @@ import yaml
 import sfc
 import functest
 
-import sfc.lib.utils as test_utils
+import sfc.lib.test_utils as test_utils
 from functest.utils.constants import CONST
 import logging
 import functest.utils.functest_utils as ft_utils
@@ -83,16 +83,12 @@ class CommonConfig(object):
             "defaults.vcpu_count", self.config_file)
         self.image_name = ft_utils.get_parameter_from_yaml(
             "defaults.image_name", self.config_file)
-        self.image_file_name = ft_utils.get_parameter_from_yaml(
-            "defaults.image_file_name", self.config_file)
         self.image_format = ft_utils.get_parameter_from_yaml(
             "defaults.image_format", self.config_file)
-        self.url = ft_utils.get_parameter_from_yaml(
-            "defaults.url", self.config_file)
+        self.image_url = ft_utils.get_parameter_from_yaml(
+            "defaults.image_url", self.config_file)
         self.dir_functest_data = ft_utils.get_functest_config(
             "general.dir.functest_data")
-        self.image_path = os.path.join(
-            self.dir_functest_data, self.image_file_name)
 
 
 class TestcaseConfig(object):
diff --git a/sfc/lib/openstack_utils.py b/sfc/lib/openstack_utils.py
index 2d4ecff3..f55f62e8 100644
--- a/sfc/lib/openstack_utils.py
+++ b/sfc/lib/openstack_utils.py
@@ -3,200 +3,207 @@ import os
 import time
 import json
 import yaml
-import functest.utils.openstack_utils as os_utils
 from tackerclient.tacker import client as tackerclient
 from functest.utils.constants import CONST
 
+from snaps.openstack.tests import openstack_tests
 
-logger = logging.getLogger(__name__)
-DEFAULT_TACKER_API_VERSION = '1.0'
+from snaps.openstack.create_image import OpenStackImage
+from snaps.config.image import ImageConfig
 
+from snaps.config.flavor import FlavorConfig
+from snaps.openstack.create_flavor import OpenStackFlavor
 
-def get_av_zones():
-    '''
-    Return the availability zone each host belongs to
-    '''
-    nova_client = os_utils.get_nova_client()
-    hosts = os_utils.get_hypervisors(nova_client)
-    return ['nova::{0}'.format(host) for host in hosts]
+from snaps.config.network import NetworkConfig, SubnetConfig, PortConfig
+from snaps.openstack.create_network import OpenStackNetwork
 
+from snaps.config.router import RouterConfig
+from snaps.openstack.create_router import OpenStackRouter
 
-def get_compute_client():
-    '''
-    Return the compute where the client sits
-    '''
-    nova_client = os_utils.get_nova_client()
-    hosts = os_utils.get_hypervisors(nova_client)
-    for compute in hosts:
-        vms = nova_client.servers.list(search_opts={'host': compute})
-        for vm in vms:
-            if "client" in vm.name:
-                return compute
-    return False
-
-
-def setup_neutron(neutron_client, net, subnet, router, subnet_cidr):
-    n_dict = os_utils.create_network_full(neutron_client,
-                                          net,
-                                          subnet,
-                                          router,
-                                          subnet_cidr)
-    if not n_dict:
-        logger.error("failed to create neutron network")
-        return False
-
-    return n_dict["net_id"]
-
-
-def create_secgroup_rule(neutron_client, sg_id, direction, protocol,
-                         port_range_min=None, port_range_max=None):
-    # We create a security group in 2 steps
-    # 1 - we check the format and set the json body accordingly
-    # 2 - we call neturon client to create the security group
-
-    # Format check
-    json_body = {'security_group_rule': {'direction': direction,
-                                         'security_group_id': sg_id,
-                                         'protocol': protocol}}
-    # parameters may be
-    # - both None => we do nothing
-    # - both Not None => we add them to the json description
-    # but one cannot be None is the other is not None
-    if (port_range_min is not None and port_range_max is not None):
-        # add port_range in json description
-        json_body['security_group_rule']['port_range_min'] = port_range_min
-        json_body['security_group_rule']['port_range_max'] = port_range_max
-        logger.debug("Security_group format set (port range included)")
-    else:
-        # either both port range are set to None => do nothing
-        # or one is set but not the other => log it and return False
-        if port_range_min is None and port_range_max is None:
-            logger.debug("Security_group format set (no port range mentioned)")
-        else:
-            logger.error("Bad security group format."
-                         "One of the port range is not properly set:"
-                         "range min: {},"
-                         "range max: {}".format(port_range_min,
-                                                port_range_max))
-            return False
-
-    # Create security group using neutron client
-    try:
-        neutron_client.create_security_group_rule(json_body)
-        return True
-    except:
-        return False
-
-
-def setup_ingress_egress_secgroup(neutron_client, protocol,
-                                  min_port=None, max_port=None):
-    secgroups = os_utils.get_security_groups(neutron_client)
-    for sg in secgroups:
-        # TODO: the version of the create_secgroup_rule function in
-        # functest swallows the exception thrown when a secgroup rule
-        # already exists and prints a ton of noise in the test output.
-        # Instead of making changes in functest code this late in the
-        # release cycle, we keep our own version without the exception
-        # logging. We must find a way to properly cleanup sec group
-        # rules using "functest openstack clean" or pretty printing the
-        # specific exception in the next release
-        create_secgroup_rule(neutron_client, sg['id'],
-                             'ingress', protocol,
-                             port_range_min=min_port,
-                             port_range_max=max_port)
-        create_secgroup_rule(neutron_client, sg['id'],
-                             'egress', protocol,
-                             port_range_min=min_port,
-                             port_range_max=max_port)
-
-
-def create_security_groups(neutron_client, secgroup_name, secgroup_descr):
-    sg_id = os_utils.create_security_group_full(neutron_client,
-                                                secgroup_name, secgroup_descr)
-    setup_ingress_egress_secgroup(neutron_client, "icmp")
-    setup_ingress_egress_secgroup(neutron_client, "tcp", 22, 22)
-    setup_ingress_egress_secgroup(neutron_client, "tcp", 80, 80)
-    setup_ingress_egress_secgroup(neutron_client, "udp", 67, 68)
-    return sg_id
-
-
-def create_instance(nova_client, name, flavor, image_id, network_id, sg_id,
-                    secgroup_name=None, fixed_ip=None,
-                    av_zone='', userdata=None, files=None):
-    logger.info("Creating instance '%s'..." % name)
-    logger.debug(
-        "Configuration:\n name=%s \n flavor=%s \n image=%s \n"
-        " network=%s\n secgroup=%s \n hypervisor=%s \n"
-        " fixed_ip=%s\n files=%s\n userdata=\n%s\n"
-        % (name, flavor, image_id, network_id, sg_id,
-           av_zone, fixed_ip, files, userdata))
-    instance = os_utils.create_instance_and_wait_for_active(
-        flavor,
-        image_id,
-        network_id,
-        name,
-        config_drive=True,
-        userdata=userdata,
-        av_zone=av_zone,
-        fixed_ip=fixed_ip,
-        files=files)
-
-    if instance is None:
-        logger.error("Error while booting instance.")
-        return None
+from snaps.config.security_group import (
+    Protocol, SecurityGroupRuleConfig, Direction, SecurityGroupConfig)
 
-    if secgroup_name:
-        logger.debug("Adding '%s' to security group '%s'..."
-                     % (name, secgroup_name))
-    else:
-        logger.debug("Adding '%s' to security group '%s'..."
-                     % (name, sg_id))
-    os_utils.add_secgroup_to_instance(nova_client, instance.id, sg_id)
+from snaps.openstack.create_security_group import OpenStackSecurityGroup
 
-    return instance
+import snaps.openstack.create_instance as cr_inst
+from snaps.config.vm_inst import VmInstanceConfig, FloatingIpConfig
 
+from snaps.openstack.utils import (
+    nova_utils, neutron_utils, glance_utils, heat_utils, keystone_utils)
 
-def assign_floating_ip(nova_client, neutron_client, instance_id):
-    instance = nova_client.servers.get(instance_id)
-    floating_ip = os_utils.create_floating_ip(neutron_client)['fip_addr']
-    instance.add_floating_ip(floating_ip)
-    logger.info("Assigned floating ip [%s] to instance [%s]"
-                % (floating_ip, instance.name))
+logger = logging.getLogger(__name__)
+DEFAULT_TACKER_API_VERSION = '1.0'
 
-    return floating_ip
 
+class OpenStackSFC:
 
-def get_nova_id(tacker_client, resource, vnf_id=None, vnf_name=None):
-    vnf = get_vnf(tacker_client, vnf_id, vnf_name)
-    try:
-        if vnf is None:
-            raise Exception("VNF not found")
-        heat = os_utils.get_heat_client()
-        resource = heat.resources.get(vnf['instance_id'], resource)
-        return resource.attributes['id']
-    except:
-        logger.error("Cannot get nova ID for VNF (id='%s', name='%s')"
-                     % (vnf_id, vnf_name))
-        return None
+    def __init__(self):
+        self.os_creds = openstack_tests.get_credentials(
+            os_env_file=CONST.__getattribute__('openstack_creds'))
+        self.creators = []
+        self.nova = nova_utils.nova_client(self.os_creds)
+        self.neutron = neutron_utils.neutron_client(self.os_creds)
+        self.glance = glance_utils.glance_client(self.os_creds)
+        self.heat = heat_utils.heat_client(self.os_creds)
 
+    def register_glance_image(self, name, url, img_format, public):
+        image_settings = ImageConfig(name=name, img_format=img_format, url=url,
+                                     public=public, image_user='admin')
 
-def get_neutron_interfaces(vm):
-    '''
-    Get the interfaces of an instance
-    '''
-    nova_client = os_utils.get_nova_client()
-    interfaces = nova_client.servers.interface_list(vm.id)
-    return interfaces
+        # TODO Remove this when tacker is part of SNAPS
+        self.image_settings = image_settings
 
+        image_creator = OpenStackImage(self.os_creds, image_settings)
+        image_creator.create()
 
-def get_client_port_id(vm):
-    '''
-    Get the neutron port id of the client
-    '''
-    interfaces = get_neutron_interfaces(vm)
-    if len(interfaces) > 1:
-        raise Exception("Client has more than one interface. Not expected!")
-    return interfaces[0].id
+        self.creators.append(image_creator)
+        return image_creator
+
+    def create_flavor(self, name, ram, disk, vcpus):
+        flavor_settings = FlavorConfig(name=name, ram=ram, disk=disk,
+                                       vcpus=vcpus)
+        flavor_creator = OpenStackFlavor(self.os_creds, flavor_settings)
+        flavor = flavor_creator.create()
+
+        self.creators.append(flavor_creator)
+        return flavor
+
+    def create_network_infrastructure(self, net_name, subnet_name, subnet_cidr,
+                                      router_name):
+        # Network and subnet
+        subnet_settings = SubnetConfig(name=subnet_name, cidr=subnet_cidr)
+        network_settings = NetworkConfig(name=net_name,
+                                         subnet_settings=[subnet_settings])
+        network_creator = OpenStackNetwork(self.os_creds, network_settings)
+        network = network_creator.create()
+
+        self.creators.append(network_creator)
+
+        # Router
+        ext_network_name = CONST.__getattribute__('EXTERNAL_NETWORK')
+
+        router_settings = RouterConfig(name=router_name,
+                                       external_gateway=ext_network_name,
+                                       internal_subnets=[subnet_name])
+
+        router_creator = OpenStackRouter(self.os_creds, router_settings)
+        router = router_creator.create()
+
+        self.creators.append(router_creator)
+
+        return network, router
+
+    def create_security_group(self, sec_grp_name):
+        rule_ping = SecurityGroupRuleConfig(sec_grp_name=sec_grp_name,
+                                            direction=Direction.ingress,
+                                            protocol=Protocol.icmp)
+
+        rule_ssh = SecurityGroupRuleConfig(sec_grp_name=sec_grp_name,
+                                           direction=Direction.ingress,
+                                           protocol=Protocol.tcp,
+                                           port_range_min=22,
+                                           port_range_max=22)
+
+        rule_http = SecurityGroupRuleConfig(sec_grp_name=sec_grp_name,
+                                            direction=Direction.ingress,
+                                            protocol=Protocol.tcp,
+                                            port_range_min=80,
+                                            port_range_max=80)
+
+        rules = [rule_ping, rule_ssh, rule_http]
+
+        secgroup_settings = SecurityGroupConfig(name=sec_grp_name,
+                                                rule_settings=rules)
+
+        sec_group_creator = OpenStackSecurityGroup(self.os_creds,
+                                                   secgroup_settings)
+        sec_group = sec_group_creator.create()
+
+        self.creators.append(sec_group_creator)
+
+        return sec_group
+
+    def create_instance(self, vm_name, flavor_name, image_creator, network,
+                        secgrp, av_zone):
+
+        port_settings = PortConfig(name=vm_name + '-port',
+                                   network_name=network.name)
+
+        instance_settings = VmInstanceConfig(
+            name=vm_name, flavor=flavor_name,
+            security_group_names=str(secgrp.name),
+            port_settings=[port_settings],
+            availability_zone=av_zone)
+
+        instance_creator = cr_inst.OpenStackVmInstance(
+            self.os_creds,
+            instance_settings,
+            image_creator.image_settings)
+
+        instance = instance_creator.create()
+
+        self.creators.append(instance_creator)
+        return instance, instance_creator
+
+    def get_av_zones(self):
+        '''
+        Return the availability zone each host belongs to
+        '''
+        hosts = nova_utils.get_hypervisor_hosts(self.nova)
+        return ['nova::{0}'.format(host) for host in hosts]
+
+    def get_compute_client(self):
+        '''
+        Return the compute where the client sits
+        '''
+        compute = nova_utils.get_server(self.nova, server_name='client')
+        return compute
+
+    def assign_floating_ip(self, router, vm, vm_creator):
+        '''
+        Assign a floating ips to all the VMs
+        '''
+        name = vm.name + "-float"
+        port_name = vm.ports[0].name
+        float_ip = FloatingIpConfig(name=name,
+                                    port_name=port_name,
+                                    router_name=router.name)
+        ip = vm_creator.add_floating_ip(float_ip)
+
+        return ip.ip
+
+    # We need this function because tacker VMs cannot be created through SNAPs
+    def assign_floating_ip_vnfs(self, router):
+        '''
+        Assign a floating ips to all the SFs
+        '''
+        stacks = self.heat.stacks.list()
+        fips = []
+        for stack in stacks:
+            servers = heat_utils.get_stack_servers(self.heat,
+                                                   self.nova,
+                                                   self.neutron,
+                                                   stack)
+            sf_creator = cr_inst.generate_creator(self.os_creds,
+                                                  servers[0],
+                                                  self.image_settings)
+            port_name = servers[0].ports[0].name
+            name = servers[0].name + "-float"
+            float_ip = FloatingIpConfig(name=name,
+                                        port_name=port_name,
+                                        router_name=router.name)
+            ip = sf_creator.add_floating_ip(float_ip)
+            fips.append(ip.ip)
+
+        return fips
+
+    def get_client_port_id(self, vm):
+        '''
+        Get the neutron port id of the client
+        '''
+        port_id = neutron_utils.get_port(self.neutron,
+                                         port_name=vm.name + "-port")
+        return port_id
 
 # TACKER SECTION #
 
@@ -210,7 +217,11 @@ def get_tacker_client_version():
 
 
 def get_tacker_client(other_creds={}):
-    sess = os_utils.get_session(other_creds)
+    creds_override = None
+    os_creds = openstack_tests.get_credentials(
+        os_env_file=CONST.__getattribute__('openstack_creds'),
+        overrides=creds_override)
+    sess = keystone_utils.keystone_session(os_creds)
     return tackerclient.Client(get_tacker_client_version(), session=sess)
 
 
diff --git a/sfc/lib/topology_shuffler.py b/sfc/lib/topology_shuffler.py
index 24c8f875..f678137a 100644
--- a/sfc/lib/topology_shuffler.py
+++ b/sfc/lib/topology_shuffler.py
@@ -1,7 +1,6 @@
 import datetime
 import random
 import logging
-import sfc.lib.utils as sfc_utils
 
 logger = logging.getLogger(__name__)
 
@@ -78,7 +77,7 @@ def get_seed():
     return seed
 
 
-def topology(vnf_names, av_zones=None, seed=None):
+def topology(vnf_names, os_sfc_util, av_zones=None, seed=None):
     '''
     Get the topology for client, server and vnfs.
     The topology is returned as a dict in the form
@@ -93,7 +92,7 @@ def topology(vnf_names, av_zones=None, seed=None):
     '''
 
     if av_zones is None:
-        av_zones = sfc_utils.get_av_zones()
+        av_zones = os_sfc_util.get_av_zones()
 
     if len(av_zones) < 2 or seed is None:
         # fall back to nova availability zone