Update restack defcore sync

It stops depending on refstack_client which cannot work till
OpenStack Queens.

JIRA: FUNCTEST-977

Change-Id: I2340643de82ab10d8fb83fb3a7fbfd1f412b2d7b
Signed-off-by: Cédric Ollivier <cedric.ollivier@orange.com>

6 files changed, 53 insertions, 12 deletions

diff --git a/docker/smoke/Dockerfile b/docker/smoke/Dockerfile
index 1ed39bdc0..f8beddb09 100644
--- a/docker/smoke/Dockerfile
+++ b/docker/smoke/Dockerfile
@@ -28,7 +28,6 @@ RUN apk --no-cache add --update libxml2 libxslt && \
     pip install --no-cache-dir --src /src -cupper-constraints.txt -cupper-constraints.opnfv.txt \
         /src/patrole /src/barbican-tempest-plugin /src/neutron-tempest-plugin \
         networking-bgpvpn networking-sfc /src/vmtp && \
-    virtualenv --system-site-packages /src/tempest/.venv && \
     rm -r upper-constraints.txt upper-constraints.opnfv.txt \
         /src/patrole /src/barbican-tempest-plugin /src/neutron-tempest-plugin /src/vmtp && \
     mkdir -p /home/opnfv/functest/data/refstack && \
diff --git a/docker/smoke/testcases.yaml b/docker/smoke/testcases.yaml
index 167471959..c262ab85d 100644
--- a/docker/smoke/testcases.yaml
+++ b/docker/smoke/testcases.yaml
@@ -58,7 +58,6 @@ tiers:
             -
                 case_name: refstack_defcore
                 project_name: functest
-                enabled: false
                 criteria: 100
                 blocking: false
                 description: >-
diff --git a/functest/ci/testcases.yaml b/functest/ci/testcases.yaml
index 3aeebcab9..fd6204e61 100644
--- a/functest/ci/testcases.yaml
+++ b/functest/ci/testcases.yaml
@@ -245,7 +245,6 @@ tiers:
             -
                 case_name: refstack_defcore
                 project_name: functest
-                enabled: false
                 criteria: 100
                 blocking: false
                 description: >-
diff --git a/functest/opnfv_tests/openstack/refstack/refstack.py b/functest/opnfv_tests/openstack/refstack/refstack.py
index 8266e281d..22e09bc82 100644
--- a/functest/opnfv_tests/openstack/refstack/refstack.py
+++ b/functest/opnfv_tests/openstack/refstack/refstack.py
@@ -11,9 +11,9 @@
 
 import logging
 import os
-import shutil
-
-from refstack_client import list_parser
+import re
+import subprocess
+import yaml
 
 from functest.opnfv_tests.openstack.tempest import tempest
 from functest.utils import config
@@ -27,8 +27,54 @@ class Refstack(tempest.TempestCommon):
     defcorelist = os.path.join(
         getattr(config.CONF, 'dir_refstack_data'), 'defcore.txt')
 
+    def _extract_refstack_data(self):
+        yaml_data = ""
+        with open(self.defcorelist) as def_file:
+            for line in def_file:
+                try:
+                    grp = re.search(r'^([^\[]*)(\[.*\])\n*$', line)
+                    yaml_data = "{}\n{}: {}".format(
+                        yaml_data, grp.group(1), grp.group(2))
+                except Exception:  # pylint: disable=broad-except
+                    self.__logger.warning("Cannot parse %s", line)
+        return yaml.load(yaml_data)
+
+    def _extract_tempest_data(self):
+        try:
+            cmd = ['stestr', '--here', self.verifier_repo_dir, 'list',
+                   '^tempest.']
+            output = subprocess.check_output(cmd)
+        except subprocess.CalledProcessError as cpe:
+            self.__logger.error(
+                "Exception when listing tempest tests: %s\n%s",
+                cpe.cmd, cpe.output)
+            raise
+        yaml_data2 = ""
+        for line in output.splitlines():
+            try:
+                grp = re.search(r'^([^\[]*)(\[.*\])\n*$', line)
+                yaml_data2 = "{}\n{}: {}".format(
+                    yaml_data2, grp.group(1), grp.group(2))
+            except Exception:  # pylint: disable=broad-except
+                self.__logger.warning("Cannot parse %s. skipping it", line)
+        return yaml.load(yaml_data2)
+
     def generate_test_list(self, **kwargs):
-        parser = list_parser.TestListParser(
-            getattr(config.CONF, 'dir_repo_tempest'))
-        nfile = parser.get_normalized_test_list(Refstack.defcorelist)
-        shutil.copyfile(nfile, self.list)
+        self.backup_tempest_config(self.conf_file, '/etc')
+        refstack_data = self._extract_refstack_data()
+        tempest_data = self._extract_tempest_data()
+        with open(self.list, 'w') as ref_file:
+            for key in refstack_data.keys():
+                try:
+                    for data in tempest_data[key]:
+                        if data == refstack_data[key][0]:
+                            break
+                    else:
+                        self.__logger.info("%s: ids differ. skipping it", key)
+                        continue
+                    ref_file.write("{}{}\n".format(
+                        key, str(tempest_data[key]).replace(
+                            "'", "").replace(", ", ",")))
+                except Exception:  # pylint: disable=broad-except
+                    self.__logger.info("%s: not found. skipping it", key)
+                    continue
diff --git a/requirements.txt b/requirements.txt
index dae7909d7..3b82370b4 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -15,6 +15,5 @@ snaps
 paramiko>=2.0.0 # LGPLv2.1+
 Jinja2>=2.10 # BSD License (3 clause)
 xtesting
-refstack-client
 os-client-config>=1.28.0 # Apache-2.0
 shade>=1.17.0 # Apache-2.0
diff --git a/upper-constraints.txt b/upper-constraints.txt
index cb24576d9..8b2486a7c 100644
--- a/upper-constraints.txt
+++ b/upper-constraints.txt
@@ -10,7 +10,6 @@ git+https://gerrit.opnfv.org/gerrit/clover#egg=clover
 git+https://gerrit.opnfv.org/gerrit/parser#egg=nfv-heattranslator&subdirectory=tosca2heat/heat-translator
 git+https://gerrit.opnfv.org/gerrit/parser#egg=nfv-toscaparser&subdirectory=tosca2heat/tosca-parser
 -e git+https://gerrit.opnfv.org/gerrit/parser#egg=nfv-parser
-git+https://github.com/openstack/refstack-client@a59189eaacda24b787ecb65e6634257beba361ec#egg=refstack-client
 cloudify-rest-client===4.3.2
 robotframework===3.0.2
 robotframework-httplibrary===0.4.2